diff --git a/contraceptive/lct_gan/eval.csv b/contraceptive/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..e73d29b3afa34c4a8f448dbaa5a19792874abd9c --- /dev/null +++ b/contraceptive/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.009699148273355183,,0.0012717798803132062,2.5979268550872803,0.031053613871335983,0.5863479971885681,0.0409364253282547,1.6693826410119073e-06,3.1333327293395996,0.028224041685461998,0.06742087006568909,0.03566202521324158,0.05481972172856331,0.022213930264115334,5.73125958442688 diff --git a/contraceptive/lct_gan/history.csv b/contraceptive/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..0940d5302833f59cc8ffdc748ec854da91739bd9 --- /dev/null +++ b/contraceptive/lct_gan/history.csv @@ -0,0 +1,20 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.01786272754015954,3.636569730948348,0.001167070859791755,0.0,0.0,0.0,0.0,0.0,0.01786272754015954,320,160,162.09109449386597,1.0130693405866622,0.5065346702933311,0.07982906211551608,0.0036570764575117208,8.543643573567897,1.3811252661610762e-05,0.0,0.0,0.0,0.0,0.0,0.0036570764575117208,80,40,38.24689245223999,0.9561723113059998,0.4780861556529999,0.013961730610299128 +1,0.005696987385198327,3.198073918344037,6.659749155254468e-05,0.0,0.0,0.0,0.0,0.0,0.005696987385198327,320,160,161.77688694000244,1.0111055433750153,0.5055527716875077,0.06436048086907249,0.003786977470736019,3.9373093709834395,1.9986069321897836e-05,0.0,0.0,0.0,0.0,0.0,0.003786977470736019,80,40,38.12403869628906,0.9531009674072266,0.4765504837036133,0.019040833081817254 +2,0.0032535013802828415,2.8198290220143503,1.6548445633081822e-05,0.0,0.0,0.0,0.0,0.0,0.0032535013802828415,320,160,158.56593680381775,0.9910371050238609,0.49551855251193044,0.05899094843493913,0.002633672622323502,4.367062080342106,6.056017390343449e-06,0.0,0.0,0.0,0.0,0.0,0.002633672622323502,80,40,36.21775007247925,0.9054437518119812,0.4527218759059906,0.029313163098959195 +3,0.0023438148911395728,2.2918122927125295,7.134696727795209e-06,0.0,0.0,0.0,0.0,0.0,0.0023438148911395728,320,160,152.8709909915924,0.9554436936974525,0.47772184684872626,0.06780749239678699,0.002467950962409304,4.88692410795129,4.36867752655612e-06,0.0,0.0,0.0,0.0,0.0,0.002467950962409304,80,40,35.697052240371704,0.8924263060092926,0.4462131530046463,0.018721673299660326 +4,0.002259014635501444,1.812988119399502,5.5367047818208335e-06,0.0,0.0,0.0,0.0,0.0,0.002259014635501444,320,160,151.4134497642517,0.9463340610265731,0.47316703051328657,0.06716944240579323,0.0023299435670196544,7.427999823173169,6.818181761379086e-06,0.0,0.0,0.0,0.0,0.0,0.0023299435670196544,80,40,36.091819047927856,0.9022954761981964,0.4511477380990982,0.025550102235138185 +5,0.002213509789987711,2.1981050524016665,6.457744553583708e-06,0.0,0.0,0.0,0.0,0.0,0.002213509789987711,320,160,155.00266909599304,0.9687666818499565,0.48438334092497826,0.06307913716664189,0.0025578231319741463,5.699148117736866,7.032264796913435e-06,0.0,0.0,0.0,0.0,0.0,0.0025578231319741463,80,40,35.88962531089783,0.8972406327724457,0.44862031638622285,0.017301402381235675 +6,0.002113006258792893,2.1890728188584476,6.114715563168734e-06,0.0,0.0,0.0,0.0,0.0,0.002113006258792893,320,160,153.502343416214,0.9593896463513374,0.4796948231756687,0.0661177773316524,0.0022539663767020103,8.082782875575992,6.678637441320801e-06,0.0,0.0,0.0,0.0,0.0,0.0022539663767020103,80,40,35.9239776134491,0.8980994403362275,0.44904972016811373,0.025397659230247883 +7,0.0018896224307241027,1.3406870565765918,4.11630951652614e-06,0.0,0.0,0.0,0.0,0.0,0.0018896224307241027,320,160,151.49836015701294,0.9468647509813308,0.4734323754906654,0.07347480687003553,0.0025110580976615894,5.483385282401798,4.007785178927748e-06,0.0,0.0,0.0,0.0,0.0,0.0025110580976615894,80,40,33.77740168571472,0.844435042142868,0.422217521071434,0.01343663605657639 +8,0.0019506988204057052,1.1689463564448361,4.559629974733976e-06,0.0,0.0,0.0,0.0,0.0,0.0019506988204057052,320,160,144.0004370212555,0.9000027313828468,0.4500013656914234,0.0659007933063549,0.0023398275739964446,4.280078241747558,9.40580381603908e-06,0.0,0.0,0.0,0.0,0.0,0.0023398275739964446,80,40,34.14411234855652,0.8536028087139129,0.42680140435695646,0.029161113313784882 +9,0.0016659495725662055,1.259914455004442,3.6357496224758216e-06,0.0,0.0,0.0,0.0,0.0,0.0016659495725662055,320,160,146.88511276245117,0.9180319547653198,0.4590159773826599,0.06927311308209028,0.0020320220184657954,1.9301895227664638,5.797358790626817e-06,0.0,0.0,0.0,0.0,0.0,0.0020320220184657954,80,40,33.82334542274475,0.8455836355686188,0.4227918177843094,0.031894830953388006 +10,0.0016075492954826132,1.0092837510064634,2.735296125088143e-06,0.0,0.0,0.0,0.0,0.0,0.0016075492954826132,320,160,143.3337082862854,0.8958356767892838,0.4479178383946419,0.07756731488425431,0.0026302734650016646,3.5820208163912866,1.3108698285635434e-05,0.0,0.0,0.0,0.0,0.0,0.0026302734650016646,80,40,33.34847378730774,0.8337118446826934,0.4168559223413467,0.026611345619312488 +11,0.0013905711543884536,1.4293391600536105,1.9943211839244628e-06,0.0,0.0,0.0,0.0,0.0,0.0013905711543884536,320,160,145.80800819396973,0.9113000512123108,0.4556500256061554,0.07521452093462813,0.0022007888529515184,4.483247433313909,7.529547302986828e-06,0.0,0.0,0.0,0.0,0.0,0.0022007888529515184,80,40,34.17486619949341,0.8543716549873352,0.4271858274936676,0.025888706676232685 +12,0.0012333092558833413,1.0602984449560355,1.4978945140242672e-06,0.0,0.0,0.0,0.0,0.0,0.0012333092558833413,320,160,144.26001048088074,0.9016250655055046,0.4508125327527523,0.06890693796813138,0.002605481748287275,5.8101766740395275,1.2162480100338935e-05,0.0,0.0,0.0,0.0,0.0,0.002605481748287275,80,40,34.0310595035553,0.8507764875888825,0.42538824379444123,0.030725552017582914 +13,0.0011501418213924809,1.3386667872641667,1.3185595447717802e-06,0.0,0.0,0.0,0.0,0.0,0.0011501418213924809,320,160,144.17591977119446,0.9010994985699654,0.4505497492849827,0.07558018262188852,0.0022788071105424024,2.7987204812981075,7.67833690556996e-06,0.0,0.0,0.0,0.0,0.0,0.0022788071105424024,80,40,34.57782983779907,0.8644457459449768,0.4322228729724884,0.028933984229661293 +14,0.0010513833095672,0.6387672975999288,1.4141241849948554e-06,0.0,0.0,0.0,0.0,0.0,0.0010513833095672,320,160,145.27083349227905,0.907942709326744,0.453971354663372,0.07942418371021631,0.0023162082296039445,3.349859969510388,8.629114276192951e-06,0.0,0.0,0.0,0.0,0.0,0.0023162082296039445,80,40,33.332454442977905,0.8333113610744476,0.4166556805372238,0.03227444588264916 +15,0.0009490251221507151,0.6831669182588915,1.0006828890851693e-06,0.0,0.0,0.0,0.0,0.0,0.0009490251221507151,320,160,145.07566261291504,0.906722891330719,0.4533614456653595,0.08901288353954442,0.002519568522711779,3.413597872712374,1.3267886331692902e-05,0.0,0.0,0.0,0.0,0.0,0.002519568522711779,80,40,35.61937355995178,0.8904843389987945,0.44524216949939727,0.03244233850882665 +16,0.0008506465366181715,0.6938899818249922,7.591354541367547e-07,0.0,0.0,0.0,0.0,0.0,0.0008506465366181715,320,160,141.91219687461853,0.8869512304663658,0.4434756152331829,0.07522430039280152,0.0022380605670150543,2.3211301649584923,8.706937260289163e-06,0.0,0.0,0.0,0.0,0.0,0.0022380605670150543,80,40,33.153270959854126,0.8288317739963531,0.41441588699817655,0.033504872786579654 +17,0.0007984611616791426,0.8550257516943558,6.954888952128042e-07,0.0,0.0,0.0,0.0,0.0,0.0007984611616791426,320,160,141.78189277648926,0.8861368298530579,0.44306841492652893,0.07633930989904911,0.0023599293230745387,2.454739641254777,1.1268823242671644e-05,0.0,0.0,0.0,0.0,0.0,0.0023599293230745387,80,40,33.33975076675415,0.8334937691688538,0.4167468845844269,0.03295552866329672 +18,0.0007362112768333872,1.1732242802433412,4.6208515920671473e-07,0.0,0.0,0.0,0.0,0.0,0.0007362112768333872,320,160,142.94105648994446,0.8933816030621529,0.44669080153107643,0.07887016502173765,0.0022324465072415477,2.106721719149411,9.551615076732606e-06,0.0,0.0,0.0,0.0,0.0,0.0022324465072415477,80,40,33.48200488090515,0.8370501220226287,0.41852506101131437,0.031034307027584872 diff --git a/contraceptive/lct_gan/mlu-eval.ipynb b/contraceptive/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ccc83044abcda562e27f07fd6fa976f2e9fb69cd --- /dev/null +++ b/contraceptive/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2556 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.179506Z", + "iopub.status.busy": "2024-03-01T17:33:51.179161Z", + "iopub.status.idle": "2024-03-01T17:33:51.213379Z", + "shell.execute_reply": "2024-03-01T17:33:51.212464Z" + }, + "papermill": { + "duration": 0.051044, + "end_time": "2024-03-01T17:33:51.215789", + "exception": false, + "start_time": "2024-03-01T17:33:51.164745", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.243896Z", + "iopub.status.busy": "2024-03-01T17:33:51.243419Z", + "iopub.status.idle": "2024-03-01T17:33:51.250437Z", + "shell.execute_reply": "2024-03-01T17:33:51.249566Z" + }, + "papermill": { + "duration": 0.02393, + "end_time": "2024-03-01T17:33:51.252575", + "exception": false, + "start_time": "2024-03-01T17:33:51.228645", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.277185Z", + "iopub.status.busy": "2024-03-01T17:33:51.276860Z", + "iopub.status.idle": "2024-03-01T17:33:51.281125Z", + "shell.execute_reply": "2024-03-01T17:33:51.280269Z" + }, + "papermill": { + "duration": 0.019169, + "end_time": "2024-03-01T17:33:51.283141", + "exception": false, + "start_time": "2024-03-01T17:33:51.263972", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.307819Z", + "iopub.status.busy": "2024-03-01T17:33:51.307160Z", + "iopub.status.idle": "2024-03-01T17:33:51.311367Z", + "shell.execute_reply": "2024-03-01T17:33:51.310516Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.01875, + "end_time": "2024-03-01T17:33:51.313388", + "exception": false, + "start_time": "2024-03-01T17:33:51.294638", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.337183Z", + "iopub.status.busy": "2024-03-01T17:33:51.336912Z", + "iopub.status.idle": "2024-03-01T17:33:51.342349Z", + "shell.execute_reply": "2024-03-01T17:33:51.341458Z" + }, + "papermill": { + "duration": 0.019601, + "end_time": "2024-03-01T17:33:51.344277", + "exception": false, + "start_time": "2024-03-01T17:33:51.324676", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "167ff1aa", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.369794Z", + "iopub.status.busy": "2024-03-01T17:33:51.369452Z", + "iopub.status.idle": "2024-03-01T17:33:51.374498Z", + "shell.execute_reply": "2024-03-01T17:33:51.373660Z" + }, + "papermill": { + "duration": 0.020317, + "end_time": "2024-03-01T17:33:51.376521", + "exception": false, + "start_time": "2024-03-01T17:33:51.356204", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"lct_gan\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/lct_gan/42\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011185, + "end_time": "2024-03-01T17:33:51.398967", + "exception": false, + "start_time": "2024-03-01T17:33:51.387782", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.423542Z", + "iopub.status.busy": "2024-03-01T17:33:51.422709Z", + "iopub.status.idle": "2024-03-01T17:33:51.432194Z", + "shell.execute_reply": "2024-03-01T17:33:51.431373Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023801, + "end_time": "2024-03-01T17:33:51.434089", + "exception": false, + "start_time": "2024-03-01T17:33:51.410288", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/lct_gan/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:51.459302Z", + "iopub.status.busy": "2024-03-01T17:33:51.458490Z", + "iopub.status.idle": "2024-03-01T17:33:53.726321Z", + "shell.execute_reply": "2024-03-01T17:33:53.725234Z" + }, + "papermill": { + "duration": 2.282713, + "end_time": "2024-03-01T17:33:53.728526", + "exception": false, + "start_time": "2024-03-01T17:33:51.445813", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:53.754971Z", + "iopub.status.busy": "2024-03-01T17:33:53.754479Z", + "iopub.status.idle": "2024-03-01T17:33:53.776671Z", + "shell.execute_reply": "2024-03-01T17:33:53.775859Z" + }, + "papermill": { + "duration": 0.037805, + "end_time": "2024-03-01T17:33:53.778843", + "exception": false, + "start_time": "2024-03-01T17:33:53.741038", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:53.803521Z", + "iopub.status.busy": "2024-03-01T17:33:53.803216Z", + "iopub.status.idle": "2024-03-01T17:33:53.813251Z", + "shell.execute_reply": "2024-03-01T17:33:53.812339Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024856, + "end_time": "2024-03-01T17:33:53.815377", + "exception": false, + "start_time": "2024-03-01T17:33:53.790521", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:53.841041Z", + "iopub.status.busy": "2024-03-01T17:33:53.840728Z", + "iopub.status.idle": "2024-03-01T17:33:54.332643Z", + "shell.execute_reply": "2024-03-01T17:33:54.331666Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.507699, + "end_time": "2024-03-01T17:33:54.335205", + "exception": false, + "start_time": "2024-03-01T17:33:53.827506", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:33:54.361276Z", + "iopub.status.busy": "2024-03-01T17:33:54.360933Z", + "iopub.status.idle": "2024-03-01T17:34:07.532200Z", + "shell.execute_reply": "2024-03-01T17:34:07.531383Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 13.187087, + "end_time": "2024-03-01T17:34:07.534752", + "exception": false, + "start_time": "2024-03-01T17:33:54.347665", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-01 17:33:58.910176: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-01 17:33:58.910280: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-01 17:33:59.040427: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:07.562056Z", + "iopub.status.busy": "2024-03-01T17:34:07.561386Z", + "iopub.status.idle": "2024-03-01T17:34:07.567813Z", + "shell.execute_reply": "2024-03-01T17:34:07.566891Z" + }, + "papermill": { + "duration": 0.022295, + "end_time": "2024-03-01T17:34:07.569876", + "exception": false, + "start_time": "2024-03-01T17:34:07.547581", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:07.595088Z", + "iopub.status.busy": "2024-03-01T17:34:07.594396Z", + "iopub.status.idle": "2024-03-01T17:34:18.381557Z", + "shell.execute_reply": "2024-03-01T17:34:18.380498Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 10.802711, + "end_time": "2024-03-01T17:34:18.384075", + "exception": false, + "start_time": "2024-03-01T17:34:07.581364", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:19.226672Z", + "iopub.status.busy": "2024-03-01T17:34:19.226306Z", + "iopub.status.idle": "2024-03-01T17:34:19.310143Z", + "shell.execute_reply": "2024-03-01T17:34:19.309075Z" + }, + "papermill": { + "duration": 0.099856, + "end_time": "2024-03-01T17:34:19.312320", + "exception": false, + "start_time": "2024-03-01T17:34:19.212464", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-01T17:34:19.341974Z", + "iopub.status.busy": "2024-03-01T17:34:19.341660Z", + "iopub.status.idle": "2024-03-01T17:34:19.821796Z", + "shell.execute_reply": "2024-03-01T17:34:19.820808Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.497925, + "end_time": "2024-03-01T17:34:19.824022", + "exception": false, + "start_time": "2024-03-01T17:34:19.326097", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:19.856179Z", + "iopub.status.busy": "2024-03-01T17:34:19.855358Z", + "iopub.status.idle": "2024-03-01T17:34:19.860242Z", + "shell.execute_reply": "2024-03-01T17:34:19.859353Z" + }, + "papermill": { + "duration": 0.02323, + "end_time": "2024-03-01T17:34:19.862233", + "exception": false, + "start_time": "2024-03-01T17:34:19.839003", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:19.889324Z", + "iopub.status.busy": "2024-03-01T17:34:19.889009Z", + "iopub.status.idle": "2024-03-01T17:34:19.896429Z", + "shell.execute_reply": "2024-03-01T17:34:19.895570Z" + }, + "papermill": { + "duration": 0.023469, + "end_time": "2024-03-01T17:34:19.898358", + "exception": false, + "start_time": "2024-03-01T17:34:19.874889", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10264072" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:19.925652Z", + "iopub.status.busy": "2024-03-01T17:34:19.925301Z", + "iopub.status.idle": "2024-03-01T17:34:20.022147Z", + "shell.execute_reply": "2024-03-01T17:34:20.021100Z" + }, + "papermill": { + "duration": 0.11297, + "end_time": "2024-03-01T17:34:20.024254", + "exception": false, + "start_time": "2024-03-01T17:34:19.911284", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 40] --\n", + "├─Adapter: 1-1 [2, 1179, 40] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 41,984\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 40] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,264,072\n", + "Trainable params: 10,264,072\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 39.93\n", + "========================================================================================================================\n", + "Input size (MB): 0.47\n", + "Forward/backward pass size (MB): 341.77\n", + "Params size (MB): 41.06\n", + "Estimated Total Size (MB): 383.29\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T17:34:20.055516Z", + "iopub.status.busy": "2024-03-01T17:34:20.054925Z", + "iopub.status.idle": "2024-03-01T18:38:46.681581Z", + "shell.execute_reply": "2024-03-01T18:38:46.680586Z" + }, + "papermill": { + "duration": 3866.645444, + "end_time": "2024-03-01T18:38:46.683970", + "exception": false, + "start_time": "2024-03-01T17:34:20.038526", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01786272754015954, 'avg_role_model_std_loss': 3.636569730948348, 'avg_role_model_mean_pred_loss': 0.001167070859791755, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01786272754015954, 'n_size': 320, 'n_batch': 160, 'duration': 162.09109449386597, 'duration_batch': 1.0130693405866622, 'duration_size': 0.5065346702933311, 'avg_pred_std': 0.07982906211551608}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036570764575117208, 'avg_role_model_std_loss': 8.543643573567897, 'avg_role_model_mean_pred_loss': 1.3811252661610762e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036570764575117208, 'n_size': 80, 'n_batch': 40, 'duration': 38.24689245223999, 'duration_batch': 0.9561723113059998, 'duration_size': 0.4780861556529999, 'avg_pred_std': 0.013961730610299128}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005696987385198327, 'avg_role_model_std_loss': 3.198073918344037, 'avg_role_model_mean_pred_loss': 6.659749155254468e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005696987385198327, 'n_size': 320, 'n_batch': 160, 'duration': 161.77688694000244, 'duration_batch': 1.0111055433750153, 'duration_size': 0.5055527716875077, 'avg_pred_std': 0.06436048086907249}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003786977470736019, 'avg_role_model_std_loss': 3.9373093709834395, 'avg_role_model_mean_pred_loss': 1.9986069321897836e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003786977470736019, 'n_size': 80, 'n_batch': 40, 'duration': 38.12403869628906, 'duration_batch': 0.9531009674072266, 'duration_size': 0.4765504837036133, 'avg_pred_std': 0.019040833081817254}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0032535013802828415, 'avg_role_model_std_loss': 2.8198290220143503, 'avg_role_model_mean_pred_loss': 1.6548445633081822e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032535013802828415, 'n_size': 320, 'n_batch': 160, 'duration': 158.56593680381775, 'duration_batch': 0.9910371050238609, 'duration_size': 0.49551855251193044, 'avg_pred_std': 0.05899094843493913}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002633672622323502, 'avg_role_model_std_loss': 4.367062080342106, 'avg_role_model_mean_pred_loss': 6.056017390343449e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002633672622323502, 'n_size': 80, 'n_batch': 40, 'duration': 36.21775007247925, 'duration_batch': 0.9054437518119812, 'duration_size': 0.4527218759059906, 'avg_pred_std': 0.029313163098959195}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023438148911395728, 'avg_role_model_std_loss': 2.2918122927125295, 'avg_role_model_mean_pred_loss': 7.134696727795209e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023438148911395728, 'n_size': 320, 'n_batch': 160, 'duration': 152.8709909915924, 'duration_batch': 0.9554436936974525, 'duration_size': 0.47772184684872626, 'avg_pred_std': 0.06780749239678699}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002467950962409304, 'avg_role_model_std_loss': 4.88692410795129, 'avg_role_model_mean_pred_loss': 4.36867752655612e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002467950962409304, 'n_size': 80, 'n_batch': 40, 'duration': 35.697052240371704, 'duration_batch': 0.8924263060092926, 'duration_size': 0.4462131530046463, 'avg_pred_std': 0.018721673299660326}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002259014635501444, 'avg_role_model_std_loss': 1.812988119399502, 'avg_role_model_mean_pred_loss': 5.5367047818208335e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002259014635501444, 'n_size': 320, 'n_batch': 160, 'duration': 151.4134497642517, 'duration_batch': 0.9463340610265731, 'duration_size': 0.47316703051328657, 'avg_pred_std': 0.06716944240579323}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023299435670196544, 'avg_role_model_std_loss': 7.427999823173169, 'avg_role_model_mean_pred_loss': 6.818181761379086e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023299435670196544, 'n_size': 80, 'n_batch': 40, 'duration': 36.091819047927856, 'duration_batch': 0.9022954761981964, 'duration_size': 0.4511477380990982, 'avg_pred_std': 0.025550102235138185}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002213509789987711, 'avg_role_model_std_loss': 2.1981050524016665, 'avg_role_model_mean_pred_loss': 6.457744553583708e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002213509789987711, 'n_size': 320, 'n_batch': 160, 'duration': 155.00266909599304, 'duration_batch': 0.9687666818499565, 'duration_size': 0.48438334092497826, 'avg_pred_std': 0.06307913716664189}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025578231319741463, 'avg_role_model_std_loss': 5.699148117736866, 'avg_role_model_mean_pred_loss': 7.032264796913435e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025578231319741463, 'n_size': 80, 'n_batch': 40, 'duration': 35.88962531089783, 'duration_batch': 0.8972406327724457, 'duration_size': 0.44862031638622285, 'avg_pred_std': 0.017301402381235675}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002113006258792893, 'avg_role_model_std_loss': 2.1890728188584476, 'avg_role_model_mean_pred_loss': 6.114715563168734e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002113006258792893, 'n_size': 320, 'n_batch': 160, 'duration': 153.502343416214, 'duration_batch': 0.9593896463513374, 'duration_size': 0.4796948231756687, 'avg_pred_std': 0.0661177773316524}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022539663767020103, 'avg_role_model_std_loss': 8.082782875575992, 'avg_role_model_mean_pred_loss': 6.678637441320801e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022539663767020103, 'n_size': 80, 'n_batch': 40, 'duration': 35.9239776134491, 'duration_batch': 0.8980994403362275, 'duration_size': 0.44904972016811373, 'avg_pred_std': 0.025397659230247883}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018896224307241027, 'avg_role_model_std_loss': 1.3406870565765918, 'avg_role_model_mean_pred_loss': 4.11630951652614e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018896224307241027, 'n_size': 320, 'n_batch': 160, 'duration': 151.49836015701294, 'duration_batch': 0.9468647509813308, 'duration_size': 0.4734323754906654, 'avg_pred_std': 0.07347480687003553}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025110580976615894, 'avg_role_model_std_loss': 5.483385282401798, 'avg_role_model_mean_pred_loss': 4.007785178927748e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025110580976615894, 'n_size': 80, 'n_batch': 40, 'duration': 33.77740168571472, 'duration_batch': 0.844435042142868, 'duration_size': 0.422217521071434, 'avg_pred_std': 0.01343663605657639}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019506988204057052, 'avg_role_model_std_loss': 1.1689463564448361, 'avg_role_model_mean_pred_loss': 4.559629974733976e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019506988204057052, 'n_size': 320, 'n_batch': 160, 'duration': 144.0004370212555, 'duration_batch': 0.9000027313828468, 'duration_size': 0.4500013656914234, 'avg_pred_std': 0.0659007933063549}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023398275739964446, 'avg_role_model_std_loss': 4.280078241747558, 'avg_role_model_mean_pred_loss': 9.40580381603908e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023398275739964446, 'n_size': 80, 'n_batch': 40, 'duration': 34.14411234855652, 'duration_batch': 0.8536028087139129, 'duration_size': 0.42680140435695646, 'avg_pred_std': 0.029161113313784882}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016659495725662055, 'avg_role_model_std_loss': 1.259914455004442, 'avg_role_model_mean_pred_loss': 3.6357496224758216e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016659495725662055, 'n_size': 320, 'n_batch': 160, 'duration': 146.88511276245117, 'duration_batch': 0.9180319547653198, 'duration_size': 0.4590159773826599, 'avg_pred_std': 0.06927311308209028}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020320220184657954, 'avg_role_model_std_loss': 1.9301895227664638, 'avg_role_model_mean_pred_loss': 5.797358790626817e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020320220184657954, 'n_size': 80, 'n_batch': 40, 'duration': 33.82334542274475, 'duration_batch': 0.8455836355686188, 'duration_size': 0.4227918177843094, 'avg_pred_std': 0.031894830953388006}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016075492954826132, 'avg_role_model_std_loss': 1.0092837510064634, 'avg_role_model_mean_pred_loss': 2.735296125088143e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016075492954826132, 'n_size': 320, 'n_batch': 160, 'duration': 143.3337082862854, 'duration_batch': 0.8958356767892838, 'duration_size': 0.4479178383946419, 'avg_pred_std': 0.07756731488425431}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026302734650016646, 'avg_role_model_std_loss': 3.5820208163912866, 'avg_role_model_mean_pred_loss': 1.3108698285635434e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026302734650016646, 'n_size': 80, 'n_batch': 40, 'duration': 33.34847378730774, 'duration_batch': 0.8337118446826934, 'duration_size': 0.4168559223413467, 'avg_pred_std': 0.026611345619312488}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013905711543884536, 'avg_role_model_std_loss': 1.4293391600536105, 'avg_role_model_mean_pred_loss': 1.9943211839244628e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013905711543884536, 'n_size': 320, 'n_batch': 160, 'duration': 145.80800819396973, 'duration_batch': 0.9113000512123108, 'duration_size': 0.4556500256061554, 'avg_pred_std': 0.07521452093462813}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022007888529515184, 'avg_role_model_std_loss': 4.483247433313909, 'avg_role_model_mean_pred_loss': 7.529547302986828e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022007888529515184, 'n_size': 80, 'n_batch': 40, 'duration': 34.17486619949341, 'duration_batch': 0.8543716549873352, 'duration_size': 0.4271858274936676, 'avg_pred_std': 0.025888706676232685}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012333092558833413, 'avg_role_model_std_loss': 1.0602984449560355, 'avg_role_model_mean_pred_loss': 1.4978945140242672e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012333092558833413, 'n_size': 320, 'n_batch': 160, 'duration': 144.26001048088074, 'duration_batch': 0.9016250655055046, 'duration_size': 0.4508125327527523, 'avg_pred_std': 0.06890693796813138}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002605481748287275, 'avg_role_model_std_loss': 5.8101766740395275, 'avg_role_model_mean_pred_loss': 1.2162480100338935e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002605481748287275, 'n_size': 80, 'n_batch': 40, 'duration': 34.0310595035553, 'duration_batch': 0.8507764875888825, 'duration_size': 0.42538824379444123, 'avg_pred_std': 0.030725552017582914}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011501418213924809, 'avg_role_model_std_loss': 1.3386667872641667, 'avg_role_model_mean_pred_loss': 1.3185595447717802e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011501418213924809, 'n_size': 320, 'n_batch': 160, 'duration': 144.17591977119446, 'duration_batch': 0.9010994985699654, 'duration_size': 0.4505497492849827, 'avg_pred_std': 0.07558018262188852}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022788071105424024, 'avg_role_model_std_loss': 2.7987204812981075, 'avg_role_model_mean_pred_loss': 7.67833690556996e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022788071105424024, 'n_size': 80, 'n_batch': 40, 'duration': 34.57782983779907, 'duration_batch': 0.8644457459449768, 'duration_size': 0.4322228729724884, 'avg_pred_std': 0.028933984229661293}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010513833095672, 'avg_role_model_std_loss': 0.6387672975999288, 'avg_role_model_mean_pred_loss': 1.4141241849948554e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010513833095672, 'n_size': 320, 'n_batch': 160, 'duration': 145.27083349227905, 'duration_batch': 0.907942709326744, 'duration_size': 0.453971354663372, 'avg_pred_std': 0.07942418371021631}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023162082296039445, 'avg_role_model_std_loss': 3.349859969510388, 'avg_role_model_mean_pred_loss': 8.629114276192951e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023162082296039445, 'n_size': 80, 'n_batch': 40, 'duration': 33.332454442977905, 'duration_batch': 0.8333113610744476, 'duration_size': 0.4166556805372238, 'avg_pred_std': 0.03227444588264916}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009490251221507151, 'avg_role_model_std_loss': 0.6831669182588915, 'avg_role_model_mean_pred_loss': 1.0006828890851693e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009490251221507151, 'n_size': 320, 'n_batch': 160, 'duration': 145.07566261291504, 'duration_batch': 0.906722891330719, 'duration_size': 0.4533614456653595, 'avg_pred_std': 0.08901288353954442}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002519568522711779, 'avg_role_model_std_loss': 3.413597872712374, 'avg_role_model_mean_pred_loss': 1.3267886331692902e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002519568522711779, 'n_size': 80, 'n_batch': 40, 'duration': 35.61937355995178, 'duration_batch': 0.8904843389987945, 'duration_size': 0.44524216949939727, 'avg_pred_std': 0.03244233850882665}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008506465366181715, 'avg_role_model_std_loss': 0.6938899818249922, 'avg_role_model_mean_pred_loss': 7.591354541367547e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008506465366181715, 'n_size': 320, 'n_batch': 160, 'duration': 141.91219687461853, 'duration_batch': 0.8869512304663658, 'duration_size': 0.4434756152331829, 'avg_pred_std': 0.07522430039280152}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022380605670150543, 'avg_role_model_std_loss': 2.3211301649584923, 'avg_role_model_mean_pred_loss': 8.706937260289163e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022380605670150543, 'n_size': 80, 'n_batch': 40, 'duration': 33.153270959854126, 'duration_batch': 0.8288317739963531, 'duration_size': 0.41441588699817655, 'avg_pred_std': 0.033504872786579654}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007984611616791426, 'avg_role_model_std_loss': 0.8550257516943558, 'avg_role_model_mean_pred_loss': 6.954888952128042e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007984611616791426, 'n_size': 320, 'n_batch': 160, 'duration': 141.78189277648926, 'duration_batch': 0.8861368298530579, 'duration_size': 0.44306841492652893, 'avg_pred_std': 0.07633930989904911}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023599293230745387, 'avg_role_model_std_loss': 2.454739641254777, 'avg_role_model_mean_pred_loss': 1.1268823242671644e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023599293230745387, 'n_size': 80, 'n_batch': 40, 'duration': 33.33975076675415, 'duration_batch': 0.8334937691688538, 'duration_size': 0.4167468845844269, 'avg_pred_std': 0.03295552866329672}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007362112768333872, 'avg_role_model_std_loss': 1.1732242802433412, 'avg_role_model_mean_pred_loss': 4.6208515920671473e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007362112768333872, 'n_size': 320, 'n_batch': 160, 'duration': 142.94105648994446, 'duration_batch': 0.8933816030621529, 'duration_size': 0.44669080153107643, 'avg_pred_std': 0.07887016502173765}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022324465072415477, 'avg_role_model_std_loss': 2.106721719149411, 'avg_role_model_mean_pred_loss': 9.551615076732606e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022324465072415477, 'n_size': 80, 'n_batch': 40, 'duration': 33.48200488090515, 'duration_batch': 0.8370501220226287, 'duration_size': 0.41852506101131437, 'avg_pred_std': 0.031034307027584872}\n", + "Epoch 19\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006516361492984402, 'avg_role_model_std_loss': 0.5988772722470912, 'avg_role_model_mean_pred_loss': 3.690359887095372e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006516361492984402, 'n_size': 320, 'n_batch': 160, 'duration': 142.92735695838928, 'duration_batch': 0.893295980989933, 'duration_size': 0.4466479904949665, 'avg_pred_std': 0.07815303717216011}\n", + "Time out: 3692.50643992424/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▇█▃▃▂▃▂▃▂▁▃▂▃▂▂▃▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▃▇▃▅▂▅▁▆▇▆▅▇▆████▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▆▂▁▃▃▂▃▄▃▃▅▅▃▅▆█▅▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▇█▃▃▂▃▂▃▂▁▃▂▃▂▂▃▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▅█▂▁▂▂▂▁▃▂▅▃▅▃▃▅▃▄▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▃▄▄▇▅█▅▃▁▃▄▅▂▃▃▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▇▆▅▄▅▅▃▂▂▂▃▂▃▁▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ██▅▄▅▅▅▂▂▂▁▂▂▃▁▄▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ██▇▅▄▆▅▄▂▃▂▂▂▂▂▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ██▅▄▅▅▅▂▂▂▁▂▂▃▁▄▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ██▇▅▄▆▅▄▂▃▂▂▂▂▂▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ██▅▄▅▅▅▂▂▂▁▂▂▃▁▄▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ██▇▅▄▆▅▄▂▃▂▂▂▂▂▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00223\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00074\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03103\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07887\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00223\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00074\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 2.10672\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 1.17322\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.83705\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.89338\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.41853\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.44669\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 33.482\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 142.94106\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/lct_gan/42/wandb/offline-run-20240301_173421-7wshd8vd\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240301_173421-7wshd8vd/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0012717791214527097, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.012063155442658336, 'pred_duration': 3.151550769805908, 'grad_duration': 2.571305990219116, 'total_duration': 5.722856760025024, 'pred_std': 0.05481971800327301, 'std_loss': 0.022213952615857124, 'mean_pred_loss': 1.6693775251042098e-06, 'pred_rmse': 0.03566201403737068, 'pred_mae': 0.0282240342348814, 'pred_mape': 0.06742086261510849, 'grad_rmse': 0.0409364178776741, 'grad_mae': 0.031053612008690834, 'grad_mape': 0.5863480567932129}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0012717791214527097, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.012063155442658336, 'avg_pred_duration': 3.151550769805908, 'avg_grad_duration': 2.571305990219116, 'avg_total_duration': 5.722856760025024, 'avg_pred_std': 0.05481971800327301, 'avg_std_loss': 0.022213952615857124, 'avg_mean_pred_loss': 1.6693775251042098e-06}, 'min_metrics': {'avg_loss': 0.0012717791214527097, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.012063155442658336, 'pred_duration': 3.151550769805908, 'grad_duration': 2.571305990219116, 'total_duration': 5.722856760025024, 'pred_std': 0.05481971800327301, 'std_loss': 0.022213952615857124, 'mean_pred_loss': 1.6693775251042098e-06, 'pred_rmse': 0.03566201403737068, 'pred_mae': 0.0282240342348814, 'pred_mape': 0.06742086261510849, 'grad_rmse': 0.0409364178776741, 'grad_mae': 0.031053612008690834, 'grad_mape': 0.5863480567932129}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0012717791214527097, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.012063155442658336, 'pred_duration': 3.151550769805908, 'grad_duration': 2.571305990219116, 'total_duration': 5.722856760025024, 'pred_std': 0.05481971800327301, 'std_loss': 0.022213952615857124, 'mean_pred_loss': 1.6693775251042098e-06, 'pred_rmse': 0.03566201403737068, 'pred_mae': 0.0282240342348814, 'pred_mape': 0.06742086261510849, 'grad_rmse': 0.0409364178776741, 'grad_mae': 0.031053612008690834, 'grad_mape': 0.5863480567932129}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:38:46.723513Z", + "iopub.status.busy": "2024-03-01T18:38:46.722723Z", + "iopub.status.idle": "2024-03-01T18:38:46.727621Z", + "shell.execute_reply": "2024-03-01T18:38:46.726667Z" + }, + "papermill": { + "duration": 0.026904, + "end_time": "2024-03-01T18:38:46.729589", + "exception": false, + "start_time": "2024-03-01T18:38:46.702685", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:38:46.765328Z", + "iopub.status.busy": "2024-03-01T18:38:46.765086Z", + "iopub.status.idle": "2024-03-01T18:38:47.049878Z", + "shell.execute_reply": "2024-03-01T18:38:47.049045Z" + }, + "papermill": { + "duration": 0.305531, + "end_time": "2024-03-01T18:38:47.052346", + "exception": false, + "start_time": "2024-03-01T18:38:46.746815", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:38:47.090440Z", + "iopub.status.busy": "2024-03-01T18:38:47.089768Z", + "iopub.status.idle": "2024-03-01T18:38:47.427766Z", + "shell.execute_reply": "2024-03-01T18:38:47.426791Z" + }, + "papermill": { + "duration": 0.35919, + "end_time": "2024-03-01T18:38:47.429837", + "exception": false, + "start_time": "2024-03-01T18:38:47.070647", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAESCAYAAAB3gfmJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHrElEQVR4nO3de1gTZ9o/8G8SSML5KAlQBLQiHvBQLCmu1Vp5Rctqcd3Verlqfa26u9jaUqu1q1K7+xOr7VvfWntwW7V9u4q6tW6rri1F3W4VURE8odYDiqeAiJwhgeT+/TFmJCUgCYEA3p/rmotk5plnngnk5pmZ5yAhIgJjjHUhUkcXgDHG7I0DG2Osy+HAxhjrcjiwMca6HA5sjLEuhwMbY6zL4cDGGOtynBxdgI7EaDTi5s2b8PDwgEQicXRxGGMNEBEqKioQFBQEqbT5OhkHtgZu3ryJkJAQRxeDMdaMa9eu4ZFHHmk2DQe2Bjw8PAAIH5ynp6eDS8MYa6i8vBwhISHi97Q5HNgaMF1+enp6cmBjrINqyW0ifnjAGOtyOLAxxrocDmyMsS6H77ExuzAajdDr9Y4uBuvknJ2dIZPJWp0PBzbWanq9Hvn5+TAajY4uCusCvL29oVarW9WWlAMbaxUiwq1btyCTyRASEvLAhpOMNYWIUF1djaKiIgBAYGCgzXlxYLPBjdIa5N+uQoCnAhGqB7ep6crq6+tRXV2NoKAguLq6Oro4rJNzcXEBABQVFSEgIMDmy1L+92qDLw9fxe8/y8KWIwWOLorDGQwGAIBcLndwSVhXYfoHWVdXZ3MeNgW2devWISwsDEqlEhqNBkeOHGk2/fbt2xEZGQmlUomoqCjs2bPHbPuOHTswevRo+Pn5QSKRIDc312z7lStXIJFILC7bt28X01nanpaWZsspNsvXVfgS363im+Um3LeW2Ys9/pasDmxbt25FcnIyUlJScPz4cQwcOBDx8fHidfEvHTp0CFOmTMGsWbOQk5ODxMREJCYm4vTp02KaqqoqDBs2DG+//bbFPEJCQnDr1i2zZfny5XB3d8fYsWPN0m7cuNEsXWJiorWn+EA+bkJgK6m2/T8KY6wNkZViYmIoKSlJfG8wGCgoKIhSU1Mtpp80aRIlJCSYrdNoNDR37txGafPz8wkA5eTkPLAcgwYNov/+7/82WweAvv766wefRBPKysoIAJWVlTWbbt/ZQgpdtIt+/f5/bD5WV1FTU0N5eXlUU1Pj6KKwLqKpv6mWfj+JiKyqsen1emRnZyMuLk5cJ5VKERcXh8zMTIv7ZGZmmqUHgPj4+CbTt0R2djZyc3Mxa9asRtuSkpLg7++PmJgYbNiwAdTM7II6nQ7l5eVmS0uINTa+FGWtIJFIsHPnTkcXw67efPNNDBo0yNHFsO5StLi4GAaDASqVymy9SqWCVqu1uI9Wq7UqfUt89tln6NOnD4YOHWq2/q233sK2bduQnp6OiRMn4k9/+hPWrl3bZD6pqanw8vISl5YOWSTeY6vmwMY6t02bNsHb29tu+S1YsAAZGRl2y89Wna65R01NDTZv3oylS5c22tZw3eDBg1FVVYXVq1fjpZdespjX4sWLkZycLL43DYvyID5uzgCAar0BtXUGKJ1b31KasY5Mr9e36Mm3u7s73N3d26FEzbOqxubv7w+ZTIbCwkKz9YWFhVCr1Rb3UavVVqV/kH/84x+orq7G9OnTH5hWo9Hg+vXr0Ol0FrcrFApxiCJrhipyVzjBWSY8ueHLUXNEhGp9vUOW5m47WLJ3714MGzYM3t7e8PPzw69//WtcunQJADB06FAsWrTILP3t27fh7OyMH3/8EQBw69YtJCQkwMXFBeHh4di8eTPCwsKwZs0amz67U6dO4emnn4aLiwv8/PwwZ84cVFZWitsPHDiAmJgYuLm5wdvbG7/61a9w9epVAMCJEycwcuRIeHh4wNPTE9HR0Th27Fizxztw4ABmzpyJsrIysRXBm2++CQAICwvDX/7yF0yfPh2enp6YM2cOAGDRokWIiIiAq6srevTogaVLl5o1y/jlpejzzz+PxMREvPPOOwgMDISfnx+SkpJa1ZSjJayqscnlckRHRyMjI0N82mg0GpGRkYF58+ZZ3Cc2NhYZGRl4+eWXxXXp6emIjY21qcCfffYZxo8fj27duj0wbW5uLnx8fKBQKGw6VlMkEgl8XOUoqtChpEqPIG8Xu+bfmdXUGdB32XcOOXbeW/Fwlbf8T7qqqgrJyckYMGAAKisrsWzZMkyYMAG5ubmYOnUqVq1ahZUrV4rND7Zu3YqgoCA8+eSTAIDp06ejuLgYBw4cgLOzM5KTk5tsHdCSssTHxyM2NhZHjx5FUVERXnjhBcybNw+bNm1CfX09EhMTMXv2bGzZsgV6vR5HjhwRyzZ16lQMHjwYH330EWQyGXJzc+Hs7NzsMYcOHYo1a9Zg2bJlOH/+PACY1bbeeecdLFu2DCkpKeI6Dw8PbNq0CUFBQTh16hRmz54NDw8PLFy4sMnj7N+/H4GBgdi/fz8uXryIyZMnY9CgQZg9e7ZNn1VLWH0pmpycjBkzZmDIkCGIiYnBmjVrUFVVhZkzZwIQftnBwcFITU0FAMyfPx8jRozAu+++i4SEBKSlpeHYsWNYv369mGdJSQkKCgpw8+ZNABA/ZLVabVazu3jxIn788cdG7eAA4Ntvv0VhYSGeeOIJKJVKpKenY8WKFViwYIG1p9givm5CYOP7bJ3XxIkTzd5v2LAB3bp1Q15eHiZNmoSXX34ZP/30kxjINm/ejClTpkAikeDcuXP44YcfcPToUQwZMgQA8Omnn6JXr142lWXz5s2ora3FF198ATc3NwDABx98gHHjxuHtt9+Gs7MzysrK8Otf/xo9e/YEAPTp00fcv6CgAK+99hoiIyMBoEXlkMvl8PLygkQisXgF9fTTT+PVV181W7dkyRLxdVhYGBYsWIC0tLRmA5uPjw8++OADyGQyREZGIiEhARkZGR0rsE2ePBm3b9/GsmXLoNVqMWjQIOzdu1d8QFBQUGDWX3Do0KHYvHkzlixZgjfeeAO9evXCzp070b9/fzHNN998IwZGAHjuuecAACkpKWLVGBD+8B555BGMHj26UbmcnZ2xbt06vPLKKyAiPProo/if//mfNvvwfPnJqEUuzjLkvRXvsGNb48KFC1i2bBmysrJQXFwsduIvKChA//79MXr0aPz973/Hk08+ifz8fGRmZuKTTz4BIPzzdXJywmOPPSbm9+ijj8LHx8emsp89exYDBw4UgxoA/OpXv4LRaMT58+cxfPhwPP/884iPj8d//dd/IS4uDpMmTRL7UyYnJ+OFF17A//3f/yEuLg6/+93vxABoK1PAbmjr1q14//33cenSJVRWVqK+vv6Bt3D69etn1jUqMDAQp06dalXZHsSmngfz5s3D1atXodPpkJWVBY1GI247cOAANm3aZJb+d7/7Hc6fPw+dTofTp0/jmWeeMdv+/PPPg4gaLQ2DGgCsWLGiUeA0GTNmDHJyclBRUYHKykrk5uZi7ty5bdYp29Tkg3sfmJNIJHCVOzlksbbF+rhx41BSUoK//e1vyMrKQlZWFgCIwy9NnToV//jHP1BXV4fNmzcjKioKUVFRdv/MWmrjxo3IzMzE0KFDsXXrVkRERODw4cMAhHtbZ86cQUJCAvbt24e+ffvi66+/btXxGgZZQGi6NXXqVDzzzDPYtWsXcnJy8Oc///mBw1X98pJYIpG0+Ugw3FfURqYmH9z7oHO6c+cOzp8/jyVLlmDUqFHo06cP7t69a5bm2WefRW1tLfbu3YvNmzdj6tSp4rbevXujvr4eOTk54rqLFy82yqOl+vTpgxMnTqCqqkpcd/DgQUilUvTu3VtcN3jwYCxevBiHDh1C//79sXnzZnFbREQEXnnlFXz//ff4zW9+g40bNz7wuHK5XOzv+yCHDh1CaGgo/vznP2PIkCHo1auX+PCio+HAZqP7jXQtP3FlHZuPjw/8/Pywfv16XLx4Efv27TNr+gMINZbExEQsXboUZ8+exZQpU8RtkZGRiIuLw5w5c3DkyBHk5ORgzpw5cHFxsamv49SpU6FUKjFjxgycPn0a+/fvx4svvohp06ZBpVIhPz8fixcvRmZmJq5evYrvv/8eFy5cQJ8+fVBTU4N58+bhwIEDuHr1Kg4ePIijR4+a3YNrSlhYGCorK5GRkYHi4mJUV1c3mbZXr14oKChAWloaLl26hPfff7/VtcK2woHNRr6uQvX6bhXX2DojqVSKtLQ0ZGdno3///njllVewevXqRummTp2KEydO4Mknn0T37t3Ntn3xxRdQqVQYPnw4JkyYID4hVCqVVpfH1dUV3333HUpKSvD444/jt7/9LUaNGoUPPvhA3H7u3DlMnDgRERERmDNnDpKSkjB37lzIZDLcuXMH06dPR0REBCZNmoSxY8di+fLlDzzu0KFD8Yc//AGTJ09Gt27dsGrVqibTjh8/Hq+88grmzZuHQYMG4dChQxbbk3YEErK28U8XVl5eDi8vL5SVlT3whug/c29gflouYnv4YcucJ9qphB1PbW0t8vPzER4ebtMXuiu5fv06QkJC8MMPP2DUqFGOLk6n1dTflDXfz07X86Cj8HMT2sZxc4+H1759+1BZWYmoqCjcunULCxcuRFhYGIYPH+7ooj30+FLURqZuVdzc4+FVV1eHN954A/369cOECRPQrVs3sbHu3//+d7F70S+Xfv36tVsZx44d22Q5VqxY0W7laG9cY7ORqR3b3Wo9iIgHWnwIxcfHIz7ecpu98ePHmzWDauhBPQLs6dNPP0VNTY3Fbb6+vu1WjvbGgc1GPveae9QZCJW6engo2++PlXV8Hh4e8PBw/HwYwcHBji6CQ/ClqI2UzjK4yoXW1Hw5yljHwoGtFUy1Ng5sjHUsHNhaoeF9NsZYx8GBrRXud4TnRrqMdSQc2FrBlzvCM9YhcWBrBfEeG1+KMht0xclcOgoObK3ga2qkW8mBjXVO9p7MBRCGLpNIJCgtLbVrvtbgwNYK9ydO5sDGWEfCga0VxGn4+B7bfUSAvsoxC0/mYrfJXHQ6HRYsWIDg4GC4ublBo9HgwIED4r5Xr17FuHHj4OPjAzc3N/Tr1w979uzBlStXMHLkSADC0FASiQTPP/+8TZ9Ha3DPg1bw5RpbY3XVwIogxxz7jZuA3O3B6e7hyVyansxl3rx5yMvLQ1paGoKCgvD1119jzJgxOHXqFHr16oWkpCTo9Xr8+OOPcHNzQ15eHtzd3RESEoKvvvoKEydOxPnz5+Hp6QkXl/af7IgDWyvwU9HOjSdzsTyZS0FBATZu3IiCggIEBQn/pBYsWIC9e/di48aN4hD9EydOFIdK79Gjh7i/qQ9qQECA3e/ftRQHtlYw3WMrramDwUiQSbkjPJxdhZqTo45tBZ7MxbJTp07BYDAgIiLCbL1Op4Ofnx8A4KWXXsIf//hHfP/994iLi8PEiRMxYMAAm47XFmy6x7Zu3TqEhYVBqVRCo9HgyJEjzabfvn07IiMjoVQqERUV1Wj6vB07dmD06NHw8/ODRCJBbm5uozyeeuop8T6AafnDH/5glqagoAAJCQlwdXVFQEAAXnvtNdTX19tyii3i7SJU9YmAshpupAsAkEiEy0FHLDyZi10mc6msrIRMJkN2djZyc3PF5ezZs/jf//1fAMALL7yAy5cvY9q0aTh16hSGDBmCtWvX2u1cW42slJaWRnK5nDZs2EBnzpyh2bNnk7e3NxUWFlpMf/DgQZLJZLRq1SrKy8ujJUuWkLOzM506dUpM88UXX9Dy5cvpb3/7GwGgnJycRvmMGDGCZs+eTbdu3RKXsrIycXt9fT3179+f4uLiKCcnh/bs2UP+/v60ePHiFp9bWVkZATDL90EGvPkdhS7aRRcKy1u8T1dSU1NDeXl5VFNT4+iiWKW4uJgA0I8//iiu+89//kMA6OuvvyYiosrKSnJzc6NvvvmG+vbtSytXrhTTnj17lgDQsWPHxHUXLlwgAPTee++1qAwNj7V+/Xry8fGhyspKcfvu3btJKpWSVqu1uP8TTzxBL774osVtzz33HI0bN+6BZfj73/9O7u7uZuvOnz/f6LN5kNdff52ioqKISPjOA6Di4uIW799QU39T1nw/rQ5sMTExlJSUJL43GAwUFBREqampFtNPmjSJEhISzNZpNBqaO3duo7T5+fnNBrb58+c3Wa49e/Y0+iP46KOPyNPTk3Q63QPOSmBLYHtq9X4KXbSLsi7fafE+XUlnDWwGg4H8/Pzo97//PV24cIEyMjLo8ccfNws2RERTp06lgQMHkkQioatXr5rlERcXR4899hhlZWXR8ePHaeTIkeTi4kJr1qxpURkaHquqqooCAwNp4sSJdOrUKdq3bx/16NGDZsyYQUREly9fptdff50OHTpEV65coe+++478/Pzoww8/pOrqakpKSqL9+/fTlStX6KeffqKePXvSwoULH1gGUxD64Ycf6Pbt21RVVSWed1hYGH311Vd0+fJlysrKohUrVtCuXbuIiGj+/Pm0d+9eunz5MmVnZ5NGo6FJkyYREdH169dJIpHQpk2bqKioiCoqKlr0eZi0e2DT6XQkk8nMfvFERNOnT6fx48db3CckJKTRf7Bly5bRgAEDGqV9UGDz9/cnPz8/6tevH73++uviL4GIaOnSpTRw4ECzfS5fvkwA6Pjx4xbLVltbS2VlZeJy7do1qwPbhHU/UeiiXfSvU7davE9X0lkDGxFReno69enThxQKBQ0YMIAOHDjQKLDt2bOHANDw4cMb7X/z5k0aO3YsKRQKCg0Npc2bN1NAQAB9/PHHLTr+L4918uRJGjlyJCmVSvL19aXZs2eLQUGr1VJiYiIFBgaSXC6n0NBQWrZsGRkMBtLpdPTcc89RSEgIyeVyCgoKonnz5rX4d/KHP/yB/Pz8CAClpKQQEZFer6dly5ZRWFgYOTs7U2BgIE2YMIFOnjxJRETz5s2jnj17kkKhoG7dutG0adPMamhvvfUWqdVqkkgkYnBuqXYPbDdu3CAAdOjQIbP1r732GsXExFjcx9nZmTZv3my2bt26dRQQENAobXOB7ZNPPqG9e/fSyZMn6csvv6Tg4GCaMGGCuH327Nk0evRos32qqqoIAO3Zs8di2VJSUghAo8WawDZr01EKXbSLNmddfXDiLqgzBzZ7M/1j/OGHHxxdlE7NHoGt0zwVnTNnjvg6KioKgYGBGDVqFC5dumTz05/FixebzSVZXl6OkJAQq/Lw5bkPHlo8mUvHZdVTUX9/f8hkMhQWFpqtLywsNGsH05BarbYqfUuZxpO/ePFis8cxbbNEoVDA09PTbLGWD7dle2jxZC4dl1U1NrlcjujoaGRkZCAxMREAYDQakZGRgXnz5lncJzY2FhkZGXj55ZfFdenp6YiNjbW50ADEJiGmdjyxsbH4f//v/6GoqAgBAQHicTw9PdG3b99WHas5vjyK7kOLJ3PpuKy+FE1OTsaMGTMwZMgQxMTEYM2aNaiqqsLMmTMBCN1MgoODkZqaCgCYP38+RowYgXfffRcJCQlIS0vDsWPHsH79ejHPkpISFBQU4OZNoWGnqXuHWq2GWq3GpUuXsHnzZjzzzDPw8/PDyZMn8corr2D48OFio8DRo0ejb9++mDZtGlatWgWtVoslS5YgKSkJCoWidZ9SM7gjPLOEJ3NxMFtu7q1du5a6d+9OcrmcYmJi6PDhw+K2ESNGNHoKsm3bNoqIiCC5XE79+vWj3bt3m23fuHGjxZv4pic0BQUFNHz4cPL19SWFQkGPPvoovfbaa41uIl65coXGjh1LLi4u5O/vT6+++irV1dW1+Lxsae6RfkZLoYt20fi1/2nxPl2J6UZvdXW1o4vCuojq6upWPzyQEFk5JEIXVl5eDi8vL5SVlbX4flv21buY+NEhhPi64D8Ln27jEnY8dXV1uHjxIoKCguDl5eXo4rAu4M6dOygqKkJERARkMpm43prvZ6d5KtpR+YkPDx7OLlVOTk5wdXUVh/SRSnkkLGYbIkJ1dTWKiorg7e1tFtSsxYGtlUz32Cp19dDVG6Bwsv2X0RlJJBIEBgYiPz9fHBuMsdbw9vZudasJDmyt5Kl0gkwqgcFIKK2ug8rz4QpsgPC0vFevXmLnccZs5ezs3KqamgkHtlaSSCTwcZWjuFKHO5V6qDyVji6SQ0ilUiiVD+e5s46Hb4jYgan3AU+czFjHwIHNDny4kS5jHQoHNjvwc7/3ZJRrbIx1CBzY7IBrbIx1LBzY7IAndWGsY+HAZgdija364Wyky1hHw4HNDsT5Rat0Di4JYwzgwGYX4ggfD2m3KsY6Gg5sdmAak43vsTHWMXBgswNf9/tjsvFgKYw5Hgc2OzDV2PT1RlTrDQ4uDWOMA5sduMhlUDoLHyW3ZWPM8Tiw2QnPfcBYx8GBzU547gPGOg4ObHbCvQ8Y6zg4sNnJ/Ua6HNgYczSbAtu6desQFhYGpVIJjUaDI0eONJt++/btiIyMhFKpRFRUFPbs2WO2fceOHRg9ejT8/PwgkUjEOUNNSkpK8OKLL6J3795wcXFB9+7d8dJLL6GsrMwsnUQiabSkpaXZcopWM3Wr4hE+GHM8qwPb1q1bkZycjJSUFBw/fhwDBw5EfHw8ioqKLKY/dOgQpkyZglmzZiEnJweJiYlITEzE6dOnxTRVVVUYNmwY3n77bYt53Lx5Ezdv3sQ777yD06dPY9OmTdi7dy9mzZrVKO3GjRtx69YtcTFN7NzWfLn3AWMdh7Vz/sXExFBSUpL43mAwUFBQEKWmplpMP2nSJEpISDBbp9FoaO7cuY3S5ufnEwDKycl5YDm2bdtGcrncbN5QAPT111+37EQssGVeUZMvMq9Q6KJdNPeLYzYfnzHWNGu+n1bV2PR6PbKzsxEXFyeuk0qliIuLQ2ZmpsV9MjMzzdIDQHx8fJPpW8o0t6CTk/m0DUlJSfD390dMTAw2bNjQbE8AnU6H8vJys8VW3NyDsY7DqslciouLYTAYoFKpzNarVCqcO3fO4j5ardZieq1Wa2VRzcvxl7/8BXPmzDFb/9Zbb+Hpp5+Gq6srvv/+e/zpT39CZWUlXnrpJYv5pKamYvny5TaXoyGfe/MecHMPxhyv081SVV5ejoSEBPTt2xdvvvmm2balS5eKrwcPHoyqqiqsXr26ycC2ePFiJCcnm+UdEhJiU7m4uQdjHYdVl6L+/v6QyWQoLCw0W19YWNjkBKdqtdqq9M2pqKjAmDFj4OHhga+//hrOzs7NptdoNLh+/Tp0OsvjpCkUCnh6epotthIDW7UeRiN3hGfMkawKbHK5HNHR0cjIyBDXGY1GZGRkIDY21uI+sbGxZukBID09vcn0TSkvL8fo0aMhl8vxzTfftGgOy9zcXPj4+EChUFh1LFuYmnsYCSiv5SejjDmS1ZeiycnJmDFjBoYMGYKYmBisWbMGVVVVmDlzJgBg+vTpCA4ORmpqKgBg/vz5GDFiBN59910kJCQgLS0Nx44dw/r168U8S0pKUFBQgJs3bwIAzp8/D0Co7anVajGoVVdX48svvzS70d+tWzfIZDJ8++23KCwsxBNPPAGlUon09HSsWLECCxYsaN0n1ELOMik8lE6oqK1HSZUe3vcCHWPMAWx57Lp27Vrq3r07yeVyiomJocOHD4vbRowYQTNmzDBLv23bNoqIiCC5XE79+vWj3bt3m23fuHEjAWi0pKSkEBHR/v37LW4HQPn5+URE9K9//YsGDRpE7u7u5ObmRgMHDqSPP/6YDAZDi8+rNc09iIiGr9pHoYt20dH8OzbtzxhrmjXfTwkRj4xoUl5eDi8vL7EpibUS1x1E7rVSrJ8WjdH9rL+HyBhrmjXfT+4rakcNHyAwxhyHA5sd3Z84mR8eMOZIHNjsyM+da2yMdQQc2OzIh7tVMdYhcGCzI9973aq49wFjjsWBzY5MNbY7HNgYcygObHbET0UZ6xg4sNmRDw8PzliHwIHNjvzuBbaK2nrUGYwOLg1jDy8ObHbkqXSGVCK85stRxhyHA5sdSaWS+5O6cCNdxhyGA5udme6z3amyPAYcY6ztcWCzM1+usTHmcBzY7IznPmDM8Tiw2RnPfcCY43FgszNfbsvGmMNxYLMz8akoX4oy5jAc2OyMa2yMOR4HNjvjblWMOZ5NgW3dunUICwuDUqmERqPBkSNHmk2/fft2REZGQqlUIioqCnv27DHbvmPHDowePRp+fn6QSCTIzc1tlEdtbS2SkpLg5+cHd3d3TJw4sdF8pQUFBUhISICrqysCAgLw2muvob6+3pZTtNn95h4c2BhzFKsD29atW5GcnIyUlBQcP34cAwcORHx8PIqKiiymP3ToEKZMmYJZs2YhJycHiYmJSExMxOnTp8U0VVVVGDZsGN5+++0mj/vKK6/g22+/xfbt2/Hvf/8bN2/exG9+8xtxu8FgQEJCAvR6PQ4dOoTPP/8cmzZtwrJly6w9xVYRL0X5HhtjjmPtFFgxMTGUlJQkvjcYDBQUFESpqakW00+aNIkSEhLM1mk0Gpo7d26jtPn5+QSAcnJyzNaXlpaSs7Mzbd++XVx39uxZAkCZmZlERLRnzx6SSqWk1WrFNB999BF5enqSTqdr0bm1dvo9IqLK2joKXbSLQhftompdvc35MMbMWfP9tKrGptfrkZ2djbi4OHGdVCpFXFwcMjMzLe6TmZlplh4A4uPjm0xvSXZ2Nurq6szyiYyMRPfu3cV8MjMzERUVBZVKZXac8vJynDlzxmK+Op1OnHy54STMreEql0HuJHysXGtjzDGsCmzFxcUwGAxmwQMAVCoVtFqtxX20Wq1V6ZvKQy6Xw9vbu8l8mjqOaZslqamp8PLyEpeQkJAWl6kpEomE77Mx5mAP9VPRxYsXo6ysTFyuXbtml3z5yShjjuVkTWJ/f3/IZLJGTyMLCwuhVlue+VytVluVvqk89Ho9SktLzWptDfNRq9WNns6ajtvUsRQKBRQKRYvL0VKmSV04sDHmGFbV2ORyOaKjo5GRkSGuMxqNyMjIQGxsrMV9YmNjzdIDQHp6epPpLYmOjoazs7NZPufPn0dBQYGYT2xsLE6dOmX2dDY9PR2enp7o27dvi49lDzwNH2OOZVWNDQCSk5MxY8YMDBkyBDExMVizZg2qqqowc+ZMAMD06dMRHByM1NRUAMD8+fMxYsQIvPvuu0hISEBaWhqOHTuG9evXi3mWlJSgoKAAN2/eBCAELUCoaanVanh5eWHWrFlITk6Gr68vPD098eKLLyI2NhZPPPEEAGD06NHo27cvpk2bhlWrVkGr1WLJkiVISkpqk1pZc3hSF8YczJbHrmvXrqXu3buTXC6nmJgYOnz4sLhtxIgRNGPGDLP027Zto4iICJLL5dSvXz/avXu32faNGzcSgEZLSkqKmKampob+9Kc/kY+PD7m6utKECRPo1q1bZvlcuXKFxo4dSy4uLuTv70+vvvoq1dXVtfi87NHcg4jovfTzFLpoF72x42Sr8mGM3WfN91NCROTAuNqhlJeXw8vLC2VlZfD09LQ5ny8yr2DZP8/gmSg1PpwabccSMvbwsub7+VA/FW0rfI+NMcfiwNYGeIQPxhyLA1sbuF9j43kPGHMEDmxtoOFTUb6FyVj748DWBkwTuhiMhPLa9h02iTHGga1NKJxkcFcITQS5vyhj7Y8DWxvhafgYcxwObG2ER/hgzHE4sLUR0wgfdziwMdbuOLC1Ea6xMeY4HNjaiA/PfcCYw3BgayNiWzausTHW7jiwtZH73aq49wFj7Y0DWxsxdaviMdkYa38c2NoId4RnzHE4sLURnveAMcfhwNZGTJeiZTV1qDcYHVwaxh4uHNjaiLerHBKJ8Lq0hh8gMNaeOLC1EZlUAm8X4XKUm3ww1r44sLUhnjiZMcewKbCtW7cOYWFhUCqV0Gg0jSYq/qXt27cjMjISSqUSUVFR2LNnj9l2IsKyZcsQGBgIFxcXxMXF4cKFC+L2AwcOQCKRWFyOHj0KALhy5YrF7YcPH7blFO3Cl5t8MOYQVge2rVu3Ijk5GSkpKTh+/DgGDhyI+Ph4s4mKGzp06BCmTJmCWbNmIScnB4mJiUhMTMTp06fFNKtWrcL777+Pjz/+GFlZWXBzc0N8fDxqa2sBAEOHDsWtW7fMlhdeeAHh4eEYMmSI2fF++OEHs3TR0Y6bJYo7wjPmINbO7RcTE0NJSUnie4PBQEFBQZSammox/aRJkyghIcFsnUajoblz5xIRkdFoJLVaTatXrxa3l5aWkkKhoC1btljMU6/XU7du3eitt94S1+Xn5xMAysnJsfaURPaaV9Rk4fYTFLpoF63N+Nku+TH2MLPm+2lVjU2v1yM7OxtxcXHiOqlUiri4OGRmZlrcJzMz0yw9AMTHx4vp8/PzodVqzdJ4eXlBo9E0mec333yDO3fuiLPPNzR+/HgEBARg2LBh+Oabb5o9H51Oh/LycrPFnny4WxVjDmFVYCsuLobBYIBKpTJbr1KpoNVqLe6j1WqbTW/6aU2en332GeLj4/HII4+I69zd3fHuu+9i+/bt2L17N4YNG4bExMRmg1tqaiq8vLzEJSQkpMm0tvBz43tsjDmCk6MLYK3r16/ju+++w7Zt28zW+/v7Izk5WXz/+OOP4+bNm1i9ejXGjx9vMa/Fixeb7VNeXm7X4MZPRRlzDKtqbP7+/pDJZCgsLDRbX1hYCLVabXEftVrdbHrTz5bmuXHjRvj5+TUZrBrSaDS4ePFik9sVCgU8PT3NFnsydaviGhtj7cuqwCaXyxEdHY2MjAxxndFoREZGBmJjYy3uExsba5YeANLT08X04eHhUKvVZmnKy8uRlZXVKE8iwsaNGzF9+nQ4Ozs/sLy5ubkIDAxs8fnZm6lb1Z1KDmyMtSerL0WTk5MxY8YMDBkyBDExMVizZg2qqqrEG/nTp09HcHAwUlNTAQDz58/HiBEj8O677yIhIQFpaWk4duwY1q9fDwCQSCR4+eWX8de//hW9evVCeHg4li5diqCgICQmJpode9++fcjPz8cLL7zQqFyff/455HI5Bg8eDADYsWMHNmzYgE8//dTaU7QbX77HxphDWB3YJk+ejNu3b2PZsmXQarUYNGgQ9u7dK978LygogFR6vyI4dOhQbN68GUuWLMEbb7yBXr16YefOnejfv7+YZuHChaiqqsKcOXNQWlqKYcOGYe/evVAqlWbH/uyzzzB06FBERkZaLNtf/vIXXL16FU5OToiMjMTWrVvx29/+1tpTtBvTPbZqvQG1dQYonWUOKwtjDxMJEZGjC9FRlJeXw8vLC2VlZXa530ZEiFjyL9QZCJmLn0agl4sdSsnYw8ma7yf3FW1DEolEvM/GT0YZaz8c2NrY/UlduJEuY+2FA1sbE2ts/ACBsXbDga2NiXMfVOocXBLGHh4c2NqYj2nug2q+FGWsvXBga2PimGz88ICxdsOBrY2Jl6J8j42xdsOBrY35uHGNjbH2xoGtjfHEyYy1Pw5sbcyH5z1grN1xYGtjDWts3HuNsfbBga2NmWpsdQZCpa7ewaVh7OHAga2NuchlcLk3qgd3q2KsfXBgawfc5IOx9sWBrR2YAtvtCu5WxVh74MDWDvoEegAA0vMsz7rFGLMvDmztYNIQYearb0/cQkUt32djrK1xYGsH0aE+6NnNDTV1Bnx74paji8NYl8eBrR1IJBI893h3AMDWowUOLg1jXR8Htnbym8eC4SyT4MT1MuTdLHd0cRjr0mwKbOvWrUNYWBiUSiU0Gg2OHDnSbPrt27cjMjISSqUSUVFR2LNnj9l2IsKyZcsQGBgIFxcXxMXF4cKFC2ZpwsLCIJFIzJaVK1eapTl58iSefPJJKJVKhISEYNWqVbacXpvwc1dgdF9hAmiutTHWxshKaWlpJJfLacOGDXTmzBmaPXs2eXt7U2FhocX0Bw8eJJlMRqtWraK8vDxasmQJOTs706lTp8Q0K1euJC8vL9q5cyedOHGCxo8fT+Hh4VRTUyOmCQ0Npbfeeotu3bolLpWVleL2srIyUqlUNHXqVDp9+jRt2bKFXFxc6JNPPmnxuZWVlREAKisrs/ZjaZF/ny+i0EW7KCplL9Xo69vkGIx1VdZ8P60ObDExMZSUlCS+NxgMFBQURKmpqRbTT5o0iRISEszWaTQamjt3LhERGY1GUqvVtHr1anF7aWkpKRQK2rJli7guNDSU3nvvvSbL9eGHH5KPjw/pdDpx3aJFi6h3795N7lNbW0tlZWXicu3atTYNbAaDkYamZlDool204/i1NjkGY12VNYHNqktRvV6P7OxsxMXFieukUini4uKQmZlpcZ/MzEyz9AAQHx8vps/Pz4dWqzVL4+XlBY1G0yjPlStXws/PD4MHD8bq1atRX3+/72VmZiaGDx8OuVxudpzz58/j7t27FsuWmpoKLy8vcQkJCWnhJ2EbqVSCyY8Lx0g7cq1Nj8XYw8yqwFZcXAyDwSDO+m6iUqmg1VpufKrVaptNb/r5oDxfeuklpKWlYf/+/Zg7dy5WrFiBhQsXPvA4DY/xS4sXL0ZZWZm4XLvW9sHmt9GPQCoBsvJLcPl2ZZsfj7GHkZOjC9BSycnJ4usBAwZALpdj7ty5SE1NhUKhsClPhUJh8762CvJ2wYiIbth//ja2HruGxWP7tOvxGXsYWFVj8/f3h0wmQ2Fhodn6wsJCqNVqi/uo1epm05t+WpMnAGg0GtTX1+PKlSvNHqfhMTqK52KENm1fZV9HncHo4NIw1vVYFdjkcjmio6ORkZEhrjMajcjIyEBsbKzFfWJjY83SA0B6erqYPjw8HGq12ixNeXk5srKymswTAHJzcyGVShEQECAe58cff0Rd3f0uS+np6ejduzd8fHysOc0293RkAPzdFSiu1CPjbJGji8NY12Ptk4m0tDRSKBS0adMmysvLozlz5pC3tzdptVoiIpo2bRq9/vrrYvqDBw+Sk5MTvfPOO3T27FlKSUmx2NzD29ub/vnPf9LJkyfp2WefNWvucejQIXrvvfcoNzeXLl26RF9++SV169aNpk+fLuZRWlpKKpWKpk2bRqdPn6a0tDRydXXtUM09Gkrdc5ZCF+2iGRuy2vxYjHUFbdrcg4ho7dq11L17d5LL5RQTE0OHDx8Wt40YMYJmzJhhln7btm0UERFBcrmc+vXrR7t37zbbbjQaaenSpaRSqUihUNCoUaPo/Pnz4vbs7GzSaDTk5eVFSqWS+vTpQytWrKDa2lqzfE6cOEHDhg0jhUJBwcHBtHLlSqvOqz0D2+XblRS6aBeFvb6LbtytbvPjMdbZWfP9lBDxQPwm5eXl8PLyQllZGTw9Pdv8eM+tz8ThyyV4JS4C8+N6tfnxGOvMrPl+cl9RBzJ1jN927BoMRv7/wpi9cGBzoDH91fByccaN0hr8dLHY0cVhrMvgwOZASmcZJgwOBsAd4xmzJw5sDmbqYpWeV4jiSp4TgTF74MDmYH0CPTEwxBt1BsKO49cdXRzGugQObB3Ac6aO8Uev8WzxjNkBB7a2oK8Gis4C5/YAmeuAPa8Bu5KBGsujjIwbGARXuQyXb1fh6BXLaRhjLddpOsF3KERAVTFwNx8oyQfuXjF/XdnENHt3LgK//wqQOZutdlc4YdyAIGw9dg1pRwsQE+7b5qfAWFfGgc0W370BHP6w+TQKL8A3DPAJB7xDgKMbgPx/A3sXAwnvNEo+OSYEW49dw55Tt5Ayrh+8XJwb58kYaxEObLbwegSABPAMBnzC7gcwnzDAN1x47fqLWlfIE8DWqcDRvwEBkcDjL5htHhzijQiVO34urMQ3uTcwLTasfc6FsS6Iu1Q10OIuG/oqQCIDnJXWHeDHd4B9fxH2nfY10GOE2eYNP+XjrV156Bfkid0vPWnDGTDWdXGXqrYmd7M+qAHAk68CUZMAMgDbpgN3LpltnjA4GHKZFGduluP0jTI7FZaxhw8HtvYkkQDj1wLB0UBtKbDlOaCmVNzs4yZHfH9hUMwtR7gnAmO24sDW3pyVwHObhftzxT8D//hvwHB/Upop99q07cy5wbU2xmzEgc0RPNRCcHNyAS5lAOlLxU1P9PBDdKgPqvQGTPnbYWRf5XZtjFmLA5ujBA0CJnwsvD78IZD9OQBhir6NMx/H42E+qKitx7TPsnCIR/5gzCoc2BypXyLw1BvC692vAlcOAgA8lc74/L9j8GQvf1TrDXh+01FknC1sOh/GmBkObI42YiHQbwJgrAO2/l7ouQDAVe6ET2cMwei+KujrjZj7f9n49sRNx5aVsU6CA5ujSSTAsx8CgYOAmhJg83NAbTkAQOEkw7qpj+HZQUGoNxLmp+Vg21GeQZ6xB7EpsK1btw5hYWFQKpXQaDQ4cuRIs+m3b9+OyMhIKJVKREVFYc+ePWbbiQjLli1DYGAgXFxcEBcXhwsXLojbr1y5glmzZiE8PBwuLi7o2bMnUlJSoNfrzdJIJJJGy+HDh205xfYldwWmbAHc1cDts8CO2YDRAABwlknxP5MGYUpMdxgJWPjVSWw8mO/gAjPWsVkd2LZu3Yrk5GSkpKTg+PHjGDhwIOLj41FUZHl+zEOHDmHKlCmYNWsWcnJykJiYiMTERJw+fVpMs2rVKrz//vv4+OOPkZWVBTc3N8THx6O2thYAcO7cORiNRnzyySc4c+YM3nvvPXz88cd44403Gh3vhx9+wK1bt8QlOjra2lN0DM8gYMpmwEkJ/LwX+GcS8PN3QGkBZBJgxYT+eGFYOABg+bd5WLf/ooML3MUYjcCZr4XFUPfg9FbnbwDKrgvHYW3O6i5VGo0Gjz/+OD744AMAwoTJISEhePHFF/H66683Sj958mRUVVVh165d4ronnngCgwYNwscffwwiQlBQEF599VUsWLAAAFBWVgaVSoVNmzbhueees1iO1atX46OPPsLly5cBCDW28PBw5OTkYNCgQdackqi9Z6my6NQ/gK9mma+TewABfUABfXDgbjesP6/AeWMIJj81GAvje0Mikdi/HIY64NYJ4Ooh4RK5x0gg9FeAzM7di4mEy3FHKjoH7HoFKDgkvPcMBjRzgejnAaVX6/KuLgFyvgSOfgqUXgU8HxEeGvWfCAQNdsy5EwHVdwAXH0Aqa//j28ia76dVf6V6vR7Z2dlYvHixuE4qlSIuLg6ZmZkW98nMzERycrLZuvj4eOzcuRMAkJ+fD61Wi7i4OHG7l5cXNBoNMjMzmwxsZWVl8PVtPLzP+PHjUVtbi4iICCxcuBDjx49v8nx0Oh10uvvDcZeXlzeZtt1E/VYY1ijvn8KYbsU/A/oK4PoRSK4fwUgAI+VC0tuZXrh8uhd69BsCiX8E4NcT8O0pfDGlVlbG9VXAtSNAwWHhC379GFBXfX/7T+8JX4SIsUCfcUDPkYCzi/XnZzQC2hPApf3A5QPC8ZSegKofoOoPBPQVXneLtK3bmjXqaoD/vAv8tEZ4eOPsKnSXK78BpC8D/r0KeGw6oPkD4BNqXd63TgJH1gOntgP1tffXl18HMj8QFp8woN9vhCCn6td2Qa6mFLh5HLieDdy4t1QVCb/PR+OAiDFAz6cbD9zQiVkV2IqLi2EwGKBSqczWq1QqnDt3zuI+Wq3WYnqtVituN61rKs0vXbx4EWvXrsU779wf/sfd3R3vvvsufvWrX0EqleKrr75CYmIidu7c2WRwS01NxfLly5s5Ywfp+6ywAEC9XhjHrShPCHRFecJy9wq6ScrQrfIYkHXMfH8npTDCiG8PwK+HEOxMQc8jUAh6VXeAgkxhuXpIqJ2RwTwfFx+ge6xQa/n5O6HmdmKzsDi7AY+OEoJcr9GAi3fT53P3KnD5XiC7/G8hn4aqbt/bduD+OokM8Hv0XsBrsHiF2CcAXNoP7E4GSoQaPyLGAs+sAtwChGCUuU6433n4QyDrY+H3Efsi8EgztzYMdcDZb4AjfxM+VxNVFKCZA0T+Grh6EDj9lfB53r0C/PQ/wuIfIQS4fr8BukXYfl71OqDwdIMgdkz4+7Gk5q5wrqe2AxIpEKIRfpcRY4CAPrZ9zkajEDTraoR/sE5y28+lFTrdsEU3btzAmDFj8Lvf/Q6zZ88W1/v7+5vVDB9//HHcvHkTq1evbjKwLV682Gyf8vJyhISEtF3hbeEkB1R9haUhXSX2H/wP9u7bh164hginQoRLCxFo1MKpvlb4Ut4+2yg7cnKBxK0bUGahL6rnI0BorBDMQocC/r3v1/wM9cKX9dwu4OwuoeZx9hthkToD4cOBPr8GeicATgog/8f7wcwUPEzkHkDYMKHWFz783ojDZ4BC03Ja+NIVnxeWMzvu76v0AnqOAvqOBx79L0Dhbt3nWXkb+P7PwMmtwnuPQGDsKiFAm77Ij00DBv8euJgh1Kwu779//617LBA7D+g99v5lXEUhkL0ROLbx/iCjUiegz3ggZg7Q/Yn7efcZJyz6KuFe6ukdwIV0oWZ+IFVYVP2FJkA9ngKM9ULNua7m3lJt+ae+WvistKcAgx6N+IQLfZSDo4FHhgiBS3tKCLAXvhf+WZr+0WUsF/6B9BoNRMQLvyNT7dx0GXv3qnBpXXoVKC24975AWAz3roIkUuFvyjdMqJ2aLeHCP842qqVaFdj8/f0hk8lQWGjeWLSwsBBqtdriPmq1utn0pp+FhYUIDAw0S/PLe2U3b97EyJEjMXToUKxfv/6B5dVoNEhPT29yu0KhgEKheGA+HZLCHSOfHgtdwGC8sjUXNbVCbUsGA4IkxQiXaBEm0Yo/wyRahEhuw6m+RgxqV6QhOOvcHxeV/VHgMQg1rkFwhxNctU5wuyuFqzwfbgoZPJROUHkoofIaDPXTsXAbsxK4lSsEuLPfCl+oSxnCsitZ+GOlBjfJJTLgkceFL2rPkcKX6xejCJvVhIiACu39IGcKeMU/A7VlQqA7swOQKe7VGscDvccIX5SmGI1A7pfA90uFAQggEYLO00uES+FfkkiAXnHCoj0t1OBObb//5fftAUTPFGq6ef8ULmUBwF0lrI9+HvAMbJyvidxNqKH1nyg07zm/Rwhyl/bdO+fTwhBXtnDxvR/AgqOBoMcAN7/G6UKHCst/LRcCkinI5f8IlF0Djn0mLE4uQPBjwv3C0gKgrqr540ukgEwuXIKXFQhL/o+N0yk8hUt8U7B7epndang2PTyIiYnB2rVrAQgPD7p374558+Y1+fCguroa3377rbhu6NChGDBggNnDgwULFuDVV18FINScAgICzB4e3LhxAyNHjkR0dDS+/PJLyGQPvuk5e/ZsZGdn4/jx4y06tw7x8MAGlbp6XL9bjTuVehRX6nC7Qoc7VXoUV+hQXKlDcaUedyp1KK2sRjdjEVS4iwsUjLuw7Rw9FE5QeSmh9lRC5alEH2ctHqv+CT3uHIB3yUkhkX+E8MChx1NC7cxS8LBWvV4IqOd2AXnfCMOxm0idgLAnhZpc7wTAo8GtjV8+HFAPAMatEb701ii/Jdw3O7bhXnBsIOQJIGa2EGRb8+WsLhHO7/QOIZA7KYV7f84u9xbXX/w0vVYKtazgaCFItKYmpK8WAtGF74RgV37jFwkkQk3Xu7sQmLxDG7zufu8erxNQWXR/2Py7V8yXilvmWco9gMXXmi23Nd9PqwPb1q1bMWPGDHzyySeIiYnBmjVrsG3bNpw7dw4qlQrTp09HcHAwUlNTAQjNPUaMGIGVK1ciISEBaWlpWLFiBY4fP47+/fsDAN5++22sXLkSn3/+OcLDw7F06VKcPHkSeXl5UCqVuHHjBp566imEhobi888/Nwtqphrf559/DrlcjsGDBwMAduzYgaVLl+LTTz/FzJkzW3RunTWwtRQRoby2HiVVelTp6lGlq0e13oAqfT2qdcLPKl09qvQGVJt+6utRWl0HbXktCstqUaU3NHuMbiiFVELwDuiOIWE+eDzMF9GhPnjEx8W+T2+JhFrc2W+FpehMg40S4fKvz3jhsung/957OOAGPP1nIGZu657u6quA3M3CpalvOPD4bKHvb1dk+py1J4XaqE+YMIK0UyuvdOpq7l3CXhGWumpg2CvN7tKmgQ0APvjgA6xevRparRaDBg3C+++/D41GAwB46qmnEBYWhk2bNonpt2/fjiVLluDKlSvo1asXVq1ahWeeeUbcTkRISUnB+vXrUVpaimHDhuHDDz9ERIRwE3XTpk1NBidT8T///HO8/fbbuHr1KpycnBAZGYnXXnsNv/3tb1t8Xl09sNlDpa4e2rJaFJbXQltWKwS8e68LK3TQltWgsLzxxM9qTyWGhPlgSKgPhoT5ok+gJ2TSlge62joDymvqoKs3Wg6SxReBc/eC3I3sxhlEjAWeWS3MP8E6pTYPbF0VBzb7uF2hQ/bVuzh2pQTHrt7F6RtlqDea/5m5K5wwuLs3okN9oHSWobymDuW1dSirqW/wug7lNfUor62Dvv7+PTuVpwJPRwbg6UgVfvWoH1zlv6h9lV0Hzu0Wgpy+Uhi5OPLXjm8vx1qFA5uNOLC1jRq9AbnXSpF9tQRHr9zF8at3UaGrf/COvyCVADKpBHWG+3+ycicphvb0w6jIAIyMDMAjPq72LDrrQDiw2YgDW/swGAk/F1bg2JUS5FwrBSAM1eTp4gwvF2d4Kp0avHaGl6uwzk3uhDqjEVmXS5BxthAZ54pw/W6NWd6Rag+M6iPU5gaFeFt1ucs6Ng5sNuLA1rkQES4UVSLjbBH2nStE9tW7aHjF6+smhybcFz5u8nuB0wkeyvuB01PpJAZUD6UTXJxlbdM9jdkFBzYbcWDr3O5W6fHvn28j41wRDpwvQkWtdZe7TlIJHvFxwWOhPoi+t/QK8OBaXwfBgc1GHNi6jjqDEceu3EXerXKU19SholZ4CFFRe/+BhGldeU0djE18CzwUThh07yFHdKgPBoV4w0PpbDkxa1Mc2GzEge3hRESo1htQVlOHnwsrcPzqXWQX3EVOQSmqf9FuTyIBeqs8xEDXN8gT4f5uUDh1nlEyOisObDbiwMYaqjcYcd4U6O4Fu2slNY3SyaQShPq5oleAO3oFeKCXyh2PBrijZzd3KJ054NkLBzYbcWBjD1JUXovjBUKgO15Qip+1FU02XZFIgO6+QsB7NMADPbu5QeWphJ+7HP7uCvi6yeEs49H5W4oDm404sDFrERGKKnT4ubACFworcaGoEheLKvBzYSXKah48Eq+3qzP83OTwc1fA/17A83NTiMFP5amA2ksJf3fFQx8E22ygScaYOYlEAtW9wQCe7NVNXE9EKK7U40JRBS4WVeJCYSXyi6vEQQlKqnQwElBaXYfS6jpcut38iBkSCe4HOk8lAjxNgxAoxNdqTyW8XZ25yQq4xmaGa2ysvRiNhNKaOtyp1OF2pQ537o3AcqdKL47GUlShQ1F5LYoqdI26pDVF4SSFylMJ9b3RVxr+VHkqEeilRDePzln74xobYx2cVCqBr5scvm5y9FJ5NJvWaCTcqdKj8N6AA4Xlugava6G9976kSg9dvREFJdUoKKluMj9T7c9U4zPVOM1fK+HTiWt/HNgY6+CkUgm6eSjQzUOB/sFNTy6jqzegqFwHrWnklXujrzR8X1RRizoD4XaFMG7fqV8OtdaAXCZFwL1gF+ChgIfSCe4KZ7grneChcIK70gnu9342fO9xL40jGzZzYGOsi1A4yRDi64oQ36YHAvhl7U97rwZY9IvXd6r00BuMuH63plF/3JZyVzgJ/X3vdV8zvRb7ALs43esHLKyLDvWxWw2RAxtjDxFran+3K3TiZe/tCh0qdfWoqK1Hpa4OlbX14vsqfb3Ze929IaYqdcK6G6UPDowKJynO/3Ws3c6TAxtjrBGFkwyP+LjaNAyUvt4odF2rrUdZjWlcvbr7r2vvvy+vEdI4yex72cqBjTFmV3InKfzcFfBzd9xESZ3vmS9jjD0ABzbGWJfDgY0x1uXYFNjWrVuHsLAwKJVKaDQaHDlypNn027dvR2RkJJRKJaKiorBnzx6z7USEZcuWITAwEC4uLoiLi8OFCxfM0pSUlGDq1Knw9PSEt7c3Zs2ahcrKSrM0J0+exJNPPgmlUomQkBCsWrXKltNjjHV2ZKW0tDSSy+W0YcMGOnPmDM2ePZu8vb2psLDQYvqDBw+STCajVatWUV5eHi1ZsoScnZ3p1KlTYpqVK1eSl5cX7dy5k06cOEHjx4+n8PBwqqmpEdOMGTOGBg4cSIcPH6b//Oc/9Oijj9KUKVPE7WVlZaRSqWjq1Kl0+vRp2rJlC7m4uNAnn3zS4nMrKysjAFRWVmbtx8IYa2PWfD+tDmwxMTGUlJQkvjcYDBQUFESpqakW00+aNIkSEhLM1mk0Gpo7dy4RERmNRlKr1bR69Wpxe2lpKSkUCtqyZQsREeXl5REAOnr0qJjmX//6F0kkErpx4wYREX344Yfk4+NDOp1OTLNo0SLq3bt3i8+NAxtjHZc130+rLkX1ej2ys7MRFxcnrpNKpYiLi0NmZqbFfTIzM83SA0B8fLyYPj8/H1qt1iyNl5cXNBqNmCYzMxPe3t4YMmSImCYuLg5SqRRZWVlimuHDh0Mul5sd5/z587h7967Fsul0OpSXl5stjLHOz6p2bMXFxTAYDFCpVGbrVSoVzp07Z3EfrVZrMb1WqxW3m9Y1lyYgIMC84E5O8PX1NUsTHh7eKA/TNh8fn0ZlS01NxfLlyxut5wDHWMdj+l5SCwYkeqgb6C5evBjJycni+xs3bqBv374ICQlxYKkYY82pqKiAl1fT3cEAKwObv78/ZDIZCgsLzdYXFhZCrVZb3EetVjeb3vSzsLAQgYGBZmkGDRokpikqKjLLo76+HiUlJWb5WDpOw2P8kkKhgEJxv3W0u7s7rl27Bg8Pjwd2xi0vL0dISAiuXbvWpcdue1jOE+Bz7eiICBUVFQgKCnpgWqsCm1wuR3R0NDIyMpCYmAgAMBqNyMjIwLx58yzuExsbi4yMDLz88sviuvT0dMTGxgIAwsPDoVarkZGRIQay8vJyZGVl4Y9//KOYR2lpKbKzsxEdHQ0A2LdvH4xGIzQajZjmz3/+M+rq6uDs7Cwep3fv3hYvQy2RSqV45JFHrPlI4Onp2Wn+MFrjYTlPgM+1I3tQTU1k7ZOJtLQ0UigUtGnTJsrLy6M5c+aQt7c3abVaIiKaNm0avf7662L6gwcPkpOTE73zzjt09uxZSklJsdjcw9vbm/75z3/SyZMn6dlnn7XY3GPw4MGUlZVFP/30E/Xq1cusuUdpaSmpVCqaNm0anT59mtLS0sjV1dWq5h7WeFieoD4s50nE59qVWB3YiIjWrl1L3bt3J7lcTjExMXT48GFx24gRI2jGjBlm6bdt20YREREkl8upX79+tHv3brPtRqORli5dSiqVihQKBY0aNYrOnz9vlubOnTs0ZcoUcnd3J09PT5o5cyZVVFSYpTlx4gQNGzaMFAoFBQcH08qVK205vRbp6n8YJg/LeRLxuXYlNgU2RlRbW0spKSlUW1vr6KK0qYflPIn4XLsSnsyFMdblcCd4xliXw4GNMdblcGBjjHU5HNgYY10OBzbGWJfDgc0G1g602Rm9+eabkEgkZktkZKSji2UXP/74I8aNG4egoCBIJBLs3LnTbDu1YODTzuBB5/n88883+h2PGTPGMYW1Mw5sVtq6dSuSk5ORkpKC48ePY+DAgYiPj2/Ul7Ur6NevH27duiUuP/30k6OLZBdVVVUYOHAg1q1bZ3H7qlWr8P777+Pjjz9GVlYW3NzcEB8fj9ra2nYuaes86DwBYMyYMWa/4y1btrRjCduQg9vRdTrWDrTZWaWkpNDAgQMdXYw2B4C+/vpr8X1LBj7tjH55nkREM2bMoGeffdYh5WlrXGOzgi0DbXZmFy5cQFBQEHr06IGpU6eioKDA0UVqcy0Z+LQrOXDgAAICAtC7d2/88Y9/xJ07dxxdJLvgwGaF5gbaNA142VVoNBps2rQJe/fuxUcffYT8/Hw8+eSTqKiocHTR2lRLBj7tKsaMGYMvvvgCGRkZePvtt/Hvf/8bY8eOhcFgcHTRWu2hHmiSNW3s2LHi6wEDBkCj0SA0NBTbtm3DrFmzHFgyZi/PPfec+DoqKgoDBgxAz549ceDAAYwaNcqBJWs9rrFZwZaBNrsKb29vRERE4OLFi44uSptqOPBpQw/D77hHjx7w9/fvEr9jDmxWaDjQpolpoE3TwJldVWVlJS5dumQ2ynFX1HDgUxPTwKdd/Xd8/fp13Llzp0v8jvlS1ErJycmYMWMGhgwZgpiYGKxZswZVVVWYOXOmo4tmVwsWLMC4ceMQGhqKmzdvIiUlBTKZDFOmTHF00VqtsrLSrFaSn5+P3Nxc+Pr6onv37nj55Zfx17/+Fb169UJ4eDiWLl2KoKAgcdTozqK58/T19cXy5csxceJEqNVqXLp0CQsXLsSjjz6K+Ph4B5baThz9WLYzam6gza5i8uTJFBgYSHK5nIKDg2ny5Ml08eJFRxfLLvbv308AGi2mAVJbMvBpZ9DceVZXV9Po0aOpW7du5OzsTKGhoTR79mxxJOzOjsdjY4x1OXyPjTHW5XBgY4x1ORzYGGNdDgc2xliXw4GNMdblcGBjjHU5HNgYY10OBzbGWJfDgY0x1uVwYGOMdTkc2BhjXc7/B80wLFhUW6WpAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:38:47.469320Z", + "iopub.status.busy": "2024-03-01T18:38:47.468612Z", + "iopub.status.idle": "2024-03-01T18:41:40.614258Z", + "shell.execute_reply": "2024-03-01T18:41:40.613445Z" + }, + "papermill": { + "duration": 173.16794, + "end_time": "2024-03-01T18:41:40.616686", + "exception": false, + "start_time": "2024-03-01T18:38:47.448746", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:41:40.657073Z", + "iopub.status.busy": "2024-03-01T18:41:40.656674Z", + "iopub.status.idle": "2024-03-01T18:41:40.678450Z", + "shell.execute_reply": "2024-03-01T18:41:40.677576Z" + }, + "papermill": { + "duration": 0.044614, + "end_time": "2024-03-01T18:41:40.680400", + "exception": false, + "start_time": "2024-03-01T18:41:40.635786", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.009699NaN0.0012722.5979270.0310540.5863480.0409360.0000023.1333330.0282240.0674210.0356620.054820.0222145.73126
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.009699 NaN 0.001272 2.597927 0.031054 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.586348 0.040936 0.000002 3.133333 0.028224 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.067421 0.035662 0.05482 0.022214 5.73126 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:41:40.718036Z", + "iopub.status.busy": "2024-03-01T18:41:40.717359Z", + "iopub.status.idle": "2024-03-01T18:41:41.073579Z", + "shell.execute_reply": "2024-03-01T18:41:41.072592Z" + }, + "papermill": { + "duration": 0.377488, + "end_time": "2024-03-01T18:41:41.075694", + "exception": false, + "start_time": "2024-03-01T18:41:40.698206", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:41:41.115828Z", + "iopub.status.busy": "2024-03-01T18:41:41.115419Z", + "iopub.status.idle": "2024-03-01T18:44:42.592226Z", + "shell.execute_reply": "2024-03-01T18:44:42.591273Z" + }, + "papermill": { + "duration": 181.515105, + "end_time": "2024-03-01T18:44:42.610235", + "exception": false, + "start_time": "2024-03-01T18:41:41.095130", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:44:42.649699Z", + "iopub.status.busy": "2024-03-01T18:44:42.649382Z", + "iopub.status.idle": "2024-03-01T18:44:42.666703Z", + "shell.execute_reply": "2024-03-01T18:44:42.666006Z" + }, + "papermill": { + "duration": 0.039835, + "end_time": "2024-03-01T18:44:42.668901", + "exception": false, + "start_time": "2024-03-01T18:44:42.629066", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:44:42.706365Z", + "iopub.status.busy": "2024-03-01T18:44:42.706086Z", + "iopub.status.idle": "2024-03-01T18:44:42.711177Z", + "shell.execute_reply": "2024-03-01T18:44:42.710319Z" + }, + "papermill": { + "duration": 0.026079, + "end_time": "2024-03-01T18:44:42.713216", + "exception": false, + "start_time": "2024-03-01T18:44:42.687137", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.44455916152562114}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:44:42.753740Z", + "iopub.status.busy": "2024-03-01T18:44:42.753407Z", + "iopub.status.idle": "2024-03-01T18:44:43.095735Z", + "shell.execute_reply": "2024-03-01T18:44:43.094877Z" + }, + "papermill": { + "duration": 0.365479, + "end_time": "2024-03-01T18:44:43.097806", + "exception": false, + "start_time": "2024-03-01T18:44:42.732327", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAE8CAYAAACYd5eXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCg0lEQVR4nO3deXxU1d0/8M+dfSazJJM9ZCUg+25ApQgojwuKUlp/+pIqgYL2EaqW0kdpX2JdUeuCtv3xtD5K4Keg1Uetj1aqD4oWF/ZNUZaQjewkmS2Z/Z7fH3dmkkkySWbJbPm+X695zcy9d+6cDMN3zjn3e87hGGMMhBAS50SxLgAhhAwFBStCSEKgYEUISQgUrAghCYGCFSEkIVCwIoQkBApWhJCEQMGKEJIQKFgRQhICBSsStoqKCnAch+rq6lgXhSQxClYkJnbu3IktW7bEuhgkgVCwIjFBwYoEi4IVISQhULAiw+Kjjz7C/PnzodFooNVqUVZWhp07dwIAFixYgA8//BA1NTXgOA4cx6G4uHjI5+Z5Hr///e+Rl5cHlUqFhQsX4tSpUyguLkZ5ebnvuPb2dmzYsAFTpkyBWq2GVqvF9ddfj+PHj/udb+/eveA4Dn/729/wxBNPID8/HwqFAldffTXOnTsXiY+DRIAk1gUgyaeiogKrVq3CpEmTsHHjRqSmpuLo0aPYvXs3br/9dvzud7+D0WjEhQsX8MILLwAA1Gr1kM+/ceNGPPPMM1iyZAmuvfZaHD9+HNdeey1sNpvfcefPn8d7772HW265BSUlJWhubsZf/vIXzJ8/H6dOnUJeXp7f8U899RREIhE2bNgAo9GIZ555BsuXL8f+/fvD/1BI+BghYdq2bRsDwKqqqpjBYGAajYbNmTOHWa1Wv+N4nvc9vuGGG1hRUVHQ79XU1MQkEglbunSp3/bf//73DABbsWKFb5vNZmNut9vvuKqqKiaXy9mjjz7q2/bZZ58xAGzChAnMbrf7tr/44osMADt58mTQ5SSRR81AElGffPIJzGYzHnzwQSgUCr99HMeFff49e/bA5XLhnnvu8dv+y1/+ss+xcrkcIpHwFXe73Whra4Narca4ceNw5MiRPsevXLkSMpnM93zevHkAhBoaiT0KViSiKisrAQCTJ08elvPX1NQAAMaMGeO3Xa/XIy0tzW8bz/N44YUXMHbsWMjlcmRkZCAzMxMnTpyA0Wjsc+7CwkK/597zdXR0RPJPICGiYEWS1pNPPon169fjyiuvxGuvvYZ//vOf+OSTTzBp0iTwPN/neLFY3O95GM38HReog51EVGlpKQDg22+/7VP76SnUJmFRUREA4Ny5cygpKfFtb2tr61MDevvtt7Fw4UK88sorftsNBgMyMjJCen8SO1SzIhF1zTXXQKPRYPPmzX2uzvWsoaSkpPTbFBvM1VdfDYlEgq1bt/pt/9Of/tTnWLFY3KdW9NZbb6G+vj7o9yWxRzUrElFarRYvvPACVq9ejbKyMtx+++1IS0vD8ePH0dXVhe3btwMAZs2ahTfffBPr169HWVkZ1Go1lixZMuj5s7Ozcd999+G5557DTTfdhOuuuw7Hjx/HRx99hIyMDL8a24033ohHH30UK1euxBVXXIGTJ0/i9ddfx+jRo4ft7yfDKMZXI0kS6Jm64PX++++zK664gimVSqbVatns2bPZrl27fPstFgu7/fbbWWpqKgMQVBqDy+ViDz30EMvJyWFKpZJdddVV7Pvvv2fp6ensF7/4he84m83Gfv3rX7Pc3FymVCrZ3Llz2ddff83mz5/P5s+f7zvOm7rw1ltv+b1PVVUVA8C2bdsW7EdChgHHGPUeksRnMBiQlpaGxx9/HL/73e9iXRwyDKjPiiQcq9XaZ5t3UPSCBQuiWxgSNdRnReJGa2sr3G53wP0ymQx6vR5vvvkmKioqsHjxYqjVauzbtw+7du3CNddcg7lz50axxCSaKFiRuFFWVuZL+uzP/PnzsXfvXkydOhUSiQTPPPMMTCaTr9P98ccfj2JpSbRRnxWJG19++WW/TTyvtLQ0zJo1K4olIvGEghUhJCHEtIP9iy++wJIlS5CXlweO4/Dee+/57WeMYdOmTcjNzYVSqcSiRYtw9uzZ2BSWEBJTMe2z6uzsxLRp07Bq1SosW7asz/5nnnkGL730ErZv346SkhI89NBDuPbaa3Hq1Kk+I/oD4XkeDQ0N0Gg0ERn1TwiJHMYYzGYz8vLyfDNkDHRwXADA3n33Xd9znudZTk4O+8Mf/uDbZjAYmFwu90su7M1mszGj0ei7nTp1igGgG93oFse3urq6QWNE3F4NrKqqQlNTExYtWuTbptPpMGfOHHz99de47bbb+n3d5s2b8cgjj/TZXldXB61WO2zlJYQEz2QyoaCgABqNZtBj4zZYNTU1ARDGgvWUnZ3t29efjRs3Yv369b7n3g9Dq9VSsCIkTg2liyZug1Wo5HI55HJ5rItBCImwuB1uk5OTAwBobm72297c3OzbRwgZOeI2WJWUlCAnJwd79uzxbTOZTNi/fz8uv/zyGJaMEBILMW0GWiwWv3XZqqqqcOzYMej1ehQWFuL+++/H448/jrFjx/pSF/Ly8rB06dLYFZqMSG63G06nM9bFSEhSqTTglNHBiGmwOnToEBYuXOh77u0YX7FiBSoqKvAf//Ef6OzsxF133QWDwYAf/ehH2L1795BzrAiJBIvFggsXLtBc7CHiOA75+flBrQ3Z73lYkv8LmEwm6HQ6GI1GuhpIguZ2u3H27FmoVCpkZmZSYnGQGGNobW1FV1cXxo4d26eGFcz/z6S7GkhIb0drO3C2xYIpo3SYkBvcD5bT6QRjDJmZmVAqlcNUwuSWmZmJ6upqOJ3OsJqDcdvBTkgknGuxYO/pVrSYbMjUhJ7SQjWq0EXqs6NgRZLawep2AMCkUTpkqCn/LpFRsCJJq6PTgSajDWIRhzkl+lgXh4SJghVJWjXtXQCAvFQlVDIJLHYX9p9vw1fnLsa4ZMmluLjYNwf+cKJgRZJWrSdYFepVAACzzYmvKttwtM4Al7vv8vEkvlGwIkmr2SisCD0qTbiKl6NVQKOQwOHiUW8IPH3ySORwOGJdhEFRsCJJiTGGWcVpmJinRaanY53jOBR4alneWleoHC4+4K13rW2gY51DPDZYCxYswLp167Bu3TrodDpkZGTgoYce8iW2FhcX47HHHsOdd94JrVaLu+66CwCwb98+zJs3D0qlEgUFBbj33nvR2dnpO29LSwuWLFkCpVKJkpISvP7660GXLVSUZ0WSEsdxmFmY1md7oV6FUw0m1LWHV7P682fnAu4ryUjB0hmjfM//+kUlnO7+c6/z05S45dIC3/NXv6yC1dF3ObJf/dslQZdx+/bt+PnPf44DBw7g0KFDuOuuu1BYWIg1a9YAAJ599lls2rQJDz/8MACgsrIS1113HR5//HG8+uqraG1t9QW8bdu2AQDKy8vR0NCAzz77DFKpFPfeey9aWlqCLlsoKFiREcVbs2ox22BzuqGQhj9mLV4VFBTghRdeAMdxGDduHE6ePIkXXnjBF6yuuuoq/PrXv/Ydv3r1aixfvhz3338/AGDs2LF46aWXMH/+fGzduhW1tbX46KOPcODAAZSVlQEAXnnlFUyYMCEqfw8FK5KUmow2iERAmkoGqbi7t0MtlyBdLUObxYF6gxWlmaGNV1u7cEzAfaJeOZB3XVka8Nje+ZKr5paEVJ7+XHbZZX4JmZdffjmee+4530Kyl156qd/xx48fx4kTJ/yadowx8DyPqqoqnDlzBhKJxG85tPHjxyM1NTViZR4IBSuSlPaebkGj0YYbpubikmz/KXNzdUqYbS502QOv/jwYmWTo3b3DdWy4UlJS/J5bLBbcfffduPfee/scW1hYiDNnzkSraP2iYEWSksEqTOeSqpL22TdvbAauHp8FUe8qUJLZv3+/3/Nvvvmm38HEXjNnzsSpU6cwZkz/tcbx48fD5XLh8OHDvmbg6dOnYTAYIlruQOhqIEk6Nqfb10mtU/YNVgqpOOkDFQDU1tZi/fr1OH36NHbt2oU//vGPuO+++wIe/8ADD+Crr77CunXrcOzYMZw9exZ///vfsW7dOgDAuHHjcN111+Huu+/G/v37cfjwYaxevTpqA7wpWJGkY/TUqlLkYsglA3egJ/MMSXfeeSesVitmz56NtWvX4r777vOlKPRn6tSp+Pzzz3HmzBnMmzcPM2bMwKZNm5CXl+c7Ztu2bcjLy8P8+fOxbNky3HXXXcjKyorGn0PNQJJ8OrqEBMdUpSzgMYdrOnDyggHTClIxo58Uh2QglUqxZcsWbN26tc++6urqfl9TVlaGjz/+OOA5c3Jy8MEHH/htu+OOO8Iq51BRzYokHWOXULPS9dNf5eV08+jocqLJk+VO4h8FK5J0zDYXAECjCNxwyNEKU2M3myhYJQpqBpKkMy5Hg1SVFHmpgTt+s7TCEJyOLifsLvegfVuJZu/evbEuQsRRsCJJp0Cv8mWqB6KSSaBRSGC2udBisg96PIk9agaSESvL0xRsMdtjXBIyFBSsSFJxuXlUtlrQYrINmpaQ7ZmTvYX6rRICBSuSVIxWJ94/1oC3Dl8YdKGCHJ0CGRr5gFcNSfygPiuSVCx24UqgdoArgV5F6Sm4Iz1l0ONIfKCaFUkq3rQF9RCCFUksFKxIUvHWrDTyoTfteJ7B5gx9BgYSHRSsSFLpcgjBSiUfWt7U0doO/Omzc9h3lla8iXcUrEhS6fLMtqCSDa0ZqJJJ4OYZLloofSHeUbAiSaU7WA2tZpWuFgY7t3U6hjYDA2OAyxGbWxAzROzYsQPp6emw2/2D8NKlS6M28DjSqBeSJJVZRWkozVQjSzO0peLTVDKIRRwcLh4mm6vf+a/8uJ3Av56LQElDMO/XgCTwTBI93XLLLbj33nvx/vvv45ZbbgEgrEzz4YcfDjirQjyjmhVJKqWZaswqSkOqamj/qcUiDmkpntpVEjUFlUolbr/9dt+qNADw2muvobCwEAsWLIhdwcJANSsy4mWkyHDRbMdFiwOjMwc5WCwVajixIA4ueXXNmjUoKytDfX09Ro0ahYqKCpSXlw+aLBuv4rpm5Xa78dBDD6GkpARKpRKlpaV47LHHknp2RxI6h4vH+VZL0NO+pHsWQR1SzYrjhKZYLG5BBpkZM2Zg2rRp2LFjBw4fPozvvvsO5eXlQZ0jnsR1zerpp5/G1q1bsX37dkyaNAmHDh3CypUrodPp+l2Bg4xsRqsTfz/WAKVMjF/MD7z8VW95qQqMy9EMOKVMolq9ejW2bNmC+vp6LFq0CAUFBYO/KE7FdbD66quvcPPNN+OGG24AICx5vWvXLhw4cCDGJSPxyJvYOdQrgV75aSrkpyXnFDG33347NmzYgJdffhk7duyIdXHCEtfNwCuuuAJ79uzxrVd2/Phx7Nu3D9dff33A19jtdphMJr8bGRm8aQvKJF5lOVg6nQ4/+clPoFarsXTp0lgXJyxxXbN68MEHYTKZMH78eIjFYrjdbjzxxBNYvnx5wNds3rwZjzzySBRLSeKFN3tdGWTNChBWuTHZXJBLREm3pHx9fT2WL18OuXxo6RzxKq5rVn/729/w+uuvY+fOnThy5Ai2b9+OZ599Ftu3bw/4mo0bN8JoNPpudXV1USwxiSVriM1AAPifE414dV8VzjSbI12smOno6MC7776LvXv3Yu3atbEuTtjiumb1m9/8Bg8++CBuu+02AMCUKVNQU1ODzZs3Y8WKFf2+Ri6XJ/wvCAmN1dcMDP5r7U0G7fCsjJMMZsyYgY6ODjz99NMYN25crIsTtrgOVl1dXRCJ/Ct/YrEYPM/HqEQkngU71KanNM8EfAbPmoPJINDagIkqroPVkiVL8MQTT6CwsBCTJk3C0aNH8fzzz2PVqlWxLhqJQ1PzdchLVYaUgpDmyXjv6EyeYJVs4jpY/fGPf8RDDz2Ee+65By0tLcjLy8Pdd9+NTZs2xbpoJA4VpaegKMSZP1M9NSuj1QWeZxCJ/BMwKRE5dJH67OI6WGk0GmzZsgVbtmyJdVFIklPLJZCKOTjdDCab0ze2UCwWmpQOhwNKZfIljUaDwyHUVr2fZajiOlgRMlQ8z1Dd1gmlTIxsjaJPzWgwHMdBo5CivdMBk9XlC1YSiQQqlQqtra2QSqV9+lDJwHieR2trK1QqFSSS8MINBSuSFLqcbvz9WAM4Drj3qrEhnWNinhYOF+83fzvHccjNzUVVVRVqamoiVdwRRSQSobCwMOwB1BSsSFKw9sheD7ZW5VVWrO93u0wmw9ixY33NGRIcmUwWkRopBSuSFHzBKoS0haEQiURQKBTDcm4yNNQAJ0mhy+kZahPGUBmeFzrXW8y0QnM8omBFkoI1yIUi+tNksuGVf1Xh/WMNkSoWiSAKViQpWMPIXvfSeobcdNrd4HnKq4o3FKxIUvAOtQlnxoQUmRhiEQeeMZg9i6WS+EEd7CQpjM/VIC1FhrzU0DvBhVwrCQxdTpiszsFXuiFRRcGKJIVIzfapUUhh6HL6lqEn8YOagYT0oPYsO99JwSruULAiSaHqYicajVa4w+wYV8uFph/1WcUfagaShOfmGd47Wg8A+MX80rASQwv0SgB6jEqjQcvxhoIVSXje6Yw5DlBIw2sshDPNDBle1AwkCc+3UIRUnLCrDZPBUbAiCc/mEKa5Dich1IsxBqPViQaDlSbcizMUrEjC840LDGOojRfPgG1fVuHNg3W+5iWJDxSsSMILZ6GI3sQiznceyrWKLxSsSMKzRngl5hS5UEPrtFPNKp7Q1UCS8EZnpkApEyNTHZn1ItVyCVpgh8VGNat4QsGKJLxcnRK5ugB5UYwBracBUz2gzgKyJgGDzFqZ4un7omZgfKFgRZIX7wZOvQe0nune1nQSmHILIA48SFnlGXJjdVKwiifUZ0USXm1bFxoMVjjdvVbqPrdHCFQiCZAzBZDIgI4aoPKzAc/nncCP+qziCwUrkvA+ONmANw/Wwdyzj6mjGqg/LDye9GNgwo3CPQA0HAFMjQHPl6tTYHaJHuNzNMNXaBI0ClYkobl5BruzV1IoY921p7wZQMYY4bF+NJA9Udhf+1XAc2ZrFZg7JgNjsylYxRMKViSheYfaiDgOconn63zxDGBuEvqlSub5v6DwCs8xZ4Gu9iiWlISLghVJaN4sc6VM1D0usO6AcJ9fBsh6DUpWZwL6EqF21fxdv+dkjMHQ5UCDIfwpZ0jkULAiCa17vUDPhW1zE2C8AHAiYNTM/l+UPUm4bzklBK1+7Pi6Bm8erEOng64IxgsKViShdfXOXvd2qmeOA+QB+pwyLgHEEqEZaG7qs5vjuofcdNEVwbhBwYokNL9xgS67UFsCgFGzAr9IIgf0pcLjtrP9HuKdwK+LalZxg4IVSWgFeiUWjMsU0gxaTwNuF6BKB3T5A78w3RusKvvd7atZOahmFS8og50ktCyNAlkaz/Jbxzwd5tmThGlDB+KtWZmbALsFkKv9dnsTQylYxQ+qWZHkYDcDhhrhcfbEwY+XqwFNjvC4o6rPbhU1A+NO3Aer+vp6/OxnP0N6ejqUSiWmTJmCQ4cOxbpYJE5c6BCG2jgavxWu7OlGAcq0ob04rUi4N9T22eUNVlaqWcWNuG4GdnR0YO7cuVi4cCE++ugjZGZm4uzZs0hLG+KXkSS9T041w9DlRLn8BGRAd1rCUKQWAbX7+w1WealKzCnRI0sb+grPJLLiOlg9/fTTKCgowLZt23zbSkpKYlgiEm+6HG7IXWYoWCsgkwCZ44f+Yl2+0LdlNQA2I6DQ+XYNOO0MiYm4bga+//77uPTSS3HLLbcgKysLM2bMwMsvvzzga+x2O0wmk9+NJCeXm4fDxSPNWg2pmBOCT++M9YFI5N39Voa64SkkiZi4Dlbnz5/H1q1bMXbsWPzzn//Ev//7v+Pee+/F9u3bA75m8+bN0Ol0vltBQUEUS0yiyTvUJt1aA7GIAzLGBX8Snef70aspyPMMHZ0OWuUmjsR1sOJ5HjNnzsSTTz6JGTNm4K677sKaNWvwn//5nwFfs3HjRhiNRt+tro5+MZOV1eGGxG2F3tUEDhyQMTb4k3jzscwNfpvdjKHiq2q8ebAOjt7zZJGYiOtglZubi4kT/S9DT5gwAbW1fTtEveRyObRard+NJKcuhxtp1lpIRRygyQaUqcGfRJsn3HdeBFwO32apWCQ0LdG9LiGJrZCC1fnz5yNdjn7NnTsXp0+f9tt25swZFBUVReX9SXyzOt3QW6sgFYuE8X6hkGsAhVZIezD7T8inkHqnN6b0hXgQUrAaM2YMFi5ciNdeew02my3SZfL51a9+hW+++QZPPvkkzp07h507d+Kvf/0r1q5dO2zvSRJHdooI09UGZGnkoQcrANDkCvcm/6agd3wgBav4EFKwOnLkCKZOnYr169cjJycHd999Nw4cOBDpsqGsrAzvvvsudu3ahcmTJ+Oxxx7Dli1bsHz58oi/F0k8ensD8jRSZGRmAymZoZ9IO0q479VvpZBQYmg8CSlYTZ8+HS+++CIaGhrw6quvorGxET/60Y8wefJkPP/882htbY1YAW+88UacPHkSNpsN33//PdasWROxc5ME1+7pjtCXDj4WcCBaqlklgrA62CUSCZYtW4a33noLTz/9NM6dO4cNGzagoKAAd955JxobA0/KT0hYGIOx/jTMNifs2jD7MDW5wmR9dgtg687L886RZadgFRfCClaHDh3CPffcg9zcXDz//PPYsGEDKisr8cknn6ChoQE333xzpMpJiL+uNtQ2NOJEYyfqEUYTEBDmak/JEB736GQvSldhzmg9CvSq8M5PIiKk4TbPP/88tm3bhtOnT2Px4sXYsWMHFi9eDJFnpduSkhJUVFSguLg4kmUlpFv7eTjdPMzyPJQqIxBMNLmApUWYMiZTSC4dnanG6Ez1IC8k0RJSsNq6dStWrVqF8vJy5Obm9ntMVlYWXnnllbAKR0ggrK0STjcPgzrf17cUFk020AjA0hz+uciwCClYffLJJygsLPTVpLwYY6irq0NhYSFkMhlWrFgRkUIS4sfthLujFjwDDMqC7vUCw6HOFu57BCs3z2CyOuHk+e4J/kjMhNRnVVpaiosXL/bZ3t7eTrMikOFnqIXL6YRdooZboReSQsOVkiVcUbRbhBuAtk47Kr6qxntH68M/PwlbSP/KgQZ2WiwWKBT0C0SGWXsVnG4eRkV+9xJc4ZLIhLnbAV/tyns10OrgaTBzHAjqX3r9+vUAhKWKNm3aBJWqu2PT7XZj//79mD59ekQLSEgfhmpPsBoVmSaglzpLGCNoaQbSS33DbXjGYHfxvuckNoIKVkePHgUg1KxOnjwJmUzm2yeTyTBt2jRs2LAhsiUkpCdHJ2BphUomwbTSqZCpAqwNGAp1DtB8yreWoHcws9PNYHO6KVjFWFDB6rPPPgMArFy5Ei+++CLNaECizzPvlCI1G9NKR0X23Jq+newKqRhOtwtWpxupkX03EqSQGvw9pxkmJKq8k+SlFUf+3N4rglYD4LQBUgWUMjHMNhdsTpomJtaGHKyWLVuGiooKaLVaLFu2bMBj33nnnbALRki/OoTltlolOXAYrEhPkUWueSZVCvOw24xC7SqtqEcnOw25ibUhByudTgfOM1hUp9MNcjQhw8BuBrraAI7DFy0K1J6rw+IpuRiXE8F+K022X7C6JFuDHJ0CGWrZ4K8lw2rIwapn04+agSQmvE1AdRYsDikAR2SvBgJCvlXrGaBTmDlk8ij6YY4XIeVZWa1WdHV1+Z7X1NRgy5Yt+PjjjyNWMEL68DQBkVrkW9Y94sFKnSXcW1oie14StpCC1c0334wdO3YAAAwGA2bPno3nnnsON998M7Zu3RrRAhLi46lZubQFsDm9wSrCS196J/HrvAjwPJxuHu2dDrR3OgZ+HRl2Ic8UOm/ePADA22+/jZycHNTU1GDHjh146aWXIlpAQgAIQ2CsHQDHoVMpDJ4XizgopBFe80SZBoglAO8CrB2obLVg+1fV2PM9DXCOtZD+pbu6uqDRCJ2aH3/8MZYtWwaRSITLLrsMNTU1ES0gIQAAk2d8niodnW6hNpUil/gu+kQMxwn9VgDQ2eK7GmhzUepCrIW8YMR7772Huro6/POf/8Q111wDAGhpaaFEUTI8jJ71H3UF6LS7AABq+TBllPfot/IFK0pdiLmQgtWmTZuwYcMGFBcXY86cObj88ssBCLWsGTNmRLSAhAAAjJ6alW4UMjVyLByfhWkFqcPzXr6aVSvk3mDldNNg5hgLqXfypz/9KX70ox+hsbER06ZN822/+uqr8eMf/zhihSMEAOB2dg+B0eUjVSnDdNUw5j2pPZ3sPWpWLp7B6WaQSSLc7CRDFvKllJycHOTk5Phtmz17dtgFIqQPcyPAuwFZCqBIHf73814RtBkhZXZIRBxcPIPV6YZMEteLmCe1kIJVZ2cnnnrqKezZswctLS3gef/Ox2it2ExGCF8TMB/gOFzo6ALHcchQyyCXDEO/lVQprNJsM4HrvAiFVAyL3SWscqOURv79yJCEFKxWr16Nzz//HHfccQdyc3Mjf0WGkJ6MF4R7XT4A4LMfWnDR4sBPZuajMH2YVp5JyRKW5epswdT8YrgZgyLSCagkKCEFq48++ggffvgh5s6dG+nyEOKPMcDkH6wsduHKXMpwXQ0EhH6rtnOApRVzxs0avvchQxZSAzwtLQ16vT7SZSGkr642YboWsQRQZ8Pp5n3Z6ynyCGev99Qj14rEh5CC1WOPPYZNmzb5jQ8kZFh4m4CaPEAkRpenViURcZAPZ2e3b9hNK+xOF9o7HTDZnMP3fmRQIf00Pffcc6isrER2djaKi4shlfp3Oh45ciQihSOku79KmBXUGzA0imHIXu9JpQdEYsDlwNEzNfi6wY3phalYOC5r+N6TDCikYLV06dIIF4OQALzDbLRCf1V3sBrmq3IisbDajaUFalc7AB1lscdYSMHq4YcfjnQ5COnL0Ql0tQuPPTUrs00YaqONRgqBOguwtCDF6QlWLgpWsRRyD6XBYMDbb7+NyspK/OY3v4Fer8eRI0eQnZ2NUaMiPJE/GZm8+VUpGULuE4DRmSlQSMXQD2cGu5enk13laANQAquDBjPHUkjB6sSJE1i0aBF0Oh2qq6uxZs0a6PV6vPPOO6itrfXNdUVIWHqlLABAlkYRvaXcPcNuFI42AIDVSTWrWArpcsr69etRXl6Os2fP+q3AvHjxYnzxxRcRKxwZ4byd69oY1dQ9NSuZwwQR7/KlTJDYCClYHTx4EHfffXef7aNGjUJTU1PYhQrkqaeeAsdxuP/++4ftPUiccLt8i416a1aMMfzQZEK9wQqej8IMCHI1IFNBKgaUznY4XDzc0Xhf0q+QgpVcLofJZOqz/cyZM8jMzAy7UP05ePAg/vKXv2Dq1KnDcn4SZyxNnsHLKmH2TgjNsI9ONuGtQ3WIWshIyYJYxGGG3ok5o/XgaZqYmAkpWN1000149NFH4XQKl5E5jkNtbS0eeOAB/OQnP4loAQHAYrFg+fLlePnll5GWlhbx85M41LMJ6MmnMlm9k+5JIBZFaTyqOgscOJRlunBFaQakYpp1IVZC+uSfe+45WCwWZGZmwmq1Yv78+RgzZgw0Gg2eeOKJSJcRa9euxQ033IBFixYNeqzdbofJZPK7kQRk7Nu53tElLNqgi+bMB7TaTdwI6WqgTqfDJ598gi+//BLHjx+HxWLBzJkzhxRMgvXGG2/gyJEjOHjw4JCO37x5Mx555JGIl4NEEWPdyaA9gpWhS6jJp0YjbcHL08nuNDXBbLFDKZNASbMvxETQwYrneVRUVOCdd95BdXU1OI5DSUkJcnJywBiL6BCIuro63Hffffjkk0/8rjoOZOPGjVi/fr3vuclkQkFBQcTKRKLA2gE4uoQscnX3BI8GT80qTRXFmpUqHeBEqGy4iD1d3+HKKaWYkk8Ln8ZCUM1AxhhuuukmrF69GvX19ZgyZQomTZqEmpoalJeXR3xK48OHD6OlpQUzZ86ERCKBRCLB559/jpdeegkSiQRud99LyXK5HFqt1u9GEoxv8HKOMNuCR0csalZiCaDSQyIWQeVso1yrGAqqZlVRUYEvvvgCe/bswcKFC/32ffrpp1i6dCl27NiBO++8MyKFu/rqq3Hy5Em/bStXrsT48ePxwAMPQCym6nhS6qcJyBiDwSrUrFKjWbMCAHUWpKIapDjaKdcqhoIKVrt27cJvf/vbPoEKAK666io8+OCDeP311yMWrDQaDSZPnuy3LSUlBenp6X22kyTiuxKY77f5+sm5MHQ5kBrtqYXV2ULNytZONasYCqoZeOLECVx33XUB919//fU4fvx42IUiI5jTKizdDvgGLwNCekxJRgpmFKZBEu30gZRMSEQcVM42qlnFUFA1q/b2dmRnZwfcn52djY6OjrALNZC9e/cO6/lJjHlrVSq9sJpNPFBnQSLmoHQacdFui3VpRqygfqLcbjckksDxTSwWw+VyhV0oMoIZaoT71EK/zWebzfihyeRbjTmqZGqI5SkAGNDZFv33JwCCrFkxxlBeXg65XN7vfrvdHpFCkRHMUCvc9wpWB6s70Gyy4capuRibrYlumTgO8tQc5BiM0KXSdzxWggpWK1asGPSYSHWukxHIae3OFO8RrHieoc0iBIkMdf8/lMNNmZqLkowLgIaCVawEFay2bds2XOUgBDDUCdnrqnRA3l17umixw8UzyCSi6A616cm7pDytdhMzNCqTDJmbZ2DDOetAgCZgg1Ho1M7VKSCK1gDm3jzLgHW2N8BJ0xvHxDAuvEaSyYkLBnx+uhVKmRg/nZU/PFnkhmrhPq3Ib3OjwQoAyEtVRv49h0qVgZMNJtidbpRMaUfOME2FRAKjmhUZlKHLgc9+aIWLZyjUq6AdjpVlbCbA0ipMB9OjZsUYQ703WOliGKzEErgVwsK+DkNj7MoxglGwIoM6WmcAzxjyUhX4t4nZw9MUaz8v3Gty/fKrDF1OmG0uSMUc8lKjNPd6AG5VhnBvbo5pOUYqagaSAfE8w9lmMwCgrFg/fAuLtlcK9+mlfpvTUmS468rRuGixRz9zvbeULADfgze3xrYcIxTVrMiAms02dNrdkEtFKEpPgdsTvPaebolcZzvvBtqrhMf60j67U+QSFKXHQTZ7Ck3EF0sUrMiALnQI/UX5aSrfVMIfn2rG0VoDWi0Ryjky1AJup9D803TPX+V088N79TFIIo0w1IyztgkLWpCoomBFBmRzuiEWcchPEzq3xSIOozxX5erarZF5k9bTwn36GN986wDw8XfNeOvQBbSY4mM8nlSlhUskh8vlBrpo2E20UZ8VGdC8sZm4bHQ6elZwCvRKVF3sRL3BillFYS7gwbuB1h+Ex1kTfJsbjVacbTGDMQxfP1mQsnVKKLLzkepqAizNgCbwoH4SeRSsyKB6r+iS40khiEiNp6NaGGYjUwGpQn5Vl8OF3d82gTFgQq4WmZrYDLHpLS9VibyxY4ALBspkjwFqBpKgZarl4DjAbHPBEu4sCM3feU46HhCJYHO68d9H6mHockKjkGDBuDhLvqTVbmKGghUJ6JvzbXjtmxp8W2/02y6TiJCeImSwh1W7cnR1NwFzpsDh4vHe0XpcNNuRIhfjJzPzoZDGz9TVjDF0yTPQ5XCBmZuAOOr8HwkoWJGAmow2tJrt/S6ZnqUVEjQvWhyhv0HjcaHPSpMDaPPwxZlWNBptUEjFWDYzH2kpUVwYYgicboa/HjLhWL0ZvMMmrMJDoob6rEhArWYhNaG/PqPLStJxRWk61PIQv0JuJ3DBsxbkqJnCOUvT0dZpx5WXZMZsKpiBSMUcRGIJuqR6OHkXxJZmYUZTEhVUsyL9sjndvv6odHXfGo5OJYVGIQ39Sl39EcDRCSh0QLaw+IdaLsH/ubQAubEcAzgAjuOglIphkWXC7WaAmcYIRhMFK9Ivo1VYoy9FLoZcEuF+I5sJqNknPC66AiYH79sVL2kKgSikInTKMuDkeYDGCEYVBSvSrw7P6sepysD9RkdqO/D+8YbgOtl5Hjj9EeByANo8tKrH4dV9VfjwRCP4fvrG4o1CKkanLBMuNwMs1MkeTRSsSL8MvtWPA08HU9PWicoWC5pNQxx2w/PA2X8KMyyIJMC4xThRbwJjAAOL3cR6QVDKxOiSpsLJOMBpA2yGWBdpxKAOdtIvqViEdLWsu7/K0iIMi+lsEZI4AYy76ALrABw1uYBiNKBIFfqgJL1qY4wJGd+VnwIdntVrJiyBQ5GOH5qEqWGmjkqNzh8WJoVEDMYJneyARWgKKsPM4idDQsGK9GtWUZowlKarHTjxN6Ctss8xmVY7jGYLFNU/AI4j3TtkKYAyFZAoAd4FWNuFfipAqFFNuBHIGo/KRhMcLh46pRQF+vjsVO+tMF0FsZhDiikfsPwgNAWzxse6WCMCBSsSWPt54Lv3AJcd4ETCXFOpRZ7FHBjEHR1osFfCjE5M1MjB2QxC08jRKdx6EomBjEuAkit9l/urLwrHjMvRxH3Hutcl2Rpckq0B6ouAMz8A5qZYF2nEoGBF+mCMAR3V4E7+t1Az0o0Cxt/YJ6dIk86jri4DjAFzp4xGilwiNBFtRiFh0mUXgpRcKyR+Srpzp3ieoaa9CwBQnBEHc1UFS5Mr3Hsz2RMk2CYyClakj+bWFpz/6FXoZS6MnzwLmPRjIej0IhWLkKqUoqPLiTaLQwhWUqVw6zEvVX9azHZYHW7IJCLkaGM7XXEweJ7B5nLDKUqFjhN1B2dlaqyLlvQoWBF/jIH/9u/gXFZ0qnOBiUv7DVRe6Wo5LHYXOh3BDWhOVUnxbxOzYXe5fZP6JYJmsw1vHKiDRiHBanWm0MFubqRgFQUUrIi/xuNwG+rg5qQwly4BxAN/Rf5tYjZkYlHQaQcKqRiTR+nCKWlMqKTC52F1uMGy8sCZmwFTvd9cXGR4UJ4V6ea0Aef3wuZ044JuFtSpGYO+RCEVJ0R+VKSo5EIt08UzOFLyhI3G+hiWaOSgYEW6XTgIOK0wcDo0aSYNz0KmAOoNVhyp7YChK4wZG2JEKhZBJhH+23QpPDOFWpppTvYooGBFBE4bcOEgGBjOqaaDceIBs9d7+uRUM/7f19Xo6Bxa8DnVYMLnp1txrM4QRoFjJ0Um1K4snFq4mMC7hYBFhhUFKyKoPwy47HDK9WiSFQsLIyuHFqwuWuy4aHEMabUbxhhq2oT8quJ4WF4rBCrPtDhWFw/o8oWNJmoKDre4DlabN29GWVkZNBoNsrKysHTpUpw+fTrWxUo+vBtoEDLQ7aMuw+gsNfLTVENeVNQ799TFIQSr9k4HzDYXJCIOo9ISI2u9txSZEKw67S5A6+m3omA17OL6auDnn3+OtWvXoqysDC6XC7/97W9xzTXX4NSpU0hJScxf5bh08SxgtwAyFTQFU3BzUXBTwmR4xg8OZdbQ6jYhEXRUmrLPQhSJojhDBZVcLARpbpSw0dQQ20KNAHEdrHbv3u33vKKiAllZWTh8+DCuvPLKfl9jt9tht3f/wptMpmEtY1Lw1KqQO33AnKpAfDUr8+A1K28TMC5WWA7RpDwdJnmfuHKF7HWbSbgptLEsWlJLqJ82o1FYuECvDzyV7ObNm6HT6Xy3goKCaBUvMXW2CTMhcByQNx1ONz/4a3rxBiuj1Qm7yx3wOKebR71nhefidFVo5Y03EhmQ4lmBh5qCwyphghXP87j//vsxd+5cTJ48OeBxGzduhNFo9N3q6uqiWMoE1HxSuE8fAyh0ePNgHbburcSFjq4hn0IpE/vmYm8boCnYZnGAAdAoJNDH2WIQwWCMocvh6r76mVoo3BtqY1eoESCum4E9rV27Ft9++y327ds34HFyuRxyefwtNhCXGANavhceZ00EYwxGqxMOF+/rRB6qTI0cCqmo35VwvHJ0CvxifilMNmfCzLLQnxazHTv31wpDbuaNFoLVhUMUrIZZQgSrdevW4YMPPsAXX3yB/Pz8WBcneZibAKtBGFKTPgadDjccLh4ijoN2iGkLXjdNyxtSJrtMIorLlWuCofLkWXXa3WCMgdMVCM3ozovChQq5OsYlTE5x3QxkjGHdunV499138emnn6KkpCTWRUouLaeE+/SxgETma9ZolZKgBxcPFqhYEs1VrvLUOnnGYHPygEzV3W9lpG6H4RLXwWrt2rV47bXXsHPnTmg0GjQ1NaGpqQlWqzXWRUt8jHWvhuwZhOtd0Waomev9cfOs36bgiQvGfld3TkRiEQelN4vds1wZUouEe2oKDpu4DlZbt26F0WjEggULkJub67u9+eabsS5a4jPVC5faJTJAXwqgx4o2IY4J3P1tI/7vZ+dQ2963c77qYidazXbYnIGvFiYS7wWF7mBFnezDLa77rJKp6RB3vB3rGZf4poHp8KxokxZisOI4Di6e4UJHF0p6zP7pcPGo8wSwkkScFbQfGoUErWY7LDZvsKJ+q+EW1zUrMkx4vjtYZXbPw5SrU6A4Q9XvcvFDUeTJnfJmqXvVtnfCxTPolNKETlnoSaMQArzZJgR4SJWAOkt43FEVo1Ilt7iuWZFhYqwTFnSQKgB990WLsmI9yooDJ9wOpkifAo4TMtnNNic0CqHv6/tGMwBgTJY6oVMWeirUq8BxHHJTe4xv1I8WZg5tPw/kTIld4ZIU1axGIl8TcFxIw2sCUcrEyNUJ86mfbbEAEGbUrPKsYjMxL3mGoozJ0mDhuCz/Zq1+tHDffl6ovZKIomA10vB8j6uA3evd2ZzuiHR+j8sRAtK39UYwxnCszgA3z5CtVSR8ftWgtPnCCj5OmzAvO4koClYjjaFaWJFFpgJSi32bTzWasHVvJXZ/G946eBNyNZBJRGizOFDZ2onSrBSMyVKjrDi5Vi32DrlpMdm6LwSJRN3N6va+i8KS8FCf1Ujj61gfL/zn8vBOMeztOA6VXCLG5aXpAIDRGSkQiTgsmZYX1jnjkZtn+Mvn5wEA/76gFAqppzmtHw20/CCsYF3S/8wgJDQUrEYS3g20eiYvzPRf8ryjM/yEUK+ZhclVi+qPRCyCUiaG1eGG2ebqEaxKhRQGcxNNGRNh1AwcSdqrhFWS5WpA5z91jjchNNQcq5GoT/oCIHy2Ws+EfBfPxKBUyYuC1UjiHQuYOcGvCeh08zB7khspWA2dNzXDZOu1so231uq9kEEigoLVSOF2dv/SZ/VuAgq1KqVM7BvzRgbnXVCjz5JimZcI98YLQjY7iQgKViNFW6UQsBTa7maKh3dVmqRPLYgwb/+edwC4j0IHaHOFweIXaYGTSKFgNVJ4m4BZE4QO4B7SVDJML0hFaWZyjNuLFp2vZuXsu9PbFGw+FcUSJTe6GjgSOG1CzQoAsib12Z2XqkReamIuixVL6Wo5phemQt9fP1/2JOD8XqEp2NUOqEIfxkQEVLMaCS6eAXgXoErvHmxLwqaWS7BwXBamFaT23SnXdA+/aToZ1XIlKwpWI4G3CZg9sU8T0Onm0WS0weGisWwR5x3M3HSSxgpGAAWrZGe3CEttAUDWxD67m0027DpQi//3TU2UC5YcvMG+vbOfVX3SxwozW9jNNG1MBFCwSnatPwCMBzQ5/fabtJi9VwIpvyoUX1e2YdeBWhy/YOi7UywBsj21q/rDUS1XMqJglcwYAxqPCY8DzK/UbLQJu7WKKBUquaR7gnzA1ahHzRSa3m2VwoKyJGQUrJKZuRGwtAIiiXB1qh/NJiFYZVOwCkmmJzetrdPR/zTcKr2wgCwA1B+KYsmSDwWrZNZ4XLjPHCdMu9uLzen2zbtOwSo0aSkycJwwyWCnI8B8YPmXCvdNJyijPQwUrJKVy9F9FTB3Wr+HtJiEpotOKaVhNiGSikW+8ZRtlgBNwdQiIaPd7QLqvoli6ZILBatk1XRCCFgqffcyUb1cMAgLO3inIiah8Q5Tag3Ub8VxQPE84XH9UeHqIAkaBatkxPPAhYPC4/yyPrlVXuNztLjyksykmhs9FnJ0QrBq8Fys6Jd+NKAbJSTn1nwVpZIlFxpuk4zazgJWg5DjM8AqK/oUWdIsjRVLxekpYGOB/DRV4IM4DiiZDxzbCTQcFZrmmpzoFTIJUM0q2TAGVO8THufNBMThz/xJBpauluPSYj1yBmtOpxUJA8kZA85+LNyTIaNglWxafwAsLcKy8PllAQ/7tt6I7xqMSbOce8IovUr4ATHWAw1HYl2ahELBKpnwbqDqC+FxwRxhBZt+MMbwzfk2fPxdM+oN1igWMHnZXW6cbjLjSG3HwAcqtMDoBcLjc58KeXBkSChYJZO6A8J0JDLVgLWqFrMdZpsLMokIhfoB+lnIkBm6nPjHyUZ8XdkGNz9I827ULKHDnXcB3/9duGpLBkXBKll0tQM1nr6q0quExTYD+LbeCAAoyUiBVExfgUjI0siRIhfD4eJR29418MEcB4y/QfhRsbQCp/5OszIMAX1Tk4HbBZx6T7hPKwKyJwc81OHi8UOTkOczZZQuSgVMfhzHYWy2BkD3j8GA5Gpg8k+FoVBt54AzH1HAGgQFq0THmPBFNzcLQ2rG3xgwrwoAjtUZ4HDxSFNJkZ9Gs4NGkjf4n2/t9F+eKxDdKGDiTcK/V+OJ7h8c0i8KVomM54VL4E3fApwImLBkwEU1O+0uHKppBwBcVpoOboCgRoKXoZYjP00JnjF8c759aC/KHAdMXAqIxMICtEd3CE160kdCBKs///nPKC4uhkKhwJw5c3DgwIFYFyn27Bbg2/8G6j2Xv8cvBtJLB3wJxwF5OiWytHKM8zRZSGTNHZMBAPiuwYi6wfquvLLGA1P/j1AzNjcDh14Bqv5FHe+9xH2wevPNN7F+/Xo8/PDDOHLkCKZNm4Zrr70WLS0tsS5abLjsQO1+4ODLQl+HSAJMvHnATHUvlUyCm6fnYdmMfKpVDZO8VCUm5WmRo1VApwoiITetGLh0ldDn6HYJib3f/Bmo9KQ3UAIpONbvJDzxY86cOSgrK8Of/vQnAADP8ygoKMAvf/lLPPjgg4O+3mQyQafTwWg0QqtNoDFwjAFuB+DoBGxGoLMVMNQKS8Dznn4NdZZwVamfYRs2pxuGLidq2jpxocOKZTNHUYCKEqebh4jjIBYJn7ehywGT1YUsrRxyiWjgfwfGhMTe858D1h45Wwqd0MelzhEW/pBrhE56iUJoQiaoYP5/xvXYQIfDgcOHD2Pjxo2+bSKRCIsWLcLXX3/d72vsdjvs9u7R7yaTaWhvVvMV0Pxd3+1+sZyFv81v+wDb3E40GTp9k+MJuxkYAJs0Dc26abhi5nxkaYS1/o7VGbD/fBsYADfP+iwAUdnaiTFZ6r5/H4m43ukgx+oMOFprAABIRBwUUjHEIg4cB3AAbptdCIVUCDj7q9pxpkUFsGuhdddAbzoFjbUeOlknSmzG/tchFImFrHiRVOi75DgAXK97j2j+YMk0wLRbI3a6uA5WFy9ehNvtRnZ2tt/27Oxs/PDDD/2+ZvPmzXjkkUeCfzO7Bei8GEoxh43TzcPsFMEpVqFLpkenNAMdykJ0SfUAx+EydP+iutw8unpN/qaUiZGrU6A0U43idEr+jBWxiINWKYXJ6oSLZ7DYA1/xs9hdvimSLyIP5zV5EKW4MCnFgJICAOYmwGYQpplxePrEeLdwwwCzPsSCIrJ9bnEdrEKxceNGrF+/3vfcZDKhoKBg8BeOmgVkjO31y+N5HM42v+1BbBNLoHVKUeTs3sVxHDjvY3B+MyZMzNOiOCMFHAARxyFFLoFMEvddkiPCvLGZmDc2Ew4XD6vTDbvTDTdj4Jkw9KlnTWxGYRrGZvW9+KGQiQBNr4HSvFvoKnA7ALdTuIF5quGB7qNIFNnwEtfBKiMjA2KxGM3NzX7bm5ubkZPT//Qacrkccnng7O2AUtKFWxzRKYChpm2qZBKoZHH9zzniySQi4QdEGbjjPahpe0RiQKTsd8rqZBTXP70ymQyzZs3Cnj17fNt4nseePXtw+eWXx7BkhJBoi/uf4vXr12PFihW49NJLMXv2bGzZsgWdnZ1YuXJlrItGCImiuA9Wt956K1pbW7Fp0yY0NTVh+vTp2L17d59Od0JIcov7PKtwJWyeFSEjQDD/P+O6z4oQQrwoWBFCEgIFK0JIQoj7DvZwebvkhjzshhASNd7/l0PpOk/6YGU2C7NiDimLnRASE2azGTrdwCnQSX81kOd5NDQ0QKPRxHTWAe+wn7q6Oroq2Qt9NoEl+2fDGIPZbEZeXh5EooF7pZK+ZiUSiZCfnx/rYvhotdqk/NJFAn02gSXzZzNYjcqLOtgJIQmBghUhJCFQsIoSuVyOhx9+OLQZIZIcfTaB0WfTLek72AkhyYFqVoSQhEDBihCSEChYEUISAgUrQkhCoGAVQcGsHP3yyy9j3rx5SEtLQ1paGhYtWpTUK02Huqr2G2+8AY7jsHTp0uEtYAwF+9kYDAasXbsWubm5kMvluOSSS/CPf/wjSqWNIUYi4o033mAymYy9+uqr7LvvvmNr1qxhqamprLm5ud/jb7/9dvbnP/+ZHT16lH3//fesvLyc6XQ6duHChSiXfPgF+9l4VVVVsVGjRrF58+axm2++OTqFjbJgPxu73c4uvfRStnjxYrZv3z5WVVXF9u7dy44dOxblkkcfBasImT17Nlu7dq3vudvtZnl5eWzz5s1Der3L5WIajYZt3759uIoYM6F8Ni6Xi11xxRXsv/7rv9iKFSuSNlgF+9ls3bqVjR49mjkcjmgVMW5QMzACvCtHL1q0yLdtsJWje+vq6oLT6YRerx+uYsZEqJ/No48+iqysLPz85z+PRjFjIpTP5v3338fll1+OtWvXIjs7G5MnT8aTTz4Jt9vd7/HJJOkHMkdDKCtH9/bAAw8gLy/P74ubDEL5bPbt24dXXnkFx44di0IJYyeUz+b8+fP49NNPsXz5cvzjH//AuXPncM8998DpdOLhhx+ORrFjhoJVHHjqqafwxhtvYO/evVAoFIO/IImZzWbccccdePnll5GRkRHr4sQdnueRlZWFv/71rxCLxZg1axbq6+vxhz/8gYIVGVwoK0d7Pfvss3jqqafwv//7v5g6depwFjMmgv1sKisrUV1djSVLlvi28TwPAJBIJDh9+jRKS0uHt9BREsr3Jjc3F1KpFGKx2LdtwoQJaGpqgsPhgEw2xNWcExD1WUVAqCtHP/PMM3jsscewe/duXHrppdEoatQF+9mMHz8eJ0+exLFjx3y3m266CQsXLsSxY8eSasbXUL43c+fOxblz53wBHADOnDmD3NzcpA5UACh1IVLeeOMNJpfLWUVFBTt16hS76667WGpqKmtqamKMMXbHHXewBx980Hf8U089xWQyGXv77bdZY2Oj72Y2m2P1JwybYD+b3pL5amCwn01tbS3TaDRs3bp17PTp0+yDDz5gWVlZ7PHHH4/VnxA1FKwi6I9//CMrLCxkMpmMzZ49m33zzTe+ffPnz2crVqzwPS8qKmIA+twefvjh6Bc8CoL5bHpL5mDFWPCfzVdffcXmzJnD5HI5Gz16NHviiSeYy+WKcqmjj6aIIYQkBOqzIoQkBApWhJCEQMGKEJIQKFgRQhICBStCSEKgYEUISQgUrAghCYGCFSEkIVCwIkmloqICqampvue///3vMX36dN/z8vLypJ4iOZlRsCIhKS8vB8dx+MUvftFn39q1a8FxHMrLy/2Oj3SQKC4uxpYtW/y23XrrrThz5kzA17z44ouoqKjwPV+wYAHuv//+iJaLDA8KViRkBQUFeOONN2C1Wn3bbDYbdu7cicLCwpiUSalUIisrK+B+nU7nV/MiiYOCFQnZzJkzUVBQgHfeece37Z133kFhYSFmzJgR1rn7q/EsXbrUV1tbsGABampq8Ktf/Qocx4HjOAB9m4G99azhlZeX4/PPP8eLL77oO0dVVRXGjBmDZ5991u91x44dA8dxOHfuXFh/FwkdBSsSllWrVmHbtm2+56+++ipWrlw57O/7zjvvID8/H48++igaGxvR2NgY9DlefPFFXH755VizZo3vHIWFhX3+JgDYtm0brrzySowZMyZSfwIJEgUrEpaf/exn2LdvH2pqalBTU4Mvv/wSP/vZz4b9ffV6PcRiMTQaDXJycgadkbU/Op0OMpkMKpXKdw6xWIzy8nKcPn3at36f0+nEzp07sWrVqkj/GSQINK0xCUtmZiZuuOEGVFRUgDGGG264IeHnTs/Ly8MNN9yAV199FbNnz8b//M//wG6345Zbbol10UY0qlmRsK1atQoVFRXYvn17xGofIpEIvadaczqdETn3UKxevdp38WDbtm249dZboVKpovb+pC8KViRs1113HRwOB5xOJ6699tqInDMzM9OvH8rtduPbb7/1O0Ymk4W9Xl6gcyxevBgpKSnYunUrdu/eTU3AOEDNQBI2sViM77//3vc4EKPR2GctwPT09H4Xgbjqqquwfv16fPjhhygtLcXzzz8Pg8Hgd0xxcTG++OIL3HbbbZDL5SE1P4uLi7F//35UV1dDrVZDr9dDJBL5+q42btyIsWPHDrjwB4kOqlmRiNBqtdBqtQMes3fvXsyYMcPv9sgjj/R77KpVq7BixQrceeedmD9/PkaPHo2FCxf6HfPoo4+iuroapaWlyMzMDKncGzZsgFgsxsSJE5GZmYna2lrfvp///OdwOBxRubpJBkdzsBMSwL/+9S9cffXVqKur67NqMok+ClaE9GK329Ha2ooVK1YgJycHr7/+eqyLREDNQEL62LVrF4qKimAwGPDMM8/EujjEg2pWhJCEQDUrQkhCoGBFCEkIFKwIIQmBghUhJCFQsCKEJAQKVoSQhEDBihCSEChYEUISwv8HZSCgXKDbVAwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:44:43.137999Z", + "iopub.status.busy": "2024-03-01T18:44:43.137731Z", + "iopub.status.idle": "2024-03-01T18:44:43.451148Z", + "shell.execute_reply": "2024-03-01T18:44:43.450219Z" + }, + "papermill": { + "duration": 0.335822, + "end_time": "2024-03-01T18:44:43.453096", + "exception": false, + "start_time": "2024-03-01T18:44:43.117274", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEqCAYAAABN+HYRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBCElEQVR4nO3deZxT5b0/8M/JnkwyyWyZmcw+A7LIvhYpKsqVWxShVqu3Lowotlfaey2XVrj9VasWQatWXF60WmG0iqgtUmvduCiiosg2CgIDDLMx+5Z9zzm/P06SWTNL9mS+79crryTnnJw8k5n55nme8zzfh+E4jgMhhMQ5QawLQAghI0HBihCSEChYEUISAgUrQkhCoGBFCEkIFKwIIQmBghUhJCFQsCKEJAQKVoSQhEDBihCSEGIarA4cOIDly5dDp9OBYRjs2bOnz36O4/DAAw8gNzcXcrkcS5Yswblz52JTWEJITMU0WFksFkyfPh3PP//8oPsff/xxPPPMM/jTn/6EQ4cOISUlBUuXLoXdbo9ySQkhscbEy0RmhmHw9ttvY+XKlQD4WpVOp8P//M//YP369QAAg8GA7OxsVFRU4JZbbhnReVmWRVNTE1QqFRiGiVTxCSFB4DgOJpMJOp0OAsHQdSdRlMo0ajU1NWhpacGSJUv829RqNebPn48vv/wyYLByOBxwOBz+542NjZg8eXLEy0sICV5DQwPy8/OHPCZug1VLSwsAIDs7u8/27Oxs/77BbN68GQ899NCA7Q0NDUhNTQ1vIQkhITEajSgoKIBKpRr22LgNVsHauHEj1q1b53/u+zBSU1MpWBESp0bSRRO3QxdycnIAAK2trX22t7a2+vcNRiqV+gMTBShCkkfcBquSkhLk5ORg3759/m1GoxGHDh3CggULYlgyQkgsxLQZaDabcf78ef/zmpoaVFZWIj09HYWFhbjvvvvw+9//HuPHj0dJSQl++9vfQqfT+a8YEkLGjpgGqyNHjmDx4sX+576+plWrVqGiogK//vWvYbFYcM8990Cv1+P73/8+PvjgA8hksrCWg+M4uN1ueDyesJ6XjJxQKIRIJKLhJSSguBlnFSlGoxFqtRoGg2HQ/iun04nm5mZYrdYYlI70plAokJubC4lEEuuikCgZ7v+zt6S7GjgaLMuipqYGQqEQOp0OEomEvtljgOM4OJ1OtLe3o6amBuPHjx92gCAZe8Z0sHI6nWBZFgUFBVAoFLEuzpgml8shFotRV1cHp9MZ1qb+8fpunGszY2qeGpNy6epwoqKvL4C+xeNEJH4P59vM2F/VjjajHVkqadjPT6KH/ktJUjtc2wUAuDRPjUwlBatERsEqQV155ZW47777Yl2MuNZtcaLFYIdQwGB+SXqsi0NCRMEqye3fvx8Mw0Cv18e6KFFX18Vf4dVp5FBIRDA73Dh0oRMHz3fEuGQkGBSsSNKq9warwnT+4onJ7sLB6k4cb9DD7WFjWTQSBApWScDhcOD+++9HQUEBpFIpxo0bh5deegm1tbX+QbdpaWlgGAbl5eXDns9kMuHWW29FSkoKcnNz8cc//nFAs/Ovf/0r5syZA5VKhZycHPzkJz9BW1ubf7+vRrdv3z7MmTMHCoUCl112GaqqqsL94wfUauCTNOalyQEAOakyqGQiON0sGvW2qJWDhAcFq0E43WzAW/9v5KGOdY3w2FDdcccdeP311/HMM8/g9OnT+POf/wylUomCggL8/e9/BwBUVVWhubkZW7duHfZ869atwxdffIF33nkHe/fuxWeffYZjx471OcblcuGRRx7BN998gz179qC2tnbQQPib3/wGTz75JI4cOQKRSITVq1eH/POOBMdxmF2chsm6VGR5O9YZhkGBt5blq3WRxDGmx1kF8vwn5wPuK8lMwcqZef7nLxyohssz+CSA/DQ5bppT4H++/Ysa2JwDp/T88t8uCbqsZ8+exZtvvom9e/f6ExWWlpb696en8x3LWq0WGo1m2POZTCa8/PLL2LlzJ66++moAwI4dO6DT6foc1zvolJaW4plnnsHcuXNhNpuhVCr9+zZt2oQrrrgCALBhwwZce+21sNvtYZ8y1R/DMJhVmDZge2G6AqeajGjooppVoqGaVYKrrKyEUCj0B4RQXbhwAS6XC/PmzfNvU6vVmDBhQp/jjh49iuXLl6OwsBAqlcr//vX19X2OmzZtmv9xbm4uAPRpLkabr2bVZrLD7qK5oImEalaDWLt4XMB9gn6zce65vCzgsf1n7qxeWBJKsQYll8vDfs7hWCwWLF26FEuXLsVrr72GrKws1NfXY+nSpXA6nX2OFYvF/se+qUwsG/nO7RaDHQIBkKaQQCzs+U5WSkXIUErQaXaiUW9DWZZyiLOQeEI1q0FIRIKAN5FQMOJjxSM8NhRTp04Fy7L49NNPB/9ZvJOCR5pRorS0FGKxGIcPH/ZvMxgMOHv2rP/5mTNn0NnZiS1btmDRokWYOHFiTGtLg9lf1YbXvqpHTYdlwL5ctRwSkQBWB9WsEgkFqwRXXFyMVatWYfXq1dizZw9qamqwf/9+vPnmmwCAoqIiMAyDd999F+3t7TCbzUOeT6VSYdWqVfjVr36FTz75BN999x3uuusuCAQCf82osLAQEokEzz77LC5cuIB33nkHjzzySMR/1tHQ21wAAI1CPGDfovGZ+M8ryjA1Xx3tYpEQULBKAtu2bcONN96Ie++9FxMnTsSaNWtgsfA1iry8PDz00EPYsGEDsrOz8fOf/3zY8z311FNYsGABrrvuOixZsgQLFy7EpEmT/J3iWVlZqKiowFtvvYXJkydjy5YteOKJJyL6M46G3eXxX8hQywcGK5lYCEH/9jyJe2M6n5XdbkdNTQ1KSkoifnUqkVksFuTl5eHJJ5/EXXfdFbH3Cdfvo9Vox85D9UiRCofsUwT4IQ6UFih2KJ8VCcnx48dx5swZzJs3DwaDAQ8//DAAYMWKFTEu2ch0W/lOfo08cBK/o3XdOHFRj+kFGswcZIgDiT8UrMaY+vr6IRd9PXXqFADgiSeeQFVVFSQSCWbPno3PPvsMmZmZ0SpmSAxWvr9KPUh/lY/Lw6Lb6kKLd5Q7iX8UrMYYnU6HysrKIfcXFhbi6NGj0StUmJnsbgCAShb4zzsnlW9mthopWCUKClZjjEgkwrhxgceRJYMJOSpoFGLoNIHHoGlT+Sk43VYXHG4PpCJhtIpHgkTBiiSdgnSFf6R6IAqJCCqZCCa7G21Gx7DHk9ijoQtkzNJ6m4JtJkeMS0JGgoIVSSpuD4vqdjPajHYMNyon25uTvY36rRICBSuSVAw2F96pbMJbRy8OO34qRy1Dpko65FVDEj+oz4okFbODvxKYOsSVQJ+ijBTcnpES6SKRMKGaFUkqvmELyhEEK5JYKFglqHhb3SZeyuOrWamkI2/asSxHua0SAAWrMax/7qlkYHXywUohHdm4qeP13Xjuk/P4/ByteBPvKFgloPLycnz66afYunUrGIYBwzCorq7GXXfdhZKSEsjlckyYMGFAvvXy8nKsXLkSmzZtgk6n82f/PHjwIGbMmAGZTIY5c+Zgz549YBimz0j3kydP4gc/+AGUSiWys7Nx++23o6OjI2B5amtro/Vx9GH1ZltQSEbWDFRIRPCwHDrMNHwh3lHDvjeOAzyu2Ly3UDwwtWgAW7duxdmzZzFlyhT/JOO0tDTk5+fjrbfeQkZGBg4ePIh77rkHubm5+PGPf+x/7b59+5Camoq9e/cC4Ge9L1++HMuWLcPOnTtRV1c3oDmn1+tx1VVX4e6778Yf//hH2Gw23H///fjxj3+Mjz/+eNDyZGVlheFDGb2eYDWymlWGkp/s3GlxUgaGOEfBqjePC/jsydi896L/AUSBswT0plarIZFIoFAokJOT49/+0EMP+R+XlJTgyy+/xJtvvtknWKWkpOAvf/mLP4Pon/70JzAMgxdffBEymQyTJ09GY2Mj1qxZ43/Nc889h5kzZ+LRRx/1b9u+fTsKCgpw9uxZXHLJJYOWJxZmF6WhLEsJrWpkS8WnKSQQChg43SyMdveg+a9IfKBglUSef/55bN++HfX19bDZbHA6nZgxY0afY6ZOneoPVAC/RNe0adP65I/qvVgEAHzzzTf45JNP+qxa41NdXY1LLgl+dZ5wG21OdaGAQVqKBB0mBzrNDgpWcYyCVW9CMV/DidV7h2DXrl1Yv349nnzySSxYsAAqlQp/+MMfcOjQoT7HpaSMflyR2WzG8uXL8dhjjw3Y51uxJpFleoNVh9mJ0ti0XskIxHWw8ng8+N3vfodXX30VLS0t0Ol0KC8vx//7f/8vMn0LDDPiplisSSSSPotAfPHFF7jssstw7733+rdVV1cPe54JEybg1VdfhcPhgFTKN516LxYBALNmzcLf//53FBcXQyQa/E+mf3liwelmcbHbihSpCNmpI880mqGUAjChkzrZ41pcXw187LHHsG3bNjz33HM4ffo0HnvsMTz++ON49tlnY120mCsuLsahQ4dQW1uLjo4OjB8/HkeOHMGHH36Is2fP4re//e2AoDOYn/zkJ2BZFvfccw9Onz6NDz/80J9P3feFsHbtWnR1deE//uM/cPjwYVRXV+PDDz/EnXfe6Q9Q/csTjeW2+jPYXPhHZRPePt44qtfpNDJMyFENmVKGxF5cB6uDBw9ixYoVuPbaa1FcXIwbb7wR11xzDb7++utYFy3m1q9fD6FQiMmTJyMrKwtLly7FDTfcgJtvvhnz589HZ2dnn1pWIKmpqfjnP/+JyspKzJgxA7/5zW/wwAMPAIC/H0un0+GLL76Ax+PBNddcg6lTp+K+++6DRqOBQCAYtDz9FzuNBt/AzpFeCfTJT1Ng2dRcTC/QRKBUJGy4OLZp0yauqKiIq6qq4jiO4yorKzmtVsu9+uqrAV9jt9s5g8HgvzU0NHAAOIPBMOBYm83GnTp1irPZbBH7GRLRq6++yonFYs5qtUb1fUP9fZxpNnJPfVTFvXm4PswlI5FiMBgC/n/2F9d9Vhs2bIDRaMTEiRMhFArh8XiwadMm3HrrrQFfs3nz5j6X8MnwXnnlFZSWliIvLw/ffPONfwxVLFZ7DoVv9Lp8lDUrgF/lxmh3QyoSQCamrKHxKK6bgW+++SZee+017Ny5E8eOHcPLL7+MJ554Ai+//HLA12zcuBEGg8F/a2hoiGKJE1NLSwtuu+02TJo0Cb/85S9x00034YUXXoh1sUbNFmQzEAD++W0ztn9eg7OtpnAXi4RJXNesfvWrX2HDhg245ZZbAPBjhOrq6rB582asWrVq0NdIpVL/VS0yMr/+9a/x61//OtbFCJlvYVO5ePR/1r7xVd3WGM1gIMOK65qV1Wr1d+D6CIXCmFxpIvFvtFNtekvzJuDTW5NvcneyiOua1fLly7Fp0yYUFhbi0ksvxfHjx/HUU09h9erVsS4aiUPT8tXQaeRBDUFIU/Dj67otFKziVVwHq2effRa//e1vce+996KtrQ06nQ4//elP/ZfWw4UbJlc3iY5Qfw9FGSkoCjLzp8ZbszLY3GBZDgIBTWiON3EdrFQqFZ5++mk8/fTTETm/WMz/gVqt1oS78pWMrFYrgJ7fSzQppSKIhQxcHg5GuwsaRWLMZBhL4jpYRZpQKIRGo0FbWxsAQKFQUIqQGOA4DlarFW1tbdBoNBAKR9/nxLIcajstkEuEyFbJRl0zYhgGKpkYXRYnjDY3Bas4NKaDFQB/ShNfwCKxo9Fogk4xY3V58I/KJjAM8F9XjQ/qHJN1qXC6WcrfHqfG/G+FYRjk5uZCq9XC5aLL1rEiFouDqlH59AxbEAbd3zS3OD3o9yeRN+aDlY9QKAzpn4XElj9YBTFsgSSGuB5nRchIWV3eqTYhTJVhWb5zvc1EKzTHIwpWJCnYRrlQxGBajHa89FkN3qlsClexSBhRsCJJwRbC6HWfVO+UG4vDA5alsXfxhoIVSQq+qTahZExIkQghFDBgOQ4m72KpJH5QBztJChNzVUhLkUCnGXk64/74sVYi6K0uGG0uWjwizlCwIkkhP02B/DRFyOdRycTQW13+ZehJ/KBmICG9KL3LzlsoWMUdClYkKdR0WNBssMETYse4Uso3/ajPKv5QM5AkPA/LYY93RZufXVEW0sDQgnQ5gHTkpdHE9nhDwYokPF86Y4YBZOLQGguhpJkhkUXNQJLw/AtFiIWUNSOJUbAiCc/u5NNchzIg1IfjOBhsLjTpbZSUMc5QsCIJzz8vMISpNj4sB+z4ogZvHG7wNy9JfKBgRRJeKAtF9CcUMP7z0Fir+ELBiiS83rmswiFFytfQLA6qWcUTuhpIEl5pVgrkEiGylOFZL1IpFaENDpjtVLOKJxSsSMLLVcuRqw4wLorjgPYqwNgIKLWA9lJAMHSDIsXb90XNwPhCwYokL9YDnNoDtJ/t2dZyAph6EyAMPElZ4Z1yY3NRsIon1GdFEl59pxVNehtcnn4rdZ/fxwcqgQjImQqIJEB3HVD9yZDn8yXwoz6r+ELBiiS8d0804Y3DDTD17mPqrgUaj/KPL/0hMOk6/h4Amo4BxuaA58tVyzCvJB0Tc1SRKzQZNQpWJKF5WA4OV79BoRzXU3vSzQQyx/GP00uB7Mn8/vqDAc+ZnSrDwnGZGJ9NwSqeULAiCc031UbAMJCKvH/OHWcBUwvfL1WyqO8LCi/zHnMOsHZFsaQkVBSsSELzjTKXSwQ98wIbvubv8+cCkn6TkpVZQHoJX7tq/W7Qc3IcB73ViSZ96ClnSPhQsCIJrWe9QO+FbVMLYLgIMAIgb9bgL8q+lL9vO8UHrUG88mUd3jjcAIuTrgjGCwpWJKFZ+49e93WqZ00ApAH6nDIvAYQivhloahmwm2F6ptxY6Ypg3KBgRRJan3mBbgdfWwKAvNmBXySSAull/OPOc4Me4kvgZ6WaVdygYEUSWkG6HFdOyOKHGbRXAR43oMgA1PlDvzDDF6yqB93tr1k5qWYVL2gEO0loWpUMWpV3+a1Kb4d59qV82tCh+GpWphbAYQakyj67fQNDKVjFD6pZkeTgMAH6Ov5x9uThj5cqAVUO/7i7ZsBuBTUD407cB6vGxkbcdtttyMjIgFwux9SpU3HkyJFYF4vEiYvd/FQbZ/NJ/sqeOg+Qp43sxWlF/L2+fsAuX7CyUc0qbsR1M7C7uxsLFy7E4sWL8f777yMrKwvnzp1DWtoI/xhJ0tt7qhV6qwvl0m8hAXqGJYyEpgioPzRosNJp5Jhfkg5tavArPJPwiutg9dhjj6GgoAA7duzwbyspKYlhiUi8sTo9kLpNkHHtgEQEZE0c+YvV+Xzflk0P2A2ATO3fNWTaGRITcd0MfOeddzBnzhzcdNNN0Gq1mDlzJl588cUhX+NwOGA0GvvcSHJye1g43SzSbLUQCxk++PQfsT4UkbSn30rfEJlCkrCJ62B14cIFbNu2DePHj8eHH36I//zP/8R//dd/4eWXXw74ms2bN0OtVvtvBQUFUSwxiSbfVJsMWx2EAgbInDD6k6i9fx/9moIsy6Hb4qRVbuJIXAcrlmUxa9YsPProo5g5cybuuecerFmzBn/6058Cvmbjxo0wGAz+W0MDfWMmK5vTA5HHhnR3CxgwQOb40Z/ENx7L1NRns4fjUHGwFm8cboCzf54sEhNxHaxyc3MxeXLfy9CTJk1Cff3ADlEfqVSK1NTUPjeSnKxOD9Js9RALGECVDcg1oz9Jqo6/t3QAbqd/s1go4JuW6FmXkMRWUMHqwoUL4S7HoBYuXIiqqqo+286ePYuioqKovD+JbzaXB+m2GoiFAn6+XzCkKkCWyg97MPVNyCcT+9Ib0/CFeBBUsBo3bhwWL16MV199FXa7Pdxl8vvlL3+Jr776Co8++ijOnz+PnTt34oUXXsDatWsj9p4kcWSnCDBDqYdWJQ0+WAGAKpe/N/ZtCvrmB1Kwig9BBatjx45h2rRpWLduHXJycvDTn/4UX3/9dbjLhrlz5+Ltt9/G66+/jilTpuCRRx7B008/jVtvvTXs70UST7qjCTqVGJlZ2UBKVvAnSs3j7/v1W8lENDA0ngQVrGbMmIGtW7eiqakJ27dvR3NzM77//e9jypQpeOqpp9De3h62Al533XU4ceIE7HY7Tp8+jTVr1oTt3CTBdXm7I9LLhp8LOJRUqlklgpA62EUiEW644Qa89dZbeOyxx3D+/HmsX78eBQUFuOOOO9DcHDgpPyEh4TgYGqtgsrvgSA2xD1OVyyfrc5gBe8+4PF+OLAcFq7gQUrA6cuQI7r33XuTm5uKpp57C+vXrUV1djb1796KpqQkrVqwIVzkJ6cvaifqmZnzbbEEjQmgCAnyu9pRM/nGvTvaiDAXml6ajIF0R2vlJWAQ13eapp57Cjh07UFVVhWXLluGVV17BsmXLIPCudFtSUoKKigoUFxeHs6yE9Oi6AJeHhUmqQ5k8DMFElQuY2/iUMVn84NLSLCVKs5TDvJBES1DBatu2bVi9ejXKy8uRm5s76DFarRYvvfRSSIUjJBCusxouDwu9Mt/ftxQSVTbQDMDcGvq5SEQEFaz27t2LwsJCf03Kh+M4NDQ0oLCwEBKJBKtWrQpLIQnpw+OCp7seLAfo5QU96wWGQpnN3/cKVh6Wg9HmgotlexL8kZgJqs+qrKwMHR0dA7Z3dXVRVgQSefp6uF0uOERKeGTp/KDQUKVo+SuKDjN/A9BpcaDiYC32HG8M/fwkZEH9lgNN7DSbzZDJ6BuIRFhXDVweFgZZfs8SXKESSfjc7YC/duW7GmhzsjSZOQ6M6je9bt06APxSRQ888AAUip6OTY/Hg0OHDmHGjBlhLSAhA+hrvcEqLzxNQB+llp8jaG4FMsr8021YjoPDzfqfk9gYVbA6fvw4AL5mdeLECUgkEv8+iUSC6dOnY/369eEtISG9OS2AuR0KiQjTy6ZBogiwNmAwlDlA6yn/WoK+ycwuDwe7y0PBKsZGFaw++eQTAMCdd96JrVu3UkYDEn3evFMyTTaml+WF99yqgZ3sMrEQLo8bNpcHmvC+GxmloBr8vdMMExJVviR5acXhP7fviqBND7jsgFgGuUQIk90Nu4vSxMTaiIPVDTfcgIqKCqSmpuKGG24Y8tjdu3eHXDBCBtXNL7fVLsqBU29DRookfM0zsZzPw2438LWrtKJenew05SbWRhys1Go1GO9kUbVaPczRhESAwwRYOwGGwYE2GerPN2DZ1FxMyAljv5Uqu0+wuiRbhRy1DJlKyfCvJRE14mDVu+lHzUASE74moFILs1MMwBneq4EAP96q/Sxg4TOHTMmjL+Z4EdQ4K5vNBqvV6n9eV1eHp59+Gh999FHYCkbIAN4mIDRF/mXdwx6slFr+3twW3vOSkAUVrFasWIFXXnkFAKDX6zFv3jw8+eSTWLFiBbZt2xbWAhLi561ZuVMLYHf5glWYl770JfGzdAAsC5eHRZfFiS6Lc+jXkYgLOlPookWLAAB/+9vfkJOTg7q6Orzyyit45plnwlpAQgDwU2Bs3QDDwCLnJ88LBQxk4jCveSJPA4QigHUDtm5Ut5vx8sFa7DtNE5xjLajftNVqhUrFd2p+9NFHuOGGGyAQCPC9730PdXV1YS0gIQAAo3d+niIDFg9fm0qRivwXfcKGYfh+KwCwtPmvBtrdNHQh1oJeMGLPnj1oaGjAhx9+iGuuuQYA0NbWRgNFSWQYvOs/qgtgcbgBAEpphEaU9+q38gcrGroQc0EFqwceeADr169HcXEx5s+fjwULFgDga1kzZ84MawEJAQAYvDUrdR6yVFIsnqjF9AJNZN7LX7Nqh9QXrFwemswcY0H1Tt544434/ve/j+bmZkyfPt2//eqrr8YPf/jDsBWOEACAx9UzBUadD41cghmKCI57Uno72XvVrNwsB5eHg0QU5mYnGbGgL6Xk5OQgJyenz7Z58+aFXCBCBjA1A6wHkKQAMk3k3893RdBugJhzQCRg4GY52FweSERxvYh5UgsqWFksFmzZsgX79u1DW1sbWLZv52O0VmwmY4S/CZgPMAwudlvBMAwylRJIRRHotxLL+VWa7UYwlg7IxEKYHW5+lRu5OPzvR0YkqGB1991349NPP8Xtt9+O3Nzc8F+RIaQ3w0X+Xp0PAPjkTBs6zE78aFY+CjMitPJMipZflsvShmn5xfBwHGThHoBKRiWoYPX+++/jX//6FxYuXBju8hDSF8cBxr7Byuzgr8ylROpqIMD3W3WeB8ztmD9hduTeh4xYUA3wtLQ0pKenh7sshAxk7eTTtQhFgDIbLg/rH72eIg3z6PXeeo21IvEhqGD1yCOP4IEHHugzP5CQiPA1AVU6QCCE1VurEgkYSCPZ2e2fdtMOh8uNLosTRrsrcu9HhhXUV9OTTz6J6upqZGdno7i4GGJx307HY8eOhaVwhPT0V/FZQX0BQyWLwOj13hTpgEAIuJ04frYOXzZ5MKNQg8UTtJF7TzKkoILVypUrw1wMQgLwTbNJ5fureoJVhK/KCYT8ajfmNijdXQDUNIo9xoIKVg8++GC4y0HIQE4LYO3iH3trViY7P9UmNRpDCJRawNyGFJc3WLkpWMVS0D2Uer0ef/vb31BdXY1f/epXSE9Px7Fjx5CdnY28vDAn8idjk298VUomP/YJQGlWCmRiIdIjOYLdx9vJrnB2AiiBzUmTmWMpqGD17bffYsmSJVCr1aitrcWaNWuQnp6O3bt3o76+3p/ripCQ9BuyAABalSx6S7l7p93InJ0AAJuLalaxFNTllHXr1qG8vBznzp3rswLzsmXLcODAgbAVrr8tW7aAYRjcd999EXsPEkd8neupMaqpe2tWEqcRAtbtHzJBYiOoYHX48GH89Kc/HbA9Ly8PLS0tIRcq0Hv++c9/xrRp0yJyfhJnPG7/YqO+mhXHcTjTYkSj3gaWjUIGBKkSkCggFgJyVxecbhaeaLwvGVRQwUoqlcJoNA7YfvbsWWRlZYVcqP7MZjNuvfVWvPjii0hLSwv7+UkcMrd4Jy8r+Oyd4Jth759owVtHGhC1kJGihVDAYGa6C/NL08FSmpiYCSpYXX/99Xj44YfhcvGXkRmGQX19Pe6//3786Ec/CmsBAWDt2rW49tprsWTJkrCfm8Sp3k1A73gqo82XdE8EoSBK81GVWjBgMDfLjcvKMiEWUtaFWAnqk3/yySdhNpuRlZUFm82GK664AuPGjYNKpcKmTZvCWsBdu3bh2LFj2Lx584iOdzgcMBqNfW4kARkGdq53W/lFG9TRzHxAq93EjaCuBqrVauzduxdffPEFvvnmG5jNZsyaNSvsNZ+Ghgb893//N/bu3dunI38omzdvxkMPPRTWcpAo47iewaC9gpXeytfkNdEYtuDj7WR3GVtgMjsgl4ggp+wLMTHqYMWyLCoqKrB7927U1taCYRiUlJQgJycHHMeFdQrE0aNH0dbWhlmzZvm3eTweHDhwAM899xwcDgeEwr5/OBs3bsS6dev8z41GIwoKCsJWJhIFtm7AaeVHkSt7EjzqvTWrNEUUa1aKDIARoLqpA/us3+HyqWWYmk8Ln8bCqIIVx3G4/vrr8d5772H69OmYOnUqOI7D6dOnUV5ejt27d2PPnj1hK9zVV1+NEydO9Nl25513YuLEibj//vsHBCqA7/yXSqVhKwOJAf/k5Rw+24JXdyxqVkIRoEiHSGiEwtUJm6s4eu9N+hhVsKqoqMCBAwewb98+LF68uM++jz/+GCtXrsQrr7yCO+64IyyFU6lUmDJlSp9tKSkpyMjIGLCdJJFBmoAcx0Fv42tWmmjWrABAqYVYUIcUZxeNtYqhUXWwv/766/jf//3fAYEKAK666ips2LABr732WtgKR8Yo/5XA/D6bfzAlF1dOyIIm2qmFldkQCQVQuLpoFHsMjapm9e233+Lxxx8PuP8HP/hBxFdk3r9/f0TPT2LMZeOXbgf8k5cBfnhMSWYKgJTolyklCyIBA4WrEyYKVjEzqppVV1cXsrOzA+7Pzs5Gd3d3yIUiY5ivVqVI51eziQdKLURCBnKXAQ6HPdalGbNGFaw8Hg9EosCVMaFQCLfbHXKhyBimr+PvNYV9Np9rNeFMi9G/GnNUSZQQSlMAcIClM/rvTwAEcTWwvLw84NU2h8MRlkKRMUxfz9/3C1aHa7vRarTjumm5GJ+tim6ZGAZSTQ5y9AaoNfQ3HiujClarVq0a9phwXQkkY5DL1jNSvFewYlkOnWY+SGQqYzMsRa7JRUnmRUBFwSpWRhWsduzYEalyEALoG/jR64oMQNpTe+owO+BmOUhEguhOtenNt6Q8rXYTMzQrk4yYh+XARTLrQIAmYJOB79TOVcsgiNYE5v68y4BZuprgovTGMRHBhddIMvn2oh6fVrVDLhHixtn5kRlFrq/l79OK+mxu1tsAADqNPPzvOVKKTJxoMsLh8qBkahdyIpAKiQyNalZkWHqrE5+caYeb5VCYrkBqJFaWsRsBczufDqZXzYrjODT6gpU6hsFKKIJHxi/s69Q3x64cYxgFKzKs4w16sBwHnUaGf5ucHZmmWNcF/l6V22d8ld7qgsnuhljIQKeJUu71ADyKTP7e1BrTcoxV1AwkQ2JZDudaTQCAucXpkVtYtKuav88o67M5LUWCey4vRYfZAVGsE9+laAGcBmtqj205xiiqWZEhtZrssDg8kIoFKMpIgccbvPZXtYWvs531AF01/OP0sgG7U6QiFGXEwWj2FErEF0sUrMiQLnbz/UX5aQp/KuGPTrXieL0e7eYwjTnS1wMeF9/8U/Xkr3J52MhefRwlgYqfasbYOvkFLUhUUbAiQ7K7PBAKGOSn8Z3bQgGDPO9VuYYuW3jepL2Kv88Y58+3DgAffdeKt45cRJsxPubjiRWpcAukcLs9gJWm3UQb9VmRIS0an4XvlWagdwWnIF2Omg4LGvU2zC4KcbUh1gO0n+Efayf5NzcbbDjXZgLHIXL9ZKOUrZZDlp0PjbsFMLcCqsCT+kn4UbAiw+q/okuOdwhBWGo83bX8NBuJAtDw46usTjc+ONkCjgMm5aYiSxUfmV91Gjl048cBF/U0kj0GqBlIRi1LKQXDACa7G+ZQsyC0fuc96URAIIDd5cHfjzVCb3VBJRPhyglxNviSVruJGQpWJKCvLnTi1a/qcLLR0Ge7RCRARgo/gj2k2pXT2tMEzJkKp5vFnuON6DA5kCIV4kez8iETx89KMhzHwSrNhNXpBmdqAeKo838soGBFAmox2NFucgy6ZLo2lR+g2WF2Bv8Gzd/wfVaqHCBVhwNn29FssEMmFuKGWflIS4niwhAj4PJweOGIEZWNJrBOO78KD4ka6rMiAbWb+KEJg/UZfa8kA5eVZUApDfJPyOMCLh7mH+fxS619rywDnRYHLr8kK2apYIYiFjIQCEWwitPhYt0Qmlv5jKYkKqhmRQZld3n8/VEZyoE1HLVCDJVMHPyVusZjgNMCyNRANr9SkVIqwo/nFCA3lnMAh8AwDORiIcySLHg8HGCiOYLRRMGKDMpg49foS5EKIRWFud/IbgTqPucfF10Go5P174qXYQqByMQCWCSZcLEsQHMEo4qCFRlUt3f1Y408cL/RsfpuvPNN0+g62VkWqHofcDuBVB3alROw/fMa/OvbZrCD9I3FG5lYCIskC24PB5ipkz2aKFiRQen9qx8HTgdT12lBdZsZrcYRTrthWeDch3yGBYEImLAM3zYawXEABy52ifVGQS4RwirWwMUxgMsO2PWxLtKYQR3sZFBioQAZSklPf5W5jZ8WY2njB3ECmNDhBtcNOOtyAVkpINPwfVCifrUxjuNHfFd/DHR7V6+ZtBxOWQbOtPCpYablaaLzg4VIJhKCY/hOdsDMNwXlIY7iJyNCwYoManZRGj+VxtoFfPsm0Fk94JgsmwMGkxmy2jOA81jPDkkKINcAIjnAugFbF99PBfA1qknXAdqJqG42wulmoZaLUZAen53q/RVmKCAUMkgx5gPmM3xTUDsx1sUaEyhYkcC6LgDf7QHcDoAR8LmmNEXexRw4CLu70eSohgkWTFZJwdj1fNPIaeFvvQmEQOYlQMnl/sv9tR38MRNyVHHfse5zSbYKl2SrgMYi4OwZwNQS6yKNGRSsyAAcxwHdtWBO/J2vGanzgInXDRhTpMpg0dCQCY4DFk4tRYpUxDcR7QZ+wKTbwQcpaSo/8FPUM3aKZTnUdVkBAMWZcZCrarRUufy9byR7ggTbREbBigzQ2t6GC+9vR7rEjYlTZgOX/pAPOv2IhQJo5GJ0W13oNDv5YCWW87deeakG02ZywOb0QCISICc1tumKR4NlOdjdHrgEGqgZQU9wlmtiXbSkR8GK9MVxYE/+A4zbBosyF5i8ctBA5ZOhlMLscMPiHN2EZo1CjH+bnA2H2+NP6pcIWk127Pq6ASqZCHcrs/gOdlMzBasooGBF+mr+Bh59AzyMGKay5YBw6D+Rf5ucDYlQMOphBzKxEFPy1KGUNCYUYv7zsDk94LQ6MKZWwNjYJxcXiQwaZ0V6uOzAhf2wuzy4qJ4NpSZz2JfIxMKEGB8VLgopX8t0sxycKTp+o6ExhiUaOyhYkR4XDwMuG/SMGi2qSyOzkCmARr0Nx+q7obeGkLEhRsRCASQi/t/GKvNmCjW3Uk72KKBgRXguO3DxMDhwOK+YAY4RDjl6vbe9p1rx1y9r0W0ZWfA51WTEp1XtqGzQh1Dg2EmR8LUrM6PkLyawHj5gkYiK62C1efNmzJ07FyqVClqtFitXrkRVVVWsi5WcGo8Cbgdc0nS0SIr5hZHlIwtWHWYHOszOEa12w3Ec6jr58VXF8bC8VhAU3rQ4NjcLqPP5jUZqCkZaXAerTz/9FGvXrsVXX32FvXv3wuVy4ZprroHFYhn+xWTkWA/QxI9Ad+R9D6VaJfLTFCNeVNSXe6pjBMGqy+KEye6GSMAgLy0xRq33lyLhg5XF4QZSvf1WFKwiLq6vBn7wwQd9nldUVECr1eLo0aO4/PLLY1SqJNRxDnCYAYkCqoKpWFE0upQwmd75gyPJGlrbyQ8EzUuTD1iIIlEUZyqgkAr5IM3k8RuNTbEt1BgQ18GqP4OBzwWenh44O6PD4YDD0fMNbzQaI16uhOetVSF3xpBjqgLx16xMw9esfE3AuFhhOUiX6tS41PfEncuPXrcb+ZssNZZFS2oJ89XGsizuu+8+LFy4EFOmTAl43ObNm6FWq/23goKCKJYyAVk6+UwIDAPoZsDlYYd/TT++YGWwueBwewIe5/KwaPSu8FycoQiuvPFGJAFSvCvwUFMwohImWK1duxYnT57Erl27hjxu48aNMBgM/ltDQ0OUSpigWk/w9xnjAJkabxxuwLb91bjYbR3xKeQSoT8Xe+cQTcFOsxMcAJVMhPQ4WwxiNDiOg9Xp7rn6qSnk7/X1sSvUGJAQzcCf//znePfdd3HgwAHk5+cPeaxUKoVUGn+LDcQljgPaTvOPtZPBcRwMNhecbtbfiTxSWSopZGLBoCvh+OSoZfjZFWUw2l0Jk2VhMG0mB3Yequen3Cwq5YPVxSMUrCIsroMVx3H4xS9+gbfffhv79+9HSUlJrIuUXEwtgE3PT6nJGAeL0wOnm4WAYZA6wmELPtdP141oJLtEJIjLlWtGQ+EdZ2VxeMBxHBh1Ad+MtnTwFyqkyhiXMDnFdTNw7dq1ePXVV7Fz506oVCq0tLSgpaUFNpst1kVLDm2n+PuM8YBI4m/WpMpFo55cPFyg4pIoV7nCW+tkOQ52FwtIFD39VgbqdoiUuA5W27Ztg8FgwJVXXonc3Fz/7Y033oh10RIfx/WshuydhOtb0WakI9cH42G5QZuC3140DLq6cyISChjIfaPYvcuVQVPE31NTMGLivhlIIsTYyF9qF0mA9DIAvVa0CXJO4Acnm3Gu1YzrputQ0i+hXk2HBe0mB+yuwFcLE4lSKoLNya+tmKWSevutDlOwiqC4rlmRCPJ1rGde4k8D0+1d0SYtyGDFMAzcLDfgSqLTzaLBmxW0fxBLVCoZ/5mZ7b6aVb9+KxJ2FKzGIpbtCVZZPXmYctUyFGcqBl0ufiSKvGOnfKPUfeq7LHCzHNRycUIPWejNF6xMdj7AQywHlFr+cXdNjEqV3OK6GUgixNDAL+gglgHpPVdY5xanY25x4NkBwylKTwHD8CPZTXYXVDK+7+t0swkAME6rTOghC70VpivAMAxyNb3mN6aX8plDuy4AOVNjV7gkRTWrscjfBJwQ1PSaQOQSIXLVfD71c218U8jm9KDGu4rNZF3yTEUZp1Vh8QRt32Zteil/33WBr72SsKJgNdawbK+rgD3r3dldnrB0fk/I4QPSyUYDOI5DZYMeHpZDdqos4cdXDSs1n1/Bx2Xn87KTsKJgNdboa/kVWSQKQFPs33yq2Yht+6vxwcnQ1sGblKuCRCRAp9mJ6nYLyrQpGKdVYm5xcq1a7Jty02a091y1Fgh6mtVdAxeFJaGhPquxxt+xPpH/5/LypRj2dRwHSyoSYkFZBgCgNDMFAgGD5dN1IZ0zHnlYDn/+9AIA4D+vLINM7G1Op5cCbWf4FaxLKI1ROFGwGktYD9DuzbSa1XfJ825L6ANCfWYVJlctajAioQByiRA2pwcmu7tXsCrjhzCYWihlTJhRM3As6arhV0mWKgF139Q5vgGhwY6xGosGDF8A+M821ZuQr+NsDEqVvChYjSW+uYBZk/o0AV0eFibv4EYKViPnG5phtPdb2cZXa/VdyCBhQcFqrPC4er7ptf2bgHytSi4R+ue8keH5FtQYsKRY1iX8veEijWYPIwpWY0VnNR+wZKk9zRQv36o0ST+0IMx8/Xu+CeB+MjWQmstPFu+g1ZjChYLVWOFrAmon8R3AvaQpJJhRoEFZVnLM24sWtb9m5Rq409cUbD0VxRIlN7oaOBa47HzNCgC0lw7YrdPIodMk5rJYsZShlGJGoQbpg/XzZV8KXNjPNwWtXYAi+GlMhEc1q7Gg4yzAugFFRs9kWxIypVSExRO0mF6gGbhTquqZftNyIqrlSlYUrMYCXxMwe/KAJqDLw6LFYIfTTXPZws43mbnlBM0VDAMKVsnOYeaX2gIA7eQBu1uNdrz+dT3++lVdlAuWHHzBvssyyKo+GeP5zBYOE6WNCQMKVsmu/QzAsYAqZ9B+kzaT70ogja8KxpfVnXj963p8c1E/cKdQBGR7a1eNR6NarmREwSqZcRzQXMk/DpBfqdVg53enyqJUqOSS4Q3yAVejzpvFN707q/kFZUnQKFglM1MzYG4HBCL+6tQgWo18sMqmYBWULO/YtE6Lc/A1AxTp/AKyANB4JIolSz4UrJJZ8zf8fdYEPu1uP3aXx593nYJVcNJSJGAYPsmgxRkgH1j+HP6+5Vsa0R4CClbJyu3suQqYO33QQ9qMfNNFLRfTNJsgiYUC/3zKTnOApqCmiB/R7nEDDV9FsXTJhYJVsmr5lg9YinR+mahBXNTzCzv4UhGT4PimKbUH6rdiGKB4Ef+48Th/dZCMGgWrZMSy/Bp2AJA/d8DYKp+JOam4/JKspMqNHgs5aj5YNXkvVgwqvRRQ5/GDc+sORqlkyYWm2ySjznOATc+P8RlilZX0FEnSLI0VS8UZKeDGA/lpisAHMQxQcgVQuRNoOs43zVU50StkEqCaVbLhOKD2c/6xbhYgDD3zJxlahlKKOcXpyBmuOZ1WxE8k5zjg3Ef8PRkxClbJpv0MYG7jl4XPnxvwsJONBnzXZEia5dwTRtlV/BeIoRFoOhbr0iQUClbJhPUANQf4xwXz+RVsBsFxHL660ImPvmtFo94WxQImL4fbg6oWE47Vdw99oCwVKL2Sf3z+Y34cHBkRClbJpOFrPh2JRDFkrarN5IDJ7oZEJEBh+hD9LGTE9FYX3jvRjC+rO+Fhh2ne5c3mO9xZN3D6H/xVWzIsClbJwtoF1Hn7qsqu4hfbDOBkowEAUJKZArGQ/gTCQauSIkUqhNPNor7LOvTBDANMvJb/UjG3A6f+QVkZRoD+UpOBxw2c2sPfpxUB2VMCHup0szjTwo/zmZqnjlIBkx/DMBifrQLQ82UwJKkSmHIjPxWq8zxw9n0KWMOgYJXoOI7/Qze18lNqJl4XcFwVAFQ26OF0s0hTiJGfRtlBw8kX/C+0W/ouzxWIOg+YfD3/+2r+tucLhwyKglUiY1n+EnjLSYARAJOWD7mopsXhxpG6LgDA98oywAwR1MjoZSqlyE+Tg+U4fHWha2QvypoATF4JCIT8ArTHX+Gb9GSAhAhWzz//PIqLiyGTyTB//nx8/fXXsS5S7DnMwMm/A43ey98TlwEZZUO+hGEAnVoObaoUE7xNFhJeC8dlAgC+azKgYbi+Kx/tRGDaj/masakVOPISUPMZdbz3E/fB6o033sC6devw4IMP4tixY5g+fTqWLl2Ktra2WBctNtwOoP4QcPhFvq9DIAImrxhypLqPQiLCihk63DAzn2pVEaLTyHGpLhU5qTKoFaMYkJtWDMxZzfc5etz8wN6vngeqvcMbaAApGG7QJDzxY/78+Zg7dy6ee+45AADLsigoKMAvfvELbNiwYdjXG41GqNVqGAwGpKYm0Bw4jgM8TsBpAewGwNIO6Ov5JeBZb7+GUstfVRpk2obd5YHe6kJdpwUXu224YVYeBagocXlYCBgGQgH/eeutThhtbmhTpZCKBEP/HjiOH9h74VPA1mvMlkzN93Epc/iFP6QqvpNeJOObkAlqNP+fcT030Ol04ujRo9i4caN/m0AgwJIlS/Dll18O+hqHwwGHo2f2u9FoHNmb1R0EWr8buL1PLOdC39Zn+xDbPC606C3+5Hj8bg4cALs4Da3q6bhs1hXQqvi1/iob9Dh0oRMcAA/LDVgAorrdgnFa5cCfj4Rd/+EglQ16HK/XAwBEAgYysRBCAQOGARgAt8wrhEzMB5xDNV0426YAuKVI9dQh3XgKKlsj1BILSuyGwdchFAj5UfECMd93yTAAmH73XtH8wpKogOk3h+10cR2sOjo64PF4kJ2d3Wd7dnY2zpw5M+hrNm/ejIceemj0b+YwA5aOYIoZMS4PC5NLAJdQAaskHRZxJrrlhbCK0wGGwffQ843q9rCw9kv+JpcIkauWoSxLieIMGvwZK0IBg1S5GEabC26Wg9kR+Iqf2eH2p0jugA4XVDoIUty4NEWPkgIAphbArufTzDi9fWKsh79hiKwPsSALb59bXAerYGzcuBHr1q3zPzcajSgoKBj+hXmzgczx/b55vI9D2dZn+yi2CUVIdYlR5OrZxTAMGN9jMH0yJkzWpaI4MwUMAAHDIEUqgkQU912SY8Ki8VlYND4LTjcLm8sDh8sDD8eB5fipT71rYjML0zBeO/Dih0wiAFT9JkqzHr6rwOMEPC7+Bs5bDQ90H0WC8IaXuA5WmZmZEAqFaG1t7bO9tbUVOTmDp9eQSqWQSgOP3g4oJYO/xRG1DBjpsE2FRASFJK5/nWOeRCTgv0DkgTveR5W2RyAEBPJBU1Yno7j+6pVIJJg9ezb27dvn38ayLPbt24cFCxbEsGSEkGiL+6/idevWYdWqVZgzZw7mzZuHp59+GhaLBXfeeWesi0YIiaK4D1Y333wz2tvb8cADD6ClpQUzZszABx98MKDTnRCS3OJ+nFWoEnacFSFjwGj+P+O6z4oQQnwoWBFCEgIFK0JIQoj7DvZQ+brkRjzthhASNb7/y5F0nSd9sDKZ+KyYIxrFTgiJCZPJBLV66CHQSX81kGVZNDU1QaVSxTTrgG/aT0NDA12V7Ic+m8CS/bPhOA4mkwk6nQ4CwdC9UklfsxIIBMjPz491MfxSU1OT8o8uHOizCSyZP5vhalQ+1MFOCEkIFKwIIQmBglWUSKVSPPjgg8FlhEhy9NkERp9Nj6TvYCeEJAeqWRFCEgIFK0JIQqBgRQhJCBSswmg0i7G++OKLWLRoEdLS0pCWloYlS5Yk9eKtwS5Uu2vXLjAMg5UrV0a2gDE02s9Gr9dj7dq1yM3NhVQqxSWXXIL33nsvSqWNIY6Exa5duziJRMJt376d++6777g1a9ZwGo2Ga21tHfT4n/zkJ9zzzz/PHT9+nDt9+jRXXl7OqdVq7uLFi1EueeSN9rPxqamp4fLy8rhFixZxK1asiE5ho2y0n43D4eDmzJnDLVu2jPv888+5mpoabv/+/VxlZWWUSx59FKzCZN68edzatWv9zz0eD6fT6bjNmzeP6PVut5tTqVTcyy+/HKkixkwwn43b7eYuu+wy7i9/+Qu3atWqpA1Wo/1stm3bxpWWlnJOpzNaRYwb1AwMA99irEuWLPFvG24x1v6sVitcLhfS09MjVcyYCPazefjhh6HVanHXXXdFo5gxEcxn884772DBggVYu3YtsrOzMWXKFDz66KPweDyDHp9Mkn5uYDQEsxhrf/fffz90Ol2fP9xkEMxn8/nnn+Oll15CZWVlFEoYO8F8NhcuXMDHH3+MW2+9Fe+99x7Onz+Pe++9Fy6XCw8++GA0ih0zFKziwJYtW7Br1y7s378fMpls+BckMZPJhNtvvx0vvvgiMjMzY12cuMOyLLRaLV544QUIhULMnj0bjY2N+MMf/kDBigwvmMVYfZ544gls2bIF//d//4dp06ZFspgxMdrPprq6GrW1tVi+fLl/G8uyAACRSISqqiqUlZVFttBREszfTW5uLsRiMYRCoX/bpEmT0NLSAqfTCYlkhAukJiDqswqDYBdjffzxx/HII4/ggw8+wJw5c6JR1Kgb7WczceJEnDhxApWVlf7b9ddfj8WLF6OysjKpkigG83ezcOFCnD9/3h/AAeDs2bPIzc1N6kAFgIYuhMuuXbs4qVTKVVRUcKdOneLuueceTqPRcC0tLRzHcdztt9/ObdiwwX/8li1bOIlEwv3tb3/jmpub/TeTyRSrHyFiRvvZ9JfMVwNH+9nU19dzKpWK+/nPf85VVVVx7777LqfVarnf//73sfoRooaCVRg9++yzXGFhISeRSLh58+ZxX331lX/fFVdcwa1atcr/vKioiAMw4Pbggw9Gv+BRMJrPpr9kDlYcN/rP5uDBg9z8+fM5qVTKlZaWcps2beLcbneUSx19lHWBEJIQqM+KEJIQKFgRQhICBStCSEKgYEUISQgUrAghCYGCFSEkIVCwIoQkBApWhJCEQMGKJJWKigpoNBr/89/97neYMWOG/3l5eXlSp0hOZhSsSFDKy8vBMAx+9rOfDdi3du1aMAyD8vLyPseHO0gUFxfj6aef7rPt5ptvxtmzZwO+ZuvWraioqPA/v/LKK3HfffeFtVwkMihYkaAVFBRg165dsNls/m12ux07d+5EYWFhTMokl8uh1WoD7ler1X1qXiRxULAiQZs1axYKCgqwe/du/7bdu3ejsLAQM2fODOncg9V4Vq5c6a+tXXnllairq8Mvf/lLMAwDhmEADGwG9te7hldeXo5PP/0UW7du9Z+jpqYG48aNwxNPPNHndZWVlWAYBufPnw/p5yLBo2BFQrJ69Wrs2LHD/3z79u248847I/6+u3fvRn5+Ph5++GE0Nzejubl51OfYunUrFixYgDVr1vjPUVhYOOBnAoAdO3bg8ssvx7hx48L1I5BRomBFQnLbbbfh888/R11dHerq6vDFF1/gtttui/j7pqenQygUQqVSIScnZ9iMrINRq9WQSCRQKBT+cwiFQpSXl6Oqqsq/fp/L5cLOnTuxevXqcP8YZBQorTEJSVZWFq699lpUVFSA4zhce+21CZ87XafT4dprr8X27dsxb948/POf/4TD4cBNN90U66KNaVSzIiFbvXo1Kioq8PLLL4et9iEQCNA/1ZrL5QrLuUfi7rvv9l882LFjB26++WYoFIqovT8ZiIIVCdm///u/w+l0wuVyYenSpWE5Z1ZWVp9+KI/Hg5MnT/Y5RiKRhLxeXqBzLFu2DCkpKdi2bRs++OADagLGAWoGkpAJhUKcPn3a/zgQg8EwYC3AjIyMQReBuOqqq7Bu3Tr861//QllZGZ566ino9fo+xxQXF+PAgQO45ZZbIJVKg2p+FhcX49ChQ6itrYVSqUR6ejoEAoG/72rjxo0YP378kAt/kOigmhUJi9TUVKSmpg55zP79+zFz5sw+t4ceemjQY1evXo1Vq1bhjjvuwBVXXIHS0lIsXry4zzEPP/wwamtrUVZWhqysrKDKvX79egiFQkyePBlZWVmor6/377vrrrvgdDqjcnWTDI9ysBMSwGeffYarr74aDQ0NA1ZNJtFHwYqQfhwOB9rb27Fq1Srk5OTgtddei3WRCKgZSMgAr7/+OoqKiqDX6/H444/HujjEi2pWhJCEQDUrQkhCoGBFCEkIFKwIIQmBghUhJCFQsCKEJAQKVoSQhEDBihCSEChYEUISAgUrQkhC+P/QFkiR4uyrXwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:44:43.494982Z", + "iopub.status.busy": "2024-03-01T18:44:43.494702Z", + "iopub.status.idle": "2024-03-01T18:44:43.641829Z", + "shell.execute_reply": "2024-03-01T18:44:43.640819Z" + }, + "papermill": { + "duration": 0.17118, + "end_time": "2024-03-01T18:44:43.644132", + "exception": false, + "start_time": "2024-03-01T18:44:43.472952", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtIklEQVR4nO3de1QUV4I/8G83j+aNKPIMCogrGkUUDogxOz4QMKuDYyZBkwgSQ3Y1/jZOr3GGJOJBnTBJlKCzRnZc8ZmNbiaOO5M4KhJ1NSJRDIqOkhFFNDwUFFogtg19f3+4VNLSjd0KNhTfzzl9sG7dun0vZX+p7qq+pRBCCBARyZDS2h0gIuouDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhItmyt3YGeSK/Xo6qqCq6urlAoFNbuDhH9hBACd+7cgZ+fH5TKzo/RGHBGVFVVISAgwNrdIKJOXLt2DU899VSndRhwRri6ugK4/wt0c3Ozcm+6j06nw4EDBxAXFwc7Oztrd4ceU1/ZnxqNBgEBAdLrtDMMOCPa35a6ubnJPuCcnJzg5uYm6xdEX9HX9qc5Hx/xJAMRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLZ4mQhRL9XS0oKLFy9Ky00/aHG8tBwenqfg4qgyqBsaGgonJ6cn3UWrY8AR9VIXL15EREREh/IPjNQtLi7G2LFju79TPQwDjqiXCg0NRXFxsbRcVt0A9WelyH5hFIb59utQty9iwBH1Uk5OTgZHZcqr9VAd/QHDR45G+OABVuxZz8GTDEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsWT3g1q9fj8DAQDg4OCA6OhrffPONybpbtmyBQqEweDg4OBjUmTdvXoc6CQkJ3T0MIuqBrDrh5a5du6BWq5Gbm4vo6Gjk5OQgPj4eZWVl8PLyMrqNm5sbysrKpGWFQtGhTkJCAjZv3iwtq1SqDnWISP6segSXnZ2NtLQ0pKamYsSIEcjNzYWTkxPy8vJMbqNQKODj4yM9vL29O9RRqVQGdTw8PLpzGETUQ1ntCO7evXsoLi5Genq6VKZUKhEbG4vCwkKT2zU1NWHw4MHQ6/UYO3Ys3nvvPTz99NMGdQ4fPgwvLy94eHhg8uTJWLVqFQYMMD2Fs1arhVarlZY1Gg0AQKfTQafTPeoQe7z2scl5jH1Ja2ur9FPO+9SSsVkt4Orq6tDW1tbhCMzb29vgVmg/NWzYMOTl5SEsLAyNjY1YvXo1xo8fj/Pnz+Opp54CcP/t6axZsxAUFITy8nK8/fbbmDZtGgoLC2FjY2O03aysLGRmZnYoP3DgQJ+41Vp+fr61u0Bd4FoTANjixIkT+P6ctXvTfVpaWsyuqxBCiG7si0lVVVXw9/fH8ePHERMTI5UvXboUR44cQVFR0UPb0Ol0GD58OObMmYOVK1carXP58mUMGTIEBw8exJQpU4zWMXYEFxAQgLq6Ori5uVk4st5Dp9MhPz8fU6dOhZ2dnbW7Q4/pTOUt/HLjKfwxLRKjB/W3dne6jUajgaenJxobGx/6+rTaEZynpydsbGxQW1trUF5bWwsfHx+z2rCzs8OYMWNw6dIlk3WCg4Ph6emJS5cumQw4lUpl9ESEnZ1dn3jh95Vxyp2tra30U87705KxWe0kg729PSIiIlBQUCCV6fV6FBQUGBzRdaatrQ2lpaXw9fU1Wef69euor6/vtA4RyZNVz6Kq1Wps3LgRW7duxYULF7BgwQI0NzcjNTUVAJCcnGxwEmLFihU4cOAALl++jNOnT+OVV17B1atX8dprrwG4fwLirbfewokTJ1BRUYGCggIkJiYiJCQE8fHxVhkjEVmPVa+DS0pKws2bN5GRkYGamhqEh4dj37590omHyspKKJU/ZvDt27eRlpaGmpoaeHh4ICIiAsePH8eIESMAADY2Njh79iy2bt2KhoYG+Pn5IS4uDitXruS1cER9kNVOMvRkGo0G7u7uZn2I2ZvpdDrs3bsXzz33nKw/s+krSq7WY+aGE9izYJys72xvyevT6l/VIiLqLgw4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItq96TgZ6slpYWg5tqN/2gxfHScnh4noKL44/3rAgNDe0TN7wm+WPA9SEXL15EREREh/IPHlguLi7G2LFjn0yniLoRA64PCQ0NRXFxsbRcVt0A9WelyH5hFIb59jOoRyQHDLg+xMnJyeDITHm1HqqjP2D4yNGyvgsT9V08yUBEssWAIyLZsnrArV+/HoGBgXBwcEB0dDS++eYbk3W3bNkChUJh8HBwcDCoI4RARkYGfH194ejoiNjYWPz973/v7mEQUQ9k1YDbtWsX1Go1li9fjtOnT2P06NGIj4/HjRs3TG7j5uaG6upq6XH16lWD9R988AHWrVuH3NxcFBUVwdnZGfHx8bh79253D4eIehirBlx2djbS0tKQmpqKESNGIDc3F05OTsjLyzO5jUKhgI+Pj/Tw9vaW1gkhkJOTg3fffReJiYkICwvDtm3bUFVVhT179jyBERFRT2K1s6j37t1DcXEx0tPTpTKlUonY2FgUFhaa3K6pqQmDBw+GXq/H2LFj8d577+Hpp58GAFy5cgU1NTWIjY2V6ru7uyM6OhqFhYWYPXu20Ta1Wi20Wq20rNFoAAA6nQ46ne6xxtmTtba2Sj/lPM6+oq/sT0vGZrWAq6urQ1tbm8ERGAB4e3sbXG3/U8OGDUNeXh7CwsLQ2NiI1atXY/z48Th//jyeeuop1NTUSG082Gb7OmOysrKQmZnZofzAgQOyvqL/WhMA2OLEiRP4/py1e0OPq6/sz5aWFrPr9qrr4GJiYhATEyMtjx8/HsOHD8d//Md/YOXKlY/cbnp6OtRqtbSs0WgQEBCAuLg4uLm5PVafe7IzlbeA0lMYN24cRg/qb+3u0GPqK/uz/R2WOawWcJ6enrCxsUFtba1BeW1tLXx8fMxqw87ODmPGjMGlS5cAQNqutrYWvr6+Bm2Gh4ebbEelUkGlUnUot7Ozg52dnVl96Y1sbW2ln3IeZ1/RV/anJWOz2kkGe3t7REREoKCgQCrT6/UoKCgwOErrTFtbG0pLS6UwCwoKgo+Pj0GbGo0GRUVFZrdJRPJh1beoarUaKSkpiIyMRFRUFHJyctDc3IzU1FQAQHJyMvz9/ZGVlQUAWLFiBcaNG4eQkBA0NDTgww8/xNWrV/Haa68BuH+GdfHixVi1ahWGDh2KoKAgLFu2DH5+fpg5c6a1hklEVmLVgEtKSsLNmzeRkZGBmpoahIeHY9++fdJJgsrKSiiVPx5k3r59G2lpaaipqYGHhwciIiJw/PhxjBgxQqqzdOlSNDc34/XXX0dDQwMmTJiAffv2dbggmIjkTyGEENbuRE+j0Wjg7u6OxsZGWZ9kKLlaj5kbTmDPgnH8sr0M9JX9acnr0+pf1SIi6i4MOCKSLQYcEclWr7rQlyxzpa4ZzdpWk+vLbzZLP9uvoTLGWWWLIE/nLu8fUXdjwMnUlbpmTFp92Ky6//bH0ofWObRkIkOuB+jsjxb/YHXEgJOp9hdBTlI4QrxcjNf5QYsvDhdi+sQYODt2/CYHAFy60YTFu0o6PRKkJ8PcP1r8g/UjBpzMhXi5YKS/u9F1Op0ONQOBsYM9ZP3VHrl42B8t/sHqiAFH1MuY+qPFP1gd8SwqEckWA46IZMvigLt8+XJ39IOIqMtZHHAhISGYNGkSduzYwRu5EFGPZnHAnT59GmFhYVCr1fDx8cE///M/d3qrPyIia7E44MLDw7F27VpUVVUhLy8P1dXVmDBhAkaOHIns7GzcvHmzO/pJRGSxRz7JYGtri1mzZuGzzz7D+++/j0uXLmHJkiUICAhAcnIyqquru7KfREQWe+SAO3XqFBYuXAhfX19kZ2djyZIlKC8vR35+PqqqqpCYmNiV/SQispjFF/pmZ2dj8+bNKCsrw3PPPYdt27bhueeek2beDQoKwpYtWxAYGNjVfSUisojFAbdhwwa8+uqrmDdvnsGdq37Ky8sLmzZteuzOERE9DosDLj8/H4MGDTK4VwIACCFw7do1DBo0CPb29khJSemyThIRPQqLP4MbMmQI6urqOpTfunULQUFBXdIpIqKuYPERnKl71DQ1NfHOVT2Itu0ulA7f44qmDEoH49Mltba2oqq1ChduXTA5f9gVTROUDt9D23YXgPFZSYh6KrMDTq1WA7h/79GMjAw4OTlJ69ra2lBUVNTp3ePpyapqvgrnoN/jbTOuwf5438edrncOAqqawxEB7y7qHdGTYXbAffvttwDuH8GVlpbC3t5eWmdvb4/Ro0djyZIlXd9DeiR+zoPRfOX/YW1SOIaYmPCytbUVXx/7Gs9MeMbkEVz5jSa8uasEfpMGd2d3ibqF2QF36NAhAEBqairWrl0r6/uFyoHKxgH6u/4IchuGEQNMT3h5xfYKhvcfbnL+MP3dRujv3oTKhh8/UO9j8UmGzZs3d2m4rV+/HoGBgXBwcEB0dLTZ32vduXMnFAoFZs6caVA+b948KBQKg0dCQkKX9ZeIeg+zjuBmzZqFLVu2wM3NDbNmzeq07u7du81+8l27dkGtViM3NxfR0dHIyclBfHw8ysrK4OXlZXK7iooKLFmyBM8++6zR9QkJCdi8ebO0rFIZn76ZiOTNrCM4d3d3KBQK6d+dPSyRnZ2NtLQ0pKamYsSIEcjNzYWTkxPy8vJMbtPW1oaXX34ZmZmZCA4ONlpHpVLBx8dHenh4eFjULyKSB7OO4H56NPTTfz+Oe/fuobi4GOnp6VKZUqlEbGwsCgsLTW63YsUKeHl5Yf78+Th69KjROocPH4aXlxc8PDwwefJkrFq1CgMGDDDZplarhVarlZY1Gg2A+59R6XQ6S4fWI7S2tko/TY2hvbyzMZrTDj0Zzdr7l+xcuv036G073hGr/bKf0hulJk8aXb7dDKXD92jWNkGnczJap6ez5P+h1W46U1dXh7a2Nnh7G1564O3tjYsXLxrd5tixY9i0aRNKSkpMtpuQkIBZs2YhKCgI5eXlePvttzFt2jQUFhbCxsbG6DZZWVnIzMzsUH7gwAGDy2F6k2tNAGCLY8eO4arxk6iS/Pz8LmmHutfpO1VwDvoYy4o7r/fxwYdf9rP3eBtqXP26sHdPTktLi9l1zQq4MWPGSG9RH+b06dNmP7kl7ty5g7lz52Ljxo3w9PQ0WW/27NnSv0eNGoWwsDAMGTIEhw8fxpQpU4xuk56eLl3nB9w/ggsICEBcXFyvPVt8vkqD1aUnMGHCBDztZ3wMOp0O+fn5mDp1qsmzqOa0Q0+Gz7Ub2L7NBtm/HIXggcaP4IpOFCF6XLTpI7ibzVD/sRTPJf8TxgaY/py7J2t/h2UOswLuwTOVXcHT0xM2Njaora01KK+trYWPj0+H+uXl5aioqMCMGTOkMr1eD+D+3HRlZWUYMmRIh+2Cg4Ph6emJS5cumQw4lUpl9ESEnZ1dr739Wvt/cFtb24eOobNxWtIOdS9nlQv0d/0R4jECI72N3zbwmu01jPIaZXJfKVsbob97C84ql167Py3pt1kBt3z58kfujCn29vaIiIhAQUGBFKB6vR4FBQVYtGhRh/qhoaEoLTW8Y/e7776LO3fuYO3atQgICDD6PNevX0d9fb3JmU+ISL6seuNntVqNlJQUREZGIioqCjk5OWhubkZqaioAIDk5Gf7+/sjKyoKDgwNGjhxpsH2/fv0AQCpvampCZmYmnn/+efj4+KC8vBxLly5FSEgI4uPjn+jYiMj6zAq4/v3747vvvoOnpyc8PDw6/Tzu1q1bZj95UlISbt68iYyMDNTU1CA8PBz79u2TTjxUVlZ2mJapMzY2Njh79iy2bt2KhoYG+Pn5IS4uDitXruS1cER9kFkB99FHH8HV1VX6t7knHMyxaNEio29JgfuXe3Rmy5YtBsuOjo7Yv39/F/WMiHo7swLup5NXzps3r7v6QkTUpSz+LqqNjQ1u3LjRoby+vt7kdWZERNZgccCZmvBSq9UaTKFERGRtZp9FXbduHYD7E17+53/+J1xcfrysva2tDf/7v/+L0NDQru8hEdEjMjvgPvroIwD3j+Byc3MN3o7a29sjMDAQubm5Xd9DIqJHZHbAXblyBQAwadIk7N69mzN0EFGPZ/GFvu0z+xIR9XQWB9yrr77a6frO5nIjInqSLA6427dvGyzrdDqcO3cODQ0NmDx5cpd1jIjocVkccH/60586lOn1eixYsMDobB5ERNZi8XVwRhtRKqFWq6UzrUREPUGXBBxwf7629umtiYh6Aovfov505lvg/nVx1dXV+PLLLw2+s0pEZG0WB1z7He7bKZVKDBw4EGvWrHnoGVYioieJ18ERkWx12WdwREQ9DQOOiGSLAUdEssWAIyLZ6rK7al2/fh0rVqzAH/7wh65qkh7DD7o2AMC57xtN1mn+QYtTNwGfq7fh7Gj8pjyXbjR1S/+InoQuC7j6+nps2rSJAddDlP9fMP1md+lDatpi+6WTD23PWWXVO0wSPRL+r5WpuKd9AABDvFzgaGf8Xhll1Y34tz+WYs0vR2GYb8c7pbdzVtkiyNO5W/pJ1J0YcDLV39kes6MGdVqn/at1QwY6Y6S/6YAj6q14koGIZMvsI7hZs2Z1ur6hoeGROrB+/Xp8+OGHqKmpwejRo/H73/8eUVFRD91u586dmDNnDhITE7Fnzx6pXAiB5cuXY+PGjWhoaMAzzzyDDRs2YOjQoY/UPyLqvcw+gnN3d+/0MXjwYCQnJ1v05Lt27YJarcby5ctx+vRpjB49GvHx8Ubvu/pTFRUVWLJkCZ599tkO6z744AOsW7cOubm5KCoqgrOzM+Lj43H37l2L+kZEvZ/ZR3CbN2/u8ifPzs5GWloaUlNTAQC5ubn48ssvkZeXh9/85jdGt2lra8PLL7+MzMxMHD161ODIUQiBnJwcvPvuu0hMTAQAbNu2Dd7e3tizZw9mz57d5WMgop7LaicZ7t27h+LiYqSnp0tlSqUSsbGxKCwsNLndihUr4OXlhfnz5+Po0aMG665cuYKamhrExsZKZe7u7oiOjkZhYaHJgNNqtdBqtdKyRqMBcH86dp1O90jj6w3aTzK0trbKepxy8bD91V7W2b6Uwz63pN9mB5y5UyGZe9OZuro6tLW1wdvb26Dc29sbFy9eNLrNsWPHsGnTJpSUlBhdX1NTI7XxYJvt64zJyspCZmZmh/IDBw7Aycmps2H0ateaAMAWJ06cwPfnrN0bepj2/XXs2DFcdTFdLz8//7Hb6MlaWlrMrmt2wG3ZsgWDBw/GmDFjIIR4pI49jjt37mDu3LnYuHEjPD09u7Tt9PR0g4k8NRoNAgICEBcXBzc3ty59rp7kTOUtoPQUxo0bh9GD+lu7O/QQ56s0WF16AhMmTMDTfh3/X+p0OuTn52Pq1Kmws7N7pDZ6g/Z3WOYwO+AWLFiATz/9FFeuXEFqaipeeeUV9O//6C8KT09P2NjYoLa21qC8trYWPj4+HeqXl5ejoqICM2bMkMr0ej0AwNbWFmVlZdJ2tbW18PX1NWgzPDzcZF9UKhVUqo5fVbKzszP5H0UObG1tpZ9yHqdcmLu/Ovt/K4d9bkm/zT6Lun79elRXV2Pp0qX4y1/+goCAALz44ovYv3//Ix3R2dvbIyIiAgUFBVKZXq9HQUEBYmJiOtQPDQ1FaWkpSkpKpMfPf/5zTJo0CSUlJQgICEBQUBB8fHwM2tRoNCgqKjLaJhHJm0UnGVQqFebMmYM5c+bg6tWr2LJlCxYuXIjW1lacP38eLi6WvalXq9VISUlBZGQkoqKikJOTg+bmZumsanJyMvz9/ZGVlQUHBweMHDnSYPt+/foBgEH54sWLsWrVKgwdOhRBQUFYtmwZ/Pz8MHPmTIv6RkS93yOfRVUqlVAoFBBCoK2t7ZHaSEpKws2bN5GRkYGamhqEh4dj37590kmCyspKKJWWfdli6dKlaG5uxuuvv46GhgZMmDAB+/btg4ODwyP1kYh6L4sCTqvVYvfu3cjLy8OxY8cwffp0/Pu//zsSEhIsDqJ2ixYtwqJFi4yuO3z4cKfbbtmypUOZQqHAihUrsGLFikfqDxHJh9kBt3DhQuzcuRMBAQF49dVX8emnn3b52UwiMu1hc/xxfr+OzA643NxcDBo0CMHBwThy5AiOHDlitN7u3bu7rHNE9CPz5vjj/H4/ZfYok5OToVAourMvRNSJh83xx/n9OrLoQl8isp6HzfHH+f064nxwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGTL6gG3fv16BAYGwsHBAdHR0fjmm29M1t29ezciIyPRr18/ODs7Izw8HNu3bzeoM2/ePCgUCoNHQkJCdw+DiHogq94ccdeuXVCr1cjNzUV0dDRycnIQHx+PsrIyeHl5dajfv39/vPPOOwgNDYW9vT2++OILpKamwsvLC/Hx8VK9hIQEbN68WVpWqYzfBJeI5M2qR3DZ2dlIS0tDamoqRowYgdzcXDg5OSEvL89o/YkTJ+IXv/gFhg8fjiFDhuDNN99EWFgYjh07ZlBPpVLBx8dHenh4eDyJ4RBRD2O1I7h79+6huLgY6enpUplSqURsbCwKCwsfur0QAl999RXKysrw/vvvG6w7fPgwvLy84OHhgcmTJ2PVqlUYMGCAyba0Wi20Wq20rNFoAAA6nQ46nc7SofUa7ffRbG1tlfU4+4q+sj8tGZvVAq6urg5tbW3w9vY2KPf29sbFixdNbtfY2Ah/f39otVrY2Njg448/xtSpU6X1CQkJmDVrFoKCglBeXo63334b06ZNQ2FhIWxsOt4NHACysrKQmZnZofzAgQNwcnJ6xBH2fNeaAMAWJ06cwPfnrN0belx9ZX+2tLSYXdeqn8E9CldXV5SUlKCpqQkFBQVQq9UIDg7GxIkTAQCzZ8+W6o4aNQphYWEYMmQIDh8+jClTphhtMz09HWq1WlrWaDQICAhAXFwc3NzcunU81nSm8hZQegrjxo3D6EH9rd0dekx9ZX+2v8Myh9UCztPTEzY2NqitrTUor62thY+Pj8ntlEolQkJCAADh4eG4cOECsrKypIB7UHBwMDw9PXHp0iWTAadSqYyeiLCzs4OdnZ2ZI+p9bG1tpZ9yHmdf0Vf2pyVjs9pJBnt7e0RERKCgoEAq0+v1KCgoQExMjNnt6PV6g8/PHnT9+nXU19fD19f3sfpLRL2PVd+iqtVqpKSkIDIyElFRUcjJyUFzczNSU1MBAMnJyfD390dWVhaA+5+VRUZGYsiQIdBqtdi7dy+2b9+ODRs2AACampqQmZmJ559/Hj4+PigvL8fSpUsREhJicBkJEfUNVg24pKQk3Lx5ExkZGaipqUF4eDj27dsnnXiorKyEUvnjQWZzczMWLlyI69evw9HREaGhodixYweSkpIAADY2Njh79iy2bt2KhoYG+Pn5IS4uDitXruS1cER9kEIIIazdiZ5Go9HA3d0djY2Nsj7JUHK1HjM3nMCeBeMQPtj0ZTTUO/SV/WnJ69PqX9UiIuouDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhItnrdbCL06FpaWgymoiqrboC25hIunHOEvr6fVB4aGirraaKo72DA9SEXL15EREREh/KXthouFxcXY+zYsU+oV0TdhwHXh4SGhqK4uFhabvpBiy8PFeKfJsXAxVFlUI9IDhhwfYiTk5PBkZlOp8PtuhuIiYqU9fxh1HfxJAMRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2bJ6wK1fvx6BgYFwcHBAdHQ0vvnmG5N1d+/ejcjISPTr1w/Ozs4IDw/H9u3bDeoIIZCRkQFfX184OjoiNjYWf//737t7GETUA1k14Hbt2gW1Wo3ly5fj9OnTGD16NOLj43Hjxg2j9fv374933nkHhYWFOHv2LFJTU5Gamor9+/dLdT744AOsW7cOubm5KCoqgrOzM+Lj43H37t0nNSwi6imEFUVFRYk33nhDWm5raxN+fn4iKyvL7DbGjBkj3n33XSGEEHq9Xvj4+IgPP/xQWt/Q0CBUKpX49NNPzW6zsbFRABCNjY1mb9Mb3bt3T+zZs0fcu3fP2l2hLvBtRZ0Y/OsvxLcVddbuSrey5PVptfng7t27h+LiYqSnp0tlSqUSsbGxKCwsfOj2Qgh89dVXKCsrw/vvvw8AuHLlCmpqahAbGyvVc3d3R3R0NAoLCzF79myjbWm1Wmi1WmlZo9EAuD9fmk6ne6Tx9QbtY5PzGPuS1tZW6aec96klY7NawNXV1aGtrQ3e3t4G5d7e3gb3DXhQY2Mj/P39odVqYWNjg48//hhTp04FANTU1EhtPNhm+zpjsrKykJmZ2aH8wIEDfeLeBPn5+dbuAnWBa00AYIsTJ07g+3PW7k33aWlpMbtur5vR19XVFSUlJWhqakJBQQHUajWCg4MxceLER24zPT0darVaWtZoNAgICEBcXBzc3Ny6oNc9k06nQ35+PqZOncoZfWXgTOUtoPQUxo0bh9GD+lu7O92m/R2WOawWcJ6enrCxsUFtba1BeW1tLXx8fExup1QqERISAgAIDw/HhQsXkJWVhYkTJ0rb1dbWwtfX16DN8PBwk22qVCqoVKoO5XZ2dn3ihd9Xxil3tra20k85709Lxma1s6j29vaIiIhAQUGBVKbX61FQUICYmBiz29Hr9dLnZ0FBQfDx8TFoU6PRoKioyKI2iUgerPoWVa1WIyUlBZGRkYiKikJOTg6am5uRmpoKAEhOToa/vz+ysrIA3P+sLDIyEkOGDIFWq8XevXuxfft2bNiwAQCgUCiwePFirFq1CkOHDkVQUBCWLVsGPz8/zJw501rDJCIrsWrAJSUl4ebNm8jIyEBNTQ3Cw8Oxb98+6SRBZWUllMofDzKbm5uxcOFCXL9+HY6OjggNDcWOHTuQlJQk1Vm6dCmam5vx+uuvo6GhARMmTMC+ffvg4ODwxMdHRNalEEIIa3eip9FoNHB3d0djY6PsTzLs3bsXzz33nKw/s+krSq7WY+aGE9izYBzCBw+wdne6jSWvT6t/VYuIqLsw4IhIthhwRCRbDDgiki0GHBHJVq/7qhYR3dfS0mLwve2y6gZoay7hwjlH6Ov7GdQNDQ3tE9+rfhADjqiXunjxIiIiIjqUv7S1Y93i4mKMHTv2CfSqZ2HAEfVSoaGhKC4ulpabftDiy0OF+KdJMXBxVHWo2xcx4Ih6KScnJ4OjMp1Oh9t1NxATFckLt/8PTzIQkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssXr4IxonwPUkrv39EY6nQ4tLS3QaDS8bkoG+sr+bH9dmjNXLwPOiDt37gAAAgICrNwTIjLlzp07cHd377QOpyw3Qq/Xo6qqCq6urlAoFNbuTrdpv//rtWvXZD01e1/RV/anEAJ37tyBn5+fwT1bjOERnBFKpRJPPfWUtbvxxLi5ucn6BdHX9IX9+bAjt3Y8yUBEssWAIyLZYsD1YSqVCsuXL4dKpXp4ZerxuD874kkGIpItHsERkWwx4IhIthhwRCRbDLgebuLEiVi8eLG1u0HUKzHgZOLw4cNQKBRoaGiwdlfIAj3tD1hP68/jYsAR9XL37t2zdhd6LAZcL6LVavHrX/8aAQEBUKlUCAkJwaZNm1BRUYFJkyYBADw8PKBQKDBv3ryHtnfnzh28/PLLcHZ2hq+vLz766KMOf8G3b9+OyMhIuLq6wsfHBy+99BJu3LghrW8/ciwoKEBkZCScnJwwfvx4lJWVdfXwZWfevHk4cuQI1q5dC4VCAYVCgfLycsyfPx9BQUFwdHTEsGHDsHbt2g7bzZw5E7/97W/h5+eHYcOGAQCOHz+O8PBwODg4IDIyEnv27IFCoUBJSYm07blz5zBt2jS4uLjA29sbc+fORV1dncn+VFRUPKlfR/cQ1KP97Gc/E2+++aYQQogXX3xRBAQEiN27d4vy8nJx8OBBsXPnTtHa2io+//xzAUCUlZWJ6upq0dDQ8NC2X3vtNTF48GBx8OBBUVpaKn7xi18IV1dX6fmEEGLTpk1i7969ory8XBQWFoqYmBgxbdo0af2hQ4cEABEdHS0OHz4szp8/L5599lkxfvz4rv5VyE5DQ4OIiYkRaWlporq6WlRXV4u7d++KjIwMcfLkSXH58mWxY8cO4eTkJHbt2iVtl5KSIlxcXMTcuXPFuXPnxLlz50RjY6Po37+/eOWVV8T58+fF3r17xT/8wz8IAOLbb78VQghx+/ZtMXDgQJGeni4uXLggTp8+LaZOnSomTZpksj+tra3W+NV0GQZcD9cecGVlZQKAyM/PN1qvPWhu375tVrsajUbY2dmJzz77TCpraGgQTk5OBgH3oJMnTwoA4s6dOwbPe/DgQanOl19+KQCIH374way+9GU//QNmyhtvvCGef/55aTklJUV4e3sLrVYrlW3YsEEMGDDA4He+ceNGg4BbuXKliIuLM2j72rVr0h9Gc/vTm/Atai9RUlICGxsb/OxnP+uS9i5fvgydToeoqCipzN3dXXq70664uBgzZszAoEGD4OrqKj1/ZWWlQb2wsDDp376+vgBg8FaWzLd+/XpERERg4MCBcHFxwR/+8IcOv+9Ro0bB3t5eWi4rK0NYWBgcHByksp/uWwA4c+YMDh06BBcXF+nRfsf78vLybhyR9XC6pF7C0dHxiT9nc3Mz4uPjER8fj08++QQDBw5EZWUl4uPjO3yw/dMZZNvn0NPr9U+0v3Kwc+dOLFmyBGvWrEFMTAxcXV3x4YcfoqioyKCes7OzxW03NTVhxowZeP/99zusa/+jJDcMuF5i1KhR0Ov1OHLkCGJjYzusb/9r3tbWZlZ7wcHBsLOzw8mTJzFo0CAAQGNjI7777jv84z/+IwDg4sWLqK+vx+9+9ztpduNTp051xXDo/9jb2xvss6+//hrjx4/HwoULpTJzjq6GDRuGHTt2QKvVSl+2P3nypEGdsWPH4vPPP0dgYCBsbY2/9B/sT2/Ht6i9RGBgIFJSUvDqq69iz549uHLlCg4fPoz//u//BgAMHjwYCoUCX3zxBW7evImmpqZO23N1dUVKSgreeustHDp0COfPn8f8+fOhVCqlI7BBgwbB3t4ev//973H58mX8+c9/xsqVK7t9rH1JYGAgioqKUFFRgbq6OgwdOhSnTp3C/v378d1332HZsmUdgsqYl156CXq9Hq+//jouXLiA/fv3Y/Xq1QB+PKJ+4403cOvWLcyZMwcnT55EeXk59u/fj9TUVCnUHuxPbz8KZ8D1Ihs2bMAvf/lLLFy4EKGhoUhLS0NzczMAwN/fH5mZmfjNb34Db29vLFq06KHtZWdnIyYmBtOnT0dsbCyeeeYZDB8+XPocZ+DAgdiyZQs+++wzjBgxAr/73e+kFw11jSVLlsDGxgYjRozAwIEDER8fj1mzZiEpKQnR0dGor683OJozxc3NDX/5y19QUlKC8PBwvPPOO8jIyAAAaX/6+fnh66+/RltbG+Li4jBq1CgsXrwY/fr1k6b+frA/D37219twuiSSNDc3w9/fH2vWrMH8+fOt3R16TJ988glSU1PR2Nholc9wewJ+BteHffvtt7h48SKioqLQ2NiIFStWAAASExOt3DN6FNu2bUNwcDD8/f1x5swZ/PrXv8aLL77YZ8MNYMDJVmVlJUaMGGFy/d/+9jcAwOrVq1FWVgZ7e3tERETg6NGj8PT0fFLdpC5UU1ODjIwM1NTUwNfXFy+88AJ++9vfWrtbVsW3qDLV2tra6ddsOjuTRiQXDDgiki2eRSUi2WLAEZFsMeCISLYYcEQkWww4sqp58+ZJkyva2dnB29sbU6dORV5enkVfE9qyZQv69evXfR01oX3ySeqZGHBkdQkJCaiurkZFRQX++te/YtKkSXjzzTcxffp0tLa2Wrt71JtZczI6opSUFJGYmNihvKCgQAAQGzduFEIIsWbNGjFy5Ejh5OQknnrqKbFgwYIOk27+9LF8+XIhhBDbtm0TERERwsXFRXh7e4s5c+aI2tpa6Xlu3bolXnrpJeHp6SkcHBxESEiIyMvLk9ZXVlaKF154Qbi7uwsPDw/x85//XFy5ckUIIcTy5cs7PO+hQ4e65fdEj4ZHcNQjTZ48GaNHj8bu3bsBAEqlEuvWrcP58+exdetWfPXVV1i6dCkAYPz48cjJyYGbmxuqq6tRXV2NJUuWAAB0Oh1WrlyJM2fOYM+ePaioqDC4X8WyZcvwt7/9DX/9619x4cIFbNiwQfomh06nQ3x8PFxdXXH06FF8/fXXcHFxQUJCAu7du4clS5bgxRdflI5Aq6urMX78+Cf7i6LOWTthqW8zdQQnhBBJSUli+PDhRtd99tlnYsCAAdLy5s2bhbu7+0Of78Ep12fMmCFSU1ON1t2+fbsYNmyY0Ov1UplWqxWOjo5i//79D+0/WR+P4KjHEkJIc5kdPHgQU6ZMgb+/P1xdXTF37lzU19ejpaWl0zYeNuX6ggULsHPnToSHh2Pp0qU4fvy4tO2ZM2dw6dIluLq6SlN89+/fH3fv3pXtFN9yw4CjHuvChQsICgpCRUUFpk+fjrCwMHz++ecoLi7G+vXrAXR+T9D2Kdfd3NzwySef4OTJk/jTn/5ksN20adNw9epV/OpXv0JVVRWmTJkivb1tampCREQESkpKDB7fffcdXnrppW4ePXUFftuaeqSvvvoKpaWl+NWvfoXi4mLo9XqsWbNGmpixfSbjdsam2jZ3yvWBAwciJSUFKSkpePbZZ/HWW29h9erVGDt2LHbt2gUvLy+4ubkZ7afcpviWGx7BkdVptVrU1NTg+++/x+nTp/Hee+8hMTER06dPR3JyMkJCQqDT6aSp07dv347c3FyDNgIDA9HU1ISCggLU1dWhpaXFrCnXMzIy8D//8z+4dOkSzp8/jy+++ALDhw8HALz88svw9PREYmIijh49Kk0T/6//+q+4fv269Lxnz55FWVkZ6urqoNPpnswvjcxj7Q8BqW9LSUmRLrGwtbUVAwcOFLGxsSIvL0+0tbVJ9bKzs4Wvr69wdHQU8fHxYtu2bR3uA/sv//IvYsCAAQaXifzXf/2XCAwMFCqVSsTExIg///nPHe4VOnz4cOHo6Cj69+8vEhMTxeXLl6U2q6urRXJysvD09BQqlUoEBweLtLQ00djYKIQQ4saNG2Lq1KnCxcWFl4n0QJwuiYhki29RiUi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEcnW/wfVmrrr8h5MTAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T18:44:43.686225Z", + "iopub.status.busy": "2024-03-01T18:44:43.685941Z", + "iopub.status.idle": "2024-03-01T18:44:43.969377Z", + "shell.execute_reply": "2024-03-01T18:44:43.968491Z" + }, + "papermill": { + "duration": 0.307354, + "end_time": "2024-03-01T18:44:43.971646", + "exception": false, + "start_time": "2024-03-01T18:44:43.664292", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAErCAYAAABKPZ9fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABqvklEQVR4nO2dd3wT9f/HX5fZPaClAwpt2busIjj4osWi+GWIMuTHcn0RcVBBAZWyFAREFBAURVDZDlTEKlSKjDIse69CKdDSUrrbzPv9kd7lLrlLL2nSJuXzfDzygFw+d/e55u6d9/5QNE3TIBAIhHqIrK4nQCAQCK6CCDgCgVBvIQKOQCDUW4iAIxAI9RYi4AgEQr2FCDgCgVBvIQKOQCDUW4iAIxAI9RYi4AgEQr2FCDgCgVBvcQsBt2LFCkRHR8PLyws9e/bE4cOHRceuXr0aDz/8MIKDgxEcHIyEhASr8ePGjQNFUbxX//79XX0ZBALBzahzAbd582YkJSUhOTkZR48eRefOnZGYmIg7d+4Ijk9LS8PIkSOxe/dupKenIyoqCo8//jhu3rzJG9e/f3/cvn2bfW3cuLE2LodAILgRVF0X2/fs2RM9evTA8uXLAQBGoxFRUVF47bXXMG3atGr3NxgMCA4OxvLlyzFmzBgAJg2usLAQ27Ztc+XUCQSCm6Ooy5NrtVpkZGRg+vTp7DaZTIaEhASkp6dLOkZ5eTl0Oh0aNGjA256WloZGjRohODgYjz76KObNm4eGDRsKHkOj0UCj0bDvjUYjCgoK0LBhQ1AU5cCVEQgEV0LTNEpKShAZGQmZTNwQrVMBl5+fD4PBgLCwMN72sLAwnD9/XtIx3nnnHURGRiIhIYHd1r9/fzz99NOIiYnBlStXMGPGDDzxxBNIT0+HXC63Osb8+fMxe/bsml0MgUCodW7cuIEmTZqIfl6nAq6mLFiwAJs2bUJaWhq8vLzY7SNGjGD/37FjR3Tq1AnNmzdHWloaHnvsMavjTJ8+HUlJSez7oqIiNG3aFDdu3EBAQIBrL4JAINhNcXExoqKi4O/vb3NcnQq4kJAQyOVy5Obm8rbn5uYiPDzc5r6LFy/GggULsGvXLnTq1Mnm2NjYWISEhODy5cuCAk6tVkOtVlttDwgIIAKOQHBjqnMh1WkUVaVSoVu3bkhNTWW3GY1GpKamolevXqL7LVy4EHPnzkVKSgq6d+9e7Xmys7Nx9+5dREREOGXeBALBM6jzNJGkpCSsXr0a69atw7lz5/DKK6+grKwM48ePBwCMGTOGF4T46KOP8P7772PNmjWIjo5GTk4OcnJyUFpaCgAoLS3F1KlTcfDgQVy7dg2pqakYNGgQWrRogcTExDq5RgKBUDfUuQ9u+PDhyMvLw8yZM5GTk4O4uDikpKSwgYesrCxelGTlypXQarV45plneMdJTk7GrFmzIJfLcfLkSaxbtw6FhYWIjIzE448/jrlz5wqaoQQCof5S53lw7khxcTECAwNRVFQk6oOjaRp6vR4Gg6GWZ0fwFJRKpWDUnlBzpDyjgBtocJ6IVqvF7du3UV5eXtdTIbgxFEWhSZMm8PPzq+up3LcQAWcnRqMRmZmZkMvliIyMhEqlIsnABCtomkZeXh6ys7PRsmVLosnVEUTA2YlWq2XLySppBW4UaxDgpUB4oHddT43gZoSGhuLatWvQ6XREwNURdR5F9VRkMhkMRhqVOgO0euLGJFhDNPu6hwi4GsAEd40kTkMguCVEwNUAedUvNBFwBIJ7QgRcDaBYAVfHE5HIf/7zH7z55pt1PY16Q1paGiiKQmFhYV1PhSACEXA1QF7lYqlvGhx5cAn1BSLgagCrwXmKCkewQqvV1vUUCC6ECLgaIJOZBJzBSKNcq6/1V02KUDQaDd555x1ERUVBrVajRYsW+Prrr3Ht2jX07dsXABAcHAyKojBu3Lhqj1dSUoJRo0bB19cXERER+OSTT6xM4u+++w7du3eHv78/wsPD8dxzz/Fa0zOaY2pqKrp37w4fHx/07t0bFy5ckHxd8+bNQ6NGjeDv748XX3wR06ZNQ1xcHPv5uHHjMHjwYHzwwQeIjIxE69atJc0NAHbs2IFWrVrB29sbffv2xbVr1yTPi1A3kDy4GlAl31ChM6DdzD9r/fxn5yTCR+XYVzhmzBikp6fjs88+Q+fOnZGZmYn8/HxERUXhxx9/xNChQ3HhwgUEBATA27v6HL+kpCTs378fv/76K8LCwjBz5kwcPXqUJ1x0Oh3mzp2L1q1b486dO0hKSsK4ceOwY8cO3rHeffddfPzxxwgNDcWECRPw/PPPY//+/dXOYf369fjggw/w+eef48EHH8SmTZvw8ccfIyYmhjcuNTUVAQEB2Llzp+S53bhxA08//TReffVVvPzyy/j333/x1ltvVTsnQt1CBFwNkHloFPXixYvYsmULdu7cyXZCjo2NZT9n2r83atQIQUFB1R6vpKQE69atw4YNG9h+e9988w0iIyN5455//nn2/7Gxsfjss8/Qo0cPlJaW8sqZPvjgA/Tp0wcAMG3aNAwYMACVlZW8pqZCLFu2DC+88ALbiWbmzJn466+/2E4zDL6+vvjqq6+gUqkkz23lypVo3rw5Pv74YwBA69atcerUKXz00UfV/n0IdQcRcDWAEXBqhQynZj0Ouax2Ezu9lY5lxx8/fhxyuZwVIjXl6tWr0Ol0iI+PZ7cFBgay5h9DRkYGZs2ahRMnTuDevXswGo0ATB1j2rVrx47jNjBlevjduXMHTZs2tTmPCxcuYOLEibxt8fHx+Pvvv3nbOnbsyBNuUuZ27tw59OzZk7ePrZ6FBPeACLgawMgziqLgpZRDKfcMl6YUk9PZlJWVITExEYmJiVi/fj1CQ0ORlZWFxMREK0e/Uqlk/28O5BidNhdfX1+H50YQ51R2EV7+7l+8078NBndpXNfTAUCCDDWCoiiPNFM7duwIo9GIPXv2CH7OaDdSW0HFxsZCqVTiyJEj7LaioiJcvHiRfX/+/HncvXsXCxYswMMPP4w2bdqIrn3rKK1bt+bNAYDVeyGkzK1t27ZWC4wfPHiw5pOuR+y9nIfbRZX4+7xzv9eaQARcDWEFnPMUDJcTHR2NsWPH4vnnn8e2bduQmZmJtLQ0bNmyBQDQrFkzUBSF7du3Iy8vz8qHZYm/vz/Gjh2LqVOnYvfu3Thz5gxeeOEFyGQyVgNr2rQpVCoVli1bhqtXr+LXX3/F3LlznXpdr732Gr7++musW7cOly5dwrx583Dy5Mlqa0KlzG3ChAm4dOkSpk6digsXLmDDhg1Yu3atU+fv6ZRU6gEA7vRTTwRcDfHUetSVK1fimWeewcSJE9GmTRu89NJLKCsrAwA0btwYs2fPxrRp0xAWFoZJkyZVe7wlS5agV69eeOqpp5CQkIAHH3wQbdu2ZQMDoaGhWLt2LbZu3Yp27dphwYIFWLx4sVOvadSoUZg+fTqmTJmCrl27IjMzE+PGjas2OCFlbk2bNsWPP/6Ibdu2oXPnzli1ahU+/PBDp87f0ymp1AFAjdKXnA3p6CuArW6hlZWVyMzMRExMDLy8vHAxtwSVOgNiQnzh76UUOeL9R1lZGRo3boyPP/4YL7zwQp3No1+/fggPD8d3331X6+e2vFfqO29uOoZtx29hQMcIrBjV1aXnIh19awmZh9Wjuopjx47h/PnziI+PR1FREebMmQMAGDRoUK3Noby8HKtWrUJiYiLkcjk2btyIXbt28fLdCK7DbKK6z8NABFwNkdXTelQulmkclpw9exaAaa3aCxcusMtB7t27FyEhIU6bR/v27XH9+nXBz7744gs8/fTT2LFjBz744ANUVlaidevW+PHHH9lcP4JrKdFUCTg3ehSIgKshsvugHjUyMhLHjx+3+XnTpk2RkZHh0nns2LEDOp1O8LOwsDB4e3tj165dLp0DQRxGg3OnH3si4GrI/WCiKhQKtGjRoq6ngWbNmtX1FAg2MAcZ6ngiHEgUtYZ4ahSVQHA2pRpGg6vjiXAgAq6GeGKiL4HgbGiaZk1Ud8qEIwKuhtwPJiqBUB0VOgMMVQ+BO/3WEwFXQ1gTlUg4wn1MKau9uZc1QwRcDSEmKoEAFHMEnDs9CUTA1ZD7xUSNjo7G0qVL2fcURWHbtm11Nh+Ce8EEGAD3MlFJmkgNuR8SfYW4ffs2goOD63oaBDeBSREB3OtZIAKuhtwPib5ChIeH1/UUQNM0DAYDFApyG9c1JRwT1Z0gJmoNkVEUQNOgteWAtqx2X3b+UkpZGEYqXBP12rVroCgKP/30E/r27QsfHx907twZ6enpvH327duHhx9+GN7e3oiKisLrr7/OdjABpC9K88cff6Bbt25Qq9XYt2+f3XMnOB93DTKQn74aIpMBlL4Crda1rf2Tz7gFqHyrH1eFlIVhasK7776LxYsXo2XLlnj33XcxcuRIXL58GQqFAleuXEH//v0xb948rFmzBnl5eZg0aRImTZqEb775BoD0RWmmTZuGxYsXIzY2lpjJbkIxx0R1I/lGBFxNkVXTTNFdkLowTE2YMmUKBgwYAACYPXs22rdvj8uXL6NNmzaYP38+Ro0axWqLLVu2xGeffYY+ffpg5cqV8PLykrwozZw5c9CvXz+nzZtQc7gmKhFwFqxYsQKLFi1CTk4OOnfujGXLlvEWMOGyevVqfPvttzh9+jQAoFu3bvjwww9542maRnJyMlavXo3CwkI8+OCDWLlyJVq2bOn0ucsoCrTCG6fHnUeHSPG+VC5B6SN5qNSFYWqC2GIxbdq0wYkTJ3Dy5EmsX7+eHUPTNIxGIzIzM9G2bVvJi9J0797daXMmOAduFNWdTNQ698Ft3rwZSUlJSE5OxtGjR9G5c2ckJiaK9utPS0vDyJEjsXv3bqSnpyMqKgqPP/44bt68yY5ZuHAhPvvsM6xatQqHDh2Cr68vEhMTUVlZ6fT5yygAFAWjwhu00sdkMtbWy820R1uLxZSWluJ///sfjh8/zr5OnDiBS5cuoXnz5uzCLwEBAVi/fj2OHDmCn3/+GYD16vOWi8YQ6h5uFNV9xJsbCLglS5bgpZdewvjx49GuXTusWrUKPj4+WLNmjeD49evXY+LEiYiLi0ObNm3w1VdfwWg0IjU1FYBJK1i6dCnee+89DBo0CJ06dcK3336LW7duuSRvi2uiutMvlyVSFoZxJV27dsXZs2fRokULq5dKpaqVRWkIroOfB+c+z0GdCjitVouMjAxeQ0KZTIaEhASrCJwY5eXl0Ol07GLFmZmZyMnJ4R0zMDAQPXv2FD2mRqNBcXEx7yUVigIouH+yr5SFYVzJO++8gwMHDmDSpEk4fvw4Ll26hF9++YVd76E2FqUhuA539cHVqYDLz8+HwWBAWFgYb3tYWBhycnIkHeOdd95BZGQkK9CY/ew55vz58xEYGMi+oqKiJF+DaelA0//dPReuuoVhXEmnTp2wZ88eXLx4EQ8//DC6dOmCmTNnskGO2liU5n7kXpkWv5+8jUqdtCUgHcVdS7XcIsjgKAsWLMCmTZuQlpZWo4d0+vTpSEpKYt8XFxfbJeRkMgoGA+3WJipg0uK4Tv6ysjLMnj0bL7/8crX7Xrt2jfeea4ZER0dbmSVBQUFW23r06IG//vpL9BwjR47EyJEjRc/zn//8x63MH0/g09RLWHvgGhYO7YRhPaTf0/ZSSioZrAkJCYFcLkdubi5ve25ubrWZ8osXL8aCBQuwa9cuXvSO2S83N5eN5DHvxfK91Go11Gq1g1fBLddy+BC1gjssDEOoXW4VVgAA7pVrqxlZM4iJKgCzOAkTIADABgx69eolut/ChQsxd+5cpKSkWKUMxMTEIDw8nHfM4uJiHDp0yOYxa4IndRRZvHgxOnfujISEBJSVlWHv3r04d+4c/Pz8RF8Ez6VMWzuLMfMFnPs8B3VuoiYlJWHs2LHo3r074uPjsXTpUpSVlWH8+PEAgDFjxqBx48aYP38+AOCjjz7CzJkzsWHDBkRHR7N+NeZhpCgKb775JubNm4eWLVsiJiYG77//PiIjIzF48GCXXIOn1KN26dJFcGGYiooKm4vKEDyX0lpYCEZvMKKC4+Nzp6egzgXc8OHDkZeXh5kzZyInJwdxcXFISUlhgwRZWVmQycyK5sqVK6HVavHMM8/wjpOcnIxZs2YBAN5++22UlZXh5ZdfRmFhIR566CGkpKS4zJlOeYiJKoa3t7dbLCpDcD61sZQfN0XE1eeylzoXcADYmkQh0tLSeO8tnd1CUBSFOXPmsD4mV8BVw+UyzzFRCbWHO5hqjAbnyrlYdhJxp+egzhN9PQ0mW7+8vJzd5kk+OELtwVRgyOXyOptDWS1ocJYCzp0eA7fQ4DwJuVyOoKAgNsvex8cHRp0GtF4HjYZCpbKaAxDuC4xGI/Ly8uDj41Nn/eoMRhplWpNvzJUyh1um5epz2QsRcA7ApKIwQq6oQoeSSj0qvBQo8bZPwhVX6GCkgSAfIhnrGzKZDE2bNq2VShEhmAgq4FrrwtoH5z4ijgg4B6AoChEREWjUqBF0Oh3W7s/Edwdv4alOkZjcL0bycYxGGv2W7gFoYMuEXmjo63guHsH9UKlUvABZbVNaS7lpjIkqo0yBNjeSb0TA1QS5XA65XA5KocLNEgPyyo12RWqNRho3i00mRLlehsa1UDJFuH8oq6UCeMZE9fdSoqhC51a+aBJkcAI+KpMTuVxrX70f9zYoqtCJjiMQHKFEUzv1ocx5ArwVLj+XvRAB5wR8VKYvtrwGBc1EwBGcTW2bqAFeyqpzuY+IIwLOCbAanMbxlYWIgCM4m7Ja6rLLmKhmAeeyU9kNEXBOwNtRE5VzJxABR3A2tWWiMpoiMVHrKYyJWkFMVIIbUVtL+VmaqCTIUM9gTNQyO01UEmQguBJefporfXBskIGYqPUSRsBV2GmiciECjuBsymorimoZZHAjI5UIOCfAjaLaE0HiDi0mAo7gZLg+OFe28mKCDIFVPriqhdTcAiLgnAATZDAYaWgNjn27RIMjOJvSWlongTGF/b3cr9yQCDgnwJioAFCuccxMJQKO4GxqYzFmmqbNJqq3+wUZalSqVVpayi7syxAQUMuru7sBSrkMKrkMWoMR5ToDgiXux/VVEAFHcDb8tUpdc44KnQGGKvM3sD4EGTIzMzFgwAD4+voiMDAQwcHBCA4ORlBQEIKDpT7a9Q9vNtDgWLIvEXAEZ1Na6Xjiub3nkFFmS8adggx2a3D/93//B5qmsWbNGoSFhdVZKxh3w1clR1GFDmV2mKjcX7pKnREavQFqRd01RyTUL2rDRGXWQ/VTKziNX11yKoewW8CdOHECGRkZaN26tSvm47E4Ws3ApbhCj1B/IuAIzqE2TFRugIHRdTzaRO3Rowdu3Ljhirl4NOZqBlKPSnAPeALORWajuVWSWYNzp2ItuzW4r776ChMmTMDNmzfRoUMHdo0CBu4izPcTztDgiIAjOAuN3gCt3hwAdJXZyERQ/b0Ubrm6nN0CLi8vD1euXGHXLQVMHW5pmgZFUTAYHH/APRlftqNITUxUIuAIzsHSF+wyE7WSY6Ky53IfCWe3gHv++efRpUsXbNy4kQQZOLDVDHZEUS3vA6LBEZyFZQTVVUKnuMpE9VMrWFng0Rrc9evX8euvv5KFgi1gTVTSUYTgBtTWYsxCJqo7aXB2BxkeffRRnDhxwhVz8Wh8HSi4t3T8EgFHcBZWAs5Fjn9uFJUJMriPeHNAg/vvf/+LyZMn49SpU+jYsaNVkGHgwIFOm5wn4V1lotqTB2cJEXAEZ1Gq4d9LrgsymKOoZh+ca87lCHYLuAkTJgAA5syZY/XZ/RxkYFsmkTQRghtQWktBBnc3Ue0WcJa1pwQTjqysRYIMBFdRW0EGs4nqnpUMdvngdDodFAoFTp8+7ar5eCw+xEQluBGWJqqrZI65VMvsqnKnWlS7BJxSqUTTpk3vWzPUFo6YqJa3AcmDIziLWtPguJUMsqogg/vIN/ujqO+++y5mzJiBgoICV8zHYyGVDAR3gvHBqRWmR9zVlQx+6noSZFi+fDkuX76MyMhINGvWDL6+vrzPjx496rTJeRJkXQaCO8GYqP5eSmhKNS4zGrnrMbBBBjcyUe0WcIMHD3bBNDwf1gdnVyUD/0Yo1xqgMxihlJNGy4SaUcqudKVAfqnGJe2S9AYju1Smv5eCbdfvTkEGuwVccnKyUyewYsUKLFq0CDk5OejcuTOWLVuG+Ph4wbFnzpzBzJkzkZGRgevXr+OTTz7Bm2++yRsza9YszJ49m7etdevWOH/+vFPnbYkzNDjA5Idr6Kd2xpQI9zEllRbrJLhA6HCTif28FLhXpjWdyo1sVIdblmdkZODcuXMAgPbt26NLly52H2Pz5s1ISkrCqlWr0LNnTyxduhSJiYm4cOECGjVqZDW+vLwcsbGxePbZZzF58mTR47Zv3x67du1i3ysUNerMLgmH0kQ4/2danhcRAUdwAsySgf5qZrV55wsdRoh6KWVQymVsLar7iDcHBNydO3cwYsQIpKWlISgoCABQWFiIvn37YtOmTQgNDZV8rCVLluCll15iO5OsWrUKv//+O9asWYNp06ZZje/Rowd69OgBAIKfMygUCoSHh9txVTXHXGxvgNFIsxElqQR4K5FfqiF+OIJT0FfZiWyQwQgUlesQ6OO8la8stcR60fDytddeQ0lJCc6cOYOCggIUFBTg9OnTKC4uxuuvvy75OFqtFhkZGUhISDBPRiZDQkIC0tPT7Z0Wj0uXLiEyMhKxsbEYNWoUsrKybI7XaDQoLi7mveyFu7JWpd5+MzWo6sYjAo7gDPQGk5RRyE1SJ+VMDjrP+Qt7L+U57RxsmVaVlsj9Sbc0U++WajD9p1M4mV3otPNLwW4Bl5KSgs8//xxt27Zlt7Vr1w4rVqzAH3/8Ifk4+fn5MBgMCAsL420PCwtDTk6OvdNi6dmzJ9auXYuUlBSsXLkSmZmZePjhh1FSUiK6z/z58xEYGMi+oqKi7D6vt5KzdKBEM5V7DzArEhEBR3AGzEpXCouA1cd/XXTaObhVDAA4HX2ttbjtJ29j4+EsfL0v02nnl4LdAs5oNFoV2AOmJGB3KON64okn8Oyzz6JTp05ITEzEjh07UFhYiC1btojuM336dBQVFbEvR1qyy2QUvJSmP6cjgQZGwJFkX4IlF3NLcKe40q59DFUSRmnhKnGm9ShmogLWi9wUVAUguF2GawOH2iW98cYbuHXrFrvt5s2bmDx5Mh577DHJxwkJCYFcLkdubi5ve25urlP9Z0FBQWjVqhUuX74sOkatViMgIID3cgRfe1NFOPdAENHgCAJk3yvH45/8g/gPU+3aj9HgXJlyVMJpdgmA1/zWUpAy93Vt++fsvvrly5ejuLgY0dHRaN68OZo3b46YmBgUFxdj2bJlko+jUqnQrVs3pKaavzij0YjU1FT06tXL3mmJUlpaiitXriAiIsJpxxTDnmqGrf/ewP4r+ez7ACLgCAKcvlnk0H76KmvK0kR1poQpsTBRbWlwTOff2l713u4oalRUFI4ePYpdu3axuWVt27blBQukkpSUhLFjx6J79+6Ij4/H0qVLUVZWxkZVx4wZg8aNG2P+/PkATIGJs2fPsv+/efMmjh8/Dj8/P7bD8JQpU/Df//4XzZo1w61bt5CcnAy5XI6RI0faPT97kZoLd/pmEab+cJLntyNBBoIQjsoDAxNkqAUT1c9LKMjAH8u4Xmo7wOpQghhFUejXrx/69etXo5MPHz4ceXl5mDlzJnJychAXF4eUlBQ28JCVlQWZzPwLdOvWLV6+3eLFi7F48WL06dMHaWlpAIDs7GyMHDkSd+/eRWhoKB566CEcPHjQrvQVRzF3FLFtou65aIpkVXDam5MgA0EIRwWCvhZNVMYHJ7OxPktxhemZqO0kYIcEXGpqKlJTU3Hnzh2rwMKaNWvsOtakSZMwadIkwc8YocUQHR1d7R9o06ZNdp3fmZg7itjW4IRC9UTAEYRwVB4wpqBSbqHBOVG+lLJ1qNWbqHXlg7NbwM2ePRtz5sxB9+7dERERQVbV4iClmqFMo0fG9XtW280mquMdgQn1D0crEMQ0OGdWNHA7iQC200Q8xge3atUqrF27FqNHj3bFfDwab041gxiHMu9CZ7D+kkmaCEGIGvvgXKjBWdW7chDT4Gq7EN9uA12r1aJ3796umIvHY178WVwL++divuD2QG8VACLgCHxq6oOzDDI4E1tRVO68dQYj+6Nf20EGuwXciy++iA0bNrhiLh6PlLVRxUplGA2uRKNnc5gIBEed8myir6WJ6lQNrioPrppKBu6PttsHGSorK/Hll19i165d6NSpk1VVw5IlS5w2OU+jujSRm4UVuJJXJvhZgLf5qyiu0CHYV+X8CRLuG8RKtZwpXtiec4JpIuYzFfEEnBMnIAG7BdzJkycRFxcHAFaLz9zvAQdzRxFhE3VvVXpIx8aBOGWRwKmUyeCjkqNca0AREXCEKrgCgaZpSc8YTdOcSgbXPJM0TVv54EQ1OM76EG4fZNi9e7cr5lEvYDS4MhENbu8lk/+tb5tGVgKOokxmKiPgCASAH/Wkab6fSwyui0MhszRRnSNgKnQG9jzmUi3z50YRDa62BRzpje1EbJmoBiONfZdNAq5Pq1DBG5XkwhEs4aaZShUOeq6Ac5EGx+TAySjzfS9Wi1pchyYqEXBOxNuGiXrqZhGKKnTw91Kgc5NAq+gWRVGkHpVgBS3yf1twBaHKRUGGYu5qWhzBJtT0si59cETAORFfG4m+jP/tweYhUMhlkAuE74kGR7CEa1I6Q4NzVqKvZZkWg3npQPN5mCRfZ55fKkTAORFb3UQYn1vP2AYArH0jABFwBGt4GpxE2WAwiPvgnIVls0sGmcC6DHwfnEumI4rdV//PP/9Ar7c2wfR6Pf755x+nTMpTYaKoQj647HsVAIBmDX0ACPtGXFXNkF+qqbYBAMFN4QgEhzQ4y24iThIw5ggqX8AJmajFnPLD2s6Ds1vA9e3bV3BV+6KiIvTt29cpk/JUzLWo1sLkZqFJwDUJrhJwtWSiFpZr0X3eLnSe/ZfTjkmoPbgmnVTtx8CpYrAMZjlLvJSKlGlRVUYqVxgXe5IGJ5aLc/fuXatV7u83xNJEiit1rNBqHOQNADwfHPPndIWAO33TtICOnlRHeCS0AxocU8Ugl1FWLYycpUEVsz44EQ1OYKwzzy8VyXlwTz/9NABTtG/cuHFQq81rdxoMBpw8efK+r1FlTFSt3giDkWaF2M0q8zTYRwnfqpyh2vLBucgFQ6gleD44icsZMD44uYzilxc4EctOIgyMgDNyflB5UVTXTEcUyQIuMDAQgEkC+/v7w9vbm/1MpVLhgQcewEsvveT8GXoQ3KUDy7V6Vn1nBBxjngKotSiq/D6vLvF0HNHgmHblchllJd+cZqJqbJuoXIrrMNFXsoD75ptvAJiaTk6ZMuW+N0eFUCtkkFEmP0O51sB++dn3ygGYzVOAH2Rg/sfkwXFV+prCFaSOLEhNqFv4PjiJJirHB2fVZddpQQZhE1XmZnlwdpdqJScnu2Ie9QKKouCjUqBUo+elipgDDBwBJ6jBmb6OonLnCTiuv1RvpKEiAs6j4Jp6koMMrA9OJqm0yxHEo6j8IANN0xa1qK6Zjxh2e2hyc3MxevRoREZGQqFQQC6X8173O94CkVQmRaQxR8DJOc4x5qYI4LRMMjrpTuBqcKQNk+fB/c6kOuj1rA/Oep0E55uotoMMZVqDQ9fgLOzW4MaNG4esrCy8//77pGW5AEL1qNkCPjhbaSI0bfqFDPSx7pRqL9zTmHwz5EfIkzDSwv+3hdlElVn74JwWRWWCDMKVDIwGZ+lPdnsTdd++fdi7dy/bMonAh11ZqxoTVSjIoFbI4aWUoVJnRFGFzkkCjmhwnoyxBqVaQlFU5+XBiaWJVFUyVJ3IMmnd7buJREVF1bqa6UmYNTjTL1y5Vo+CMi0AvomqFAgyAPZFUrefvIVpP56EziCeP8BVsIXWgiC4N9wfpSt5paxzX8o+QkEGV1cymH+3RTQ4WwctygYKMp0zQWY+9u6wdOlSTJs2DdeuXXPqROoLlitrMSkiAV4KBHBC6kIaHGCfgJu04Rg2HbmBzUduiI7h3tBEg/M8DJwvcPTXh9Fr/t/V78PR4FzlQGIFnKWJygYZTO8la3A5p4CvEoD1zwDl1pVSjmK3iTp8+HCUl5ejefPm8PHxsWpZLlTGdT/BrFbPmKjmAIMPb5yCF2Qwb3ckF+5OcaXoZ9z7SW+UmClKcBssg02lEmqKuQLOMi3IGd089AYju/ZvdWkizH3MdKsWPP2Vv4HNYwBtCRDaBtCVA2hQ43kCDgi4pUuXOuXE9RWmUoExUbMF/G+AczQ4BluKGfeGJhqc52HD+yCKrURfZ8AVsn5eliKEnybCBCOCqrpVW2lwx9YDv70OGPVAs4eAEd8D3sFOm6vdAm7s2LFOO3l9xLJlEpPkayngxHrlO9L00pbjlq/BEQHnaRgccJrZLLZ3wi3AmKdeSpnVql2W3USY+zjIR4VbRZXmH2OaBvYsBNI+NL3v8Aww+HNAoYYzcahS8cqVK3jvvfcwcuRI3LlzBwDwxx9/4MyZM06dnCfio+SnibAmapC4Bsf9na2pBlepM+D63TLOZ9I1uCPXCjBoxX4cv1Eo+dwE1+JIPiQ3imqZxiUm4M7dLsakDUdx+U5Jtce3teAza6JWWQ7FrIBTmrcbdMCvr5mF20OTgadXO124AQ4IuD179qBjx444dOgQfvrpJ5SWlgIATpw4QaocAPiomTQR000gVIcKiDcilCrguJFs7v/f2noCfRal4XyOqYsI937WVxNFfXZVOk7cKMRzqw/aHEeoPRzR4IwOBBm+/Ocqtp+8jTc2Ha/2h1CsTAsw/1hbpokwAs7LUAFsHAEc+w6gZMCAj4GEWS7rCmH3UadNm4Z58+Zh586dUKnMS9s9+uijOHiQPBiWUVRzkq8NH5xAkKG6ppfclA/u7XijwGQSZ901/UvbocExCHUkJtQNNdXgrGpRRThd1XH6zK1irD903eZYcwTVWsCJBRkCvVUIxT0s174HXN4FKH2AERuAHi+y+565VQSN3rn3nt0C7tSpUxgyZIjV9kaNGiE/P98pk/JkuJUMlToD8ks1AKwFnFAlAyBdg9NyvM9cwcWYpMxNzlUAdCSK6nE4EhjiVTJY+eCsj1ehNeBKXin7ftGfF9j7VgixTiIAJ9GXMVGrtL3muIGf1cloTV8FfEKAcduB1k+w+1XqDBiy4gA6zfoLOUXiWQH2YreACwoKwu3bt622Hzt2DI0bN3bKpDwZJk2kXGtgKxj81ApWcDGILefG5MpVJ+A0OvMvHU/AVckwRsBxnw8SRfU8HDFReT44WKaJWHMupxhGGgjxU6ND4wCUVOoxf8d50ePbMlEZjBwNrid1DqPPvowmVD6yEAG8uBNo3I03PuP6PWgNRgT7qBAW4DxfnN0CbsSIEXjnnXeQk5MDiqJgNBqxf/9+TJkyBWPGjHHaxDwVJk2kXKtn/W+Ng7ytnL28YnvOdqY8q7qWSRq9WRvjanOsBle1jfuLXZ0PjuB+OGKiGu2Mop6pMk87Ng7A3EEdAAA/Hs3GkWvCOa0lGuFml4DZlcbcd/GlafhWNR9qfQn+NbbCC4oPgQaxVvsduGKy/no3b+jU+na7BdyHH36INm3aICoqCqWlpWjXrh0eeeQR9O7dG++9957TJuapcNNExPxvgBNMVI6Aq+T4zJgbmBFmjmhwpKOS+1ATDU4muCaD9fGYtvbtIwPRpWkwRvSIAgC8v+00+0PJxVYUlV2TwUgD+5ZinmEJ1JQeOY0TMUo7A/cQIDjnA1fuAgB6NW8o5RIlY7eAU6lUWL16Na5cuYLt27fj+++/x/nz5/Hdd9851C5pxYoViI6OhpeXF3r27InDhw+Ljj1z5gyGDh2K6OhoUBQlmnRszzGdDTdNhG10KSDghNZkAPhBBlu/3lwNrpLjmDVY+uA4N7TUSgaxJGRC7eNIoq+h6nsWbHgpwJnbJg2uQ2OT8Hm7fxsE+ShxPqcE69KtAw6MiWqd5Gv6cZTBiKiD7wO7TFkVX+ufwPW+y6GBCkJGckmlDiezTXOocwHH0LRpUzz55JMYNmwYWrZs6dAxNm/ejKSkJCQnJ+Po0aPo3LkzEhMT2dw6S8rLyxEbG4sFCxYgPDzcKcd0Nr6cNBGhLiIM1WlwRhooFVidi4EbbeK2ZjIHGRgT1byPVA2OtMByH2qeB8f/zFIh1OqNuJBjyn1rHxkIAGjgq8I7/dsAAD7ZedGqFJBZUStAQMB5QYsvlJ+g0fnvQYPCHN1ozNWPRoCvl+l6BC7nyLUCGIw0mjX0sUqnqimSKhmSkpIwd+5c+Pr6IikpyebYJUuWSD75kiVL8NJLL2H8+PEAgFWrVuH333/HmjVrMG3aNKvxPXr0QI8ePQBA8HNHjulshE1U6y9NIRf+bfFSyqFSyKDVG1FUruMV6HPhmag68/8tTVRHKhnIOg7uQ40rGaoJMly6UwKdgUagt5L3Qzy8exQ2HbmBEzcKsfivC1j4TGf2M7FOIijNwycV76Kt/BKMMhXyHl+ONdv84K9WsD/oQlHcA5dN5mlvJ2tvgEQBd+zYMeh0Ovb/Ytjzy6/VapGRkYHp06ez22QyGRISEpCeni75OM44pkajgUZjDosXFxc7dH6AnyZyU6SKAeBrcJY3YaC3EnklGhRV6BAlNmeOgKvQiWtw9lQyMBAT1X1wRIMzcHxw1X2VZ1j/WwDv+ZXJKLzxWAs8v/ZfZFy/x9unRKjZZf5lYP1QtDVewz3aD7ceXwNtZDyAAwjwVlp1GeFi9r+F2HOZkpAk4Hbv3i34/5qQn58Pg8GAsLAw3vawsDCcPy8eonbFMefPn4/Zs2c7dE5LfJSmP6neSCOnSrUXMlFtCRFGwNlK9uWaqJUCAo5JBObeT7b6xnEh8s19qEmQQUoU9fQtk++rfaS18z/Ix5TIr7W4b0os25VnHTJVJ1QU4DYVhlGaqfggtDu0VYIw0FvJ3lOWddP3yrQ4e9skZHvFOl+DI6tmApg+fTqKiorY140b4v3VqsNbxQ+0eCllaOCrshqnEAkyANIiqXwTlSPgqjYb2Dw4+zU4MfOZUPvUJNHXtOiM7YUDz9wyCZcOjQOtjqOqug+49xpgkQd37jfg24FARQEQ2QVv+C3EVToSNGj2/g3wVpjnYXE5B6+atLdWYX4I9Xd+LaokDY5Z9FkKP/30k6RxISEhkMvlyM3N5W3Pzc0VDSC46phqtZq3kHVNUClkUMopVoNqEuwjaLrLRRJ9AWkCTsxEpS3y4Lg3lFQfnFDkrajcOS3UCfbhSItvngZn8Zll0OnsLXOKiCVqhUnAaSwEHFPJ0OTiOmDfLAA00Ko/8MwaFK3IAFAC0NwyLXENjjFPe7vAPAUkanCBgYHsKyAgAKmpqfj333/ZzzMyMpCamsouDi0FlUqFbt26ITU1ld1mNBqRmpqKXr162XEJrj2mTQx6IG0BUMn32THVDICweQoAShvFxZIEnE44yMDIMH0NNDhLBW7HqdvoPOcvfH/Qdo0iwfk4psFx+sHZWFUrM78UFToDvJVyxIRYr3OsUlhrcDRNo7RSi3cV3yN0X7LpiN2fB4avB1S+5pXtaXM9dYCX0lyEb3EOJsHX2ekhDJI0OGbRZwB45513MGzYMKxatYrNezMYDJg4cSICAoST+MRISkrC2LFj0b17d8THx2Pp0qUoKytjI6BjxoxB48aNMX/+fACmIMLZs2fZ/9+8eRPHjx+Hn58fWrRoIemYTuWv94BDK01q+qitQEAkANPCM0yjP6EAA2DZLomPJAFnqC7I4LwoKlOIfe6248EXgmM4Ul3H3BryaoIMjHnaLjJAdBEkgC/gKirKsFT+GZ6SHzJteCzZ1O6o6p7h1qIWczQ4SkCDyy2uxJW8MlAU8EBMHQo4LmvWrMG+fft4Sb1yuRxJSUno3bs3Fi1aJPlYw4cPR15eHmbOnImcnBzExcUhJSWFDRJkZWVBxtF0bt26hS5durDvFy9ejMWLF6NPnz5IS0uTdEyn0nk4cPpHIPe0qZ/8qB+AsHbwUXM1OOG8HrFaVEBa00vuTafVG9lV65kHggko8DQ4iUGGW0WVmLf9LF5PaIkALyVropBCr9rHkQWeuIm+VmkinOMxP1xCAQbArMHpjbTp/qq8B8X6EXhKfghaWg7l0FWgOg3j7WNeNtBcbhjgrWRbp3MvJ73KPO0QGegy94fdAk6v1+P8+fNo3bo1b/v58+dhdKBbxaRJkzBp0iTBzxihxRAdHS3pC7d1TKcS2cVUOLz+WSD/IrCmPzD8OzZVBBCuYgBsR1GZBEpbAs6yhKZSb4CPSsEKNMa04f617Ono+9W+TJRpDZj/dEc2YksWU6t9HPmb20z05fyfKdHqIOB/A8wCDgB0dzOh3jQMqruXUEz7IImaiq8shBvAr0Xl+uCYaXCvh1t/6irsFnDjx4/HCy+8gCtXriA+Ph4AcOjQISxYsMA1ZqC7ExwNPP8nsGkUkHUA+H4oEv3fxGmYNE0xHxw/imqdBwfYFnCWKR8VWr6A0xloFFfqMOe3s+wYe1uWM00zzT4+IuFqG0eCDFJaltM0jTNMikhjEQ2uyhnbkboK5TdvAOV50PpGYmjBmygPFK5e4vraiiuqKh68FWzgins9rqo/5WK3gFu8eDHCw8Px8ccfs22TIiIiMHXqVLz11ltOn6BH4NMAGP0zsG0CcOZnvFa0CBXy4fjcMNCGgBMPMjBpJcx6qkJoLTqDVFaZkUxiqMFoxEd/nGfLxUzb7HtYmOGsiUrkW63jyJ+cX2wvbClk36tAcaUeSjmFlo38Bcco5RT6yo5hhfIzyMo1QFhHHOu9Cpc2XkcbkVZJ5jUZLDQ4iyyRGwXlyL5XAYWMQo9o56ygJYTdAk4mk+Htt9/G22+/zWb82xtcqJcovYCha4CAxkD6cryt3Iwo+V2EeCcKDleILPwMABGBJqF420bjP0sTlalH5ZZqXbpTbLGPfY8L4w5ges/V9qrkBMeCDNx2SZaeEOY7ZfxvrcP9eaYoFypjLb5SLoacolHZtA+8nvsehVfKAVxna66t9uGsbM/64LysgwzHqtb9aN84UPRYzqBGGZ0BAQFEuHGRyYDED/Bz+Bsw0hRGynZBtnkUoCm1GmrLB8dEXgvKtLxCei6WJmqlhRDSGWkrjctgp4+UORbR4OoOR4IMem6ir0gt6tV808JErcIEtDeaBnbNBra/CTlFY4u+D249uQ7wCkB5VQMIH5Vw5yCur42fB8cPMnB70LkSh0TnDz/8gC1btiArKwtaLd+MOnr0qFMm5skcDR+GlCwZlqlXQHXpT2DtAFMaiV8jdoxYNxHA5LPwVclRpjXgVlEFmof6WY3RWZqorIAzvTcYjVbmjb0+OEYeskEGu/YmOANHflRs+eAYmFZeUZZRfr0W+OVV4NQWAMAXsmGYrx+EFPDXGhETcMxtbaAt0kQ4Y2iaxilWwEnPnXUEuzW4zz77DOPHj0dYWBiOHTuG+Ph4NGzYEFevXsUTTzxR/QHuA/y8FPjT2ANfx34KeDcAbh8HvnoMyLvIjuH54CxuQoqi2OgrU7BvibUGx08L0Rloq6fDlg9OSFNgtjDHJhpc7ePISvS2Gl4yh7tRYLqvohpwBFxFIfD90ybhJlMAg1ZgnXIEAIpNSyrXMALOtolaWqlnf2wDOBocYLoPGRNZqETMmdgt4D7//HN8+eWXWLZsGVQqFd5++23s3LkTr7/+OoqKilwxR49jUFwk+rYOxUN9BwAv7gKCY4DCLODrfsB1U1cTW6VaABBZZabeKpQm4JhkX5rV4KwfDVsanKVGaDqWhYlKdDinciGnBLeLhL9fBkfWCeK1LBcxUa0WJC/KNqU5XdsLqPyA57YAXf7PqpqhOg2OOVthlfamUsjgpZTzBO31gnIUV+qhkstEAxzOwm4Bl5WVhd69ewMAvL29UVJiapY3evRobNy40bmz81DahAfgm/Hx6NgkEGjY3CTkGncHKguBbwcBZ362aJdkTXUCzjJgUGHpgzMYBXxw4gJKqNuv2QdH8uCcza3CCiQu/Qe95v9tc5xjGpy5VEsoyGAw0mx0PaqBD5BzypSonncO8AsHxv8BtHgMgHW5VnU+OEZTKyo3ua6YfobcaO6pqu69bSLEAxzOwu6jh4eHo6DAtBhF06ZN2bVQMzMzHXKI3hf4hgBjfwNaDwAMGmDreDQ5vwa2vFpMoOFmoXAk1bKFjWWQQW+grR4OW+2ShDQ4Nk2ENVHJ9+ssmBbd1eFYqRbXB2etwd0pqYTOQEMhoxCelw6seQIouQ2EtjX9GEd0YserLArumQXNvUVMVObXupD1v5nGcQUtc+2uNk8BBwTco48+il9//RWAKel38uTJ6NevH4YPHy64XiqhCpUPMPw7oMdLAGjEZHyIZMW3kEFY6JgFXLng55YaXKXOAJqmOUEGoSiqDQ1OQPhZRVFF9ybYi+QFju38o9M0bbFsIB8jTbP+t+f9DkC+8VlAWwJEPww8nwIE8VusMsm+GgsT1bc6E7XcXKYFWGhwNwsBuD7AADgQRf3yyy/ZkqxXX30VDRs2xIEDBzBw4ED873//c/oE6xUyOfDkIiCoKbDzfYxX/IkIqgAz8YbVULOJKqzBCaWJWC7ybCnPbPngBD9jNThiojobbg+/Mo1eNBfMntzDo1n38PK3/yK/1GQeCq1sbzQC2QVleF3+E5K0P5g2dnwWGLQCUFi3DGML7g2WQQbbJmohJ4Jq2m4eU12JmDOxS4PT6/WYN28ecnJy2G0jRozAZ599htdeew0qlXVjR4IFFAU8+LpplSFagf7yI/gSc4Cyu7xhkUGmRTpuF1UItq1mbjhvdhUvY7WtkWxpcELmK9HgXAe3xdW0n06JjhP6m4u1MZ+3/Swr3ICqXEsLFU5G69Dq8LtIUlYJt4cmA0O+FBRugIAPTlddFNX0r5UPjjORCp0BSjmFVuHW6U/Oxi4Bp1AosHDhQuj14qs9EaRR2nIgRmuno4j2QRwumiKsBVfZz8MCvCCjTL6x/FKN1f6Micq0ja7UG3gam85AW/nMbGpwIj44o5FmhamlNkHTNK7klVp1fCUAheVamz5Lrgb324lbouOENLiu83ZiZdoVq+2W369CJuNpTr6owBeKReiQ+wsMNIXdLaYBCbPMFfICMAJuytYTWPznBZRrJAYZLDQ4y3SV1uH+rHboSuz2wT322GPYs2ePK+ZyX6GQyXCYbountbNxC6FAwRXgq35AdgYAQCmXITzApMVlC0RSGY2LEXB3ijVYsz+T/VzIp2arkkEsisoLZlg8a3+eycVjH+/BhO8zRI97P7L7/B3EzdmJ97adFh3D1eBsISQjC8t1+CjFeo0RbqNVgN/wMhT3sFk1F31kJ1FJqfGyLglF7cdUe35ulHP57sus6ekjWqpVNcdqBFxt+N8AB3xwTzzxBKZNm4ZTp06hW7du8PXldwIdOHCg0yZXn2FKta7QjTGG+gC7wlcAt0+Yqh6eWQO0eRKRQd64VVSJW4UV6No0mLe/WcCZbqAfj2bzPjcYaaubylYtqnAeHL9zsGVUdtUekxbx9/naWXPWU1j05wUAwPpDWfhgSEfBMZUSgwz2RK7vlPA1fSbI0ILKxlrVQjSh8pFPB+Ad1XtIrWiCV0QaQXBRW7R3vlZV4iWmwTGYgwxMFJV/M9ZGBBVwQMBNnDgRgPD6pxRFwWCQGB26z1FyEn3vUsHAuB3A1nHA5Z3A5lHAEwsRGdQduH5PMBdOZ2GiWn1uNEIh59+EthN9hTU47oNo+azdKxfvdnI/I0UkcU1UR49F0zSnuJ22uk8UMgqKrP34STULAVQ5rhgjME73Nm5oTM1feVUMIljmqTH3UHUmKoM5yGAh4GohwAA4YKIajUbRFxFu0rEqtlf7ASM3AV3HALQR2DEFzxV/DQpGwUgqI5CYMLwlBoN1moj9eXA0X4OzFHA22jndz0jRuqSaqLaiqExbfMDUmMFycZjwrN+h2vQMAqhy/GtshaHaWbhBm4SbSiFDqF/1Cy2pRRJxqwsyMJiDDGYUMgqtw11bwcBA1oerI7i1qGyOkFwB/PczoO97AIAHbn+HT5UrkFNgnRTK/JIGiGpwtNXDIdaZBBDLg+Pna1kej/uA1UcqdQZM+C4Dmw5nOf3YGgsNblnqJba9EBdbspKrBTJr8Fbthf/Jf0PbA2+CMmixwxCPUdoZKIRZqDQJ8mbbiNtCrNJALA9OTIPjbm4V5g8vpesDDIAdJmpFRQVSU1Px1FNPATCtJcpdDV4ul2Pu3Lnw8vJy/izrIaLtkigK6DMVCGwC4y+TMFCejpjsqUDFdsDb7IdjIpeMD84SoVrUMhsCzvLXH6jywXFXVBLdu36y+cgNpJzJQcqZHIyIbyppn8z8MpzPKWHfG4y04HddYSHgPt55EbeKKjD/6U687bYqGbjRa6Z7bouGXhhb9DlGK3aZPnhgIpIP97ESqGKt9C0RE3CW6/+y2y0El1Cib20FGAA7NLh169bhiy++YN8vX74cBw4cwLFjx3Ds2DF8//33WLlypUsmWR+x1S4JABA3EtkDvkUJ7Y2O+tPA14mmgv0q2CCDSDRLZzBaBRXKNOIaV6nAZzRN8zS4+y3Rt9hGy3gxxq45zHvfadaf+HDHOatxQj8oJ25Ya+q2zF3ud1NSqYMXNPhQtxCjFbtgBIWKR+cC/eejcUNrc1CK/w0AVHJhQSZmoj7/UAzPPxfIcaEwt3yHJm4o4NavX4+XX36Zt23Dhg3YvXs3du/ejUWLFmHLli1On2B9xVZHX4agDo/jWW0ybtMNgPwLpoLo2ycAmE1UsSCD3kBb5acxhdJClAqYm0aatvAViT9s7207JWjmejJivdRskVXAL60r0xrw5T9XrcYJBRmEvktbPypcIakpysUm1TzEaw8CcjWoZ9fC+5HXAQBNBYSZWCt9S4Q0OLVCJmqBdGsWjPUv9oS/WoFgHyVvtXrGfO0gsoqXK5As4C5fvoyOHc0hby8vL96SfvHx8eyapYTqsbUmA0OAlxI31bEYopkNTYM2QGku8M2TwKVd0EkwUS0L8ss04iZqiYAGZ+mD4z5slprF9wez8Mmui6hPiK1n4AyEBZz1d2mrmwgr4PIvo8/e5xAnu4IyWQAw9ldQ7Qez44QEnFWjSxGEBFx1Lca7NA3GgemP4u+3/sPztSV2CEf3ZsFoX0sRVMAOAVdYWMjzueXl5SE6Opp9bzQaeZ8TbMNb+NnGc9Q4yBs5aIjDfTcAMX0AbSmwYRgGGk0+FrEoqs5otNLgKnQG0XItMQ2OG0XlBhmE/Hlr9l0TvxAPxIXyTTCKKhQwqtYHl3UI+LofAiqykWUMxRctVwFNH+CNi4sKstpXsokqIOAs/WxC+HspEezLL91c8VxX/PBKb5e3SOIi+UxNmjTB6dPimdknT55EkyZNnDKp+4FqfXBVMEX3N8qVpoWlO40AaAM+kH+JyYof4K8WvtloWrhjhZiZWqoRjuCJBRmE0hcqqjqaAMAvx29i76U80etyd24WVmBhygXRz8/dLsbyvy9JzmezRCjRV9hEFZdwfpkpwLcDgYoC3PJpi6e1c6ANam417rG2jZDUrxVvm1QTVShNxFfknnNHJAu4J598EjNnzkRlpXVOVkVFBWbPno0BAwY4dXL1GcF20gKEB5qi0jnFlYBCBQxZBfrhKQCANxQ/ofWhd6CEsNASym0rF4mkCgUZjDaCDGLP3ZW8Uly/W4Y3Nh3H6K8PCw/yAF5db3ttkSc+3YvFf13E52lXcC2/DF/vy5Qs7PZdysfVvDKr7ULauNjfeZw8Be33vgroK4FW/bG82VLkI5CtHOBCURRef4y/jmlDX2mNMYQEnGgvODdE8kxnzJiBLVu2oHXr1pg0aRJatTL9Ily4cAHLly+HXq/HjBkzXDbR+ohSJqvyk4lLOiYZky24pyho+8xA8t+FmKdYg+CLP2CN8jxe0b2JUlRvdohFUoVy2miab0pJCaLuv3xXeKUmD4NZ9Lo6TmYX4rPUSwAg2BRBiP/7+pDgdqHUC8u/OQUj3lWsx4uKP0wbur8APLEQBRtMwScxnywXP7VCsn9RJRfQ4Kop03InJGtwYWFhOHDgANq2bYtp06ZhyJAhGDJkCKZPn4527dph3759CAsLc+Vc6x22lg5kCKmKQuVx6gx1BhqbDI/iRd1boJU+eFh+GltVcxCGgmqPJ6rBifngeBpc9SJu/+V8wcJ9T8O6VaQwXB/Zv9eq//vb4lp+GUZ9dRAHruRzjm8+gRpaLFMuY4Xb6bZvAgM+xp1yPTKy7gEQT/zm0s6OKKaQvyzYx3Paotnl7YuJiUFKSgry8vJw8OBBHDx4EHl5eUhJSUFsbKyr5lhvYfxwtn5MQ/1MNxNXO2AiqGnGLjCM/R15dCDayrLws3omWlO2s+5zRBaTFs6Dg3iploisO3ytwO7lCd0RqQEGrtCXS3Q7iLHl32zsv3wXz602a3jM4YNQgu9VH+Ip+SFoaTle176KUzEvgAYw4suD7A+gWNCJy8DOkZLnZCngJie0wpsJLUVGux8OhTMaNGiA+Ph4xMfHo0GDBs6e031DdStrAUCIpYkKU4QUMD2EiiZdMcI4F5eNkYikCrBVNRu9ZGdEj7fj9G3B7eIanHg3EbHj2OpaUt/gRqUVMplEvU86RppGE+oOflTNQg/ZRRTTPhijm45fjQ9CqzfiZHYRz59nS4Ob0Kc5nuwYjmHdo0THWMI1UdUKGd5IaImWHuSCILWodYiUXDhWwJWYC9uZ4IGy6uYrUEZgqHYWDhnbIICqwDrlAgyW7RM83p+ncwQjqfYEGWjaekEbBr2RrhcJv1IFldFCg7OsxawJF3JKEK25gJ9VM9Fcdhs36YYYqp2Fg8Z2AExR8pQzObx9Amz44KY90Qafj+pmV5qGupZqRl0FEXB1CGui2hjDZIJX6AxsgIARIMyvq7dSjiL4YYx2GrYbHoCKMmCp6nNMlG8DY0vGhvgiqoE3yrQG7Dyba3WeEqFCb1gEGWhTCVj/pXsx0UaUsdJOv507ItUJz/Q9A0zfp7ME3J6Lefjos6X40pCMUKoYZ4zNMEQzB5docyqWRmdEymkLASfBRLUHoSCDJ+HZs/dwpAQZfNUKNrGSMVOZOlSm3IvJFtdAhdd0k/CF3pSu87ZyCz5UfA05DKAoYEhcYwDAz8du8s5B07S4D44rrEAj4/o9XMgtwYErd63GM3CF4uTNx6u9RndE7Ju5XVTBSwfhFtZL8cFJFfi3dq3AauXH8KE0+MfQEcO0M3EH/Kanp24WITOfn24iVrrnKLWZlOsKPHv2Hg4joKp7KEL8TYEGxpGs1fNNVG45DA0Z5utHYaZuLIw0hecUf+NL5RJ4oxKDu5gE3N5L+byobLnWIJoxz/XBGWlpQpkrALYdv4WreaXV7uN2CFzmpdwS9Jr/N/p9ItyyXyGvXsBZls9ZQ2OqYhNG3vkEcorGFn0fPK+bijJYJ+b+JaCJS6kysAduHpwrKztchVsIuBUrViA6OhpeXl7o2bMnDh+2nSC6detWtGnTBl5eXujYsSN27NjB+3zcuHGgKIr36t+/vysvwSGkCAvAOtDApGEoZYwGZ/01fmtIxATdm6iklXhMfgxLyt5FrFcZOkcFwWCksf2keaETIe2NgRtFBW0WqrawLEO6KdCR2N0R+mYYc5BZV9QSuUxWrYkqlHzNoIQenyg/x6sK07rDn+iG4m39y9Db0Xjb2fWzRIOrIZs3b0ZSUhKSk5Nx9OhRdO7cGYmJibhzR7jP/4EDBzBy5Ei88MILOHbsGAYPHozBgwdblZH1798ft2/fZl8bN26sjcuxC6WEIANgFnB5VUvCMSaqsurmE+vN9ZexB0Zq38Nd2h+tjJeBrxMwrqVJSP7KWcmpxEbjSksTVSkh8muZ0e+hbji7UQgstGyJ2ApkASjDOuUCDJHvh46W48vgt/CpYSiqC3e4WgBxfXC2lp10V+pcwC1ZsgQvvfQSxo8fj3bt2mHVqlXw8fHBmjVrBMd/+umn6N+/P6ZOnYq2bdti7ty56Nq1K5YvX84bp1arER4ezr6Cg4MFj1eXyNkgg+2b2BxJFTFRbSy/doxuaVq5SxYBFGZhYMY4dKfO4/Ids9nIaHBCmqBlkEGaBmch4Krdw/1wRBOSEmQQahsfgbvYqpqN3vKzKKW98LxuKn5XPGbzOMtGdsGiZzph/Ys97Z6nPag594Qt7dNdqVMBp9VqkZGRgYSEBHabTCZDQkIC0tPTBfdJT0/njQeAxMREq/FpaWlo1KgRWrdujVdeeQV374o7xTUaDYqLi3mv2kAhQRsCzJFUSxOVicJ6VVM6c50Ox2S/RUDj7pBrirBeNR8Pa/exgo3JgWvoa92j37JluZQoobUG53kPhiOWXk5xZbX5JZYaXFvquilBW5aNXDoIw7QzsdfYCaUCUW0u0Q198Wz3KPSINuehSvR42AWJotaA/Px8GAwGqxKvsLAw5OTkCO6Tk5NT7fj+/fvj22+/RWpqKj766CPs2bMHTzzxhOiiOPPnz0dgYCD7ioqSnghZE+QSKhkA62oGRgtgzBMpjuUiWSAw9jeg9QCoKR0+V32Gyj2fAjTNdhJp6GddgmPdTcR5C6q4M0KCvLor33spv9ofAG6Q4SHZKWxRzUE4dQ8XjE0wRDMHZ+loAEBRhe31LoROU12fNkdQEAHnfowYMQIDBw5Ex44dMXjwYGzfvh1HjhxBWlqa4Pjp06ejqKiIfd24caNW5im1ZRLrgythBBzN21/ItLSEoihA5QMM/w6/qEzraoQcmAOkTENJhem4QjWGPAFH2+5PxmC53oDn6W/SE30tqe4rZX6cnpHvwTfKhfCnKpBuaIdntcm4hRB2XHXt0oUEqZ8LBJynU6cCLiQkBHK5HLm5/HB3bm4uwsPDBfcJDw+3azwAxMbGIiQkBJcvXxb8XK1WIyAggPeqDaRUMgDmgvt8yyCDXLoGxz4OMjl+Dn8d83SjTO8PrUKPw5Ohhhb+Xgqr9jj8IIO0gIFV2yAPlHBCGpKtZRcZxDS4NfsyMWxVOgpKNXhd/hMWK7+AkjJgm6E3xureQTH4C6hXl04idOsQAWdNnQo4lUqFbt26ITU1ld1mNBqRmpqKXr16Ce7Tq1cv3ngA2Llzp+h4AMjOzsbdu3cRERHhnIk7CTYPrppxlmkiliaqlCXYuM9dRJA3vjIMwI7WHwByFaLz/sZG1TxEKMusonI8c9NGiRZvHws/k5R93J33tp3Csr+FfyC5iFmoc7afxdFrd2DYNglJyh8AACv0AzFZNxFa2F99ICRIXWGiejp1bqImJSVh9erVWLduHc6dO4dXXnkFZWVlGD9+PABgzJgxmD59Ojv+jTfeQEpKCj7++GOcP38es2bNwr///otJkyYBAEpLSzF16lQcPHgQ165dQ2pqKgYNGoQWLVogMTGxTq5RDOl5cCbTsVxrQLlWL2Ci2ingAk1Jo/+oHgFGb0OF3B9dZZfxypVXECvnp+dwl5sz0rY1OEY4Otrl1r3gfzffH5S2NqpY9NUXFfhauRgPl6bAQFP41OsVLNKPAO3gIyh0FqLBWVPnAm748OFYvHgxZs6cibi4OBw/fhwpKSlsICErKwu3b5s7YPTu3RsbNmzAl19+ic6dO+OHH37Atm3b0KFDBwCm9VlPnjyJgQMHolWrVnjhhRfQrVs37N27F2p19St51ybmdkm2BZ2fWsH62fJLtFYmqiQBx3kkmC7Bt4oqgegH8Wmz5cimQ9Cg8gbWGmYgjjJrKpbdRGwJOLVcWMB5YBDV4ax9od1CcQ+bVXPRR34SlZQaL+newp/eNet+LXTPuErAeXIk1S1E/qRJk1gNzBKhwMCzzz6LZ599VnC8t7c3/vzzT2dOz2VI1eAoikKInxrZ9yqQV6ph+8HZ5YPjaXBVbdCLTBn5l4yNMUQzB382WoYGxeewUTUPr+smYaexu1WQwZa5qVbKUKKpJwLOwf0sTccWVDbWqhaiCZWPPDoAyxp9gL+zGiKuhgm63FsnNsQXV/PLMKRr4xodUwyVQiahxMw98VzRXA+wJwTPjaQyDSWZqgJvlYQoKuf/jIl6u6r5ZVGFDnkIwpG+3+Owohu8KS1WKT/BaPlfvGPQ1ZmorAZn6YPzPBzV4LiC5wHZWfykmoUmVD6uGCPwtHY2MtWtAdRcK+IK0m2THsRPE3vj8Xau6ajtyeVanjvzeoDUNBGAH2jQWpqoNioZWDgPBKPBlVTqUarRo6gqJcHPPwgfBiZjo74v5BSNucq1mKbYCAqm81UXRWV6h9UPH5xjMKbjf2UHsE65AAFUOf41tsJQ7SzcoMPYz2sqNLhuiQAvJbo2DXbZOq6ebKJ67szrAVJNVAAI9Tcn+zIdcxkNsLpKBoCvwfmqFWzn15yiClbABXoroVAoMV3/IhbphgEAJih+w6fKFVBBZ7PRJcDV4PgCbvORLMz57axHVTRIXZPBej8a/5P/hmWq5VBTeuwwxGOUdgYK4V/1uQkpNb22CPJxbt83WxANjuAQUtZkYOCursWmiTD94CRocJbnYMzUW4WVPAFnqj2ksMIwGJO1r0BHyzFQno7vVPPhayzl+eQsYeoWLdNEdp27gzX7M232kHM3HFGGZDDiLf2XmK40NXb4Sv8EXtW9Dg3MCdRMB2ApNb22kBJYchY1FcZ1CRFwdYhcYqIvwEn2LdFamahi3US4WN6iEUEmM/X63TJWaAV4K3nmyM/Gh01JqLQ3esrOY0HhFLy56lfRczBJwmIdM4qqyc53J+x9pL2gwRfKTzBEnwIjTWGObjTm6UdbpYEw7ac8SStSS3GBuCme81euh9jzy8j1wVmZqFJLtTgwfjimIy1FAf5qhdXNfMDYAc9qk3GbboCWspv4WZ2M9lSm4Dmqe2jd1UIVMp3t8Wc1RBE2qeahnzwDGqgwUfcG1hieEBzLVIbUxK9lj2vDGXiSMLbEc2deD5BabA9we8JZm6h2lWpVwZioF6oEXICXEjIZJXgzX6CbYohmNs4Zo9CIKsQW1Rz0kZ2wGled2eWOFQ2r/7mKHh+kOtx1OIa6jZ9UyYiTXcE92g9TvOcgxRgvOr7SCRpcoJPXXagOIuAIDmFfFLUqyFCiMVcy2JPoa3EqJtmXEXDMQyN2M+egIYZpk7HP0B6+lAZfKxdhmHw3b0x1tbXuqMF9sOMc8ks1mPXbWd52KT86XamL+FGVjGayO7hubISntbNxVtHW5j6MBlcTH1xQLQs4y/pkT8JzZ14PYHxwUiJ2jA+uTGtAcVWvMLsqGURM1JKqnnCMgLN1M5fAB+N17+BHw8NQUEYsVK7GZMVWMJlu1QlsN5RvLAYj329YnYBLlB3GBtUHaECV4rgx1pTjRkdU2y5JY5Gk7QjOXjmrOkiaCMEh7PHBmfxjpq+LWZ1e6QQTlaE6DY5BBwXe0k3AZ/rBAIA3FD+bOmNAX20DT3dLE+HOx3Kxals/OuPlf2Cl8lN4UTrsNHQ1tYVHoGm/ar5SJoWmJmZfbaaIAMREJTiIPc5iplwLAG5XLeLCaAFSBKV1mogX732AtykvTihiZq3VUViiH4Zpuhehp2V4Rv4P1igXwpcul3IpLN/sz8Torw+hokor3X85v9b6/u+5mIfu83ax7y3PKySoKBjxnuI7JCu/g4yi8Z0+ARN0k1EB89+yOg2O9cHVIPWC+OCk4xa1qPcr9uTBASYz9WZhhalIHmYBJyXiZ6mR+KoVkMso9sG2pcF5KeWC+W+bDI8ih26AFcpP8bD8NJpnvY40TEYuGliNFWJ2ld9r/aHr+CEjG+dzSjB7YHuM7R0taf+aMHYNf+U2vaWAsxivhhZLlJ9jgNy03wLdCKwy/FdgpG2YRbHtFRqN/NW4U9XwtLYFHPHBERzCnjw4wNy6nEHqmg6AsBD14ty4ATZ8cLZM4DRjHIZr30ceHYhIzRXT+gKUcGshmgZ+PpaN97ed5mlMxZV6Nl2Fu9pXbWKtwZn/YEEowfeqDzFAfhhaWo7Xta9ilWEghIRbdRocYxXb64P7e8p/2P8TDU46njvzeoA9UVTAnCrCYI/zV1DAcQSXrSBDdXl2p+lYDNHOwR11M0RSBdiqmo1esjNW42jQmLz5BL47eJ23LquRI1xqw6FdUKa12iZmGjeh7uBH1Sz0kF1EMe2DMbrp+NX4oOixpWrjUoQG9+/OvVVahflLO4mTUMlJoi/BAaR29GVgVtdisEcLEHKaCwk4MRO1OrLpUHzffjUOGdsggKrAOuUCDJbt443hOvK5i0EbOc5+tYSk5Zpwt1SDrnN3Wm230uAAdKSu4mfVTDSX3cZNuiGGamfhoLGdU+Yh5bvjrpFBgcIXo7vhlf80x4COtduZWqkgpVoEB6ipBldTE5UrTGxrcNJ+wdvFNsUY7TRsNzwAFWXAUtXnmCjfBiZBZOoPJ9mx5RpzQb6Brj0N7si1e4LbL+SWYGHKeTbK+YDhCDar5iKUKsYZYzMM0czBJbqJ0+Yh5Tq5pihFAYntw/FO/zaQ1XIlg9qD00RIkKEOYfPgJNo1NTFRheBGTJmHSUiYSUlDAYAHYhtCAxVe003CTboh/qf4HW8rt6AJlY/39eNhgPk45VpOK3SO9mS5IpezsSUbPk+7Ah+VHJMC9mJuxYeQU0bsMXTCRN0bKIO3+I4cpGbCSDFReRpcHSpRxAdHcAj7NbiaBBmETFRrDc5HZf2bJzWdJajqgaQhw3z9KMzUjYWRpvCc4m+sVn4MH1SyY8u15nU/uc1i917Kx4yfT0k6nyPY+jGhYETrM58A2ydDDiM26/+DF3RTJAs3QHoysxQTlZvvJmXBbVdBiu0JDmFv0XRIjXxw1ngJaHC+avO2h1uGIMhHiT6tQu2aJ8O3hkRM0L2JSlqJR+XHsUk1F6EoBGBqtslgtFB7NhyStsCLI4j9xZXQY4lyJfrdXQ8A+EQ3FO/oX4LeTiNHajKzFK0oiOeDqzuIBkdwCHuDDJYmql0CTuAkXA2QFXAcDe71x1ri2Pv90LtFQ8nniQnhr+/5l7GHKdOf9kcnWSZ+UiWjOXWT1zqpNiscUs/nWm0LQBnWKRdgiHw/dLQcU3Uv41PDUDgiVqReipTkbHfR4IiAIziE1IWfGQK8FLybjfuQTE1sbXNfocdDw1k7wd+L0eDMAo6CyaTTGaQLoGUju1htO0a3NNVqGsMQJcvDj6pZqLxijrDW1oImBWVabDx8g7ctAnexVTUbveVnUUp74XndVGw1/Mfhc+iN0q5FitDg+j7r1AfnwUEGz515PYA1USXevBRFsZ19Ab4G92rfFjg563F8M74HFj/bWXBfS7ir1jNzEVp6rrDcOm9MDLEk1Ot0OIZqZ+OosQWCqDKsV83Hk7KDAGAldACgpNL5zTH/992/vPftqGumxGRZNnLpIAzTzsReY6caneNKXpmkcQFe1SfrcjVsV623IAWiwREcwt4gAwD4cLr3NvTlBx0CvJTo27oR206He3hBDU6g/MqH44NjOvM+ENsQLRv5YVh38TSJsb2aAbA2o7kUIADPad/FX4ZuUFM6fK76DC/If4eQa77jrL+Q/Mtph83Xy3dKUaYx+/nKNHpeishDslPYrJqLcOoeLhibYIhmDs7S0Q6dyxGkVCM4cn+4AiLgCA4hd6DgmukkAgCNArwExzC//NwbU0gBEFr9iuuDK6tK5fBSyvHX5Eew8BlrzZBh1sD2AEzt03+e2Ft0XCXUmKCbjLX6xwEA7yvXI1nxLWSwFrbr0q/joY92I6+qBlMqh67eRcKSPZjwfQYA4EZBOdonm9fKfUa+B98oF8KfqkC6oR2e1SbjFkLsOkdNkaTB2enCcBW1uf6Ds3GPv+B9ipLtByedEo5WInrcKtOV7zsRMlGthQo3sstN5ajOROJ+3qVpsM2xRsgwSz8W83SjAADjFX/ic+WnUMPaFL5ZWIG+i9NsHo+LwUhj+Jcm03fvpXwAwNZ/GROYxuvyn0ztnSgDfjY8aFpzAr4iR3Mdfl7VR2fdZbGXXrEN0TYiAAM7R9b1VOyGCLg6hFksxp48o4+f7QyKAj4ZLq5NMUJKpbDtpLa1QhYAlGlcmXRL4SvDAEzSvgYNrUB/+RFsVM1DMIqtRpZKEOoM2fesWzbRABTQ4yPFaiQpfwAArNAPRJLuFWhRu4XrDNWlCE15vJVdC4O7EpVChh2vP4TPBAJI7o57/AXvU+KigvD6oy0w40nbba65DO3WBGdmJ2JIF3F/GPPLz+05JvQ4PVvlU+sVK5wG0rVZkOR5Ocp2Yy/8n3YGCmlfdJVdxo+qWWhKWadyHM4sqNYfN+vXM3hq2T6r7Qp9Gb5WLsZwRRoMNIV3dc9jkX6E1YpXruaBWH4bKf8qLe6xNo0Q4KVAl6ZBAIC2EQGY9GhLt/HBAXUb5KgJpFSrDpHLKCQ9bju9QwihagMuLUL9EeqvxoMtQrA1IxuAsAaX1K8VujUNxgPN+QLu3/cSkFNUiTbhAXbPzRGO0G0wVDsL61QfIVaWg59UyXhROwXH6RbsmGFfpOOpThFY/lxX0eOsPXDNemPxbYw4/T+EyS+inFZjku41/G0UP4Yrmf90J7y37RQebmlKnP5r8iPYeykfg+IioZTJQFHAmVvFaB7qB6Dma6cSiAZXLwn0UeLg9MewiJMuItRNRK2Q4/H24VYO7xA/NTo0DnT5PLlcoRtjiGYOThmjEUIVY6NqHvrJ+Gkd20/eFt3/l+M3rba1pLKBr/shrPwi8ugAjNC+V2fCDTB1UV7/4gOY0Kd51XtvDOseBbVCDpmMAkVR6NA4kHVd2FOKRxCGCLh6iqWPxxMsjDwEYbh2Jv42xMGb0mKV8hOMlv8lad83Nh3nvX9AdhY/qmYBRTdwHZF4WjsbJ+nmTp1vh8YmDddfrcCupEcQLhLVZrC3M667RFE9GfIXvE9wloCb8WQb5xxIhHJ44SXdW9ig7ws5RWOuci2mKTaCEkgjEWOg7ADWKRcggCrHJXUHDKpMxg06rEbz6t28If7TOhQT+jRnKwzeG9AOb/dvjZ9ffRAtGvljcJfGovt/Naa73X4sd4miejLEB3ef4Cx/zvMPxqBFIz90aByI+A9SRce1bOSHS3eEF1OOj2mAw5kFovsaIMcM/Yu4SYdiqnILJih+QySVjym6CTiVXYRQfzW7risfGhPkv2GachMAYIchHpOLJkIDlcBY+3iuZ1M81cmUJvHiwzG4frcc3ZoF4wFOgGbSoy2wZl8m1AoZYkJ9cTK7CC8+FIP3nnKsSWZ8TAPIZRRa13IH3/oERbvbWm5uQHFxMQIDA1FUVISAgNpxtLuKr/Zexbr0a9j0ci80DpLe9kcKv524hdc2HgMAXFswgPdZYbkWH6VcwMbD1p1Bnu7aGD8dtfaZCTFEthcLlV9CSRlwyNgGL2mTUAw/q/N1mLkD7xi/xmiFaaWsr/RP4AP9KKdESgd0jMDy57rYpYEVVehw8Opd/Kd1aI3aDVXqDFDKZXZ3nqnvSH1G3cJEXbFiBaKjo+Hl5YWePXvi8OHDNsdv3boVbdq0gZeXFzp27IgdO3bwPqdpGjNnzkRERAS8vb2RkJCAS5cuufIS3JYXH47F3rcfdbpwA4BuzUwJvULPfZCPCvOf7ojzc/tjwdMd8dPE3ujYOBALn+mEsb2iBY/3wZAOVtt+Nj5sSsalvdFTdh4/qmajMfL4g7Rl+IRehNGKXTDSFGbrRmOefnSNhVtDXxVG9WyKT0fE2W1eBnorkdg+vMa91LyUciLcakCda3CbN2/GmDFjsGrVKvTs2RNLly7F1q1bceHCBTRq1Mhq/IEDB/DII49g/vz5eOqpp7BhwwZ89NFHOHr0KDp0MD0gH330EebPn49169YhJiYG77//Pk6dOoWzZ8/Cy8u2IxioXxqcq7lZWIEALwXbjUQqd4or0cBXha/3ZWL+H+cBAOfn9keb91MEx7emsrBWtRARVAHu0EG40m8Nej30GIwld1D6zVAEFJxEJa3Em7pXkWKMd/h6Vv1fV7SPDMSNe+Xo3qyBR9dh1mekPqN1LuB69uyJHj16YPny5QAAo9GIqKgovPbaa5g2bZrV+OHDh6OsrAzbt29ntz3wwAOIi4vDqlWrQNM0IiMj8dZbb2HKlCkAgKKiIoSFhWHt2rUYMWJEtXMiAq72MBpprD1wDb1bNESrRv6InbFDdGw47uIb1UK0ld1AGa3Gz41excO536OZ7A4KaD+8qJ2Co3Qr3j7xMQ3QrIEP5DIKswe1R6XWiJV7rmBo18Z4c/NxnLlVjF1Jj+BkdhEGxTUm2pKH4BECTqvVwsfHBz/88AMGDx7Mbh87diwKCwvxyy+/WO3TtGlTJCUl4c0332S3JScnY9u2bThx4gSuXr2K5s2b49ixY4iLi2PH9OnTB3Fxcfj000+tjqnRaKDRmAu6i4uLERUVRQRcHXA1rxQGI40reWWY8H0GYkN90TOmAdtSyR/lWKn8BA/JzcsSXjc2wjjdO8ikTatNxYb4YtXobjhxoxB9Woeikb+w1k7TNPRGmiTUeiBSBVydRlHz8/NhMBgQFsYP4YeFheH8+fOC++Tk5AiOz8nJYT9ntomNsWT+/PmYPXu2Q9dAcC6xVVn8LcP8eYGEOYM6oN+SPbh2F5iIGUg2rMJQ+V4cN8ZisnwGjP4heLNLY4yMb4qwqny06tYPpSiKpGLUc0iaCIDp06cjKSmJfc9ocAT3QSmXIW1qX/Y9bXwSd6+dRJvINvhbrfbYWkmCa6lTARcSEgK5XI7cXH5xdW5uLsLDwwX3CQ8Ptzme+Tc3NxcRERG8MVyTlYtarYZaLd6okeB+UDIZGsbG1fU0CG5OnTofVCoVunXrhtRUc8Ko0WhEamoqevXqJbhPr169eOMBYOfOnez4mJgYhIeH88YUFxfj0KFDosckEAj1FLqO2bRpE61Wq+m1a9fSZ8+epV9++WU6KCiIzsnJoWmapkePHk1PmzaNHb9//35aoVDQixcvps+dO0cnJyfTSqWSPnXqFDtmwYIFdFBQEP3LL7/QJ0+epAcNGkTHxMTQFRUVkuZUVFREA6CLioqce7EEAsEpSH1G69wHN3z4cOTl5WHmzJnIyclBXFwcUlJS2CBBVlYWZJyi4969e2PDhg147733MGPGDLRs2RLbtm1jc+AA4O2330ZZWRlefvllFBYW4qGHHkJKSoqkHDgCgVB/qPM8OHeE5MERCO6NR5VqEQgEgisgAo5AINRbiIAjEAj1ljoPMrgjjFuyuNh6hScCgVD3MM9mdSEEIuAEKCkpAQBSzUAguDklJSUIDBRfP4REUQUwGo24desW/P3960UJEFN6duPGjXoVFSbX5Vk487pomkZJSQkiIyN5aWSWEA1OAJlMhiZNxNcd9VQCAgLq1QPDQK7Ls3DWddnS3BhIkIFAINRbiIAjEAj1FiLg7gPUajWSk5PrXccUcl2eRV1cFwkyEAiEegvR4AgEQr2FCDgCgVBvIQKOQCDUW4iAIxAI9RYi4OoBBQUFGDVqFAICAhAUFIQXXngBpaWlNveprKzEq6++ioYNG8LPzw9Dhw61WuuCoiir16ZNm1x2HStWrEB0dDS8vLzQs2dPHD582Ob4rVu3ok2bNvDy8kLHjh2xYwd/TVWapjFz5kxERETA29sbCQkJuHTpksvmL4azr2vcuHFW30v//v1deQmC2HNdZ86cwdChQxEdHQ2KorB06dIaH1MSLu0rTKgV+vfvT3fu3Jk+ePAgvXfvXrpFixb0yJEjbe4zYcIEOioqik5NTaX//fdf+oEHHqB79+7NGwOA/uabb+jbt2+zL6lt3+1l06ZNtEqlotesWUOfOXOGfumll+igoCA6NzdXcPz+/ftpuVxOL1y4kD579iz93nvvCbauDwwMpLdt20afOHGCHjhwoF2t652BK65r7NixdP/+/XnfS0FBQW1dEk3T9l/X4cOH6SlTptAbN26kw8PD6U8++aTGx5QCEXAeztmzZ2kA9JEjR9htf/zxB01RFH3z5k3BfQoLC2mlUklv3bqV3Xbu3DkaAJ2ens5uA0D//PPPLps7l/j4ePrVV19l3xsMBjoyMpKeP3++4Phhw4bRAwYM4G3r2bMn/b///Y+maZo2Go10eHg4vWjRIvbzwsJCWq1W0xs3bnTBFQjj7OuiaZOAGzRokEvmKxV7r4tLs2bNBAVcTY4pBjFRPZz09HQEBQWhe/fu7LaEhATIZDIcOnRIcJ+MjAzodDokJCSw29q0aYOmTZsiPT2dN/bVV19FSEgI4uPjsWbNmmrb0ziCVqtFRkYGbz4ymQwJCQlW82FIT0/njQeAxMREdnxmZiZycnJ4YwIDA9GzZ0/RYzobV1wXQ1paGho1aoTWrVvjlVdewd27d51/ASI4cl11cUyAFNt7PDk5OWjUqBFvm0KhQIMGDZCTkyO6j0qlQlBQEG97WFgYb585c+bg0UcfhY+PD/766y9MnDgRpaWleP311516Dfn5+TAYDOxCQ9z5nD9/XvQahMYz82f+tTXG1bjiugCgf//+ePrppxETE4MrV65gxowZeOKJJ5Ceng65XO78C7HAkeuqi2MCRMC5LdOmTcNHH31kc8y5c+dcOof333+f/X+XLl1QVlaGRYsWOV3AEexjxIgR7P87duyITp06oXnz5khLS8Njjz1WhzNzP4iJ6qa89dZbOHfunM1XbGwswsPDcefOHd6+er0eBQUFCA8PFzx2eHg4tFotCgsLedtzc3NF9wGAnj17Ijs7GxqNpsbXxyUkJARyudwqimtrPuHh4TbHM//ac0xn44rrEiI2NhYhISG4fPlyzSctAUeuqy6OCRAB57aEhoaiTZs2Nl8qlQq9evVCYWEhMjIy2H3//vtvGI1G9OzZU/DY3bp1g1KpRGpqKrvtwoULyMrKQq9evUTndPz4cQQHBzu9WFqlUqFbt268+RiNRqSmporOp1evXrzxALBz5052fExMDMLDw3ljiouLcejQIZvX6ExccV1CZGdn4+7du4iIiHDOxKvBkeuqi2MCIGki9YH+/fvTXbp0oQ8dOkTv27ePbtmyJS9NJDs7m27dujV96NAhdtuECRPopk2b0n///Tf977//0r169aJ79erFfv7rr7/Sq1evpk+dOkVfunSJ/vzzz2kfHx965syZLrmGTZs20Wq1ml67di199uxZ+uWXX6aDgoLonJwcmqZpevTo0fS0adPY8fv376cVCgW9ePFi+ty5c3RycrJgmkhQUBD9yy+/0CdPnqQHDRpUJ2kizryukpISesqUKXR6ejqdmZlJ79q1i+7atSvdsmVLurKy0m2vS6PR0MeOHaOPHTtGR0RE0FOmTKGPHTtGX7p0SfIxHYEIuHrA3bt36ZEjR9J+fn50QEAAPX78eLqkpIT9PDMzkwZA7969m91WUVFBT5w4kQ4ODqZ9fHzoIUOG0Ldv32Y//+OPP+i4uDjaz8+P9vX1pTt37kyvWrWKNhgMLruOZcuW0U2bNqVVKhUdHx9PHzx4kP2sT58+9NixY3njt2zZQrdq1YpWqVR0+/bt6d9//533udFopN9//306LCyMVqvV9GOPPUZfuHDBZfMXw5nXVV5eTj/++ON0aGgorVQq6WbNmtEvvfRSjYSAo9hzXcw9aPnq06eP5GM6AmmXRCAQ6i3EB0cgEOotRMARCIR6CxFwBAKh3kIEHIFAqLcQAUcgEOotRMARCIR6CxFwBAKh3kIEHIFAqLcQAUfwGIRadddVu26CZ0DaJRE8iv79++Obb77hbRMr/tfpdFAqlbxtWq0WKpXK7vM6uh+hbiEaHMGjUKvVCA8P572Cg4MBmBbJWblyJQYOHAhfX1988MEHmDVrFuLi4vDVV18hJiYGXl5eAICsrCwMGjQIfn5+CAgIwLBhw3itesT2I3gWRMAR6hWzZs3CkCFDcOrUKTz//PMAgMuXL+PHH3/ETz/9hOPHj8NoNGLQoEEoKCjAnj17sHPnTly9ehXDhw/nHctyP4LnQUxUgkexfft2+Pn58bbNmDEDM2bMAAA899xzGD9+PO9zrVaLb7/9FqGhoQBM/dVOnTqFzMxMREVFAQC+/fZbtG/fHkeOHEGPHj0E9yN4HkTAETyKvn37YuXKlbxtDRo0YP/PXXyHoVmzZjwhde7cOURFRbHCDQDatWuHoKAgnDt3jhVwlvsRPA8i4Ageha+vL1q0aGHzcynbpJ6L4NkQHxzhvqNt27a4ceMGbty4wW47e/YsCgsL0a5duzqcGcHZEA2O4FFoNBqrZf8UCgVCQkIkHyMhIQEdO3bEqFGjsHTpUuj1ekycOBF9+vQRNHEJngvR4AgeRUpKCiIiInivhx56yK5jUBSFX375BcHBwXjkkUeQkJCA2NhYbN682UWzJtQVpGU5gUCotxANjkAg1FuIgCMQCPUWIuAIBEK9hQg4AoFQbyECjkAg1FuIgCMQCPUWIuAIBEK9hQg4AoFQbyECjkAg1FuIgCMQCPUWIuAIBEK9hQg4AoFQb/l/AKqREwpTUdoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020682, + "end_time": "2024-03-01T18:44:44.013129", + "exception": false, + "start_time": "2024-03-01T18:44:43.992447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4257.086008, + "end_time": "2024-03-01T18:44:46.757977", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/lct_gan/42/mlu-eval.ipynb", + "output_path": "eval/contraceptive/lct_gan/42/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/contraceptive/lct_gan/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "lct_gan" + }, + "start_time": "2024-03-01T17:33:49.671969", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/lct_gan/model.pt b/contraceptive/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a146f21de3d89c856240dc8871687dfd28a5678 --- /dev/null +++ b/contraceptive/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12c803933371fdedc1397d36f37b854713f585ec63c4a4887467f850e2b255cf +size 41106197 diff --git a/contraceptive/lct_gan/params.json b/contraceptive/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..e7e3ed1fbd2a79cbb81b9d27c1806668999b506e --- /dev/null +++ b/contraceptive/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.775, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "lct_gan", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": false, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "relu6", "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/realtabformer/eval.csv b/contraceptive/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..cf8b3bfafad561ceb927ba121cc52d1dcdb97dd2 --- /dev/null +++ b/contraceptive/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.01544937376803275,,0.0014923845270062802,2.446577548980713,0.11075861752033234,1.6372919082641602,0.24003435671329498,1.4633540104114218e-06,4.698302984237671,0.03089020401239395,0.07129628211259842,0.03863139450550079,0.05503246188163757,0.02127235010266304,7.144880533218384 diff --git a/contraceptive/realtabformer/history.csv b/contraceptive/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..b71c5e34e22583e5ed9d49922efee3bc9cba24c4 --- /dev/null +++ b/contraceptive/realtabformer/history.csv @@ -0,0 +1,20 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.015214953777865503,3.156637710683768,0.0015362251463382382,0.0,0.0,0.0,0.0,0.0,0.015214953777865503,320,160,150.5653796195984,0.9410336226224899,0.47051681131124495,0.06246557859708446,0.0029828296956111444,3.72726428431983,7.1439977092975506e-06,0.0,0.0,0.0,0.0,0.0,0.0029828296956111444,80,40,35.29301643371582,0.8823254108428955,0.44116270542144775,0.017451438040006907 +1,0.005820814266519392,3.788629152714401,0.00012637018700479842,0.0,0.0,0.0,0.0,0.0,0.005820814266519392,320,160,151.99716687202454,0.9499822929501534,0.4749911464750767,0.054377025530902755,0.003795744390345135,4.469785842269539,2.3026626298489063e-05,0.0,0.0,0.0,0.0,0.0,0.003795744390345135,80,40,35.12023663520813,0.8780059158802033,0.43900295794010163,0.025538454634443042 +2,0.003438323737486826,2.1381948816152487,2.1238802406123302e-05,0.0,0.0,0.0,0.0,0.0,0.003438323737486826,320,160,151.25162959098816,0.945322684943676,0.472661342471838,0.059417175995986324,0.0027740994402847717,3.3004495511856193,8.8964574990494e-06,0.0,0.0,0.0,0.0,0.0,0.0027740994402847717,80,40,35.27286076545715,0.8818215191364288,0.4409107595682144,0.02031386639282573 +3,0.003000541084111319,1.2077899141461088,1.181568532097677e-05,0.0,0.0,0.0,0.0,0.0,0.003000541084111319,320,160,151.62242531776428,0.9476401582360268,0.4738200791180134,0.0704218547190976,0.003368616230500265,3.7329810831716825,2.0277994416362245e-05,0.0,0.0,0.0,0.0,0.0,0.003368616230500265,80,40,35.269885778427124,0.8817471444606781,0.4408735722303391,0.022317990543524503 +4,0.0021543572441999003,2.36580728989997,3.905246718544086e-06,0.0,0.0,0.0,0.0,0.0,0.0021543572441999003,320,160,151.42908096313477,0.9464317560195923,0.47321587800979614,0.06814513673386387,0.0023375894830166997,3.1858106834027735,5.140880977472922e-06,0.0,0.0,0.0,0.0,0.0,0.0023375894830166997,80,40,35.21497082710266,0.8803742706775666,0.4401871353387833,0.022702036050031894 +5,0.0020639134972753937,1.8771122450190234,4.7617895551466114e-06,0.0,0.0,0.0,0.0,0.0,0.0020639134972753937,320,160,150.65398001670837,0.9415873751044274,0.4707936875522137,0.06839547897311604,0.002713540389231639,2.751661246550566,8.476993637529517e-06,0.0,0.0,0.0,0.0,0.0,0.002713540389231639,80,40,35.15806221961975,0.8789515554904938,0.4394757777452469,0.0216164964978816 +6,0.001974041026574014,2.6116740645130543,4.903630427295745e-06,0.0,0.0,0.0,0.0,0.0,0.001974041026574014,320,160,150.98297429084778,0.9436435893177986,0.4718217946588993,0.06690819428837073,0.0026584940679640567,2.928696505277807,1.0981135633864048e-05,0.0,0.0,0.0,0.0,0.0,0.0026584940679640567,80,40,34.98962068557739,0.8747405171394348,0.4373702585697174,0.028631291013152805 +7,0.0017634188792953864,1.5246528687675955,4.08627352063845e-06,0.0,0.0,0.0,0.0,0.0,0.0017634188792953864,320,160,151.10478925704956,0.9444049328565598,0.4722024664282799,0.06988477217641957,0.0025102962197934174,3.828912246527557,9.336235920053004e-06,0.0,0.0,0.0,0.0,0.0,0.0025102962197934174,80,40,35.14225649833679,0.8785564124584198,0.4392782062292099,0.025317877356610553 +8,0.0017970734881543216,1.9675439566956825,4.507927208244523e-06,0.0,0.0,0.0,0.0,0.0,0.0017970734881543216,320,160,151.0646207332611,0.9441538795828819,0.47207693979144094,0.06440023747350096,0.0029548812297832683,2.1843356087258696,1.6957902914016554e-05,0.0,0.0,0.0,0.0,0.0,0.0029548812297832683,80,40,35.090479135513306,0.8772619783878326,0.4386309891939163,0.028445404235390014 +9,0.0016425128245685983,2.024512654822502,3.285046532060373e-06,0.0,0.0,0.0,0.0,0.0,0.0016425128245685983,320,160,150.84772562980652,0.9427982851862907,0.47139914259314536,0.0747756733842209,0.002334522669548278,2.7116857997084027,6.992434950987836e-06,0.0,0.0,0.0,0.0,0.0,0.002334522669548278,80,40,35.245197772979736,0.8811299443244934,0.4405649721622467,0.026337641538702883 +10,0.0016268517200273892,1.4290221301512553,2.7465491623539574e-06,0.0,0.0,0.0,0.0,0.0,0.0016268517200273892,320,160,150.6063461303711,0.9412896633148193,0.47064483165740967,0.07696705762027704,0.0022422952166834876,3.8237469222483185,4.734226487249082e-06,0.0,0.0,0.0,0.0,0.0,0.0022422952166834876,80,40,35.0414342880249,0.8760358572006226,0.4380179286003113,0.02215462920921709 +11,0.001684735846095009,1.8539249592111176,3.617836458871815e-06,0.0,0.0,0.0,0.0,0.0,0.001684735846095009,320,160,151.0113205909729,0.9438207536935806,0.4719103768467903,0.06602817026396224,0.0023927180209284415,1.9305211880035131,8.37876484468536e-06,0.0,0.0,0.0,0.0,0.0,0.0023927180209284415,80,40,35.019232988357544,0.8754808247089386,0.4377404123544693,0.02876366543350741 +12,0.001648270361597781,1.466464621800828,3.6850406459681182e-06,0.0,0.0,0.0,0.0,0.0,0.001648270361597781,320,160,150.67579007148743,0.9417236879467964,0.4708618439733982,0.07282405201085566,0.0024106145921905407,1.8117562619359986,8.840941997867446e-06,0.0,0.0,0.0,0.0,0.0,0.0024106145921905407,80,40,35.25731110572815,0.8814327776432037,0.44071638882160186,0.028296888258773835 +13,0.0014949767563791738,2.0110324508543216,2.594263938809541e-06,0.0,0.0,0.0,0.0,0.0,0.0014949767563791738,320,160,150.59481382369995,0.9412175863981247,0.47060879319906235,0.07127468679950652,0.0034006003257673,2.3089766705088124,2.330103268749495e-05,0.0,0.0,0.0,0.0,0.0,0.0034006003257673,80,40,35.14684081077576,0.8786710202693939,0.43933551013469696,0.03264807362284046 +14,0.0014807669692402214,1.486915388038304,2.468103663008994e-06,0.0,0.0,0.0,0.0,0.0,0.0014807669692402214,320,160,150.8731460571289,0.9429571628570557,0.47147858142852783,0.06568786399493547,0.002275905742408213,2.4593560876942546,5.3657121410810585e-06,0.0,0.0,0.0,0.0,0.0,0.002275905742408213,80,40,35.17850923538208,0.879462730884552,0.439731365442276,0.028419802509597504 +15,0.0013959772029608075,1.5552767487895864,2.032669835160851e-06,0.0,0.0,0.0,0.0,0.0,0.0013959772029608075,320,160,145.97118186950684,0.9123198866844178,0.4561599433422089,0.0743303620764891,0.0021730058373577777,3.265071701107611,5.249506042540042e-06,0.0,0.0,0.0,0.0,0.0,0.0021730058373577777,80,40,32.64319920539856,0.816079980134964,0.408039990067482,0.02472462045188877 +16,0.0013986327389147845,1.0404113516639566,2.472861648920811e-06,0.0,0.0,0.0,0.0,0.0,0.0013986327389147845,320,160,142.02302479743958,0.8876439049839974,0.4438219524919987,0.0766021506049583,0.0021833765216797475,1.9410757024995746,5.971337235900851e-06,0.0,0.0,0.0,0.0,0.0,0.0021833765216797475,80,40,33.03069186210632,0.8257672965526581,0.41288364827632906,0.029009606450563295 +17,0.0013606195244022957,1.2219324406304888,2.558043809543567e-06,0.0,0.0,0.0,0.0,0.0,0.0013606195244022957,320,160,142.88847541809082,0.8930529713630676,0.4465264856815338,0.07105417825841868,0.0025992857859819195,4.460525457162658,1.0201318598324071e-05,0.0,0.0,0.0,0.0,0.0,0.0025992857859819195,80,40,32.683080196380615,0.8170770049095154,0.4085385024547577,0.02676110131196765 +18,0.0013010777075521673,0.9639209467314742,2.0914453604690185e-06,0.0,0.0,0.0,0.0,0.0,0.0013010777075521673,320,160,141.19299387931824,0.882456211745739,0.4412281058728695,0.07996181348535174,0.0021221282840997446,2.53250820556988,6.020639418136131e-06,0.0,0.0,0.0,0.0,0.0,0.0021221282840997446,80,40,32.62534475326538,0.8156336188316345,0.40781680941581727,0.028227102017262952 diff --git a/contraceptive/realtabformer/mlu-eval.ipynb b/contraceptive/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..863445229dccabfd5e574a7ef4ec0f4f5c890130 --- /dev/null +++ b/contraceptive/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2563 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.338469Z", + "iopub.status.busy": "2024-03-01T09:07:56.338121Z", + "iopub.status.idle": "2024-03-01T09:07:56.372428Z", + "shell.execute_reply": "2024-03-01T09:07:56.371512Z" + }, + "papermill": { + "duration": 0.049508, + "end_time": "2024-03-01T09:07:56.374820", + "exception": false, + "start_time": "2024-03-01T09:07:56.325312", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.403990Z", + "iopub.status.busy": "2024-03-01T09:07:56.403100Z", + "iopub.status.idle": "2024-03-01T09:07:56.412062Z", + "shell.execute_reply": "2024-03-01T09:07:56.411069Z" + }, + "papermill": { + "duration": 0.026577, + "end_time": "2024-03-01T09:07:56.414227", + "exception": false, + "start_time": "2024-03-01T09:07:56.387650", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.441828Z", + "iopub.status.busy": "2024-03-01T09:07:56.441456Z", + "iopub.status.idle": "2024-03-01T09:07:56.446308Z", + "shell.execute_reply": "2024-03-01T09:07:56.445341Z" + }, + "papermill": { + "duration": 0.021682, + "end_time": "2024-03-01T09:07:56.448533", + "exception": false, + "start_time": "2024-03-01T09:07:56.426851", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.475086Z", + "iopub.status.busy": "2024-03-01T09:07:56.474797Z", + "iopub.status.idle": "2024-03-01T09:07:56.479421Z", + "shell.execute_reply": "2024-03-01T09:07:56.478371Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.021034, + "end_time": "2024-03-01T09:07:56.481637", + "exception": false, + "start_time": "2024-03-01T09:07:56.460603", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.510323Z", + "iopub.status.busy": "2024-03-01T09:07:56.509980Z", + "iopub.status.idle": "2024-03-01T09:07:56.516453Z", + "shell.execute_reply": "2024-03-01T09:07:56.515542Z" + }, + "papermill": { + "duration": 0.023343, + "end_time": "2024-03-01T09:07:56.518756", + "exception": false, + "start_time": "2024-03-01T09:07:56.495413", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "faef7c07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.547738Z", + "iopub.status.busy": "2024-03-01T09:07:56.547348Z", + "iopub.status.idle": "2024-03-01T09:07:56.552780Z", + "shell.execute_reply": "2024-03-01T09:07:56.551757Z" + }, + "papermill": { + "duration": 0.023159, + "end_time": "2024-03-01T09:07:56.555149", + "exception": false, + "start_time": "2024-03-01T09:07:56.531990", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"realtabformer\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/realtabformer/3\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.013015, + "end_time": "2024-03-01T09:07:56.581179", + "exception": false, + "start_time": "2024-03-01T09:07:56.568164", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.607237Z", + "iopub.status.busy": "2024-03-01T09:07:56.606900Z", + "iopub.status.idle": "2024-03-01T09:07:56.616814Z", + "shell.execute_reply": "2024-03-01T09:07:56.616022Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.025043, + "end_time": "2024-03-01T09:07:56.618854", + "exception": false, + "start_time": "2024-03-01T09:07:56.593811", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/realtabformer/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:56.645717Z", + "iopub.status.busy": "2024-03-01T09:07:56.645378Z", + "iopub.status.idle": "2024-03-01T09:07:58.858292Z", + "shell.execute_reply": "2024-03-01T09:07:58.857375Z" + }, + "papermill": { + "duration": 2.22873, + "end_time": "2024-03-01T09:07:58.860317", + "exception": false, + "start_time": "2024-03-01T09:07:56.631587", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:58.884811Z", + "iopub.status.busy": "2024-03-01T09:07:58.884308Z", + "iopub.status.idle": "2024-03-01T09:07:58.895711Z", + "shell.execute_reply": "2024-03-01T09:07:58.894678Z" + }, + "papermill": { + "duration": 0.025785, + "end_time": "2024-03-01T09:07:58.897772", + "exception": false, + "start_time": "2024-03-01T09:07:58.871987", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:58.922747Z", + "iopub.status.busy": "2024-03-01T09:07:58.922238Z", + "iopub.status.idle": "2024-03-01T09:07:58.929493Z", + "shell.execute_reply": "2024-03-01T09:07:58.928757Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021647, + "end_time": "2024-03-01T09:07:58.931696", + "exception": false, + "start_time": "2024-03-01T09:07:58.910049", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:58.958947Z", + "iopub.status.busy": "2024-03-01T09:07:58.958666Z", + "iopub.status.idle": "2024-03-01T09:07:59.060750Z", + "shell.execute_reply": "2024-03-01T09:07:59.059835Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.118212, + "end_time": "2024-03-01T09:07:59.063105", + "exception": false, + "start_time": "2024-03-01T09:07:58.944893", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:07:59.087774Z", + "iopub.status.busy": "2024-03-01T09:07:59.087278Z", + "iopub.status.idle": "2024-03-01T09:08:03.766937Z", + "shell.execute_reply": "2024-03-01T09:08:03.766114Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.694674, + "end_time": "2024-03-01T09:08:03.769395", + "exception": false, + "start_time": "2024-03-01T09:07:59.074721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-01 09:08:01.341406: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-01 09:08:01.341469: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-01 09:08:01.343143: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:03.795981Z", + "iopub.status.busy": "2024-03-01T09:08:03.795367Z", + "iopub.status.idle": "2024-03-01T09:08:03.801326Z", + "shell.execute_reply": "2024-03-01T09:08:03.800622Z" + }, + "papermill": { + "duration": 0.021111, + "end_time": "2024-03-01T09:08:03.803263", + "exception": false, + "start_time": "2024-03-01T09:08:03.782152", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:03.827480Z", + "iopub.status.busy": "2024-03-01T09:08:03.827161Z", + "iopub.status.idle": "2024-03-01T09:08:12.049493Z", + "shell.execute_reply": "2024-03-01T09:08:12.048459Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.237193, + "end_time": "2024-03-01T09:08:12.051943", + "exception": false, + "start_time": "2024-03-01T09:08:03.814750", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:12.567977Z", + "iopub.status.busy": "2024-03-01T09:08:12.567646Z", + "iopub.status.idle": "2024-03-01T09:08:12.646794Z", + "shell.execute_reply": "2024-03-01T09:08:12.645795Z" + }, + "papermill": { + "duration": 0.095394, + "end_time": "2024-03-01T09:08:12.649237", + "exception": false, + "start_time": "2024-03-01T09:08:12.553843", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-01T09:08:12.680189Z", + "iopub.status.busy": "2024-03-01T09:08:12.679291Z", + "iopub.status.idle": "2024-03-01T09:08:13.186481Z", + "shell.execute_reply": "2024-03-01T09:08:13.185534Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.525003, + "end_time": "2024-03-01T09:08:13.189040", + "exception": false, + "start_time": "2024-03-01T09:08:12.664037", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:13.225847Z", + "iopub.status.busy": "2024-03-01T09:08:13.225030Z", + "iopub.status.idle": "2024-03-01T09:08:13.229918Z", + "shell.execute_reply": "2024-03-01T09:08:13.229020Z" + }, + "papermill": { + "duration": 0.023528, + "end_time": "2024-03-01T09:08:13.231899", + "exception": false, + "start_time": "2024-03-01T09:08:13.208371", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:13.263727Z", + "iopub.status.busy": "2024-03-01T09:08:13.262850Z", + "iopub.status.idle": "2024-03-01T09:08:13.272486Z", + "shell.execute_reply": "2024-03-01T09:08:13.271126Z" + }, + "papermill": { + "duration": 0.03102, + "end_time": "2024-03-01T09:08:13.275530", + "exception": false, + "start_time": "2024-03-01T09:08:13.244510", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10911264" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:13.310953Z", + "iopub.status.busy": "2024-03-01T09:08:13.310566Z", + "iopub.status.idle": "2024-03-01T09:08:13.401012Z", + "shell.execute_reply": "2024-03-01T09:08:13.400073Z" + }, + "papermill": { + "duration": 0.109543, + "end_time": "2024-03-01T09:08:13.403229", + "exception": false, + "start_time": "2024-03-01T09:08:13.293686", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 16128] --\n", + "├─Adapter: 1-1 [2, 1179, 16128] --\n", + "│ └─Embedding: 2-1 [2, 1179, 24, 672] (48,384)\n", + "│ └─TensorInductionPoint: 2-2 [24, 1] 24\n", + "│ └─Sequential: 2-3 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 689,152\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 16128] (recursive)\n", + "│ └─Embedding: 2-4 [2, 294, 24, 672] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [24, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-7 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,959,648\n", + "Trainable params: 10,911,264\n", + "Non-trainable params: 48,384\n", + "Total mult-adds (M): 42.71\n", + "========================================================================================================================\n", + "Input size (MB): 0.28\n", + "Forward/backward pass size (MB): 721.87\n", + "Params size (MB): 43.84\n", + "Estimated Total Size (MB): 765.99\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T09:08:13.434509Z", + "iopub.status.busy": "2024-03-01T09:08:13.434148Z", + "iopub.status.idle": "2024-03-01T10:12:35.609452Z", + "shell.execute_reply": "2024-03-01T10:12:35.608422Z" + }, + "papermill": { + "duration": 3862.193385, + "end_time": "2024-03-01T10:12:35.611608", + "exception": false, + "start_time": "2024-03-01T09:08:13.418223", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.015214953777865503, 'avg_role_model_std_loss': 3.156637710683768, 'avg_role_model_mean_pred_loss': 0.0015362251463382382, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015214953777865503, 'n_size': 320, 'n_batch': 160, 'duration': 150.5653796195984, 'duration_batch': 0.9410336226224899, 'duration_size': 0.47051681131124495, 'avg_pred_std': 0.06246557859708446}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0029828296956111444, 'avg_role_model_std_loss': 3.72726428431983, 'avg_role_model_mean_pred_loss': 7.1439977092975506e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029828296956111444, 'n_size': 80, 'n_batch': 40, 'duration': 35.29301643371582, 'duration_batch': 0.8823254108428955, 'duration_size': 0.44116270542144775, 'avg_pred_std': 0.017451438040006907}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005820814266519392, 'avg_role_model_std_loss': 3.788629152714401, 'avg_role_model_mean_pred_loss': 0.00012637018700479842, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005820814266519392, 'n_size': 320, 'n_batch': 160, 'duration': 151.99716687202454, 'duration_batch': 0.9499822929501534, 'duration_size': 0.4749911464750767, 'avg_pred_std': 0.054377025530902755}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003795744390345135, 'avg_role_model_std_loss': 4.469785842269539, 'avg_role_model_mean_pred_loss': 2.3026626298489063e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003795744390345135, 'n_size': 80, 'n_batch': 40, 'duration': 35.12023663520813, 'duration_batch': 0.8780059158802033, 'duration_size': 0.43900295794010163, 'avg_pred_std': 0.025538454634443042}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003438323737486826, 'avg_role_model_std_loss': 2.1381948816152487, 'avg_role_model_mean_pred_loss': 2.1238802406123302e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003438323737486826, 'n_size': 320, 'n_batch': 160, 'duration': 151.25162959098816, 'duration_batch': 0.945322684943676, 'duration_size': 0.472661342471838, 'avg_pred_std': 0.059417175995986324}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027740994402847717, 'avg_role_model_std_loss': 3.3004495511856193, 'avg_role_model_mean_pred_loss': 8.8964574990494e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027740994402847717, 'n_size': 80, 'n_batch': 40, 'duration': 35.27286076545715, 'duration_batch': 0.8818215191364288, 'duration_size': 0.4409107595682144, 'avg_pred_std': 0.02031386639282573}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003000541084111319, 'avg_role_model_std_loss': 1.2077899141461088, 'avg_role_model_mean_pred_loss': 1.181568532097677e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003000541084111319, 'n_size': 320, 'n_batch': 160, 'duration': 151.62242531776428, 'duration_batch': 0.9476401582360268, 'duration_size': 0.4738200791180134, 'avg_pred_std': 0.0704218547190976}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003368616230500265, 'avg_role_model_std_loss': 3.7329810831716825, 'avg_role_model_mean_pred_loss': 2.0277994416362245e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003368616230500265, 'n_size': 80, 'n_batch': 40, 'duration': 35.269885778427124, 'duration_batch': 0.8817471444606781, 'duration_size': 0.4408735722303391, 'avg_pred_std': 0.022317990543524503}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021543572441999003, 'avg_role_model_std_loss': 2.36580728989997, 'avg_role_model_mean_pred_loss': 3.905246718544086e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021543572441999003, 'n_size': 320, 'n_batch': 160, 'duration': 151.42908096313477, 'duration_batch': 0.9464317560195923, 'duration_size': 0.47321587800979614, 'avg_pred_std': 0.06814513673386387}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023375894830166997, 'avg_role_model_std_loss': 3.1858106834027735, 'avg_role_model_mean_pred_loss': 5.140880977472922e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023375894830166997, 'n_size': 80, 'n_batch': 40, 'duration': 35.21497082710266, 'duration_batch': 0.8803742706775666, 'duration_size': 0.4401871353387833, 'avg_pred_std': 0.022702036050031894}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020639134972753937, 'avg_role_model_std_loss': 1.8771122450190234, 'avg_role_model_mean_pred_loss': 4.7617895551466114e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020639134972753937, 'n_size': 320, 'n_batch': 160, 'duration': 150.65398001670837, 'duration_batch': 0.9415873751044274, 'duration_size': 0.4707936875522137, 'avg_pred_std': 0.06839547897311604}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002713540389231639, 'avg_role_model_std_loss': 2.751661246550566, 'avg_role_model_mean_pred_loss': 8.476993637529517e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002713540389231639, 'n_size': 80, 'n_batch': 40, 'duration': 35.15806221961975, 'duration_batch': 0.8789515554904938, 'duration_size': 0.4394757777452469, 'avg_pred_std': 0.0216164964978816}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001974041026574014, 'avg_role_model_std_loss': 2.6116740645130543, 'avg_role_model_mean_pred_loss': 4.903630427295745e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001974041026574014, 'n_size': 320, 'n_batch': 160, 'duration': 150.98297429084778, 'duration_batch': 0.9436435893177986, 'duration_size': 0.4718217946588993, 'avg_pred_std': 0.06690819428837073}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026584940679640567, 'avg_role_model_std_loss': 2.928696505277807, 'avg_role_model_mean_pred_loss': 1.0981135633864048e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026584940679640567, 'n_size': 80, 'n_batch': 40, 'duration': 34.98962068557739, 'duration_batch': 0.8747405171394348, 'duration_size': 0.4373702585697174, 'avg_pred_std': 0.028631291013152805}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017634188792953864, 'avg_role_model_std_loss': 1.5246528687675955, 'avg_role_model_mean_pred_loss': 4.08627352063845e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017634188792953864, 'n_size': 320, 'n_batch': 160, 'duration': 151.10478925704956, 'duration_batch': 0.9444049328565598, 'duration_size': 0.4722024664282799, 'avg_pred_std': 0.06988477217641957}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025102962197934174, 'avg_role_model_std_loss': 3.828912246527557, 'avg_role_model_mean_pred_loss': 9.336235920053004e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025102962197934174, 'n_size': 80, 'n_batch': 40, 'duration': 35.14225649833679, 'duration_batch': 0.8785564124584198, 'duration_size': 0.4392782062292099, 'avg_pred_std': 0.025317877356610553}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017970734881543216, 'avg_role_model_std_loss': 1.9675439566956825, 'avg_role_model_mean_pred_loss': 4.507927208244523e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017970734881543216, 'n_size': 320, 'n_batch': 160, 'duration': 151.0646207332611, 'duration_batch': 0.9441538795828819, 'duration_size': 0.47207693979144094, 'avg_pred_std': 0.06440023747350096}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0029548812297832683, 'avg_role_model_std_loss': 2.1843356087258696, 'avg_role_model_mean_pred_loss': 1.6957902914016554e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029548812297832683, 'n_size': 80, 'n_batch': 40, 'duration': 35.090479135513306, 'duration_batch': 0.8772619783878326, 'duration_size': 0.4386309891939163, 'avg_pred_std': 0.028445404235390014}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016425128245685983, 'avg_role_model_std_loss': 2.024512654822502, 'avg_role_model_mean_pred_loss': 3.285046532060373e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016425128245685983, 'n_size': 320, 'n_batch': 160, 'duration': 150.84772562980652, 'duration_batch': 0.9427982851862907, 'duration_size': 0.47139914259314536, 'avg_pred_std': 0.0747756733842209}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002334522669548278, 'avg_role_model_std_loss': 2.7116857997084027, 'avg_role_model_mean_pred_loss': 6.992434950987836e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002334522669548278, 'n_size': 80, 'n_batch': 40, 'duration': 35.245197772979736, 'duration_batch': 0.8811299443244934, 'duration_size': 0.4405649721622467, 'avg_pred_std': 0.026337641538702883}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016268517200273892, 'avg_role_model_std_loss': 1.4290221301512553, 'avg_role_model_mean_pred_loss': 2.7465491623539574e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016268517200273892, 'n_size': 320, 'n_batch': 160, 'duration': 150.6063461303711, 'duration_batch': 0.9412896633148193, 'duration_size': 0.47064483165740967, 'avg_pred_std': 0.07696705762027704}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022422952166834876, 'avg_role_model_std_loss': 3.8237469222483185, 'avg_role_model_mean_pred_loss': 4.734226487249082e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022422952166834876, 'n_size': 80, 'n_batch': 40, 'duration': 35.0414342880249, 'duration_batch': 0.8760358572006226, 'duration_size': 0.4380179286003113, 'avg_pred_std': 0.02215462920921709}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001684735846095009, 'avg_role_model_std_loss': 1.8539249592111176, 'avg_role_model_mean_pred_loss': 3.617836458871815e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001684735846095009, 'n_size': 320, 'n_batch': 160, 'duration': 151.0113205909729, 'duration_batch': 0.9438207536935806, 'duration_size': 0.4719103768467903, 'avg_pred_std': 0.06602817026396224}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023927180209284415, 'avg_role_model_std_loss': 1.9305211880035131, 'avg_role_model_mean_pred_loss': 8.37876484468536e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023927180209284415, 'n_size': 80, 'n_batch': 40, 'duration': 35.019232988357544, 'duration_batch': 0.8754808247089386, 'duration_size': 0.4377404123544693, 'avg_pred_std': 0.02876366543350741}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001648270361597781, 'avg_role_model_std_loss': 1.466464621800828, 'avg_role_model_mean_pred_loss': 3.6850406459681182e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001648270361597781, 'n_size': 320, 'n_batch': 160, 'duration': 150.67579007148743, 'duration_batch': 0.9417236879467964, 'duration_size': 0.4708618439733982, 'avg_pred_std': 0.07282405201085566}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024106145921905407, 'avg_role_model_std_loss': 1.8117562619359986, 'avg_role_model_mean_pred_loss': 8.840941997867446e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024106145921905407, 'n_size': 80, 'n_batch': 40, 'duration': 35.25731110572815, 'duration_batch': 0.8814327776432037, 'duration_size': 0.44071638882160186, 'avg_pred_std': 0.028296888258773835}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014949767563791738, 'avg_role_model_std_loss': 2.0110324508543216, 'avg_role_model_mean_pred_loss': 2.594263938809541e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014949767563791738, 'n_size': 320, 'n_batch': 160, 'duration': 150.59481382369995, 'duration_batch': 0.9412175863981247, 'duration_size': 0.47060879319906235, 'avg_pred_std': 0.07127468679950652}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0034006003257673, 'avg_role_model_std_loss': 2.3089766705088124, 'avg_role_model_mean_pred_loss': 2.330103268749495e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034006003257673, 'n_size': 80, 'n_batch': 40, 'duration': 35.14684081077576, 'duration_batch': 0.8786710202693939, 'duration_size': 0.43933551013469696, 'avg_pred_std': 0.03264807362284046}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014807669692402214, 'avg_role_model_std_loss': 1.486915388038304, 'avg_role_model_mean_pred_loss': 2.468103663008994e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014807669692402214, 'n_size': 320, 'n_batch': 160, 'duration': 150.8731460571289, 'duration_batch': 0.9429571628570557, 'duration_size': 0.47147858142852783, 'avg_pred_std': 0.06568786399493547}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002275905742408213, 'avg_role_model_std_loss': 2.4593560876942546, 'avg_role_model_mean_pred_loss': 5.3657121410810585e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002275905742408213, 'n_size': 80, 'n_batch': 40, 'duration': 35.17850923538208, 'duration_batch': 0.879462730884552, 'duration_size': 0.439731365442276, 'avg_pred_std': 0.028419802509597504}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013959772029608075, 'avg_role_model_std_loss': 1.5552767487895864, 'avg_role_model_mean_pred_loss': 2.032669835160851e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013959772029608075, 'n_size': 320, 'n_batch': 160, 'duration': 145.97118186950684, 'duration_batch': 0.9123198866844178, 'duration_size': 0.4561599433422089, 'avg_pred_std': 0.0743303620764891}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021730058373577777, 'avg_role_model_std_loss': 3.265071701107611, 'avg_role_model_mean_pred_loss': 5.249506042540042e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021730058373577777, 'n_size': 80, 'n_batch': 40, 'duration': 32.64319920539856, 'duration_batch': 0.816079980134964, 'duration_size': 0.408039990067482, 'avg_pred_std': 0.02472462045188877}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013986327389147845, 'avg_role_model_std_loss': 1.0404113516639566, 'avg_role_model_mean_pred_loss': 2.472861648920811e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013986327389147845, 'n_size': 320, 'n_batch': 160, 'duration': 142.02302479743958, 'duration_batch': 0.8876439049839974, 'duration_size': 0.4438219524919987, 'avg_pred_std': 0.0766021506049583}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021833765216797475, 'avg_role_model_std_loss': 1.9410757024995746, 'avg_role_model_mean_pred_loss': 5.971337235900851e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021833765216797475, 'n_size': 80, 'n_batch': 40, 'duration': 33.03069186210632, 'duration_batch': 0.8257672965526581, 'duration_size': 0.41288364827632906, 'avg_pred_std': 0.029009606450563295}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013606195244022957, 'avg_role_model_std_loss': 1.2219324406304888, 'avg_role_model_mean_pred_loss': 2.558043809543567e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013606195244022957, 'n_size': 320, 'n_batch': 160, 'duration': 142.88847541809082, 'duration_batch': 0.8930529713630676, 'duration_size': 0.4465264856815338, 'avg_pred_std': 0.07105417825841868}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025992857859819195, 'avg_role_model_std_loss': 4.460525457162658, 'avg_role_model_mean_pred_loss': 1.0201318598324071e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025992857859819195, 'n_size': 80, 'n_batch': 40, 'duration': 32.683080196380615, 'duration_batch': 0.8170770049095154, 'duration_size': 0.4085385024547577, 'avg_pred_std': 0.02676110131196765}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013010777075521673, 'avg_role_model_std_loss': 0.9639209467314742, 'avg_role_model_mean_pred_loss': 2.0914453604690185e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013010777075521673, 'n_size': 320, 'n_batch': 160, 'duration': 141.19299387931824, 'duration_batch': 0.882456211745739, 'duration_size': 0.4412281058728695, 'avg_pred_std': 0.07996181348535174}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021221282840997446, 'avg_role_model_std_loss': 2.53250820556988, 'avg_role_model_mean_pred_loss': 6.020639418136131e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021221282840997446, 'n_size': 80, 'n_batch': 40, 'duration': 32.62534475326538, 'duration_batch': 0.8156336188316345, 'duration_size': 0.40781680941581727, 'avg_pred_std': 0.028227102017262952}\n", + "Epoch 19\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013113658649871241, 'avg_role_model_std_loss': 1.157548566752187, 'avg_role_model_mean_pred_loss': 2.71372324170595e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013113658649871241, 'n_size': 320, 'n_batch': 160, 'duration': 141.17731380462646, 'duration_batch': 0.8823582112789154, 'duration_size': 0.4411791056394577, 'avg_pred_std': 0.06990279755846132}\n", + "Time out: 3691.47634100914/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▅█▄▆▂▃▃▃▄▂▂▂▂▆▂▁▁▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▃▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▅▂▃▃▃▆▅▆▅▃▆▆█▆▄▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▃▁▂▅▅▅▄▅▄▇▇▄▆▆▄▆▇▆█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▅█▄▆▂▃▃▃▄▂▂▂▂▆▂▁▁▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▂█▃▇▁▂▃▃▆▂▁▂▃█▁▁▁▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▆█▅▆▅▃▄▆▂▃▆▁▁▂▃▅▁█▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▆█▄▂▄▃▅▂▃▄▂▃▂▄▂▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ██████▇█▇█▇▇███▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▇████▇▇▇▇▇▇▇▇▇▇▄▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ██████▇█▇█▇▇███▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▇████▇▇▇▇▇▇▇▇▇▇▄▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ██████▇█▇█▇▇███▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▇████▇▇▇▇▇▇▇▇▇▇▄▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00212\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.0013\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.02823\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07996\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00212\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.0013\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 2.53251\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.96392\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.81563\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.88246\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.40782\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.44123\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 32.62534\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 141.19299\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/realtabformer/3/wandb/offline-run-20240301_090815-dmvtx8gf\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240301_090815-dmvtx8gf/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.001492384186390257, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.009758013392212732, 'pred_duration': 4.686195373535156, 'grad_duration': 2.4317636489868164, 'total_duration': 7.117959022521973, 'pred_std': 0.05503244325518608, 'std_loss': 0.021272432059049606, 'mean_pred_loss': 1.4633540104114218e-06, 'pred_rmse': 0.0386313833296299, 'pred_mae': 0.030890200287103653, 'pred_mape': 0.07129626721143723, 'grad_rmse': 0.24003435671329498, 'grad_mae': 0.11075862497091293, 'grad_mape': 1.6372917890548706}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.001492384186390257, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.009758013392212732, 'avg_pred_duration': 4.686195373535156, 'avg_grad_duration': 2.4317636489868164, 'avg_total_duration': 7.117959022521973, 'avg_pred_std': 0.05503244325518608, 'avg_std_loss': 0.021272432059049606, 'avg_mean_pred_loss': 1.4633540104114218e-06}, 'min_metrics': {'avg_loss': 0.001492384186390257, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.009758013392212732, 'pred_duration': 4.686195373535156, 'grad_duration': 2.4317636489868164, 'total_duration': 7.117959022521973, 'pred_std': 0.05503244325518608, 'std_loss': 0.021272432059049606, 'mean_pred_loss': 1.4633540104114218e-06, 'pred_rmse': 0.0386313833296299, 'pred_mae': 0.030890200287103653, 'pred_mape': 0.07129626721143723, 'grad_rmse': 0.24003435671329498, 'grad_mae': 0.11075862497091293, 'grad_mape': 1.6372917890548706}, 'model_metrics': {'realtabformer': {'avg_loss': 0.001492384186390257, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.009758013392212732, 'pred_duration': 4.686195373535156, 'grad_duration': 2.4317636489868164, 'total_duration': 7.117959022521973, 'pred_std': 0.05503244325518608, 'std_loss': 0.021272432059049606, 'mean_pred_loss': 1.4633540104114218e-06, 'pred_rmse': 0.0386313833296299, 'pred_mae': 0.030890200287103653, 'pred_mape': 0.07129626721143723, 'grad_rmse': 0.24003435671329498, 'grad_mae': 0.11075862497091293, 'grad_mape': 1.6372917890548706}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:12:35.651891Z", + "iopub.status.busy": "2024-03-01T10:12:35.651152Z", + "iopub.status.idle": "2024-03-01T10:12:35.655515Z", + "shell.execute_reply": "2024-03-01T10:12:35.654755Z" + }, + "papermill": { + "duration": 0.026966, + "end_time": "2024-03-01T10:12:35.657479", + "exception": false, + "start_time": "2024-03-01T10:12:35.630513", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:12:35.693775Z", + "iopub.status.busy": "2024-03-01T10:12:35.693180Z", + "iopub.status.idle": "2024-03-01T10:12:35.985277Z", + "shell.execute_reply": "2024-03-01T10:12:35.984429Z" + }, + "papermill": { + "duration": 0.312743, + "end_time": "2024-03-01T10:12:35.987715", + "exception": false, + "start_time": "2024-03-01T10:12:35.674972", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:12:36.026081Z", + "iopub.status.busy": "2024-03-01T10:12:36.025721Z", + "iopub.status.idle": "2024-03-01T10:12:36.311184Z", + "shell.execute_reply": "2024-03-01T10:12:36.310203Z" + }, + "papermill": { + "duration": 0.307179, + "end_time": "2024-03-01T10:12:36.313430", + "exception": false, + "start_time": "2024-03-01T10:12:36.006251", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBiklEQVR4nO3de1xU5b4/8M/MMBdAGG7CACHgBVE0LAzCNDPJ0dDEPHnJreQ2tTZ2KvJoVELt9g5v7e3xcjLrt7V2KWillRplqLkTxERUCCUvKCQMiMgM94GZ5/fHgmUjA87AwDDyfb9e67WGtZ611jMw82U91yVgjDEQQoiNEFo7A4QQYg4KWoQQm0JBixBiUyhoEUJsCgUtQohNoaBFCLEpFLQIITbFztoZ6Cl6vR4lJSVwcnKCQCCwdnYIIXdgjKG6uho+Pj4QCtu/n+ozQaukpAR+fn7WzgYh5C6Ki4tx3333tbu/zwQtJycnANwvxNnZ2cq5IYTcSaPRwM/Pj/+utqfPBK3WIqGzszMFLUJ6sbtV31BFPCHEplDQIoTYFApahBCb0mfqtEjX6HQ6NDU1WTsbxIaJxWKIRKIun4eCFukQYwwqlQpVVVXWzgq5B7i4uEChUHSpryQFLdKh1oDl6ekJBwcH6phLOoUxhrq6OpSXlwMAvL29O30uClp3UNc1Ib9UA6EAiBjobu3sWJVOp+MDlrt73/5dkK6zt7cHAJSXl8PT07PTRUWqiL9DTvEtzP3oBN7+Nt/aWbG61josBwcHK+eE3CtaP0tdqR+loHUHj35SAEBlbaOVc9J7UJGQWIolPksUtO7g5igBAFTWakHP/CCk96GgdYfWoNWkY6hubLZybgghd6KgdQeZWARHCVdBeLNGa+XcEFslEAiwb98+a2fDot5++22MGjXK2tmgoGWMW7/WIiLVaxHbtWPHDri4uFjsfMuXL0d6errFztdZFLSMcHfkKuPpTov0BVqtaZ/zfv369YquLxS0jHD/Q2U8uY0xhjpts1UWcxtF0tLSMHbsWLi4uMDd3R1Tp07F5cuXAQBjxozBypUrDdLfuHEDYrEYx44dAwCUlpYiOjoa9vb2CAwMxM6dOxEQEIANGzZ06neXm5uLxx9/HPb29nB3d8eSJUtQU1PD7z969CjCw8Ph6OgIFxcXPPLII7h27RoA4OzZs5gwYQKcnJzg7OyMsLAwnDp1qsPrHT16FAsXLoRarYZAIIBAIMDbb78NAAgICMC7776LBQsWwNnZGUuWLAEArFy5EkFBQXBwcMDAgQOxatUqg64JdxYPn3vuOcTExGD9+vXw9vaGu7s74uLiun24F3UuNaK1Mv4mBS0D9U06DE/83irXzv+rEg4S0z+utbW1iI+Px/3334+amhokJiZixowZOHPmDObNm4e1a9di9erVfBN8amoqfHx8MG7cOADAggULUFFRgaNHj0IsFiM+Pp7vzW2u2tpaKJVKREZG4pdffkF5eTmef/55LFu2DDt27EBzczNiYmKwePFi7Nq1C1qtFidPnuTzNm/ePDzwwAP44IMPIBKJcObMGYjF4g6vOWbMGGzYsAGJiYkoKCgAwN0ptVq/fj0SExORlJTEb3NycsKOHTvg4+OD3NxcLF68GE5OTlixYkW71zly5Ai8vb1x5MgRXLp0CbNnz8aoUaOwePHiTv2uTEFBy4jWOi0qHtqumTNnGvz8r3/9C/3790d+fj5mzZqFV155BT///DMfpHbu3Im5c+dCIBDgwoUL+PHHH/HLL79g9OjRAICPP/4YQ4YM6VRedu7ciYaGBnz66adwdHQEAGzevBnTpk3DmjVrIBaLoVarMXXqVAwaNAgAMGzYMP74oqIi/M///A+Cg4MBwKR8SCQSyOVyCAQCKBSKNvsff/xxvPbaawbb3nrrLf51QEAAli9fjpSUlA6DlqurKzZv3gyRSITg4GBER0cjPT2dglZPu108pIr4P7IXi5D/V6XVrm2OixcvIjExEVlZWaioqIBerwfABYARI0Zg0qRJ+PzzzzFu3DgUFhYiMzMTH374IQCgoKAAdnZ2ePDBB/nzDR48GK6urp3K+/nz5xEaGsoHLAB45JFHoNfrUVBQgEcffRTPPfcclEolnnjiCURFRWHWrFn8+Lz4+Hg8//zz+Pe//42oqCg888wzfHDrrNZg/EepqanYuHEjLl++jJqaGjQ3N991lt+QkBCD4Tje3t7Izc3tUt7uhuq0jOAr4ql4aEAgEMBBYmeVxdye1NOmTUNlZSU++ugjZGVlISsrC8DtSud58+bhiy++QFNTE3bu3ImRI0di5MiRFv+dmWr79u3IzMzEmDFjkJqaiqCgIJw4cQIAV5f066+/Ijo6GocPH8bw4cOxd+/eLl3vjwEUADIzMzFv3jw8+eST2L9/P3JycvDmm2/etZL+zmKqQCDg/0F0FwpaRtzu8kBByxbdvHkTBQUFeOuttzBx4kQMGzYMt27dMkgzffp0NDQ0IC0tDTt37sS8efP4fUOHDkVzczNycnL4bZcuXWpzDlMNGzYMZ8+eRW1tLb/t+PHjEAqFGDp0KL/tgQceQEJCAjIyMjBixAjs3LmT3xcUFIRXX30VP/zwA55++mls3779rteVSCTQ6XQm5TEjIwP+/v548803MXr0aAwZMoRvCOhtKGgZ0Vo8pDot2+Tq6gp3d3ds27YNly5dwuHDhxEfH2+QxtHRETExMVi1ahXOnz+PuXPn8vuCg4MRFRWFJUuW4OTJk8jJycGSJUtgb2/fqbFz8+bNg0wmQ2xsLPLy8nDkyBG89NJLmD9/Pry8vFBYWIiEhARkZmbi2rVr+OGHH3Dx4kUMGzYM9fX1WLZsGY4ePYpr167h+PHj+OWXXwzqvNoTEBCAmpoapKeno6KiAnV1de2mHTJkCIqKipCSkoLLly9j48aNXb6b6y4UtIyg8Ye2TSgUIiUlBdnZ2RgxYgReffVVrFu3rk26efPm4ezZsxg3bhwGDBhgsO/TTz+Fl5cXHn30UcyYMYNvSZPJZGbnx8HBAd9//z0qKyvx0EMP4b/+678wceJEbN68md9/4cIFzJw5E0FBQViyZAni4uKwdOlSiEQi3Lx5EwsWLEBQUBBmzZqFKVOm4J133rnrdceMGYMXXngBs2fPRv/+/bF27dp20z711FN49dVXsWzZMowaNQoZGRlYtWqV2e+1R7BO2Lx5M/P392dSqZSFh4ezrKysDtPv3r2bDR06lEmlUjZixAh24MABg/1ffvkle+KJJ5ibmxsDwHJycto9l16vZ5MnT2YA2N69e03Os1qtZgCYWq2+a9q6xmbmv3I/81+5n2nqtSZf415TX1/P8vPzWX19vbWzYnXFxcUMAPvxxx+tnRWb1tFnytTvqNl3WqmpqYiPj0dSUhJOnz6N0NBQKJXKdvuwZGRkYO7cuVi0aBFycnIQExODmJgY5OXl8Wlqa2sxduxYrFmz5q7X37BhQ7dPlWIvEcGhZfwh1Wv1TYcPH8Y333yDwsJCZGRkYM6cOQgICMCjjz5q7awRcyNleHg4i4uL43/W6XTMx8eHJScnG00/a9YsFh0dbbAtIiKCLV26tE3awsLCDu+0cnJymK+vLystLe3WOy3GGHtkdTrzX7mfZV+rNPka95q+fKeVlpbGQkJCmL29PfP09GQxMTHs6tWrjDHGPvvsM+bo6Gh0GT58eI/lcfLkye3m4+9//3uP5cMclrjTMqufllarRXZ2NhISEvhtQqEQUVFRyMzMNHpMZmZmm0pQpVJp9gj4uro6PPvss9iyZYvRznJ3amxsRGPj7X5WGo3GrOu5O0rw+616qozvo5RKJZRK433SnnrqKURERBjdd7ee6pb08ccfo76+3ug+Nze3HstHTzMraFVUVECn08HLy8tgu5eXFy5cuGD0GJVKZTS9SqUyK6OvvvoqxowZg+nTp5uUPjk52aTKyva4UQdT0g4nJyc4OTlZOxvw9fW1dhaswiZaD7/55hscPnzYrMGqCQkJUKvV/FJcXGzWNd2ogykhvZJZQcvDwwMikQhlZWUG28vKytotsikUCrPSG3P48GFcvnwZLi4usLOzg50dd4M4c+ZMPPbYY0aPkUqlcHZ2NljM4dHawZSKh4T0KmYFLYlEgrCwMIOJwPR6PdLT0xEZGWn0mMjIyDYThx06dKjd9Ma8/vrrOHfuHM6cOcMvAPDPf/7TpJ7BneFG09MQ0iuZPWA6Pj4esbGxGD16NMLDw7FhwwbU1tZi4cKFALgpPXx9fZGcnAwAePnllzF+/Hi8//77iI6ORkpKCk6dOoVt27bx56ysrERRURFKSkoAgJ9KQ6FQGCx3GjBgAAIDA81/1yZoDVoVFLQI6VXMDlqzZ8/GjRs3kJiYCJVKhVGjRiEtLY2vbC8qKoJQePsGbsyYMdi5cyfeeustvPHGGxgyZAj27duHESNG8Gm++eYbPugBwJw5cwAASUlJ/MRlPc2dplwmpFcSMNY3xqloNBrI5XKo1WqT6rfOFFchZstx+MhlyEiY2AM57H0aGhpQWFiIwMDATg1f6csEAgH27t2LmJgYa2elV+noM2Xqd9QmWg+twf0Ps5f2kbhO7jGWfrAFwE3jLBAIUFVVZdHzmoOCVjtai4eNzXrUaU2b3oMQ0v0oaLXDQWIHmZj79VCv+BaMAdpa6yz0YAuLPdiisbERy5cvh6+vLxwdHREREYGjR4/yx167dg3Tpk2Dq6srHB0dERISgoMHD+Lq1auYMGECAG76H4FAgOeee65Tv4+uoOmWO+DuKMX1qnrcrG3EAHcHa2fH+prqgPd8rHPtN0oAiePd07WgB1u0/2CLZcuWIT8/HykpKfDx8cHevXsxefJk5ObmYsiQIYiLi4NWq8WxY8fg6OiI/Px89OvXD35+fvjyyy8xc+ZMFBQUwNnZGfb29p36nXQFBa0OuDlKcL2qnvpq2SB6sIXxB1sUFRVh+/btKCoqgo8P9w9o+fLlSEtLw/bt2/Hee++hqKgIM2fO5KefHjhwIH9865hGT09Pi9eXmYqCVgda67VoKE8LsQN3x2Ota5uBHmxhXG5uLnQ6HYKCggy2NzY28g9i/e///m+8+OKL+OGHHxAVFYWZM2fi/vvv79T1ugPVaXWAesXfQSDgimjWWOjBFhZ5sEVNTQ1EIhGys7MNRpicP38e//u//wsAeP7553HlyhXMnz8fubm5GD16NDZt2mSx99pVFLQ6cHuueOpgakvowRYcYw+2eOCBB6DT6VBeXo7BgwcbLH8sRvr5+eGFF17AV199hddeew0fffQRf04AJj8woztQ0OoAzfRgm+jBFhxjD7YICgrCvHnzsGDBAnz11VcoLCzEyZMnkZycjAMHDgAAXnnlFXz//fcoLCzE6dOnceTIEf56/v7+EAgE2L9/P27cuGHQAtpjumV6wl7I3JlLGWMs9WQR81+5n8X+q+M58O9Vtjxz6aFDh9iwYcOYVCpl999/Pzt69Gib2W4PHjzIALBHH320zfElJSVsypQpTCqVMn9/f7Zz507m6enJtm7datL177zWuXPn2IQJE5hMJmNubm5s8eLFrLq6mjHGmEqlYjExMczb25tJJBLm7+/PEhMTmU6nY42NjWzOnDnMz8+PSSQS5uPjw5YtW2by3+SFF15g7u7uDABLSkpijDGm1WpZYmIiCwgIYGKxmHl7e7MZM2awc+fOMcYYW7ZsGRs0aBCTSqWsf//+bP78+ayiooI/51//+lemUCiYQCBgsbGxJuWjlSVmLqVhPB1IP1+GRZ+cwv33yfHNsrHdnMPeh4bx3Pb777/Dz88PP/74IyZO7JvDuizBEsN4qPWwA270/MM+6/Dhw6ipqcHIkSNRWlqKFStW0IMtegmq0+qAO1+nRRXxfU1TUxPeeOMNhISEYMaMGejfvz/f0fTzzz9Hv379jC4hISE9lscpU6a0m4/33nuvx/LR0+hOqwNuLf20Gpr0qNM2w0FCv66+gh5s0XvRt7ADjhIRJHZCaJv1uFmjhYMb/boIPdjC2qh42AGBQAAP6mBKU/MQi7HEZ4mC1l249eu7Qau1qFNXV2flnJB7RetnqSvFaCrv3EVrB9OKPtgrXiQSwcXFhZ/dwMHBoVOdKwlhjKGurg7l5eVwcXGBSCTq9LkoaN2Fex8vHrYO7ejstCyE/JGLi4tZjw80hoLWXfT1QdMCgQDe3t7w9PREU1OTtbNDbJhYLO7SHRbPrD74LTZv3sz8/f2ZVCpl4eHhLCur42Euu3fvZkOHDmVSqZSNGDGCHThwwGD/l19+yZ544gnm5ubGALCcnByD/Tdv3mTLli1jQUFBTCaTMT8/P/bSSy+xqqoqk/PcmWE8jDG25chF5r9yP3tt9xmzjiOEmMfU76jZFfGpqamIj49HUlISTp8+jdDQUCiVynaLDxkZGZg7dy4WLVqEnJwcxMTEICYmBnl5eXya2tpajB07FmvWrDF6jpKSEpSUlGD9+vXIy8vDjh07kJaWhkWLFpmbfbPRTA+E9DLmRsPw8HAWFxfH/6zT6ZiPjw9LTk42mn7WrFksOjraYFtERARbunRpm7SFhYVG77SM2b17N5NIJKypqcno/oaGBqZWq/mluLi4U3daP/yqYv4r97OnNv3HrOMIIebpljstrVaL7OxsREVF8duEQiGioqKQmZlp9JjMzEyD9ADX27i99KZqHVRpZ2e8Wi45ORlyuZxf/Pz8OnUdN0eavZSQ3sSsoFVRUQGdTsc/TbqVl5cXVCqV0WNUKpVZ6U3Nx7vvvoslS5a0myYhIQFqtZpfiouLO3Wtvt56SEhvY3OthxqNBtHR0Rg+fDj/SCRjpFIppFJpl6/XOk98nVaHeq0O9hILtH4QQjrNrDstDw8PiEQilJWVGWwvKytrt++FQqEwK31HqqurMXnyZDg5OWHv3r09Mji1n9QOElHL8w9ptgdCrM6soCWRSBAWFob09HR+m16vR3p6OiIjI40eExkZaZAeAA4dOtRu+vZoNBpMmjQJEokE33zzTY9NSicQCPp8Xy1CehOzi4fx8fGIjY3F6NGjER4ejg0bNqC2thYLFy4EwD3k0tfXF8nJyQCAl19+GePHj8f777+P6OhopKSk4NSpU9i2bRt/zsrKShQVFaGkhHs8VevDJRUKBRQKBR+w6urq8Nlnn0Gj0UCj0QAA+vfvb5kOax1wc5RApWmgynhCeoPONE1u2rSJDRgwgEkkEhYeHs5OnDjB7xs/fnybeaN3797NgoKCmEQiYSEhIW06l27fvp0BaLO0zml95MgRo/sBsMLCQpPy3NnOpYwx9qePTzD/lfvZF6eKzT6WEGIamiP+Dp2ZI77VKyk52HemBG8+OQyLHx149wMIIWYz9TtKU9OYgJ/pgSriCbE6ClomaO32UEkPuCDE6ihomYBaDwnpPShomYCG8hDSe1DQMoFHH55ymZDehoKWCVor4ml6GkKsj4KWCVqLh7VaHRqadFbODSF9GwUtEzjL7CAWcQ90oCIiIdZFQcsEAoEArg5Ur0VIb0BBy0Tu/VrqtShoEWJVFLRMRHPFE9I7UNAyEXUwJaR3oKBlIupgSkjvQEHLRPxc8TT+kBCroqBlIqqIJ6R3oKBlotvFQ6qIJ8SaKGiZyJ3GHxLSK1DQMpEb1WkR0itQ0DJRa0V8dWMzGptp/CEh1tKpoLVlyxYEBARAJpMhIiICJ0+e7DD9nj17EBwcDJlMhpEjR+LgwYMG+7/66itMmjQJ7u7uEAgEOHPmTJtzNDQ0IC4uDu7u7ujXrx9mzpzZ5nmK3clZJoadkBt/eKu2qceuSwgxZHbQSk1NRXx8PJKSknD69GmEhoZCqVSivLzcaPqMjAzMnTsXixYtQk5ODmJiYhATE4O8vDw+TW1tLcaOHYs1a9a0e91XX30V3377Lfbs2YOffvoJJSUlePrpp83NfqcJhQK4ttxtVVCveEKsx9zH/ISHh7O4uDj+Z51Ox3x8fFhycrLR9LNmzWLR0dEG2yIiItjSpUvbpC0sLGQAWE5OjsH2qqoqJhaL2Z49e/ht58+fZwBYZmam0es2NDQwtVrNL8XFxZ1+hFgr5T9/Yv4r97OfCso7fQ5CiHGmPkLMrDstrVaL7OxsREVF8duEQiGioqKQmZlp9JjMzEyD9ACgVCrbTW9MdnY2mpqaDM4THByMAQMGtHue5ORkyOVyfvHz8zP5eu2hoTyEWJ9ZQauiogI6nQ5eXl4G2728vKBSqYweo1KpzErf3jkkEglcXFxMPk9CQgLUajW/FBcXm3y99tBQHkKsz87aGeguUqkUUqnUouf0aOkVX0kdTAmxGrPutDw8PCASidq02pWVlUGhUBg9RqFQmJW+vXNotVpUVVV16Txdxd9pUV8tQqzGrKAlkUgQFhaG9PR0fpter0d6ejoiIyONHhMZGWmQHgAOHTrUbnpjwsLCIBaLDc5TUFCAoqIis87TVVQ8JMT6zC4exsfHIzY2FqNHj0Z4eDg2bNiA2tpaLFy4EACwYMEC+Pr6Ijk5GQDw8ssvY/z48Xj//fcRHR2NlJQUnDp1Ctu2bePPWVlZiaKiIpSUlADgAhLA3WEpFArI5XIsWrQI8fHxcHNzg7OzM1566SVERkbi4Ycf7vIvwVTuVBFPiPV1pmly06ZNbMCAAUwikbDw8HB24sQJft/48eNZbGysQfrdu3ezoKAgJpFIWEhICDtw4IDB/u3btzMAbZakpCQ+TX19PfvLX/7CXF1dmYODA5sxYwYrLS01Oc+mNqd2JOvKTea/cj97bN2RTp+DEGKcqd9RAWOMWTFm9hiNRgO5XA61Wg1nZ+dOneNSeQ2i/vETnGV2OPe20sI5JKRvM/U7SmMPzdBaPNQ0NEPbrLdybgjpmyhomUFuL4aodfxhHdVrEWINFLTMIBQK4OogBkDdHgixFgpaZnJ3bO1gSkGLEGugoGUmmnaZEOuioGUmt37UK54Qa6KgZSbqYEqIdVHQMhMN5SHEuihomcmdZnogxKooaJnJnWZ6IMSqKGiZiWYvJcS6KGiZyZ3qtAixKgpaZmq901LXN6FJR+MPCelpFLTM5OIgQcvwQxp/SIgVUNAyk0gogKsDVcYTYi0UtDqBKuMJsR4KWp1AHUwJsR4KWp3g3jL+sLKGOpgS0tMoaHUCTU9DiPV0Kmht2bIFAQEBkMlkiIiIwMmTJztMv2fPHgQHB0Mmk2HkyJE4ePCgwX7GGBITE+Ht7Q17e3tERUXh4sWLBml+++03TJ8+HR4eHnB2dsbYsWNx5MiRzmS/y1qLhxUUtAjpcWYHrdTUVMTHxyMpKQmnT59GaGgolEolysvLjabPyMjA3LlzsWjRIuTk5CAmJgYxMTHIy8vj06xduxYbN27E1q1bkZWVBUdHRyiVSjQ0NPBppk6diubmZhw+fBjZ2dkIDQ3F1KlToVKpOvG2u+Z28ZCCFiE9ztzH/ISHh7O4uDj+Z51Ox3x8fFhycrLR9LNmzWLR0dEG2yIiItjSpUsZY4zp9XqmUCjYunXr+P1VVVVMKpWyXbt2McYYu3HjBgPAjh07xqfRaDQMADt06JBJ+bbEI8RafXv2OvNfuZ8980FGl89FCOGY+h01605Lq9UiOzsbUVFR/DahUIioqChkZmYaPSYzM9MgPQAolUo+fWFhIVQqlUEauVyOiIgIPo27uzuGDh2KTz/9FLW1tWhubsaHH34IT09PhIWFGb1uY2MjNBqNwWIpNHspIdZjVtCqqKiATqeDl5eXwXYvL692i2kqlarD9K3rjtIIBAL8+OOPyMnJgZOTE2QyGf7xj38gLS0Nrq6uRq+bnJwMuVzOL35+fua81Q559KOKeEKsxSZaDxljiIuLg6enJ/7zn//g5MmTiImJwbRp01BaWmr0mISEBKjVan4pLi62WH5a77Ru1TWhmcYfEtKjzApaHh4eEIlEKCsrM9heVlYGhUJh9BiFQtFh+tZ1R2kOHz6M/fv3IyUlBY888ggefPBB/N///R/s7e3xySefGL2uVCqFs7OzwWIprg4SCPjxh00WOy8h5O7MCloSiQRhYWFIT0/nt+n1eqSnpyMyMtLoMZGRkQbpAeDQoUN8+sDAQCgUCoM0Go0GWVlZfJq6ujous0LD7AqFQuj1PX+nIxIK4GLPPf+QioiE9DBza/hTUlKYVCplO3bsYPn5+WzJkiXMxcWFqVQqxhhj8+fPZ6+//jqf/vjx48zOzo6tX7+enT9/niUlJTGxWMxyc3P5NKtXr2YuLi7s66+/ZufOnWPTp09ngYGBrL6+njHGtR66u7uzp59+mp05c4YVFBSw5cuXM7FYzM6cOWNSvi3ZesgYY4+vP8L8V+5nxy/dsMj5COnrTP2Omh20GGNs06ZNbMCAAUwikbDw8HB24sQJft/48eNZbGysQfrdu3ezoKAgJpFIWEhICDtw4IDBfr1ez1atWsW8vLyYVCplEydOZAUFBQZpfvnlFzZp0iTm5ubGnJyc2MMPP8wOHjxocp4tHbSe2ZrB/FfuZ9+evW6R8xHS15n6HRUwxph17/V6hkajgVwuh1qttkj91oufZeO7PBXeeSoEsWMCup5BQvo4U7+jNtF62BvRTA+EWAcFrU66/dBW6mBKSE+ioNVJNBEgIdZBQauTWh/aSlMuE9KzKGh1Ej1KjBDroKDVSW79qHhIiDVQ0Oqk2+MPtdDp+0SvEUJ6BQpandT6GDHGgCp6/iEhPYaCVieJRUK4OND4Q0J6GgWtLuDniqcWREJ6DAWtLnCnvlqE9DgKWl3gRr3iCelxFLS6wK3l+YfUV4uQnkNBqws8WvpqUa94QnoOBa0uoPGHhPQ8ClpdQI8SI6TnUdDqAndHepQYIT2NglYXUPGQkJ5HQasLPP4waFpP4w8J6RGdClpbtmxBQEAAZDIZIiIicPLkyQ7T79mzB8HBwZDJZBg5ciQOHjxosJ8xhsTERHh7e8Pe3h5RUVG4ePFim/McOHAAERERsLe3h6urK2JiYjqTfYtxbbnT0jOgqp6ef0hITzA7aKWmpiI+Ph5JSUk4ffo0QkNDoVQqUV5ebjR9RkYG5s6di0WLFiEnJwcxMTGIiYlBXl4en2bt2rXYuHEjtm7diqysLDg6OkKpVKKhoYFP8+WXX2L+/PlYuHAhzp49i+PHj+PZZ5/txFu2HLFICGeZHQDqYEpIjzH3MT/h4eEsLi6O/1mn0zEfHx+WnJxsNP2sWbNYdHS0wbaIiAi2dOlSxhj3+DCFQsHWrVvH76+qqmJSqZTt2rWLMcZYU1MT8/X1ZR9//LG52eVZ+hFirR5bxz3/8MTlCouel5C+xtTvqFl3WlqtFtnZ2YiKiuK3CYVCREVFITMz0+gxmZmZBukBQKlU8ukLCwuhUqkM0sjlckRERPBpTp8+jevXr0MoFOKBBx6At7c3pkyZYnC3dqfGxkZoNBqDpTtQZTwhPcusoFVRUQGdTgcvLy+D7V5eXlCpVEaPUalUHaZvXXeU5sqVKwCAt99+G2+99Rb2798PV1dXPPbYY6isrDR63eTkZMjlcn7x8/Mz562arHXQdAUFLUJ6hE20Hur1egDAm2++iZkzZyIsLAzbt2+HQCDAnj17jB6TkJAAtVrNL8XFxd2SN/fWFkQaykNIjzAraHl4eEAkEqGsrMxge1lZGRQKhdFjFApFh+lb1x2l8fb2BgAMHz6c3y+VSjFw4EAUFRUZva5UKoWzs7PB0h08Wp7KU1RZ1y3nJ4QYMitoSSQShIWFIT09nd+m1+uRnp6OyMhIo8dERkYapAeAQ4cO8ekDAwOhUCgM0mg0GmRlZfFpwsLCIJVKUVBQwKdpamrC1atX4e/vb85bsLjIge4AgMMXytCs01s1L4T0CebW8KekpDCpVMp27NjB8vPz2ZIlS5iLiwtTqVSMMcbmz5/PXn/9dT798ePHmZ2dHVu/fj07f/48S0pKYmKxmOXm5vJpVq9ezVxcXNjXX3/Nzp07x6ZPn84CAwNZfX09n+bll19mvr6+7Pvvv2cXLlxgixYtYp6enqyystKkfHdX62FTs46Neud75r9yP/v54g2LnpuQvsTU76jZQYsxxjZt2sQGDBjAJBIJCw8PZydOnOD3jR8/nsXGxhqk3717NwsKCmISiYSFhISwAwcOGOzX6/Vs1apVzMvLi0mlUjZx4kRWUFBgkEar1bLXXnuNeXp6MicnJxYVFcXy8vJMznN3BS3GGFv5xVnmv3I/e+OrcxY/NyF9hanfUQFjrE+MP9FoNJDL5VCr1Rav3zpaUI7ntv8Cj35SZL0xESKhwKLnJ6QvMPU7ahOth73dmEEecJbZoaKmEaeuGu+CQQixDApaFiCxE+KJ4VxL53d5xvurEUIsg4KWhUwZwQWttDwVzfhASDeioGUhY4d4oJ/UDipNA3KKq6ydHULuWRS0LEQmFuHxYE8AQFpeqZVzQ8i9i4KWBT05kisiHsxVoY80yhLS4yhoWdD4IE/Yi0W4XlWPvOvdM6sEIX0dBS0LspeIMCG4PwDgIBURCekWFLQsbMoIbnD3d7mlVEQkpBtQ0LKwCcGekNgJcfVmHS6oqq2dHULuORS0LKyf1A7jg7gi4ne5VEQkxNIoaHWD1lZE6h1PiOVR0OoGjwd7QSwS4GJ5DS6VUxGREEuioNUN5PZijB3sAQD4LpfutgixJApa3aS1FfEgFREJsSgKWt3kieFeEAkFOF+qwdWKWmtnh5B7BgWtbuLqKMGYQdz88VQhT4jlUNDqRpNHtLYiUtcHQiyFglY3mjRcAaEAOPe7Gr/fokeMEWIJnQpaW7ZsQUBAAGQyGSIiInDy5MkO0+/ZswfBwcGQyWQYOXIkDh48aLCfMYbExER4e3vD3t4eUVFRuHjxotFzNTY2YtSoURAIBDhz5kxnst9j+jtJ8VCAGwBuckBCSNeZHbRSU1MRHx+PpKQknD59GqGhoVAqlSgvLzeaPiMjA3PnzsWiRYuQk5ODmJgYxMTEIC8vj0+zdu1abNy4EVu3bkVWVhYcHR2hVCrR0NDQ5nwrVqyAj4+Pudm2midHtrQiUu94QizD3Mf8hIeHs7i4OP5nnU7HfHx8WHJystH0s2bNYtHR0QbbIiIi2NKlSxlj3OPDFAoFW7duHb+/qqqKSaVStmvXLoPjDh48yIKDg9mvv/7KALCcnByT892djxDriEpdz/xX7mf+K/ez0qr6ux9ASB9l6nfUrDstrVaL7OxsREVF8duEQiGioqKQmZlp9JjMzEyD9ACgVCr59IWFhVCpVAZp5HI5IiIiDM5ZVlaGxYsX49///jccHBzumtfGxkZoNBqDxRq8nGUI83cFQDOaEmIJZgWtiooK6HQ6eHl5GWz38vKCSmW8zkalUnWYvnXdURrGGJ577jm88MILGD16tEl5TU5Ohlwu5xc/Pz+TjusOU0bQWERCLMUmWg83bdqE6upqJCQkmHxMQkIC1Go1vxQXF3djDjvW2vXh5NVK3KhutFo+CLkXmBW0PDw8IBKJUFZWZrC9rKwMCoXC6DEKhaLD9K3rjtIcPnwYmZmZkEqlsLOzw+DBgwEAo0ePRmxsrNHrSqVSODs7Gyzd4vx+YOs44Py37Sa5z9UBoffJwRjw/a90t0VIV5gVtCQSCcLCwpCens5v0+v1SE9PR2RkpNFjIiMjDdIDwKFDh/j0gYGBUCgUBmk0Gg2ysrL4NBs3bsTZs2dx5swZnDlzhu8ykZqair///e/mvAXL+u17YE8soDoHfLkYKMtvN+mUllZE6vpASBeZW8OfkpLCpFIp27FjB8vPz2dLlixhLi4uTKVSMcYYmz9/Pnv99df59MePH2d2dnZs/fr17Pz58ywpKYmJxWKWm5vLp1m9ejVzcXFhX3/9NTt37hybPn06CwwMZPX1xlvbCgsLrd96ePkIY3/tz1iSM2Pv+XHrjWGMNWiMJr9aUcP8V+5nAxMOsMqaRsvkgZB7SLe0HgLA7NmzsX79eiQmJmLUqFE4c+YM0tLS+Ir0oqIilJbebiUbM2YMdu7ciW3btiE0NBRffPEF9u3bhxEjRvBpVqxYgZdeeglLlizBQw89hJqaGqSlpUEmk3U1JnePohPArrmArhEYGg3EZQFOPsDNi8C3LwNG5ob3d3fEcG9n6PQMh/LLjJyUEGIKAWN94+kLGo0GcrkcarW6a/Vb108Dn04HGjXAoInA3F2AnZQLZNufBJgOiH4feOj5NoduSr+I9w/9hseG9seOheFdeDeE3HtM/Y7aROthr1H2K/DZ01zA8n8EmP0ZF7AAYMDDwBPvcK/TErjgdofWeq3jlyqoFZGQTqKgZaqKi8CnMUD9LcB3NPBsKiC5o5Nr5DIgeCqg03IV9PW3DHYP9uyHoV5OaNIxPPHPn7DjeCGadPqOr1v4H24h947T/wb+OQK4lH73tKQNClqmuHUV+OQpoLYcUIwE/vQFIHVqm04gAKZvAVwDgKoiYN9f2tRvvT8rFEM8+6Gqrglvf5sP5YZjSD9f1vYZic2NwIHXgE+mAp9Mow/4veJaJrD/FUBdDOxdCtRWWDtHNoeC1t2or3MBq7oE8BgKzN8H2Lu2n97eBXjmE0AkBQoOAhkbDXaP8JXju5fH4W8xI+DuKMGVG7VY9Mkp/On/ZSG/pGWo0a1rwL+UwC8ftxzFgK8WA5qS7niHpKdUq7g7cH0zIBABtTfabbgh7aOg1ZGacq7Sveoa4BoILPgacPS4+3E+o4Apq7nXP74DXMsw2G0nEuJPD/vjyP88hhfGD4JEJMTxSzcRvek/2P6vD6Df+ihQksMFx9mfA4r7gbqbwBd/BnTNln+fpPs1a4HdsUBNGdB/GLDwICAUAxf2A2d2Wjt3NoWCVnvqKrk6rJsXAef7gNhvAGdv048PWwiMnMW1Jn7xZ6DmRpskzjIxXp8SjPTXxuOpkZ5YLkrBwqLXIWysgsppBBr+/BMwbCrwzA5A4gQUZQKH37XYWyQ96Ie3gOITgNQZmPM513AzoWVY2ncrubtrYhIKWsY0aIDPZgLlvwL9vLiA5TLAvHMIBMDUf3JFyupS4MtFgF5nNKmfuBobtW8jzu4bAMD2ZiXG3ViBCR9fwt6c36F3HQhM38wlPr6B64lPbMfZVODkh9zrGR8C7oO414+8AvhFANpqrv5Tf5dGGQKAglZb2lpg5yyg5DRg78YVCVs/ZOaS9gNmfQqIHYDCn4Cf1rRNc/Vn4MNxwLWfAUk/sP/aDo9nNsDTxQml6ga8mnoWs7dlQjMoGghfwh2zdylQZb0B4N2m8grXAHEvUeVy9VYAMG45EPzk7X1CETBjKyB25P7+J7ZYJ482hoLWnXI+44phUjkwfy/gOaxr5/MMBqZu4F7/tPZ2K6BeD/znH1zLYE0Z4DkcWHIUghFPY1qoD9JfG4+Vk4PRT2qHX67ewuJPTqFhwjuAzwNcV4ovFnL1JPeCyitAyjxg4wPAB48AN36zdo4so/4WkPonoLme64g84Y22adwGApPf416n/7XD8auEQz3i78QYkP4OMPRJwM+Cvda/fRnI3gE4uAOx33If0N/SuH2hc4Hof7Tt9wUg77oac7adQE1jMyYN98L/PekOu4/HAw1qrl+Y0ooDxruqQQMcWwdkbeX6trWSOHF3IMOmWi9vXaXXA7tmAxd/4KoWlvwEOLgZT8sYsGsO93nwGgksPgzYSXo2v70A9YjvLIEAiHrbsgELACavud0K+MEY7gMqkgLTNgIxHxgNWADXReKjBaMhsRPih/wyvHFUAza9pRiRuRm4cMCy+ewJeh0XwDc9yHUJ0Wm5O5GF3wH+Y7k6ntR5wOG/2W49z7G1XMCyk3EjJ9oLWAD3mZu2kauOKMsFjib3XD5tEN1p9aTKK8CHjwGNaq4LxaxPAO9Qkw79/lcVXvwsG3oGLB0/EAmCf3N1IDI5sPQY16HVXL+fAvL3ccHT3rX9xZL/9QuPccOcyloebOI+BFC+BwRN4n7WNQGHEoET/8f9PGQS8PRHXP83W/Hb91y9KMD9Qxr1rGnH5X8D7J4PCITAwjRgQET35bEXMvU7SkGrp5XkAJcPA6MXmf1F3P1LMVZ8eQ4A8NbkQXj+Yhxw/RTg8yDw57Tb4yA7whhw7ThXLLty1LQLS/q1BDAXrjXVK4S7a/QO5epkhKK7n6PyCvDDKq5fEsAF28cSuIHlInHb9GdTgW//G2hu4K4xZ2fX6xd7QuUVYNtjXPF99CJg6j/MO37vC8DZXdw/oReOc405fQQFrTv0mqDVRVt/uozV310AAGyO7o+pGbO4Ct/wpcCTa9s/kDGuEeDYOq6/EAAI7YCQGVxAqr9lZKkCcJePh9iRC2Le97cEsvu5RoXWAHpnvZVABDy0iAtYHRWZAKDkDJA6H1AXcdeZ8QEwfLopvybr0NYB/+8J7i7yvoeA5w6af5faoOYaI9TFwIOxwFMb735MT2lQAxmbuM9R+GKuLlYgsNjpKWjd4V4JWgDw3sHz2HbsCoQC4IuJ1Xjw56Xcjmc+AUJiDBPr9UDBAeDYeqD0DLdNJAUenA888nLH/c/0eq4o2xrE6m5xAUSVC5Se42a9aK5ve5zQjuuf5hUCXDnCDVcBuHor5Xtci6qpam8CXzzHFSsBYGw88Phbpt3d9STGgK+WALm7Acf+XJHduZPP5yz8D9eqDAbMTQWGTrZoVs3W1MANKfvP+0B95e3t/o9w0zBZ6A6YgtYd7qWgxRjDii/OYU/275DYCXE09DB8ft3G9bZe+hNXnNI1A7/u5T5oN85zB4odgNF/5lodzend3x69Drh5iQtgqrMt63NtZrfg662GPNG5/8y6ZuDHJK7hAeCC38yP736n1pOyPgS+W8HdScZ+AwSM7dr5vn+Te7+OnsBfMk0bPmZpeh1wNoVrGFC39Av0CAKClMAv/w9oquP+QUUuA8avACSOXbocBa073EtBCwCadXq8+PlpHMovg6sU+FnxDziWneKKaA89z/Wcr7zCJZY6cx1TH/4L4OjevRljDNBcb7kTywOcFFwxwli9lblyvwC+Xsbd3bkGtIzLHHHXw0xWXQaUnr0dgOtuckVanZZrIGjzuun2a9Yy2kH5HhAZ1/W8NDVwdWM3zgPDpgGz/m3RoliHGAMKvuO65bT+w3P25Yr0oXMBkR03i8l3r3N38QAg9wOmrDXsPGsmClp3uNeCFgA0NOmw4F8ncbKwEsMdq/GtOAGihj/cvtu7AZF/AR5abFutbx1R5QIpz3JfGjt7wDcMkN8HuPhxa/l9gHwAt26nGwkY444vPdsSpM5x65ouToMd9hzXkdhSwaX0LPDRREDfBMRsBUbNtcx5O3ItA/jxbaA4i/tZ5gKMe42rwxLbt01f8B1wcAVXbQBw/RunrDF/2BsoaLVxLwYtANA0NGHOhyeQX6rB084X8L5uNQT2rsCYl7hB2/di61NdJTcI/cqRjtM5uLcEMT9uEYpaAtQ5oKGqbXqBkCvKeodyDQry+7j6P5GEu7sQSVoW8e3XwpbtYlnHUxZ11rH13CB5oRjoHwy4+nN3mX9c5H7c9bui7FduRpKLLeNa7eyBh1/k6j3v9g9PW8s1tmRs4qbdETtwxcWH48xqiOjWoLVlyxasW7cOKpUKoaGh2LRpE8LD2++MuWfPHqxatQpXr17FkCFDsGbNGjz55O3bSMYYkpKS8NFHH6GqqgqPPPIIPvjgAwwZMgQAcPXqVbz77rs4fPgwVCoVfHx88Kc//QlvvvkmJBLTfin3atACgBvVjXhmawau3qzD6P4MW/78KDxdnCDoqeKENej13PjQykKuvkX9++11VTHXQbUjQjHgNfx21w3vUK7hoIv1Mhana+Z61l/6sYNEAq7S3+UPAU3qxBVZ9ToukDA9t9brWrY3c79DfTM3V9z5/QAYVyf34AJg/Erz6z3LL3ATV177mfu5fzA30iPgEZMO77aglZqaigULFmDr1q2IiIjAhg0bsGfPHhQUFMDT07NN+oyMDDz66KNITk7G1KlTsXPnTqxZswanT5/mn8izZs0aJCcn45NPPkFgYCBWrVqF3Nxc5OfnQyaTIS0tDampqZg7dy4GDx6MvLw8LF68GPPnz8f69est+guxVcWVdZj5QQbKW+aed5SI4ONif3uRy/jXvi728JJLIbXrZS1wllRf1RLIWoNZMTdW0yuEC1D9g21nqAxjQMVv3PQ1t67eXqqucUG7qdYy1wmZAUx4C/AY3LW8nk3hpuKpa5mVNfRZYNK7d21M6LagFRERgYceegibN3MtOXq9Hn5+fnjppZfw+uuvt0k/e/Zs1NbWYv/+/fy2hx9+GKNGjcLWrVvBGIOPjw9ee+01LF++HACgVqvh5eWFHTt2YM6cOUbzsW7dOnzwwQe4cuWKSfm+14MWAFxQaRD3+WlcvmHah7i/k9QgoHnLZfBtCWzeLjJ4OEohFN7Dd2v3Asa4BgM+mBVy66Z6rugqEAFC4R9e23HFZIHw9muRhGsR9HnAcvmqq+Qq8rN3AGDAnF13raQ39TtqZ04+tFotsrOzkZCQwG8TCoWIiopCZmam0WMyMzMRHx9vsE2pVGLfvn0AgMLCQqhUKkRFRfH75XI5IiIikJmZ2W7QUqvVcHNrv8m7sbERjY23pznRaDR3fX+2LljhjPTXHkO9VodSdT1KqhpQUlWPEnU9t275+XpVPRqb9bhR3Ygb1Y04284sNxKREAq5DD4uMvjIuWDm7+6AyEHuuM+1nUpu0rMEAu4OxtEDuG+0tXNzm4MbMG0D8MCfgPPfdKlV8U5mBa2KigrodDr+waytvLy8cOHCBaPHqFQqo+lVKhW/v3Vbe2nudOnSJWzatKnDomFycjLeeeedjt/QPcpeIsLA/v0wsL/xSnjGGCprtShVN+D3W/UoVdejVN2A61VccCutakBZdQO0Oj2KKutQVFnX5hwDPRwxdogHxg72QOQgdzjJLNClgdx77htt8WBqVtDqDa5fv47JkyfjmWeeweLFi9tNl5CQYHCHp9Fo4Ofn1xNZ7PUEAgHc+0nh3k+KEb5yo2madHqUaRpQUtWAUnU9H9DySzQ4+7saVypqcaWiFp9mXoNIKMADfi4YO8QD44b0R+h9ctiJaAIR0j3MCloeHh4QiUQoKzPsz1JWVgaFQmH0GIVC0WH61nVZWRm8vb0N0owaNcrguJKSEkyYMAFjxozBtm3bOsyrVCqFVGrCAGJilFgkxH2uDkaLgZqGJmRevomfL1bg50sVKKyoxalrt3Dq2i1s+PEinGR2iBzojnFDPBDo0Q912mbUaXUtSzPqtTrUNem4tbYZtVrudUOTDj4u9hjpK8cIXzmGezvDXnIPNxaQTjEraEkkEoSFhSE9PR0xMTEAuIr49PR0LFu2zOgxkZGRSE9PxyuvvMJvO3ToECIjIwEAgYGBUCgUSE9P54OURqNBVlYWXnzxRf6Y69evY8KECQgLC8P27dshFNJ/cmtxlomhDFFAGcL9wymurMPPlyrw88UKHL9cgaq6JvyQX4Yf8jvXWfOL7N8BAEIBMMTTCSG+zhjpK8dIXzmG+zjDQdL9BYTGZh0YA6R2wnu764gN6lSXh9jYWHz44YcIDw/Hhg0bsHv3bly4cAFeXl5YsGABfH19kZzMTWSWkZGB8ePHY/Xq1YiOjkZKSgree++9Nl0eVq9ebdDl4dy5c3yXh+vXr+Oxxx6Dv78/PvnkE4hEt//7tneHd6e+0HrYG+j0DHnX1XwQu1Wnhb1EBAeJCPZiOzi0vm5ZO0jsYC/mXkvshLhaUYvc62rkXtegoqbtfPFCATCofz+M9JVjmLczHKV2sBMJIBYJYCcUQiwScq9FQoiF3NpOJIBEJIROz3CrTouquibcqtPiVl0Tqv6wrqy9va9Oq+Ov5yixg4NUxK8dJHZwlIjgIG1ZS+zQT2qHAA9HjPSVY1B/Ryoed0K3tB4CXBeGGzduIDExESqVCqNGjUJaWhpfkV5UVGRwFzRmzBjs3LkTb731Ft544w0MGTIE+/bt4wMWAKxYsQK1tbVYsmQJqqqqMHbsWKSlpUEm43r5Hjp0CJcuXcKlS5dw3333GeSnj3TotxkioQChfi4I9XNB3ITO9/dhjKG8uhG5v6uRe12NvOvcury6ERfLa3CxvAbIuW7BnBunZ0B1YzOqG5sBmPbQDZlYiGHezhjhw90dhvg6I8jLCWIKZBZBw3iITSnXNCCvRI3c3zW4dKMGjU06NOn0aNYzbq1jaNIzNDXr0axv/ZlbCwC4OEjg6ijm1g5iuDpI7njNrV0dJBAKgTqtDrWNzYZrbTPqGlvWLds1DU34TVWDX0vUqNW2fVScRCREsLcTRvjKMcJHjmBvJzhK7CAScneJ3FoIOyF3x2gnEnCLUAhRH+krR2MP70BBi/QEvZ6h8GYt8q6r8WuJBrm/q5FXokZ1Q+efDC5oKaJ6OUvhLbeHQi6Dt1zGr72cZfCW28PVQWzT9W8UtO5AQYtYC2MMRZV1yLuuQe51NX4tUeNSeQ20zdwdYrNOjyY9g65l6SyJnZALZs4y9JPagbVcm1uD/xn8zwyt336ZWATHljo6R6ndHa+5+rzW7f2kdnBxEMPFXmzRujsKWnegoEVsgV7PoGOML9bqWtY1Dc1QaRqgUjegVP2HtaYeKnUDKmqs8wxMub2YK1o7SvhitVtL8duN3ybGcB/nu3ZA7raKeEJI9xEKBRBCALEIsMftVnJPJ7Q7wgHgumiUaxpRquY6Azc2tTx6TQAIwHUo5ta3p/sSQMC/bmjSoaZRh7rGZtS01tk1NqNW24zalvq72kbudXVDEzQtxV11fRPU9U24erPtqIk/+uovY/DgAMtM3UNBi5B7gNROBD83B/i59cyY0GadHur6Jtxq6SLCdRfRorL2dveR1m4lt2q18HC0XEdvClqEELPZiYT8ULCeRh1HCCE2hYIWIcSmUNAihNgUClqEEJtCQYsQYlMoaBFCbAoFLUKITekz/bRaRyv1hQdcEGKLWr+bdxtZ2GeCVnU19/BOmieekN6turoacrnxZxcAfWjAtF6vR0lJCZyc7v7k5daHYBQXF9/zg6v7ynvtK+8TsN33yhhDdXU1fHx8OpxOvc/caQmFwjaznt6Ns7OzTf3Ru6KvvNe+8j4B23yvHd1htaKKeEKITaGgRQixKRS0jJBKpUhKSuoTz03sK++1r7xP4N5/r32mIp4Qcm+gOy1CiE2hoEUIsSkUtAghNoWCFiHEplDQIoTYFApad9iyZQsCAgIgk8kQERGBkydPWjtLFvf2229zj5T6wxIcHGztbFnEsWPHMG3aNPj4+EAgEGDfvn0G+xljSExMhLe3N+zt7REVFYWLFy9aJ7NddLf3+txzz7X5O0+ePNk6mbUgClp/kJqaivj4eCQlJeH06dMIDQ2FUqlEeXm5tbNmcSEhISgtLeWXn3/+2dpZsoja2lqEhoZiy5YtRvevXbsWGzduxNatW5GVlQVHR0colUo0NDT0cE677m7vFQAmT55s8HfetWtXD+awmzDCCw8PZ3FxcfzPOp2O+fj4sOTkZCvmyvKSkpJYaGiotbPR7QCwvXv38j/r9XqmUCjYunXr+G1VVVVMKpWyXbt2WSGHlnPne2WMsdjYWDZ9+nSr5Kc70Z1WC61Wi+zsbERFRfHbhEIhoqKikJmZacWcdY+LFy/Cx8cHAwcOxLx581BUVGTtLHW7wsJCqFQqg7+xXC5HRETEPfk3BoCjR4/C09MTQ4cOxYsvvoibN29aO0tdRkGrRUVFBXQ6Hby8vAy2e3l5QaVSWSlX3SMiIgI7duxAWloaPvjgAxQWFmLcuHH8nGP3qta/Y1/4GwNc0fDTTz9Feno61qxZg59++glTpkyBTqezdta6pM9MTUNumzJlCv/6/vvvR0REBPz9/bF7924sWrTIijkjljRnzhz+9ciRI3H//fdj0KBBOHr0KCZOnGjFnHUN3Wm18PDwgEgkQllZmcH2srIyKBQKK+WqZ7i4uCAoKAiXLl2ydla6VevfsS/+jQFg4MCB8PDwsPm/MwWtFhKJBGFhYUhPT+e36fV6pKenIzIy0oo56341NTW4fPkyvL29rZ2VbhUYGAiFQmHwN9ZoNMjKyrrn/8YA8Pvvv+PmzZs2/3em4uEfxMfHIzY2FqNHj0Z4eDg2bNiA2tpaLFy40NpZs6jly5dj2rRp8Pf3R0lJCZKSkiASiTB37lxrZ63LampqDO4kCgsLcebMGbi5uWHAgAF45ZVX8Le//Q1DhgxBYGAgVq1aBR8fH8TExFgv053U0Xt1c3PDO++8g5kzZ0KhUODy5ctYsWIFBg8eDKVSacVcW4C1my97m02bNrEBAwYwiUTCwsPD2YkTJ6ydJYubPXs28/b2ZhKJhPn6+rLZs2ezS5cuWTtbFnHkyBEGoM0SGxvLGOO6PaxatYp5eXkxqVTKJk6cyAoKCqyb6U7q6L3W1dWxSZMmsf79+zOxWMz8/f3Z4sWLmUqlsna2u4zm0yKE2BSq0yKE2BQKWoQQm0JBixBiUyhoEUJsCgUtQohNoaBFCLEpFLQIITaFghYhxKZQ0CKE2BQKWoQQm0JBixBiU/4/lqNSE6Bb9HoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:12:36.353226Z", + "iopub.status.busy": "2024-03-01T10:12:36.352847Z", + "iopub.status.idle": "2024-03-01T10:15:26.520418Z", + "shell.execute_reply": "2024-03-01T10:15:26.519620Z" + }, + "papermill": { + "duration": 170.189982, + "end_time": "2024-03-01T10:15:26.522913", + "exception": false, + "start_time": "2024-03-01T10:12:36.332931", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:15:26.563076Z", + "iopub.status.busy": "2024-03-01T10:15:26.562348Z", + "iopub.status.idle": "2024-03-01T10:15:26.584238Z", + "shell.execute_reply": "2024-03-01T10:15:26.583326Z" + }, + "papermill": { + "duration": 0.043931, + "end_time": "2024-03-01T10:15:26.586215", + "exception": false, + "start_time": "2024-03-01T10:15:26.542284", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.015449NaN0.0014922.4465780.1107591.6372920.2400340.0000014.6983030.030890.0712960.0386310.0550320.0212727.144881
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.015449 NaN 0.001492 2.446578 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.110759 1.637292 0.240034 0.000001 4.698303 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.03089 0.071296 0.038631 0.055032 0.021272 \n", + "\n", + " total_duration \n", + "realtabformer 7.144881 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:15:26.624013Z", + "iopub.status.busy": "2024-03-01T10:15:26.623759Z", + "iopub.status.idle": "2024-03-01T10:15:27.007626Z", + "shell.execute_reply": "2024-03-01T10:15:27.006640Z" + }, + "papermill": { + "duration": 0.404864, + "end_time": "2024-03-01T10:15:27.009710", + "exception": false, + "start_time": "2024-03-01T10:15:26.604846", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:15:27.049664Z", + "iopub.status.busy": "2024-03-01T10:15:27.049319Z", + "iopub.status.idle": "2024-03-01T10:18:31.321002Z", + "shell.execute_reply": "2024-03-01T10:18:31.320085Z" + }, + "papermill": { + "duration": 184.310461, + "end_time": "2024-03-01T10:18:31.339969", + "exception": false, + "start_time": "2024-03-01T10:15:27.029508", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:18:31.379114Z", + "iopub.status.busy": "2024-03-01T10:18:31.378787Z", + "iopub.status.idle": "2024-03-01T10:18:31.395719Z", + "shell.execute_reply": "2024-03-01T10:18:31.394853Z" + }, + "papermill": { + "duration": 0.038625, + "end_time": "2024-03-01T10:18:31.397591", + "exception": false, + "start_time": "2024-03-01T10:18:31.358966", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:18:31.436477Z", + "iopub.status.busy": "2024-03-01T10:18:31.436203Z", + "iopub.status.idle": "2024-03-01T10:18:31.441158Z", + "shell.execute_reply": "2024-03-01T10:18:31.440312Z" + }, + "papermill": { + "duration": 0.027315, + "end_time": "2024-03-01T10:18:31.443082", + "exception": false, + "start_time": "2024-03-01T10:18:31.415767", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.4290029579087308}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:18:31.483385Z", + "iopub.status.busy": "2024-03-01T10:18:31.482777Z", + "iopub.status.idle": "2024-03-01T10:18:31.813548Z", + "shell.execute_reply": "2024-03-01T10:18:31.812612Z" + }, + "papermill": { + "duration": 0.352957, + "end_time": "2024-03-01T10:18:31.815561", + "exception": false, + "start_time": "2024-03-01T10:18:31.462604", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAE8CAYAAACYd5eXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABD0UlEQVR4nO3dd3hUZdr48e/0TNqkkAoJCQEpooA0ERVUXhUr69oLIIu4Cssq6+9V9lpRbKir2F92bQR3LbvrgroW1opdkCaoSE1CCIEEUiZt+vn9cZiBIX0yyZTcn4u5MnPmnDN3Dsmd53nOUzSKoigIIUSY04Y6ACGE6AhJVkKIiCDJSggRESRZCSEigiQrIUREkGQlhIgIkqyEEBFBkpUQIiJIshJCRARJViJg9957LxqNJqjnLCwsRKPRsH79+g7t/+c//5kBAwag0+kYOXJkUGMR4UWSlQiqhx56iLfeeqtHPuvDDz/kf//3f5k4cSLLly/noYce6pHPFaGhD3UAIro89NBDXH755UybNq3bP+vTTz9Fq9Xy0ksvYTQau/3zRGhJyaqXaGhoCHUIQVdRUYHZbA5aolIUhaampqCcqysaGxtDHUJYkmQVhbxtST///DPXXnstycnJnH766QD8/e9/Z/To0ZjNZlJSUrj66qspLS31O/7LL7/kiiuuIDc3F5PJRE5ODrfffnu7v8gajYaGhgZWrFiBRqNBo9Ewc+ZMAEpKSrj11lsZPHgwZrOZ1NRUrrjiCoqLi1s8V2NjIzfffDOpqakkJiYyffp0qqur/T5r+fLlNDQ0+D6rsLAQAJfLxf33309BQQEmk4m8vDz++Mc/Yrfb/T4jLy+Piy66iP/+97+MGTMGs9nMX//6V9asWYNGo+Gf//wnixcvpm/fviQkJHD55ZdTW1uL3W7ntttuIz09nfj4eG688cZm5+7otZ48eTLDhw9nw4YNnHnmmcTGxvLHP/6xzevcW0k1MIpdccUVDBo0iIceeghFUXjwwQe5++67ufLKK5k9ezaVlZU888wznHnmmWzatImkpCQA/vWvf9HY2Mgtt9xCamoq69at45lnnmHfvn3861//avXz/va3vzF79mzGjRvHnDlzACgoKADg+++/55tvvuHqq6+mX79+FBcXs2zZMiZPnszPP/9MbGys37nmzZtHUlIS9957L9u3b2fZsmWUlJT4Esnf/vY3nn/+edatW8eLL74IwGmnnQbA7NmzWbFiBZdffjl/+MMfWLt2LUuWLGHbtm2sWrXK73O2b9/ONddcw80338xNN93E4MGDfe8tWbIEs9nMXXfdxa5du3jmmWcwGAxotVqqq6u59957+e677ygsLCQ/P59Fixb5ju3otQY4fPgwU6dO5eqrr+b6668nIyOjk//TvYQios4999yjAMo111zj21ZcXKzodDrlwQcf9Nt369atil6v99ve2NjY7JxLlixRNBqNUlJS0uxzjhUXF6fMmDGj2fEtnfPbb79VAOWVV17xbVu+fLkCKKNHj1YcDodv+6OPPqoAyttvv+3bNmPGDCUuLs7vnJs3b1YAZfbs2X7b77jjDgVQPv30U9+2/v37K4CyevVqv30/++wzBVCGDx/uF8M111yjaDQaZerUqX77T5gwQenfv7/vdWeu9aRJkxRA+ctf/tLs+gh/Ug2MYr/97W99z1euXInH4+HKK6/k0KFDvkdmZiaDBg3is88+8+1rNpt9zxsaGjh06BCnnXYaiqKwadOmgGI59pxOp5PDhw8zcOBAkpKS2LhxY7P958yZg8Fg8L2+5ZZb0Ov1vP/++21+jvf9BQsW+G3/wx/+AMB7773ntz0/P5/zzjuvxXNNnz7dL4bx48ejKAqzZs3y22/8+PGUlpbicrmAzl1rAJPJxI033tjm9yWkGhjV8vPzfc937tyJoigMGjSoxX2P/aXcu3cvixYt4p133vFrJwKora0NKJampiaWLFnC8uXLKSsrQzlmgtqWznl8nPHx8WRlZbXaxuVVUlKCVqtl4MCBftszMzNJSkqipKTEb/ux1+h4ubm5fq8tFgsAOTk5zbZ7PB5qa2tJTU3t1LUG6Nu3r9zN7ABJVlHs2NKMx+NBo9HwwQcfoNPpmu0bHx8PgNvt5n/+53+oqqrizjvvZMiQIcTFxVFWVsbMmTPxeDwBxfK73/2O5cuXc9tttzFhwgQsFgsajYarr7464HO2paOdVY+9Rsdr6Tq1td2bgDt6rTsSgzhKklUvUVBQgKIo5Ofnc8IJJ7S639atW9mxYwcrVqxg+vTpvu0fffRRhz6ntSTx5ptvMmPGDB5//HHfNpvNRk1NTYv779y5k7POOsv3ur6+nvLyci644II2P79///54PB527tzJ0KFDfdsPHjxITU0N/fv379D30RUdvdaic6TNqpe47LLL0Ol0LF682K8KBmqJ4PDhw8DRUsOx+yiKwlNPPdWhz4mLi2sxAel0umaf+8wzz+B2u1s8z/PPP4/T6fS9XrZsGS6Xi6lTp7b5+d5k9uSTT/ptX7p0KQAXXnhhe99Cl3X0WovOkZJVL1FQUMADDzzAwoULKS4uZtq0aSQkJFBUVMSqVauYM2cOd9xxB0OGDKGgoIA77riDsrIyEhMT+fe//92s7ao1o0eP5uOPP2bp0qVkZ2eTn5/P+PHjueiii/jb3/6GxWJh2LBhfPvtt3z88cekpqa2eB6Hw8E555zDlVdeyfbt2/m///s/Tj/9dC655JI2P3/EiBHMmDGD559/npqaGiZNmsS6detYsWIF06ZN8yutdZeOXmvROZKsepG77rqLE044gSeeeILFixcDamPxueee60sCBoOB//znP8yfP58lS5YQExPDr371K+bNm8eIESPa/YylS5cyZ84c/vSnP9HU1MSMGTMYP348Tz31FDqdjldffRWbzcbEiRP5+OOPW70T9+yzz/Lqq6+yaNEinE4n11xzDU8//XSH2qJefPFFBgwYQGFhIatWrSIzM5OFCxdyzz33dOJqdU1HrrXoHI1yfDlVCCHCkLRZCSEigiQrIUREkGQlhIgIkqyEEBFBkpUQIiJIshJCRISo72fl8XjYv38/CQkJQV/cQAjRNYqiUFdXR3Z2Nlpt22WnqE9W+/fvbzZKXggRXkpLS+nXr1+b+0R9skpISADUi5GYmBjiaIQQx7JareTk5Ph+T9sS9cnKW/VLTEyUZCVEmOpIE400sAshIoIkKyFERJBkJYSICFHfZiVEMLjdbr/JAEXHGQyGVqeC7gxJVkK0o76+nn379jWb9VN0jEajoV+/fs3mnu8sSVZCtMHtdrNv3z5iY2NJS0uTjsWdpCgKlZWV7Nu3j0GDBnWphBXSZPXFF1/w5z//mQ0bNlBeXs6qVauYNm0aoK4t96c//Yn333+fPXv2YLFYmDJlCg8//DDZ2dmhDFtEkF8OWNlSWsuYvGQGpHX+L7vT6URRFNLS0mQVmgClpaVRXFyM0+nsUrIKaQN7Q0MDI0aM4Lnnnmv2XmNjIxs3buTuu+9m48aNrFy5ku3bt8uUsKLDrDYn//3xIGU1TawtqupSNU5KVIEL1rULaclq6tSpra5WYrFYmi3/9OyzzzJu3Dj27t3bbAFKIY73S3kdHkUh3qTn6rE5knAiXES1WdXW1qLRaEhKSmp1H7vdjt1u9722Wq09EJkIR6VVjQCMzU+RRBUFIqaflc1m48477+Saa65pc9jMkiVLsFgsvocMYu6dPB6F8tomAPolS1tTd8rLy2u2TmN3iIhk5XQ6ufLKK1EUhWXLlrW578KFC6mtrfU9SktLeyhKEU5qm5w43QoGnYbEGANvby5j+ddFON3BX6pe9IywrwZ6E1VJSQmffvppu4ORTSYTJpOph6IT4epwgwOA5DgjBp2G8lobTQ431Q0O0hNjQhxd+HE4HBiNxlCH0aawLll5E9XOnTvbXL1XiOOlxZuYPDiNEf2S0Gg0pMSpv4jeJNZVDpen1YfruNJbW/seX9Jrbb/Omjx5MvPmzWPevHlYLBb69OnD3Xff7bsjmpeXx/3338/06dNJTExkzpw5AHz11VecccYZmM1mcnJymD9/Pg0NDb7zVlRUcPHFF2M2m8nPz+fVV1/tdGyBCmnJqr6+nl27dvleFxUVsXnzZlJSUsjKyuLyyy9n48aNvPvuu7jdbg4cOABASkpK2P8VEKFliTUwKjfZ9zo51khZdRM1jcEZMvPcZ7tafS+/TxzTRvX1vX7+i9043S13m+iXbOaKMUfbVV/+uogmh7vZfrf/zwmdjnHFihX85je/Yd26daxfv545c+aQm5vLTTfdBMBjjz3GokWLfCtV7969m/PPP58HHniAl19+mcrKSl/CW758OQAzZ85k//79fPbZZxgMBubPn09FRUWnYwtESJPV+vXrOeuss3yvFyxYAMCMGTO49957eeeddwAYOXKk33GfffYZkydP7qkwRRRIjFF/1OtsvWd8X05ODk888QQajYbBgwezdetWnnjiCV+yOvvss/nDH/7g23/27Nlcd9113HbbbQAMGjSIp59+mkmTJrFs2TL27t3LBx98wLp16xg7diwAL730EkOHDu2R7yekyWry5MltdtSTsVgiULsq6okz6UhPiEGn1ZBoNgBgtbmCcv65Zw1s9T3tcb0k5pxZ0Oq+x/eomDUxvyth+Tn11FP9umxMmDCBxx9/HLdbLbmNGTPGb/8ffviBLVu2+FXtFEXB4/FQVFTEjh070Ov1jB492vf+kCFD2uxKFExh38AuRGe5PQrvbtmPosCcMwcQZ9IfTVZNwSlZGfUdb+7trn27Ki4uzu91fX09N998M/Pnz2+2b25uLjt27Oip0FokyUpEnXqbC0UBvVZDrFEdi5YYoyfGoMNs1KEoSq/oJLp27Vq/1999912bg4lPOeUUfv75ZwYObLnUOGTIEFwuFxs2bPBVA7dv305NTU1Q425NWN8NFCIQ1iPtUgkxel9SSogxcMvkAq4Zl9srEhXA3r17WbBgAdu3b+f111/nmWee4fe//32r+99555188803zJs3j82bN7Nz507efvtt5s2bB8DgwYM5//zzufnmm1m7di0bNmxg9uzZPTbAW5KViDq1R6p63qpfbzV9+nSampoYN24cc+fO5fe//72vi0JLTj75ZD7//HN27NjBGWecwahRo1i0aJHfLCfLly8nOzubSZMmcdlllzFnzhzS09N74tuRaqCIPnVHGtETY3p3sjIYDDz55JMtjvooLi5u8ZixY8fy4YcftnrOzMxM3n33Xb9tN9xwQ5fi7ChJViLqNNjVZBVn8v/x/mbXIXZV1jM2L4WhWbIsW6SRaqCIOg0ONVnFH5esGhxuDtc7fNVEEVmkZCWizqicZHJTYslO8h8DGGdS74I1OoLT1yqcrVmzJtQhBJ0kKxF1clNjyU2NbbY9zqj+uNfbmw9nEeFPqoGi1/CVrOzRX7KKRpKsRFRxuj3sPFhHWU1Ts/dij5SsGloYKCzCnyQrEVXqbC7e3VLO25vLmr3n7c1uc0qyikTSZiWiiq/bgrH5j3aMQUe8SU+MQYvbo6A7fsSxCGuSrERUaTxSxfOWoo4VY9Bx05kDejokESRSDRRRxdvH6vgOoSLySbISUaXxSLcEcwslKxHZ5M+PiCq+klULbVYAn22voLSqkYkD+1AQwHLyInSkZCWiird3ekttVqDOdXW43kF9oDOGKgq4HKF5dGLm3FdeeYXU1FS/BX8Bpk2b1mMDj4NNSlYiqpySm0x+n3j6JrU8x5LZoCaxpkC7L7id8OXjgYbXNWf8AfQdWyjliiuuYP78+bzzzjtcccUVgLoyzXvvvdfmrArhTEpWIqr0T41jZE4SyXEt/1J727ICTlYRwmw2c+211/pWpQH4+9//Tm5ubsQutiIlK9GrxBwpWdkC7cWuM6glnFDQdW5+rptuuomxY8dSVlZG3759KSwsZObMmRE7U6okKxE1PB6FPYfqiTHoyLaY0bbQ6dNbDbS5AkxWGk2Hq2KhNmrUKEaMGMErr7zCueeey08//cR7770X6rACJslKRA2by81/figH4PfnDGpxH1810NH5VY4j0ezZs3nyyScpKytjypQp5OTktH9QmAppm9UXX3zBxRdfTHZ2NhqNhrfeesvvfUVRWLRoEVlZWZjNZqZMmcLOnTtDE6wIezanmoBMBm2LpSpQ7xLGm/SYjb2jufbaa69l3759vPDCC8yaNSvU4XRJSP/HGhoaGDFiBM8991yL7z/66KM8/fTT/OUvf2Ht2rXExcVx3nnnYbPZejhSEQm8A5Rj9K13CM1IjOGmMwfwq1H9eiqskLJYLPz6178mPj6eadOmhTqcLglpNXDq1KlMnTq1xfcUReHJJ5/kT3/6E5deeimg9h3JyMjgrbfe4uqrr+7JUEUE8CUrg/ReP1ZZWRnXXXcdJpMp1KF0SdiWhYuKijhw4ABTpkzxbbNYLIwfP55vv/221ePsdjtWq9XvIXoHb3eE3lLFa091dTWrVq1izZo1zJ07N9ThdFnYNrAfOHAAgIyMDL/tGRkZvvdasmTJEhYvXtytsYnw5G2zaqsaCLBq0z7qbC4uHdEXS2z0Ltc1atQoqqureeSRRxg8eHCow+mysE1WgVq4cCELFizwvbZarRF9B0R0nL2D1cDD9Q7qbC5sLjcWojdZtbY2YKQK22SVmZkJwMGDB8nKyvJtP3jwICNHjmz1OJPJFPF1cxGYgvR44kx6Ulrpve5lMujUZBXlvdijTdhW7vPz88nMzOSTTz7xbbNaraxdu5YJEyaEMDIRrjISYxiRk0ROSvOVbY5l0qs/9nZXx/taKZ0YRCz8BevahbRkVV9fz65du3yvi4qK2Lx5MykpKeTm5nLbbbfxwAMPMGjQIPLz87n77rvJzs6O+FuwIrR8ycrZfrLS6dQqpcPhwGxueXC0aJvD4QCOXstAhTRZrV+/nrPOOsv32tvWNGPGDAoLC/nf//1fGhoamDNnDjU1NZx++umsXr2amJiY1k4perG9hxvRaCA90YSpjUZ2b5uWvQNDbvR6PbGxsVRWVmIwGNBqw7YyEpY8Hg+VlZXExsai13ct3WiUKC/fWq1WLBYLtbW1JCYmhjoc0Y1e/HIPdTYX14zLJdPS+h+0Ndsr2LS3hrF5KZw+qE+753U4HBQVFeHx9I4hOsGm1WrJz8/HaGzeltiZ38+wbWAXorO8bVAxhrZLP3EmPfEmfYdXtzEajQwaNMhXnRGdYzQag1IilWQlooLL7cHhS1Ztt42MzUthbF5Kp86v1Wql+SHEpAIuooLtSKLSaI42oIvoIv+rIiocOy4wUieXE22TaqCICkdnXGj/7++hejufbDuISa9j2qi+3R2aCBJJViIq2JwdXy9QUWB/ja3VFXBEeJJkJaJCWnwMZw9J79D0MN67hXaXB0VRpNoYISRZiahgiTUwIjapQ/t6O4y6PQpOt4JRL8kqEkgDu+h1DDoN2iOlqY70YhfhQZKViAoVVhulVY002NtfaVmj0WAydH4wswgtSVYiKqwtquLNDfvYVVHfof29fbFkmpjIIW1WIip0dv71OJMel1shukfGRhdJViIqeHuwmzuYrK4cI7PHRhqpBoqo4F0Ovr1BzCJyyf+siHiKoviqgSZZhitqSTVQRDyXR8HlURufOlqy2rKvhl/K6xiUEc+o3OTuDE8EiZSsRMTzrheo02ow6jr2I11vd1FW00RNo7M7QxNBJCUrEfGMOi1nD0nH5en40Jmji0ZI14VIIclKRLwYg44ROUmdOsao887DLp1CI4VUA0WvZDxSsnJIsooYUrISEa+20YnV5iTRbMBi7tgKy4GsHShCS0pWIuJtP1jHmxv2sXbP4Q4fIyWryCPJSkS8pk4OtQE1WRl0GvQ6mR4mUkg1UES8zswS6tUn3sS8swd1V0iiG4R1ycrtdnP33XeTn5+P2WymoKCA+++/nyhfl1V00tH516X3ejQL65LVI488wrJly1ixYgUnnngi69ev58Ybb8RisTB//vxQhyfChN3ZscVNRWQL62T1zTffcOmll3LhhRcCkJeXx+uvv866detCHJkIJ4G0WQG8t6WcBruL807MxBLbsbuIInTC+k/RaaedxieffMKOHTsA+OGHH/jqq6+YOnVqq8fY7XasVqvfQ0S3zs5l5VVe20RZTZMv2YnwFtYlq7vuugur1cqQIUPQ6XS43W4efPBBrrvuulaPWbJkCYsXL+7BKEWonVbQh0aHi4SYzv04m/Ra6pAhN5EirEtW//znP3n11Vd57bXX2LhxIytWrOCxxx5jxYoVrR6zcOFCamtrfY/S0tIejFiEwkn9LIwfkNrpkpV3lRvpaxUZwrpk9f/+3//jrrvu4uqrrwbgpJNOoqSkhCVLljBjxowWjzGZTJhMpp4MU0Qoo/RijyhhXbJqbGxEq/UPUafT4fHID5dQ2ZxuSqsaqWpwdPpYSVaRJaxLVhdffDEPPvggubm5nHjiiWzatImlS5cya9asUIcmwsRBq42VG8vok2DihlP7d+pYkwy5iShhnayeeeYZ7r77bm699VYqKirIzs7m5ptvZtGiRaEOTYQJ7528ji4UcSzvkBsF6WQcCTRKlHcHt1qtWCwWamtrSUxMDHU4Isg2l9bw2S8VDMqI56KTszt1rKJ0fLI+0T068/sZ1m1WQrSnK0NtJFFFFklWIqI1BTCIWUSmsG6zEqI9dmfg6wUetNr4bs9h4ox6pgzLCHZoIsgCKlnt2bMn2HEIEZBAxwWCehdwT2UDZTVNwQ5LdIOAktXAgQM566yz+Pvf/47NZgt2TEJ02El9LZxWkEpGYkynj5WuC5EloGS1ceNGTj75ZBYsWEBmZiY333yzzIQgQmJgegLjB6TSJ77zoxa8w21kbGBkCChZjRw5kqeeeor9+/fz8ssvU15ezumnn87w4cNZunQplZWVwY5TiKDz9mB3uhXcnqjuwRMVunQ3UK/Xc9lll/Gvf/2LRx55hF27dnHHHXeQk5PD9OnTKS8vD1acQjTj8SiUVjVSWWcPaPZYb7ICcLqlKhjuupSs1q9fz6233kpWVhZLly7ljjvuYPfu3Xz00Ufs37+fSy+9NFhxCtFMk9PNmxv28eraEgLp2qzTajAcWTDCO9uoCF8BdV1YunQpy5cvZ/v27VxwwQW88sorXHDBBb5Bx/n5+RQWFpKXlxfMWIXw4+0QatLr0GoD6+Cplq48OGVwfNgLKFktW7aMWbNmMXPmTLKyslrcJz09nZdeeqlLwQnRlqYu9LHymn36gIATnehZASWrjz76iNzc3GbTtyiKQmlpKbm5uRiNxlbnnBIiGGxHqm6BDGL2kkQVOQL6k1RQUMChQ4eaba+qqiI/P7/LQQnREYHOvS4iU0Alq9buvNTX1xMT0/nOeUIEIhjJauPeakqrGjkx28LA9PhghSa6QaeS1YIFCwB1tPqiRYuIjY31ved2u1m7di0jR44MaoBCtKZDbVb2Otj7HdQfBHMK5J4KsSm+tw/V2dlT2UCWxdzd4You6lSy2rRpE6CWrLZu3YrRaPS9ZzQaGTFiBHfccUdwIxSiFXmpcZj0OjISW+m9XlsGW/8JziNDwmpKoeJnOPkqSMoBjva1kiE34a9Tyeqzzz4D4MYbb+Spp56SyexESOWkxJKTEtvym03VRxNVQgb0HQPlP0DtPvhpJYz5DZjiZchNBAmogX358uWSqET48nhg27tqokrMgpHXQ9bJMOJqiE8DRyPs+hiQklUk6XDJ6rLLLqOwsJDExEQuu+yyNvdduXJllwMToj3ltU3otBpSYo3odcf83T1wpASlN8KwS9WvADoDDLkINhRCxTbIGY9Jr5bMZIWb8NfhZGWxWHzTwFoslm4LSIiO+s8P+2mwu7nu1FzSE47chXbZoehL9XnemWBO9j8oIRPSh8LBn6F0LaY+5wBSsooEHU5Wy5cvb/G5EKGgKIqvU6hf14XSteBoUJNU31NaPjjnVDVZVW4nJmUCAC6ZdSHsBdTPqqmpCUVRfF0XSkpKWLVqFcOGDePcc88NaoBCtMTh9vimdfH1YHfZYd969fmAyaBtpf9VQoZ6N7CmlL5N25l/zhnopCd72Auogf3SSy/llVdeAaCmpoZx48bx+OOPc+mll7Js2bKgBihES7ylKr1Wg8HbXrV/k5qw4vpA2uC2T5A1EgBt5TZ0kqciQsAzhZ5xxhkAvPnmm2RmZlJSUsIrr7zC008/HdQAhWhJk+O4VW3cLtj3vfo8Zxy0t8xWn0Gg00NjFdQd6MZIRbAElKwaGxtJSEgA4MMPP+Syyy5Dq9Vy6qmnUlJSEtQAy8rKuP7660lNTcVsNnPSSSexfv36oH6GiDyNDhcAscYjLRkVP4O9HkzxkDG8/RPoTZA6CEVR+H7tl7y9uUz6WoW5gBeMeOuttygtLeW///2vr52qoqIiqP2vqqurmThxIgaDgQ8++ICff/6Zxx9/nOTk5PYPFlGt8UjJKtZbsirboH7tO6b1tqrjpQ9Do9HQVL6NPRX1vqqlCE8BNbAvWrSIa6+9lttvv51zzjmHCRPUOyoffvgho0aNClpwjzzyCDk5OX53H2VWBwGQnmhi4sA+JJr1YC1Xq3Jandr5s6NS8kGrJ9ZThdlZfaRkZei2mEXXBFSyuvzyy9m7dy/r169n9erVvu3nnHMOTzzxRNCCe+eddxgzZgxXXHEF6enpjBo1ihdeeKHNY+x2O1ar1e8hok96Qgzj8lMYkpkI+zeqG9MGgzGu4yfRGSC5P3qthmTbXulrFeYCnmIxMzOTUaNG+U3AN27cOIYMGRKUwEBdTHXZsmUMGjSI//73v9xyyy3Mnz+fFStWtHrMkiVLsFgsvkdOTk7Q4hFhyNmktlcBZLfSr6otqQXotFqSm/ZKL/Ywp1ECWBakoaGBhx9+mE8++YSKigo8x81fHawVm41GI2PGjOGbb77xbZs/fz7ff/893377bYvH2O127Ha777XVaiUnJ4fa2loZzxhFKqw20EBy1RYMez5RuyuMnd3+XcDjNdXw86pHqG1ykfw/dzAkV5aR70lWqxWLxdKh38+A2qxmz57N559/zg033EBWVpZvGE6wZWVlMWzYML9tQ4cO5d///nerx5hMJkymzi94KSLLhz8fpLLOzg2GjfQBtVQVyM+hOQm3ORWaDkBNCUiyClsBJasPPviA9957j4kTJwY7Hj8TJ05k+/btftt27NhB//79u/VzRfizOd3EOg4T464Es0kd7xcgR2IuVB1AW1sMjAtajCK4AkpWycnJpKSktL9jF91+++2cdtppPPTQQ1x55ZWsW7eO559/nueff77bP1uEL0VRaHS46duwE0OyBlILwNjKvFYdcNLwEeg1u9AaqoIYpQi2gBrY77//fhYtWkRjY2Ow4/EzduxYVq1axeuvv87w4cO5//77efLJJ7nuuuu69XNFeLO7PLjdbtIad6pTw2Sc1KXzGVPz0Gp16oR9ttogRSmCLaCS1eOPP87u3bvJyMggLy8Pg8G/b8rGjRuDEhzARRddxEUXXRS084nI1+RwY7GVEaPY0BlT1ZJVV+hN6tQx1v1QXdK5vlqixwSUrKZNmxbkMITouEanm7SGnRi0Gsg4seM91ltRXttEaU0CWbWN5NRIsgpXASWre+65J9hxCNFhTU2NpDQVYzBq1WTVRQ12Fz82JhHb5CSnuhgUJbA7i6JbBdwptKamhhdffJGFCxdSVaU2TG7cuJGysrKgBSdES9IcZeQlG0lNS4eErC6fz6TXUWfMxKlo1cHQTdVBiFIEW0Alqy1btjBlyhQsFgvFxcXcdNNNpKSksHLlSvbu3eub60qI7mCp340lyQy5I4NSAjLqtXi0eqyGNMAGtaV+awuK8BBQyWrBggXMnDmTnTt3+q3AfMEFF/DFF18ELTghmnE74fAu9Xla4H2rjmU6ssJNjeFIh9Ca0qCcVwRXQMnq+++/5+abb262vW/fvhw4IBOZiW50eDf1jTbqtHE4zOlBOaV3Oa4qfTqKoqgr44iwE1CyMplMLc5msGPHDtLS0roclBCtqtxG0aEGPqxIZl9NU1BO6V3otM6UgUtBbbOy1wfl3CJ4AkpWl1xyCffddx9OpxMAjUbD3r17ufPOO/n1r38d1ACF8DlSBXS4PBw2DyDOFFCTazM6rQaDToNba8IVk6pulNJV2AkoWT3++OPU19eTlpZGU1MTkyZNYuDAgSQkJPDggw8GO0YhVNXFKC4n9ZpYGox9gpasAH5z+gB+d/ZAzGl56gZJVmEnoP9ti8XCRx99xNdff80PP/xAfX09p5xyClOmTAl2fEIcdXgXTo9CVUx/NFoNsYaudQY9lm/hCUs/KNuo3hEUYaXTycrj8VBYWMjKlSspLi5Go9GQn59PZmYmiqJ023QxopdTFDi8G4fLQ7U5lzijHm13rPVn6ad+rT+oLuull+mGwkWnqoGKonDJJZcwe/ZsysrKOOmkkzjxxBMpKSlh5syZ/OpXv+quOEVvV18B9jocipY6U1ZQq4AAm0trWP1jOaWNBohJVJOjdX9QP0N0Taf+xwsLC/niiy/45JNPOOuss/ze+/TTT5k2bRqvvPIK06dPD2qQQlC1G4B6cz88Tj1xpuBVAQH2VTey82A9mRYzOZYcsP2ktlulyAIl4aJTJavXX3+dP/7xj80SFcDZZ5/NXXfdxauvvhq04ITwOawmq/i+gzl9UB91oYgg8nZfsDvdR6uC0sgeVjqVrLZs2cL555/f6vtTp07lhx9+6HJQQvhxNIJVHXOakjOUsXkpDM5MCOpHeDuGOtyeo8nKWgYeWUQiXHQqWVVVVZGR0foc1RkZGVRXyyBQEWTVRWobUlwfiLF0y0d4h9zYnR6I7QN6o9qvq6GiWz5PdF6nkpXb7Uavb72ZS6fT4XK5uhyUEH6OVAFJHcjew41U1tlxezq9KFObTMeWrLRaSJSqYLjpVAO7oijMnDmz1dVjjl0CS4ig8HigSl3aTUkZwDubynC6FW6cmEdSrDFoH+OtBtqc6rL0WPqqn1u7D/qNCdrniMB1KlnNmDGj3X3kTqAIqrr96kKmehNNsVk43cVoNJAQE9xl3mOOdDD1LXQqjexhp1PJavny5d0VhxAt81YBUwZgtamJJM6oRxfkDqG5KbHMOXOAL2mRkA0aLdjr1EUkuqmtTHRcwDOFCtEjvHNXpRZQZ1MHzieag9shFMCg0xJnOiYJ6o0Qf2QKGildhQVJViJ82axqz3WN5kjJ6kiyCnIVsFW+qqBM1R0OJFmJ8HWk1zoJWWCMw9qk3mkOdnsVqDePPvulgtU/lmN3eRvZvf2tpGQVDiIqWT388MNoNBpuu+22UIcieoKvy4K6LqC1G6uBGo2Gn/bXsq28DpvjSCN7Yl/1a32FOqhZhFTEJKvvv/+ev/71r5x8sqzp1iu4XVBdrD5PHQjAyJwkJg7sQ3aSuVs+0tu4bvOWrGIS1YZ1GdQcFiIiWdXX13PdddfxwgsvkJycHOpwRE+o3av2IDfFQ7w6aqJ/ahzj8lPoE98907aYvMnK29cK1P5WII3sYSAiktXcuXO58MILOzS5n91ux2q1+j1EBDqsdgQlpaDHFhyN8Q65cR0zHjDxmHGCIqSCX/kPsjfeeIONGzfy/fffd2j/JUuWsHjx4m6OSnQrRfHrsgBQ0+jgUL2DtHgTltjuuRvYcsnquEHN2oj4+x6VwvrKl5aW8vvf/55XX33Vb33CtixcuJDa2lrfo7RUpqeNOI1V6gozWh0k5wGwu7KB//ywn692Heq2j43xDbk5pmQVl6b2uXI5oKGy2z5btC+sS1YbNmygoqKCU045xbfN7XbzxRdf8Oyzz2K329Hp/CdhM5lMrY5dFBHCW6pKyvVNK1zd4AAguZtKVXBMA/uxJSutVr0rWFWktlsltD7riOheYZ2szjnnHLZu3eq37cYbb2TIkCHceeedzRKViBK+KuAg36bKerXrQJ+E7vtDNC4/hdH9k48OufHyJivrPmB0t32+aFtYJ6uEhASGDx/uty0uLo7U1NRm20WUcDYdvfOWOgAAj0fhUJ2arNK66U4g0DxJecmg5rAQ1m1WoheqKgLFo060Z1a7qVQ3OnB5FIx6LUndWA1sVWK2ekfSZlUfIiTCumTVkjVr1oQ6BNGdDu9Uvx7pCArHVAHjjd261FtNo4NNe2sw6LScPqjP0Tf0JnVQc91B9a5gTHDnfxcdIyUrET7cLr9ZQb0OWtVklZ7QsTvCgbK7PGwureGXAy2UnmTm0JCLuJKViGLVxeoYPFP80XYiYHT/ZDISTSSZgzczaEti9C3cDfSy9IOyDZKsQkiSlQgfldvUr2lD/Xqtx5v0QV96qyUmg1rRcLoVXG4Pet0xFQ/LsYOaHWrfK9GjpBoowoPbBYd2qM/Th4QkBJNe68uRfkNuQB3QbEpQG//rZFBzKEiyEj3K5fawZV8NH/50gHVFVTQ6jqyGVF2kllhMCUenZgE2lFSzrqjKNz1Md9JoNL7FTptaqwqCVAVDRKqBosfYnG7e2lRGea3Nt219SRUXDM8i78CRzr/pQ3xVQEVR2FhSTb3dRVqCqUdmCDUbtNicbpocrSSrim0yc2iISMlK9Bi7y4PT7SHGoGNsXgrpiSbsTg/vbtjFwaIf1Z0yj85XtreqkXq7C6NeS05y98xhdbxYo/r3u82SlXWfrNQcAlKyEj3GYjZw1dhc6u0uUuKMTPCk8vG2g1T/8gNFNVYS+gwl1rtIA2oVEGBYdqJ/Y3c3Ov+kTPRaje/OoJ+4dNAZ1Opq46GjC0qIHiElK9GjjHotKXHqnTSdVsO5Q9MYayyhID2e2P5HB6xvP1BHyeFGtBoNo3KSeiy+xBgDsUY92paW+vIOagZptwoBSVai25VWNbJxb/XRhRiOoTm0k4J4J2nJFkg/EYBPfznIBz+WAzAmLzmoKy93mcwcGjJSDRTdbm1RFaVVjTTa3f7DWBQF9n6rPu872td3qbLOjqKo1b8JA1J7NNYDtTa2lVuxxBo4JbeFKbQtMnNoqEiyEt2qqsFBaZVanTs557hVjSu3Q90B0OnVZHXEGYPSMOm1pHbjDAutqWlysLm0hn7J5paTVWJf9W5lU426WrMpocdj7K2kGih8nG4P64qq+O9PB9h5sA5FUbp8Tu84u/6psf5dD9xO2POZ+jxnPBjjfG9lJ5lDkqgAzC1NwHcsvUmdPRSkC0MPk2QlALWz5sqN+/h61yF+3m/l3S3lfv2hAqEoCtsP1AEwJOu4Esjuz9TSiSkeck7t0ucEk9moJqvGlvpZeUnn0JCQaqAAYF1RFftrbMQYdBSkxZEab+zy+nwHrXZqGp0YdBoG9Ik/+kb5D+qgYIDBF4TVODtvPyub04OiKC1PSWPpB2UbZaXmHibJStDocLFxr9qnacrQdAZlBKcdxlsFLEiLx6jXqg3qZRtg1yfqDrmn+lavCRfeaqBHUbA5Pb6Slh9v94W6g+osEXqZ878nSLISbN1Xi9OtkJEYw8D0eL/3bE43lTV15CjlUF0CjYfBbVcTj8GstjUZ48CYoFbpTAlgjAdTIg6XBy0wNNkDB7bC/k1H23myR8KAyT39rbZLp9VgMmixOz00Od0tJytzkjqLaVM11JRCn4HN9xFBJ8lKsLeqEYARORa/as/huibWfPoBfWs3kd3PjK6Ts3Seq9VxttaJdofm6JQvOj3kT4Z+Y3ps8dLOijXosDs9NDpcvg6szSTnqcmquliSVQ+RZCX49Sn92FfdRKblmJk4HQ0k71xFfs0W7C4PBx1JZA8coS7lbohV93E2gbNRvYXvqFe/2o989bjA40av1arr/8WmQp8TIGtE2E8L/KtT+mHUaYkxtHH/Kbm/WlKsLuq5wHo5SVYCrVZDbmrs0Q2OBtj8GtqGQ2QkJ7LGfRI7E0/ihoL8Ds2B7nF7qG+sJ9GAmqj0MerXCGExd2B2h6T+asmw4ZCaoE3x7R8jukS6LvRyzfpSuV2w9V/qL6Epnj6TbqIm6UQON7goPtzYoXPuq7Hx0ncHePuXOrU9K4ISVYcZY48OZK4uDmkovYUkq17ManPy0ldFfPZLxdGktesjsJaDIQZGXofJksHwvmrP841HZkFoz/aDat+qeFNkFtz3VTfy2S8VbNlX0/aOR5a2l2TVMyRZ9WIlhxqps7moqLOp1bvDu2H/ZrV6M+xSiE0BYGROEhqN2hBfeWSx0da4PQq7KuoBOCFIXSB6WnWDk82lNRQdamh7R2+yqilR746KbiXJqhcrrVardf1T49T+QjtWq2/0GwMpA3z7WcwGBqbHo9VoKK9tavOcRYfqsTndxJl09O1ip9JQiTOp1dYGexu92AEsOWoV12aFxqoeiKx3C+tktWTJEsaOHUtCQgLp6elMmzaN7du3hzqsqKAoCmXVauLpm2SGkm/UXzpzEuSd2Wz/0wf24cbT8zi5X1Kb5/1pv9oRdGhWYstzQkUAb/W1we5qe0ed4ejQm6o93RyVCOtk9fnnnzN37ly+++47PvroI5xOJ+eeey4NDe0Uz0W7rDYX9XYXOq2GzBgnlK1X3xg4pcXhL0mxxnbnQK+zOX1VpxOzLW3uG87ivMnK4cLjaad6512M9fCubo5KhHUL6OrVq/1eFxYWkp6ezoYNGzjzzOZ//UXH7a9RS1XpCSYMpd+odwEt/fxWQm5NhdVGotlAjMH/Lt9P+60oCvRNNrfemTICmA06tBoNHkWh0elu+0ZB6kB1+FBtqQy96WZhnayOV1tbC0BKSkqr+9jtduz2o43AVmsLS4ELX7LKjXVA+RZ144DJ7fYq/2b3IdbuqWJMXjJnDErze29sXgqJMQYSYiLqx6oZrVZDnElHnc1Fvc3VdrKKTVEfjVVQVRSyNQ97g7CuBh7L4/Fw2223MXHiRIYPH97qfkuWLMFisfgeOTk5PRhl5Ig36UmNN5Jn+0VduDMlH5Lav1aZiWov9017a6io859CRqfVMCw7kZyU2JYOjSjeqmB9e+1WcHQwtlQFu5VGCcYMaz3glltu4YMPPuCrr76iX79+re7XUskqJyeH2tpaEhPDe5hHj3M54Ntn1erLyVd2aAYERVF454f97KlswGI2cPGIbKobHeQkx7Y86DdCWW1OjDrtkVWa27lRUF0Mm19XO4qeNj9sxzyGI6vVisVi6dDvZ0SU1+fNm8e7777LF1980WaiAjCZTJhM0m7QIQe2qokqNsWvq0JbNBoN5w7L5PV1e6ltcvL370oAtXvDFWP6kdADC5H2hE4tqGrJUW9KOBrVCfk6UEIVnRfW1UBFUZg3bx6rVq3i008/JT8/P9QhRYU6mxO323N0Ary+oztVGjAbdVw5Nof8PupUxEa9lmHZiRHbY73LtDpIHaQ+r5SuNd0lrH+65s6dy2uvvcbbb79NQkICBw4cAMBisWA2R2aHw3Dw/tZyHBW7uIhykhPjIfOkTp8j3qRn2qi+ON0etBoNugjtU9Waw/V2tuyrxaTXctrAPu0fkD4UDv4Eldtg4DlSFewGYV2yWrZsGbW1tUyePJmsrCzf4x//+EeoQ4tYLreHg1Y7fWq2qLNiZo7o0u12g04bdYkK1OXjN5fW8MuROeTblZyvVgXt9TI3ezcJ65JVhLT9R5SDdXYM9mrSnWWYDCnQb3T7B/VC3mli6mxqx9B2e+Pr9Op8XQd+hMpfpN2qG4R1yUoE3/6aJrLqfiQhxoCmzyB1el7RTLxJj06rdgyt60j3BYC0oerXim3g8XRfcL2UJKte5sDhGvo07FQ7bvYbE+pwwpZGoyHxSOdWa5OzYwel5KtT6zgaZAbRbiDJqhdRFAXnvk3oFCdxKVnqbJeiVYlHqoK1HU1WWh1kHOmwXP5DN0XVe0my6kUO19tIqdmKTqshvuBUuWPVDm+7ldXWwWQFkHmy+vXwLrWEJYJGklUvYq4rYlCih4yUJLSZrQ9ZEipvyare1sE2K4CEDEjIBI9b7coggias7waK4Io7uIm4JLO6uKguOnqad6eT+lo4qa8Fk76Tf9OzToa6A+qsq/3GSgk2SKRk1VvUV0DNXtBooe8poY4mIsQYdMQYdB1a0cdPxnC1z1XjYZmUL4gkWfUS9uLvOFRvpympAGIid2K8iKA3QdZI9XnpupCGEk0kWfUG9nqsxZvZWVHP6prcUEcTUTaUVPHWpjL2VXdsGTIf73jL6mK1VCu6TJJVb1C2gdpGO3WmDPr0lcHgnVFea6PoUAMHrW2v6tOMOQnSBqvPS74Oely9kSSraOd2ouzfRG2jk/KEk+ifEhfqiCKKd3rmqgZH5w/uP1H9WvGLlK6CQJJVtDuwlaaGOuo0cVjj8slOigl1RBElNU4d5H2ovpMlK1BXbPZOc1z0RRCj6p0kWUUzjxv2fkdtk1qq6psSh14n/+WdkZGoJqvKOjsudwDj/fLOUNuuDu2EmtIgR9e7yE9uNDuwFWy1HHIYqIgbQm4UzI3e0yxmA2ajDrdH4VB9AFXBuD5He7Xv/FAGOHeBJKto5XFDyTe4PQrbDCfi0eopSIsPdVQRR6PR+BbJaG816lYNmKR2Z6ivgP2bghhd7yLJKlrt3wS2WnQx8Vx0/lQuOCmL5Aheyy+UMhJjMBm0BDy7mjFOTVgARWugqTpIkfUuMtwmGjkaofhL9Xne6cSazQyWWaADdkr/JMbnp7Q/AV9bskbBwZ/VWUS3vQsjrwOtlBU6Q65WNCr+Epw2lLi0oz2pRcBMel3XEhWoiWnoReownNp9UPJVcILrRSRZRZvqYijbCMCPsWN5Y/0+9lTWhzamKKEoCjanO/ATmJNh0Hnq8+Kv1RlFRYdJsoomThv88j4AnqyRbKhNpLzWRoO9C79gAoCymiZe/rqYf23Y17W1ATKHQ85Y9fm2d6G6JDgB9gKSrKKFxwPb3gFbLcRY2JkwhupGJzEGHSdkyl3ArkqNM9Jgd3Gozk5lXQAdRI814GxIHQgeF2z9pySsDpJkFQ0UBXZ/Aod3g06Pa+ilfFusVv1OyU3CpI+eZd1DJcag83X92Fxa07WTabVw4q/UVbDdLtjyT7VPnGiTJKtI5/HAzo9g33r19eAL+f5wDNWNTuJMOkbmJoU0vGhySv8kALaV11EdyFjBY+n0MPzX0GeQWsLa9q5ahXfauh5olIqIZPXcc8+Rl5dHTEwM48ePZ906mSMIUKt8P7yuLgOv0cDg89mjzWVt0WEAJp2QLqWqIMqymBmQFodHUfjw5wN4PF1c19KbsPKODHgu/wG+f0HtI+eRdsbjhX2y+sc//sGCBQu455572LhxIyNGjOC8886joqIXj2K3WWHPGlj3vDr7p84Awy6F7FEcsNpQFBje18IJGdJWFWyTT0jHqNeyv8bGhz8f6PoJNRrIPxNGXafeLbTXw/bV8N0y2PM5NBxSq/kCjRLmyx6PHz+esWPH8uyzzwLg8XjIycnhd7/7HXfddVe7x1utViwWC7W1tSQmJnZ3uMHl8YCrSe3k2VSlDteoLgZrGSgKLo+Hxti+HO57Nvm56qR6iqLwY5mVYdmJUbmsezjYXVnPe1vKGd0/mYkD+wD4Bjl3aaC42wXlm6HkG/+VcUwJ6grPcWkQ20d9bYwFQ5xaOotgnfn9DOvv1OFwsGHDBhYuXOjbptVqmTJlCt9++22Lx9jtduz2o3drrFZrxz6s5JuWVyPxy+VKULftPFh3ZJkn5chmxfsPDR7G9Iv37b+7sp6aRodvyEetKYt9sSdS5c7DuNvBb/sp6LQaNBoNJ/WTaYu7U0FaPJed0tc3ZhBga1kta7ZXYjJoMWi1aLUa9FqNb62Ii0/O9g13+qG0hi37avzOGWvU8+vR/dSFZ7NGwqEdcPBH9Y+TvU7t/d4Sre7IQw8a79dj/0gd87y1ueS7a0ELYwKMuCpopwvrZHXo0CHcbjcZGRl+2zMyMvjll19aPGbJkiUsXry48x9mr1eL3D3IZa/H0coCmhoA5chEeYYY6o1myjyx1BvTqTHnYNcnAGAyaMlIiKHJ6SbeFNb/nVGlX7L/DBYHatWGcbvTg53mMyu4j/kj1ehwN5vBISHmmGN0esgYpj7cTrXHe125ugBFwyG11OVsVNu1vA+62ODfHWKCG1PU/XQvXLiQBQsW+F5brVZycnLaP7DvaPXOTEt/ldr9SxXYtj71NhJdajlKo1FH+GuO7K5oNJCSDHozaLXkNTnJdnnU/QCtRkOsSScN6GHi/OGZnDUknQa7C7dHwa0ouNxHE1RizNGlz4ZlJdIvWR2s6c1hrQ4T1BnUZelTjpuOWlHAZQO345ik5VIfxw65bql032x7N9EGN72EdbLq06cPOp2OgwcP+m0/ePAgmZmZLR5jMpkwmUyd/7C4VPXRgzozY4t3dWARnjQajW/prvZYYg1YYrv4/6nRgMGsPnqJsL4baDQaGT16NJ988olvm8fj4ZNPPmHChAkhjEwI0dPCumQFsGDBAmbMmMGYMWMYN24cTz75JA0NDdx4442hDk0I0YPCPlldddVVVFZWsmjRIg4cOMDIkSNZvXp1s0Z3IUR0C/t+Vl0V0f2shIhynfn9DOs2KyGE8JJkJYSICJKshBARIewb2LvK2yTX4WE3Qoge4/297EjTedQnq7q6OoCO9WIXQoREXV0dFkvbY1qj/m6gx+Nh//79JCQkoOmuAZsd4B32U1paKncljyPXpnXRfm0URaGuro7s7Gy07SxNFvUlK61WS79+/UIdhk9iYmJU/tAFg1yb1kXztWmvROUlDexCiIggyUoIEREkWfUQk8nEPffcE9iMEFFOrk3r5NocFfUN7EKI6CAlKyFERJBkJYSICJKshBARQZKVECIiSLIKos6sHP3CCy9wxhlnkJycTHJyMlOmTInqlaYDXVX7jTfeQKPRMG3atO4NMIQ6e21qamqYO3cuWVlZmEwmTjjhBN5///0eijaEFBEUb7zxhmI0GpWXX35Z+emnn5SbbrpJSUpKUg4ePNji/tdee63y3HPPKZs2bVK2bdumzJw5U7FYLMq+fft6OPLu19lr41VUVKT07dtXOeOMM5RLL720Z4LtYZ29Nna7XRkzZoxywQUXKF999ZVSVFSkrFmzRtm8eXMPR97zJFkFybhx45S5c+f6XrvdbiU7O1tZsmRJh453uVxKQkKCsmLFiu4KMWQCuTYul0s57bTTlBdffFGZMWNG1Carzl6bZcuWKQMGDFAcDkdPhRg2pBoYBN6Vo6dMmeLb1t7K0cdrbGzE6XSSkpLSXWGGRKDX5r777iM9PZ3f/OY3PRFmSARybd555x0mTJjA3LlzycjIYPjw4Tz00EO43e6eCjtkon4gc08IZOXo4915551kZ2f7/eBGg0CuzVdffcVLL73E5s2beyDC0Ank2uzZs4dPP/2U6667jvfff59du3Zx66234nQ6ueeee3oi7JCRZBUGHn74Yd544w3WrFlDTExMqMMJqbq6Om644QZeeOEF+vTpE+pwwo7H4yE9PZ3nn38enU7H6NGjKSsr489//rMkK9G+QFaO9nrsscd4+OGH+fjjjzn55JO7M8yQ6Oy12b17N8XFxVx88cW+bR6PBwC9Xs/27dspKCjo3qB7SCA/N1lZWRgMBnS6oys/Dx06lAMHDuBwODAajd0acyhJm1UQBLpy9KOPPsr999/P6tWrGTNmTE+E2uM6e22GDBnC1q1b2bx5s+9xySWXcNZZZ7F58+aomvE1kJ+biRMnsmvXLl8CB9ixYwdZWVlRnagA6boQLG+88YZiMpmUwsJC5eeff1bmzJmjJCUlKQcOHFAURVFuuOEG5a677vLt//DDDytGo1F58803lfLyct+jrq4uVN9Ct+nstTleNN8N7Oy12bt3r5KQkKDMmzdP2b59u/Luu+8q6enpygMPPBCqb6HHSLIKomeeeUbJzc1VjEajMm7cOOW7777zvTdp0iRlxowZvtf9+/dXgGaPe+65p+cD7wGduTbHi+ZkpSidvzbffPONMn78eMVkMikDBgxQHnzwQcXlcvVw1D1PpogRQkQEabMSQkQESVZCiIggyUoIEREkWQkhIoIkKyFERJBkJYSICJKshBARQZKVECIiSLISUaWwsJCkpCTf63vvvZeRI0f6Xs+cOTOqp0iOZpKsREBmzpyJRqPht7/9bbP35s6di0ajYebMmX77BztJ5OXl8eSTT/ptu+qqq9ixY0erxzz11FMUFhb6Xk+ePJnbbrstqHGJ7iHJSgQsJyeHN954g6amJt82m83Ga6+9Rm5ubkhiMpvNpKent/q+xWLxK3mJyCHJSgTslFNOIScnh5UrV/q2rVy5ktzcXEaNGtWlc7dU4pk2bZqvtDZ58mRKSkq4/fbb0Wg0aDQaoHk18HjHlvBmzpzJ559/zlNPPeU7R1FREQMHDuSxxx7zO27z5s1oNBp27drVpe9LBE6SleiSWbNmsXz5ct/rl19+mRtvvLHbP3flypX069eP++67j/LycsrLyzt9jqeeeooJEyZw0003+c6Rm5vb7HsCWL58OWeeeSYDBw4M1rcgOkmSleiS66+/nq+++oqSkhJKSkr4+uuvuf7667v9c1NSUtDpdCQkJJCZmdnujKwtsVgsGI1GYmNjfefQ6XTMnDmT7du3+9bvczqdvPbaa8yaNSvY34boBJnWWHRJWloaF154IYWFhSiKwoUXXhjxc6dnZ2dz4YUX8vLLLzNu3Dj+85//YLfbueKKK0IdWq8mJSvRZbNmzaKwsJAVK1YErfSh1Wo5fqo1p9MZlHN3xOzZs303D5YvX85VV11FbGxsj32+aE6Sleiy888/H4fDgdPp5LzzzgvKOdPS0vzaodxuNz/++KPfPkajscvr5bV2jgsuuIC4uDiWLVvG6tWrpQoYBqQaKLpMp9Oxbds23/PW1NbWNlsLMDU1tcVFIM4++2wWLFjAe++9R0FBAUuXLqWmpsZvn7y8PL744guuvvpqTCZTQNXPvLw81q5dS3FxMfHx8aSkpKDVan1tVwsXLmTQoEFtLvwheoaUrERQJCYmkpiY2OY+a9asYdSoUX6PxYsXt7jvrFmzmDFjBtOnT2fSpEkMGDCAs846y2+f++67j+LiYgoKCkhLSwso7jvuuAOdTsewYcNIS0tj7969vvd+85vf4HA4euTupmifzMEuRCu+/PJLzjnnHEpLS5utmix6niQrIY5jt9uprKxkxowZZGZm8uqrr4Y6JIFUA4Vo5vXXX6d///7U1NTw6KOPhjoccYSUrIQQEUFKVkKIiCDJSggRESRZCSEigiQrIUREkGQlhIgIkqyEEBFBkpUQIiJIshJCRIT/D5ZSDsfGxkT5AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:18:31.857102Z", + "iopub.status.busy": "2024-03-01T10:18:31.856683Z", + "iopub.status.idle": "2024-03-01T10:18:32.222650Z", + "shell.execute_reply": "2024-03-01T10:18:32.221779Z" + }, + "papermill": { + "duration": 0.389174, + "end_time": "2024-03-01T10:18:32.224694", + "exception": false, + "start_time": "2024-03-01T10:18:31.835520", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEmCAYAAAA6OrZqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAd0lEQVR4nO3deXxTZdo//s/J2qRN0n1vaVlEkH0VcAFkVFAEnRH3oeA6gvNgf8wIMyMMOArOqI886gsfUVl+ijyOAuO44IICgiNbKYJCoaUbpSttkqbZc+7vH6dJG7omzd7r/SKvpidnuRrSq/d9n3vhGGMMhBAS4kTBDoAQQnqDkhUhJCxQsiKEhAVKVoSQsEDJihASFihZEULCAiUrQkhYoGRFCAkLkmAH4G88z+PSpUtQqVTgOC7Y4RBC2mGMobm5Genp6RCJui87RXyyunTpErKysoIdBiGkG5WVlcjMzOx2n4hPViqVCoDwZqjV6iBHQwhpT6/XIysry/V72p2IT1bOqp9araZkRUiI6k0TDTWwE0LCAiUrQkhYoGRFCAkLEd9mRfqOMQa73Q6HwxHsUEgYkkqlEIvFfT4PJSvSLavViurqahiNxmCHQsIUx3HIzMxETExMn85DyYp0ied5lJaWQiwWIz09HTKZjDrWEo8wxlBfX4+LFy9iyJAhfSphBTVZHThwAP/4xz9w/PhxVFdXY9euXZg/fz4AwGaz4S9/+Qs+//xzXLhwARqNBrNmzcL69euRnp4ezLD7DavVCp7nkZWVBaVSGexwvGKyOWCyOhAtE0Mu7XtVhHguKSkJZWVlsNlsfUpWQW1gb2lpwejRo/HGG290eM1oNKKgoADPPvssCgoKsHPnThQVFeGOO+4IQqT9W0/DIEKVg+ehN9lgc/AwWO2g5QaCw1el8aCWrGbPno3Zs2d3+ppGo8HXX3/ttu3111/HpEmTUFFRgezs7ECESMKY2cYDAEQch3glVWHDXVi1Wel0OnAch9jY2C73sVgssFgsru/1en0AIiOhyGoXklW0XEyJKgKETfnebDbjmWeewX333dftsJl169ZBo9G4HjSIuX9ijMHmEJKVVOybj3lZWRk4jkNhYWFAznPo0CGMHDkSUqnU1Zbbn4VFsrLZbFiwYAEYY9i4cWO3+65cuRI6nc71qKysDFCUJJQ4eAYGgAMgFnHQGq1oMFjA+7Ddat++feA4Dlqt1mfnbC8/Px9jxoxBaWkptmzZ4pdrhJOQrwY6E1V5eTm+/fbbHgcjy+VyyOXyAEVHQpWdF5KSWMSBA2AwmSGVyuDgGUTi8KgSlpSU4Iknnuhx6pTuWK1WyGQyH0bVNcYYHA4HJBL/pJWQLlk5E9X58+fxzTffICEhIdghkVZWO9/lw95a/erNvrZe7uupW2bdhNXP5GP1yj8gKSkJ9915B8788jNumzMHMTExSElJwUMPPYSGhgbXMXv27MF1112H2NhYJCQk4Pbbb0dJSUmn5y8rK8OMGTMAAHFxceA4Dnl5eR6d5+zZs5g6dSqioqIwYsQI7N+/33VujuNw+fJlLF68GBzHuUpW+/fvx6RJkyCXy5GWloYVK1bAbre7zjl9+nQsXboUy5YtQ2JiIm655RZXCfDLL7/E2LFjoVAoMHPmTNTV1eGLL77AsGHDoFarcf/997t1/uV5HuvWrUNubi4UCgVGjx6Njz76yPW687xffPEFxo8fD7lcjoMHD3r8f9VbQS1ZGQwGFBcXu74vLS1FYWEh4uPjkZaWht/85jcoKCjAp59+CofDgZqaGgBAfHx8wP5akM698V1xl6/lJkZj/tgM1/dvHSiBzdF59SszToG7J7S1K757qBQma8dhPU//6iqP4uM44P33/n/87ne/w6FDh1BV04DfzJ2NvEWL8T8bXoXJZMIzzzyDBQsW4NtvvwUgdKXJz8/HqFGjYDAYsGrVKtx5550oLCzs0H0jKysLH3/8MX7961+jqKgIarUaCoXCo/P84Q9/wKuvvorhw4fjlVdewdy5c1FaWoqsrCxUV1dj6NChWLt2Le655x5oNBpUVVVhzpw5yMvLw7Zt23D27Fk8+uijiIqKwl//+lfXebdu3er6uQGguroaAPDXv/4Vr7/+OpRKJRYsWIAFCxZALpdj+/btMBgMuPPOO/Haa6/hmWeeASC0/7733nt48803MWTIEBw4cAAPPvggkpKScOONN7qut2LFCrz00ksYOHAg4uLiPPp/8ggLou+++44B6PBYuHAhKy0t7fQ1AOy7777r9TV0Oh0DwHQ6nf9+kAhlMpnYL7/8wkwmU4fXXvmqqMvHroKLbvu+tvdcl/t+eLTCbd+N+4o73c9TN954Ixs7dqzr+2dXr2HTZ85iWqPVta2yspIBYEVFnZ+/vr6eAWCnTp1ijDHXZ/LEiROMsbbPb1NTU7exdHWe9evXu/ax2WwsMzOTvfjii65tGo2Gbd682fX9n/70JzZ06FDG87xr2xtvvMFiYmKYw+Ho9OduH+c333zj2rZu3ToGgJWUlLi2Pf744+yWW25hjDFmNpuZUqlkP/zwg9u5Hn74YXbfffe5nXf37t3d/vzdfY48+f0Maslq+vTp3XbU6+41ElxLZgzu8jXRFU1Cj90wqMt9r+xRsHhabl/CcuEZw9ix48AYA8dxOH3qJxz6fj8ykjr+5S8pKcFVV12F8+fPY9WqVTh8+DAaGhrA80L1s6KiAiNGjOj1tXt7nilTprieSyQSTJgwAWfOnOnyvGfOnMGUKVPcumFMmzYNBoMBFy9edPU9HD9+fKfHjxo1yvU8JSUFSqUSAwcOdNt25MgRAEBxcTGMRiN+9atfuZ3DarVi7NixbtsmTJjQZcy+FPIN7CQ0ySS9b+70175dYYzBzjOI5VHgGSDmgJYWA26+dQ5WrX0B8THuTQhpaWkAgLlz52LAgAHYtGkT0tPTwfM8RowYAavV6tH1fXUeb0VHR3e6XSqVup5zHOf2vXObM7EaDAYAwGeffYaMjAy3/a68gdXV9XyNkhWJOA6+rUTuLOWNGzcOH330MQbkDkCyWtmhk+jly5dRVFSETZs24frrrweAHhuLne2m7afO8eQ8P/74I2644QYAgN1ux/Hjx7F06dIurzds2DB8/PHHrtIiIPTFUqlUfbpj2Jnhw4dDLpejoqLCrX0qmEL6biAh3nD2peI4zvVL/dTSpdBpm7D0kTwcO3YMJSUl+PLLL7Fo0SI4HA7ExcUhISEBb731FoqLi/Htt98iPz+/2+sMGDAAHMfh008/RX19PQwGg0fneeONN7Br1y6cPXsWS5YsQVNTExYvXtzl9Z588klUVlbiqaeewtmzZ/Gvf/0Lq1evRn5+vs/Hb6pUKixfvhxPP/00tm7dipKSEhQUFOC1117D1q1bfXqt3qJkRSKOs2TVvuyUnp6OQ4cOweFw4Oabb8bIkSOxbNkyxMbGQiQSQSQSYceOHTh+/DhGjBiBp59+Gv/4xz+6vU5GRgbWrFmDFStWICUlBUuXLvXoPOvXr8f69esxevRoHDx4EJ988gkSExO7vd7nn3+OI0eOYPTo0XjiiSfw8MMP4y9/+YvH71FvPPfcc3j22Wexbt06DBs2DLfeeis+++wz5Ob6pl3RUxyL8FZsvV4PjUYDnU5Hq9t4yGw2o7S0FLm5uYiKigp2OL1mMNvRYrVDIRVDrZD2fADxq+4+R578flKbFYk4zmqg6Ip2KYPZBrOdR7RMAoWM5rYKN1QNJBHHlaxEV24XqoiOyK5MRCwqWZGIo5CJIROLILtitgXnnUGep2QVjihZkYgjl4g7/WSLWrOVL2deIIFD1UDSbzjbsChZhSdKViSi8IzBbHN0OlODM1k5PJ/EgYQASlYkovA8g85kg9bUcWiLs80qwnvrRCxKViSidNVtwblNxHEQizhKWGGIGthJRHHe6Os0WYk4JKloFtlwRSUrElGcQ23umP0rLFu2LLjBtDN9+vSQiiccUbIiEcU1iNkH5wrUlC6kdyhZkYjCM+D3v3sUB78/gA0bNrhmXigpKcHDDz+MATnCfOJXXTUUGzZscDs2Ly8P8+fPx/PPP4/09HQMHToUAPDDDz9gzJgxiIqKwoQJE7B79+4OS2mdPn0as2fP7nR+97y8POzfv98tnrKyskC9JRGD2qyIZxgDHLbgXFss7Ti16BV4xvC39S+h/EIxRo0cibVr1wIQFnXIzMzElve2I1odh9MnjuCpJ3+HtLQ0LFiwwHX83r17oVarXauB6/V6zJ07F3PmzMH27dtRXl7eoTqn1Woxc+ZMPPLII/jv//7vDvO7b9iwAefOncOIESNc8SQlJfnwjekfKFkRzzhswPcvB+fa1/9/gKT7hUKUMjEyUhIgl8mhVCqRmprqem3NmjXQm2ww2RwYPnQwCo4ewYcffuiWrKKjo/H222+7JtZ78803wXEcNm3ahKioKAwfPhxVVVV49NFHXce8/vrrGDt2LF544QXXtnfffRdZWVk4d+4crrrqKshksg7xEM9QsiIRRS4RZlPorAD2xhtv4O133kFFRSUsZhOsVivGjBnjts/IkSPdVk4qKirCqFGj3KY2mTRpktsxJ0+exHfffYeYmJgO13TO7076jpIV8YxYKpRwgnVtL+3YsQPLly/HCy/+HSPHTkRCrBpvvr4Bhw8fdtvPm/nEDQYD5s6dixdffLHDa8753UnfUbIinuG4HqtiwcIYg8XOQ8QJ86O3nxv90KFDmDp1Kp544knozTbIJKIuFzBtb+jQoXjvvfdgsVhcCyUcPXrUbZ9x48bh448/Rk5OTperEV8ZD/Ec3Q0kEYNngM5kQ5PRhgEDBuDw4cMoKytDQ0MDhgwZgmPHjmHv11+ipPg8nl+zukPS6cz9998Pnufx2GOP4cyZM/jyyy/x0ksvAYBrfvclS5agsbER9913H44ePdphfncAyMnJcYvHuYoM6b2gJqsDBw5g7ty5SE9PB8dx2L17t9vrjDGsWrUKaWlpUCgUmDVrFs6fPx+cYEnIY66FIoTVjsViMYYPH46kpCTccsstuOuuu/DgA/djzk03oKmpEU8++WSP51Sr1fj3v/+NwsJCjBkzBn/+85+xatUqAHC1Y/U0vzsALF++3C2eiooKP70LEazHZVD96PPPP2d//vOf2c6dOxkAtmvXLrfX169fzzQaDdu9ezc7efIku+OOO1hubm6nK7t2hVZk9l53K+mGIovNwWp0JlavN/v1Ou+99x6TSqXMaDT69TqRIiJWZJ49ezZmz57d6WuMMbz66qv4y1/+gnnz5gEAtm3bhpSUFOzevRv33ntvIEMlYYC1K1n50rZt2zBw4EBkZGTg5MmTrj5UCoXCtxci3QrZNqvS0lLU1NRg1qxZrm0ajQaTJ0/Gf/7zny6Ps1gs0Ov1bg/SP7TNve7bbFVTU4MHH3wQw4YNw9NPP427774bb731lk+vQXoWsncDa2pqAAApKSlu21NSUlyvdWbdunVYs2aNX2Mjock140IP+zUZrXDwDLEKKSTinv9e//GPf8Qf//jHvgdI+iRkS1beWrlyJXQ6netRWVkZ7JBIgLB2KzF3x+5gcPAMNKNVeAnZkpVzWEJtba1bx7ra2toOvY7bk8vlrv4wpH+RS8UQiThIeqgGijihFEZzsYeXkC1Z5ebmIjU1FXv37nVt0+v1OHz4MKZMmRLEyPofFia/1FKxCEqZBDJJ9wuYOkteYfJjhT1ffX6CWrIyGAwoLi52fV9aWorCwkLEx8cjOzsby5Ytw9/+9jcMGTIEubm5ePbZZ5Geno758+cHL+h+RCoVhrcYjcaIuvPlWj+QslVAOOcFE4v7tgp2UJPVsWPHMGPGDNf3+fn5AICFCxdiy5Yt+OMf/4iWlhY89thj0Gq1uO6667Bnzx63QaXEf8RiMWJjY1FXVwcAUCqVPbYHBZPVLvQWl4hFnU5r7GSz2mCz8zDDDhEfsi0hEYHnedTX10OpVHY5FKm3OBYuZXwv6fV6aDQa6HQ6qNXqYIcTdhhjqKmpgVarDXYoPWo228EYQ7RcAnE37VbOpbpkEhGipH37a096JhKJkJub6zabhZMnv5/0Z4V0i+M4pKWlITk5GTZbkCbd66X3D5fDZudx17gMqBVdD7Y+VaVF0aVmDEmJwbDsuABG2D/JZDLXsKO+oGRFekUsFve5zcGf7A4eeisHQAx1THS3JaaJg1IxcRBNghduQvZuICGeMLeuwMxxgFxCH+tIRP+rJCKYbULjepRUHNI3AYj3qBpIIoIrWfWiVNVgsGDvmVrIJWLMH5vh79CIj1CyIhHBmawUsp7b1RgDLmnNUPZiXxI6KFmRiJAUE4WZVyf3qitClFQofVnsPBhjVG0ME5SsSETQKKUYrYzt1b7OFXAcPIPNwSCTULIKB9TATvodqZhz9XC32GkRh3BByYpEhDq9GZWNRrRY7D3uy3Ec5O2qgiQ8ULIiEeFwaSM+On4RxXWGXu3v7IvlbJgnoY/arEhEaN/Pqjei5RLYHYymiQkjlKxIRHD2YFf0MlktmJDlz3CIH1A1kEQEs9VZsqKPdKSi/1kS9hhjrmqgnKZ8iVhUDSRhz84z2FuXtultyeqni1qcrRamiRlL08SEBSpZkbBnai1ViUUcZL1YWgsADBY7qrQmaI2hPUcXaUMlKxL2ZGIRZl6dDDvf+6Ezzq4L1Ck0fFCyImEvSirG6KxYj46RtU4kSJ1CwwdVA0m/JGstWVkpWYUNKlmRsKcz2qA326BWSKFRSHt1TFs1kJJVuKCSFQl7RbXN+Oj4RRy+cLnXx1DJKvxQsiJhz+ThUBtASFZSMQeJmKaHCRdUDSRhz5NZQp0SY+RYOnOIv0IifhDSJSuHw4Fnn30Wubm5UCgUGDRoEJ577jlE+LqsxENt869T7/VIFtIlqxdffBEbN27E1q1bcc011+DYsWNYtGgRNBoNfv/73wc7PBIiLDah3YnGBUa2kE5WP/zwA+bNm4fbbrsNAJCTk4MPPvgAR44cCXJkJJR402YFAJ/9VI0Wix23XJMKjbJ3dxFJ8IT0n6KpU6di7969OHfuHADg5MmTOHjwIGbPnt3lMRaLBXq93u1BIpunc1k5VetMqNKaXMmOhLaQLlmtWLECer0eV199NcRiMRwOB55//nk88MADXR6zbt06rFmzJoBRkmCbOigRRqsdqijPPs5yiQjNoCE34SKkS1Yffvgh3n//fWzfvh0FBQXYunUrXnrpJWzdurXLY1auXAmdTud6VFZWBjBiEgwjMzWYPDDB45KVc5Ub6msVHkK6ZPWHP/wBK1aswL333gsAGDlyJMrLy7Fu3TosXLiw02PkcjnkcnkgwyRhSka92MNKSJesjEYjRCL3EMViMXiePlxEYLY5UNloRGOL1eNjKVmFl5AuWc2dOxfPP/88srOzcc011+DEiRN45ZVXsHjx4mCHRkJErd6MnQVVSFTJ8dC1Azw6Vk5DbsJKSCer1157Dc8++yyefPJJ1NXVIT09HY8//jhWrVoV7NBIiHDeyevtQhHtOYfcMFAn43DAsQjvDq7X66HRaKDT6aBWq4MdDvGxwkotvjtbhyEpMbh9VLpHxzLW+8n6iH948vsZ0m1WhPSkL0NtKFGFF0pWJKyZvBjETMJTSLdZEdITi8379QJr9Wb8eOEyomUSzBqe4uvQiI95VbK6cOGCr+MgxCvejgsEhLuAF+pbUKU1+Tos4gdeJavBgwdjxowZeO+992A2m30dEyG9NjJDg6mDEpCijvL4WOq6EF68SlYFBQUYNWoU8vPzkZqaiscff5xmQiBBMThZhckDE5AY4/moBedwGxobGB68SlZjxozBhg0bcOnSJbz77ruorq7GddddhxEjRuCVV15BfX29r+MkxOecPdhtDgYHH9E9eCJCn+4GSiQS3HXXXfjnP/+JF198EcXFxVi+fDmysrLw29/+FtXV1b6Kk5AOeJ6hstGI+maLV7PHOpMVANgcVBUMdX1KVseOHcOTTz6JtLQ0vPLKK1i+fDlKSkrw9ddf49KlS5g3b56v4iSkA5PNgY+OX8T7h8vhTddmsYiDtHXBCOdsoyR0edV14ZVXXsHmzZtRVFSEOXPmYNu2bZgzZ45r0HFubi62bNmCnJwcX8ZKiBtnh1C5RAyRyLsOnkLpioeNBseHPK+S1caNG7F48WLk5eUhLS2t032Sk5Pxzjvv9Ck4Qrpj6kMfK6dHrhvodaIjgeVVsvr666+RnZ3dYfoWxhgqKyuRnZ0NmUzW5ZxThPiCubXq5s0gZidKVOHDqz9JgwYNQkNDQ4ftjY2NyM3N7XNQhPSGt3Ovk/DkVcmqqzsvBoMBUVGed84jxBu+SFYFFU2obDTimnQNBifH+Co04gceJav8/HwAwmj1VatWQalUul5zOBw4fPgwxowZ49MACelKr9qsLM1AxY+AoRZQxAPZ1wLKeNfLDc0WXKhvQZpG4e9wSR95lKxOnDgBQChZnTp1CjKZzPWaTCbD6NGjsXz5ct9GSEgXchKiIZeIkaLuove6rgo49SFgax0Spq0E6n4BRt0DxGYBaOtrRUNuQp9Hyeq7774DACxatAgbNmygyexIUGXFK5EVr+z8RVNTW6JSpQAZE4Dqk4DuIvDzTmDCw4A8hobchBGvGtg3b95MiYqELp4HznwqJCp1GjDmQSBtFDD6XiAmCbAageJvAFDJKpz0umR11113YcuWLVCr1bjrrru63Xfnzp19DoyQnlTrTBCLOMQrZZCI2/3drWktQUlkwPB5wlcAEEuBq28Hjm8B6s4AWZMhlwglM1rhJvT1OllpNBrXNLAajcZvARHSW/8+eQktFgceuDYbyarWu9B2C1D6vfA85wZAEed+kCoVSB4G1P4CVB6GPPEmAFSyCge9TlabN2/u9DkhwcAYc3UKdeu6UHkYsLYISSpjXOcHZ10rJKv6IkTFTwEA2GnWhZDnVT8rk8kExpir60J5eTl27dqF4cOH4+abb/ZpgIR0xurgXdO6uHqw2y3AxWPC84HTAVEX/a9UKcLdQG0lMkxF+P1N10NMPdlDnlcN7PPmzcO2bdsAAFqtFpMmTcLLL7+MefPmYePGjT4NkJDOOEtVEhEHqbO96tIJIWFFJwJJQ7s/QdoYAICo/gzElKfCgtczhV5//fUAgI8++gipqakoLy/Htm3b8D//8z8+DbCqqgoPPvggEhISoFAoMHLkSBw7dsyn1yDhx2S9YlUbhx24eFR4njUJ6GmZrcQhgFgCGBuB5ho/Rkp8xatkZTQaoVKpAABfffUV7rrrLohEIlx77bUoLy/3WXBNTU2YNm0apFIpvvjiC/zyyy94+eWXERcX1/PBJKIZrXYAgFLW2pJR9wtgMQDyGCBlRM8nkMiBhCFgjOHo4e/xr8Iq6msV4rxqsxo8eDB2796NO++8E19++SWefvppAEBdXZ1P+1+9+OKLyMrKcmvQp4HSBACMrSUrpbNkVXVc+Joxoeu2qislDwdXdwam6jO4wEbDbONdnURJ6PGqZLVq1SosX74cOTk5mDx5MqZMEe6ofPXVVxg7dqzPgvvkk08wYcIE3H333UhOTsbYsWOxadMmn52fhK9ktRzTBifi6jQVoK8WqnIisdD5s7ficwGRBEreAIWtiUpWIc6rktVvfvMbXHfddaiursbo0aNd22+66SbceeedPgvuwoUL2LhxI/Lz8/GnP/0JR48exe9///tu58qyWCywWCyu7/V6vc/iIaEjWRXV1rfqbGu/qqShgCy69ycRS4G4AZCU1yPOXAGr3YNERwLO6xWZU1NTkZqa6rZt0qRJfQ6oPZ7nMWHCBLzwwgsAgLFjx+L06dN48803u0xW69atw5o1a3waBwlhNpPQXgUA6V30q+pOwiCIRQWIM1VQL/YQ51U1sKWlBc8++yymTp2KwYMHY+DAgW4PX0lLS8Pw4cPdtg0bNgwVFRVdHrNy5UrodDrXo7Ky0mfxkNBRpzejrtkM26VTwp3A6ERAk+n5ieIHQSzioLLUwmZq8X2gxGe8Klk98sgj2L9/Px566CGkpaW5huH42rRp01BUVOS27dy5cxgwYECXx8jlcsjlni94ScLLV7/Uor7ZgoekBUgEhFKVN59DRSwcigTAVANoy4HsFF+HSnzEq2T1xRdf4LPPPsO0adN8HY+bp59+GlOnTsULL7yABQsW4MiRI3jrrbfw1ltv+fW6JPSZbQ4orZcR5agHFHJhvJ+XrOpsoLEGIl0ZAN82ZRDf8SpZxcXFIT4+vucd+2jixInYtWsXVq5cibVr1yI3NxevvvoqHnjgAb9fm4QuxhiMVgcyWs5DGscBCYMAWRfzWvXCyBGjIeGKIZI2+jBK4mteJavnnnsOq1atwtatW92mNvaH22+/Hbfffrtfr0HCi8XOw+FwIMl4HpJEBZAysk/nkyXkCN0eTE2AWQdE0awiocirZPXyyy+jpKQEKSkpyMnJgVQqdXu9oKDAJ8ER0hmT1QGNuQpRzAyxLEEoWfWFRC5MHaO/BDSVe9ZXiwSMV8lq/vz5Pg6DkN4z2hxIajkPqYgDUq7pfY/1LlTrTKjUqpCmMyJLS8kqVHmVrFavXu3rOAjpNZPJiHhTGaQykZCs+qjFYsdpYyyUJhuymsoAxry7s0j8yut1t7VaLd5++22sXLkSjY1Cw2RBQQGqqqp8FhwhnUmyViEnToaEpGRAldbn88klYjTLUmFjImEwtKnJB1ESX/OqZPXTTz9h1qxZ0Gg0KCsrw6OPPor4+Hjs3LkTFRUVrrmuCPEHjaEEmlgFkD3GJyUgmUQEXiSBXpoEwAzoKt3WFiShwauSVX5+PvLy8nD+/Hm3FZjnzJmDAwcO+Cw4Qjpw2IDLxcLzJO/7VrUnb13hRitt7RCqpVEPocirZHX06FE8/vjjHbZnZGSgpoYmMiN+dLkEBqMZzaJoWBXJPjmlczmuRkkyGGPCyjgk5HiVrORyeaezGZw7dw5JSUl9DoqQLtWfQWlDC76qi8NFrcknp3TOYdUsT4GdQWizshh8cm7iO14lqzvuuANr166FzWYDAHAch4qKCjzzzDP49a9/7dMACXFprQJa7TwuKwYiWu71pCFuxCIOUjEHh0gOe1SCsJFKVyHHq2T18ssvw2AwICkpCSaTCTfeeCMGDx4MlUqF559/3tcxEiJoKgOz22DglGiRJfosWQHAw9cNxFMzB0ORlCNsoGQVcrz639ZoNPj6669x6NAhnDx5EgaDAePGjcOsWbN8HR8hbS4Xw8YzNEYNACfioJT6bgpi18ITmkygqkC4I0hCisfJiud5bNmyBTt37kRZWRk4jkNubi5SU1PBGPPbdDGkn2MMuFwCq51HkyIb0TIJRP5Y6885J5ahVljWS0LTDYUKj6qBjDHccccdeOSRR1BVVYWRI0fimmuuQXl5OfLy8nw6pTEhbgx1gKUZViZCszzNp1VAACis1GLP6WpUGqVAlFpIjvpLPr0G6RuP/se3bNmCAwcOYO/evZgxY4bba99++y3mz5+Pbdu24be//a1PgyQEjSUAAIMiE7xNgmi5b1ehudhkxPlaA1I1CmRpsgDzz0K7VTytphQqPCpZffDBB/jTn/7UIVEBwMyZM7FixQq8//77PguOEJfLQrKKyRiK64Yk4upU3y35BrR1X7DYHG1VQWpkDykeJauffvoJt956a5evz549GydPnuxzUIS4sRoBvTDmND5rGCbmxGNoqsqnl3B2DLU6+LZkpa8CeFpEIlR4lKwaGxuRktL1HNUpKSloaqJBoMTHmkqFNqToRL9NjOcccmOx8YAyEZDIhH5dLXV+uR7xnEfJyuFwQCLpuplLLBbDbrf3OShC3LRWAZEwGBWXjahvtsDBM59eQt6+ZCUSAWqqCoYajxrYGWPIy8vrcvWY9ouLEuITPA80XgAAsPiB+OREFWwOhkXTchCrlPnsMs5qoNnWuiqzJkO4ru4ikDnBZ9ch3vMoWXW1sGh7dCeQ+FTzJWEhU4kcJmUabI4ycBygipL2fKwHolo7mLoWOqVG9pDjUbLavHmzv+IgpHPOKmD8QOjNQiKJlkkg9nGH0Ox4JR67YaAraUGVDnAiwNJMi0iECK9nCiUkIJxzVyUMQrNZGDivVvi2QygASMUiRMvbJUGJDIhpnYKGSlchgZIVCV1mvdBzneNaS1atycrHVcAuuaqCNFV3KKBkRUJXa691qNIAWTT0JuFOs6/bqwDh5tF3Z+uw53Q1LHZnI7uzvxWVrEJBWCWr9evXg+M4LFu2LNihkEBwdVkQ1gXU+7EayHEcfr6kw5nqZpitrY3s6gzhq6FOGNRMgipsktXRo0fxv//7vxg1itZ06xccdqCpTHieMBgAMCYrFtMGJyI9VuGXSzob183OklWUWmhYp0HNISEskpXBYMADDzyATZs2IS4uLtjhkEDQVQg9yOUxQIwwamJAQjQm5cYjMcY/07bIncnK2dcKEPpbAdTIHgLCIlktWbIEt912W68m97NYLNDr9W4PEoYuCx1BET8oYAuORjmH3NjbjQdUtxsnSILK95V/H9uxYwcKCgpw9OjRXu2/bt06rFmzxs9REb9izK3LAgBojVY0GKxIipFDo/TP3cDOS1ZXDGoWhcXf94gU0u98ZWUl/uu//gvvv/++2/qE3Vm5ciV0Op3rUVlJ09OGHWOjsMKMSAzE5QAASupb8O+Tl3CwuMFvl41yDblpV7KKThL6XNmtQEu9365NehbSJavjx4+jrq4O48aNc21zOBw4cOAAXn/9dVgsFojF7pOwyeXyLscukjDhLFXFZrumFW5qsQIA4vxUqgLaNbC3L1mJRMJdwcZSod1K1fWsI8S/QjpZ3XTTTTh16pTbtkWLFuHqq6/GM8880yFRkQjhqgIOcW2qNwhdBxJV/vtDNCk3HuMHxLUNuXFyJiv9RQDj/XZ90r2QTlYqlQojRoxw2xYdHY2EhIQO20mEsJna7rwlDAQA8DxDQ7OQrJL8dCcQQMck5USDmkNCSLdZkX6osRRgvDDRnkLoptJktMLOM8gkIsT6sRrYJXW6cEfSrBceJChCumTVmX379gU7BOJPl88LX1s7ggLtqoAxMr8u9aY1WnGiQgupWITrhiS2vSCRC4Oam2uFu4JRvp3/nfQOlaxI6HDY3WYFdarVC8kqWdW7O8Lesth5FFZqcbamk9ITzRwadGFXsiIRrKlMGIMnj2lrJwIwfkAcUtRyxCp8NzNoZ6IkndwNdNJkAlXHKVkFESUrEjrqzwhfk4a59VqPkUt8vvRWZ+RSoaJhczDYHTwk4nYVD037Qc1Woe8VCSiqBpLQ4LADDeeE58lXByUEuUTkypFuQ24AYUCzXCU0/jfToOZgoGRFAsru4PHTRS2++rkGR0obYbS2robUVCqUWOSqtqlZABwvb8KR0kbX9DD+xHGca7FTU1dVQYCqgkFC1UASMGabA7tPVKFaZ3ZtO1beiDkj0pBT09r5N/lqVxWQMYaC8iYYLHYkqeQBmSFUIRXBbHPAZO0iWdWdoZlDg4RKViRgLHYeNgePKKkYE3PikayWw2Lj8enxYtSWnhZ2Sm2br6yi0QiDxQ6ZRISsOP/MYXUlpUz4+91tyUp/kVZqDgIqWZGA0SikuGdiNgwWO+KjZZjCJ+CbM7VoOnsSpVo9VInDoHQu0gChCggAw9PV7o3dfnTryFRIRJzrzqCb6GRALBWqq8aGtgUlSEBQyYoElEwiQny0cCdNLOJw87AkTJSVY1ByDJQD2gasF9U0o/yyESKOw9is2IDFp46SQimTQNTZUl/OQc0AtVsFASUr4neVjUYUVDS1LcTQDtdwHoNibEiK0wDJ1wAAvj1biy9OVwMAJuTE+XTl5T6jmUODhqqBxO8OlzaistEIo8XhPoyFMaDiP8LzjPGuvkv1zRYwJlT/pgxMCGisNTozzlTroVFKMS67kym0NTRzaLBQsiJ+1dhiRWWjUJ0blXXFqsb1RUBzDSCWCMmq1fVDkiCXiJDgxxkWuqI1WVFYqUVmnKLzZKXOEO5WmrTCas1yVcBj7K+oGkhcbA4eR0ob8eXPNThf2wzGWJ/P6RxnNyBB6d71wGEDLnwnPM+aDMiiXS+lxyqCkqgAQNHZBHztSeTC7KEAdWEIMEpWBIDQWXNnwUUcKm7AL5f0+PSnarf+UN5gjKGophkAcHXaFSWQku+E0ok8Bsi6tk/X8SWFTEhWxs76WTlR59CgoGogAQAcKW3EJa0ZUVIxBiVFIyFG1uf1+Wr1FmiNNkjFHAYmxrS9UH1SGBQMAEPnhNQ4O2c/K7ONB2Os8ylpNJlAVQGt1BxglKwIjFY7CiqEPk2zhiVjSIpv2mGcVcBBSTGQSURCg3rVcaB4r7BD9rWu1WtChbMayDMGs413lbTcOLsvNNcKs0RIaM7/QKBkRXDqog42B0OKOgqDk2PcXjPbHKjXNiOLVQNN5YDxMuCwCIlHqhDammTRgEwlVOnkKkAWA8jVsNp5iAAMi+OBmlPApRNt7TzpY4CB0wP9o/ZILOIgl4pgsfEw2RydJytFrDCLqakJ0FYCiYM77kN8jpIVQUWjEQAwOkvjVu253GzCvm+/QIbuBNIzFRB7OEvnzSIxZopsEJ3j2qZ8EUuA3OlA5oSALV7qKaVUDIuNh9Fqd3Vg7SAuR0hWTWWUrAKEkhXBr8dl4mKTCamadjNxWlsQd34XcrU/wWLnUWuNRfrg0cJS7lKlsI/NBNiMwi18q0H4amn9ytsB3gGJSCSs/6dMABKvAtJGh/y0wHeOy4RMLEKUtJv7T3EDhJJiU2ngAuvnKFkRiEQcshOUbRusLUDhdohaGpASp8Y+x0icV4/EQ4NyezUHOu/gYTAaoJZCSFSSKOFrmNAoejG7Q+wAoWTY0iAkaHlMz8eQPqGuC/1ch75UDjtw6p/CL6E8Bok3Pgpt7DW43GJH2WVjr855UWvGOz/W4F9nm4X2rDBKVL0mU7YNZG4qC2oo/QUlq35Mb7bhnYOl+O5sXVvSKv4a0FcD0ihgzAOQa1IwIkPoeV7QOgtCT4pqhb5VMfLwLLhfbDLiu7N1+OmitvsdW5e2p2QVGJSs+rHyBiOazXbUNZuF6t3lEuBSoVC9GT4PUMYDAMZkxYLjhIb4+tbFRrvi4BmK6wwAgKt81AUi0JpabCis1KK0oaX7HZ3JSlsu3B0lfhXSyWrdunWYOHEiVCoVkpOTMX/+fBQVFQU7rIhR2SRU6wYkRAv9hc7tEV7InADED3Ttp1FIMTg5BiKOQ7XO1O05SxsMMNsciJaLkdHHTqXBEi0Xqq0tlm56sQOAJkuo4pr1gLExAJH1byGdrPbv348lS5bgxx9/xNdffw2bzYabb74ZLS09/MUjPWKMoapJSDwZsQqg/Afhl04RC+Tc0GH/6wYnYtF1ORiVGdvteX++JHQEHZam7nxOqDDgrL62WOzd7yiWtg29abzg56hISDcq7Nmzx+37LVu2IDk5GcePH8cNN3T8hSK9pzfbYbDYIRZxSI2yAVXHhBcGz+p0+Etv5pRqNttcVadr0jU97B26op3JymoHz7Puk27CYKGz7OViIGtigCLsn0K6ZHUlnU4HAIiPjw9yJOHvklYoVSWr5JBW/iDcBdRkuq2E3JU6vbnTWQl+vqQHY0BGnKLrzpRhQCEVQ8RxYAwwdjX7gpPz/dJVClVp4jchXbJqj+d5LFu2DNOmTcOIESO63M9iscBiafvQ6PWdLAVOXMkqW2kFqn8SNg6c3mOv8h9KGnD4QiMm5MTh+iFJbq9NzImHOkoKVVTYfKw6JRJxiJaL0Wy2w2C2d39XUxkvPIyNQGNp0NY87A/CpmS1ZMkSnD59Gjt27Oh2v3Xr1kGj0bgeWVlZAYowvMTIJUiIkSHHfFZYuDM+F4jt+b1KVQu93E9UaFHX7D6FjFjEYXi6Glnxys4ODSvOqqChp3YroG0w9uViP0ZEOOaLGdb8bOnSpfjXv/6FAwcOIDc3t9t9OytZZWVlQafTQa0O7WEeAWe3Av95Xai+jFrQqxkQGGP45OQlXKhvgUYhxdzR6WgyWpEVp+x80G+Y0pttkIlFras093CjoKkMKPxA6Cg69fchO+YxFOn1emg0ml79foZ0eZ0xhqeeegq7du3Cvn37ekxUACCXyyGX05QdvVJzSkhUyni3rgrd4TgONw9PxQdHKqAz2fDej+UAhO4Nd0/IhCoAC5EGgkcLqmqyhJsSVqMwIV8vSqjEcyFdDVyyZAnee+89bN++HSqVCjU1NaipqYHJ1H1fH9K9ZrMNDgffNgFexniPSgMKmRgLJmYhN1GYilgmEWF4ujpse6z3mUgMJAwRntdTP0B/CelqYFfF782bNyMvL69X5/CkmNlf/N/RCljrinE7vkecOgaYstTrCeRsDh4ijoM4TPtUdeWywYKfLuogl4gwdXBizwc0nAdOfSQMaJ6ylKqCvRRR1UDiW3YHj1q9BUO0P0GRJAZSR/dppktpgFZKDjSTzYHCSi00CmnvklVcrlAVtBioKugnkflJI12qbbZAamlCsq0KcqkYyBzf80H9kHOamGaz0DG0R2KJMF8XANSf9WNk/Rclq37mktaEtObTUEVJwSUOEabnJR3EyCUQizjwjKG5N90XACBpmPC17gzA8/4Lrp+iZNXP1FzWIrHlvNBxM3NCsMMJWRzHQd3auVVvsvXuoPhcYWodawvNIOoHlKz6EcYYbBdPQMxsiI5PE2a7JF1St1YFdb1NViIxkNI6uqL6pJ+i6r8oWfUjlw1mxGtPQSziEDPoWrpj1QNnu5Xe3MtkBQCpo4Svl4uFEhbxGUpW/YiiuRRD1DxS4mMhSu16fCUROEtWBnMv26wAQJUCqFIB3gHU/uynyPqnkO66QHwruvYEomMVwuKi4sjoae5PIzM0GJmhgVzi4d/0tFFAc40w62rmRCrB+giVrPoLQx2grQA4EZAxLtjRhIUoqRhRUnGvVvRxkzJC6HNlvEyT8vkQJat+wlL2IxoMFphiBwFR4TsxXliQyIG0McLzyiNBDSWSULLqDywG6MsKcb7OgD3a7GBHE1aOlzdi94kqXGzq3TJkLs7xlk1lQqmW9Bklq/6g6jh0Rgua5SlIzOh55grSplpnRmlDC2r1Hs4CqogFkoYKz8sP+Tyu/oiSVaRz2MAunYDOaEO1aiQGxEcHO6Kw4pyeubHF6vnBA6YJX+vOUunKByhZRbqaUzC1NKOZi4Y+OhfpsVHBjiisJEQLg7wbDF7Mrx6T3DbNcekBH0bVP1GyimS8A6j4ETqTUKrKiI+GJEJnSfCXFLWQrOqbLbA7vBjvl3O90HbVcB7QVvo4uv6FPrmRrOYUYNahwSpFXfTVyI6AudEDTaOQQiETw8EzNBi8qApGJ7b1aj//FQ1w7gNKVpGKdwDlP8DBM5yRXgNeJMGgpJhgRxV2OI5zLZLR02rUXRp4o9CdwVAHXDrhw+j6F0pWkerSCcCsgzgqBrffOhtzRqYhLozX8gumFHUU5FIRvJ4KUhYtJCwAKN0HmJp8FFn/QsNtIpHVCJR9LzzPuQ5KhQJDFcENKZyNGxCLybnx3a/M3JO0sUDtL8Isomc+BcY8AIiorOAJerciUdn3gM0MFp3U1pOaeE0uEfctUQFCYhp2uzAMR3cRKD/om+D6EUpWkaapDKgqAACcVk7EjmMXcaHeENyYIgRjDOaelpPvjiIOGHKL8LzskDCjKOk1SlaRxGYGzn4OAODTxuC4To1qnRktlj78ghEAQJXWhHcPleGfxy/2bSGT1BFA1kTh+ZlPgaZy3wTYD1CyihQ8D5z5BDDrgCgNzqsmoMloQ5RUjKtS6S5gXyVEy9BisaOh2YL6Zi86iLY3cCaQMBjg7cCpDylh9RIlq0jAGFCyF7hcAoglsA+bh/+UCVW/cdmxkEsiZ1n3YImSil1dPwortX07mUgEXHOnsAq2ww789KHQJ450i5JVuON54PzXwMVjwvdDb8PRy1FoMtoQLRdjTHZsUMOLJOMGxAIAzlQ3o8mbsYLtiSXAiF8DiUOEEtaZT4UqvM3c90AjVFgkqzfeeAM5OTmIiorC5MmTceQIzREEQKjynfxAWAae44Cht+KCKBuHSy8DAG68KplKVT6UplFgYFI0eMbw1S81vVtPsDvOhJXTOuC5+iRwdJPQR46ndsYrhXyy+r//+z/k5+dj9erVKCgowOjRo3HLLbegrq4fj2I364EL+4Ajbwmzf4qlwPB5QPpY1OjNYAwYkaHBVSnUVuVr069KhkwiwiWtGV/9UtP3E3IckHsDMPYB4W6hxQAU7QF+3Ahc2A+0NAjVfAKOhfga7ZMnT8bEiRPx+uuvAwB4nkdWVhaeeuoprFixosfj9Xo9NBoNdDod1Gq1v8P1LZ4H7Cahk6epURiu0VQG6KsAxmDneRiVGbicMRO52cKkeowxnK7SY3i6GuK+9g0inSqpN+Czn6oxfkAcprUuLe8c5NyngeIOO1BdCJT/4L4yjlwlLEcfnQQoE4XvZUpAGi2UzsKYJ7+fIf2TWq1WHD9+HCtXrnRtE4lEmDVrFv7zn/90eozFYoHF0na3Rq/X9+5i5T90vhqJWy5nPt12vra5dZkn1rqZOf+BA48JmTGu/UvqDdAara4hHzp5Gi4qr0GjIweyEiueyGQQizhwHIeRmTRtsT8NSorBXeMyXGMGAeBUlQ77iuohl4ogFYkgEnGQiDjXWhFzR6W7hjudrNTip4tat3MqZRL8enymsPBs2hig4RxQe1r442RpFnq/d0Ykbn1IAM75tf0fqXbPu5pL3l8LWshUwOh7fHa6kE5WDQ0NcDgcSElJcduekpKCs2fPdnrMunXrsGbNGs8vZjEIRe4AslsMsHaxgCYHAKx1ojxpFAwyBap4JQyyZGgVWbBIVAAAuVSEFFUUTDYHYuQh/d8ZUTLj3GewqNEJDeMWGw8LOs6s4Gj3R8podXSYwUEV1e4YsQRIGS48HDahx3tztbAARUuDUOqyGYV2LecDfWzw94co38YUcZ/ulStXIj8/3/W9Xq9HVlZWzwdmjBfuzHT2V6nHv1TebUs0mKG2C+UojhNG+HOtuzOOA+LjAIkCEImQY7Ih3c4L+wEQcRyUcjE1oIeIW0ekYsbVyWix2OHgGRyMwe5oS1DqqLalz4anqZEZJwzWdOawLocJiqXCsvTxV0xHzRhgNwMOa7ukZRce7Ydcd1a677DdT0S+TS8hnawSExMhFotRW1vrtr22thapqamdHiOXyyGXyz2/WHSC8AggT2Zsca4OTEITx3Gupbt6olFKoVH28f+T4wCpQnj0EyF9N1Amk2H8+PHYu3evaxvP89i7dy+mTJkSxMgIIYEW0iUrAMjPz8fChQsxYcIETJo0Ca+++ipaWlqwaNGiYIdGCAmgkE9W99xzD+rr67Fq1SrU1NRgzJgx2LNnT4dGd0JIZAv5flZ9Fdb9rAiJcJ78foZ0mxUhhDhRsiKEhAVKVoSQsBDyDex95WyS6/WwG0JIwDh/L3vTdB7xyaq5uRkAeteLnRASFM3NzdBouh/TGvF3A3mex6VLl6BSqcD5a8BmLziH/VRWVtJdySvQe9O1SH9vGGNobm5Geno6RD0sTRbxJSuRSITMzMxgh+GiVqsj8kPnC/TedC2S35ueSlRO1MBOCAkLlKwIIWGBklWAyOVyrF692rsZISIcvTddo/emTcQ3sBNCIgOVrAghYYGSFSEkLFCyIoSEBUpWhJCwQMnKhzxZOXrTpk24/vrrERcXh7i4OMyaNSuiV5r2dlXtHTt2gOM4zJ8/378BBpGn741Wq8WSJUuQlpYGuVyOq666Cp9//nmAog0iRnxix44dTCaTsXfffZf9/PPP7NFHH2WxsbGstra20/3vv/9+9sYbb7ATJ06wM2fOsLy8PKbRaNjFixcDHLn/efreOJWWlrKMjAx2/fXXs3nz5gUm2ADz9L2xWCxswoQJbM6cOezgwYOstLSU7du3jxUWFgY48sCjZOUjkyZNYkuWLHF973A4WHp6Olu3bl2vjrfb7UylUrGtW7f6K8Sg8ea9sdvtbOrUqeztt99mCxcujNhk5el7s3HjRjZw4EBmtVoDFWLIoGqgDzhXjp41a5ZrW08rR1/JaDTCZrMhPj7eX2EGhbfvzdq1a5GcnIyHH344EGEGhTfvzSeffIIpU6ZgyZIlSElJwYgRI/DCCy/A4XAEKuygifiBzIHgzcrRV3rmmWeQnp7u9sGNBN68NwcPHsQ777yDwsLCAEQYPN68NxcuXMC3336LBx54AJ9//jmKi4vx5JNPwmazYfXq1YEIO2goWYWA9evXY8eOHdi3bx+ioqKCHU5QNTc346GHHsKmTZuQmJgY7HBCDs/zSE5OxltvvQWxWIzx48ejqqoK//jHPyhZkZ55s3K000svvYT169fjm2++wahRo/wZZlB4+t6UlJSgrKwMc+fOdW3jeR4AIJFIUFRUhEGDBvk36ADx5nOTlpYGqVQKsbht5edhw4ahpqYGVqsVMpnMrzEHE7VZ+YC3K0f//e9/x3PPPYc9e/ZgwoQJgQg14Dx9b66++mqcOnUKhYWFrscdd9yBGTNmoLCwMKJmfPXmczNt2jQUFxe7EjgAnDt3DmlpaRGdqABQ1wVf2bFjB5PL5WzLli3sl19+YY899hiLjY1lNTU1jDHGHnroIbZixQrX/uvXr2cymYx99NFHrLq62vVobm4O1o/gN56+N1eK5LuBnr43FRUVTKVSsaVLl7KioiL26aefsuTkZPa3v/0tWD9CwFCy8qHXXnuNZWdnM5lMxiZNmsR+/PFH12s33ngjW7hwoev7AQMGMAAdHqtXrw584AHgyXtzpUhOVox5/t788MMPbPLkyUwul7OBAwey559/ntnt9gBHHXg0RQwhJCxQmxUhJCxQsiKEhAVKVoSQsEDJihASFihZEULCAiUrQkhYoGRFCAkLlKxIRNmyZQtiY2Nd3//1r3/FmDFjXN/n5eVF9KyjkYySFfFKXl4eOI7DE0880eG1JUuWgOM45OXlue3v6ySRk5ODV1991W3bPffcg3PnznV5zIYNG7BlyxbX99OnT8eyZct8GhfxD0pWxGtZWVnYsWMHTCaTa5vZbMb27duRnZ0dlJgUCgWSk5O7fF2j0biVvEj4oGRFvDZu3DhkZWVh586drm07d+5EdnY2xo4d26dzd1bimT9/vqu0Nn36dJSXl+Ppp58Gx3HgOA5Ax2rgldqX8PLy8rB//35s2LDBdY7S0lIMHjwYL730kttxhYWF4DgOxcXFffq5iPcoWZE+Wbx4MTZv3uz6/t1338WiRYv8ft2dO3ciMzMTa9euRXV1Naqrqz0+x4YNGzBlyhQ8+uijrnNkZ2d3+JkAYPPmzbjhhhswePBgX/0IxEOUrEifPPjggzh48CDKy8tRXl6OQ4cO4cEHH/T7dePj4yEWi6FSqZCamtrjJIed0Wg0kMlkUCqVrnOIxWLk5eWhqKjItSSWzWbD9u3bsXjxYl//GMQDNFMo6ZOkpCTcdttt2LJlCxhjuO2228J+OuL09HTcdtttePfddzFp0iT8+9//hsViwd133x3s0Po1KlmRPlu8eDG2bNmCrVu3+qz0IRKJcOXsRTabzSfn7o1HHnnEdfNg8+bNuOeee6BUKgN2fdIRJSvSZ7feeiusVitsNhtuueUWn5wzKSnJrR3K4XDg9OnTbvvIZLI+L0HV1TnmzJmD6OhobNy4EXv27KEqYAigaiDpM7FYjDNnzried0Wn03VYXishIaHTedVnzpyJ/Px8fPbZZxg0aBBeeeUVaLVat31ycnJw4MAB3HvvvZDL5V5VP3NycnD48GGUlZUhJiYG8fHxEIlErrarlStXYsiQId3OpU8Cg0pWxCfUajXUanW3++zbtw9jx451e6xZs6bTfRcvXoyFCxfit7/9LW688UYMHDgQM2bMcNtn7dq1KCsrw6BBg5CUlORV3MuXL4dYLMbw4cORlJSEiooK12sPP/wwrFZrQO5ukp7RtMaEdOH777/HTTfdhMrKyg4LkZLAo2RFyBUsFgvq6+uxcOFCpKam4v333w92SARUDSSkgw8++AADBgyAVqvF3//+92CHQ1pRyYoQEhaoZEUICQuUrAghYYGSFSEkLFCyIoSEBUpWhJCwQMmKEBIWKFkRQsICJStCSFigZEUICQv/D/75Uc7bwO/iAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:18:32.265970Z", + "iopub.status.busy": "2024-03-01T10:18:32.265684Z", + "iopub.status.idle": "2024-03-01T10:18:32.479903Z", + "shell.execute_reply": "2024-03-01T10:18:32.478895Z" + }, + "papermill": { + "duration": 0.237404, + "end_time": "2024-03-01T10:18:32.482074", + "exception": false, + "start_time": "2024-03-01T10:18:32.244670", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAu0UlEQVR4nO3de1hT15438G/CJdwRiFxFAbGiVYjCgFg73oDQiwfHTou0FUSr53iZqc1LPcUqPqhTar0caofKHKcI6pnqab2cnmpRijLHC96gWLRAC4JoJQgoRKDGkKz3D4ddUxJIEAxsfp/nyQNZWXtlrWzyZe/snbUFjDEGQgjhIaGpO0AIIf2FAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb5mbugMDkUajwe3bt2Fvbw+BQGDq7hBCHsMYw/379+Hp6QmhsPttNAo4HW7fvg1vb29Td4MQ0o2bN29ixIgR3dahgNPB3t4ewKMX0MHBwcS96T8qlQonTpxAVFQULCwsTN0d8oSGyvpUKBTw9vbm3qfdoYDToXO31MHBgfcBZ2NjAwcHB16/IYaKobY+Dfn4iA4yEEJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RaeJEDJItbe3o7y8nLvf+osS50qr4CS+DDtrkVbdgIAA2NjYPO0umhwFHCGDVHl5OYKDg7uUf6SjblFRESZPntz/nRpgKOAIGaQCAgJQVFTE3a+oa4bsi1Jsf3UixnoM61J3KKKAI2SQsrGx0doqE95oguj0Lxg3IQiSUS4m7NnAQQcZCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwlskDLiMjAz4+PrCyskJYWBguXryot252djYEAoHWzcrKSqvOwoULu9SJjo7u72EQQgYgk054eeDAAchkMmRmZiIsLAzp6emQSqWoqKiAq6urzmUcHBxQUVHB3RcIBF3qREdHY/fu3dx9kUjUpQ4hhP9MugW3fft2LFmyBImJiRg/fjwyMzNhY2ODrKwsvcsIBAK4u7tzNzc3ty51RCKRVh0nJ6f+HAYhZIAy2Rbcw4cPUVRUhOTkZK5MKBQiIiIChYWFepdrbW3FqFGjoNFoMHnyZHzwwQd49tlnteoUFBTA1dUVTk5OmDVrFjZt2gQXF/1TOCuVSiiVSu6+QqEAAKhUKqhUqt4OccDrHBufxziUdHR0cD/5vE6NGZvJAq6xsRFqtbrLFpibm5vWpdAeN3bsWGRlZSEwMBAtLS3YunUrpk6dimvXrmHEiBEAHu2ezps3D76+vqiqqsKaNWvwwgsvoLCwEGZmZjrbTUtLQ2pqapfyEydODIlLreXl5Zm6C6QP3GwFAHOcP38eP181dW/6T3t7u8F1BYwx1o990ev27dvw8vLCuXPnEB4ezpWvXr0a//u//4sLFy702IZKpcK4ceMQFxeHjRs36qxz/fp1jB49Gt9++y1mz56ts46uLThvb280NjbCwcHByJENHiqVCnl5eYiMjISFhYWpu0Oe0JXau/jXXZfx5ZIQBI10NnV3+o1CoYBYLEZLS0uP70+TbcGJxWKYmZmhvr5eq7y+vh7u7u4GtWFhYYFJkyahsrJSbx0/Pz+IxWJUVlbqDTiRSKTzQISFhcWQeOMPlXHynbm5OfeTz+vTmLGZ7CCDpaUlgoODkZ+fz5VpNBrk5+drbdF1R61Wo7S0FB4eHnrr3Lp1C01NTd3WIYTwk0mPospkMuzatQs5OTkoKyvDsmXL0NbWhsTERABAfHy81kGIDRs24MSJE7h+/TqKi4vx5ptv4saNG3jrrbcAPDoA8e677+L8+fOoqalBfn4+YmJi4O/vD6lUapIxEkJMx6TnwcXGxqKhoQEpKSmQy+WQSCTIzc3lDjzU1tZCKPw1g+/du4clS5ZALpfDyckJwcHBOHfuHMaPHw8AMDMzw/fff4+cnBw0NzfD09MTUVFR2LhxI50LR8gQZLKDDAOZQqGAo6OjQR9iDmYqlQrHjh3Diy++yOvPbIaKkhtNmLvzPI4sm8LrK9sb8/40+Ve1CCGkv1DAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S2TB1xGRgZ8fHxgZWWFsLAwXLx4UW/d7OxsCAQCrZuVlZVWHcYYUlJS4OHhAWtra0REROCnn37q72EQQgYgkwbcgQMHIJPJsH79ehQXFyMoKAhSqRR37tzRu4yDgwPq6uq4240bN7Qe/+ijj7Bjxw5kZmbiwoULsLW1hVQqxYMHD/p7OISQAcakAbd9+3YsWbIEiYmJGD9+PDIzM2FjY4OsrCy9ywgEAri7u3M3Nzc37jHGGNLT07F27VrExMQgMDAQe/bswe3bt3HkyJGnMCJCyEBibqonfvjwIYqKipCcnMyVCYVCREREoLCwUO9yra2tGDVqFDQaDSZPnowPPvgAzz77LACguroacrkcERERXH1HR0eEhYWhsLAQ8+fP19mmUqmEUqnk7isUCgCASqWCSqV6onEOZJ1j4/MYh5KOjg7uJ5/XqTFjM1nANTY2Qq1Wa22BAYCbmxvKy8t1LjN27FhkZWUhMDAQLS0t2Lp1K6ZOnYpr165hxIgRkMvlXBu/bbPzMV3S0tKQmprapfzEiROwsbExdmiDTl5enqm7QPrAzVYAMMf58+fx81VT96b/tLe3G1zXZAHXG+Hh4QgPD+fuT506FePGjcN//dd/YePGjb1uNzk5GTKZjLuvUCjg7e2NqKgoODg4PFGfBzKVSoW8vDxERkbCwsLC1N0hT+hK7V2g9DKmTJmCoJHOpu5Ov+ncwzKEyQJOLBbDzMwM9fX1WuX19fVwd3c3qA0LCwtMmjQJlZWVAMAtV19fDw8PD602JRKJ3nZEIhFEIpHO9ofCG3+ojJPvzM3NuZ98Xp/GjM1kBxksLS0RHByM/Px8rkyj0SA/P19rK607arUapaWlXJj5+vrC3d1dq02FQoELFy4Y3CYhhD9Muosqk8mQkJCAkJAQhIaGIj09HW1tbUhMTAQAxMfHw8vLC2lpaQCADRs2YMqUKfD390dzczO2bNmCGzdu4K233gLw6AjrqlWrsGnTJowZMwa+vr5Yt24dPD09MXfuXFMNkxBiIiYNuNjYWDQ0NCAlJQVyuRwSiQS5ubncQYLa2loIhb9uZN67dw9LliyBXC6Hk5MTgoODce7cOYwfP56rs3r1arS1tWHp0qVobm7GtGnTkJub2+WEYEII/wkYY8zUnRhoFAoFHB0d0dLSwquDDO3t7VpHqFt/UeLoqUK8NDMcdta/fgYZEBAwJI4e803JjSbM3XkeR5ZNgWSUi6m702+MeX8OqqOo5MmUl5cjODi4S/lHv7lfVFSEyZMnP51OEdKPKOCGkICAABQVFXH3K+qaIfuiFNtfnYixHsO06hHCBxRwQ4iNjY3WlpnwRhNEp3/BuAlBvN6lIUOXyWcTIYSQ/kJbcIQMItWNbWhTduh8rKqhjfvZedKvLrYic/iKbfulfwMNBRwhg0R1Yxtmbi3osd7/+7K0xzqnkmYMiZCjgCNkkOjcckuPlcDf1a7r478o8XVBIV6eEQ5b665fPQSAyjutWHWgRO9WIN9QwBEyyPi72mGCl2OXcpVKBflwYPIoJ15/F9UYdJCBEMJbFHCEEN4yOuCuX7/eH/0ghJA+Z3TA+fv7Y+bMmdi3bx9dyIUQMqAZHXDFxcUIDAyETCaDu7s7fv/733d7qT9CCDEVo4+iSiQSfPzxx9i2bRu++uorZGdnY9q0aXjmmWewaNEiLFiwAMOHD++PvhIjdXdSKEAnhhL+6/VpIubm5pg3bx5eeuklfPrpp0hOTkZSUhLWrFmD1157DZs3b9aaNpw8XYaeFArQiaGEv3odcJcvX0ZWVhb2798PW1tbJCUlYfHixbh16xZSU1MRExNDu64m1NNJoQCdGEr4z+iA2759O3bv3o2Kigq8+OKL2LNnD1588UVu5l1fX19kZ2fDx8enr/tKekHfSaEAnRhK+M/ogNu5cycWLVqEhQsX6t0FdXV1xWefffbEnSOEkCdhdMDl5eVh5MiRWtdKAADGGG7evImRI0fC0tISCQkJfdZJQgjpDaNPExk9ejQaGxu7lN+9exe+vr590ilCCOkLRgecvmvUtLa20pWrCCEDisG7qDKZDMCja4+mpKRoXXVJrVbjwoUL3V49nhBCnjaDA+67774D8GgLrrS0FJaWltxjlpaWCAoKQlJSUt/3kBBCesnggDt16hQAIDExER9//DGvrhfKR0r1Awitfka1ogJCK93nwXV0dOB2x22U3S3T+02GakUrhFY/Q6l+AED36SaEDFRGH0XdvXt3n3YgIyMDW7ZsgVwuR1BQED755BOEhob2uNz+/fsRFxeHmJgYHDlyhCtfuHAhcnJytOpKpVLk5ub2ab8HutttN2Dr+wnWGHCu9ae5n3b7uK0vcLtNgmC49VHvCHk6DAq4efPmITs7Gw4ODpg3b163dQ8dOmTwkx84cAAymQyZmZkICwtDeno6pFIpKioq4Orqqne5mpoaJCUl4fnnn9f5eHR0tFYQi0S6z9LnM0/bUWir/jd8HCvBaD3fZOjo6MDZM2fx3LTn9G7BVd1pxdsHSuA5c1R/dpeQfmFQwDk6OkIgEHC/95Xt27djyZIlSExMBABkZmbi6NGjyMrKwnvvvadzGbVajTfeeAOpqak4ffo0mpubu9QRiURwd3fvs34ORiIzK2geeMHXYSzGu+j/JkO1eTXGOY/T+00GzYMWaB40QGRGR8jJ4GNQwD2+NdRXu6gPHz5EUVERkpOTuTKhUIiIiAgUFhbqXW7Dhg1wdXXF4sWLcfr0aZ11CgoK4OrqCicnJ8yaNQubNm2Ci4v+CxsrlUoolUruvkKhAPAoAFQqlbFDGxA6Ojq4n/rG0Fne3RgNaYc8HT2ti6GyPo3pt8kuOtPY2Ai1Wg03N+3Pddzc3FBeXq5zmTNnzuCzzz5DSUmJ3najo6Mxb948+Pr6oqqqCmvWrMELL7yAwsJCmJmZ6VwmLS0NqampXcpPnDihdTrMYHKzFQDMcebMGdzQvYfKycvL65N2SP+6fv8hhFb3cPj0X1Forb9ezjc5eh+T/wIIrcxw6vRJ3LC31FtvIGtvbze4rkEBN2nSJG4XtSfFxcUGP7kx7t+/jwULFmDXrl0Qi8V6682fP5/7feLEiQgMDMTo0aNRUFCA2bNn61wmOTmZO88PeLQF5+3tjaioqEF7tPjabQW2lp7HtGnT8Kyn7jGoVCrk5eUhMjJS7y6qIe2Qp+NoxWXYqjfgkBpAa+/bsfUFfCV/xotjQ/qsb09T5x6WIQwKuLlz5/a2L3qJxWKYmZmhvr5eq7y+vl7n52dVVVWoqanBnDlzuDKNRgPg0dx0FRUVGD16dJfl/Pz8IBaLUVlZqTfgRCKRzgMRFhYWg3aWjc6DBubm5j2OobtxGtMO6V/eDn7dHjgy5qCR90y/Qbs+jem3QQG3fv36XndGH0tLSwQHByM/P58LUI1Gg/z8fKxcubJL/YCAAJSWak/MuHbtWty/fx8ff/wxvL29dT7PrVu30NTURJNvkkGvpwNHdNCoK5Ne+FkmkyEhIQEhISEIDQ1Feno62trauKOq8fHx8PLyQlpaGqysrDBhwgSt5YcNGwYAXHlraytSU1PxyiuvwN3dHVVVVVi9ejX8/f0hlUqf6tgIIaZnUMA5Ozvjxx9/hFgshpOTU7efx929e9fgJ4+NjUVDQwNSUlIgl8shkUiQm5vLHXiora3tMi1Td8zMzPD9998jJycHzc3N8PT0RFRUFDZu3Dgkz4UjZKgzKOD+9Kc/wd7envvd0AMOhli5cqXOXVLg0eke3cnOzta6b21tjePHj/dRzwghg51BAff45JULFy7sr74QQkifMno+ODMzM9y5c6dLeVNTk97zzAghxBT6bMJLpVKpNYUSIYSYmsFHUXfs2AHg0YSX//3f/w07u1/Pw1Gr1fjHP/6BgICAvu8hIYT0ksEB96c//QnAoy24zMxMrd1RS0tL+Pj4IDMzs+97SAghvWRwwFVXVwMAZs6ciUOHDsHJyanfOkUIIX3B6BN9O2f2JYSQgc7ogFu0aFG3j2dlZfW6M4QQ0peMDrh79+5p3VepVLh69Sqam5sxa9asPusYIYQ8KaMD7vDhw13KNBoNli1bpnM2D0IIMRWjz4PT2YhQCJlMxh1pJYSQgaBPAg54NF9b53TIhBAyEBi9i/r4zLfAo/Pi6urqcPToUa3vrBJCiKkZHXCdV7jvJBQKMXz4cGzbtq3HI6yEEPI00XlwhBDe6rPP4AghZKChgCOE8BYFHCGEtyjgCCG81WcBd+vWLSxdurSvmiOEkCfWZwHX1NSEzz77rK+aI4SQJ0a7qIQQ3qKAI4TwFgUcIYS3DP4mw7x587p9vLm5uVcdyMjIwJYtWyCXyxEUFIRPPvkEoaGhPS63f/9+xMXFISYmBkeOHOHKGWNYv349du3ahebmZjz33HPYuXMnxowZ06v+EUIGL4O34BwdHbu9jRo1CvHx8UY9+YEDByCTybB+/XoUFxcjKCgIUqlU53VXH1dTU4OkpCQ8//zzXR776KOPsGPHDmRmZuLChQuwtbWFVCrFgwcPjOobIWTwM3gLbvfu3X3+5Nu3b8eSJUuQmJgIAMjMzMTRo0eRlZWF9957T+cyarUab7zxBlJTU3H69GmtLUfGGNLT07F27VrExMQAAPbs2QM3NzccOXIE8+fP7/MxEEIGLqO/bN9XHj58iKKiIiQnJ3NlQqEQERERKCws1Lvchg0b4OrqisWLF+P06dNaj1VXV0MulyMiIoIrc3R0RFhYGAoLC/UGnFKphFKp5O4rFAoAj6ZjV6lUvRqfqXXOzdfR0aF3DJ3l3Y3RkHbI09HTuhgq69OYfhsccIZOhWToRWcaGxuhVqvh5uamVe7m5oby8nKdy5w5cwafffYZSkpKdD4ul8u5Nn7bZudjuqSlpSE1NbVL+YkTJ2BjY9PdMAasm60AYI4zZ87ghl33dfPy8vqkHdK/DF0XfF+f7e3tBtc1OOCys7MxatQoTJo0CYyxXnXsSdy/fx8LFizArl27IBaL+7Tt5ORkrYk8FQoFvL29ERUVBQcHhz59rqfl2m0Ftpaex7Rp0/Csp+4xqFQq5OXlITIyEhYWFr1uhzwdPa2LobI+O/ewDGFwwC1btgyff/45qqurkZiYiDfffBPOzs696iAAiMVimJmZob6+Xqu8vr4e7u7uXepXVVWhpqYGc+bM4co0Gg0AwNzcHBUVFdxy9fX18PDw0GpTIpHo7YtIJIJIJOpSbmFhofcPZaAzNzfnfvY0hu7GaUw7pH8Zui74vj6N6bfBR1EzMjJQV1eH1atX4+9//zu8vb3x2muv4fjx473aorO0tERwcDDy8/O5Mo1Gg/z8fISHh3epHxAQgNLSUpSUlHC33/3ud5g5cyZKSkrg7e0NX19fuLu7a7WpUChw4cIFnW0SQvjNqIMMIpEIcXFxiIuLw40bN5CdnY3ly5ejo6MD165dg52dcTv1MpkMCQkJCAkJQWhoKNLT09HW1sYdVY2Pj4eXlxfS0tJgZWWFCRMmaC0/bNgwANAqX7VqFTZt2oQxY8bA19cX69atg6enJ+bOnWtU3wghg1+vj6IKhUIIBAIwxqBWq3vVRmxsLBoaGpCSkgK5XA6JRILc3FzuIEFtbS2EQuO+bLF69Wq0tbVh6dKlaG5uxrRp05CbmwsrK6te9ZEQMngZFXBKpRKHDh1CVlYWzpw5g5dffhn/+Z//iejoaKODqNPKlSuxcuVKnY8VFBR0u2x2dnaXMoFAgA0bNmDDhg296g9f/KJ69E/n6s8teuu0/aLE5QbA/cY92Fp3/QwSACrvtPZL/wh5GgwOuOXLl2P//v3w9vbGokWL8Pnnn/f50UzSd6r+L5jeO1TaQ01z7K281GN7tiKTnTJJ/k9P/7ToH1ZXAmbgEQKhUIiRI0di0qRJEAgEeusdOnSozzpnKgqFAo6OjmhpaRm0p4ncbXuIE9fkGO1qB2sLM511Kupa8P++LMW2f52IsR6OetuyFZnDV2zbX10lBtp/sdaAf1iGOZU0Y9CuU2Penwb/W46Pj+822MjA4mxrifmhI7ut03lW++jhtpjgpT/gyMAQ9eyj06D0/dOif1hdGXWiLyHEdHr6p0X/sLqi+eAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQnjL5AGXkZEBHx8fWFlZISwsDBcvXtRb99ChQwgJCcGwYcNga2sLiUSCvXv3atVZuHAhBAKB1i06Orq/h0EIGYBMejXfAwcOQCaTITMzE2FhYUhPT4dUKkVFRQVcXV271Hd2dsb777+PgIAAWFpa4uuvv0ZiYiJcXV0hlUq5etHR0di9ezd3XyTSfRFcQgi/mXQLbvv27ViyZAkSExMxfvx4ZGZmwsbGBllZWTrrz5gxA//yL/+CcePGYfTo0Xj77bcRGBiIM2fOaNUTiURwd3fnbk5OTk9jOISQAcZkW3APHz5EUVERkpOTuTKhUIiIiAgUFhb2uDxjDCdPnkRFRQU2b96s9VhBQQFcXV3h5OSEWbNmYdOmTXBxcdHbllKphFKp5O4rFAoAgEqlgkqlMnZog0bndTQ7Ojp4Pc6hYqisT2PGZrKAa2xshFqthpubm1a5m5sbysvL9S7X0tICLy8vKJVKmJmZ4dNPP0VkZCT3eHR0NObNmwdfX19UVVVhzZo1eOGFF1BYWAgzs65XAweAtLQ0pKamdik/ceIEbGxsejnCge9mKwCY4/z58/j5qql7Q57UUFmf7e3tBtc16WdwvWFvb4+SkhK0trYiPz8fMpkMfn5+mDFjBgBg/vz5XN2JEyciMDAQo0ePRkFBAWbPnq2zzeTkZMhkMu6+QqGAt7c3oqKi4ODg0K/jMaUrtXeB0suYMmUKgkY6m7o75AkNlfXZuYdlCJMFnFgshpmZGerr67XK6+vr4e7urnc5oVAIf39/AIBEIkFZWRnS0tK4gPstPz8/iMViVFZW6g04kUik80CEhYUFLCwsDBzR4GNubs795PM4h4qhsj6NGZvJDjJYWloiODgY+fn5XJlGo0F+fj7Cw8MNbkej0Wh9fvZbt27dQlNTEzw8PJ6ov4SQwceku6gymQwJCQkICQlBaGgo0tPT0dbWhsTERABAfHw8vLy8kJaWBuDRZ2UhISEYPXo0lEoljh07hr1792Lnzp0AgNbWVqSmpuKVV16Bu7s7qqqqsHr1avj7+2udRkIIGRpMGnCxsbFoaGhASkoK5HI5JBIJcnNzuQMPtbW1EAp/3chsa2vD8uXLcevWLVhbWyMgIAD79u1DbGwsAMDMzAzff/89cnJy0NzcDE9PT0RFRWHjxo10LhwhQ5CAMcZM3YmBRqFQwNHRES0tLbw+yFByowlzd57HkWVTIBml/zQaMjgMlfVpzPvT5F/VIoSQ/kIBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLdMHnAZGRnw8fGBlZUVwsLCcPHiRb11Dx06hJCQEAwbNgy2traQSCTYu3evVh3GGFJSUuDh4QFra2tERETgp59+6u9hEEIGIHNTPvmBAwcgk8mQmZmJsLAwpKenQyqVoqKiAq6url3qOzs74/3330dAQAAsLS3x9ddfIzExEa6urpBKpQCAjz76CDt27EBOTg58fX2xbt06SKVS/PDDD7CysnraQxxQ2tvbUV5ezt2vqGuGUl6JsqvW0DQN48oDAgJgY2Njgh4S0seYCYWGhrIVK1Zw99VqNfP09GRpaWkGtzFp0iS2du1axhhjGo2Gubu7sy1btnCPNzc3M5FIxD7//HOD22xpaWEAWEtLi8HLDAZFRUUMQI+3oqIiU3eV9MJ3NY1s1B+/Zt/VNJq6K/3KmPenybbgHj58iKKiIiQnJ3NlQqEQERERKCws7HF5xhhOnjyJiooKbN68GQBQXV0NuVyOiIgIrp6joyPCwsJQWFiI+fPn9/1ABpGAgAAUFRVx91t/UeLoqUK8NDMcdtYirXqE8IHJAq6xsRFqtRpubm5a5W5ublq7Ub/V0tICLy8vKJVKmJmZ4dNPP0VkZCQAQC6Xc238ts3Ox3RRKpVQKpXcfYVCAQBQqVRQqVTGDWwAs7CwwMSJE7n7KpUK9xrvIGRSECwsLLTq8mncQ0VHRwf3k8/rz5ixmfQzuN6wt7dHSUkJWltbkZ+fD5lMBj8/P8yYMaPXbaalpSE1NbVL+YkTJ4bEZ1F5eXmm7gLpAzdbAcAc58+fx89XTd2b/tPe3m5wXZMFnFgshpmZGerr67XK6+vr4e7urnc5oVAIf39/AIBEIkFZWRnS0tIwY8YMbrn6+np4eHhotSmRSPS2mZycDJlMxt1XKBTw9vZGVFQUHBwcejO8QUGlUiEvLw+RkZFdtuDI4HOl9i5QehlTpkxB0EhnU3en33TuYRnCZAFnaWmJ4OBg5OfnY+7cuQAAjUaD/Px8rFy50uB2NBoNt3vp6+sLd3d35Ofnc4GmUChw4cIFLFu2TG8bIpEIIpGoS7mFhcWQeOMPlXHynbm5OfeTz+vTmLGZdBdVJpMhISEBISEhCA0NRXp6Otra2pCYmAgAiI+Ph5eXF9LS0gA82pUMCQnB6NGjoVQqcezYMezduxc7d+4EAAgEAqxatQqbNm3CmDFjuNNEPD09uRAlhAwdJg242NhYNDQ0ICUlBXK5HBKJBLm5udxBgtraWgiFv56L3NbWhuXLl+PWrVuwtrZGQEAA9u3bh9jYWK7O6tWr0dbWhqVLl6K5uRnTpk1Dbm7ukD8HjpChSMAYY6buxECjUCjg6OiIlpYW3n8Gd+zYMbz44ou83qUZKkpuNGHuzvM4smwKJKNcTN2dfmPM+9PkX9UihJD+QgFHCOEtCjhCCG9RwBFCeIsCjhDCW4Puq1qEkEcMnf4KGLpTYFHAETJIlZeXIzg4uEv56zld6xYVFWHy5MlPoVcDCwUcIYOUodNfddYdiijgCBmkbGxstLbKOqe/Cg8NoRO3/w8dZCCE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCW3QenA6dc4Aac3GLwUilUqG9vR0KhYLOm+KBobI+O9+XhszVSwGnw/379wEA3t7eJu4JIUSf+/fvw9HRsds6NGW5DhqNBrdv34a9vT0EAoGpu9NvOi+PePPmTV5PzT5UDJX1yRjD/fv34enpqXXNFl1oC04HoVCIESNGmLobT42DgwOv3xBDzVBYnz1tuXWigwyEEN6igCOE8BYF3BAmEomwfv16iESiniuTAY/WZ1d0kIEQwlu0BUcI4S0KOEIIb1HAEUJ4iwJugKqpqYFAIEBJSclTaefs2bOYOHEiLCwsMHfu3Cd6TkIGCgq4QaKgoAACgQDNzc390r5MJoNEIkF1dTWys7P75TlIVzNmzMCqVatM3Q3OQOvPk6KA6wcPHz40dReMVlVVhVmzZmHEiBEYNmxYr9p4muNmjKGjo+OpPd9ANhj/3p4aRp7Y9OnT2YoVK9jbb7/NXFxc2IwZM1hpaSmLjo5mtra2zNXVlb355pusoaGBW+abb75hzz33HHN0dGTOzs7spZdeYpWVldzj1dXVDAD77rvvuN8fvyUkJBjVzueff87Cw8OZSCRizz77LCsoKNB6/PHb7t27GWOMFRQUsH/6p39ilpaWzN3dnf3xj39kKpWq23GfOnWKAWC5ublMIpEwKysrNnPmTFZfX8+OHTvGAgICmL29PYuLi2NtbW1cW2q1mn3wwQfMx8eHWVlZscDAQPbFF19wj3e2e+zYMTZ58mRmYWHBTp061Zer8alLSEjo8tpXVlayRYsWca/DM888w9LT07ssFxMTwzZt2sQ8PDyYj48PY4yxs2fPsqCgICYSiVhwcDA7fPgw9zfUqbu/S139qa6uflovR7+ggOsD06dPZ3Z2duzdd99l5eXl7Pz582z48OEsOTmZlZWVseLiYhYZGclmzpzJLfPll1+ygwcPsp9++ol99913bM6cOWzixIlMrVYzxrQDrqOjgx08eJABYBUVFayuro41Nzcb1c6IESPYl19+yX744Qf21ltvMXt7e9bY2Mg6OjpYXV0dc3BwYOnp6ayuro61t7ezW7duMRsbG7Z8+XJWVlbGDh8+zMRiMVu/fr3ecZeXl3NBNGXKFHbmzBlWXFzM/P392fTp01lUVBQrLi5m//jHP5iLiwv78MMPubY2bdrEAgICWG5uLquqqmK7d+9mIpGIC+LOdgMDA9mJEydYZWUla2pq6u9V26+am5tZeHg4W7JkCaurq2N1dXXswYMHLCUlhV26dIldv36d7du3j9nY2LADBw5wyyUkJDA7Ozu2YMECdvXqVXb16lXW0tLCnJ2d2ZtvvsmuXbvGjh07xp555hmtgLt37163f5e6+tPR0WGKl6bPUMD1genTp7NJkyZx9zdu3MiioqK06ty8eZMLKF0aGhoYAFZaWsoY0w44xn59g9+7d6/bvuhr5/EwUalUbMSIEWzz5s1cmaOjI7flxhhja9asYWPHjmUajYYry8jIYHZ2dlx4/nbcj/fz22+/5crS0tIYAFZVVcWV/f73v2dSqZQxxtiDBw+YjY0NO3funFZbixcvZnFxcVrtHjlypNvxDzbTp09nb7/9drd1VqxYwV555RXufkJCAnNzc2NKpZIr27lzJ3NxcWG//PILV7Zr1y6tvyFD/i4N6c9gQp/B9ZHg4GDu9ytXruDUqVOws7Pjbp1XFq+qqgIA/PTTT4iLi4Ofnx8cHBzg4+MDAKitrTXqeQ1tJzw8nPvd3NwcISEhKCsr09tuWVkZwsPDtaaLeu6559Da2opbt27pHPfjAgMDud/d3NxgY2MDPz8/rbI7d+4AACorK9He3o7IyEit12zPnj3c69UpJCREb5/5IiMjA8HBwRg+fDjs7Ozw5z//ucv6nDhxIiwtLbn7FRUVCAwMhJWVFVcWGhqqtYwhf5d8Q9Ml9RFbW1vu99bWVsyZMwebN2/uUs/DwwMAMGfOHIwaNQq7du2Cp6cnNBoNJkyYYPQHxn3VTm89Pu7HPT6jrEAg6DLDrEAggEajAfDo9QKAo0ePwsvLS6veb79Xqe/5+GL//v1ISkrCtm3bEB4eDnt7e2zZsgUXLlzQqteb18GQv0u+oYDrB5MnT8bBgwfh4+MDc/OuL3FTUxMqKiqwa9cuPP/88wCAM2fOdNtm539rtVrdq3bOnz+Pf/7nfwYAdHR0oKioCCtXrtT7fOPGjcPBgwfBGOO24s6ePQt7e/s+nytv/PjxEIlEqK2txfTp0/u07YHO0tJSa52ePXsWU6dOxfLly7kyQ7auxo4di3379kGpVHL/FC5duqRVp6e/S139GexoF7UfrFixAnfv3kVcXBwuXbqEqqoqHD9+HImJiVCr1XBycoKLiwv+/Oc/o7KyEidPnoRMJuu2zVGjRkEgEODrr79GQ0MDWltbjWonIyMDhw8fRnl5OVasWIF79+5h0aJFep9v+fLluHnzJv7t3/4N5eXl+Nvf/ob169dDJpP1OIuqsezt7ZGUlIR33nkHOTk5qKqqQnFxMT755BPk5OT06XMNND4+Prhw4QJqamrQ2NiIMWPG4PLlyzh+/Dh+/PFHrFu3rktQ6fL6669Do9Fg6dKlKCsrw/Hjx7F161YA4P5B9fR3qas/nVvZgxUFXD/w9PTE2bNnoVarERUVhYkTJ2LVqlUYNmwYhEIhhEIh9u/fj6KiIkyYMAHvvPMOtmzZ0m2bXl5eSE1NxXvvvQc3NzesXLnSqHY+/PBDfPjhhwgKCsKZM2fw1VdfQSwWd/t8x44dw8WLFxEUFIQ//OEPWLx4MdauXftEr40+GzduxLp165CWloZx48YhOjoaR48eha+vb78830CRlJQEMzMzjB8/HsOHD4dUKsW8efMQGxuLsLAwNDU1aW3N6ePg4IC///3vKCkpgUQiwfvvv4+UlBQA4D6X6+nvUld/jP1MeKCh6ZII4am//OUvSExMREtLC6ytrU3dHZOgz+AI4Yk9e/bAz88PXl5euHLlCv74xz/itddeG7LhBlDAEcIbcrkcKSkpkMvl8PDwwKuvvor/+I//MHW3TIp2UQkhvEUHGQghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcMSkFi5cCIFAwH0h383NDZGRkcjKyjLqa0LZ2dm9non4SSxcuJCuYTGAUcARk4uOjkZdXR1qamrwzTffYObMmXj77bfx8ssv07Tk5MmYdDY6MuR1Tr/9W/n5+QwA27VrF2OMsW3btrEJEyYwGxsbNmLECLZs2TJ2//59xtivk2E+fuuceXjPnj0sODiY2dnZMTc3NxYXF8fq6+u557l79y57/fXXmVgsZlZWVszf359lZWVxj9fW1rJXX32VOTo6MicnJ/a73/2Om8Z7/fr1XZ53sE+jzje0BUcGpFmzZiEoKAiHDh0CAAiFQuzYsQPXrl1DTk4OTp48idWrVwMApk6divT0dDg4OKCurg51dXVISkoCAKhUKmzcuBFXrlzBkSNHUFNTg4ULF3LPs27dOvzwww/45ptvUFZWhp07d3KTEKhUKkilUtjb2+P06dM4e/Ys7OzsEB0djYcPHyIpKQmvvfYatwVaV1eHqVOnPt0XinTP1AlLhjZ9W3CMMRYbG8vGjRun87EvvviCubi4cPd3797NHB0de3y+S5cuMQDc1t+cOXNYYmKizrp79+7tMm27Uqlk1tbW7Pjx4z32n5gebcGRAYs9Ntnmt99+i9mzZ8PLywv29vZYsGABmpqa0N7e3m0bRUVFmDNnDkaOHAl7e3tuQs3OaYCWLVuG/fv3QyKRYPXq1Th37hy37JUrV1BZWQl7e3tuim9nZ2c8ePCAt1N88w0FHBmwysrK4Ovri5qaGrz88ssIDAzEwYMHUVRUhIyMDADdXxO0ra0NUqkUDg4O+Mtf/oJLly7h8OHDWsu98MILuHHjBt555x3cvn0bs2fP5nZvW1tbERwcjJKSEq3bjz/+iNdff72fR0/6As0mQgakkydPorS0FO+88w6Kioqg0Wiwbds2bmLGv/71r1r1dU21XV5ejqamJnz44Yfw9vYGAFy+fLnLcw0fPhwJCQlISEjA888/j3fffRdbt27F5MmTceDAAbi6usLBwUFnP/k2xTff0BYcMTmlUgm5XI6ff/4ZxcXF+OCDDxATE4OXX34Z8fHx8Pf3h0qlwieffILr169j7969yMzM1GrDx8cHra2tyM/PR2NjI9rb2zFy5EhYWlpyy3311VfYuHGj1nIpKSn429/+hsrKSly7dg1ff/01xo0bBwB44403IBaLERMTg9OnT6O6uhoFBQX493//d+7KYj4+Pvj+++9RUVGBxsZGqFSqp/OiEcOY+kNAMrQ9fjV1c3NzNnz4cBYREcGysrK4668yxtj27duZh4cHs7a2ZlKplO3Zs6fLdWL/8Ic/MBcXF63TRP7nf/6H+fj4MJFIxMLDw9lXX33V5Vqh48aNY9bW1szZ2ZnFxMSw69evc23W1dWx+Ph4JhaLmUgkYn5+fmzJkiWspaWFMcbYnTt3WGRkJLOzs6PTRAYgmg+OEMJbtItKCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4Qwlv/HyFckB+VOkxgAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T10:18:32.525075Z", + "iopub.status.busy": "2024-03-01T10:18:32.524768Z", + "iopub.status.idle": "2024-03-01T10:18:32.727276Z", + "shell.execute_reply": "2024-03-01T10:18:32.726378Z" + }, + "papermill": { + "duration": 0.226259, + "end_time": "2024-03-01T10:18:32.729379", + "exception": false, + "start_time": "2024-03-01T10:18:32.503120", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhhElEQVR4nO2dd3gU1dfHv7M1CWm0NAgQeg81NOldmoiCiAoIKFJUkK50NCAIvFJVBCwgoijykyYgvfeaUEIwlDQS0pOt8/6x2c3M7MzuzGaTzSb38zx5kp1y5+5m5zvnnnvOuRRN0zQIBALBzZC5ugMEAoHgCES8CASCW0LEi0AguCVEvAgEgltCxItAILglRLwIBIJbQsSLQCC4JUS8CASCW6JwdQeKG6PRiGfPnsHHxwcURbm6OwQCgQNN08jMzERISAhkMmH7qsyJ17NnzxAaGurqbhAIBDs8fvwYVatWFdxf5sTLx8cHgOmD8fX1dXFvCAQCl4yMDISGhlruVSHKnHiZh4q+vr5EvAiEEow9tw5x2BMIBLeEiBeBQHBLiHgRCAS3pMz5vNwFmqah1+thMBhc3RUCwanI5XIoFIpChyoR8SqBaLVaxMfHIycnx9VdIRCKBC8vLwQHB0OlUjncBhGvEobRaERsbCzkcjlCQkKgUqlIMC2h1EDTNLRaLZKTkxEbG4s6derYDES1BRGvYuZsTArWH3uARYMaI6xSOav9Wq0WRqMRoaGh8PLyckEPCYSixdPTE0qlEv/99x+0Wi08PDwcaoc47IuZ70/F4uT95/jndoLN4xx9GhEI7oAzvt/kDilmohMyAAAGsu4JgVAoiHgVIxl5Ojx5kQsAINpFIBQOIl7FyL2ETMvfZMU5cTx69AgUReHatWvF0s7p06fRpEkTKJVKvPLKK4W6Zlnl2LFjoCgKaWlpRXodIl7FSBRDvIxEuxyiqG+MqVOnolmzZoiNjcXWrVuL5BoE50DEqxiJjs+w/G0sA5aXVqt1dRckExMTg27duqFq1arw9/d3qI3ifN/mYGZn4w7/OyJexUi0g5YXTdPI0epd8iNleNulSxdMmjQJH3/8MSpVqoTevXvj1q1b6Nu3L7y9vREYGIi3334bz58/t5xz4MABvPTSS/D390fFihXRv39/xMTE8Lb/6NEjdO3aFQBQvnx5UBSFUaNGSWonOjoa7du3h4eHBxo3bozjx49b2qYoCikpKXj33XdBUZTF8jp+/DgiIiKgVqsRHByMWbNmsQSD732bLcSDBw+iefPm8PT0RLdu3ZCUlIT9+/ejQYMG8PX1xZtvvskKRjYajYiMjERYWBg8PT0RHh6O33//3bLf3O7+/fvRsmVLqNVqnDp1yu7/ZsmSJQgICICPjw/Gjh2LWbNmoVmzZpb9o0aNwiuvvILPP/8cISEhqFevHgDgp59+QqtWreDj44OgoCC8+eabSEpKYrW9b98+1K1bF56enujatSsePXpktz/OgMR5FRNGI427DPGS4rHP1RnQcN7BIuiVfe4s6g0vlfivyQ8//IAPPvgAp0+fRlpaGrp164axY8di1apVyM3NxcyZMzF06FD8+++/AIDs7GxMnToVTZs2RVZWFubNm4fBgwfj2rVrVtPpoaGh2LVrF4YMGYK7d+/C19cXnp6ektqZPn06Vq9ejYYNG2LlypUYMGAAYmNjERoaivj4eNSrVw+LFi3CsGHD4Ofnh6dPn+Lll1/GqFGj8OOPPyI6Ohrjxo2Dh4cHFixYwPu+ASA+Ph4AsGDBAqxduxZeXl4YOnQohg4dCrVaje3btyMrKwuDBw/GmjVrMHPmTABAZGQkfv75Z2zcuBF16tTBiRMn8NZbb6Fy5cro3Lmz5XqzZs3CihUrULNmTZQvX97m/2Tbtm34/PPPsX79enTo0AE7duzAV199hbCwMNZxR44cga+vLw4dOmTZptPpsHjxYtSrVw9JSUmYOnUqRo0ahX379gEw1cV79dVXMXHiRLz33nu4dOkSPvnkE/tfFCdAxKuYeJqWiyxNwdO6tPq86tSpgy+//BKA6WnfvHlzfPHFF5b9mzdvRmhoKO7du4e6detiyJAhrPM3b96MypUr486dO2jcuDFrn1wuR4UKFQAAAQEBrGGd2HYmTZpkOXbDhg04cOAAvv/+e8yYMQNBQUGgKAp+fn4ICgoCAKxfvx6hoaFYu3YtKIpC/fr18ezZM8ycORPz5s2zCCPzfQMF4rVkyRJ06NABADBmzBjMnj0bMTExqFmzJgDgtddew9GjRzFz5kxoNBp88cUXOHz4MNq1awcAqFmzJk6dOoVvvvmGJV6LFi1Cz5497f9DAKxZswZjxozB6NGjAQDz5s3DP//8g6ysLNZx5cqVw6ZNm1gpO++++67l75o1a+Lrr79G69atkZWVBW9vb2zYsAG1atXCV199BQCoV68ebt68iWXLlonqW2Eg4lVMRDH8XYA0n5enUo47i3o7u0uiry2Fli1bWv6+fv06jh49Cm9vb6vjYmJiULduXdy/fx/z5s3D+fPn8fz5cxiNRgBAXFyclXjZQmw7ZlEAAIVCgVatWiEqKkqw3aioKLRr146VotWhQwdkZWXhyZMnqFatmtX7ZtK0aVPL34GBgfDy8rIIl3nbhQsXAAAPHjxATk6OlShptVo0b96cta1Vq1aCfeZy9+5dTJgwgbUtIiLCYv2aadKkiVWu4eXLl7FgwQJcv34dL168YH2uDRs2RFRUFNq0acM6h/kZFyVEvIoJpr8LkGZ5URQlaejmSsqVK0h5ysrKwoABA3ifwsHBwQCAAQMGoHr16vjuu+8QEhICo9GIxo0bS3YYO6sdR2G+byZKpdLyN0VRrNfmbWZBMFtCe/fuRZUqVVjHqdVqUdcrDNw2s7Oz0bt3b/Tu3Rvbtm1D5cqVERcXh969e5cIh7573BGlAHNkvVJOQWegQaOUjhsZtGjRArt27UKNGjWgUFh/1VJSUnD37l1899136NixIwDYdT6bLQNmqSAp7Zw7dw6dOnUCAOj1ely+fBmTJk0SvF6DBg2wa9cu0DRtsb5Onz4NHx8fm4tDOELDhg2hVqsRFxfHGiIWlnr16uHixYt45513LNsuXrxo97zo6GikpKRg6dKllkVrLl26xDqmQYMG2LNnD2vbuXPnnNBr+5DZxmIiOt5kedUJMC0qUAYiJTBx4kSkpqZi+PDhuHjxImJiYnDw4EGMHj0aBoMB5cuXR8WKFfHtt9/iwYMH+PfffzF16lSbbVavXh0UReHvv/9GcnIysrKyJLWzbt06/Pnnn4iOjsbEiRPx4sULll+Hy4QJE/D48WNMnjwZ0dHR+OuvvzB//nxMnTrV6fmnPj4+mDZtGqZMmYIffvgBMTExuHLlCtasWYMffvjB4XYnT56M77//Hj/88APu37+PJUuW4MaNG3arlVSrVg0qlQpr1qzBw4cPsWfPHixevJh1zPjx43H//n1Mnz4dd+/exfbt24stPo6IVzGQqzUgNiUbANAwxLToh7G0euwZhISE4PTp0zAYDOjVqxeaNGmCjz/+GP7+/pDJZJDJZNixYwcuX76Mxo0bY8qUKVi+fLnNNqtUqYKFCxdi1qxZCAwMxKRJkyS1s3TpUixduhTh4eE4deoU9uzZg0qVKtm83r59+3DhwgWEh4dj/PjxGDNmDD777LNCfTZCLF68GHPnzkVkZCQaNGiAPn36YO/evVYzg1IYMWIEZs+ejWnTpqFFixaIjY3FqFGj7FZzqFy5MrZu3YrffvsNDRs2xNKlS7FixQrWMdWqVcOuXbuwe/duhIeHY+PGjawJmqKEostYnkpGRgb8/PyQnp5ebKsHXX+chkHrTqOStxqvt6qKDcdi8G6HMMwb0NDq2Ly8PMTGxiIsLMzhUiEEgj169uyJoKAg/PTTTy65vq3vudh7lPi8igGzv6tBsA/MhnpZ8HkRSgY5OTnYuHEjevfuDblcjl9++QWHDx9mxXO5I2TYaIdjd5Nw+E5iodqIyvd31Q/ygSzfz1C27F1CUdKoUSN4e3vz/mzbtg0URWHfvn3o1KkTWrZsif/973/YtWsXevTo4equFwpiedlAbzBi/M+XYTQC1+f3gqdKWsyTGbPlVT/IF//l+77KQm4joXjYt28fdDod777AwEB4enri8OHDxdyrooeIlw30Rhp5OlMMTrZW75B40TRtifGqH+yDuFRTHhsRL4KzqF69uqu74BLIsNEGTH3RGYwOtZGYoUFajg5yGYXaAd4wz04T7SIQCgcRLxswrSOt3jHxisofMtaqXA5qhdzi8yoDkRIEQpFCxMsGzhCvaIuz3jTlK7NYXkS9CITCQMTLBkzrSOvgsNHirA82RdZTZLaRQHAKRLxsQDvR8mqQb3mZfV7EYU8gFA4iXjZgWV4OiJdGb0BMsqlSgNnyIj4vYWrUqIHVq1dbXlMUhd27d7usP4SSDQmVsAHT8tIZpKtNTFI29EYafp5KBPmaUiCIz0s88fHxdquEEsouRLxswPZ5GYQPFMBcgLB+kI/F12WJsC9890o95mqmroSmaRgMBt6SPgTXQoaNNiisz6sgp9E6uVSSz4umAW22a34kWoiZmZkYMWIEypUrh+DgYKxatQpdunTBxx9/LKkdgD1sNC+Q8ccff6Br167w8vJCeHg4zp49yzrn1KlT6NixIzw9PREaGooPP/wQ2dnZlv32FpRwdIELQvFDHic2YFte0m0lS2R9kI9lm0M+L10O8EWI5Os7hTnPAJX4qp1Tp07F6dOnsWfPHgQGBmLevHm4cuUKa6WawvDpp59ixYoVqFOnDj799FMMHz4cDx48gEKhQExMDPr06YMlS5Zg8+bNSE5OxqRJkzBp0iRs2bIFgP0FJcxIWeCC4BqIeNmgsHFeloRshuUlK8WzjZmZmfjhhx+wfft2dO/eHQCwZcsWhIQ4T3inTZuGfv36AQAWLlyIRo0a4cGDB6hfvz4iIyMxYsQIi5VXp04dfP311+jcuTM2bNgADw8PuwtKmJGywAXBNRDxsoGR5bCXJl7JmRo8z9KAooC6gQU3hczisZfQmNLLZAG5AqWX6EMfPnwInU6HiIgIyzY/Pz/LGoDOgLmghbkOflJSEurXr4/r16/jxo0b2LZtm+UYmqZhNBoRGxuLBg0a2F1QwoyUBS4IroGIlw3oQoRK3E80WV3VK3ixFs8w1/OSZHlRlKShW2mGu6AFANYCFu+//z4+/PBDq/OqVasmaUGJoljgguBciHjZoDDiZV6j0d+LvZSU5YYrhcPGmjVrQqlU4uLFi5YlwdLT03Hv3j3LohdFSYsWLXDnzh3Url2bd//NmzftLihBcB/IbKMNWD4vicNG85ncNQ5Kc5Cqj48PRo4cienTp+Po0aO4ffs2xowZA5lMZnexB2cwc+ZMnDlzBpMmTcK1a9dw//59/PXXX5bVgcQsKEFwH4h42aAwDnvzqdxbVlbKS+KsXLkS7dq1Q//+/dGjRw906NABDRo0KJZ6/E2bNsXx48dx7949dOzYEc2bN8e8efMsEwZiFpQguA9k2GiDwiVmm07mWhxUKY+w9/HxYTnMs7OzsXDhQrz33nt2z3306BHrNfMzqlGjhtVn5u/vb7WtdevW+OeffwSvMXz4cAwfPlzwOl26dCm1/5vSBhEvG7DSgxy0vGQc06s0+7wA4OrVq4iOjkZERATS09OxaNEiAMCgQYNc3DNCaYOIlw0KY3kZLcNGtnqVZp+XmRUrVuDu3btQqVRo2bIlTp48iaioKPTt21fwHPNS9wSCWIh42YC5PJlknxf4nV6OhHm5E82bN8fly5ettufm5uLatWvF3yFCqcWlDvsTJ05gwIABCAkJEVX+xJx3xv1JSEgokv4ZGXolebZRwGFf2n1eQnh6eqJ27dqCPwSCVFwqXtnZ2QgPD8e6desknXf37l3Ex8dbfgICAoqkf4Wabcz/LRwqYTpCozdg1JYL+O7EQ/b5ZUzcCGULZ3y/XTps7Nu3r00/iBABAQHw9/d3foc4FCZI1fzP4fq8CqLCTa9vPc3AsbvJiEnOwrhONS0R5Dk5OfD09HSw5wRCySYnx7QEIDNjQipu6fNq1qwZNBoNGjdujAULFqBDhw5Fcp3C5DZaZhs5tm2Bzyvf8tKZ6oSZxUwul8Pf399SpsXLy6tYAjwJhOKApmnk5OQgKSkJ/v7+kMsdW8gZcDPxCg4OxsaNG9GqVStoNBps2rQJXbp0wfnz59GiRQveczQaDTQajeV1RkaG6OsVLsJewPICe7ZRk2/RMa9lLsLHrDNFIJQm/P39C11s0q3Eq169eqwKBe3bt0dMTAxWrVqFn376ifecyMhILFy40KHrFaaGvcVhLzTbyPB5MY83nUMhODgYAQEBgsu4EwjuilKpLJTFZcatxIuPiIgIm5UuZ8+ejalTp1peZ2RkWJJy7cO0vKQ5GIX8kRRl3/IyI5fLnfJPJhBKI24vXteuXbPUdeJDrVZDrVY71HahLK/831x/FdfyyjP7vMjkIoEgCZeKV1ZWFh48eGB5HRsbi2vXrqFChQqoVq0aZs+ejadPn+LHH38EAKxevRphYWFo1KgR8vLysGnTJvz77782c9kKg9HIDJWQtgCHWZyE04NMr82WFwmNIBCk4VLxunTpErp27Wp5bR7ejRw5Elu3bkV8fDzi4uIs+7VaLT755BM8ffoUXl5eaNq0KQ4fPsxqw5kwrSGpS5/Zryphnm0UHjYSCARhXCpe9jL4t27dyno9Y8YMzJgxo4h7VUBhVg+iBapKcHMbLQ57RztJIJRRSD0vGxQmMdtuepA5zsvssCdOLwJBEkS8bGAsTEmc/N/cUAluhL3ZYU9GjQSCNIh42YCpJxpHLS+B2caC3Ebi8yIQHIGIlw24idlSZgQLIuzZmH1e5qbMDnsiXQSCNIh42YArVnoJfilLMUKrYWN+2xaflznOi8gXgSAFIl42MHJGipJmHIWqSnByG/MsoRKO9ZFAKKsUKlQiKyvLsuCnGV9fX4Gj3Q+uNaTVG1FOZLC+cD0vdtsFuY1EvQgEKUi2vGJjY9GvXz+UK1cOfn5+KF++PMqXLw9/f3+UL1++KProMrjWkJSyOIKJ2Zw60AUR9o70kEAou0i2vN566y3QNI3NmzcjMDCwVNea4lpDGgnDRksxQjLbSCAUCZLF6/r167h8+TKrNE1phSsnUgJVC1YP4sIfYU98XgSCNCQPG1u3bo3Hjx8XRV9KHFxrSNKwMf+3PcvL7LAHiN+LQJCCZMtr06ZNGD9+PJ4+fYrGjRtb1aBu2rSp0zrnarjWkJTZxoIa9mys4rwY1Spo2tpHRiAQ+JEsXsnJyYiJicHo0aMt2yiKAk3ToCgKBoO00jElGa4lJLWOPWAjzotTVQIwWWMynoEmgUCwRrJ4vfvuu2jevDl++eWXUu+w5w4bpTnsTb9ldqtKMMXLgU4SCGUUyeL133//Yc+ePWViodDCBKkKpQdRAnFezG0EAsE+kh323bp1w/Xr14uiLyUOa4e99PQgrnpZfF4wDR2ZDnsCgSAeyZbXgAEDMGXKFNy8eRNNmjSxctgPHDjQaZ1zNVahEg4MG60XnTXvp61CL4jlRSCIR7J4jR8/HgCwaNEiq32l3WGvlfDeCiqpsrczfV5cHxrxeREI4pEsXtxcxtKMVXqQXkJJHDs17I00zZppNG8jEAjikOTz0ul0UCgUuHXrVlH1p0RhNdvoQKgEd7aRYsR5mauomqHLznOBQCg0ksRLqVSiWrVqpWpoaIvCBKmaa9JbxXlZ2qatho00KUlIIIhG8mzjp59+ijlz5iA1NbUo+lOiKEyQqnBJnALLS8NZC5L4vAgE8Uj2ea1duxYPHjxASEgIqlevjnLlyrH2X7lyxWmdczXcFX0cmW3ker0KHPbWlhfxeREI4pEsXq+88koRdKNkUqhQCYHZxoJQCRCHPYFQCCSL1/z584uiHyUSZxQjlAmIl5GmkccZNhKXF4EgHofLQF++fBlRUVEAgEaNGqF58+ZO61RJoVDFCPN/c4NUWT4vK8tLeh8JhLKKZPFKSkrCG2+8gWPHjsHf3x8AkJaWhq5du2LHjh2oXLmys/voMqxq2EuyvOwFqdJWDvtdV55gYtfSnzNKIDgDybONkydPRmZmJm7fvo3U1FSkpqbi1q1byMjIwIcfflgUfXQZ1kGqjqQHsWGWsOdacssP3iUFCQkEkUi2vA4cOIDDhw+jQYMGlm0NGzbEunXr0KtXL6d2ztUUyvKyOOz5A71MEfbW8XKkICGBIA7JlpfRaLRKxgZMAaylLXWIawQ5FirBRsaKsLduj8w4EgjicKgkzkcffYRnz55Ztj19+hRTpkxB9+7dndo5V8P1WzkSpCpUjBCwTg9inkcgEGwjWbzWrl2LjIwM1KhRA7Vq1UKtWrUQFhaGjIwMrFmzpij66DLMPi8PhRyAY5VUhRadBWAdKgFieREIYpHs8woNDcWVK1dw+PBhREdHAwAaNGiAHj16OL1zrsYsJGqlDLk6g1MW4GCGTuRqrdsj2kUgiMOhOC+KotCzZ0/07NnT2f0pUbAtL51Tchsphq2bq9Nbn0fEi0AQhUPideTIERw5cgRJSUlWTvrNmzc7pWMlAZpheQGOxnkJ+7xytXw+L6JeBIIYJIvXwoULsWjRIrRq1QrBwcFlYvUgtSJfvJzt8+KdbZTWRwKhrCJZvDZu3IitW7fi7bffLor+lCjMAuShNDnspSzAIZQexPJ58cZ5EfUiEMQgebZRq9Wiffv2RdGXEgd3tlFSMUKB9CDmaz7xIpYXgSAOyeI1duxYbN++vSj6UuLg+rwcCpXgbLcX5+Woy+uva08x58+bMBD1I5QRJA8b8/Ly8O233+Lw4cNo2rSpVbT9ypUrndY5V1Pg8zIPG6VnENjyefE57G3FeaXlaHHp0Qt0rlcZSjn7ufPRjmsAgDZhFTCoWRXJ/SQQ3A3J4nXjxg00a9YMAKwW4ihtznvLsFHpiMPedLLQAhyAgM/LRpvDvjmHu4mZ+KRnXUzuXof3mLQcneg+EgjujGTxOnr0aFH0o0TCtbykJWabEKoqAfAPG21ZXncTMwEA/7vxTFC85NzqhwRCKUWyz6ssYdYRs8/LYKRF+5RogShVyl6cl4jmudYcE6niteNCHP68+kTSOQRCScDhSqplAfPQzzzbCJj8XnKZXOgUC0aB9CDAZH0ZacdDJWwNz6Vo1/MsDWb9cRMA8HKTYIuFSSC4A8TysoGRY3kB4mcchdKDTNvM1VSFz7OFLX2yZZVxYVp+ZJaS4G64VLxOnDiBAQMGICQkBBRFYffu3XbPOXbsGFq0aAG1Wo3atWtj69atRdY/boQ9IH7GsSBUwlpMbFlHYqpKyGz816QMG5k6R8SL4G5IFq8TJ05Ar7dOKNbr9Thx4oSktrKzsxEeHo5169aJOj42Nhb9+vVD165dce3aNXz88ccYO3YsDh48KOm6YjHfz3KKgkoudcbRPNtovcfWsE+Mz4tPEM1IES/msaWsjiShDCDZ59W1a1fEx8cjICCAtT09PR1du3aFwcATeClA37590bdvX9HHb9y4EWFhYfjqq68AmErxnDp1CqtWrULv3r1FtyMWS7iDjIJSTkFrcMDy4tGSQlteNs6XMmyUM44ldcQI7oZky4umaV7LISUlxWr1bGdz9uxZq7phvXv3xtmzZwXP0Wg0yMjIYP2IhXlDqyQmZxeIl/VnxbWcVIyAU1Ea4qTZRmbfDES8CG6GaMvr1VdfBWD6wo8aNQpqtdqyz2Aw4MaNG0We85iQkIDAwEDWtsDAQGRkZCA3Nxeenp5W50RGRmLhwoUOXc88bJRRlEW8xDrsbVkyXH1RK2SWGDJxoRLs18wZSimWF/NQYnkR3A3R4uXn5wfAdKP4+PiwhEKlUqFt27YYN26c83tYSGbPno2pU6daXmdkZCA0NFTUucxVr83pOKKHjfm/+YeNBRvVChlkDDUSU8+LK1BMX7ujQarE50VwN0SL15YtWwAANWrUwLRp04p8iMhHUFAQEhMTWdsSExPh6+vLa3UBgFqtZlmJUmCm+Dg6bOSzhJib1AoZ67U4hz0bPUN55BIcAcxrkWEjwd2Q7LCfP39+UfRDFO3atcO+fftY2w4dOoR27doVyfWYZW0ss42iLS/hIFWmr0mtlEPPaFOcw55jeRmF94npo6kNIl4E90Kywz4xMRFvv/02QkJCoFAoIJfLWT9SyMrKwrVr13Dt2jUAplCIa9euIS4uDoBpyPfOO+9Yjh8/fjwePnyIGTNmIDo6GuvXr8fOnTsxZcoUqW9DFHw+L9GVJUTONposL+awUQScNtmWl2PDRhLnRXA3JFteo0aNQlxcHObOnVvoMtCXLl1C165dLa/NvqmRI0di69atiI+PtwgZAISFhWHv3r2YMmUK/u///g9Vq1bFpk2biiRMAiiwgmRMy0uiw54/SLVgm4dSzkrQFpMexNUnpuUll/L/4Bk2Xo17gUV/38Hc/g3Rolp58W0RCMWMZPE6deoUTp48aSmLUxi6dOli82bli57v0qULrl69Wuhri4EZ7iB1tlFMehBgjt5nWF4OJGYz/VUyCZYX81LmYeOQDWdgpE2/YyP7iW6LQChuJA8bQ0NDy0yddablVTDbKLGqBA+2HPZiRm9cQWQOGx21g80CaL5+GfkXE9wYyeK1evVqzJo1C48ePSqC7pQs+Cwv0bON+b/5HOhsn5ecJTgOhUpIDHNIyszDiXvJLIHSS1hchEAoCUgeNg4bNgw5OTmoVasWvLy8rMpAp6amOq1zrsbIGyohLv2JZsxUcmHFeSllrNeOxFsxh41iJKjTl0eRpzPis34NCq5LTC2CmyFZvFavXl0E3SiZFMw2FjjsxQwb9Qaj4AIcpvYKtipknDgvBywvg0Srybxe5PF7yQVtkNlGgpshWbxGjhxZFP0okbCCVEXGeeXpDOi24hiepecBsF/XX0axBU5UkCqnSZblJUGDmNYWsbwI7oZD9bxiYmLw2WefYfjw4UhKSgIA7N+/H7dv33Zq51wNK0hV5GzjuYcpFuEyn8uFWY+LotgC59Bso4NWE/M8BxZGIhBcimTxOn78OJo0aYLz58/jjz/+QFZWFgDg+vXrLo2+LwqMDIe92NxGrvjwWV5M8ZFRlOQEaW40hKPixTyNDBsJ7oZk8Zo1axaWLFmCQ4cOQaVSWbZ369YN586dc2rnXA0rSFXkbCPXZ2XP52WyvJjni0HY8hLjM7McS4aNBDdGsnjdvHkTgwcPttoeEBCA58+fO6VTJQ0pidnc2ULeIFXWfooVhS8kIuyyN5xrOig87GEjES+CeyFZvPz9/REfH2+1/erVq6hSpXSt1MxOzDYpht1hI+c1X3oQU9AozmshHWKKi3WQqhOGjcTyIrgZksXrjTfewMyZM5GQkACKomA0GnH69GlMmzaNlURdGjBbUVIsL272gb04L4qiWK+FsheYAmXTYS9Bg/jSgwgEd0GyeH3xxReoX78+QkNDkZWVhYYNG6JTp05o3749Pvvss6Loo8tgBqmaHfYaiZYXX6oh22HPCZWw0xfu+UAhHPZk2EhwYyTHealUKnz33XeYO3cubt26haysLDRv3hx16vAvP+/OMBfRsJTEkWp5iRg2Mg8RsoBYQ0OnzTYS8SK4Lw6vmF2tWjVUq1bNmX0pcfCWxJEYKsE33UhxQyWY5wu0y4yity4DLS09qOA8RvvE50VwM0SJ19SpU7F48WKUK1eOVQ+ej5UrVzqlYyWBgrI2EmYbuXFePMewhpIUW4yEZg5tiYvDDnsybCS4MaLE6+rVq9DpdJa/hShMYcKSiJEnPcjebCNXZOwFqVJgB6kKmU6sWC7ONYysfTa7xz6PxHkR3BhR4nX06FHev0s7rMRskZaXniNu/DXsC/42OeyZlhd/uwYbAuUcn5dDTRCchM5gxJ1nGWhcxc/hUt5lDYdyG8sKNN9so13xYguJjOcTplihEtwIe4Fho1HYSnJ02MhshoRKuJb1R2MwaN1p/HHliau74jaIsrzMC86K4Y8//nC4MyUNvsRse8NGnZFredkuRmgaNtq3vPQ2xIvtsBcvQizLiwwbXUpCRi4AIC41x8U9cR9EWV5+fn6WH19fXxw5cgSXLl2y7L98+TKOHDliWZi2tGDWIZbD3o54cS0ve0GqMhm3JI4Yy0t4nxRIYnbJwfxvZy7GQrCNKMvLvOAsAMycORNDhw7Fxo0bLUudGQwGTJgwAb6+vkXTSxfhyOpBYpZGozivmENLwVAJGw57Z5TEIQ77koG5UCTBPpJ9Xps3b8a0adNYazTK5XJMnToVmzdvdmrnSgrsdRtt3+Rc/5P9kjjsoWVhLS8pGkSTINUSA7G8pCNZvPR6PaKjo622R0dHw+hIAfYSDO+K2RJnG/kmjlgR9lyHvYjZRq6V5Ki/igwbSx55Ihd4ITgQYT969GiMGTMGMTExiIiIAACcP38eS5cuxejRo53eQVfCXDFbKTJUgmuZ2Vt0luIcIeywNwoew67nJR4x6UF/XHmCvTfi8X/Dm8Nb7XBCBsEO5okWYnmJR/K3ccWKFQgKCsJXX31lKY0THByM6dOn45NPPnF6B10JX5Cq1mAETdOCAblcEeBfdLbgb5lVGWh+EWHNKDrJ5yUmPWjqzusAgG+Ox+CTXvUcuk5pJk9nwNBvzqJ1jQqY27+hU9ojiEOyeMlkMsyYMQMzZsxARkYGAJQ6R70Z2mJ5FYRKACbrSqXgFy/rUAlruCVxxCw6y5zFtBo2OhznxWjTThvPszQOXaO0c/B2Am48SceNJ+mFEi/zv0JDHPaiKdQ4oLSKlhk+nxdgmlFkihkTMaES3G3sl/ZzG7muRVszkbaQEmEvdqXwsoazJ2nzRK4LSnBQvH7//Xfs3LkTcXFx0Gq1rH1XrlxxSsdKAgXiRbHESqs3opya/xyuw57P9rKqKiEiSNWWw94pZaDttEEc+vw4K5XH/OkSy0s8kmcbv/76a4wePRqBgYG4evUqIiIiULFiRTx8+BB9+/Ytij66jIJhIwW5jLJ8UW0Fquo4Nzl/McKCvymK/VpIQ/Q2wiGKIz3I0WuUdpydh1gSLa9vjsdg+LfnkKstWX2TLF7r16/Ht99+izVr1kClUmHGjBk4dOgQPvzwQ6SnpxdFH10G0+cFAMr8Ova2ZhytErMlxnkJWVFGW6ESzphttGN5WVuUBMC6tpqjlOQ4ry2nH+HswxTcelay7m/J4hUXF4f27dsDADw9PZGZmQkAePvtt/HLL784t3cuhjnbCIgrSGjl8+I5hm15setAC0mIzdxGRyPsJTjsic+rAKYF4nTLq4QNG41G2jJZU9KS9yWLV1BQEFJTUwGYqqma12qMjY2V5Cx2B5gOewBQKUxZBbYsL+6wkf/BTLH+EpPbyLa82PucsnqQ3WFjybqpXMWCPbfRYN4BXH+cBgBQOM3nVTLjvNJydZbvV0m7uyWLV7du3bBnzx4ApoDVKVOmoGfPnhg2bBjveo7uDDNIFRC3/Jn1sNH6GK7lxV49iL/dO/EZjGNsOOylfMNExHlZ9pewp66r2HrmEQBg5aF7AACZjPm/K/xnpNEbS5QRwAyRKUHdAuDAbOO3335rSQOaOHEiKlasiDNnzmDgwIF4//33nd5BV0JbWV72o+ytIuztVVLlpAfx+bzOPUzBmn8fMI5h73c4MVtCbqOYhPOyhNkSZVpeBiMNhdxBS4zx8Wv0Rngo5cLHFiPJmQzxKmG2lyTLS6/XY8mSJUhISLBse+ONN/D1119j8uTJUKlUTu+gK7GyvESIF3d4xevzkrH328tt/PvGM06/bDnsHY3zsuewL1lfXFdj/jyYDyJnzciWpKEjS7wK8/Z0ucC9g4XvEANJ4qVQKPDll19Cr9c7tRMllYJKqqbXSkcc9jyWF3N20bR6kO3ZRq5YOsvyYoVK2Plmcn15ZR2zUDEd9oUpK8Q8syQ57Z0iXg+PARvaA7+8ATy76pR+AQ74vLp3747jx487rQMlGfP9SkmwvLjDK3s17K3LQPO1yd5qldvoBGeEPQE0EIc9iwLxst5mC5qmMXXnNXz6503BY0qS5cXyeUkdNmanAH+OB34cBKQ+BLwDgTznhVtI9nn17dsXs2bNws2bN9GyZUuUK1eOtX/gwIFO65yrUdAaADJpoRIiZhutcxttO32517M5bHRQx+y5tMiwkY15YkbO8AGICSWIT8/DH1eeAgDm9m9o8W0x/+8lKVDVIcuLpoHrO4CDc4DcVAAUEDEO6DYX8HBeSqFk8ZowYQIA/vUZKYqCwVByPvhCoc3B7/R0HFK0gFwfAaCcqDr21qsH2athzw2V4OlKIYaNGXk6TN5+FQPDQzCkZVXB4+wOG4nDnoX5M2f+L8UM35kPAaGPvEQNGxmWl6hhcUoM8PcUIDZ/dBbQCBjwf0Boa6f3TbJ4lbaCg4JE/40wKh7vKfZCv+06MHA1VHJ/APYc9vbTg2ytHsT3/ecKh5SqEifuJeP4vWSk5epsipf9YSOxvJiY/yfMh5PUz4g53Gf7vEqOAcCebbSBXguc+Ro4sRzQ5wEKD6DzTKD9ZECuLJK+kaXPhGg6FO8bZuAJXQmKzCfAttfwwfPPURlpdtKDOP9iu/W8OHFePF8R7vVsrdvI3ZeQngcA0Nkpomg3PYiIFws+oZLqezQIDMVLknixSiEJvb3HF4BvOwP/LjYJV82uwISzQMepRSZcgATLKzc3F0eOHEH//v0BALNnz4ZGU/DG5HI5Fi9eDA8PD+f30kUcpVvgpKY+LrQ7D+9r36FV1lEcVp/HzbhPgDZTeRdlFLf0mXCEPZ9GWA8bxTvsn6WZxMtehLzdxGzi82LBly4l5jNiPpyY/xPmv9De2qDFhd5gREp2QdUYqwdrXjpweCFwaTMAGvCqCPSOBJoOFUotcSqixeuHH37A3r17LeK1du1aNGrUCJ6engBMNexDQkIwZcqUoumpK6CBHHggs8tCeLcejic/voeqeffwUtRiYOth01i+Mru6qLilz9j7KTtOLynDRu7Z5vUA7VlOBiNtsdL4MN9oOVo9dAYafp5F90R1B/gsLzE+ITFliEqK5ZWarWV9HS1/0zQQtQfYNwPIyo/5bDYC6LUE8KpQbP0TPWzctm0b3nvvPda27du34+jRozh69CiWL1+OnTt3Or2DroSVmB3SHBvrfovFuregk3kAcWeBDR2Ao18AuoKb3tphbw3TGjP5v2zX89JyBNFWMUIu8fmCZM8qMNI0Rm6+ILjfbGk0mn8Q4Qv/QbambMT6CWEWc7YlJVG8BB46JaWmV1Imu3quzkAD6U+AHW8CO98xCVeFWsA7e4BX1hercAESxOvBgwdo0qSJ5bWHhwdkjGFTREQE7ty541An1q1bhxo1asDDwwNt2rTBhQvCN9HWrVst4QXmn6IaqnITsxUKFb43vIwtzX4F6vQGjDrg+DJgYwcg9iQAnnpePB57VoS9VT0v+5aXzdxGDvHmYaO9xXKNNO4mZgrvzz/ffKnY59k22yvt8AmVmFAJ5nlCD5SSEirB9HfJYMT5X5Ygb3Ur4O4+QKYEOk0HPjgD1Ozskv6JFq+0tDSWjys5ORk1atSwvDYajaz9Yvn1118xdepUzJ8/H1euXEF4eDh69+6NpKQkwXN8fX0RHx9v+fnvv/8kX1cMQulBzxWBwJu/Aq9vNQXepTwAfugP7J6Icnp2EB5/kCq3kqr1NZnYC5VgT78zbw4jkjLzHfaFnE3k3qwlLUm3uEnL0eGNb8+yfF+FsryYcV4lZNhonmlsRD3Cn6p5mK/8CR50LoxVI4DxJ4FunwFK1/m4RYtX1apVcevWLcH9N27cQNWqwlPxQqxcuRLjxo3D6NGj0bBhQ2zcuBFeXl42F7ClKApBQUGWn8DAQMnXtQfzy2QWL1YxQooCGg0GJl4AWo42HXjtZ+wyfoxBslMwDwTs+rzAWXSWpy/2fF5ClldylsYidPbEyZ6/hntjkhW2gXMPU3EkKtHyWlSclyifV8kYNr5IT8NsxTb8pfoM4bKHyKC9MEc3Bld77AACGri6e+LF6+WXX8a8efOQl2ft1M3NzcXChQvRr18/SRfXarW4fPkyevToUdAhmQw9evTA2bNnBc/LyspC9erVERoaikGDBuH27duSrisG5vfQLC2q/FXCWRHvnv7AgNXAuweByvVRkcrA/6nW40flUoRSieCzvdg+L25ithiHPXu/0E1jnmnka4MLifNyDObnKuYzYqZZCfm8SoTldf8wXj//Ot5X7IWCMuJvQxt01yzHdkN3XH+SYf/8YkD0bOOcOXOwc+dO1KtXD5MmTULdunUBAHfv3sXatWuh1+sxZ84cSRd//vw5DAaDleUUGBjIuyo3ANSrVw+bN29G06ZNkZ6ejhUrVqB9+/a4ffs2r+Wn0WhYw1nzcm324LO8bOY2VmsLvH8SXy2YiEnyP9FJfhP/yGYi9WYCUHU6K96FaXlZxXnxfP+5U+c0TYOmaYzachEAZ01HxnHM2UO7DnuJD/uLj1IRHuov7aRSCPP/JSbOi/l/EPR5udLyykoCDswCbu1CeQBP6YqYqxuNf40tLIdcf5Lmsu4xES1egYGBOHPmDD744APMmjWLUeuKQs+ePbF+/foiGb5xadeuHdq1a2d53b59ezRo0ADffPMNFi9ebHV8ZGQkFi5cKPk6LMsr3z61V8OeliuxRv8K/ja0weeK79FefgdVLi0DHu81hVVUbWVqz8baZ3xBqnzDxhc5Ohy/lwwAqB/kw9uf+PRcy992LSuJw8Ale6PQqkYFNCvjAsb81MRZXgXHCBWRdInD3mgErv4EHJprit+iZNjrNQjTUwYgB2y/lrmKrKuRFGEfFhaGAwcOIDk5GefOncO5c+eQnJyMAwcOoGbNmpIvXqlSJcjlciQmJrK2JyYmIigoSFQbSqUSzZs3x4MHD3j3z549G+np6Zafx48fi2rXyGN5qe3kNpr9GbF0MN7UfYpPtOOhV5cHEm8Bm3oAe6cBeRmcBTgoB4JU2TcBcxjLvB/iGZYXN3iWiyPDwouxqZLPKW2wLC8RnyFz4kTIwV/sw8bke6YJp/99aBKuoKbAuH+xUjYKOfCw8ts+SslBWo6Wv61ixKH0oAoVKiAiIgIRERGoUMHx2A6VSoWWLVviyJEjlm1GoxFHjhxhWVe2MBgMuHnzJoKDg3n3q9Vq+Pr6sn7EwPxSmod59krisIcBFHYZO+Hua0eApm8AoIGL3wHr2qB+2rGCoyhwqkpYt8v9jhtpmiVYORr2l/1BUhaSMvNYw0aatn1zOeKAJylDbEtZus+L8dBhtFNscV56DXA00hTq899pQOllCjQddxQIaW6ZbazIs0jptRJgfRVqxWxnMHXqVIwcORKtWrVCREQEVq9ejezsbIwebZrBe+edd1ClShVERkYCABYtWoS2bduidu3aSEtLw/Lly/Hff/9h7NixTu0Xn+Vlrxghn3Vj9KwEvPoNEP6GKdv+RSyGZs6Gn7IV5utGggLbByZGRGga0DCezsyA0cSMPIz/+TIAoEU1f3b/DEbIZfzlhR2xvMiMo3TLi/mAE5pDKRbL69Fp4O+PgeemWvyo3RPo9xVQvjoAQKM3ICPP9L0K8FGzcxwBXH+cji71Aoq+nzZwuXgNGzYMycnJmDdvHhISEtCsWTMcOHDA4j+Li4tjBcO+ePEC48aNQ0JCAsqXL4+WLVvizJkzaNiwoVP7xbwxxdaw53PAWoyqWvnJqieWw3BqNXrLL6G97DaiHn+I63QvyX1jOvGztQXiFZ1QMCERz0n3sXVzOSJeZAaSjVSfl1C+aZ7egAO34rHq0H383/BmqB/kvBpYyH0BHJoHXPnR9LpcANB3KdDoVda09/Ms07BQKafg71Uw2SSXUTAY6RLhtHe5eAHApEmTMGnSJN59x44dY71etWoVVq1aVeR9YodKiCtGaHd5MKUn0H0etqa1QLPrC9BSdh8R0csQ6LEH0dQ7iKKri4rS5ooX8xQlo7QnV7xszTg6YkUR8WIjOc5LoBpIns6I8T9fAQDM+P0G9kx6qfCdo2ng1i7TTGK2aaIHLUYCPRcCnuWtDjcPGSt7q1k+2sZV/HD9cRquP04DTdO8Zc6LC1ISRwgen5fSAcuLb0Xl5+Vq4zXtfHymGw2twhvV86LwP9WnmKX4BXKDcHK0GSNn2MiEKV4Au8a6Lae92BLGTIh4cTIaJFte/Mdr9PwuAYd58QjY9hqwa4xJuCrVA0bvBwZ+zStcAEO8fNQsh33zUH8o5RRSsrV48iKX99zigoiXALyzjXI7s422ho0MZBRAQ4afDT3xd8fduO7TGQrKiPGK/2HE5aHAg8MF/eD5gtMcy4sJdwXnIF8Py7a7CZmiFrUVgitWzqid7+4wPwEx6y2yVj63YXmZKdQSaAY9cPprYF1b03dKrgK6zDGl9lRvb/NUs4+rkreaZV35eirRINg0jHX10JGIlwCO+Lz4LBt7NezzPALxU7XFGKP9BE/pivDTPAN+HgLsGgtkJfMOUY20cM0nroBW8lZZ1hYcsek8fr3IHyoiJEQsy81ALC8uUvWbOcMoJlTCYfF6ehn4rospbkufC1R/yZRE3WUmoLCePeTCsrwY2z2UMoRX9Qfg+ngvIl4CFKwcVBDKYJltlOKw50sPYsV5mX6OGFuil+ZLXAl+wxQVe/M3YG0rGC//CG7Go8nnxT9s5PrdFHIZayi54XgM73lCI0qFjWEnES82Yj4NQZ8X42ym5eUpVbw0mcD+Waa4woSbgIc/MHAtMOpvoFId0c0IDRvVCrkls+L6Y+etBOQIRLwEKFizseA/Z7G8BBzffMNJPsuL4uw3C1w2PHE0bCow9ggQ1ATIS4PXgY/xq2oxalFPGX0TTiHhWkdyihK1tqCQELHES0/Ei1vhSOpyYEJVJZikZBeEJXgoJdyid/ebhojnNwC0EWjyOjDpEtDibcmVTYUsL7VChmahfgCAm0/T7ZZaKkqIeAlQUA6nYFvBsFHI6uFz2Fsfxy4DzS2JQwNVWgDjjgG9lsCo8EQbWTT2qWbjI/kuqKADIBwLxP0yyWQFaU2A8DBHjP+KO1Qti+KlkAvfMmKGkOw4L36fl2RXYkY88OvbpkVdM54A/tWBt3YBQzYB3pUlNmaC6fNifl/VChlqVvKGt1qBXJ0B9xKzHGrfGRDxEsBSiJDx3FFZHPZS0jr4atgz9lpVlcj/Q64A2k/Gf8P+xVFDONSUHlOUu7BPNRsRVJSweHEERSGTQcGIkxO6MYQc9syt7Zf+y9pXFh32Cs7TSLrPq+CEH88+Qq9Vx/EsTXjWLtdWwKrRCFzcBKyLMJVlpuRAh4+ACeeA2j2EzxOBeckz7rDRQymHTEahaVWT9eVKpz0RLwHMXzGKz/ISMJUP30m02sbrsGfcANxFZ7kaku1VBaN1MzBJOxnJtB9qy55hp3ox2t1eCF9YP/W4Q1eZjIKCZXkJDBsFttu6OcXMUJY2rMSL9be02cYrcWm4l5iFpfujBc/M1QqIV+IdYEsfYO8ngCYDCGkBvHcM6LkIUHnZ7Yc9mHFezAewOb+3wO+VVuhrOQoRLwHMNybL55VveRmMtNWQSaM3YNeVJwCAwc2rWLbb8zRQnGO4N4DJMU/hb2M7Uz0lfVcAQKOE3TiinoaBsjOAjfw6OcW+4YT0RmgIaOuGLIu5jbaGjWIw8MyM2EoHyuX6NnW5wJFFwDcdgcfnAZU30GcZMPYwENy0UH0zk63RIydfNK0c9vkTCOYZR1fmOBLxEsBscTAftOYgVcDawvnndiJe5OgQ6KtGt/oFOV98EcisqhIygWFjPswk3Qx4Y45+HF7XzEOyRw1UpjLwtWotflAuQ1UqKb9fHPGSUawbTkiMBIeNxPJiYWvYKMrnxfOZUZSt1bMZwvbwOLChPXDyK8CoB+q9DEw8D7QdDwjkrDqC2d/lqZSjnFrBugc88u8Bcymke4mZyNG6ZjEWIl4CGPlmGxkiwHVem+OnhrYKtQwvAX7Li10GmmKXgaa5lpf1k/oiXR/Lqn+Hr3SvQUMr0Fl+A4dUM/Ce/H8wGnSca1GsG06qw97W/VgmLS+rGRjHZxvNUKDw5EUO7/G5WgOQnQL8+QHw40Ag9SHgEwwM/Ql4YzvgJ730uj2YM43m/pkxW15Bfh4I8vWAkQZuPXVNZVUiXgJwVw4C2LN2zFivuJQcnHrwHBRlEi/mF5wvPYg120hxq0qwjxWK57ryLAdrDK+ir3YpzhkbwJPSYo7yF8yPn4CmVEEsl5zj8xLSG6FJCFtR42XRYS+X27C8RJzPJ/gHbicgOoFv5SYa3bX/AmtbAde3A6CA1mNN1lbDgUWysKtp0RaOeLHivAokIzw/ZMJVfq8SkZhdErGESnCc6yq5DFqDkTVs/PVSHADgpdqVEFrBCw+SsxjnWLfN3MZ12FsNGwUCYh8mm5Yee0iH4A3tZ3hdfhyfKrahuu4hdqvm4QdDL6zQDzU57Fkre0uL87KlT0LL1ZdmFJxV0gsz22iL6lQCPld8j5dwG8gFENDQVI03NELaBSWQmJGHPqtP4EWOyXo3OethFSphJjzUHwdvJ+Kai2YcieUlgKXMNWc7N0VIbzDit0smR/3wiGoA+IYWbChWnBcbbhCpuMJ0FH4zdEF3zQoc9+gKGUVjtOIgDqmnIzz7jKg4LyFsHX7gdgIy83Q2jih9WM82MmO1RMw22hF8BfSYIP8LB1Uz8ZL8NvJoJYzd5gHvnyhS4QKA1YfvW4QLKLC8mF9SZrpSMxenCRHxEsD8FeMO+7jhEv9GJyEpU4OK5VTo0cBUg0zOGRZy4S7AYcv6l1LPPAV+WOk9DW9rZ+E/YwBCqFS89/RTTEtbgkCYSjZLLX1j74ac/cdN7L8ZX2ZW0OYmvkuFb7bRTAvqHv5WfYoZyl/hQelwytAIvbXLkNvmI9YCLkUFsxYcYApQBcB6gjEtr8ZV/UBRwJMXuVbFCosDIl4CFPi82F9W7iIcZkf9kJZVLcIm5ww1uVj7vITTd2xZXnUDva226Qw0Thqbord2GdbrB8IAOdrkncZh9XS8Lf8HFC2tSqc9qfv7Rjw+2HYFn+y8Lqldd0Vhw+clBj6flw9ysEixBb+rFqK+7DFSaW9M0X6At3Rz8B8dxBuouv18HH4695+0i9vhLsfvZra8mHGNaobl5euhRK3Kpu+gK6wvIl4CmB+Q3Act0/KKT8/F0bumEIVhrUMtx8hZDnvrtll+NADNGeWarX1ewmJjjrVhcife9PTMgxpf6t/A/9XehBhVffhQuVis3Iqt9FzUp+IE2+Qi9uY8cDtBdJvujJXPS+L5bJ8Xjd6yCzikno53FIcgo2j8buiE7poV+NPYEebxGjdQVWcwYu5ftzB39y08THZeek4O5zoW8WL4XT0U7PfvygoTRLwE4AuVABjVVPUmX5eRBiLCKlieQADH8uIJlvBiPL0oikK/JsEIq1QOAF+QqrDlJWbdxGSv2lhW5WvM041EJu2JpriP/6k+xQzFDqjh3BVgxPh83B1nxXkFIwXfKVfiG9VqBFEvEGsMxHDtp5imG48XYJd95gaxMoOkD962zupwhIR06yKYfOLFDdI1J2lfe1L8FSaIeAnAF6QKFJTF0eiNliHj8IhQ1jHsYaN12+XUctZ+iqIsUfnWoRL84lW1vCcqeavsvg8ZRUGhUOBHQ2/00CzHYToCSsqACYo9+Ec1Ay/JbgqeK1WMhmw445CApefqii3gNT1Xh7+uPRVOu7GDPZ9XUmYedl56LBg1b9DrMEp+AIfU09FTfhk6Wo41+lfQR7sMZ42NeK/FHTYyP+KDTrJ4T95Pttpm/n4JpcMB7DSh4n54kVAJAYL8PPBZvwYop2Z/RGaH5dHoJDxNy4WvhwJ9G7OXXZNzhoVcvFQFbZotO/Nx1hH2/DcB35CRD4WsIBQjERXwEf0J2uvOY5FyK6rLkvCzKhJ/GF7CEt1bSOU88aV+F6/EpSE9Vwd/L/uiaiY6IQN9Vp9E13qVsWV00c6mAcB7P17C+dhUvNayKla8Hi75fK4lzl36rOfKE0jP1eFBUhYmdqkNtVJWMEOXcBMfPZqIGkrTavCXjHUxWzcW92n+QFNvtQLpuToroWX6Ra89TkNCeh6C/Dy4p0vi5P3nVtvMDnuh+nUAUD/IFyqFDOm5OvyXkoMa+SOI4oBYXgJU9lFjbMealvAHM2af1++XC/IYudUu5ewQeiuYgmjebfaD0TSNE/eS8SDJ5MsQsrxMAYL2Z75kMoqTOwkcMrZCD81ybNH3hpGm8Kr8FI6op+F1+TEwvTjF8Rz96azJ6Xz0rvWTvyg4n79Q7u+XnyDDCWEeTIH/5LfrSM81tbnr8hN0X3kcQzacAbQ5wD9zgW86o4YmGhm0Jz7VvYvXtfOshIuZneGd/z3hWl7cSZ1/7hTO+jIaaZx6wBYvXw+F5XttS7xUChkahbimLDQRL4mYh41Z+aEBb3DEDYDdCHvusJHJ6ZjneGfzBXz4y1UANsSrqr+oAGs5RXHqrJt+Z8MTC/UjMVi7EHeM1VGeysJy5bfYrvwcYVQ8AMfWDxSK1BeC+fnkaPWC6wMUBUM3ngXAroFmMNIYteUCPt97R1QbQqEnKdlaPM/SoGLCSdDr2wJnvgZoAy6V64QemhXYZugBmuf2Y6agmcWL+3/gjrAP3CqceN2Jz0BqNtv/aYnxgu1hI+C6JG0iXhJRcSKMzYsRMJHZCEIFgHI8w0bz78epptpOCRkmByp3trFhsC8iwiqgGWdBWSEoCiwTykjTqF6xoGTKdbo2BmoXI1I3HLm0Cu3ld3BANQuT5X/gjQ0nRF2Did3l3/j6l0/DeQfRcdlRydd0lOiETNx8ko6mC//Bxvzy2OcepuDY3WR8dzIW+27G2w1HEHLVVUQ6VivX4kfVMlBp/wG+VYHhO7C+8jwkobxge8yAYm8PfsuLaxKfj03Fi2zHJ19O5Pu7yjPWZ7TEeMG25QUUJGkX94wjES+JMJ+Mw1uH8h7DjuPis7wKxIsvhxIomB7nxnmtH9ECO99vB7VCLmLQaP3FM9K0VV10PRT4xjAAPbVf4rihKdSUDp8of8fK1EloRUWLuAqjLRs5kkfvJiE+nV14j/sezKJdXMzfcws5WgOW7je9T6aVMWHbFczdfQuxz7MFz7e2vGgMlR/FEfU0vCI/AwNNIafF+6Z8xHp97SazM9cbMH9PcrXW/0Mz9QJ9YDDSOBzl+KzjqXx/V8c6BVVXmZaXrRlvoMBpf+tZhl2hcyZEvCRiLovjpZKjf3gI7zH2HfYF4mF+qnInsXJ1BhiNtCXCvnPdypjbv6Fkhyj3i2drWPeEDsBI3Ux8qJ2EZNoXdWRP8bt6Eb5QbOItfMiH0M15OCoJo7dcRLtIdjVWPnEvzlI7XH8l3/8rxUb0OLOvNaln2KFagi+V38GfysZtY3W8ol2M5x0WAGpTKI2tCHuA/eDzEeHz6tM4CAB/yERSZh4GrT2Flf/cFbxejlaPS49eAAA61qlk2c4aNtrJ8qhR0Qu+Hgpo9UarQFcmjrghbEHESyLe+UO+geEhFp8EF3bitfV+ZoqFOTCQLx4sT2/AszSTJfJ22+oY81IY5zr2bS+N3mgVO8ZfwcDSKvYY26OHZgV26LsAAN5U/Isj6unoLzsLe258bg391GwtVh++h98u8S+5xoetxXGdDfNBIoStZGojTUMFHT6U/4H9qlloK4tCDq3GEt0IDNQuwU26JrIZ9a7s5TYyxd/sG7Xl8+rbxCReJ+4nW6VobT71CNefpGPN0QeIiucvW3M+NhVagxFV/D1Rs3LBg1GKz4uiKIv1xZeknaPVI3J/FLquOGaZ0HAGRLwkMvqlGhjRpho+6VVP8Bh7QapM0TEPD6uU9wQADGpWYM1df5yO2OfZUMlliKhZwbodEf3V6A3SF3QAkA5vzNK/h6GauYgxBqMylY61qjXYrFyOKhCeGeRadh//eg2rD9/HPzwlsp9nabD1zCO7bRQlnir70ULDvj0HncGIuJQcnH2YwtpXK+cG9qlmY6ryd6gpPY4awtFL+yU2GfpBnp+PyIxct1dVgmlVmUNq+CLsAdODsV6gD6pX9IJWb8TxewX/lxytHr9cMGVS0DSw/CC/9XXynmnI2KluJTC/UVJ8XoCw3+vQnUT0XHkC3xx/iPj0PPx945ndtsRCxEsi9YN88fngJqwnExe+GUYhzF/s3o2C8Nv4dlj+WrjFMjOXle5UtxJ8PRxLzC3sCj8X6Aboq12K1fpXoaEV6Ca/hkPqGRgr3ws5rIcBXIf9iXvCQjftN/58SO4Sa0WJF3fYKPC/OxqdhB4rj1te+yILXyi+w4z4Kagte4Zk2heTtJMxWjcDT+jKCKtUzpI1kas1YMeFOIz94aLVrB4XpuXqmW8VcoeNp/PDGuoEeIOiKPRpZLK+mLOOu648RXquqbKvXEbh3+gkXMgPE2FiDk5l+rsAtuUl5ivETRN68iIHY3+4hHE/XsLTtFxU8ffEpndaYUSb6vYbEwkRryLAaikzGwTkf0nkMgqta1SASiGzfGn33zSFLHCDYPmuIwRNO7CUFgctlFitfw0vayNx3lgfXpQGnym34S/VXDSmHrKOlWI1HROI7SrOcAlPEcNGAMjW6vOHTzT6y87iiHo63lSYZka367uiu2YF/ja2g9l6mdS1tqXtHK0es/64icNRSXhow/kPsIeN5okVbs7hnusm62VAU5OV3itfvI5GJ0GrN8JopLHldCwAYHznWpa826X7o1hR8PHpubiflAUZBbSvVZH1farsLfxw5qNpfprQg+QsfH3kPnquPIHDUYlQyCiM71wLh6Z2Qo+GgZLatAeJsC8CbFWJMPPTmAiciUlhDRPNeCnlSIMO2VoDlHKqUP90GtIXRhUihq7CKnzYWPYIf6nmYquhD77Sv44ceDhlEVJ7PhZnwvR5GYy04FBcozOiKpWMxYrN6Co3WYwPjCHYETQNm+KCrI4f0rIq/soXmRc54sMYmJZyxfz0nATGDO3zLA3OxJiGrgPyJ4yah/ojwEeNpEwNzsQ8Bw1TsUoftQKvtwpFtkaPP648wZW4NPxzJxG988XOPMvYtKo//L1UoOmCSZkAGyMLPgJ8PFDF3xNP03Kx8tA9AEBEjQpYMrgx6gb6SGpLLMTyKgKYjnyhJ3vHOpUxs0993tVoPBjnvFS7Evw8+YeMQpbXsiFNLH/bs7o61a3MOt4eNGTYaeiK7pqv8JehPeQUjTGK/Tikno7usssWy+HrI/ex/bxw9QpbM0/F6fNihq1ka/W8Dxs5DKj1YAv+Uc1AV/l1aGgFVumG4GVtJO6qGwu2XSvfAX7lvzTR/WGKlznZn2mt7b+VAIORRtOqfpaZZ5mMQq9GpgfcwdsJ2HzKZHUNax0Kb7UCgb4eeLeDabJn+cG7lgeMOSWoU/4s44vsAmd6hXLiU7zMtKpR3nLu8tea4tf32xaZcAHE8ioSPJRy/PpeWxhpdh6jWJhxWC834R8y2mJY62qYuasg4dqWgFEAb6CtPZ7DDx/pJuEPQ0csUWxGqCwZ36u+QtLx23go+9zy9BUixYbvpziHjcxsiByNwcpH2Jh6iKXKTWh87xFAAeeN9TFHNwYxtDmRXvjDrZd/457gSXoWQs8jXo9Tc5CnM8BDKcf/OENGM30aBePnc3H43/V4ZGn0kFHAyPY1LPvf71wL287H4UFSFsb/fAWNq/gW+LvqmvxdWYzZSkeWeJvzcgO0qlEB/ZsEo7wD4icVYnkVEW1qVkS7WhUdOtc8lFHIKPRqaD0kMcM3k8mHLTuGogpXHfS4MRy9tMuwUd8felqGgMcHUO2XznhLfggUhEXIVuxUcQY6Mj+bLI3eIh5eyMNcxU8mv57sEbJlPpihG4c3tJ9ZhAsoqPvGR90gk3jF85SbEcKcJwiYqjr4eihgpIH/UnIQn56Li49MTvd+TdkPtTY1K8DPU2kRoF4NgxBaoSCTws9TicndagMADkclWko+e6sVlpnCXo0C0TDYF+93qslqe/2IFvBRK7BlVGubfQ/09cDbbasXi3ABxPIqkZgDJzvUrgQ/L3GzjB3rVMLJ+89RO4BbXZWGraAKCmwfXbuaFa3CAeyRCw8s1b+JPYb22B70C/xf3MQS5Ra8Ij+NOboxuEezMxH0BqPNssHFaXkxLadsjR4GI41usitYpNyKqpRpWPWXoT12VZiAE8+sn/XZNtYsrMP5XwT5evBmEIzrGIaT958jOiETk7vVQfWKXqhYTgWKolArwBtX49IQk5yFZ2m5oGmTLynE35PVhlIuQ/cGAfjjylMAwLucmEAAeLdDGKqW98SDpCw8eZGLZ+l56N8k2BLV76VSYN9HHa3Oe7lJMPo0CmIV0SwJEPEqgdSq7I2T95/jtZa21+Rj+o1WvB6e74xlO/dp2vas5NG7yfioR13L600jW6HR/IM2r+ulklvNgAHAHboGXkqZg2/qXUH4vTVoJbuHvao5+MbQH2v0g6GB6YmsM9B4nmlr2Fg0Pi++QE3mqE+b9gzhZxfgZZXp/T82VsZn+ndx3BgOnxQFAGuhumGjCJ+Ph9LixAaAmpXL8YrXp/0aIkerR+zzbDQM9mWFa9SsZBKvK/+9sMTKDQjndyUMDA/BH1eeIjzUH61rWOdPymQU+gjMXNujpAkXQIaNJZJpvethz6QOltkkIdIY0coBPmq83bY6AnzYdZ1o2F/NKIfh6xATcR75qrCDP0tLY8TN5uipWY5/DC2hpAyYpPgLB1Qz0V52C4BpNvF5dvFbXn3/76TVtq1nHoGCESPkh9FsTy9UeXYQelqGjfr+6KVdhuNGU82vTJELjIRW8MSO99paXtcLKnBYMyPYzZh1ykulQKMQP6s4s1oBpnM2nYpFXGoOqvh7YmCzKtxmAABd6gVg+7g2+H5kK1HZF+4OsbxKIN5qBZqKKDaYxlimytaXdVbf+rj83wuMeSkM9YN9EbkvipUi1LiqKUYnwEdt90vftmYFUUG48aiI93SfoLfhIhYqtyJMlojtqi+wy9ARuozm+PKAcL5dcYZK+GfFYINqE1rJ7gE6INWvEd5KGoE7dA3pbXkpcXJGN9a2uoE++DfatM5BWCXrBVPsPViY5cW9VHJsGtlKcPYZANrXqiS4r7RBxMuNETMh5K1WoHrFcjg/p7tFmDrXrYwas/YCALrXD4CvhxLX5/eCh9J+g2GVvCU5+A8aW+OMphGmKX7F2/LDGCI/iYyNbTBE9iZ2MRaZYFIcEfZqaDFRsRvj5f+DijIgi/ZAVIOPEBM2HHf+tF/La++HL6FqeS+EL/zHso0brQ8A9YIKxCe0vKfVfu6CHlyY4hX5ahOHZoZLK2TY6Ma82aY6OtaphC8GWw/jVg0LR/Nq/pjzcgMAwpZZ+9qmJ7WfpxJqhf0hY50Ab94VkWyRCS/M14/GEO0CRBlD4Utn4CvVRmxTfoEa+YUPmRSFz4sZWd5Odhv7VbPwoWI3VJQBhwwt0VOzHK9fC8eOS+Jy7xqF+MHPU4nfx7ezbPPiSdRnxjkF+nqgTVgFBPl6YFxHk0N98SvCcWIAWLXXHJ29Lq0Qy8uN8VYr8NOYNrz7BjevisHNbTv8AX5r4MPudbD+6AMsHdLUKv/w7XbVLcMgqVyl62CA9nOMk+/DR4pd6CC/jYOyWfg//WB8Z+gPXf7XsSh8XofuJMIfmfhUsQ2vK0xFFhNpf8zTjcJBY2uYLUCp1UBb1aiAhsG+uBOfgaGtrD/vWpW9oZLLTJUbypv8YXojbUmbqWgnDUcpl2H72DbIyNNb+TPLOkS8yigrh4bj9rMM9ORJPZrasy4md6sNhYzCrafpOPXguaWmvlIuk5R4zkUPBTYYBmKvsQ2WKDajk/wmZih3YpD8DGbrxuIKXdf5Pi+aRt7l7TiiXomKVCaMNIWfDT2wXD8MmfCyfz6HhQPZq/z8Mq4trjx+gU6c5GbAFPby9fBmyMjVWyo1mKul2hMuM2brmMCGosvCYnsMMjIy4Ofnh/T0dPj6Ev+BGA7cSsD4ny8DAB4t7YcjUYkY88MlJ7RMY5DsNOYpf0JFyjSB8LO+O5S9F2JYR/EpS2ayNXo8S8tFHWZKSupD4O+pwENTEnW0MRRzdGNwha4r0Io1+z7siJe/Ns1U1g/ywYGPO0nuG0E8Yu9R4vMi2KVFdX8ABZH4zOn/3RM7FKJlCn8ZX0J3zQrs1HcGALylOIIuh/vjp+//D9r8OLYlf99B5P4ou63N3X0LPVedwNG7STDotNAeWwGsbwc8PAo9pcKXumEYoP3crnC9066gbMvKoeFoyIh6Ly9hWTdC0UIsL4IoEtLzUE4th09+XbEbT9Lg76lCtYpeePIiB1q9Ed2+MtW7mt67nmDxO1u0ld3B54rvUUtmcuLfKtcOwW+uQ8s1pvrytxf2tlpHk4l5BnVw5WeYrl2PEI2pXM8D75YYkzIC/9HCqVZmejUMxNz+DdHxS5OlFhv5MiiKwugtF3D0bjK+H9kK3Rs4t7QLgY3Ye5SIF8Fp3HmWAX8vJUL8PbH9fBy+O/kQaoXMTtlpNmpoMUHxFz6Q74GKMkBDeWCZ9nVsNfTGoU+6YufFx3irbXVW3h4ApOVo8dKivzA9PyRDRtFIpb2xRPcW/hAIyWCyYEBDdKhdyTLk3H31KXw9FehW3yRUeoMR8el5VtclOB+3Eq9169Zh+fLlSEhIQHh4ONasWYOICOHVk3/77TfMnTsXjx49Qp06dbBs2TK8/PLLoq5FxKt4MRhpHI1OgqdKjv87ch9TetRFu1oVsef6M6RmabDgf/wxVbWpJ/hC+T0iZCYL7oYxDLN1Y3GbDkONil44Nr2r5dinablY9OUyLFD+gGDKlLi8y9ARS3Qj8IKzCni/psE4G5OC1GwtOtetjPuJmRjUvApm9qlfRJ8AQSpuI16//vor3nnnHWzcuBFt2rTB6tWr8dtvv+Hu3bsICAiwOv7MmTPo1KkTIiMj0b9/f2zfvh3Lli3DlStX0Lix7ZgZgIhXSeL2s3T0+/qU4H4KRgyTH8McxXb4Ujkw0BQ2G/pilf417JzcA42r+AHpT3F+3Ri00ZoWkI01BuJT/RicMbK/C8tfa4p2tSqiso8aORoDZDIKfp5K0DRdJlJp3Am3Ea82bdqgdevWWLt2LQDAaDQiNDQUkydPxqxZs6yOHzZsGLKzs/H3339btrVt2xbNmjXDxo0b7V6PiFfJQWcwYsqv1/D3DZOP6+cxbbD26H2ce8iutV4ZaZin/BED5OcAAE/oSpinG4U25TPxZtaP8KFyoaPlVgngZqb2rIsPu9cpnjdFKDRi71GXxnlptVpcvnwZs2fPtmyTyWTo0aMHzp49y3vO2bNnMXXqVNa23r17Y/fu3bzHazQaaDQFScAZGfxLQBGKH6VchrVvtsCK102zih5KOV6qUwlGI43Np2Nx7mEKkjI1uPEEmKz7ELsMnbBEuRlVqefYrFoBZAOggMvGOpitG4t7dCiiF/dBQnoevjkRg8HNq6JRiK9NJz/BfXHpf/X58+cwGAwIDGTP3gQGBiI6mn+l5oSEBN7jExISeI+PjIzEwoULndNhQpHAXfhVJqMwtmNNjO1oKor3IlsLbw8F/rzaFD1/r48pil0YI9+HbHjgS/0b2Gbojs/6N8aA8GB4KOWoUakcIl9t6oq3QihGSv0jafbs2SxLLSMjA6GhoTbOIJQ0zJU5h7YKxdBWoQCG4FHMXfj7+WNJpUAscW33CC7CpeJVqVIlyOVyJCayFyRNTExEUBB/TE5QUJCk49VqNdRqaSuhEEo+NWoJL/pLKBu4NMJepVKhZcuWOHLkiGWb0WjEkSNH0K5dO95z2rVrxzoeAA4dOiR4PIFAKJ24fNg4depUjBw5Eq1atUJERARWr16N7OxsjB49GgDwzjvvoEqVKoiMjAQAfPTRR+jcuTO++uor9OvXDzt27MClS5fw7bffuvJtEAiEYsbl4jVs2DAkJydj3rx5SEhIQLNmzXDgwAGLUz4uLg4yRsG29u3bY/v27fjss88wZ84c1KlTB7t37xYV40UgEEoPLo/zKm5InBeBULIhVSUIBEKphogXgUBwS4h4EQgEt8TlDvvixuziI2lCBELJxHxv2nPHlznxysw01ZYiUfYEQskmMzMTfn5+gvvL3Gyj0WjEs2fP4OPjI6kUijmt6PHjx249S0neR8mCvA9raJpGZmYmQkJCWGFSXMqc5SWTyVC1qv0lwYTw9fV16y+ZGfI+ShbkfbCxZXGZIQ57AoHglhDxIhAIbgkRL5Go1WrMnz/f7StUkPdRsiDvw3HKnMOeQCCUDojlRSAQ3BIiXgQCwS0h4kUgENwSIl4EAsEtIeKVT2pqKkaMGAFfX1/4+/tjzJgxyMrKsnnOt99+iy5dusDX1xcURSEtLc0p7RYGR66Xl5eHiRMnomLFivD29saQIUOs1gmgKMrqZ8eOHU7r97p161CjRg14eHigTZs2uHDhgs3jf/vtN9SvXx8eHh5o0qQJ9u3bx9pP0zTmzZuH4OBgeHp6okePHrh//77T+msLZ7+XUaNGWX32ffr0Kcq3AEDa+7h9+zaGDBmCGjVqgKIorF69utBt2oUm0DRN03369KHDw8Ppc+fO0SdPnqRr165NDx8+3OY5q1atoiMjI+nIyEgaAP3ixQuntFsYHLne+PHj6dDQUPrIkSP0pUuX6LZt29Lt27dnHQOA3rJlCx0fH2/5yc3NdUqfd+zYQatUKnrz5s307du36XHjxtH+/v50YmIi7/GnT5+m5XI5/eWXX9J37tyhP/vsM1qpVNI3b960HLN06VLaz8+P3r17N339+nV64MCBdFhYmNP6XJzvZeTIkXSfPn1Yn31qamqJeh8XLlygp02bRv/yyy90UFAQvWrVqkK3aQ8iXjRN37lzhwZAX7x40bJt//79NEVR9NOnT+2ef/ToUV7xKmy7UnHkemlpabRSqaR/++03y7aoqCgaAH327FnLNgD0n3/+6fQ+0zRNR0RE0BMnTrS8NhgMdEhICB0ZGcl7/NChQ+l+/fqxtrVp04Z+//33aZqmaaPRSAcFBdHLly+37E9LS6PVajX9yy+/FME7KMDZ74WmTeI1aNCgIumvEFLfB5Pq1avzildh2uSDDBthWoXb398frVq1smzr0aMHZDIZzp8/X+Ladeb1Ll++DJ1Ohx49eli21a9fH9WqVbNatXzixImoVKkSIiIisHnzZrslS8RgXjWdeX0xq6YzjwdMq6abj4+NjUVCQgLrGD8/P7Rp00awTWdQFO/FzLFjxxAQEIB69erhgw8+QEpKivPfQD6OvA9XtFnmErP5SEhIQEBAAGubQqFAhQoVBFfidmW7zrxeQkICVCoV/P39Wdu5q5AvWrQI3bp1g5eXF/755x9MmDABWVlZ+PDDDwvV56JYNd38W8rK6s6gqFaA79OnD1599VWEhYUhJiYGc+bMQd++fXH27FnI5XJuky55H65os1SL16xZs7Bs2TKbx0RFRRVTbxynJLyPuXPnWv5u3rw5srOzsXz58kKLF8E+b7zxhuXvJk2aoGnTpqhVqxaOHTuG7t27u7BnrqVUi9cnn3yCUaNG2TymZs2aCAoKQlJSEmu7Xq9Hamqq4ErcYnBWu0X5PoKCgqDVapGWlsayvmytQg4Abdq0weLFi6HRaAqVz1YUq6abfycmJiI4OJh1TLNmzRzuqz2KYwV4wPS/rlSpEh48eFAk4uXI+3BFm6Xa51W5cmXUr1/f5o9KpUK7du2QlpaGy5cvW879999/YTQa0aZNG4ev76x2i/J9tGzZEkqlkrUK+d27dxEXF2dzFfJr166hfPnyhU7ELYpV08PCwhAUFMQ6JiMjA+fPny/SldWLawX4J0+eICUlhSXMzsSR9+GKNslsYz59+vShmzdvTp8/f54+deoUXadOHVaIwZMnT+h69erR58+ft2yLj4+nr169Sn/33Xc0APrEiRP01atX6ZSUFNHtloT3MX78eLpatWr0v//+S1+6dIlu164d3a5dO8v+PXv20N999x198+ZN+v79+/T69etpLy8vet68eU7p844dO2i1Wk1v3bqVvnPnDv3ee+/R/v7+dEJCAk3TNP3222/Ts2bNshx/+vRpWqFQ0CtWrKCjoqLo+fPn84ZK+Pv703/99Rd948YNetCgQcUWKuHM95KZmUlPmzaNPnv2LB0bG0sfPnyYbtGiBV2nTh06Ly+vxLwPjUZDX716lb569SodHBxMT5s2jb569Sp9//590W1KhYhXPikpKfTw4cNpb29v2tfXlx49ejSdmZlp2R8bG0sDoI8ePWrZNn/+fBqA1c+WLVtEt1sS3kdubi49YcIEunz58rSXlxc9ePBgOj4+3rJ///79dLNmzWhvb2+6XLlydHh4OL1x40baYDA4rd9r1qyhq1WrRqtUKjoiIoI+d+6cZV/nzp3pkSNHso7fuXMnXbduXVqlUtGNGjWi9+7dy9pvNBrpuXPn0oGBgbRaraa7d+9O371712n9tYUz30tOTg7dq1cvunLlyrRSqaSrV69Ojxs3zuEbvqjeh/l7xf3p3Lmz6DalQkriEAgEt6RU+7wIBELphYgXgUBwS4h4EQgEt4SIF4FAcEuIeBEIBLeEiBeBQHBLiHgRCAS3hIgXgUBwS4h4EUoEfKWOi6vcMcE9KdVVJQjuRZ8+fbBlyxbWNqHEb51OB6VSydqm1WqhUqkkX9fR8wiuhVhehBKDWq1GUFAQ66d8+fIATAuAbNiwAQMHDkS5cuXw+eefY8GCBWjWrBk2bdqEsLAweHh4AADi4uIwaNAgeHt7w9fXF0OHDmWVYhE6j+BeEPEiuA0LFizA4MGDcfPmTbz77rsAgAcPHmDXrl34448/cO3aNRiNRgwaNAipqak4fvw4Dh06hIcPH2LYsGGstrjnEdwPMmwklBj+/vtveHt7s7bNmTMHc+bMAQC8+eabGD16NGu/VqvFjz/+iMqVKwMw1cK6efMmYmNjERoaCgD48ccf0ahRI1y8eBGtW7fmPY/gfhDxIpQYunbtig0bNrC2VahQwfI3c2ERM9WrV2cJUFRUFEJDQy3CBQANGzaEv78/oqKiLOLFPY/gfhDxIpQYypUrh9q1a9vcL2ab2GsR3Bvi8yKUKho0aIDHjx/j8ePHlm137txBWloaGjZs6MKeEZwNsbwIJQaNRmO1NJlCoUClSpVEt9GjRw80adIEI0aMwOrVq6HX6zFhwgR07tyZd9hJcF+I5UUoMRw4cADBwcGsn5deeklSGxRF4a+//kL58uXRqVMn9OjRAzVr1sSvv/5aRL0muApSBppAILglxPIiEAhuCREvAoHglhDxIhAIbgkRLwKB4JYQ8SIQCG4JES8CgeCWEPEiEAhuCREvAoHglhDxIhAIbgkRLwKB4JYQ8SIQCG4JES8CgeCW/D9x05Yu+9z8KQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020919, + "end_time": "2024-03-01T10:18:32.771285", + "exception": false, + "start_time": "2024-03-01T10:18:32.750366", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4240.608803, + "end_time": "2024-03-01T10:18:35.514598", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/realtabformer/3/mlu-eval.ipynb", + "output_path": "eval/contraceptive/realtabformer/3/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/contraceptive/realtabformer/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "realtabformer" + }, + "start_time": "2024-03-01T09:07:54.905795", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/realtabformer/model.pt b/contraceptive/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc359c4c633413aa75af552b52a0205fd07948de --- /dev/null +++ b/contraceptive/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4cc609d6c10be1c4d1dc21442aa2a3a961cbd654795f076579f799dd433c742 +size 43889419 diff --git a/contraceptive/realtabformer/params.json b/contraceptive/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..9b5504bdd1d230e6cb79a7c9f8ae16d8a42852a8 --- /dev/null +++ b/contraceptive/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.775, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "realtabformer", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": false, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "relu6", "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/eval.csv b/contraceptive/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..7889f8d888e843c312acd3df2adf3fadabc17666 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.004686037855951365,0.016378744196746925,0.0026090041155702806,3.8580918312072754,0.06953004002571106,0.8769555687904358,0.09042102098464966,1.2404520020936616e-05,1.3648459911346436,0.03967232629656792,0.0928136557340622,0.05107840895652771,0.06657693535089493,7.981087151165411e-07,5.222937822341919 diff --git a/contraceptive/tab_ddpm_concat/history.csv b/contraceptive/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..5974f34d2806d7f08e963a2f28106ce8d74f0f88 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.017509687443816802,0.25492126335856824,0.000845979718699752,0.0,0.0,0.0,0.0,0.0,0.017509687443816802,320,80,74.85561537742615,0.9356951922178268,0.2339237980544567,0.12247967834118753,0.03225579813006334,0.3893812867692759,0.0018670448790572892,0.0,0.0,0.0,0.0,0.0,0.03225579813006334,80,20,16.974793434143066,0.8487396717071534,0.21218491792678834,0.11351076629944146 +1,0.015060588270716834,0.5294836329319879,0.0005396637374993886,0.0,0.0,0.0,0.0,0.0,0.015060588270716834,320,80,74.75039911270142,0.9343799889087677,0.23359499722719193,0.10789964701980352,0.017869547638110817,3.109420410258463,0.0007849385737095816,0.0,0.0,0.0,0.0,0.0,0.017869547638110817,80,20,17.005717754364014,0.8502858877182007,0.21257147192955017,0.032646807050332426 +2,0.007901813013450009,0.43976500204076957,0.00010274562480983643,0.0,0.0,0.0,0.0,0.0,0.007901813013450009,320,80,74.73881554603577,0.9342351943254471,0.23355879858136178,0.09000834664329886,0.006841135048307479,1.7945492254511919,8.046349178982836e-05,0.0,0.0,0.0,0.0,0.0,0.006841135048307479,80,20,16.90992760658264,0.8454963803291321,0.21137409508228303,0.052934233518317345 +3,0.005526901292250841,0.4796540130246029,5.3269670587949184e-05,0.0,0.0,0.0,0.0,0.0,0.005526901292250841,320,80,74.69570064544678,0.9336962580680848,0.2334240645170212,0.09062463160371408,0.004396481180447154,1.441257982449315,1.9934287330158895e-05,0.0,0.0,0.0,0.0,0.0,0.004396481180447154,80,20,16.831034421920776,0.8415517210960388,0.2103879302740097,0.034367192443460225 +4,0.003952335390204098,0.6800493642964284,3.501202619586863e-05,0.0,0.0,0.0,0.0,0.0,0.003952335390204098,320,80,74.82494044303894,0.9353117555379867,0.23382793888449668,0.08449668972752988,0.0030278531834483148,1.419872753619893,1.2108854497228094e-05,0.0,0.0,0.0,0.0,0.0,0.0030278531834483148,80,20,16.83168315887451,0.8415841579437255,0.21039603948593139,0.04602950892876834 +5,0.003957326662930427,0.3222652507973578,1.749390277871613e-05,0.0,0.0,0.0,0.0,0.0,0.003957326662930427,320,80,74.4447557926178,0.9305594474077225,0.2326398618519306,0.09507375009125099,0.003036417685507331,1.8372398112704105,1.3633386806094494e-05,0.0,0.0,0.0,0.0,0.0,0.003036417685507331,80,20,16.73884344100952,0.836942172050476,0.209235543012619,0.03600916846189648 +6,0.0028476251969550503,0.2714852012659293,1.3130850977921548e-05,0.0,0.0,0.0,0.0,0.0,0.0028476251969550503,320,80,75.16001582145691,0.9395001977682114,0.23487504944205284,0.09719131344463676,0.0032441405899589883,2.57110884013091,1.2244734485200582e-05,0.0,0.0,0.0,0.0,0.0,0.0032441405899589883,80,20,16.80543065071106,0.840271532535553,0.21006788313388824,0.034793011099100116 +7,0.002179265605263936,0.2912421387520652,5.355932938649715e-06,0.0,0.0,0.0,0.0,0.0,0.002179265605263936,320,80,75.13848423957825,0.9392310529947281,0.23480776324868202,0.09003764551598578,0.002960549862473272,1.5272669666737784,1.1626163023858993e-05,0.0,0.0,0.0,0.0,0.0,0.002960549862473272,80,20,16.840531826019287,0.8420265913009644,0.2105066478252411,0.048068627482280135 +8,0.0019942367394833126,0.8764173788223844,5.071989225379896e-06,0.0,0.0,0.0,0.0,0.0,0.0019942367394833126,320,80,74.68324661254883,0.9335405826568604,0.2333851456642151,0.0842734721081797,0.003437347624276299,1.6128702243404405,2.132158708016774e-05,0.0,0.0,0.0,0.0,0.0,0.003437347624276299,80,20,16.703343152999878,0.8351671576499939,0.20879178941249849,0.04150933439377695 +9,0.001910439515268081,0.5296208621499737,3.807974610924676e-06,0.0,0.0,0.0,0.0,0.0,0.001910439515268081,320,80,74.98403477668762,0.9373004347085953,0.2343251086771488,0.0939617162453942,0.0029005830438109115,1.4297594713909347,9.182002216068242e-06,0.0,0.0,0.0,0.0,0.0,0.0029005830438109115,80,20,16.704230070114136,0.8352115035057068,0.2088028758764267,0.040789688983932135 +10,0.002364715466683265,0.23049106563653615,1.08880691394031e-05,0.0,0.0,0.0,0.0,0.0,0.002364715466683265,320,80,74.73340845108032,0.934167605638504,0.233541901409626,0.09482922677416354,0.0025556790380505843,1.4470476474137044,9.480406802708785e-06,0.0,0.0,0.0,0.0,0.0,0.0025556790380505843,80,20,16.752872228622437,0.8376436114311219,0.20941090285778047,0.04689050167798996 +11,0.001990120611480961,0.20637534558109127,5.3637341571725695e-06,0.0,0.0,0.0,0.0,0.0,0.001990120611480961,320,80,74.96153736114502,0.9370192170143128,0.2342548042535782,0.09388396987924352,0.0026564171042991803,1.8061061197324306,1.1139397728709977e-05,0.0,0.0,0.0,0.0,0.0,0.0026564171042991803,80,20,16.754515647888184,0.8377257823944092,0.2094314455986023,0.046416288684122266 +12,0.0018798561781295576,0.3383319207922398,4.4709591399128e-06,0.0,0.0,0.0,0.0,0.0,0.0018798561781295576,320,80,74.77418828010559,0.9346773535013199,0.23366933837532997,0.0905981837247964,0.0026210575761069776,2.1850536189552257,5.391822381461964e-06,0.0,0.0,0.0,0.0,0.0,0.0026210575761069776,80,20,16.748401641845703,0.8374200820922851,0.20935502052307128,0.03947516868356615 +13,0.0018263132704305462,0.4754561664466223,3.583063472956116e-06,0.0,0.0,0.0,0.0,0.0,0.0018263132704305462,320,80,74.7354383468628,0.9341929793357849,0.23354824483394623,0.0871307724271901,0.002742944849887863,2.296998679006356,8.635369825379935e-06,0.0,0.0,0.0,0.0,0.0,0.002742944849887863,80,20,16.92655324935913,0.8463276624679565,0.21158191561698914,0.04202657011337578 +14,0.0017013913855407736,0.2523655897014123,2.6748898654643803e-06,0.0,0.0,0.0,0.0,0.0,0.0017013913855407736,320,80,75.46495079994202,0.9433118849992752,0.2358279712498188,0.09202192013035529,0.00283771293470636,1.8566916088265089,1.0541326899016213e-05,0.0,0.0,0.0,0.0,0.0,0.00283771293470636,80,20,17.10892963409424,0.8554464817047119,0.21386162042617798,0.05005494304932654 +15,0.0015473646866666969,0.26786694799376515,3.7729344502605496e-06,0.0,0.0,0.0,0.0,0.0,0.0015473646866666969,320,80,74.63104557991028,0.9328880697488785,0.23322201743721963,0.09204029910615645,0.00325308749161195,1.8810293299167824,1.5295879933319156e-05,0.0,0.0,0.0,0.0,0.0,0.00325308749161195,80,20,16.859638690948486,0.8429819345474243,0.21074548363685608,0.04019828836899251 diff --git a/contraceptive/tab_ddpm_concat/mlu-eval.ipynb b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..a195c6e38e804922095ad900d42fa42d8972dc57 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2479 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.299772Z", + "iopub.status.busy": "2024-02-29T20:24:17.299436Z", + "iopub.status.idle": "2024-02-29T20:24:17.332143Z", + "shell.execute_reply": "2024-02-29T20:24:17.331451Z" + }, + "papermill": { + "duration": 0.047487, + "end_time": "2024-02-29T20:24:17.334124", + "exception": false, + "start_time": "2024-02-29T20:24:17.286637", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.359785Z", + "iopub.status.busy": "2024-02-29T20:24:17.359451Z", + "iopub.status.idle": "2024-02-29T20:24:17.366321Z", + "shell.execute_reply": "2024-02-29T20:24:17.365528Z" + }, + "papermill": { + "duration": 0.02161, + "end_time": "2024-02-29T20:24:17.368190", + "exception": false, + "start_time": "2024-02-29T20:24:17.346580", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.392128Z", + "iopub.status.busy": "2024-02-29T20:24:17.391590Z", + "iopub.status.idle": "2024-02-29T20:24:17.395659Z", + "shell.execute_reply": "2024-02-29T20:24:17.394854Z" + }, + "papermill": { + "duration": 0.018308, + "end_time": "2024-02-29T20:24:17.397516", + "exception": false, + "start_time": "2024-02-29T20:24:17.379208", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.421170Z", + "iopub.status.busy": "2024-02-29T20:24:17.420894Z", + "iopub.status.idle": "2024-02-29T20:24:17.424803Z", + "shell.execute_reply": "2024-02-29T20:24:17.424006Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018081, + "end_time": "2024-02-29T20:24:17.426726", + "exception": false, + "start_time": "2024-02-29T20:24:17.408645", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.450615Z", + "iopub.status.busy": "2024-02-29T20:24:17.449717Z", + "iopub.status.idle": "2024-02-29T20:24:17.455310Z", + "shell.execute_reply": "2024-02-29T20:24:17.454632Z" + }, + "papermill": { + "duration": 0.019442, + "end_time": "2024-02-29T20:24:17.457111", + "exception": false, + "start_time": "2024-02-29T20:24:17.437669", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4a39259d", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.482190Z", + "iopub.status.busy": "2024-02-29T20:24:17.481582Z", + "iopub.status.idle": "2024-02-29T20:24:17.486940Z", + "shell.execute_reply": "2024-02-29T20:24:17.486107Z" + }, + "papermill": { + "duration": 0.019901, + "end_time": "2024-02-29T20:24:17.488758", + "exception": false, + "start_time": "2024-02-29T20:24:17.468857", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tab_ddpm_concat/3\"\n", + "param_index = 1\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010913, + "end_time": "2024-02-29T20:24:17.510570", + "exception": false, + "start_time": "2024-02-29T20:24:17.499657", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.533681Z", + "iopub.status.busy": "2024-02-29T20:24:17.533442Z", + "iopub.status.idle": "2024-02-29T20:24:17.542009Z", + "shell.execute_reply": "2024-02-29T20:24:17.541229Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022345, + "end_time": "2024-02-29T20:24:17.543896", + "exception": false, + "start_time": "2024-02-29T20:24:17.521551", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tab_ddpm_concat/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:17.569033Z", + "iopub.status.busy": "2024-02-29T20:24:17.568743Z", + "iopub.status.idle": "2024-02-29T20:24:19.735944Z", + "shell.execute_reply": "2024-02-29T20:24:19.735006Z" + }, + "papermill": { + "duration": 2.182796, + "end_time": "2024-02-29T20:24:19.738044", + "exception": false, + "start_time": "2024-02-29T20:24:17.555248", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:19.764864Z", + "iopub.status.busy": "2024-02-29T20:24:19.764385Z", + "iopub.status.idle": "2024-02-29T20:24:19.775482Z", + "shell.execute_reply": "2024-02-29T20:24:19.774579Z" + }, + "papermill": { + "duration": 0.026856, + "end_time": "2024-02-29T20:24:19.777545", + "exception": false, + "start_time": "2024-02-29T20:24:19.750689", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:19.801415Z", + "iopub.status.busy": "2024-02-29T20:24:19.801150Z", + "iopub.status.idle": "2024-02-29T20:24:19.808219Z", + "shell.execute_reply": "2024-02-29T20:24:19.807337Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021364, + "end_time": "2024-02-29T20:24:19.810227", + "exception": false, + "start_time": "2024-02-29T20:24:19.788863", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:19.833931Z", + "iopub.status.busy": "2024-02-29T20:24:19.833610Z", + "iopub.status.idle": "2024-02-29T20:24:19.936285Z", + "shell.execute_reply": "2024-02-29T20:24:19.935372Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.117078, + "end_time": "2024-02-29T20:24:19.938537", + "exception": false, + "start_time": "2024-02-29T20:24:19.821459", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:19.965113Z", + "iopub.status.busy": "2024-02-29T20:24:19.964662Z", + "iopub.status.idle": "2024-02-29T20:24:24.575496Z", + "shell.execute_reply": "2024-02-29T20:24:24.574680Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.626471, + "end_time": "2024-02-29T20:24:24.577879", + "exception": false, + "start_time": "2024-02-29T20:24:19.951408", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 20:24:22.201582: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 20:24:22.201642: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 20:24:22.203391: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:24.603940Z", + "iopub.status.busy": "2024-02-29T20:24:24.603370Z", + "iopub.status.idle": "2024-02-29T20:24:24.609312Z", + "shell.execute_reply": "2024-02-29T20:24:24.608472Z" + }, + "papermill": { + "duration": 0.021509, + "end_time": "2024-02-29T20:24:24.611262", + "exception": false, + "start_time": "2024-02-29T20:24:24.589753", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:24.637565Z", + "iopub.status.busy": "2024-02-29T20:24:24.636890Z", + "iopub.status.idle": "2024-02-29T20:24:33.501995Z", + "shell.execute_reply": "2024-02-29T20:24:33.500904Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.880952, + "end_time": "2024-02-29T20:24:33.504445", + "exception": false, + "start_time": "2024-02-29T20:24:24.623493", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.74,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:34.043923Z", + "iopub.status.busy": "2024-02-29T20:24:34.043553Z", + "iopub.status.idle": "2024-02-29T20:24:34.126797Z", + "shell.execute_reply": "2024-02-29T20:24:34.125613Z" + }, + "papermill": { + "duration": 0.100884, + "end_time": "2024-02-29T20:24:34.129493", + "exception": false, + "start_time": "2024-02-29T20:24:34.028609", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T20:24:34.164150Z", + "iopub.status.busy": "2024-02-29T20:24:34.163804Z", + "iopub.status.idle": "2024-02-29T20:24:34.623226Z", + "shell.execute_reply": "2024-02-29T20:24:34.622278Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.479089, + "end_time": "2024-02-29T20:24:34.625391", + "exception": false, + "start_time": "2024-02-29T20:24:34.146302", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:34.661052Z", + "iopub.status.busy": "2024-02-29T20:24:34.660649Z", + "iopub.status.idle": "2024-02-29T20:24:34.665126Z", + "shell.execute_reply": "2024-02-29T20:24:34.664272Z" + }, + "papermill": { + "duration": 0.025637, + "end_time": "2024-02-29T20:24:34.667047", + "exception": false, + "start_time": "2024-02-29T20:24:34.641410", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:34.693128Z", + "iopub.status.busy": "2024-02-29T20:24:34.692884Z", + "iopub.status.idle": "2024-02-29T20:24:34.699532Z", + "shell.execute_reply": "2024-02-29T20:24:34.698756Z" + }, + "papermill": { + "duration": 0.02194, + "end_time": "2024-02-29T20:24:34.701376", + "exception": false, + "start_time": "2024-02-29T20:24:34.679436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11282952" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:34.727471Z", + "iopub.status.busy": "2024-02-29T20:24:34.727228Z", + "iopub.status.idle": "2024-02-29T20:24:34.804688Z", + "shell.execute_reply": "2024-02-29T20:24:34.803872Z" + }, + "papermill": { + "duration": 0.092848, + "end_time": "2024-02-29T20:24:34.806652", + "exception": false, + "start_time": "2024-02-29T20:24:34.713804", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 10] --\n", + "├─Adapter: 1-1 [2, 1179, 10] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 11,264\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 10] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 524,544\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,282,952\n", + "Trainable params: 11,282,952\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 42.96\n", + "========================================================================================================================\n", + "Input size (MB): 0.12\n", + "Forward/backward pass size (MB): 365.70\n", + "Params size (MB): 45.13\n", + "Estimated Total Size (MB): 410.95\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:24:34.837133Z", + "iopub.status.busy": "2024-02-29T20:24:34.836862Z", + "iopub.status.idle": "2024-02-29T20:51:23.802107Z", + "shell.execute_reply": "2024-02-29T20:51:23.801127Z" + }, + "papermill": { + "duration": 1609.000137, + "end_time": "2024-02-29T20:51:23.820977", + "exception": false, + "start_time": "2024-02-29T20:24:34.820840", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.017509687443816802, 'avg_role_model_std_loss': 0.25492126335856824, 'avg_role_model_mean_pred_loss': 0.000845979718699752, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.017509687443816802, 'n_size': 320, 'n_batch': 80, 'duration': 74.85561537742615, 'duration_batch': 0.9356951922178268, 'duration_size': 0.2339237980544567, 'avg_pred_std': 0.12247967834118753}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.03225579813006334, 'avg_role_model_std_loss': 0.3893812867692759, 'avg_role_model_mean_pred_loss': 0.0018670448790572892, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03225579813006334, 'n_size': 80, 'n_batch': 20, 'duration': 16.974793434143066, 'duration_batch': 0.8487396717071534, 'duration_size': 0.21218491792678834, 'avg_pred_std': 0.11351076629944146}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.015060588270716834, 'avg_role_model_std_loss': 0.5294836329319879, 'avg_role_model_mean_pred_loss': 0.0005396637374993886, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015060588270716834, 'n_size': 320, 'n_batch': 80, 'duration': 74.75039911270142, 'duration_batch': 0.9343799889087677, 'duration_size': 0.23359499722719193, 'avg_pred_std': 0.10789964701980352}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.017869547638110817, 'avg_role_model_std_loss': 3.109420410258463, 'avg_role_model_mean_pred_loss': 0.0007849385737095816, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.017869547638110817, 'n_size': 80, 'n_batch': 20, 'duration': 17.005717754364014, 'duration_batch': 0.8502858877182007, 'duration_size': 0.21257147192955017, 'avg_pred_std': 0.032646807050332426}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007901813013450009, 'avg_role_model_std_loss': 0.43976500204076957, 'avg_role_model_mean_pred_loss': 0.00010274562480983643, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007901813013450009, 'n_size': 320, 'n_batch': 80, 'duration': 74.73881554603577, 'duration_batch': 0.9342351943254471, 'duration_size': 0.23355879858136178, 'avg_pred_std': 0.09000834664329886}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006841135048307479, 'avg_role_model_std_loss': 1.7945492254511919, 'avg_role_model_mean_pred_loss': 8.046349178982836e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006841135048307479, 'n_size': 80, 'n_batch': 20, 'duration': 16.90992760658264, 'duration_batch': 0.8454963803291321, 'duration_size': 0.21137409508228303, 'avg_pred_std': 0.052934233518317345}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005526901292250841, 'avg_role_model_std_loss': 0.4796540130246029, 'avg_role_model_mean_pred_loss': 5.3269670587949184e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005526901292250841, 'n_size': 320, 'n_batch': 80, 'duration': 74.69570064544678, 'duration_batch': 0.9336962580680848, 'duration_size': 0.2334240645170212, 'avg_pred_std': 0.09062463160371408}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004396481180447154, 'avg_role_model_std_loss': 1.441257982449315, 'avg_role_model_mean_pred_loss': 1.9934287330158895e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004396481180447154, 'n_size': 80, 'n_batch': 20, 'duration': 16.831034421920776, 'duration_batch': 0.8415517210960388, 'duration_size': 0.2103879302740097, 'avg_pred_std': 0.034367192443460225}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003952335390204098, 'avg_role_model_std_loss': 0.6800493642964284, 'avg_role_model_mean_pred_loss': 3.501202619586863e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003952335390204098, 'n_size': 320, 'n_batch': 80, 'duration': 74.82494044303894, 'duration_batch': 0.9353117555379867, 'duration_size': 0.23382793888449668, 'avg_pred_std': 0.08449668972752988}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030278531834483148, 'avg_role_model_std_loss': 1.419872753619893, 'avg_role_model_mean_pred_loss': 1.2108854497228094e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030278531834483148, 'n_size': 80, 'n_batch': 20, 'duration': 16.83168315887451, 'duration_batch': 0.8415841579437255, 'duration_size': 0.21039603948593139, 'avg_pred_std': 0.04602950892876834}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003957326662930427, 'avg_role_model_std_loss': 0.3222652507973578, 'avg_role_model_mean_pred_loss': 1.749390277871613e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003957326662930427, 'n_size': 320, 'n_batch': 80, 'duration': 74.4447557926178, 'duration_batch': 0.9305594474077225, 'duration_size': 0.2326398618519306, 'avg_pred_std': 0.09507375009125099}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003036417685507331, 'avg_role_model_std_loss': 1.8372398112704105, 'avg_role_model_mean_pred_loss': 1.3633386806094494e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003036417685507331, 'n_size': 80, 'n_batch': 20, 'duration': 16.73884344100952, 'duration_batch': 0.836942172050476, 'duration_size': 0.209235543012619, 'avg_pred_std': 0.03600916846189648}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0028476251969550503, 'avg_role_model_std_loss': 0.2714852012659293, 'avg_role_model_mean_pred_loss': 1.3130850977921548e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028476251969550503, 'n_size': 320, 'n_batch': 80, 'duration': 75.16001582145691, 'duration_batch': 0.9395001977682114, 'duration_size': 0.23487504944205284, 'avg_pred_std': 0.09719131344463676}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032441405899589883, 'avg_role_model_std_loss': 2.57110884013091, 'avg_role_model_mean_pred_loss': 1.2244734485200582e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032441405899589883, 'n_size': 80, 'n_batch': 20, 'duration': 16.80543065071106, 'duration_batch': 0.840271532535553, 'duration_size': 0.21006788313388824, 'avg_pred_std': 0.034793011099100116}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002179265605263936, 'avg_role_model_std_loss': 0.2912421387520652, 'avg_role_model_mean_pred_loss': 5.355932938649715e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002179265605263936, 'n_size': 320, 'n_batch': 80, 'duration': 75.13848423957825, 'duration_batch': 0.9392310529947281, 'duration_size': 0.23480776324868202, 'avg_pred_std': 0.09003764551598578}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002960549862473272, 'avg_role_model_std_loss': 1.5272669666737784, 'avg_role_model_mean_pred_loss': 1.1626163023858993e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002960549862473272, 'n_size': 80, 'n_batch': 20, 'duration': 16.840531826019287, 'duration_batch': 0.8420265913009644, 'duration_size': 0.2105066478252411, 'avg_pred_std': 0.048068627482280135}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019942367394833126, 'avg_role_model_std_loss': 0.8764173788223844, 'avg_role_model_mean_pred_loss': 5.071989225379896e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019942367394833126, 'n_size': 320, 'n_batch': 80, 'duration': 74.68324661254883, 'duration_batch': 0.9335405826568604, 'duration_size': 0.2333851456642151, 'avg_pred_std': 0.0842734721081797}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003437347624276299, 'avg_role_model_std_loss': 1.6128702243404405, 'avg_role_model_mean_pred_loss': 2.132158708016774e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003437347624276299, 'n_size': 80, 'n_batch': 20, 'duration': 16.703343152999878, 'duration_batch': 0.8351671576499939, 'duration_size': 0.20879178941249849, 'avg_pred_std': 0.04150933439377695}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001910439515268081, 'avg_role_model_std_loss': 0.5296208621499737, 'avg_role_model_mean_pred_loss': 3.807974610924676e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001910439515268081, 'n_size': 320, 'n_batch': 80, 'duration': 74.98403477668762, 'duration_batch': 0.9373004347085953, 'duration_size': 0.2343251086771488, 'avg_pred_std': 0.0939617162453942}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0029005830438109115, 'avg_role_model_std_loss': 1.4297594713909347, 'avg_role_model_mean_pred_loss': 9.182002216068242e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029005830438109115, 'n_size': 80, 'n_batch': 20, 'duration': 16.704230070114136, 'duration_batch': 0.8352115035057068, 'duration_size': 0.2088028758764267, 'avg_pred_std': 0.040789688983932135}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002364715466683265, 'avg_role_model_std_loss': 0.23049106563653615, 'avg_role_model_mean_pred_loss': 1.08880691394031e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002364715466683265, 'n_size': 320, 'n_batch': 80, 'duration': 74.73340845108032, 'duration_batch': 0.934167605638504, 'duration_size': 0.233541901409626, 'avg_pred_std': 0.09482922677416354}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025556790380505843, 'avg_role_model_std_loss': 1.4470476474137044, 'avg_role_model_mean_pred_loss': 9.480406802708785e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025556790380505843, 'n_size': 80, 'n_batch': 20, 'duration': 16.752872228622437, 'duration_batch': 0.8376436114311219, 'duration_size': 0.20941090285778047, 'avg_pred_std': 0.04689050167798996}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001990120611480961, 'avg_role_model_std_loss': 0.20637534558109127, 'avg_role_model_mean_pred_loss': 5.3637341571725695e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001990120611480961, 'n_size': 320, 'n_batch': 80, 'duration': 74.96153736114502, 'duration_batch': 0.9370192170143128, 'duration_size': 0.2342548042535782, 'avg_pred_std': 0.09388396987924352}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026564171042991803, 'avg_role_model_std_loss': 1.8061061197324306, 'avg_role_model_mean_pred_loss': 1.1139397728709977e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026564171042991803, 'n_size': 80, 'n_batch': 20, 'duration': 16.754515647888184, 'duration_batch': 0.8377257823944092, 'duration_size': 0.2094314455986023, 'avg_pred_std': 0.046416288684122266}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018798561781295576, 'avg_role_model_std_loss': 0.3383319207922398, 'avg_role_model_mean_pred_loss': 4.4709591399128e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018798561781295576, 'n_size': 320, 'n_batch': 80, 'duration': 74.77418828010559, 'duration_batch': 0.9346773535013199, 'duration_size': 0.23366933837532997, 'avg_pred_std': 0.0905981837247964}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026210575761069776, 'avg_role_model_std_loss': 2.1850536189552257, 'avg_role_model_mean_pred_loss': 5.391822381461964e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026210575761069776, 'n_size': 80, 'n_batch': 20, 'duration': 16.748401641845703, 'duration_batch': 0.8374200820922851, 'duration_size': 0.20935502052307128, 'avg_pred_std': 0.03947516868356615}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018263132704305462, 'avg_role_model_std_loss': 0.4754561664466223, 'avg_role_model_mean_pred_loss': 3.583063472956116e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018263132704305462, 'n_size': 320, 'n_batch': 80, 'duration': 74.7354383468628, 'duration_batch': 0.9341929793357849, 'duration_size': 0.23354824483394623, 'avg_pred_std': 0.0871307724271901}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002742944849887863, 'avg_role_model_std_loss': 2.296998679006356, 'avg_role_model_mean_pred_loss': 8.635369825379935e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002742944849887863, 'n_size': 80, 'n_batch': 20, 'duration': 16.92655324935913, 'duration_batch': 0.8463276624679565, 'duration_size': 0.21158191561698914, 'avg_pred_std': 0.04202657011337578}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017013913855407736, 'avg_role_model_std_loss': 0.2523655897014123, 'avg_role_model_mean_pred_loss': 2.6748898654643803e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017013913855407736, 'n_size': 320, 'n_batch': 80, 'duration': 75.46495079994202, 'duration_batch': 0.9433118849992752, 'duration_size': 0.2358279712498188, 'avg_pred_std': 0.09202192013035529}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00283771293470636, 'avg_role_model_std_loss': 1.8566916088265089, 'avg_role_model_mean_pred_loss': 1.0541326899016213e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00283771293470636, 'n_size': 80, 'n_batch': 20, 'duration': 17.10892963409424, 'duration_batch': 0.8554464817047119, 'duration_size': 0.21386162042617798, 'avg_pred_std': 0.05005494304932654}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015473646866666969, 'avg_role_model_std_loss': 0.26786694799376515, 'avg_role_model_mean_pred_loss': 3.7729344502605496e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015473646866666969, 'n_size': 320, 'n_batch': 80, 'duration': 74.63104557991028, 'duration_batch': 0.9328880697488785, 'duration_size': 0.23322201743721963, 'avg_pred_std': 0.09204029910615645}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00325308749161195, 'avg_role_model_std_loss': 1.8810293299167824, 'avg_role_model_mean_pred_loss': 1.5295879933319156e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00325308749161195, 'n_size': 80, 'n_batch': 20, 'duration': 16.859638690948486, 'duration_batch': 0.8429819345474243, 'duration_size': 0.21074548363685608, 'avg_pred_std': 0.04019828836899251}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▅▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▇▄▃▂▂▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▁▃▁▂▁▁▂▂▂▂▂▂▂▃▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▅▂▂▁▃▃▂▁▃▃▃▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▅▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▇▄▃▂▂▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▄▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▅▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▁█▅▄▄▅▇▄▄▄▄▅▆▆▅▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▂▄▃▄▆▂▂▂█▄▁▁▂▄▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▆▆▅▃▃▂▃▃▁▁▂▂▂▅█▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▄▃▃▃▄▁▆▆▃▅▃▅▃▃█▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▆▆▅▃▃▂▃▃▁▁▂▂▂▅█▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▄▃▃▃▄▁▆▆▃▅▃▅▃▃█▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▆▆▅▃▃▂▃▃▁▁▂▂▂▅█▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▄▃▃▃▄▁▆▆▃▅▃▅▃▃█▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00325\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00155\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.0402\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.09204\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00325\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00155\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 2e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 1.88103\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.26787\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.84298\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.93289\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.21075\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.23322\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 16.85964\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 74.63105\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/tab_ddpm_concat/3/wandb/offline-run-20240229_202436-6gs8ggtf\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_202436-6gs8ggtf/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.0026090041735591157, 'avg_g_mag_loss': 0.027198189083769916, 'avg_g_cos_loss': 0.003037689876903717, 'pred_duration': 1.3711330890655518, 'grad_duration': 3.8808236122131348, 'total_duration': 5.2519567012786865, 'pred_std': 0.06657693535089493, 'std_loss': 7.981087151165411e-07, 'mean_pred_loss': 1.2404520930431318e-05, 'pred_rmse': 0.05107840895652771, 'pred_mae': 0.03967232629656792, 'pred_mape': 0.0928136557340622, 'grad_rmse': 0.09042102098464966, 'grad_mae': 0.06953004002571106, 'grad_mape': 0.876955509185791}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0026090041735591157, 'avg_g_mag_loss': 0.027198189083769916, 'avg_g_cos_loss': 0.003037689876903717, 'avg_pred_duration': 1.3711330890655518, 'avg_grad_duration': 3.8808236122131348, 'avg_total_duration': 5.2519567012786865, 'avg_pred_std': 0.06657693535089493, 'avg_std_loss': 7.981087151165411e-07, 'avg_mean_pred_loss': 1.2404520930431318e-05}, 'min_metrics': {'avg_loss': 0.0026090041735591157, 'avg_g_mag_loss': 0.027198189083769916, 'avg_g_cos_loss': 0.003037689876903717, 'pred_duration': 1.3711330890655518, 'grad_duration': 3.8808236122131348, 'total_duration': 5.2519567012786865, 'pred_std': 0.06657693535089493, 'std_loss': 7.981087151165411e-07, 'mean_pred_loss': 1.2404520930431318e-05, 'pred_rmse': 0.05107840895652771, 'pred_mae': 0.03967232629656792, 'pred_mape': 0.0928136557340622, 'grad_rmse': 0.09042102098464966, 'grad_mae': 0.06953004002571106, 'grad_mape': 0.876955509185791}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.0026090041735591157, 'avg_g_mag_loss': 0.027198189083769916, 'avg_g_cos_loss': 0.003037689876903717, 'pred_duration': 1.3711330890655518, 'grad_duration': 3.8808236122131348, 'total_duration': 5.2519567012786865, 'pred_std': 0.06657693535089493, 'std_loss': 7.981087151165411e-07, 'mean_pred_loss': 1.2404520930431318e-05, 'pred_rmse': 0.05107840895652771, 'pred_mae': 0.03967232629656792, 'pred_mape': 0.0928136557340622, 'grad_rmse': 0.09042102098464966, 'grad_mae': 0.06953004002571106, 'grad_mape': 0.876955509185791}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:51:23.858964Z", + "iopub.status.busy": "2024-02-29T20:51:23.858098Z", + "iopub.status.idle": "2024-02-29T20:51:23.862294Z", + "shell.execute_reply": "2024-02-29T20:51:23.861566Z" + }, + "papermill": { + "duration": 0.025401, + "end_time": "2024-02-29T20:51:23.864119", + "exception": false, + "start_time": "2024-02-29T20:51:23.838718", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:51:23.898742Z", + "iopub.status.busy": "2024-02-29T20:51:23.898458Z", + "iopub.status.idle": "2024-02-29T20:51:24.204643Z", + "shell.execute_reply": "2024-02-29T20:51:24.203859Z" + }, + "papermill": { + "duration": 0.326548, + "end_time": "2024-02-29T20:51:24.207482", + "exception": false, + "start_time": "2024-02-29T20:51:23.880934", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:51:24.244534Z", + "iopub.status.busy": "2024-02-29T20:51:24.244231Z", + "iopub.status.idle": "2024-02-29T20:51:24.514752Z", + "shell.execute_reply": "2024-02-29T20:51:24.513854Z" + }, + "papermill": { + "duration": 0.291584, + "end_time": "2024-02-29T20:51:24.516959", + "exception": false, + "start_time": "2024-02-29T20:51:24.225375", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA78UlEQVR4nO3de1xUdf4/8NfMMBduM8N9QLl4ATFFMoiJbtTKhi5ltO5qLKvkz9R2tSy2MtuUtt0Vy/zqN/NbW9tqtRnqrvX9rpobIuomCMqlMJS8IJAwICL3yzAzn98fB44MDDgzXIaR9/PxOI+ZOecz53wYZ16e8zmf8zkCxhgDIYTYCaGtK0AIIZag0CKE2BUKLUKIXaHQIoTYFQotQohdodAihNgVCi1CiF1xsHUFRovBYEBVVRVcXV0hEAhsXR1CSB+MMTQ3N8PPzw9C4cD7U+MmtKqqquDv72/rahBCbqGyshITJ04ccPm4CS1XV1cA3Acil8ttXBtCSF9NTU3w9/fnf6sDGTeh1XNIKJfLKbQIGcNu1XxDDfGEELtCoUUIsSsUWoQQuzJu2rTI0Oj1enR1ddm6GsSOicViiESiIa+HQosMijEGjUaDhoYGW1eF3AaUSiVUKtWQ+kpSaJFB9QSWt7c3nJycqGMusQpjDG1tbaitrQUA+Pr6Wr0uCq2+OpqAmrOAQQdMetDWtbEpvV7PB5aHh4etq0PsnKOjIwCgtrYW3t7eVh8qUkN8X1e+AXbOA/79e1vXxOZ62rCcnJxsXBNyu+j5Lg2lfZRCqy+PKdxj/WWAhs8HcOvOfoSYazi+SxRafbkFARAA2hag9Zqta0MI6YNCqy8HKaDovrD6+iXb1oUQ0g+Flikek7nHegotYh2BQIAvv/zS1tUYVq+//jruvPNOW1eDQssk917tWoTYqV27dkGpVA7b+l588UVkZmYO2/qsRV0eTHHv3tOiw0MyDmi1WkgkkluWc3FxgYuLyyjUaHC0p2UKfwaRQqs3xhjatDqbTJbeCP3w4cO4//77oVQq4eHhgUcffRSXLnH/nvfeey/Wrl1rVP7atWsQi8U4ceIEAKC6uhrx8fFwdHTEpEmTsHv3bgQFBWHbtm1WfXbFxcX4yU9+AkdHR3h4eGDFihVoaWnhlx87dgxRUVFwdnaGUqnEfffdh/LycgDAt99+i4cffhiurq6Qy+WIiIjAmTNnBt3esWPHsHTpUjQ2NkIgEEAgEOD1118HAAQFBeGPf/wjlixZArlcjhUrVgAA1q5di5CQEDg5OWHy5MlYv369UdeEvoeHTz31FBISEvD222/D19cXHh4eWLVq1Yhf7kV7Wqbwh4dlXLcHOuUPAGjv0uOODf+2ybZL3oiDk8T8r2traytSUlIwa9YstLS0YMOGDXjiiSdQVFSEpKQkvPXWW9i0aRN/Cn7Pnj3w8/PDAw88AABYsmQJ6urqcOzYMYjFYqSkpPC9uS3V2tqKuLg4REdH4/Tp06itrcXTTz+N1atXY9euXdDpdEhISMDy5cvx+eefQ6vVIi8vj69bUlISZs+ejffeew8ikQhFRUUQi8WDbvPee+/Ftm3bsGHDBpSWlgKA0V7S22+/jQ0bNiA1NZWf5+rqil27dsHPzw/FxcVYvnw5XF1d8fLLLw+4naysLPj6+iIrKwsXL17EokWLcOedd2L58uVWfVbmoNAyxS0QEAi5bg8ttYCrj61rRCy0YMECo9d/+9vf4OXlhZKSEixcuBDPP/88vvnmGz6kdu/ejcTERAgEApw/fx5HjhzB6dOnERkZCQD461//iuDgYKvqsnv3bnR0dOCTTz6Bs7MzAODdd9/FY489hjfffBNisRiNjY149NFHMWUK9x/m9OnT+fdXVFTgpZdeQmhoKACYVQ+JRAKFQgGBQACVStVv+U9+8hP87ne/M5r32muv8c+DgoLw4osvIj09fdDQcnNzw7vvvguRSITQ0FDEx8cjMzOTQmvUOUgBxUSgoYI7RKTQAgA4ikUoeSPOZtu2xIULF7Bhwwbk5uairq4OBoMBABcAM2fOxCOPPILPPvsMDzzwAMrKypCTk4O//OUvAIDS0lI4ODjgrrvu4tc3depUuLm5WVX3c+fOITw8nA8sALjvvvtgMBhQWlqKBx98EE899RTi4uLw05/+FLGxsVi4cCF/fV5KSgqefvppfPrpp4iNjcUvf/lLPtys1RPGve3ZswfvvPMOLl26hJaWFuh0uluO8jtjxgyjy3F8fX1RXFw8pLrdCrVpDYTOIPYjEAjgJHGwyWRpT+rHHnsM9fX1+PDDD5Gbm4vc3FwAXKMzwB1y/eMf/0BXVxd2796NsLAwhIWFDftnZq6dO3ciJycH9957L/bs2YOQkBCcOnUKANeW9P333yM+Ph5Hjx7FHXfcgS+++GJI2+sdoACQk5ODpKQk/OxnP8OBAwdQWFiI3//+9/znNZC+h6kCgYD/D2KkWBVaO3bsQFBQEGQyGdRqNfLy8gYtv2/fPoSGhkImkyEsLAyHDh0yWv76668jNDQUzs7OcHNzQ2xsLP8l61FfX4+kpCTI5XIolUosW7bMqCFz2NEZRLt1/fp1lJaW4rXXXsOcOXMwffp03Lhxw6jM448/jo6ODhw+fBi7d+9GUlISv2zatGnQ6XQoLCzk5128eLHfOsw1ffp0fPvtt2htbeXnnTx5EkKhENOmTePnzZ49G+vWrUN2djZmzpyJ3bt388tCQkLwwgsv4Ouvv8bPf/5z7Ny585bblUgk0Ov1ZtUxOzsbgYGB+P3vf4/IyEgEBwfzJwLGGotDa8+ePUhJSUFqaioKCgoQHh6OuLi4ARsps7OzkZiYiGXLlqGwsBAJCQlISEjA2bNn+TIhISF49913UVxcjG+++QZBQUF45JFHcO3azctokpKS8P333yMjIwMHDhzAiRMn+LMeI4LOINotNzc3eHh44IMPPsDFixdx9OhRpKSkGJVxdnZGQkIC1q9fj3PnziExMZFfFhoaitjYWKxYsQJ5eXkoLCzEihUr4OjoaNW1c0lJSZDJZEhOTsbZs2eRlZWFZ599FosXL4aPjw/Kysqwbt065OTkoLy8HF9//TUuXLiA6dOno729HatXr8axY8dQXl6OkydP4vTp00ZtXgMJCgpCS0sLMjMzUVdXh7a2tgHLBgcHo6KiAunp6bh06RLeeeedIe/NjRhmoaioKLZq1Sr+tV6vZ35+fiwtLc1k+YULF7L4+HijeWq1mq1cuXLAbTQ2NjIA7MiRI4wxxkpKShgAdvr0ab7MV199xQQCAbt69apZ9e5ZZ2Njo1nl2fmvGEuVM/befeaVvw21t7ezkpIS1t7ebuuqWCwjI4NNnz6dSaVSNmvWLHbs2DEGgH3xxRd8mUOHDjEA7MEHH+z3/qqqKjZv3jwmlUpZYGAg2717N/P29mbvv/++Wdvvu63vvvuOPfzww0wmkzF3d3e2fPly1tzczBhjTKPRsISEBObr68skEgkLDAxkGzZsYHq9nnV2drInn3yS+fv7M4lEwvz8/Njq1avN/jd55plnmIeHBwPAUlNTGWOMBQYGsq1bt/Yr+9JLLzEPDw/m4uLCFi1axLZu3coUCgW/PDU1lYWHh/Ovk5OT2eOPP260jjVr1rCYmJgB6zPYd8rc36hFodXZ2clEIpHRPwZjjC1ZsoTNnz/f5Hv8/f37fUAbNmxgs2bNGnAbmzdvZgqFgl27do0xxthHH33ElEqlUbmuri4mEonY/v37Ta6no6ODNTY28lNlZaVloVVbyoXWn3wZMxjMe89txp5Da7j1fH96/iMl1hmO0LLo8LCurg56vR4+PsZn03x8fKDRaEy+R6PRmFX+wIEDcHFxgUwmw9atW5GRkQFPT09+Hd7e3kblHRwc4O7uPuB209LSoFAo+Mniu0u7BXHdHrpagZYay95L7N7Ro0fxf//3fygrK0N2djaefPJJBAUF4cEHx/fAkGPBmDl7+PDDD6OoqAjZ2dmYO3cuFi5caHVnPgBYt24dGhsb+amystKyFThIbo72QGcQx52uri68+uqrmDFjBp544gl4eXnxHU0/++wz/pKWvtOMGTNGrY7z5s0bsB4bN24ctXqMNov6aXl6ekIkEqGmxnjPo6amxmQHNgBQqVRmlXd2dsbUqVMxdepU3HPPPQgODsZHH32EdevWQaVS9QswnU6H+vr6AbcrlUohlUot+fP6c58MNJRzZxAD7x3auohdiYuLQ1yc6T5p8+fPh1qtNrnsVj3Vh9Nf//pXtLe3m1zm7u4+avUYbRaFlkQiQUREBDIzM5GQkAAAMBgMyMzMxOrVq02+Jzo6GpmZmXj++ef5eRkZGYiOjh50WwaDAZ2dnfw6GhoakJ+fj4iICADc7rvBYBjwyzMsPKYAl7PoDCIx4urqCldXV1tXAxMmTLB1FWzC4h7xKSkpSE5ORmRkJKKiorBt2za0trZi6dKlALhrtiZMmIC0tDQAwJo1axATE4MtW7YgPj4e6enpOHPmDD744AMA3HVZf/7znzF//nz4+vqirq4OO3bswNWrV/HLX/4SANfPZe7cuVi+fDnef/99dHV1YfXq1XjyySfh5+c3XJ9Ff9TBlJAxx+LQWrRoEa5du4YNGzZAo9HgzjvvxOHDh/nG9oqKCgiFN5vK7r33XuzevRuvvfYaXn31VQQHB+PLL7/EzJkzAQAikQjnz5/Hxx9/jLq6Onh4eODuu+/Gf/7zH6P2gc8++wyrV6/GnDlzIBQKsWDBArzzzjtD/fsHx3cwpdAiZKwQMDY+7t7Q1NQEhUKBxsbGW15Pxau7ALwbCYidgVevjrvRHjo6OlBWVoZJkyZBJpPZujrkNjDYd8rc3+iYOXs4JikDqdsDIWMMhdZgend7oGsQCRkTKLRuha5BJFa4HW9sMVZQaN1KT2M8nUEkdma4b2wBcMM4CwQCNDQ0DOt6LUGhdSs93R7o8JCQMYFC61Y8eo0XP94xBmhbbTPRjS2G7cYWnZ2dePHFFzFhwgQ4OztDrVbj2LFj/HvLy8vx2GOPwc3NDc7OzpgxYwYOHTqEK1eu4OGHHwbADf8jEAjw1FNPWfV5DAUNt3wrvQ8Px/tNLrragI0j2Jl3MK9WARLnW5frRje2GPjGFqtXr0ZJSQnS09Ph5+eHL774AnPnzkVxcTGCg4OxatUqaLVanDhxAs7OzigpKYGLiwv8/f3xz3/+EwsWLEBpaSnkcjkcHR2t+kyGgkLrVnp3e2jWAHJfW9eImIFubGH6xhYVFRXYuXMnKioq+KtJXnzxRRw+fBg7d+7Exo0bUVFRgQULFvDDT0+ePJl/f881jd7e3sPeXmYuCq1bcZAAygDgxhVub2s8h5bYidvjsdW2LUA3tjCtuLgYer0eISEhRvM7Ozvh4eEBAHjuuefwm9/8Bl9//TViY2OxYMECzJo1y6rtjQRq0zIHf4g4zhvjBQLuEM0WE93YYlhubNHS0gKRSIT8/HwUFRXx07lz5/Df//3fAICnn34aly9fxuLFi1FcXIzIyEhs37592P7WoaLQMgedQbQrdGMLjqkbW8yePRt6vR61tbX8UFA9U+/DSH9/fzzzzDPYv38/fve73+HDDz/k1wnA7BtmjAQKLXN40GgP9oRubMExdWOLkJAQJCUlYcmSJdi/fz/KysqQl5eHtLQ0HDx4EADw/PPP49///jfKyspQUFCArKwsfnuBgYEQCAQ4cOAArl27NrJ3xBrIyIwEPfZYfGOL3koPc+PF/8+9w1+xMcyex4inG1twTN3YQqvVsg0bNrCgoCAmFouZr68ve+KJJ9h3333HGGNs9erVbMqUKUwqlTIvLy+2ePFiVldXx6/zjTfeYCqVigkEApacnGxWPXoMxxjxNMqDOeouAu9G3GyIHifdHmiUh5t+/PFH+Pv748iRI5gzZ46tq2O3hmOUBzp7aA5lACAQcf2UqNvDuHD06FG0tLQgLCwM1dXVePnll+nGFmMEtWmZw0ECKHtuckGN8eMB3dhi7KI9LXO5T+H6al2/BATdb+vakBFGN7YYuyi0zOUxBbiUSWcQCd3Ywsbo8NBc47iD6Tg5V0NGwXB8lyi0zMV3MB0/e1o9hzptbW02rgm5XfR8l4ZyGE2Hh+bq3cF0nIz2IBKJoFQq+dENnJycrOpcSQhjDG1tbaitrYVSqYRIJLJ6XRRa5urp9qBrB5qrAbmNhmgZZT2Xdlg7LAshvSmVygHvCm8uCi1zicTdoz2UcWcQx0loCQQC+Pr6wtvbG11dXbauDrFjYrF4SHtYPSi0LOExhQut+svApAdsXZtRJRKJhuULR8hQWdUQv2PHDgQFBUEmk0GtViMvL2/Q8vv27UNoaChkMhnCwsJw6NAhfllXVxfWrl2LsLAwODs7w8/PD0uWLEFVlfG4TUFBQfywsT3Tpk2brKm+9cbxGURCxgqLQ2vPnj1ISUlBamoqCgoKEB4ejri4uAHbPLKzs5GYmIhly5ahsLAQCQkJSEhIwNmzZwFwZxMKCgqwfv16FBQUYP/+/SgtLcX8+fP7reuNN95AdXU1Pz377LOWVn9oaIgaQmzPoku0GWNRUVFs1apV/Gu9Xs/8/PxYWlqayfILFy5k8fHxRvPUajVbuXLlgNvIy8tjAFh5eTk/LzAwkG3dutXsenZ0dLDGxkZ+qqystH6Uhx4/fM2N9rAj2vp1EEJMMneUB4v2tLRaLfLz8xEbG8vPEwqFiI2NRU5Ojsn35OTkGJUHuEskBioPgL+DSN8xqDdt2gQPDw/Mnj0bmzdvhk6nG3AdaWlpUCgU/OTv72/GX3gLvW9y0T18LyFkdFkUWnV1ddDr9fDx8TGa7+PjA41GY/I9Go3GovIdHR1Yu3YtEhMTjYaneO6555Ceno6srCysXLkSGzduxMsvvzxgXdetW4fGxkZ+qqysNPfPHFjfbg+EkFE3ps4ednV1YeHChWCM4b333jNa1nvkyVmzZkEikWDlypVIS0uDVCrtty6pVGpy/pCIxIBbILenVX8ZUIzPa78IsSWL9rQ8PT0hEolQU1NjNL+mpmbADmMqlcqs8j2BVV5ejoyMjFsO1KdWq6HT6XDlyhVL/oShozOIhNiURaElkUgQERGBzMxMfp7BYEBmZiaio6NNvic6OtqoPABkZGQYle8JrAsXLuDIkSP8rYwGU1RUBKFQCG9vb0v+hKGjM4iE2JTFh4cpKSlITk5GZGQkoqKisG3bNrS2tmLp0qUAuDvzTpgwAWlpaQCANWvWICYmBlu2bEF8fDzS09Nx5swZfPDBBwC4wPrFL36BgoICHDhwAHq9nm/vcnd3h0QiQU5ODnJzc/lbg+fk5OCFF17Ar3/9a6vvRWc1uskFIbZlzanJ7du3s4CAACaRSFhUVBQ7deoUvywmJqbfYPd79+5lISEhTCKRsBkzZrCDBw/yy8rKyhgAk1NWVhZjjLH8/HymVquZQqFgMpmMTZ8+nW3cuJF1dHSYXech3diiN77bwz1DWw8hxAjd2KKPId3Yorfrl4DtdwEOMuDVakBIo/sQMhzM/Y3SL85SykBA6ADoOqjbAyE2QKFlKZED118LoDOIhNgAhZY16AwiITZDoWWN3pfzEEJGFYWWNajbAyE2Q6FlDTo8JMRmKLSs4T6Je7xRRqM9EDLKKLSsYdTtoerW5Qkhw4ZCyxoiBy64ADpEJGSUUWhZi84gEmITFFrW4s8g0p4WIaOJQsta/BlE2tMiZDRRaFmLDg8JsQkKLWt5dIcWdXsgZFRRaFlLEXCz20PTVVvXhpBxg0LLWr27PdAhIiGjhkJrKOgMIiGjjkJrKOgaREJGHYXWUPBnEMtsWw9CxhEKraHwoHsgEjLaKLSGoufwsJ66PRAyWii0hkLhz3V70HdStwdCRgmF1lCIHAC3IO45HSISMiqsCq0dO3YgKCgIMpkMarUaeXl5g5bft28fQkNDIZPJEBYWhkOHDvHLurq6sHbtWoSFhcHZ2Rl+fn5YsmQJqqqMx6mqr69HUlIS5HI5lEolli1bhpaWFmuqP7zoDCIho8ri0NqzZw9SUlKQmpqKgoIChIeHIy4uDrW1tSbLZ2dnIzExEcuWLUNhYSESEhKQkJCAs2fPAgDa2tpQUFCA9evXo6CgAPv370dpaSnmz59vtJ6kpCR8//33yMjIwIEDB3DixAmsWLHCij95mNE1iISMLktvXR0VFcVWrVrFv9br9czPz4+lpaWZLL9w4UIWHx9vNE+tVrOVK1cOuI28vDwGgJWXlzPGGCspKWEA2OnTp/kyX331FRMIBOzq1atm1dvcW25bLPcDxlLljO1+cnjXS8g4Y+5v1KI9La1Wi/z8fMTGxvLzhEIhYmNjkZOTY/I9OTk5RuUBIC4ubsDyANDY2AiBQAClUsmvQ6lUIjIyki8TGxsLoVCI3Nxck+vo7OxEU1OT0TQieva06PCQkFFhUWjV1dVBr9fDx8fHaL6Pjw80Go3J92g0GovKd3R0YO3atUhMTIRcLufX4e3tbVTOwcEB7u7uA64nLS0NCoWCn/z9/c36Gy3m3nu0B/3IbIMQwhtTZw+7urqwcOFCMMbw3nvvDWld69atQ2NjIz9VVlYOUy37UPgDQjGg11K3B0JGgYMlhT09PSESiVBTU2M0v6amBiqVyuR7VCqVWeV7Aqu8vBxHjx7l97J61tG3oV+n06G+vn7A7UqlUkilUrP/Nqv1dHu4foE7RFQGjPw2CRnHLNrTkkgkiIiIQGZmJj/PYDAgMzMT0dHRJt8THR1tVB4AMjIyjMr3BNaFCxdw5MgReHh49FtHQ0MD8vPz+XlHjx6FwWCAWq225E8YGXQGkZDRY2kLf3p6OpNKpWzXrl2spKSErVixgimVSqbRaBhjjC1evJi98sorfPmTJ08yBwcH9vbbb7Nz586x1NRUJhaLWXFxMWOMMa1Wy+bPn88mTpzIioqKWHV1NT91dnby65k7dy6bPXs2y83NZd988w0LDg5miYmJZtd7xM4eMsbYV69wZxAPvzr86yZknDD3N2pxaDHG2Pbt21lAQACTSCQsKiqKnTp1il8WExPDkpOTjcrv3buXhYSEMIlEwmbMmMEOHjzILysrK2MATE5ZWVl8uevXr7PExETm4uLC5HI5W7p0KWtubja7ziMaWj3dHj5bNPzrJmScMPc3KmCMMVvt5Y2mpqYmKBQKNDY2GrWXDYuLmcDffw54TgNWD351ACHENHN/o2Pq7KHd6hnBlLo9EDLiKLSGQ+9uD40/2ro2hNzWKLSGg1DUa7QHOoNIyEii0BoudJMLQkYFhdZw4a9BpD0tQkYShdZwoQ6mhIwKCq3hQoeHhIwKCq3h4hHMPdZfBnSdtq0LIbcxCq0+GGPYduQH5JXVW/ZGxURApgAMOuDa+ZGpHCGEQquvj7OvYNuRC1j56RmUX281/40CAaCaxT3XFI9M5QghFFp9Lbo7ALMmKnCjrQvLPj6Dpo4u89+sCuMeKbQIGTEUWn04SkT4cEkkVHIZLta2YNVnBdDpzbwRKx9aZ0eugoSMcxRaJvjIZfhrciQcxSL850Id/nigxLw39t7TGh/XoRMy6ii0BjBzggJbF90JAPg4pxyf5Fy59Zs8p3HXIHY2Ag0VI1o/QsYrCq1BzJ2pwtq5oQCAP/yrBMd/uDb4GxwkgDdXntq1CBkZFFq38EzMZCy4ayL0BobVnxXgQk3z4G/wocZ4QkYShdYtCAQCbPz5TEQFuaO5U4dlH59Bfat24DfQGURCRhSFlhmkDiK8vzgCAe5OqKhvwzOf5qNTN8Bgfz2hVUOhRchIoNAyk7uzBB8lR8JV6oC8K/V4df9ZmBypWjWTe2yoANobRrWOhIwHFFoWCPZxxbtJd0EkFOCfBT/i/eMmRnRwdAMU3fc+rKH+WoQMNwotC8WEeCH1sTsAAG8ePo/DZzX9C1G7FiEjhkLLCkuig7AkOhAA8MKeIpy92mhcoOcQkUKLkGFHoWWlDY/egQeCPdHepcfTH59BTVPHzYW0p0XIiLEqtHbs2IGgoCDIZDKo1Wrk5Q1+r799+/YhNDQUMpkMYWFhOHTokNHy/fv345FHHoGHhwcEAgGKior6reOhhx6CQCAwmp555hlrqj8sHERC7Ei6C1O9XaBp6sDTH59Bu7b7jGJPaF07D+gG6R5BCLGYxaG1Z88epKSkIDU1FQUFBQgPD0dcXBxqa2tNls/OzkZiYiKWLVuGwsJCJCQkICEhAWfP3mykbm1txf33348333xz0G0vX74c1dXV/PTWW29ZWv1hJZeJ8VFyJNycxCi+2ojf7SuCwcAAZSAglXO3FKv7waZ1JOS2Y+mtq6OiotiqVav413q9nvn5+bG0tDST5RcuXMji4+ON5qnVarZy5cp+ZcvKyhgAVlhY2G9ZTEwMW7NmjaXV5Zl7y21r5F6+zqa+epAFrj3Atnxdys382zzGUuWMFe4e9u0Rcjsy9zdq0Z6WVqtFfn4+YmNj+XlCoRCxsbHIyckx+Z6cnByj8gAQFxc3YPnBfPbZZ/D09MTMmTOxbt06tLW1DVi2s7MTTU1NRtNIiZrkjo1PcIeE7x+/xB0mUrsWISPCotCqq6uDXq+Hj4+P0XwfHx9oNCZO/QPQaDQWlR/Ir371K/z9739HVlYW1q1bh08//RS//vWvByyflpYGhULBT/7+/hZtz1K/iJiICUpHaHUGnLp8HfDpOYP43Yhul5DxxsHWFTDXihUr+OdhYWHw9fXFnDlzcOnSJUyZMqVf+XXr1iElJYV/3dTUNKLBJRAI8NA0L3yWW4Gs0lo8fHfP5TxnubG1BIIR2zYh44lFe1qenp4QiUSoqakxml9TUwOVSmXyPSqVyqLy5lKr1QCAixcvmlwulUohl8uNppH20DRvAEBWaS2Y1zRA6AC03wCaro74tgkZLywKLYlEgoiICGRmZvLzDAYDMjMzER0dbfI90dHRRuUBICMjY8Dy5urpFuHr6zuk9Qyne6d4QCISorK+HZcb9NyggAC1axEyjCw+PExJSUFycjIiIyMRFRWFbdu2obW1FUuXLgUALFmyBBMmTEBaWhoAYM2aNYiJicGWLVsQHx+P9PR0nDlzBh988AG/zvr6elRUVKCqqgoAUFpaCoDbS1OpVLh06RJ2796Nn/3sZ/Dw8MB3332HF154AQ8++CBmzZo15A9huDhLHaCe7I7/XKhD1vlaTFGFAbXfc6E1bZ6tq0fI7cGaU5Pbt29nAQEBTCKRsKioKHbq1Cl+WUxMDEtOTjYqv3fvXhYSEsIkEgmbMWMGO3jwoNHynTt3MgD9ptTUVMYYYxUVFezBBx9k7u7uTCqVsqlTp7KXXnrJou4LI9nlobcPT1xigWsPsKQPTzF2cjvX7SE9aUS3ScjtwNzfqICx8XEHhqamJigUCjQ2No5o+9alay2Ys+U4JCIhvl0ihePnTwBuQcCab0dsm4TcDsz9jdK1h8NssqczAtydoNUbkNvmx828cQXoGLl+YoSMJxRaw0wgEODhaV4AgK+vdAHyCdyCmu9tWCtCbh8UWiOgp+vDsfO1YDRMDSHDikJrBNwz2QNSByGqGjtQ79pzSzHqGU/IcKDQGgGOEhGip3gAAPI7JnIzaU+LkGFBoTVCHgrh2rUOXuPCC7XnAL3OhjUi5PZAoTVCetq1Dv0oBZM4A/pO4PoFG9eKEPtHoTVCgjydMdnTGV0GARpc6XIeQoYLhdYIiunu+nCOBXEzqDGekCGj0BpBD/eM+tDYPZ4Y7WkRMmQUWiMoapI7HMUinGrr7mCqKebG1iKEWI1CawTJxCLcO8UDP7CJMEAEtF0Hmi0bsZUQYoxCa4Q9FOqNTkhw1aHX3hYhxGoUWiOsp79WQWf3UM/UGE/IkFBojTB/dycEe7vge0MAN4P2tAgZEgqtUfDQNC+U8N0eKLQIGQoKrVHw8DRvnOve02L1l4HOFhvXiBD7RaE1CiKD3NEhcYeGuUEABtSW2LpKhNgtCq1RIHEQ4r6pnvzeFjXGE2I9Cq1R8nCoN0pYIPeC2rUIsRqF1ih5aJoXSgxBAABdFe1pEWItCq1R4qtwRIfHHQAAQc33NLYWIVai0BpFIdNnoZVJITJ0AvWXbF0dQuySVaG1Y8cOBAUFQSaTQa1WIy8vb9Dy+/btQ2hoKGQyGcLCwnDo0CGj5fv378cjjzwCDw8PCAQC/pb3vXV0dGDVqlXw8PCAi4sLFixYgJqaGmuqbzMPhapwnnGN8YZqatcixBoWh9aePXuQkpKC1NRUFBQUIDw8HHFxcaitrTVZPjs7G4mJiVi2bBkKCwuRkJCAhIQEnD17li/T2tqK+++/H2+++eaA233hhRfwr3/9C/v27cPx48dRVVWFn//855ZW36buCnTDBcEkAMC1C6dtXBtC7JSlt66Oiopiq1at4l/r9Xrm5+fH0tLSTJZfuHAhi4+PN5qnVqvZypUr+5UtKytjAFhhYaHR/IaGBiYWi9m+ffv4eefOnWMAWE5Ojln1NveW2yPtsx2pjKXKWdnWR2xaD0LGGnN/oxbtaWm1WuTn5yM2NpafJxQKERsbi5ycHJPvycnJMSoPAHFxcQOWNyU/Px9dXV1G6wkNDUVAQMCA6+ns7ERTU5PRNBZ4B98NAFA0nrdxTQixTxaFVl1dHfR6PXx8fIzm+/j4QKMxPU6URqOxqPxA65BIJFAqlWavJy0tDQqFgp/8/f3N3t5ImnVXNPRMADfWgOs1FbauDiF257Y9e7hu3To0NjbyU2Vlpa2rBADw9nBHlYgbW+t84Ukb14YQ+2NRaHl6ekIkEvU7a1dTUwOVSmXyPSqVyqLyA61Dq9WioaHB7PVIpVLI5XKjaaxoceP6a9VdLLBxTQixPxaFlkQiQUREBDIzM/l5BoMBmZmZiI6ONvme6Ohoo/IAkJGRMWB5UyIiIiAWi43WU1paioqKCovWM1a4Bs0GAEjqvodOb7BxbQixLw6WviElJQXJycmIjIxEVFQUtm3bhtbWVixduhQAsGTJEkyYMAFpaWkAgDVr1iAmJgZbtmxBfHw80tPTcebMGXzwwQf8Ouvr61FRUYGqqioAXCAB3B6WSqWCQqHAsmXLkJKSAnd3d8jlcjz77LOIjo7GPffcM+QPYbT5TrsbyAeCDWX49scGRAS627pKhNgPa05Nbt++nQUEBDCJRMKioqLYqVOn+GUxMTEsOTnZqPzevXtZSEgIk0gkbMaMGezgwYNGy3fu3MkA9JtSU1P5Mu3t7ey3v/0tc3NzY05OTuyJJ55g1dXVZtd5rHR5YIwx1qRhLFXO9BsUbNvBQlvXhpAxwdzfqICx8XFPq6amJigUCjQ2No6J9q2OtMmQdV5HinwL/ivlaVtXhxCbM/c3etuePRzrhL6zAACy6yWobeqwcW0IsR8UWjYimRAOAJguKMexH67ZuDaE2A8KLVtRhQEA7hCW41ip6es2CSH9UWjZSndohQoqcPKHWnRR1wdCzEKhZSseU8EcHOEs6IS79ioKym/YukaE2AUKLVsRiiDw4XrG3yEoR1YptWsRYg4KLVvqPkScLizH0fM1GCe9TwgZEgotW+oOrZnCCvxQ04KtRy7YuEKEjH0UWrak4vpqRTleBQC8k3kB+86MjdEoCBmrKLRsyfsOAAI4ddbipfvdAADr9hfj5MU629aLkDGMQsuWpC6A+2QAwG+mdWB+uB90BoZnPs1HqabZxpUjZGyi0LK17nYtYW0xNv9yFqKC3NHcqcPSnXmooct7COmHQsvWukMLmrOQOojwl8URmOzpjKrGDiz7+DRaO+mmroT0RqFla92N8dBw90F0c5Zg59K74e4swdmrTXj280IaKJCQXii0bK17tAdcOw9cyAAABHo446/JkZA6CHH0fC3+8K8S6sNFSDcKLVtzVQF3JQNgwL6lQE0JAOCuADdsW3QnBALg01Pl+OibMtvWk5AxgkJrLPjZ20Dg/YC2Gdi9CGjhLumZF+aL3/9sOgDgz4fO4avialvWkpAxgUJrLHCQAIs+5bo/NFYAe5KALu7M4bL7J2FJdCAYA57fU4SCCrqwmoxvFFpjhZM78Ku9gEwBVOYC//cswBgEAgE2PHoH5oR6o1NnwPKPz6D8equta0uIzVBojSWewcDCTwCBCCjeC5x4GwDgIBLincTZmDlBjuutWizdeRoNbVobV5YQ26DQGmsmPwTEb+GeZ/0JOLsfAOAsdcDfku+Gn0KGy3WtWPFJPjq69LarJyE2QqE1FkUuBe5ZxT3/8jfA1XwAgLdchp1Lo+AqdUDelXq89I/vYDBQVwgyvlBojVWP/BEIjgN0HcDniUDjjwCAaSpXvL84Ag5CAf71bRXe+ncp9eEi44pVobVjxw4EBQVBJpNBrVYjLy9v0PL79u1DaGgoZDIZwsLCcOjQIaPljDFs2LABvr6+cHR0RGxsLC5cMB5bKigoCAKBwGjatGmTNdW3D0IR8IuPAO8ZQEsNsPtJoLMFAHDfVE+k/Zy7/Of945fwXHoRXe5Dxg2LQ2vPnj1ISUlBamoqCgoKEB4ejri4ONTWmr6jTHZ2NhITE7Fs2TIUFhYiISEBCQkJOHv2LF/mrbfewjvvvIP3338fubm5cHZ2RlxcHDo6jC8YfuONN1BdXc1Pzz77rKXVty9SV+BX6YCzF1BTDOxfDhi4dqxfRvrjD/Nn8Htc89/9Bj/U0MgQZByw9NbVUVFRbNWqVfxrvV7P/Pz8WFpamsnyCxcuZPHx8Ubz1Go1W7lyJWOMMYPBwFQqFdu8eTO/vKGhgUmlUvb555/z8wIDA9nWrVstrS7P3Ftuj0kVeYy94cVYqpyxf79mtOh02XWm/vMRFrj2AAt97Su2v6DSRpUkZGjM/Y1atKel1WqRn5+P2NhYfp5QKERsbCxycnJMvicnJ8eoPADExcXx5cvKyqDRaIzKKBQKqNXqfuvctGkTPDw8MHv2bGzevBk63cCHRJ2dnWhqajKa7Jb/3UDC/3DPs98BCj7hF0UGuePgc/fjgWBPtHfp8cKeb/HqF8V0ZpHctiwKrbq6Ouj1evj4+BjN9/HxgUajMfkejUYzaPmex1ut87nnnkN6ejqysrKwcuVKbNy4ES+//PKAdU1LS4NCoeAnf39/8//QsSjsF0DMK9zzAy8AZf/hF3m4SLFraRTWzAmGQADszq3AL97PRsX1NhtVlpCRYzdnD1NSUvDQQw9h1qxZeOaZZ7BlyxZs374dnZ2dJsuvW7cOjY2N/FRZeRuMvf7QK8DMBYBBB+xdDFy/xC8SCQV44ach+HhpFNycxDh7tQnx2/+DjJIaG1aYkOFnUWh5enpCJBKhpsb4h1BTUwOVSmXyPSqVatDyPY+WrBMA1Go1dDodrly5YnK5VCqFXC43muyeQAA8vgOYEAm03wB2L+Qee3kwxAsHn3sAdwUo0dyhw/JPziDtq3M0Jhe5bVgUWhKJBBEREcjMzOTnGQwGZGZmIjo62uR7oqOjjcoDQEZGBl9+0qRJUKlURmWampqQm5s74DoBoKioCEKhEN7e3pb8CfZP7Ag8uRuQTwSuXwTSk/hRIXr4KR2RviIa/+++SQCAvxy/jF99mEvDN5Pbg6Ut/Onp6UwqlbJdu3axkpIStmLFCqZUKplGo2GMMbZ48WL2yiuv8OVPnjzJHBwc2Ntvv83OnTvHUlNTmVgsZsXFxXyZTZs2MaVSyf73f/+Xfffdd+zxxx9nkyZNYu3t7YwxxrKzs9nWrVtZUVERu3TpEvv73//OvLy82JIlS8yut12fPTSlupixP/txZxTfnMRY8T8YMxj6FTv0XRWbseEwC1x7gEX88Wt28sI1G1SWkFsz9zdqcWgxxtj27dtZQEAAk0gkLCoqip06dYpfFhMTw5KTk43K7927l4WEhDCJRMJmzJjBDh48aLTcYDCw9evXMx8fHyaVStmcOXNYaWkpvzw/P5+p1WqmUCiYTCZj06dPZxs3bmQdHR1m1/m2Cy3GGKv6lrH/uY8LrlQ5Y+lJjDXX9Ct2+VoLi9t6nAWuPcAmvXKAbc/8gen1/QOOEFsy9zcqYGx8XAPS1NQEhUKBxsbG26N9q4dOC3zzX8CJzVwDvaM78LPNXIO9QMAX6+jSY8P/nsXeM9zlQDP85PB0kYIBRpcB9TxlYDefM+41AHg4S/H0A5MwO8BtVP48Mn6Y+xul0LpdaIq5i6u7b5CB0EeB+P8CXI27kuw9U4n1X55Fp25oDfPzZqrwYtw0TPFyGdJ6COlBodXHbR9aAKDvAr7ZChx/CzB0AY5uwLzNXB+vXntdFdfbcPpKPYCbs/lHCIxec89vvvjPD9fwz4IfYWBcN4tFd/vj+TnB8JbLRvZvI7c9Cq0+xkVo9dCcBf73t0D1t9zrafHAo//F3URjGJRqmrH53+dx5Bx3valMLMSy+ydhZcwUyGXiYdnGkOi0QGMloAwARGOgPsQsFFp9jKvQAri9rpPbgGNvcntdMiUw7y1g1kLj3aghOH2lHpu+Oo/8cq6vmJuTGKsenorF0YGQOoiGZRtmab8BVJ4GKk8BFae48cd0HYDYGQi8F5gcA0x6EPAJA4R205/69tHVznXVuQUKrT7GXWj1qPke+PK3QHUR9zpkHvDoVkDuOyyrZ4wh4/tq/OWrPLTXX4W3oAHTnFrx+BQBQl3aIGzRAM3V3LA6Sn/AbRJ3Aw/3SdxztyBA4mTJBoEbV7hx9CtygIpc4Nq5/uWEDtyJid4c3YFJDwCTYrjJY8qwBTjp1tXOfeeqCoGqIu7x2nkg5Vy/9tW+KLT6GLehBQB6HZD938CxTYBey9084/4XuL0vsO5ThgxGpwv7ze9+1LUDzTVcEDVruKlF0z8gLOHq2x1mk24+9jyXugKa77g9qIpTXFi1mLg0yX0KEHAPN/nfwwVSbQlw+ThQdhwozwa0LcbvkU/oDrAHub0xuZ/1f8N4pOsEas72Cqgi7jNnJi7WT/oHEPzTQVdHodXHuA6tHjUlXFtXVeHIrN/ZCwYXFSq7FDhdL8FVnRK1TAmljz8SokIQLKkH6suAG2XcY30Z0Nk4+DoFov4/AqEY8LsT8FcDAdHco4vX4OvRdwFXC7gAu3wc+DGPC/DePIK5APO7E/CaDnhNA2Tj9LvSm76LC/wbV4z3oGrPcU0PfTl5An6zu6c7uUdX31vu1VJo9UGh1U2vA/I+AK50jxIh6G7jEQgACG796CABXFRco76rb/ekAly8jRq9G9q0+J9jl7Ar+wq03d0rgr1dMG+mCnNn+mK6ryt3nrL9Rp8gu3zzeUv3KB8yZXdAdYeU32yz2kgGpW3j2sAuHwfKTnCHz8xENxD5RC68vKcDXqHdj9O4PcCxymAAOpu4qaPvYyMXQNrW7qnneZvx/K5er/uGe2+O7v0DSj7BqsNuCq0+KLRs42pDO7Zl/IAvi66iS3/zqxbo4YS5M1WYN9MX4RMVRt0qeNo2oO069yMY6Qb09gag/CRw5SRQ+z1wrZQ7BB6Iwp8LLz7IQrn/AHoHQWdzr9ctXLueqddg3N6jUMQFv9Dh5muhQ695vSaRA3dDX1PB1DlCY8fJlMbh5Deb+xyGqV2QQqsPCi3baurowtFztThUXI3jP1wz6tzqp5AhrjvAIgLdIBKOkcbx9htceNWe4xqTex5NtamNRSIp134pkwNS+c1HqSsgceFOgEicuefiXs8lzt3Lep47c2diHSQjWl0KrT4otMaO1k4djpVew1dnq5F1vhat2pttVp4uUsTN8MG8mb5QT3aHWDQGuyi01UNfex7XLhehsbwYwrrzcGsrB4RCMLEzRDJXSJ3kcHRRQCh1AaQ9P35X7lHq0h0I3cEhEHInMvQ67tGg49qKDHquPcmgM570XdxyB9nNMJIpup8rbs5zkNr6k7IIhVYfFFpjU0eXHv+5UIevzlbjSEkNmjpunoVUOonxk2neCPBwgqeLFJ4uUni5SvjnzlKHUatnfasWRZU3UFjRgIKKG/i2shEtt7gDklgkwBQvF0z3lWO6rytCVXJM95XDy9W+wmS0UGj1QaE19ml1BuRcvo6viqvxdUkN6lsHaQAG4CgWwbNXiHm6SOHlIoGnqxQezlI4S0VwFIvgJHGAo0QIWc9zsQgysdB0OxoAnd6A85pmFFY2oLD8BgorG1BW19qvnLNEhHB/Je4KcMPMCQo0tGlxrroJ5zTNOFfdhOYO06Hm6SLFdF9XTPeVI1TlCpVcBheZA1ykDvyjo1g0YP1uVxRafVBo2Red3oC8K/U4dbke15o7UdfSa2rWon0YbtzhKBbBUSLiH50kXFD8oGk2uf4pXs6YHeCGuwLcMDtAiRAf1wHb3xhjqGrswLmqJpzXNOFcdTPOaZpQVtcKc35xQgG4EOsVZM5SB7j2hJtUDJlYCImDEGKRENLuR4mDEBKREOLuR4mDABKRqLucABIHIZwk3HrkMjEkDmPn8JtCqw8KrdtLa6eOD7Frzdp+oVbfqkWrVof2Lj06tHq0denRrtWbPbqFq8wBd3bvRc0OUOJOfyWUTkNviG7X6vFDDbcndl7TjPOaJtS3atHSoUNzpw6tnToYRvEXKXUQQu4o5kPMVeYAuaMY8j6vXWUOUDiKoXSSwM1JAncnCVxlDhAO40kTCq0+KLQIAOgNDB1derRp9ejo0qO9+3m7Vo/2Lh20OgOmeLlgipfLsP4gzcUYQ3uX3ijEep63dOjQqtWhuYObOnV6aHUGdOkN0OoM0OoN0OoYtHoDuvjXfZcb0KbV37I9zhxCAaB0kkDpJIabkwRuPY/OxvOUThLM8JPD9RYX05v7Gx29lkxCxgCRUADn7kOtsUggEMBJ4gAniQNG8u4HegNDS4cOTR1d3NSuQ3NHF5o6uh/5111o7tChsb0Lje1daGjrwo02Ldq0ehgYd4KCa3vs3+bX2/7f3ou7hmngyLH5L0cIGVEioQAKJzEUTtYN3dOp0/MBdqO1Cw1tWtS3abl5rVrcaOPm3eie5+k8fGdMKbQIIRaTOojgIxfBxwaDP46dUweEEGIGCi1CiF2h0CKE2BUKLUKIXaHQIoTYFQotQohdodAihNiVcdNPq+dqpaamERrVkRAyJD2/zVtdWThuQqu5uRkA4O/vb+OaEEIG09zcDIVCMeDycXPBtMFgQFVVFVxdXW85TlFTUxP8/f1RWVlJF1f3Qp/LwOizMc2Sz4UxhubmZvj5+UE4yD0Bxs2ellAoxMSJEy16j1wupy+gCfS5DIw+G9PM/VwG28PqQQ3xhBC7QqFFCLErFFomSKVSpKamQiqlGxD0Rp/LwOizMW0kPpdx0xBPCLk90J4WIcSuUGgRQuwKhRYhxK5QaBFC7AqFFiHErlBo9bFjxw4EBQVBJpNBrVYjLy/P1lWyuddffx0CgcBoCg0NtXW1Rt2JEyfw2GOPwc/PDwKBAF9++aXRcsYYNmzYAF9fXzg6OiI2NhYXLlywTWVH2a0+m6eeeqrfd2ju3LlWbYtCq5c9e/YgJSUFqampKCgoQHh4OOLi4lBbW2vrqtncjBkzUF1dzU/ffPONras06lpbWxEeHo4dO3aYXP7WW2/hnXfewfvvv4/c3Fw4OzsjLi4OHR0do1zT0XerzwYA5s6da/Qd+vzzz63bGCO8qKgotmrVKv61Xq9nfn5+LC0tzYa1sr3U1FQWHh5u62qMKQDYF198wb82GAxMpVKxzZs38/MaGhqYVCpln3/+uQ1qaDt9PxvGGEtOTmaPP/74sKyf9rS6abVa5OfnIzY2lp8nFAoRGxuLnJwcG9ZsbLhw4QL8/PwwefJkJCUloaKiwtZVGlPKysqg0WiMvj8KhQJqtZq+P92OHTsGb29vTJs2Db/5zW9w/fp1q9ZDodWtrq4Oer0ePj4+RvN9fHyg0WhsVKuxQa1WY9euXTh8+DDee+89lJWV4YEHHuDHKCPgvyP0/TFt7ty5+OSTT5CZmYk333wTx48fx7x586DX6y1e17gZmoZYb968efzzWbNmQa1WIzAwEHv37sWyZctsWDNiL5588kn+eVhYGGbNmoUpU6bg2LFjmDNnjkXroj2tbp6enhCJRKipqTGaX1NTA5VKZaNajU1KpRIhISG4ePGirasyZvR8R+j7Y57JkyfD09PTqu8QhVY3iUSCiIgIZGZm8vMMBgMyMzMRHR1tw5qNPS0tLbh06RJ8fX1tXZUxY9KkSVCpVEbfn6amJuTm5tL3x4Qff/wR169ft+o7RIeHvaSkpCA5ORmRkZGIiorCtm3b0NraiqVLl9q6ajb14osv4rHHHkNgYCCqqqqQmpoKkUiExMREW1dtVLW0tBjtGZSVlaGoqAju7u4ICAjA888/jz/96U8IDg7GpEmTsH79evj5+SEhIcF2lR4lg3027u7u+MMf/oAFCxZApVLh0qVLePnllzF16lTExcVZvrFhOQd5G9m+fTsLCAhgEomERUVFsVOnTtm6Sja3aNEi5uvryyQSCZswYQJbtGgRu3jxoq2rNeqysrIYgH5TcnIyY4zr9rB+/Xrm4+PDpFIpmzNnDistLbVtpUfJYJ9NW1sbe+SRR5iXlxcTi8UsMDCQLV++nGk0Gqu2ReNpEULsCrVpEULsCoUWIcSuUGgRQuwKhRYhxK5QaBFC7AqFFiHErlBoEULsCoUWIcSuUGgRQuwKhRYhxK5QaBFC7Mr/B7J8GWEvmS4aAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:51:24.555532Z", + "iopub.status.busy": "2024-02-29T20:51:24.554714Z", + "iopub.status.idle": "2024-02-29T20:52:56.127454Z", + "shell.execute_reply": "2024-02-29T20:52:56.126441Z" + }, + "papermill": { + "duration": 91.594797, + "end_time": "2024-02-29T20:52:56.130022", + "exception": false, + "start_time": "2024-02-29T20:51:24.535225", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:52:56.169853Z", + "iopub.status.busy": "2024-02-29T20:52:56.169020Z", + "iopub.status.idle": "2024-02-29T20:52:56.189468Z", + "shell.execute_reply": "2024-02-29T20:52:56.188631Z" + }, + "papermill": { + "duration": 0.042595, + "end_time": "2024-02-29T20:52:56.191313", + "exception": false, + "start_time": "2024-02-29T20:52:56.148718", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0046860.0163790.0026093.8580920.069530.8769560.0904210.0000121.3648460.0396720.0928140.0510780.0665777.981087e-075.222938
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.004686 0.016379 0.002609 3.858092 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.06953 0.876956 0.090421 0.000012 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 1.364846 0.039672 0.092814 0.051078 0.066577 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 7.981087e-07 5.222938 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:52:56.227345Z", + "iopub.status.busy": "2024-02-29T20:52:56.227067Z", + "iopub.status.idle": "2024-02-29T20:52:56.620316Z", + "shell.execute_reply": "2024-02-29T20:52:56.619376Z" + }, + "papermill": { + "duration": 0.413742, + "end_time": "2024-02-29T20:52:56.622445", + "exception": false, + "start_time": "2024-02-29T20:52:56.208703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:52:56.661150Z", + "iopub.status.busy": "2024-02-29T20:52:56.660832Z", + "iopub.status.idle": "2024-02-29T20:54:33.509851Z", + "shell.execute_reply": "2024-02-29T20:54:33.508865Z" + }, + "papermill": { + "duration": 96.871564, + "end_time": "2024-02-29T20:54:33.512526", + "exception": false, + "start_time": "2024-02-29T20:52:56.640962", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:54:33.551579Z", + "iopub.status.busy": "2024-02-29T20:54:33.550692Z", + "iopub.status.idle": "2024-02-29T20:54:33.567817Z", + "shell.execute_reply": "2024-02-29T20:54:33.567105Z" + }, + "papermill": { + "duration": 0.038334, + "end_time": "2024-02-29T20:54:33.569869", + "exception": false, + "start_time": "2024-02-29T20:54:33.531535", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:54:33.606611Z", + "iopub.status.busy": "2024-02-29T20:54:33.606318Z", + "iopub.status.idle": "2024-02-29T20:54:33.611340Z", + "shell.execute_reply": "2024-02-29T20:54:33.610539Z" + }, + "papermill": { + "duration": 0.025713, + "end_time": "2024-02-29T20:54:33.613336", + "exception": false, + "start_time": "2024-02-29T20:54:33.587623", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.4453968018069303}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:54:33.651935Z", + "iopub.status.busy": "2024-02-29T20:54:33.651124Z", + "iopub.status.idle": "2024-02-29T20:54:34.000147Z", + "shell.execute_reply": "2024-02-29T20:54:33.999197Z" + }, + "papermill": { + "duration": 0.37059, + "end_time": "2024-02-29T20:54:34.002126", + "exception": false, + "start_time": "2024-02-29T20:54:33.631536", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIfElEQVR4nO3deXxU5b348c+ZfSaZmewbJCTsCiKRTbQILgWRUqheb+sKImorXmq53pdyf68rLm3Rtm51oa0L6HWht1atrQsuFXGpCiiKiOwkIQtZZ5JJJrM+vz9OZkhMAklI5swkz/v1mtdkzpw5852TyTfP85xnUYQQAkmSJA3ptA5AkiRJJiJJkjQnE5EkSZqTiUiSJM3JRCRJkuZkIpIkSXMyEUmSpDmZiCRJ0pxMRJIkaU4mogRTWFjID37wgwF9D0VRuOOOO0643x133IGiKAMaizQ0yEQUYx9//DF33HEHLpdL61CkBPDYY4+xYcMGrcMYcDIRxdjHH3/MnXfeKROR1CMyEUmSJMWITEQxdMcdd/Bf//VfABQVFaEoCoqicPjwYdavX895551HVlYWZrOZU089lXXr1nV7rLfeeovJkydjsVg49dRTeemll3odj8/n4xe/+AWZmZnY7XZ++MMfcuTIkS73/fDDD5k2bRoWi4VRo0bxxz/+scv9FEXhpptu4rnnnmPcuHFYLBamTJnCli1bOp0LRVHYu3cvV155JU6nk8zMTP7nf/4HIQRlZWUsWrQIh8NBTk4O9913X68/H8Abb7zB7NmzsdvtOBwOpk2bxvPPP99hn7/85S9MmTIFq9VKRkYGV155JeXl5R32Wbp0KcnJyZSXl7N48WKSk5PJzMzklltuIRQKddg3HA7z0EMPcdppp2GxWMjMzOTCCy9k27Zt0X168vsuLCxk165dvP/++9Hvypw5c/p0HuKekGLmyy+/FJdddpkAxAMPPCD+93//V/zv//6v8Hg8Ytq0aWLp0qXigQceEA8//LCYO3euAMQjjzzS4RgjRowQY8eOFSkpKeK2224T999/vzjttNOETqcTb731Vq/iufLKKwUgLr/8cvHII4+Iiy++WEyaNEkAYs2aNdH9vvrqK2G1WkVBQYFYu3atuPvuu0V2dnZ03/YAMXHiRJGRkSHuuusuce+994oRI0YIq9Uqdu7cGd1vzZo1AhCTJ08Wl112mXjsscfEggULBCDuv/9+MW7cOPGzn/1MPPbYY+Lss88WgHj//fd79fnWr18vFEUREydOFL/61a/Eo48+KpYvXy6uuuqqDvsAYtq0aeKBBx4Qt912m7BaraKwsFA0NDRE91uyZImwWCxiwoQJYtmyZWLdunXikksuEYB47LHHOrzv0qVLBSDmz58vHnzwQfG73/1OLFq0SDz88MPRfXry+3755ZfF8OHDxfjx46Pfld7+jhOFTEQx9tvf/lYA4tChQx22t7S0dNp33rx5YuTIkR22jRgxQgDir3/9a3Sb2+0Wubm5ori4uMdx7NixQwDixhtv7LD98ssv75SIFi9eLCwWiygpKYlu++abb4Rer+8yEQFi27Zt0W0lJSXCYrGIH/3oR9FtkUR0/fXXR7cFg0ExfPhwoSiKuOeee6LbGxoahNVqFUuWLOnx53O5XMJut4sZM2YIr9fb4blwOCyEEMLv94usrCwxceLEDvv84x//EIC4/fbbo9uWLFkiAHHXXXd1OFZxcbGYMmVK9PE///lPAYiVK1d2iinyvkL0/Pc9YcIEMXv27B584sQmq2Zxwmq1Rn92u93U1tYye/ZsDh48iNvt7rBvXl4eP/rRj6KPHQ4HV199NV988QVVVVU9er/XX38dgJUrV3bYfvPNN3d4HAqF2LRpE4sXL6agoCC6/ZRTTmHevHldHnvmzJlMmTIl+rigoIBFixaxadOmTtWY5cuXR3/W6/VMnToVIQTXXnttdHtKSgrjxo3j4MGDPfpsAG+//TZNTU3cdtttWCyWDs9Fuhxs27aN6upqbrzxxg77LFiwgPHjx/Paa691Ou5Pf/rTDo9nzZrVIa6//vWvKIrCmjVrOr22fVeH3vy+hwKZiOLERx99xAUXXEBSUhIpKSlkZmby3//93wCdvpijR4/u1H9n7NixABw+fLhH71dSUoJOp2PUqFEdto8bN67D45qaGrxeL2PGjOl0jO/uG9HVvmPHjqWlpYWampoO29snNwCn04nFYiEjI6PT9oaGhu4/0HccOHAAgIkTJ3a7T0lJCdD15xg/fnz0+YhIe097qampHeI6cOAAeXl5pKWlHTe+3vy+hwKD1gFI6pf3/PPPZ/z48dx///3k5+djMpl4/fXXeeCBBwiHw1qHOGD0en2PtgEIjWc17i6u3hrKv+/uyEQUY131RP773/+Oz+fj1Vdf7VBCeO+997o8xv79+xFCdDjW3r17AfVKS0+MGDGCcDjMgQMHOpQI9uzZ02G/zMxMrFYr+/bt63SM7+4b0dW+e/fuxWazdSpRDJRISe/rr79m9OjRXe4zYsQIQP0c5513Xofn9uzZE32+t++7adMm6uvruy0V9eb3PVR6rsuqWYwlJSUBdOjQGPlP2/4/vtvtZv369V0eo6Kigpdffjn6uLGxkWeeeYbJkyeTk5PTozjmz58PwO9///sO2x988MEOj/V6PfPmzeOVV16htLQ0un337t1s2rSpy2P/61//4vPPP48+Lisr429/+xtz587tt1LFicydOxe73c7atWtpbW3t8FzkPE+dOpWsrCz+8Ic/4PP5os+/8cYb7N69mwULFvT6fS+55BKEENx5552dnou8b29+30lJSUOi86ssEcVYpBH3//2//8dPfvITjEYj55xzDiaTiYULF3LDDTfg8Xh4/PHHycrKorKystMxxo4dy7XXXsvWrVvJzs7mqaee4ujRo90mrq5MnjyZyy67jMceewy3281ZZ53Fu+++y/79+zvte+edd/Lmm28ya9YsbrzxRoLBIA8//DATJkzgq6++6rT/xIkTmTdvHitXrsRsNvPYY49FjxMrDoeDBx54gOXLlzNt2jQuv/xyUlNT+fLLL2lpaeHpp5/GaDRy7733cs011zB79mwuu+wyjh49ykMPPURhYSG/+MUvev2+5557LldddRW///3v2bdvHxdeeCHhcJgPPviAc889l5tuuom5c+f2+Pc9ZcoU1q1bxy9/+UtGjx5NVlZWp9LboKDdBbuh6+677xbDhg0TOp0uein/1VdfFZMmTRIWi0UUFhaKe++9Vzz11FOdLvWPGDFCLFiwQGzatElMmjRJmM1mMX78ePGXv/yl13F4vV6xcuVKkZ6eLpKSksTChQtFWVlZp8v3Qgjx/vvviylTpgiTySRGjhwp/vCHP0QvwbcHiBUrVohnn31WjBkzRpjNZlFcXCzee++9DvtFXltTU9Nh+5IlS0RSUlKnWGfPni0mTJjQ68/46quvirPOOktYrVbhcDjE9OnTxQsvvNBhnz//+c+iuLhYmM1mkZaWJq644gpx5MiRHsXV1TkIBoPit7/9rRg/frwwmUwiMzNTzJ8/X2zfvr1DXD35fVdVVYkFCxYIu90ugEF7KV8RQq5rJvUfRVFYsWIFjzzyiNahSAlEthFJkqQ52UY0CJ2oU6PVasXpdMYomv5XU1PTqWNkeyaT6YT9eKT4IhPRIJSbm3vc55csWZLQU0tMmzatU2fD9mbPns3mzZtjF5B00mQiGoTefvvt4z6fl5c3YO8diybH5557Dq/X2+3zqampAx6D1L9kY7UkSZqTjdWSJGkuoatm4XCYiooK7Hb7kOkKL0mJRAhBU1MTeXl56HTdl3sSOhFVVFSQn5+vdRiSJJ1AWVkZw4cP7/b5hE5EdrsdUD+kw+HQOBpJkr6rsbGR/Pz86N9qdxI6EUWqYw6HQyYiSYpjJ2o6kY3VkiRpTiYiSZI0JxORJEmaS+g2IknqD0IIgsHgccevSV3T6/UYDIaT7j4jE5E0pPn9fiorK2lpadE6lIRls9nIzc3FZDL1+RgyEUlDVjgc5tChQ+j1evLy8jCZTLJjbC8IIfD7/dTU1HDo0CHGjBlz3E6LxyMTkZSQWvxBth1uoMLlxWY2cPpwJyPSk3p1DL/fTzgcJj8/H5vNNkCRDm5WqxWj0UhJSQl+v7/TGnI9pWljdWFhYXRN7/a3FStWaBmWFOfcLQFe+KyM7SUNVLpbOVDt4aXPy/lwX22fRv/39b+4pOqP86dpiWjr1q0dGgi//vprvv/973PppZdqGJUUz4QQbNpVRaM3QIrNyIyidI42trKjzMXWw/U4rAYmDU/ROkyplzRNRN9d4+qee+5h1KhRzJ49W6OIpHinKArnnZLFP7+tZv7EHOwWI6fmOXBYDZS7WhmbffyhBFJ8ipsyqd/v59lnn2XZsmXdNhj6fD4aGxs73KShJyPZzL9PzcduMUa3nVGQysJJuViMsVk3bSgpLCzstN5df4ubRPTKK6/gcrlYunRpt/usXbsWp9MZvcmR90PL8dp/Iu2LEaGwnO8vkcRNInryySeZP3/+cacxXb16NW63O3orKyuLYYSSloQQvLj9CO/uPorHF+x2P1eLn1e/rODlL8pjGF388/v9WodwXHGRiEpKSnjnnXdYvnz5cfczm83RkfZyxP3QUuPxcaTBy66KRvTH6euj1ykcqmmmrL6Fo42t3e53Iv5guNtbMBTu8b6BHuzbF3PmzOGmm27ipptuwul0kpGRwf/8z/9ES42FhYXcfffdXH311TgcDq6//noAPvzwQ2bNmoXVaiU/P5+VK1fS3NwcPW51dTULFy7EarVSVFTEc88916f4eisu+hGtX7+erKysPq01Lg0N+6s9ABRmJGE1dd8OZLcYGZudzLdVTeyqcJPt6Fu/lkff67z0dkRRRhKLi4dFH/9pywECoa6rgsNTrVw69VgTwlMfHcLr7ziU5BffH9unGJ9++mmuvfZaPvvsM7Zt28b1119PQUEB1113HQC/+93vuP3221mzZg0ABw4c4MILL+SXv/wlTz31FDU1NdFkFlmufOnSpVRUVPDee+9hNBpZuXIl1dXVfYqvNzRPROFwmPXr17NkyRIMBs3DkeLUgRr1v/bozOQT7ntqnoNvq5rYU+XhnDGZGPRxUfDvd/n5+TzwwAMoisK4cePYuXMnDzzwQDQRnXfeefznf/5ndP/ly5dzxRVXcPPNNwMwZswYfv/73zN79mzWrVtHaWkpb7zxBp999hnTpk0D1CaTU045ZcA/i+Z/+e+88w6lpaUsW7ZM61CkOOX2Bqht8qFTFIoyTtx7Oj/Vht1ioKk1yOG6ZkZn9f6S/opzR3f7nO47NcPrzxnV7b7frUUuO7uo17F058wzz+zQQD9z5kzuu+++aN+8qVOndtj/yy+/5KuvvupQ3RJCRIe67N27F4PBwJQpU6LPjx8/npSUlH6LuTuaJ6K5c+fGZC0sKXGV1asDUnOdluNWyyJ0OoXRWcl8UeriQE3fEpHJ0PNS1EDte7KSkjombY/Hww033MDKlSs77VtQUMDevXtjFVonmiciSTqRIw1qIhqeau34hBBQ8QVUfglGG4w6D5LVTrKjMtVEdKi2mXBYoPtuMWYQ+PTTTzs8/uSTTxgzZgx6fdfJ+owzzuCbb75h9OiuS3vjx48nGAyyffv2aNVsz549uFyufo27K4Oz8iwNKik2Exl2M/lp3xmYWvov2LsJmqqg/iDseBa8DQDkpVgZmZnE9KI0woO0xF1aWsqqVavYs2cPL7zwAg8//DA///nPu93/1ltv5eOPP+amm25ix44d7Nu3j7/97W/cdNNNAIwbN44LL7yQG264gU8//ZTt27ezfPlyrFZrt8fsLzIRSXHvzJHpXHXmiI6JqKkKDn2g/lxwJtizIdAK+94B1Mv4iyYP44yC1EHbWH311Vfj9XqZPn06K1as4Oc//3n0Mn1XJk2axPvvv8/evXuZNWsWxcXF3H777R367q1fv568vDxmz57NxRdfzPXXX09WVtaAf5aEXnK6sbERp9OJ2+2WfYqGmq/+D+oOQOY4mPAjaKmHrU+ACMOUpeDIPeEhWltbOXToEEVFRX2evkIrc+bMYfLkyQM+9KInjncee/o3Ojj/VUiDRmNroFMHQjzVahJSdDByjnppKikdssarz1d+2eH131Q09rnjoBQbMhFJce0fX1by2OYDlNQd6/1LxQ71PmMM2NKObc89Xb2v3gUhdRjIX7YdYdOuKird3tgELPWJvGomxa1gKEytx0coLEixts2HHArA0a/Vn/Mmd3xByggw28HXBK4SSB/F8FQr33gDHGnw9noGx3i2efNmrUPoV7JEJMWtWo+fUFhgNelxWNv+Z9YfhKAPLA5I/U7nQEWB9LZL03XqEI1hKeoVn3KXLBHFM5mIpLgVGbSa7TAf60Fcs0e9zxzXudsydEpEuU618bSmyUdYTg0St2QikuJWNBHZ267EhEPRBEPGuK5flFKgNmK3NoLXRarNhMmgwx8MU98S31NhDGUyEUlxq7rJB0BWZAS9q0Stlpls4BjW9YsMJrDnqD+7y9DpFLLsZgCq3H2fFkQaWDIRSXEpHBY0NKslmMxkNZFQu0+9zxgLx1s5IqVt2g2XOnFeTlv17GTmJ5IGlrxqJsWlkBCcOSqd+mZ/u4bqQ+p9evcj4wFwFgCfgltNRONzHOQ6LeQ4B36ogtQ3MhFJccmo1zGtsF0fIW+DelN0ajvQ8Tjbqm0t9RDwkmm3ktlWPZPik6yaSYmhoUS9d+SB4QRJxWgFa4r6c1PVgIYl9Q+ZiKS4VOn20tDsP3bJveGwep9a2LMDRBqs2xJRucvLpwfrqJD9ieKSTERSXHpr11E2fHyY0voWdd6h3iai5LZE5FET0TcVjXx8oI7Dtc3HeRHqewX9sb/1Yuz5M888Q3p6Oj6fr8P2xYsXc9VVV/X4OPFEthFJcScYCuNqCQCQnmxSB7kGvKA3qlWznvhOiSgjWR0iUuPxdfcKVSgAH9zXp7hPyqz/VLse9MCll17KypUrefXVV6PLs1dXV/Paa6/x1ltvDWSUA0bzElF5eTlXXnkl6enpWK1WTjvtNLZt26Z1WJKGGloChIXAbNSRbDYcKw2ljABdD1dyjSQirwsCXjLaugDUeRK/U6PVauXyyy+PrrwB8Oyzz1JQUMCcOXO0C+wkaFoiamho4Oyzz+bcc8/ljTfeIDMzk3379pGamqplWJLGXG09oFNtJnVoh6tUfSJ1RM8PEmmw9rrAc5SMZLVvkdsbwBcMYTZ0k9D0RrV0Emt644n3aee6665j2rRplJeXM2zYMDZs2MDSpUu7Xa493mmaiO69917y8/M7ZPaiov5b5UBKTC6vWi1LtRnVtpO2/kA4e7nEeFKmmoia67CmFpJsNuDxBalv9pPbXZ8iRelxFUlLxcXFnH766TzzzDPMnTuXXbt28dprr2kdVp9pWjV79dVXmTp1KpdeeilZWVkUFxfz+OOPd7u/z+ejsbGxw00afCI9qlNsJmiuVYd16I2QnN27AyVlqPfNNQCkJakJpr458atnoK5TtmHDBtavX88FF1xAfn4vE3Uc0TQRHTx4kHXr1jFmzBg2bdrEz372M1auXMnTTz/d5f5r167F6XRGb4l84qXuRRqqU22mY6Uhx7DjD+voiq1jIkpNMnY4fqK7/PLLOXLkCI8//njCrwuoaSIKh8OcccYZ/PrXv6a4uJjrr7+e6667jj/84Q9d7r969Wrcbnf0VlZWFuOIpViYUpjKzFHp5Dgs4D6ibnQO7/2BktSlhWiuASE4oyCVq2aOYEZR2vFflyCcTieXXHIJycnJLF68WOtwToqmbUS5ubmceuqpHbadcsop/PWvf+1yf7PZjNksu+oPdqMykxkVWVr6ZBKRLV1t8wn6wO8hxdb7hRbjXXl5OVdccUXC/11omojOPvts9uzZ02Hb3r17GTGiF1dHpMGr1a3eFF33034cj94A1jRoqVNLRebBk4gaGhrYvHkzmzdv5rHHHtM6nJOmaSL6xS9+wVlnncWvf/1r/v3f/53PPvuMP/3pT/zpT3/SMixJQ/XNftzeAOnJJhzucnVjclbfr2QlpbclolpIG8nnpQ3UNvmYOSqd3l0wjy/FxcU0NDRw7733Mm5cN5PEJRBNE9G0adN4+eWXWb16NXfddRdFRUU8+OCDXHHFFVqGJWloT1UTnxysY0Keg7nGSLXsJC5KJGVCzd5og/XX5W7qPH7GZtvJSe5h58g4dPjwYa1D6FeaD/H4wQ9+wA9+8AOtw5DiRLQzY5IJak+ifSgicuWspU49rs1EncdPQ4ufnGQ5P1G80HyIhyS1F+3MaAY8aimmJ6u2dsuWrt631KvHtalVvIZ281cn8GLHcaE/zp9MRFJciSSI1LBLXT7alATmk1hO3No2XCjgBX8LKTa1ZcjtDWA0qj+3tLScTMhDXuT8Rc5nX2heNZOkiNZACF9AXRraEahWNzryul42qKcMpmOLLnrrcVrVPkTulgB6vZ6UlBSqq9X3stlsCTtWSwtCCFpaWqiuriYlJQW9vu9tbjIRSXGjsa1aZjPpMTa3zazY02k/jseWpiailnocKeowkcbWIOGwICdHHaUfSUZS76WkpETPY1/JRCTFjcZWNRE5rEZoqlQ32k+ifSjCmqZONeutx55tQKcohMKCZn8Qu8VIbm4uWVlZBAKDY+hHLBmNxpMqCUXIRCTFjWyHhQsn5mAMtcI+l7qxPxJRuwZrnU7hqpkjSDYbMBmONZHq9fp++YOS+kYmIilu2C1GTsk1Ql3b1TJbOhgtJ39gW9vYMq965SwyCl+KH/KqmRR/GivU+5O5bN9e5MpZS0Ov5oaWYkeWiKS48W1VIxaDnuGucvWLae+HhmoAS4o6xWw4CK1uKv1mdpU34rQZO66dJmlGloikuCCE4J1vjvLy50cIuNrGmPVXiUinU5MRgLceT2uQneVuDlR7+uf40kmTiUiKCy3+EIGQwBL2YBY+tQSTlNV/bxBpJ2ppwGlVO95FrtJJ2pOJSIoLkaSQSQM6RVEbqvX92HLQrsHa0ZaImn0h/MFw/72H1GcyEUlxodEbBCAdt7rBfnId5DqJNFh7G7AY9ViM6qV6t1eWiuKBTERSXIgkhLSweom91xPln0i0jcgFgMOqlraaZPUsLshEJMWFyPAOR2iAElGkRNTqgnAYu0WtnjW1Bvv3faQ+kYlIiguNrQEMIS9JokUd5Jrcjw3VoI7g1+khHAJ/E3aLWiJq9stEFA9kPyIpLpw9OoNGWz2OUqNaejH082TwOh1YnOq8RN4GZo7M56xR6d2v+CrFlCwRSXEh22FhjLVZbUTu79JQRLt2IotRL5NQHJGJSIofnqPqfX+3D0W0u3ImxRdNE9Edd9yBoigdbuPHj9cyJEkDja0BvjrioqG6bY7qAUtEKep9q4tAKKz25P7iCKGwHH+mNc3biCZMmMA777wTfWwwaB6SFGPVja28t6uc82rLSc1zxKREZNApfFPZSCgs8PiC0d7WkjY0/6s3GAwnPbublNgaW4PYAnWY9QqYk9XbQGjXRqQAdosBV0uAptaATEQa07yNaN++feTl5TFy5EiuuOIKSktLu93X5/PR2NjY4SYlPk9rkCR/nTpR2UCVhuBY1Szog4BX9iWKI5omohkzZrBhwwbefPNN1q1bx6FDh5g1axZNTU1d7r927VqcTmf0lp9/EgvvSXGjqTWILdCA2aCDpIyBeyO98Vhpq9UV7UvUKId5aE7TRDR//nwuvfRSJk2axLx583j99ddxuVz83//9X5f7r169GrfbHb2VlZXFOGJpIDS1BrBGE1HmwL5Zu3YihywRxQ3N24jaS0lJYezYsezfv7/L581mM2ZzP3d0kzTX5A0wLNCAyWAe+ERkSQHKwOvCblFXkG3yyRKR1jRvI2rP4/Fw4MABcnP7aUIsKe6FwgJ/axOGcCsmg/7YRPcDpYsSUWQtNUk7mpaIbrnlFhYuXMiIESOoqKhgzZo16PV6LrvsMi3DkmJIAS4eZ8bgT8aYnKa24wykdn2JhqVaufHcUbKHdRzQNBEdOXKEyy67jLq6OjIzM/ne977HJ598QmbmABfPpbih0ynk6j2QbB7YhuqIaInIhV6noNfJJBQPNE1EGzdu1PLtpXjRUqveD3T7EBzrS+RrglBg4EtgUo/EVWO1NPQcaWhBHCkhLRgkKRaJyGhVR/YHfdDqZmuNjiMNLUwdkUZ+mm3g31/qUlw1VktDz96qRkpKS6j3+GNTIlKUY+1E3gaONrZyuLaFWo9v4N9b6pZMRJKmvB4X+nAAo9FwbIL7gdZuqEeyOTJlrOxLpCWZiCRNBRurATAkp6szKMZCu2lj5TCP+CATkaSpkEdtqDY5B3CM2XdFq2YuHG3DPDyyU6OmZCKSNOMPhjF41URkjmUiilbNGmSJKE7IRCRpxuMLYgvUo9cpsU1E0U6NbuxmfTQWOUGadmQikjTT5PVjDbjUwa62GHRmjDA7QdFBOIiNFjURGvR4A6HYxSB1IPsRSZrJNrWSkmND6PTHGpBjIbKih7cBpdXNz+aMwqiX/5O1JM++pBmLvwGn1UhKeo6aHGKpXYO1TELak78BSTvNNep9LDoyfle7BmtJezIRSZopLSvhaGMrLcYYVssi2o3C31/t4eUvjvDpwbrYxyEBfUxEBw8e7O84pCGo/EgpB2ubceucsX/zdr2rvf4Qh2tbqGpsjX0cEtDHRDR69GjOPfdcnn32WVpb5S9P6j0RDqF46wGwpWqwikuH3tVtc1fLvkSa6VMi+vzzz5k0aRKrVq0iJyeHG264gc8++6y/Y5MGsdbGOkQ4RFgxkOQY4FkZuxKpmvlbsBvUy/ZNrbJ3tVb6lIgmT57MQw89REVFBU899RSVlZV873vfY+LEidx///3U1NT0d5zSINPSUAlAyJqOQYsZEg1mdUoQIBkPoE4Z6wvKvkRaOKnGaoPBwMUXX8xf/vIX7r33Xvbv388tt9xCfn4+V199NZWVlf0VpzTItLrUde6FFlfMItpKReZAE2aj+qcgh3po46QS0bZt27jxxhvJzc3l/vvv55ZbbuHAgQO8/fbbVFRUsGjRov6KUxpkAm2j7vXJGiaidg3WkTFnHpmINNGnRHT//fdz2mmncdZZZ1FRUcEzzzxDSUkJv/zlLykqKmLWrFls2LCBzz//vMfHvOeee1AUhZtvvrkvIUkJJtTUNv2HI0u7INo1WDssBixGPf6QXNFDC30a4rFu3TqWLVvG0qVLu136JysriyeffLJHx9u6dSt//OMfmTRpUl/CkRJNKEi+1UdmnoNgUZF2cbTrXf2DSXnodYp2sQxxfSoRvf3229x6662dkpAQIrp2vclkYsmSJSc8lsfj4YorruDxxx8nNVWDjm1S7HnrMenAkZxMWpoGV8wi2vWulklIW31KRKNGjaK2trbT9vr6eop6+R9uxYoVLFiwgAsuuOCE+/p8PhobGzvcpATUfmiHomECaDcdCGFZJdNSn6pmQnQ9b4vH48FisfT4OBs3buTzzz9n69atPdp/7dq13HnnnT0+vhSfhKeGI/UtBM028kNhDFoNOjXZ1elpwyHcrjr+WaJOoP+j4uHaxDOE9SoRrVq1CgBFUbj99tux2Y4tvxIKhfj000+ZPHlyj45VVlbGz3/+c95+++0eJ6/Vq1dHYwBobGwkPz+/5x9Aigs+dxVHXF4O63QUaFki0unU6llLHYrPzeHaMHqdghACRcu4hqBeJaIvvvgCUEtEO3fuxGQyRZ8zmUycfvrp3HLLLT061vbt26muruaMM86IbguFQmzZsoVHHnkEn8+HXt+xo5vZbMZsNvcmZCkO+d1qHyJsmdq3zVhToKWOpHATipJEKCxo8YdIMsupumKpV2f7vffeA+Caa67hoYcewuFw9PmNzz//fHbu3Nlh2zXXXMP48eO59dZbOyUhaZAI+gg2q1NvGBxxsLR4W4O1vtVFstlJU2uQptagTEQx1qezvX79+pN+Y7vdzsSJEztsS0pKIj09vdN2aRBprsUXDOPX27Am2bWOpsN0IHbLqLZEFCDH2fO2Tunk9TgRXXzxxWzYsAGHw8HFF1983H1feumlkw5MGqSaa/AHw3iNGdHezJrq1Lu6VY7C10CPE5HT6Yw24DmdAzN/zObNmwfkuFIcaa7FHwrTYkwlMx6qP+1LRM7IGmcyEcVaj78J7atj/VE1k4ao5hp8wTAttrToPECaipSIAq04DCEsRtk2qYU+fRO8Xi9CiOjl+5KSEl5++WVOPfVU5s6d268BSoNMcw1jspJJG38qKalWraMBgwlMSeBvZlJ6mNOLRmkd0ZDUp55kixYt4plnngHA5XIxffp07rvvPhYtWsS6dev6NUBpEPE3g78Zs0FPXm4+NlMclIggWj1TWt3axjGE9XmGxlmzZgHw4osvkpOTQ0lJCc888wy///3v+zVAaRBpbhsWZE1RSyLxol2DtaSNPiWilpYW7Hb10utbb73FxRdfjE6n48wzz6SkpKRfA5QGEU813kCIPU1mvq2Ko3GC7Rqs//5lBc/86zDuFjltbCz1efL8V155hbKyMjZt2hRtF6qurj6pTo7SIOc5Sos/yFduK1+VxVE1KDIvkddFfbOfOo+fRjl/dUz1KRHdfvvt3HLLLRQWFjJjxgxmzpwJqKWj4uLifg1QGkQ8VfiCYZqN6STHwxWziEjVrNVFsjmyoodMRLHUp2/Dv/3bv/G9732PyspKTj/99Oj2888/nx/96Ef9Fpw0iISC0FyHPximOSkj+gcfF6JVs0bsyWpfOTllbGz1+duQk5NDTk7H9aimT59+0gFJg1RzDYgwXkz49Unx0YcowpQMOgOEg6Tq1HX65CT6sdWnb0NzczP33HMP7777LtXV1YS/M6mUXAlW6sRTBYBbnw6KEl+JSFHUUlFzLU7FA1hp8smqWSz16duwfPly3n//fa666ipyc3Pl3C3SiTWpU3/UKWrDcFyMM2vPmgrNtThEM2CVJaIY61MieuONN3jttdc4++yz+zseabDyVBEWgnolBSC+2ogg2mCdJDxYjNlYtFj0cQjr07chNTWVtLS0/o5FGqzCYfDUoCiw6HtTaNTZsZni7A+9rcHaLpr42Rw5zCPW+nT5/u677+b222+npaWlv+ORBqPmaggHUQwWHGlZDE+1xV91Xvau1lSfSkT33XcfBw4cIDs7m8LCQozGjvX93iysKA0BjeXqvSNP21U7jqdd72qEiN84B6k+JaLFixf3cxjSoOZWE1GlSOfg/lryU20UpNtO8KIYi5SIgn4+21fOt3VBzihIZeKwgZl7S+qoT4lozZo1/R2HNJg1VgBQEkzhs0P1CEH8JSK9Acx28DURaK6nzmOhocWvdVRDRp8XlHK5XDzxxBOsXr2a+vp6QK2SlZeX91tw0iDgbwavOll+jaKu6hpXfYjaa6uepSrNALi9si9RrPTpG/HVV19xwQUX4HQ6OXz4MNdddx1paWm89NJLlJaWRucqkqRItYykDNwBPRDEYY2zPkQRlhSgDKfSDKTLRBRDfSoRrVq1iqVLl7Jv374OiyNedNFFbNmypcfHWbduHZMmTcLhcOBwOJg5cyZvvPFGX0KS4pWrbVoYZ350IGm8l4jswgOoJaLuVjWW+lefEtHWrVu54YYbOm0fNmwYVVVVPT7O8OHDueeee9i+fTvbtm3jvPPOY9GiRezatasvYUnxqOEwAH5HPr6AOhTIEW+9qiPapgOxhdVE5AuE8QXDx3uF1E/6lIjMZjONjZ0nttq7dy+ZmT1fNG/hwoVcdNFFjBkzhrFjx/KrX/2K5ORkPvnkk76EJcUbX5M6K6Oi4DYPA8Bq0mMyaLTW/Ym0XTkz+NzRnt8uOUFaTPTpG/HDH/6Qu+66i0BA/SUpikJpaSm33norl1xySZ8CCYVCbNy4kebm5uj8Rt/l8/lobGzscJPiWENbtSw5i8aQ+ocdt9UyONaXyNdEVpKeLIeZkKyaxUSfEtF9992Hx+MhMzMTr9fL7NmzGT16NHa7nV/96le9OtbOnTtJTk7GbDbz05/+NLoaSFfWrl2L0+mM3vLz8/sSvhQrDYfU+9RCitKTWPa9IuZNyDn+a7RktIHeCEKwaHwyV8wYwbCUOFhpZAhQxEm0xn300Ud8+eWXeDwezjjjDC644IJeH8Pv91NaWorb7ebFF1/kiSee4P333+8yGfl8Pnw+X/RxY2Mj+fn5uN1uOUVtvAmH4KOHIOiD4isgpUDriHpm6xPgqYFJ/w7pcszZyWpsbMTpdJ7wb7TX5eRwOMyGDRt46aWXOHz4MIqiUFRURE5ODkKIXo8hMplMjB49GoApU6awdetWHnroIf74xz922tdsNmM2m3sbsqQFV6mahEw2cAzXOpqes6SoiahtzFlfvtNS7/WqaiaE4Ic//CHLly+nvLyc0047jQkTJlBSUsLSpUv7ZZrYcDjcodQjJaiaPep9xljQ6fj4QC0f7a+N/745bVfOmlw1PPOvwzzzL7kqTSz0qkS0YcMGtmzZwrvvvsu5557b4bl//vOfLF68mGeeeYarr766R8dbvXo18+fPp6CggKamJp5//nk2b97Mpk2behOWFG9CAahu64KROR6AnUfctPhDjMlKhnjt0AjRBmuj302dx4+iQDAUxqCP0yt9g0Svzu4LL7zAf//3f3dKQgDnnXcet912G88991yPj1ddXc3VV1/NuHHjOP/889m6dSubNm3i+9//fm/CkuJN9W4I+tU/6tRCAqEwLf4QQPz2qo5oKxGZA25MBh1CyPmrY6FXJaKvvvqK3/zmN90+P3/+/F6t9Prkk0/25u2lRCAElG9Tf849HRSFprYe1SaDDnO89iGKsKoT/imtLhwWPbWeMG5vgNSkOFqZdhDq1beivr6e7Ozsbp/Pzs6moaHhpIOSEljdfnV+ar1RTURAY1u7kMNiiP+GX4uzbUWPEBkGLwCueG/XGgR6lYhCoRAGQ/eFKL1eTzAoi7FDVigIBzerPw+bAqYk4FjVJu6rZaBOiGZTq2fpbaPw5XQgA69XVTMhBEuXLu32Erq82jXEHdqsDukw2aDgzOjmuB/s+l3WNPDUkK5rApy4ZCIacL36ZixZsuSE+/T0ipkUR4QA9xGoP3hsqlSDBWzpkJQBydlqcjne60s+hrKt6uOxF4LxWI/kY1WzBCgRAdjUdqIUmsiwm0mxyvahgdarRLR+/fqBikPSiqcG9r5xbN6g7pjtYM+B5Cw1MRltgFBLQFVfQWOlul/ROZA5rsNL503I4axRGRgNcd4+FGFTJ3BL1zVz1ZkjNA5maEiQsrI0IOoPwtcvqf1+9Aa182FyDuj04PeoSaa5Vp1h0dek3mr3dX0svRFGnQfDzuj0lE6n4LQlSGkIolfOaKnTNo4hRCaioaqp6lgSSimAUxaCpZuxQEEfeI6Cp1p9XXONug3U16QWQs5paqlpMGirmuHzQNCH0JsIhYXs1DiAZCIaioJ+2PWKmoRSC9UBnrrjLHhoMKvJqg8DV+ub/Xx6sI4sh5kpIxJkUU6jVW0T87ewdfd+Pq0xMbUwjTNHpmsd2aAlU/xQdHiLWt0y22HCj46fhE5SrcfHt1VN7K/2DNh7DIi26pk16CYQEjQ0yytnA0kmoqHG2wBHtqs/j5sPRsvx9z9JkRkOnYl25amtwToVNYE2yJkaB5RMREPN4Y9AhCGtKCbz7URG2zsToTNje23tRA6hzgLa0OKXE+kPIJmIhhKvC45+rf5cdE5M3jLSGTAlka6aQbRqlhRqRKco+INhPD45amCgyEQ0lFR8rnY+TCtS16GPgcQtEalVM523HqdFbUNraJbVs4EiE9FQEQpC5Vfqz3md+/oMhGDoWCki8UpEKaDoIBQg26ImoBqPHMI0UOTl+6Gi5lsIeNV+P+mjY/KWHl8QnaKg1ytYjQN3ZW5A6PRqO1FzLSOsXgLGrMQr1SUQmYiGiupv1Puc00AXm4Jwis3EinNH4/EF43/6j67Y0qG5llPtPk4tiE1VdqiSVbOhwN8C9W1L+2RNiOlb63VK4pYkktoWC22p1TaOIUAmoqGgdo96yT45C5Jk7+AeiySi5hqEEDS1BvDLJagHhKaJaO3atUybNg273U5WVhaLFy9mz549WoY0OFXvVu+zul64cqBs2lXFm19XJW6v5KQM9b65lr9uP8ITHxyipK5Z25gGKU0T0fvvv8+KFSv45JNPePvttwkEAsydO5fmZvnL7jcBL7jK1J+zxsfsbYUQ7K/2sLuykXCidgS0pqqN1qEAaXp12lh55WxgaNpY/eabb3Z4vGHDBrKysti+fTvnnBObDneDXv0htVqWlBFdoSIWPL4g/mAYnaKQYkuw4R0RkStnnhqyDU2AnTpPgpbu4lxcXTVzu90ApKV1PUq7qyWnpROo26/ex3j55EjnP6fVgF6XgFfMImwZ4Kkhg0bATq0sEQ2IuGmsDofD3HzzzZx99tlMnDixy33Wrl2L0+mM3vLz82McZYIJh6H+gPpzjPoORdS3De1IS07wJcLbGqxTUP9Jur2ywXogxE0iWrFiBV9//TUbN27sdp/Vq1fjdrujt7KyshhGmIAayyHQqs4nFOP15+vaSg5piVoti2hLRGZfPXaLASFkO9FAiIuq2U033cQ//vEPtmzZwvDh3f/BmM3mblcQkboQKQ2ljYxZJ8aImib1jzXTnuC/r+iVszoyHUaaWoMcbWxlWIr1+K+TekXTRCSE4D/+4z94+eWX2bx5M0VFRVqGM/i4StX7tJExfVshBGGhLhGW8InImqrOxx0KMN4RJDUplRzHwM7hNBRpmohWrFjB888/z9/+9jfsdjtVVVUAOJ1OrFb5H+ekBP3HVtY4wRSvb+2qIhgWnDbMSX7acZYN6iFFUbh8RgH+YBijPoEbqkHNpslZ4C5nXFIz43LkP8uBoGkb0bp163C73cyZM4fc3Nzo7c9//rOWYQ0O7jL1sr3FqY4kb1Pu8rJlb02HXStcXvZUNfHi9iO8881RQuH+6fdjMugSc4zZdyXnqPeeo9rGMYhpXjWTBkikWtZWGgqGwvzrYB3bSxoQAooykqKln/NPyWZPVRNfV7jZWe6m2R9k4aQ8dIl82b0/JWep956jtAZC1DT5cNqMibNgZAKIm6tmUj9rl4gaWwP8eVsZ2w6rSWhCnoP05GNXs/LTbFxwajaLJg/DoFM4WNPMB/v7PtDzH19V8OetpRxpaDnZTxEf7MdKRG/tquLF7UfYdzTBFgOIczIRDUZBv7r+GFCpZPHCp6VUN/qwmvQsPD2PuRNysJk6F4aLMpKYN1H9o/u8pIFDtb0faiOEoLS+hQpXK8bBsg6YLUOdJC3QSp5F7R9V3diqcVCDyyD5pkgdtLUP1YWsvLiriRZ/iEy7mcumFzA6K/m4Lx2bbae4IAW9TomuWd8b9c1+fAG1kToj0TszRugN0VkLcvVqx8ajMhH1q7joRyT1s7ZqmTdpGKFmwcjMJOZPzMVk6Nn/nbNHZ3DaMCfpfUgklW71DzTLYUnsoR3flZyjDvUQLsBGQ0uA1kAIS6LNPBmnZIloMGpLRMOLxnPJGcP5waS8HichAKNe16ckBMcSUa5zkPW1Sc4GwOyrjc6/XeWWpaL+IhPRINPS0kzQ3dZ/yJlPfprtpEomlW4vHx/oecN1pIE6b7D1PLariYimKnKd6mercHs1DGhwkYloEBFC8OH2Hew8Uk+TktSh/1BfeHxB/m/rET49WE9Z/YmvgLlbArhaAugUheGpgywRJeeonRt9TQy3qW1nlS5ZIuovMhENIl+XN9JUdRB/MIw4QW/qnkg2GzhtuAOALftqTtjvKxgOMzormcIMG2bDIGs7MZiiA2BHGNycNz6Lc8ZmahzU4CET0SDRGgjx4f5aHK0V5KfZcOT0z/xDZ45Mx2TQUd3o49uqpuPum55sZuHpeSyaPKxf3jvuONTPZfcf5fT8lMQfRxdHZCIaJD47VE/A5yWTBnIdlhOOL+spm8nAtEJ1orqP9tcSCA3huXgiq+M2VmgbxyAkE9Eg0NQaYEeZC7uvisI0K4otTR1j1k+KC1KwWww0tQb5otTV5T7VTa3UJ+ok+T3VViKiqQqP18fOI26+LndrG9MgIRPRIPB5qYtQWDDSWKfOD91PpaEIo17H2aPVeXm2Hq6nuW0Z6QghBO/urubpjw/z1RFXv753XLGlqZPMhYM0HD3CO7uPsu1wvdZRDQoyESU4IQSutmlZJ9ja/jv3cyICGJ9jpygjiTnjMrGZOjZEH2nwUuVuRa9TTthzO6EpSrRUlEUdAA0tAVr8weO9SuoBmYgSnKIoLJo8jMunZJEeVv84BiIRqe+Tx4Q8Z4epPQKhMJv3VAMwcZijyzFsg0pbO5G5pZKMtoHDFS7Zn+hkDfJvzdCRHW7rdGhNBYtjQN6jfQKqb/bzwb4aPL4gtR4/VpOemSMzBuR940pK24INrlKGpc6g1uOnrMHL6Cy7tnElOJmIEpjbG8Bs0KnjnVwl6sYBKA19lxCCt3ZVRYdzmI06Fp6eh9U0yPoOdcUxTF3vzOehwObnS6C8QZaITpZMRAnsw321HKzxcN4pWUyIYSJSFIW5E3LYd7QJvU5hXI4d+1CZJExvVKtnrjKGiWrAQa3HJwfAniTZRpSgWgMhDtR4CIYFmZYweNR2GlJHxOT905JMzBiZztTCtKGThCJS1HNsbSknLUltJ4qsWiL1jSwRJaj91R5CYUGG3UxWqAaEAFs6mGVbxYCLlDpdpVw0cR52q1GWhk6SpiWiLVu2sHDhQvLy8lAUhVdeeUXLcBLKnrbhFuOy7cfah2JUGhryHMNAZwCfh0x9s0xC/UDTRNTc3Mzpp5/Oo48+qmUYCafFH6SsbbqNsdnJ0HBYfSJFJqKY0BvA2dbLOnLupZOiadVs/vz5zJ8/X8sQEtK+ox6EgGyHhRRDAJrbLt3HoKFaapM2EhpKoP4AO8Qo9lY1Ma0ojaKMJK0jS0gJ1Vjt8/lobGzscBuK9h5tq5blJB9brSM5E0wnvzii1ENpbbMbNJRQ5/ZQ7vJS2oM5m6SuJVQiWrt2LU6nM3rLz8/XOiRNXDgxh9njMhmTbVf/KwOkFGoa05CTlKF2HA0HKTKoPdpL63q/6omkSqhEtHr1atxud/RWVlamdUiasFuMnFGQqi7wJxuqtaEo0VLRsHAligK1Hj9Nrb1f+URKsERkNptxOBwdbkOa1wUt9eqaW86hWTrUVLqaiMzug2S3TZJWUierZ32RUIloqGtsDfDS50eOzYFTf0C9d+SBcZCtmpEIUkaoV9Ba3Yy2qdUy2U7UN5omIo/Hw44dO9ixYwcAhw4dYseOHZSWlmoZVtzad7SJkroWdle2NdLXH1Lv0/tnWliplwymaPWsSKjNBCV1LYTDx5/bW+pM00S0bds2iouLKS4uBmDVqlUUFxdz++23axlW3NpTpa63PjbbDqHgsT4saSO1C2qoyzoFgLTmAzgsBvLTrPiCQ3g63T7StB/RnDlzTrgyhKRytfg52tiKTlEYk52sLisdCoApKbr4n6SBtFGgN6BrdbNsigUlMq+11CuyjShB7D2qloby06zq5GP1B9Un0kaqV3AkbRhMkD4aAKV6t8bBJC6ZiBLEnrZOjGOz7eoA19q96hNtfwSShrJOVe+P7kKEgtS1TQsi9ZxMRAmg1uOjtsl3bE5oT7V66V5vkO1D8SB9tNqr3d/Mlk8+5Zl/lUR7v0s9IxNRAgiFBflpNgozktSR3jXfqk+kjVSrBpK2dHrIOQ2A4a17AHWaFqnn5HxECSDbYeHfpgw/dlk4Ui3LGKddUFJHuZOh9FPyQpWYQh7K6hU5a2MvyBJRAtHpFLVa1lyr/heW7UPxw5YGKQVYjTrGhvYSFoJDtXLsWU/JRBTnDtc242m/oGHVV+p9+ijZmzre5E8HYGxwH/qwj2+rhubsEH0hE1Ec8wfDvLazkic/OKTOiRwOwdFd6pM5k7QNTuosfTQkZZBtU8j2fEtJXYscBNtDMhHFsf3VHvzBMA6rQV3Mr+4A+FvUKzTyaln8URTIn4HFqGd8YDdKKBjt/yUdn0xEcWxnuQuAU3Md6uKGFV+oT2RPVNuIpPiTPQEsTgodcGlOJZPzU7SOKCHIRBSnKlxeKlzqevIThznBU6P2plYUyCvWOjypOzo9FJ2D02okr3EH+qAcjd8TMhHFqc9LGwAYn2MnyWyAI1vVJzLGqFdopPiVPQHs2RD0w8HNhORo/BOSiSgO1Xl80Q5xZ4xIVSc/q9qpPpk/Q8PIpB5RFBj9fQRw5JtP+L+33qfOIxdgPB6ZiOJQrcePQacwKiuZjGQzHNoCIqxesncO1zo8qSdS8lGGT6XZHyK3ajOf7S3XOqK4JntWx6FxOXaGp1oJhoU6OX71bvW/bNFsrUOTeqNoNsMq9tCw/zCtu1+lcuR15KbI5Ya6IktEcSrJbMBpFLDndXVD7mS13UFKHAYTyVP+nXRnEs7WI3z7wUuEQnLStK7IRBRHPj5Qy+HIsAAhYM9r6ih7iwNGnatpbFIfJWcx7MxLMeh1WKq/ZM/Hr6i/W6kDmYjixNflbj49WM/fdlTgavbBgX9C9bfq5eBTFoLBrHWIUh/Zhk8kc8oiANx7P6bi4+chKBuv24uLRPToo49SWFiIxWJhxowZfPbZZ1qHFDNCCHaUuXhn91EApo9IJqX0bShrOwdj5sqlpAeBwtPORj9hIULRYWnYB1ufgNp9snTURvPG6j//+c+sWrWKP/zhD8yYMYMHH3yQefPmsWfPHrKysrQOb0A1NPv5YH8tB6o9IMKc7ahlWt27anVMUWDsPMibrHWYUj+ZOv0cKnJySatq+x3vfJFAUjbGgumQMXZIzy2lCI1nr58xYwbTpk3jkUceASAcDpOfn89//Md/cNtttx33tY2NjTidTtxud9wvthgKCxTapvIIh/hodynfHK7A6q/H4a+mOKmG4UlhFBQwJ8P4H0BakdZhSwMh6IeSD2k59Bk7S2txWo04k21Y0guwZhRgcmShs6WC2Q4GC+gMCTsveU//RjUtEfn9frZv387q1auj23Q6HRdccAH/+te/Ou3v8/nw+Y7VrRsbezjNQv0h2P9O9GFZfQsVbu93isVtPwvBxGFOkkzqWK7yBi9lDe0akL+z/6m5DhwWAwhBpbuVw3XNKAgEoAj1XggIhcNMGu7EadFD0M+oxlZ0tc2k2ozkp9lIMhnAaIXh02DYVDnFx2BmMMGo8/g2PIZS14dkNO+jodoN1TsBteOqQacwLseuLiuu01PTIihxtQI6BAooCgIdQlEAhbHZdlJtRgBqPD4O1nQ/F9KorGQyktTSV12zn/01HlJsJsZlJffuc0z6sXohpR9omohqa2sJhUJkZ3e8LJ2dnc23337baf+1a9dy55139v6Ngj51MrGI5hbCHm/3+7eEIdR2alq9CO9xxgv5FFDULwB+L/hbIikteq+gnuiQryXa6JyebMbpcGJNzYHkLEgtgtRCOZh1CDljTD75ORdzsLqJ2upyAvWl6JoqsQQbMQcb1YQDEA4R8rcSaOk+uYjkVhBtVTuPj7DnOKP+bV6g7eJHs49wkwcRMkJSa+8+gOi/rgiatxH1xurVq1m1alX0cWNjI/n5PVjzPSUfTv9JtHjr9AbQ+cKdSrtK2wZjsgUMOkDB6Qsy0h8ERVGrTSjR1ykoWKxGMKjJIyUQZow/2Pa8ujeKgk5RMBv0WEx6dZ16gwWjwYJRFxfXCiQNZdrNZNrNMCoDOJ1QWOALhmgNhLGa9EAAQj7sLS0UNPtAhFFEGAirRe22Uro1yQht09La/CEKmrufB8mWZASTHoTAFlD3NRsVSOrllVlT/3XO1DQRZWRkoNfrOXr0aIftR48eJScnp9P+ZrMZs7kPl7FNSR3aWxyp0NMCZXLbrSesbTdJ6iu9TsFmMmCLtlvrwWghyeIkqYdjnZPabj1hBYb1Osr+p+m/ZJPJxJQpU3j33Xej28LhMO+++y4zZ87UMDJJkmJJ86rZqlWrWLJkCVOnTmX69Ok8+OCDNDc3c80112gdmiRJMaJ5Ivrxj39MTU0Nt99+O1VVVUyePJk333yzUwO2JEmDl+b9iE5GIvUjkqShqKd/o/KyjSRJmpOJSJIkzclEJEmS5jRvrD4ZkeatHg/1kCQppiJ/mydqik7oRNTU1ATQs97VkiRppqmpCafT2e3zCX3VLBwOU1FRgd1ujw7P0EpkuElZWZm8gteOPC/dGwrnRghBU1MTeXl56I4zpCmhS0Q6nY7hw+NrVQuHwzFov1QnQ56X7g32c3O8klCEbKyWJElzMhFJkqQ5mYj6idlsZs2aNX2bHWAQk+ele/LcHJPQjdWSJA0OskQkSZLmZCKSJElzMhFJkqQ5mYgkSdKcTES90JsVaR9//HFmzZpFamoqqampXHDBBYN2Bdu+rtS7ceNGFEVh8eLFAxughnp7blwuFytWrCA3Nxez2czYsWN5/fXXYxSthoTUIxs3bhQmk0k89dRTYteuXeK6664TKSkp4ujRo13uf/nll4tHH31UfPHFF2L37t1i6dKlwul0iiNHjsQ48oHV2/MScejQITFs2DAxa9YssWjRotgEG2O9PTc+n09MnTpVXHTRReLDDz8Uhw4dEps3bxY7duyIceSxJxNRD02fPl2sWLEi+jgUCom8vDyxdu3aHr0+GAwKu90unn766YEKURN9OS/BYFCcddZZ4oknnhBLliwZtImot+dm3bp1YuTIkcLv98cqxLghq2Y9EFmR9oILLohuO96KtF1paWkhEAiQltbDNWESQF/Py1133UVWVhbXXnttLMLURF/OzauvvsrMmTNZsWIF2dnZTJw4kV//+teEQqFYha2ZhB70Giu9XZG2K7feeit5eXkdvpiJri/n5cMPP+TJJ59kx44dMYhQO305NwcPHuSf//wnV1xxBa+//jr79+/nxhtvJBAIsGbNmliErRmZiGLgnnvuYePGjWzevBmLZeiuad/U1MRVV13F448/TkZGhtbhxJ1wOExWVhZ/+tOf0Ov1TJkyhfLycn7729/KRCT1fkXa9n73u99xzz338M477zBp0qSBDDPmenteDhw4wOHDh1m4cGF0Wzisrp9uMBjYs2cPo0aNGtigY6Qv35nc3FyMRiN6vT667ZRTTqGqqgq/34/JZOrydYOBbCPqgb6uSPub3/yGu+++mzfffJOpU6fGItSY6u15GT9+PDt37mTHjh3R2w9/+EPOPfdcduzYMahm2uzLd+bss89m//790eQMsHfvXnJzcwd1EgLk5fue2rhxozCbzWLDhg3im2++Eddff71ISUkRVVVVQgghrrrqKnHbbbdF97/nnnuEyWQSL774oqisrIzempqatPoIA6K35+W7BvNVs96em9LSUmG328VNN90k9uzZI/7xj3+IrKws8ctf/lKrjxAzMhH1wsMPPywKCgqEyWQS06dPF5988kn0udmzZ4slS5ZEH48YMUIAnW5r1qyJfeADrDfn5bsGcyISovfn5uOPPxYzZswQZrNZjBw5UvzqV78SwWAwxlHHnpwGRJIkzck2IkmSNCcTkSRJmpOJSJIkzclEJEmS5mQikiRJczIRSZKkOZmIJEnSnExEkiRpTiYiKaFs2LCBlJSU6OM77riDyZMnRx8vXbp0UE89O1jJRCR1aenSpSiKwk9/+tNOz61YsQJFUVi6dGmH/fs7ARQWFvLggw922PbjH/+YvXv3dvuahx56iA0bNkQfz5kzh5tvvrlf45L6n0xEUrfy8/PZuHEjXq83uq21tZXnn3+egoICTWKyWq1kZWV1+7zT6exQYpISg0xEUrfOOOMM8vPzeemll6LbXnrpJQoKCiguLj6pY3dVUlm8eHG0lDVnzhxKSkr4xS9+gaIoKIoCdK6afVf7ktnSpUt5//33eeihh6LHOHToEKNHj+Z3v/tdh9ft2LEDRVHYv3//SX0uqW9kIpKOa9myZaxfvz76+KmnnuKaa64Z8Pd96aWXGD58OHfddReVlZVUVlb2+hgPPfQQM2fO5Lrrroseo6CgoNNnAli/fj3nnHMOo0eP7q+PIPWCTETScV155ZV8+OGHlJSUUFJSwkcffcSVV1454O+blpaGXq/HbreTk5Nzwpkwu+J0OjGZTNhstugx9Ho9S5cuZc+ePdE1xgKBAM8//zzLli3r748h9ZCcKlY6rszMTBYsWMCGDRsQQrBgwYKEn286Ly+PBQsW8NRTTzF9+nT+/ve/4/P5uPTSS7UObciSJSLphJYtW8aGDRt4+umn+63UoNPp+O5UWIFAoF+O3RPLly+PNsSvX7+eH//4x9hstpi9v9SRTETSCV144YX4/X4CgQDz5s3rl2NmZmZ2aPcJhUJ8/fXXHfYxmUwnvaZXd8e46KKLSEpKYt26dbz55puyWqYxWTWTTkiv17N79+7oz91xu92d1itLT0/vclL88847j1WrVvHaa68xatQo7r//flwuV4d9CgsL2bJlCz/5yU8wm819qhIWFhby6aefcvjwYZKTk0lLS0On00XbilavXs2YMWOOuwiCNPBkiUjqEYfDgcPhOO4+mzdvpri4uMPtzjvv7HLfZcuWsWTJEq6++mpmz57NyJEjOffcczvsc9ddd3H48GFGjRpFZmZmn+K+5ZZb0Ov1nHrqqWRmZlJaWhp97tprr8Xv98fkKqB0fHLOamnI+uCDDzj//PMpKyvrtCKrFFsyEUlDjs/no6amhiVLlpCTk8Nzzz2ndUhDnqyaSUPOCy+8wIgRI3C5XPzmN7/ROhwJWSKSJCkOyBKRJEmak4lIkiTNyUQkSZLmZCKSJElzMhFJkqQ5mYgkSdKcTESSJGlOJiJJkjT3/wFq1JH+ZxxDjQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:54:34.039667Z", + "iopub.status.busy": "2024-02-29T20:54:34.039338Z", + "iopub.status.idle": "2024-02-29T20:54:34.354702Z", + "shell.execute_reply": "2024-02-29T20:54:34.353779Z" + }, + "papermill": { + "duration": 0.336297, + "end_time": "2024-02-29T20:54:34.356751", + "exception": false, + "start_time": "2024-02-29T20:54:34.020454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABE3UlEQVR4nO2deXxU5dXHf3f2mcySdbKQhCSEfZclIiqoVEVFLFapKyh1xfYVSmt9bUG0inuhby1tVYjWClpFa9WKigUUBFkMshkgZIPsy2yZfe7z/nFnJglkmUySuTPJ+X4+9zOZe5+598zNzG+e5zznOYdjjDEQBEGIiERsAwiCIEiICIIQHRIigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhIggCNGRiW1Ab+B5HlVVVdDpdOA4TmxzCII4B8YYrFYrMjIyIJF03u+JaSGqqqpCVlaW2GYQBNENlZWVyMzM7PR4TAuRTqcDILxJvV4vsjUEQZyLxWJBVlZW8LvaGTEtRIHhmF6vJyEiiCimO9cJOasJghAdEiKCIESHhIggCNEhISIIQnRIiAiCEB0SIoIgREdUIcrJyQHHcedtS5cuFdMsIgawu73YeaIem7+twIeHqlDe2CK2SUQvEDWOaN++ffD5fMHnR44cwY9+9CPcdNNNIlpFRDtmuwfvHjwDi8MT3FdSZ8O0nETMzE+i5T4xiKhClJKS0u75M888g2HDhmHWrFkiWUREO4wxbD1aA4vDg3iNHAW5Sai1OFFUacK+sibo1TJMyIwX20yih0RNZLXb7cabb76J5cuX0y8a0Skcx+Hy0UZ8+UMd5o5Lg04lx5gMPfRqGc6anBiR2vVSAiI6iRoh+uCDD2AymbB48eJO27hcLrhcruBzi8USAcuIaCNZq8TNU9svdr4gOwEXZHe/lICITqJm1uy1117D3LlzkZGR0WmbNWvWwGAwBDdaeT+46KoEX2CiI4CPp3J9sQQXDQUWy8vLkZeXhy1btmD+/PmdtuuoR5SVlQWz2UyLXgc4jDG8e+AMEuMUKMhLglbZcWfeZHdj58kGuL08fjKl87QTRGSwWCwwGAzdfkejYmi2ceNGGI1GXHvttV22UyqVUCqVEbKKiCbqbS6caXag2uzERcOSO20nlXAorW8BzxhqLU6k6lURtJIIF9GHZjzPY+PGjVi0aBFksqjQRSIKOVVnAwDkJMdBrZB22k6nkmNEqhYAcLTKHBHbiN4juhB98cUXqKiowN133y22KUQUU1IvBCzmp2i7bTsmQxgCFNfY4PXx/WoX0TeI3gW58soru3RCEoTZ4UGD1QUJxyE3Oa7b9lkJGuhUMlidXpQ1tiDfSFP60Y7oPSKC6I7KJjsAIN2g6nJYFkAi4ZBvFHpOgZ4UEd2QEBFRz5lmQYgyE9TtDzAGnD0I7N8IHHobsNUHDw3zD+FKG1rA01R+1CP60IwguiNeo0CyTomsRE37AxXfAKd3tD63VgFTFgPqBGTEq5GXEoesRA14xiABBTpGM1ERRxQuocYoEAMQaw1w4HWA8UD2hUBzKWCtBZLygQm0aDpaCPU7SkMzIjYp3SmIUMpIIG82MHo+wEmAxlOApVps64geQkJERDUWp+f8KXhbHdBYIghP3myA44C4JMA4Sjhefajd649VWeD20jR+NENCREQ1Hx2qxp+3l7RPfFZVJDwmDwc0ia370ycKj3VHAZ8XAPDP/Wew9WgNqs2OyBhMhAUJERG1eH08Gmwu+HiGeLVC2OnzALVHhL8zJrV/QfxQQKkDvG7AVA6gdabtTDMJUTRDQkRELQ02N3w8g1ohhV7tn+BtOg14XYBKDyTktn8BxwnOakDwFQEYEi8I0VkTCVE0Q0JERC21FicAIFWvbE3xUV8sPKaMFITnXM4RonSDsOi13uqieKIohoSIiFqCQqTzr6DnfUGBQfLIjl8Uny04sZ0WwGFCgkYBhUwCt5dHk90dAauJcCAhIqKWOquQe8oYSOVhKheGZQoNoB/S8YtkCkCXJvxtroREwsGoE1LH1Jid/W0yESYkRERUwvMMzS1CDyZF689B1XBSeEweAUi6+OjG+zN3mioBAGn+4Vmgh0VEH7TEg4hKfIzhwmFJaGpxt3FUlwqPAT9QZxiyAewFzIIQjUrTI92gQppB3fXrCNEgISKiErlUgmk5bWKEHM3CxkkEP1BXGPzDNnsT4HEgRadGio4ye0YzNDQjYoNmIS4I+gxA1o2oyNWAOl7421rTr2YRfQMJERGVVJsdaG5xt065N5cJjwk5oZ0g4LD2C9FZkwN7TzeiiuKJohISIiIq+exoLQp3l6GiyS7kHeqpEGn9QmQThOhYlQW7SxpR1kCJ0qIREiIi6vD6eJjsQl37JK1CWOTqcQBSuTA0C4VzekTJWmGJSL3N1dkrCBERXYjOnj2L22+/HUlJSVCr1Rg/fjz2798vtlmEiDTbPeAZg1IuEeqXBXpD8UMBSfepYgG0CpHDBHgcSPaHADTaKKgxGhF11qy5uRkzZ87EZZddhv/85z9ISUnByZMnkZCQIKZZhMiY/BHQCRqFsLTDVCEcSBga+kkCDmuHCbDVIlkrxBaZHR64vD4oZSEKGhERRBWiZ599FllZWdi4cWNwX25ubhevIAYDJocwLEvQyAX/kD8eCIYelhiPSxGEqKUR6oQcaJUy2FxeNLW4kU4xRVGFqEOzDz/8EFOnTsVNN90Eo9GIyZMn45VXXum0vcvlgsViabcRA49ARHW8RgG0NAjLOqRyQJvasxPF+SvCtghJ9RPjBD9RUwsNz6INUYXo9OnTWL9+PYYPH46tW7figQcewC9+8Qu8/vrrHbZfs2YNDAZDcMvK6uEvJBETBBzVCRpFa29IP6TrZR0doWkvRAlx8nbnJ6IHUZPnKxQKTJ06Fbt37w7u+8UvfoF9+/bhm2++Oa+9y+WCy9U662GxWJCVlUXJ8wcYJfU21FtdGJ2mh6HsP0DtUSDnYiD3kp6dyFoL7N8gBEBevAwmhwdeniFeLYdMKvo8zaAg1OT5ovqI0tPTMWbMmHb7Ro8ejffee6/D9kqlEkolheoPdIalaIN1yWA+IzwaMnt+Ik2SkLPI6wLcNsRrqOJrtCLqz8LMmTNRXFzcbt+JEycwdGgPZkeIgYvTLGycpPO0H10hlQFq/3q1lvqu2xKiIqoQLVu2DHv27MHTTz+NU6dO4a233sLf/vY3LF26VEyzCBFpanGjtKEFFqcHMJ8VdmqNQp6hcIhLEh5bGgAAByua8dnRGlid5CeKJkQVomnTpuH999/Hpk2bMG7cODz55JNYu3YtbrvtNjHNIkSkuMaKD747iz0ljW2GZb2YlIhLER79PaIjZ804WmWhwMYoQ/Q0INdddx2uu+46sc0gooRgMGOcAmjohX8oQGDmzN4onFejQKPNjWa7GzmI642pRB9CUwdEVBEMZlQCsPn9Ovr08E+o8Q/N7E3CeTXCEK+Z8ldHFSRERFQREIgE3iSUlFbEAcpehGao/cuFPA7AbUe8RoglMjvIRxRNkBARUYPT44PLI5SG1nvqhJ36jI7LBoWKTCEUXQQARxMMar8QUVBjVEFCREQNFn8vRaOQQt7iz6wYatqPrgiUpbY3Qe8XIovTS3XOoggSIiJqsPin1PVqOWCtFnbqeuEfChCIJXI0QaeUQcJx8PEMLW5v789N9Amiz5oRRIBUvQpXj0uD3OcETpqEnX0hRG0c1hIJhztmDIVWKYNCRr/D0QIJERE16FRyjE6XA43+2TJNEiBX9f7EmtYeEdC6Cp+IHugngYg+LFXCY2+m7dsSmDmzNwv5jYiog3pERNTwQ40FKpkUmaazwgdT1weOagBQxQspZnkv4DSj2q3E0bMWGDTy9rXTCNGgHhERFTDG8MWxWrx/8Aw8Jv8as77qEUkkghgBgKMJNqcXh8+aUVJn65vzE72GhIiICuxuHzw+BhVvg5K5hB5MnLHvLhCcwm8OxhJZaOFr1EBCREQFAVFIQTMkHCc4qqV96Dlo47AOxBK1uHxwe/m+uwYRNiRERFRgcQgxPUkwCzsC5YD6ioDD2tEMlVwKlVyo4kFLPaIDEiIiKggIQiIvTLH3OFF+dwR9RCYAgF4t9LYoL1F0QEJERAWB5R16Xz8JUaBH5DQBPA+dShieWZ0UXR0NkBARUYHF6YHM50AcswuLXLV96KgGhBX8EinA+wC3FTqV0COiZR7RAcUREVHBzPxkWDRN0FfIhd6LrI+LJEgkgMog5CVyNGNGXhYuGpZEFV+jBOoREVFBql6F4eoWwYnc172hAG38RCq5lEQoihBViB5//HFwHNduGzVqlJgmEWJiqxUe+9o/FKDNzBkRXYg+NBs7diy++OKL4HOZTHSTiAhjcXpQ1tCCrLozSAD6UYjihUenCR4fjx3F9bC6PLh+4hBIJb1Ivkb0GtG/9TKZDGlpfRwzQsQUdRYn/nv0LC5vOIuEDH1EekQyCYdj1Rb4eAabyxuMtibEQXQf0cmTJ5GRkYG8vDzcdtttqKioENskIsJYnF5oPI1QSjlAqRW2/qCNj4gDgjNnFEskPqIKUUFBAQoLC/Hpp59i/fr1KC0txSWXXAKr1dphe5fLBYvF0m4jYh+b04s4d6OQqKy/ekNA69DM6wI8DooliiJEHZrNnTs3+PeECRNQUFCAoUOH4p133sGSJUvOa79mzRqsXr06kiYSEcDq9ELjaYZSKQHikvvvQlK50Nty2QCnKdgjstAyD9ERfWjWlvj4eIwYMQKnTp3q8Pijjz4Ks9kc3CorKyNsIdEfWJ0eqD3NUMokrZVZ+4s2fiI99YiihqgSIpvNhpKSEqSnd5yHRqlUQq/Xt9uI2Mfq8EDjaYZCJu1/IWrjJwr6iFzUIxIbUYVoxYoV2LFjB8rKyrB79278+Mc/hlQqxS233CKmWUQE8fEMbqcVMt4pCFEg0X1/0UGPKFBLjRAPUX1EZ86cwS233ILGxkakpKTg4osvxp49e5CS0s+/ikTUwAFYMFIJmVsLuTZR8OP0J21iiYYkqPHgZcMowjoKEFWINm/eLObliShAIuGQLrUBWmX/OqoDBHtEJkglHKQSEqFoIKp8RMQgxd4gPPa3fwho9RG5rICPfEPRguiR1cTg5kyzHexMORK9XsRFQojkamFlv9cFOM3YVy/BmWY7pg5NRFaipv+vT3QI9YgIUTlRY0F5RTmabO7I9Ig4rtVP5GhGrcWJsgY7Gmyu/r820SkkRISoOGwmSHkP5HJZa4L7/qbNFL5WGVjmQbFEYkJCRIiK11IHAJBpk4QMipGgTdpYWuYRHZAQEaLiswmOaoWhH9eYnUtwaGaC3h/UaKOgRlEhISJEw+3lIXMIQqSMpBAFh2bN1COKEkiICNGwubzQeJoglXCRFaJgUKMZOqU0aIuPZ5GzgWgHCREhGlaHG2qPSVjsqolAMGMApQHgJADvhQZ2QQhlUjg8vsjZQLSD4ogI0UhVOBGfpgGTSFsdyJEgUNHD0QzOacYDs4dBLqXfZDGhu0+IhsrdDINajvikNEEcIkkbhzWJkPjQf4AQj5Z64TESgYzn0sZhTYgPDc2iFJ/PB49nYE8pV52pgsIuQ5wkEWqnM7IXl8cDEg1gt6H8bCNO1Fph1CkxMSuCQ8QBgFwuh1Ta+/ivsITo9OnTyMvL6/XFifNhjKGmpgYmk0lsU/odlzITMGbA5I6DtLQ0shf3aQB9AeCTwt1UAyN8kNk5lJaaImvHACA+Ph5paWnguPBLMoUlRPn5+Zg1axaWLFmCn/zkJ1CpVGEbQLQnIEJGoxEajaZX/9xohjEGu6kWYAxKfTJksgiX8/F5hGEZJ4FbkQCLywspxyEhThFZO2IYxhjsdjvq6oTo+M4yq4ZCWEJ08OBBbNy4EcuXL8dDDz2EhQsXYsmSJZg+fXrYhhDCcCwgQklJ/ZypUGR4rwc+mRTgOMTFaSMvuEwB+GwAAJlaBQfzgONAP6o9RK1WAwDq6upgNBrDHqaF5ayeNGkS1q1bh6qqKmzYsAHV1dW4+OKLMW7cOLz00kuor68Py5jBTsAnpNEM/HQUvD8XEONk4vT6OImwAZBAiB9iDOAZBTX2lMDntTc+zV7NmslkMixYsAD//Oc/8eyzz+LUqVNYsWIFsrKycOedd6K6uro3px+0DNThWFsCQgSpiPMl/kW2EsYjcMspurrn9MXntVdCtH//fjz44INIT0/HSy+9hBUrVqCkpASff/45qqqqMH/+/F4bSAxMmM+/tksivhCBF/xDAMCTEIlCWEL00ksvYfz48bjoootQVVWFN954A+Xl5fj973+P3NxcXHLJJSgsLMTBgwdDPuczzzwDjuPw8MMPh2MSEWMwnwf3/Xw5brntjj497+zZs7v9DOXk5GDt2rWtIsh8kEo4cBxAMiQOYQnR+vXrceutt6K8vBwffPABrrvuOkjOiYw1Go147bXXQjrfvn378Ne//hUTJkwIxxwiCghFAIIwBoWEQSbhIJWIOAzlAj0iHwxqOYw6FVRySqYf4PHHH8ekSZMicq2whOjzzz/HI488ct50HWMMFRUVAACFQoFFixZ1ey6bzYbbbrsNr7zyChISKJhsUMB7IYHgW+A4EYP72wzNBoNfLpoJ61MwbNgwNDQ0nLe/qakJubm5PTrX0qVLce2112LOnDndtnW5XLBYLO02QnwWL16MHTt2YN26dX5x4VBSUoIlS5YgNzcXarUaI0eOxLp164QX8H7/kF+EVq9ejZSUFOj1etx///1wu90hXbelpQV33nkntFot0tPT8eKLL57Xpq6uDvPmzYNarUZubi7+8Y9/tB70CxGnS8X6P/8Zc+fOhVqtRl5eHt59991gs7KyMnAch3feeQeXXHIJ1Go1pk2bhhMnTmDfvn2YOnUqtFot5s6d26MZ4w0bNmDs2LFQKpVIT0/HQw89FDxWUVGB+fPnQ6vVQq/X4+abb0ZtbW3weKC38ve//x05OTkwGAz46U9/CqvVGmzD8zyee+455OfnQ6lUIjs7G0899VTw+COPPIIRI0ZAo9EgLy8Pv/vd74IzX4WFhVi9ejUOHToU/J8WFhaG/N56DAsDjuNYbW3tefvLysqYRqMJ+TybNm1i48aNYw6HgzHG2KxZs9j//M//dNp+1apVDMIwvt1mNpt7/B6iEYfDwY4dOxa8H21xeXydbh6vL+S27hDa9hSTycRmzJjB7rnnHlZdXc2qq6uZ0+lkK1euZPv27WOnT59mb775JtNoNOztt99mvMPCXE1n2O233sK0Wi1buHAhO3LkCPvoo49YSkoK+9///d+QrvvAAw+w7Oxs9sUXX7Dvv/+eXXfddUyn07X7DM2dO5dNnDiRffPNN2z//v3soosuYmq1mv3hD39gjOcZs1QzACwpKYmtfXk9+7boMPvtb3/LpFIpO3bsGGOMsdLSUgaAjRo1in366afs2LFj7MILL2RTpkxhs2fPZl9//TU7ePAgy8/PZ/fff39Itv/5z39mKpWKrV27lhUXF7Nvv/1WsIkx5vP52KRJk9jFF1/M9u/fz/bs2cOmTJnCZs2aFXz9qlWrmFarZQsWLGCHDx9mO3fuZGlpae3u3a9//WuWkJDACgsL2alTp9hXX33FXnnlleDxJ598ku3atYuVlpayDz/8kKWmprJnn32WMcaY3W5nv/zlL9nYsWOD/1O73d7he+nqc2s2m0P6jvZIiJYtW8aWLVvGJBIJu++++4LPly1bxn7xi1+wgoICdtFFF4V0roqKCmY0GtmhQ4eC+7oTIqfTycxmc3CrrKwcNEL00mfFnW7vHzzTru3/bTvRadt39lW0a7t++6nz2oRDd/87xhhbunQpu/HGG5nP1sisdRXsp7fcwhITE1lLS0urPevXM61Wy3y+rgXRarUyhULB3nnnneC+xsZGplarg3YUFxczAOzbb78Ntjl+/DgDEPzSM2sdA8Duu+ceVmN2sBqzg/E8zwoKCtgDDzzAGGsVoldffTV4nk2bNjEAbNu2bcF9a9asYSNHjuzS7gAZGRnsscce6/DYZ599xqRSKauoaP1fHT16tN17WbVqFdNoNMxisQTb/OpXv2IFBQWMMcYsFgtTKpXthKc7nn/+eTZlypTg81WrVrGJEyd2+7q+EKIezZ1+9913gV4UDh8+DIWiNRxeoVBg4sSJWLFiRUjnOnDgAOrq6nDBBRcE9/l8PuzcuRN/+tOf4HK5zovSVCqVUCqVPTGZEJGXX34ZGzZsQEVFBRwOB9xuNyZNmgQWLGzIYeLEie0COGfMmAGbzYbKykoMHTq003OXlJTA7XajoKAguC8xMREjR44MPj9+/DhkMhmmTJkS3Ddq1CjEx8e3nsg/PJtx4bTgLp4JdhQVFbW7ZtvJlNRUIaPk+PHj2+0LLHfoirq6OlRVVeGKK67o8Pjx48eRlZWFrKys4L4xY8YgPj4ex48fx7Rpgq05OTnQ6XTBNunp6cHrHz9+HC6Xq9NrAMDbb7+NP/7xjygpKYHNZoPX64Ver+/W/v6gR0L03//+FwBw1113Yd26db0y+oorrsDhw4fb7bvrrrswatQoPPLII32yoncgsfSy/E6PnTvxdO+lwzpte65P9u6ZPfPphcrmzZuxYsUKvPjii5gxYwZ0Oh2ef/557N27F4z3Z0IU01EdIOAnYjwkHAeesU5jieTy1vVwAef2uft4nu/2koFlEb2l7bXPvX531/jmm29w2223YfXq1bjqqqtgMBiwefPmDv1skSCsT8LGjRt7rZw6nQ7jxo1rt8XFxSEpKQnjxo3r1bkHIgqZpNNNdk5ir67anpsErKM2YdmnUMDna021umvXLlx00UV48MEHMXnyZOTn56OkpARgDDxjYJwEHMfh0KFDcDgcwdft2bMHWq22XW+gI4YNGwa5XI69e/cG9zU3N+PEiRPB56NGjYLX68WBAweC+4qLi9tnNvDHEu3Z+y0Ct8bHGPbs2YPRo0eHcyu6RafTIScnB9u2bevw+OjRo1FZWYnKysrgvmPHjsFkMmHMmDEhXWP48OFQq9WdXmP37t0YOnQoHnvsMUydOhXDhw9HeXl5uzbn/k/7k5B7RAsWLEBhYSH0ej0WLFjQZdstW7b02jAitsjJycHevXtRVlYGrVaL4cOH44033sDWrVuRm5uLv//979i3bx9yc4YKa7o4GTgOcLvdWLJkCX7729+irKwMq1atwkMPPXReXNq5aLVaLFmyBL/61a+QlJQEo9GIxx57rN3rRo4ciauvvhr33Xcf1q9fD5lMhocffrh9b8EfS/TPLf/CmMkFuKBgBv793jv49ttvQ46DC4fHH38c999/P4xGI+bOnQur1Ypdu3bh5z//OebMmYPx48fjtttuw9q1a+H1evHggw9i1qxZmDp1akjnV6lUeOSRR/DrX/8aCoUCM2fORH19PY4ePYolS5Zg+PDhqKiowObNmzFt2jR8/PHHeP/999udIycnB6WlpSgqKkJmZiZ0Ol2/uUZC/vkzGAzB7qjBYOhyC5ft27cLEa9EzLFixQpIpVKMGTMGKSkpuOqqq7BgwQIsXLgQBQUFaGxsxIMPPggAYGDgOSk4CEP04cOH49JLL8XChQtx/fXX4/HHHw/pms8//zwuueQSzJs3D3PmzMHFF1/czh8ECL33jIwMzJo1CwsWLMC9994Lo9HY2sA/NFv9vyuw5b1/4vKLpmHTP97Epk2bQu59hMOiRYuwdu1a/PnPf8bYsWNx3XXX4eTJkwCEIda//vUvJCQk4NJLL8WcOXOQl5eHt99+u0fX+N3vfodf/vKXWLlyJUaPHo2FCxcGfUjXX389li1bhoceegiTJk3C7t278bvf/a7d62+88UZcffXVuOyyy5CSkoJNmzb1zZvvAI6x2F1ubLFYYDAYYDabRXOy9SVOpxOlpaXIzc0duOko7I1wOBywS7SI0+rEj2RmPDiJFO//4zVceeMdsLl9UMulwXpnRPd09bkN9Tsa1opDh8MBxlhwtqO8vBzvv/8+xowZgyuvvDKcUxKDBd4LpVwCqUoFaTQkrW/jMFfLGDTKAfoDEOWE9UmYP38+3njjDQCAyWTC9OnT8eKLL2L+/PlYv359nxpIDCB4H8DzkICDQq7ocp1ZRUUFtFptp1tgKVFfwvF955jtyvavvvqqz64zUAg7Q+Mf/vAHAMC7776LtLQ0fPfdd3jvvfewcuVKPPDAA31qJDFACCztkEi7nbrPyMg4L47n3ON9BbM3Ax6HIJR9RFe2DxkypM+uM1AIS4jsdnswkOqzzz7DggULIJFIcOGFF543BUgQQXgveMbg5iVgHsEX0xkymQz5+Z3HTvUpgcWvzAeT3Q0vzxCvlp8XFtETImb7ACGsO52fn48PPvgAlZWV2Lp1a9AvVFdXNyCcxkQ/4fOAZ4Ddy8HhjqLyzm1W4Xt5Bh/PKGVshAlLiFauXIkVK1YgJycHBQUFmDFjBgChdzR58uQ+NZAYQPAeIXJZIoM0mrJucP6BAe8LZmqklLGRJayh2U9+8hNcfPHFqK6uxsSJE4P7r7jiCvz4xz/uM+OIAQRjAO8VHiQySMRMiHYubYZmEikAH+AjHYooYScMTktLQ1paWrt9VE6I6BTeAzCABwcGabDnERVwEmERHmOQQVirRbmrI0tYQtTS0oJnnnkG27ZtQ11d3XkL/U6fPt0nxhEDCH+yfB8nAzhEV4+I44Rekc8LKXgAEvjIRxRRwhKin/3sZ9ixYwfuuOMOpKenU5pNont4IfWHF8IwSNRc1R3BSSFY5xci6hFFlLCE6D//+Q8+/vhjzJw5s6/tIWKU2bNnY9KkSZ2vFfR5wODvEQGQ9POPV7f2nItEBsAFCXzgOFm/20e0JywhSkhIQGJiYl/bQgxU/I5qAIjXauCD5LwcSoCwEr9tsr2I4ndYS8HDqKNlHpEmrOn7J598EitXroTdbu9re4gYpNvk+RoNRl4wE39c/yqkUhkUMik4jsPixYtxww034KmnnkJGRkYwu+Lu3bsxadIkqFQqTJ06FR988AE4jmsXrXzkyBHMnTsXWq0WqampuOOOO4IFHTqyp6ysrOs3wbXGEhGRJ6we0YsvvoiSkhKkpqYiJyfnvExxPSmsSHQDY4Av/JriYSOVn5/OsRPWrVuHEydOYNy4cXjiiScACL3mzMxM/POf/0SSTo3dX23Hvf/za6TnjsTNN98cfO22bdug1+vx+eefAxBWa8+bNw/XXHMN3nrrLZSXl59XL81kMuHyyy/Hz372M/zhD3+Aw+HAI488gptvvhlffvllh/akpKR0/SYkrTXOwFjI753oG8ISohtuuKGPzSA6xecBvhIhfeclvwRkoQ2TDAYDFAoFNBpNu5CO1atXC384TMi9eQG+3n8Ib23ajPkLboRSJnzx4+Li8OqrrwaHZH/5y1/AcRxeeeUVqFQqjBkzBmfPnsU999wTPO+f/vQnTJ48GU8//XRw34YNG5CVlYUTJ05gxIgRHdrTJcGqrwwtLg8cXoY4hRRqhYglsQcRYd3lVatW9bUdxAAkmDy/vBwOpwNutwdjx0+A28sHhWj8+PHt/ELFxcWYMGFCu7w258anHTp0CP/973+h1WrPu2ZJSQlGjBjRc2MDU/i8D4z3wsdL4KWZs4gRttybTCa8++67KCkpwa9+9SskJibi4MGDSE1NpdXFfYlULvROxLhuLwgmz3/hecyYMAI6rRa//78N2L9/X7up+7i4uB6f22azYd68eXj22WfPO3Zu9eEe4RcimsKPPGEJ0ffff485c+bAYDCgrKwM99xzDxITE7FlyxZUVFQEcxV1x/r167F+/fqgI3Hs2LFYuXIl5s6dG45ZAxOOC3mIJCadJs+/92eAoxmQylBaKgS6dhVVPXLkSLz55ptwuVzB/Mj79u1r1+aCCy7Ae++9h5ycHMhkHX+Ew0r8zrXOnAG03iyShDVrtnz5cixevBgnT55s14W+5pprsHPnzpDPk5mZiWeeeQYHDhzA/v37cfnll2P+/Pk4evRoOGYRItI2eX5DQwOGDx+O/fv3Y+unn+DEqRL87vfP4buDQjWNrqKqb731VvA8j3vvvRfHjx/H1q1b8cILLwBoLeGzdOlSNDU14ZZbbsG+fftQUlKCrVu34q677gqKz7n2hFLmp3UKXziHTyhAGvY9IUInLCHat28f7rvvvvP2DxkyBDU1NSGfJzA7Mnz4cIwYMQJPPfUUtFot9uzZE45ZhIh0mjz/jrtQcPl1aGgyY/GSewF0HVWt1+vx73//G0VFRZg0aRIee+wxrFy5EgCCP3oZGRnYtWsXfD4frrzySowfPx4PP/ww4uPjg1U8zrUnpIyOfoe1hAmixZiwEREg5Hq0bUhJSWEHDx5kjDGm1WpZSUkJY0wolZuZmRnOKZnX62WbNm1iCoWCHT16NKTXhFrONlboqnRvTOLzMmauYsxSxdweD6sxO1itpefv7c0332RyubzT2ut9hscl2GutZXUWJ6sxO5jb03Xpa0KEktMBrr/+ejzxxBN45513AAhd5oqKCjzyyCO48cYbe3Suw4cPY8aMGXA6ndBqtcEk/B3hcrngcrmCzy0WSzjmE5HC5xYeJXL4mNALCmXV/RtvvIG8vDwMGTIEhw4dCsYI9VWF1E5pkw5ELgF8jAN1iCJDWEOzF198ETabDSkpKXA4HJg1axby8/Oh0+nw1FNP9ehcI0eORFFREfbu3YsHHngAixYtwrFjxzpsu2bNmnb107qrBkqIjNf/oyFVQCmTIEmrgF7d/WxcTU0Nbr/9dowePRrLli3DTTfdhL/97W/9bCzapAMB4lVSJGmVYVe+JXpGr+qa7dq1C4cOHYLNZsMFF1yAOXPm9NqgOXPmYNiwYfjrX/963rGOekRZWVlU1ywaYQyw1QqPmkRA1j8VQvuclnohZYkmAZDF+P8gQohS14zneRQWFmLLli0oKysDx3HIzc1FWloaGGO9TgnC83w7sWmLUqnst5K3RB/jcwsiJJEA0ugPPwjiTwcSqOjRF59pont6JESMMVx//fX45JNPMHHiRIwfPx6MMRw/fhyLFy/Gli1b8MEHH4R8vkcffRRz585FdnY2rFYr3nrrLWzfvh1bt27t6fsgog2vU3iUqQCOg80prJdTKaSQdVPXXlT86UB8Pi+abcIPYrKWfvz6mx4JUWFhIXbu3Ilt27bhsssua3fsyy+/xA033IA33ngDd955Z0jnq6urw5133onq6moYDAZMmDABW7duxY9+9KOemDXgCCnmJZphvFAnDAgObxweH3gGKMUuMd0dfoc1x/uCAY3UK+qavvi89shHdOWVV+Lyyy/Hb37zmw6PP/3009ixY0fEejShjj9jBZ7ncfLkSUilUqSkpEChUMTmF8DtANxWYZijSQQPoKlFmEFL1CiiK03suXjdgNMExsnQxOLAgF7XOBuoMMbgdrtRX18Pn8+H4cOHB+O4AvSLj+j777/Hc8891+nxuXPn4o9//GNPTkm0QSKRIDc3F9XV1aiqqhLbnPBx2QDmE3pDMgt4nsHm8oLjAKuqd2vY+h3GAy4rAMDGxYHnGZoVUhKiLtBoNMjOzj5PhHpCj4SoqakJqampnR5PTU1Fc3Nz2MYQwhqp7OxseL3enq+Vigaay4ATuwGJHJj4U0ChwdlmO746VosEjQLzR0f5gmjGgH0bAObFTs0VOG2RoSAvCcPTY7/H3R9IpVLIZLJe99x7JEQ+n6/TRYYBo7xeynDXWziOg1wuPy/hXNTj8wKVXwG8Hci8ENAL6YTtPhccvBTpalVshCWolYDNgnjOAQdvgMmN2LA7hunxrNnixYs7nULvbNqdGCSUbgdaGgCFBsi+MLjb4p8x06liJMmYOhGw1SNJYgVggMnuFtuiAU+PPhmLFi3qtk2oM2ZEFMEYYD4DNJ0GnCbhuUwFaJKAuGRAmyqIS1evL98NVPrTdYy4GpC3LsewOAQh0ke7fyiARujJxcOKZJ0S8eoYioOKUXokRBs3buwvOwixsNUDJ/4DmM923U6pA3RpgNYoCJNcA4AJPaCa7wFLtdAu91IgZWS7l141Ng0XDUuGXBbFs2Vt0SQBAJIkLbjjwqEiGzM4iJG+MtEvNJ0GjmwR8mJLZUDyCECbJsTSuG2CyLQ0CInNXFZhazjZ8bmkcmDY5cCQC847JJFwMGhipDcECEMzALA3imvHIIKEaLBirWkVofhsYPQ8QNXJzJDXJawbs9UJr2upb13QqtIDCTlA2nih1zQQ8A/N4LIBXheYVAEfz2gKvx8hIRqMeN3A0Q8EEUrIASbc3JoCoyNkSkGs4rN7fKmmFjf2nm6EUa/ElKExUpRTrhZ8Ym479h0/hb31CkzNScSFeUliWzZgIYkfjJTtFIZbSh0w9sddi1AvabC58EONFafqbP12jX7BPzxTe83w+BiaW2jmrD8hIRpsOJqBM0LuaIycC8j7Nz7GZBdmzAyxNvPkd1gnQBDQZrsIRS4HESREg42yXcIyhsRcIGlYv1/O7AgIUQw5q4Ggn0jPhCygzXY3JdLvR0iIBhMOE1B7RPg799KIXDIQDBgfS7NmQHBoFuezQMJxcHt52Fy0aqC/ICEaTFQdFIIPE3MBfUZELhm7PSJhaCZxNMGgEnxozS00POsvSIgGCz4vUP298HfG+bE+/YHX19qLiL0eUbyQw9rnQapKEKB6Gy1h6i9o+n6wUP+DkKxMpQeS8iNySZvLCwnHQSrloI72hGjnIhFyKaGlAUPVDnjkxtjr1cUQJESDhTp/ZZS08UIe6QgQr1Fg6WX5/lxEMbK8oy2aJKClAWN0LozJjsxQdrBCQ7PBgNsONJUKfxvHRvTSUgkXuz2JuBTh0d4grh2DABKiwUBDsTBlrzUCcRQdHDIBIWqpB2MMVqcHbm+M5xOPUkQVojVr1mDatGnQ6XQwGo244YYbUFxcLKZJA5O648KjseMKuv3F1qM1+PRITexGJcclC48tDXjvwBm8+lUpyhtbxLVpgCKqEO3YsQNLly7Fnj178Pnnn8Pj8eDKK69ESwv9s/sMjwMwVQp/G0dF7LKMMZyqs+F4tQV8rAYCqhMEp7XPg0SpUJWEZs76B1Gd1Z9++mm754WFhTAajThw4AAuvTQyAXcDnqZSYVgWlyx8sSKEzeWF28tDwnGI18TY8o4AgZkzWz1SZVYAOjTaYrR3F+VE1ayZ2WwGACQmdrxKu6OS00Q3NJ4SHiOwnKMtgeA/g1oGaTSXD+oOTTJgq0cyLAB0aKAeUb8QNc5qnufx8MMPY+bMmRg3blyHbdasWQODwRDcsrKyImxljMHzQFOJ8HeEYocCNPmXdiTGepVUv8M6HsKPpNlBDuv+IGqEaOnSpThy5Ag2b97caZtHH30UZrM5uFVWVkbQwhjEchbwOIV8QvrMiF660d9zSIzVYVkAvxApXU3QqWRgjPxE/UFUDM0eeughfPTRR9i5cycyMzv/wiiVyk4riBAdEOgNJeZFLIgxQL1V+LKm6GL8/xWcOWtEil4Oq9OLWosTQ+LVXb+O6BGiChFjDD//+c/x/vvvY/v27cjNzRXTnIGHqUJ4TMyL6GUZY+AZwHEDQIjUCUI+bp8Ho/ReJMQlIE1PNc76GlGFaOnSpXjrrbfwr3/9CzqdDjU1NQAAg8EAtZp+cXqF191aWaObFK+fHa2Bl2cYP8SArMQuygaFCMdxuLUgG24vD7k0hh3VgKCmWiNgPouRcS0YmUY/lv2BqD6i9evXw2w2Y/bs2UhPTw9ub7/9tphmDQzMlcK0vcogrCT3c9bkwM4T9e2aVpkcKK6x4t0DZ/DFsVr4+L6J+1HIJLG5xuxctGnCo61WXDsGMKIPzYh+IjAs8/eGvD4e35xuxIHyZjAG5CbHBXs/V4xORXGNFUeqzDh81owWtxfzJmRAEsvT7n2J1ig82mrh9PhQb3XBoJHHTsHIGCBqZs2IPqaNEFmcHry9vxL7ywQRGpuhR5K2dTYrK1GDOWNSMX/SEMgkHE7Xt+CrU+Ev9Pzo+yq8va8CZ5rtvX0X0YGutUf02dEavHvgDE7WxlgxgCiHhGgg4nUL9ccAVHNGbNpbgTqLC2qFFPMmZuDKsWnQKM7vDOcmx+GqccKX7mB5M0ober7UhjGGiiY7qkxOyAdKHTBNspAkzeNEhkqIj6qzOEU2amAxQD4pRDv8/qFGnxrvHrXC7vYhRafELdOzkW/UdvnSEak6TM6Oh1TCBWvW94SmFjdcHsFJnRzrwYwBpLJg1oJ0qRDYWEtC1KdERRwR0cf4h2WOuCHwtTDkpcRh7rh0KGSh/e7MzE/G+CEGJIUhJNVm4Qtq1Ktie2nHuWjThKUezARAg2a7B06PD6pYyzwZpVCPaCDiF6LM3FG48YJMXDchI2QRAgC5VBKWCAGtQpRuGGCxNtpUAIDS1RDMv11jpl5RX0FCNMCw21vgNfvjhwxZyErU9KpnUm12YHdJ6I7rgIM6Y6BFHusEIYK1BukG4b1VmR0iGjSwICEaQDDG8PWBIhw+0wQrF9cufigcbC4v3tl3BntPN6GyqfsZMLPdA5PdAwnHITNhgAmRNk0IbnRZkakRfGfVJuoR9RUkRAOII2ctsNachtvLg3UTTR0KWqUM4zP1AICdJ+u7jfvy8jzyjVrkJGuglA0w34lMEVwAO1RmxuWjjLh0RIrIRg0cSIgGCE6PD1+faoDeWYWsRA30aX2Tf+jCvCQoZBLUWVz4ocbaZdskrRLzJmZg/qQhfXLtqEMvvC+duxYTs+Jjfx1dFEFCNED4trQJHpcDKWhGul7V7fqyUNEoZJiWIySq23WqAR7fIM7FE6iOa6kS144BCAnRAMDq9KCo0gSdqwY5iWpwmkRhjVkfMTk7HjqVDFanF99VmDpsU2d1oilWk+SHir9HBGsNbA4XDp8x48hZs7g2DRBIiAYABytM8PEMefJGIT90H/WGAsilEszMF/Ly7CtrQou/jHQAxhi2Ha/D67vL8P0ZU59eO6rQJApJ5ngvmmvP4Ivjtdhf1iS2VQMCEqIYhzEGkz8t61iN/9e5j4UIAEal6ZCbHIfZI1OgUbR3RJ9pdqDG7IRUwnUbuR3TcFywV2REIwCg2e6B3e3t6lVECJAQxTgcx2H+pCG4dYoRSbzw5egPIRKuk4GxGYZ2qT08Ph7bi+sAAOOG6Dtcwzag8PuJlPZqJPsXDleZKJ6otwzwT83gIZX3Bx2qEwCVvl+u0VaAmlrc+OpkPWwuLxpsbqgVUszIS+6X60YV8f6CDaYKDEkoQIPNjcpmB/KNOnHtinFIiGIYs8MDpUwirHcylQs7+6E3dC6MMXx2tCa4nEMpl2DexAyoFQMsdqgj9EOEemcuG7I1bhwCcLaZekS9hYQohvn6ZANO19tw+WgjxkZQiDiOw5Vj03Cy1gqphMPINB10gyVJmFQuDM9MlRjC6gDo0WBz0QLYXkI+ohjF6fGhpN4GL8+QouIBm+CnQcLQiFw/MU6BgrwkTM1JHDwiFCBeuMdq+1kkxgl+okDVEiI8RBWinTt3Yt68ecjIyADHcfjggw/ENCemOFVng49nSNYpYfTVA4wBmiRASb6KfifQ6zRV4Jpxabh/1rA+KTowmBFViFpaWjBx4kS8/PLLYpoRkxT7l1uMTNW1+oci1Bsa9OiHABIZ4LIhRdpCQ7I+QFQf0dy5czF37lwxTYhJ7G4vKv3pNkakaoEjZcKBeBKiiCCVAYYhQHM50FwWzN5IhA/5iGKQk7U2MAak6lWIl3mAFv/UfQQc1YSfQNHKphIUVZrwzr7KsHJ8EwIxJUQulwsWi6XdNhg5UesflqVpW6t1aFMABfkpIkaiP7tBczkazTacNTlQEULOJqJjYkqI1qxZA4PBENyysrLENkkUrh6XhlkjUzA8VScMDwAgPkdUmwYdcclC4CjvRa5MiGivaKQeUbjElBA9+uijMJvNwa2yslJsk0RBp5LjguwEocAfOarFgeOCvaIhfDU4DmiwuWF19rzyCRFjQqRUKqHX69ttgxqHCbA3CTW3DIOzdygqSYIQKc2nkepPklbeSMOzcBBViGw2G4qKilBUVAQAKC0tRVFRESoqKsQ0K2qxOD3YcvBMaw6cphLhUZ8ByAdY1YxYIH6oMIPmNCNfIwzLyE8UHqIK0f79+zF58mRMnjwZALB8+XJMnjwZK1euFNOsqOVkrRXljXYcr/Y76ZtKhcekvkkLS/QQmSI4PMtlgpugvNEOnu86tzdxPqLGEc2ePbvbhOxEK8U1Qr31Eak6wOcVYliA1qlkIvIYRwP1xUhsKYFeNRypBhVcXn5wLADuQ2jRa4xgsrtRa3FCwnEYnqoVykr7PIAiLlj8jxCBxGGAVAaJ04y7p6jABfJaEz0ippzVg5kTtUJvKCtRLSQfazotHEjME2ZwCHGQKYCkfAAAV3dcZGNiFxKiGKHYH8Q4IlUnLHBtOCEc8H8JCBExjhEea4+C+bxo9KcFIUKHhCgGaLC50GB1teaEttUJU/dSGfmHooGkfCGq3d2CnXv24o1vyoPR70RokBDFAD6eIStRg5zkOGGld/0PwoHEPGFoQIiLRAqkjQcAZDqLAQhpWojQIWd1DJCqV+EnUzJbp4UDw7LkkeIZRbQnfRJQsRcZvmoofDZUNnGUtbEHUI8ohpBIOGFY1tIg/AqTfyh60CQC8dlQyyUY4TsBnjFajd8DSIiinLKGFtjaFjSs+V54TBpG0dTRRtZ0AMAI70lIeRd+qBmc2SHCgYQoinF7eXx8uBqvfVUq5ETmfUDtUeFg2gRxjSPOJykfiEtGqoZDqu0HlDfaaRFsiJAQRTGn6mxwe3no1TKhmF9jCeC2CzM0NFsWfXAckFUAlVyKUZ7j4HzeYPwX0TUkRFHM4bMmAMCYdL1Q3LDqO+FA6jjBR0REH6ljAZUBOXrgprRqTMqKF9uimICEKEqpMjlQZRLqyY8bYgBs9UI0NccBGZPFNo/oDIkUyL0UBrUcGZYiSL20Gj8USIiilIMVzQCAUWk6xCllwJl9woHk4cIMDRG9pI4FdKmA1w2c3g4frcbvFhKiKKTR5goGxF0wNEFIflZzWDiYVSCiZURIcByQ/yMwAGeO7cE7n+1Ao40KMHYFCVEU0mBzQybhMMyoRbJWCZTuBBgvTNkbMsU2jwiF+CxwmVPR4vYhvWY7vj1xVmyLohqKrI5CRqbpkJmghpdnQnL8uuPCr2zuLLFNI3pC7iwMqSpG86kyOI9/iOq8e5AeHye2VVEJ9YiilDilDAY5A4o/EXakTxL8DkTsIFNAO+VmJBniYHCewQ9fbYHPx4ttVVRCQhRF7C5pQFlgWQBjQPHHwip7lR4YdpmothFhojViyIU3QSaVQFV3CMW7PxD+t0Q7SIiihCNnzdh7ugn/KqqCqcUFlHwJ1P0gTAePngfIlGKbSISJJnMcUqbMBwCYT+xG1e63AC85r9sSFUL08ssvIycnByqVCgUFBfj222/FNiliMMZQVGnCF8drAQDTh2oRX/E5UOm/B8OvpFLSA4Cc8TMhHTsPjJNA1XwS2Pcq0HCSekd+RHdWv/3221i+fDn+8pe/oKCgAGvXrsVVV12F4uJiGI1Gsc3rV5pb3PjqVANK6mwA4zFT34BpjduE4RjHASOuAjImiW0m0UdMnX4pqtLSkVjj/x8ffheeuFTIs6cDySMGdW4pjolcRqOgoADTpk3Dn/70JwAAz/PIysrCz3/+c/zmN7/p8rUWiwUGgwFmsznqiy36eAYO/lQevA+7jlfgWFkV1O4m6N11mBxXj8w4Hhw4QKkFRl0HJOaKbTbRH3jdQPnXsJd+i8MVDTCo5TBoNVAlZUOdnA2F3giJJgFQ6gCZCpDIYjYveajfUVF7RG63GwcOHMCjjz4a3CeRSDBnzhx8880357V3uVxwuVrH1hZLiGkWmkqBU18En1Y22VFldpzTLfb/zRjGDTEgzl8O5myzA5XNbRzI57Qfk66HXiUDGEO12YmyxhZwYGAAOCY8Mgb4eB4TMg0wqKSA141hFickDS1I0MiRlahBnEIGyNVA5jRgyFRK8TGQkSmAYZfjB344KkxfI7nlJJrrzEDdYQBC4KpMwmFkmk4oKy6Rot7OUG5yApCAgQM4DgwSMI4DwGFEqg4JGjkAoN7mwun6znMhDTNqkRwn9L4aW9w4VW9DvEaBkUZtz97HhIXCREofIKoQNTQ0wOfzITW1/bR0amoqfvjhh/Par1mzBqtXr+75hbwuIZlYgBY7eJuj8/Z2HvD5b43TAeboYr2QiwM44QMAtwNw2wOSFnzkINxon8sedDonaZUw6A1QJ6QBWiOQkAsk5NBi1kHEBcOzkJW2AKfrrGioOwtPUwUk1mqovBYovRZBcACA98HndsJj71xcmNYJMP/QzuYCb+ti1b/GAcA/+dHiAm+1gfnkQJyzZ2+A9V0ogug+op7w6KOPYvny5cHnFosFWVkh1HyPzwIm/jTYvTU4PJC4+PN6u5x/h1yrAmQSABwMLi/y3F6A44RhE7jg6zhwUKnlgEwQj3gPj+Fur/+40BocBwnHQSmTQqWQCnXqZSrIZSrIJVExV0CISIpOiRSdEhiWDGAifDyDy+uD0xMo0ugBfC7o7HZkt7gAxoNjPABe6Gr7e+nqODngT0urcfuQ3dJ5HiRNnBxQSAHGoPEIbZVyDojr4cysou+CM0UVouTkZEilUtTW1rbbX1tbi7S0tPPaK5VKKJVhTGMr4tr5W/QJQKgdSq1/CwW1fyOIcJFKOGgUMmiCfmspIFchTmVAXIhrneP8WyioAQzpsZV9j6g/yQqFAlOmTMG2bduC+3iex7Zt2zBjxgwRLSMIIpKIPjRbvnw5Fi1ahKlTp2L69OlYu3YtWlpacNddd4ltGkEQEUJ0IVq4cCHq6+uxcuVK1NTUYNKkSfj000/Pc2ATBDFwET2OqDfEUhwRQQxGQv2O0rQNQRCiQ0JEEITokBARBCE6ojure0PAvRXyUg+CICJK4LvZnSs6poXIarUCQGjR1QRBiIbVaoXBYOj0eEzPmvE8j6qqKuh0uuDyDLEILDeprKykGbw20H3pnMFwbxhjsFqtyMjIgKSLJU0x3SOSSCTIzIyuqhZ6vX7Afqh6A92Xzhno96arnlAAclYTBCE6JEQEQYgOCVEfoVQqsWrVqvCyAwxg6L50Dt2bVmLaWU0QxMCAekQEQYgOCRFBEKJDQkQQhOiQEBEEITokRD2gJxVpX3nlFVxyySVISEhAQkIC5syZM2Ar2IZbqXfz5s3gOA433HBD/xooIj29NyaTCUuXLkV6ejqUSiVGjBiBTz75JELWiggjQmLz5s1MoVCwDRs2sKNHj7J77rmHxcfHs9ra2g7b33rrrezll19m3333HTt+/DhbvHgxMxgM7MyZMxG2vH/p6X0JUFpayoYMGcIuueQSNn/+/MgYG2F6em9cLhebOnUqu+aaa9jXX3/NSktL2fbt21lRUVGELY88JEQhMn36dLZ06dLgc5/PxzIyMtiaNWtCer3X62U6nY69/vrr/WWiKIRzX7xeL7vooovYq6++yhYtWjRghain92b9+vUsLy+Pud3uSJkYNdDQLAQCFWnnzJkT3NdVRdqOsNvt8Hg8SEwMsSZMDBDufXniiSdgNBqxZMmSSJgpCuHcmw8//BAzZszA0qVLkZqainHjxuHpp5+Gz+eLlNmiEdOLXiNFTyvSdsQjjzyCjIyMdh/MWCec+/L111/jtddeQ1FRUQQsFI9w7s3p06fx5Zdf4rbbbsMnn3yCU6dO4cEHH4TH48GqVasiYbZokBBFgGeeeQabN2/G9u3boVIN3pr2VqsVd9xxB1555RUkJyeLbU7UwfM8jEYj/va3v0EqlWLKlCk4e/Ysnn/+eRIioucVadvywgsv4JlnnsEXX3yBCRMm9KeZEaen96WkpARlZWWYN29ecB/PC/XTZTIZiouLMWzYsP41OkKE85lJT0+HXC6HVCoN7hs9ejRqamrgdruhUCg6fN1AgHxEIRBuRdrnnnsOTz75JD799FNMnTo1EqZGlJ7el1GjRuHw4cMoKioKbtdffz0uu+wyFBUVDahMm+F8ZmbOnIlTp04FxRkATpw4gfT09AEtQgBo+j5UNm/ezJRKJSssLGTHjh1j9957L4uPj2c1NTWMMcbuuOMO9pvf/CbY/plnnmEKhYK9++67rLq6OrhZrVax3kK/0NP7ci4Dedasp/emoqKC6XQ69tBDD7Hi4mL20UcfMaPRyH7/+9+L9RYiBglRD/i///s/lp2dzRQKBZs+fTrbs2dP8NisWbPYokWLgs+HDh3KAJy3rVq1KvKG9zM9uS/nMpCFiLGe35vdu3ezgoICplQqWV5eHnvqqaeY1+uNsNWRh9KAEAQhOuQjIghCdEiICIIQHRIigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhIiIKQoLCxEfHx98/vjjj2PSpEnB54sXLx7QGR8HKiRERIcsXrwYHMfh/vvvP+/Y0qVLwXEcFi9e3K59XwtATk4O1q5d227fwoULceLEiU5fs27dOhQWFgafz549Gw8//HCf2kX0PSRERKdkZWVh8+bNcDgcwX1OpxNvvfUWsrOzRbFJrVbDaDR2etxgMLTrMRGxAQkR0SkXXHABsrKysGXLluC+LVu2IDs7G5MnT+7VuTvqqdxwww3BXtbs2bNRXl6OZcuWgeM4cBwH4Pyh2bm07ZktXrwYO3bswLp164LnKC0tRX5+Pl544YV2rysqKgLHcTh16lSv3hcRHiRERJfcfffd2LhxY/D5hg0bcNddd/X7dbds2YLMzEw88cQTqK6uRnV1dY/PsW7dOsyYMQP33HNP8BzZ2dnnvScA2LhxIy699FLk5+f31VsgegAJEdElt99+O77++muUl5ejvLwcu3btwu23397v101MTIRUKoVOp0NaWlq3Ceg6wmAwQKFQQKPRBM8hlUqxePFiFBcXB0v7eDwevPXWW7j77rv7+m0QIUIZGokuSUlJwbXXXovCwkIwxnDttdfGfJrXjIwMXHvttdiwYQOmT5+Of//733C5XLjpppvENm3QQj0iolvuvvtuFBYW4vXXX++zXoNEIsG5GWg8Hk+fnDsUfvaznwUd8Rs3bsTChQuh0Wgidn2iPSRERLdcffXVcLvd8Hg8uOqqq/rknCkpKe38Pj6fD0eOHGnXRqFQ9LqUTmfnuOaaaxAXF4f169fj008/pWGZyNDQjOgWqVSK48ePB//uDLPZfF6ZoKSkpA5zUV9++eVYvnw5Pv74YwwbNgwvvfQSTCZTuzY5OTnYuXMnfvrTn0KpVIY1JMzJycHevXtRVlYGrVaLxMRESCSSoK/o0UcfxfDhw7vMPU70P9QjIkJCr9dDr9d32Wb79u2YPHlyu2316tUdtr377ruxaNEi3HnnnZg1axby8vJw2WWXtWvzxBNPoKysDMOGDUNKSkpYdq9YsQJSqRRjxoxBSkoKKioqgseWLFkCt9sdkVlAomsoVSwxaPnqq69wxRVXoLKy8rxCiERkISEiBh0ulwv19fVYtGgR0tLS8I9//ENskwY9NDQjBh2bNm3C0KFDYTKZ8Nxzz4ltDgHqEREEEQVQj4ggCNEhISIIQnRIiAiCEB0SIoIgRIeEiCAI0SEhIghCdEiICIIQHRIigiBEh4SIIAjR+X9QhJvHzcR+MAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:54:34.396579Z", + "iopub.status.busy": "2024-02-29T20:54:34.396284Z", + "iopub.status.idle": "2024-02-29T20:54:34.614556Z", + "shell.execute_reply": "2024-02-29T20:54:34.613626Z" + }, + "papermill": { + "duration": 0.240522, + "end_time": "2024-02-29T20:54:34.616564", + "exception": false, + "start_time": "2024-02-29T20:54:34.376042", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyXUlEQVR4nO3de1wTV/4//lcIEMIdRG6WOxa0CiisiKsftUVAV4tLL2gvXGrpFut+6qZql1ZhUbd8ar2g/VjZn1tEbbfa9mPdbmtRN0rXC3jBqoiIgly0XAQVwkUjJOf7hz+mjQkYMJgweT8fjzxwZs6cnGHCy5k5kzMCxhgDIYTwkIm+G0AIIYOFAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb5nquwGGSKlUoq6uDjY2NhAIBPpuDiHkVxhjaGtrg7u7O0xM+j5Go4DToK6uDh4eHvpuBiGkD9euXcMTTzzRZxkKOA1sbGwA3P8F2tra6rk1g6erqwsHDhxAVFQUzMzM9N0c8oiMZX/KZDJ4eHhwf6d9oYDToOe01NbWlvcBZ2lpCVtbW17/QRgLY9uf2lw+ok4GQghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS36DYRQoaozs5OXLp0iZtuvyPH8ZJKODidhrVYpFI2MDAQlpaWj7uJekcBR8gQdenSJYSGhqrNX6OhbHFxMcaPHz/4jTIwFHCEDFGBgYEoLi7mpsvrWyD5qgTrXxiLADd7tbLGiAKOkCHK0tJS5ajMpOYmREfuYNSYYIR4DdNjywwHdTIQQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BZ92d6IaDt+mLGOHUb4hwLOiGg7fpixjh1G+IcCzohoO36YsY4dRvhH79fgNm/eDG9vb1hYWCA8PBwnT57stWxeXh4EAoHKy8LCQqVMUlKSWpmYmJjB3owhoWf8sJ7XqDHBELn6Y9SYYJX5dHpK+EKvR3C7d++GRCJBTk4OwsPDkZ2djejoaJSXl8PZ2VnjOra2tigvL+emBQKBWpmYmBhs27aNmxaJRGplCCH8p9cjuPXr1yMlJQXJyckYPXo0cnJyYGlpidzc3F7XEQgEcHV15V4uLi5qZUQikUoZBweHwdwMQoiB0tsR3L1791BcXIy0tDRunomJCSIjI1FYWNjreu3t7fDy8oJSqcT48ePxwQcf4KmnnlIpU1BQAGdnZzg4OODpp5/G6tWrMWxY70M4y+VyyOVyblomkwEAurq60NXVNdBNNHjd3d3cTz5vp7Ewlv3Zn23TW8A1NzdDoVCoHYG5uLio3MrwawEBAcjNzUVQUBBaW1uxdu1aTJo0CaWlpXjiiScA3D89jYuLg4+PDyorK/Hee+9h5syZKCwshFAo1FhvVlYWMjMz1eYfOHCA19ejrrUDgCmKiorw8wV9t4Y8KmPZn52dnVqXHVK9qBEREYiIiOCmJ02ahFGjRuFvf/sbVq1aBQCYN28et3zs2LEICgqCn58fCgoK8Mwzz2isNy0tDRKJhJuWyWTw8PBAVFQUbG1tB2lr9O9c7S2g5DQmTpyIYE9HfTeHPCJj2Z89Z1ja0FvAOTk5QSgUorGxUWV+Y2MjXF1dtarDzMwM48aNQ0VFRa9lfH194eTkhIqKil4DTiQSaeyIMDMzg5mZmVZtGYpMTU25n3zeTmNhLPuzP9umt04Gc3NzhIaGQiqVcvOUSiWkUqnKUVpfFAoFSkpK4Obm1muZ69ev4+bNm32WIYTwk157USUSCbZu3Yrt27ejrKwMqamp6OjoQHJyMgAgISFBpRNi5cqVOHDgAK5evYozZ87glVdeQU1NDV5//XUA9zsgli5diqKiIlRXV0MqlSI2Nhb+/v6Ijo7WyzYSQvRHr9fg4uPj0dTUhPT0dDQ0NCAkJAT5+flcx0NtbS1MTH7J4Nu3byMlJQUNDQ1wcHBAaGgojh8/jtGjRwMAhEIhzp8/j+3bt6OlpQXu7u6IiorCqlWr6F44QoyQgDHG9N0IQyOTyWBnZ4fW1lZedzKcrbmJuVuKsDd1Ij0JnQeMZX/25+9T71/VIoSQwUIBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLf0HnCbN2+Gt7c3LCwsEB4ejpMnT/ZaNi8vDwKBQOVlYWGhUoYxhvT0dLi5uUEsFiMyMhJXrlwZ7M0ghBggvQbc7t27IZFIkJGRgTNnziA4OBjR0dG4ceNGr+vY2tqivr6ee9XU1KgsX7NmDTZt2oScnBycOHECVlZWiI6Oxt27dwd7cwghBkavAbd+/XqkpKQgOTkZo0ePRk5ODiwtLZGbm9vrOgKBAK6urtzLxcWFW8YYQ3Z2NpYvX47Y2FgEBQVhx44dqKurw969ex/DFhFCDImpvt743r17KC4uRlpaGjfPxMQEkZGRKCws7HW99vZ2eHl5QalUYvz48fjggw/w1FNPAQCqqqrQ0NCAyMhIrrydnR3Cw8NRWFiIefPmaaxTLpdDLpdz0zKZDADQ1dWFrq6uR9pOQ9bd3c395PN2Ggtj2Z/92Ta9BVxzczMUCoXKERgAuLi44NKlSxrXCQgIQG5uLoKCgtDa2oq1a9di0qRJKC0txRNPPIGGhgaujgfr7FmmSVZWFjIzM9XmHzhwAJaWlv3dtCHjWjsAmKKoqAg/X9B3a8ijMpb92dnZqXVZvQXcQERERCAiIoKbnjRpEkaNGoW//e1vWLVq1YDrTUtLg0Qi4aZlMhk8PDwQFRUFW1vbR2qzITtXewsoOY2JEyci2NNR380hj8hY9mfPGZY29BZwTk5OEAqFaGxsVJnf2NgIV1dXreowMzPDuHHjUFFRAQDceo2NjXBzc1OpMyQkpNd6RCIRRCKRxvrNzMy0astQZGpqyv3k83YaC2PZn/3ZNr0FnLm5OUJDQyGVSjF37lwAgFKphFQqxaJFi7SqQ6FQoKSkBLNmzQIA+Pj4wNXVFVKplAs0mUyGEydOIDU1dTA2w6BVNXegQ97d6/LKpg7uZ88fhyZWIlP4OFnpvH2EDDa9nqJKJBIkJiYiLCwMEyZMQHZ2Njo6OpCcnAwASEhIwIgRI5CVlQUAWLlyJSZOnAh/f3+0tLTgo48+Qk1NDV5//XUA93tYFy9ejNWrV2PkyJHw8fHBihUr4O7uzoWosahq7sD0tQValX3n65KHljm8ZBqFHBly9Bpw8fHxaGpqQnp6OhoaGhASEoL8/Hyuk6C2thYmJr/cyXL79m2kpKSgoaEBDg4OCA0NxfHjxzF69GiuzLJly9DR0YE33ngDLS0tmDx5MvLz89VuCOa7niO37PgQ+Dtbay5zR47vCgoxe1oErMTqp+gAUHGjHYt3n+3zSJAQQ6X3ToZFixb1ekpaUFCgMr1hwwZs2LChz/oEAgFWrlyJlStX6qqJQ5q/szXGjLDTuKyrqwsNw4HxXg68vmZDjJfev6pFCCGDhQKOEMJbFHCEEN7S+zU4Qoj2+rr1h277UUcBR8gQoe2tP3Tbzy8o4AgZIh526w/d9qOOAo6QIaa3W3/oth911MlACOEtCjhCCG9RwBFCeKvfAXf16tXBaAchhOhcvwPO398f06dPx2effUYPciGEGLR+B9yZM2cQFBQEiUQCV1dX/OEPf+jzUX+EEKIv/Q64kJAQbNy4EXV1dcjNzUV9fT0mT56MMWPGYP369WhqahqMdhJCSL8NuJPB1NQUcXFx+Oqrr/Dhhx+ioqICS5YsgYeHBxISElBfX6/LdhJCSL8NOOBOnz6NhQsXws3NDevXr8eSJUtQWVmJgwcPoq6uDrGxsbpsJyGE9Fu/v8mwfv16bNu2DeXl5Zg1axZ27NiBWbNmcSPv+vj4IC8vD97e3rpuKyGE9Eu/A27Lli147bXXkJSUpPLkql9zdnbGp59++siNI4SQR9HvgDt48CA8PT1VnpUAAIwxXLt2DZ6enjA3N0diYqLOGkkIIQPR72twfn5+aG5uVpt/69Yt+Pj46KRRhBCiC/0OOMaYxvnt7e1G9+QqQohh0/oUVSKRALj/1Kr09HRYWlpyyxQKBU6cONHn0+PJ4yVX3IWJxc+okpXDxELzYwO7u7tR112HsltlvY4AWyVrh4nFz5Ar7gLQ/HQuQgyV1gH3008/Abh/BFdSUgJzc3Numbm5OYKDg7FkyRLdt5AMSF1HDax8PsZ7WnzJ5JP8T/pcbuUD1HWEIBQuOmodIY+H1gF3+PBhAEBycjI2btwIW1tbnTRg8+bN+Oijj9DQ0IDg4GB8/PHHmDBhwkPX27VrF+bPn4/Y2Fjs3buXm5+UlITt27erlI2OjkZ+fr5O2jtUuFt5oaPqj9gYHwK/Xh783N3djWNHj+G3k3/b6xFc5Y12vL37LNynew1mcwkZFP3uRd22bZvO3nz37t2QSCTIyclBeHg4srOzER0djfLycjg7O/e6XnV1NZYsWYIpU6ZoXB4TE6PSTpFI8/DNfCYSWkB5dwR8bAMweljvD36uMq3CKMdRvY4Aq7zbCuXdJoiEdH2VDD1aBVxcXBzy8vJga2uLuLi4Psvu2bNH6zdfv349UlJSkJycDADIycnB999/j9zcXPz5z3/WuI5CocDLL7+MzMxMHDlyBC0tLWplRCIRXF1dtW4HIYSftAo4Ozs7CAQC7t+6cO/ePRQXFyMtLY2bZ2JigsjISBQWFva63sqVK+Hs7IwFCxbgyJEjGssUFBTA2dkZDg4OePrpp7F69WoMGzas1zrlcjnkcjk3LZPJANw/wunq6urvphmE7u5u7mdv29Azv69t1KYe8ng8bF8Yy/7sT7u1Crhfn+7p6hS1ubkZCoUCLi6qF65dXFxw6dIljescPXoUn376Kc6ePdtrvTExMYiLi4OPjw8qKyvx3nvvYebMmSgsLIRQKNS4TlZWFjIzM9XmHzhwQKW3eCi51g4Apjh69ChqNF+C4xw8eFAn9ZDBdbXtHkwsbuObI1+iUNx7ue0/bO91WcMdwMRCiMNHDqHGxrzXcoass7NT67JD5qlabW1tePXVV7F161Y4OTn1Wm7evHncv8eOHYugoCD4+fmhoKAAzzzzjMZ10tLSuNtggPtHcB4eHoiKitJZZ8rjVlonw9qSIkyePBlPuWvehq6uLhw8eBAzZszo9RqcNvWQx+P78tOwUqzEHgWA9oHXY+UD+IT8f5gVEKaztj1OPWdY2tAq4MaNG8edoj7MmTNntCrn5OQEoVCIxsZGlfmNjY0ar59VVlaiuroac+bM4eYplUoA94duKi8vh5+fn9p6vr6+cHJyQkVFRa8BJxKJNHZEmJmZDdnHr/X0ipqamj50G/razv7UQwaXh61vnz3j/ekV95juO2T3Z3/arVXAzZ07d6Bt6ZW5uTlCQ0MhlUq5+pVKJaRSKRYtWqRWPjAwECUlqk/sXr58Odra2rBx40Z4eHhofJ/r16/j5s2bvQ4MQMhQ8bCeceoVV6dVwGVkZAzKm0skEiQmJiIsLAwTJkxAdnY2Ojo6uF7VhIQEjBgxAllZWbCwsMCYMWNU1re3twcAbn57ezsyMzPx3HPPwdXVFZWVlVi2bBn8/f0RHR09KNtACDFcer0GFx8fj6amJqSnp6OhoQEhISHIz8/nOh5qa2vVRi3pi1AoxPnz57F9+3a0tLTA3d0dUVFRWLVqlVHeC0eIsdMq4BwdHXH58mU4OTnBwcGhz+txt27d6lcDFi1apPGUFLh/u0df8vLyVKbFYjH279/fr/cnhPCXVgG3YcMG2NjYcP/WtsOBEEL0SauA+/XglUlJSYPVFkII0al+jwcnFApx48YNtfk3b97s9UZaQgjRB50NeCmXy1WGUCKEEH3Tuhd106ZNAO4PePn3v/8d1ta/3GioUCjwn//8B4GBgbpvISGEDJDWAbdhwwYA94/gcnJyVE5Hzc3N4e3tjZycHN23kBBCBkjrgKuqqgIATJ8+HXv27IGDg8OgNYoQQnSh3zf69ozsSwghhq7fAffaa6/1uTw3N3fAjSGEEF3qd8Ddvn1bZbqrqwsXLlxAS0sLnn76aZ01jBBCHlW/A+6bb75Rm6dUKpGamqpxuCJCCNGXft8Hp7ESExNIJBKup5UQQgyBTgIOuD8gZc9474QQYgj6fYr666G9gfv3xdXX1+P7779X+c4qIYToW78DrucJ9z1MTEwwfPhwrFu37qE9rOTxudOlAABc+Lm11zIdd+Q43QS41tyGlVjzeHkVNx5h8H9C9Izug+Opyv8/mP68p+QhJU2xs+LUQ+uzEg2Z5xMRwqFPLU9FPXX/wT1+ztYQm2ke5aW8vhXvfF2Cdc+PRYBb78+7tRKZwsfJalDaSchgooDjKUcrc8yb4NlnmZ5OIb/hVhgzQjcP9CbEkOisF5UQQgwNBRwhhLd0FnDXr1/HG2+8oavqCCHkkeks4G7evIlPP/1UV9URQsgjo1NUQghvUcARQnhL7wG3efNmeHt7w8LCAuHh4Th58qRW6+3atQsCgQBz585Vmc8YQ3p6Otzc3CAWixEZGYkrV64MQssJIYZO6/vg4uLi+lze0tLS7zffvXs3JBIJcnJyEB4ejuzsbERHR6O8vBzOzs69rlddXY0lS5ZgypQpasvWrFmDTZs2Yfv27fDx8cGKFSsQHR2NixcvwsLCot9tJIQMXVofwdnZ2fX58vLyQkJCQr/efP369UhJSUFycjJGjx6NnJwcWFpa9jkqsEKhwMsvv4zMzEz4+vqqLGOMITs7G8uXL0dsbCyCgoKwY8cO1NXVYe/evf1qGyFk6NP6CG7btm06feN79+6huLgYaWlp3DwTExNERkaisLCw1/VWrlwJZ2dnLFiwAEeOHFFZVlVVhYaGBkRGRnLz7OzsEB4ejsLCQsybN09jnXK5HHK5nJuWyWQA7o9W3NXVNaDtGwp6vsnQ3d3N6+3ki4ftr555fe1LPuzz/rRbb1/Vam5uhkKhgIuLi8p8FxcXXLp0SeM6R48exaeffoqzZ89qXN7Q0MDV8WCdPcs0ycrKQmZmptr8AwcOwNLSsq/NGNKutQOAKYqKivDzBX23hjxMz/46evQoaqx7L3fw4MFHrsOQdXZ2al1W64DTdiikwXroTFtbG1599VVs3boVTk5OOq07LS1NZZw7mUwGDw8PREVFwdbWVqfvZUjO1d4CSk5j4sSJCPZ01HdzyEOU1smwtqQIkydPxlPu6p/Lrq4uHDx4EDNmzICZmdmA6hgKes6wtKF1wOXl5cHLywvjxo0DY2xADfs1JycnCIVCNDY2qsxvbGyEq6urWvnKykpUV1djzpw53DylUgkAMDU1RXl5ObdeY2Mj3NzcVOoMCQnptS0ikQgikfp4aGZmZr1+UPjA1NSU+8nn7eQLbfdXX59bPuzz/rRb64BLTU3FF198gaqqKiQnJ+OVV16Bo+PA/9c3NzdHaGgopFIpd6uHUqmEVCrFokWL1MoHBgaipER1bLPly5ejra0NGzduhIeHB8zMzODq6gqpVMoFmkwmw4kTJ5CamjrgthJChiate1E3b96M+vp6LFu2DP/617/g4eGBF198Efv37x/wEZ1EIsHWrVuxfft2lJWVITU1FR0dHUhOTgYAJCQkcJ0QFhYWGDNmjMrL3t4eNjY2GDNmDMzNzSEQCLB48WKsXr0a3377LUpKSpCQkAB3d3e1++UIIfzXr04GkUiE+fPnY/78+aipqUFeXh4WLlyI7u5ulJaWwtq6f1ct4+Pj0dTUhPT0dDQ0NCAkJAT5+flcJ0FtbS1MTPp3L/KyZcvQ0dGBN954Ay0tLZg8eTLy8/PpHjhCjNCAe1FNTEwgEAjAGINCoRhwAxYtWqTxlBQACgoK+lw3Ly9PbZ5AIMDKlSuxcuXKAbeJEMIP/Qo4uVyOPXv2IDc3F0ePHsXs2bPxv//7v4iJien3kRYhpH8e9iAheoiQOq0DbuHChdi1axc8PDzw2muv4YsvvtD57RqEkN5p9yAheojQr2m9lTk5OfD09ISvry9+/PFH/PjjjxrL7dmzR2eNI4T84mEPEqKHCKnTOuASEhIgEAgGsy2EkD487EFC9BAhdf260ZcQQoYS6hkghPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8pfeA27x5M7y9vWFhYYHw8HCcPHmy17J79uxBWFgY7O3tYWVlhZCQEOzcuVOlTFJSEgQCgcorJiZmsDeDEGKA9PpwxN27d0MikSAnJwfh4eHIzs5GdHQ0ysvL4ezsrFbe0dER77//PgIDA2Fubo7vvvsOycnJcHZ2RnR0NFcuJiYG27Zt46ZFIs0PwSWE8Jtej+DWr1+PlJQUJCcnY/To0cjJyYGlpSVyc3M1lp82bRp+//vfY9SoUfDz88Pbb7+NoKAgHD16VKWcSCSCq6sr93JwcHgcm0MIMTB6O4K7d+8eiouLkZaWxs0zMTFBZGQkCgsLH7o+YwyHDh1CeXk5PvzwQ5VlBQUFcHZ2hoODA55++mmsXr0aw4YN67UuuVwOuVzOTctkMgBAV1cXurq6+rtpQ0bPczS7u7t5vZ3Gwlj2Z3+2TW8B19zcDIVCARcXF5X5Li4uuHTpUq/rtba2YsSIEZDL5RAKhfjkk08wY8YMbnlMTAzi4uLg4+ODyspKvPfee5g5cyYKCwshFKo/DRwAsrKykJmZqTb/wIEDsLS0HOAWGr5r7QBgiqKiIvx8Qd+tIY/KWPZnZ2en1mX1eg1uIGxsbHD27Fm0t7dDKpVCIpHA19cX06ZNAwDMmzePKzt27FgEBQXBz88PBQUFeOaZZzTWmZaWBolEwk3LZDJ4eHggKioKtra2g7o9+nSu9hZQchoTJ05EsKejvptDHpGx7M+eMyxt6C3gnJycIBQK0djYqDK/sbERrq6uva5nYmICf39/AEBISAjKysqQlZXFBdyDfH194eTkhIqKil4DTiQSaeyIMDMzg5mZmZZbNPSYmppyP/m8ncbCWPZnf7ZNb50M5ubmCA0NhVQq5eYplUpIpVJERERoXY9SqVS5fvag69ev4+bNm3Bzc3uk9hJChh69nqJKJBIkJiYiLCwMEyZMQHZ2Njo6OpCcnAwASEhIwIgRI5CVlQXg/rWysLAw+Pn5QS6XY9++fdi5cye2bNkCAGhvb0dmZiaee+45uLq6orKyEsuWLYO/v7/KbSSEEOOg14CLj49HU1MT0tPT0dDQgJCQEOTn53MdD7W1tTAx+eUgs6OjAwsXLsT169chFosRGBiIzz77DPHx8QAAoVCI8+fPY/v27WhpaYG7uzuioqKwatUquheOECMkYIwxfTfC0MhkMtjZ2aG1tZXXnQxna25i7pYi7E2diBCv3m+jIUODsezP/vx96v2rWoQQMlgo4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFnpBQKBU4XHkXHxR9xuvAoFAqFvptEiM5RwBmhPXv2wN/fHynz56L5Xx8hZf5c+Pv7Y8+ePfpuGiE6RQFnRDo7O7F27Vo8//zzqK+vV1lWX1+P559/HmvXrkVnZ6eeWkiIblHAGZHS0lIsXboUjDG1h2XL5XIwxrB06VKUlpbqqYWE6JZen4tKHq+mpibu37/97W/h6jYCV6prMdLbEw31P+PYsWNq5QgZyijgjEhPgLm7u6OoqIjrWDh/ughCoRBubm6or6/HsWPHMGvWLH02lRCd0Psp6ubNm+Ht7Q0LCwuEh4fj5MmTvZbds2cPwsLCYG9vDysrK4SEhGDnzp0qZRhjSE9Ph5ubG8RiMSIjI3HlypXB3owhoba2FgBQV1cHJycn5OTkYNu2bcjJyYGTkxN3Xa6nHCFDnV4Dbvfu3ZBIJMjIyMCZM2cQHByM6Oho3LhxQ2N5R0dHvP/++ygsLMT58+eRnJyM5ORk7N+/nyuzZs0abNq0CTk5OThx4gSsrKwQHR2Nu3fvPq7NMlju7u4AADMzM1RVVcHPzw8lJSXw8/NDVVUVzMzMVMoRMuQxPZowYQJ76623uGmFQsHc3d1ZVlaW1nWMGzeOLV++nDHGmFKpZK6uruyjjz7ilre0tDCRSMS++OILretsbW1lAFhra6vW6wwFb775JgPAADALCwvu3w9Ov/nmm/puKhmAn6qbmde737Gfqpv13ZRB1Z+/T71dg7t37x6Ki4uRlpbGzTMxMUFkZCQKCwsfuj5jDIcOHUJ5eTk+/PBDAEBVVRUaGhoQGRnJlbOzs0N4eDgKCwsxb948jXXJ5XKVXkWZTAYA6OrqQldX14C2z9A9eET74DRft5vPuru7uZ983n/92Ta9BVxzczMUCgVcXFxU5ru4uODSpUu9rtfa2ooRI0ZALpdDKBTik08+wYwZMwAADQ0NXB0P1tmzTJOsrCxkZmaqzT9w4AAsLS213iZDp+1p+t27d7Fv375Bbg3RtWvtAGCKoqIi/HxB360ZPP25T3PI9aLa2Njg7NmzaG9vh1QqhUQiga+vL6ZNmzbgOtPS0iCRSLhpmUwGDw8PREVFwdbWVgetNgxCoRB5eXkPLffcc88hOjp68BtEdOpc7S2g5DQmTpyIYE9HfTdn0PScYWlDbwHn5OQEoVCIxsZGlfmNjY1wdXXtdT0TExP4+/sDAEJCQlBWVoasrCxMmzaNW6+xsRFubm4qdYaEhPRap0gkgkgkUptvZmbGXXjnA21O/XvKzZ49e5BbQ3TN1NSU+8mnz+2D+rNteutFNTc3R2hoKKRSKTdPqVRCKpUiIiJC63qUSiV3/czHxweurq4qdcpkMpw4caJfdfJVTU2NTssRYuj0eooqkUiQmJiIsLAwTJgwAdnZ2ejo6EBycjIAICEhASNGjEBWVhaA+9fKwsLC4OfnB7lcjn379mHnzp3YsmULAEAgEGDx4sVYvXo1Ro4cCR8fH6xYsQLu7u6YO3euvjbTYCiVSgD3j4KHDx+ucvTs4uKCpqYmKJVKrhwhQ51eAy4+Ph5NTU1IT09HQ0MDQkJCkJ+fz3US1NbWwsTkl4PMjo4OLFy4ENevX4dYLEZgYCA+++wzxMfHc2WWLVuGjo4OvPHGG2hpacHkyZORn58PCwuLx759hqbnK1hKpVLjpYEHyxEy1AkYY0zfjTA0MpkMdnZ2aG1t5VUnQ2xsLL799tuHlnv22Wfxz3/+8zG0iOjS2ZqbmLulCHtTJyLEa5i+mzNo+vP3qfevapHHx8nJSaflCDF0FHBG5KefftJpOUIMHQWcEfn55591Wo4QQ0cBZ0R6vsqjq3KEGLoh900GMnC/7kl2cnLCtGnTcOvWLTg6OqKgoADNzc1q5QgZyijgjIi9vT3q6uoA3P8u8Ndff91rOWL4Ojs7Vb63XV7fAnlDBcouiKG8aa9SNjAwkFffq9YWBZwR+e1vf4uLFy9qVY4YvkuXLiE0NFRt/kvb1csWFxdj/Pjxj6FVhoUCzohMmTIFW7du1aocMXyBgYEoLi7mptvvyPH94UL8bnoErMUitbLGiALOiHh4eOi0HNEvS0tLlaOyrq4u3G6+gYgJYbz+sn1/UMAZkSlTpsDb2xtCoRDV1dUqT7M3NTWFl5cXlEolHcER3qCAMyJCoRDr1q3D888/j1mzZsHHxweXL1/Gk08+iaqqKuzbtw9ff/01hEKhvptKiE5QwBmZuLg4fP3113jnnXfw/fffA7g/crGPjw++/vprxMXF6bmFhOgOBZwRiouLQ2xsLA4fPowffvgBM2fOxPTp0+nIjfAOBZyREgqFmDp1Kjo6OjB16lQKN8JL9FUtQghvUcARQniLAo4Qwlt0DU6DnkGO+/N4sqGoq6sLnZ2dkMlkdGMoDxjL/uz5u9RmMHIKOA3a2toA0B39hBiytrY22NnZ9VmGnsmggVKpRF1dHWxsbCAQCPTdnEHT84Dra9eu8erZE8bKWPYnYwxtbW1wd3dXeSiVJnQEp4GJiQmeeOIJfTfjsbG1teX1H4SxMYb9+bAjtx7UyUAI4S0KOEIIb1HAGTGRSISMjAyIRKKHFyYGj/anOupkIITwFh3BEUJ4iwKOEMJbFHCEEN6igNOBpKQkzJ07V6d1Tps2DYsXL+6zjLe3N7Kzs3X6voTwCQXcA7QJFjK0/OUvf0FISIi+m6GRoX3eDK09j4oCjpAh7t69e/pugsGigPuVpKQk/Pjjj9i4cSMEAgEEAgEqKyuxYMEC+Pj4QCwWIyAgABs3btS4fmZmJoYPHw5bW1u8+eabWn/wOjo6kJCQAGtra7i5uWHdunVqZW7cuIE5c+ZALBbDx8cHn3/+uVoZgUCALVu2YObMmRCLxfD19VV5en11dTUEAgG+/PJLTJkyBWKxGL/5zW9w+fJlnDp1CmFhYbC2tsbMmTPR1NSk5W8NyM3NxVNPPQWRSAQ3NzcsWrSIW1ZbW4vY2FhYW1vD1tYWL774IhobG7nlPUdXO3fuhLe3N+zs7DBv3jxuwAPg/neD16xZA39/f4hEInh6euKvf/0rt/zdd9/Fk08+CUtLS/j6+mLFihXo6uoCAOTl5SEzMxPnzp3j9mleXp7W2zaYBvp567kk8te//hXu7u4ICAgAABw/fhwhISGwsLBAWFgY9u7dC4FAgLNnz3LrXrhwATNnzoS1tTVcXFzw6quvorm5udf2VFdXP65fx+BghNPS0sIiIiJYSkoKq6+vZ/X19ezu3bssPT2dnTp1il29epV99tlnzNLSku3evZtbLzExkVlbW7P4+Hh24cIF9t1337Hhw4ez9957T6v3TU1NZZ6enuzf//43O3/+PJs9ezazsbFhb7/9Nldm5syZLDg4mBUWFrLTp0+zSZMmMbFYzDZs2MCVAcCGDRvGtm7dysrLy9ny5cuZUChkFy9eZIwxVlVVxQCwwMBAlp+fzy5evMgmTpzIQkND2bRp09jRo0fZmTNnmL+/P3vzzTe1avsnn3zCLCwsWHZ2NisvL2cnT57k2qRQKFhISAibPHkyO336NCsqKmKhoaFs6tSp3PoZGRnM2tqaxcXFsZKSEvaf//yHubq6qvzuli1bxhwcHFheXh6rqKhgR44cYVu3buWWr1q1ih07doxVVVWxb7/9lrm4uLAPP/yQMcZYZ2cne+edd9hTTz3F7dPOzk6ttm2wPern7dVXX2UXLlxgFy5cYK2trczR0ZG98sorrLS0lO3bt489+eSTDAD76aefGGOM3b59mw0fPpylpaWxsrIydubMGTZjxgw2ffr0XtvT3d2tj1+NzlDAPWDq1KkqwaLJW2+9xZ577jluOjExkTk6OrKOjg5u3pYtW5i1tTVTKBR91tXW1sbMzc3Zl19+yc27efMmE4vFXDvKy8sZAHby5EmuTFlZGQOgFnAPBlN4eDhLTU1ljP0ScH//+9+55V988QUDwKRSKTcvKyuLBQQE9NnuHu7u7uz999/XuOzAgQNMKBSy2tpabl5paanKtmRkZDBLS0smk8m4MkuXLmXh4eGMMcZkMhkTiUQqgfYwH330EQsNDeWmMzIyWHBwsNbrP04D/by5uLgwuVzOzduyZQsbNmwYu3PnDjdv69atKgG3atUqFhUVpVL3tWvXGABWXl6udXuGEhpNRAubN29Gbm4uamtrcefOHdy7d0/tonVwcDAsLS256YiICLS3t+PatWvw8vLqte7Kykrcu3cP4eHh3DxHR0futAMAysrKYGpqitDQUG5eYGAg7O3t1eqLiIhQm/71KQoABAUFcf92cXEBAIwdO1Zl3o0bN3ptc48bN26grq4OzzzzjMblZWVl8PDwUBlXb/To0bC3t0dZWRl+85vfALjfG2xjY8OVcXNz496/rKwMcrm81/cAgN27d2PTpk2orKxEe3s7uru7h/RoGtp83saOHQtzc3Nuury8HEFBQbCwsODmTZgwQWWdc+fO4fDhw7C2tlZ7z8rKSjz55JO63RADQNfgHmLXrl1YsmQJFixYgAMHDuDs2bNITk4e0hd2fz3aa894dw/OUyqVD61HLBbrvD0Pvv/D3qOwsBAvv/wyZs2ahe+++w4//fQT3n///SG7f7T9vFlZWfW77vb2dsyZMwdnz55VeV25cgX/9V//patNMCgUcA8wNzeHQqHgpo8dO4ZJkyZh4cKFGDduHPz9/VFZWam23rlz53Dnzh1uuqioCNbW1g8dFdjPzw9mZmY4ceIEN+/27du4fPkyNx0YGIju7m4UFxdz88rLy9HS0qJWX1FRkdr0qFGj+mzDQNnY2MDb2xtSqVTj8lGjRuHatWu4du0aN+/ixYtoaWnB6NGjtXqPkSNHQiwW9/oex48fh5eXF95//32EhYVh5MiRqKmpUSnz4D41JAP9vD0oICAAJSUlkMvl3LxTp06plBk/fjxKS0vh7e0Nf39/lVdPYBry72ogKOAe4O3tjRMnTqC6uhrNzc0YOXIkTp8+jf379+Py5ctYsWKF2gcHuN9Vv2DBAly8eBH79u1DRkYGFi1a9NARR62trbFgwQIsXboUhw4dwoULF5CUlKSyXkBAAGJiYvCHP/wBJ06cQHFxMV5//XWNRzdfffUVcnNzcfnyZWRkZODkyZMqvZq69pe//AXr1q3Dpk2bcOXKFZw5cwYff/wxACAyMhJjx47Fyy+/jDNnzuDkyZNISEjA1KlTERYWplX9FhYWePfdd7Fs2TLs2LEDlZWVKCoqwqeffgrgfgDW1tZi165dqKysxKZNm/DNN9+o1OHt7Y2qqiqcPXsWzc3NKiGgbwP9vD3opZdeglKpxBtvvIGysjLs378fa9euBfDLUfpbb72FW7duYf78+Th16hQqKyuxf/9+JCcnc6H2YHu0OZI3aPq+CGhoysvL2cSJE5lYLGYA2KVLl1hSUhKzs7Nj9vb2LDU1lf35z39WuWidmJjIYmNjWXp6Ohs2bBiztrZmKSkp7O7du1q9Z1tbG3vllVeYpaUlc3FxYWvWrFG72FtfX89+97vfMZFIxDw9PdmOHTuYl5eXWifD5s2b2YwZM5hIJGLe3t4qvW89nQw9F50ZY+zw4cMMALt9+zY3b9u2bczOzk7r31lOTg4LCAhgZmZmzM3Njf3xj3/kltXU1LBnn32WWVlZMRsbG/bCCy+whoYGbrmmDoANGzYwLy8vblqhULDVq1czLy8vZmZmxjw9PdkHH3zALV+6dCn3e4+Pj2cbNmxQaf/du3fZc889x+zt7RkAtm3bNq23bbA9yuftQceOHWNBQUHM3NychYaGsn/84x9cnT0uX77Mfv/73zN7e3smFotZYGAgW7x4MVMqlRrbU1VVNci/gcFFwyXxiEAgwDfffKPzr42Roenzzz9HcnIyWltbdXa9dKihXlRCeGLHjh3w9fXFiBEjcO7cObz77rt48cUXjTbcAAq4QVdbW9vnBfWLFy/C09PzMbaofzTdUtDjhx9+wJQpUx5ja0hfGhoakJ6ejoaGBri5ueGFF15Q+caHMaJT1EHW3d3d59ddvL29YWpquP/PVFRU9LpsxIgRRn10QAwfBRwhhLfoNhFCCG9RwBFCeIsCjhDCWxRwhBDeooAjepWUlMQNrmhmZgYXFxfMmDEDubm5/fqaUF5ensbRVQbbYDyPg+gOBRzRu5iYGNTX16O6uho//PADpk+fjrfffhuzZ89Gd3e3vptHhjJ9fk+MkN6+VymVShkAbqDLdevWsTFjxjBLS0v2xBNPsNTUVNbW1sYY++X7tL9+ZWRkMMYY27FjBwsNDWXW1tbMxcWFzZ8/nzU2NnLvc+vWLfbSSy8xJycnZmFhwfz9/Vlubi63vLa2lr3wwgvMzs6OOTg4sGeffZb7fmZGRoba+x4+fHhQfk9kYOgIjhikp59+GsHBwdizZw8AwMTEBJs2bUJpaSm2b9+OQ4cOYdmyZQCASZMmITs7G7a2tqivr0d9fT2WLFkCAOjq6sKqVatw7tw57N27F9XV1UhKSuLeZ8WKFbh48SJ++OEHlJWVYcuWLXBycuLWjY6Oho2NDY4cOYJjx47B2toaMTExuHfvHpYsWYIXX3yROwKtr6/HpEmTHu8vivRN3wlLjFtvR3CMMRYfH89GjRqlcdlXX33Fhg0bxk1rOwLKqVOnGADu6G/OnDksOTlZY9mdO3eygIAAbqQNxhiTy+VMLBaz/fv3P7T9RP/oCI4YLMYYN5bZv//9bzzzzDMYMWIEbGxs8Oqrr+LmzZvo7Ozss47i4mLMmTMHnp6esLGxwdSpUwHc/44wAKSmpmLXrl0ICQnBsmXLcPz4cW7dc+fOoaKiAjY2NrC2toa1tTUcHR1x9+5drQahJPpHAUcMVllZGXx8fFBdXY3Zs2cjKCgI//d//4fi4mJs3rwZQN/PBO3o6EB0dDRsbW3x+eef49SpU9xgmD3rzZw5EzU1NfjTn/7EPV+i5/S2vb0doaGhakN8X758GS+99NIgbz3RBcP9ljcxaocOHUJJSQn+9Kc/obi4GEqlEuvWreNGOv7yyy9VymsaavvSpUu4efMm/ud//ocbOv706dNq7zV8+HAkJiYiMTERU6ZMwdKlS7F27VqMHz8eu3fvhrOzc68PseHbEN98Q0dwRO/kcjkaGhrw888/48yZM/jggw8QGxuL2bNnIyEhAf7+/ujq6sLHH3+Mq1evYufOncjJyVGpw9vbG+3t7ZBKpWhubkZnZyc8PT1hbm7Orfftt99i1apVKuulp6fjn//8JyoqKlBaWorvvvuOe4bFyy+/DCcnJ8TGxuLIkSOoqqpCQUEB/vu//xvXr1/n3vf8+fMoLy9Hc3Mz98BpYiD0fRGQGLfExETuFgtTU1M2fPhwFhkZyXJzc1WeKbt+/Xrm5ubGxGIxi46OZjt27FAbav3NN99kw4YNU7lN5B//+Afz9vZmIpGIRUREsG+//VbtWaGjRo1iYrGYOTo6stjYWHb16lWuzvr6epaQkMCcnJyYSCRivr6+LCUlhbW2tjLGGLtx4wabMWMGs7a2pttEDBANl0QI4S06RSWE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhrf8H6xGjdUi3daIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:54:34.659357Z", + "iopub.status.busy": "2024-02-29T20:54:34.659046Z", + "iopub.status.idle": "2024-02-29T20:54:34.932127Z", + "shell.execute_reply": "2024-02-29T20:54:34.931215Z" + }, + "papermill": { + "duration": 0.29791, + "end_time": "2024-02-29T20:54:34.934235", + "exception": false, + "start_time": "2024-02-29T20:54:34.636325", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABiqElEQVR4nO2dd3xT1fvH3xlN92K1FFrKqGwBGbWAIlotiALqDxGRJYp+AVHLHjJEBWUrKF9URP2KLBFUEIUKKFBWKXsPKasts7vNur8/QtOkTdKkpG1azvv1yqvNvefenNwkn/uc8zzneWSSJEkIBAJBJURe3h0QCASC0kIInEAgqLQIgRMIBJUWIXACgaDSIgROIBBUWoTACQSCSosQOIFAUGkRAicQCCotyvLugCui1+u5evUqvr6+yGSy8u6OQCAohCRJZGRkEBISglxu3U4TAmeBq1evEhoaWt7dEAgExXDp0iVq165tdX+5C9yiRYuYNWsWycnJtGjRgs8++4x27dpZbHvs2DEmT55MQkICFy9eZN68ebzzzjtmbWbMmMHatWs5efIknp6etG/fno8//piGDRva3SdfX1/AcPH8/PxK/N4EAkHpkJ6eTmhoqPG3ao1yFbiVK1cSGxvL4sWLiYyMZP78+cTExHDq1Clq1KhRpH12djb16tWjV69evPvuuxbPuX37doYNG0bbtm3RarVMmDCBp556iuPHj+Pt7W1Xv/KHpX5+fkLgBAIXptgpJKkcadeunTRs2DDjc51OJ4WEhEgzZswo9tg6depI8+bNK7ZdamqqBEjbt2+3u19paWkSIKWlpdl9jEAgKDvs/Y2WmxdVrVaTkJBAdHS0cZtcLic6Opr4+HinvU5aWhoAVapUsdomLy+P9PR0s4dAIKj4lJvA3bhxA51OR1BQkNn2oKAgkpOTnfIaer2ed955hw4dOtCsWTOr7WbMmIG/v7/xIRwMAkHloNydDKXJsGHDOHr0KDt27LDZbvz48cTGxhqf509glgSdTodGoynRsQKBwIBCoUCpVN5zmFa5CVy1atVQKBSkpKSYbU9JSSE4OPiezz98+HB+++03/v77b5tuZAB3d3fc3d3v+TUzMzO5fPkyksghKhDcM15eXtSsWROVSlXic5SbwKlUKlq3bk1cXBw9e/YEDEPKuLg4hg8fXuLzSpLEW2+9xc8//8y2bduoW7euk3psG51Ox+XLl/Hy8qJ69eoiQFggKCGSJKFWq7l+/ToXLlwgIiLCZjCvLcp1iBobG8uAAQNo06YN7dq1Y/78+WRlZTFo0CAA+vfvT61atZgxYwZgcEwcP37c+P+VK1c4ePAgPj4+NGjQADAMS5cvX8769evx9fU1zuf5+/vj6elZau9Fo9EgSRLVq1cvlddJTc8lT6undqCnEE9BpcfT0xM3NzcuXryIWq3Gw8OjZCcqA4+uTT777DMpLCxMUqlUUrt27aTdu3cb93Xq1EkaMGCA8fmFCxckoMijU6dOxjaW9gPSN998Y3efShImkpOTIx0/flzKycmx+xhHOHr5jnTo0m0pT6MtlfMLBK6Grd+Uvb/RcncyDB8+3OqQdNu2bWbPw8PDi53fKm5/RUUq9FcgEBSPyCZSQZCK/CMQCIpDCFxFwcUs04EDBxqdQ87iscceK7K2uDDh4eHMnz/fqa8rKB/s+bzvFSFwFYTSlLey+KIJypapU6fSsmXL8u5GuSMErgJgOq94IzOPK7ezK+1co6Bi4epB7ULgSglJkshWa53yyFJrydXoyNXouHInhyt3ckjLUVtt74j4DRw4kO3bt7NgwQJkMhkymYxz584xePBg6tati6enJw0bNmTBggUWj582bRrVq1fHz8+PN998E7VabdfrZmVl0b9/f3x8fKhZsyZz5swp0iY1NZVnn30WT09P6tatyw8//FCkjUwm44svvqBr1654enpSr1491qxZY9z/77//IpPJWLVqFY888gienp60bduW06dPs2/fPtq0aYOPjw9du3bl+vXrdl41WLp0KU2bNsXd3Z2aNWuaOcqSkpLo0aMHPj4++Pn58eKLL5oFtOdbV99//z3h4eH4+/vz0ksvkZGRYWyj1+v55JNPaNCgAe7u7oSFhfHhhx8a948dO5YHHngALy8v6tWrx3vvvWcUm2XLljFt2jQOHTpk/EyXLVtW7Hs6efIkHTt2xMPDgyZNmrBlyxZkMhnr1q0zu5YrV66kU6dOeHh48MMPP3Dz5k369OlDrVq18PLyonnz5vz4449m57bn8y4Nyt2LWlnJ0ehoMvmPcnnt4+/H4KWy76NdsGABp0+fplmzZrz//vsABAYGUrt2bVavXk3VqlXZtWsXQ4YMoWbNmrz44ovGY+Pi4vDw8GDbtm38+++/DBo0iKpVq5r9EK0xevRotm/fzvr166lRowYTJkzgwIEDZsOqgQMHcvXqVbZu3YqbmxsjRowgNTW1yLnee+89Zs6cyYIFC/j+++956aWXOHLkCI0bNza2mTJlCvPnzycsLIxXX32Vl19+GV9fXxYsWICXlxcvvvgikydP5osvvii271988QWxsbHMnDmTrl27kpaWxs6dOwGDMOWL2/bt29FqtQwbNozevXubRQWcO3eOdevW8dtvv3H79m1efPFFZs6cabx248eP58svv2TevHl07NiRa9eucfLkSePxvr6+LFu2jJCQEI4cOcLrr7+Or68vY8aMoXfv3hw9epRNmzaxZcsWwBAHagudTkfPnj0JCwtjz549ZGRkMHLkSIttx40bx5w5c2jVqhUeHh7k5ubSunVrxo4di5+fHxs2bKBfv37Ur1/fmNvRns+7VHB+9ErFxxlxcFl5GqnO2N/K5ZGVp3Ho/Xbq1El6++23bbYZNmyY9MILLxifDxgwQKpSpYqUlZVl3PbFF19IPj4+kk6ns3mujIwMSaVSSatWrTJuu3nzpuTp6Wnsx6lTpyRA2rt3r7HNiRMnJMAsTRYgvfnmm2bnj4yMlP7zn/9IklQQO/nVV18Z9//4448SIMXFxRm3zZgxQ2rYsKHNfucTEhIiTZw40eK+P//8U1IoFFJSUpJx27Fjx8zey5QpUyQvLy8pPT3d2Gb06NFSZGSkJEmSlJ6eLrm7u0tffvmlXf2RJEmaNWuW1Lp1a+PzKVOmSC1atLD7+N9//11SKpXStWvXjNs2b94sAdLPP/8sSVLBtZw/f36x5+vWrZs0cuRISZLs+7wtUSni4Cornm4Kjr8f45Rz6fR6TlzLMNvWoIYPHm4Kq699ryxatIilS5eSlJRETk4OarW6yN22RYsWeHl5GZ9HRUWRmZnJpUuXqFOnjtVznzt3DrVaTWRkpHFblSpVzLIunzhxAqVSSevWrY3bGjVqREBAQJHzRUVFFXl+8OBBs20PPvig8f/8DDbNmzc322bJOixMamoqV69e5YknnrC4/8SJE4SGhpola2jSpAkBAQGcOHGCtm3bAgZvsGk22po1axpf/8SJE+Tl5Vl9DTAki/300085d+4cmZmZaLXae0rOeurUKUJDQ83WgVvLrN2mTRuz5zqdjo8++ohVq1Zx5coV1Go1eXl5xu+GPZ93aSEErpSQyWR2DxOLQ6vTFxEzL5XSqsDdKytWrGDUqFHMmTOHqKgofH19mTVrFnv27CmV1ysL3NzcjP/nL3UrvE2v1xd7HmctwzN97cKvX9xrxMfH07dvX6ZNm0ZMTAz+/v6sWLGizOa1CmfGnjVrFgsWLGD+/Pk0b94cb29v3nnnHbvnY0sT4WQQoFKp0Ol0xuc7d+6kffv2DB06lFatWtGgQQPOnTtX5LhDhw6Rk5NjfL579258fHyKTTVVv3593NzczATz9u3bnD592vi8UaNGaLVaEhISjNtOnTrFnTt3ipxv9+7dRZ6bzr85E19fX8LDw4mLi7O4v3Hjxly6dIlLly4Ztx0/fpw7d+7QpEkTu14jIiICT09Pq6+xa9cu6tSpw8SJE2nTpg0RERFcvHjRrE3hz7Q4GjZsyKVLl8ycIfv27bPr2J07d9KjRw9eeeUVWrRoQb169cw+S3s+79JCWHAVAEs+UWdGiYSHh7Nnzx7+/fdffHx8iIiI4LvvvuOPP/6gbt26fP/99+zbt69IZha1Ws3gwYOZNGkS//77L1OmTGH48OHFZn7w8fFh8ODBjB49mqpVq1KjRg0mTpxodlzDhg3p0qULb7zxBl988QVKpZJ33nnHonWzevVq2rRpQ8eOHfnhhx/Yu3cvX3/9tXMujgWmTp3Km2++SY0aNejatSsZGRns3LmTt956i+joaJo3b07fvn2ZP38+Wq2WoUOH0qlTpyJDO2t4eHgwduxYxowZg0qlokOHDly/fp1jx44xePBgIiIiSEpKYsWKFbRt25YNGzbw888/m50jPDycCxcucPDgQWrXro2vr6/NlGBPPvkk9evXZ8CAAXzyySdkZGQwadIkoPi6BxEREaxZs4Zdu3YRGBjI3LlzSUlJMQq6PZ93aSEsuAqL8xRu1KhRKBQKmjRpQvXq1YmJieH555+nd+/eREZGcvPmTYYOHVrkuCeeeIKIiAgeffRRevfuTffu3Zk6dapdrzlr1iweeeQRnn32WaKjo+nYsaPZfBvAN998Q0hICJ06deL5559nyJAhFosRTZs2jRUrVvDggw/y3Xff8eOPP9ptLZWEAQMGMH/+fD7//HOaNm3KM888w5kzZwCDGKxfv57AwEAeffRRoqOjqVevHitXrnToNd577z1GjhzJ5MmTady4Mb179zbO0XXv3p13332X4cOH07JlS3bt2sV7771ndvwLL7xAly5d6Ny5M9WrVy8StlEYhULBunXryMzMpG3btrz22mtMnDgRoNhMHpMmTeKhhx4iJiaGxx57jODg4CKrXOz5vEsDmSSJiNHCpKen4+/vT1pamt0Tt7m5uVy4cIG6deuWPLWLFdRaPSeTzetERNTwwdNJc3wVGZlMxs8//+z0ZWMCw9CzY8eOnD17lvr165f569v6Tdn7GxW/kApB0XuQuCsJnM3PP/9snKI4e/Ysb7/9Nh06dCgXcXMWYohaAahoYpaUlISPj4/VR1JSUnl30Sa2+v7PP/+Ud/dKxA8//GD1PTVt2hSAjIwMhg0bRqNGjRg4cCBt27Zl/fr15dzze0NYcBUBi16GMu+F3YSEhBSJQyu831mUxgyLrb7XqlXL6a9XFnTv3t0sDs2U/JCV/v37079//7LsVqkjBK4CUMH0DaVSaUwhXxGpyH23hq+vr1lg8f2CGKIKBIJKixC4CoDwcwsEJUMIXIVAeFEFgpIgBK4CYFHMhMIJBMUiBK4iIBROICgRQuAqAK7sRS1cBMY0A6xAUN6IMBGBU7l27RqBgYHl3Q2BABACVyFwFWvNHkwTJpYXkiSh0+lQKsXX+36n3IeoixYtIjw8HA8PDyIjI9m7d6/VtseOHeOFF14gPDwcmUxmtT6mI+csNSQJ1FnOeeRlItNkmz0kW+0djCvJyMigb9++eHt7U7NmTebNm1fiUoKWipSsXbuWzp074+XlRYsWLYiPjzc7ZseOHcaCMKGhoYwYMYKsrCzj/u+//542bdrg6+tLcHAwL7/8sln23W3btiGTyfj9999p3bo17u7u7Nixw+G+Cyof5XqLW7lyJbGxsSxevJjIyEjmz59PTEwMp06dspgWJzs7m3r16tGrVy/effddp5yz1NBkw0fOWZLkAzQvtpUJE66Cyrv4dneJjY1l586d/PLLLwQFBTF58mSnFgSZOHEis2fPJiIigokTJ9KnTx/Onj2LUqnk3LlzdOnShQ8++IClS5dy/fp1hg8fzvDhw/nmm28AQ2m66dOn07BhQ1JTU4mNjWXgwIFs3LjR7HXGjRvH7NmzqVevnhgmC4BytuDmzp3L66+/zqBBg2jSpAmLFy/Gy8uLpUuXWmzftm1bZs2axUsvvWQ1eZ+j57zfycjI4Ntvv2X27Nk88cQTNGvWjG+++cahbLDFMWrUKLp168YDDzzAtGnTuHjxImfPngVgxowZ9O3bl3feeYeIiAjat2/Pp59+ynfffUdubi4Ar776Kl27dqVevXo8/PDDfPrpp/z+++9kZmaavc77779vTNxYpUoVp/VfUHEpNwtOrVaTkJDA+PHjjdvkcjnR0dFFhjClfc68vDzy8vKMz9PT0622tRs3L4Ml5QTSczRcvJVtti2sihf+nm6WD3DzsrzdAufPn0ej0ZgVGPH393dqQRDTgi81a9YEDMVbGjVqxKFDhzh8+LBZzVNJktDr9Vy4cIHGjRuTkJDA1KlTOXToELdv3zbWLkhKSjJLbGlvxlzB/UO5CdyNGzfQ6XTGCkf5BAUFmdV/LItzzpgxg2nTppXoNa0ikzk0TLSJToNUWMtU3qCyInAuhqWCL/kilZmZyRtvvMGIESOKHBcWFkZWVhYxMTHExMTwww8/UL16dZKSkoiJiSlS1KRwMRSBQLiZMBTZjY2NNT5PT08vtnBKWWLZZeAc32q9evVwc3Nj3759hIWFAZCWlsbp06d59NFHnfIatnjooYc4fvy41QweR44c4ebNm8ycOdP4mezfv7/U+yWoHJTbHFy1atVQKBRmVXwAUlJSShxqUNJzuru74+fnZ/ZwKSx4RfV6yFHr7jkfmq+vLwMGDGD06NFs3brVWNhELpcXW2zEGYwdO5Zdu3YxfPhwDh48yJkzZ1i/fj3Dhw8HDFacSqXis88+4/z58/zyyy9Mnz691PslqByUm8CpVCpat25tVhpNr9cTFxdXpJBveZ7TFbAkYVfu5HAmNYNs9b07A+bOnUtUVBTPPPMM0dHRdOjQgcaNGzu9toQlHnzwQbZv387p06d55JFHaNWqFZMnTzYmxaxevTrLli1j9erVNGnShJkzZzJ79uxS75egkmCz7n0ps2LFCsnd3V1atmyZdPz4cWnIkCFSQECAlJycLEmSJPXr108aN26csX1eXp6UmJgoJSYmSjVr1pRGjRolJSYmSmfOnLH7nPaQlpYmAVJaWprdx+Tk5EjHjx+XcnJy7D7GXm5n5UmHLt22+LiVlef018vMzJT8/f2lr776yunnFgjsxdZvyt7faLnOwfXu3Zvr168zefJkkpOTadmyJZs2bTI6CZKSksxqJ169epVWrVoZn8+ePZvZs2fTqVMntm3bZtc5KyK2BqHOyBWXmJjIyZMnadeuHWlpabz//vsA9OjR495PLhCUI6JsoAVcrWzgrSw1l29nW9xXO9CLKt6qezp/YmIir732GqdOnTIO8+fOncudO3fo2rWr1eMKx6EJBM5ElA0UIDnBm9qqVSsSEhKKbM/JybFZgEUgcHWEwFUAbIpYKdrfnp6elbIAi+D+odwX21c2SmXEXz76JhCUK874LQmBcxIKhQKgSHS9MxAiJrgfyc42zDubroRxFDFEdRJKpRIvLy+uX7+Om5ubmff3XtHkqZG0loVTnScnVykkUFB5kCSJ7OxsUlNTCQgIMBoPJUEInJOQyWTUrFmTCxcucPHiRaeeOyNXS1qOxuK+PE8laR4VY02qQOAIAQEB95xAVQicE1GpVERERDh9mLpyXxJL/r5icd9rj9SjT+Mwp76eQFDeuLm53ZPllo8QOCcjl8udHgeXrVNwJcPykqxsnfNfTyCoLAgnQwVAb8ObpNOL+TeBwBpC4CoAehsiJgROILCOELgKgM6GBWfLuhMI7neEwFUAhAUnEJQMIXAVAFsWnK19AsH9jhC4CoBOb32fLetO4FwkSeKN7/czctWh8u6KwE6EwFUAbHtRy7Aj9zkXb2bzx7EUfjpwGY248BUCIXAVAFvzbMLJUHZohbVc4RACVwGwJXDCyVA+lH45HoEzEAJXAbA5RBUWXBkirnVF456WamVmZhoL+ObjciX3KgE2h6jCgiszTO8l4qpXDBy24C5cuEC3bt3w9vbG39+fwMBAAgMDCQgIIDAwsDT6eN9jS8PEELXsEFe64uGwBffKK68gSRJLly4lKCioTIoD3+/YDPQVQ9RyQVz2ioHDAnfo0CESEhJo2LBhafRHYAGbS7WEBVdmCFGreDg8RG3bti2XLl0qjb4IrGDbgivDjgiMOKOamaD0cdiC++qrr3jzzTe5cuUKzZo1K5Iv/cEHH3Ra5wQGhAXnGghRq3g4LHDXr1/n3LlzDBo0yLhNJpMhSRIymQydznJiRkHJEXFwrocYrlYMHB6ivvrqq7Rq1Yr4+HjOnz/PhQsXzP46yqJFiwgPD8fDw4PIyEj27t1rs/3q1atp1KgRHh4eNG/enI0bN5rtz8zMZPjw4dSuXRtPT0+aNGnC4sWLHe6XKyHi4FwDcakrHg5bcBcvXuSXX35xSkHglStXEhsby+LFi4mMjGT+/PnExMRw6tQpatSoUaT9rl276NOnDzNmzOCZZ55h+fLl9OzZkwMHDtCsWTMAYmNj+euvv/jf//5HeHg4f/75J0OHDiUkJITu3bvfc5/LAxEHJxCUDIctuMcff5xDh5yTTWHu3Lm8/vrrDBo0yGhpeXl5sXTpUovtFyxYQJcuXRg9ejSNGzdm+vTpPPTQQyxcuNDYZteuXQwYMIDHHnuM8PBwhgwZQosWLYq1DF0ZW+u6hQVXdohLXfFw2IJ79tlneffddzly5AjNmzcv4mSw10pSq9UkJCQwfvx44za5XE50dDTx8fEWj4mPjyc2NtZsW0xMDOvWrTM+b9++Pb/88guvvvoqISEhbNu2jdOnTzNv3jyrfcnLyyMvL8/4PD093a73UFaImgyuhxC7ioHDAvfmm28C8P777xfZ54iT4caNG+h0OoKCgsy2BwUFcfLkSYvHJCcnW2yfnJxsfP7ZZ58xZMgQateujVKpRC6X8+WXX/Loo49a7cuMGTOYNm2aXf0uD2wJnMgmUnYIL2rFw+Ehql6vt/pwBQ/qZ599xu7du/nll19ISEhgzpw5DBs2jC1btlg9Zvz48aSlpRkfrhbnJ7yorocQu4qBQxacRqPB09OTgwcPGif1S0q1atVQKBSkpKSYbU9JSbFazTo4ONhm+5ycHCZMmMDPP/9Mt27dAENc3sGDB5k9ezbR0dEWz+vu7o67u/s9vZ/SxKYFJ/IulhnCWK54OGTBubm5ERYW5hRLTaVS0bp1a+Li4ozb9Ho9cXFxREVFWTwmKirKrD3A5s2bje01Gg0ajQa53PxtKRSKIllPKhI2LTjxqysXxGWvGDg8RJ04cSITJkzg1q1b9/zisbGxfPnll3z77becOHGC//znP2RlZRmDiPv372/mhHj77bfZtGkTc+bM4eTJk0ydOpX9+/czfPhwwJCqqVOnTowePZpt27Zx4cIFli1bxnfffcdzzz13z/0tL2xpsxiiCgTWcdjJsHDhQs6ePUtISAh16tTB29vbbP+BAwfsPlfv3r25fv06kydPJjk5mZYtW7Jp0yajIyEpKcnMGmvfvj3Lly9n0qRJTJgwgYiICNatW2c2XF6xYgXjx4+nb9++3Lp1izp16vDhhx8anSMVEVEX1fUQV71i4LDA9ezZ06kdGD58uNECK8y2bduKbOvVqxe9evWyer7g4GC++eYbZ3XPJRBOBtdA3EsqHg4L3JQpU0qjHwIbiDg410MSalchKHHK8oSEBE6cOAFA06ZNadWqldM6JTBHVNVyDURoSMXDYYFLTU3lpZdeYtu2bQQEBABw584dOnfuzIoVK6hevbqz+3jfI4aoAkHJcNiL+tZbb5GRkcGxY8e4desWt27d4ujRo6SnpzNixIjS6ON9j+2VDGXYkfscUXSm4uGwBbdp0ya2bNlC48aNjduaNGnCokWLeOqpp5zaOYEBUXRGICgZJVqqVXiBPRiCgCtyMK0rYzNluRC4MsP0Soupz4pBidIlvf3221y9etW47cqVK7z77rs88cQTTu2cwICIgxMISobDArdw4ULS09MJDw+nfv361K9fn7p165Kens5nn31WGn287xFOBtdAEpNwFQ6H5+BCQ0M5cOAAW7ZsMaY1aty4sdWF7IJ7x9YQVVhwZYe40kXJyNWwbOe/PNsihPBq3sUfUMaUKA5OJpPx5JNP8uSTTzq7PwIL2BqiCguu7DA34MR1B1iTcJk5m09z+XYOH/+f61XUK5HAxcXFERcXR2pqahHHgrV044KSYzNluRA4QTlyNjUTgEy1tpx7YhmHBW7atGm8//77tGnThpo1ayKTyUqjXwITRBycq1BwscXMgIGkW9mA6xY/cljgFi9ezLJly+jXr19p9EdgAeFkELgqF28aBE7rot9Dh72oarWa9u3bl0ZfBFawGQcnTIkyQzhRzdHo9Fy5kwO4rgXnsMC99tprLF++vDT6IrCCzTg4F/1iVUbElTbn6p0c4wjCVW+0Dg9Rc3NzWbJkCVu2bOHBBx8ssqph7ty5TuucwICobO96iHRJBcNTcN2pEocF7vDhw7Rs2RKAo0ePmu0TDofSwdYKOEky/NjEtS99hKaZc/FWgcC5ajymwwK3devW0uiHwAbFWWk6vYRSIQSuLHHNn3PZcvFGlvF/rc41r4jDc3CCsqc4818MU8sGMSw1pyJYcELgXBx7nAgiiUvZILKJmJNUAebghMC5OPZYZ8KCKx3UWvM7h7jMBUiSZAzyBXDREaoQOFfHnjujq949KzKfxp3hgUm/s//fgvq/putP7/e1qNcz8sjRFBSAd9VwJYcF7u+//0arLbruTKvV8vfffzulU4IC7JnbcNUvV0Vm7ubTAEz99VjBRnGZjZjOv4Hr3mQdFrjOnTtbrGqflpZG586dndIpQQF2WXBi7FQmSFaf3H/kx8Ap5QbvfaUROGsxVzdv3ixS5V5w79jjQBAWXOlhtjxLXGYjSTcNISJhVbwA173J2i1wzz//PM8//zwymYyBAwcanz///PP06NGDmJiYEq1RXbRoEeHh4Xh4eBAZGcnevXtttl+9ejWNGjXCw8OD5s2bs3HjxiJtTpw4Qffu3fH398fb25u2bduSlJTkcN9cAXuGqK765apsmM/B3d/kD1Hr3k1y6ao3WbsFzt/fH39/fyRJwtfX1/jc39+f4OBghgwZwv/+9z+HXnzlypXExsYyZcoUDhw4QIsWLYiJiSE1NdVi+127dtGnTx8GDx5MYmIiPXv2pGfPnmYrKs6dO0fHjh1p1KgR27Zt4/Dhw7z33nt4eHg41DdXwS4vqot+uSoDwoKzTP4QtV51g8C56k3W7pUM33zzDQDh4eGMGjXKKcPRuXPn8vrrrzNo0CDAkIppw4YNLF26lHHjxhVpv2DBArp06cLo0aMBmD59Ops3b2bhwoUsXrwYgIkTJ/L000/zySefGI+rX7/+Pfe1vBBxcK6Da/6Ey4ckowXnA7juTdbhObgpU6Y4RdzUajUJCQlmtRzkcjnR0dHEx8dbPCY+Pr5I7YeYmBhje71ez4YNG3jggQeIiYmhRo0aREZGsm7dOpt9ycvLIz093ezhKog4uPLFPLhXJLwEQx2GW1lqAMKr3Z2DqywCl5KSQr9+/QgJCUGpVKJQKMwe9nLjxg10Oh1BQUFm24OCgkhOTrZ4THJyss32qampZGZmMnPmTLp06cKff/7Jc889x/PPP8/27dut9mXGjBlmQ+7Q0FC730dpI+LgXAdxlQ3kD0+reKvw9zRkE3LV76DDi+0HDhxIUlIS7733nsulLM+vD9GjRw/effddAFq2bMmuXbtYvHgxnTp1snjc+PHjiY2NNT5PT093GZGzy4t6P5sTZYmdRWf+Pn2drDwtXZvXLINOlT35w9OwKl4o7oaJuOp30GGB27FjB//8848xZVJJqVatGgqFgpSUFLPtKSkpBAcHWzwmODjYZvtq1aqhVCpp0qSJWZvGjRuzY8cOq31xd3fH3d29JG+j1BFOBtfB3tUL/ZcaIgH2THiCIL+K6dyyRb4FV6eqV+WLgwsNDXVKVgWVSkXr1q2Ji4szbtPr9cTFxREVFWXxmKioKLP2AJs3bza2V6lUtG3bllOnTpm1OX36NHXq1LnnPpcHYohavlibd7PnJ3A7W10KPSp/km4ZYuDqVPFCLnNtgXPYgps/fz7jxo3jv//9L+Hh4ff04rGxsQwYMIA2bdrQrl075s+fT1ZWltGr2r9/f2rVqsWMGTMAePvtt+nUqRNz5syhW7durFixgv3797NkyRLjOUePHk3v3r159NFH6dy5M5s2beLXX39l27Zt99TX8sKupVouOjyobNhzme+HlEr5FlxYVW/jELXSCFzv3r3Jzs6mfv36eHl5FUlZbmkZl61zXb9+ncmTJ5OcnEzLli3ZtGmT0ZGQlJSEXF5gZLZv357ly5czadIkJkyYQEREBOvWraNZs2bGNs899xyLFy9mxowZjBgxgoYNG/LTTz/RsWNHR9+qSyAsONdBsvK/WZv74KMwHaIaLTgXfeMlsuCcyfDhwxk+fLjFfZasrl69etGrVy+b53z11Vd59dVXndG9csce8XI1C06vl5DLXcf55Czsuc6u9Uk4H7VWz7U0QyWtOlW8jOUCXTUW02GBGzBgQGn0Q2AFe7RL50Jfro83nWT1/ktsGPFIpZtgN5+Ds/zBVPYh6uXb2egl8HRTUN3XndSMPMB1LbgS5YM7d+4ckyZNok+fPsZlVb///jvHjh0r5kiBo1Q0L+oX285xI1PNoq1ny7srTsH88gsL7qJJiIhMJnP5OTiHBW779u00b96cPXv2sHbtWjIzMwE4dOgQU6ZMcXoH73cq4hAVoPINUO3zorrgR+FUkowOBsMKBoVJHKwrLrh3WODGjRvHBx98wObNm1GpVMbtjz/+OLt373Zq5wR2ZhNxwS9WZcSeq1zZM/0aHQx30ySZzrVqXfB76LDAHTlyhOeee67I9ho1anDjxg2ndEpQgEh46TrYFyZS+v0oT4wxcPkWnInAueJIwmGBCwgI4Nq1a0W2JyYmUqtWLad0SlCAfdlEXO+L5Xo9KhmiDoM5BSEihoQbpkNUVxxJOCxwL730EmPHjiU5ORmZTIZer2fnzp2MGjWK/v37l0Yf72sqmpOhMlMR5uAW/nWGL/8+Xyrn1usLKmlZsuBccSThsMB99NFHNGrUiNDQUDIzM2nSpAmPPvoo7du3Z9KkSaXRx/uaiupkqCyYiZo97cvRykvL0TD7z9PM+P0EeVpd8Qc4SGpGHnlaPQq5jJAAT6DQENUFb7QOx8GpVCq+/PJL3nvvPY4ePUpmZiatWrUiIiKiNPp332Ofk6EMOiIwX5dqRcjK816TmWeodqeXIFetx11pf/oye7h4tw5DrQBP3BQG28g0ntsVnQwOC1w+YWFhhIWFObMvAgvYI16uODRwwS6VCEffRnm+7Rx1gdWWrdHij5uN1o5zsdDwFEAmkyGXGUS1wlpwsbGxTJ8+HW9vb7O8aZaYO3euUzomMCDqoroO9s3Bld9nkWtSiNlU7JyFMQauipfZdoVchl4nueSN1i6BS0xMRKPRGP+3hislv6ws2CNewslQNtgzv1aen0S2iaiZVp13FpYsOODugnvJJb+Hdgnc1q1bLf4vKH1ETQbXwR6HQ3l+FDmlbsHl10I1r8milMvIwzUX3JdoLaqg7LDLi+qCd87KEjPmcKGZ8hQ4tbbg/7K04OSumzLJLgvu+eeft/uEa9euLXFnBEURhZ9dB7N8cFauuennJSvjFbmlacGlZWu4k22YprI0Bwegc0ETzi4LzrTilJ+fH3Fxcezfv9+4PyEhgbi4OPz9/Uuto/cr9nhRXdGCq4zY40CorHNwF+8u0arm4463u7ldpDCmLXfqSzoFuyy4/KLPAGPHjuXFF19k8eLFxjKBOp2OoUOH4ufnVzq9vI8RTobyxZ4svmbty9GaNrXanG3BmWbxLYzchVMmOTwHt3TpUkaNGmVWA1WhUBAbG8vSpUud2jmBvU6GMuiIoFDZwGKblPk8pFmYiJMtOOMSrSpFBU7pwqUDHRY4rVbLyZMni2w/efKksS6pwHlUVCdDpcFM1OwYotoRK1dalOoQNd+DasmCc2ZlLZ22+DYO4PBKhkGDBjF48GDOnTtHu3btANizZw8zZ840VsMSOA/hZHAd7Ar0Nc0+UsYfS2k6GWwNUfOdDPe0VCvnDvw5EdTZ0OubYpvbi8MCN3v2bIKDg5kzZ44xbVLNmjUZPXo0I0eOdFrHBAZEVS3Xwa6r7KDF50xKcw6uoJq9d5F991zd/uRG2BALGdcAGTw6CoKalrSrZjgscHK5nDFjxjBmzBjS09MBhHOhFLFHu1xxiFoZjUp76jOYh5KUZm+KklNKc3C5Gh3J6bmAFSfD3WgYh2+0WTdh01g4strwvGoD6L7QaeIG97DYHoSwlQV2eVEro5q4CI46Dcrzo8guJQvu8u1sJAm8VQqqequK7DdacPYKnCTBsZ9h42jIvgEyObR/Cx4bD26eTus3lFDg1qxZw6pVq0hKSkKtVpvtO3DggFM6JjBgj3i5ogVXkZAkibOpmdSr7mOW36xoO8v/m7Upxzm40vKimlayt7TeXHG3OLtdN9qMZNgwEk7+Znheown0WAi1Wjutv6Y47EX99NNPGTRoEEFBQSQmJtKuXTuqVq3K+fPn6dq1a4k6sWjRIsLDw/Hw8CAyMpK9e/fabL969WoaNWqEh4cHzZs3Z+PGjVbbvvnmm8hkMqcXrC4rRE2G0uerfy7w5Ly/GfvT4SL78uPa8rQ6O+PgTP4v4zm40vKiFi40U5i7qeFsOxkkCQ4uh0XtDOImV0KncTBke6mJG5RA4D7//HOWLFnCZ599hkqlYsyYMWzevJkRI0aQlpbmcAdWrlxJbGwsU6ZM4cCBA7Ro0YKYmBhjvdXC7Nq1iz59+jB48GASExPp2bMnPXv25OjRo0Xa/vzzz+zevZuQkBCH++Uq2BfoWwYdcZCKJLmfxp0BYE3CZYv7fz10lYaTNrFyX5Jxm11xcGU9B2eaD86JQ9TCacoLk7+Swep39c4l+OH/YN1/IDcNarY0CFvn8aAsOuR1Jg4LXFJSEu3btwfA09OTjIwMAPr168ePP/7ocAfmzp3L66+/zqBBg2jSpAmLFy/Gy8vLatDwggUL6NKlC6NHj6Zx48ZMnz6dhx56iIULF5q1u3LlCm+99RY//PADbm7OTfxXltg1RBUW3D2hVNgYlgJv/WhIEXb0Snqx5zLP+lu2mA5Rc51qwVmPgQMbKxn0etj3NXz+MJzdAgp3iJ4Kr8VBcDOn9c8WDgtccHAwt27dAgxZffNroV64cMHhZSpqtZqEhASio6MLOiSXEx0dTXx8vMVj4uPjzdoDxMTEmLXX6/X069eP0aNH07Sp8zwy5YFYqlX6KBWOJ9Wxp+hMWS/bKi0nQ34WkfCqRUNEwMSCM32/N8/Bt88awj/UmRAaCf/ZCR3fBcU9+TYdwuFXevzxx/nll19o1aoVgwYN4t1332XNmjXs37/foawjADdu3ECn0xEUFGS2PSgoyOJqCYDk5GSL7ZOTk43PP/74Y5RKJSNGjLCrH3l5eeTl5Rmf54e/uAKiqlbpo7ThWLgXyvpTMZ13c9YQVaeXuHwrByiaRSSfgmwigF4HexZD3HTQ5oCbFzwxBdq9DnLn1oiwB4cFbsmSJcYlWcOGDaNq1ars2rWL7t2788Ybbzi9g46SkJDAggULOHDggN0ZhmfMmMG0adNKuWclw65sIi44RHXBLlnF1hDVGvYUnSnPOThnDVGT03NR6/S4KQoqaRUmX+A8087A0n5weZ9hR91H4dlPoUpdp/SlJDhkm2u1Wj744AMza+mll17i008/5a233kKlcmzCsFq1aigUClJSUsy2p6SkEBwcbPGY4OBgm+3/+ecfUlNTCQsLQ6lUolQquXjxIiNHjiQ8PNziOcePH09aWprxcenSJbv6r9NLnL+eyaW7JnxpYF9VrQqkJi6Im9x5eV+lQpFzZYVWp0dtcjd0lhc1f/6tdqCX1RAalUzHUMU6Om193iBu7n7w7ALo/0u5ihs4KHBKpZJPPvkErdY5C2JVKhWtW7cmLi7OuE2v1xMXF0dUVJTFY6KioszaA2zevNnYvl+/fhw+fJiDBw8aHyEhIYwePZo//vjD4jnd3d3x8/Mze9jD7D9P8fic7SwppUK7IOqilgX2xr45ur0sP5bCgpaj0TllDtBaoRkj1w4zLeUtxritQqHXQMRTMHQ3tB4ILlCjxeEh6hNPPMH27dutWkOOEhsby4ABA2jTpg3t2rVj/vz5ZGVlGRfu9+/fn1q1ajFjxgwA3n77bTp16sScOXPo1q0bK1asYP/+/SxZsgSAqlWrUrVqVbPXcHNzIzg4mIYNGzqlz/k0DPIF4MS10puzq6gWnAt8t+2mJE4Ga5SP/VZU4CQJ8rR6PNzubd7LWppytHnw9yzYMY86ei23JR9OtZzIwz3/41IfvsMC17VrV8aNG8eRI0do3bo13t7mnpXu3bs7dL7evXtz/fp1Jk+eTHJyMi1btmTTpk1GR0JSUhJykyFE+/btWb58OZMmTWLChAlERESwbt06mjUrG7ezKY1qGgTuZHIGer1kdJc7k4oaB1eRcLMZJlL8XJv59vJZyZCrNnwJ3JVy8rSG/3PUunsWOIsW3OX9sH4YXDc4Avd7Pcqbt/owslZHHnYhcYMSCNzQoUMBy/VPZTIZOp3jY//hw4czfPhwi/u2bdtWZFuvXr3o1auX3ef/999/He6TPdSv7oNKISczT8uVOzmEWjPj74GKGgfngl2yiq0hqqPYU7ehNMjWGKaNfD2USLla1Fo92Rodgfd43vxU5XWqehtSGW39EHZ/DpIevKtDtzl8lVCbG7eSXXIk4bBtrtfrrT5KIm4VGTeFnAY1fAA47oRh6q5zN1iw5YzZF8WuyvYu+MWqSNhyMtiz5tRa+zIdot71oHq4KfC8a7XdayycJEnGZVqN8g7B4g4Qv9Agbg++BMP2QpMeJmEirvc9LLuIu0pK45p+HL+Wzolr6cQ0tez5tYf9/97i5S/3ANC2biDt61cDRDaRssCZFhzltNg+X8y8VAq0Oom0HM09h4rcydagz83gfeUKQtdvNmz0qwXPzIcHnjK2c+WaDHYLXE5ODnFxcTzzzDOAIbTCNDhWoVAwffp0PDw8nN9LF6ZxzXt3NGh1eoYvTzQ+v5lZkKFFZBMpfUoUB2ePF7UMbbh8J4OnmwLN3SId9xoqcuvw7/zhPpbashuGDa0HwpPvg4e/Wbv8y+eKUyV2C9y3337Lhg0bjAK3cOFCmjZtiqenIfjv5MmThISE8O6775ZOT12UxjUNISUnrmWU+BynUzKNCQUBMnILwnAq7lItV+yTZdxKyYtalpcg22SImi9wJV7NkHMb/phE/YP/AxmkKIIJ6rsE6nWy2NyVLTi7P9kffviBIUOGmG1bvnw5W7duZevWrcyaNYtVq1Y5vYOuTr7AJd3KJjOvZPGBhy7fMXuenqsx/l9RnQwViZLEwdnTvjzCRLxUCjxV9zAHd3IDLIqEg/9DQsZSbRcWPPCtVXGDgqVurjhVYrfAnT17lubNmxufe3h4mIVvtGvXjuPHjzu3d66OJocq8R/RwNcgSKeSSzZMPZh0x+x5eo6JwNlxV3Rmapz7EVthIo5SXgkv88XMU1XgZHBoDi7rBqweBCtehswUqBrBwroLeV/bn5rVqto81OGMvmWI3QJ3584dszm369evmwX76vV6s/33Bdtmwo55rNG/y1PyfRwv4TD14KU7QIE1aGrB2bLO8r/It7LUVtsIikdZgqVarjsHpzRacHbd+CQJjqwxJKI8thZkCkPGjzd38E9OfcB6mqR88ssG3lNVrVLC7k+2du3aFpNK5nP48GFq167tlE5VGBo+DVUjCNDdYolqHi12v2u4EzpAZp6W06kGYXw0wuA5NZ+Ds35slbv58W+6oMC54GjFKiVxMlij3JZqGS04eUGYSHEWXPo1g8X202DIvglBzeD1OEPONjcP8xg4G1QKC+7pp59m8uTJ5ObmFtmXk5PDtGnT6Natm1M75/KERcKbOzj9wOtoJTkP3omDhW0Nd0Q7v91HLqchSRDi70H9uzF1ZkNUG+ep6mMQuNtZapf8clUUSmuxffnMwSmLH6JKEiT+zzDXdmojyN3gsQnw+lYIaWU8NiXdMCKzlqo8H2PhZxe8q9ntRZ0wYQKrVq2iYcOGDB8+nAceeACAU6dOsXDhQrRaLRMmTCi1jrosbh7InphCjyPhzFH9l0Y5SYY74pE18Mxc8LOdLj1/eNoyLAA/D0Pm4XQbXlSZrEA7A70MAqfVS6TnagjwKt30z5UVUwtOkiSzNFvWViPYly6pDIeoJl5Utcpg9merLTi97iTBr2/Dub8Mz0Megh6LIKiJWbP8NOW+HkoCvGxnxDbLB+di2C1wQUFB7Nq1i//85z+MGzfO+OHJZDKefPJJPv/88yKJKO8X6lbz5oyiPs/mfcDeTkcI3DcfTv8Oi3ZBzAfQqp/VBcgHL90GoEXtAPw8DB+HLQtOLpMZt6mUcnzdlWTkabmZpRYCV0JME15q9ZLTnA5lac/kz7cZ4uDy16KaKI5eD/u/hi1TDRl2lR7QeQI8PMxiht0jlw31VepX9yk2r6LyXgs/lyIOrWSoW7cumzZt4tatW5w9exaABg0aUKVKlVLpXEVBqZDTMMiXI1fS2F17MF0fet6wGPnqAfjlLTj6kyHxX2CdIsceumT4IrUMDcDb3fBxmM7BFfaiKmQy8us7yTAMUzPytNzKUlO/eim9wUqOaTYRrU7CnvXpdqVRKsvF9iZhIkaByx+i3jxn+B5e3Gl4HhZlKLBcrYHV8+04a5hLbl/ftgcVCuLgtDrXE7gSTT5UqVKFdu3a0a5du/te3PJpFHx3RUNyhsHcH7wZnpxuuFOe3wafR8Ge/5p5DZLTcklOz0Uhl9G8tr/JENW6F9X0Ziph4mjIvM882E7EdIiqseXVsQPzObgyXGx/dzjq6abA664XNTcvD3Z9Bl+0N4ibmzd0nQUDN9oUN0mSjALX8a7jyxYWazK4CM6bXb3PKVjRcDcWTqGEDiPgP7ugTgfQZMHvY2DZ03DDYP3mD08fCPLFS6XE9+4QNVutM96FC1tw8kLDhSre7oBrelIrCmZD1EJWiLWfrNXt5Zzw0lOlwMNNQYTsMiP+HQZ/TgJtLtR7DIbGQ+QQKMapciolg+sZeXi6KWhdp/h8JJViJYPANkUELp+q9WHAb/D0bFD5QFK84Y66Yz6Hkm4C0DLUsLYvX+CgYJha2KAoHHRf9a4FdytTCFxJkWEqcEUtOEdSnJlaMWUrcIZ+eyn0tLrwJRtU46mbdxLc/aH7Z9BvncUpEkv8c9pgvUXWq4K7svjxusKFvahC4JxEk5p+yGVw+XYOX++4YL5TLjdUFRoaD/UfB10ebJnC/yUOpKEsiZahAYBhLsj77vAi4+4wtYiToZDC5YeKuIIFV17JHp2JppAVYk9iS7PtVv6/V9JzNVy+bb32R45aS1PZBR6O+z+anvoMlUzHfvdIGLYbHurvkEr/kz88bVD88BQKhviuGKokBM5J+Hu5MeKJCACm/3acZTsvFG0UEAavrIUenyN5+FNfc4ZfVRN5InkpaA0C5ed5dx4ux2DBFT9EdSWBK+8e3DsWLTgHji+tMJHHZm2j48db6fjxX3y/+6L5Tk0uL6V/w3rVe3jfPoFaFcgI9XCmek8qNkypMLkaHXsvGEYWj0TY57WqFCsZBMXz9hMRDH3MsLxl6q/H+T7+36KNZDJo1Zdzvf7iT11rVDId1RLmwZLH4MoB4zA139GQP+Tp0jQYTzcFX7zyUMGpKLDgbmWVv5PB9b7e9mHqDNAUmYMr3lKztsdZ1yNHrTMux7t8O4f31pmsKLq0F/3ijrwq/YxSpkfdsAeHe/zBL/r2xmGrIxy4eJtcjZ4avu48EORj1zH5TmhhwVVyZDIZo2Ma8kanegC8t/4YP+y5aLFtwi0VQzSxLAgcD15VIfUYfPUEQzXf4Y7aOETNF7je7UI5Oi3GmAgzn6r5TgYXmIMr60rujrJq/yX6L91rM+tLtlrLjN9PmG2zt74ulI6T4dz1zKIb1VmwaTx8/RTym2e4LvkzQj8St5e+ReVviEctSTaRv88UeE/tfd+uvJJBCJyTkclkjOvSiNcfMdSDnPjzUX7cm1SknWEFg4ycB3oaUj83+z+Q9PTMXsNG1Xh0/8YDBdHhCpmsSFofszARVxiimv3vel/2MWsO8/fp6zbLPC7aepb/bjffb+lnbj2VufVnJeVsqrnARcmPGRxVuz8HJFLqPU903ixOBnZCJpPZvxbVAjvOXgfgETvCQ/IRKcvvM2QyGROeboxegq93XGD82iPIZdC7bZixTeLdFEktQwPAuxr839fQ7AUy175FffU16u5/FeTxqHTRgPWcZabrUQsvMyprXPAGbpEMkzhDMO/3vdbWKA0L7szdZAw+ZDNO+SOvKOPgNuBXG56dz9bbDUk7foRWdyvPe5RQ4G5m5nHsquH9d7DTwQAmi+1d8AsgLLhSQiaTMalbYwZ1CAdg3NojrN5/CTAMg06nGL60+R5UABo9zf6nN7FC+xhyJNi7hMUZw+kgP1LEuZBPvgWn1UtGx0R54YpWmyVkNtwGuRbmrSxfemtrUZ0/B3cmJZPH5Af5w32sQdwA2rxq8MpHPMmVOzkA1LorcMZAX43eoXmxneduIkmGoPUavvaXHhAW3H2KTCZj8jNN0Oslvo2/yJifDiOXyagd6IlegmA/D4L9zb9IjeqGMlA7hI1SFN9W+x810y7xg2oGqbvPQcgs8AwoOD/grlQY16PeyMrDv5iF0aWJC97ALWKrxkzhDBwOZ/S9h2Mtkn2L55M+pIvKsDj+or4GY7VDWPHMaGMTo8AFGgQuPx8cQK5Wh5fKvp/5jjOOD0/BJA7OBQVOWHCljEwmY2r3przycBiSBKPWHGLeltNAIevtLkF+7lTzUfG3rjlHevzBT0pDCqoaZ1bB5w/Dqd+LHFPl7jA1Nd3ck3rpVjY/7k3ivKVJ6vuYwhaZ6c8yz5IFZ8HiK5OElyd+RVoUSRftX+glGV9pu9JFPZPdevPMH1dum1twHibBufY6GiRJYofRweDYomaxkuE+RyaT8X73ZrwcaRC53edvAYYUSZbaNg0xrGw4nKplnttr9MqbTK5fXci4Bj++xAK3hQRSMFdUv7rBnT/2p8PGSuTpuRoe+WQr49ceYexPh0v5HRoor2VKzkRdKA7O0bfhlJTlmddh1QBY+QqyrFTO6Gvxf+opfKDtRw5Fh46Fh6hyuQwPN8NP2955uHPXs7ialotKKadduGPrywtWMjh0WBEkSSIlvWi+yXvBJQRu0aJFhIeH4+HhQWRkJHv37rXZfvXq1TRq1AgPDw+aN2/Oxo0bjfs0Gg1jx46lefPmeHt7ExISQv/+/bl69Wppvw2byOUyPujRjJfahhq3tagdYLFts1qGZV/Hrqah10vskxpx5vlN0OFtkMnpodjFZvcxtM7cBpLE1GebElbFi6Rb2byweBcnk9O5dKsg6v10StlYcBVlDs7afKZVLDS3+k4ly210eonXvt3HJ5tOWn8dSYLDqw3pw4+vA5mCMw+8wTPqDzkgPWDxEJ1eIjnNIAr5Q1TA4eLP+cPTtuGBZkNce7iXjL6XbmWzcl8Sb/2YSNsPt9B94Q6nhhuVu8CtXLmS2NhYpkyZwoEDB2jRogUxMTGkpqZabL9r1y769OnD4MGDSUxMpGfPnvTs2dOYTj07O5sDBw7w3nvvceDAAdauXcupU6fo3r17Wb4ti8jlMj56rjnDOzfguVa1aBtueSFzs7sW3NEr6cbYIpmbp6Em5WtbOKkPpZosnTdSp8PKVwhTpbPmzSgaBftyPSOPqb8cM4uLS8vRlLjilyOUt9W2/fR1nl7wD0evpNluWHiIaqPfjg67zOfgCp7tOX+TLSdS+XzbOTYdTWboDwnm3tz0q/DjS7D2Nci5BUHNYchWfq3+GnlYz/OXmpGLVi+hlMvMHAOOhooYs4c0cDznliNOhltZan47fJXxa4/w6CdbeeSTrYz96Qi/HrrKjUw16TlasxKa90q5Oxnmzp3L66+/zqBBgwBYvHgxGzZsYOnSpYwbN65I+wULFtClSxdGjzZMsk6fPp3NmzezcOFCFi9ejL+/P5s3bzY7ZuHChbRr146kpCTCwsKKnLMskctljIppaLNNs1oGgTuVnIGXu+GLarQ6arXmWfWHDFWsZ4TbOhQnf4N//6FGzAw+7NmVFxbHc+lWDjcKpU+6cjuHhndTOpUW5W2/DVhqsPxf+3Y/uyc8YbWdLS9qYW5lqXFXFrUD7MoHZ0KeydD3zf8lAFDD14OpzzaBA98Zsn7kpRvSh3caCx3fAYUbV3ccMvRZZvnc+fNvwf4eZqFEjpQO1Oj0xJ/LX57lmIMBbAtcVp6Wvf/eYtfZG+w8e7NIGI5CLqNlaAAdGlSjQ/2qtAoLRGXhepeUchU4tVpNQkIC48ePN26Ty+VER0cTHx9v8Zj4+HhiY2PNtsXExLBu3Tqrr5OWloZMJiMgIMDi/ry8PLOKYOnp9xYLda/UDvTEz0NJeq4Wdbbhh2H65dWgZIHuBdLDY5ii/xyuHYT1Q2kc9hi1eI70nJpFVjZcuZNd6gLnKnFQd3JsBz0XdTI4r9/W5uAUFobFqUmn4PsJhnyBALVaG9KH12hsbJN/owoN9DKmETel8PxbPsbKWnZYcIlJd8hS66jiraLJ3aw4jmC6kkGj03Po0h12nL3BrrM3Sbx0u8jyt0bBvrSvX42OEVVpV7cqPu6lJ0PlKnA3btxAp9MVSXUeFBTEyZOW5yqSk5Mttk9OTrbYPjc3l7Fjx9KnTx/8/Cx/eDNmzGDatGkleAelg0wmo1ktf3bdvatCwXo/U66414e+cRC/ELZ+hFfSNv5w381MbR9S0mqZt717py9NXETfKG6k5GgotKUpO6vZRKx4UU3PIUNPP8Vmxt9YCTdyDUlRH58EDw8Fufn81/WMuwJXxdO2wAUWErj8wjN2WHD5828dGlQrkq3GHvJvvgkXb9Pq/c1FpkNqBXjSsUE12jeoSvv61aju6+7wa5SUch+iliYajYYXX3wRSZL44osvrLYbP368mVWYnp5OaGio1fZlQauwADOBszoxrlAahjONuiGtH4bPpT184PYNp44msEU2gH+lmgBcvmMucIlJt1HIZTxoxdFRIlxE4Irrh+M+BgfWopr+b/IkP5FmXdk1PnZbQjv5KcOOOh0M+dqq1rd4PqPABXoBBd8HvV5CLpcZb1y1i1hwhp+2PXNw+emRHnFg9YIppjffzDwtgV5utG9QjQ71q9GhQVXCqniV2wqbchW4atWqoVAoSElJMduekpJCcHCwxWOCg4Ptap8vbhcvXuSvv/6yar0BuLu74+5edncVe2hTpwpwzvjcdIiqUshR6/Tm6aSrRSAbtIkPp43kHX6gYd5hNqnG8Z1nP2beeZzLJhZcrkbHK1/tQa3T89fIxwgtpiycvbiKF7W4fuTfLG5k5rH7/E00Wsf7bT2jr+Uhal5eHkMUvxKrXIOHTEOm5MFMbR+mD5iDTG7Za6nTS8Y1xoU/I61eQiWXGS24kMICdzdMpLjiz2nZGg7drexmT3pyS5jefJvX8mfdsA5WlxaWNeXqRVWpVLRu3Zq4uDjjNr1eT1xcHFFRURaPiYqKMmsPsHnzZrP2+eJ25swZtmzZQtWqxRfOcDUeCjP3sJp+ibaNfozP+rSib2ShDK1yORs9nyVG/TE7dE3xkGkYkruUtaopyK8XDPnTczVkqXVodBKLtp51Wp/Nh2flR3FD5fwr+dznOxm+PJGllnL3mbZ34LdqMeFlyjEit77EBLcf8ZBp+FvXnJi8j/mf7kmuZ2osnMXA7Ww1Or2ETAYhAebxb/kT+leLG6IWY8HFn7+BXoL61b2LiGRJeO+ZJi4jbuACYSKxsbF8+eWXfPvtt5w4cYL//Oc/ZGVlGb2q/fv3N3NCvP3222zatIk5c+Zw8uRJpk6dyv79+xk+fDhgELf/+7//Y//+/fzwww/odDqSk5NJTk5GrS7/jBv24u/lZpaPy/RLExLgybMtQix+kfw93bgs1eAVzQTGaF5HrfSlpfwcc2+/Bds/AZ2GXJNycmsSLpvFzN0LrmG/2dGPu4p16ZZz5yW/+uc8g77ZZ3yuUefCto/hv50IvHOUNMmL0Zoh9NeM4wqGcIyvd16g+8IdRVPdUzA8reKlwrvQciutXo8kSUVWMeRjHKIWY8H9c3f1gr3JLS3RKiyQetW8ebVDXdrVda0iVOUucL1792b27NlMnjyZli1bcvDgQTZt2mR0JCQlJXHt2jVj+/bt27N8+XKWLFlCixYtWLNmDevWraNZs2YAXLlyhV9++YXLly/TsmVLatasaXzs2rWrXN5jSWldp+DLYu9d0d8zfy2qjFW6zpx/MY7NuodwQwtbP4QlndFfPWhsr9VLdJn/N9lqLSnpucYhliRJfPDbcYcsPFsBmiNXHWLCz0es7r96J4dhPxwg4eItu1+vuH7M33Ka/24/V2S/w04Gi69RdNsHGwryyDWTnefJf3rDto9AryGpemeezJvFat1jZmf87/bzHL6cxozfizrV8gWuuq+7MUNIPlqdRFqOwRIHS0NU+7yo+fFvJQkPycff042/Rj3G5GebFN+4jHEJJ8Pw4cONFlhhtm3bVmRbr1696NWrl8X24eHhLp940V7a1Ak05pKzN/q+cBXy0Dr16aIZSXddPHP9fkCZcoQ6a59htLIbi/QvkK13I0uto8nkPwAY1rk+o2MacexqOl/drS3xxqP1zGqHWsPaBHtKei4/HbgMwISnG1sMC3hn5UH2XrjFhiPX+HdmN7veq61+pKTnMn/LGQAGdahrFlvl6Hy3I98md9S8o/yJ1xUbUGbrDclMn57FppstSL10yupxaTlFh6r5ISLVfd2L9EGrl0jNMATEVvNRFRFAT9XdpVo2LLikm9lcvJmNUi4jsl7Fm8axh3K34ATWMS3ZZu+0RoEFB94qxd07uYxf9O2ZVf87aPocMknHMOUvbHSfwEOy02bHL9pqsHhMs8iaFqLOJ78OpynWQiRM4+N0VhYsFk7qeC9IkvncU2GngyNeUbAc32fJkdFadoqNqvH8R/krSpmevT6d7yYzfYHcYhwZ7hZuIEYLzscdjdZ8jaxOb314ChgziNiag/vnbnLLh8ICSzUWrTwRAufC1KnqRUzTIDo0qGrM+1YcpgJX1cfcM7z+jAZ6LePII5+TKgUQLl1hjWoak5Xf4Yn58phVd3PXAVwvtCpi0dazNJn8B//cjZ/Kx54gV2vBwIV/wM6k8Es6OgdeXFzdsX+v8e//hrNa9T715ddIlQIYon6XmV6jDclMgTyt7aGipej9fIGr5utOYKHPX6vXW42Bg4Kkl4W9qLkaHTvP3mD2H6f4YpvhZlZS72lFoHLKdiVBJpPx335tHDrGNB9c4YDK5PRcctQ6Ltd4nL55nzDPfxVP5G3hVeUmouUJjNUOIV7flN3nb7LzbEHM1VPz/ubAe08aRXbWH4ah1shVh9g7MbrgBcyyiRQ8MQ0etZa3v3AWj3vF1EorLKqODlEtLiK/u+n20T/xXTWUMPl1kMEqbSc+0PYlHR9CTW4MhdMwFV565aYo2qn8G0t1H3ceCgtgdExD47Vff/Aqd7INTrMQ/6ICZ7oWNUet47fDV1l74AoJF2+bXWulXEZMU8shWZUBIXCVDFMLrk4VryI/5tUJl/B0U5COD19XG823F1ozw+0rwuTX+VH1Icu1jxN/LIjC/Lg3iWGdG5htS80wt+wshkhgPn9ozYJzROAkyTDBHuBln1V7r2nKLPVZoc6AX94n8MC3BMrhslSNCZrB/K1vYWxzPSPPmEY+t5AFF1bFi4s3C7zXbgo5x6+m06CGj9GaM3UyyGQyhnVuwJK/z5OWo2HWH6eMRb8tWXD5WX0TLt4m8qMtpJtMMwT7eRBVvypR9arSMaKaU8JDXBUhcJUMM4Gr6l1k/+T1x4zWgre7ks36FjyV9wljlSvor9zMy8q/uJl4mMPyQWzVtzIe92ncGZrX8ufRB6yHE1jLB2eqsXorOuaIX+ijjSf48p8LfNH3Ibo2r2mxjamwF7XgHJ2DM3/eWZ5Ik3XvgNqQ8eY77ZN8rH2JLMyFIlejZ9mufzl8Oa3IXFhooLnAbT2Vyp/HU3iySRDDOjcgNNDTTODyUZpYw/lBwA1qFC3vlz9EvWUMFPakT7swujQNpm4173Kt3VGWCIGrZAR4Flg14dXMo9/b1AnkTGqm0WPn52EQwyw8mawdxAbdw8x0W0JdUvhGNYu1uo68r+nHHXzJ0+rpv3Qv7/doavW1rTkWzPKiOcHD/eU/Bu/uhxtPWBU4U/R6yVhnFhwfouYTQAbvuX3PC4odoAaq1CO+6VQmb7b+M5r263GL2wuvTMhfkL75eAqbjxtW6uRrmanAFQ4XahjkS4f6RefQHqoTQNMQP2r6e9L34TA6RVQv0TrTio4QuEpGYQvO9E79fo9mNKjhw9ZTqew8e4OXI8OM4RsAe6TGdFXPJFa5hsGKjTyv2MEj8sNM1gzid30kYLAATbmVpeZOtpp61X2srmQwnY8rj+LAegk+uxsyAo57UQG6yPcy3W0p1WXp6CQZX+ueZp/Xm4TlVANsr4SwRO1Cw0q5DJrXDjAum8rvNxjm4PIpLHDvREdYFK4avh5sGPGIw/2qbAiBq2SYZmMNr2puJcjlBm9dTNNgqxPLubjzkbYvG3WRfOL2Xx6QX+EL1QJ+17VlsmYQ1wkwa/9/X+zi0u1s/hnzuPlQ0KrYlfSdFcVeS0wvFWS9deQ4gGqk8b7bNzytMOSaO62vxRjNGxyUGsDZDMOjBJjeiAC+HtCWprX8aPdhnM22nibxbg1q+FRqB4EzEGEilYzwql40CvalY4NqRSbhHbFcDkoN6KGdwafanmgkBV0V+9jsPprn5X9jKlnnb2Sh0Ulcvp1tV8EVZ1Y/t/V+9IWsRlMrx76rINFTvoPN7qN5WrEXjaTgU21PnlF/ZBA3HA83McXXo8C28FIp6NyoBjV8PXiyibmDx8NNbtZ30xUHa96Mui+HnY4gLLhKhlIhZ+OIRyxaKYVLFBZHveAqzL36Ir/rIpnl9l+ayf9lrmoxz+rimaB5jWsURL8XjrcydSaYaprOmpehBNiyxExHwnoJTKMwirPggrnJR25f87jiIABH9eGM0QzhuBRu1q55LX8OXS4mPfpd+kfVIStPZ5wSMBU400y4C15qyadxZ3FXylmTcJkuzcwttBfbhvJt/EUAu73I9zNC4Cohhe/q/4zpTJ5WV2RYZI3wql48VCeQ1nUCmfjzUU5Ideipfp8hig28rfyJzopD/Ckfwwzty/yo64yEnByNjr9OFtTRMAv6NfnfyeFuVjGz4CRzC876sjeJPoq/GK9cjp8shzxJyQLt8yzRPYPWwk+ldhUvuwWuT7swrqXlmAhcwWdh2lcvlZJxXRsB8O6TRQvNNA3xZ/lrkXhV0pUHzkZcpfsAe/K91fB1N8a1vfJwHV57pB4bjxQkOdCi5HNdD/7Qt+ETtyW0lp/hI7eveVYez1jt6+SodUz5pcABYW0kWlxhEkdS7dhqKRUSOEspw00JlaUwU/kVHRSG93BA34AxmiGclWpbPcbPw/6fj0Iuw9OtoL3p0ihHC9u0L2FiyvsRMQcnAMytmrrVDPFzXhbKx52TatFLPYVpmn5kS+5EKY7zh2osR9Z8hJwC88zsJ2s2RLX9Y7YU0V8SzIaoelCanPenA1eM/ZCjZ5Did/5QjaOD4hg5korpmlf4P/VUm+IGBpGKsnORulxmfj193JWM6WIoPvTJ/7WwdpjgHhEWnAAwX0mQX9WrqrflLMd65Hyj68oW/UN8HfgdD2Qn8p7b/+im2M1ozRuck2qx+XgKz32+k5VDohyKg3OzI2tJPraCVYsMUU3anriWjkanp77sCh+7fUkbuSHhQLyuCeO0r3FRss8z6ePuxrevtuNGZh5VvFWs2n+JpTsu8O/dAF7T5VhymcxM4Pw83Bj6WAN6twktsmZY4DyEBScACiLewTBcBUMB6t5tQhnxeAMOTX6qyDGXpCC+rruA8ZrBZEiePCQ/y0bVeIYq1qFES2LSHZ7+9B/zbCLFWHAqBwTOFqa+DL0kmQ19FeiQ7ZzHRtUE2shPkyF5MlHzKi9rJtgtbgDe7gpUSjkhAZ54uCnoHxXOt6+2I8Tfg8nPNCHSJPmjXCYzS2nkfbccpBC30kVYcALAMHzK94TmW0YymYyP/+9Bq8e88FBtPN2VLNM9wTZdSz6863kc47aKboo9jNa8wfHUcPb9e9t4THGlBR2y4GzsM32dOZtPs+GwYT6xkSyJT9z+i/u2CyCDbboWTNAM5irFz2tN6taYQ5fT+PXQVQDyLGRAqVPVm13jDTVZvVQKdp83JPBUyGXU9PegaYgfQX4eduXXE9w7QuAEALzWsS5yuYxudix9yif2qQf4Lv5fAK5RlVc1o+mp28kUt+9oKr/IetV7LNY9y6gfNYDBa6i1kg8uHzel8+O6Nhy+hhtahinXMUyxHjeZjjTJi/c1/flJ/wj2Rsa99kg9AKPA5WfzsMbTD9bky3/Oo1IqCPY3iNpvb3W8b9aBugLiNiIAoIq3ineiHyAiyHpx6AFR5kVuijoEZKzTd+TJvFls0LXDTabjLeU6flNNoKXMkPrcmRacLV0yfZ0HZef4VTWRd5RrcZPp2KRrS3TeLH7SP2r7JCZsH/2Y8f/HG9UA4MU2tktL+nm4ETfyMTaO6Gh8X0LcyhZhwd3nvBv9AFtPpfJi2+LrwE7t3pS+D9fhqXl/A4b5Mkt6dQN/hmne4VedYf3mA/Ir/KSawlJdVyR18yLtTUM6HJ2D0+j0ZKuLxvjtvXALd9S8q1zD64oNKGQSNyQ/JmsGslEfiT3C1ijYl3XDOpCRqzVb8P5l/zbcyVbbPX8mRK38EAJ3n/N2dARvR0fY1VYmk1HHZH2rl0pps/7FJn07duc1NmbgeF25kawNR0nRzKNqs8eN81CmHtyTyRnM23zaYpBrkf4AMfP+5vyNLPaZJt4ENm38md9VS6gnTwZgna490zT9uY31+rj5fNH3IerX8CHY3wMPN0WRegcKuUw4ByoIQuAEDuGuVLBr3ONIGBbuF9a3aj4qbmQWzE3dwZeRmqH8qoviI7evCclKwvvnF1i7PoYm/edxJk1Gp4bmOeYWxJ2xS+DAsBYWYOfd6lBe5DJGuYL+is3IZRLJUiATNa8Sp29d7LkaBfuy6Z1H7XpdQcVAzMEJHCYkwNNY6KRw1Ef96kWTLwJs07fiqbxP+EFr8DA+r/8D328eYc3KZay4WznMFEtFbQpjOvT79K8zdJAf4Q/VWAYq/0Quk1ihfYyn8j6xKm6dCwlruoXKVoKKjRA4wT0RWc+80O+EpxtbbZuJFxO1g+mjnshFfQ1qyW7yrepjqmx+B3/Mq2rdzCy+SHe+vPmRxZDb8/hBNYNQ+XUuS9V4RT2ecdohpFM0qzHAjrGd+WZQO8599DRjuzRCKZeJFQWVEJlUWYqIOpH09HT8/f1JS0vDz6/4OZv7GUmS+ONYMk1D/An0VuHjriR83AYAHqztT2p6HsnpuUWO8ySXkcrVvKrYhFwmcV3yZ5LmVf7QtwXg56HtaRUWWOQ4wHh+gCfkCXzotpRgmSHWbpn2KT7RvkQ2RTOnfNW/DW3Dq5gV5sknT6vDXVl0aZrANbH3Nyrm4AT3hEwmo0sz89i5j55rzvwtp/noueY0q+VPRq6Go1fSWbX/Et1bhDBo2T5y8OADbb+7iTWX0EB+lf+q5vGbLpIpmoHFWnCBpDPF7Tt6KnYBcF4fzFjNEPZJjYq0Xf5aZLEL1IW4VU5cYoi6aNEiwsPD8fDwIDIykr1799psv3r1aho1aoSHhwfNmzdn48aNZvslSWLy5MnUrFkTT09PoqOjOXPmjJWzCZzNy5Fh7J0YbVzT6uvhRlT9qszr3ZLOjWpw6oMuxrYHpAfopv6IhdoeaCU5zyj2sNl9NBuWLyArt+ic2J2sPLrJd7PZfQw9FbvQSTIWa5+hq3qmRXGr4q0S2TfuY8pd4FauXElsbCxTpkzhwIEDtGjRgpiYGFJTUy2237VrF3369GHw4MEkJibSs2dPevbsydGjR41tPvnkEz799FMWL17Mnj178Pb2JiYmhtzcokMlQdnjrlTgblLoOA8VC6SX6KGeznF9HarIMpmnXMTZT59Be7ugZoQuPZndM7uxSPUp1WTpnNLX5jn1+8zUvkweBckf+7QLNVb/+rzvQ2X3xgQuR7nPwUVGRtK2bVsWLlwIgF6vJzQ0lLfeeotx48YVad+7d2+ysrL47bffjNsefvhhWrZsyeLFi5EkiZCQEEaOHMmoUaMASEtLIygoiGXLlvHSSy8V2ycxB1f6JKfl8s+Z6/xy6CpvPR5B6zqBjFp9iF8TL/KG4jdGKNfiLtOSLnnykbYvgzs1pMbOqfjLstBICj7X9WCRtge/vPM4O87cQCaTMf234/h7unFoylPkaXUkp+VaLJ0oqPhUiDk4tVpNQkIC48ePN26Ty+VER0cTHx9v8Zj4+HhiY2PNtsXExLBu3ToALly4QHJyMtHRBYGf/v7+REZGEh8fb1Hg8vLyyMsrKGKcnp5+L29LYAfB/h70ahNKL5PlTnNfbEF1X3eW7+/FnzltmOX2X1rKzzHT7SvYBcjgiD6cMZo3OCHV4fxHTyOXy2gUbPiCN67pS0QNw1Izd6VCiJugfAXuxo0b6HQ6goLMC20EBQVx8uRJi8ckJydbbJ+cnGzcn7/NWpvCzJgxg2nTppXoPQich0wmY8LTjY2hJtPWR/Hr3sWMUq5GjsR37i/RrNd7rAqtgpdKWSQ1e3sL9UEF9zfCiwqMHz/ezCpMT08nNLT4tZmC0mVKjwehx+ekXBuHQpJ4PaRO8QcJBCaUq8BVq1YNhUJBSkqK2faUlBSCgy0nHgwODrbZPv9vSkoKNWvWNGvTsmVLi+d0d3fH3V2sLXRVgmqGlXcXBBWUcvWiqlQqWrduTVxcQbFbvV5PXFwcUVFRFo+Jiooyaw+wefNmY/u6desSHBxs1iY9PZ09e/ZYPadAIKikSOXMihUrJHd3d2nZsmXS8ePHpSFDhkgBAQFScnKyJEmS1K9fP2ncuHHG9jt37pSUSqU0e/Zs6cSJE9KUKVMkNzc36ciRI8Y2M2fOlAICAqT169dLhw8flnr06CHVrVtXysnJsatPaWlpEiClpaU5980KBAKnYO9vtNzn4Hr37s3169eZPHkyycnJtGzZkk2bNhmdBElJScjlBYZm+/btWb58OZMmTWLChAlERESwbt06mjVrZmwzZswYsrKyGDJkCHfu3KFjx45s2rQJDw/HCh8LBIKKTbnHwbkiIg5OIHBt7P2NlvtKBoFAICgthMAJBIJKixA4gUBQaSl3J4Mrkj8tKZZsCQSuSf5vszgXghA4C2RkZACI1QwCgYuTkZGBv7+/1f3Ci2oBvV7P1atX8fX1LdeSb/lLxi5dunRfe3PFdTAgrkMBkiSRkZFBSEiIWRhZYYQFZwG5XE7t2rXLuxtG/Pz87vsvNIjrkI+4DgZsWW75CCeDQCCotAiBEwgElRYhcC6Mu7s7U6ZMue8znYjrYEBcB8cRTgaBQFBpERacQCCotAiBEwgElRYhcAKBoNIiBE4gEFRahMC5ELdu3aJv3774+fkREBDA4MGDyczMtHnMkiVLeOyxx/Dz80Mmk3Hnzp2y6ayTWbRoEeHh4Xh4eBAZGcnevXtttl+9ejWNGjXCw8OD5s2bs3HjxjLqaeniyHU4duwYL7zwAuHh4chkMubPn192Ha0gCIFzIfr27cuxY8fYvHkzv/32G3///TdDhgyxeUx2djZdunRhwoQJZdRL57Ny5UpiY2OZMmUKBw4coEWLFsTExJCammqx/a5du+jTpw+DBw8mMTGRnj170rNnT44ePVrGPXcujl6H7Oxs6tWrx8yZM60WabrvKd3M6QJ7OX78uARI+/btM277/fffJZlMJl25cqXY47du3SoB0u3bt0uxl6VDu3btpGHDhhmf63Q6KSQkRJoxY4bF9i+++KLUrVs3s22RkZHSG2+8Uar9LG0cvQ6m1KlTR5o3b14p9q5iIiw4FyE+Pp6AgADatGlj3BYdHY1cLmfPnj3l2LPSRa1Wk5CQQHR0tHGbXC4nOjqa+Ph4i8fEx8ebtQeIiYmx2r4iUJLrICgeIXAuQnJyMjVq1DDbplQqqVKlCsnJyeXUq9Lnxo0b6HQ6Y5GhfIKCgqy+7+TkZIfaVwRKch0ExSMErpQZN24cMpnM5uPkyZPl3U2BoFIi0iWVMiNHjmTgwIE229SrV4/g4OAik8larZZbt25V6gnkatWqoVAoSElJMduekpJi9X0HBwc71L4iUJLrICgeYcGVMtWrV6dRo0Y2HyqViqioKO7cuUNCQoLx2L/++gu9Xk9kZGQ5voPSRaVS0bp1a+Li4ozb9Ho9cXFxREVFWTwmKirKrD3A5s2brbavCJTkOgjsoLy9HIICunTpIrVq1Uras2ePtGPHDikiIkLq06ePcf/ly5elhg0bSnv27DFuu3btmpSYmCh9+eWXEiD9/fffUmJionTz5s3yeAslYsWKFZK7u7u0bNky6fjx49KQIUOkgIAAKTk5WZIkSerXr580btw4Y/udO3dKSqVSmj17tnTixAlpypQpkpubm3TkyJHyegtOwdHrkJeXJyUmJkqJiYlSzZo1pVGjRkmJiYnSmTNnyustuBxC4FyImzdvSn369JF8fHwkPz8/adCgQVJGRoZx/4ULFyRA2rp1q3HblClTJKDI45tvvin7N3APfPbZZ1JYWJikUqmkdu3aSbt37zbu69SpkzRgwACz9qtWrZIeeOABSaVSSU2bNpU2bNhQxj0uHRy5Dvnfh8KPTp06lX3HXRSRLkkgEFRaxBycQCCotAiBEwgElRYhcAKBoNIiBE4gEFRahMAJBIJKixA4gUBQaRECJxAIKi1C4AQCQaVFCJygwjBw4ECL2Vi6dOlS3l0TuCgim4igQtGlSxe++eYbs23WKr1rNBrc3NzMtqnValQqlcOvW9LjBOWLsOAEFQp3d3eCg4PNHoGBgQDIZDK++OILunfvjre3Nx9++CFTp06lZcuWfPXVV9StWxcPDw8AkpKS6NGjBz4+Pvj5+fHiiy+apSqydpygYiEETlCpmDp1Ks899xxHjhzh1VdfBeDs2bP89NNPrF27loMHD6LX6+nRowe3bt1i+/btbN68mfPnz9O7d2+zcxU+TlDxEENUQYXit99+w8fHx2zbhAkTjFXFXn75ZQYNGmS2X61W891331G9enXAkDvuyJEjXLhwgdDQUAC+++47mjZtyr59+2jbtq3F4wQVDyFwggpF586d+eKLL8y2ValSxfi/adGefOrUqWMmUidOnCA0NNQobgBNmjQhICCAEydOGAWu8HGCiocQOEGFwtvbmwYNGtjcb882e19LULERc3CC+47GjRtz6dIlLl26ZNx2/Phx7ty5Q5MmTcqxZwJnIyw4QYUiLy+vSBk9pVJJtWrV7D5HdHQ0zZs3p2/fvsyfPx+tVsvQoUPp1KmTxSGuoOIiLDhBhWLTpk3UrFnT7NGxY0eHziGTyVi/fj2BgYE8+uijREdHU69ePVauXFlKvRaUFyJluUAgqLQIC04gEFRahMAJBIJKixA4gUBQaRECJxAIKi1C4AQCQaVFCJxAIKi0CIETCASVFiFwAoGg0iIETiAQVFqEwAkEgkqLEDiBQFBpEQInEAgqLf8POUr0vKbhRWsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020681, + "end_time": "2024-02-29T20:54:34.975237", + "exception": false, + "start_time": "2024-02-29T20:54:34.954556", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1821.816751, + "end_time": "2024-02-29T20:54:37.716390", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tab_ddpm_concat/3/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tab_ddpm_concat/3/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 1, + "path": "eval/contraceptive/tab_ddpm_concat/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T20:24:15.899639", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/model.pt b/contraceptive/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..a367a1207bdd197cd305fe6194407487a04cc9d9 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec4624dc56aa9865a93acbdcdeae70f85f9456a946fb2e5ed9cd8b5dc9f4c19 +size 45181003 diff --git a/contraceptive/tab_ddpm_concat/params.json b/contraceptive/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..d71ed91584dff9f6f27f24f1c2580277efebd435 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mse", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.74, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "tab_ddpm_concat", "mse_mag": false, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "tanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "softsign", "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tvae/eval.csv b/contraceptive/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..2319da723270496e5373283267bd47809805fb07 --- /dev/null +++ b/contraceptive/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.014026033173837974,,0.0012284577070317652,2.706435203552246,0.03164428099989891,0.6164292693138123,0.0400746688246727,9.105955882660055e-07,3.2376320362091064,0.02793470025062561,0.06429528445005417,0.03504936024546623,0.057808149605989456,0.011085247620940208,5.9440672397613525 diff --git a/contraceptive/tvae/history.csv b/contraceptive/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..15834c75fee1a336628ceadcd9cceb31f7e973aa --- /dev/null +++ b/contraceptive/tvae/history.csv @@ -0,0 +1,20 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.02241888580356317,1.4905488636076916,0.0030782962097319457,0.0,0.0,0.0,0.0,0.0,0.02241888580356317,320,160,142.862961769104,0.8928935110569001,0.44644675552845003,0.0961261961127093,0.007483271204910125,7.415935450342414,9.700103195409149e-05,0.0,0.0,0.0,0.0,0.0,0.007483271204910125,80,40,32.79719305038452,0.8199298262596131,0.40996491312980654,0.030276008496821306 +1,0.004170822119840522,2.427984166694133,5.6218089488430103e-05,0.0,0.0,0.0,0.0,0.0,0.004170822119840522,320,160,140.515380859375,0.8782211303710937,0.43911056518554686,0.06761386157022571,0.0027781161175880697,5.97971810359972,8.342038965802879e-06,0.0,0.0,0.0,0.0,0.0,0.0027781161175880697,80,40,32.894118309020996,0.8223529577255249,0.41117647886276243,0.024398993137219806 +2,0.0032142372105653295,3.1775326946756253,9.600223262965901e-06,0.0,0.0,0.0,0.0,0.0,0.0032142372105653295,320,160,144.2568175792694,0.9016051098704339,0.4508025549352169,0.06473975269825587,0.002925431027142622,6.108939102519116,8.98504544019768e-06,0.0,0.0,0.0,0.0,0.0,0.002925431027142622,80,40,35.61764717102051,0.8904411792755127,0.44522058963775635,0.028732989538184484 +3,0.003577234379551553,2.9195899648459376,4.348199776086897e-05,0.0,0.0,0.0,0.0,0.0,0.003577234379551553,320,160,150.63309359550476,0.9414568349719048,0.4707284174859524,0.06363038770923594,0.0032030581480285035,5.47701075857707,1.4309088606778708e-05,0.0,0.0,0.0,0.0,0.0,0.0032030581480285035,80,40,32.49244570732117,0.8123111426830292,0.4061555713415146,0.021739204511686695 +4,0.002611448067193578,1.8328958396290929,8.810737564255572e-06,0.0,0.0,0.0,0.0,0.0,0.002611448067193578,320,160,143.45851230621338,0.8966157019138337,0.44830785095691683,0.07633217830557441,0.0030096135813437288,5.497269465320635,8.17212617150176e-06,0.0,0.0,0.0,0.0,0.0,0.0030096135813437288,80,40,34.19865131378174,0.8549662828445435,0.42748314142227173,0.01728544359702937 +5,0.002066187719401569,1.418562725057735,4.818185051591941e-06,0.0,0.0,0.0,0.0,0.0,0.002066187719401569,320,160,141.1322615146637,0.8820766344666481,0.44103831723332404,0.07070515162549781,0.002357064618456661,3.038762490750969,4.724545272427605e-06,0.0,0.0,0.0,0.0,0.0,0.002357064618456661,80,40,32.73992657661438,0.8184981644153595,0.40924908220767975,0.019966062564344612 +6,0.0018150892569863686,1.9370867185192977,4.895915542963466e-06,0.0,0.0,0.0,0.0,0.0,0.0018150892569863686,320,160,142.5343050956726,0.8908394068479538,0.4454197034239769,0.06771241171363726,0.002098434802974225,2.5296149099483842,6.119135131865683e-06,0.0,0.0,0.0,0.0,0.0,0.002098434802974225,80,40,33.130537033081055,0.8282634258270264,0.4141317129135132,0.036213114765996576 +7,0.0017754018189464205,1.0608564720709155,4.110462002784865e-06,0.0,0.0,0.0,0.0,0.0,0.0017754018189464205,320,160,150.93800163269043,0.9433625102043152,0.4716812551021576,0.07812782935689029,0.002651070246429299,5.481469099184153,8.274616622792885e-06,0.0,0.0,0.0,0.0,0.0,0.002651070246429299,80,40,36.68884253501892,0.917221063375473,0.4586105316877365,0.0201021930330171 +8,0.0016320147636861293,1.572569604070008,3.280935549836465e-06,0.0,0.0,0.0,0.0,0.0,0.0016320147636861293,320,160,152.10332083702087,0.9506457552313805,0.47532287761569025,0.07706006977591642,0.0021084082123252303,4.821968620683037,7.140439676618648e-06,0.0,0.0,0.0,0.0,0.0,0.0021084082123252303,80,40,35.41457486152649,0.8853643715381623,0.44268218576908114,0.032409553838078864 +9,0.0014390503529739362,1.1130779689192227,1.9856122689985296e-06,0.0,0.0,0.0,0.0,0.0,0.0014390503529739362,320,160,148.04402089118958,0.9252751305699348,0.4626375652849674,0.07865846673303167,0.002113264991157848,2.781704166371675,4.972169583404757e-06,0.0,0.0,0.0,0.0,0.0,0.002113264991157848,80,40,33.858819007873535,0.8464704751968384,0.4232352375984192,0.02809684935346013 +10,0.001374225791067829,1.163778030275184,1.7583196497888975e-06,0.0,0.0,0.0,0.0,0.0,0.001374225791067829,320,160,145.61796760559082,0.9101122975349426,0.4550561487674713,0.072609595393169,0.0023332797987222877,2.608034198338737,1.0012352827615257e-05,0.0,0.0,0.0,0.0,0.0,0.0023332797987222877,80,40,33.24501919746399,0.8311254799365997,0.41556273996829984,0.03622158533107722 +11,0.0013136249404567478,1.105370874132261,2.0836581030414523e-06,0.0,0.0,0.0,0.0,0.0,0.0013136249404567478,320,160,144.06322360038757,0.9003951475024223,0.45019757375121117,0.07791482849102067,0.0020939761153385915,7.381703513306002,3.89069975454473e-06,0.0,0.0,0.0,0.0,0.0,0.0020939761153385915,80,40,33.347615242004395,0.8336903810501098,0.4168451905250549,0.01896329457867978 +12,0.0013007374736943688,0.8016777972321465,1.6661171600203944e-06,0.0,0.0,0.0,0.0,0.0,0.0013007374736943688,320,160,143.7109453678131,0.8981934085488319,0.44909670427441595,0.07403813572964282,0.0021091806715048734,2.195618169948898,6.716770201746968e-06,0.0,0.0,0.0,0.0,0.0,0.0021091806715048734,80,40,33.3885440826416,0.8347136020660401,0.41735680103302003,0.03174531738768564 +13,0.0011258274745216568,0.9933245053406304,1.2559000061217402e-06,0.0,0.0,0.0,0.0,0.0,0.0011258274745216568,320,160,143.82260847091675,0.8988913029432297,0.44944565147161486,0.07367398725546082,0.002973305231353152,2.332661612354639,1.9470425126388857e-05,0.0,0.0,0.0,0.0,0.0,0.002973305231353152,80,40,33.963603019714355,0.8490900754928589,0.42454503774642943,0.035911593766650186 +14,0.0010081856245165,1.8124559902420032,1.1379342504010126e-06,0.0,0.0,0.0,0.0,0.0,0.0010081856245165,320,160,144.61974716186523,0.9038734197616577,0.45193670988082885,0.07099440268893886,0.002199153335527626,2.3161073656544886,7.577638564465472e-06,0.0,0.0,0.0,0.0,0.0,0.002199153335527626,80,40,33.57773303985596,0.8394433259963989,0.41972166299819946,0.02861409220568021 +15,0.0010586415690795547,1.0111776571605908,1.3155730025755604e-06,0.0,0.0,0.0,0.0,0.0,0.0010586415690795547,320,160,143.99448657035828,0.8999655410647392,0.4499827705323696,0.07437738951684877,0.0024275374956232556,2.65694752669535,1.2085388890881177e-05,0.0,0.0,0.0,0.0,0.0,0.0024275374956232556,80,40,33.66980719566345,0.8417451798915863,0.42087258994579313,0.034823847954976374 +16,0.0008538674877796026,1.5765032050085541,5.374222879224455e-07,0.0,0.0,0.0,0.0,0.0,0.0008538674877796026,320,160,143.34916639328003,0.8959322899580002,0.4479661449790001,0.08046830528701321,0.0022838078687641428,2.0230109165978774,9.47888851559331e-06,0.0,0.0,0.0,0.0,0.0,0.0022838078687641428,80,40,33.02995800971985,0.8257489502429962,0.4128744751214981,0.030621698120376094 +17,0.0008248503026644372,0.35676954403032646,7.142933498651121e-07,0.0,0.0,0.0,0.0,0.0,0.0008248503026644372,320,160,143.77559542655945,0.8985974714159966,0.4492987357079983,0.08080137882643612,0.0023548384517198427,4.805163549015765,1.1883367263940125e-05,0.0,0.0,0.0,0.0,0.0,0.0023548384517198427,80,40,33.76482057571411,0.8441205143928527,0.42206025719642637,0.03005029430896684 +18,0.0007936206464748352,1.0760348675862972,7.879423535514518e-07,0.0,0.0,0.0,0.0,0.0,0.0007936206464748352,320,160,144.43712854385376,0.902732053399086,0.451366026699543,0.0707601236276787,0.0022624223027378322,4.505487352633622,1.0153568444593031e-05,0.0,0.0,0.0,0.0,0.0,0.0022624223027378322,80,40,33.66985249519348,0.841746312379837,0.4208731561899185,0.03378975939194788 diff --git a/contraceptive/tvae/mlu-eval.ipynb b/contraceptive/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..358170172bd7e8ddf2731f152fe1aa38a9b6b2dc --- /dev/null +++ b/contraceptive/tvae/mlu-eval.ipynb @@ -0,0 +1,2563 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.441740Z", + "iopub.status.busy": "2024-02-29T22:23:56.441354Z", + "iopub.status.idle": "2024-02-29T22:23:56.475164Z", + "shell.execute_reply": "2024-02-29T22:23:56.474284Z" + }, + "papermill": { + "duration": 0.049332, + "end_time": "2024-02-29T22:23:56.477091", + "exception": false, + "start_time": "2024-02-29T22:23:56.427759", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.502389Z", + "iopub.status.busy": "2024-02-29T22:23:56.502039Z", + "iopub.status.idle": "2024-02-29T22:23:56.508713Z", + "shell.execute_reply": "2024-02-29T22:23:56.507881Z" + }, + "papermill": { + "duration": 0.021493, + "end_time": "2024-02-29T22:23:56.510656", + "exception": false, + "start_time": "2024-02-29T22:23:56.489163", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.534297Z", + "iopub.status.busy": "2024-02-29T22:23:56.534007Z", + "iopub.status.idle": "2024-02-29T22:23:56.538072Z", + "shell.execute_reply": "2024-02-29T22:23:56.537225Z" + }, + "papermill": { + "duration": 0.018128, + "end_time": "2024-02-29T22:23:56.539980", + "exception": false, + "start_time": "2024-02-29T22:23:56.521852", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.563822Z", + "iopub.status.busy": "2024-02-29T22:23:56.563564Z", + "iopub.status.idle": "2024-02-29T22:23:56.567349Z", + "shell.execute_reply": "2024-02-29T22:23:56.566540Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018066, + "end_time": "2024-02-29T22:23:56.569241", + "exception": false, + "start_time": "2024-02-29T22:23:56.551175", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.592724Z", + "iopub.status.busy": "2024-02-29T22:23:56.592470Z", + "iopub.status.idle": "2024-02-29T22:23:56.597579Z", + "shell.execute_reply": "2024-02-29T22:23:56.596646Z" + }, + "papermill": { + "duration": 0.019073, + "end_time": "2024-02-29T22:23:56.599378", + "exception": false, + "start_time": "2024-02-29T22:23:56.580305", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7b9b0dd4", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.626607Z", + "iopub.status.busy": "2024-02-29T22:23:56.625833Z", + "iopub.status.idle": "2024-02-29T22:23:56.631327Z", + "shell.execute_reply": "2024-02-29T22:23:56.630534Z" + }, + "papermill": { + "duration": 0.02201, + "end_time": "2024-02-29T22:23:56.633199", + "exception": false, + "start_time": "2024-02-29T22:23:56.611189", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/2\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011329, + "end_time": "2024-02-29T22:23:56.657143", + "exception": false, + "start_time": "2024-02-29T22:23:56.645814", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.681290Z", + "iopub.status.busy": "2024-02-29T22:23:56.680693Z", + "iopub.status.idle": "2024-02-29T22:23:56.689901Z", + "shell.execute_reply": "2024-02-29T22:23:56.689107Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023313, + "end_time": "2024-02-29T22:23:56.691767", + "exception": false, + "start_time": "2024-02-29T22:23:56.668454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.716278Z", + "iopub.status.busy": "2024-02-29T22:23:56.715756Z", + "iopub.status.idle": "2024-02-29T22:23:59.037889Z", + "shell.execute_reply": "2024-02-29T22:23:59.036865Z" + }, + "papermill": { + "duration": 2.337143, + "end_time": "2024-02-29T22:23:59.040406", + "exception": false, + "start_time": "2024-02-29T22:23:56.703263", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.067269Z", + "iopub.status.busy": "2024-02-29T22:23:59.066864Z", + "iopub.status.idle": "2024-02-29T22:23:59.078105Z", + "shell.execute_reply": "2024-02-29T22:23:59.077378Z" + }, + "papermill": { + "duration": 0.026705, + "end_time": "2024-02-29T22:23:59.080082", + "exception": false, + "start_time": "2024-02-29T22:23:59.053377", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.105174Z", + "iopub.status.busy": "2024-02-29T22:23:59.104896Z", + "iopub.status.idle": "2024-02-29T22:23:59.112132Z", + "shell.execute_reply": "2024-02-29T22:23:59.111304Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021579, + "end_time": "2024-02-29T22:23:59.114101", + "exception": false, + "start_time": "2024-02-29T22:23:59.092522", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.139461Z", + "iopub.status.busy": "2024-02-29T22:23:59.139015Z", + "iopub.status.idle": "2024-02-29T22:23:59.244628Z", + "shell.execute_reply": "2024-02-29T22:23:59.243810Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.120502, + "end_time": "2024-02-29T22:23:59.247058", + "exception": false, + "start_time": "2024-02-29T22:23:59.126556", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.273328Z", + "iopub.status.busy": "2024-02-29T22:23:59.273055Z", + "iopub.status.idle": "2024-02-29T22:24:04.033228Z", + "shell.execute_reply": "2024-02-29T22:24:04.032395Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.775985, + "end_time": "2024-02-29T22:24:04.035818", + "exception": false, + "start_time": "2024-02-29T22:23:59.259833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 22:24:01.603445: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 22:24:01.603521: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 22:24:01.605533: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:04.062123Z", + "iopub.status.busy": "2024-02-29T22:24:04.061563Z", + "iopub.status.idle": "2024-02-29T22:24:04.067353Z", + "shell.execute_reply": "2024-02-29T22:24:04.066647Z" + }, + "papermill": { + "duration": 0.020608, + "end_time": "2024-02-29T22:24:04.069312", + "exception": false, + "start_time": "2024-02-29T22:24:04.048704", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:04.095589Z", + "iopub.status.busy": "2024-02-29T22:24:04.095238Z", + "iopub.status.idle": "2024-02-29T22:24:12.594652Z", + "shell.execute_reply": "2024-02-29T22:24:12.593579Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.515633, + "end_time": "2024-02-29T22:24:12.597235", + "exception": false, + "start_time": "2024-02-29T22:24:04.081602", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.112927Z", + "iopub.status.busy": "2024-02-29T22:24:13.112643Z", + "iopub.status.idle": "2024-02-29T22:24:13.187354Z", + "shell.execute_reply": "2024-02-29T22:24:13.186375Z" + }, + "papermill": { + "duration": 0.090401, + "end_time": "2024-02-29T22:24:13.189442", + "exception": false, + "start_time": "2024-02-29T22:24:13.099041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.218592Z", + "iopub.status.busy": "2024-02-29T22:24:13.217918Z", + "iopub.status.idle": "2024-02-29T22:24:13.658575Z", + "shell.execute_reply": "2024-02-29T22:24:13.657634Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.457774, + "end_time": "2024-02-29T22:24:13.660801", + "exception": false, + "start_time": "2024-02-29T22:24:13.203027", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.691177Z", + "iopub.status.busy": "2024-02-29T22:24:13.690867Z", + "iopub.status.idle": "2024-02-29T22:24:13.695119Z", + "shell.execute_reply": "2024-02-29T22:24:13.694245Z" + }, + "papermill": { + "duration": 0.022428, + "end_time": "2024-02-29T22:24:13.697307", + "exception": false, + "start_time": "2024-02-29T22:24:13.674879", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.724488Z", + "iopub.status.busy": "2024-02-29T22:24:13.724194Z", + "iopub.status.idle": "2024-02-29T22:24:13.731124Z", + "shell.execute_reply": "2024-02-29T22:24:13.730276Z" + }, + "papermill": { + "duration": 0.022748, + "end_time": "2024-02-29T22:24:13.733118", + "exception": false, + "start_time": "2024-02-29T22:24:13.710370", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10270216" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.760413Z", + "iopub.status.busy": "2024-02-29T22:24:13.760117Z", + "iopub.status.idle": "2024-02-29T22:24:13.843758Z", + "shell.execute_reply": "2024-02-29T22:24:13.842854Z" + }, + "papermill": { + "duration": 0.099517, + "end_time": "2024-02-29T22:24:13.845628", + "exception": false, + "start_time": "2024-02-29T22:24:13.746111", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 48,128\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,270,216\n", + "Trainable params: 10,270,216\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 39.96\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 341.77\n", + "Params size (MB): 41.08\n", + "Estimated Total Size (MB): 383.39\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.877406Z", + "iopub.status.busy": "2024-02-29T22:24:13.877040Z", + "iopub.status.idle": "2024-02-29T23:27:38.382139Z", + "shell.execute_reply": "2024-02-29T23:27:38.381146Z" + }, + "papermill": { + "duration": 3804.523997, + "end_time": "2024-02-29T23:27:38.384713", + "exception": false, + "start_time": "2024-02-29T22:24:13.860716", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02241888580356317, 'avg_role_model_std_loss': 1.4905488636076916, 'avg_role_model_mean_pred_loss': 0.0030782962097319457, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02241888580356317, 'n_size': 320, 'n_batch': 160, 'duration': 142.862961769104, 'duration_batch': 0.8928935110569001, 'duration_size': 0.44644675552845003, 'avg_pred_std': 0.0961261961127093}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007483271204910125, 'avg_role_model_std_loss': 7.415935450342414, 'avg_role_model_mean_pred_loss': 9.700103195409149e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007483271204910125, 'n_size': 80, 'n_batch': 40, 'duration': 32.79719305038452, 'duration_batch': 0.8199298262596131, 'duration_size': 0.40996491312980654, 'avg_pred_std': 0.030276008496821306}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004170822119840522, 'avg_role_model_std_loss': 2.427984166694133, 'avg_role_model_mean_pred_loss': 5.6218089488430103e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004170822119840522, 'n_size': 320, 'n_batch': 160, 'duration': 140.515380859375, 'duration_batch': 0.8782211303710937, 'duration_size': 0.43911056518554686, 'avg_pred_std': 0.06761386157022571}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027781161175880697, 'avg_role_model_std_loss': 5.97971810359972, 'avg_role_model_mean_pred_loss': 8.342038965802879e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027781161175880697, 'n_size': 80, 'n_batch': 40, 'duration': 32.894118309020996, 'duration_batch': 0.8223529577255249, 'duration_size': 0.41117647886276243, 'avg_pred_std': 0.024398993137219806}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0032142372105653295, 'avg_role_model_std_loss': 3.1775326946756253, 'avg_role_model_mean_pred_loss': 9.600223262965901e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032142372105653295, 'n_size': 320, 'n_batch': 160, 'duration': 144.2568175792694, 'duration_batch': 0.9016051098704339, 'duration_size': 0.4508025549352169, 'avg_pred_std': 0.06473975269825587}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002925431027142622, 'avg_role_model_std_loss': 6.108939102519116, 'avg_role_model_mean_pred_loss': 8.98504544019768e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002925431027142622, 'n_size': 80, 'n_batch': 40, 'duration': 35.61764717102051, 'duration_batch': 0.8904411792755127, 'duration_size': 0.44522058963775635, 'avg_pred_std': 0.028732989538184484}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003577234379551553, 'avg_role_model_std_loss': 2.9195899648459376, 'avg_role_model_mean_pred_loss': 4.348199776086897e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003577234379551553, 'n_size': 320, 'n_batch': 160, 'duration': 150.63309359550476, 'duration_batch': 0.9414568349719048, 'duration_size': 0.4707284174859524, 'avg_pred_std': 0.06363038770923594}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032030581480285035, 'avg_role_model_std_loss': 5.47701075857707, 'avg_role_model_mean_pred_loss': 1.4309088606778708e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032030581480285035, 'n_size': 80, 'n_batch': 40, 'duration': 32.49244570732117, 'duration_batch': 0.8123111426830292, 'duration_size': 0.4061555713415146, 'avg_pred_std': 0.021739204511686695}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002611448067193578, 'avg_role_model_std_loss': 1.8328958396290929, 'avg_role_model_mean_pred_loss': 8.810737564255572e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002611448067193578, 'n_size': 320, 'n_batch': 160, 'duration': 143.45851230621338, 'duration_batch': 0.8966157019138337, 'duration_size': 0.44830785095691683, 'avg_pred_std': 0.07633217830557441}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030096135813437288, 'avg_role_model_std_loss': 5.497269465320635, 'avg_role_model_mean_pred_loss': 8.17212617150176e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030096135813437288, 'n_size': 80, 'n_batch': 40, 'duration': 34.19865131378174, 'duration_batch': 0.8549662828445435, 'duration_size': 0.42748314142227173, 'avg_pred_std': 0.01728544359702937}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002066187719401569, 'avg_role_model_std_loss': 1.418562725057735, 'avg_role_model_mean_pred_loss': 4.818185051591941e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002066187719401569, 'n_size': 320, 'n_batch': 160, 'duration': 141.1322615146637, 'duration_batch': 0.8820766344666481, 'duration_size': 0.44103831723332404, 'avg_pred_std': 0.07070515162549781}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002357064618456661, 'avg_role_model_std_loss': 3.038762490750969, 'avg_role_model_mean_pred_loss': 4.724545272427605e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002357064618456661, 'n_size': 80, 'n_batch': 40, 'duration': 32.73992657661438, 'duration_batch': 0.8184981644153595, 'duration_size': 0.40924908220767975, 'avg_pred_std': 0.019966062564344612}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018150892569863686, 'avg_role_model_std_loss': 1.9370867185192977, 'avg_role_model_mean_pred_loss': 4.895915542963466e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018150892569863686, 'n_size': 320, 'n_batch': 160, 'duration': 142.5343050956726, 'duration_batch': 0.8908394068479538, 'duration_size': 0.4454197034239769, 'avg_pred_std': 0.06771241171363726}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002098434802974225, 'avg_role_model_std_loss': 2.5296149099483842, 'avg_role_model_mean_pred_loss': 6.119135131865683e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002098434802974225, 'n_size': 80, 'n_batch': 40, 'duration': 33.130537033081055, 'duration_batch': 0.8282634258270264, 'duration_size': 0.4141317129135132, 'avg_pred_std': 0.036213114765996576}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017754018189464205, 'avg_role_model_std_loss': 1.0608564720709155, 'avg_role_model_mean_pred_loss': 4.110462002784865e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017754018189464205, 'n_size': 320, 'n_batch': 160, 'duration': 150.93800163269043, 'duration_batch': 0.9433625102043152, 'duration_size': 0.4716812551021576, 'avg_pred_std': 0.07812782935689029}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002651070246429299, 'avg_role_model_std_loss': 5.481469099184153, 'avg_role_model_mean_pred_loss': 8.274616622792885e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002651070246429299, 'n_size': 80, 'n_batch': 40, 'duration': 36.68884253501892, 'duration_batch': 0.917221063375473, 'duration_size': 0.4586105316877365, 'avg_pred_std': 0.0201021930330171}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016320147636861293, 'avg_role_model_std_loss': 1.572569604070008, 'avg_role_model_mean_pred_loss': 3.280935549836465e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016320147636861293, 'n_size': 320, 'n_batch': 160, 'duration': 152.10332083702087, 'duration_batch': 0.9506457552313805, 'duration_size': 0.47532287761569025, 'avg_pred_std': 0.07706006977591642}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021084082123252303, 'avg_role_model_std_loss': 4.821968620683037, 'avg_role_model_mean_pred_loss': 7.140439676618648e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021084082123252303, 'n_size': 80, 'n_batch': 40, 'duration': 35.41457486152649, 'duration_batch': 0.8853643715381623, 'duration_size': 0.44268218576908114, 'avg_pred_std': 0.032409553838078864}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014390503529739362, 'avg_role_model_std_loss': 1.1130779689192227, 'avg_role_model_mean_pred_loss': 1.9856122689985296e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014390503529739362, 'n_size': 320, 'n_batch': 160, 'duration': 148.04402089118958, 'duration_batch': 0.9252751305699348, 'duration_size': 0.4626375652849674, 'avg_pred_std': 0.07865846673303167}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002113264991157848, 'avg_role_model_std_loss': 2.781704166371675, 'avg_role_model_mean_pred_loss': 4.972169583404757e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002113264991157848, 'n_size': 80, 'n_batch': 40, 'duration': 33.858819007873535, 'duration_batch': 0.8464704751968384, 'duration_size': 0.4232352375984192, 'avg_pred_std': 0.02809684935346013}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001374225791067829, 'avg_role_model_std_loss': 1.163778030275184, 'avg_role_model_mean_pred_loss': 1.7583196497888975e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001374225791067829, 'n_size': 320, 'n_batch': 160, 'duration': 145.61796760559082, 'duration_batch': 0.9101122975349426, 'duration_size': 0.4550561487674713, 'avg_pred_std': 0.072609595393169}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023332797987222877, 'avg_role_model_std_loss': 2.608034198338737, 'avg_role_model_mean_pred_loss': 1.0012352827615257e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023332797987222877, 'n_size': 80, 'n_batch': 40, 'duration': 33.24501919746399, 'duration_batch': 0.8311254799365997, 'duration_size': 0.41556273996829984, 'avg_pred_std': 0.03622158533107722}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013136249404567478, 'avg_role_model_std_loss': 1.105370874132261, 'avg_role_model_mean_pred_loss': 2.0836581030414523e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013136249404567478, 'n_size': 320, 'n_batch': 160, 'duration': 144.06322360038757, 'duration_batch': 0.9003951475024223, 'duration_size': 0.45019757375121117, 'avg_pred_std': 0.07791482849102067}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020939761153385915, 'avg_role_model_std_loss': 7.381703513306002, 'avg_role_model_mean_pred_loss': 3.89069975454473e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020939761153385915, 'n_size': 80, 'n_batch': 40, 'duration': 33.347615242004395, 'duration_batch': 0.8336903810501098, 'duration_size': 0.4168451905250549, 'avg_pred_std': 0.01896329457867978}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013007374736943688, 'avg_role_model_std_loss': 0.8016777972321465, 'avg_role_model_mean_pred_loss': 1.6661171600203944e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013007374736943688, 'n_size': 320, 'n_batch': 160, 'duration': 143.7109453678131, 'duration_batch': 0.8981934085488319, 'duration_size': 0.44909670427441595, 'avg_pred_std': 0.07403813572964282}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021091806715048734, 'avg_role_model_std_loss': 2.195618169948898, 'avg_role_model_mean_pred_loss': 6.716770201746968e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021091806715048734, 'n_size': 80, 'n_batch': 40, 'duration': 33.3885440826416, 'duration_batch': 0.8347136020660401, 'duration_size': 0.41735680103302003, 'avg_pred_std': 0.03174531738768564}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011258274745216568, 'avg_role_model_std_loss': 0.9933245053406304, 'avg_role_model_mean_pred_loss': 1.2559000061217402e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011258274745216568, 'n_size': 320, 'n_batch': 160, 'duration': 143.82260847091675, 'duration_batch': 0.8988913029432297, 'duration_size': 0.44944565147161486, 'avg_pred_std': 0.07367398725546082}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002973305231353152, 'avg_role_model_std_loss': 2.332661612354639, 'avg_role_model_mean_pred_loss': 1.9470425126388857e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002973305231353152, 'n_size': 80, 'n_batch': 40, 'duration': 33.963603019714355, 'duration_batch': 0.8490900754928589, 'duration_size': 0.42454503774642943, 'avg_pred_std': 0.035911593766650186}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010081856245165, 'avg_role_model_std_loss': 1.8124559902420032, 'avg_role_model_mean_pred_loss': 1.1379342504010126e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010081856245165, 'n_size': 320, 'n_batch': 160, 'duration': 144.61974716186523, 'duration_batch': 0.9038734197616577, 'duration_size': 0.45193670988082885, 'avg_pred_std': 0.07099440268893886}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002199153335527626, 'avg_role_model_std_loss': 2.3161073656544886, 'avg_role_model_mean_pred_loss': 7.577638564465472e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002199153335527626, 'n_size': 80, 'n_batch': 40, 'duration': 33.57773303985596, 'duration_batch': 0.8394433259963989, 'duration_size': 0.41972166299819946, 'avg_pred_std': 0.02861409220568021}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010586415690795547, 'avg_role_model_std_loss': 1.0111776571605908, 'avg_role_model_mean_pred_loss': 1.3155730025755604e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010586415690795547, 'n_size': 320, 'n_batch': 160, 'duration': 143.99448657035828, 'duration_batch': 0.8999655410647392, 'duration_size': 0.4499827705323696, 'avg_pred_std': 0.07437738951684877}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024275374956232556, 'avg_role_model_std_loss': 2.65694752669535, 'avg_role_model_mean_pred_loss': 1.2085388890881177e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024275374956232556, 'n_size': 80, 'n_batch': 40, 'duration': 33.66980719566345, 'duration_batch': 0.8417451798915863, 'duration_size': 0.42087258994579313, 'avg_pred_std': 0.034823847954976374}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008538674877796026, 'avg_role_model_std_loss': 1.5765032050085541, 'avg_role_model_mean_pred_loss': 5.374222879224455e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008538674877796026, 'n_size': 320, 'n_batch': 160, 'duration': 143.34916639328003, 'duration_batch': 0.8959322899580002, 'duration_size': 0.4479661449790001, 'avg_pred_std': 0.08046830528701321}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022838078687641428, 'avg_role_model_std_loss': 2.0230109165978774, 'avg_role_model_mean_pred_loss': 9.47888851559331e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022838078687641428, 'n_size': 80, 'n_batch': 40, 'duration': 33.02995800971985, 'duration_batch': 0.8257489502429962, 'duration_size': 0.4128744751214981, 'avg_pred_std': 0.030621698120376094}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008248503026644372, 'avg_role_model_std_loss': 0.35676954403032646, 'avg_role_model_mean_pred_loss': 7.142933498651121e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008248503026644372, 'n_size': 320, 'n_batch': 160, 'duration': 143.77559542655945, 'duration_batch': 0.8985974714159966, 'duration_size': 0.4492987357079983, 'avg_pred_std': 0.08080137882643612}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023548384517198427, 'avg_role_model_std_loss': 4.805163549015765, 'avg_role_model_mean_pred_loss': 1.1883367263940125e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023548384517198427, 'n_size': 80, 'n_batch': 40, 'duration': 33.76482057571411, 'duration_batch': 0.8441205143928527, 'duration_size': 0.42206025719642637, 'avg_pred_std': 0.03005029430896684}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007936206464748352, 'avg_role_model_std_loss': 1.0760348675862972, 'avg_role_model_mean_pred_loss': 7.879423535514518e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007936206464748352, 'n_size': 320, 'n_batch': 160, 'duration': 144.43712854385376, 'duration_batch': 0.902732053399086, 'duration_size': 0.451366026699543, 'avg_pred_std': 0.0707601236276787}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022624223027378322, 'avg_role_model_std_loss': 4.505487352633622, 'avg_role_model_mean_pred_loss': 1.0153568444593031e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022624223027378322, 'n_size': 80, 'n_batch': 40, 'duration': 33.66985249519348, 'duration_batch': 0.841746312379837, 'duration_size': 0.4208731561899185, 'avg_pred_std': 0.03378975939194788}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007593693141508595, 'avg_role_model_std_loss': 0.6788086620792548, 'avg_role_model_mean_pred_loss': 5.805468950893806e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007593693141508595, 'n_size': 320, 'n_batch': 160, 'duration': 144.87138056755066, 'duration_batch': 0.9054461285471916, 'duration_size': 0.4527230642735958, 'avg_pred_std': 0.08014883432224451}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002318795861719991, 'avg_role_model_std_loss': 2.6745841488044872, 'avg_role_model_mean_pred_loss': 1.0226613121978867e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002318795861719991, 'n_size': 80, 'n_batch': 40, 'duration': 33.691651344299316, 'duration_batch': 0.842291283607483, 'duration_size': 0.4211456418037415, 'avg_pred_std': 0.03418012205511332}\n", + "Time out: 3627.9625329971313/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▂▂▂▂▁▁▂▁▁▁▁▁▂▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▆▄▅▃▁▂█▂▇▅█▂▆█▅▇▆▆▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▂▁▁▄▃▂▄▄▄▃▄▃▃▃▃▅▅▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▂▂▂▂▁▁▂▁▁▁▁▁▂▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▂▁▁▁▁▁▁▁▁▁▂▁▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▆▆▅▆▂▂▅▅▂▂█▁▁▁▂▁▅▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▄▆█▇▅▄▅▃▄▃▃▃▂▃▅▃▄▁▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▂▂▆▁▄▁▂█▆▃▂▂▂▃▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▂▁▃▇▃▁▂▇█▆▄▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▂▂▆▁▄▁▂█▆▃▂▂▂▃▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▂▁▃▇▃▁▂▇█▆▄▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▂▂▆▁▄▁▂█▆▃▂▂▂▃▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▂▁▃▇▃▁▂▇█▆▄▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00226\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00079\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03379\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07076\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00226\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00079\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 4.50549\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 1.07603\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.84175\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.90273\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.42087\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.45137\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 33.66985\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 144.43713\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/tvae/2/wandb/offline-run-20240229_222415-ekgllb0j\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_222415-ekgllb0j/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'pred_duration': 3.313055992126465, 'grad_duration': 2.6909384727478027, 'total_duration': 6.003994464874268, 'pred_std': 0.057808153331279755, 'std_loss': 0.01108523365110159, 'mean_pred_loss': 9.105955314225866e-07, 'pred_rmse': 0.03504936397075653, 'pred_mae': 0.02793470025062561, 'pred_mape': 0.06429529935121536, 'grad_rmse': 0.040074676275253296, 'grad_mae': 0.03164428845047951, 'grad_mape': 0.6164292097091675}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'avg_pred_duration': 3.313055992126465, 'avg_grad_duration': 2.6909384727478027, 'avg_total_duration': 6.003994464874268, 'avg_pred_std': 0.057808153331279755, 'avg_std_loss': 0.01108523365110159, 'avg_mean_pred_loss': 9.105955314225866e-07}, 'min_metrics': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'pred_duration': 3.313055992126465, 'grad_duration': 2.6909384727478027, 'total_duration': 6.003994464874268, 'pred_std': 0.057808153331279755, 'std_loss': 0.01108523365110159, 'mean_pred_loss': 9.105955314225866e-07, 'pred_rmse': 0.03504936397075653, 'pred_mae': 0.02793470025062561, 'pred_mape': 0.06429529935121536, 'grad_rmse': 0.040074676275253296, 'grad_mae': 0.03164428845047951, 'grad_mape': 0.6164292097091675}, 'model_metrics': {'tvae': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'pred_duration': 3.313055992126465, 'grad_duration': 2.6909384727478027, 'total_duration': 6.003994464874268, 'pred_std': 0.057808153331279755, 'std_loss': 0.01108523365110159, 'mean_pred_loss': 9.105955314225866e-07, 'pred_rmse': 0.03504936397075653, 'pred_mae': 0.02793470025062561, 'pred_mape': 0.06429529935121536, 'grad_rmse': 0.040074676275253296, 'grad_mae': 0.03164428845047951, 'grad_mape': 0.6164292097091675}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:38.426125Z", + "iopub.status.busy": "2024-02-29T23:27:38.425799Z", + "iopub.status.idle": "2024-02-29T23:27:38.430193Z", + "shell.execute_reply": "2024-02-29T23:27:38.429306Z" + }, + "papermill": { + "duration": 0.0277, + "end_time": "2024-02-29T23:27:38.432267", + "exception": false, + "start_time": "2024-02-29T23:27:38.404567", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:38.470452Z", + "iopub.status.busy": "2024-02-29T23:27:38.470104Z", + "iopub.status.idle": "2024-02-29T23:27:38.772666Z", + "shell.execute_reply": "2024-02-29T23:27:38.771746Z" + }, + "papermill": { + "duration": 0.324535, + "end_time": "2024-02-29T23:27:38.775289", + "exception": false, + "start_time": "2024-02-29T23:27:38.450754", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:38.818515Z", + "iopub.status.busy": "2024-02-29T23:27:38.818096Z", + "iopub.status.idle": "2024-02-29T23:27:39.115381Z", + "shell.execute_reply": "2024-02-29T23:27:39.114363Z" + }, + "papermill": { + "duration": 0.321667, + "end_time": "2024-02-29T23:27:39.117472", + "exception": false, + "start_time": "2024-02-29T23:27:38.795805", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3PElEQVR4nO3dfVxUZd4/8M+ZZx5nBIRhFAEVwgdE0yA0tZINjSxc73S52TR/JtpKD1Kra5tQu/uKsrrX1bxrq12929U0N2t31SxDzVUQFEExldRQUBgQieFxmGHm+v1xYGRggJkBmRn4vl+v85qZc6455zrAfLjOOddch2OMMRBCiIsQOLoChBBiCwotQohLodAihLgUCi1CiEuh0CKEuBQKLUKIS6HQIoS4FJGjKzBQjEYjysvL4eXlBY7jHF0dQkgnjDHU19dDpVJBIOi+PTVkQqu8vBxBQUGOrgYhpBdlZWUYOXJkt8uHTGh5eXkB4H8g3t7eDq4NIaSzuro6BAUFmT6r3RkyodV+SOjt7U2hRYgT6+30DZ2IJ4S4FAotQohLodAihLiUIXNOi/SNwWCAXq93dDWICxOLxRAKhX1eD4UW6RFjDGq1GrW1tY6uChkEFAoFlEpln/pKUmiRHrUHlr+/P9zd3aljLrELYwxNTU2oqqoCAAQGBtq9LgqtTjRNelyoqINQwCE61MfR1XEog8FgCixfX19HV4e4ODc3NwBAVVUV/P397T5UpBPxnZy9UYukj04i/Z/nHV0Vh2s/h+Xu7u7gmpDBov1vqS/nRym0OpG7iQEAdc100rkdHRKS/tIff0sUWp20h5aGQosQp0Sh1Ul7aDXqDNAbjA6uDSGkMwqtTrzbQgugQ0RiP47j8OWXXzq6Gv3qtddew+TJkx1dDQqtzoQCDl5S/qIqHSISV7Z9+3YoFIp+W9/LL7+MrKysflufvSi0LPCm81pkCNHpdFaV8/T0dIquLxRaFlBoWcYYQ5Ou1SGTrTdCP3jwIB544AEoFAr4+vrisccew9WrVwEA06dPx7p168zK37p1C2KxGMeOHQMAVFRUICEhAW5ubggNDcXOnTsREhKCTZs22fWzKyoqwsMPPww3Nzf4+voiJSUFDQ0NpuVHjx5FdHQ0PDw8oFAoMGPGDFy/fh0AcPbsWTz00EPw8vKCt7c3pk6ditOnT/e4vaNHj2LZsmXQaDTgOA4cx+G1114DAISEhOD3v/89lixZAm9vb6SkpAAA1q1bh/DwcLi7u2P06NHYsGGDWdeEzoeHTz/9NBITE/HOO+8gMDAQvr6+WL169V3/uhd1LrVA7kaHh5Y06w0Yn/61Q7Z94XfxcJdY/+fa2NiItLQ0TJo0CQ0NDUhPT8eCBQtQWFiI5ORkbNy4EW+++abpEvzu3buhUqkwc+ZMAMCSJUtQXV2No0ePQiwWIy0tzdSb21aNjY2Ij49HbGwsTp06haqqKjzzzDNITU3F9u3b0draisTERKxYsQKffvopdDod8vLyTHVLTk7GlClT8P7770MoFKKwsBBisbjHbU6fPh2bNm1Ceno6iouLAfAtpXbvvPMO0tPTkZGRYZrn5eWF7du3Q6VSoaioCCtWrICXlxfWrl3b7XaOHDmCwMBAHDlyBFeuXMHixYsxefJkrFixwq6flTUotCygvlqub+HChWav//rXv2L48OG4cOECFi1ahBdffBHHjx83hdTOnTuRlJQEjuNw6dIlfPvttzh16hSmTZsGAPj4448RFhZmV1127twJrVaLTz75BB4eHgCA9957D/Pnz8dbb70FsVgMjUaDxx57DGPGjAEAjBs3zvT+0tJS/PrXv0ZERAQAWFUPiUQCuVwOjuOgVCq7LH/44Yfx0ksvmc179dVXTc9DQkLw8ssvY9euXT2G1rBhw/Dee+9BKBQiIiICCQkJyMrKotAaaNRXyzI3sRAXfhfvsG3b4vLly0hPT0dubi6qq6thNPLdV0pLSzFx4kQ88sgj2LFjB2bOnImSkhLk5OTgz3/+MwCguLgYIpEI9957r2l9Y8eOxbBhw+yq+8WLFxEVFWUKLACYMWMGjEYjiouLMWvWLDz99NOIj4/Hz372M8TFxWHRokWm7+elpaXhmWeewd/+9jfExcXhySefNIWbvdrDuKPdu3dj8+bNuHr1KhoaGtDa2trrKL8TJkww+zpOYGAgioqK+lS33tA5LQsotCzjOA7uEpFDJlt7Us+fPx81NTX46KOPkJubi9zcXAB3TjonJyfjH//4B/R6PXbu3InIyEhERkb2+8/MWtu2bUNOTg6mT5+O3bt3Izw8HCdPngTAn0v6/vvvkZCQgMOHD2P8+PH44osv+rS9jgEKADk5OUhOTsajjz6Kffv2oaCgAL/97W97PUnf+TCV4zjTP4i7hULLAgot13b79m0UFxfj1VdfxZw5czBu3Dj89NNPZmWeeOIJaLVaHDx4EDt37kRycrJp2T333IPW1lYUFBSY5l25cqXLOqw1btw4nD17Fo2NjaZ5J06cgEAgwD333GOaN2XKFKxfvx7Z2dmYOHEidu7caVoWHh6ONWvW4JtvvsHPf/5zbNu2rdftSiQSGAwGq+qYnZ2N4OBg/Pa3v8W0adMQFhZmuhDgbCi0LKDQcm3Dhg2Dr68vPvzwQ1y5cgWHDx9GWlqaWRkPDw8kJiZiw4YNuHjxIpKSkkzLIiIiEBcXh5SUFOTl5aGgoAApKSlwc3Oz67tzycnJkMlkWLp0Kc6fP48jR47gueeew1NPPYWAgACUlJRg/fr1yMnJwfXr1/HNN9/g8uXLGDduHJqbm5GamoqjR4/i+vXrOHHiBE6dOmV2zqs7ISEhaGhoQFZWFqqrq9HU1NRt2bCwMJSWlmLXrl24evUqNm/e3OfW3N1CoWUBdXlwbQKBALt27UJ+fj4mTpyINWvW4O233+5SLjk5GWfPnsXMmTMxatQos2WffPIJAgICMGvWLCxYsMB0JU0mk9lcH3d3d3z99deoqanBfffdh//6r//CnDlz8N5775mWX7p0CQsXLkR4eDhSUlKwevVqrFy5EkKhELdv38aSJUsQHh6ORYsWYd68eXj99dd73e706dOxatUqLF68GMOHD8fGjRu7Lfv4449jzZo1SE1NxeTJk5GdnY0NGzbYvK8DgWO2doBxUXV1dZDL5dBoNL2eXDxaXIWnt53CuEBvfPXCzAGqofPRarUoKSlBaGioXR/WweTGjRsICgrCt99+izlz5ji6Oi6rp78paz+jdPXQAuryQA4fPoyGhgZERkaioqICa9euRUhICGbNmuXoqg15dHhoAYUW0ev1eOWVVzBhwgQsWLAAw4cPN3U03bFjBzw9PS1OEyZMGLA6zps3r9t6vPHGGwNWj4FGLS0L2kOrvqUVBiODUECD4A018fHxiI+33Cft8ccfR0xMjMVlvfVU708ff/wxmpubLS7z8Rm8Q4VTaFnQeXiaYR4SB9aGOBsvLy94eXk5uhoYMWKEo6vgEHYdHm7duhUhISGQyWSIiYlBXl5ej+X37NmDiIgIyGQyREZG4sCBA6Zler0e69atQ2RkJDw8PKBSqbBkyRKUl5ebraOmpgbJycnw9vaGQqHA8uXLzb5w2p/EQgE8JHwvX7qCSIhzsTm0du/ejbS0NGRkZODMmTOIiopCfHx8t18mzc7ORlJSEpYvX46CggIkJiYiMTER58/zN45oamrCmTNnsGHDBpw5cwZ79+5FcXExHn/8cbP1JCcn4/vvv8ehQ4ewb98+HDt2zPTt9LuB+moR4qSYjaKjo9nq1atNrw0GA1OpVCwzM9Ni+UWLFrGEhASzeTExMWzlypXdbiMvL48BYNevX2eMMXbhwgUGgJ06dcpU5quvvmIcx7GbN29aVW+NRsMAMI1GY1X5+D9+x4LX7WPfFVdZVX4wam5uZhcuXGDNzc2OrgoZJHr6m7L2M2pTS0un0yE/Px9xcXGmeQKBAHFxccjJybH4npycHLPyAH+Ss7vyAExjALWPupiTkwOFQmH2Jc+4uDgIBALTd8o6a2lpQV1dndlkC2ppEeKcbAqt6upqGAwGBAQEmM0PCAiAWq22+B61Wm1Tea1Wi3Xr1iEpKcnUwUytVsPf39+snEgkgo+PT7fryczMhFwuN01BQUFW7WM7Ci1CnJNT9dPS6/VYtGgRGGN4//33+7Su9evXQ6PRmKaysjKb3k+hRfpiMN7YwlnYFFp+fn4QCoWorKw0m19ZWWlxoDEAUCqVVpVvD6zr16/j0KFDZt34lUpllxP9ra2tqKmp6Xa7UqkU3t7eZpMtqIMpcXX9fWMLgB/GmeM41NbW9ut6bWFTaEkkEkydOtXsjhxGoxFZWVmIjY21+J7Y2Ngud/A4dOiQWfn2wLp8+TK+/fbbLoPnx8bGora2Fvn5+aZ5hw8fhtFo7LaTX19RS4sQJ2Xr2f9du3YxqVTKtm/fzi5cuMBSUlKYQqFgarWaMcbYU089xX7zm9+Yyp84cYKJRCL2zjvvsIsXL7KMjAwmFotZUVERY4wxnU7HHn/8cTZy5EhWWFjIKioqTFNLS4tpPXPnzmVTpkxhubm57Pjx4ywsLIwlJSVZXW9brx7+X3YJC163j63622mrtzHYdLnSYzQy1tLgmMlotKnuX331FZsxYwaTy+XMx8eHJSQksCtXrjDGGIuNjWVr1641K19VVcVEIhH77rvvGGOMlZeXs0cffZTJZDIWEhLCduzYwYKDg9kf//hHq7YPgH3xxRem1+fOnWMPPfQQk8lkzMfHh61YsYLV19eblh85coTdd999zN3dncnlcjZ9+nR27do1xhhjhYWF7MEHH2Senp7My8uL3XvvvWZX0i05cuQIA2A2ZWRkMMYY02q17KWXXmIqlYq5u7uz6OhoduTIEdN7r127xh577DGmUCiYu7s7Gz9+PNu/fz8rKSnpss6lS5da9fNo1x9XD23uEb948WLcunUL6enpUKvVmDx5Mg4ePGg62V5aWgqB4E4Dbvr06di5cydeffVVvPLKKwgLC8OXX36JiRMnAgBu3ryJf/3rXwDQ5UaQR44cwYMPPggA2LFjB1JTUzFnzhwIBAIsXLgQmzdvtrX6VqOWlgX6JuANlWO2/Uo5IPHovVwburFF9ze2SE1NxYULF7Br1y6oVCp88cUXmDt3LoqKihAWFobVq1dDp9Ph2LFj8PDwwIULF+Dp6YmgoCB8/vnnWLhwIYqLi+Ht7Q03Nze7fiZ9YdfXeFJTU5Gammpx2dGjR7vMe/LJJ/Hkk09aLB8SEmLV7aF8fHzMRnK822hMLddGN7awfGOL0tJSbNu2DaWlpVCp+H9AL7/8Mg4ePIht27bhjTfeQGlpKRYuXGgafnr06NGm97d/p9Hf37/fz5dZi7572A1qaVkgdudbPI7atg3oxhaWFRUVwWAwIDw83Gx+S0uL6Vzy888/j2effRbffPMN4uLisHDhQkyaNMmu7d0NTtXlwZl4yyi0uuA4/hDNERPd2KJfbmzR0NAAoVCI/Px8FBYWmqaLFy/iT3/6EwDgmWeewY8//oinnnoKRUVFmDZtGrZs2dJv+9pXFFrdMA1Po+WHpyGug25swbN0Y4spU6bAYDCgqqoKY8eONZs6HkYGBQVh1apV2Lt3L1566SV89NFHpnUCsPqGGXcDhVY35B2Gp6nXUmvLldCNLXiWbmwRHh6O5ORkLFmyBHv37kVJSQny8vKQmZmJ/fv3AwBefPFFfP311ygpKcGZM2dw5MgR0/aCg4PBcRz27duHW7du3bWRVnpk0/VKF2ZrlwfGGIt49SsWvG4fu1bdcBdr5rxc+QvThw4dYuPGjWNSqZRNmjSJHT16tEs3hAMHDjAAbNasWV3eX15ezubNm8ekUikLDg5mO3fuZP7+/uyDDz6wavudt9VTlwe1Ws0SExNZYGAgk0gkLDg4mKWnpzODwcBaWlrYL37xCxYUFMQkEglTqVQsNTXV6t/JqlWrmK+vr1mXB51Ox9LT01lISAgTi8UsMDCQLViwgJ07d44xxlhqaiobM2YMk0qlbPjw4eypp55i1dXVpnX+7ne/Y0qlknEc55AuD3Rjix7c/0YW1HVa/Ct1BiaNVNzdCjohurHFHXRji/5BN7a4y+RuYqjrtHQyfgiiG1s4Lzqn1QPq9jB00Y0tnBe1tHpAHUyHLrqxhfOi0OoBtbSIJXRjC8eiw8MeUGjxhsi1GjIA+uNviUKrB0N9TK32Q52mpiYH14QMFu1/S305jKbDwx7I3fgfz1BtaQmFQigUCtPoBu7u7nZ1riSEMYampiZUVVVBoVBAKBTavS4KrR7I3enwsP2rHfYOy0JIRwqFotvRhq1FodWDO4eHrQ6uieNwHIfAwED4+/tDrx+64U36TiwW96mF1Y5Cqwd0Iv4OoVDYL39whPQVnYjvAYUWIc6HQqsH7Z1L67R6GGl4GkKcAoVWD9pbWowB9S1D97wWIc6EQqsHUpEQMjH/IxqqfbUIcTYUWr2g81qEOBcKrV5QaBHiXCi0ekGhRYhzodDqBYUWIc6FQqsXNKYWIc6FQqsX1NIixLlQaPWCbtpKiHOh0OoFtbQIcS4UWr0Y6gMBEuJsKLR6QS0tQpwLhVYvaCBAQpwLhVYvqKVFiHOh0OpFx3NaNDwNIY5HodWL9tAyMqBBR8PTEOJoFFq9kImFkIj4H5OmiQ4RCXE0Ci0r0HktQpwHhZYVqK8WIc6DQssK1NIixHnYFVpbt25FSEgIZDIZYmJikJeX12P5PXv2ICIiAjKZDJGRkThw4IDZ8r179+KRRx6Br68vOI5DYWFhl3U8+OCD4DjObFq1apU91bcZhRYhzsPm0Nq9ezfS0tKQkZGBM2fOICoqCvHx8d3egTg7OxtJSUlYvnw5CgoKkJiYiMTERJw/f95UprGxEQ888ADeeuutHre9YsUKVFRUmKaNGzfaWn27UGgR4kSYjaKjo9nq1atNrw0GA1OpVCwzM9Ni+UWLFrGEhASzeTExMWzlypVdypaUlDAArKCgoMuy2bNnsxdeeMHW6ppoNBoGgGk0Gpvfm/HP8yx43T721lcX7d4+IaRn1n5GbWpp6XQ65OfnIy4uzjRPIBAgLi4OOTk5Ft+Tk5NjVh4A4uPjuy3fkx07dsDPzw8TJ07E+vXr0dTU1G3ZlpYW1NXVmU32ooEACXEeIlsKV1dXw2AwICAgwGx+QEAALl26ZPE9arXaYnm1Wm1TRf/7v/8bwcHBUKlUOHfuHNatW4fi4mLs3bvXYvnMzEy8/vrrNm2jO6arh1rqXEqIo9kUWo6UkpJieh4ZGYnAwEDMmTMHV69exZgxY7qUX79+PdLS0kyv6+rqEBQUZNe26ZwWIc7DptDy8/ODUChEZWWl2fzKykoolUqL71EqlTaVt1ZMTAwA4MqVKxZDSyqVQiqV9mkb7Si0CHEeNp3TkkgkmDp1KrKyskzzjEYjsrKyEBsba/E9sbGxZuUB4NChQ92Wt1Z7t4jAwMA+rcca1LmUEOdh8+FhWloali5dimnTpiE6OhqbNm1CY2Mjli1bBgBYsmQJRowYgczMTADACy+8gNmzZ+Pdd99FQkICdu3ahdOnT+PDDz80rbOmpgalpaUoLy8HABQXFwPgW2lKpRJXr17Fzp078eijj8LX1xfnzp3DmjVrMGvWLEyaNKnPP4TeUEuLECdiz6XJLVu2sFGjRjGJRMKio6PZyZMnTctmz57Nli5dalb+s88+Y+Hh4UwikbAJEyaw/fv3my3ftm0bA9BlysjIYIwxVlpaymbNmsV8fHyYVCplY8eOZb/+9a9t6r7Qly4PFbXNLHjdPjZ6/X5mNBptfj8hpHfWfkY5xtiQGCSqrq4OcrkcGo0G3t7eNr23WWfAuPSDAIDzr8fDU+oy1y8IcRnWfkbpu4dWkIkFkAjbhqehQ0RCHIpCywocx93pYEpjahHiUBRaVvJ24w8JqaVFiGNRaFmJriAS4hwotKxEfbUIcQ4UWlailhYhzoFCy0oUWoQ4BwotK1FoEeIcKLSsRKFFiHOg0LISDQRIiHOg0LIStbQIcQ4UWlaiLg+EOAcKLStRS4sQ50ChZaWOoTVEBsYgxClRaFmpPbRajQxNOoODa0PI0EWhZSV3iRAiAQeADhEJcSQKLStxHEfntQhxAhRaNqDQIsTxKLRsQB1MCXE8Ci0bUEuLEMej0LIBdTAlxPEotGxAoUWI41Fo2YAODwlxPAotG1BoEeJ4FFo2oNAixPEotGxAXR4IcTwKLRtQS4sQx6PQssGdG7a2OrgmhAxdFFo26NjlgYanIcQxKLRs0B5aOoMRWr3RwbUhZGii0LKBp1QEIQ1PQ4hDUWjZgOM4eMvaz2tRaBHiCBRaNqIriIQ4FoWWjSi0CHEsCi0bUQdTQhyLQstG1NIixLEotGxEoUWIY1Fo2YjG1CLEsewKra1btyIkJAQymQwxMTHIy8vrsfyePXsQEREBmUyGyMhIHDhwwGz53r178cgjj8DX1xccx6GwsLDLOrRaLVavXg1fX194enpi4cKFqKystKf6fUItLUIcy+bQ2r17N9LS0pCRkYEzZ84gKioK8fHxqKqqslg+OzsbSUlJWL58OQoKCpCYmIjExEScP3/eVKaxsREPPPAA3nrrrW63u2bNGvz73//Gnj178N1336G8vBw///nPba1+n1FoEeJgzEbR0dFs9erVptcGg4GpVCqWmZlpsfyiRYtYQkKC2byYmBi2cuXKLmVLSkoYAFZQUGA2v7a2lonFYrZnzx7TvIsXLzIALCcnx+J2tVot02g0pqmsrIwBYBqNxtpdtejAuXIWvG4f+/n/nujTeggh5jQajVWfUZtaWjqdDvn5+YiLizPNEwgEiIuLQ05OjsX35OTkmJUHgPj4+G7LW5Kfnw+9Xm+2noiICIwaNarb9WRmZkIul5umoKAgq7fXE2ppEeJYNoVWdXU1DAYDAgICzOYHBARArVZbfI9arbapfHfrkEgkUCgUVq9n/fr10Gg0pqmsrMzq7fWE+mkR4lgiR1fgbpFKpZBKpf2+XmppEeJYNrW0/Pz8IBQKu1y1q6yshFKptPgepVJpU/nu1qHT6VBbW9un9fQHuXvb8DStRmj1hgHdNiHExtCSSCSYOnUqsrKyTPOMRiOysrIQGxtr8T2xsbFm5QHg0KFD3Za3ZOrUqRCLxWbrKS4uRmlpqU3r6Q+eEhHaRqeh1hYhDmDz4WFaWhqWLl2KadOmITo6Gps2bUJjYyOWLVsGAFiyZAlGjBiBzMxMAMALL7yA2bNn491330VCQgJ27dqF06dP48MPPzSts6amBqWlpSgvLwfABxLAt7CUSiXkcjmWL1+OtLQ0+Pj4wNvbG8899xxiY2Nx//339/mHYAuBgIO3mxi1TXpomvUI8JYN6PYJGfLsuTS5ZcsWNmrUKCaRSFh0dDQ7efKkadns2bPZ0qVLzcp/9tlnLDw8nEkkEjZhwgS2f/9+s+Xbtm1jALpMGRkZpjLNzc3sV7/6FRs2bBhzd3dnCxYsYBUVFVbX2drLqdaYtfEwC163j+WV3O7zugghPGs/oxxjQ2Ow87q6Osjlcmg0Gnh7e/dpXY+/dxznbmjw8ZJpiBsf0PsbCCG9svYzSt89tIPp+4daOqdFyECj0LID9dUixHEotOxAfbUIcRwKLTt4yyi0CHEUCi07UEuLEMeh0LIDDQRIiONQaNmBWlqEOA6Flh0otAhxHAotO1BoEeI4FFp2oNAixHEotOzQHlpavREtrTQ8DSEDiULLDl4yETganoYQh6DQsoNAwMFLyo/qQ90eCBlYFFp2ah/BlFpahAwsCi070cl4QhyDQqszoxGoOAdc7/kWZxRahDgGhVZnRXuAP88Evnm1x2Km0Gqi0CJkIFFodRYyg38sPwNoNd0Wu9PSah2IWhFC2lBodSYfCfiMBpixx0NEGgiQEMeg0LIkdBb/WHKs2yJ0TosQx6DQsqQ9tK5RaBHibCi0LAmZyT+qi4CmGotFaEwtQhyDQssST3/Afzz//Np/LBahlhYhjkGh1Z321lY357UotAhxDAqt7vRyMp5CixDHoNDqTsgMABxQ/QNQr+6yuD20mvUG6FqNA1w5QoYuCq3uuA0DAqP45yVdz2t5td1GDKDWFiEDiUKrJ6ZDxO+6LBIKOHjJ+OFpKLQIGTgUWj3p5bxW+01b67QUWoQMFAqtnoy6HxCIgNrrwE/Xuyymk/GEDDwKrZ5IvYARU/nnFvprUQdTQgYehVZvejhEpJYWIQOPQqs3HTuZMma2iMbUImTgUWj1JigaEEqB+grg9hWzRTROPCEDj0KrN2I3PriALoeIdHhIyMCj0LJG6Gz+sVNo0UCAhAw8Ci1rmMbX+g9/44s21NIiZODZFVpbt25FSEgIZDIZYmJikJeX12P5PXv2ICIiAjKZDJGRkThw4IDZcsYY0tPTERgYCDc3N8TFxeHy5ctmZUJCQsBxnNn05ptv2lN92424FxB7AE23gaoLptkUWoQMPJtDa/fu3UhLS0NGRgbOnDmDqKgoxMfHo6qqymL57OxsJCUlYfny5SgoKEBiYiISExNx/vx5U5mNGzdi8+bN+OCDD5CbmwsPDw/Ex8dDq9Waret3v/sdKioqTNNzzz1na/XtIxQDwbH88w6HiNRPixAHYDaKjo5mq1evNr02GAxMpVKxzMxMi+UXLVrEEhISzObFxMSwlStXMsYYMxqNTKlUsrffftu0vLa2lkmlUvbpp5+a5gUHB7M//vGPtlbXRKPRMABMo9HYt4LjmxjL8GZs5y9Ms3681cCC1+1j4zd8ZXe9CCE8az+jNrW0dDod8vPzERcXZ5onEAgQFxeHnBzLd67JyckxKw8A8fHxpvIlJSVQq9VmZeRyOWJiYrqs880334Svry+mTJmCt99+G62t3d++q6WlBXV1dWZTn5jOax0HDPx221tajToD9AYanoaQgSCypXB1dTUMBgMCAgLM5gcEBODSpUsW36NWqy2WV6vVpuXt87orAwDPP/887r33Xvj4+CA7Oxvr169HRUUF/ud//sfidjMzM/H666/bsns9U04CZHL+Xojqs8CIqfCW3fnx1TXr4esp7b/tEUIscpmrh2lpaXjwwQcxadIkrFq1Cu+++y62bNmClpYWi+XXr18PjUZjmsrKyvpWAYEQCH6Af952XkskFMBTSsPTEDKQbAotPz8/CIVCVFZWms2vrKyEUqm0+B6lUtlj+fZHW9YJADExMWhtbcW1a9csLpdKpfD29jab+sz0PcQ7X56mK4iEDCybQksikWDq1KnIysoyzTMajcjKykJsbKzF98TGxpqVB4BDhw6ZyoeGhkKpVJqVqaurQ25ubrfrBIDCwkIIBAL4+/vbsgt90x5apTlAqw4AdTAlZKDZdE4L4A/Tli5dimnTpiE6OhqbNm1CY2Mjli1bBgBYsmQJRowYgczMTADACy+8gNmzZ+Pdd99FQkICdu3ahdOnT+PDDz8EAHAchxdffBF/+MMfEBYWhtDQUGzYsAEqlQqJiYkA+JP5ubm5eOihh+Dl5YWcnBysWbMGv/zlLzFs2LB++lFYwX8c4O4HNFUDN/OB4FjI3ejwkJCBZHNoLV68GLdu3UJ6ejrUajUmT56MgwcPmk6kl5aWQiC404CbPn06du7ciVdffRWvvPIKwsLC8OWXX2LixImmMmvXrkVjYyNSUlJQW1uLBx54AAcPHoRMJgPAH+rt2rULr732GlpaWhAaGoo1a9YgLS2tr/tvG44DQmcC33/Bn9cKjqW+WoQMMI6xTuOtDFJ1dXWQy+XQaDR9O791+q/AvjX8Sfll+7H2H2fx2ekbePmRcKQ+HNZ/FSZkiLH2M+oyVw+dRvuXp2/kAfpmU0vrTGkt3UqMkAFAoWUrn9GA9wjAoAPKcjE7nL8QcPhSFZI+OonKOm0vKyCE9AWFlq04zmwI5gfC/PCXpdPgJRMh//pPeGzLcZy6VuPYOhIyiFFo2aPjEMwA5owLwL9SH0B4gCdu1bcg6cOT+CTnGobI6UJCBhSFlj1C20Lr5hmgpZ6f5eeBL341AwmTAtFqZEj/5/d4ec85aPUGB1aUkMGHQsseilHAsFCAGYDrd77U7SEV4b2kKXjl0QgIOODzMzew8P1slNU02bUZxhguqetwrbqxv2pOiMuj0LKX6bzWd2azOY5Dyqwx+PvyGPh4SPB9eR0ef+84jl+utmq1Wr0BRy5V4bdfFGH6m4cxd9N/8LM/focduV1vFkvIUET9tOxV9A/g8+X86A+rut7IFQBu1jbj2b/n49wNDQQcsHZuBFbOGg2O48zKVdVrceRSFb69WIXjl6vR3OGQUiTg0Grkf0VJ0UF47fEJkIqEfa8/IU7G2s8ohZa96iuBd8MBcMDaHwF3H4vFtHoDNnx5HnvybwAA5k1U4u0no3D9diOyLlYh62Ilzt7QmL0nUC7DwxH+iBsXgNgxvvjriRK8/XUxGAMmBynwwS+nQimX9X0fCHEiFFqd9HtoAcDWGODWJWDx34Fx87stxhjDjtxSvP7v76E3MEhEgi4dUaNGyjFnXAAejvDHBJU3OM0NoCwXKMsDDC24YlTh3QLgnDYAek8V/veX0zAtxHJQEuKKrP2M2vzdQ9JB6Cw+tEqO9RhaHMfhl/cHY1ygN579ez6q6lsgEwvwwNjhiBvnj4fDfeDfdBkoOwxkn+TDqu6m2TrGAngfAGRAk16KH/+qwo8jxiM0Ygq44fcAw+/hO74KxXd1lwlxNGpp9cXFfwO7fwkMjwBW51r1ltomHX64fhOTuR8gKT8NlJ0EbuQD+k5XCDkhoIwERt0PSDyB6h+A6h/Abl8FZ+zmy9kCER9c4+YD058H3BR92z9ydzTcAn48AoTH86PhEgDU0hoYwTMAcHxrq74S8GobMlrfDGhuALWlgKaMf6wtAzRlUNSWIbruJoBO/yukciDoPiDofmBUDKC6F5B6dtkkZ9CD1ZTg6++OoagwD6O5m5gkrcQYrhwCfSMfbv95l/9i98yXgfueAcR0/stp/PA18OWv+OGNvAKBhHeBiARH18qlUEurrz6YCajP8WFj1PPh1Gj5dmpmhoXcCaig+/nWmsC2HijHfriF5z4tgKZZDz8PMT5aMAJTUAx89xYfpAAgDwIe+i0waRE/ZDRxDH0zcCgdyOPHkYNAzP+9AMD4J4B5GwGv7kfqHQroRHwndy20DqUDJ/7Udb7Ekw8MRVDb46i256MAn1DAw69fNl96uwkpfzuNS+p6iAQc0uePx1PRI8Gd2wUceePOuTH/CUDca0DYz/jvTw6E21f5R98xA7M9Z1X5PfD5M3du9BvzLPDgb/i/mxN/4jspS+XAI78H7l0ycL8fJ0Oh1cldC62mGuD0X/g7UCvawkkeBLgNG7A/viZdK9Z9XoR/ny0HADwc4Y9nHxyDaSoZuFMf8YeL2rZuFSEzgbjXgZFT705lGAOuHgayNwM/HuXnjX4ImJ4KjJkztD6QjAG5f+b/sRlaAA9/IPF9IKzDLfUqzgH/eg6oKORfh8wE5v9pSAY9hVYndy20nARjDB//pwSZX11EW19URAUpsGJmKOaOlkKUvYn/ABna7l40/gng4XTAb2z/VMCgB87vBbK3AJVF/DxOCIABrK17h/94IHY1EPkkIBrkt1trqOLPXV05xL8OewR44n8Bz+FdyxpagdwPgMN/AFqbAaGUb4lNf86+q8HNP/GBOYD/OPsDhVYngz202l2pasBfjv+Iz8/cNPUFGznMDf9vRigW3yOAx4mNwNmdfJBwQv5wZMICQDUFkNnxc2mpB/L/Dzj5PlDHd6CF2INf7/3PAmhrbZz5BNA18Ms9A4DoFcC05d12yu1XjAH1FYC2jm/B3O1uIT98A/zzV0DjLT6AHvkDv7+9BUhNCT8q7o9H+NcBkcDjm4ER91ouzxhQex1QF/FTxTn+sePvQT6SnxRBbc9H3XntpQKE3VyLa9UBLXV8C72ljv/ZtT+2NgMyBf+7c/MB3H3552L3PoUkhVYnQyW02lU3tOCTnOv4W841/NTEn/D1lonw3zHBeCa8GX65bwE/fNXhHRzf12vENP7QccRU/jxYd3/UdRV86+D0NqCl7dDTwx+ISbEcRs21wJn/A05+ANTzh7EQuQFTkoH7f9V/h0N6LX8RovI8oD7PP1ae51sfAB8iyolAYBQQOBlQTQaGjwNEkv7Z9qF0IO/P/Gv/8cDCvwAB461fB2PA2V3A1+v5OnMC/ucz69f8leiO4aQuuvOztwcn4IPLWwUYW83DqdWOwSxFsg4hNqzt0Zefd+8SPih7QKHVyVALrXbNOgM+P3MDfzlegpK20SLEQg7zo1R4fuwthFzdCdw4DWhKu75Z5MZ/qEe0hdjIaYCuiT8EPLf7ztUv3zD+nNWkX/TevcKg528Mkr2Fv+oKAOD4y/6xqXyYgPEf3m4fwT/qm/iAUhfxJ7srzwPVl/kT251xQkDsdqe115FQwgdMYBS/v4GTgYAJvR/CMgYYDfz2qn8A9qZ0ONm+ij93aG93k4ZbwMHfAOf/0XM5gZi/S5RyEt+vL3ASX3ehBNDc5INOc8P8sbaMv0Bj0PVeD4knIPXmW+HtjyIZ3wJrqgGaa4Cm272vK+Uo35rvAYVWJ0M1tNoZjQzfXqzEx/8pQV6HkVUfGOuHB+8Zjql+eow3XoZUXQDcPA3cLOj9v3jQ/cCM54HweTZ31wBjwLX/ANnvAZe/tmOPeuA2DAiYyH+IAybwz4dH8B/kn0r4k97lhUDFWf651sJ+CkR8K8HY2hZMxjsBZTTw8zv3tQMAj+FtJ9t/1j/78sPXwL40/pBPKuf3qT2clJGA3z32tRKNRr5rjuYGH2BCSddwknpb102GMf6fQVNbgDXX3Hne/vjgesvn8zqg0OpkqIdWR4VltfjoPz/iq6IK00l7ABBwQHiAF6JGKjBppBfu86rB6JZLEJXn8/d5rDzPf2AjEoAZLwBB0f1ToVvFQM5WvvVmy2EJJ+BbecqJfDAFTOSfewVaf26FMeCna3cCrLyQf2w/nLRF+Dz+HJRnP99A2KAHGqv5flwudGLdVhRanVBodVVW04R/Ft5EYZkGZ2/U4lZ9S5cyEpEAE1TeiBqpwORAKcJ9xXDz9oNUJIBMLIRUJIBUJIBI2A9Dsxn0QGtL2weTs+7xbnyIGeNbINpa/rBSIORbXpyAf25pnkAMSNz7vy5DCIVWJxRaPWOMQV2nxdkyDc7dqMW5G/xjnbbVqveLBBwfYGIhZG2PUpEACncxVAo3jGibVAo3jBjmBpXcDW4S6qFP7qDvHhKbcByHQLkbAuVumDuR/zqJ0chwvaYJ527UorCMD7Lrt5vQojegpdUIneHO8DqtRoZWnQGNOuvHxPfxkLQFmQwjFO5QKWQI9fPAmOGeCPJxh1AweA+FiP2opUXsZjAy6FqN0LaFWEurAVr9nUet3oCaRh1u1jajvLb5zuNPzb2Gm0QkwGg/D4zx98TY4Z4Y6++JMcM9MXq4B2RiaqENRtTSInedUMDBTSK0+TCPMYY6bStu/sSHWLmGD7IbPzXjx+pG/HirAS2tRlxS1+OSut7svRwHBA1zx1h/T4z284DCXQxPqQgeUhG8ZCJ4SsXwkArNnntIRBBQq23QoNAiA47jOMjdxJC7iTFe1fU/qsHIcPOnZly5VY8rVQ1mU522FaU1TSitacJhG7bJB5sQHlIR/1zCB13XeUJTCEpFAgg4DkIBB4GAg4ADhBz/XNj2un25UMAhwFsGXw9Jl3sAkP5FoUWcjlDAYZSvO0b5uuPhiADTfMYYqht0fIDdakDp7UbUa1tR39KKBm0rGlpa0djSivq25w0trTC09elofw10vULan7ykIoT4eSDEzwOhvu4dnntgmEc/9LondE6LDF6MMbS0Gk0h1tjS+dGAJl3HeQY0dliuNxhhYPwFCYORwcj4iX+OO8+NDDoDw+3GFvT0aZK7iU1hFuzrgQBvGfy9pPD3lsLfSwY/T0n/dB1xUXROiwx5HMdBJhZCJhZiuNfdH1VCqzegtKYJJdWNuFbdiGu3G9ueN0Fdp4WmWY+zZbU4W1bbTX0BXw8JhnvxYTbcS8qHmpcUnjIxtHoDmnUGNOkMaNYb0Kxr7fCcf2zSGaDVG+AlEyE8wKvD5Alfz8Exsga1tAgZAM06A67d5sOs5HYjymqaUFXXgqr6FlTVa1HdoDMdyt4tvh4SU4CFK9vCzN8LcnfnuBkKdS7thEKLODODkaGmUYeqei2q6ltwq67lzvP6FjS0tMJdIoSbWAg3iajDc2Gn5yLIxAJUN7Tgh8oGXK6sR3FlPcpqmrvdtr+XFHI3McRCAcQiASRCDhKRgH8tFEAiFEAs5PjnbfMlovb57fP4zsUdl3csN2mkHF6ynsORDg8JcSFCAYfhbYeEE+7C+pt0rbhS1YBidT0uVzXgh8p6XK5swM3a5rbW3t29QPGv1BmYNFLRL+ui0CJkCHCXiDBppKJLcNRr9fjxViMada3QGxj0rUboDfy3HfQGvvOw3tBhXiuDzmAwLWtpL9/hUdfpUW8wwkPaf1FDoUXIEOYlEyMqSOHoathk6F5fJYS4JAotQohLsSu0tm7dipCQEMhkMsTExCAvL6/H8nv27EFERARkMhkiIyNx4MABs+WMMaSnpyMwMBBubm6Ii4vD5cuXzcrU1NQgOTkZ3t7eUCgUWL58ORoaLAydSwgZ1GwOrd27dyMtLQ0ZGRk4c+YMoqKiEB8fj6oqy3dVzs7ORlJSEpYvX46CggIkJiYiMTER58+fN5XZuHEjNm/ejA8++AC5ubnw8PBAfHw8tNo7o1gmJyfj+++/x6FDh7Bv3z4cO3YMKSkpduwyIcSlMRtFR0ez1atXm14bDAamUqlYZmamxfKLFi1iCQkJZvNiYmLYypUrGWOMGY1GplQq2dtvv21aXltby6RSKfv0008ZY4xduHCBAWCnTp0ylfnqq68Yx3Hs5s2bFrer1WqZRqMxTWVlZQwA02g0tu4yIWQAaDQaqz6jNrW0dDod8vPzERd35w65AoEAcXFxyMnJsfienJwcs/IAEB8fbypfUlICtVptVkYulyMmJsZUJicnBwqFAtOmTTOViYuLg0AgQG5ursXtZmZmQi6Xm6agoJ5vX0QIcQ02hVZ1dTUMBgMCAgLM5gcEBECtVlt8j1qt7rF8+2NvZfz9zW8WIBKJ4OPj0+12169fD41GY5rKysqs3EtCiDMbtP20pFIppNI7XxBlbd9Wqqurc1SVCCE9aP9ssl6+WWhTaPn5+UEoFKKystJsfmVlJZRKpcX3KJXKHsu3P1ZWViIwMNCszOTJk01lOp/ob21tRU1NTbfb7ay+nh8Bkw4TCXFu9fX1kMvl3S63KbQkEgmmTp2KrKwsJCYmAgCMRiOysrKQmppq8T2xsbHIysrCiy++aJp36NAhxMbGAgBCQ0OhVCqRlZVlCqm6ujrk5ubi2WefNa2jtrYW+fn5mDp1KgDg8OHDMBqNiImJsaruKpUKZWVl8PLy6nVkybq6OgQFBaGsrGzQf7l6qOzrUNlPwHX3lTGG+vp6qFSqXgvaZNeuXUwqlbLt27ezCxcusJSUFKZQKJharWaMMfbUU0+x3/zmN6byJ06cYCKRiL3zzjvs4sWLLCMjg4nFYlZUVGQq8+abbzKFQsH++c9/snPnzrEnnniChYaGsubmZlOZuXPnsilTprDc3Fx2/PhxFhYWxpKSkmytvlWsvYoxGAyVfR0q+8nY4N9Xm0OLMca2bNnCRo0axSQSCYuOjmYnT540LZs9ezZbunSpWfnPPvuMhYeHM4lEwiZMmMD2799vttxoNLINGzawgIAAJpVK2Zw5c1hxcbFZmdu3b7OkpCTm6enJvL292bJly1h9fb091e/VYP+ldzRU9nWo7Cdjg39fh8x4WrYYSmNvDZV9HSr7CQz+faXvHloglUqRkZFhdvVxsBoq+zpU9hMY/PtKLS1CiEuhlhYhxKVQaBFCXAqFFiHEpVBoEUJcCoUWIcSlUGh1YuuorK7otddeA8dxZlNERISjq9Uvjh07hvnz50OlUoHjOHz55Zdmy5kVo+S6it729emnn+7ye547d65jKtuPKLQ6sHVUVlc2YcIEVFRUmKbjx487ukr9orGxEVFRUdi6davF5daMkusqettXAJg7d67Z7/nTTz8dwBreJQ7tj+9kbB2V1VVlZGSwqKgoR1fjrgPAvvjiC9Nra0bJdVWd95UxxpYuXcqeeOIJh9TnbqKWVht7RmV1ZZcvX4ZKpcLo0aORnJyM0tJSR1fprrNmlNzB5ujRo/D398c999yDZ599Frdv33Z0lfqMQquNPaOyuqqYmBhs374dBw8exPvvv4+SkhLMnDnTNObYYGXNKLmDydy5c/HJJ58gKysLb731Fr777jvMmzcPBoPB0VXrk0E7cinp3rx580zPJ02ahJiYGAQHB+Ozzz7D8uXLHVgz0p9+8YtfmJ5HRkZi0qRJGDNmDI4ePYo5c+Y4sGZ9Qy2tNvaMyjpYKBQKhIeH48qVK46uyl3VcZTcjobC7xgARo8eDT8/P5f/PVNotek4Kmu79lFZ20dZHawaGhpw9epVs+GuB6OOo+S2ax8ld7D/jgHgxo0buH37tsv/nunwsIO0tDQsXboU06ZNQ3R0NDZt2oTGxkYsW7bM0VXrVy+//DLmz5+P4OBglJeXIyMjA0KhEElJSY6uWp81NDSYtSRKSkpQWFgIHx8fjBo1Ci+++CL+8Ic/ICwsDKGhodiwYQNUKpVp+HBX0tO++vj44PXXX8fChQuhVCpx9epVrF27FmPHjkV8fLwDa90PHH350tn0NCrrYLF48WIWGBjIJBIJGzFiBFu8eDG7cuWKo6vVL44cOcIAdJnaR9O1ZpRcV9HTvjY1NbFHHnmEDR8+nInFYhYcHMxWrFhhGhbdldF4WoQQl0LntAghLoVCixDiUii0CCEuhUKLEOJSKLQIIS6FQosQ4lIotAghLoVCixDiUii0CCEuhUKLEOJSKLQIIS7l/wNbHvnPnjL2AAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:39.159910Z", + "iopub.status.busy": "2024-02-29T23:27:39.158900Z", + "iopub.status.idle": "2024-02-29T23:30:35.025439Z", + "shell.execute_reply": "2024-02-29T23:30:35.024321Z" + }, + "papermill": { + "duration": 175.890298, + "end_time": "2024-02-29T23:30:35.028077", + "exception": false, + "start_time": "2024-02-29T23:27:39.137779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:30:35.068851Z", + "iopub.status.busy": "2024-02-29T23:30:35.068532Z", + "iopub.status.idle": "2024-02-29T23:30:35.089170Z", + "shell.execute_reply": "2024-02-29T23:30:35.088192Z" + }, + "papermill": { + "duration": 0.043655, + "end_time": "2024-02-29T23:30:35.091488", + "exception": false, + "start_time": "2024-02-29T23:30:35.047833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.014026NaN0.0012282.7064350.0316440.6164290.0400759.105956e-073.2376320.0279350.0642950.0350490.0578080.0110855.944067
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.014026 NaN 0.001228 2.706435 0.031644 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.616429 0.040075 9.105956e-07 3.237632 0.027935 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.064295 0.035049 0.057808 0.011085 5.944067 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:30:35.133977Z", + "iopub.status.busy": "2024-02-29T23:30:35.133704Z", + "iopub.status.idle": "2024-02-29T23:30:35.486775Z", + "shell.execute_reply": "2024-02-29T23:30:35.485861Z" + }, + "papermill": { + "duration": 0.376469, + "end_time": "2024-02-29T23:30:35.489194", + "exception": false, + "start_time": "2024-02-29T23:30:35.112725", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:30:35.531701Z", + "iopub.status.busy": "2024-02-29T23:30:35.530777Z", + "iopub.status.idle": "2024-02-29T23:33:40.420559Z", + "shell.execute_reply": "2024-02-29T23:33:40.419643Z" + }, + "papermill": { + "duration": 184.93115, + "end_time": "2024-02-29T23:33:40.440396", + "exception": false, + "start_time": "2024-02-29T23:30:35.509246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tvae/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:40.481967Z", + "iopub.status.busy": "2024-02-29T23:33:40.481633Z", + "iopub.status.idle": "2024-02-29T23:33:40.499780Z", + "shell.execute_reply": "2024-02-29T23:33:40.498832Z" + }, + "papermill": { + "duration": 0.041954, + "end_time": "2024-02-29T23:33:40.501851", + "exception": false, + "start_time": "2024-02-29T23:33:40.459897", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:40.541536Z", + "iopub.status.busy": "2024-02-29T23:33:40.541203Z", + "iopub.status.idle": "2024-02-29T23:33:40.546741Z", + "shell.execute_reply": "2024-02-29T23:33:40.545643Z" + }, + "papermill": { + "duration": 0.028052, + "end_time": "2024-02-29T23:33:40.548749", + "exception": false, + "start_time": "2024-02-29T23:33:40.520697", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.42948681272958456}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:40.590218Z", + "iopub.status.busy": "2024-02-29T23:33:40.589936Z", + "iopub.status.idle": "2024-02-29T23:33:40.956533Z", + "shell.execute_reply": "2024-02-29T23:33:40.955442Z" + }, + "papermill": { + "duration": 0.389471, + "end_time": "2024-02-29T23:33:40.958584", + "exception": false, + "start_time": "2024-02-29T23:33:40.569113", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCAUlEQVR4nO3deZhT9b0/8PfJyb7PvjHDADMsirKjQhFRrrValUur+NMqSBHvFWot1z5Kn1uoW8FWLdbr5V69sviIYItYrdatKiq1goIsog7bDDPMvmaZ7Dnf3x8nyUxmTTJJThI+r+fJk5mTk5NvMjOf+a6fL8cYYyCEEAnJpC4AIYRQICKESI4CESFEchSICCGSo0BECJEcBSJCiOQoEBFCJEeBiBAiOQpEhBDJUSAihEiOAhGJq88++wy/+c1v0NXVJXVRSBqhQETi6rPPPsNDDz1EgYhEhQIRIURyFIhI3PzmN7/BL3/5SwDAmDFjwHEcOI6DXq/HggUL+p0vCAJKSkrw4x//OHTsiSeewJw5c5CTkwONRoMZM2Zg9+7dA77eSy+9hBkzZkCj0SA7Oxu33HIL6urqEvPmSEJxlAaExMvRo0exceNG7Ny5E3/4wx+Qm5sLADh9+jQefvhh1NfXo7CwMHT+J598gvnz5+PPf/5zKBiVlpbihhtuwAUXXACPx4Ndu3bhwIEDePPNN3HdddeFnvvYY4/h17/+NW6++WbMnz8fra2teOaZZ6DX6/HVV1/BbDYn9b2TEWKExNHvf/97BoBVV1eHjlVVVTEA7Jlnngk795577mF6vZ45HI7Qsd5fM8aYx+NhkydPZldeeWXoWE1NDeN5nj322GNh5x47dozJ5fJ+x0nqo6YZSbjx48dj6tSpeOWVV0LH/H4/du/ejeuvvx4ajSZ0vPfXnZ2dsFgsmDdvHg4dOhQ6vmfPHgiCgJtvvhltbW2hW2FhISorK/HRRx8l542RuJFLXQByfliyZAl+9atfob6+HiUlJdi7dy9aWlqwZMmSsPPefPNNPProozh8+DDcbnfoOMdxoa9PnjwJxhgqKysHfC2FQpGYN0EShgIRSYolS5Zg7dq1+POf/4z77rsPf/rTn2AymXDNNdeEzvn0009xww034PLLL8d///d/o6ioCAqFAlu3bsXLL78cOk8QBHAch7fffhs8z/d7Lb1en5T3ROKHAhGJq941l97GjBmD2bNn45VXXsHq1auxZ88eLFq0CCqVKnTOq6++CrVajXfffTfs+NatW8OuNW7cODDGMGbMGIwfPz4xb4QkFfURkbjS6XQAMOCExiVLluDzzz/Hli1b0NbW1q9ZxvM8OI6D3+8PHaupqcFf/vKXsPMWL14Mnufx0EMPgfUZ9GWMob29PT5vhiQNDd+TuPriiy8we/ZsXHvttbjlllugUChw/fXXQ6fT4dy5cygrK4Ner4dCoUBTU1NYf86HH36Iq666CvPmzcOtt96KlpYWPPvssygsLMTRo0fDgs7GjRuxdu1azJkzB4sWLYLBYEB1dTVee+01rFy5Evfff78Ub5/ESsIRO5KhHnnkEVZSUsJkMlm/ofy5c+cyAGzFihUDPveFF15glZWVTKVSsYkTJ7KtW7ey9evXs4F+VV999VX2ve99j+l0OqbT6djEiRPZqlWrWFVVVaLeGkkQqhERQiRHfUSEEMlRICKESI4CESFEchSICCGSo0BECJEcBSJCiOTSeomHIAhoaGiAwWAYdGkBIUQ6jDHYbDYUFxdDJhui3iPlJCafz8f+8z//k5WXlzO1Ws3Gjh3LHn74YSYIQkTPr6urYwDoRje6pfitrq5uyL9lSWtEjz/+ODZv3ozt27fjwgsvxJdffok777wTJpMJ995777DPNxgMAIC6ujoYjcZEF5cQEiWr1YrS0tLQ3+pgJA1En332GW688cZQCtDy8nLs3LkTBw4ciOj5weaY0WikQERIChuu60TSzuo5c+bggw8+wIkTJwAAR44cwb59+/CDH/xgwPPdbjesVmvYjRCS/iStET344IOwWq2YOHEieJ6H3+/HY489httuu23A8zds2ICHHnooyaUkhCSapDWiP/3pT9ixYwdefvllHDp0CNu3b8cTTzyB7du3D3j+2rVrYbFYQjfaOoaQzCDp6vvS0lI8+OCDWLVqVejYo48+ipdeegnffffdsM+3Wq0wmUywWCzUR0RixhiDz+cLS8hGIsPzPORy+aB9QJH+jUraNHM4HP3mFvA8D0EQJCoROd94PB40NjbC4XBIXZS0pdVqUVRUBKVSGfM1JA1E119/PR577DGUlZXhwgsvxFdffYWnnnoKy5cvl7JY5DwhCAKqq6vB8zyKi4uhVCppYmwUGGPweDxobW1FdXU1Kisrh560OARJA9EzzzyDX//617jnnnvQ0tKC4uJi3H333Vi3bp2UxSJpptHiRJ5eBTkf3R+Bx+OBIAgoLS2FVqtNUOkym0ajgUKhwNmzZ+HxeKBWq2O6jqSByGAwYNOmTdi0aZOUxSBpxOMT0Gx1oTRbDBwOjw+vfVUPo1qBf51WAp0q+l/pWP+LE1E8Pj/6CZC08vdvm7HnUD2+bRTnkFmcXvAch1abG3890gC/INnYCxkBCkQkbdS0daOqyQYAMGvF3T+KTBrcPLMUKoUMjRYXDtV2SllEEiMKRCRt7K8W9yubUmpCkUkTOp6lU+KK8fkAgAPVHXB6aBg+nsrLyxPefUKBiKSFRosTDV0u8DIOs8qz+z0+qciAPIMKHp+Ao+e6kl9AMiIUiEha+KZB7BOaUGgYsEOa4zjMGJ0FAPi6wdpvB9jzncfjkboIQ6JARFKeIDCcbLEDACYVDj47tyJfD5NGgTG5Wnj9IwtEHp8w6M3nFyI+1xvBubG44oorsHr1aqxevRomkwm5ubn49a9/HQrA5eXleOSRR3DHHXfAaDRi5cqVAIB9+/Zh3rx50Gg0KC0txb333ovu7u7QdVtaWnD99ddDo9FgzJgx2LFjR0zli1ZaZ2gk54f2bg/8AoNGyWNUlmbQ8xS8DHfOLY/LpMRnPzo16GNjcnVYNK0k9P1zn5weNPCNytLgppmloe+3/KO6Xx/WL/5lfExl3L59O37605/iwIED+PLLL7Fy5UqUlZXhrrvuAgA88cQTWLduHdavXw8AOH36NK655ho8+uij2LJlC1pbW0PBbOvWrQCAZcuWoaGhAR999BEUCgXuvfdetLS0xFS+aFAgIikvz6DCysvHotPhgUw2dJA5n2ZGl5aW4g9/+AM4jsOECRNw7Ngx/OEPfwgFoiuvvBL/8R//ETp/xYoVuO2223DfffcBACorK/HHP/4R8+fPx+bNm1FbW4u3334bBw4cwKxZswAAL7zwAiZNmpTw90KBiKQFBS9DviGyWbuCwNBic8OsVUCt4GN6vVULKgZ9rG8sXHn5uEHP7RsXl88dE1N5BnLppZeGBd7LLrsMTz75ZGjx7syZM8POP3LkCI4ePRrW3GKMhZa6nDhxAnK5HDNmzAg9PnHiRJjN5riVeTAUiEhKY4xFXcvZ81U96joc+P6FhbigOLasDEp55N2niTp3pHQ6Xdj3drsdd99994BpmMvKykIJCqVAgYiktKpmG76o7sAFxabQqNhw8g0q1HU40GR1xhyI0sH+/fvDvv/8889RWVkJnh+4Fjh9+nR88803qKgYuLY3ceJE+Hw+HDx4MNQ0q6qqQldXV1zLPRAaNSMpraHLiTa7B3a3L+LnFJrEJlyz1Z2oYqWE2tparFmzBlVVVdi5cyeeeeYZ/PznPx/0/AceeACfffYZVq9ejcOHD+PkyZN4/fXXsXr1agDAhAkTcM011+Duu+/G/v37cfDgQaxYsQIazeADBPFCgYiktCaLGEwKjZGv6i4I9CW12twZvfbsjjvugNPpxOzZs7Fq1Sr8/Oc/Dw3TD+Tiiy/Gxx9/jBMnTmDevHmYNm0a1q1bh+Li4tA5W7duRXFxMebPn4/Fixdj5cqVyM/PT/h7oaYZSVk+v4A2e/SByKiRQ63g4fL60WZ3oyCK56YThUKBTZs2YfPmzf0eq6mpGfA5s2bNwnvvvTfoNQsLC/Hmm2+GHbv99ttHVM5IUI2IpKw2e8/8IaMm8v+ZHMehwKgCINaKSOqjQERSVpPVBUCsDUU7cpatE9OWtnen9tIGIqKmGUlZzYFAFEvTalyeHjqVHCXmxHe0SmHv3r1SFyGuKBCRlKVV8sjWKZEfaGZFozRbG8riSFIfBSKSsuZV5mFeZZ7UxSBJQH1EJGO129041WKPag4SkQYFIpKSfH5hxDmF/v5tM/56pAENXc44lYokCgUikpK+qOnEf+89jf1n2mO+hkkj5rW2OL3xKhZJEApEJCW1d7vh8QmQ87Gn9TAGA5GDAlGqo0BEUlK7XZz/k6OLfsQsyKgWA5HVRYEo1VEgIinHLzB0BWoxOfrY91Onpln6oEBEUo7N5YXAGOQyDvoYdm4NMgX2PrO5fBAyePFrJqBARFJOsAZj0ipGlPpVr5SDl3HwCwx2T4RD+IwBPk/yb1GMEL744ovIycmB2x2+jm7RokVJWaCaCDShkaScUCAKNK0GxRjQeARoPg4otUD5PECXG3pYJuNwxYQ8qBU8VJFmRvR7gU+fjLXosZv3H4A8smboTTfdhHvvvRdvvPEGbrrpJgDi7htvvfXWkCvrUxnViEjK0SrlGJunQ/Fw68RqPgWq3ga6aoGW74BDLwLd4cP9F48yY3yBASp5bLmrU5FGo8Gtt94a2nkDAF566SWUlZXhiiuukK5gI0A1IpJyKvL1qMjXD31S51ng7Gfi12WXiN/bmoCqvwHTftI/a32keIVYO0k2fpjaXx933XUXZs2ahfr6epSUlGDbtm1YtmxZ2u5iQoGIpB/GgNMfiPdFU4BxVwIuC3DgOcByDug4A+SIO2tYXV60WN3QKPnIVuJzXMRNJClNmzYNU6ZMwYsvvoirr74ax48fx1tvvSV1sWJGgYikFMYYXF4BaoVs8P/ubScBW7MYMMZeIR5Tm4CiacC5L4D6g6FAdLrFjr1Vrags0GdcSpAVK1Zg06ZNqK+vx8KFC1FaWjr8k1IU9RGRlOL0+vE/H5/G/3x8ZvB80/UHxfvi6WIndVDJdPG+/TTgsgIADGrxf63dlXkLX2+99VacO3cOzz//PJYvXy51cUaEAhFJKcERMwXPgR9oV1dHB9BZIzahiqeFP6bNBkyBraDbxD26dIF5SJm4At9kMuFHP/oR9Ho9Fi1aJHVxRoQCEUkpwRnVgw7dNx0T77PHAhpz/8fzAtsjt34HAKEJkd1uf0ZOaqyvr8dtt90GlSr2pTCpgAIRSSlDziFiLBRgUDB54AvkVgYuVA94XdAp5eA4QGAMDq8/ASWWRmdnJ1577TXs3bsXq1atkro4I0ad1SSlDBmI7C1i00wmB3IG2ZteYxabaI4OwFIHWW4ldEo57G4fut2+ES0ZSSXTpk1DZ2cnHn/8cUyYMEHq4oxYZvxUSMYIpuwIrhML0/qteJ8zdugh9qzynr6k3EroVGIgsrt9KIh7iaUx2L5l6YoCEUkpwRqRWdMn0DAmzp4GevqBBmMeDdQfEgMRgEvGZsPnZxm70WImoEBEUgZjDBUFelgcXpj71oicneJNxofmCA3KXCbed7cBHgfG5Q09S3ukKWnPd/H4/CgQkZTBcRwWTBhkn/X20+K9aRQgH2aESKnt6SeyNQ4auBQKMdg5HA5oNJk12TGZHA4HgJ7PMxYUiEh66Dgj3mcPUxsKMhaLgchaD7t+NJosLih5GcpyeiZA8jwPs9mMlpYWAIBWq03btVpSYIzB4XCgpaUFZrMZPB/7wmIKRCRlODw+yDgOakWfX2i/V1xhDwzfLAsyFgNNXwPWBpzTOPD2sSaMytKEBSIAKCwsBIBQMCLRM5vNoc8xVpIHovr6ejzwwAN4++234XA4UFFRga1bt2LmzJlSF40k2f7qDhyu7cIlY7MxZ1xPXiF0ngUEH6A2AtqcyC5mDMywtjZAN0oMbM4B5hFxHIeioiLk5+fD66WUstFSKBQjqgkFSRqIOjs7MXfuXCxYsABvv/028vLycPLkSWRlZUlZLCIRa2DEzKDq09fQu1kWadNJlw/wcsDnhtbfBQBweAaf0MjzfFz+oEhsJA1Ejz/+OEpLS8MSPI0ZM0bCEhEpDbq8o7NavM8eG/nFZDJAXwhYzkHraQOggdPjh19gA69hI5KSdInHG2+8gZkzZ+Kmm25Cfn4+pk2bhueff37Q891uN6xWa9iNZAbGWKhGFBaIXFax05njeoblI6UXR+DUrnbIAjWpgZpnRHqSBqIzZ85g8+bNqKysxLvvvot///d/x7333ovt27cPeP6GDRtgMplCt3TOv0LC2d0++AQGGceFUncA6Omk1hcAiignJAYCEdfdAo1S/FV3ZOAq/EwgaSASBAHTp0/Hb3/7W0ybNg0rV67EXXfdhf/5n/8Z8Py1a9fCYrGEbnV1dUkuMUmUYLPMqJFD1rvp1HVWvM8aHf1F9YEFHfZmaAIjcUP1ExHpSNpHVFRUhAsuuCDs2KRJk/Dqq68OeL5KpUr7dAdkYIMudg3WiMwxBCJdntik8zgwp1INnzwHeQb6/UlFkgaiuXPnoqqqKuzYiRMnMHp0DL90JK1l6ZSYWmpGlq7XGjNnl3jjZOKM6mjxCnG4v7sN4zTdQM7I5rqQxJE0EP3iF7/AnDlz8Nvf/hY333wzDhw4gOeeew7PPfeclMUiEigxa/rnlA7WhgyFwy/rGIwuT1xzZm+JfDIkSTpJ+4hmzZqF1157DTt37sTkyZPxyCOPYNOmTbjtttukLBZJFaFmWZSjZb0F+om6OxpwqsWGc52OOBSMxJvkM6t/+MMf4oc//KHUxSASa7G5YFQrepZ3MDayjuqgwMhZZ2sD/trWiIp8PUZlaYd5Ekk2ShVLJOfy+rHj81ps3nsabl9gVMvVJc4h4mSAMYb+oaDAkhC1twtgAs0jSlEUiIjkghMZtUq+Z2toS714bygc2YaHahPAy6HgGNQ+K1wUiFISBSIiuQGH7q2BQBTcHihWHAdocyHnOWi8XXDSPKKURIGISG7AQGQ5J96PpFkWpMuFXMZB6+2EyytQRsYURIGISK6rb8J8nxvobhW/HmmNCAC0uVDwMmh8nRAYg9snjPyaJK4oEBHJ9asRWRvEUTO1CVAZRv4CulxxDZvfAgDUT5SCJB++J6RfIAo2y+JRGwJCI2eTTB4UX1QAjZLyDqUaCkREclPLzOhyeJAdXN4R6qiOQ/8QAKjNgEyOAh1QYBIAOQWiVEOBiEhuelmvjJyC0BOI4tFRDYhJ0rTZ4jKP7nZAQxlAUw31EZHU4mgDfB5xwaouL37X1eXC4fHhXH0tWm3u+F2XxAUFIiKpLocHLTYXPMGRrNCwfYlYk4kXbS6arW4c+vYUTjTb4nddEhcUiIikjpyzYMfntfjnmXbxgK1JvDcWxfeFtNmQ8xzUPgtNakxBFIiIpLocHgCAOThiZm8W7/Vxzh2kyRbnEnktcPkoEKUaCkREUsGhe7NWAQh+MXcQEFo1HzeaLMhlHOSCCx6nPb7XJiNGgYhIhjEGS2BWtVmjBBzt4kaKcmX8R7bkSsg0RvF1HZ3xvTYZMQpERDK2vjt3hJplBZFvpBgFuV7cPZY5OuJ+bTIyFIiIZCyhDRUDO3fYegWiBOD14gxr3kmBKNXQhEYimZ7+ocCMantiA5HKkIuxuTqMzuHAGAOXgFoXiQ0FIiKZQpMa8ypzYVArxEWuCQ5Ecn0uCoxqQOVISNOPxI4CEZFMrl6FXH1gdw5np5j+Q8YDutzEvGCwA9zZKQY+CkYpg/qISGqwt4j3ulwxGCWC2gybR0CHtRvdVho5SyUUiIgkBIHhRLMNbXa3mDExOKM6Qc0yAIBMhioLj6pmO9paGxP3OiRqFIiIJGwuH9462oid+2vBGHpqRPGeUd0HCzTPfPb2hL4OiQ4FIiKJ9m5xBbxZpxSH7u3BGlGcZ1T3wTTZAAAhOIObpAQKREQSHd3iGrMcnRLwdANuu9h5nOBAxGnFQESTGlMLBSIiiWAgytIqe4btNVmx73EfIZlWHJHjKBClFApERBKhGpFe2WtGdWJrQwAg04s1IrgsgN+X8NcjkYkpEJ05cybe5SDnEcYY2gOBKFunTFzqjwEo1Ab4ZQr4BUHc1pqkhJgCUUVFBRYsWICXXnoJLpcr3mUiGa7b44fHJ4DjAnmI7MmrERVnaTF6VCmKTBpxtT9JCTEFokOHDuHiiy/GmjVrUFhYiLvvvhsHDhyId9lIhlLyMlw/pRiXj8+DnPnEmc6AuM99gmXrlCgtKRG3LnLSpMZUEVMgmjp1Kp5++mk0NDRgy5YtaGxsxPe+9z1MnjwZTz31FFpbW+NdTpJBlHIZKvL14u4d9mZxuYVKDyh1ySlAYAgf1GGdMkbUWS2Xy7F48WL8+c9/xuOPP45Tp07h/vvvR2lpKe644w40NtLsVTKM0ETGBM6o7sUvMDR5NejodoNR0yxljCgQffnll7jnnntQVFSEp556Cvfffz9Onz6N999/Hw0NDbjxxhvjVU6SQQ7VduJMqx1ev5DwFfd9+QWGN064UNVsh7+bAlGqiGn1/VNPPYWtW7eiqqoK1157LV588UVce+21kAW2fxkzZgy2bduG8vLyeJaVZACX14+Pq8Sm+7/NHweFPQlrzHpR8By8KjMAwO+yQ+5zJ3zuEhleTIFo8+bNWL58OZYtW4aiooG3fcnPz8cLL7wwosKRzBPc3NCglkMjR0+yfENyAhHHcVCoNPDKNPAJDCpnZ1I6ycnQYgpE77//PsrKykI1oCDGGOrq6lBWVgalUomlS5fGpZAkczRZxekeBUa1GIQEv5gsX21OWhlUch4uhQk+v0PssKZAJLmY+ojGjRuHtrb+iwY7OjowZsyYEReKZK6GLicAoNisSXiy/MGoFTK45EZxUiPlr04JMQUixtiAx+12O9Rq9YgKRDIXYwwNXWKNqMSsSVrqj77UCh4uuQk+gdEQfoqIqmm2Zs0aAGI7e926ddBqtaHH/H4/9u/fj6lTp8a1gCRzdHR74PL6oeA55BlUwJnkpP7oSyWXoUsRCEQ0qTElRBWIvvrqKwDif7Zjx45BqVSGHlMqlZgyZQruv//++JaQZIxGi1gbKjRpwHPoaZoluY9mUpERZcpxMFV/IS7zoPzVkosqEH300UcAgDvvvBNPP/00jEZjQgpFMtOFxUYUmdTw+gM1EZ8HkMkBbU5SyzE6RweYy4F6uZiw3+sElNphn0cSJ6ZRs61bt8a7HOQ8wHEccoK7drTUiPeJTJY/FF4BqAyA2yZ2WFMgklTEgWjx4sXYtm0bjEYjFi9ePOS5e/bsGXHBSIZL8ozq3lxeP1ptbuihRxZsYoe1aVTSy0F6RDxqZjKZQjtjmkymIW+x2LhxIziOw3333RfT80lq+/C7Zrx1tBEttkDaGAkDUUOXE7sPnsOR9kBNjDqsJRdxjah3cyzeTbMvvvgC//u//4uLL744rtclqcHrF/Btow0en4AppYF/VKGO6uQHIpVCDEA2ziAeoLlEkotpHpHT6YTD4Qh9f/bsWWzatAnvvfde1Ney2+247bbb8PzzzyMrKyuW4pAUd6a1Gx6fAINaLs4fctt7kuXrkjt0DwBqufhrb5MFBltoLpHkYgpEN954I1588UUAQFdXF2bPno0nn3wSN954IzZv3hzVtVatWoXrrrsOCxcuHPZct9sNq9UadiOp76taselzQbFRbN6HkuVni8s7kkwdqBFZOT0YmFgjGmSSLkmOmDM0zps3DwCwe/duFBYW4uzZs3jxxRfxxz/+MeLr7Nq1C4cOHcKGDRsiOn/Dhg1hfVGlpaWxFJ8kUUOXE40WF3gZhymjzOLBJKaGHYgqUCNy8Qb4GCcm0XfbJCkLEcUUiBwOBwwGsX393nvvYfHixZDJZLj00ktx9uzZiK5RV1eHn//859ixY0fEy0LWrl0Li8USutXV1cVSfJJEX9V2AQAmFBqgUwW6JCXsqAYAOS+DgufAOB4+RaB5Rv1Ekoo5ef5f/vIX1NXV4d1338XVV18NAGhpaYl4kuPBgwfR0tKC6dOnQy6XQy6X4+OPP8Yf//hHyOVy+P3+fs9RqVQwGo1hN5K6LE4vTraINY3pZb36/4JrzCToqA4KNs88ykDnOY2cSSqmCY3r1q3Drbfeil/84he46qqrcNlllwEQa0fTpk2L6BpXXXUVjh07FnbszjvvxMSJE/HAAw+A5yWY5EbiSqPgMX98HlptbnFtGSDOZA52DktUIwKAS8fmQGAMKmse4KijDmuJxRSIfvzjH+N73/seGhsbMWXKlNDxq666Cv/6r/8a0TUMBgMmT54cdkyn0yEnJ6ffcZKelHIZppX1GQkN1oZUhuQlyx/A5JJATag+F2gG1YgkFlMgAoDCwkIUFoYvVpw9e/aIC0QynMT9Q/1oAoGSakSSiikQdXd3Y+PGjfjggw/Q0tICQRDCHo91J9i9e/fG9DySWhhjePd4E0ZlaTGh0AAF36sr0hbY2UXirIhWlxcWhxd6ZkAWIO76KgiAjHZhl0JMgWjFihX4+OOPcfvtt6OoqCi09IMQQEwH+22jDada7BhfYAh/0BbIQSRxIDpc24WDZzsxo8yMy2VyQPABbktPDYkkVUyB6O2338Zbb72FuXPnxrs8JAN81yiOlFXk66GU96ph+L092zxLHIiCo2YunwBozGL+bEcHBSKJxFQPzcrKQnZ2drzLQjKAIDBUNYuBaGJhn+kVwV1dlTpAqZegdD2CkxrdPgHQBn6XqcNaMjEFokceeQTr1q0LW29GCCA2y5weP1QKGUqz++T4sfXKyChxcz5UI/L6qcM6BcTUNHvyySdx+vRpFBQUoLy8HAqFIuzxQ4cOxaVwJP3UtHUDAEZn68DL+gQbe2r0DwHiTh5AsGkWrBFRIJJKTIFo0aJFcS4GyRTV7WIgKs8dIONhcMQsybt2DCRYI3J7/dQ0SwExBaL169fHuxwkA/gFBoVMBrmMQ3lOn8mKfh/QnRod1QCglgcCUe8akcsilpOPeXodiVHMn3hXVxd2796N06dP45e//CWys7Nx6NAhFBQUoKSkJJ5lJGmCl3G4eVYpfH4Bcr5P96O9GWCCmBtaZRj4AkmkUfKYW5ELtUIGptCCkyvFZP6uLjGPNkmqmALR0aNHsXDhQphMJtTU1OCuu+5CdnY29uzZg9ra2lCuInJ+6heEAMDaIN4bSyTvqAbE5Sezx/Qa+dVkiZ3pjg4KRBKIadRszZo1WLZsGU6ePBmWwuPaa6/FJ598ErfCkfTi9QuDP2g9J94bU7S2TB3WkoopEH3xxRe4++67+x0vKSlBU1PTiAtF0o9fYHjukzN46fOzcHr6pHBhDLAEApEpdQJRu92Nug6HWF7qsJZUTIFIpVINmKb1xIkTyMvLG3GhSPpps7vh8QmwuXyhofEQtzWQo1oGGIqkKeAA3j3ejN0Hz6HB4uypEdFcIknEFIhuuOEGPPzww/B6vQDEjfNqa2vxwAMP4Ec/+lFcC0jSQ6vNDQDIN6j6rz201Iv3+nxxY8MUEQyYbm/v2dUUiKQQUyB68sknYbfbkZeXB6fTifnz56OiogIGgwGPPfZYvMtI0kAwEIUSoPVmDQSiFNvEsGe9Wa/Z1W67OHpGkiqmUTOTyYT3338f//jHP3DkyBHY7XZMnz49op04iHQEgaHZ5kKWVhn6I4yXIQNRVyCPeYp1VIdmV3v9gEIj3rxOsVaUAnOdzidRByJBELBt2zbs2bMHNTU14DgOY8aMQWFhIRhjlBIkRfkFhlcPnkN9lxMGtRw3zSyFSROfZhJjDK32QQKRpxuwt4pfZ42Oy+vFiyo4qdEbGO3TZovNSGcnBaIki6ppxhjDDTfcgBUrVqC+vh4XXXQRLrzwQpw9exbLli2LOE0sSb5j9RbUdzkBADaXD/tOtsXt2hanFx6fALmMQ7a2zz5lnYHakD5P0tSwAwmrEQHUYS2hqGpE27ZtwyeffIIPPvgACxYsCHvsww8/xKJFi/Diiy/ijjvuiGshycgdPdcFALh4lAlHz1lwqsWObrevZ4ufEWAMmFRkgMAAWd+Frp014n1W+YhfJ95UvZd5ANRhLaGoakQ7d+7Er371q35BCACuvPJKPPjgg9ixY0fcCkfi57qLinDp2BzMrchFkUkNgTGcbLHH5dpZOiWumVyEay/qMzTPWK9ANCYurxVPhSY15lbkYnJJIG8S1YgkE1UgOnr0KK655ppBH//BD36AI0eOjLhQJP5y9CpcNi4HagWPSUVGVOTr49ZHNKjuNnEhqUwOmFJvV95cvQqzx2SjIj+w9i04ckY1oqSLql7e0dGBgoLBd18oKChAZyfNTE11U0rNmFJqjtv17G4ftAq+f7Os7YR4n1UuyR73UQsGIq8L8DjEBbokKaKqEfn9fsjlg8cunufh8/lGXCgSP4wxfPRdC443WOAbai3YCK6//bMa/NdHp9Dl6DP/JhiIcivj/rrxIAgMLTYX6jocYIyJwTKYGYCWeiRVVDUixhiWLVsGlWqAuSIA3G53XApF4qe924PDdV1QymWY1CuHNGMMFqc4M97cd6QrCt0ePzw+ARwH6Ht3fDs6xB07OA7IqYj5+onkZww7Pq8FANyzYJzYea3NBtw2sXmWQuviMl1UgWjp0qXDnkMjZqmlyeICIC696N102l/dgX+ebseFxUZcfWHsc2Y6u8VakFGtCE//0RTYTjxrDKCSNlH+YBS8mMTNJzC4vIIYiDTZ4pQD6rBOqqgC0datWxNVDpIg7YFA0XeiYa5e/L7ZNrJabJdDrFVl63rVqgQBaP5a/LrwohFdP9HUCh52t09MGatRUIe1RGhbywzXFgg0wcATVGgS80h12D1D5xEaRmegX8is7TUC11YFuKyAQg3kjo/52smgCi58Dc0lyhHvg/uvkaSgQJTh2rvFQJSjD+8H0qvk0Ch5CIyFmlexCAairGA/E2NA7T/Fr0tmpnz+52Du6tDsal0wEHWKNTuSFBSIMpjD40O3W/wDy9H1H2AINqfaRxCIgk2zUCBqPi6mXOUVQMmMmK+bLKrQMo9A0FGbxeAp+MT81SQpKBBlsI5AgDFpFOFbPwfkBmpJ7fbYA9GEQgPGFxiQrVeKC1xPfyA+MHpOWszDCUsFAoijfMHmWXf81uORoaV2vZmMyKgsLe6ePzZUK+orO1BLCjbfAIj70zMm1mgiyKRw6djAH63PDRzdI04E1OUCpZeMuPzJUJGvR5ZWiVFZmp6D2txAIv02AKndx5UpKBBlOK1SDq1y4B9ziVmDS8Zko0TWDnzzujhs7RE3SISMF5sp2mwxsGhzA/c54VkWGQMsdcDJ98R0H3IlcOFi8flpYFyeHuP6ZjfWBQ50tya9POcrCkTnsTwtjzzffqDxaP8HBb84cuRoB9pO9hznOEBtAtRmOD1ecM4OqAQHOHBiU+yim3o6fNNVcDshapolDQWiDPb3b5qhlMswY3RW/3Qffi9w9BWgq04MLoUXi3N+dLkAxwNeh7jMwdEhNlG628QagtcJOLsAZxca2+xotrpRnG3A6AtmA2MuT7mcQ8Px+AR0OT1gDCgwBrbGCg3hd4gjZzLqSk00CkQZyi8wfN1gAWPA9NFZ4Q8yBnz7V6CrDm5Ogbay66ApGBc+KVGuBDRmILtP+g5PtxiU3Dac+rYFtRo5TFMuAErTc1PCRosTew7VI9egwu2XBjJIqs1ixoDgyFkwTxFJGAr1Gcrq9IIxQMFz0Cn79Nc0HgZaqwAZjy90V+JPp3l819R/e6gBKXViytfCyaiVl8OmKkSWMb1qQb1pgqNmvfdik8l6mpfUPEsKCkQZKjjR0KRVhucRd9uAU4Eh9rFXQJMv1ng6opxLFNzDDOg1hygNqQNB2un1iyvwg7SBGp6DAlEyUCDKUF3BlfV9k5/V/EPsHzIWAaNmhZpj0c6u7nKK52uUfNx3BEmmYI3IL7CeZR4AdVgnGQWiDGXpO+MZEDtfGwMZNMddCXA9ye47HV4IAut7mUH1zKhOnQ0TY6HgZaHJnqFlHgDViJKMAlGGCtZYwhaj1h0AmCDmBzKXAQAMajnkMg5+gcHq8kZ8/WBTbiS5jFJFsEbn7B2IQjWidlpzlgQUiDKUPdB/E8pL7XH05AgqnR06TybjYA40z6LpJxqVpcGs8myMy0vfjuqgYPPM2bvDuu/IGUkoGr7PUD+5dDS6PX6og2vMGg+Lf1SGglBtKChHp0SbzR3q4I7EqCwtRmWl/lqySFxUYkJlgT68GSuTiXuxWRsBewsN4ScYBaIMxXFcT+pWxnpqQyUz+60hm1xswugcLUrMGpyPLhplGvgBfUEgEDUD+ROTW6jzDAWi84G1Xuyo5hVAXv8/qLKc6Go2Hp+AZqsLWTpleJ7qTKPLF+/tLdKW4zwgaR/Rhg0bMGvWLBgMBuTn52PRokWoqqqSskgZ4WSzDX890oDjDRbxQLA2lDchLtv6tNrd2H3wHF75om7E10oFLq8fLVYX2u190ubqg4GoOfmFOs9IGog+/vhjrFq1Cp9//jnef/99eL1eXH311eju7payWGmv0eLCqRY72uwewO8DWr4VHxgif3RdhwNH6rrCh7AHEZxzlO5D90Enm+3Ysb8W+071GaoPBiK3TezsJwkjab36nXfeCft+27ZtyM/Px8GDB3H55ZdLVKr0F9wmyKRRAF1nxVxBKj1gHj3oc977phlWpxc5euWwndCh9LC69B+6BwCNcoB5RAAgV4nJ9J2dYq2o77o7EjcpNXxvsYhNiezsgUco3G43rFZr2I30FxaIQpscjh8y0Vm2TqzddHYPP5coOMyfnQFziICeeUQOzwC1QT31EyVDygQiQRBw3333Ye7cuZg8efKA52zYsAEmkyl0Ky1Nvf3UpdZ740STmo94t9Xg0HVHBEP4waZZdobUiIKJ45wDNUv1gS3WqZ8ooVImEK1atQpff/01du3aNeg5a9euhcViCd3q6jKjszSeXF4BnsCaKaOnWezbkKuGbJYBPcn1h1tz5hcYLE5xsqQ5Q/qIghMa3V4B/r7LXIKBqJtqRImUEmOvq1evxptvvolPPvkEo0aNGvQ8lUo16HbXRBSsDelVcsg7TokHcyqGTd2aFWiaDTe7usvhgcAYlHJZxgzdq+QycJw43crl9YcnkdMH08a2ix3/Kb49UrqS9FNljOFnP/sZXnvtNezduxdjxlBn4Eg5PD7IZRxManlPitcINjkMNrOsLi+8fgEKfuDKslYpx8JJBfD4hfD0ImlMJuOgVvBwevxwePoEIpVR3CjS6xIzVBqLpCtoBpM0EK1atQovv/wyXn/9dRgMBjQ1NQEATCYTNJrzc5bvSI3N02P1lRXw2NqAg51iTSh77LDP0yjEdB4urx+dDg/yDeqBz1Pyg89ETmMzRmeBg/j+wnAcYCgCOqoBWyMFogSRNBBt3rwZAHDFFVeEHd+6dSuWLVuW/AJlCI7joLLWit8YSyKaxMhxHP7lggKoFbK0TnQWq1nlQ6wlCwYiawNQMj15hTqPSN40IwnSWS3eRzH3pSJfP+w5p1rs0Ch55BtUgzbfMo6xRLy3NUpbjgxGPW8Z5vXD9eA5AQvbz0DNAciKX78bYwzvHm+Cxyfg9stGI1efOQMHLq8fVqcXcl7Wf1pCsDnmaBcnh8oz532nivPkX9r5wS8w1LQ50FR7BjLBCyg0gKEw4uc7PX58XW/BwbMdAz5udfrg8QngZVzGNd+ON1iwY38t9p9p7/+gUifu5cYY1YoShAJRBrG7fBAYQ46nHgqeE5tlUYxsubx+vP9NM/55un3AZnOr3QUAyNErwcsyY8QsKDipsXug2dVAT63I2pCkEp1fKBBlkOAcokJ/o7jzapTNMpNGAV7GwetnsAYyPPbWahPnGGVSkyxIFwhEDk//9w0AMBSL9xSIEoICUQaxOL3g/S5kCYHmRVZ5VM+XybjQivqBZli3BtJkZGQgUonD9t3uwWpEgUBkaxSbaCSuKBBlEIvTC7O7Hiq5TEz+rjZGfY3givr2bne/x1pt4rF8QyYGIrFG5PL64fMPkCzfUAhwMsBtB9y02DreKBBlEIvTC5PznJinOsaUFcGJjE2W8EDU7fbB6vSC44C8DAxEKrks1O/lGGjxK68Q830DQBetcYw3CkQZhDEB2Z56qBR8zMP2RSYxEDVanGHH1QoeS2aVYuGkgrTeUHEwHMdBG5hV7RiseWYKZHuwUCCKN5pHlEF+WKEG61CCcXy/nToiVWBUQ8ZxsLl8sLt9oYWtvIxDsVmD4gxOsD99dBYEgUGrGiTQmsvEveGoRhR3FIgySWc1OHDgzGViUyIGSrkMi6eXIFev6r/uKsNNL8sa+gTTKHE6hKNd7CtSDT8TnUSGmmaZpCP6ZR0DKc3WhgUhh8eHd4834VSLbUTXTXsKTc8OsJZz0pYlw1AgyhDfNXTi8NdHUN/ljHrYfjCMMTDGcKa1G980WLG/euAZ15nC4xPQYnWhxeoa/CRToMlL/URxRYEoQ9hbzsLpdMEJVU9WwRH4rsmKl/bX4mSLPbQtUUVeZjdFTrWIu3l8erJt8JPMgQ7rzpqklOl8QX1EGcLbdhoAIMsZG9WyjsF02D1os7nx1lFxbRUv4zC5JPPyEPVmUIt/DjbXEBsImEeLn293G+CyxjRXi/RHNaIMwQX+Q6vyx8XletNHZ4X+MAFg5uis8MyFGcioFjv4bS7f4ClqlNqehcTBVCtkxDL7N+s8wdx2cLYmMAC6woq4XFOt4HHTzFIcPdcFk0aBizK8NgSIyzw4DvAJDE6vP7QQtp/ssYC1Eeg4AxRNSW4hMxTViDJAd/NpCIzBqcyB0TjMEHQUTBoF5lXm4eJR5ozJTz0UOS8LLX61DbDoNySYerezBhAGWA5CokaBKAM4m8TdOnzmcsgyLD1Hsukj6ScyFIvJ0bwuyk8UJxSI0h1jkFlqoFHw4HOGT5JPhhbsFxsoDUqITNYzRaLjdOILdR6gPqJ0192GHLkHOeW5YJdQYveRmlhoRJFJjVFZwyxlya0EWqvE25jLk1O4DEaBKN11nBHvTWXgYlzWQXpEsoEAAHHTSk4mDuN3twO6nMQWLMNR0yzNsfaTYGAR7V1G4kih6WmetVVJWpRMQIEonXkc8LTX4ouaTrxWp6HtmeJAEBhabC6cbLYN/3nmBXbQbaVANFIUiNJZxxk43V7Y+CzYOP15McSeaAzAzv11ePNoI+zuITqsAXErb44DbE2AszMp5ctUFIjSWftJOLx+dGpGI0eXeVkTpcDLOBg1Ytdpl2OIIXxA3GYomPep+XiCS5bZKBClK78P6DiDbrcPHZrRyNVn1j5jUgru2TZsIAKAwovE+6ZjlFR/BCgQpStLLeDzoMunRLcyDwVGtdQlyhim4E4mjv47mfSTO0FMQufsotQgI0CBKF21fAefIKBBLmYNpEAUP8EaUUSBSK4E8ieJXzd9ncBSZTYKROnI7wNav0O324827TiYNIrzLq1rIuUEtlRqs0cQiICe5lnLccDrHPpcMiAKROmosxrwuQGVHrmjKlCeq5W6RBkluF2S1emFa6CthfoylQL6PPEfROORBJcuM9HM6nQUGKExlV2MGytLJS5M5lEreMyfkBfagntYHAeMmgV89zeg/iAwara4Ho1EjD6tdON1Ae0nxa8LLpC2LBlselkWxuXpoeAj/BPJv1BMmuay0kzrGFAgSjfNXwN+HzzqHFjkuVKXhgTxcqA4sOi4Zh8N5UeJAlE6YQxo+AoAUK2sxJZ/1OCdrykfTiL4/ALOtNrxZU0UO5eMminmKepuA1q+SVzhMhAFonRiOSf+kvNynMBoAECOnmZUJ4LXz/D64QZ8erINDs8wSz2CFBqg9BLx6+pPASGCjm4CgAJReqnbDwDw5U7CWYv4Sz46m0bMEkGj5EOz1Ru6ohiSHzVL7Ctydood1yQiFIjShb0FaDsJcBxq9RfD62cwahShoWYSfyWB5GjnOqMIRHJlT6K06k/EzmsyLApE6aJmn3ifNwEnbeJ/6nF5Olpxn0CjssTa5tl2R3RPLJoKmEoAvxc48S51XEeAAlE66KoVc95wHNyjLsOpFjuAKLIJkpiUZWvByzh0dHvQbndH/kSOA8ZfA8h4oP0UNdEiQIEo1Ql+4OT74tdFU1Ht1MLjE5CtU6LEPExeZTIiagWPskAfXDD4R0yfD4y7Uvz69IdAFy2IHQrNrE51NZ+K/UMKNTBmHiYotJDLOPgFULMsCSoL9Khu60ZrNDWioJIZQNdZoPUE8PVuYNrtgI7mfg2EAlEqazsF1H4ufj3+B4BSBw5ARb5B0mKdT8YXGJClVaLIFEN2A44DJt0AuF8GrA3A4ZeBi5cAhoL4FzTNUdMsVXXVAd+8JnZ0Fk+F01wZ+XwWEjcKXoZisyb22ievAC76sdhU83QDh18CWr6NbyEzQEoEomeffRbl5eVQq9W45JJLcODAAamLJB3GxLw2R3aJq7mzx8A3diHeOtaI7Z+djb6vgsSN1eXF8QZL9E9U6oCpt4lpZX0e4PhfgG9eB1wxXCtDSR6IXnnlFaxZswbr16/HoUOHMGXKFHz/+99HS0uL1EVLPmsjcGw38O1fAcEH5FbCMu4G7DnchLoOBwTGYNbS3mVS6Hb78PL+Wrz/TTMO13VFv2OKQg1M+X/A6MvEJlvzN8D+54Dv3gIs9ef9ED/HJN6D5pJLLsGsWbPwX//1XwAAQRBQWlqKn/3sZ3jwwQeHfK7VaoXJZILFYoHRaExGceODMTGBlqcbcLSJu0B0nAbsrQAAt5+hJWcmvlFMxonmbvgEBqVchhumFKOUZlJLZm9VC76q7QIgTnacXGxCsVkNo1oBWSTpQoJsTcCpD8RpGUFKHZA1GtAXALo8QGUAlHpx2UgaD0pE+jcqaWe1x+PBwYMHsXbt2tAxmUyGhQsX4p///Ge/891uN9zuntELqzXCWasd1cCpv/c/HhaDWZ9jbIDzwo+darGjy+kOncIBgf+U4nkzyrIQ/P083WpDu90DgIEXvAATwq44tdQMOS8H8ifh4+4KHG+TAxCbYSVZGiycVIBsHSXIl9L88XnQq+T45+l21Hc6UR+Ycc1xwI+mjwr9k/i63oKDZzsxWGz6lwsKUTj1VsBaj/rj+6DoPI18dIu1pOYBFsvK5OLqfplCnJuEwIU5rs/X6P91Il10M6COTwVA0kDU1tYGv9+PgoLwUYSCggJ89913/c7fsGEDHnrooehfyOcWF4vGmc9phXeInR44ryr0SyF4nPB7xNSjwaWQfpkSTrkJDkU2LpowG/KC8YBCA83JVuQLDhSbNJhQaECRSU1D9SmA4zjMLM/G+EIDvq63oKbNgTa7G36BQa3oSdXr9PrR0T14mlmvOPcCMI1CW+k1EIp9yDfaAFsjYG8GHO2A296TdlbwiTe4EvwOoxT4ZxoPaTV8v3btWqxZsyb0vdVqRWlpBBkKzaXAlFv6/KcY4D9JRMd6Hsvt9sDgE0LHOA7gOFnP6Ua1uD86gAKnFya/IJ7Jq8AUGnAyeehchVaJ4L/QeZV5w78nIhmjWoE543IxZ5y4M6zL54dK3hOIJhQaUGhUD9rt03t94Ng8nVgrVucC2WPCT/T7AL9bXCoi+MR7Fvg31ruWPuDXSaDUxe1Skgai3Nxc8DyP5ubmsOPNzc0oLCzsd75KpYJKFcMiT6Wu/w85DnKjqJUadUAa9WKRCMlkHLTK8D8jo1oBozqyQQXDUOfxgSbZeUDSUTOlUokZM2bggw8+CB0TBAEffPABLrvsMglLRghJJsnD7Zo1a7B06VLMnDkTs2fPxqZNm9Dd3Y0777xT6qIRQpJE8kC0ZMkStLa2Yt26dWhqasLUqVPxzjvv9OvAJoRkLsnnEY1E2s4jIuQ8EenfqOQzqwkhhAIRIURyFIgIIZKTvLN6JILdWxEv9SCEJFXwb3O4rui0DkQ2mw0AIptdTQiRjM1mg8lkGvTxtB41EwQBDQ0NMBgMkq/FCi43qauroxG8XuhzGdz58NkwxmCz2VBcXAyZbPCeoLSuEclkMowaNUrqYoQxGo0Z+0s1EvS5DC7TP5uhakJB1FlNCJEcBSJCiOQoEMWJSqXC+vXrY8sOkMHocxkcfTY90rqzmhCSGahGRAiRHAUiQojkKBARQiRHgYgQIjkKRFGIZkfa559/HvPmzUNWVhaysrKwcOHCjN3BNtadenft2gWO47Bo0aLEFlBC0X42XV1dWLVqFYqKiqBSqTB+/Hj87W9/S1JpJcRIRHbt2sWUSiXbsmULO378OLvrrruY2Wxmzc3NA55/6623smeffZZ99dVX7Ntvv2XLli1jJpOJnTt3LsklT6xoP5eg6upqVlJSwubNm8duvPHG5BQ2yaL9bNxuN5s5cya79tpr2b59+1h1dTXbu3cvO3z4cJJLnnwUiCI0e/ZstmrVqtD3fr+fFRcXsw0bNkT0fJ/PxwwGA9u+fXuiiiiJWD4Xn8/H5syZw/7v//6PLV26NGMDUbSfzebNm9nYsWOZx+NJVhFTBjXNIhDckXbhwoWhY0PtSDsQh8MBr9eL7OzsRBUz6WL9XB5++GHk5+fjpz/9aTKKKYlYPps33ngDl112GVatWoWCggJMnjwZv/3tb+H3+wc8P5Ok9aLXZIl2R9qBPPDAAyguLg77xUx3sXwu+/btwwsvvIDDhw8noYTSieWzOXPmDD788EPcdttt+Nvf/oZTp07hnnvugdfrxfr165NRbMlQIEqCjRs3YteuXdi7dy/UarXUxZGMzWbD7bffjueffx65ublSFyflCIKA/Px8PPfcc+B5HjNmzEB9fT1+//vfUyAi0e9I29sTTzyBjRs34u9//zsuvvjiRBYz6aL9XE6fPo2amhpcf/31oWOCIO6fLpfLUVVVhXHjxiW20EkSy+9MUVERFAoFeL5n++pJkyahqakJHo8HSqUyoWWWEvURRSDWHWl/97vf4ZFHHsE777yDmTNnJqOoSRXt5zJx4kQcO3YMhw8fDt1uuOEGLFiwAIcPH86oTJux/M7MnTsXp06dCgVnADhx4gSKiooyOggBoOH7SO3atYupVCq2bds29s0337CVK1cys9nMmpqaGGOM3X777ezBBx8Mnb9x40amVCrZ7t27WWNjY+hms9mkegsJEe3n0lcmj5pF+9nU1tYyg8HAVq9ezaqqqtibb77J8vPz2aOPPirVW0gaCkRReOaZZ1hZWRlTKpVs9uzZ7PPPPw89Nn/+fLZ06dLQ96NHj2YA+t3Wr1+f/IInWDSfS1+ZHIgYi/6z+eyzz9gll1zCVCoVGzt2LHvssceYz+dLcqmTj9KAEEIkR31EhBDJUSAihEiOAhEhRHIUiAghkqNARAiRHAUiQojkKBARQiRHgYgQIjkKRCStbNu2DWazOfT9b37zG0ydOjX0/bJlyzI69WymokBEBrRs2TJwHId/+7d/6/fYqlWrwHEcli1bFnZ+vANAeXk5Nm3aFHZsyZIlOHHixKDPefrpp7Ft27bQ91dccQXuu+++uJaLxB8FIjKo0tJS7Nq1C06nM3TM5XLh5ZdfRllZmSRl0mg0yM/PH/Rxk8kUVmMi6YECERnU9OnTUVpaij179oSO7dmzB2VlZZg2bdqIrj1QTWXRokWhWtYVV1yBs2fP4he/+AU4jgPHcQD6N8366l0zW7ZsGT7++GM8/fTToWtUV1ejoqICTzzxRNjzDh8+DI7jcOrUqRG9LxIbCkRkSMuXL8fWrVtD32/ZsgV33nlnwl93z549GDVqFB5++GE0NjaisbEx6ms8/fTTuOyyy3DXXXeFrlFWVtbvPQHA1q1bcfnll6OioiJeb4FEgQIRGdJPfvIT7Nu3D2fPnsXZs2fxj3/8Az/5yU8S/rrZ2dngeR4GgwGFhYXDZsIciMlkglKphFarDV2D53ksW7YMVVVVoT3GvF4vXn75ZSxfvjzeb4NEiFLFkiHl5eXhuuuuw7Zt28AYw3XXXZf2+aaLi4tx3XXXYcuWLZg9ezb++te/wu1246abbpK6aOctqhGRYS1fvhzbtm3D9u3b41ZrkMlk6JsKy+v1xuXakVixYkWoI37r1q1YsmQJtFpt0l6fhKNARIZ1zTXXwOPxwOv14vvf/35crpmXlxfW7+P3+/H111+HnaNUKke8p9dg17j22muh0+mwefNmvPPOO9Qskxg1zciweJ7Ht99+G/p6MBaLpd9+ZTk5OQMmxb/yyiuxZs0avPXWWxg3bhyeeuopdHV1hZ1TXl6OTz75BLfccgtUKlVMTcLy8nLs378fNTU10Ov1yM7OhkwmC/UVrV27FpWVlUNugkASj2pEJCJGoxFGo3HIc/bu3Ytp06aF3R566KEBz12+fDmWLl2KO+64A/Pnz8fYsWOxYMGCsHMefvhh1NTUYNy4ccjLy4up3Pfffz94nscFF1yAvLw81NbWhh776U9/Co/Hk5RRQDI0yllNzluffvoprrrqKtTV1fXbkZUkFwUict5xu91obW3F0qVLUVhYiB07dkhdpPMeNc3IeWfnzp0YPXo0urq68Lvf/U7q4hBQjYgQkgKoRkQIkRwFIkKI5CgQEUIkR4GIECI5CkSEEMlRICKESI4CESFEchSICCGS+/+RzMc0HhwMagAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:41.003005Z", + "iopub.status.busy": "2024-02-29T23:33:41.002647Z", + "iopub.status.idle": "2024-02-29T23:33:41.377966Z", + "shell.execute_reply": "2024-02-29T23:33:41.376921Z" + }, + "papermill": { + "duration": 0.400345, + "end_time": "2024-02-29T23:33:41.380250", + "exception": false, + "start_time": "2024-02-29T23:33:40.979905", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEnCAYAAAANc04FAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAfElEQVR4nO29ebxT5bX//9nZmeczzwNwAFFmEQoUR+qAY+1Vb7UWcK7YXuXnrXJvi1ctoq1ypcMPqy2DV5BaSmutVapYsTiggoAgMhw4A2ceM5zM2fv7x5PknJwxyUmyk5z1fr3y2snOzs46+ySfrGc961mLE0VRBEEQhITIpDaAIAiChIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCckiICIKQHBIigiAkRy61AaNBEAQ0NjbCYDCA4zipzSEIoh+iKMJms6G4uBgy2TB+jyghPp9P/MlPfiJWVlaKarVaHD9+vPjEE0+IgiBE9Pr6+noRAN3oRrcUv9XX1w/7XZbUI3rmmWewYcMGbNmyBeeddx4+//xzLF++HCaTCT/60Y9GfL3BYAAA1NfXw2g0JtpcgiCixGq1oqysLPRdHQpJheijjz7C9ddfj6uvvhoAUFlZiVdffRWffvppRK8PDseMRiMJEUGkMCOFTiQNVi9YsAC7d+/GiRMnAACHDh3C3r17cdVVVw16vNvthtVqDbsRBJH+SOoRPfroo7BarTjnnHPA8zz8fj/WrFmD2267bdDj165di8cffzzJVhIEkWgk9Yhee+01bN26Fdu2bcOBAwewZcsWPPvss9iyZcugx69atQoWiyV0q6+vT7LFBEEkAk4UpSuMVlZWhkcffRQrVqwI7fvZz36GV155BV9//fWIr7darTCZTLBYLEPGiERRhM/ng9/vj5vdxEB4nodcLqc0CiKMSL6jgMRDM4fDMSC3gOd5CIIQl/N7PB40NTXB4XDE5XzE8Gi1WhQVFUGpVEptCpFmSCpE1157LdasWYPy8nKcd955+OKLL7Bu3Trccccdoz63IAg4c+YMeJ5HcXExlEol/VonCFEU4fF40NbWhjNnzmDixInDJ68RRD8kFaJf/epX+OlPf4r7778fra2tKC4uxr333ovVq1eP+twejweCIKCsrAxarTYO1hLDodFooFAoUFtbC4/HA7VanbT3brI4kadXQc6T+KUrkgqRwWDA888/j+effz5h70G/zMkjGdfa4xPQYnWhLJv9uDg8Pvz5iwYY1Qp8e1YJdKq0XrU0ZqFvKZFWvHusBTsPNOBYE8shszi94DkObTY33jjUCL8g2dwLMQpIiIi0oaa9B8ebbQAAs1YBACgyaXDznDKoFDI0WVw4UNclpYlEjJAQpSAXX3wxHnzwQanNSDn2nekAAMwoM6HIpAntz9IpcfGkfADAp2c64fRQqka6QUJEpAVNFicau13gZRwuqMwe8PyUIgPyDCp4fAIOn+1OvoHEqCAhSjGWLVuGPXv2YP369eA4DhzHobS0FBs2bAg77osvvoBMJkNtbS0AYN26dZg2bRp0Oh3Kyspw//33w263h71m7969WLRoETQaDcrKyvCjH/0IPT09SfvbRsNXjSwmNLnQMGhAmuM4nF+RBQA40miFhHm6RAyMSSHy+IQhbz6/EPGx3giOjZb169dj/vz5uPvuu9HU1ISmpiZ897vfxbZt28KO27p1KxYuXIiKigoAbMbql7/8JY4ePYotW7bgvffew49//OPQ8dXV1bjyyivxne98B4cPH8Yf/vAH7N27Fw888EDUNiYbQRBxspWJ6pTCobNzq/L1MGkUGJerhddPQpROSLrEY7QMlz7ucrlw5swZjBs3bkBOy/++c2LIc47L1eGGWSWhx79+7+SQH+rSLA1umlMWevzCnuoB8YmHvjUp4r8nyMUXX4yZM2eG0hoOHjyI2bNno6amBuXl5RAEAeXl5fjJT36C++67b9Bz7NixA/fddx/a29sBAHfddRd4nsdvf/vb0DF79+7FRRddhJ6enrjk/Qx3zUdDm82N1z6vBy/jcM+i8ZDJhk5MFUWREldTiEiXeIxJjyjdmDlzJqZMmRLyivbs2YPW1lbcdNNNoWPeffddXHbZZSgpKYHBYMDtt9+Ojo6O0PKWQ4cOYfPmzdDr9aHbFVdcEcpAT2XyDCrcc+F43Di7ZFgRAkaue0OkJmMy+2vFJVVDPtf/c37PhROGPLb/Z/6OheNGY9aw3Hbbbdi2bRseffRRbNu2DVdeeSVycnIAADU1Nbjmmmvwgx/8AGvWrEF2djb27t2LO++8Ex6PB1qtFna7Hffee++glS/Ly8sTZne8UPAy5Bsi87IEQUSrzQ2zVgG1gk+wZUQ8GJNCpJRH7ggm6thhz6NUDqgWcOutt+InP/kJ9u/fjx07duCFF14IPbd//34IgoDnnnsulN382muvhb1+9uzZ+Oqrr1BVNbQIpyKxDLV2ftGA+k4HrjivEOcWU+XOdICGZilIZWUl9u3bh5qaGrS3t0MQBFRWVmLBggW488474ff7cd1114WOr6qqgtfrxa9+9SucPn0a//d//xcmVADwyCOP4KOPPsIDDzyAgwcP4uTJk3j99ddTPlh9vMWG//u4BvtrI09UzDeoAADNVmeizCLiDAlRCvLwww+D53mce+65yMvLQ11dHQA2PDt06BC+/e1vQ6PpTeibMWMG1q1bh2eeeQZTp07F1q1bsXbt2rBzTp8+HXv27MGJEyewaNEizJo1C6tXr0ZxcXFS/7Zoaex2ot3ugd3ti/g1hSY2hGuxuhNlFhFnxuSsGZEYEnHNt+2rQ4vVhSXTijC5cPhOEEEsDi82fngGvIzDikuqwI8Q4CYSB82aEWmPzy+g3c68mkJj5MJm1MihVvDwC2Lo9URqQ0JEpCztdg/8ggiNkodRE/m8CsdxKDCyOFGbjYQoHSAhIlKWZqsLAPOGop05y9axcrUdPZ6420XEnzE5fU+kBy0BISqIYlgWZEKeHjqVHCVmzcgHE5JDQkSkLFolj2ydEvmBYVY0lGVrQ1UcidSHhIhIWRZNzMOiiXlSm0EkAYoRERlLh92NU632qHKQCGkgISJSEp9fGHVNoXePteCNQ41o7KYM61SHhIhIST6r6cL//3419p3uiPkcJg2ra21xeuNlFpEgSIiIlKSjxw2PT4Ccjz0r2hgUIgcJUapDQpSCpFrxfCns6bCz/J8cXfQzZkGMaiZEVhcJUapDQpSheDzpm8jnF0R0B7yYHL0y5vPQ0Cx9ICFKMQYrnl9dXY0777wT48aNg0ajweTJk7F+/foBr7vhhhuwZs0aFBcXY/LkyQCAjz76CDNnzoRarcacOXPwl7/8BRzH4eDBg6HXHjlyBFdddRX0ej0KCgpw++23h0rMDmZPTU1NQq+BzeWFIIqQyzjoR9G51RTofWZz+SBQ48WUZmzlEYki4Jfg15FXDCznOATr16/HiRMnMHXqVDzxxBMAgKysLJSWluKPf/wjcnJy8NFHH+Gee+5BUVERbr755tBrd+/eDaPRiHfeeQcAW/l87bXXYsmSJdi2bRtqa2sHDLG6u7tx6aWX4q677sL//u//wul04pFHHsHNN9+M9957b1B78vISm9sT9GBMWsWoSr/qlXLwMg5+QYTd4wsN1YjUY2wJkd8L/Ou55L/vov8PkEc2xDCZTFAqldBqtSgsLAztf/zxx0P3x40bh48//hivvfZamBDpdDr87ne/g1LJ3uuFF14Ax3F46aWXoFarce6556KhoQF333136DW//vWvMWvWLDz11FOhfRs3bkRZWRlOnDiBSZMmDWpPIgkJkWYE4RBFoOkQ0HIUUGqBykWALjf0tEzG4eLJeVAreKjiVD2TSAxjS4jSmN/85jfYuHEj6urq4HQ64fF4MHPmzLBjpk2bFhIhADh+/DimT58eVhto7ty5Ya85dOgQ/vnPf0Kv1w94z+rqakyaFH0XktGiVcoxPk+H4pHWidX8C6j5sPdx5xlg9lJAlxPaNb3UnBgjibgytoSIVzDvRIr3HQXbt2/Hww8/jOeeew7z58+HwWDAL37xC+zbty/sOJ1OF/W57XY7rr32WjzzzDMDnisqKorZ5tFQla9HVf5AYQyjqxao/YjdL5/HHtuageN/B2Z9L+KhMJEajC0h4riIh0hS0r94/ocffogFCxbg/vvvD+2rrq4e8TyTJ0/GK6+8ArfbDZWKTYN/9tlnYcfMnj0bf/rTn1BZWQm5fPCPw2DF/CVFFIHq3WxbNAOYcCngsgCfvghYzgKdp4Ec1n3F6vKi1eqGRsnTSvwUhgbOKUj/4vkTJ07E559/jl27duHEiRP46U9/OkBQBuPWW2+FIAi45557cOzYMezatQvPPvssgN7+XytWrEBnZye++93v4rPPPkN1dTV27dqF5cuXh8RnsGL+iUIURTg9/uGXd7SfBGwt7Edl/MVsn9oEFM1i9xv2hw6tbrXjjUON+KIu8uL7RPIhIUpB+hfPv+KKK3DjjTfilltuwbx589DR0RHmHQ2F0WjEG2+8gYMHD2LmzJn47//+b6xevRoAQnGj4uJifPjhh/D7/bj88ssxbdo0PPjggzCbzaHWREMV808ETq8fL+ypxgt7TsM/1JR7UGiKZ7MgdZCS2WzbUQ24rAAAg5p5eXYXLXxNZah4/hhj69atWL58OSwWS1gnkHgQj2veZHFi+6f1MKjluGvR+IEHODqBfb9lw+x59wEac/jzB14GLA3AxG8BpXNGPh+RUCItnj+2YkRjkJdffhnjx49HSUkJDh06FMoRircIxYtgRvWQU/fNX7Jt9viBIgQAeVOYELV9DZTOCSVE9rj9EARxxJbVhDSQEGU4zc3NWL16NZqbm1FUVISbbroJa9askdqsIRk2h0gUmcAAQMHUwU+QOxE49S4TI68LOqUKHAcIogiH1z+qTG0icdB/JcP58Y9/jB//+MdSmxExwwqRvZUNzWRyIGeI1tkaM6DNZsdZ6iHLnQidUg6724cet4+EKEWhYDWRUgRLdgTXiYXRdoxtc8YPn4aRVcm2XTUAAF1AfKhSY+pCPw9EShH0iMyafkIjikBrYFiWN2X4k5grgIYDISGaNz4bPr8YUzcQIjlkvBCl8aRg2jHaay2KIqoK9LA4vDD394icXewm40PJikNiLmfbnnbA48CEvBGytAnJyVghUijYB9nhcKTsDFGm4XA4APRe+2jhOA6XTM4f/MmOQCa5qRSQj1AsTantjRPZmkYWLkJyMlaIeJ6H2WxGa2srAECr1Y6qpAQxNKIowuFwoLW1FWazGTzPx/9NOk+zbXaEomIsZkJkbYBdX4FmiwtKXobyHOp1lopkrBABCJWtCIoRkVjMZvOoSoU4PD7IOA5qRT8h83uB7kA2d6TejbEYaD4CWBtxVuPAW182ozRLQ0KUokguRA0NDXjkkUfw1ltvweFwoKqqCps2bcKcOXNGfW6O41BUVIT8/Hx4vVQuNJEoFIpRe0L7znTiYF035o3PxoIJvXWF0FULCD5AbQS0OUOfoC/GEra1NkJXyuxyelNo4S4RhqRC1NXVhYULF+KSSy7BW2+9hby8PJw8eRJZWVlxfR+e5xMzXCDiijUwY2ZQ9Ysx9R2WRTq81uUDvBzwuaH1dwMAHB4SolRFUiF65plnUFZWhk2bNoX2jRs3TkKLCCkZcnlH1xm2zY5irZhMBugLActZaD3tADRwevzwCyJ4WuaRckia0PjXv/4Vc+bMwU033YT8/HzMmjULL7300pDHu91uWK3WsBuRGYiiGPKIwoTIZWVBZ47rnZaPFD2bgVO7OiALeFI0PEtNJBWi06dPY8OGDZg4cSJ27dqFH/zgB/jRj36ELVu2DHr82rVrYTKZQreysrIkW0wkCrvbB58gQsZxodIdAHqD1PoCQBFlQmJAiLieVmiU7KPuoOzqlERSIRIEAbNnz8ZTTz2FWbNm4Z577sHdd9+NF154YdDjV61aBYvFErrV19cn2WIiUQSHZUaNPHyFfHct22ZVRH9SfQHb2lugCczEUZwoNZE0RlRUVIRzzz03bN+UKVPwpz/9adDjVSpVqOQpkVkMudg16BGZYxAiXR4b0nkcWDBRDZ88B3kG+vykIpIK0cKFC3H8+PGwfSdOnEBFRQwfOiKtydIpMbPMjCxdnzVmzm5242QsozpaeAWb7u9pxwRND5CTnHZIRPRIKkQPPfQQFixYgKeeego333wzPv30U7z44ot48cUXpTSLkIASs2ZgcfugN2QoHHlZx1Do8tiaM3srLfVIYSSNEV1wwQX485//jFdffRVTp07Fk08+ieeffx633XablGYRqUJoWBblbFlfAnGins5GnGq14WyXIw6GEfFG8szqa665Btdcc43UZhAS02pzwahW9C7vEMXRBaqDBGbOutoa8UZ7E6ry9SjNomUeqQYVRiMkx+X1Y+snddjwfjXcvsCslqub5RBxMsAYQ3woSGBJiNrbDYgC5RGlKCREhOQEExm1Sh4qecAjsjSwraFwdE0x1SaAl0PBiVD7rHCREKUkJESE5Aw6dW8NCJGpZHQn5zhAmws5z0Hj7YaT8ohSEhIiQnIGFSLLWbYdzbAsiC4XchkHrbcLLq9AVTtTEBIiQnK6+xfM97mBnjZ2f7QeEQBoc6HgZdD4uiCIIty+xLXMJmKDhIiQnAEekbWRzZqpTYDKMPo30OWyNWx+CwBQnCgFkXz6niAGCFFwWBYPbwgIzZxNMXlQPK0AGiXVpko1SIgIyZlZbka3w4Ps4PKOUKA6DvEhAFCbAZkcBTqgwCQAchKiVIOEiJCc2eV9KnIKQq8QxSNQDbAiadpstsyjpwPQxLcCKDF6KEZEpBaOdsDnYQtWdXnxO68uFw6PD2cb6tBmc8fvvERcICEiJKXb4UGrzQVPcCYrNG1fwjyZeKHNRYvVjQPHTuFEiy1+5yXiAgkRISmHzlqw9ZM6fHy6g+2wNbOtsSi+b6TNhpznoPZZKKkxBSEhIiSl2+EBAJiDM2b2FrbVx7l2kCab5RJ5LXD5SIhSDRIiQlKCU/dmrQIQ/Kx2EBBaNR83NFmQyzjIBRc8Tnt8z02MGhIiQjJEUYQlkFVt1igBRwdrpChXxn9mS66ETGNk7+voiu+5iVFDQkRIhq1/547QsKwg8kaKUSDXs+6xoqMz7ucmRgcJESEZllBDxUDnDlsfIUoAvJ5lWPNOEqJUgxIaCcnojQ8FMqrtiRUilSEX43N1qMjhIIoiuAR4XURskBARklFoUmPRxFwY1Aq2yDXBQiTX56LAqAZUjoQM/YjYISEiJCNXr0KuPtCdw9nFyn/IeECXm5g3DAbAnV1M+EiMUgaKERGpgb2VbXW5TIwSgdoMm0dAp7UHPVaaOUslSIgISRAEESdabGi3u1nFxGBGdYKGZQAAmQzHLTyOt9jR3taUuPchooaEiJAEm8uHNw834dV9dRBF9HpE8c6o7ocYGJ757B0JfR8iOkiICEno6GEr4M06JZu6twc9ojhnVPdD1GQDAIRgBjeREpAQEZLQ2cPWmOXolICnB3DbWfA4wULEaZkQUVJjakFCREhCUIiytMreaXtNVuw97iNEpmUzchwJUUpBQkRIQsgj0iv7ZFQn1hsCAJmeeURwWQC/L+HvR0RGTEJ0+vTpeNtBjCFEUURHQIiydcrElf4YBIXaAL9MAb8gsLbWREoQkxBVVVXhkksuwSuvvAKXyxVvm4gMp8fjh8cngOMCdYjsyfOIirO0qCgtQ5FJw1b7EylBTEJ04MABTJ8+HStXrkRhYSHuvfdefPrpp/G2jchQlLwM184oxoWT8iAXfSzTGWB97hNMtk6JspIS1rrISUmNqUJMQjRz5kysX78ejY2N2LhxI5qamvDNb34TU6dOxbp169DW1hZvO4kMQimXoSpfz7p32FvYcguVHlDqkmNAYAofFLBOGUYVrJbL5bjxxhvxxz/+Ec888wxOnTqFhx9+GGVlZfj+97+PpibKXiVGIJTImMCM6j74BRHNXg06e9wQaWiWMoxKiD7//HPcf//9KCoqwrp16/Dwww+juroa77zzDhobG3H99dfHy04igzhQ14XTbXZ4/ULCV9z3xy+I+OsJF4632OHvISFKFWJafb9u3Tps2rQJx48fx5IlS/Dyyy9jyZIlkAXav4wbNw6bN29GZWVlPG0lMgCX1489x9nQ/b6LJkBhT8Iasz4oeA5elRkA4HfZIfe5E567RIxMTEK0YcMG3HHHHVi2bBmKigZv+5Kfn4/f//73ozKOyDyCzQ0Najk0cvQWyzckR4g4joNCpYFXpoFPEKFydiUlSE4MT0xC9M4776C8vDzkAQURRRH19fUoLy+HUqnE0qVL42IkkTk0W1m6R4FRzURI8LNi+Wpz0mxQyXm4FCb4/A4WsCYhkpyYYkQTJkxAe/vARYOdnZ0YN27cqI0iMpfGbicAoNisSXix/KFQK2RwyY0sqZHqV6cEMQmRKIqD7rfb7VCr1aMyiMhcRFFEYzfziErMmqSV/uiPWsHDJTfBJ4g0hZ8iRDU0W7lyJQA2zl69ejW0Wm3oOb/fj3379mHmzJlxNZDIHDp7PHB5/VDwHPIMKuB0ckp/9Ecll6FbERAiSmpMCaISoi+++AIA+2X78ssvoVQqQ88plUrMmDEDDz/8cHwtJDKGJgvzhgpNGvAceodmSY7RTCkyolw5AaYzn7FlHlS/WnKiEqJ//vOfAIDly5dj/fr1MBqNCTGKyEzOKzaiyKSG1x/wRHweQCYHtDlJtaMiRweYK4EGOSvY73UCSu2IryMSR0yzZps2bYq3HcQYgOM45AS7drTWsG0ii+UPB68AVAbAbWMBaxIiSYlYiG688UZs3rwZRqMRN95447DH7ty5c9SGERlOkjOq++Ly+tFmc0MPPbJgYwFrU2nS7SB6iXjWzGQyhTpjmkymYW+x8PTTT4PjODz44IMxvZ5Ibd77ugVvHm5Cqy1QNkZCIWrsdmLH/rM41BHwxChgLTkRe0R9h2PxHpp99tln+O1vf4vp06fH9bxEauD1CzjWZIPHJ2BGWeCHKhSoTr4QqRRMgGycge2gXCLJiSmPyOl0wuFwhB7X1tbi+eefxz/+8Y+oz2W323HbbbfhpZdeQlZWVizmECnO6bYeeHwCDGo5yx9y23uL5euSO3UPAGo5+9jbZIHJFsolkpyYhOj666/Hyy+/DADo7u7G3Llz8dxzz+H666/Hhg0bojrXihUrcPXVV2Px4sUjHut2u2G1WsNuROrzRR0b+pxbbGTD+1Cx/Gy2vCPJqAMekZXTQ4TIPKIhknSJ5BBzhcZFixYBAHbs2IHCwkLU1tbi5Zdfxi9/+cuIz7N9+3YcOHAAa9eujej4tWvXhsWiysrKYjGfSCKN3U40WVzgZRxmlJrZziSWhh0MVcAjcvEG+ESOFdF32ySxhWDEJEQOhwMGAxtf/+Mf/8CNN94ImUyGb3zjG6itrY3oHPX19fiP//gPbN26NeJlIatWrYLFYgnd6uvrYzGfSCJf1HUDACYXGqBTBUKSEgaqAUDOy6DgOYgcD58iMDyjOJGkxFw8/y9/+Qvq6+uxa9cuXH755QCA1tbWiJMc9+/fj9bWVsyePRtyuRxyuRx79uzBL3/5S8jlcvj9/gGvUalUMBqNYTcidbE4vTjZyjyN2eV94n/BNWYSBKqDBIdnHmUgeE4zZ5ISU0Lj6tWrceutt+Khhx7CZZddhvnz5wNg3tGsWbMiOsdll12GL7/8Mmzf8uXLcc455+CRRx4Bz0uQ5EbEFY2Cx0WT8tBmc7O1ZQDLZA4GhyXyiADgG+NzIIgiVNY8wFFPAWuJiUmI/u3f/g3f/OY30dTUhBkzZoT2X3bZZfj2t78d0TkMBgOmTp0atk+n0yEnJ2fAfiI9UcplmFXebyY06A2pDMkrlj8IU0sCnlBDLtAC8ogkJiYhAoDCwkIUFoYvVpw7d+6oDSIyHInjQwPQBISSPCJJiUmIenp68PTTT2P37t1obW2FIAhhz8faCfb999+P6XVEaiGKInYdbUZplhaTCw1Q8H1CkbZAZxeJqyJaXV5YHF7oRQOyANb1VRAAGXVhl4KYhOiuu+7Cnj17cPvtt6OoqCi09IMgAFYO9liTDada7ZhUYAh/0haoQSSxEB2s68b+2i6cX27GhTI5IPgAt6XXQyKSSkxC9NZbb+HNN9/EwoUL420PkQF83cRmyqry9VDK+3gYfm9vm2eJhSg4a+byCYDGzOpnOzpJiCQiJj80KysL2dnZ8baFyAAEQcTxFiZE5xT2S68IdnVV6gClXgLregkmNbp9AqANfJYpYC0ZMQnRk08+idWrV4etNyMIgA3LnB4/VAoZyrL71fix9anIKPFwPuQRef0UsE4BYhqaPffcc6iurkZBQQEqKyuhUCjCnj9w4EBcjCPSj5r2HgBARbYOvKyf2NhTIz4EsE4eQHBoFvSISIikIiYhuuGGG+JsBpEpnOlgQlSZO0jFw+CMWZK7dgxG0CNye/00NEsBYhKixx57LN52EBmAXxChkMkgl3GozOmXrOj3AT2pEagGALU8IER9PSKXhdnJx5xeR8RIzFe8u7sbO3bsQHV1Nf7zP/8T2dnZOHDgAAoKClBSUhJPG4k0gZdxuPmCMvj8AuR8v/CjvQUQBVYbWmUY/ARJRKPksbAqF2qFDKJCC06uZMX8Xd2sjjaRVGISosOHD2Px4sUwmUyoqanB3XffjezsbOzcuRN1dXWhWkXE2GSACAGAtZFtjSWSB6oBtvxk7rg+M7+aLBZMd3SSEElATLNmK1euxLJly3Dy5MmwEh5LlizBBx98EDfjiPTC6xeGftJ6lm2NKeotU8BaUmISos8++wz33nvvgP0lJSVobm4etVFE+uEXRLz4wWm88kktnJ5+JVxEEbAEhMiUOkLUYXejvtPB7KWAtaTEJEQqlWrQMq0nTpxAXl7eqI0i0o92uxsenwCbyxeaGg/htgZqVMsAQ5E0Bg7CrqMt2LH/LBotzl6PiHKJJCEmIbruuuvwxBNPwOv1AmCN8+rq6vDII4/gO9/5TlwNJNKDNpsbAJBvUA1ce2hpYFt9PmtsmCIEBdPt7ZtdTUIkBTEJ0XPPPQe73Y68vDw4nU5cdNFFqKqqgsFgwJo1a+JtI5EGBIUoVACtL9aAEKVYE8Pe9WZ9sqvddjZ7RiSVmGbNTCYT3nnnHXz44Yc4dOgQ7HY7Zs+eHVEnDkI6BEFEi82FLK0y9CWMF8MKUXegjnmKBapD2dVeP6DQsJvXybyiFMh1GktELUSCIGDz5s3YuXMnampqwHEcxo0bh8LCQoiiSCVBUhS/IOJP+8+iodsJg1qOm+aUwaSJzzBJFEW02YcQIk8PYG9j97Mq4vJ+8UIVTGr0Bmb7tNlsGOnsIiFKMlENzURRxHXXXYe77roLDQ0NmDZtGs477zzU1tZi2bJlEZeJJZLPlw0WNHQ7AQA2lw97T7bH7dwWpxcenwC5jEO2tl+fsq6AN6TPk7Q07GCEeUQABawlJCqPaPPmzfjggw+we/duXHLJJWHPvffee7jhhhvw8ssv4/vf/35cjSRGz+Gz3QCA6aUmHD5rwalWO3rcvt4WP6NAFIEpRQYIIiDrv9C1q4ZtsypH/T7xRtV3mQdAAWsJicojevXVV/Ff//VfA0QIAC699FI8+uij2Lp1a9yMI+LH1dOK8I3xOVhYlYsikxqCKOJkqz0u587SKXHl1CIsmdZval4U+wjRuLi8VzwpNKmxsCoXU0sCdZPII5KMqITo8OHDuPLKK4d8/qqrrsKhQ4dGbRQRf3L0KsyfkAO1gseUIiOq8vVxixENSU87W0gqkwOm1OvKm6tXYe64bFTlB9a+BWfOyCNKOlH55Z2dnSgoGLr7QkFBAbq6KDM11ZlRZsaMMnPczmd3+6BV8AOHZe0n2DarUpIe91ETFCKvC/A42AJdIilE5RH5/X7I5UNrF8/z8Pl8ozaKiB+iKOKfX7fiaKMFvuHWgo3i/Fs+qsGv/3kK3Y5++TdBIcqdGPf3jQeCIKLV5kJ9pwOiKDKxDFYGoKUeSSUqj0gURSxbtgwq1SC5IgDcbndcjCLiR0ePBwfru6GUyzClTw1pURRhcbLMeHP/ma4o6PH44fEJ4DhA3zfw7ehkHTs4Dsipivn8icQvitj6SR0A4P5LJrDgtTYbcNvY8CyF1sVlOlEJ0dKlS0c8hmbMUotmiwsAW3rRd+i070wnPq7uwHnFRlx+Xuw5M109zAsyqhXh5T+aA+3Es8YBKmkL5Q+FgmdF3HyCCJdXYEKkyWYpBxSwTipRCdGmTZsSZQeRIDoCQtE/0TBXzx632EbnxXY7mFeVrevjVQkC0HKE3S+cNqrzJxq1gofd7WMlYzUKClhLBLW1zHDaA0ITFJ4ghSZWR6rT7hm+jtAIdAXiQmZtnxm49uOAywoo1EDupJjPnQxUwYWvoVyiHLYN9l8jkgIJUYbT0cOEKEcfHgfSq+TQKHkIohgaXsVCUIiygnEmUQTqPmb3S+akfP3nYO3qUHa1LihEXcyzI5ICCVEG4/D40ONmX7Ac3cAJhuBwqmMUQhQcmoWEqOUoK7nKK4CS82M+b7JQhZZ5BERHbWbiKfhY/WoiKZAQZTCdAYExaRThrZ8D5Aa8pA577EI0udCASQUGZOuVbIFr9W72RMWCtMjDCSsFArBZvuDwrCd+6/GI4Ultv5kYFaVZWtx70fiQV9Sf7ICXFBy+AWD96UWReTQRVFL4xvjAl9bnBg7vZImAulygbN6o7U8GVfl6ZGmVKM3S9O7U5gYK6bcDSO0YV6ZAQpThaJVyaJWD/5tLzBrMG5eNElkH8NXrbNrawxokQsazYYo2mwmLNjewzQmvsiiKgKUeOPkPVu5DrgTOu5G9Pg2YkKfHhP7VjXWBHT1tSbdnrEJCNIbJ0/LI8+0Dmg4PfFLws5kjRwfQfrJ3P8cBahOgNsPp8YJzdkIlOMCBY0OxaTf1BnzTlWA7IRqaJQ0Sogzm3a9aoJTLcH5F1sByH34vcPgPQHc9E5fC6SznR5cLcDzgdbBlDo5ONkTpaWcegtcJOLsBZzea2u1osbpRnG1AxblzgXEXplzNoZHw+AR0Oz0QRaDAGGiNFZrC72QzZzIKpSYaEqIMxS+IONJogSgCsyuywp8UReDYG0B3PdycAu3lV0NTMCE8KVGuBDRmILtf+Q5PDxMltw2njrWiTiOHaca5QFl6NiVssjix80ADcg0q3P6NQAVJtZlVDAjOnAXrFBEJg6Q+Q7E6vRBFQMFz0Cn7xWuaDgJtxwEZj890l+K1ah5fNw9sDzUoSh0r+Vo4FXXySthUhcgyppcX1BdNcNasby82max3eEnDs6RAQpShBBMNTVpleB1xtw04FZhiH38xNPnM4+mMMpco2MMM6JNDlIaoAyLt9PrZCvwg2oCH5yAhSgYkRBlKd3Blff/iZzUfsviQsQgovSA0HIs2u7rbyY7XKPm4dwRJJkGPyC+Ivcs8AApYJxkSogzF0j/jGWDB16ZABc0JlwJcb7H7LocXgiD2P82Q9GZUp07DxFhQ8LJQsmdomQdAHlGSISHKUIIeS9hi1PpPAVFg9YHM5QAAg1oOuYyDXxBhdXkjPn9wKDeaWkapQtCjc/YVopBH1EFrzpIACVGGYg/Eb0J1qT2O3hpBZXNDx8lkHMyB4Vk0caLSLA0uqMzGhLz0DVQHCQ7PnH0D1v1nzoiEQtP3Gcr3vlGBHo8f6uAas6aD7EtlKAh5Q0FydEq029yhAHcklGZpUZqV+mvJImFaiQkTC/Thw1iZjPViszYB9laawk8wJEQZCsdxvaVbRbHXGyqZM2AN2dRiEypytCgxazAWmVZqGvwJfUFAiFqA/HOSa9QYg4RoLGBtYIFqXgHkDfxCledE59l4fAJarC5k6ZThdaozDV0+29pbpbVjDCBpjGjt2rW44IILYDAYkJ+fjxtuuAHHjx+X0qSM4GSLDW8casTRRgvbEfSG8ibHpa1Pm92NHfvP4g+f1Y/6XKmAy+tHq9WFDnu/srn6oBC1JN+oMYakQrRnzx6sWLECn3zyCd555x14vV5cfvnl6OnpkdKstKfJ4sKpVjva7R7A7wNaj7EnhqkfXd/pwKH67vAp7CEI5hyl+9R9kJMtdmzdV4e9p/pN1QeFyG1jwX4iYUjqV7/99tthjzdv3oz8/Hzs378fF154oURWpT/BNkEmjQLormW1glR6wFwx5Gv+8VULrE4vcvTKEYPQofKwuvSfugcAjXKQPCIAkKtYMX1nF/OK+q+7I+JGSk3fWyxsKJGdPfgMhdvthtVqDbsRAwkTolCTw0nDFjrL1jHvpqtn5Fyi4DR/dgbkEAG9eUQOzyDeoJ7iRMkgZYRIEAQ8+OCDWLhwIaZOnTroMWvXroXJZArdyspSr5+61PRtnGhS8xF3Ww1OXXdGMIUfHJplZ4hHFCwc5xxsWKoPtFinOFFCSRkhWrFiBY4cOYLt27cPecyqVatgsVhCt/r6zAiWxhOXV4AnsGbK6GlhsQ25athhGdBbXH+kNWd+QYTFyZIlzRkSIwomNLq9Avz9l7kEhaiHPKJEkhJzrw888AD+9re/4YMPPkBpaemQx6lUqiHbXROMoDekV8kh7zzFduZUjVi6NSswNBspu7rb4YEgilDKZRkzda+Sy8BxLN3K5fWHF5HTB8vGdrDAf4q3R0pXJL2qoijihz/8If785z/j/fffx7hxFAwcLQ6PD3IZB5Na3lviNYImh8FhltXlhdcvQMEP7ixrlXIsnlIAj18ILy+SxshkHNQKHk6PHw5PPyFSGVmjSK+LVag0FklnaAYjqRCtWLEC27Ztw+uvvw6DwYDm5mYAgMlkgkYzNrN8R8v4PD0euLQKHls7sL+LeULZ40d8nUbBynm4vH50OTzIN6gHP07JD52JnMacX5EFDuzvC4PjAEMR0HkGsDWRECUISYVow4YNAICLL744bP+mTZuwbNmy5BuUIXAcB5W1jj0wlkSUxMhxHL51bgHUCllaFzqLlQsqh1lLFhQiayNQMjt5Ro0hJB+aEQmi6wzbRpH7UpWvH/GYU612aJQ88g2qIYdvGYexhG1tTdLakcFQ5C3DeP1gA3hOwOKO01BzALLiF3cTRRG7jjbD4xNw+/wK5OozZ+LA5fXD6vRCzssGpiUEh2OODpYcKs+cvztVGCM/aWMDvyCipt2B5rrTkAleQKEBDIURv97p8eNIgwX7azsHfd7q9MHjE8DLuIwbvh1ttGDrvjrsO90x8EmljvVyE0XyihIECVEGYXf5IIgicjwNUPAcG5ZFMbPl8vrxzlct+Li6Y9Bhc5vdBQDI0SvByzJjxixIMKmxZ7DsaqDXK7I2JsmisQUJUQYRzCEq9DexzqtRDstMGgV4GQevX4Q1UOGxL202lmOUSUOyILqAEDk8A/9uAIChmG1JiBICCVEGYXF6wftdyBICw4usyqheL5NxoRX1g2VYtwXKZGSkEKnYtH2PeyiPKCBEtiY2RCPiCglRBmFxemF2N0All7Hi72pj1OcIrqjv6HEPeK7NxvblGzJRiJhH5PL64fMPUizfUAhwMsBtB9y02DrekBBlEBanFybnWVanOsaSFcFExmZLuBD1uH2wOr3gOCAvA4VIJZeF4l6OwRa/8gpW7xsAummNY7whIcogRFFAtqcBKgUf87R9kYkJUZPFGbZfreBxywVlWDylIK0bKg4Fx3HQBrKqHUMNz0yBag8WEqJ4Q3lEGcQ1VWqInUqIHD+gU0ekFBjVkHEcbC4f7G5faGErL+NQbNagOIML7M+uyIIgiNCqhhBacznrDUceUdwhIcokus6AAwfOXM6GEjGglMtw4+wS5OpVA9ddZTizy7OGP8BUytIhHB0sVqQaOROdiAwammUSndEv6xiMsmxtmAg5PD7sOtqMU622UZ037VFoejvAWs5Ka0uGQUKUIXzd2IWDRw6hodsZ9bT9UIiiCFEUcbqtB181WrHvzOAZ15mCxyeg1epCq9U19EGmwJCX4kRxhYQoQ7C31sLpdMEJVW9VwVHwdbMVr+yrw8lWe6gtUVVeZg9FTrWybh7/Otk+9EHmQMC6qyYpNo0VKEaUIXjbqwEAspzxUS3rGIpOuwftNjfePMzWVvEyDlNLMq8OUV8MavZ1sLmGaSBgrmDXt6cdcFljytUiBkIeUYbABX6hVfkT4nK+2RVZoS8mAMypyAqvXJiBGNUswG9z+YYuUaPU9i4kDpZaIUZNZn+yxgii2w7O1gwRgK6wKi7nVCt43DSnDIfPdsOkUWBahntDAFvmwXGATxDh9PpDC2EHkD0esDYBnaeBohnJNTJDIY8oA+hpqYYginAqc2A0jjAFHQUmjQKLJuZheqk5Y+pTD4ecl4UWv9oGWfQbIlh6t6sGEAZZDkJEDQlRBuBsZt06fOZKyDKsPEey0UcSJzIUs+JoXhfVJ4oTJETpjihCZqmBRsGDzxm5SD4xPMG42GBlUELIZL0pEp3ViTdqDEAxonSnpx05cg9yKnMhzqPC7qPlnEIjikxqlGaNsJQldyLQdpzdxl2YHOMyGBKidKfzNNuaysHFuKyD6CWSBgIAWNNKTsam8Xs6AF1OYg3LcGholuaIHSchQoyodxkRRxSa3uFZ+3FJTckESIjSGY8Dno46fFbThT/Xa6g9UxwQBBGtNhdOtthGvp55gQ66bSREo4WEKJ3pPA2n2wsbnwUbpx8TU+yJRgTw6r56/O1wE+zuYQLWAGvlzXGArRlwdiXFvkyFhCid6TgJh9ePLk0FcnSZVzVRCngZB6OGhU67HcNM4QOszVCw7lPL0QRbltmQEKUrfh/QeRo9bh86NRXI1WdWnzEpCfZsG1GIAKBwGts2f0lF9UcBCVG6YqkDfB50+5ToUeahwKiW2qKMwRTsZOIY2MlkALmTWRE6ZzeVBhkFJETpSuvX8AkCGuWsaiAJUfwIekQRCZFcCeRPYfebjyTQqsyGhCgd8fuAtq/R4/ajXTsBJo1izJV1TSQ5gZZK7fYIhAjoHZ61HgW8zuGPJQaFhCgd6ToD+NyASo/c0ipU5mqltiijCLZLsjq9cA3WWqg/pjJAn8d+IJoOJdi6zIQyq9ORwAyNqXw6rp9YJrExmYdaweOiyXmhFtwjwnFA6QXA138HGvYDpXPZejQiYuhqpRteF9Bxkt0vOFdaWzKY2eVZmJCnh4KP8CuSfx4rmuayUqZ1DJAQpRstRwC/Dx51DizyXKmtIYLwcqA4sOi4Zi9N5UcJCVE6IYpA4xcAgDPKidj4YQ3ePkL1cBKBzy/gdJsdn9dE0bmkdA6rU9TTDrR+lTjjMhASonTCcpZ9yHk5TqACAJCjp4zqROD1i3j9YCP+dbIdDs8ISz2CKDRA2Tx2/8y/ACGCQDcBgIQovajfBwDw5U5BrYV9yCuyacYsEWiUfChbvbE7iin50gtYrMjZxQLXRESQEKUL9lag/STAcajTT4fXL8KoUYSmmon4UxIojna2Kwohkit7C6Wd+YAFr4kRISFKF2r2sm3eZJy0sV/qCXk6WnGfQEqzmLdZ2+GI7oVFMwFTCeD3Aid2UeA6AkiI0oHuOlbzhuPgLp2PU612AFFUEyRiojxbC17GobPHgw67O/IXchww6UpAxgMdp2iIFgEkRKmO4AdOvsPuF83EGacWHp+AbJ0SJeYR6ioTo0Kt4FEeiMEFxT9i9PnAhEvZ/er3gG5aEDsclFmd6tT8i8WHFGpg3CJMVmghl3HwC6BhWRKYWKDHmfYetEXjEQUpOR/orgXaTgBHdgCzbgd0lPs1GCREqUz7KaDuE3Z/0lWAUgcOQFW+QVKzxhKTCgzI0ipRZIqhugHHAVOuA9zbAGsjcHAbMP0WwFAQf0PTHBqapSrd9cBXf2aBzuKZcJonRp7PQsQNBS9DsVkTu/fJK4Bp/8aGap4e4OArQOux+BqZAaSEEP3mN79BZWUl1Go15s2bh08//VRqk6RDFFldm0Pb2Wru7HHwjV+MN79swpaPaqOPVRBxw+ry4mijJfoXKnXAzNtYWVmfBzj6F+Cr1wFXDOfKUCQXoj/84Q9YuXIlHnvsMRw4cAAzZszAFVdcgdbWVqlNSz7WJuDLHcCxNwDBB+ROhGXCddh5sBn1nQ4IogizlnqXSUGP24dt++rwzlctOFjfHX3HFIUamPFdoGI+G7K1fAXsexH4+k3A0jDmp/g5UeIeNPPmzcMFF1yAX//61wAAQRBQVlaGH/7wh3j00UeHfa3VaoXJZILFYoHRaEyGufFBFFkBLU8P4GhnXSA6qwF7GwDA7RfRmjMHXymm4kRLD3yCCKVchutmFKOMMqkl4/3jrfiirhsAS3acWmxCsVkNo1oBWSTlQoLYmoFTu1laRhClDsiqAPQFgC4PUBkApZ4tG0njSYlIv6OSBqs9Hg/279+PVatWhfbJZDIsXrwYH3/88YDj3W433O7e2QurNcKs1c4zwKl3B+4P02Cx3z5xkOPC951qtaPb6Q4dwgGBX0p23PnlWQh+PqvbbOiwewCI4AUvIAphZ5xZZoaclwP5U7CnpwpH2+UA2DCsJEuDxVMKkK2jAvlSctGkPOhVcnxc3YGGLicaAhnXHAd8Z3Zp6EfiSIMF+2u7MJQ2fevcQhTOvBWwNqDh6F4ouqqRjx7mJbUMslhWJmer+2UKlpuEwIk5rt99DLyfSKbdDKjj4wBIKkTt7e3w+/0oKAifRSgoKMDXX3894Pi1a9fi8ccfj/6NfG62WDTO+JxWeIfp9MB5VaEPheBxwu9hpUeDSyH9MiWcchMcimxMmzwX8oJJgEIDzck25AsOFJs0mFxoQJFJTVP1KQDHcZhTmY1JhQYcabCgpt2BdrsbfkGEWtFbqtfp9aOzZ+gys16WewGYStFediWEYh/yjTbA1gTYWwBHB+C295adFXzsBleC/8IoCfyYxoO0mr5ftWoVVq5cGXpstVpRVhZBhUJzGTDj3/v9UgzySxLRvt7ncns8MPiE0D6OAzhO1nu4Uc36owMocHph8gvsSF4FUaEBJ5OHjlVolQj+hC6amDfy30RIhlGtwIIJuVgwgXWGdfn8UMl7hWhyoQGFRvWQYZ++6wPH5+mYV6zOBbLHhR/o9wF+N1sqIvjYVgz8jPX10ge9nwSUuridSlIhys3NBc/zaGlpCdvf0tKCwsLCAcerVCqoVDEs8lTqBv6T40BuFF6pUQekURSLiBCZjINWGf41MqoVMKojm1QwDHccHxiSjQEknTVTKpU4//zzsXv37tA+QRCwe/duzJ8/X0LLCIJIJpLL7cqVK7F06VLMmTMHc+fOxfPPP4+enh4sX75catMIgkgSkgvRLbfcgra2NqxevRrNzc2YOXMm3n777QEBbIIgMhfJ84hGQ9rmERHEGCHS76jkmdUEQRAkRARBSA4JEUEQkiN5sHo0BMNbES/1IAgiqQS/myOFotNaiGw2GwBEll1NEIRk2Gw2mEymIZ9P61kzQRDQ2NgIg8Eg+Vqs4HKT+vp6msHrA12XoRkL10YURdhsNhQXF0MmGzoSlNYekUwmQ2lpqdRmhGE0GjP2QzUa6LoMTaZfm+E8oSAUrCYIQnJIiAiCkBwSojihUqnw2GOPxVYdIIOh6zI0dG16SetgNUEQmQF5RARBSA4JEUEQkkNCRBCE5JAQEQQhOSREURBNR9qXXnoJixYtQlZWFrKysrB48eKM7WAba6fe7du3g+M43HDDDYk1UEKivTbd3d1YsWIFioqKoFKpMGnSJPz9739PkrUSIhIRsX37dlGpVIobN24Ujx49Kt59992i2WwWW1paBj3+1ltvFX/zm9+IX3zxhXjs2DFx2bJloslkEs+ePZtkyxNLtNclyJkzZ8SSkhJx0aJF4vXXX58cY5NMtNfG7XaLc+bMEZcsWSLu3btXPHPmjPj++++LBw8eTLLlyYeEKELmzp0rrlixIvTY7/eLxcXF4tq1ayN6vc/nEw0Gg7hly5ZEmSgJsVwXn88nLliwQPzd734nLl26NGOFKNprs2HDBnH8+PGix+NJlokpAw3NIiDYkXbx4sWhfcN1pB0Mh8MBr9eL7OzsRJmZdGK9Lk888QTy8/Nx5513JsNMSYjl2vz1r3/F/PnzsWLFChQUFGDq1Kl46qmn4Pf7Bz0+k0jrRa/JItqOtIPxyCOPoLi4OOyDme7Ecl327t2L3//+9zh48GASLJSOWK7N6dOn8d577+G2227D3//+d5w6dQr3338/vF4vHnvssWSYLRkkREng6aefxvbt2/H+++9DrVZLbY5k2Gw23H777XjppZeQm5srtTkphyAIyM/Px4svvgie53H++eejoaEBv/jFL0iIiOg70vbl2WefxdNPP413330X06dPT6SZSSfa61JdXY2amhpce+21oX2CwPqny+VyHD9+HBMmTEis0Ukils9MUVERFAoFeL63ffWUKVPQ3NwMj8cDpVKZUJulhGJEERBrR9qf//znePLJJ/H2229jzpw5yTA1qUR7Xc455xx8+eWXOHjwYOh23XXX4ZJLLsHBgwczqtJmLJ+ZhQsX4tSpUyFxBoATJ06gqKgoo0UIAE3fR8r27dtFlUolbt68Wfzqq6/Ee+65RzSbzWJzc7MoiqJ4++23i48++mjo+KefflpUKpXijh07xKamptDNZrNJ9SckhGivS38yedYs2mtTV1cnGgwG8YEHHhCPHz8u/u1vfxPz8/PFn/3sZ1L9CUmDhCgKfvWrX4nl5eWiUqkU586dK37yySeh5y666CJx6dKloccVFRUigAG3xx57LPmGJ5horkt/MlmIRDH6a/PRRx+J8+bNE1UqlTh+/HhxzZo1os/nS7LVyYfKgBAEITkUIyIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCckiICIKQHBIigiAkh4SISCs2b94Ms9kcevw///M/mDlzZujxsmXLMrriY6ZCQkQMyrJly8BxHO67774Bz61YsQIcx2HZsmVhx8dbACorK/H888+H7bvllltw4sSJIV+zfv16bN68OfT44osvxoMPPhhXu4j4Q0JEDElZWRm2b98Op9MZ2udyubBt2zaUl5dLYpNGo0F+fv6Qz5tMpjCPiUgPSIiIIZk9ezbKysqwc+fO0L6dO3eivLwcs2bNGtW5B/NUbrjhhpCXdfHFF6O2thYPPfQQOI4Dx3EABg7N+tPXM1u2bBn27NmD9evXh85x5swZVFVV4dlnnw173cGDB8FxHE6dOjWqv4uIDRIiYljuuOMObNq0KfR448aNWL58ecLfd+fOnSgtLcUTTzyBpqYmNDU1RX2O9evXY/78+bj77rtD5ygvLx/wNwHApk2bcOGFF6KqqipefwIRBSRExLB873vfw969e1FbW4va2lp8+OGH+N73vpfw983OzgbP8zAYDCgsLByxAN1gmEwmKJVKaLXa0Dl4nseyZctw/PjxUGsfr9eLbdu24Y477oj3n0FECFVoJIYlLy8PV199NTZv3gxRFHH11VenfZnX4uJiXH311di4cSPmzp2LN954A263GzfddJPUpo1ZyCMiRuSOO+7A5s2bsWXLlrh5DTKZDP0r0Hi93ricOxLuuuuuUCB+06ZNuOWWW6DVapP2/kQ4JETEiFx55ZXweDzwer244oor4nLOvLy8sLiP3+/HkSNHwo5RKpWjbqUz1DmWLFkCnU6HDRs24O2336ZhmcTQ0IwYEZ7ncezYsdD9obBYLAPaBOXk5Axai/rSSy/FypUr8eabb2LChAlYt24duru7w46prKzEBx98gH//93+HSqWKaUhYWVmJffv2oaamBnq9HtnZ2ZDJZKFY0apVqzBx4sRha48TiYc8IiIijEYjjEbjsMe8//77mDVrVtjt8ccfH/TYO+64A0uXLsX3v/99XHTRRRg/fjwuueSSsGOeeOIJ1NTUYMKECcjLy4vJ7ocffhg8z+Pcc89FXl4e6urqQs/deeed8Hg8SZkFJIaHSsUSY5Z//etfuOyyy1BfXz+gESKRXEiIiDGH2+1GW1sbli5disLCQmzdulVqk8Y8NDQjxhyvvvoqKioq0N3djZ///OdSm0OAPCKCIFIA8ogIgpAcEiKCICSHhIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCckiICIKQnP8HIaoZ92HQ/HgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:41.424789Z", + "iopub.status.busy": "2024-02-29T23:33:41.424484Z", + "iopub.status.idle": "2024-02-29T23:33:41.629345Z", + "shell.execute_reply": "2024-02-29T23:33:41.628265Z" + }, + "papermill": { + "duration": 0.228528, + "end_time": "2024-02-29T23:33:41.631475", + "exception": false, + "start_time": "2024-02-29T23:33:41.402947", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArXUlEQVR4nO3de1hU16E+/neGy3BHFLmKcjOiSQCFB9SaxgsCyS9Ga5oQk8hFq+do7ImdWFtSxQe14TRRQ+wxcuoJoqaJ9mJsTpJ66URzvOANg6IVDAioEVBUGIE4Dsz6/eGX3YzM4AyCA5v38zzz4F6z9p612Mzr3rP2rK0QQggQEcmQ0tYNICLqKQw4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLbsbd2A3shgMODq1atwd3eHQqGwdXOI6AeEELh9+zYCAgKgVHZ+jMaAM+Hq1asICgqydTOIqBOXL1/GkCFDOq3DgDPB3d0dwL1foIeHh41b03P0ej327t2LxMREODg42Lo59JD6y/7UarUICgqS3qedYcCZ0H5a6uHhIfuAc3FxgYeHh6zfEP1Ff9uflnx8xEEGIpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFs8TIRoj6qpaUFpaWl0nLT9zocKamAl/dJuDmrjOpGRETAxcXlUTfR5hhwRH1UaWkpYmJiOpS/Y6JuUVERxowZ0/ON6mUYcER9VEREBIqKiqTlspoGqP9cgnUvPokR/gM61O2PGHBEfZSLi4vRUZmy+gZUB7/HyCeiED1skA1b1ntwkIGIZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLZsH3IYNGxAcHAwnJyfEx8fj+PHjZusWFBRAoVAYPZycnIzqpKend6iTnJzc090gol7IphNe7tixA2q1Gnl5eYiPj0dubi6SkpJQVlYGHx8fk+t4eHigrKxMWlYoFB3qJCcnY/PmzdKySqXqUIeI5M+mR3Dr1q3DvHnzkJGRgVGjRiEvLw8uLi7Iz883u45CoYCfn5/08PX17VBHpVIZ1fHy8urJbhBRL2WzI7i7d++iqKgImZmZUplSqURCQgIKCwvNrtfU1IRhw4bBYDBgzJgxePvtt/H4448b1Tlw4AB8fHzg5eWFyZMnY/Xq1Rg0yPwUzjqdDjqdTlrWarUAAL1eD71e39Uu9nrtfZNzH/uT1tZW6aec96k1fbNZwNXX16Otra3DEZivr6/RrdB+aMSIEcjPz0dkZCQaGxuxZs0ajB8/HufOncOQIUMA3Ds9nTlzJkJCQlBRUYG33noLzzzzDAoLC2FnZ2dyuzk5OcjOzu5Qvnfv3n5xq7V9+/bZugnUDS43AYA9jh49iu/O2ro1PaelpcXiugohhOjBtph19epVBAYG4siRIxg3bpxUvnTpUnz99dc4duzYA7eh1+sxcuRIzJo1C6tWrTJZ5+LFiwgLC8M//vEPTJkyxWQdU0dwQUFBqK+vh4eHh5U96zv0ej327duHqVOnwsHBwdbNoYd0+tJN/HTTSfxlXiyihg60dXN6jFarhbe3NxobGx/4/rTZEZy3tzfs7OxQV1dnVF5XVwc/Pz+LtuHg4IDRo0ejvLzcbJ3Q0FB4e3ujvLzcbMCpVCqTAxEODg794o3fX/opd/b29tJPOe9Pa/pms0EGR0dHxMTEQKPRSGUGgwEajcboiK4zbW1tKCkpgb+/v9k6V65cwY0bNzqtQ0TyZNNRVLVajU2bNmHLli04f/48FixYgObmZmRkZAAAUlNTjQYhVq5cib179+LixYs4deoUXnvtNVRXV+NnP/sZgHsDEL/85S9x9OhRVFVVQaPRYPr06QgPD0dSUpJN+khEtmPT6+BSUlJw/fp1ZGVloba2FtHR0di9e7c08HDp0iUolf/K4Fu3bmHevHmora2Fl5cXYmJicOTIEYwaNQoAYGdnhzNnzmDLli1oaGhAQEAAEhMTsWrVKl4LR9QP2WyQoTfTarXw9PS06EPMvkyv1+PLL7/Es88+K+vPbPqL4uobmLHxKHYtGCvrO9tb8/60+Ve1iIh6CgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki2bB9yGDRsQHBwMJycnxMfH4/jx42brFhQUQKFQGD2cnJyM6gghkJWVBX9/fzg7OyMhIQHffvttT3eDiHohmwbcjh07oFarsWLFCpw6dQpRUVFISkrCtWvXzK7j4eGBmpoa6VFdXW30/DvvvIP169cjLy8Px44dg6urK5KSknDnzp2e7g4R9TI2Dbh169Zh3rx5yMjIwKhRo5CXlwcXFxfk5+ebXUehUMDPz096+Pr6Ss8JIZCbm4tly5Zh+vTpiIyMxNatW3H16lXs2rXrEfSIiHoTe1u98N27d1FUVITMzEypTKlUIiEhAYWFhWbXa2pqwrBhw2AwGDBmzBi8/fbbePzxxwEAlZWVqK2tRUJCglTf09MT8fHxKCwsxMsvv2xymzqdDjqdTlrWarUAAL1eD71e/1D97E1aWlpQVlYmLTd9r8ORkgq4DzgKN2eVVD5ixAi4uLjYoon0EFpbW6Wfcvq7vZ81fbNZwNXX16Otrc3oCAwAfH19UVpaanKdESNGID8/H5GRkWhsbMSaNWswfvx4nDt3DkOGDEFtba20jfu32f6cKTk5OcjOzu5QvnfvXlm90SsqKvDmm292KH/nvuW1a9ciLCzs0TSKus3lJgCwx9GjR/HdWVu3pue0tLRYXNdmAdcV48aNw7hx46Tl8ePHY+TIkfjv//5vrFq1qsvbzczMhFqtlpa1Wi2CgoKQmJgIDw+Ph2pzb9LS0oIJEyZIyxdqGvHLT/+Jd38yCo/5e0rlPILrm05fugmUnMTYsWMRNXSgrZvTY9rPsCxhs4Dz9vaGnZ0d6urqjMrr6urg5+dn0TYcHBwwevRolJeXA4C0Xl1dHfz9/Y22GR0dbXY7KpUKKpWqQ7mDgwMcHBwsaktf4Onpibi4OGnZsfoGVIV38UT0GEQPG2TDllF3sLe3l37K6e/2ftb0zWaDDI6OjoiJiYFGo5HKDAYDNBqN0VFaZ9ra2lBSUiKFWUhICPz8/Iy2qdVqcezYMYu3SUTyYdNTVLVajbS0NMTGxiIuLg65ublobm5GRkYGACA1NRWBgYHIyckBAKxcuRJjx45FeHg4Ghoa8O6776K6uho/+9nPANwbYV28eDFWr16N4cOHIyQkBMuXL0dAQABmzJhhq24SkY3YNOBSUlJw/fp1ZGVloba2FtHR0di9e7c0SHDp0iUolf86yLx16xbmzZuH2tpaeHl5ISYmBkeOHMGoUaOkOkuXLkVzczPmz5+PhoYGTJgwAbt37+5wQTARyZ9CCCFs3YjeRqvVwtPTE42NjbIaZLhfcfUNzNh4FLsWjOVncDLQX/anNe9Pm39Vi4iopzDgiEi2GHBEJFsMOCKSrT71TQai/q6yvhnNulaTz1Vcb5Z+tl/0a4qryh4h3q490r7ehgFH1EdU1jdj0poDD6z35l9KHlhn/5KJ/SLkGHBEfUT7kVtuSjTCfdw6Pv+9Dp8fKMRzE8fB1bnjVw8BoPxaExbvKDZ7FCg3DDiiPibcxw1PBHp2KNfr9agdDIwZ5iXr76Jag4MMRCRbDDgiki2rT1EvXryI0NDQnmgLdbPORtwAjrqR/FkdcOHh4Xj66acxd+5c/PSnP+WX2HspS0fcAI66kXxZHXCnTp3C5s2boVarsWjRIqSkpGDu3LlGEymS7T1oxA3gqBvJn9UBFx0djffffx9r167FZ599hoKCAkyYMAGPPfYY5syZg9mzZ2Pw4ME90VbqAnMjbgBH3Uj+ujzIYG9vj5kzZ+LPf/4zfve736G8vBxLlixBUFAQUlNTUVNT053tJCKyWpcD7uTJk1i4cCH8/f2xbt06LFmyBBUVFdi3bx+uXr2K6dOnd2c7iYisZvUp6rp167B582aUlZXh2WefxdatW/Hss89KM++GhISgoKAAwcHB3d1WIiKrWB1wGzduxJw5c5Cenm5056of8vHxwYcffvjQjSMiehhWB9y+ffswdOhQo3slAIAQApcvX8bQoUPh6OiItLS0bmskEVFXWP0ZXFhYGOrr6zuU37x5EyEhId3SKCKi7mB1wJm7R01TUxMv+iWiXsXiU1S1Wg3g3r1Hs7Ky4OLiIj3X1taGY8eOdXr3eCKiR83igPvmm28A3DuCKykpgaOjo/Sco6MjoqKisGTJku5vIRFRF1kccPv37wcAZGRk4P3335f1/UKJSB6s/gxu8+bN3RpuGzZsQHBwMJycnBAfH4/jx49btN727duhUCgwY8YMo/L09HQoFAqjR3Jycre1l4j6DouO4GbOnImCggJ4eHhg5syZndbduXOnxS++Y8cOqNVq5OXlIT4+Hrm5uUhKSkJZWRl8fHzMrldVVYUlS5bgqaeeMvl8cnIyNm/eLC2rVKa/SE5E8mbREZynpycUCoX0784e1li3bh3mzZuHjIwMjBo1Cnl5eXBxcUF+fr7Zddra2vDqq68iOzvb7Lx0KpUKfn5+0sPLy8uqdhGRPFh0BPfDo6Ef/vth3L17F0VFRcjMzJTKlEolEhISUFhYaHa9lStXwsfHB3PnzsXBgwdN1jlw4AB8fHzg5eWFyZMnY/Xq1Rg0aJDZbep0Ouh0OmlZq9UCuDfbhl6vt7ZrvUJra6v001wf2ss766Ml26FH40H7or/sT2vabbObztTX16OtrQ2+vr5G5b6+vigtLTW5zqFDh/Dhhx+iuLjY7HaTk5Mxc+ZMhISEoKKiAm+99RaeeeYZFBYWws7OzuQ6OTk5yM7O7lC+d+9eo8th+pLLTQBgj0OHDqHa9HRwkn379nXLdqhnXbx9F0qnW/j04J9Q6Gy+3pa/bzH7XO33gNLJDvsPfoVqd0ez9XqzlpYWi+taFHCjR4+WTlEf5NSpUxa/uDVu376N2bNnY9OmTfD29jZb7+WXX5b+/eSTTyIyMhJhYWE4cOAApkyZYnKdzMxM6To/4N4RXFBQEBITE/vsaPG5q1qsKTmKCRMm4PEA033Q6/XYt28fpk6danY+OEu2Q4/GF2Un4dq2EjvbADR1fTuuIUBI9B/w7IjYbmvbo9R+hmUJiwLu/pHK7uDt7Q07OzvU1dUZldfV1cHPz69D/YqKClRVVWHatGlSmcFgAHBvbrqysjKEhYV1WC80NBTe3t4oLy83G3AqlcrkQISDg0OfnQiy/R4L9vb2D+xDZ/20ZjvUs4I8QtFc+XO8nxKNMBOzNLe2tuLwocP40YQfmb3HRsW1JryxoxhBk0L77P60pt0WBdyKFSu63BhzHB0dERMTA41GIwWowWCARqPBokWLOtSPiIhASYnxvQOWLVuG27dv4/3330dQUJDJ17ly5Qpu3LhhduYTudK13YHS6TtUasugdDJ9btna2oqrrVdx/uZ5s2+ISm0TlE7fQdd2B4B1g0jUvVR2TjDcCUSIxwiMGmT6vqiV9pUYOXCk2RAw3GmE4c51qOz6x9cqbXrjZ7VajbS0NMTGxiIuLg65ublobm5GRkYGACA1NRWBgYHIycmBk5MTnnjiCaP1BwwYAABSeVNTE7Kzs/HCCy/Az88PFRUVWLp0KcLDw5GUlPRI+2ZrV5ur4Rrye7xlwWWFH+z+oNPnXUOAq83RiIFvp/WIehuLAm7gwIG4cOECvL294eXl1enncTdv3rT4xVNSUnD9+nVkZWWhtrYW0dHR2L17tzTwcOnSpQ7TMnXGzs4OZ86cwZYtW9DQ0ICAgAAkJiZi1apV/e5auADXYZ2ezgDWndIETBrWk80l6hEWBdx7770Hd3d36d+WDjhYYtGiRSZPSYF7l3t0pqCgwGjZ2dkZe/bs6aaW9W0POp0BeEpD8mdRwP1w8sr09PSeagsRUbey+ruodnZ2uHbtWofyGzdumL3OjIjIFrptwkudTmc0hRIRka1ZPIq6fv16APcmvPyf//kfuLn964PrtrY2/N///R8iIiK6v4VERF1kccC99957AO4dweXl5Rmdjjo6OiI4OBh5eXnd30Iioi6yOOAqKysBAJMmTcLOnTs5QwcR9XpWX+jbPrMvEVFvZ3XAzZkzp9PnO5vLjYjoUbI64G7dumW0rNfrcfbsWTQ0NGDy5Mnd1jAioodldcB9+umnHcoMBgMWLFhgcjYPIiJbsfo6OJMbUSqhVqulkVYiot6gWwIOuDdfW/t0yEREvYHVp6g/nPkWuHddXE1NDb744guj76wSEdma1QHXfof7dkqlEoMHD8batWsfOMJKRPQo8To4IpKtbvsMjoiot2HAEZFsMeCISLYYcEQkW90WcFeuXMH8+fO7a3NERA+t2wLuxo0b+PDDD7trc0RED42nqEQkWww4IpItBhwRyZbF32SYOXNmp883NDR0qQEbNmzAu+++i9raWkRFReH3v/894uLiHrje9u3bMWvWLEyfPh27du2SyoUQWLFiBTZt2oSGhgb86Ec/wsaNGzF8+PAutY+I+i6Lj+A8PT07fQwbNgypqalWvfiOHTugVquxYsUKnDp1ClFRUUhKSjJ539UfqqqqwpIlS/DUU091eO6dd97B+vXrkZeXh2PHjsHV1RVJSUm4c+eOVW0jor7P4iO4zZs3d/uLr1u3DvPmzUNGRgYAIC8vD1988QXy8/Px61//2uQ6bW1tePXVV5GdnY2DBw8aHTkKIZCbm4tly5Zh+vTpAICtW7fC19cXu3btwssvv9ztfSCi3svqL9t3l7t376KoqAiZmZlSmVKpREJCAgoLC82ut3LlSvj4+GDu3Lk4ePCg0XOVlZWora1FQkKCVObp6Yn4+HgUFhaaDTidTgedTicta7VaAPemY9fr9V3qn621z83X2tpqtg/t5Z310ZLt0KPxoH3RX/anNe22OOAsnQrJ0pvO1NfXo62tDb6+vkblvr6+KC0tNbnOoUOH8OGHH6K4uNjk87W1tdI27t9m+3Om5OTkIDs7u0P53r174eLi0lk3eq3LTQBgj0OHDqHarfO6+/bt65btUM+ydF/IfX+2tLRYXNfigCsoKMCwYcMwevRoCCG61LCHcfv2bcyePRubNm2Ct7d3t247MzPTaCJPrVaLoKAgJCYmwsPDo1tf61E5d1WLNSVHMWHCBDweYLoPer0e+/btw9SpU+Hg4NDl7dCj8aB90V/2Z/sZliUsDrgFCxbgk08+QWVlJTIyMvDaa69h4MCBXWogAHh7e8POzg51dXVG5XV1dfDz8+tQv6KiAlVVVZg2bZpUZjAYAAD29vYoKyuT1qurq4O/v7/RNqOjo822RaVSQaVSdSh3cHAw+4fS29nb20s/H9SHzvppzXaoZ1m6L+S+P61pt8WjqBs2bEBNTQ2WLl2K//3f/0VQUBBeeukl7Nmzp0tHdI6OjoiJiYFGo5HKDAYDNBoNxo0b16F+REQESkpKUFxcLD2ef/55TJo0CcXFxQgKCkJISAj8/PyMtqnVanHs2DGT2yQiebNqkEGlUmHWrFmYNWsWqqurUVBQgIULF6K1tRXnzp2Dm5t1J/VqtRppaWmIjY1FXFwccnNz0dzcLI2qpqamIjAwEDk5OXBycsITTzxhtP6AAQMAwKh88eLFWL16NYYPH46QkBAsX74cAQEBmDFjhlVtI6K+r8ujqEqlEgqFAkIItLW1dWkbKSkpuH79OrKyslBbW4vo6Gjs3r1bGiS4dOkSlErrvmyxdOlSNDc3Y/78+WhoaMCECROwe/duODk5damNfdX3+nv75Ox3jWbrNH+vw8nrgF/1Lbg6dzxFB4Dya0090j6iR8GqgNPpdNi5cyfy8/Nx6NAhPPfcc/iv//ovJCcnWx1E7RYtWoRFixaZfO7AgQOdrltQUNChTKFQYOXKlVi5cmWX2iMXFf8vmH69s+QBNe2xrfzEA7fnqrLZFUVEXWbxX+3ChQuxfft2BAUFYc6cOfjkk0+6fTSTuk/i4/cGXMJ83ODsYGeyTllNI978SwnW/vRJjPD3NLstV5U9Qrxde6SdZLkHHZXziLwjiwMuLy8PQ4cORWhoKL7++mt8/fXXJuvt3Lmz2xpHXTfQ1REvxw3ttE77RZ9hg13xRKD5gKPewbKjch6R/5DFvUxNTYVCoejJthBRJx50VM4j8o6sutCXiGznQUflPCLviPPBEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItmwfchg0bEBwcDCcnJ8THx+P48eNm6+7cuROxsbEYMGAAXF1dER0djW3bthnVSU9Ph0KhMHokJyf3dDeIqBey6c0Rd+zYAbVajby8PMTHxyM3NxdJSUkoKyuDj49Ph/oDBw7Eb37zG0RERMDR0RGff/45MjIy4OPjg6SkJKlecnIyNm/eLC2rVKZvgktE8mbTI7h169Zh3rx5yMjIwKhRo5CXlwcXFxfk5+ebrD9x4kT85Cc/wciRIxEWFoY33ngDkZGROHTokFE9lUoFPz8/6eHl5fUoukNEvYzNjuDu3r2LoqIiZGZmSmVKpRIJCQkoLCx84PpCCHz11VcoKyvD7373O6PnDhw4AB8fH3h5eWHy5MlYvXo1Bg0aZHZbOp0OOp1OWtZqtQAAvV4PvV5vbdf6jPb7aLa2tsq6n/1Ff9mf1vTNZgFXX1+PtrY2+Pr6GpX7+vqitLTU7HqNjY0IDAyETqeDnZ0dPvjgA0ydOlV6Pjk5GTNnzkRISAgqKirw1ltv4ZlnnkFhYSHs7DreDRwAcnJykJ2d3aF87969cHFx6WIPe7/LTQBgj6NHj+K7s7ZuDT2s/rI/W1paLK5r08/gusLd3R3FxcVoamqCRqOBWq1GaGgoJk6cCAB4+eWXpbpPPvkkIiMjERYWhgMHDmDKlCkmt5mZmQm1Wi0ta7VaBAUFITExER4eHj3aH1s6fekmUHISY8eORdTQgbZuDj2k/rI/28+wLGGzgPP29oadnR3q6uqMyuvq6uDn52d2PaVSifDwcABAdHQ0zp8/j5ycHCng7hcaGgpvb2+Ul5ebDTiVSmVyIMLBwQEODg4W9qjvsbe3l37KuZ/9RX/Zn9b0zWaDDI6OjoiJiYFGo5HKDAYDNBoNxo0bZ/F2DAaD0edn97ty5Qpu3LgBf3//h2ovEfU9Nj1FVavVSEtLQ2xsLOLi4pCbm4vm5mZkZGQAAFJTUxEYGIicnBwA9z4ri42NRVhYGHQ6Hb788kts27YNGzduBAA0NTUhOzsbL7zwAvz8/FBRUYGlS5ciPDzc6DISIuofbBpwKSkpuH79OrKyslBbW4vo6Gjs3r1bGni4dOkSlMp/HWQ2Nzdj4cKFuHLlCpydnREREYGPPvoIKSkpAAA7OzucOXMGW7ZsQUNDAwICApCYmIhVq1bxWjiifkghhBC2bkRvo9Vq4enpicbGRlkPMhRX38CMjUexa8FYRA8zfxkN9Q39ZX9a8/60+Ve1iIh6CgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki2bB9yGDRsQHBwMJycnxMfH4/jx42br7ty5E7GxsRgwYABcXV0RHR2Nbdu2GdURQiArKwv+/v5wdnZGQkICvv32257uBhH1QjYNuB07dkCtVmPFihU4deoUoqKikJSUhGvXrpmsP3DgQPzmN79BYWEhzpw5g4yMDGRkZGDPnj1SnXfeeQfr169HXl4ejh07BldXVyQlJeHOnTuPqltE1FsIG4qLixOvv/66tNzW1iYCAgJETk6OxdsYPXq0WLZsmRBCCIPBIPz8/MS7774rPd/Q0CBUKpX45JNPLN5mY2OjACAaGxstXqcv+qaqXgz71efim6p6WzeFukF/2Z/WvD9tdgR39+5dFBUVISEhQSpTKpVISEhAYWHhA9cXQkCj0aCsrAw//vGPAQCVlZWora012qanpyfi4+Mt2iYRyYu9rV64vr4ebW1t8PX1NSr39fVFaWmp2fUaGxsRGBgInU4HOzs7fPDBB5g6dSoAoLa2VtrG/dtsf84UnU4HnU4nLWu1WgCAXq+HXq+3rmN9SGtrq/RTzv3sL/rL/rSmbzYLuK5yd3dHcXExmpqaoNFooFarERoaiokTJ3Z5mzk5OcjOzu5QvnfvXri4uDxEa3u3y00AYI+jR4/iu7O2bg09rP6yP1taWiyua7OA8/b2hp2dHerq6ozK6+rq4OfnZ3Y9pVKJ8PBwAEB0dDTOnz+PnJwcTJw4UVqvrq4O/v7+RtuMjo42u83MzEyo1WppWavVIigoCImJifDw8OhK9/qE05duAiUnMXbsWEQNHWjr5tBD6i/7s/0MyxI2CzhHR0fExMRAo9FgxowZAACDwQCNRoNFixZZvB2DwSCdXoaEhMDPzw8ajUYKNK1Wi2PHjmHBggVmt6FSqaBSqTqUOzg4wMHBwfJO9TH29vbSTzn3s7/oL/vTmr7Z9BRVrVYjLS0NsbGxiIuLQ25uLpqbm5GRkQEASE1NRWBgIHJycgDcO5WMjY1FWFgYdDodvvzyS2zbtg0bN24EACgUCixevBirV6/G8OHDERISguXLlyMgIEAK0f6spaXF6PPNspoG6GrLcf6sMww3BkjlERERsj41p/7DpgGXkpKC69evIysrC7W1tYiOjsbu3bulQYJLly5BqfzXQG9zczMWLlyIK1euwNnZGREREfjoo4+QkpIi1Vm6dCmam5sxf/58NDQ0YMKECdi9ezecnJweef96m9LSUsTExHQof2WL8XJRURHGjBnziFpF1HMUQghh60b0NlqtFp6enmhsbJTVZ3D3H8E1fa/DF/sL8f9NGgc353+dovMIrm8qrr6BGRuPYteCsYgeNsjWzekx1rw/+9woKnWdi4uL0ZGZXq/HrfprGBcXK+vPbKj/svl3UYmIegoDjohkiwFHRLLFgCMi2eIgA1EfZel1jUD/HRlnwBH1UZZe1wj032sbGXBEfVRERASKioqkZXPXNbbX7Y8YcER9FK9rfDAOMhCRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxevgTGifA9Sam1v0RXq9Hi0tLdBqtbxuSgb6y/5sf19aMlcvA86E27dvAwCCgoJs3BIiMuf27dvw9PTstA6nLDfBYDDg6tWrcHd3h0KhsHVzekz77REvX74sq6nZ+6v+sj+FELh9+zYCAgKM7tliCo/gTFAqlRgyZIitm/HIeHh4yPoN0d/0h/35oCO3dhxkICLZYsARkWwx4PoxlUqFFStWQKVSPbgy9Xrcnx1xkIGIZItHcEQkWww4IpItBhwRyRYDjohkiwEnExMnTsTixYtt3QyyUm/bb72tPQ+LAUfUx929e9fWTei9BPV5aWlpAoDRIzAwUHzwwQdG9U6dOiUUCoWoqqoSQgixdu1a8cQTTwgXFxcxZMgQsWDBAnH79m2jdQ4ePCgmTJggnJycxJAhQ8TPf/5z0dTU9Mj6Jmem9lt5ebmYM2eOCA4OFk5OTuKxxx4Tubm5HdabPn26WL16tfD39xfBwcFCCCEOHz4soqKihEqlEjExMeLTTz8VAMQ333wjrVtSUiKSk5OFq6ur8PHxEa+99pq4fv262fZUVlY+ql9Hj2DAyUBDQ4MYN26cmDdvnqipqRE1NTViyZIlYsKECUb13nzzTaOy9957T3z11VeisrJSaDQaMWLECLFgwQLp+fLycuHq6iree+89ceHCBXH48GExevRokZ6e/sj6Jmem9tudO3dEVlaWOHHihLh48aL46KOPhIuLi9ixY4e0XlpamnBzcxOzZ88WZ8+eFWfPnhWNjY1i4MCB4rXXXhPnzp0TX375pXjssceMAu7WrVti8ODBIjMzU5w/f16cOnVKTJ06VUyaNMlse1pbW23xq+k2DDiZePrpp8Ubb7whLX/zzTdCoVCI6upqIYQQbW1tIjAwUGzcuNHsNv785z+LQYMGSctz584V8+fPN6pz8OBBoVQqxffff9+9Hein7t9vprz++uvihRdekJbT0tKEr6+v0Ol0UtnGjRvFoEGDjPbLpk2bjAJu1apVIjEx0Wjbly9fFgBEWVmZxe3pS/gZnExFR0dj5MiR+PjjjwEAX3/9Na5du4YXX3xRqvOPf/wDU6ZMQWBgINzd3TF79mzcuHEDLS0tAIDTp0+joKAAbm5u0iMpKQkGgwGVlZU26Vd/sGHDBsTExGDw4MFwc3PDH/7wB1y6dMmozpNPPglHR0dpuaysDJGRkXBycpLK4uLijNY5ffo09u/fb7Q/2+94X1FR0YM9sh0GnIy9+uqrUsB9/PHHSE5OxqBBgwAAVVVVeO655xAZGYm//vWvKCoqwoYNGwD860PrpqYm/Nu//RuKi4ulx+nTp/Htt98iLCzMNp2Sue3bt2PJkiWYO3cu9u7di+LiYmRkZHQYSHB1dbV6201NTZg2bZrR/iwuLsa3336LH//4x93VhV6F88HJhKOjI9ra2ozKXnnlFSxbtgxFRUX4y1/+gry8POm5oqIiGAwGrF27Vpo08E9/+pPR+mPGjME///lPhIeH93wH+qn799vhw4cxfvx4LFy4UCqz5OhqxIgR+Oijj6DT6aQv2584ccKozpgxY/DXv/4VwcHBsLc3/dY39XfUl/EITiaCg4Nx7NgxVFVVob6+HgaDAcHBwRg/fjzmzp2LtrY2PP/881L98PBw6PV6/P73v8fFixexbds2owAEgF/96lc4cuQIFi1aJP1P/7e//Q2LFi161N2Trfv32/Dhw3Hy5Ens2bMHFy5cwPLlyzsElSmvvPIKDAYD5s+fj/Pnz2PPnj1Ys2YNAEizUr/++uu4efMmZs2ahRMnTqCiogJ79uxBRkaGFGqm/o76NFt/CEjdo6ysTIwdO1Y4OzsbDe9/8MEHAoBITU3tsM66deuEv7+/cHZ2FklJSWLr1q0CgLh165ZU5/jx42Lq1KnCzc1NuLq6isjISPHb3/72EfVK/u7fb6WlpSI9PV14enqKAQMGiAULFohf//rXIioqSlqn/TKR+x0+fFhERkYKR0dHERMTIz7++GNpm+0uXLggfvKTn4gBAwYIZ2dnERERIRYvXiwMBoPJ9vT1y0Q4XRKRTP3xj39ERkYGGhsb4ezsbOvm2AQ/gyOSia1btyI0NBSBgYE4ffo0fvWrX+Gll17qt+EGMOCIZKO2thZZWVmora2Fv78/XnzxRfz2t7+1dbNsiqeoRCRbHEUlItliwBGRbDHgiEi2GHBEJFsMOCKSLQYc2VR6ejoUCgUUCgUcHBzg6+uLqVOnIj8/36qvCRUUFGDAgAE911Az0tPTMWPGjEf+umQZBhzZXHJyMmpqalBVVYW///3vmDRpEt544w0899xzaG1ttXXzqC+z7TfFqL8z971KjUYjAIhNmzYJITqfXn3//v0dptpesWKFEEKIrVu3ipiYGOHm5iZ8fX3FrFmzRF1dnfQ6N2/eFK+88orw9vYWTk5OIjw8XOTn50vPX7p0Sbz44ovC09NTeHl5ieeff176fuaKFSs6vO7+/ft75PdEXcMjOOqVJk+ejKioKOzcuRMAoFQqsX79epw7dw5btmzBV199haVLlwIAxo8fj9zcXHh4eKCmpgY1NTVYsmQJAECv12PVqlU4ffo0du3ahaqqKqSnp0uvs3z5cvzzn//E3//+d5w/fx4bN26Et7e3tG5SUhLc3d1x8OBBHD58GG5ubkhOTsbdu3exZMkSvPTSS9IRaE1NDcaPH/9of1HUOVsnLPVv5o7ghBAiJSVFjBw50uRz90+vvnnzZuHp6fnA1ztx4oQAIB39TZs2TWRkZJisu23bNjFixAhppg0hhNDpdMLZ2Vns2bPnge0n2+MRHPVaQghpLrMHTa9uTlFREaZNm4ahQ4fC3d0dTz/9NABIU4AvWLAA27dvR3R0NJYuXYojR45I654+fRrl5eVwd3eXpvgeOHAg7ty5I9spvuWGAUe91vnz5xESEmLR9OqmNDc3IykpCR4eHvjjH/+IEydO4NNPPzVa75lnnkF1dTV+8Ytf4OrVq5gyZYp0etvU1ISYmJgOU3xfuHABr7zySg/3nroDZxOhXumrr75CSUkJfvGLX1g0vbqpqbZLS0tx48YN/Od//ieCgoIAACdPnuzwWoMHD0ZaWhrS0tLw1FNP4Ze//CXWrFmDMWPGYMeOHfDx8YGHh4fJdsptim+54REc2ZxOp0NtbS2+++47nDp1Cm+//TamT5+O5557DqmpqRZNrx4cHIympiZoNBrU19ejpaUFQ4cOhaOjo7TeZ599hlWrVhmtl5WVhb/97W8oLy/HuXPn8Pnnn2PkyJEA7t20x9vbG9OnT8fBgwdRWVmJAwcO4D/+4z9w5coV6XXPnDmDsrIy1NfXQ6/XP5pfGlnG1h8CUv/2w7up29vbi8GDB4uEhASRn58v2trapHqWTK/+7//+72LQoEFGl4l8/PHHIjg4WKhUKjFu3Djx2WefdbhX6MiRI4Wzs7MYOHCgmD59urh48aK0zZqaGpGamiq8vb2FSqUSoaGhYt68eaKxsVEIIcS1a9ekKd3By0R6Hc4HR0SyxVNUIpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLL1/wM4eUGHCSMD8QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:41.678110Z", + "iopub.status.busy": "2024-02-29T23:33:41.677272Z", + "iopub.status.idle": "2024-02-29T23:33:41.967168Z", + "shell.execute_reply": "2024-02-29T23:33:41.965990Z" + }, + "papermill": { + "duration": 0.315853, + "end_time": "2024-02-29T23:33:41.969289", + "exception": false, + "start_time": "2024-02-29T23:33:41.653436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEmCAYAAAAXyJnQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmHklEQVR4nO2dd3gU1frHP7MtvVCTgIGEIkVpIiBYEI2CV6/KtQAiCqKIiA1RAZXqvaAgYkG46gVURBQb/FRQjBSBUIw0KdKlJSGUdJJt8/tj2cn2kuwmm+R8nmefJDNnZs5sdr/znvO+530lWZZlBAKBoA6jqu4OCAQCQXUjhFAgENR5hBAKBII6jxBCgUBQ5xFCKBAI6jxCCAUCQZ1HCKFAIKjzCCEUCAR1Hk11dyAUMZvNnD59mpiYGCRJqu7uCAQCB2RZprCwkCZNmqBSVd6eE0LogtOnT5OcnFzd3RAIBF44ceIEl112WaXPI4TQBTExMYDlTY6Nja3m3ggEAkcKCgpITk5WvquVRQihC6zD4djYWCGEAkEIE6ipK+EsEQgEdR4hhAKBoM4jhFAgENR5xBxhHUaWZYxGIyaTqbq7IgggWq0WtVpd3d2oUQghrKPo9XqysrIoKSmp7q4IAowkSVx22WVER0dXd1dqDEII6yBms5mjR4+iVqtp0qQJOp1OBI7XEmRZJjc3l5MnT9K6dWthGfqIEMIaznfbT/H9rtPMGdiF6DDf/p16vR6z2UxycjKRkZFB7qGgqmnUqBHHjh3DYDAIIfQR4Syp4SzadIxf9p1h29Hzfh8biKVJgtBDWPf+I74JNRyT2VJ7q8xoruaeCAQ1FyGENRwZixAazUIIBYKKIoSwhmMtxmo0iaqsdYWhQ4dy9913V3c3ahVCCGs4ihCa64YQ3njjjTz77LPV3Q1BLUMIYS3BaBJD45qEwWCo7i4IbAgJIZw7dy4pKSmEh4fTo0cPtm7d6rbthx9+yPXXX0+9evWoV68eaWlpTu1lWWbixIkkJSURERFBWloaBw8eDPZtVAtWO9BQSYtQlmVK9MZqecmyb30fOnQo69at4+2330aSJCVweN68eXbttm/fjkql4u+//wZg9uzZdOjQgaioKJKTkxk1ahRFRUV2x2zYsIHrr7+eiIgIkpOTefrppykuLvapX1lZWdx+++1ERESQmprKkiVLSElJYc6cOUobSZKYN28ed955J1FRUfz73//GZDIxfPhwUlNTiYiIoE2bNrz99tt25zaZTIwZM4b4+HgaNGjAiy++6PP7JfCdao8j/OKLLxgzZgzz58+nR48ezJkzh759+/LXX3/RuHFjp/Zr165l0KBB9OrVi/DwcF5//XVuvfVW9uzZQ9OmTQF44403eOedd/j4449JTU3l1VdfpW/fvuzdu5fw8PCqvsWgYv1SVNYivGgw0X7iT4Hokt/sndqXSJ33j+Lbb7/NgQMHuPLKK5k6dSoAb775JkuWLOGJJ55Q2n322Wdce+21NG/eHLCECb3zzjukpqZy5MgRRo0axYsvvsj7778PwOHDh+nXrx+vvfYaCxYsIDc3l9GjRzN69GgWLlzotV8PPfQQZ8+eZe3atWi1WsaMGcOZM2ec2k2ePJkZM2YwZ84cNBoNZrOZyy67jGXLltGgQQM2bdrEiBEjSEpK4v7771fub9GiRSxYsIB27drx5ptv8u2333LTTTd5f2MFPiPJ1fx46dGjB926deO9994DUAJ9n3rqKcaNG+f1eJPJRL169Xjvvfd46KGHkGWZJk2a8PzzzzN27FgA8vPzSUhIYNGiRQwcONDrOQsKCoiLiyM/Pz/k8xH2m7Oe/dmFvHJ7Ox69voVPx5SWlnL06FFSU1OVB0OJ3hjyQgiWOcLOnTsr1taOHTu46qqrOHbsGM2aNcNsNtOsWTNeeeUVRo4c6fIcX331FSNHjuTs2bMAPProo6jVav773/8qbTZs2EDv3r0pLi72+PDcv38/7dq1Y9u2bVx99dUAHDp0iNatW/PWW28p85mSJPHss8/y1ltveby/0aNHk52dzVdffQVAkyZNeO6553jhhRcAMBqNpKam0rVrV7777juX53D1/61tBPo7Wq0WoV6vJzMzk/HjxyvbVCoVaWlpZGRk+HSOkpISDAYD9evXB+Do0aNkZ2eTlpamtImLi6NHjx5kZGS4FMKysjLKysqUvwsKCip6S1WO9TFmqKTXOEKrZu/UvgHoUcWuXVE6d+5Mu3btWLJkCePGjWPdunWcOXOG++67T2nzyy+/MH36dPbv309BQQFGo5HS0lJKSkqIjIxk586d7Nq1i88++0w5RpZlZSliu3bt3F7/r7/+QqPRcNVVVynbWrVqRb169ZzaWoXSlrlz57JgwQKOHz/OxYsX0ev1dO7cGbA8wLOysujRo4fSXqPRcPXVV4vhcYCp1jnCs2fPYjKZSEhIsNuekJBAdna2T+d46aWXaNKkiSJ81uP8Oef06dOJi4tTXjWpXokSR1jJobEkSUTqNNXyquxKiMGDB7NkyRIAlixZQr9+/WjQoAEAx44d44477qBjx458/fXXZGZmMnfuXMDyIAYoKiri8ccfZ8eOHcpr586dHDx4kJYtW1aqb7ZERUXZ/b106VLGjh3L8OHD+fnnn9mxYwfDhg1T+iWoOkLCWVJRZsyYwdKlS/n2228rNQQYP348+fn5yuvEiRMB7GVwUSzCOhI+o9PpnNKGPfDAA/z5559kZmby1VdfMXjwYGVfZmYmZrOZN998k2uuuYbLL7+c06dP2x1/1VVXsXfvXlq1auX00ul0HvvTpk0bjEYj27dvV7YdOnSICxcueL2XjRs30qtXL0aNGkWXLl1o1aoVhw8fVvbHxcWRlJTEli1blG1Go5HMzEyv5xb4R7UKYcOGDVGr1eTk5Nhtz8nJITEx0eOxs2bNYsaMGfz888907NhR2W49zp9zhoWFKfVJalqdEqv81ZXwmZSUFLZs2cKxY8c4e/YsZrOZlJQUevXqxfDhwzGZTNx5551K+1atWmEwGHj33Xc5cuQIn376KfPnz7c750svvcSmTZsYPXo0O3bs4ODBgyxfvpzRo0d77U/btm1JS0tjxIgRbN26le3btzNixAgiIiK8WrqtW7fm999/56effuLAgQO8+uqrbNu2za7NM888w4wZM/juu+/Yv38/o0aNIi8vz/c3TOAT1SqEOp2Orl27kp6ermwzm82kp6fTs2dPt8e98cYbTJs2jVWrVjnNu6SmppKYmGh3zoKCArZs2eLxnDUdUx2xCMeOHYtaraZ9+/Y0atSI48ePA5bh8c6dO+nfvz8RERFK+06dOjF79mxef/11rrzySj777DOmT59ud86OHTuybt06Dhw4wPXXX0+XLl2YOHEiTZo08alPn3zyCQkJCdxwww3079+fxx57jJiYGK+jlMcff5x//etfDBgwgB49enDu3DlGjRpl1+b5559nyJAhPPzww/Ts2ZOYmBj69+/vU78EfiBXM0uXLpXDwsLkRYsWyXv37pVHjBghx8fHy9nZ2bIsy/KQIUPkcePGKe1nzJgh63Q6+auvvpKzsrKUV2FhoV2b+Ph4efny5fKuXbvku+66S05NTZUvXrzoU5/y8/NlQM7Pzw/szQaBm2atkZu/9L08ZcUen4+5ePGivHfvXp/fD4F/nDhxQgbkX375pVquXxf+v4H+jlZ7HOGAAQPIzc1l4sSJZGdn07lzZ1atWqU4O44fP26XLmrevHno9Xruvfdeu/NMmjSJyZMnA/Diiy9SXFzMiBEjyMvL47rrrmPVqlW1MpRAGRqLpAvVxq+//kpRUREdOnQgKyuLF198kZSUFG644Ybq7prAR6pdCAEleNUVa9eutfv72LFjXs8nSRJTp05Vgm5rNQEKnxG45rfffuO2225zu7+oqAiDwcCECRM4cuQIMTEx9OrVi88++wytVluFPRVUhpAQQkHFscqfSViEQeHqq69mx44dHtv07duXvn2rJwZTEBiEENZwZGWJnbAIg0FERAStWrWq7m4IgkyNjiMUBC7pgkBQlxFCWEuoK3GEAkEwEEJYw6lriVkFgmAghLCGE6i1xgJBXUYIYQ1HWIQCQeURQljDKU/DJSxCb7jKGu0up5+gbiHCZ2oJInzGf7KyslzmDRTUPYQQ1nCUOEIxNPYbbxmOqgJZljGZTGg04qtYnYihcS2h0muNZRn0xdXz8iPbcmFhIYMHDyYqKoqkpCTeeuutCpf4tB0aHzt2DEmS+Oabb+jTpw+RkZF06tTJKVO6tyJPn376KVdffTUxMTEkJibywAMP2NUvWbt2LZIksXLlSrp27UpYWBgbNmzwu++CwCIeQzWc8nyElbQIDSXwH9/STgWcCadBF+W9HTBmzBg2btzIihUrSEhIYOLEifzxxx9KevvK8vLLLzNr1ixat27Nyy+/zKBBgzh06BAajcanIk8Gg4Fp06bRpk0bzpw5w5gxYxg6dCg//vij3XXGjRvHrFmzaNGihRiehwBCCGs4dclZUlhYyMcff8ySJUu4+eabAVi4cKHPeQN9YezYsdx+++0ATJkyhSuuuIJDhw7Rtm1bpk+fzuDBgxXrs3Xr1rzzzjv07t2befPmER4eziOPPKKcq0WLFrzzzjt069aNoqIioqOjlX1Tp07llltuCVi/BZVDCGENxxpHWOnErNpIi2VWHWgjfWp25MgRDAYD3bt3V7bFxcXRpk2bgHXFNtt5UlISAGfOnKFt27Y+FXnKzMxk8uTJ7Ny5kwsXLmC+NGVx/Phx2rdvrxznqpCToPoQQljDCVQVOyTJ5+FpbcY2dZY11b5VzKxFnp5++mmn45o1a0ZxcbGSieazzz5TMmj37dvXqSCTYyEnQfUihDBEOF+sZ/66w9zX9TJaJ8T4fFxdSszaokULtFot27Zto1mzZoCl5OWBAweqJAmqbZEnV+zevZtz584xY8YMpRLi77//HvR+CSqP8BqHCOO/2cUH64/Qd856v45TVpbUgTjCmJgYHn74YV544QXWrFnDnj17GD58OCqVqtIlQX3BW5GnZs2aodPplEJRK1asYNq0aUHvVyhiMstsPHSWglJDdXfFJ4QQhgg7TuQBUNGpvrrgLAGYPXs2PXv25I477iAtLY1rr72Wdu3aVUkZBm9Fnho1asSiRYtYtmwZ7du3Z8aMGcyaNSvo/QpFft6TzeCPtjBj5f7q7opPiKFxiOBHKJ3jkUDdqWIXExNj56woLi5mypQpjBgxwuuxjmUeZJs3PSUlxe5vgPj4eKdt3bp14+eff3Z7jUGDBjFo0CC317nxxhudzlkbOX6+BIC/zxV7aRkaCCEMESr61ahrBd63b9/O/v376d69O/n5+UpdmrvuuquaeyawpbjMCEBeSc0YGgshDBEqaiTUtQLvALNmzeKvv/5S6mL/9ttv7Nu3z2uRJUHVUVRmAoQQCvymYkpoHWaZZTCbZVSq4DsNqpMuXbqQmZnptP3ixYteiywJqg6rRZh/UQihwA8qOrK1PcxoltHVciF0hyiyFFoUXRLCojIjBpMZrTq0/bKh3bs6REUn0G0P8zeWsC5M2tdFQuH/ahVCqBlWoRDCECEQH11fV5dYV0+UlJQE4KqCUMO6ikWtVldbH4pthLAmzBOKoXGIUGFnic2BvjpM1Go18fHxSnqoyMjIKglIFgQfs9lMbm4ukZGR1Zrj0N4i1HtoGRoIIQwRKjw0tvndn1hCa1JS21x5gtqBSqWiWbNm1fpwKxIWoaAiVHhax+Y4f2IJJUkiKSmJxo0bYzCE/gdV4Ds6nQ6VqnpnvcTQWFAhAqCDFYolVKvV1TqXJKidFF+KIwTIE84Sga9U3GtcflylU3EJBAGgzGhCb/NQzi8J/TlCIYQhQmDiCOvO6hJB6GJrDYKwCAV+IFd4ZUn573UhFZcg9LGdH4SaMUcohDBECEQMrCjpKQgFihyFUFiEAl+puLPE/zhCgSCYOFqEYo5Q4DsVDqgu/104SwShQKGwCAUVxVzFAdUCQbCwWoQNo3WAmCMU+EElE1QDYBBeY0EIYBXCpvERABSUGkL+IS2EMESo+BI72znC0P6wCeoG1qSsTS4JoSxDYYgXcRJCGCJUOI7QLnxGWISC6qeo1GIRxkfqiA6zLF4L9eFxpYSwqKiIgoICu5e/zJ07l5SUFMLDw+nRowdbt25123bPnj3cc889pKSkIEkSc+bMcWozefJkJEmye7Vt29bvftVERPiMIBQo1luEMDpMTVyEJeVbqDtM/BbCo0ePcvvttxMVFUVcXBz16tWjXr16xMfHU69ePb/O9cUXXzBmzBgmTZrEH3/8QadOnejbt6/bjCglJSW0aNGCGTNmKNlTXHHFFVeQlZWlvDZs2OBXv2oSYmWJINSwxhFGhWmIj7wkhCEeQuN30oUHH3wQWZZZsGABCQkJlUr1M3v2bB577DGGDRsGwPz58/nhhx9YsGAB48aNc2rfrVs3unXrBuByvxWNRuNRKGsTYq2xINSwDo2jbYQw1LNU+y2EO3fuJDMzkzZt2lTqwnq9nszMTMaPH69sU6lUpKWlkZGRUalzHzx4kCZNmhAeHk7Pnj2ZPn06zZo1c9u+rKyMsrIy5e+KDPGrC/vsM0IIBdWP1WscHaYhPqJmhND4PTTu1q0bJ06cqPSFz549i8lkIiEhwW57QkIC2dnZFT5vjx49WLRoEatWrWLevHkcPXqU66+/nsLCQrfHTJ8+nbi4OOWVnJxc4etXNZWpWSIQBAPboXGcMjQObSH02yL86KOPGDlyJKdOneLKK69U6l9Y6dixY8A6VxFsa9t27NiRHj160Lx5c7788kuGDx/u8pjx48czZswY5e+CgoIaJYZWgmkRlhpMSBKEaUTuQoFnyp0lGuIvOUsu1LY5wtzcXA4fPqzM64El27Esy0iShMlk8nB0OQ0bNkStVpOTk2O3PScnJ6Dze/Hx8Vx++eUcOnTIbZuwsDDCwsICds2qwjH2MFgWocFkpvPUn9GpVeyYeGutr50sqBzKHGG4hgbRlu/VueLQFkK/h8aPPPIIXbp0ISMjgyNHjnD06FG7n76i0+no2rUr6enpyjaz2Ux6ejo9e/b0t1tuKSoq4vDhwyQlJQXsnKFKsJwl2fmllBrMFJQauWjw7UEnqLtYA6qjdBoaxViEMLewtDq75BW/LcK///6bFStWBKSY9pgxY3j44Ye5+uqr6d69O3PmzKG4uFixNh966CGaNm3K9OnTAYuDZe/evcrvp06dYseOHURHRyv9GTt2LP/85z9p3rw5p0+fZtKkSajVagYNGlTp/npj46GzfP3HSV7+RzvlSRhMHBejBGtobBsYUNE10YK6g62zpNGl78GZwjJPh1Q7fgvhTTfdxM6dOwMihAMGDCA3N5eJEyeSnZ1N586dWbVqleJAOX78uF0RmtOnT9OlSxfl71mzZjFr1ix69+7N2rVrATh58iSDBg3i3LlzNGrUiOuuu47NmzfTqFGjSvfXGx/9doQ1f+USG65l8p1XBP16jpJkCtLQWGWjhCJmW+AJk1lWRg1RYWobi7CWCeE///lPnnvuOXbv3k2HDh2cnCV33nmnX+cbPXo0o0ePdrnPKm5WUlJSvK7JXbp0qV/XDyRlRosQffn7CZ5Na018pC6o13N8L/ypYucPtkJY0TXRgrqBbVLW6HANWo3FkCksNVJqMBGuDU1nm99COHLkSACmTp3qtM8fZ0ltxJpho0Rv4rMtx3myT+WtZk84SpLRZKbMaCL/ooHGMeEBu46tb0TooMAT1mGxVi0RplGjU6sI06goM5rJLSwjuX5kNffQNX47S8xms9tXXRZBsJ8/W7TpGGXG4L4fjqJkMMmM+CSTa2f8Sk5B4CanJbuhsVBCgXuKbWIIwfLZUYbHRaE7PPZLCA0GAxqNhj///DNY/anR2OZcyy0sY/mO00G9nmPBJ6PZzKEzRRhMMifOlwTsOio7Z0nATiuohSjB1LrywaZVCM8U1BIh1Gq1NGvWrM5bfu6wOm27p9QH4MP1R6p0Ts1klpVYwmBlohFzhAJPWIUwJtxGCKNrmUUI8PLLLzNhwgTOnz8fjP7UaMyXxGfwNc2IDtNw8EwRaw/kBu16robG1hCaYGUEFjIo8ITj0BigcWzoe479dpa89957HDp0iCZNmtC8eXOioqLs9v/xxx8B61xNwyo+8ZE6BnZL5qMNR/lw/RH6tGns8ThHK8tsliu0esNoMmO4lJw1kEJoeyYxRyjwhBJMHWZrEVocd7VKCO++++4gdKN2YBUJtSQx7LpUFm46xqbD5/jzVD5XNo3zcJzzeVR4F0Ini9AsKwIYLItQzBEKPFEeTF0eJlMTYgn9FsJJkyYFox+1Aqv4qFSWwjW3d0hixc7TfPTbEeYM7OL2OMc1wiZZ9ukf4+gsMZlkJZYwkHOEtoJrFkoo8ECRzaoSKzVhmV2FU/VnZmayePFiFi9ezPbt2wPZpxqLVQjVl8JNHru+BQD/tyuL03kXvR5nxdfRp9MSO7NZqVsSPItQCKHAPUUu5ghrpUV45swZBg4cyNq1a4mPjwcgLy+PPn36sHTp0ipZyhaqmKxD40vzex0ui6NniwZkHDnHwo1Hefn29q6PcxAtX0XMsVWZ0awMXQM7R1h+LmEQCjxR7MIibGwTR2jNUhVq+G0RPvXUUxQWFrJnzx7Onz/P+fPn+fPPPykoKODpp58ORh9rDOVD4/J/9IgbLFbh51tPUOCmpKGjaPlqdTk6WcoM5UPsYKXkEhahwBPWFFy2FmGDS4XeDSY5ZFP2+y2Eq1at4v3336ddu3bKtvbt2zN37lxWrlwZ0M7VNMwOQ2OA3pc3onXjaIrKjCzdetzlcc5CWLHrl9qsZAmoYIk5QoGPuJojDNOoldoloZqFpkJL7BwTLYAl2Npcx1PFOw6NwWIdWucKF248poS32B3nKIQVHBqX2uQKDFZKLqGDAk/YZqe2RQmqri1CeNNNN/HMM89w+nT58rFTp07x3HPPcfPNNwe0czUNq8apHWIA7+rShIbRYWTll/LDriyn4xw9vL4Pje3/tk2aKuIIaxfbjp3n1/053htWM67iCCH0HSZ+C+F7771HQUEBKSkptGzZkpYtW5KamkpBQQHvvvtuMPpYYzC7sAjBMjQY2qs5AB+4WHbn5CzxIjayLLM/u8DJuryot50jFF7j2sR98zN4ZNHvZOeHbggKQNGlefCoMPt0W41DXAj99honJyfzxx9/8Msvv7B//34A2rVrR1paWsA7V9NQnCUuvGKDezRn7prD7M0qYNPhc1zbqqHTcVa8ac33u7J46vPtPHiNfYlS26FxIAXLPo4wYKcVVIAzhaUkxgUuxVqgKb5kEcaE2U+fhXoGGr+FECypdW655RZuueWWQPenRqM4S1wsj6sXpeP+qy/j44y/+WD9ETsh9HdofPxSZpntx/PstlfNHKGwCKuTUH/7y9ca21uE5RloQtOirZAQpqenk56ezpkzZ5wcJAsWLAhIx2oiJpsldq545LpUPt38N+sO5PJXdiFtEmMAZ3HxNr9nFbnj5+xTbdkKavDiCEP8m1gLsZ1KCeV3X5Zlitw5S0LcIvR7jnDKlCnceuutpKenc/bsWS5cuGD3qsvYLrFzRfMGUfS70lKq9IP15RX/HK03b1pjjREstEmL7tSXIAmWEMKqx/YtD+U0aCV6k9LX6HB7IbRmTK81c4Tz589n0aJFDBkyJBj9qdG4c5bYMuzaVH7cnc3qvdlAJ7vjrHiz5nwp2xlQi9B2jjB0v4e1FtnN76GGdViskiBC63poHKpC6LdFqNfr6dWrVzD6UuMxugiodqRdUiwABaVGJfjU3zlCo4tYROc2QbIIhRJWObafhxA2CO2yUzsuo7PGEV4oMaA3hp7HzW8hfPTRR1myZEkw+lKjkWVZ+ZB6yiUYHaYh5tL8iTUUwrEMpzet8SU0JpBDY9szBWvILXBPTXnLi93EEALERWjRqi3fi7MhOE/o99C4tLSUDz74gF9++YWOHTs6rTKZPXt2wDpXk7AdinqyCAES48IpPFNEdn4prRpH42jgebMIXa1Oce5PcJ66NeVLWZuw/zyE7j+gsMwSQ+g4PwgW48C6qCC3sIwm8RFV3T2P+C2Eu3btonPnzgBORZxCMatEVWFrKXnLLp0YF87BM0VkXwolcEyQ4H1o7P3LENh8hMJrHCqE8tvvySIES1C1VQhDDb+FcM2aNcHoR43HVss03oQw1uJBy8635Cj0Nw2XLyIXrLk8MUVY9ZhrSPiMq+zUtoRyCE2FE7MK7LG1CD15jQGSLq0MyFLmCCsWPuO5jchQXVuwD5+pvn54w1UpT1tC2XMshDBA2IqZqyV2tiTGWeZHrEXY/c1H6MvQWGSorj3Ye41D9/1XUnC5mCOEcs/xmRBM2S+EMEDYWkreLMLEOMsHwp1F6D2OsGotQluEQVh1bD5yjhPnS2pcHKHjqhIroWwRVmiJncAZO2eJF59RYqzFIsx2I4T+hs+oJBeV8IKkWMGyNAX27DyRx8APNlt+n3irsj2EDUKX9UpsaRTCq0v8tgjXr1+P0ei8tMtoNLJ+/fqAdKomYhUeleTde26dIzxXrKfMaPI7oNpqEVoFV6N2/jcGa44wlIdmtYk/jpcvV3WsVhiq+GwR1gZnSZ8+fTh//rzT9vz8fPr06ROQTtVEXGWndkd8pBadxvLWnykocxI+b5H3VqusaT2LZSnh7KkWdY1rNu6WNYayKLpK029LYyUDTVnIPVD9FkJ3VajOnTtHVFRUQDpVk9h0+Cwmm8Lq3hwlYLEYbT3Hjs6PTzKOeTze2r5Fw+hL5wONOnhCaPvlEytLqga7ecEas8TOcxxhw0vOkjKj2WPCkOrA5znCf/3rX4DlSzx06FDCwsKUfSaTiV27dtW5Ncgf/XaE137Yx5BrmvPo9amAbxYhWGIJ/z5XQlb+RUW0LqsXQVZ+KT/tyWHdgVx6X+66NKrhUvhMi0ZRrDuQi4R06brllmSwLMJQe5LXBcx21mHovv/e4ggjdGpiwjQUlhnJLSwjNty59lF14bNFGBcXR1xcHLIsExMTo/wdFxdHYmIiI0aMYPHixcHsa8iRFBeBJMGnm/9m7ppDgPfldeXHWizCnIJSxcpqlxTLwz1TAJiyYg9lNlXpbLFahO2SYtGqJepH6ZwswkCW85RryBexNmGfg9B2ZU919MY3XJXydKRRbGh6jn22CBcuXAhASkoKY8eOrZPDYEdu75jEmcL2TPm/vXz5+0nA+/I6Kwk2Q+OISwGoakni2Vtas2LnaY6cLWbBhmM8cWNLp2OtzpLE2HCWjexFbLiG+/+72a6NDxE2FUKk6q96asqDyNscIVhiCY/kFoecEPo9Rzhp0iQhgjYMuzaVx3u3UP72trzOSpKyzK60PMW/WiI2XMv429oC8O6vB8m6tAzPFqtHWKOW6JwcT4tG0UpmDyuBTLogss9ULzVhZY8syxRcKtzkUQitDpOaLoQ5OTkMGTKEJk2aoNFoUKvVdq+6yEt923J35yYAhGt9ew+sq0uyC0rLhe2SiPbv0pSuzetRojcx++cDTsda8xFqbcJmnIfGYo6wJuPOCgxRHeRMYRmFpUZUEiTXj3TbLlSDqv0OqB46dCjHjx/n1VdfJSkpqU5nnLGiUkm8cW8nmjeIon2TWJ+OsVYiy84vVaw36/yiSiXxzM2teWjBVrYecw5VUhLA2lifGof6AIHNUB36X8Tahu28oJ1FHqL/gL2nCwBo2SjaozFQa4Rww4YN/Pbbb0oqLoEFnUbFc7dc7nN7q7PkTGGZknrfVtjaXirsdOJ8CXqjWYk7hHJnidZG/KoqjjBUv4i1GdvhcKha5HuzLELozRBQapeEWFC130Pj5OTkgP4z5s6dS0pKCuHh4fTo0YOtW7e6bbtnzx7uueceUlJSkCSJOXPmVPqc1UXD6DDUKgmTWea/6w4D9muIG8WEEaVTY5bLy3dasXqEbYfDjqtLAhtHaPN7iH4Raxvu3uZQfQ5ZLcL2SZ6FMFQtQr+FcM6cOYwbN45jx45V+uJffPEFY8aMYdKkSfzxxx906tSJvn37cubMGZftS0pKaNGiBTNmzCAxMTEg56wu1CqJhEsfioJLYQff7Tit7JckidRGFqfU0bPFdsdaLUhbB4mjs0QkXajZ2L7NtnOEoeqs8tUitGagyQ2xDDR+C+GAAQNYu3YtLVu2JCYmhvr169u9/GH27Nk89thjDBs2jPbt2zN//nwiIyPd1kbu1q0bM2fOZODAgXYB3ZU5Z3ViDaFxR+qllSNHzxbZbbc6SzQehsaBDLOwPZUYGlcN7nIQhqJFXlxm5Ng5y8O6nY8W4blivU9FyKoKv+cI3Q1H/UWv15OZmcn48eOVbSqVirS0NDIyMqr0nGVlZZSVlZvqBQUFvl/03GFo4Bzr5wtJceFs97A/tYHF++ZkEdqEz1hxdJYErYpdCH4RayP2QdQ2FmEIPoj2Zxciy5a1xNZldO6oH6VTsiWdL9bTONazMVBV+C2EDz/8cEAufPbsWUwmEwkJCXbbExIS2L9/f5Wec/r06UyZMsX/Cx74CZY+ADeOg+vHWhb9+oE1HZc7rEPjI7n2QmhSwm3ch88E9gtTM9a61lbsh8nV1g23+DosBsuUUMPoMM4UlnGmsMxeCK0frmqIRKlQYtbDhw/zyiuvMGjQIGXubeXKlezZsyegnasqxo8fT35+vvI6ceKEbwdm7wKzEX59DX5+xW+VsCZodUf50LhcCGW5PMGDR2dJkBRLWIRVg7vhcCi+/746Sqy4TMdlNsOPY+GXyYHunk/4LYTr1q2jQ4cObNmyhW+++YaiIsv81c6dO5k0aZLP52nYsCFqtZqcnBy77Tk5OW4dIcE6Z1hYGLGxsXYvn7jhBeg73fJ7xnuwYjSYfM+qYQ2qtuI4z5fawGIRniksU5YvGWyGvLbhM9oghs/YzRGG4BexthPqK0v2+WERgo0QFlwSQpMBvh0B2z6CjW9D1s6g9NMTfgvhuHHjeO2111i9ejU6nU7ZftNNN7F582YPR9qj0+no2rUr6enpyjaz2Ux6ejo9e/b0t1tBO6dXeo6Cu94HSQXbF8NXQ8HoW2hAkoOzROtg1cVFamkQZXmPj12yCm2TKdhahI5ZbwKZdMEWoYNVj332merrhytMZpn92X5ahNE2FqHhIiwdDLuXgUoD93wESZ2C1l93+C2Eu3fvpn///k7bGzduzNmzZ/0615gxY/jwww/5+OOP2bdvH0888QTFxcUMGzYMgIceesjO8aHX69mxYwc7duxAr9dz6tQpduzYwaFDh3w+Z1DoMhju/xTUOtj3f7BkAJQVeT0s0WGi2NU65dSG9iE0thahxi58xv5fGUgdtJujCrVvYi3FffaZwL7/s3/+i0cWbauwB/fo2WJKDWYidWqaN/AtB4HVIizIOweL74GDP4EmHAYugQ73VqgflcVvZ0l8fDxZWVmkpqbabd++fTtNmzb161wDBgwgNzeXiRMnkp2dTefOnVm1apXi7Dh+/Dgqm+Hf6dOn6dKli/L3rFmzmDVrFr1792bt2rU+nTNotLsDBi+Dzx+AI2vg07vhgS8h0n1IUWJcOAmxYeRcGiJYM07bktowit//vqAIoe0HVuvBWRIsi1AMjasG27f5g3VHlN8DLYQLNh6jqMzI51uP8+6vh2jeIJIb2zSmT5vGtEuK8bqE1uooaZsY43MuzsYxYdSngMH7J0HZQQiLhQe+gObVl8/UbyEcOHAgL730EsuWLUOSJMxmMxs3bmTs2LE89NBDfndg9OjRjB492uU+q7hZSUlJ8SmOytM5g0qLG+HhFZan3MltsOh2GPItxLien9SqVax85gaumrYagDCNs4Ge4mARWgOlVZJ9yq/grjUu/10YhFXPN9tPKb8H0iI3mszK3PPKP7MVT+62YxeY+dNfJMaG06dtI25pn0CfNo1diuLflz6XrRpH+3zdZPUFlumm0KwsCyIbwpBvqmU4bIvfQ+P//Oc/tG3bluTkZIqKimjfvj033HADvXr14pVXXglGH2sWl10Nw1ZCdCKc2QsL+sL5o26b148qn2d1FTbQ4pIQHnEQQkfh82dlyY4Tedzwxhp+2pPt+V5cEIoBvbURd+9yIMNDi2zS5eeVWFJoXd+6IWntGhOhVZNdUMrnW0/wyKLf3X5WrOeIj9S53O/E2UNc99tgWqqyyJEawiOrql0EoQJCqNPp+PDDDzl8+DDff/89ixcvZv/+/Xz66ad1Ng2XEwntYfhPUC8FLhyDBf3gzD6vh7kaWCjL7HKLkGW5fFWJg/A5Dks8WYSPf/o7x8+X8PinmV77BA41S4RJWCW4e94E8kGUf9Gg/J5XogegT5vGfPRwN7ZPvIVPHumuOEBO5bleElfoQzJWhaydsKAvYcWnOWxOYrBpCjRsXcm7CAwVrmvcrFkzmjVrFsi+1C7qpcAjP8Gn/S2W4cLbYPBXFovRD1IuTUAXlBo5X6xXnCWOjhVHZ4mnlSXequR5Quhg1eCuWl0g5wgLLpZbhBdK7JOqhmvV3HB5I77fdZq9WQWUGlyXjfAlPT8Af2+65EQswJTQkfv/foJzxFFcZvR+bBXgUw/GjBnDtGnTiIqKYsyYMR7bzp49OyAdqxXEJMLQH2DJ/ZY5w4/vhEFLLHOJPhKuVdM0PoJTeRc5eraY6HDLv8xR+ByF8aLBxL9/2MvLt7d3Oqfjsd4I9bWutRF3b3Mgl+daM0qD5fMCzoIWprGM8srcPDytQ+MYT2J24Gf48iEwXoTm16Ie9DkX/5MBehNni8pqjhBu374dg8Gg/O4OkaTVBZH1Ych38MWDFm/yZ/fBvQug3T+dmrp7+5rVj+RU3kVOXrioTEo7Do1dFXn/8LejdkJYXGYkTKPyWwhtCcWVDXWJwFqEBqdtUQ4V6MK1ls9KmReL0PqAdmL3V/Dt45YVWK37wv0fgzaCRjFh/H2uhDOFZT6H3QQTn4RwzZo1Ln8X+EhYtCU84OtHYd8Ky9Pxzvcs8Yc+EBth+TcVlhmVnIXenCWOFJQauHbGrzSKCbOzBHzBPvuMX4cKKog7uQuk19jV5yDGQdCs2abdDY09zhFu+x/88Dwgw5X3Qv/5oLaU8GwUbRHCUMlLWHHTQOAfmjC4dyF0eRBkMywfBRnv+3RodJjlw1NUanS5zhi811M+ef4ihaVGjuQWU1ha8eLawiKsIty8z4Gco7WdI7TiOEwtF0J3Q+NLc4u2AirL8Nub8MMYQIarh8O/PlREEEIvQatPFqG1uLsvfPPNNxXuTK1HrbFYguHxlrXJP42H0jygI659xhasT+niMqPPzhJH3E2++4Jd/QwhhFWCW4swSF5jK1E6xznCS0NjNzW2rUNjZY5QlmH1RNj0juXv68fCTa84zfs0DjEh9MkitC3mHhsbS3p6Or///ruyPzMzk/T0dOLi4oLW0VqDJMGtr1k+HADrXmeS5hMkzG6l0DrsKCozKitGvDlLHAnU90esLKleAjpH6MPQOMyDRSjLcnkt43ANmE3wf0+Xi+Ctr8HNr7qc/K6RFqG1uDvASy+9xP3338/8+fOVuEGTycSoUaN8z9pS15EkS+aa8Hj48QWGaX4iTipmKS+5bG4ddhSWGpWwGF+cJYFCrCypetzXLAm2s8RhaHzJIix1YRGWGc3KCCVabYKvHoG931kSkPzzbbjK/Uqz8vrGoZGy32+/9YIFC9iwYYNd8LRarWbMmDH06tWLmTNnBrSDtZruj0F4PIavH+df6g0k5xnAsAK09skYyi1Cg1tniTeLMFBfIDE0rl4CGz5jP0eocxFR4MlZYrUGIygl+pshcORXUGktGWSuuNvjtV3mJKxG/DYjjEajy2zP+/fvxxykhf61mo73McIwhlJZSzf9FvjsXii1LxVgHa5YhsbOhZvA2UJ0JGBDY2ESVgnu5nQD+SBytAhdeX7L5widv9tFpUZiKWJJ2OtIR34FbSQM/tKrCAI0ir5U1rMmDY1tGTZsGMOHD+fw4cN0794dgC1btjBjxozgprqqxawxd+Fh/TgWhs8i8thv8MmdMPhriGoA2FiEpR7CZ1TenCWBQehg1VAlQ+NS70LoyWtceuE0X+heo510HMLjLCunkrv7dO3GsRaL8GyRHrNZtksgUh34LYSzZs0iMTGRN998k6ysLACSkpJ44YUXeP755wPewbrCFrkdk+q9wczSyXB6OyzsZwnEjmuqzNsUlrkPn/FuEVbCa2w3RyiUsCpwm3QhgIMuR6+xqxUeViF0Cqi+8DfNV9xDhOo456R6NBj2AyRc4fO160fpkCTLCONCiZ4GXoo+BRu/h8YqlYoXX3yRU6dOkZeXR15eHqdOneLFF18USRcqyVFtKxi2CmIvg7MHLJlrzh22swiNbsJn3DlLrAIYKEtO6GDVUDXOEvs5wugw5++vdWWJ3Rzhmf2woC8RhX9zwtyIl+vN9EsEwRL1UP9SxpozITA8rpSr0a/6HgLfaHS5JTVRg1aQfwIW9KV+4V/ApThCszX7jG/OknIBDEwcoZgjrF4CNUeoN5qV9cVWXFmETmuNT2VaRiuFWRTEtORe/SSKoyqWfCWUQmgqtNr5q6++4ssvv+T48ePo9Xq7fX/88UdAOlYXUcKt4pMtluHi/pC9m8Rv76Gr9ByZ+jZK5hgnZ4kbITSazahV6oBZcmJoXDW4c5YEKo6z0EUMoes5QhuL8Oh6+HwQ6IugaVdWXj6bnJWnuKqCSRMaxYSxP7swJITQb4vwnXfeYdiwYSQkJLB9+3a6d+9OgwYNOHLkCLfddlsw+lg3iW5kyVzTrCeqsgIW66bTW7VTSaDpvNbY9b9y9s8H+CTjGOeL9S73+4J99pkKn0bgD26HxoE5vWPoDHh2llxn2gqL77WIYOoN8NByzsnRbo/zhVAKofFbCN9//30++OAD3n33XXQ6HS+++CKrV6/m6aefJj8/Pxh9rDNIjmtLwuPgwW+g1S1ESHo+1M6i6akfAffOEkfD8L/rjzBx+R6mr3Rd4N461NpzOp+bZq1l5e4sj30UQ+OqobJJFw6dKeS++Zv47WCuy/2+BFODJXzmX6r1zNW8BaYyaHsHPLAMwmK8Z57xghJUXVADhfD48eP06mUpshIREUFhYSEAQ4YM4fPPPw9s7wSgi4SBS1glXYtOMnHvsckMVP/qNBS2Jl1w5zQ55+apa41LHLk4kyNni3niM+epDdlFe0H14OvUxPPLdrHt2AWG/G+ry/2u1hm7suyidnzEbN18NJIZQ4eBcN/HSsC/T7kIPWBX1rOa8VsIExMTOX/+PGDJUm2tZXz06FGx6iBYaHTMCB/DYuPNqJCZof2IPufsHzrWobFjoXcr7vTLGpfoa0YavcjDVSW4+y75+hwq8pJqzdU6YzshlGVYOwPtz5ZyuguM/ci/dY4lcYj1GmU+Zqd2Q7mzpPqX2fkthDfddBMrVqwALMHVzz33HLfccgsDBgxwWe9Y4AceQgEjwsN4xfgIyyLuB+C2rHmwepIyaWe1EN2l43I3pDUYZY/7wf5LaahEmn8BrP3rDCfOl1T4eF+HxnERWo/7PabgMpth1XhYOx2Ad0z3MdU4hFKj/bUDNTQOBWeJ33fwwQcfKEvpnnzySRo0aMCmTZu48847efzxxwPeQYEFy/BD4l3pAQ4a1EzQfg4b51jSeN0+u9widDM0djekslp4vn7BDEGwCGVZrhPZzTccPMvQhdsAODbjdo9tKxtHGOtNCF1YhFFhajAZYcVTsHOJZWO/1/loVSoYjE6rS4r8KdzkgsYxobPMzi+L0Gg08tprr5GdXV7ab+DAgbzzzjs89dRT6HQ+lvQT+I31qXuhRM8Hpn/yY+p4QILMRfD1cJrGqInQqt3Wl3X3BbKm9fIUlmG7J9BD47/PFdPt3+nMW3s4oOcNRbYdO+9z28qW8/RuEVqE0LaWdpzWZMmevnMJSGro/1+4ZmT56hKHDDTKHGElLcKCUqPbDNhVhV9CqNFoeOONNzAaK57hWOAeTzaR9alrncvbm9gf7ltoyfax51vqrXiYjc9fwyfDXa/1dDtHeGm442u+jMpUwHPFjJX7OVtUxuurXHu1axPesojbUlmL0FYIjS4eXlaLsOElh0UUF+ny2wj46wdQh8GAxdBpIOB+vbEyNA7zLLruiA3XoLskxNVtFfo9R3jzzTezbt26YPRF4AHHeRiNWoIr+sMDSy1ZPw79Qv2vBxBmKHR5vLs5QKuF58kitB02B3poXJcCtP0RQncUlRo5klvktV1seLk4ufIQ51+aI6wfpSOeQj7T/Zu47AzQRcODX0Hbfyht3RVw8qumsQskSQoZz7Hfd3Dbbbcxbtw4du/eTdeuXYmKsq9Adeeddwasc4JyHEMUlPCZVmmW5AxL7oMTm2HRHTRkFGfxLVu4VSA9OUsWbjqm/C68xhXHn2lQdytL1h3I5aY31/HtqF50aVbP7fG2mpt30eCU1MA6NG4ZXsCbuqlcrjqFKbwe6iHfQNOr7Npal9k5JmdV0vRXcGgMluHxqbyL1W4R+n0Ho0aNAlzXL5YkCZOpesf6tRXHp65dvGCzHjD0R0sx+ZzdLNNN5kH9BE7RyOt5fall8sOu8iBrg7HuWHCBRuWHEnozlD/67ShzB7sXQtvnWl6J86qiglIDzaVsXj3zBg1U2WTJ9QkfuJx6TTs6tS23CMsfgkZT+VrlilqEEDq1S/weGpvNZrcvIYKVw9P3xGlo7DjMSrzSkqwhrhmpqhy+CptCS+lUwPsYaIvQaTVNLSaQKff+PO15FZftlIN1WaYtDYsO8pVuCg0M2Rw1J3CffhIRTV1nkFHmCG0swuKy8t8rU6C9PGV/DRNCQfDwJAqOHzaXYTINWsLwnzhobkqSdJ4vdVO5Ujri8Zr+TtGJOMKK459F6Pkfk5VnCUJ2F/Zku/WCoxCe2MqbJRNoJOVzLqo19+snkSU1tvMg26IMjW0swsKycq+zzs1xvhAqsYQ+S/nFixdJT0/njjvuAGD8+PGUlZV3Xq1WM23aNMLDw92dQlAJnOYI3SVijW3CQMOr/E/7Bp1VR/hc928eMzzPZnN7l839FUIxR1hxbIWwsrGTepOZ/9t5mgnf7GbmfZ3od2Wi3X57i9BmaHwoHb54kFhK+N18OSevW0Tu8mPEhWnc9ifMRU7CyobOWAkVIfRZyj/++GP++9//Kn+/9957bNq0ie3bt7N9+3YWL17MvHnzgtJJgfPQ2FNq/gJVHIP1L7PJ1J4Y6SIfa1/nZlVmQPqhN5kDupSyDsRRK9h6jQ1eAgJ9eYe3HD1HYZmRySv2OMfh2ZxA8Rrv+Q6WDABDCetMHRmiH0dcPcs8sqd5vnDHnITYhs5UUghDxGvssxB+9tlnjBgxwm7bkiVLWLNmDWvWrGHmzJl8+eWXAe9gXcLjHKHDB85TKIZaJVFMBMMML/KzqSthkoH/at/ibtUGp7lFfwu/y7LIQFNRbN96b2FIvjxrrG2yC0pZsPGo3T5bi/BCiR7++AS+GgZmA6WX38mjhrGUSuFc06IBvVo2YEjP5m6v4ypLdWFZ5ZbXWWkcaxlBnq0pFuGhQ4fo0KGD8nd4eDgqG6uke/fu7N27N7C9Eyg4DkE81Six5iosQ8cThmf52nQdGsnMHN37DNP8XOm+eLNm/KEuWYS2Q0+jV4vQ+3tsHZ4CzFt7mAs2OSdtn1VdTy62LJuTzXDVw5y86T0MaIgJ0xChU7PksWsY2bul2+u4CqgOmEVoMzSuzqQtPgthXl6e3Zxgbm4uKSkpyt9ms9luvyCwOEbvu1tTDPaWhwk1Yw0jWWjsC8DLqoWMVn+LdexUkc9eoFaXFJcZ+XF3tveGtQRb0Q/EXKttst3CUiPvrTmk/G35v8qM1XxB/7PzLRuvfQb++TYFesu14yJ9WxFidaK4miOs6KoSKw2jLcty9Sazy8DvqsJnIbzsssv4888/3e7ftWsXl112WUA6JXDGa/iM7T4HkZRRMcX4EG8Z7gFgrHYZr2gWI1GxL2OgHCbf7Qh8eE8oY+vhNXpZ0+jLA8oqhF2bW+IJP834W8lsI5tNTNMsZLRmuaXxzZPglqkgSUowte3qE0+UrzV2tggr6ywJ06iV5YDV6TDxWQj/8Y9/MHHiREpLnXOHXbx4kSlTpnD77Z4zagg842mYGKlV2+33bBG6OpHE26Z7eFc3HIBHNSt5Q/MBmP1fNx6oZXZ1barRdm41EIHpViG8rlVDrmvVEL3JzKyf/wKTgX8dm8IQzS+YZYk3NCPh+jHKcdY0/b4LofMSu8our7MlFDzHPgvhhAkTOH/+PG3atGHmzJksX76c5cuX88Ybb9CmTRsuXLjAhAkTgtnXWo+nOEKVSiJKV/6h8zRH6EEj+T7ibp7Xj8Qoq7hPs57kX0aB0b8PYKATL1i5qK/dAfm204IGbxahD+ezCqEkwbjb2gLw046jFC66nw4XVmOQ1TxtGM0nhpvsjlMswgjfRMxVQHVlcxHa0jgEgqp9FsKEhAQ2bdpEu3btGDduHP3796d///6MHz+e9u3bs2HDBhISEoLZ1zqP7dPXsXiTLZ72hWlVfG2+gVGGZyiTNcT9vQr5s/uIxNnSdzd5bTCZyS0s4530g+QUBC67cJ9ZawN2rlDEn+QVvgyNbYeqVzaNY2CHWD7WvU7MiV/RS2E8Znie7809KSoz2j288v0cGpfPEdoMjS8FVNc5ixAgNTWVVatWkZuby+bNm9m8eTO5ubmsWrWKFi1aVLgTc+fOJSUlhfDwcHr06MHWra7rLFhZtmwZbdu2JTw8nA4dOvDjjz/a7R86dCiSJNm9+vXrV+H+hQq2T1/PFqH7fdYP9c/mbgw1vIRJG4V0dB2f6f5DHOVZTV797k96z1zrsuxjmdHMox9vY/bqAzz1+faK3IpLsgMoqqGIbYYfb15jv+tQF+UyNW8cPVT7KZAjGBs+mbXmzspuW0eENQWXt+StVlzlI6xsUlZbQiGWsEJrY+rXr0/37t3p3r079evXr1QHvvjiC8aMGcOkSZP4448/6NSpE3379uXMmTMu22/atIlBgwYxfPhwtm/fzt13383dd9/t5Mjp168fWVlZyqs2FJaytwg9hc94D60ByDBfweHbliCH16OL6hBf6qbSmAsArNqTzfHzJfyV7ZzWy2Ays/OkZa3r1qO+Jxt1xFUvT14o4fOtx52SgNYGbOcIvTmc/PHmx5Rmw8J+6HL/pFhTj0H6V1lxwT4uMP9iuYfZmqbfW/JWKy6X2AUofAZqoEUYDGbPns1jjz3GsGHDaN++PfPnzycyMpIFCxa4bP/222/Tr18/XnjhBdq1a8e0adO46qqreO+99+zahYWFkZiYqLzq1XOfqSNU8BZTZ+uh8+gs8SCEjqPmi406kz9wOdlyPdqoTvKVbjLms0eUOSDHZJxgH0dYz8cQDF+5ZfZ6xn+zm7lral/Galsh9G4R+kYL6TT37X4Uzh2CuGSMD6/keFgrp3a2640Vi9DH+T1PS+wCMkcYW8eFUK/Xk5mZSVpamrJNpVKRlpZGRkaGy2MyMjLs2gP07dvXqf3atWtp3Lgxbdq04YknnuDcuXNu+1FWVkZBQYHdKxSxswg9BlS73+fokJGBi/Uu5179JI6ZE2imykVa2I9k4zEAJdWSLbbzTdaneaCwXm/jobMBPW8oYArwHOEV0lGW6aYQW5YDDS+HR1YRl9yOUTc6C6FtBppyZ4mPQ2PFInR2llS0lKctjaItq0vOVGM1u2oVwrNnz2IymZycLAkJCXZ1UWzJzs722r5fv3588sknpKen8/rrr7Nu3Tpuu+02t2nCpk+fTlxcnPJKTk6u5J0FB1+dJZ7mCB2tTlmW0RvNnJQbc59+IvvMyUjFOXypm0oX6aDLWhK2X+JAC6Ftv2obtsvevA6NvcwRdpf28bnuNRpIheREtYVhKyHOEsc77NoUkuLsk58U2M0RVjB8xjaOMIAWoRgaB4mBAwdy55130qFDB+6++26+//57tm3bxtq1a122Hz9+PPn5+crrxIkTVdthH4m2GxpXzFniCusHPJd6DNC/SklCV+KlYhbr/kPM6d+c2hfry2MPG0a7F0JZlln1Zxa7T7rOnedpKqD2yaB/Q2NPz4E+qu18optBrHSRLea2fN1hHkQ1VPaHa9W8eX8nbri8EW0TYwD70Be/LcIgLrGDciG8UGIIWmiWN6pVCBs2bIharSYnJ8due05ODomJiS6PSUxM9Ks9QIsWLWjYsCGHDh1yuT8sLIzY2Fi7Vyhi+6HzJHae8t457pOxH+oWEM3OPotYb+pAlFTGtVtG0Vdl78W35sIDiI/QsvnIOU7nXXS61rLMk4xc/AePfrLNbX98ZevR8xw7W1zp81Qntl7jigal36naxAfa2YRLBn4xdeEh/Tj0GufKhb1aNuSTR7orQmgbo+lvHKE10sAaUG02yxTpA2cRxkdolemcc8XVYxVWqxDqdDq6du1Kenq6ss1sNpOenk7Pnj1dHtOzZ0+79gCrV6922x7g5MmTnDt3jqSkpMB0PEh4y09nK4SenCUe5wgddm0/nmc35AE4p9fwqGEsP5i6o8XI+9q3uU+9Vtl/ykb0tp/IY+AHm+k141ena32xzWJZ5xSUkX/RwPFzvhc2t7WIDuYUcv9/M7ixhscZmkx+zBG62PagejVztHPRSia+NV3LSMNzlOG5hG6EzmLNWYVQlmXFWeKr19gxoLrEYFL+PzGVXGsMFudedQ+Pq31oPGbMGD788EM+/vhj9u3bxxNPPEFxcTHDhg0D4KGHHmL8+PFK+2eeeYZVq1bx5ptvsn//fiZPnszvv//O6NGjASgqKuKFF15g8+bNHDt2jPT0dO666y5atWpF3759q+UeA0Wkj+EzHucIHf6e9v1ep+HIhRIDerQ8ZXiapcYbUUsyM7UfMFxtidc8eaFcCHe5GfaCfRnJrtNWc8PMNZy84JsY2grBntOh6bzyF3uL0PPQ2L66n8wo9Xe8pl2ISpL52HgLYwxPYPQhr7JVxKxOqFKDWbm2v2uNDSYZk1lWhsVqlaTMH1YWJWV/QfUIYeXt2koyYMAAcnNzmThxItnZ2XTu3JlVq1YpDpHjx4/bpfvq1asXS5Ys4ZVXXmHChAm0bt2a7777jiuvvBKwZMretWsXH3/8MXl5eTRp0oRbb72VadOmERYWnIn9qiLcJiW6Y2IFWzw7S5z3OU7c511aumVGxTjjY+QTxeOaH3hVu5g4qYiVF4b71F+7AOJL82OZf1/gsnqRlr74WK+ktuQ/tEu64G1oLJf/Ml6zhMc1PwCQ1+1ZDurvQd7i2zx2pM5eCK3WoFolKfu8YSt2ZUaT3aqSymTZtqW6g6qrXQgBRo8erVh0jrhycNx3333cd999LttHRETw008/BbJ7VYa3j1SYtvyDW1FniatdrixC215NNz5AvhzNi9oveFrzHY3yLjKBIcgeBhRr/jrDn6ecLbmKOINrS+1jX+YIp32/l9V7c0iuH4EKM//RfMRAzVrLPsNgXr19Clf9cZLFPgphhNY+9KU884zvImYNqLacxxzQYGordX5oLPAd25ASz2uNPX3AXViEDkLoXP5R4n3TXbxiGIZZlhjET8zWzkOD+8w1wxa6dpD4LGo27WqyEP60J5sHPtxMdn4pttrnamh8vljP/zYc5fj5ErYdyuY97TsM1KzFJEu8YBjB/0yW7E7+FIGyDmtLLs0R5vvpMQbLg9X64C01mJQKdpVNwWVLdQthSFiEAt+wHSJWeI7QxS7H5WwXXNTBBVhsuoUCOYo3tfPor95IDCU8aXjG64S9LT7roM3vNXlk/Pinlloxk1fssXNOuLIIfzuYC0AEpfxX+xY3qHdTJmt42jCan8zdlXb+jEYdnSX+OkqshGvUGExGyozmgCZcsFLd9Y2FRRhCePuA2wqhp2V0gR0a27PC3IsRhjGUylrS1Nv5WPc60VgcILaF4N1hvQNLFTfPbT/NOMak5X/WqDnC3MIy+s1Zz4IN9jVELpTolXlSwO53K5IkEUsRi3XTuUG9m2I5jEcML9iJoLWdr0Q4OEus64x9dZRYCbMZYhcGMAWXlfL6xtWzukQIYQ3C1yGiZ6+xD84SNxahlTXmLgzRj6dAjuAalWWFQ30KeHLJH177ZjSZ+Xzrcbq+9gt/nvJcpPzV5Xv4OONvMo64Xx4Zarz760H2Zxcy9Xv7+j1qlWT3/5v5019KoPnRs8Ws+jOL8NIzfKGbRlfVQfLkKB7UT2CjuQOO+BMvb3WIlDo4S3yNIbRim64/kJlnrChD42pylgghDCG8fb5dWRGuUHuYP3RlTDjNEbqpHWG7rnSb3JZB+lc4K8fSQXWMZbopJOFdsIr1JsZ/s5vzxXo+23LcbTtbzS+oxloW/uJqSSJYhNDRsh252DJs7jNrLa99toouvzxAO9UJzsjxDNC/yna5tctzVWaO0N80/eXnKc9JGKg0/bZY1xtXVxEnIYQ1CLOvQujhe+J6jtBeCN0V0bH1WgPskVO5Xz+RU3IDWqqyWBY2Bc66Xr1jpaTMt9IA/pYZDRXchQRJkmTnNYby9bqtpZN8rZtMI8MpjpsbcY9+En/Jzdxewx+L0GlobF1n7O8coU1OwmBahFq1ShHtqkQ4S2oQgbEIvXuN3T2QwzTO5z0iN+G+skl8qptOS1UWLOwHD37j9vpFet+E0Jaa5DV2Z6ypJecHmVolwclMvtRNpZ5UxEE5mcH6cZzBc8o4v+YIrUNjq9e4xL8UXFZs1xsXBqiCnS0ROjX7p/VTrlPVCIswhPD2AffVaeA5DZczvlal07kQQoDTNOR+/UT+NKdAcS4sup2rpf0u21on671hq30bD9WcOUJ3/0KV5Dw07i7/CZ/cST2piB3mlgzQv+JVBMH7FIotthbhkdwiMo9bEu/66zVuEGWJDDicWxTQeiW2VJcIghDCGoWvlpEnj7IrsS1zkXzVFZ6CuM8RxyD9K5iTe0JZAZ/qZnCjyjmNv6/D+9qGymGO8FbVNt4xvwb6IjaYruAB/cucl2N8O1cF5gjzLhroN+c3Dp0pIkyjoksz/xIV927TCIBf9uXYDI2rT7gCjRDCGkT7pDif2nmzCB0FTe8mT6Mj7ixCK4VEknfPUmh9KxGSng+1s7lDZZ8w94vffVsRUYNGw25Z9Wd5jky1zRzhPar1zNPOQYcRc5s7eMTwIiWEuzuNEx5mPpyweo1l2WL539imET8/dwNXNvXts2Tl5raWJa87TuQpWYACOTSuboQQhhDenvPXtmrAu4O68PNzN3hs5y2OcMwtbey2+ZoDztNqFislsg4GLmG5qRdaycQ72vd4QJ3u9bjagMksc9QmVZjVKwzlXuNh6pW8qZuPWpL5QX0TJXf/Dz3+CYo/c4TxkTraJsZwWb0I5j/YlYVDu9G8QZRf1wNIjAun42VxyDIcsQphgIfG1UntuZM6gCRJ/LNTE6/tEuPcWxeSJDGydwteX1U+h+erEOo8FUy+RKnBBOpInjWMokCOZIjmF/6j/R9xFDPPdKdP14GamZj1pa93sfmI62JWKgnuzvuEe7SLAfif8TY+iXiM7kb/kxb4MzRWqyR+ePp6VJJ/AuqKtHYJdtmGAuk1rm6ERVgLGdorhQ5uhj4Szl8IV84SVx5iXzJfW0MfZFS8ahzGe8a7AHhJu5Rxms+pmRLnG19lnnS5XcLMvbnvck+hRQRnGe5jmvFBVGq127hDV3RKjr90Pv9Qq6SAZIm5uV1ju78DGUdY3QghrIWEa9XMfeAq1zt9CKgG1yn4fZmbKtGbbBwiErOMA/i34QEARmr+j+maj1Dh3QKtLTVLNBiZpZ1P7wuWkKKJhod5z9QfkFBJrotjuePDIV0B/yzCQNI+KZYmNqMNYREKgkIgP98RbnLNufoSOQZUgyXg1tEq9OULuGTLcWassg+d+dB0By8aHsMkSwzSrOEd7btoPWSugdB3luSXGHg3/SAnzrtPNBuGnnnat7lHvQETaubEvsAnpvLkwGqVZJdC3xuNYy0i5GdJmoAhSRJp7csLp9WmOUIhhCFF4D7h7pJuurqCqy9jTLjGKa7Ll6Hxip2n+WD9EaftX5r6MNrwNHpZzR3qLXyknUUE7hfYh3oQ9YTvdvPm6gPcPXejy/1RXGSh9g1uUWdSJmv5pNlrrAnrY9dGJUl+WYRWApUMtSKktSsXwiidEEJBEOiRWj9g54pwE5xqLaZti7UWRZSNeMaGa5zOoa7kF3CluQfDDS9QIofRW72LT3UziKXIZdtQF8LNhy1B3ueKnRNU1KOAJbp/00u9l0I5gocNL3Eg/jqnGEqVJPk1R1h+XMX6HAiuadGAtHYJDLmmud/VEkMZIYQhwPoX+vDGPR0Zem1KwM7pLqjaVeiEtUyj7VAnOkzDeYcveSAskd/MHXlQP558OZKrVQf4QvcajchzahfiOuj0/lrTbiVyji910+ikOsJ5OZoH9C+z2dzestbYQQj3ZhUoa38rc+2qRKdR8dHDVzPt7iurrQ/BQAhhCNCsQST3d0v2WJkuUDSvH+m0zWqV2E5+x4RrnbzJgereH/LlDNBPJFeOo53qOF/qpnCZlGvXxjFBQahhG7ReZjQx9fu9pEhZfBU2hdaqU5yW63OffhK75RaAxZp2ZeV+tvlvv6+dGOt78LXAN4QQ1mIe793Cyfub0tCVRehKCDUehz5Drmleqb7tl5txr34SJ8yNSFXlsEw3hVZSefhJqA+Nbd+bUr2ZdtLfLNNN5TLpLEfNCdxXNonDclO79q7Wiv9xae2vP1xWL4Km8REV67jAJUIIazHjb2vHt6N6KX+Ha1VKSvR3BnUB4MqmscrQOMrBInQMoD5jk0a9SQC+iH/Lidyrn8QBc1OSpPN8qZtKR+kwEPpDY1shNP69maW6aTSS8tlrbs59+smcopFde1dJF8B7WU9XSJJEz5YN/O+0wC1CCOsQKQ2ilHk+azzY0dxiSvTO+eWiwzVOa4ubxJWLn+cCUb6TQ33u109kh7kF9aUiluj+TU/VnpBPzmAVwhtUO6n39b3ESSVsM1/OQP0rnMU5mF2tCuxwv5cQwoAihLCWYzux3rxB+fxg5+R4ujavR7HepBRHsnWWxIZrnOYsr2nZgDkDOvPTszcENOYxjxgG619mo+kKoqVSFmnf4FrTFtdtS/QeY+/eWn2AySv2BK5zblBLErerNvORdhYqYylrTZ0s5QtwvY7XnUVYUWwtQl9TmwncI4SwlmOrVyk2HmONWsWcAZ3t0u87zhFG6Bw+HrLM3V2a0ibRt3RR/lBMhKVQkelqwiQD040z6a/6zald56mr6TTlZ5fnkGWZt9MPsmjTMSVDSiCQZdlJxG43/My72nfRSSbOp9zOY4bnKcU5NMnKrpP5Tit4mrlwXPlKko11vvVYzcnXGKoIIazl2FpuzRrYf/GS60fyWn9LGESkTk2kzlYUtfRq0dCufbAHq2XoGGV4hq9N16PBzFu6eTys/smpnd5kdrkE75d9Z5TfS42BS/c+bNE2bn5zrVL2NOvHGTx78T1UkswS4038ec1sDF7yl2QcOWc3xwpweULlHihdm1tyCtoGOQsqRu0JDRe4xHZZXIqLGMK7Ols8m7HhWjsPZky4hgm3t+Pv88VKRhVb7QnW6gYTasYaHqdME8MD8o9M0X5MHMW8c2l9rpUyo9lp5cvvf5dnfjH7llDHK3qjmbV/WUJ79p8uoNOBt0na+hYA84z/5HXjQP7rp+aqVRL9rkykef1IftmXU+G+LRzWjd8OnHVKhiDwH2ER1nJsw1CaN3A9FLurc1P6tG1sl28wJlxDXISWpSN60ukyy+R/3ysSg9vZS8ioeF0eymzDvQCM0X7FRM2nSDbJGlzNExbZBCcHqviTtc6uCjOpW16BDRYRnGEYyOvGQYD/c38rn7meuQ9cReuEaGXbsGtT6H5pZdHkf7Znx8Rb7I5xlQQjNlzL7R2TqjXFfW1BWIS1HNuQGNt5JVdobDJXx9iUe/xm1LWU6I1221zZg2ntEipl4dgiA++Y/kUBkUzWfsIjmlXESiW8ZHgME2ouGkxO1T2KbCrkBcpBm1NQhhYjs7XvE7tnMyAx3vAIn5tuVtr4G/NonSts3bh8aNwuKZaJd7Qnp6CMhNgwJ4t75TPXV/wmBF4RQljLiQ3X8v1T1xGuVXtdG2q739ZxolZJdiIIljWnjjSND9yKB6uRtcjUjwI5kje0H3Cvej0xlPC0YbTLko9FFViu5o3c8xf4QPsmfdQ7MaAhq88cPl9pvybcX4uw+JJgt2ocjSRZRDtCq0aSJJdJddPaNVbKXQqCgxga1wGubBpHq8bRXttZYwOjdN5Fs32TWA7/5x922x7qlVLhPjpia919Y76BJwzPUiZr6av+nQXameTlnefd9IO8m35QqcMccIvwYh6d1z1CH/VOLso6HtU/zz9+dZ6P8/da1hRp4Vq14jl2ly0IfCuRIKgc4h0WKFiF0NH6c4daJdHaRmBbNopm68s3ezii4qw2X81Qw4sUyeFcq95DvWX38r/Vmby5+gAjP7XUBim2qZlc0eDlQ2eKeHLJH+w7eIiy//2DxLztFMiRPKgfzzpzJzuxteLv0NjWgfXkja244fJGdPOQeUjjoXqgIDAIIRQoqC8FUPuTcHPiP9sDMLJ3SwAaxwQvIUCG2Vr2MpqWhr/4UjeVBM6TccQSR2c7NDb6WKvZFpNZJm32Onbs2kXYp7cTdnYPuXIsA/Svkim3cXucwc9r2Vrb93dL5pNHuhPr4eHT8TL/Ks4J/EcIoUBBq1iEvgvh9a0bsXdqX8bd1tbl/igPQ76KsEtuyf36iZyhHperTvGVbgrNJIuDpqisfN7Q6Oe8ndksM/Onv2gpnWJZ2BRaqLIpCEvkPv0k9smeE0z4k2UafEtwC/DD09cx7ra2DLs21a/zC/xHCKFAwTp31SBK59dxkQ6Zils0Ko9XHH5d4L/Eh+TLuN8whWPmBJJVuXylm8Kva3/lbFF5wLLRj2QGsixz46y1bFi/mi91U2kineeQuQmftPuAY3KS1+Mn/99ev/rva7q1K5rEMbJ3yypJz1bXEe+wQOHmdgk8cWNLnk27vFLn+XR4D+X3pCClizpmamix1szNaCzl0XXNYK6SDij7jX5EVBvNMkkXMvlc928aSIXsMqdyn34isza7zp5dWVLcxHMKqg8hhAKF6DANL/Vry5VuSoH6SlXlysslngH6V/jdfDlxUgmLddO5XrUL8M8i3PrTEj7WzSBGushmczse0L/MBWKD1e1qrTkicI0QQkFQsNZfubW993WwzepHsnPSrRW6TgHRDNGPY72pA5FSGf/TzuQ21RZ+//uCx1ReJrNsmdvb9SU9tjxFuGRgtekqHta/RBG+WWzdUhxDut0zqHuyz20FVY8QQkFQ+Pyxa9g/rR8NXCwNc2TiHe2Ji/AtZMcVFwnnUcNYvjf1QCeZeE/7Duc2/I/Pth53e8x98zfx1vQXMX8zAo1k5hvTdZZYRXyfH51xT0en5LXumPCPdgztlcI3NolyBaGDEEJBUFCpJGUN7L/7lxf6Geoi6NpaPsCbM7X35Y3c7tOj5WnDU3xu7INakpmp/YC//+91Vuw8bddu/YFcOk3+iV6nFjJB/ggVMouMt/K8YSRGNwutMl9J48vHe9pta5MQQ/P6kXRKLp9GaBjtWkQfuTaVmHAtk++8gqua+W5FCqqOkFhiN3fuXGbOnEl2djadOnXi3XffpXv37m7bL1u2jFdffZVjx47RunVrXn/9df7xj/JVDrIsM2nSJD788EPy8vK49tprmTdvHq1bt66K2xE4MLhHcwb3aE6pwUS4Vk27pBgWbDjG87deTphWrax6ad04hr9yCu2OzRh/Ez2n/wqglBlwhxkV442Pkk8UIzXf84r2M95bVsSRpLm0uLSu96EFW3hZ8xmPaX8E4G3jv3jLeA/uakovG9mTBtFhTiEvq569HkmS+OSRHqw/mEtsuJaeLRuwP7uArLxSyoxmYiM0XN28vlOmb0HoUe3/oS+++IIxY8YwadIk/vjjDzp16kTfvn05c+aMy/abNm1i0KBBDB8+nO3bt3P33Xdz99138+effypt3njjDd555x3mz5/Pli1biIqKom/fvpSWui8oLgg+VgtxQLdm/PTcDdx6RaKdlffBQ1257cpEVoy+lo8euprlT15LUlwEcx+4iqG9UnjixpZO53z3Uu2VciRmGB/gdcNAAEZrlrPxnaGU6g1gMvKG5gMe01hEcKphCG8Z78WVCHZPrc+WCTfTLcUy1xkfqVOG79e2aqA4PCJ0avpekahkjG6bGEufto3pd2UivVo2FCJYQ5BkVxkuq5AePXrQrVs33nvvPQDMZjPJyck89dRTjBs3zqn9gAEDKC4u5vvvv1e2XXPNNXTu3Jn58+cjyzJNmjTh+eefZ+zYsQDk5+eTkJDAokWLGDhwoNc+FRQUEBcXR35+PrGxwfMeCvzn1/05RGg19EitT5nRTIROzdT/28uCjUed2g5W/8I0zUJUksxyUy90GLhNvQ2TLDHO+BjLTDcqbR+8phmLN1vmFG/vmMTcB65yOp8syxw6U0TzBlFC4KqZQH9Hq3VorNfryczMZPz48co2lUpFWloaGRkZLo/JyMhgzJgxdtv69u3Ld999B8DRo0fJzs4mLS1N2R8XF0ePHj3IyMhwKYRlZWWUlZUH4xYUFFTmtgRB5Ka25V5oawD4q3e0Y81fZzh6tphJ/2xPnzaNufWt9XxmSqNAjmS2dh53qTcBUCZreNrwFD+ZuwFwTYv6PNwzhZvbJfDa3R0oNZgIcyNykiTRupJZpQWhSbUK4dmzZzGZTCQk2IdYJCQksH//fpfHZGdnu2yfnZ2t7Lduc9fGkenTpzNlypQK3YOg+pEkidXP3UDeRYOSwDT9+d7c+tZ6Tib9g8dORjBP+zZmJEYYxrDR3IGrmsXzxr2daNEwyq7AlUhyWjcJCWdJdTN+/Hg7K7OgoIDkZBH3VZPQqFV2WZyT60eyb1q/S39dy9nsh1BJKhY3bioCmgVOVKsQNmzYELVaTU6OfVbjnJwcEhNdp4VPTEz02N76Mycnh6SkJLs2nTt3dnnOsLAwwsJE4svaTMPEZtXdBUEIU60zvjqdjq5du5Kenq5sM5vNpKen07NnT5fH9OzZ0649wOrVq5X2qampJCYm2rUpKChgy5Ytbs8pEAjqOHI1s3TpUjksLExetGiRvHfvXnnEiBFyfHy8nJ2dLcuyLA8ZMkQeN26c0n7jxo2yRqORZ82aJe/bt0+eNGmSrNVq5d27dyttZsyYIcfHx8vLly+Xd+3aJd91111yamqqfPHiRZ/6lJ+fLwNyfn5+YG9WIBAEhEB/R6t9jnDAgAHk5uYyceJEsrOz6dy5M6tWrVKcHcePH0dlk6q8V69eLFmyhFdeeYUJEybQunVrvvvuO668snz1wosvvkhxcTEjRowgLy+P6667jlWrVhEeHrykoQKBoOZS7XGEoYiIIxQIQptAf0dFVKhAIKjzCCEUCAR1HiGEAoGgzlPtzpJQxDptKpbaCQShifW7GSgXhxBCFxQWWlJBidUlAkFoU1hYSFxc5cudCq+xC8xmM6dPnyYmJsZpOZZ1+d2JEydqnUdZ3FvNpC7emyzLFBYW0qRJE7vwuooiLEIXqFQqLrvsMo9tYmNja92Hzoq4t5pJXbu3QFiCVoSzRCAQ1HmEEAoEgjqPEEI/CQsLY9KkSbUyW424t5qJuLfKI5wlAoGgziMsQoFAUOcRQigQCOo8QggFAkGdRwihQCCo8wghdOD8+fMMHjyY2NhY4uPjGT58OEVFRR6P+eCDD7jxxhuJjY1FkiTy8vICct5AU5E+lJaW8uSTT9KgQQOio6O55557nGrGSJLk9Fq6dGkwbwWAuXPnkpKSQnh4OD169GDr1q0e2y9btoy2bdsSHh5Ohw4d+PHHH+32y7LMxIkTSUpKIiIigrS0NA4ePBjMW3BLoO9t6NChTv+jfv36uTlb8PDnvvbs2cM999xDSkoKkiQxZ86cSp/TLQHJc12L6Nevn9ypUyd58+bN8m+//Sa3atVKHjRokMdj3nrrLXn69Ony9OnTZUC+cOFCQM4baCrSh5EjR8rJyclyenq6/Pvvv8vXXHON3KtXL7s2gLxw4UI5KytLeflaFqGiLF26VNbpdPKCBQvkPXv2yI899pgcHx8v5+TkuGy/ceNGWa1Wy2+88Ya8d+9e+ZVXXnFZ4iEuLk7+7rvv5J07d8p33nmnXyUeAkUw7u3hhx+W+/XrZ/c/On/+fFXdkizL/t/X1q1b5bFjx8qff/65nJiYKL/11luVPqc7hBDasHfvXhmQt23bpmxbuXKlLEmSfOrUKa/Hr1mzxqUQVva8gaAifcjLy5O1Wq28bNkyZdu+fftkQM7IyFC2AfK3334btL67onv37vKTTz6p/G0ymeQmTZrI06dPd9n+/vvvl2+//Xa7bT169JAff/xxWZZl2Ww2y4mJifLMmTOV/Xl5eXJYWJj8+eefB+EO3BPoe5NlixDeddddQemvr/h7X7Y0b97cpRBW5py2iKGxDRkZGcTHx3P11Vcr29LS0lCpVGzZsiXkzhvsPmRmZmIwGEhLS1O2tW3blmbNmpGRkWHX9sknn6Rhw4Z0796dBQsWBCw9kiv0ej2ZmZl2/VKpVKSlpTn1y0pGRoZde4C+ffsq7Y8ePUp2drZdm7i4OHr06OH2nMEgGPdmZe3atTRu3Jg2bdrwxBNPcO7cucDfgBsqcl9VeU6RdMGG7OxsGjdubLdNo9FQv359srOzQ+68we5DdnY2Op2O+Ph4u+0JCQl2x0ydOpWbbrqJyMhIfv75Z0aNGkVRURFPP/10wO8D4OzZs5hMJqXAl22/9u/f7/KY7Oxsl+2t92H96alNVRCMewPo168f//rXv0hNTeXw4cNMmDCB2267jYyMDNRqdeBvxIGK3FdVnrNOCOG4ceN4/fXXPbbZt29fFfUmsITCvb366qvK7126dKG4uJiZM2cGTQgF/jNw4EDl9w4dOtCxY0datmzJ2rVrufnmm6uxZ6FBnRDC559/nqFDh3ps06JFCxITEzlz5ozddqPRyPnz50lMTKzw9YN1XgjuvSUmJqLX68nLy7OzCnNycjz2u0ePHkybNo2ysrKgrBFt2LAharXayXvtqV+JiYke21t/5uTkkJSUZNemc+fOAey9Z4Jxb65o0aIFDRs25NChQ1UihBW5r6o8Z52YI2zUqBFt27b1+NLpdPTs2ZO8vDwyMzOVY3/99VfMZjM9evSo8PWDdd5g31vXrl3RarWkp6cr2/766y+OHz9Oz5493fZpx44d1KtXL2gL5XU6HV27drXrl9lsJj093W2/evbsadceYPXq1Ur71NRUEhMT7doUFBSwZcsWj/caaIJxb644efIk586dsxP9YFKR+6rSc/rlWqkD9OvXT+7SpYu8ZcsWecOGDXLr1q3tQkxOnjwpt2nTRt6yZYuyLSsrS96+fbv84YcfyoC8fv16efv27fK5c+d8Pm+o3tvIkSPlZs2ayb/++qv8+++/yz179pR79uyp7F+xYoX84Ycfyrt375YPHjwov//++3JkZKQ8ceLEoN7L0qVL5bCwMHnRokXy3r175REjRsjx8fFydna2LMuyPGTIEHncuHFK+40bN8oajUaeNWuWvG/fPnnSpEkuw2fi4+Pl5cuXy7t27ZLvuuuuagufCeS9FRYWymPHjpUzMjLko0ePyr/88ot81VVXya1bt5ZLS0tD9r7Kysrk7du3y9u3b5eTkpLksWPHytu3b5cPHjzo8zl9RQihA+fOnZMHDRokR0dHy7GxsfKwYcPkwsJCZf/Ro0dlQF6zZo2ybdKkSTLg9Fq4cKHP560KKnJvFy9elEeNGiXXq1dPjoyMlPv37y9nZWUp+1euXCl37txZjo6OlqOiouROnTrJ8+fPl00mU9Dv591335WbNWsm63Q6uXv37vLmzZuVfb1795Yffvhhu/ZffvmlfPnll8s6nU6+4oor5B9++MFuv9lsll999VU5ISFBDgsLk2+++Wb5r7/+Cvp9uCKQ91ZSUiLfeuutcqNGjWStVis3b95cfuyxx/wWi0Dgz31ZP4+Or969e/t8Tl8RabgEAkGdp07MEQoEAoEnhBAKBII6jxBCgUBQ5xFCKBAI6jxCCAUCQZ1HCKFAIKjzCCEUCAR1HiGEAoGgziOEUFBjcJVuvrpSzgtqF3Ui+4yg9tCvXz8WLlxot81dcgeDwYBWq7Xbptfr0el0fl+3oscJagbCIhTUKMLCwkhMTLR71atXD7AUkZo3bx533nknUVFR/Pvf/2by5Ml07tyZjz76iNTUVMLDwwE4fvw4d911F9HR0cTGxnL//ffbpXNyd5ygdiKEUFCrmDx5Mv3792f37t088sgjABw6dIivv/6ab775hh07dmA2m7nrrrs4f/4869atY/Xq1Rw5coQBAwbYncvxOEHtRQyNBTWK77//nujoaLttEyZMYMKECQA88MADDBs2zG6/Xq/nk08+oVGjRoAlV9/u3bs5evQoycnJAHzyySdcccUVbNu2jW7durk8TlB7EUIoqFH06dOHefPm2W2rX7++8rttcSorzZs3txOzffv2kZycrIggQPv27YmPj2ffvn2KEDoeJ6i9CCEU1CiioqJo1aqVx/2+bPP1WoK6gZgjFNQ52rVrx4kTJzhx4oSybe/eveTl5dG+fftq7JmguhAWoaBGUVZW5lReU6PR0LBhQ5/PkZaWRocOHRg8eDBz5szBaDQyatQoevfu7XJoLaj9CItQUKNYtWoVSUlJdq/rrrvOr3NIksTy5cupV68eN9xwA2lpabRo0YIvvvgiSL0WhDoiVb9AIKjzCItQIBDUeYQQCgSCOo8QQoFAUOcRQigQCOo8QggFAkGdRwihQCCo8wghFAgEdR4hhAKBoM4jhFAgENR5hBAKBII6jxBCgUBQ5xFCKBAI6jz/D8GJmdE3Q+jiAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.02314, + "end_time": "2024-02-29T23:33:42.014235", + "exception": false, + "start_time": "2024-02-29T23:33:41.991095", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4189.757421, + "end_time": "2024-02-29T23:33:44.759053", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/2/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tvae/2/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/contraceptive/tvae/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "tvae" + }, + "start_time": "2024-02-29T22:23:55.001632", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tvae/model.pt b/contraceptive/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7257f0a06bb0dc3c6e5ea84000212836d46e57d --- /dev/null +++ b/contraceptive/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36d7ca6e137da75d51ef8ea5a79b9d92615b195432807d283d4d140daf9b0271 +size 41130645 diff --git a/contraceptive/tvae/params.json b/contraceptive/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..e6d7f4e22f14ab25b505b2384c6e1a63e74e1aef --- /dev/null +++ b/contraceptive/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.775, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "tvae", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": false, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "relu6", "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/lct_gan/eval.csv b/insurance/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..6d7f463600ea5aa04da5ed6fb069ab12de6639fb --- /dev/null +++ b/insurance/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.07935211913926261,0.1306050524190255,0.0007405445374794104,0.5596821308135986,0.03549230098724365,0.7786027193069458,0.054690830409526825,9.762088666320778e-07,0.8873205184936523,0.020644349977374077,0.3238646686077118,0.027212947607040405,0.15517421066761017,1.0350788215873763e-05,1.447002649307251 diff --git a/insurance/lct_gan/history.csv b/insurance/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..69017c60bbced244bdaf471a147e901dc0d9cf5f --- /dev/null +++ b/insurance/lct_gan/history.csv @@ -0,0 +1,27 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.07503844532329822,3.76196769104788,0.039352887886764186,0.0,0.0,0.0,0.0,0.0,0.07503844532329822,320,40,42.94257736206055,1.0735644340515136,0.1341955542564392,0.13469835626892745,0.008877724732155912,3.33818835544007,2.9330407841143823e-06,0.0,0.0,0.0,0.0,0.0,0.008877724732155912,80,10,9.165157794952393,0.9165157794952392,0.1145644724369049,0.037592777702957395 +1,0.02608004305366194,2.0130314562969813,0.005282479949307373,0.0,0.0,0.0,0.0,0.0,0.02608004305366194,320,40,42.331037521362305,1.0582759380340576,0.1322844922542572,0.10117223353590817,0.005078719957964495,0.2115427433644072,3.133896269957859e-05,0.0,0.0,0.0,0.0,0.0,0.005078719957964495,80,10,9.217864990234375,0.9217864990234375,0.11522331237792968,0.11006196644157171 +2,0.007246867158391979,3.0597032676599154,9.03313408463391e-05,0.0,0.0,0.0,0.0,0.0,0.007246867158391979,320,40,42.494264125823975,1.0623566031455993,0.13279457539319992,0.07669011817779392,0.0074968072643969205,2.55860044410183,0.00017504165297967944,0.0,0.0,0.0,0.0,0.0,0.0074968072643969205,80,10,9.166083335876465,0.9166083335876465,0.11457604169845581,0.09575922545045615 +3,0.006846483054687269,2.2157006146561513,0.0007793176604440372,0.0,0.0,0.0,0.0,0.0,0.006846483054687269,320,40,42.40406584739685,1.0601016461849213,0.13251270577311516,0.08761264618951828,0.0020610511739505453,0.43892243231239264,7.913704455120296e-06,0.0,0.0,0.0,0.0,0.0,0.0020610511739505453,80,10,9.511794328689575,0.9511794328689576,0.1188974291086197,0.08304880987852811 +4,0.002302334751948365,0.6305191363795544,1.4314678949647008e-05,0.0,0.0,0.0,0.0,0.0,0.002302334751948365,320,40,43.27270483970642,1.0818176209926604,0.13522720262408255,0.08970329142175615,0.004081522431806661,0.026551660033874214,2.7209868290256623e-05,0.0,0.0,0.0,0.0,0.0,0.004081522431806661,80,10,9.23831558227539,0.923831558227539,0.11547894477844238,0.11656681830063462 +5,0.0013704985673030023,0.32384693517769847,7.155363357548236e-06,0.0,0.0,0.0,0.0,0.0,0.0013704985673030023,320,40,42.453025341033936,1.0613256335258483,0.13266570419073104,0.09745097612030804,0.0033582281379494817,0.619899958840142,2.357043052825247e-06,0.0,0.0,0.0,0.0,0.0,0.0033582281379494817,80,10,9.264163732528687,0.9264163732528686,0.11580204665660858,0.06247350247576833 +6,0.0025572317323167225,1.3281221274127346,4.255108958927875e-06,0.0,0.0,0.0,0.0,0.0,0.0025572317323167225,320,40,42.2554829120636,1.05638707280159,0.13204838410019876,0.08552498414646834,0.001253202352381777,0.32618888739889373,2.2838767717248133e-06,0.0,0.0,0.0,0.0,0.0,0.001253202352381777,80,10,9.205610513687134,0.9205610513687134,0.11507013142108917,0.07330623050220311 +7,0.0017415513455489417,0.39128143024250334,1.98570894781383e-06,0.0,0.0,0.0,0.0,0.0,0.0017415513455489417,320,40,42.568729639053345,1.0642182409763337,0.1330272801220417,0.08582097220933065,0.002552773474599235,0.38444976235623474,9.21505393498695e-06,0.0,0.0,0.0,0.0,0.0,0.002552773474599235,80,10,9.204639673233032,0.9204639673233033,0.11505799591541291,0.09078566757962107 +8,0.001197526408395788,0.5185240543789404,6.638705742609621e-06,0.0,0.0,0.0,0.0,0.0,0.001197526408395788,320,40,42.46436643600464,1.061609160900116,0.1327011451125145,0.0921793600777164,0.0011125051009003074,0.1445327332803572,3.7028677351003125e-06,0.0,0.0,0.0,0.0,0.0,0.0011125051009003074,80,10,9.251813411712646,0.9251813411712646,0.11564766764640808,0.08213724349625409 +9,0.0011423767211454106,0.24242009912380066,5.1029623472642616e-06,0.0,0.0,0.0,0.0,0.0,0.0011423767211454106,320,40,42.30474233627319,1.0576185584068298,0.13220231980085373,0.086794763058424,0.0021649388814694247,1.9614427807347965,1.2043508045372908e-05,0.0,0.0,0.0,0.0,0.0,0.0021649388814694247,80,10,9.148065567016602,0.9148065567016601,0.11435081958770751,0.0991100890096277 +10,0.0008830112970827031,0.4022787155074184,5.028790277500362e-07,0.0,0.0,0.0,0.0,0.0,0.0008830112970827031,320,40,42.54185461997986,1.0635463654994965,0.13294329568743707,0.09175913570215925,0.0017155635854578578,1.2685116354904722,2.10015661070706e-06,0.0,0.0,0.0,0.0,0.0,0.0017155635854578578,80,10,9.253859281539917,0.9253859281539917,0.11567324101924896,0.0978053328813985 +11,0.001937569323945354,0.6675240401481404,3.123376906712105e-06,0.0,0.0,0.0,0.0,0.0,0.001937569323945354,320,40,42.27454662322998,1.0568636655807495,0.1321079581975937,0.09221202009357513,0.0027879032801138236,0.09599128968548029,1.215036173931594e-05,0.0,0.0,0.0,0.0,0.0,0.0027879032801138236,80,10,9.17811369895935,0.917811369895935,0.11472642123699188,0.11065587596967816 +12,0.0013319606783625203,0.16566794978843974,6.807524444540914e-07,0.0,0.0,0.0,0.0,0.0,0.0013319606783625203,320,40,42.627525091171265,1.0656881272792815,0.1332110159099102,0.10023473438341171,0.0013272355950903147,0.5792492911004956,6.966086060211652e-08,0.0,0.0,0.0,0.0,0.0,0.0013272355950903147,80,10,9.216788053512573,0.9216788053512573,0.11520985066890717,0.07118493653833866 +13,0.0007021169698418816,0.11813758928258485,2.759127278142287e-06,0.0,0.0,0.0,0.0,0.0,0.0007021169698418816,320,40,42.255826234817505,1.0563956558704377,0.1320494569838047,0.09224181645549834,0.0008028386626392602,0.06161341504857774,6.375153506842091e-07,0.0,0.0,0.0,0.0,0.0,0.0008028386626392602,80,10,9.26439380645752,0.9264393806457519,0.11580492258071899,0.0892744664568454 +14,0.0006613305880819098,0.18104081245551243,6.271647721827747e-07,0.0,0.0,0.0,0.0,0.0,0.0006613305880819098,320,40,42.603567361831665,1.0650891840457917,0.13313614800572396,0.09589193011634052,0.0005620345647912473,0.01514620759198806,4.4228354818542924e-07,0.0,0.0,0.0,0.0,0.0,0.0005620345647912473,80,10,9.197651863098145,0.9197651863098144,0.1149706482887268,0.09222434270195663 +15,0.0007842243476261501,0.3221512252403457,1.0985442627384213e-06,0.0,0.0,0.0,0.0,0.0,0.0007842243476261501,320,40,42.46329689025879,1.0615824222564698,0.13269780278205873,0.09709920620080084,0.001301504473667592,2.9538385085063057,4.959147403504893e-07,0.0,0.0,0.0,0.0,0.0,0.001301504473667592,80,10,9.24899411201477,0.924899411201477,0.11561242640018463,0.1033931726939045 +16,0.013570401900506113,0.36384937064023576,0.0018209778673778428,0.0,0.0,0.0,0.0,0.0,0.013570401900506113,320,40,42.59553337097168,1.064888334274292,0.1331110417842865,0.12845125668682159,0.16798840463161469,0.3928926819935441,0.06907801991328597,0.0,0.0,0.0,0.0,0.0,0.16798840463161469,80,10,9.341678619384766,0.9341678619384766,0.11677098274230957,0.28858067095279694 +17,0.22693241573870182,0.459286569285905,0.10300876491237432,0.0,0.0,0.0,0.0,0.0,0.22693241573870182,320,40,42.3380069732666,1.058450174331665,0.13230627179145812,0.32175534069538114,0.4309734970331192,0.6810152728110552,0.33519675582647324,0.0,0.0,0.0,0.0,0.0,0.4309734970331192,80,10,9.211932897567749,0.921193289756775,0.11514916121959687,0.35243880599737165 +18,0.2022466917289421,0.7070020012586611,0.11807100524520138,0.0,0.0,0.0,0.0,0.0,0.2022466917289421,320,40,42.41661763191223,1.0604154407978057,0.1325519300997257,0.34771558828651905,0.023606129095423967,0.21133080043364316,0.0015780384962681636,0.0,0.0,0.0,0.0,0.0,0.023606129095423967,80,10,9.170459747314453,0.9170459747314453,0.11463074684143067,0.16341671436093747 +19,0.005479642776481342,0.5328093900557633,8.112759967222604e-05,0.0,0.0,0.0,0.0,0.0,0.005479642776481342,320,40,42.17777228355408,1.0544443070888518,0.13180553838610648,0.1161046927794814,0.0007801399522577412,0.05935813882520051,5.190229413365443e-07,0.0,0.0,0.0,0.0,0.0,0.0007801399522577412,80,10,9.320383310317993,0.9320383310317993,0.11650479137897492,0.0829056172631681 +20,0.0016211183414270637,0.5920635455369594,2.3483921812016834e-06,0.0,0.0,0.0,0.0,0.0,0.0016211183414270637,320,40,42.64041256904602,1.0660103142261506,0.13325128927826882,0.08777563656913116,0.0008631729724584147,0.09347999086021445,1.679538957422011e-06,0.0,0.0,0.0,0.0,0.0,0.0008631729724584147,80,10,9.24825143814087,0.9248251438140869,0.11560314297676086,0.07839330667629837 +21,0.0005253879406154737,0.10636353976760801,2.6855408757735233e-07,0.0,0.0,0.0,0.0,0.0,0.0005253879406154737,320,40,42.83327293395996,1.070831823348999,0.13385397791862488,0.09573199808364734,0.0005467957133078016,0.23533951704739592,2.0549961012861218e-07,0.0,0.0,0.0,0.0,0.0,0.0005467957133078016,80,10,9.286863803863525,0.9286863803863525,0.11608579754829407,0.08401567195542156 +22,0.0006425162649065896,0.10582681066216537,6.702316121443182e-07,0.0,0.0,0.0,0.0,0.0,0.0006425162649065896,320,40,42.88295650482178,1.0720739126205445,0.13400923907756807,0.09285907801240682,0.003129586172872223,0.14476592368632737,2.8466294405604663e-05,0.0,0.0,0.0,0.0,0.0,0.003129586172872223,80,10,9.417258024215698,0.9417258024215698,0.11771572530269622,0.10630810875445604 +23,0.0012267698623873002,0.3422558290479515,5.092052370217958e-06,0.0,0.0,0.0,0.0,0.0,0.0012267698623873002,320,40,42.59943246841431,1.0649858117103577,0.1331232264637947,0.0916012367233634,0.001950129849865334,1.6070946810563327,1.2394382595015685e-05,0.0,0.0,0.0,0.0,0.0,0.001950129849865334,80,10,9.218074083328247,0.9218074083328247,0.11522592604160309,0.06952399904839694 +24,0.0015580077400954907,0.5951609104130398,1.101507371748138e-05,0.0,0.0,0.0,0.0,0.0,0.0015580077400954907,320,40,42.20673155784607,1.0551682889461518,0.13189603611826897,0.08924343169201165,0.0014282745076343417,1.1416928244575502,5.442704249958296e-07,0.0,0.0,0.0,0.0,0.0,0.0014282745076343417,80,10,9.161205768585205,0.9161205768585206,0.11451507210731507,0.08108144407160581 +25,0.001158999443759967,0.673237547548581,1.6942943679773905e-06,0.0,0.0,0.0,0.0,0.0,0.001158999443759967,320,40,42.81976580619812,1.070494145154953,0.13381176814436913,0.09180414919974282,0.003335691889151349,0.1234515317961268,9.410348545632954e-05,0.0,0.0,0.0,0.0,0.0,0.003335691889151349,80,10,9.541585206985474,0.9541585206985473,0.11926981508731842,0.10400733416900039 diff --git a/insurance/lct_gan/mlu-eval.ipynb b/insurance/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..9afd0fd616081c83f95b0463feddbfd2d15ee25f --- /dev/null +++ b/insurance/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2674 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:55.980748Z", + "iopub.status.busy": "2024-02-29T12:41:55.980389Z", + "iopub.status.idle": "2024-02-29T12:41:56.013514Z", + "shell.execute_reply": "2024-02-29T12:41:56.012639Z" + }, + "papermill": { + "duration": 0.048201, + "end_time": "2024-02-29T12:41:56.015613", + "exception": false, + "start_time": "2024-02-29T12:41:55.967412", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.040986Z", + "iopub.status.busy": "2024-02-29T12:41:56.040638Z", + "iopub.status.idle": "2024-02-29T12:41:56.047244Z", + "shell.execute_reply": "2024-02-29T12:41:56.046384Z" + }, + "papermill": { + "duration": 0.021542, + "end_time": "2024-02-29T12:41:56.049125", + "exception": false, + "start_time": "2024-02-29T12:41:56.027583", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.072553Z", + "iopub.status.busy": "2024-02-29T12:41:56.072280Z", + "iopub.status.idle": "2024-02-29T12:41:56.076151Z", + "shell.execute_reply": "2024-02-29T12:41:56.075362Z" + }, + "papermill": { + "duration": 0.017935, + "end_time": "2024-02-29T12:41:56.078076", + "exception": false, + "start_time": "2024-02-29T12:41:56.060141", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.101677Z", + "iopub.status.busy": "2024-02-29T12:41:56.101357Z", + "iopub.status.idle": "2024-02-29T12:41:56.105462Z", + "shell.execute_reply": "2024-02-29T12:41:56.104647Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018392, + "end_time": "2024-02-29T12:41:56.107632", + "exception": false, + "start_time": "2024-02-29T12:41:56.089240", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.132779Z", + "iopub.status.busy": "2024-02-29T12:41:56.132429Z", + "iopub.status.idle": "2024-02-29T12:41:56.138350Z", + "shell.execute_reply": "2024-02-29T12:41:56.137479Z" + }, + "papermill": { + "duration": 0.021063, + "end_time": "2024-02-29T12:41:56.140226", + "exception": false, + "start_time": "2024-02-29T12:41:56.119163", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2646260c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.167711Z", + "iopub.status.busy": "2024-02-29T12:41:56.166754Z", + "iopub.status.idle": "2024-02-29T12:41:56.172026Z", + "shell.execute_reply": "2024-02-29T12:41:56.171127Z" + }, + "papermill": { + "duration": 0.021124, + "end_time": "2024-02-29T12:41:56.174023", + "exception": false, + "start_time": "2024-02-29T12:41:56.152899", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"lct_gan\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/lct_gan/2\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.0118, + "end_time": "2024-02-29T12:41:56.197555", + "exception": false, + "start_time": "2024-02-29T12:41:56.185755", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.222174Z", + "iopub.status.busy": "2024-02-29T12:41:56.221850Z", + "iopub.status.idle": "2024-02-29T12:41:56.231486Z", + "shell.execute_reply": "2024-02-29T12:41:56.230651Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024384, + "end_time": "2024-02-29T12:41:56.233470", + "exception": false, + "start_time": "2024-02-29T12:41:56.209086", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/lct_gan/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:56.258505Z", + "iopub.status.busy": "2024-02-29T12:41:56.258188Z", + "iopub.status.idle": "2024-02-29T12:41:58.610430Z", + "shell.execute_reply": "2024-02-29T12:41:58.609479Z" + }, + "papermill": { + "duration": 2.367122, + "end_time": "2024-02-29T12:41:58.612552", + "exception": false, + "start_time": "2024-02-29T12:41:56.245430", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:58.640394Z", + "iopub.status.busy": "2024-02-29T12:41:58.639422Z", + "iopub.status.idle": "2024-02-29T12:41:58.651843Z", + "shell.execute_reply": "2024-02-29T12:41:58.651095Z" + }, + "papermill": { + "duration": 0.028052, + "end_time": "2024-02-29T12:41:58.653855", + "exception": false, + "start_time": "2024-02-29T12:41:58.625803", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:58.677919Z", + "iopub.status.busy": "2024-02-29T12:41:58.677663Z", + "iopub.status.idle": "2024-02-29T12:41:58.685003Z", + "shell.execute_reply": "2024-02-29T12:41:58.684140Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021603, + "end_time": "2024-02-29T12:41:58.686934", + "exception": false, + "start_time": "2024-02-29T12:41:58.665331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:58.711050Z", + "iopub.status.busy": "2024-02-29T12:41:58.710792Z", + "iopub.status.idle": "2024-02-29T12:41:58.813475Z", + "shell.execute_reply": "2024-02-29T12:41:58.812734Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.1174, + "end_time": "2024-02-29T12:41:58.815932", + "exception": false, + "start_time": "2024-02-29T12:41:58.698532", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:41:58.843222Z", + "iopub.status.busy": "2024-02-29T12:41:58.842456Z", + "iopub.status.idle": "2024-02-29T12:42:03.450984Z", + "shell.execute_reply": "2024-02-29T12:42:03.450183Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.624356, + "end_time": "2024-02-29T12:42:03.453351", + "exception": false, + "start_time": "2024-02-29T12:41:58.828995", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 12:42:01.052754: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 12:42:01.052810: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 12:42:01.054457: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:03.478386Z", + "iopub.status.busy": "2024-02-29T12:42:03.477808Z", + "iopub.status.idle": "2024-02-29T12:42:03.484201Z", + "shell.execute_reply": "2024-02-29T12:42:03.483544Z" + }, + "papermill": { + "duration": 0.020941, + "end_time": "2024-02-29T12:42:03.486155", + "exception": false, + "start_time": "2024-02-29T12:42:03.465214", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:03.512422Z", + "iopub.status.busy": "2024-02-29T12:42:03.512135Z", + "iopub.status.idle": "2024-02-29T12:42:11.955354Z", + "shell.execute_reply": "2024-02-29T12:42:11.954221Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.459386, + "end_time": "2024-02-29T12:42:11.958044", + "exception": false, + "start_time": "2024-02-29T12:42:03.498658", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.77,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.RReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:12.465760Z", + "iopub.status.busy": "2024-02-29T12:42:12.465462Z", + "iopub.status.idle": "2024-02-29T12:42:12.537486Z", + "shell.execute_reply": "2024-02-29T12:42:12.536552Z" + }, + "papermill": { + "duration": 0.08776, + "end_time": "2024-02-29T12:42:12.539858", + "exception": false, + "start_time": "2024-02-29T12:42:12.452098", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T12:42:12.570494Z", + "iopub.status.busy": "2024-02-29T12:42:12.570173Z", + "iopub.status.idle": "2024-02-29T12:42:13.013319Z", + "shell.execute_reply": "2024-02-29T12:42:13.012260Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.460597, + "end_time": "2024-02-29T12:42:13.015412", + "exception": false, + "start_time": "2024-02-29T12:42:12.554815", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:13.070814Z", + "iopub.status.busy": "2024-02-29T12:42:13.070443Z", + "iopub.status.idle": "2024-02-29T12:42:13.074413Z", + "shell.execute_reply": "2024-02-29T12:42:13.073661Z" + }, + "papermill": { + "duration": 0.035244, + "end_time": "2024-02-29T12:42:13.076511", + "exception": false, + "start_time": "2024-02-29T12:42:13.041267", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:13.131950Z", + "iopub.status.busy": "2024-02-29T12:42:13.131563Z", + "iopub.status.idle": "2024-02-29T12:42:13.140556Z", + "shell.execute_reply": "2024-02-29T12:42:13.139807Z" + }, + "papermill": { + "duration": 0.045465, + "end_time": "2024-02-29T12:42:13.142649", + "exception": false, + "start_time": "2024-02-29T12:42:13.097184", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9631361" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:13.187726Z", + "iopub.status.busy": "2024-02-29T12:42:13.187391Z", + "iopub.status.idle": "2024-02-29T12:42:13.278191Z", + "shell.execute_reply": "2024-02-29T12:42:13.277246Z" + }, + "papermill": { + "duration": 0.108973, + "end_time": "2024-02-29T12:42:13.280766", + "exception": false, + "start_time": "2024-02-29T12:42:13.171793", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 29] --\n", + "├─Adapter: 1-1 [2, 1071, 29] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 30,720\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 29] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─RReLU: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,631,361\n", + "Trainable params: 9,631,361\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.31\n", + "Forward/backward pass size (MB): 307.47\n", + "Params size (MB): 38.53\n", + "Estimated Total Size (MB): 346.31\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T12:42:13.312205Z", + "iopub.status.busy": "2024-02-29T12:42:13.311907Z", + "iopub.status.idle": "2024-02-29T13:06:34.619283Z", + "shell.execute_reply": "2024-02-29T13:06:34.618279Z" + }, + "papermill": { + "duration": 1461.343801, + "end_time": "2024-02-29T13:06:34.639603", + "exception": false, + "start_time": "2024-02-29T12:42:13.295802", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.07503844532329822, 'avg_role_model_std_loss': 3.76196769104788, 'avg_role_model_mean_pred_loss': 0.039352887886764186, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.07503844532329822, 'n_size': 320, 'n_batch': 40, 'duration': 42.94257736206055, 'duration_batch': 1.0735644340515136, 'duration_size': 0.1341955542564392, 'avg_pred_std': 0.13469835626892745}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008877724732155912, 'avg_role_model_std_loss': 3.33818835544007, 'avg_role_model_mean_pred_loss': 2.9330407841143823e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008877724732155912, 'n_size': 80, 'n_batch': 10, 'duration': 9.165157794952393, 'duration_batch': 0.9165157794952392, 'duration_size': 0.1145644724369049, 'avg_pred_std': 0.037592777702957395}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02608004305366194, 'avg_role_model_std_loss': 2.0130314562969813, 'avg_role_model_mean_pred_loss': 0.005282479949307373, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02608004305366194, 'n_size': 320, 'n_batch': 40, 'duration': 42.331037521362305, 'duration_batch': 1.0582759380340576, 'duration_size': 0.1322844922542572, 'avg_pred_std': 0.10117223353590817}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005078719957964495, 'avg_role_model_std_loss': 0.2115427433644072, 'avg_role_model_mean_pred_loss': 3.133896269957859e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005078719957964495, 'n_size': 80, 'n_batch': 10, 'duration': 9.217864990234375, 'duration_batch': 0.9217864990234375, 'duration_size': 0.11522331237792968, 'avg_pred_std': 0.11006196644157171}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007246867158391979, 'avg_role_model_std_loss': 3.0597032676599154, 'avg_role_model_mean_pred_loss': 9.03313408463391e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007246867158391979, 'n_size': 320, 'n_batch': 40, 'duration': 42.494264125823975, 'duration_batch': 1.0623566031455993, 'duration_size': 0.13279457539319992, 'avg_pred_std': 0.07669011817779392}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0074968072643969205, 'avg_role_model_std_loss': 2.55860044410183, 'avg_role_model_mean_pred_loss': 0.00017504165297967944, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0074968072643969205, 'n_size': 80, 'n_batch': 10, 'duration': 9.166083335876465, 'duration_batch': 0.9166083335876465, 'duration_size': 0.11457604169845581, 'avg_pred_std': 0.09575922545045615}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006846483054687269, 'avg_role_model_std_loss': 2.2157006146561513, 'avg_role_model_mean_pred_loss': 0.0007793176604440372, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006846483054687269, 'n_size': 320, 'n_batch': 40, 'duration': 42.40406584739685, 'duration_batch': 1.0601016461849213, 'duration_size': 0.13251270577311516, 'avg_pred_std': 0.08761264618951828}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020610511739505453, 'avg_role_model_std_loss': 0.43892243231239264, 'avg_role_model_mean_pred_loss': 7.913704455120296e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020610511739505453, 'n_size': 80, 'n_batch': 10, 'duration': 9.511794328689575, 'duration_batch': 0.9511794328689576, 'duration_size': 0.1188974291086197, 'avg_pred_std': 0.08304880987852811}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002302334751948365, 'avg_role_model_std_loss': 0.6305191363795544, 'avg_role_model_mean_pred_loss': 1.4314678949647008e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002302334751948365, 'n_size': 320, 'n_batch': 40, 'duration': 43.27270483970642, 'duration_batch': 1.0818176209926604, 'duration_size': 0.13522720262408255, 'avg_pred_std': 0.08970329142175615}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004081522431806661, 'avg_role_model_std_loss': 0.026551660033874214, 'avg_role_model_mean_pred_loss': 2.7209868290256623e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004081522431806661, 'n_size': 80, 'n_batch': 10, 'duration': 9.23831558227539, 'duration_batch': 0.923831558227539, 'duration_size': 0.11547894477844238, 'avg_pred_std': 0.11656681830063462}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013704985673030023, 'avg_role_model_std_loss': 0.32384693517769847, 'avg_role_model_mean_pred_loss': 7.155363357548236e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013704985673030023, 'n_size': 320, 'n_batch': 40, 'duration': 42.453025341033936, 'duration_batch': 1.0613256335258483, 'duration_size': 0.13266570419073104, 'avg_pred_std': 0.09745097612030804}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0033582281379494817, 'avg_role_model_std_loss': 0.619899958840142, 'avg_role_model_mean_pred_loss': 2.357043052825247e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033582281379494817, 'n_size': 80, 'n_batch': 10, 'duration': 9.264163732528687, 'duration_batch': 0.9264163732528686, 'duration_size': 0.11580204665660858, 'avg_pred_std': 0.06247350247576833}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0025572317323167225, 'avg_role_model_std_loss': 1.3281221274127346, 'avg_role_model_mean_pred_loss': 4.255108958927875e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025572317323167225, 'n_size': 320, 'n_batch': 40, 'duration': 42.2554829120636, 'duration_batch': 1.05638707280159, 'duration_size': 0.13204838410019876, 'avg_pred_std': 0.08552498414646834}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001253202352381777, 'avg_role_model_std_loss': 0.32618888739889373, 'avg_role_model_mean_pred_loss': 2.2838767717248133e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001253202352381777, 'n_size': 80, 'n_batch': 10, 'duration': 9.205610513687134, 'duration_batch': 0.9205610513687134, 'duration_size': 0.11507013142108917, 'avg_pred_std': 0.07330623050220311}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017415513455489417, 'avg_role_model_std_loss': 0.39128143024250334, 'avg_role_model_mean_pred_loss': 1.98570894781383e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017415513455489417, 'n_size': 320, 'n_batch': 40, 'duration': 42.568729639053345, 'duration_batch': 1.0642182409763337, 'duration_size': 0.1330272801220417, 'avg_pred_std': 0.08582097220933065}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002552773474599235, 'avg_role_model_std_loss': 0.38444976235623474, 'avg_role_model_mean_pred_loss': 9.21505393498695e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002552773474599235, 'n_size': 80, 'n_batch': 10, 'duration': 9.204639673233032, 'duration_batch': 0.9204639673233033, 'duration_size': 0.11505799591541291, 'avg_pred_std': 0.09078566757962107}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001197526408395788, 'avg_role_model_std_loss': 0.5185240543789404, 'avg_role_model_mean_pred_loss': 6.638705742609621e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001197526408395788, 'n_size': 320, 'n_batch': 40, 'duration': 42.46436643600464, 'duration_batch': 1.061609160900116, 'duration_size': 0.1327011451125145, 'avg_pred_std': 0.0921793600777164}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011125051009003074, 'avg_role_model_std_loss': 0.1445327332803572, 'avg_role_model_mean_pred_loss': 3.7028677351003125e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011125051009003074, 'n_size': 80, 'n_batch': 10, 'duration': 9.251813411712646, 'duration_batch': 0.9251813411712646, 'duration_size': 0.11564766764640808, 'avg_pred_std': 0.08213724349625409}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011423767211454106, 'avg_role_model_std_loss': 0.24242009912380066, 'avg_role_model_mean_pred_loss': 5.1029623472642616e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011423767211454106, 'n_size': 320, 'n_batch': 40, 'duration': 42.30474233627319, 'duration_batch': 1.0576185584068298, 'duration_size': 0.13220231980085373, 'avg_pred_std': 0.086794763058424}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021649388814694247, 'avg_role_model_std_loss': 1.9614427807347965, 'avg_role_model_mean_pred_loss': 1.2043508045372908e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021649388814694247, 'n_size': 80, 'n_batch': 10, 'duration': 9.148065567016602, 'duration_batch': 0.9148065567016601, 'duration_size': 0.11435081958770751, 'avg_pred_std': 0.0991100890096277}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008830112970827031, 'avg_role_model_std_loss': 0.4022787155074184, 'avg_role_model_mean_pred_loss': 5.028790277500362e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008830112970827031, 'n_size': 320, 'n_batch': 40, 'duration': 42.54185461997986, 'duration_batch': 1.0635463654994965, 'duration_size': 0.13294329568743707, 'avg_pred_std': 0.09175913570215925}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017155635854578578, 'avg_role_model_std_loss': 1.2685116354904722, 'avg_role_model_mean_pred_loss': 2.10015661070706e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017155635854578578, 'n_size': 80, 'n_batch': 10, 'duration': 9.253859281539917, 'duration_batch': 0.9253859281539917, 'duration_size': 0.11567324101924896, 'avg_pred_std': 0.0978053328813985}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001937569323945354, 'avg_role_model_std_loss': 0.6675240401481404, 'avg_role_model_mean_pred_loss': 3.123376906712105e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001937569323945354, 'n_size': 320, 'n_batch': 40, 'duration': 42.27454662322998, 'duration_batch': 1.0568636655807495, 'duration_size': 0.1321079581975937, 'avg_pred_std': 0.09221202009357513}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027879032801138236, 'avg_role_model_std_loss': 0.09599128968548029, 'avg_role_model_mean_pred_loss': 1.215036173931594e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027879032801138236, 'n_size': 80, 'n_batch': 10, 'duration': 9.17811369895935, 'duration_batch': 0.917811369895935, 'duration_size': 0.11472642123699188, 'avg_pred_std': 0.11065587596967816}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013319606783625203, 'avg_role_model_std_loss': 0.16566794978843974, 'avg_role_model_mean_pred_loss': 6.807524444540914e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013319606783625203, 'n_size': 320, 'n_batch': 40, 'duration': 42.627525091171265, 'duration_batch': 1.0656881272792815, 'duration_size': 0.1332110159099102, 'avg_pred_std': 0.10023473438341171}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0013272355950903147, 'avg_role_model_std_loss': 0.5792492911004956, 'avg_role_model_mean_pred_loss': 6.966086060211652e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013272355950903147, 'n_size': 80, 'n_batch': 10, 'duration': 9.216788053512573, 'duration_batch': 0.9216788053512573, 'duration_size': 0.11520985066890717, 'avg_pred_std': 0.07118493653833866}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007021169698418816, 'avg_role_model_std_loss': 0.11813758928258485, 'avg_role_model_mean_pred_loss': 2.759127278142287e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007021169698418816, 'n_size': 320, 'n_batch': 40, 'duration': 42.255826234817505, 'duration_batch': 1.0563956558704377, 'duration_size': 0.1320494569838047, 'avg_pred_std': 0.09224181645549834}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008028386626392602, 'avg_role_model_std_loss': 0.06161341504857774, 'avg_role_model_mean_pred_loss': 6.375153506842091e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008028386626392602, 'n_size': 80, 'n_batch': 10, 'duration': 9.26439380645752, 'duration_batch': 0.9264393806457519, 'duration_size': 0.11580492258071899, 'avg_pred_std': 0.0892744664568454}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006613305880819098, 'avg_role_model_std_loss': 0.18104081245551243, 'avg_role_model_mean_pred_loss': 6.271647721827747e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006613305880819098, 'n_size': 320, 'n_batch': 40, 'duration': 42.603567361831665, 'duration_batch': 1.0650891840457917, 'duration_size': 0.13313614800572396, 'avg_pred_std': 0.09589193011634052}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0005620345647912473, 'avg_role_model_std_loss': 0.01514620759198806, 'avg_role_model_mean_pred_loss': 4.4228354818542924e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005620345647912473, 'n_size': 80, 'n_batch': 10, 'duration': 9.197651863098145, 'duration_batch': 0.9197651863098144, 'duration_size': 0.1149706482887268, 'avg_pred_std': 0.09222434270195663}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007842243476261501, 'avg_role_model_std_loss': 0.3221512252403457, 'avg_role_model_mean_pred_loss': 1.0985442627384213e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007842243476261501, 'n_size': 320, 'n_batch': 40, 'duration': 42.46329689025879, 'duration_batch': 1.0615824222564698, 'duration_size': 0.13269780278205873, 'avg_pred_std': 0.09709920620080084}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001301504473667592, 'avg_role_model_std_loss': 2.9538385085063057, 'avg_role_model_mean_pred_loss': 4.959147403504893e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001301504473667592, 'n_size': 80, 'n_batch': 10, 'duration': 9.24899411201477, 'duration_batch': 0.924899411201477, 'duration_size': 0.11561242640018463, 'avg_pred_std': 0.1033931726939045}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013570401900506113, 'avg_role_model_std_loss': 0.36384937064023576, 'avg_role_model_mean_pred_loss': 0.0018209778673778428, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013570401900506113, 'n_size': 320, 'n_batch': 40, 'duration': 42.59553337097168, 'duration_batch': 1.064888334274292, 'duration_size': 0.1331110417842865, 'avg_pred_std': 0.12845125668682159}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.16798840463161469, 'avg_role_model_std_loss': 0.3928926819935441, 'avg_role_model_mean_pred_loss': 0.06907801991328597, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.16798840463161469, 'n_size': 80, 'n_batch': 10, 'duration': 9.341678619384766, 'duration_batch': 0.9341678619384766, 'duration_size': 0.11677098274230957, 'avg_pred_std': 0.28858067095279694}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.22693241573870182, 'avg_role_model_std_loss': 0.459286569285905, 'avg_role_model_mean_pred_loss': 0.10300876491237432, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.22693241573870182, 'n_size': 320, 'n_batch': 40, 'duration': 42.3380069732666, 'duration_batch': 1.058450174331665, 'duration_size': 0.13230627179145812, 'avg_pred_std': 0.32175534069538114}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.4309734970331192, 'avg_role_model_std_loss': 0.6810152728110552, 'avg_role_model_mean_pred_loss': 0.33519675582647324, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.4309734970331192, 'n_size': 80, 'n_batch': 10, 'duration': 9.211932897567749, 'duration_batch': 0.921193289756775, 'duration_size': 0.11514916121959687, 'avg_pred_std': 0.35243880599737165}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.2022466917289421, 'avg_role_model_std_loss': 0.7070020012586611, 'avg_role_model_mean_pred_loss': 0.11807100524520138, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.2022466917289421, 'n_size': 320, 'n_batch': 40, 'duration': 42.41661763191223, 'duration_batch': 1.0604154407978057, 'duration_size': 0.1325519300997257, 'avg_pred_std': 0.34771558828651905}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.023606129095423967, 'avg_role_model_std_loss': 0.21133080043364316, 'avg_role_model_mean_pred_loss': 0.0015780384962681636, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.023606129095423967, 'n_size': 80, 'n_batch': 10, 'duration': 9.170459747314453, 'duration_batch': 0.9170459747314453, 'duration_size': 0.11463074684143067, 'avg_pred_std': 0.16341671436093747}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005479642776481342, 'avg_role_model_std_loss': 0.5328093900557633, 'avg_role_model_mean_pred_loss': 8.112759967222604e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005479642776481342, 'n_size': 320, 'n_batch': 40, 'duration': 42.17777228355408, 'duration_batch': 1.0544443070888518, 'duration_size': 0.13180553838610648, 'avg_pred_std': 0.1161046927794814}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007801399522577412, 'avg_role_model_std_loss': 0.05935813882520051, 'avg_role_model_mean_pred_loss': 5.190229413365443e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007801399522577412, 'n_size': 80, 'n_batch': 10, 'duration': 9.320383310317993, 'duration_batch': 0.9320383310317993, 'duration_size': 0.11650479137897492, 'avg_pred_std': 0.0829056172631681}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016211183414270637, 'avg_role_model_std_loss': 0.5920635455369594, 'avg_role_model_mean_pred_loss': 2.3483921812016834e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016211183414270637, 'n_size': 320, 'n_batch': 40, 'duration': 42.64041256904602, 'duration_batch': 1.0660103142261506, 'duration_size': 0.13325128927826882, 'avg_pred_std': 0.08777563656913116}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008631729724584147, 'avg_role_model_std_loss': 0.09347999086021445, 'avg_role_model_mean_pred_loss': 1.679538957422011e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008631729724584147, 'n_size': 80, 'n_batch': 10, 'duration': 9.24825143814087, 'duration_batch': 0.9248251438140869, 'duration_size': 0.11560314297676086, 'avg_pred_std': 0.07839330667629837}\n", + "Epoch 21\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005253879406154737, 'avg_role_model_std_loss': 0.10636353976760801, 'avg_role_model_mean_pred_loss': 2.6855408757735233e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005253879406154737, 'n_size': 320, 'n_batch': 40, 'duration': 42.83327293395996, 'duration_batch': 1.070831823348999, 'duration_size': 0.13385397791862488, 'avg_pred_std': 0.09573199808364734}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0005467957133078016, 'avg_role_model_std_loss': 0.23533951704739592, 'avg_role_model_mean_pred_loss': 2.0549961012861218e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005467957133078016, 'n_size': 80, 'n_batch': 10, 'duration': 9.286863803863525, 'duration_batch': 0.9286863803863525, 'duration_size': 0.11608579754829407, 'avg_pred_std': 0.08401567195542156}\n", + "Epoch 22\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006425162649065896, 'avg_role_model_std_loss': 0.10582681066216537, 'avg_role_model_mean_pred_loss': 6.702316121443182e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006425162649065896, 'n_size': 320, 'n_batch': 40, 'duration': 42.88295650482178, 'duration_batch': 1.0720739126205445, 'duration_size': 0.13400923907756807, 'avg_pred_std': 0.09285907801240682}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003129586172872223, 'avg_role_model_std_loss': 0.14476592368632737, 'avg_role_model_mean_pred_loss': 2.8466294405604663e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003129586172872223, 'n_size': 80, 'n_batch': 10, 'duration': 9.417258024215698, 'duration_batch': 0.9417258024215698, 'duration_size': 0.11771572530269622, 'avg_pred_std': 0.10630810875445604}\n", + "Epoch 23\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012267698623873002, 'avg_role_model_std_loss': 0.3422558290479515, 'avg_role_model_mean_pred_loss': 5.092052370217958e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012267698623873002, 'n_size': 320, 'n_batch': 40, 'duration': 42.59943246841431, 'duration_batch': 1.0649858117103577, 'duration_size': 0.1331232264637947, 'avg_pred_std': 0.0916012367233634}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001950129849865334, 'avg_role_model_std_loss': 1.6070946810563327, 'avg_role_model_mean_pred_loss': 1.2394382595015685e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001950129849865334, 'n_size': 80, 'n_batch': 10, 'duration': 9.218074083328247, 'duration_batch': 0.9218074083328247, 'duration_size': 0.11522592604160309, 'avg_pred_std': 0.06952399904839694}\n", + "Epoch 24\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015580077400954907, 'avg_role_model_std_loss': 0.5951609104130398, 'avg_role_model_mean_pred_loss': 1.101507371748138e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015580077400954907, 'n_size': 320, 'n_batch': 40, 'duration': 42.20673155784607, 'duration_batch': 1.0551682889461518, 'duration_size': 0.13189603611826897, 'avg_pred_std': 0.08924343169201165}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0014282745076343417, 'avg_role_model_std_loss': 1.1416928244575502, 'avg_role_model_mean_pred_loss': 5.442704249958296e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014282745076343417, 'n_size': 80, 'n_batch': 10, 'duration': 9.161205768585205, 'duration_batch': 0.9161205768585206, 'duration_size': 0.11451507210731507, 'avg_pred_std': 0.08108144407160581}\n", + "Epoch 25\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001158999443759967, 'avg_role_model_std_loss': 0.673237547548581, 'avg_role_model_mean_pred_loss': 1.6942943679773905e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001158999443759967, 'n_size': 320, 'n_batch': 40, 'duration': 42.81976580619812, 'duration_batch': 1.070494145154953, 'duration_size': 0.13381176814436913, 'avg_pred_std': 0.09180414919974282}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003335691889151349, 'avg_role_model_std_loss': 0.1234515317961268, 'avg_role_model_mean_pred_loss': 9.410348545632954e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003335691889151349, 'n_size': 80, 'n_batch': 10, 'duration': 9.541585206985474, 'duration_batch': 0.9541585206985473, 'duration_size': 0.11926981508731842, 'avg_pred_std': 0.10400733416900039}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▄█▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train ▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▇▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▃▂▂▃▂▂▂▂▂▂▃▂▂▂▂▇█▄▂▂▂▃▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▂▂▁▁▁▂▁▁▁▁▁▁▂▁▁▂▂▇█▂▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▄█▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train ▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▇▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂█▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train ▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▇█▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▁▆▂▁▂▂▂▁▅▄▁▂▁▁▇▂▂▁▁▁▁▁▄▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▅▇▅▂▁▃▂▂▁▂▂▁▁▁▁▁▂▂▂▂▁▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▁▂▁▇▃▃▂▂▃▁▃▂▂▃▂▃▄▂▁▄▃▃▆▂▁█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▆▂▃▂█▃▁▃▃▂▃▂▄▁▄▃▄▂▃▁▄▅▆▄▁▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▁▂▁▇▃▃▂▂▃▁▃▂▂▃▂▃▄▂▁▄▃▃▆▂▁█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▆▂▃▂█▃▁▃▃▂▃▂▄▁▄▃▄▂▃▁▄▅▆▄▁▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▁▂▁▇▃▃▂▂▃▁▃▂▂▃▂▃▄▂▁▄▃▃▆▂▁█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▆▂▃▂█▃▁▃▃▂▃▂▄▁▄▃▄▂▃▁▄▅▆▄▁▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00334\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00116\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.10401\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.0918\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00334\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00116\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 9e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.12345\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.67324\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.95416\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.07049\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.11927\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.13381\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 9.54159\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 42.81977\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/lct_gan/2/wandb/offline-run-20240229_124214-gelsvnyc\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_124214-gelsvnyc/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.0007405610326770926, 'avg_g_mag_loss': 0.1445552625342968, 'avg_g_cos_loss': 0.08170349127554655, 'pred_duration': 0.9051024913787842, 'grad_duration': 0.5714690685272217, 'total_duration': 1.4765715599060059, 'pred_std': 0.15517398715019226, 'std_loss': 1.0358485269534867e-05, 'mean_pred_loss': 9.762113677425077e-07, 'pred_rmse': 0.02721325121819973, 'pred_mae': 0.02064499258995056, 'pred_mape': 0.3238719403743744, 'grad_rmse': 0.05469190701842308, 'grad_mae': 0.03549487516283989, 'grad_mape': 0.7790535688400269}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0007405610326770926, 'avg_g_mag_loss': 0.1445552625342968, 'avg_g_cos_loss': 0.08170349127554655, 'avg_pred_duration': 0.9051024913787842, 'avg_grad_duration': 0.5714690685272217, 'avg_total_duration': 1.4765715599060059, 'avg_pred_std': 0.15517398715019226, 'avg_std_loss': 1.0358485269534867e-05, 'avg_mean_pred_loss': 9.762113677425077e-07}, 'min_metrics': {'avg_loss': 0.0007405610326770926, 'avg_g_mag_loss': 0.1445552625342968, 'avg_g_cos_loss': 0.08170349127554655, 'pred_duration': 0.9051024913787842, 'grad_duration': 0.5714690685272217, 'total_duration': 1.4765715599060059, 'pred_std': 0.15517398715019226, 'std_loss': 1.0358485269534867e-05, 'mean_pred_loss': 9.762113677425077e-07, 'pred_rmse': 0.02721325121819973, 'pred_mae': 0.02064499258995056, 'pred_mape': 0.3238719403743744, 'grad_rmse': 0.05469190701842308, 'grad_mae': 0.03549487516283989, 'grad_mape': 0.7790535688400269}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0007405610326770926, 'avg_g_mag_loss': 0.1445552625342968, 'avg_g_cos_loss': 0.08170349127554655, 'pred_duration': 0.9051024913787842, 'grad_duration': 0.5714690685272217, 'total_duration': 1.4765715599060059, 'pred_std': 0.15517398715019226, 'std_loss': 1.0358485269534867e-05, 'mean_pred_loss': 9.762113677425077e-07, 'pred_rmse': 0.02721325121819973, 'pred_mae': 0.02064499258995056, 'pred_mape': 0.3238719403743744, 'grad_rmse': 0.05469190701842308, 'grad_mae': 0.03549487516283989, 'grad_mape': 0.7790535688400269}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:06:34.680448Z", + "iopub.status.busy": "2024-02-29T13:06:34.680130Z", + "iopub.status.idle": "2024-02-29T13:06:34.684279Z", + "shell.execute_reply": "2024-02-29T13:06:34.683537Z" + }, + "papermill": { + "duration": 0.027317, + "end_time": "2024-02-29T13:06:34.686263", + "exception": false, + "start_time": "2024-02-29T13:06:34.658946", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:06:34.724114Z", + "iopub.status.busy": "2024-02-29T13:06:34.723566Z", + "iopub.status.idle": "2024-02-29T13:06:34.983440Z", + "shell.execute_reply": "2024-02-29T13:06:34.982646Z" + }, + "papermill": { + "duration": 0.281322, + "end_time": "2024-02-29T13:06:34.985810", + "exception": false, + "start_time": "2024-02-29T13:06:34.704488", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:06:35.028483Z", + "iopub.status.busy": "2024-02-29T13:06:35.027626Z", + "iopub.status.idle": "2024-02-29T13:06:35.304518Z", + "shell.execute_reply": "2024-02-29T13:06:35.303633Z" + }, + "papermill": { + "duration": 0.300942, + "end_time": "2024-02-29T13:06:35.306789", + "exception": false, + "start_time": "2024-02-29T13:06:35.005847", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAESCAYAAAAv/mqQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1lklEQVR4nO2de3xU5bX3v3smc8nkTgIJwWBQoYAIURAKVq01lWOVFkpb9OUI5rzCaUt6qqmtpWrQ9m2Dl2OpyivHetC3VgrHHtEe9YOXSPSoEZSLUIEcL0CAkJCA5DJJ5vq8f0xmMkNmJjPJJAMz6/v57M9M9n727DU7M79Zz3rWsx5NKaUQBEEYBnTxNkAQhORBBEcQhGFDBEcQhGFDBEcQhGFDBEcQhGFDBEcQhGFDBEcQhGEjJd4GRILb7aahoYGMjAw0TYu3OYIg+KGUor29ncLCQnS68D7MOSE4DQ0NFBUVxdsMQRDCcOTIEc4777ywbc4JwcnIyAA8bygzMzPO1giC4E9bWxtFRUW+72k4zgnB8XajMjMzRXAE4SwlknCHBI0FQRg2RHAEQRg2RHAEQRg2zokYjjBwXC4XDocj3mYI5zhGo7HfIe9IEMFJUJRSNDY2cvr06XibIiQAOp2OcePGYTQaB/U6IjgJildsRo0ahcVikYRJYcB4E2+PHz/O2LFjB/VZEsFJQFwul09scnNz423OuYOrp+upN8TXjrOQkSNH0tDQgNPpxGAY+P2RoHEC4o3ZWCyWOFtyDqEUNNdB8wFQ7nhbc9bh7Uq5XK5BvY54OAmMdKOiwO0Ct6P3uV5+i/2J1WdJ7qogACin33PxcIYKERxBAI9X40UEZ8gQwREECBAczWjhxRdfjJ8tQ8B9991HSUlJvM0QwREEANzO/tsMI8888wzZ2dkxe70777yT6urqmL3eQJGgsSAAqMGNvsQLu90eUTJeeno66enpw2BReMTDSRKUUnTanXHZol3cdcuWLXzta18jOzub3NxcbrzxRj7//HMA5syZw1133RXQvrm5GYPBwDvvvAPA8ePHueGGG0hNTWXcuHFs2LCB4uJi1qxZE/qi7tCCs3fvXr7xjW+QmppKbm4uy5cvp6Ojw3e8pqaGmTNnkpaWRnZ2NldccQWHDx8G4OOPP+aaa64hIyODzMxMpk+fzkcffRT2/dfU1FBWVkZrayuapqFpGvfddx8AxcXF/OY3v2HJkiVkZmayfPlyAO666y4mTJiAxWLhggsu4N577w2Y0nJml+rWW29l/vz5PPzww4wePZrc3FxWrFgx5NNgxMNJErocLiZXvhaXa+/79Vwsxsg/alarlYqKCqZOnUpHRweVlZUsWLCA3bt3s3jxYh588EFWr17tG6rdtGkThYWFXHnllQAsWbKElpYWampqMBgMVFRUcOLEifAXDSE4VquVuXPnMnv2bD788ENOnDjBbbfdRnl5Oc888wxOp5P58+ezbNky/vKXv2C329m+fbvPtsWLF3PppZfyxBNPoNfr2b17d7+Jc3PmzGHNmjVUVlZSV1cHEOCdPPzww1RWVrJq1SrfvoyMDJ555hkKCwvZu3cvy5YtIyMjg1/84hchr7N161ZGjx7N1q1b+eyzz1i0aBElJSUsW7Ys/L0aBCI4wlnHwoULA/5ev349I0eOZN++ffzgBz/g9ttv59133/UJzIYNG7j55pvRNI0DBw7w5ptv8uGHHzJjxgwAnnrqKcaPHx/+oiG6VBs2bKC7u5s//elPpKWlAfD4448zb948HnjgAQwGA62trdx4441ceOGFAEyaNMl3fn19PT//+c+ZOHEiQP924Emyy8rKQtM0CgoK+hz/xje+wc9+9rOAfffcc4/veXFxMXfeeScbN24MKzg5OTk8/vjj6PV6Jk6cyA033EB1dbUIjjB4Ug169v16btyuHQ2ffvoplZWVbNu2jZaWFtxuzzB1fX09U6ZM4brrruO5557jyiuv5ODBg9TW1vJv//ZvANTV1ZGSksJll13me72LLrqInJyc8BcNETTev38/06ZN84kNwBVXXIHb7aauro6rrrqKW2+9lblz5/LNb36T0tJSfvCDHzB69GgAKioquO2223j22WcpLS3l+9//vk+YBopXSP3ZtGkTjz76KJ9//jkdHR04nc5+q2NefPHF6PW9/5vRo0ezd+/eQdnWHxLDSRI0TcNiTInLFm2W6rx58zh16hR//OMf2bZtG9u2bQM8AVLwdFP++te/4nA42LBhA5dccgmXXHLJ4G5QmBhOfzz99NPU1tYyZ84cNm3axIQJE/jggw8AT+zkk08+4YYbbuCtt95i8uTJbN68eVCm+osfQG1tLYsXL+Zb3/oWL7/8Mrt27eLuu+/23a9QnNm10zTNJ+5DxYAEZ+3atRQXF2M2m5k1axbbt2+P6LyNGzeiaRrz588fyGWFJODkyZPU1dVxzz33cO211zJp0iS+/PLLgDbf+c536O7uZsuWLWzYsIHFixf7jn3lK1/B6XSya9cu377PPvusz2v0IYTgTJo0iY8//hir1erb995776HT6fjKV77i23fppZeycuVK3n//faZMmcKGDRt8xyZMmMAdd9zB66+/zne/+12efvrpfu+D0WiMeN7S+++/z/nnn8/dd9/NjBkzGD9+vC9ofbYRteBs2rSJiooKVq1axc6dO5k2bRpz587tNyh36NAh7rzzTl+/WxCCkZOTQ25uLk8++SSfffYZb731FhUVFQFt0tLSmD9/Pvfeey/79+/n5ptv9h2bOHEipaWlLF++nO3bt7Nr1y6WL19OampqeE8rRAxn8eLFmM1mli5dyt///ne2bt3KT37yE2655Rby8/M5ePAgK1eupLa2lsOHD/P666/z6aefMmnSJLq6uigvL6empobDhw/z3nvv8eGHHwbEeEJRXFxMR0cH1dXVtLS00NnZGbLt+PHjqa+vZ+PGjXz++ec8+uijg/aihoqoBeeRRx5h2bJllJWVMXnyZNatW4fFYmH9+vUhz3G5XCxevJj777+fCy64YFAGC4mNTqdj48aN7NixgylTpnDHHXfw0EMP9Wm3ePFiPv74Y6688krGjh0bcOxPf/oT+fn5XHXVVSxYsMA3YmM2m0NfOISHY7FYeO211zh16hSXX3453/ve97j22mt5/PHHfccPHDjAwoULmTBhAsuXL2fFihX88z//M3q9npMnT7JkyRImTJjAD37wA66//nruv//+fu/DnDlz+OEPf8iiRYsYOXIkDz74YMi23/72t7njjjsoLy+npKSE999/n3vvvbffa8QDTUWRJGG327FYLPz1r38N6BYtXbqU06dP89JLLwU9b9WqVezZs4fNmzdz6623cvr06bCp4zabDZvN5vvbu+5Na2urLBMTAd3d3Rw8eJBx48aF/5IlCUePHqWoqIg333yTa6+9tm8DpeD47t6/TVmQKz+M/oT7TLW1tZGVlRXR9zOqUaqWlhZcLhf5+fkB+/Pz8zlw4EDQc959913+/d//nd27d0d8naqqqoh+BQQhGG+99RYdHR1ccsklHD9+nF/84hcUFxdz1VVXBT/hzO6UTN4cMoZ0lKq9vZ1bbrmFP/7xj+Tl5UV83sqVK2ltbfVtR44cGUIrhUTD4XDwq1/9iosvvpgFCxYwcuRIXxLgc88950vz922Z2aSPv4KLr/me5wWGQXCuv/76vnb0bL/73e+G/PrxIioPJy8vD71eT1NTU8D+pqamoAlKn3/+OYcOHWLevHm+fd5ht5SUFOrq6oLmJJhMJkwmUzSmCYKPuXPnMndu8Jyjb3/728yaNStwp70LvvwCg6Hn6zAMgvPUU0/R1dUV9NiIESOG/PrxIirBMRqNTJ8+nerqal8Mx+12U11dTXl5eZ/2EydO7JNIdM8999De3s4f/vAHioqKBm65IAyAjIyMvmtgd7fBqeEtwDVmzJghv8bZSNSZxhUVFSxdupQZM2Ywc+ZM1qxZg9VqpaysDPDMYxkzZgxVVVWYzWamTJkScL53yv2Z+wUhbnhjOJrOIzYSwxkyohacRYsW0dzcTGVlJY2NjZSUlLBlyxZfILm+vj4mC2YJwrDhHRLXG8BpE8EZQqIaFo8X0Qy7CTIsHjXtTdDeAMYMsLd79o0uASlC7yNWw+LiigiC8vNwenfGxZRERwRHELwzxf0FZ4gnMSYrIjiC4I3h6FIADW3MZbz44tk5F+lcRwRHEHyCo/eMVIFnukMciXURdfCULtU0jdOnT8f0daNBBEcQlJ+Hc5YITqIigpMsKAV2a3y2s72IujeGo/l5OPTGcM6mIuo2m40777yTMWPGkJaWxqxZs6ipqfGde/jwYebNm0dOTg5paWlcfPHFvPrqqxw6dIhrrrkG8JQA0TSNW2+9NawdQ4GUGE0WHJ3wu8L4XPtXDWBM679dD8NeRD1Ml+psK6JeXl7Ovn372LhxI4WFhWzevJl/+Id/YO/evYwfP54VK1Zgt9t55513SEtLY9++faSnp1NUVMR//ud/snDhQurq6sjMzCQ1NTXi/0msEMERzjqGtYi6Un5dqr6CczYVUa+vr+fpp5+mvr6ewkLPj8edd97Jli1bePrpp/nd735HfX09Cxcu9JVc9a8/5Z2jNWrUqJjHhyJFBCdZMFg8nka8rh0Fw1pE3T+rWEsBXWCX6mwqor53715cLhcTJkwI2G+z2cjNzQXgX/7lX/jRj37E66+/TmlpKQsXLmTq1KkDut5QIDGcZEHTPN2aeGxncxF132oNmkdsfB5O5Hk4w1VEvaOjA71ez44dO9i9e7dv279/P3/4wx8AuO222/jiiy+45ZZb2Lt3LzNmzOCxxx4b0PWGBHUO0NraqgDV2toab1POCbq6utS+fftUV1dXvE2JmpaWFgWod955x7fvv//7vxWgNm/erJRSqqOjQ6Wlpam//e1vavLkyWr16tW+tvv371eA+uijj3z7Pv30UwWo3//+930vaLMqdWynUsf3eP4+ddBzrQ1PK6WUevLJJ1VOTo7q6OjwnfLKK68onU6nGhsbg76Hr371q+onP/lJ0GM33XSTmjdvXr/34bnnnlPp6ekB++rq6vrcm/745S9/qS655BKllFLvvfeeAlRLS0vE53sJ95mK5vspHo5wVjHsRdT94zfQZ5TqbCqiPmHCBBYvXsySJUt44YUXOHjwINu3b6eqqopXXnkFgNtvv53XXnuNgwcPsnPnTrZu3eq73vnnn4+mabz88ss0NzcHjLQNG1FLXRwQDyc6zmUPRyml3njjDTVp0iRlMpnU1KlTVU1NTYCHo5RSr776qgLUVVdd1ef8hoYGdf311yuTyaTOP/98tWHDBjVq1Ci1bt26vhfr/NLj4Zyo8/x9+qjnWn9+ytdkz5496pprrlFms1mNGDFCLVu2TLW3tyullGpsbFTz589Xo0ePVkajUZ1//vmqsrJSuVwuZbPZ1E033aSKioqU0WhUhYWFqry8POL/yw9/+EOVm5urALVq1SqllFJ2u11VVlaq4uJiZTAY1OjRo9WCBQvUnj0eD628vFxdeOGFymQyqZEjR6pbbrklwKP59a9/rQoKCpSmaWrp0qUR2aFU7DwcmS2egMhs8UDCFlG3tkDrETBlQu6F0HYcOhrBkgfZUiDOS1yKqAvCuUBURdT7dKl6ul1SE2dIkBiOkHBEVUS94ALSx19BetHFXHzxxZ5sYxhywZEi6oKQIERVRL3tOHSdAstIDCPGDGhYfCBIEXVBSAL6FFE/lQLd6ZA5BtJHQVfPGuRDLDjJWkRdulQJzDkwHhB/VE/i35nD4hLDCSBWnyURnATEOzmws7MzzpacA3gnbmoiOOHwZnnr9fpBvY50qRIQvV5Pdna2b4a0xWIJnvQmgM0BbgUOF2jdYHeAU3mmPHR3x9u6swK3201zczMWi4WUlMFJhghOguKdZRy2LIMArY2eofE2PeiN4HJAe7PH42mXr4cXnU7H2LFjB/3DJXc0QdE0jdGjRzNq1CgcDke8zTk7UQr+7/8C5YClL0NGAbQegy0/A70JfvRevC08azAajTFZb04EJ8HR6/WD7ncnLHYrtH/heZ6ZByYzODOh44hnn9HoV65CiAVyN4Xkpeu051GX0luR0OhXu8chQfdYI4IjJC/dpz2P5qzeKQ0pfmU3RXBijgiOkLx4PRxzdu8+na63QqHdOtwWJTwiOELy4vVwUrMD93sFRzycmCOCIyQvwTwc6I3j2EVwYo0IjpC8hPRwegLIDulSxRoRHCF56W71PIqHM2yI4AjJi7dLJTGcYUMER0hefMPi2YH7RXCGDBEcIXnxBY2zAvdLl2rIEMERkhcJGg87IjhC8iLD4sOOCI6QvEji37AjgiMkLyE9nJ4ulUxtiDkDEpy1a9dSXFyM2Wxm1qxZbN++PWTbF154gRkzZpCdnU1aWholJSU8++yzAzZYEGKCoxtcNs9z8XCGjagFZ9OmTVRUVLBq1Sp27tzJtGnTmDt3bsjKciNGjODuu++mtraWPXv2UFZWRllZGa+99tqgjReEAePtTmk6MGYEHpMYzpARteA88sgjLFu2jLKyMiZPnsy6deuwWCysX78+aPuvf/3rLFiwgEmTJnHhhRfy05/+lKlTp/Luu+8O2nhBGDDe7pQps2+RLRmlGjKiEhy73c6OHTsoLS3tfQGdjtLSUmpra/s9XylFdXU1dXV1wZdd7cFms9HW1hawCUJMCRUwBjD01MRxBF+oThg4UQlOS0sLLpeL/Pz8gP35+fk0NjaGPK+1tZX09HSMRiM33HADjz32GN/85jdDtq+qqiIrK8u3FRXJovJCjAkVMAbpUg0hwzJKlZGRwe7du/nwww/57W9/S0VFBTU1NSHbr1y5ktbWVt925MiR4TBTSCbCejjSpRoqoiqinpeXh16vp6mpKWB/U1OTb1mSYOh0Oi666CIASkpK2L9/P1VVVXz9618P2t5kMmEymaIxTRCiQzycuBCVh2M0Gpk+fTrV1dW+fW63m+rqambPnh3x67jdbmw2WzSXFoTY4i1NEdbDEcGJNVEvE1NRUcHSpUuZMWMGM2fOZM2aNVitVsrKygBYsmQJY8aMoaqqCvDEY2bMmMGFF16IzWbj1Vdf5dlnn+WJJ56I7TsRhGgINVMc/Dwcq2ftKlm1NGZELTiLFi2iubmZyspKGhsbKSkpYcuWLb5Acn19fcCCWVarlR//+MccPXqU1NRUJk6cyJ///GcWLVoUu3chCNESqhYO9Cb+KRe47JAi3ftYoSmlVLyN6I+2tjaysrJobW0lMzMz3uYIicBfboa6V+HG38OMfwo85nLAb/I8z39xECwjht++c4hovp8yl0pITsIFjfUG0Bk8zyWOE1NEcITkJNywOPjNp5Lkv1gigiMkJ+E8HAgMHAsxQwRHSE4i9nCkSxVLRHCE5MNp7xWSfj0cEZxYIoIjJB/epD/oW0Ddi0xvGBJEcITkw9udMmWCTh+8jXg4Q4IIjpB89BcwBr8Yjng4sUQER0g+fAHjEN0p8KtrLB5OLBHBEZKPqDwcEZxYIoIjJB/9DYmDCM4QIYIjJB+ReDgSNB4SRHCE5EM8nLghgiMkH+Fq4XiRxfCGBBEcIfnwdanCjFKJhzMkiOAIyYevvGhO6DYSwxkSRHCE5COiYXGZ2jAUiOAIyUckQWPxcIYEERwh+YjKwxHBiSUiOEJy4XKCvd3zPOyweM9yvzJKFVNEcITkwua3Tn24USqjlBgdCkRwhOSi60vPoyHNUyw9FN4ulbML3O6htytJEMERkotIAsbQ6+GAxHFiiAiOkFxEEjAGSEntfS6CEzNEcITkIlIPR6frzTaWwHHMEMERkotIPRyQ6Q1DgAiOkFxE6uGAJP8NASI4QnLhnUcVxMNZ/+5BHq3+FKWUZ4fUNY45KfE2QBCGFW+X6gwPp8vu4jev7EMpyLEYuGV2sV8MRzycWCEejpBc+GrhBCb9tXTY8Do2/+eV/Xx2or23Jo7EcGKGCI6QXIQIGp+y2n3PbU43t2/ajds7NC6CEzNEcITkIkTQ2Cs4Y7JTybYY+PuxNv7ny54MY+lSxQwRHCG5COHhnOwRnAtGplG14BIA9pxweA5K0DhmiOAIyUUID+fLHsEZkWbk+ktG873p59GpTADYujqG0cDERgRHSB7cbujumS0ewsMZkWYEYNW8yaSYPUHjDw4cGTYTEx0RHCF5sLUBPUNRZ4xSnbLaAMjtEZwMs4FrLikG4NiJFl7Zc3y4rExoRHCE5MHbnUoxg8EccOiUz8Mx+faNGZkLgEWz8avNe2ls7R4WMxMZERwheQgzj6q3S+VXI6cn8a8g1U1rl4M7n/8Yt1sNsZGJzYAEZ+3atRQXF2M2m5k1axbbt28P2faPf/wjV155JTk5OeTk5FBaWhq2vSAMGWHmUX0ZxMPxJv5NzTdiNuh497MWnnn/0NDamOBELTibNm2ioqKCVatWsXPnTqZNm8bcuXM5ceJE0PY1NTXcfPPNbN26ldraWoqKirjuuus4duzYoI0XhKiIyMMx9u7s8XAs2LjnhskArN5ygE+b2ofSyoQmasF55JFHWLZsGWVlZUyePJl169ZhsVhYv3590PbPPfccP/7xjykpKWHixIk89dRTuN1uqqurQ17DZrPR1tYWsAnCoAnh4didbtq7nUBv0BgImC2+eNZYrhyfh93pZvMu+bEcKFEJjt1uZ8eOHZSWlva+gE5HaWkptbW1Eb1GZ2cnDoeDESNGhGxTVVVFVlaWbysqKorGTEEITggP58tOj3ej0yAr1T+G07sYnqZpfPUCTxC5ud02xIYmLlEJTktLCy6Xi/z8/ID9+fn5NDY2RvQad911F4WFhQGidSYrV66ktbXVtx05InkQQgzwLfGbHbD7ZIdHcHIsRnQ6rffAGfVw8tI93k9LhwjOQBnW8hSrV69m48aN1NTUYDabQ7YzmUyYTKaQxwVhQPhmimcH7PZ6OAHxG+izGF5euucz2dJhRxgYUQlOXl4eer2epqamgP1NTU0UFBSEPffhhx9m9erVvPnmm0ydOjV6SwVhsPi6VIFJf0EDxuDn4VhBKXJ7BOekeDgDJqouldFoZPr06QEBX28AePbs2SHPe/DBB/nNb37Dli1bmDFjxsCtFYTBEGqmeI+A5Kaf6eH0lKdQLnDZ/bpU9t6qgEJURN2lqqioYOnSpcyYMYOZM2eyZs0arFYrZWVlACxZsoQxY8ZQVVUFwAMPPEBlZSUbNmyguLjYF+tJT08nPT09hm9FEPqhn1o4IbtUAHYreekez8juctNuc5JpDrOQnhCUqAVn0aJFNDc3U1lZSWNjIyUlJWzZssUXSK6vr0en63WcnnjiCex2O9/73vcCXmfVqlXcd999g7NeEKIhlIfjjeFYzhCcFCPoUsDtBEcXZssI0k0pdNictLTbRHAGwICCxuXl5ZSXlwc9VlNTE/D3oUOHBnIJQYg90Xo44PFybK2+wHFuupEOm5OTVjsXjBxCWxMUmUslJAdK9TssPiI9yMioMXAxPN9IleTiDAgRHCE5sHd4gr8Q0sPJDerhBC6G523TYpWh8YEggiMkB97ulM7QO/rUQ9gu1ZnJfxni4QwGERwhOfAPGGu92cRutwqd+AcB0xsA8nranLSK4AwEERwhOQgRMG7tcuAtcZNz5igVhPFwpEs1EERwhOQgxJC4N8s4w5yCMSXI1+GM5X57pzeIhzMQRHCE5KCfIfGgAWPos9xvrq9LJR7OQBDBEZKDkAvgeTyVoPEb6O1SOboACRoPFhEcITmwtngeLbkBu09ZPYvdhRScPkFjj+C025x0O1yxtzPBEcERkoP2nmVeMkYH7I7Yw+npUmWmpmDQe0a5pFsVPSI4QnLQ1uB5PENwTgYrnu7PGYl/mqaRmyZlKgaKCI6QHLT3VKTMPNPD6Sdo3LNyg3dqA0BehlT+GygiOEJyELJLFSbpD/p4OIDPw5HKf9EjgiMkPrZ2z1wqiF5wzojhgOTiDAYRHCHxaevxbkyZYAos+ha5h+PXpeqp/HdSPJyoEcEREp92b8A4sO62Uip0PWMvBvFwYokIjpD4eAPGZ3SnrHYXdqcbCFLP2IsxcOUG6A0ai4cTPSI4QuLjHRLPLAzYfapHMMwGHRZjiOKXYYPG4uFEiwiOkPj4PJzALlXIWsb+SNA4pojgCImPL4ZzhofjzTIO1Z2C3qkNzi5we7pf3qDxKasdl1uWi4kGERwh8fGOUp3h4fhqGYfKMoZeDwd83SpvgNmtelftFCJDBEdIfHxZxmd6OP1kGQOk+JUj7RGcFL2OHItniRgJHEeHCI6Q2Ljd0BF8lKrfHBwAnc5vaNw/F0fiOANBBEdIbKzNnoXs0CB9VMChiAQHeouu+49Upct8qoEggiMkNt45VOmjQB+4UmbkguOdwBlspEq6VNEggiMkNiEmbQL9Zxl7MQab3iAlKgaCCI6Q2IQRnIiCxuCX/Nfl25UnXaoBIYIjJDbeIfHM0ILTv4cTpCaOz8ORLlU0iOAIiY0v6S9QcGxOFx02JxBJDCfI9AYZpRoQIjhCYhNi4uaXPcXT9TqNTLPhzLMCCTq9wdulEg8nGkRwhMQmRJfKu1RvjsWITqedeVYgZ6zcAIF5OErJ9IZIEcEREpt+Sov2GzCGoB6ONw/H5nT7umZC/4jgCImLoxu6TnmeDyTL2EuQxD+LMQWLUQ9I4DgaRHCExMXr3ehNkJoTcCg6wek7SgUyvWEgiOAIiYv/0jBaYJwmKsEx9h2lAv/pDeLhRIoIjpC4hKiDA1FkGUPQxD8QD2cgiOAIiUuIOjjQW140ZC1jf4Ik/oGs3jAQBiQ4a9eupbi4GLPZzKxZs9i+fXvItp988gkLFy6kuLgYTdNYs2bNQG0VhOjwxnAy+3o40cVwgnepxMOJnqgFZ9OmTVRUVLBq1Sp27tzJtGnTmDt3LidOnAjavrOzkwsuuIDVq1dTUND3l0YQhoxw86giqWfsJciwOPhNb7CK4ERK1ILzyCOPsGzZMsrKypg8eTLr1q3DYrGwfv36oO0vv/xyHnroIW666SZMpjClHAUh1oQong5+Hk4kXaogiX/gFzRuly5VpEQlOHa7nR07dlBaWtr7AjodpaWl1NbWxswom81GW1tbwCYIURNieRiXW/lqEUc1ShXCw5EuVeREJTgtLS24XC7y8/MD9ufn59PY2Bgzo6qqqsjKyvJtRUVFMXttIUlQyq9LFejhnO60452NkBNJlypkDEdKVETLWTlKtXLlSlpbW33bkSNH4m2ScK7RfRqc3Z7nIbKMM80pGPQRfAX8axr7zZvyejht3U5sTtegTU4GQiw3GJy8vDz0ej1NTU0B+5uammIaEDaZTBLvEQaHd0g8Nad3akIPvnlU6RF+xrxdKuUClx1SPOdlmg2k6DScbsUpq53RWalhXkSAKD0co9HI9OnTqa6u9u1zu91UV1cze/bsmBsnCAMmgkp/EcVvoDdoDAHdKp1Ok8BxlETl4QBUVFSwdOlSZsyYwcyZM1mzZg1Wq5WysjIAlixZwpgxY6iqqgI8geZ9+/b5nh87dozdu3eTnp7ORRddFMO3Igh+xKKWsZcUI+hSPKs/2DsD5mXlpploarPRIkPjERG14CxatIjm5mYqKytpbGykpKSELVu2+ALJ9fX16HS9jlNDQwOXXnqp7++HH36Yhx9+mKuvvpqamprBvwNBCEZbDGoZ+2NIA1tr38BxhgmOQ0u7CE4kRC04AOXl5ZSXlwc9dqaIFBcXS4EiYfhpD154C3oFJycawTFaPIJz5vSGntfwek1CeM7KUSpBGDSxWK3Bn1BD4xk9uTji4USECI6QmHiT/mIRNIYwyX/i4USDCI6QmPjXwjmDqIPG4OfhnDG9IU2yjaNBBEdIPFxOsPZMJg5SC+dUz4iSVywiwhDCw+npUjVLlyoiEkZwlFI8t+0wNz1ZK782yY71BCg3aHpIyws4pJTyCxr3szyMP8YQEzglaBwVCSM4mqax6cMjfPDFKV7Zczze5gjxxL/wlk4fcKjD5sTh8oyaDsjDOaPq38geD+eU1Y7bLaOx/ZEwggPwnZIxALy4+1icLRHiiq+0aOiyFKkGPalGfZ/jIQkRNPbGgVxuxekuR/S2JhkJJTjzpo5Gp8Gu+tPUn+zs/wQhMQmx2iYMMGAMIWviGPQ6si2erpl05fsnoQRnVKaZORd6+uwviZeTvISogwNR1jL2J4SHA71xHBGc/kkowQH4TonnQ/bi7mOS4ZysRFDpL6I6OP6ESPwD/0JcEjjuj4QTnLlTCjCm6Pi82conDVIpMCkJszyMt5ZxVFnGEHLlBvCrbSweTr8knOBkmg2UThoFSLcqaQm3PMyAYzh9l/v1IpX/IifhBAd6R6v+9nEDLhmqTD58WcZBFsDriKJ4uj8hEv/A38ORLlV/JKTgfP0rI8k0p9DUZmPbwZPxNkcYTuxWz6xuCDGPyptlPMAulaNvlypXiqlHTEIKjilFz7cu8XzY/ra7Ic7WCMOKtztlSANTRp/Dgw4aB/VwPK/VLB5OvySk4AB8u2e06tW9x6XAdTLhXwdH0/oc9gWNox4W93o4XX0O5UrQOGISVnC+Oi6Xgkwzbd1Oth5ojrc5wnARpg4O9ObhjIhmWgOEnC0OMNKvSyWpGOFJWMHR6TSfl/O3j2W0KmkIIzjdDhdWu8fbjXqUKlziX4+31O1w02kXbzocCSs4AN+e5hGcN/efoK1b5rkkBW39lxZN0WlkmqOsruud2uDsArc74FCaKYVUg2delgSOw5PQgnNxYSYXjUrH7nSz5e+xWxlUOIvxJf2Fr2WsBYnvhMXr4UDQXBzfcjESOA5LQguOpml8p8fLkdGqJCHMxM0B1TL2kuK3yF3Y6Q3i4YQjoQUHepMA3/+8hRNt3XG2RhhyfF2qYJX+BphlDKDT9YpO0OkNPYW4xMMJS8ILzthcC5eNzcat4L+kMFdio5Rf0LjvtIYBl6bwYoxkAqd4OOFIeMGBXi9H5lYlOJ0nwd0zOJAebB7VALOMvXgDx2GnN4jghCMpBOeGqaPR6zT2HG3li+aOeJsjDBXeOjiWPM/yvGcwoAXw/Anj4UjQODKSQnDy0k1cOd5bmEuCxwlLmKVhYJBBY4ioJk6zeDhhSQrBgd7CXC9JYa7EJUwdHPAPGkeZZewlTE2cXF/QWAQnHEkjONdNLsBs0HHoZCd7jrbG2xxhKAhTBwdiEDQO4+GMlKp/EZE0gpNmSuGbkz0fxL/uOBpna4QhoT30kDj4daminbjpJez0Bo/gtHY5sDvdfY4LHpJGcAAWXuYZrfrztsMyYpWIhJlH5XS5Od3pGcGKujSFlzATOLNTDeh1nuzlU7IoXkiSSnCunjCSJbPPRyn42X98TE3diXibJMSSMILjv2ZUjiWKFTf9Sc3xPH76hmc5YT90Os3XVZNcnNAkleBomsZ98y5m3rRCnG7Fj/68kx2Hv4y3WUKsiGDiZrbFQIp+gB/7Gf8ExnSor4Wtv+1zWJL/+iepBAc8v0T/+v1pXDVhJF0OF//0zIf8T1N7vM0SBovTBp0tnufBFsDrGGTAGCD3Qvj2o57n7z4C//NawGGZ3tA/SSc4AMYUHev+8TIuHZtNa5eDW/59G0dOyUqd5zQdTZ5HvREsuX0ODzoHx8uUhXD5Ms/zF5bD6XrfIfFw+icpBQfAYkzh6VsvZ0J+Ok1tNpas3y4flHMZ/yHxIKUnmts9E3cHHDD2Z+5vofBS6D4Nz5eB0yNmPg9HgsYhSVrBAci2GPnTP81iTHYqB1usLF2/nXYp1HVuEiZg/MLOozz4Wh0AxXlpg79Wigm+/wyYs+DYR/BGJeC3ekO7/HCFIrEE57M3Yd/fen/tIqAgy8yfb5tFbpqRTxrauO3/fUS3Q8pEnnMEERyrzcnP/uNjKv7jYzrtLmZfkMuPrr4wNtfLKYb56zzPtz0B+16S6Q0REGWdxbOc9x6Fg297nmeeB0WXw3k9W8FUMJj7nuPoZpw6ygvXtrLxtbcZfeQ47/7eQHvRNTSMmI1mSsWo12HwbRrGFB2TR3uqCUZdOU4YGrwTN3uS/vY1tFH+l5180WxFp8HtpRNYcc1FvlyZmDDxWzDnX+D9R+Glcs775guABI3DMSDBWbt2LQ899BCNjY1MmzaNxx57jJkzZ4Zs//zzz3Pvvfdy6NAhxo8fzwMPPMC3vvWtARsdkoJLoPMUnPgE2o7CJ0fhk82eYzoDjJ4KY6Z7RjROfQGnDkLbMUBxPnCXhueOdAJ1r9KljLztnsbrrulUuy+jlfSAy43KMDHnwlzmXJTHFRflMSY7FSFO9EzcVOkF/Ln2EL95ZT92p5uCTDN/uKmEWRf0DSTHhGsr4eiHUF/LtNqfYuLnHPmykxd2HuWysTmcn2uRHyU/NBXlTMZNmzaxZMkS1q1bx6xZs1izZg3PP/88dXV1jBo1qk/7999/n6uuuoqqqipuvPFGNmzYwAMPPMDOnTuZMmVKRNdsa2sjKyuL1tZWMjMz+z/B1gENu+Dodjj6kecDYQ2zVIwxA0aMgxEX0JRSSH1TC+O/fIdse28dZBc6PjVP5aPUK3hPfzkHGtvJczVTqLVQqJ2iUGvhItNpig1fkuc6QYqz07MetTENzWDxZKkaLZ59hjTPRMDM0ZA1FrKLIKsIe/oYWpWF1i4HrV123AosRj1pxhQsJs9jKjZ03V96hNXe4YkjWPI8IzP6KH8/XA7PvCCdwROX0On7ae8EW5snWNrd2ru5HGAZ4bEjbaTHliDlIYaUZ26EQ//N+vy7+fXhiwG4duIoHvr+tMENhUdCWwOs+xp0nuQvrmtZ6fjfvkO5aUYuHZvNpWNzuGxsDtOKsrAYB/A773Z5RsROfg4nP/N8ns2ZnmREczakZvs9z/F8viIVOqU8n6e2Y5734ntsgIIpMHtF+LcfxfczasGZNWsWl19+OY8//jgAbreboqIifvKTn/DLX/6yT/tFixZhtVp5+eWXffu++tWvUlJSwrp16yK6ZtSCcyZKwenDHvFp2OX5Z4y4wLPljIO0vL7/HKWgcQ8ceMWzNf09+usOgDZl4ZjK46jKw4aBHDrI0TrI1trJoYNULbS73q7LoEOXjTUlG6shhy5DDpqmYXa1k+rqwOyyYnZ1YHZ3YHZ1YHQHllx1o8OlM+DSjLh1Rlw6A26dAZ3LjtHZjskdeepAh5ZOmy6bjpQsOlOycaSkofQm0Js84pZiRksxoRnM6AxmtBQj6PQorWfTpaA0PWh6lE4Pmg7NaUPn7ELv7ETv6kLn7ETv7ELv7GJswyuYHG3cZL+HHdrF3PUPE/nfXxs3fN7F52/Bs98FFO8WLGFPRxbHWztRPSs8aCg0FCkajMwwYjaZMBiMmEwmjEYjJqMJs9mE2Wgi1Wwk1dZCattBzG0HsbQfJLWjHp078gENF3rsulScmgGHZsSuGXBgwK4ZseN5rldORrhPkutuwaiCf64aRn6NwhWvhL1WNN/PqKTWbrezY8cOVq5c6dun0+koLS2ltrY26Dm1tbVUVFQE7Js7dy4vvvhiyOvYbDZstt7AW1tbWzRm9kXTPEG+nGK45HuRnzN6mme75leeLtiBVz3iU18LegNkjoGs8yCrCFvaaD63ZbOjNY2aRhN1X4LebcOCjVRspGr+z+2k00mhdooxWgtjtGbGaC2M0DrI1DrJ1OqZRH1I0xxKz2nS6VBmMrVOcuhApyky3O1kuNvBeQQGUL5Zhxud24YBG4SJm3coM21YaFcW2rDgUCnkaO3kaW3k0E6K5iZddZDu6gDXURimGKpbaTgyi/nPf5zD1POyh+eiXi78Blx9F7y9mq81/omvAeh7tjPp6tmixKYMHFQFHFQFnFDZpGtdZGMlS7OSTQdZmpUsOjBqLvS4SHVHV2yuWWXSqEZwXOVyXI2gUeVi1k3hp9GbGpKoBKelpQWXy0V+fn7A/vz8fA4cOBD0nMbGxqDtGxtDL9tSVVXF/fffH41pQ8+IC2BOuWdzdHsSzHS9g3wmYHLPdkvPPpvThdXmoqPbSbvNQUe3kw6bZ+u0uzzrGaUacKcaaLcY0OltZHQ3om87Cq31ni6MZQSkjkBZRmAzZGPVZ2JVqVgdLjrtTo473NjsDlTnSehsQes8ib6rBX3XKQzdJ3ErsKWk061Px5aSgU3f+7xbl4ZNZ0a5nWhOO7jtnkeXHc1lR3Pb0Zw2NGMqKZZsDJZsTOk5pFvMZJoNZJhTyEw1YDbosTldHHO4+cLuwNn5JaqjGay99ihHJ8rRDc5ulNOO5uxGc9l6Njt6t90jeMqFDjd65UKHC51yex5x49AM2LVUbJoZm86MQ2fGpnke7Toz9pFTePq788k0D3Cu1GC5+heebu2xnYDW6zVrOtA8Pk6Xw017txOX04HLacfldOB2OVAuJ8rl8HRP3U5Oa1kc04/huL6QhpTzOJ5yHid1eaDTo9M0dDoNU4oOs0GP2aDvea7DrNeRrreTiZVUzY5JefwZIw5M2DHgxKDsGHGApqPDlE+7cRTtKXnYSMHuUjhcbhxONwaXm7EjY5BG4MdZOUq1cuXKAK+ora2NoqKiOFp0BsFGu4JgStFjStFHEUNIA0ZAweQ+RzTA3LMFD38Gr3IXH0YCE+JtxPCj08NVPw95WAMsPVskzIiFTWcZUQlOXl4eer2epqamgP1NTU0UFAQvelRQUBBVewCTyYTJNMCqbIIgnLVElfhnNBqZPn061dXVvn1ut5vq6mpmz54d9JzZs2cHtAd44403QrYXBCFxibpLVVFRwdKlS5kxYwYzZ85kzZo1WK1WysrKAFiyZAljxoyhqqoKgJ/+9KdcffXV/Ou//is33HADGzdu5KOPPuLJJ5+M7TsRBOGsJ2rBWbRoEc3NzVRWVtLY2EhJSQlbtmzxBYbr6+vR+QVT58yZw4YNG7jnnnv41a9+xfjx43nxxRcjzsERBCFxiDoPJx4MOg9HEIQhI5rvZ2JN3hQE4axGBEcQhGHjrMzDORNvr2/QGceCIMQc7/cykujMOSE47e2emsNnVfKfIAgBtLe3k5WVFbbNORE0drvdNDQ0kJGREXYynjcj+ciRIxJcjhFyT2NLIt5PpRTt7e0UFhYGjFAH45zwcHQ6Heedd17E7TMzMxPmn3m2IPc0tiTa/ezPs/EiQWNBEIYNERxBEIaNhBIck8nEqlWrZOJnDJF7GluS/X6eE0FjQRASg4TycARBOLsRwREEYdgQwREEYdgQwREEYdgQwREEYdhIKMFZu3YtxcXFmM1mZs2axfbt2+Nt0jnDO++8w7x58ygsLETTtD7L+CilqKysZPTo0aSmplJaWsqnn34aH2PPAaqqqrj88svJyMhg1KhRzJ8/n7q6uoA23d3drFixgtzcXNLT01m4cGGf+t+JRsIIzqZNm6ioqGDVqlXs3LmTadOmMXfuXE6cOBFv084JrFYr06ZNY+3atUGPP/jggzz66KOsW7eObdu2kZaWxty5c+nuHsACWEnA22+/zYoVK/jggw944403cDgcXHfddVitVl+bO+64g//6r//i+eef5+2336ahoYHvfve7cbR6GFAJwsyZM9WKFSt8f7tcLlVYWKiqqqriaNW5CaA2b97s+9vtdquCggL10EMP+fadPn1amUwm9Ze//CUOFp57nDhxQgHq7bffVkp57p/BYFDPP/+8r83+/fsVoGpra+Nl5pCTEB6Od0XQ0tJS377+VgQVIufgwYM0NjYG3N+srCxmzZol9zdCWltbARgxYgQAO3bswOFwBNzTiRMnMnbs2IS+pwkhOOFWBA23wqcQGd57KPd3YLjdbm6//XauuOIK3+IBjY2NGI1GsrOzA9om+j09J8pTCMK5zIoVK/j73//Ou+++G29T4k5CeDgDWRFUiBzvPZT7Gz3l5eW8/PLLbN26NaCmU0FBAXa7ndOnTwe0T/R7mhCCM5AVQYXIGTduHAUFBQH3t62tjW3btsn9DYFSivLycjZv3sxbb73FuHHjAo5Pnz4dg8EQcE/r6uqor69P7Hsa76h1rNi4caMymUzqmWeeUfv27VPLly9X2dnZqrGxMd6mnRO0t7erXbt2qV27dilAPfLII2rXrl3q8OHDSimlVq9erbKzs9VLL72k9uzZo77zne+ocePGqa6urjhbfnbyox/9SGVlZamamhp1/Phx39bZ2elr88Mf/lCNHTtWvfXWW+qjjz5Ss2fPVrNnz46j1UNPwgiOUko99thjauzYscpoNKqZM2eqDz74IN4mnTNs3bpVAX22pUuXKqU8Q+P33nuvys/PVyaTSV177bWqrq4uvkafxQS7l4B6+umnfW26urrUj3/8Y5WTk6MsFotasGCBOn78ePyMHgakHo4gCMNGQsRwBEE4NxDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2BDBEQRh2Pj/1RJ9yPHTqfYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:06:35.348888Z", + "iopub.status.busy": "2024-02-29T13:06:35.348545Z", + "iopub.status.idle": "2024-02-29T13:07:26.702212Z", + "shell.execute_reply": "2024-02-29T13:07:26.701154Z" + }, + "papermill": { + "duration": 51.37756, + "end_time": "2024-02-29T13:07:26.704760", + "exception": false, + "start_time": "2024-02-29T13:06:35.327200", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:07:26.747179Z", + "iopub.status.busy": "2024-02-29T13:07:26.746496Z", + "iopub.status.idle": "2024-02-29T13:07:26.766712Z", + "shell.execute_reply": "2024-02-29T13:07:26.765820Z" + }, + "papermill": { + "duration": 0.043433, + "end_time": "2024-02-29T13:07:26.768758", + "exception": false, + "start_time": "2024-02-29T13:07:26.725325", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0793520.1306050.0007410.5596820.0354920.7786030.0546919.762089e-070.8873210.0206440.3238650.0272130.1551740.000011.447003
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.079352 0.130605 0.000741 0.559682 0.035492 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.778603 0.054691 9.762089e-07 0.887321 0.020644 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.323865 0.027213 0.155174 0.00001 1.447003 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:07:26.808278Z", + "iopub.status.busy": "2024-02-29T13:07:26.808002Z", + "iopub.status.idle": "2024-02-29T13:07:27.270693Z", + "shell.execute_reply": "2024-02-29T13:07:27.269810Z" + }, + "papermill": { + "duration": 0.484984, + "end_time": "2024-02-29T13:07:27.272767", + "exception": false, + "start_time": "2024-02-29T13:07:26.787783", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:07:27.315442Z", + "iopub.status.busy": "2024-02-29T13:07:27.315124Z", + "iopub.status.idle": "2024-02-29T13:08:20.918770Z", + "shell.execute_reply": "2024-02-29T13:08:20.917945Z" + }, + "papermill": { + "duration": 53.628117, + "end_time": "2024-02-29T13:08:20.921192", + "exception": false, + "start_time": "2024-02-29T13:07:27.293075", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:08:20.964645Z", + "iopub.status.busy": "2024-02-29T13:08:20.964272Z", + "iopub.status.idle": "2024-02-29T13:08:20.981733Z", + "shell.execute_reply": "2024-02-29T13:08:20.980987Z" + }, + "papermill": { + "duration": 0.041566, + "end_time": "2024-02-29T13:08:20.983748", + "exception": false, + "start_time": "2024-02-29T13:08:20.942182", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:08:21.024340Z", + "iopub.status.busy": "2024-02-29T13:08:21.024024Z", + "iopub.status.idle": "2024-02-29T13:08:21.029282Z", + "shell.execute_reply": "2024-02-29T13:08:21.028479Z" + }, + "papermill": { + "duration": 0.027882, + "end_time": "2024-02-29T13:08:21.031356", + "exception": false, + "start_time": "2024-02-29T13:08:21.003474", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.06832179765130643}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:08:21.073329Z", + "iopub.status.busy": "2024-02-29T13:08:21.073044Z", + "iopub.status.idle": "2024-02-29T13:08:21.409287Z", + "shell.execute_reply": "2024-02-29T13:08:21.408346Z" + }, + "papermill": { + "duration": 0.360199, + "end_time": "2024-02-29T13:08:21.411526", + "exception": false, + "start_time": "2024-02-29T13:08:21.051327", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABB90lEQVR4nO3deXxU5b348c+ZfZLMZCEJSSAhLAGRHQTEhUWpItZKbb2tYgUVtbdYtZT7qtz+6toW9eWCXS719irR61q9bq1FpSq4AgKCCAqCCQRICJA9s885vz/OzCQhCUySSWaSfN+v17wmOTk582Qg33yf53yf51E0TdMQQogEYYh3A4QQojkJSkKIhCJBSQiRUCQoCSESigQlIURCkaAkhEgoEpSEEAlFgpIQIqFIUBJCJBQJSuKUiouLURSF0tLSeDdF9BMSlETMPffcc6xatSrezRC9lAQlEXMSlERXSFASQiQUCUqiw9auXcusWbNwOBw4nU6mTp3Kc889B8Ds2bN58803OXDgAIqioCgKhYWFUV9bVVXuvvtu8vLySEpKYs6cOezevZvCwkIWL14cOa+qqorly5czbtw4UlJScDqdXHLJJezYsaPF9davX4+iKPztb3/jd7/7HYMHD8Zms3HhhReyb9++WLwdIsZM8W6A6F2Ki4u5/vrrGTNmDCtWrCAtLY3PP/+ct956i6uvvppf//rX1NbWcujQIR599FEAUlJSor7+ihUrePDBB7nsssu4+OKL2bFjBxdffDEej6fFed9++y2vvfYaV155JUOHDuXo0aM8/vjjzJo1i927d5OXl9fi/Pvvvx+DwcDy5cupra3lwQcfZOHChWzatKnrb4qILU2IU1izZo0GaCUlJVpNTY3mcDi06dOna263u8V5qqpGPr700ku1IUOGdPi1KioqNJPJpC1YsKDF8bvvvlsDtEWLFkWOeTweLRgMtjivpKREs1qt2r333hs59v7772uANnr0aM3r9UaOP/bYYxqg7dy5s8PtFN1Lum8iauvWraO+vp477rgDm83W4muKonT5+u+++y6BQICf/exnLY7//Oc/b3Wu1WrFYND/+waDQU6cOEFKSgqjRo1i27Ztrc6/7rrrsFgskc/PP/98QM+4RGKRoCSitn//fgDGjh3bLdc/cOAAACNGjGhxPCMjg/T09BbHVFXl0UcfpaioCKvVSmZmJllZWXzxxRfU1ta2unZBQUGLz8PXq66ujuWPIGJAgpLolX7/+9+zbNkyZs6cyTPPPMPbb7/NunXrGDNmDKqqtjrfaDS2eR1NVoNOODLQLaI2fPhwAL788stW2Uxzne3KDRkyBIB9+/YxdOjQyPETJ060ymhefvll5syZwxNPPNHieE1NDZmZmZ16fZEYJFMSUbvoootwOBysXLmy1d2w5hlHcnJym12o07nwwgsxmUysXr26xfE//elPrc41Go2tspyXXnqJw4cPd/h1RWKRTElEzel08uijj7JkyRKmTp3K1VdfTXp6Ojt27MDlcvHUU08BMGXKFF588UWWLVvG1KlTSUlJ4bLLLjvt9QcOHMhtt93Gww8/zPe+9z3mzZvHjh07WLt2LZmZmS0ysO9+97vce++9XHfddZxzzjns3LmTZ599lmHDhnXbzy96SJzv/okE17wkIOyNN97QzjnnHM1ut2tOp1ObNm2a9vzzz0e+3tDQoF199dVaWlqaBnSoPCAQCGi/+c1vtJycHM1ut2sXXHCB9tVXX2kDBgzQfvrTn0bO83g82i9/+UstNzdXs9vt2rnnnqt9+umn2qxZs7RZs2ZFzguXBLz00kstXqekpEQDtDVr1nT0LRHdTNE0GekTia2mpob09HR++9vf8utf/zrezRHdTMaUREJxu92tjoUn986ePbtnGyPiQsaURI84duwYwWCw3a9bLBYyMjJ48cUXKS4uZv78+aSkpPDRRx/x/PPPc9FFF3Huuef2YItFvEhQEj1i6tSpkeLItsyaNYv169czfvx4TCYTDz74IHV1dZHB79/+9rc92FoRTzKmJHrExx9/3GbXLCw9PZ0pU6b0YItEopKgJIRIKDLQLYRIKL16TElVVY4cOYLD4YjJLHUhRGxpmkZ9fT15eXmRVR1Op1cHpSNHjpCfnx/vZgghTqOsrIzBgwdHdW6vDkoOhwPQf2Cn0xnn1gghTlZXV0d+fn7kdzUavToohbtsTqdTgpIQCawjwysy0C2ESCgSlIQQCUWCkhAiofTqMaVoaJpGIBA45bwr0Taj0YjJZJJyC9Gj+nRQ8vl8lJeX43K54t2UXispKYnc3NwWO4EI0Z36bFBSVZWSkhKMRiN5eXlYLBb5i98Bmqbh8/k4duwYJSUlFBUVRV38JkRX9Nmg5PP5UFWV/Px8kpKS4t2cXslut2M2mzlw4AA+n6/VXm+ibQ3eAO9/XYndbGT2qCxMRgnmHdFng1KY/HXvGnn/Ou69ryvZX9kAQHqymSlDMuLcot5F/scJEUMef5CSY42Rz3cfqYtja3qnPp8pCdGTDpxwoWoayVYjwzJTyM9IQtM0Gc/sAMmU+rnCwsLIGtii68pr9YXsigY6mHvmQEblyAoWHSVBSYgY0gCLycBAh9wU6CzpvvUBPp9P6ogSxJxR2cwemYWqgS+gcqzBiz+gUpiZHO+m9Rr9MlPyBdR2H4GgGvW5/ijO7YzZs2dzyy23cMstt5CamkpmZia/+c1vIttUFxYWct9993HttdfidDq56aabAPjoo484//zzsdvt5Ofnc+utt9LY2DToWllZyWWXXYbdbmfo0KE8++yznWqfODVFUTAaFMpr3fztszLW76mMd5N6lX6ZKf35/X3tfm1oZjILJg2KfP7fH+zHH2x7GfPB6XauPKtpkbknPy7B7Ws5neUX3xnZqTY+9dRT3HDDDWzevJktW7Zw0003UVBQwI033gjAQw89xJ133sldd90FwP79+5k3bx6//e1vefLJJzl27FgksK1ZswaAxYsXc+TIEd5//33MZjO33norlZXyC9NdMpL17LXG7SeoahgNMrYUjX4ZlHqD/Px8Hn30URRFYdSoUezcuZNHH300EpQuuOACfvnLX0bOX7JkCQsXLuT2228HoKioiD/84Q/MmjWL1atXc/DgQdauXcvmzZuZOnUqAE888QSjR4/u8Z+tr/qqvI7NJVUUZadwzohMUqwmTAaFgKrR4AmQmmSOdxN7hX4ZlJbOGdHu107+Y3bTzOHtnnvyTZXrzx3alWa1cPbZZ7e4azNjxgwefvjhyMTis846q8X5O3bs4IsvvmjRJdM0LTLdZu/evZhMphbbGJ1xxhmkpaXFrM39XVWjj6pGH56A/m+kKAoOm4lql586j1+CUpT6ZVCymKIfSuuuc7sqObnlwGlDQwM333wzt956a6tzCwoK2Lt3b081rd+qdfsBSLU33XRw2s2RoCSi0y+DUm+wadOmFp9v3LiRoqIijEZjm+dPnjyZ3bt3M2JE21ngGWecQSAQYOvWrZHu2549e6ipqYlpu/uzpqDUlBE5bfrHde5AXNrUG/XLu2+9wcGDB1m2bBl79uzh+eef549//CO33XZbu+f/6le/4pNPPuGWW25h+/btfPPNN7z++uvccsstAIwaNYp58+Zx8803s2nTJrZu3cqSJUuw2+099SP1eQ0ePfA4bE1/68MfS6YUvbgHpcOHD3PNNdcwYMAA7HY748aNY8uWLfFuVtxde+21uN1upk2bxtKlS7ntttsit/7bMn78eDZs2MDevXs5//zzmTRpEnfeeSd5eXmRc9asWUNeXh6zZs3iiiuu4KabbiI7O7snfpw+T1U1lPpDFFRvJCVYGzk+NCuZuaMHMmFwWvwa18vEtftWXV3Nueeey5w5c1i7di1ZWVl88803pKenx7NZCcFsNrNq1SpWr17d6mulpaVtfs/UqVN555132r1mTk4O//jHP1oc+8lPftKldgpdY91xRh99E6MWJGl3DUz/KZgsZDtsZEt1d4fENSg98MAD5OfnR+poAIYOjd0dLCF6zKGtOMwARhRfI1TugrxJ8W5VrxTX7tsbb7zBWWedxZVXXkl2djaTJk3ir3/9a7vne71e6urqWjyESASOxgOMH5zG+NFn6AeOfwPoZRllVS6+rqhrNVtAtC2uQenbb79l9erVFBUV8fbbb/Pv//7v3HrrrTz11FNtnr9y5UpSU1Mjj766Zff69etl5n5v4q4GVxUYjDDiQv1YbRmEpgW99vlh1u6soNEnm1dEI65BSVVVJk+ezO9//3smTZrETTfdxI033shf/vKXNs9fsWIFtbW1kUdZWVkPt1iINtSV688p2eAcDEYTBHzgOoGiKCRZ9VGSRq+UBUQjrkEpNzeXM888s8Wx0aNHc/DgwTbPt1qtkS26ZatukTAajvLt8QbePaSwp7IRHLn68bojACRb9Noyl0+CUjTiGpTOPfdc9uzZ0+LY3r17GTJkSJxaJEQnNFTi8ascDaaiahqk5OjHG/XJzk2ZknTfohHXoPSLX/yCjRs38vvf/559+/bx3HPP8d///d8sXbo0ns0SomMajuIPqjRaBpBsMUFSaKMAVzXQlCk1SqYUlbgGpalTp/Lqq6/y/PPPM3bsWO677z5WrVrFwoUL49ksIaLnc4GvEX9QxWXOwG4xQtIA/WuuEwAkhzIll2RKUYn73Lfvfve7fPe73413M4ToHE8NGhqNShKqwawHJcuAyNcIBvTsCcmUohX3oCREr+auJhDU8BgdANjNRlCSwWTR78B5ahmc7uA7Zw4kTZYuiYoEJSG6wl2DP6jiNTmxmY1Nq0tanRA4Dp4a0gcMID1Z1lCPVtwn5ArRq3lq0ACbcwADUpoFHlua/uyVWQcd1b+CkqbpKXVPP7S21/huy9NPP82AAQPwer0tji9YsEAmzyYidw3JFhMXTR7FvzVbrx1bqIbOU4uqahw40cjXFXWoavT/F/qr/tV9C/rhw4d7/nXP/6U+xhCFK6+8kltvvZU33niDK6+8EtB3IXnzzTdPuQKAiBNPjf5sP2llC1tq6Ot6pvTKtsMAFMxKIsnSv37tOqp/ZUq9gN1u5+qrr26xcsIzzzxDQUEBs2fPjl/DRGvBAHjr9Y/taS2/Zm3KlAwGBatZ/1Xz+GVS7un0r5BtNOtZSzxetwNuvPFGpk6dyuHDhxk0aBDFxcUsXrxYtn9ONN460DRKa3xs+KySyUMyGDc4lCGFM6XQmJLdbMTrV3H7pVbpdPpXUFKUqLtR8TRp0iQmTJjA008/zUUXXcSuXbt48803490scbJQltSInSqXH7/aLAuyOkLnNICqYjcbqcHfal9A0Vr/Ckq9yJIlS1i1ahWHDx9m7ty5fXaZll7N1wDoQQkgydJsUwdLiv5HUFPB78Jm1r/mkUzptGRMKUFdffXVHDp0iL/+9a9cf/318W6OaItXD0ouJRSUzM3+xhsMYA5tyuBrjAQl6b6dngSlBJWamsoPfvADUlJSWLBgQbybI9oS6r41aHrwsVlO+nWypOjPvgZ9+gmSKUVDum8J7PDhwyxcuBCr1Rrvpoi2+OrR0GjQ9I0BWt3qt6QAleBrZOTAPAYkW8h2yL/l6UhQSkDV1dWsX7+e9evX81//9V/xbo5oj7eBQFDDa9R3K7abT9oo1BLaxdjXQG6undxU2WMvGhKUEtCkSZOorq7mgQceYNSoUfFujmiPtx5V03A40zHYLE3z3sIiQamx59vWi0lQSkDt7esmEoimga8Bq8nIFWePal08CU1lAb4GvIEgFbUeAqrG8KyUHm1qbyNBSYjO8LtADQ1ah4PPycKZkreBek+AV7Ydxm4xMnyWBKVT6fN337QOTIYVrcn7145QOQCWJH1rpbY0675ZTfqvmtevynt6Gn02KJnN+tQOl8sV55b0buH3L/x+ipBQ4eRhl5Hij0v4ZN/x1uc0KwmwmvTApWoa/qAEpVPps903o9FIWloalZWhHSWSkmTuWAdomobL5aKyspK0tDSMxnaygf6q2RSTapcfT6CN+qNwphT0Y8aP0aAQVDU8gSAWU5/NB7qszwYlgJwcfaubcGASHZeWlhZ5H0Uzfj2D9Co20MBmaiNoGy1gMIEaQPG7sZoMuHxBvH4VbD3c3l6kTwclRVHIzc0lOzsbv98f7+b0OmazWTKk9vj0oORGL4a0nlyjBPrcN7Ndz6qaB6W2sioREdegdPfdd3PPPfe0ODZq1Ci+/vrrmL6O0WiUXy4RW6FMya1Y9UzJ3E53LBKUXNjMZsAvayqdRtwzpTFjxvCvf/0r8rnJFPcmCXF6oaDkCk0xsbWVKUHTuJLfxZQhhXgDKlky1eSU4h4BTCaTjFmI3idUpe1SzaCcIiiFVwrwuynKb6eeSbQQ91sA33zzDXl5eQwbNoyFCxdy8ODBds/1er3U1dW1eAgRF6FMyZLkwGEztZ73FmZO0p9lqknU4hqUpk+fTnFxMW+99RarV6+mpKSE888/n/r6+jbPX7lyJampqZGHLHwm4kLTwO8G4LKzRrDk/GFktLevWzgo+d3UefyUHm+kss7TQw3tnRQtgcpLa2pqGDJkCI888gg33HBDq697vd4WWw/V1dWRn59PbW0tTqezJ5sq+jO/Bz56VP945vJTr8F+eBvsfRsyi9jiuIAPvznO6FwH88bm9kxb46yuro7U1NQO/Y7GfUypubS0NEaOHMm+ffva/LrVapW1hUT8hbpumCyn3xSiWaYUHnfyBuTu26nEfUypuYaGBvbv309ubv/4KyJ6qdD4UL1qpvjjEt7YcaT9cyMD3a4W899E++IalJYvX86GDRsoLS3lk08+4fvf/z5Go5Grrroqns0S4tRC40lebFS7/NS5T1GY26wkIDz/rc0pKSIirt23Q4cOcdVVV3HixAmysrI477zz2LhxI1lZWfFslhCn5tczJZ8htDZ3e3feoFmm5MEW+m2TTOnU4hqUXnjhhXi+vBCd42s2741TVHMDmOyhrZY0rJp+k0ammZxaQo0pCdErhAa6PehlAG1Oxg0zGMCkBy8relDyBzUCQcmW2pNQd9+E6BXCQUk5zRSTMHOSPiFX8zBzZGZkbEm0TYKSEB0V6r65QisEnLL7BpFxJcXvYcqQwu5sWZ8gQUmIjgoNdBusyThUU+v93k4WHuwOuLu5YX2DBCUhOiqUKc0Ylc8MRxSTyUNjSvg9HG/w0uAJkOmwkmKVX7+2yEC3EB3RbN5bpFr7dMyhoBRws37PMV79/DCHqmXt+PZIUBKiIwIe0EJ3zqINSqamWqVwVbcs9NY+yR+F6IhQ100zWfnfzYewGA18b2LeqceVmmVKkflvfqlVao8EJSE6IjTIHTTaOdHgA8BkOE2Ho3mmlBya/yaTctsl3TchOiI0nuQ36tmP0aBgNp5m665mmVJkUq4EpXZJUBKiI3zheW9NU0xOu59g80wpsnyJdN/aI0FJiI4IVXP7lCgm44a1lSnJQHe7ZExJiI6ITMYNVXNHM2UknCkFAwxMMTFzZCZpSe0snyskKAnRISfNe7OebooJgMkKigE0lQxLkIwhGd3Zwl5Pum9CdEQoKGnmJBw2U3RV2YqiBybQ1/cWpySZkhAdERroLhqUTdG4odF/n9kOfjeqz8XRWjdev8qQAUmnHyTvhyRTEqIjwpsGRFvNHRaa/xbwuXlhcxmvfn4YfzBhNhJKKBKUhIhW83lvlg4GpdBKAWbVgyGUHUlZQNskKAkRLb9bD0zAW3vreGHzQcqqopxYG8qUlIA3sv6SFFC2TYKSENGKdN1sHGsMUF7rQY12L9dmaypJVfepSVASIlqhQW7MSZEJtVEVT0KLNZWsMin3lBImKN1///0oisLtt98e76YI0bZm6yh5wkEp2vW2JVOKWkIEpc8++4zHH3+c8ePHx7spQrQvvEKAyR65cxZV8SS0zJRMsn33qcQ9KDU0NLBw4UL++te/kp6eHu/mCNG+0BQTf2gTSkUhkvWcVrNMaVSOg5kjM8lLs3VHK3u9uAelpUuXcumllzJ37tzTnuv1eqmrq2vxEKLHhAa6vaEVAqwmY/TFj80ypRHZKUwZkkG2Q4JSW+K+Q+62bdv47LPPojp/5cqV3HPPPd3cKiHaEQpKQaMNh80U/SA3yI4mHdCpTOnbb7/t8guXlZVx22238eyzz2KzRfcXY8WKFdTW1kYeZWVlXW6HEFELdd8y0tJZcv4wrjl7SPTfG86UAj48Xh8VtR4q62UeXFs6FZRGjBjBnDlzeOaZZ/B4OvfGbt26lcrKSiZPnozJZMJkMrFhwwb+8Ic/YDKZCAZb3y61Wq04nc4WDyF6TLhOqaPV3NAUlIADlVU8v/kgH+w9HqOG9S2dCkrbtm1j/PjxLFu2jJycHG6++WY2b97coWtceOGF7Ny5k+3bt0ceZ511FgsXLmT79u0YjbK1sUgwkTql5I5/r8EQWSnAhr62t0wzaVungtLEiRN57LHHOHLkCE8++STl5eWcd955jB07lkceeYRjx46d9hoOh4OxY8e2eCQnJzNgwADGjh3bmWYJ0X1UVd9eCdhx1MsLmw+yo6ymY9cIjStZw0FJVp9sU5fuvplMJq644gpeeuklHnjgAfbt28fy5cvJz8/n2muvpby8PFbtFCK+AqF5b4rCCZ+R8loPjb5Ax64R6sJZtXCmJEGpLV0KSlu2bOFnP/sZubm5PPLIIyxfvpz9+/ezbt06jhw5wuWXX96h661fv55Vq1Z1pUlCdI/QIDcmG95QLOrQ3TdonSkFgmjRzp3rRzpVEvDII4+wZs0a9uzZw/z583n66aeZP38+htD+V0OHDqW4uJjCwsJYtlWI+AlVc2NJxhMaC4q6cDIslClZNC+QhKaBL6hGKryFrlNBafXq1Vx//fUsXryY3NzcNs/Jzs7miSee6FLjhEgYkXlv9siW253NlIxBD0aDQlDV8AYkKJ2sU0Fp3bp1FBQURDKjME3TKCsro6CgAIvFwqJFi2LSSCHizte04qTH1cEVAsKaral09rABGA1gMcZ9UkXC6VRQGj58OOXl5WRnZ7c4XlVVxdChQ9usMRKiV2vefQtnSh3tvjWr6p42UnY0aU+nglJ7g3MNDQ1RV2cL0av4wruY2LGYDKia1ulMSXY0ObUOBaVly5YBoCgKd955J0lJTZWtwWCQTZs2MXHixJg2UIiEEMqUFEsyN5zXgV1MmmuWKdV7/DR6g6REu01TP9Khd+Pzzz8H9Exp586dWCxNu3xaLBYmTJjA8uXLY9tCIRJBZEypE9XcYc0ypQ+/Oc6einpmj8piUoEs2dNch4LS+++/D8B1113HY489JnPPRP/RlXlvYbL6ZFQ6lTeuWbMm1u0QIrGF5r2Vuw1s2HuQgU4bc87IPs03nSSyUoAXq1GCUnuiDkpXXHEFxcXFOJ1OrrjiilOe+8orr3S5YUIkjGAAAl4A6oMWymtrMHXmVn44KGkaNoMfkM0D2hJ1UEpNTY2sspeamtptDRIi4YS7booBt2oGOlHNDWA0gdEMQT92ZP5be6IOSs27bNJ9E/1Ks/EkT6CT1dxhZjsE/c2WL5GgdLJOlZO63W5crqadQQ8cOMCqVat45513YtYwIRJGs/3emoJSJyuxwysFhIKSR7pvrXTqnb388st5+umnAaipqWHatGk8/PDDXH755axevTqmDRQi7iKZUnLTfm9dyZSAVLPK9KEZTMxPi0ED+5ZOrzx5/vnnA/Dyyy+Tk5PDgQMHePrpp/nDH/4Q0wYKEXfN5711dBPKk4UyJYfJzzkjMhk7SMZnT9apoORyuXA4HAC88847XHHFFRgMBs4++2wOHDgQ0wYKEXeReW9JGA0KFpOh8923cK2STDVpV6c3DnjttdcoKyvj7bff5qKLLgKgsrJSCipF39Osmvu74/NYOmcERQMdnbtWaJ1uze+mulHf1SSoykJvzXUqKN15550sX76cwsJCpk+fzowZMwA9a5o0aVJMGyhE3PmaMqUuM4V21w14eGbjAZ7ffJAGbweX1e3jOlXR/cMf/pDzzjuP8vJyJkyYEDl+4YUX8v3vfz9mjRMiIfi7sIvJyczhqm4PVrOBgDcY2tXE3PVr9xGdnp6ck5NDTk5Oi2PTpk3rcoOESDih7ptHsfLq5oPYzUYun5gX/ZbdzZnCY0purCYjjd6g7Gpykk4FpcbGRu6//37effddKisrUdWWb2osdtAVIiFoWqQkwK3YqaitwmIydC4gQctMSSbltqlTQWnJkiVs2LCBn/zkJ+Tm5nb+H0iIRBfwgqqXAbg1fakee2drlKBlppQUDkpSQNlcp4LS2rVrefPNNzn33HO79OKrV69m9erVlJaWAjBmzBjuvPNOLrnkki5dV4iYCRdOmiy4VT2I2C1dCErNMyVZKaBNnbr7lp6eTkZG19cYHjx4MPfffz9bt25ly5YtXHDBBVx++eXs2rWry9cWIiaabdXt9ukZTUwyJTWIzaBfT8aUWupUULrvvvu48847W8x/64zLLruM+fPnU1RUxMiRI/nd735HSkoKGzdubPN8r9dLXV1di4cQ3SqcKZntXZ9iAvoqAQb9+4emGZk+NIPB6fautrJP6VT37eGHH2b//v0MHDiQwsJCzOaWtzO3bdvW4WsGg0FeeuklGhsbI3VPJ1u5ciX33HNPZ5osROf4GvRnawruUFDqUvdNUfSpJr5GhqYaGTo4MwaN7Fs6FZQWLFgQswbs3LmTGTNm4PF4SElJ4dVXX+XMM89s89wVK1ZENi8AqKurIz8/P2ZtEaIVbygoWVJQ0KeYdKn7BvpUE18jBNxdb18f1KmgdNddd8WsAaNGjWL79u3U1tby8ssvs2jRIjZs2NBmYLJarVit1pi9thCn5Wva7+28wkzOK8psd4uxqIUm5fq9LhoafaiaxoAU+X8d1untOWtqavif//kfVqxYQVVVFaB32w4fPtyh61gsFkaMGMGUKVNYuXIlEyZM4LHHHutss4SIrUhQSokc6nIJTGhS7vHqWoo/KWXtlxVdu14f06lM6YsvvmDu3LmkpqZSWlrKjTfeSEZGBq+88goHDx6MrLXUGaqq4vV6O/39QsSUr15/bhaUuiyUKVk0WX2yLZ3KlJYtW8bixYv55ptvWuyIO3/+fD744IOor7NixQo++OADSktL2blzJytWrGD9+vUsXLiwM80SIvaadd/+b+shXvv8cNcn0JrDQUn/4yvFky11KlP67LPPePzxx1sdHzRoEBUV0aeilZWVXHvttZSXl5Oamsr48eN5++23+c53vtOZZgkRW6oamfemmpMpq65A06DL8xdCtUqRoORXUVUNg0FmRkAng5LVam2zRmjv3r1kZWVFfZ0nnniiMy8vRM/wu0BTQVHwGuyEx7e7VKcEkUzJHOq+AfiCKjZDF6/bR3Sq+/a9732Pe++9F79f37tKURQOHjzIr371K37wgx/EtIFCxE2kmtuOO6BHJKvZgLGrGU0oUzIEPJiN+rWkqrtJp4LSww8/TENDA1lZWbjdbmbNmsWIESNwOBz87ne/i3UbhYgPX1ONUqRwsqtZEjSb/6YvXwIyrtRcp7pvqamprFu3jo8//pgdO3bQ0NDA5MmTmTt3bqzbJ0T8NA9KsZj3FmZqWqd7/OBUgqqGrStV4n1Mh4OSqqoUFxfzyiuvUFpaiqIoDB06lJycHDRNk2VMRN/R7M5bJCjFIng0y5SmDxvQ9ev1MR3qvmmaxve+9z2WLFnC4cOHGTduHGPGjOHAgQMsXrxYlsIVfUs4KFlTCGoaFpOBJEunF2ttEs6UggH9IVro0DtcXFzMBx98wLvvvsucOXNafO29995jwYIFPP3001x77bUxbaQQceFtKpycODiNiflpqLHYecRk1SfmahoeTyNu7FhMBpKtMQh4fUCHMqXnn3+e//zP/2wVkAAuuOAC7rjjDp599tmYNU6IuGrWfQuLSS2RokS2WvrsmyMUf1LKjkM1Xb9uH9GhoPTFF18wb968dr9+ySWXsGPHji43SoiEEM6UrJ3c4+1UQl04m6KX1chUkyYdyherqqoYOHBgu18fOHAg1dXVXW6UEHGnac3WUnLw9x1HCKgqs0Zmk5Fs6fr1zTZwgw0vYJM6pWY6lCkFg0FMpvbjmNFoJBCQgTvRB/ga9Q0DFAUsDsqqXZQed3V92ZKwcKZEOFOSOqWwDmVKmqaxePHidtc0ktn9os9oNsgd0JoqrmM2GB0qC7AiKwWcrEPv8KJFi057jtx5E32CNzS30+rAFarmNhqUyF5tXRbKlKyRlQIkKIV1KCitWbOmu9ohRGJpNsjt8upBKclijF1x8MnLl/il+xYmhRFCtCWSKTlx+fRx0pgUToaZkwCwKz4m5qeRJNNMIiQoCdGW5pmSrylTiplQULJpHuackR276/YBMeogC9HHhIOSzUlQ1bCaDTEOSk3bd4uWJFMSoi3NMqUJ2WlMyE+LXTkANFWJ+xrx+IO4fEEcNhNmo+QJ8g4IcTJNa7OaO6YrYIQzpYCHZz4t4alPSjnR4Dv19/QTEpSEOFmLwskY7mLSXGhMCU0jxagXULrlDhwg3TchWosUTiaDwcjr2w+jaTBrZBbpsZhiAmAw6pNyA15SDH7AGFmzqb+TTEmIk3lq9WerE03TOHjCRcnxRmK+fmFoXClF0WuVJFPSxTUorVy5kqlTp+JwOMjOzmbBggXs2bMnnk0Soiko2dPwBlQCoTWUUmK93lFoXMlu0MeSPBKUgDgHpQ0bNrB06VI2btzIunXr8Pv9XHTRRTQ2NsazWaK/89Toz7ZU6j164aTdYsQU6ztjoXGlJEKZknTfgDiPKb311lstPi8uLiY7O5utW7cyc+bMOLVK9HvuGv3ZlkZjaDfcblkVMlzVHZqUK903XUINdNfW6mlzRkZGm1/3er0tViJoa0NMIbos3H2zpdLg1oOSozuCkkUPSulmPxPz08hytL36Rn+TMAPdqqpy++23c+655zJ27Ng2z1m5ciWpqamRR35+fg+3UvR5mtZiTKmhBzKlAZYgc87IZuyg1Ni/Ri+UMEFp6dKlfPnll7zwwgvtnrNixQpqa2sjj7Kysh5soegXvPWgBkAxhO6+6bvixnyQG5pqlfyu2F+7F0uI7tstt9zCP/7xDz744AMGDx7c7nlWq7XdBeaEiIlIOYADDEZmDB/AjOEDYjvFJCx0903zufD4grj9QdKTzP1+78S4ZkqapnHLLbfw6quv8t577zF06NB4NkeIpjtv9rQWh7slUITqlFRfI3/ZsJ+nPimVxd6Ic6a0dOlSnnvuOV5//XUcDgcVFRWAvi243W6PZ9NEf9VskLvbhTIlY8CDxajgC2p4/EFssdgavBeLa6a0evVqamtrmT17Nrm5uZHHiy++GM9mif6sWTmAL6Dy9KelvPb5YQLBbshgzKGVAjSVZKM+oC5lAXHOlLqlny5EVzQrnKzz+DnR4KPBG4h94SSA0QRGMwT9pBj8VMv8NyCB7r4JkRBcJ/TnpAHUufXZ+06bufteLzz/zeABJFMCCUpCNPG7wRe6PZ+UQV1oionT3o1BKVQWkKzI/LcwCUpChLmq9GdrCpiszTKlbhzlCGVKSeiZkku6b4lRpyREQmjWdQOo8+hBKbU7M6XQypY5tgATM9LIS5O7zhKUhAhzhzKlcFBy90D3LZQpDUpSGSS7mgDSfROiSThTsusTwi0mAxaToZsHukPL7foauu81ehnJlIQIC48pJelB6YdT9ClP3Vq6EgpKmrchMtUkI1ZL7vZSkikJAaCq4K7WPw5138K6dS5aqPvmc9fzlw37efrTUlS1f9fvSVASAvSiSTUIBlPPTDEJs+qZkjnoQkFF06RWSYKSENDszlsGKAqflVbx9KelbC+r6d7XNSeDomAAHAb9bl+jL9C9r5ngJCgJAdBwVH9O0e+AnWjwcqLBh7875rw1ZzBECihTjfqqqi6vZEpCiIZK/TlZD0pVjXrWkp7UjXfewkLjSg6DHpQkUxJCQOMx/TklC03TqHbp0z7Sk3rgTliogDK8/1t/r+qWoCREwNd05y1lIC5fEF9ARVG6uZo7LLIppRuQoCRBSQjXcX3DAEsyWJKpatSzJKfN3D1LlpwsVKs00OpjYn4ag/r5VBMpnhQiPJ4UGuQ+3qB3owak9FARo80JQI41QI5MNZFMSYimQe4sAMxGA5kOKwOdtp55faselPDW9szrJTjJlIQ4qRxg7KDUnt2DLVSsqXnq8PqD1HsC/XpjSsmURP+mBqFe37ACR1582hDKlFS/h/9+7yue2XigXy/2JkFJ9G+Nx/TNJ01WSMogqGo9P/fMZAGzDaOi4Azdgav39N9aJQlKon+rO6I/O/NAUdh/rIE/v7+Pt3dV9Gw7QtlSulEPSuHtwvujuAalDz74gMsuu4y8vDwUReG1116LZ3NEfxQOSo5cAI7WeQioGsae3qU2NK6UZghnSv6eff0EEteg1NjYyIQJE/jzn/8cz2aI/qy+XH926uNJ5bX6Wtm5aT105y0slCk5FX3jgv7cfYvr3bdLLrmESy65JJ5NEP1ZwNu0OoAjl6CqUVkXCkqpPVzAGKpVcsiYUu8qCfB6vXi93sjndXV1cWyN6PVqD+mV3LZUsKZwos6DP6hhNRt6ZiJuc6FMKVlrBKT71musXLmS1NTUyCM/Pz/eTRK9WW2Z/pxWAMCRcNct1da9q022JTSm5MDFxPw0zshx9uzrJ5BeFZRWrFhBbW1t5FFWVhbvJonerOag/hwOSjV61ynHGYe5Z/Z0AFI0F3OK0hk3uAeLNxNMr+q+Wa1WrNb+W+kqYijgg7rQIHeannEXZCThD6oMGZDU8+0x2/VaqYAX3DWQktXzbUgQvSooCREzdYdAU/UBZlsaEIfpJc0pip4t1VfgazhOjeogxWYiydL/fkXj+hM3NDSwb9++yOclJSVs376djIwMCgoK4tgy0ec177r19PhRe0JB6dNd+9imGvnOmQPjFyTjKK5BacuWLcyZMyfy+bJlywBYtGgRxcXFcWqV6BeqS/Xn0HhS6fFG0pMtPbOoW3tC+82lo29MGV79sr+Ja1CaPXt29270J0RbvA1N40kZw/AFVP6+4wgBVePaGUMYkBKnccvQYHdqKCjVuPpnWUCvuvsmRExUfas/OwaC1cHBqkYCqkZakjm+u9OG78Cp9QDUSKYkRD9RtV9/HjACgH2VesHisKyUnq9Pai4pE4BkzYVR9VDjUtA0Lb5tigPJlET/ogahqkT/OGM4gaDKt8f17tLwrOQ4Ngww28DmxGoykBKoIaBq1PfD1QIkKIn+peaAXgtkSQJHLt8eb8TrV3HYTOT19Hy3tiRnoSgKWQZ9adzqxv7XhZOgJPqXyq/056wzwGDgq3J9/uToXCcGQwJ0k0LrhJ+Z6mPmyEzSemLfuQQjY0qi/wgG4Nge/ePs0fgCKoeq9aklo3MTZK5ZKCgNT3LDkIw4NyY+JCiJ/qPqW73rZk2B1HwsisKS84dSVuWO71235kKbF9BwVF/BoJ8NcoMEJREPqqovrtZ4DDy1eqAIevVpH82Zk/XZ8848fWVIQxdHG8q368/ZZ0Z+2a0mIyOyU7p23VhKygSjCS3gperYEY6pDoqyHRgToWvZQyQoiZ7jqYOyTagVuwh4G1EAo0HBEE02YEmGQZNh0Fn6XaqOclc31SflTaLRGyDJYky82+0Ggx6Aaw7yr42fc8Q+kgFnW/vVlksSlET3U1Uo20T91++z72gNXr+K32Cl0ZKFx+TEZLVjsSUxNEvPCgACapBDR4+RFKjD7jqCzV2PueRDOPI5FF0EWaM61obD2/TuUMYwgrZ0XvykFIvJwLyxOWTGq4K7Pc48lJoyBhurOQJU1HokKIm+45uj9WjoC5c5bHGY1+X3wK5XoboUq1Gj0pDN4axJ1NoGgdKyO5aSmkFRvl5A2ODy8ereUgAULUiGq4Qi13YGmivIrH0Rx4gZKMMvAIPx9G3wNerBDGDQFL44VEOt20+K1YQzHu/J6TgHA5vIUaoAOFzj7lfrK0lQ6u1UFY7vgRP79HV4DCZ9+sTAcZCSxReHajlYpS9GPyI7hfOLeu42c8Wx49RteoaRyW4wmrGc8R3OnDSC81KsJFmMaBp4Ayr1Hj/13kCLAKFpUJiZRKM3iNsX5IQygqqkQvJrt5J3ZAd5jRsY0lAJY76v1xydysGNEPSDYyCNKUPY+KW+QsD0YRlYTAlYFRPaxCCDWkxBD+W1CRg4u5EEpd6suhT2vg0u/S9qUNU4VO1iUPq3mA5ugpyxjMg4C1/QxtE6D/sqGzhY5eLC0dndvtzqkePV7H3nr1g9Jzg+OIfMc64BRw7NFzBWFLBbjNgtRrJP+v70ZAvfnzQ48rnHH6S81sM3RzPYX5LDWNs2ffmRbU/B2B+2vyha/VE4tAUArXAm7+05hscfJNtpZWxegmYf1hRIzsShVpLWcIjjxhE0egMkW/vHr2v/+Cn7Gk2Dkg/g4Kf6x2Y7nqwxrCszUmts5EzPUSYnHYOKL5lgO8iEMxdwwjiE976u5FC1m7U7K3D5gkwuSO+W5h2pquerdWtI8pwgKcWJ85zrwDGgS9e0mY0MzUxmaGYynpFZ2LwT4cuXwV3DwXdXUz9sPqPHTGlZABnwwldv6Hf1skaxrTGDfZXHMSgK3zlzYGIUS7YnYximxuMUKEc5zggOnHBxZl6C1FJ1swTMXcUpqUH46u9w4BM9IOVNxDPlJl6uHsk+bTCNGWeSc+5CmLJIX5/HUwfbn2OAp4wfTB7M5CF6INqw5xglxxtj3rwj1Y3sXPe/JLmOkJKczOiLb8TSxYB0MpvZqGdGkxfRkJTHkeN11G1+kffXvkjFiWr9JE8tfPEiNB4HSzL7087hw2+OAzBrVBbZjh7e162jMoYBMIQK0LRu+bdKVJIp9SZBP3z5in5rWzHAqEvwZo3h1W2HOVbvJcli5Mqz8kOFgHaYshh2v6GPN335fxjOuJSZRWOwmQw0eAMUxngt6sPVLj5/928MaPgWZ5KVkXMXY07LjelrtGBJIvmsheQEX+fYno0kVWzh29e3cSwtkzyrlySzgt2eBOOuZJA9i8xDXgal25nQGwaNU/PBZGGgzc33h6vkDjm5g9t3KVovXmWtrq6O1NRUamtrcTr7eGob8OndleoDYDTBmCvwpw3l1c8Pc7jajc1s5IdTBre+dawG4es34egufRCn6CIYNLnFkhiqqqEodKlmx+UL8Pab/0dO1Wek2i0UzVmIOXdMV37i6GkarvKvKflsLXXHj0QO19nymHnpNdjT9F9obyCIxWhIvNqk9nz9JpR/AbkT4Iz58W5Np3Tmd1Qypd4g4IWdL0FNGRjNMP5HkJbPO1+Uc7jajcVk4IrJg9quZTEYYfRlYLLB4a36wHjQh1JwNqAPjv9zZzkDki3MGD6g07+wScd3co5hN1V2M0XnfR9TTwUkAEUhKW80Yy4fzYljFew9cIRSl4mqYBJjtBTCw+VWUxTlA4kkZ5welI59BSPmgilBpsJ0MwlKic7vhi/+BnVH9C14xv8IUgcBMLUwnYo6D5eMzWGg8xRjJIoCRd/R/1Mf+BT2v68HuqEzKT3RyL7KBvYBGnBOBwOTqmoYjn8Ne99moNNG9rg5KAVTu/Yzd8GArBxmZOUwI24tiKHUfEjKQHOd4Kut69mijmTBpEGJWVsVQzLQncjc1bDtf/WAZLbBhKvQQjUsANlOG4vPKSQvLYp1gBQFhs3WH6APlO/7F8MHJDFrlH47fXNJFZ/sPxH1uum7jtTyz3+tw7/zVX3QPXcCytBZHfwhRbsUBQpmoKAQKN1ITV0Dm7+tinerup0EpUR1Yj9sexpcJ8DqgIkLqbdk8vLWQxyqdkVO6/BEzSEz9HEl0Ot3djzP5IFmZjcLTGu/rMDtC7Z7CY8/yLpdFez++E0GlL1DRY1bH/cYOa9fzmrvVgPHgD2dQicU1Gxi15E6jtZ54t2qbpUQQenPf/4zhYWF2Gw2pk+fzubNm+PdpPjxufRxny/+pn/sGIg2+Vr2Ntp5ZuNBDlW7eWfXUYJqF+5PDJ4CYxbo41M1B+Gz/2GS6QAXjMrCoCjsqajnqU9LOXCi6Ta0pmlUNfrY+O0JXlz/Oernz5Jft438dDuDx8+CUZd0fRa/aM1ghFHzcNrMjGE/A+q/5p87y3H5+u4yuXEfU3rxxRdZtmwZf/nLX5g+fTqrVq3i4osvZs+ePWRn95/boHhqoXyHnr0EvABog6ZQ4pzKli9rORxajGyg08b8cTldX8oiezQkZ8PuV6HhGOxZy4SkDAYPHsO6SieVfnuLNYbe2H6I44f2kd24h6LG/SRbDAwZnEHahMv0AVnRfdILYcgMhqofU3foI/YHG/i/LSrzxw+O33ZQ3SjuJQHTp09n6tSp/OlPfwJAVVXy8/P5+c9/zh133HHK7+2VJQFqEPwufZJo43F9TaHqUoKhfciMigIpWZSkzWBduY1Gr96NMhsVpgzJYGphOiZjDDMSNajPoC/9sCkYahr12HGmZuh/qQNe9pYepLq+EYfNTLbDSkbBmRhGfieyLZDoZpoG36zDXbqZ3UfqqCeJ2tSRzD//bMwpmWB1oGokXJV6rysJ8Pl8bN26lRUrVkSOGQwG5s6dy6efftrqfK/Xi9frjXxeV1cX3QtVlcC+f7U+3iIea5FjJxq9fHusIXwERdPQAAUNTdMYmpVCViiLqHH52HO0Xv9+rfW1hgxIIsdpAzRqXV6+OXKiVRM0TSOowaDCURSMPQ8yR2Gu9dBYegir2cDYvFQmFqR1z10XgxHyp+pjQkd3wtHdKHVHcGoefYA9ZFiGBUNOOoasIsibDM5uLIoUrSkKjLwIuzOXsbb3KDlcwdDgbsw7DwCgAp+W1qMZzRiMZn2NKkUB9OdMhy2ymF1Q1dhyoLrdl0pPtjAymoXvxv0b2GKfDMQ1KB0/fpxgMMjAgQNbHB84cCBff/11q/NXrlzJPffc0/EXCnj1rCRabi+BxoZ2v6y5g3rdD6B5faie+vav5VfAHwpSAT/+oB7e/AYbbnMqbnM6DZZsauz5KEPzKcjWp2Tkpdr5weTBDEq398yqgyYLDJqiPwJefZKvt06PmgYTpqQMsKXJuFG85YzDmjWaM459jXp8HzQeBXcNXp8fg+oD1Qf+Zn8fwwxWaNCHANA01PpT3MVTLZDkPn1bTl4pNEbiPqbUEStWrGDZsmWRz+vq6sjPzz/Fd4Sk5cOEH590Zyj0cRvH7H6VgkZfy3odRQl9WSHFbgaLCRQFuz/IME8g9N1K5FRN0T9LtobOBeyqQlHACCZbi2ubDAZsFgOWZt0yg0GhIMbTQKJmsoYyIcmGEpLRBDljMeSM1T9Xg9j8Lia6Pfi9bnyBAEFNBVVDC2X3dpMCdj3TVjQoKGz/Dp7NbIBo1iy3dM8+eXENSpmZmRiNRo4ePdri+NGjR8nJyWl1vtVqxWrtxMCeJRkyhkZ9ehKQ1M5KGCezhR5RNQPIjLoVQkTJYESxOrBbHUSzc50BGBTbOdIxFddc3GKxMGXKFN59993IMVVVeffdd5kxo0/U5AohOiju3bdly5axaNEizjrrLKZNm8aqVatobGzkuuuui3fThBBxEPeg9KMf/Yhjx45x5513UlFRwcSJE3nrrbdaDX4LIfqHuNcpdUWvrFMSoh/pzO+o3N8VQiQUCUpCiIQiQUkIkVDiPtDdFeHhsKinmwghelT4d7MjQ9e9OijV1+vTO6Kq6hZCxE19fT2pqdFt2NCr776pqsqRI0dwOBy9ZzH4GAtPtSkrK5M7kJ0g71/XnO790zSN+vp68vLyMEQ5b7JXZ0oGg4HBgwef/sR+wOl0yi9VF8j71zWnev+izZDCZKBbCJFQJCgJIRKKBKVezmq1ctddd3Vu9QQh718Xdcf716sHuoUQfY9kSkKIhCJBSQiRUCQoCSESigQlIURCkaDUC1VVVbFw4UKcTidpaWnccMMNNDS0v/sKwOzZs1EUpcXjpz/9aQ+1OL46ugPzSy+9xBlnnIHNZmPcuHH885//7KGWJqaOvH/FxcWt/p/ZbNGuYh+iiV5n3rx52oQJE7SNGzdqH374oTZixAjtqquuOuX3zJo1S7vxxhu18vLyyKO2traHWhw/L7zwgmaxWLQnn3xS27Vrl3bjjTdqaWlp2tGjR9s8/+OPP9aMRqP24IMPart379b+3//7f5rZbNZ27tzZwy1PDB19/9asWaM5nc4W/88qKio69JoSlHqZ3bt3a4D22WefRY6tXbtWUxRFO3z4cLvfN2vWLO22227rgRYmlmnTpmlLly6NfB4MBrW8vDxt5cqVbZ7/b//2b9qll17a4tj06dO1m2++uVvbmag6+v6tWbNGS01N7dJrSvetl/n0009JS0vjrLPOihybO3cuBoOBTZs2nfJ7n332WTIzMxk7diwrVqzA5XJ1d3PjKrwD89y5cyPHTrUDM+jvb/PzAS6++OJ2z+/LOvP+ATQ0NDBkyBDy8/O5/PLL2bVrV4det1dPyO2PKioqyM7ObnHMZDKRkZFBRUVFu9939dVXM2TIEPLy8vjiiy/41a9+xZ49e3jllVe6u8lx09EdmEF/f9s6/1TvbV/Vmfdv1KhRPPnkk4wfP57a2loeeughzjnnHHbt2hX15HkJSgnijjvu4IEHHjjlOV999VWnr3/TTTdFPh43bhy5ublceOGF7N+/n+HDh3f6ukI0N2PGjBZ7Np5zzjmMHj2axx9/nPvuuy+qa0hQShC//OUvWbx48SnPGTZsGDk5OVRWVrY4HggEqKqqanNX4fZMnz4dgH379vXZoNTRHZgBcnJyOnR+X9aZ9+9kZrOZSZMmsW/fvqhfV8aUEkRWVhZnnHHGKR8Wi4UZM2ZQU1PD1q1bI9/73nvvoapqJNBEY/v27QDk5ubG+kdJGJ3ZgXnGjBktzgdYt25dv9yxORY7WAeDQXbu3Nmx/2ddGiYXcTFv3jxt0qRJ2qZNm7SPPvpIKyoqalEScOjQIW3UqFHapk2bNE3TtH379mn33nuvtmXLFq2kpER7/fXXtWHDhmkzZ86M14/QY1544QXNarVqxcXF2u7du7WbbrpJS0tLi9ym/slPfqLdcccdkfM//vhjzWQyaQ899JD21VdfaXfddVe/LwnoyPt3zz33aG+//ba2f/9+bevWrdqPf/xjzWazabt27Yr6NSUo9UInTpzQrrrqKi0lJUVzOp3addddp9XX10e+XlJSogHa+++/r2maph08eFCbOXOmlpGRoVmtVm3EiBHaf/zHf/SLOiVN07Q//vGPWkFBgWaxWLRp06ZpGzdujHxt1qxZ2qJFi1qc/7e//U0bOXKkZrFYtDFjxmhvvvlmD7c4sXTk/bv99tsj5w4cOFCbP3++tm3btg69nixdIoRIKDKmJIRIKBKUhBAJRYKSECKhSFASQiQUCUpCiIQiQUkIkVAkKAkhEooEJSFEQpGgJHqV4uJi0tLSIp/ffffdTJw4MfL54sWLWbBgQY+3S8SOBCXRpsWLF7e7jvfSpUtRFKXFqgbdEQwKCwtZtWpVi2M/+tGP2Lt3b7vf89hjj1FcXBz5fPbs2dx+++0xbZfoXhKURLvy8/N54YUXcLvdkWMej4fnnnuOgoKCuLTJbre3WuSuudTU1BaZlOh9JCiJdk2ePJn8/PwWq1O+8sorFBQUMGnSpC5du60MZsGCBZHsa/bs2Rw4cIBf/OIXkV0xoHX37WTNM7bFixezYcMGHnvsscg1SkpKGDFiBA899FCL79u+fTuKonRo3R/RPSQoiVO6/vrrWbNmTeTzJ598kuuuu67bX/eVV15h8ODB3HvvvZSXl1NeXt7hazz22GPMmDGDG2+8MXKNgoKCVj8TwJo1a5g5cyYjRoyI1Y8gOkmCkjila665ho8++ogDBw5w4MABPv74Y6655ppuf92MjAyMRiMOh4OcnJxOrfyYmpqKxWIhKSkpcg2j0cjixYvZs2dPZP8yv9/Pc889x/XXXx/rH0N0giyHK04pKyuLSy+9lOLiYjRN49JLLyUzMzPezeqSvLw8Lr30Up588kmmTZvG3//+d7xeL1deeWW8myaQTElE4frrr6e4uJinnnoqZtmEwWDg5KW8/H5/TK4djSVLlkQG8desWcOPfvQjkpKSeuz1RfskKInTmjdvHj6fD7/fz8UXXxyTa2ZlZbUYJwoGg3z55ZctzrFYLASDwS69TnvXmD9/PsnJyaxevZq33npLum4JRLpv4rSMRmNkeyej0djuebW1tZENCcIGDBhAfn5+q3MvuOACli1bxptvvsnw4cN55JFHqKmpaXFOYWEhH3zwAT/+8Y+xWq2d6jYWFhayadMmSktLSUlJISMjA4PBEBlbWrFiBUVFRf1yY4BEJZmSiIrT6cTpdJ7ynPXr1zNp0qQWj3vuuafNc6+//noWLVrEtddey6xZsxg2bBhz5sxpcc69995LaWkpw4cPJysrq1PtXr58OUajkTPPPJOsrCwOHjwY+doNN9yAz+frkbuJInqyRrfotz788EMuvPBCysrKWu0CK+JHgpLod7xeL8eOHWPRokXk5OTw7LPPxrtJohnpvol+5/nnn2fIkCHU1NTw4IMPxrs54iSSKQkhEopkSkKIhCJBSQiRUCQoCSESigQlIURCkaAkhEgoEpSEEAlFgpIQIqFIUBJCJJT/D1eaE6sXeWRzAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:08:21.454296Z", + "iopub.status.busy": "2024-02-29T13:08:21.453996Z", + "iopub.status.idle": "2024-02-29T13:08:21.801402Z", + "shell.execute_reply": "2024-02-29T13:08:21.800436Z" + }, + "papermill": { + "duration": 0.370999, + "end_time": "2024-02-29T13:08:21.803407", + "exception": false, + "start_time": "2024-02-29T13:08:21.432408", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAPUlEQVR4nO3deXxU5b348c+ZPZNMNshKEhL2fROhuLAodcFaqa21ViuIS3tL763l0irt/dGqL4v2qhVbX9jbVqJVRG3V2lql1AUtCiIIIgoIJCRAQgjZJ7Of8/vjzEwSSMIkmWQmyff9es1ryMyZM08G5sv3ec73eR5F0zQNIYSIE4ZYN0AIIVqToCSEiCsSlIQQcUWCkhAirkhQEkLEFQlKQoi4IkFJCBFXJCgJIeKKKdYN6AlVVTlx4gQOhwNFUWLdHCHEGTRNo7GxkdzcXAyGyHKgfh2UTpw4QX5+fqybIYQ4h/LycvLy8iI6tl8HJYfDAei/cHJycoxbI4Q4U0NDA/n5+eHvaiT6dVAKddmSk5MlKAkRx7oyvCID3UKIuCJBSQgRVyQoCSHiSr8eU4qEpmn4/X4CgUCsmzKomc1mjEZjrJsh+oEBHZS8Xi8VFRU0NzfHuimDnqIo5OXlkZSUFOumiDg3YIOSqqqUlJRgNBrJzc3FYrFIgWWMaJrGqVOnOHbsGKNHj5aMSXRqwAYlr9eLqqrk5+djt9tj3ZxBLyMjg9LSUnw+34APSk0eP2/vryLBbGT+2AxMRhm67YoBG5RCIi1tF71rMGWpb+2v4nBVEwBpiWbOG54e4xb1L/KNFSKK3L4AJaec4Z8/O9EQw9b0TwM+UxKiLx093YyqaSRajYwYmkR+uh1N0wZVpthTkinFofnz53PnnXfGuhmiGyrqXQCMznKwcEIWY7NlBYuukqDUj73zzjsoikJdXV2smyKCNMBiMpDlsMW6Kf2WdN+EiKIFYzOZPyYDVQOvX+VUkwefX6VwaGKsm9ZvDMpMyetXO7z5A2rEx/oiOLanPB4Pd911F/n5+VitVkaNGsUf//hHSktLWbBgAQBpaWkoisLSpUvPeb7GxkZuvPFGEhMTycnJ4de//vVZ3cU//elPzJw5E4fDQXZ2Nt/+9repqqoKPx/K0N58801mzpyJ3W7nggsu4MCBAz3+fQcCRVEwGhQq6l28sKOcdw5UnftFImxQZkqPv32ow+eKhiayePqw8M//9+5hfIH2dzbPS0vgupkti8w9ubUEl7ftdJYffXlMj9p6880388EHH/DYY48xdepUSkpKqK6uJj8/n7/85S98/etf58CBAyQnJ5OQkHDO861YsYKtW7fy6quvkpWVxerVq9m1axfTpk0LH+Pz+bjvvvsYO3YsVVVVrFixgqVLl/KPf/yjzbl+9rOf8fDDD5ORkcH3vvc9li1bxtatW3v0+w4k6YkWAOpcPgKqhtEgY0uRGJRBqb84ePAgL7zwAps3b2bhwoUAjBgxIvx8erpe/5KZmUlqauo5z9fY2MhTTz3Fhg0buPTSSwFYv349ubm5bY5btmxZ+M8jRozgscce4/zzz6epqanNNJH777+fefPmAXD33Xdz1VVX4Xa7sdkG53jK5xUNfFhSw+jMJC4YNZQkqwmTQcGvajS5/aTYzbFuYr8wKIPS8gWjOnzuzP/M7pg7ssNjz7yosuzCop406yy7d+/GaDSGv/g9deTIEXw+H7NmzQo/lpKSwtixY9sct3PnTn7xi1+wZ88eamtrUVW9G1pWVsaECRPCx02ZMiX855ycHACqqqooKCiISnv7mxqnlxqnF7dfz5YVRcFhM1Hb7KPB7ZOgFKFBGZQspsiH0nrr2EhE0h2LNqfTyeWXX87ll1/Os88+S0ZGBmVlZVx++eV4vd42x5rNLV+y0GXvUAAbjOpdPgBSEizhx5ITzOGgJCIzKAe6+4vJkyejqipbtmxp93mLRf/HH+myLCNGjMBsNrNjx47wY/X19Rw8eDD88/79+zl9+jQPPPAAF198MePGjWszyC061hKUWoJ1sk3/c4PLH5M29UcSlOJYYWEhS5YsYdmyZbzyyiuUlJTwzjvv8MILLwAwfPhwFEXh73//O6dOnaKpqanT8zkcDpYsWcKPf/xj3n77bfbt28ett96KwWAIZzoFBQVYLBZ+85vfcOTIEV599VXuu+++Xv9dB4Imtx54HLaWDkjoz5IpRS7mQen48ePcdNNNDBkyhISEBCZPnsxHH30U62bFjXXr1vGNb3yD73//+4wbN47bb78dp1OfWzVs2DDuuece7r77brKysvjBD35wzvM98sgjzJkzh6985SssXLiQCy+8kPHjx4cHpzMyMiguLubFF19kwoQJPPDAAzz00EO9+jsOBKqqoTQeo6B2G0mB+vDjRRmJLByfxdS81Ng1rp9RNE1r/3p3H6itrWX69OksWLCA//iP/yAjI4MvvviCkSNHMnJkxwPMIQ0NDaSkpFBfX3/WbiZut5uSkhKKiooG7dWgSDidToYNG8bDDz/Mrbfe2mvvM9D/PhrrTvHJSw9h1ALMHleAMvt7YLKc+4UDXGff0Y7EdKD7wQcfJD8/n/Xr14cfKyqK7hUs0dbHH3/M/v37mTVrFvX19dx7770AXHPNNTFuWT93bCcOM4ARxeuEqn2QOz3WreqXYtp9e/XVV5k5cybXXXcdmZmZTJ8+nd///vcdHu/xeGhoaGhzEy3KyspISkrq8FZWVgbAQw89xNSpU1m4cCFOp5P33nuPoUOHxrj1/ZvDeZQpealMGT9Of6D6C0BfdbO8ppn9lQ1nzRYQ7YtppnTkyBHWrVvHihUr+OlPf8qOHTv4r//6LywWC0uWLDnr+DVr1nDPPffEoKX9Q25uLrt37+70+YKCAnbu3Nl3jRoMXLXQXAMGI4y6FD5+FurLITgy8srHx/GrGssuKiIlIebDuHEvpkFJVVVmzpzJL3/5SwCmT5/Op59+yhNPPNFuUFq1ahUrVqwI/xzaEljoTCYTo0Z1XBgqeklDhX6flAnJeWA0gd8LzadREodit5pocPlwevxtygVE+2IatnNyctpUCAOMHz8+3M04k9VqDW/RLVt1i7jRdJIj1U28eUzhQJUTHHp1Ow0nAEi06GuSN3ulVikSMQ1KF1544Vkzyw8ePMjw4cNj1CIhuqGpCrdP5WQgBVXTIClbf9ypF53arXqHxOmRvQcjEdOg9KMf/Yht27bxy1/+kkOHDrFhwwb+7//+j+XLl8eyWUJ0TdNJfAEVp2UIiRYT2IMbBTTXAi2ZklMypYjENCidf/75vPzyyzz33HNMmjSJ++67j0cffZQbb7wxls0SInLeZvA68QVUms3pJFiMYB+iP9d8GoDEYKbULJlSRGI+IfcrX/kKX/nKV2LdDCG6x12HhoZTsaMazHpQsgwJP0fAr2dPSKYUqZgHJSH6NVct/oCG2+gAIMFsBCVRr+b2e8FdT16agy9PyCJVli6JiBRNxKF4280k3toTV1x1+AIqHlMyNrNRX11SUcAavDLsriMt0cKkYSnkpclOzZGQoDRAnbn2kegl7jo0wJY8hCFJrea62VL1e4/MOuiqwRWUNE1Pqfv61oU5z0uXLmXLli2sXbsWRVFQFIXDhw9z6623UlRUREJCAmPHjmXt2rVnvW7x4sXcf//95ObmhleTfP/995k2bRo2m42ZM2fyyiuvoChKm8rvTz/9lCuvvJKkpCSysrL4zne+Q3V1dYftKS0t7fFfxYDhqiPRYuKyGWP5Zqv12rGFMqV6VFXj6Gkn+ysbUNWYzX/vNwbXmFLAB+893Pfve/F/RzxjfO3atRw8eJBJkyaFJ8umpaWRl5fHiy++yJAhQ3j//fe54447yMnJ4Zvf/Gb4tW+++SbJycls3rwZ0Cver776ahYtWsSGDRs4evToWd2wuro6LrnkEm677TZ+/etf43K5uOuuu/jmN7/JW2+91W57MjIyovChDBDuOv0+Ia3t47aU4PN6pvTSruMAFMyzY7cMrq9dV8mnE2dSUlKwWCzY7Xays7PDj7ee81dUVMQHH3zACy+80CYoJSYm8oc//CG8IuUTTzyBoij8/ve/x2azMWHCBI4fP87tt98efs1vf/tbpk+fHp7qA/Dkk0+Sn5/PwYMHGTNmTLvtEUDAD55G/c8JqW2fs7ZkSgaDgtVswONTcftU7LKiSacGV1AymvWsJRbv20OPP/44Tz75JGVlZbhcLrxeb5ttkUBfPjcUkAAOHDjAlClT2qxf1HrTAIA9e/bw9ttvt9mlJOTw4cOMGdOzLaIGNE8DaBqldV627KhixvB0JucFM6RQphQcU0owG/H4VFw+qVU6l8EVlBSlXy68tXHjRlauXMnDDz/MnDlzcDgc/O///i/bt29vc1xiYtd3YW1qauLqq6/mwQcfPOu50A4logPBLMlJAjXNPnytN02wOoLHNIGqkmA2UofvrH0BxdkGV1DqJywWS5vNALZu3coFF1zA97///fBjhw8fPud5xo4dyzPPPIPH48FqtQK02TQAYMaMGfzlL3+hsLAQk6n9fw5ntkcEefU10Z3ou87Yg9NJALAk6f8Jair4mrGZ9efckimd0+C6+tZPFBYWsn37dkpLS6murmb06NF89NFHbNq0iYMHD/L//t//Oyu4tOfb3/42qqpyxx138Pnnn7Np06bwetuhjQKWL19OTU0NN9xwAzt27ODw4cNs2rSJW265JRyIzmzPYN5GqQ2PHpSalWBQMrcK6gYDmINbZHmd4aAk3bdzk6AUh1auXInRaGTChAlkZGRw+eWXc+2113L99dcze/ZsTp8+3SZr6khycjJ/+9vf2L17N9OmTeNnP/sZq1evBgiPM+Xm5rJ161YCgQCXXXYZkydP5s477yQ1NRWDwdBuezpaWmbQCXbfmjQ9+NgsZ3ydLMFxOm+TPv0EyZQiEdONA3pKNg7oumeffZZbbrmF+vr6Pt3sckD+fex7Ga3qc56vHUuFYzK3zx1BkrVVtrTneag5AuOuosI+mtNNXjIdVjKTB8jvH4F+t3GA6H1PP/00I0aMYNiwYezZsydcgxSL3XcHHE8T/oCGx6hfYEgwG9s+bwleePA2kZOTQE6KfOaRkKA0wFVWVrJ69WoqKyvJycnhuuuu4/777491swYGTyOqpuFITsNgs+jz3loLByVn37etH5OgNMD95Cc/4Sc/+UmsmzHwaBp4m7CajFz7pbFnF09CS1mAtwmPP0BlvRu/qjEy4+yaMNFCgpIQ3eFrBjU4aB0KPmcKZUqeJhrdfl7adZwEi5GR8yQodWbAX33rx+P4A8qA+3sIlgNgsetbK7WnVffNatK/ah6fOvA+iygbsEHJbNandjQ3N8e4JQJallIxGjv4Avc3wcLJ481GireW8P6h6rOPaVUSYDXpv7eqafgCEpQ6M2C7b0ajkdTUVKqqgjtK2O3hgkHRt1RV5dSpU9jt9g6rxvudVlNMapt9uP3t1B+FMqWADzM+jAaFgKrh9gewmAZsPtBjA+RfSPtCs9pDgUnEjsFgoKCgYOD8x+DTM3CPYgMNbKZ2MkCjBQwmUP0oPhdWk4FmbwCPT4XBU6rUZQM6KCmKQk5ODpmZmfh8vlg3Z1CzWCzhCvEBwasHJRf6nELrmTVKoM99MyfoWVXroNReViXCYhqUfvGLX7RZJwj0SaT79++P6vsYjcaBM5Yh4kMwU3IpVj1TMncQcMNBqRmb2Qz4cPtk7mBnYp4pTZw4kX/961/hnwfMmIMY2IJBqVnT+2G29jIlaBlX8jVz3vBCPH6VDIe1L1rYb8U8AphMJlnRUPQ/wSrtZtUMSidBKbRSgM/F6PwO6plEGzHv5H/xxRfk5uYyYsQIbrzxxk5noHs8HhoaGtrchIiJYKZksTtw2Exnz3sLMQe3VZKpJhGLaVCaPXs2xcXFvPHGG6xbt46SkhIuvvhiGhsb2z1+zZo1pKSkhG/5+fntHidEr9I08LkAuHrmKG67eATpiR2saBoKSj4XDW4fpdVOqhrcfdTQ/imuli6pq6tj+PDhPPLII9x6661nPe/xePB4POGfGxoayM/P79KyCEL0mM8N//61/ue5Kztfg/34Lji4CYaO5iPHJbz3RTXjcxxcMWlwLDXc75cuSU1NZcyYMRw6dKjd561Wa3hZVyFiJth1w2Q596YQrTKl0LiTxy9X3zoT8zGl1pqamjh8+LAsWC/iW3B8qFE1U7y1hFf3nOj42PBAd3Ob+W+iYzENSitXrmTLli2Ulpby/vvv87WvfQ2j0cgNN9wQy2YJ0bngeJIHG7XNPhpcnRTmtioJCM1/a3dKigiLafft2LFj3HDDDZw+fZqMjAwuuugitm3bJjuwivjm0zMlryG4NndHV96gVabkxhb8tkmm1LmYBqWNGzfG8u2F6B5vq3lvdFLNDWBKCG61pGHV9Is0Ms2kc3E1piREvxAc6HajlwG0Oxk3xGAAkx68rOhByRfQ8AckW+pIXF19E6JfCAUl5RxTTELMdn1CruZm7pih4bEl0T4JSkJ0VbD71hxcIaDT7huEx5UUn5vzhhf2ZssGBAlKQnRVcKDbYE3EoZqwW87xNQoNdvtdvdywgUGCkhBdFcyU5ozNZ44jgsnkwTElfG6qmzw0uf0MdVjbblwpwmSgW4iuaDXvLVytfS7mYFDyu3jnwCle/vg4x2pl7fiOSFASoiv8btCCV84iDUqmllqlUFW3LPTWMckfheiKYNdNM1n504fHsBgNfHVabufjSq0ypfD8N5/UKnVEgpIQXREc5A4YEzjdpG8bZTrX2uOtM6XE4Pw3mZTbIem+CdEVwfEkn1HPfowGBbPxHDu0tMqUwpNyJSh1SIKSEF3hDc17a5lics5to1pnSuHlS6T71hEJSkJ0RbCa26tEMBk3pL1MSQa6OyRjSkJ0RXgybrCaO5IpI6FMKeAnK8nE3DFDSbV3sHyukKAkRJecMe/Neq4pJgAmKygG0FTSLQHSh6f3Zgv7Pem+CdEVwaCkme04bKbIqrIVRQ9MoK/vLTolmZIQXREc6B49LJPRk4sif505AXwuVG8zJ+tdeHwqw4fYzz1IPghJpiREV4Q2DYi0mjskOP/N73Wx8cNyXv74OL5A3GwkFFckKAkRqdbz3ixdDErBlQLMqhtDMDuSsoD2SVASIlI+lx6YgDcONrDxwzLKayKcWBvMlBS/J7z+khRQtk+CkhCRCnfdbJxy+qmod6NGupdrqzWVpKq7cxKUhIhUcJAbsz08oTai4klos6aSVSbldipugtIDDzyAoijceeedsW6KEO1rtY6SOxSUIl1vWzKliMVFUNqxYwe/+93vmDJlSqybIkTHQisEmBLCV84iKp6EtpmSSbbv7kzMg1JTUxM33ngjv//970lLS4t1c4ToWHCKiS+4CaWiEM56zqlVpjQ228HcMUPJTbX1Riv7vZgHpeXLl3PVVVexcOHCcx7r8XhoaGhocxOizwQHuj3BFQKsJmPkxY+tMqVRmUmcNzydTIcEpfbEfIfcXbt2sWPHjoiOX7NmDffcc08vt0qIDgSDUsBow2EzRT7IDbKjSRd0K1M6cuRIj9+4vLycH/7whzz77LPYbJH9j7Fq1Srq6+vDt/Ly8h63Q4iIBbtv6alp3HbxCG760vDIXxvKlPxe3B4vlfVuqhplHlx7uhWURo0axYIFC3jmmWdwu7v3we7cuZOqqipmzJiByWTCZDKxZcsWHnvsMUwmE4HA2ZdLrVYrycnJbW5C9JlQnVJXq7mhJSgBR6tqeO7DMt49WB2lhg0s3QpKu3btYsqUKaxYsYLs7Gy++93v8uGHH3bpHJdeeil79+5l9+7d4dvMmTO58cYb2b17N0ajbG0s4ky4Timx6681GMIrBdjQ1/aWaSbt61ZQmjZtGmvXruXEiRM8+eSTVFRUcNFFFzFp0iQeeeQRTp06dc5zOBwOJk2a1OaWmJjIkCFDmDRpUneaJUTvUVV9eyVgz0kPGz8sY095XdfOERxXsoaCkqw+2a4eXX0zmUxce+21vPjiizz44IMcOnSIlStXkp+fz80330xFRUW02ilEbPmD894UhdNeIxX1bpxef9fOEezCWbVQpiRBqT09CkofffQR3//+98nJyeGRRx5h5cqVHD58mM2bN3PixAmuueaaLp3vnXfe4dFHH+1Jk4ToHcFBbkw2PMFY1KWrb3B2puQPoEU6d24Q6VZJwCOPPML69es5cOAAixYt4umnn2bRokUYgvtfFRUVUVxcTGFhYTTbKkTsBKu5sSTiDo4FRVw4GRLMlCyaB7CjaeANqOEKb6HrVlBat24dy5YtY+nSpeTk5LR7TGZmJn/84x971Dgh4kZ43ltCeMvt7mZKxoAbo0EhoGp4/BKUztStoLR582YKCgrCmVGIpmmUl5dTUFCAxWJhyZIlUWmkEDHnbVlx0t3cxRUCQlqtqfSlEUMwGsBijPmkirjTraA0cuRIKioqyMzMbPN4TU0NRUVF7dYYCdGvte6+hTKlrnbfWlV1zxojO5p0pFtBqaPBuaampoirs4XoV7yhXUwSsJgMqJrW7UxJdjTpXJeC0ooVKwBQFIXVq1djt7dUtgYCAbZv3860adOi2kAh4kIwU1Isidx6URd2MWmtVabU6Pbh9ARIinSbpkGkS5/Gxx9/DOiZ0t69e7FYWnb5tFgsTJ06lZUrV0a3hULEg/CYUjequUNaZUrvfVHNgcpG5o/NYHqBLNnTWpeC0ttvvw3ALbfcwtq1a2XumRg8ejLvLURWn4xIt/LG9evXR7sdQsS34Ly3CpeBLQfLyEq2sWBc5jledIbwSgEerEYJSh2JOChde+21FBcXk5yczLXXXtvpsS+99FKPGyZE3Aj4we8BoDFgoaK+DlN3LuWHgpKmYTP4ANk8oD0RB6WUlJTwKnspKSm91iAh4k6o66YYcKlmoBvV3ABGExjNEPCRgMx/60jEQal1l026b2JQaTWe5PZ3s5o7xJwAAV+r5UskKJ2pW+WkLpeL5uaWnUGPHj3Ko48+yj//+c+oNUyIuNFqv7eWoNTNSuzQSgHBoOSW7ttZuvXJXnPNNTz99NMA1NXVMWvWLB5++GGuueYa1q1bF9UGChFz4UwpsWW/t55kSkCKWWV2UTrT8lOj0MCBpdsrT1588cUA/PnPfyY7O5ujR4/y9NNP89hjj0W1gULEXOt5b13dhPJMwUzJYfJxwaihTBom47Nn6lZQam5uxuFwAPDPf/6Ta6+9FoPBwJe+9CWOHj0a1QYKEXPheW92jAYFi8nQ/e5bqFZJppp0qNsbB7zyyiuUl5ezadMmLrvsMgCqqqqkoFIMPK2qub8yJZflC0YxOsvRvXMF1+nWfC5qnfquJgFVFnprrVtBafXq1axcuZLCwkJmz57NnDlzAD1rmj59elQbKETMeVsypR4zBXfX9bt5ZttRnvuwjCZPF5fVHeC6VdH9jW98g4suuoiKigqmTp0afvzSSy/la1/7WtQaJ0Rc8PVgF5MzmUNV3W6sZgN+TyC4q4m55+ceILo9PTk7O5vs7Ow2j82aNavHDRIi7gS7b27FyssflpFgNnLNtNzIt+xuzRQaU3JhNRlxegKyq8kZuhWUnE4nDzzwAG+++SZVVVWoatsPNRo76AoRFzQtXBLgUhKorK/BYjJ0LyBB20xJJuW2q1tB6bbbbmPLli185zvfIScnp9t/QevWrWPdunWUlpYCMHHiRFavXs2VV17ZrfMJEXV+D6h6GYBL05fqSehujRK0zZTsoaAkBZStdSsovf7667z22mtceOGFPXrzvLw8HnjgAUaPHo2maTz11FNcc801fPzxx0ycOLFH5xYiKkKFkyYLLlUPIgmWHgSl1pmSrBTQrm4FpbS0NNLTe77G8NVXX93m5/vvv59169axbds2CUoiPrTaqtvl1TOaqGRKagCbQT+fjCm11a2SgPvuu4/Vq1e3mf/WU4FAgI0bN+J0OsMlBmfyeDw0NDS0uQnRq0KZkjmh51NMQF8lwKC/vijVyOyidPLSEnraygGlW5nSww8/zOHDh8nKyqKwsBCzue3lzF27dkV8rr179zJnzhzcbjdJSUm8/PLLTJgwod1j16xZwz333NOdJgvRPd4m/d6ahCsYlHrUfVMUfaqJ10lRipGivKFRaOTA0q2gtHjx4qg1YOzYsezevZv6+nr+/Oc/s2TJErZs2dJuYFq1alV48wKAhoYG8vPzo9YWIc7iCQYlSxIK+hSTHnXfQJ9q4nWC39Xz9g1AihZnm5kvXLiQkSNH8rvf/e6cxzY0NJCSkkJ9fb1MbxG9Y/8/oGIPFF0MhRcB+sYZ3S4JANj1J6g/hm/8NTQ5RqFqGkOSrFFqcHzpzne029tz1tXV8Yc//IFVq1ZRU1MD6N2248ePd/eUAKiqisfj6dE5hIia8BSTpPBDPQpIEJ6UW11bT/H7pbz+aWXPzjfAdKv79sknn7Bw4UJSUlIoLS3l9ttvJz09nZdeeomysrLwWkvnsmrVKq688koKCgpobGxkw4YNvPPOO2zatKk7zRIi+ryN+n2roNRjweVLLJqsPtmebmVKK1asYOnSpXzxxRdtdsRdtGgR7777bsTnqaqq4uabb2bs2LFceuml7Nixg02bNvHlL3+5O80SIvq8Ldt1/2XnMV75+HjPJ9CaQ0FJ7xFI8WRb3cqUduzY0e6Yz7Bhw6isjDwV/eMf/9idtxeib6hqeN6bak6kvLYSTYMedt7CtUrhoORTUVUNg6HHZx4QupUpWa3WdmuEDh48SEZGRo8bJURc8DWDpoKi4DEkELok1KM6JQhnSuZg9w3AG5AuXEi3gtJXv/pV7r33Xnw+fe8qRVEoKyvjrrvu4utf/3pUGyhEzISruRNw+fWIZDUbMPY0owlmSga/G7NRP5dUdbfoVlB6+OGHaWpqIiMjA5fLxbx58xg1ahQOh4P7778/2m0UIja8LTVK4cLJnmZJ0Gr+m758Cci4UmvdGlNKSUlh8+bNbN26lT179tDU1MSMGTNYuHBhtNsnROy0DkrRmPcWYmpZp3tKXgoBVcPWkyrxAabLQUlVVYqLi3nppZcoLS1FURSKiorIzs7ueVGZEPGk1ZW3cFCKRvBolSnNHjGk5+cbYLrUfdM0ja9+9avcdtttHD9+nMmTJzNx4kSOHj3K0qVLZSlcMbCEgpI1iYCmYTEZsFu6vVhri1CmFPDrN9FGlz7h4uJi3n33Xd58800WLFjQ5rm33nqLxYsX8/TTT3PzzTdHtZFCxISnpXByWl4q0/JTUaOx84jJqk/M1TTcbicuErCYDCRaoxDwBoAuZUrPPfccP/3pT88KSACXXHIJd999N88++2zUGidETLXqvoVEpZZIUcJbLe344gTF75ey51hdz887QHQpKH3yySdcccUVHT5/5ZVXsmfPnh43Soi4EMqUrN3c460zwS6cTdHLamSqSYsu5Ys1NTVkZWV1+HxWVha1tbU9bpQQMadprdZScvC3PSfwqyrzxmSSnmjp+fnNNnCBDQ9gkzqlVrqUKQUCAUymjuOY0WjE75eBOzEAeJ36hgGKAhYH5bXNlFY3E7WVfkKZEqFMSeqUQrqUKWmaxtKlS7Fa21/7RZYcEQNGq0Fuv9ZScR21wehgWYAVWSngTF36hJcsWXLOY+TKmxgQPMG5nVYHzcFqbqNBCe/V1mPBTMkaXilAglJIl4LS+vXre6sdQsSXVoPczR49KNktxugVB5+5fIlPum8hUhghRHvCmVIyzV59nDQqhZMhZjsACYqXafmp2GWaSZgEJSHa0zpT8rZkSlETDEo2zc2CcZnRO+8AEKUOshADTCgo2ZIJqBpWsyHKQall+27RlmRKQrSnVaY0NTOVqfmp0SsHgJYqca8Tty9AszeAw2bCbJQ8QT4BIc6kae1Wc0d1BYxQpuR388wHJTz1fimnm7ydv2aQkKAkxJnaFE5GcReT1oJjSmgaSUa9gNIlV+AA6b4JcbZw4WQiGIz8dfdxNA3mjckgLRpTTAAMRn1Srt9DksEHGMNrNg12kikJcSZ3vX5vTUbTNMpON1NS7STq6xcGx5WSFL1WSTIlXUyD0po1azj//PNxOBxkZmayePFiDhw4EMsmCdESlBJS8fhV/ME1lJKivd5RcFwpwaCPJbklKAExDkpbtmxh+fLlbNu2jc2bN+Pz+bjssstwOp2xbJYY7Nx1+r0thUa3XjiZYDFiivaVseC4kp1gpiTdNyDGY0pvvPFGm5+Li4vJzMxk586dzJ07N0atEoOeq06/t6XiDO6G2yurQoaquoOTcqX7pourge76ej1tTk9Pb/d5j8fTZiWC9jbEFKLHQt03WwpNLj0oOXojKFn0oJRm9jEtP5UMR/urbww2cTPQraoqd955JxdeeCGTJk1q95g1a9aQkpISvuXn5/dxK8WAp2ltxpSa+iBTGmIJsGBcJpOGpUT/PfqhuAlKy5cv59NPP2Xjxo0dHrNq1Srq6+vDt/Ly8j5soRgUPI2g+kExBK++6bviRn2QG1pqlXzN0T93PxYX3bcf/OAH/P3vf+fdd98lLy+vw+OsVmuHC8wJERXhcgAHGIzMGTmEOSOHRHeKSUjw6pvmbcbtDeDyBUizmwf93okxzZQ0TeMHP/gBL7/8Mm+99RZFRUWxbI4QLVfeElLbPNwrgSJYp6R6nTyx5TBPvV8qi70R40xp+fLlbNiwgb/+9a84HA4qKysBfVvwhISEWDZNDFatBrl7XTBTMvrdWIwK3oCG2xfAFo2twfuxmGZK69ato76+nvnz55OTkxO+Pf/887FslhjMWpUDeP0qT39QyisfH8cf6IUMxhxcKUBTSTTqA+pSFhDjTKlX+ulC9ESrwskGt4/TTV6aPP7oF04CGE1gNEPAR5LBR63MfwPi6OqbEHGh+bR+bx9Cg0ufvZ9sM/fe+4XmvxncgGRKIEFJiBY+F3iDl+ft6TQEp5gkJ/RiUAqWBSQqMv8tRIKSECHNNfq9NQlM1laZUi+OcgQzJTt6ptQs3bf4qFMSIi606roBNLj1oJTSm5lScGXLbJufaemp5KbKVWcJSkKEuIKZUigoufqg+xbMlIbZVYbJriaAdN+EaBHKlBL0CeEWkwGLydDLA93B5Xa9Tb33Hv2MZEpChITGlOx6UPrGefqUp14tXQkGJc3TFJ5qkh6tJXf7KcmUhABQVXDV6n8Odt9CenUuWrD75nU18sSWwzz9QSmqOrjr9yQoCQF60aQaAIOpb6aYhFj1TMkcaEZBRdOkVkmCkhDQ6spbOigKO0prePqDUnaX1/Xu+5oTQVEwAA6DfrXP6fX37nvGOQlKQgA0ndTvk/QrYKebPJxu8uLrjTlvrRkM4QLKFKO+qmqzRzIlIURTlX6fqAelGqeetaTZe/HKW0hwXMlh0IOSZEpCCHCe0u+TMtA0jdpmfdpHmr0ProQFCyhD+78N9qpuCUpC+L0tV96Ssmj2BvD6VRSll6u5Q8KbUroACUoSlIRortY3DLAkgiWRGqeeJSXbzL2zZMmZgrVKWVYv0/JTGTbIp5pI8aQQofGk4CB3dZPejRqS1EdFjLZkALKtfrJlqolkSkK0DHJnAGA2GhjqsJKVbOub97fqQQlPfd+8X5yTTEmIM8oBJg1L6ds92ILFmpq7AY8vQKPbP6g3ppRMSQxuagAa9Q0rcOTGpg3BTEn1ufm/tz7nmW1HB/VibxKUxODmPKVvPmmygj2dgKr1/dwzkwXMNoyKQnLwClyje/DWKklQEoNbwwn9PjkXFIXDp5p4/O1DbNpX2bftCGZLaUY9KIW2Cx+MYhqU3n33Xa6++mpyc3NRFIVXXnklls0Rg1EoKDlyADjZ4Mavahj7epfa4LhSqiGUKfn69v3jSEyDktPpZOrUqTz++OOxbIYYzBor9PtkfTypol5fKzsntY+uvIUEM6VkRd+4YDB332J69e3KK6/kyiuvjGUTxGDm97SsDuDIIaBqVDUEg1JKHxcwBmuVHDKm1L9KAjweDx6PJ/xzQ0NDDFsj+r36Y3olty0FrEmcbnDjC2hYzYa+mYjbWjBTStScgHTf+o01a9aQkpISvuXn58e6SaI/qy/X71MLADgR6rql2Hp3tcn2BMeUHDQzLT+VcdnJffv+caRfBaVVq1ZRX18fvpWXl8e6SaI/qyvT70NBqU7vOmUnx2DuWUIaAElaMwtGpzE5rw+LN+NMv+q+Wa1WrNbBW+kqosjvhYbgIHeqnnEXpNvxBVSGD7H3fXvMCXqtlN8DrjpIyuj7NsSJfhWUhIiahmOgqfoAsy0ViMH0ktYURc+WGivxNlVTpzpIspmwWwbfVzSmv3FTUxOHDh0K/1xSUsLu3btJT0+noKAghi0TA17rrltfjx91JBiUPth3iF2qkS9PyIpdkIyhmAaljz76iAULFoR/XrFiBQBLliyhuLg4Rq0Sg0JtqX4fHE8qrXaSlmjpm0XdOhLcby4NfWPK0OqXg01Mg9L8+fN7d6M/IdrjaWoZT0ofgdev8rc9J/CrGjfPGc6QpBiNWwYHu1OCQamueXCWBfSrq29CREXNEf3ekQVWB2U1TvyqRqrdHNvdaUNX4NRGAOokUxJikKg5rN8PGQXAoSq9YHFERlLf1ye1Zh8KQKLWjFF1U9esoGlabNsUA5IpicFFDUBNif7n9JH4AypHqvXu0siMxBg2DDDbwJaM1WQgyV+HX9VoHISrBUhQEoNL3VG9FshiB0cOR6qdeHwqDpuJ3L6e79aexAwURSHDoC+NW+scfF04CUpicKn6XL/PGAcGA59X6PMnx+ckYzDEQTcpuE74hBQvc8cMJbUv9p2LMzKmJAaPgB9OHdD/nDker1/lWK0+tWR8TpzMNQsGpZF2FwxPj3FjYkOCkhg8ao7oXTdrEqTkY1EUbru4iPIaV2yvurUW3LyAppP6CgaDbJAbJCiJWFBVfXE15ylw1+uBIuDRp320Zk7UZ88n5+orQxp6ONpQsVu/z5wQ/rJbTUZGZSb17LzRZB8KRhOa30PNqROcUh2MznRgjIeuZR+RoCT6jrsByrejVu7D73GiAEaDgiGSbMCSCMNmwLCZ+lWqrnLVttQn5U7H6fFjtxjj73K7waAH4Loy/rXtY04kjGHIl6yDasslCUqi96kqlG+ncf/bHDpZh8en4jNYcVoycJuSMVkTsNjsFGXoWQGAXw1w7OQp7P4GEppPYHM1Yi55D058DKMvg4yxXWvD8V16dyh9BAFbGs+/X4rFZOCKSdkMjVUFd0eSc1Hqyskz1nICqKx3S1ASA8cXJxvR0Bcuc9hiMK/L54Z9L0NtKVajRpUhk+MZ06m3DQOlbXcsKSWd0fl6AWFTs5eXD5YCoGgB0ptLGN28myxzJUPrn8cxag7KyEvAYDx3G7xOPZgBDDuPT47VUe/ykWQ1kRyLz+RckvOA7WQrNQAcr3MNqvWVJCj1d6oK1Qfg9CF9HR6DSZ8+kTUZkjL45Fg9ZTX6YvSjMpO4eHTfXWauPFVNw/ZnGJPoAqMZy7gvM2H6KC5KsmK3GNE08PhVGt0+Gj3+NgFC06BwqB2nJ4DLG+C0MooaeyH59TvJPbGHXOcWhjdVwcSv6TVHnSnbBgEfOLJwJg1n26f6CgGzR6RjMcVhVUxwE4N06jEF3FTUx2Hg7EUSlPqz2lI4uAma9f9RA6rGsdpmhqUdwVS2HbInMSp9Jt6AjZMNbg5VNVFW08yl4zN7fbnVE9W1HPzn77G6T1Odl83QC24CRzatFzBWFEiwGEmwGMk84/VpiRa+Nj0v/LPbF6Ci3s0XJ9M5XJLNJNsuffmRXU/BpG90vCha40k49hEAWuFc3jpwCrcvQGaylUm5cZp9WJMgcSgOtYrUpmNUG0fh9PhJtA6Or+vg+C0HGk2Dkneh7AP9z+YE3BkT2VxupN7oZIL7JDPsp6DyU6baypg6YTGnjcN5a38Vx2pdvL63kmZvgBkFab3SvBM1jXy+eT1292nsSckkX3ALOIb06Jw2s5GioYkUDU3EPSYDm2cafPpncNVR9uY6GkcsYvzE89oWQPo98Pmr+lW9jLHscqZzqKoag6Lw5QlZ8VEs2ZH0EZic1RQoJ6lmFEdPNzMhN05qqXpZHOauolNqAD7/Gxx9Xw9IudNwn3cHf64dwyEtD2f6BLIvvBHOW6Kvz+NugN0bGOIu5+sz8pgxXA9EWw6coqTaGfXmnah1snfzn7A3nyApMZHxl9+OpYcB6Uw2s1HPjGYsocmey4nqBho+fJ63X3+eytO1+kHuevjkeXBWgyWRw6kX8N4X1QDMG5tBpqOP93XrqvQRAAynEjStV/6u4pVkSv1JwAefvqRf2lYMMPZKPBkTeXnXcU41erBbjFw3Mz9YCJgA5y2Fz17Vx5s+/QuGcVcxd/REbCYDTR4/hVFei/p4bTMfv/kCQ5qOkGy3MmbhUsypOVF9jzYsdhJn3kh24K+cOrANe+VHHPnrLk6lDiXX6sFuVkhIsMPk6xiWkMHQYx6GpSUwtT8MGqfkg8lCls3F10aq5Aw/s4M7cClaP15lraGhgZSUFOrr60lOHuCprd+rd1dqj4LRBBOvxZdaxMsfH+d4rQub2cg3zss7+9KxGoD9r8HJffogzujLYNiMNktiqKqGotCjmp1mr59Nr/2F7JodpCRYGL3gRsw5E3vyG0dO02iu2E/JjtdpqD4RfrjBlsvcq24iIVX/Qnv8ASxGQ/zVJnVk/2tQ8QnkTIVxi2Ldmm7pzndUMqX+wO+BvS9CXTkYzTDlekjN55+fVHC81oXFZODaGcPar2UxGGH81WCywfGd+sB4wItS8CVAHxz/x94KhiRamDNySLe/sPbqvVxg+IyaBDOjL/oapr4KSACKgj13PBOvGc/pU5UcPHqC0mYTNQE7E7UkQsPlVlME5QPxJHuyHpROfQ6jFoIpTqbC9DIJSvHO54JPXoCGE/oWPFOuh5RhAJxfmEZlg5srJ2WTldzJGImiwOgv6/+oj34Ah9/WA13RXEpPOzlU1cQhQAMu6GJgUlUNQ/V+OLiJrGQbmZMXoBSc37PfuQeGZGQzJyObOTFrQRSl5IM9Ha35NJ/vfIeP1DEsnj4sPmurokgGuuOZqxZ2/UkPSGYbTL0BLVjDApCZbGPpBYXkpkawDpCiwIj5+g30gfJD/2LkEDvzxuqX0z8sqeH9w6cjXjd934l6/vGvzfj2vqwPuudMRSma18VfUnRIUaBgDgoK/tJt1DU08eGRmli3qtdJUIpXpw/Drqeh+TRYHTDtRhotQ/nzzmMcq20OH9bliZrD5+jjSqDX7+x5jhlZZua3Ckyvf1qJyxvo8BRuX4DN+yr5bOtrDCn/J5V1Ln3cY8wVg3JWe6/KmggJaRQmQ0HddvadaOBkgzvWrepVcRGUHn/8cQoLC7HZbMyePZsPP/ww1k2KHW+zPu7zyQv6nx1ZaDNu5qAzgWe2lXGs1sU/950koPbg+kTeeTBxsT4+VVcGO/7AdNNRLhmbgUFROFDZyFMflHL0dMtlaE3TqHF62XbkNM+/8zHqx8+S37CL/LQE8qbMg7FX9nwWvzibwQhjryDZZmYihxnSuJ9/7K2g2Ttwl8mN+ZjS888/z4oVK3jiiSeYPXs2jz76KJdffjkHDhwgM3PwXAbFXQ8Ve/Tsxe8BQBt2HiXJ5/PRp/UcDy5GlpVsY9Hk7J4vZZE5HhIz4bOXoekUHHidqfZ08vImsrkqmSpfQps1hl7dfYzqY4fIdB5gtPMwiRYDw/PSSZ16tT4gK3pPWiEMn0ORupWGY//mcKCJv3yksmhKXuy2g+pFMS8JmD17Nueffz6//e1vAVBVlfz8fP7zP/+Tu+++u9PX9suSADUAvmZ9kqizWl9TqLaUQHAfMqOiQFIGJalz2Fxhw+nRu1Fmo8J5w9M5vzANkzGKGYka0GfQl77XEgw1jUYSSE5J1/+n9ns4WFpGbaMTh81MpsNKesEEDGO+HN4WSPQyTYMvNuMq/ZDPTjTQiJ36lDEsuvhLmJOGgtWBqhF3Ver9riTA6/Wyc+dOVq1aFX7MYDCwcOFCPvjgg7OO93g8eDye8M8NDQ2RvVFNCRz619mPt4nHWvix004PR041hR5B0TQ0QEFD0zSKMpLICGYRdc1eDpxs1F+vnX2u4UPsZCfbAI36Zg9fnDh9VhM0TSOgwbDCsRRMugiGjsVc78ZZegyr2cCk3BSmFaT2zlUXgxHyz9fHhE7uhZOfoTScIFlz6wPsQSPSLRiy0zBkjIbcGZDci0WR4myKAmMuIyE5h0m2tyg5XklR4DPMe48CoAIflDaiGc0YjGZ9jSpFAfT7oQ5beDG7gKrx0dHaDt8qLdHCmEgWvpv8TbBFPxmIaVCqrq4mEAiQlZXV5vGsrCz2799/1vFr1qzhnnvu6fob+T16VhIplwe/s6nDpzVXQK/7ATSPF9Xd2PG5fAr4gkHK78MX0MObz2DDZU7BZU6jyZJJXUI+SlE+BZn6lIzclAS+PiOPYWkJfbPqoMkCw87Tb36PPsnX06BHTYMJkz0dbKkybhRr2ZOxZoxn3Kn9qNWHwHkSXHV4vD4MqhdUL/ha/f8YYrBCkz4EgKahNnZyFU+1gN117racuVJolMR8TKkrVq1axYoVK8I/NzQ0kJ+f38krglLzYeq3zrgyFPxzO48l+FQKnN629TqKEnxaISnBDBYTKAoJvgAj3P7gq5XwoZqi/5RoDR4LJKgKo/1GMNnanNtkMGCzGLC06pYZDAoFUZ4GEjGTNZgJSTYUl4wmyJ6EIXuS/rMawOZrZprLjc/jwuv3E9BUUDW0YHafYFIgQc+0FQ0KCju+gmczGyCSNcstvbNPXkyD0tChQzEajZw8ebLN4ydPniQ7O/us461WK1ZrNwb2LImQXhTx4XbA3sFKGGeyBW8RNQMYGnErhIiQwYhidZBgdRDJznUGYFh050hHVUxzcYvFwnnnncebb74ZfkxVVd58803mzBkQNblCiC6KefdtxYoVLFmyhJkzZzJr1iweffRRnE4nt9xyS6ybJoSIgZgHpeuvv55Tp06xevVqKisrmTZtGm+88cZZg99CiMEh5nVKPdEv65SEGES68x2V67tCiLgiQUkIEVckKAkh4krMB7p7IjQcFvF0EyFEnwp9N7sydN2vg1Jjoz69I6KqbiFEzDQ2NpKSEtmGDf366puqqpw4cQKHw9F/FoOPstBUm/LycrkC2Q3y+fXMuT4/TdNobGwkNzcXQ4TzJvt1pmQwGMjLyzv3gYNAcnKyfKl6QD6/nuns84s0QwqRgW4hRFyRoCSEiCsSlPo5q9XKz3/+8+6tniDk8+uh3vj8+vVAtxBi4JFMSQgRVyQoCSHiigQlIURckaAkhIgrEpT6oZqaGm688UaSk5NJTU3l1ltvpamp491XAObPn4+iKG1u3/ve9/qoxbHV1R2YX3zxRcaNG4fNZmPy5Mn84x//6KOWxqeufH7FxcVn/Tuz2SJdxT5IE/3OFVdcoU2dOlXbtm2b9t5772mjRo3Sbrjhhk5fM2/ePO3222/XKioqwrf6+vo+anHsbNy4UbNYLNqTTz6p7du3T7v99tu11NRU7eTJk+0ev3XrVs1oNGq/+tWvtM8++0z7n//5H81sNmt79+7t45bHh65+fuvXr9eSk5Pb/DurrKzs0ntKUOpnPvvsMw3QduzYEX7s9ddf1xRF0Y4fP97h6+bNm6f98Ic/7IMWxpdZs2Zpy5cvD/8cCAS03Nxcbc2aNe0e/81vflO76qqr2jw2e/Zs7bvf/W6vtjNedfXzW79+vZaSktKj95TuWz/zwQcfkJqaysyZM8OPLVy4EIPBwPbt2zt97bPPPsvQoUOZNGkSq1atorm5ubebG1OhHZgXLlwYfqyzHZhB/3xbHw9w+eWXd3j8QNadzw+gqamJ4cOHk5+fzzXXXMO+ffu69L79ekLuYFRZWUlmZmabx0wmE+np6VRWVnb4um9/+9sMHz6c3NxcPvnkE+666y4OHDjASy+91NtNjpmu7sAM+ufb3vGdfbYDVXc+v7Fjx/Lkk08yZcoU6uvreeihh7jgggvYt29fxJPnJSjFibvvvpsHH3yw02M+//zzbp//jjvuCP958uTJ5OTkcOmll3L48GFGjhzZ7fMK0dqcOXPa7Nl4wQUXMH78eH73u99x3333RXQOCUpx4r//+79ZunRpp8eMGDGC7Oxsqqqq2jzu9/upqalpd1fhjsyePRuAQ4cODdig1NUdmAGys7O7dPxA1p3P70xms5np06dz6NChiN9XxpTiREZGBuPGjev0ZrFYmDNnDnV1dezcuTP82rfeegtVVcOBJhK7d+8GICcnJ9q/Stzozg7Mc+bMaXM8wObNmwfljs3R2ME6EAiwd+/erv0769EwuYiJK664Qps+fbq2fft27d///rc2evToNiUBx44d08aOHatt375d0zRNO3TokHbvvfdqH330kVZSUqL99a9/1UaMGKHNnTs3Vr9Cn9m4caNmtVq14uJi7bPPPtPuuOMOLTU1NXyZ+jvf+Y529913h4/funWrZjKZtIceekj7/PPPtZ///OeDviSgK5/fPffco23atEk7fPiwtnPnTu1b3/qWZrPZtH379kX8nhKU+qHTp09rN9xwg5aUlKQlJydrt9xyi9bY2Bh+vqSkRAO0t99+W9M0TSsrK9Pmzp2rpaena1arVRs1apT24x//eFDUKWmapv3mN7/RCgoKNIvFos2aNUvbtm1b+Ll58+ZpS5YsaXP8Cy+8oI0ZM0azWCzaxIkTtddee62PWxxfuvL53XnnneFjs7KytEWLFmm7du3q0vvJ0iVCiLgiY0pCiLgiQUkIEVckKAkh4ooEJSFEXJGgJISIKxKUhBBxRYKSECKuSFAS/UpxcTGpqanhn3/xi18wbdq08M9Lly5l8eLFfd4uET0SlES7li5d2uGSucuXL0dRlDYTiHsjGBQWFvLoo4+2eez666/n4MGDHb5m7dq1FBcXh3+eP38+d955Z1TbJXqXBCXRofz8fDZu3IjL5Qo/5na72bBhAwUFBTFpU0JCwlnrSbWWkpLSJpMS/Y8EJdGhGTNmkJ+f32YhuJdeeomCggKmT5/eo3O3l8EsXrw4nH3Nnz+fo0eP8qMf/Si8AD2c3X07U+uMbenSpWzZsoW1a9eGz1FSUsKoUaN46KGH2rxu9+7dKIrSpSU2RO+QoCQ6tWzZMtavXx/++cknn+SWW27p9fd96aWXyMvL495776WiooKKiooun2Pt2rXMmTOH22+/PXyOgoKCs34ngPXr1zN37lxGjRoVrV9BdJMEJdGpm266iX//+98cPXqUo0ePsnXrVm666aZef9/09HSMRiMOh4Ps7OxuLbKWkpKCxWLBbreHz2E0Glm6dCkHDhwIbxXk8/nYsGEDy5Yti/avIbpBVp4UncrIyOCqq66iuLgYTdO46qqrGDp0aKyb1SO5ublcddVVPPnkk8yaNYu//e1veDwerrvuulg3TSCZkojAsmXLKC4u5qmnnopaNmEwGDhz1RyfzxeVc0fitttuCw/ir1+/nuuvvx673d5n7y86JkFJnNMVV1yB1+vF5/Nx+eWXR+WcGRkZbcaJAoEAn376aZtjLBYLgUCgR+/T0TkWLVpEYmIi69at44033pCuWxyR7ps4J6PRGN5JxWg0dnhcfX19eO3vkCFDhpCfn3/WsZdccgkrVqzgtddeY+TIkTzyyCPU1dW1OaawsJB3332Xb33rW1it1m51GwsLC9m+fTulpaUkJSWRnp6OwWAIjy2tWrWK0aNHD8o1uOOVZEoiIsnJySQnJ3d6zDvvvMP06dPb3O655552j122bBlLlizh5ptvZt68eYwYMYIFCxa0Oebee++ltLSUkSNHkpGR0a12r1y5EqPRyIQJE8jIyKCsrCz83K233orX6+2Tq4kicrIcrhi03nvvPS699FLKy8vP2nBRxI4EJTHoeDweTp06xZIlS8jOzubZZ5+NdZNEK9J9E4POc889x/Dhw6mrq+NXv/pVrJsjziCZkhAirkimJISIKxKUhBBxRYKSECKuSFASQsQVCUpCiLgiQUkIEVckKAkh4ooEJSFEXJGgJISIK/8fFxiUbZ0CEC4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:08:21.847341Z", + "iopub.status.busy": "2024-02-29T13:08:21.847043Z", + "iopub.status.idle": "2024-02-29T13:08:22.009657Z", + "shell.execute_reply": "2024-02-29T13:08:22.008722Z" + }, + "papermill": { + "duration": 0.186924, + "end_time": "2024-02-29T13:08:22.011719", + "exception": false, + "start_time": "2024-02-29T13:08:21.824795", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2IUlEQVR4nO3deVwT97o/8E8SdmSRsgQtCoKK/RXFYkU87qzSekC0deEoWqutra0trnhv9Xi8vdaKKLUeuZ5T8WqxeqoWraKSulIXRNRWEFEU1CqIihBZDCF8f394M4fIlmggZOZ5v168JN95ZvIMkzzO8p3viBhjDIQQwnNiQydACCHtgYodIUQQqNgRQgSBih0hRBCo2BFCBIGKHSFEEKjYEUIEgYodIUQQTAydQEdXX1+Pe/fuwcbGBiKRyNDpEEIaYIzhyZMn6NKlC8TilvfdqNi14t69e3BzczN0GoSQFty5cwevvvpqizFU7FphY2MD4Nkf09bW1sDZtB2lUon09HSEhITA1NTU0OmQlySU7SmXy+Hm5sZ9T1tCxa4V6kNXW1tb3hc7Kysr2Nra8vrLIRRC257anGKiCxSEEEGgYkcIEQQqdoQQQaBiRwgRBCp2hBBBoGJHCBEE6npCCA9UV1fj6tWr3OvKGgVOX76Bzo7n0cnSXCPW29sbVlZW7Z2iwVGxI4QHrl69Cj8/v0btXzcRm52djTfeeKPtk+pgqNgRwgPe3t7Izs7mXucXlyP2x8tIeMcHvV3tG8UKERU7gSh8WIUqRR33uqamGoUF17jXKpUKly7fQLVJBiQSCdfu4dULlpb/PuSxNjeBh6N1+yRNtGZlZaWxtya+9QjmGTXo83o/+HZ/xYCZdRxU7ATg6v1HCN+4W6Ot9uEdPNof3+q8r7w9H2aOmgMhpM0eB28X+gIR40LFTgCul92Atcd6jTZrD6Dzm15azJ3axPJ8qdgRo0PFTgC6WHdHVeEnSJzgC0/nTk3G1NXV4dSvp/CnIX+CiUnTH4sbpZWYu/MSuozs3pbpEi09f2qioRsPqrh/m9uegLBOS1CxEwBziQXqn3aFh21vvPaKXZMxSqUShSaF6OPQp9lRMuqfVqD+6QOYSyzaMl2ihcKHVRgZf7zVuHm7Lrcac2z+CEEUPCp2hBgh9R7dugm+8Gpib72qRoH9x8/g7REBsH6un51aQWklPtt5qdm9Q76hYkeIEfNy7oTXuzbeW1cqlShxAt7o3lkQ49lpg24XI4QIAu3ZEWKEFKqnEFvcRaE8H2KLxoexdXV1uFd3D3llec1eoCiUV0JscRcK1VMATZ/L5RMqdoQYoXtVt2DtsR5LzrUc9/dDf29xurUHcK/KF35w0WN2HZPRFbsNGzZg9erVKCkpQb9+/bB+/XoMHDiwydjc3FwsXboU2dnZuHXrFtauXYvPPvusfRMmpA201p2IuhI1ZlTFbufOnYiNjUVSUhL8/f2xbt06hIaGIj8/H87Ozo3iq6ur0aNHD7zzzjv4/PPPDZAxIW2jte5E1JWoMaO6QJGQkICZM2di+vTpeO2115CUlAQrKyts3ry5yfg333wTq1evxsSJE2Fu3vTld0KIMBjNnl1tbS2ys7MRFxfHtYnFYgQFBeHMmTMGzKzjq1GqAAA5dyuajamqUeD8A0B663GL/bIIMVZGU+wePnwIlUoFFxfNE6kuLi4agxa+LIVCAYVCwb2Wy+UAnh0WKJVKvb1Pe7pW/KzILd7TWm96E2wryGp1eeYSZrR/C76oq6vj/m1qW6jbWtpOrS3DGOiSt9EUu/aycuVKLF++vFF7enq68Y7uqgQm9hDB2ZLBrJkTF/drgG0FJpjiVQcXy+YXZS4BrmSewJW2yZRo6U4lAJjg119/xa2mb3cGAMhkspdeRkdWXV2tdazRFDtHR0dIJBLcv39fo/3+/fuQSqV6e5+4uDjExsZyr+VyOdzc3BASEgJbW1u9vU97e7eV6b/dLsO2gvOIGDkI/bo5tEtO5MXl3pMj/vJZDBkyBP+vS+PPpVKphEwmQ3BwcLMXKFpbhjFQH3lpw2iKnZmZGfz8/HDkyBFERkYCAOrr63HkyBHMmTNHb+9jbm7e5MUMU1NTXt92o+6eYGJiwuv15Attt1dLn1s+bHNd8jaaYgcAsbGxiImJwYABAzBw4ECsW7cOVVVVmD59OgBg6tSp6Nq1K1auXAng2UWNK1eucL/fvXsXly5dQqdOneDlpc1YboQQvjCqYjdhwgQ8ePAAS5cuRUlJCXx9fXHo0CHuosXt27chFv/7pNS9e/fQv39/7nV8fDzi4+MxfPhwHD9+vL3TJ4QYkFEVOwCYM2dOs4etzxcwd3d3MMbaIStCSEdnVJ2KCSHkRVGxI4QIAhU7QoggULEjhAiC0V2gIIS0fr8z3evcGBU7QozQjf8rVC3f76zdvc7W5sIoA8JYS0J4JuT/PbtF0tO5EyxNJY2m5xdXYN6uy1gz3ge9XZsfcp2eG0sI6dAcrM0wcWC3ZqerRzTxdLJu8uljQkQXKAghgkDFjqC2thYp321EmSwJKd9tRG1traFTIkTvqNgJ3MKFC2FtbY34FV/gyYX9iF/xBaytrbFw4UJDp0aIXtE5OwFbuHAhVq9eDRcXF4T8eRz2FzG87S5C+r7dWL16NQDg66+/NnCWhOiHiNGd8i2Sy+Wws7NDRUWFUQ/e+bzy8nI4OjrC0tISVlZWKC0t5aY5OzujuroaNTU1ePjwIezt7Q2XKHkhl249QuTGs0idPQi+3V8xdDptRpfvJx3GCtSKFSugUqlQWVmpUegAoLS0FJWVlVCpVFixYoWBMiREv6jYCVRlpXa957WNI6Sjo2InUPX19XqNI6Sjo2InUNeuXdNrHCEdHV2NFaiLFy9yv4tEIkyePBl+fn7Izs7G9u3buRGeG8YRYsyo2AnUkydPNF6npKQgJSUFACAWi7li93wcIcaKip1AiUQirqCFhobCy8sL165dQ69evVBQUIBDhw5xcYTwARU7gbK1tUVFxbOx0NSFDQDS09MbxRHCB0Z3gWLDhg1wd3eHhYUF/P39ce7cuRbjf/zxR3h7e8PCwgI+Pj5IS0trp0w7tvnz5+s1jpCOzqiK3c6dOxEbG4tly5bhwoUL6NevH0JDQxt1ilU7ffo0Jk2ahBkzZuDixYuIjIxEZGQkcnJy2jnzjkfbe1/pHlnCF0Z1u5i/vz/efPNNfPvttwCe9QFzc3PDJ598gsWLFzeKnzBhAqqqqrB//36ubdCgQfD19UVSUpJW78nX28WAf98b25wFCxbQvbFGorq6GlevXuVe5xeXI/bHy0h4xwe9Xe01Yr29vWFlZdXOGbYNXb6fRnPOrra2FtnZ2YiLi+PaxGIxgoKCcObMmSbnOXPmDGJjYzXaQkNDkZqa2uz7KBQKKBQK7rVcLgcAKJVKKJXKl1iDjufLL7+ESqVCYmIiVCoV1y6RSDB37lx8+eWXvFtnvsrJyYG/v3+j9sn/2zg2MzMT/fv3b4es2p4un0+jKXYPHz6ESqWCi4uLRruLi4vG/2gNlZSUNBlfUlLS7PusXLkSy5cvb9Senp7Om/8NGxo2bBgGDRqEQ4cOoaSkBFKpFGFhYTAzM6Pzm0ZEoVBgzZo13GtlPVD2FHCwAEyfO1lVVFSE4uLids6wbVRXV2sdazTFrr3ExcVp7A3K5XK4ubkhJCSEd4exDb311luQyWQIDg6GqampodMhL0mpVApie6qPvLRhNMXO0dEREokE9+/f12i/f/8+pFJpk/NIpVKd4gHA3Nwc5uaNHz1namrK6w+NmlDWUyj4vj11WTejuRprZmYGPz8/HDlyhGurr6/HkSNHEBAQ0OQ8AQEBGvEAIJPJmo0nhPCX0ezZAUBsbCxiYmIwYMAADBw4EOvWrUNVVRWmT58OAJg6dSq6du2KlStXAgDmzp2L4cOHY82aNXjrrbewY8cOnD9/Hps2bTLkahBCDMCoit2ECRPw4MEDLF26FCUlJfD19cWhQ4e4ixC3b9+GWPzvndXBgwdj+/bt+M///E8sWbIEPXv2RGpqKl5//XVDrQIhxECMqp+dIfC5n11DSqUSaWlpCA8P5/U5HqEQyvakYdkJIeQ5VOwIIYJAxY4QIghU7AghgkDFjhAiCFTsCCGCQMWOECIIVOwIIYJAxY4QIghU7AghgkDFjhAiCFTsCCGCQMWOECIIVOwIIYJAxY4QIghU7AghgkDFjhAiCFTsCFQqFU6cOIGTJ0/ixIkTGg/MJoQvqNgJ3J49e+Dp6Yng4GAkJCQgODgYnp6e2LNnj6FTI0SvdC52N2/ebIs8iAHs2bMH48aNQ2lpqUZ7aWkpxo0bRwWP8IrOxc7LywsjR47E999/j6dPn7ZFTqQdqFQqfPjhhwCAUaNGITExEXPmzEFiYiJGjRoFAJg9ezYd0hLe0LnYXbhwAX379kVsbCykUik++OADnDt3ri1y01BWVobo6GjY2trC3t4eM2bMQGVlZYvzbNq0CSNGjICtrS1EIhHKy8vbPE9jcfz4cTx48AB9+vRBTk4O5s6di2+//RZz585FTk4OvL29UVpaiuPHjxs6VUL0Qudi5+vri8TERNy7dw+bN29GcXExhgwZgtdffx0JCQl48OBBW+SJ6Oho5ObmQiaTYf/+/Th58iRmzZrV4jzV1dUICwvDkiVL2iQnY6YuYnl5eejbty8yMjLwww8/ICMjA3379sXVq1c14ggxeuwlPX36lCUkJDBzc3MmEomYubk5mzJlCrt3797LLppz5coVBoBlZWVxbQcPHmQikYjdvXu31fmPHTvGALDHjx/r/N4VFRUMAKuoqNB53o5syZIlDAALCAhgtbW1TCaTsdjYWCaTyVhtbS3z9/dnANiSJUsMnSp5AbW1tSw1NZXV1tYaOpU2pcv30+RFi+T58+exefNm7NixA9bW1pg/fz5mzJiBP/74A8uXL0dERITeDm/PnDkDe3t7DBgwgGsLCgqCWCxGZmYmxo4dq5f3AQCFQgGFQsG9lsvlAJ49dFipVOrtfQzNzs4OAHDv3j307NkTt27dAgAkJCSge/fuYP/37HQ7OzterbdQqLcZ37edLuunc7FLSEhAcnIy8vPzER4ejq1btyI8PBxi8bMjYg8PD2zZsgXu7u66LrpZJSUlcHZ21mgzMTGBg4MDSkpK9PY+ALBy5UosX768UXt6ejqsrKz0+l6GVFxcDAC4desW7OzsMHjwYFhYWODp06fIzc1FRUUFF5eWlmbIVMlLkMlkhk6hTVVXV2sdq3Ox27hxI9577z1MmzYNrq6uTcY4Ozvju+++a3VZixcvxqpVq1qMycvL0zXFlxIXF4fY2FjutVwuh5ubG0JCQmBra9uuubQlCwsLrFu3DgBQUVGB06dPNxkXFhbGXZ0lxkOpVEImkyE4OBimpqaGTqfNqI+8tKFzsZPJZOjWrRu3J6fGGMOdO3fQrVs3mJmZISYmptVlzZs3D9OmTWsxpkePHpBKpY36gtXV1aGsrAxSqVTXVWiRubk5zM3NG7Wbmpry6kNjYqLdpjcxMeHVegsN3z63z9Nl3XQudp6eniguLm50WFlWVgYPDw+d+mU5OTnBycmp1biAgACUl5cjOzsbfn5+AICjR4+ivr4e/v7+uq0AAfDsXJ2aSCSCl5cXRCIRGGMoKCjgztk1jCPEmOnc9UT9JXheZWUlLCwsXjqhpvTp0wdhYWGYOXMmzp07h1OnTmHOnDmYOHEiunTpAgC4e/cuvL29NS6KlJSU4NKlSygoKAAAXL58GZcuXUJZWVmb5GlMTp48yf3OGMP169dx7do1XL9+XWMbN4wjxJhpvWenPo8lEomwdOlSjZP1KpUKmZmZ8PX11XuCaikpKZgzZw4CAwMhFosxbtw4fPPNN9x0pVKJ/Px8jROWSUlJGhcbhg0bBgBITk5u9fCZ786ePcv97uzsjOjoaFRVVcHa2hopKSncaYOGccQ41NTUIDY2FmfPnsWhQ4eQkJAAS0tLQ6dlcCLW3K7ac0aOHAkAOHHiBAICAmBmZsZNMzMzg7u7O+bPn4+ePXu2TaYGIpfLYWdnh4qKCl5doOjVqxeuX78OAHjrrbcQEhKC69evo2fPnkhPT8eBAwcAAD179sS1a9cMmSrRQWRkJPbu3duoPSIiAqmpqe2fUBvT6fupaye+adOm8a6DbUv42ql40KBBDAATi8VMIpEwANyPiYkJE4vFDAAbNGiQoVMlWoqIiNDYjs//REREGDpFvWvTTsXJycm6zkI6oL59++Ls2bOor6+HRCLBhAkTYGVlherqauzatQv19fVcHOn4ampqmtyja2jv3r2oqakR7CGtVsUuKioKW7Zsga2tLaKiolqMpWGBjEOvXr2431UqFXbu3NlqHOm4PvnkE63j/vnPf7ZxNh2TVldj7ezsIBKJuN9b+iHGwcfHB0Dz/ZTU/fDUcaRj27Vrl8br0NBQfPXVVwgNDW0xTki02rNreOhKh7H88OjRIwDN31tYV1enEUc6NvXtfQBQVVUFU1NTpKWlITY2FkqlEtbW1o3ihIaGZReo5m71e9E40nE8f3eMtnfL8J1WXU/69+/PHca25sKFCy+dVEfC164nNTU1Wg1sUF1dLdgT2sak4ffT1NQUc+fOhYeHBwoLC5GYmKixB6/FV95o6PL91KrkR0ZG6iMv0oE07JDt6OiIKVOmcJ2Kt23bhocPH3JxixYtMlSaREt+fn7Izs4G8OzURHx8fLNxQqV1p2Kh4uuenY+PD3JycmBra4vKykquqwkAiMViWFtb48mTJ3j99ddx+fJlA2ZKtFFRUQF7e/tW48rLy3l1IVGX7yedsxMo9fM45HJ5o1FezM3N8eTJE4040rHZ2dnB09OzxRhPT09eFTpdaVXsHBwcuMOazp07w8HBodkfYhwa3tY3atQojWdQNBy/jm+3//FZQUFBswXP09OTGxBDqLQ6Z7d27VrY2Nhwv2t7sYJ0XMHBwTh27BgAIDMzE5cvX4aFhQUuX76MzMxMjThiPAoKClBRUYHRo0dz9zofPHhQ0Ht0aloVu4YDcQp9tBC+aPgYyocPH+Kjjz5qNY4YBzs7O5w4cQJpaWkIDw/n9eCdutD5nJ1EImk0ajDwrPOpRCLRS1Kk7T0/0vTLxhHS0elt8E6FQqEx7BPp2EaMGAEA8Pb2Rrdu3TSmde/eHd7e3hpxhBg7rbtWq/tliUQi/POf/0SnTp24aSqVCidPnuS+IKTjGzFiBJydnXH16lW89dZbmDdvXqPx7JydnanYEd7QutitXbsWwLM9u6SkJI1DVvXgnUlJSfrPkLQJiUSCjRs3Yvz48Th69Cg3WCcAWFlZQSQSYePGjXRqgvCG1sWusLAQwLMRi/fs2YPOnTu3WVKkfURFRWHXrl2YN28eioqKuHYXFxfEx8e3OpwXIcZE5zuE1d0VCD9ERUUhIiICx44dw8GDBzF69GiMHDmS9ugI7+hc7N57770Wp2/evPmFkyGGIZFIMHz4cFRVVWH48OFU6Agv6VzsHj9+rPFaqVQiJycH5eXl9OR4QkiHpXOx++mnnxq11dfXY/bs2a3em/cyysrK8Mknn+Dnn3/mHqWYmJiocVX4+fhly5YhPT0dt2/fhpOTEyIjI7FixQrqTf6c2tparF+/HkePHkVBQQE++eQT6kZE+EdfT/m5evUqk0ql+lpcI2FhYaxfv37s7NmzLCMjg3l5ebFJkyY1G3/58mUWFRXF9u3bxwoKCtiRI0dYz5492bhx43R6X74+XUxtwYIFzMTEpNHTxRYsWGDo1MhLqK2tZampqay2ttbQqbQpXb6feit2Bw4cYI6OjvpanIYrV64wACwrK4trO3jwIBOJROzu3btaL+df//oXMzMzY0qlUut5+FzsFixYwAAwFxcXlpSUxJKTk1lSUhJzcXFhAKjgGTEqdo3pfBgbGxv7/J4hiouLceDAAY17aPXpzJkzsLe3x4ABA7i2oKAgiMViZGZmYuzYsVotRz3mVUvDVCsUCigUCu61XC4H8OzcZHPPazBGtbW1WLt2LZydnVFYWAjGGGQyGYKDgzF16lS4u7tj7dq1WLZsGR3SGiH1Z5VPn9mm6LJ+Ohe7ixcvarwWi8VwcnLCmjVrWr1S+6JKSkrg7Oys0WZiYgIHBweUlJRotYyHDx9ixYoVmDVrVotxK1euxPLlyxu1p6enazWMubHYt28f6urqMH78eKSnp3PtMpkMADBu3Dhs3LgRn376Kf785z8bKk3yktTbk6+qq6u1jjVoP7vFixdj1apVLcbk5eW99PvI5XK89dZbeO211/DXv/61xdi4uDiNvVe5XA43NzeEhITwaqTiw4cPA3i2DaRSKZRKJbdnZ2pqCl9fX2zcuBHm5uYIDw83cLZEF7W1tdiwYQNOnDiB4cOH4+OPP+bt3rn6yEsbBn3s0Lx581odMqpHjx6QSqWNRlqpq6tDWVkZpFJpi/M/efIEYWFhsLGxwU8//dTqcDfm5uaNRu4Fnj3EhE9D5agffn348GG8//77XLt6PdXFsFevXrxab75buHAhEhISoFKpAABpaWlYsmQJYmNj8fXXXxs4O/3T6bPZ5mcQ9UB9geL8+fNc2+HDh1u9QFFRUcEGDRrEhg8fzqqqql7ovfl6gUKhUDATExPm4uLCHj9+zMaMGcO6d+/OxowZwx4/fsxcXFyYiYkJUygUhk6VaEl9wam5Hz5ecDLI1di2FhYWxvr3788yMzPZr7/+ynr27KnR9eSPP/5gvXv3ZpmZmYyxZ38Ef39/5uPjwwoKClhxcTH3U1dXp/X78rXYMSbMLwdfKRQKJhKJWtyeIpGId/956fL9NJqRGVNSUuDt7Y3AwECEh4djyJAh2LRpEzddqVQiPz+fO2F54cIFbrhxLy8vuLq6cj937twx1Gp0KMePH3+p6aTj+Oabb1p9HixjTOMRmkKjt0cp/vHHH/jb3/6mUYD4gK+PUqysrOSeK9KSJ0+eNHuXCuk4AgICcPbs2VbjBg0ahDNnzrRDRu3DII9SfPToEb777jt9LY60scmTJ+s1jhjWzZs39RrHRwa9GksMR9sHX9MDso2D+jm/APDKK69g5MiRKCsrg4ODA44dO4ZHjx41ihMaKnYCVVtbq9c4YlgN7yR49OgRdu3a1Wqc0BjNBQqiX9qcr9MljpCOTus9u9aG6C4vL3/ZXEg7qqmp0WscMSypVIo//vhDqzih0rrYtTYGnJ2dHaZOnfrSCZH2IRKJ9BpHDKtv375aFbu+ffu2QzYdk9bFLjk5uS3zIIS8hJZG8nmROD6ic3YCpW3fOepjZxy07QPKp76iutK6zGs7fBM9cMc4+Pj4IDc3V6s40vFNmTIF33//PSwsLFBbW4v6+npumlgshpmZGZ4+fYopU6YYMEvD0rrYbdmyBd27d0f//v1bvS2FdHx+fn7YsWOHVnGk4wsMDIStrS3kcjmcnZ0xdOhQPH78GJ07d0ZGRgZKS0tha2uLwMBAQ6dqMFoXu9mzZ+OHH35AYWEhpk+fjr/85S9wcHBoy9xIG3J1ddVrHDEsiUSC5ORkjBs3Dg8ePMDu3bu5aeqLTMnJyYJ+TKbW5+w2bNiA4uJiLFy4ED///DPc3Nzw7rvv4vDhw7SnZ4S6du2q1zhieFFRUdi9ezfc3Nw02rt164bdu3e32n2M7154IIBbt25hy5Yt2Lp1K+rq6pCbm8vLk9l8HQigtrYW1tbWsLa2hr29PW7dusVNc3d3x+PHj1FVVYWqqirejnLLVyqVCseOHcPBgwcxevRojBw5krd7dLp8P1/4OrRYLIZIJAJjjBsVlRiP06dPo66uDnK5HEOHDkVsbCyuX7+Onj17QiaT4cCBA2CM4fTp0xgxYoSh0yU6kEgkGD58OKqqqjB8+HDeFjpd6dT1RKFQ4IcffkBwcDB69eqFy5cv49tvv8Xt27d5uVfHZ8XFxQCAbdu2IScnB3PnzsW3336LuXPnIjc3F9u2bdOII8TYab1n99FHH2HHjh1wc3PDe++9hx9++AGOjo5tmRtpQ+oLD56enigoKGh02HPu3DmNOEKMndbn7MRiMbp164b+/fu3eAvRnj179JZcR8DXc3YqlQpeXl7w8fFBamoqVCoV0tLSEB4eDolEgsjISOTk5OD69et0GGSElEoltz35/MCkNjlnN3XqVLpPkkckEgnWrFmD8ePHIzIyEgsWLEBNTQ3Onj2L1atXY//+/di1axcVOsIbOnUqJvwSFRWFXbt2Yd68eRg2bBjX7uHhgV27dgm+qwLhF+HeFUwAPCt4ERERgumqQISLih2hrgpEEIxm1JOysjJER0fD1tYW9vb2mDFjBiorK1uc54MPPoCnpycsLS3h5OSEiIgIXL16tZ0yJoR0JEZT7KKjo5GbmwuZTIb9+/fj5MmTmDVrVovz+Pn5ITk5GXl5edxtbSEhIdQJmhABMorD2Ly8PBw6dAhZWVkYMGAAAGD9+vUIDw9HfHw8unTp0uR8DYuhu7s7/uu//gv9+vVDUVERPD092yV3QkjHYBTF7syZM7C3t+cKHQAEBQVBLBYjMzMTY8eObXUZVVVVSE5OhoeHR6MbpRtSKBRQKBTca7lcDuBZvyU+P5lJvW58XkchEcr21GX9jKLYlZSUwNnZWaPNxMQEDg4OKCkpaXHev//971i4cCGqqqrQu3dvyGSyFm9sX7lyJZYvX96oPT09HVZWVi+2AkZEJpMZOgWiR3zfntXV1VrHGrTYLV68GKtWrWoxJi8v76XeIzo6GsHBwSguLkZ8fDzeffddnDp1ChYWFk3Gx8XFITY2lnstl8vh5uaGkJAQXt1B8TylUgmZTIbg4GBe97gXCqFsT/WRlzYMWuzmzZuHadOmtRjTo0cPSKVSlJaWarTX1dWhrKys1UfD2dnZwc7ODj179sSgQYPQuXNn/PTTT5g0aVKT8ebm5jA3N2/UbmpqyusPjZpQ1lMo+L49dVk3gxY7JycnODk5tRoXEBCA8vJyZGdnc8OEHz16FPX19fD399f6/RhjYIxpnJMjhAiDUXQ96dOnD8LCwjBz5kycO3cOp06dwpw5czBx4kTuSuzdu3fh7e3NjdZx8+ZNrFy5EtnZ2bh9+zZOnz6Nd955B5aWlggPDzfk6hBCDMAoih0ApKSkwNvbG4GBgQgPD8eQIUOwadMmbrpSqUR+fj53wtLCwgIZGRkIDw+Hl5cXJkyYABsbG5w+fbrRxQ5CCP8ZxdVYAHBwcMD27dubne7u7q7xLIwuXbogLS2tPVIjhBgBo9mzI4SQl0HFjhAiCFTsCCGCQMWOECIIVOwIIYJAxY4QIghU7AghgkDFjhAiCFTsCCGCQMWOECIIVOwIIYJAxY4QIghU7AghgkDFjhAiCFTsCCGCQMWOECIIVOwIIYJAxY4QIghU7AghgkDFjhAiCEZT7MrKyhAdHQ1bW1vY29tjxowZqKys1GpexhhGjx4NkUiE1NTUtk2UENIhGU2xi46ORm5uLmQyGfbv34+TJ09i1qxZWs27bt06iESiNs6QENKRGcWjFPPy8nDo0CFkZWVhwIABAID169cjPDwc8fHx3IOym3Lp0iWsWbMG58+fh6ura3ulTAjpYIyi2J05cwb29vZcoQOAoKAgiMViZGZmYuzYsU3OV11djcmTJ2PDhg2QSqVavZdCoYBCoeBey+VyAM8ewq1UKl9iLTo29brxeR2FRCjbU5f1M4piV1JSAmdnZ402ExMTODg4oKSkpNn5Pv/8cwwePBgRERFav9fKlSuxfPnyRu3p6emwsrLSPmkjJZPJDJ0C0SO+b8/q6mqtYw1a7BYvXoxVq1a1GJOXl/dCy963bx+OHj2Kixcv6jRfXFwcYmNjuddyuRxubm4ICQmBra3tC+ViDJRKJWQyGYKDg2FqamrodMhLEsr2VB95acOgxW7evHmYNm1aizE9evSAVCpFaWmpRntdXR3KysqaPTw9evQobty4AXt7e432cePGYejQoTh+/HiT85mbm8Pc3LxRu6mpKa8/NGpCWU+h4Pv21GXdDFrsnJyc4OTk1GpcQEAAysvLkZ2dDT8/PwDPill9fT38/f2bnGfx4sV4//33Ndp8fHywdu1ajBkz5uWTJ4QYFaM4Z9enTx+EhYVh5syZSEpKglKpxJw5czBx4kTuSuzdu3cRGBiIrVu3YuDAgZBKpU3u9XXr1g0eHh7tvQqEEAMzmn52KSkp8Pb2RmBgIMLDwzFkyBBs2rSJm65UKpGfn6/TCUtCiHAYxZ4dADg4OGD79u3NTnd3dwdjrMVltDadEMJfRrNnRwghL4OKHUFNTQ0+/fRT/PWvf8Wnn36KmpoaQ6dEiN5RsRO4yMhIWFlZISkpCZcuXUJSUhKsrKwQGRlp6NQI0SsqdgIWGRmJvXv3Njlt7969VPAIr1CxE6iamppmC53a3r176ZDWCKlUKpw4cQInT57EiRMnoFKpDJ1Sh0DFTqAa3hKnjzjSMezZswdeXl4IDg5GQkICgoOD4eXlhT179hg6NYOjYidQR44c0WscMbw9e/Zg/Pjx8PHxQUZGBn744QdkZGTAx8cH48ePF3zBEzHqfNYiuVwOOzs7VFRU8GogAKlUivv377ca5+Li0uLIMqRjUKlU8PLygo+PD1JTU6FSqZCWlobw8HBIJBJERkYiJycH169fh0QiMXS6eqPL95P27ASqc+fOeo0jhpWRkYGioiIsWbIEdXV1+Oabb7Bp0yZ88803qKurQ1xcHAoLC5GRkWHoVA3GaO6gIPpVUVGh1zhiWMXFxQCAHTt2YMiQIdxFibS0NCxatAhz5szRiBMi2rMTKG3vIaZ7jY2D+pEDiYmJjW6LZIwhMTFRI06IqNgJVF1dnV7jiGE1HOps9OjRGhcoRo8e3WSc0FCxE6imBih9mThiWBs3buR+Z4xh586dSE5Oxs6dOzX29BrGCQ2dsxMoqVSKsrIyreJIx6e+8ODv74+0tDSuPT09nWvPzMxERkaGYPtO0p6dQA0ZMkSvccSwbGxsAACZmZlNTle3q+OEiIqdQFGx45dJkybpNY6PqNgJlJubm17jiGH9/vvveo3jIyp2AjV06FC4u7vD09MTYrHmx0AikcDT0xMeHh4YOnSogTIkukhNTdVrHB/RBQqBkkgkWLNmDcaPH4/w8HB4eHjg2rVr6NWrFwoLC5GWloZdu3bx6tYiPissLOR+t7S01BitpuHrhnFCQ8VOwKKiorBr1y7MmzcPBw4cAPDs6p2Hhwd27dqFqKgoA2dItNWwP2RpaSn+53/+B0ePHsWoUaPwwQcfcBcmhNxvkgYCaAVfBwJoSKVS4dixYzh48CBGjx6NkSNH0h6dkfHw8EBRUREAQCwWo76+npvW8LW7uzuv9u54ORBAWVkZoqOjYWtrC3t7e8yYMQOVlZUtzjNixAiIRCKNnw8//LCdMjYeEokEw4cPx7BhwzB8+HAqdEao4Z0RDQvd86/pDgojEB0djdzcXMhkMuzfvx8nT57ErFmzWp1v5syZKC4u5n6+/vrrdsiWkPY1ffp0vcbxkVGcs8vLy8OhQ4eQlZWFAQMGAADWr1+P8PBwxMfHo0uXLs3Oa2VlRXcBEN57fm/uZeP4yCiK3ZkzZ2Bvb88VOgAICgqCWCxGZmYmxo4d2+y8KSkp+P777yGVSjFmzBh88cUXsLKyajZeoVBAoVBwr+VyOQBAqVRCqVTqYW06JvW68Xkd+Wzbtm1axwUFBbVxNu1Hl8+rURS7kpISODs7a7SZmJjAwcGhxVF0J0+ejO7du6NLly74/fffsWjRIuTn57c4PPXKlSuxfPnyRu3p6ektFkm+kMlkhk6BvICLFy9yv5uammoUgYavL168qHHvrLHTZQgygxa7xYsXY9WqVS3G5OXlvfDyG57T8/HxgaurKwIDA3Hjxg14eno2OU9cXJzGjdJyuRxubm4ICQnh7dVY4Nn/kDKZDMHBwTA1NTV0OkRHiYmJuHr1KiwtLVFaWopff/2V255DhgyBs7Mzampq4OrqivDwcEOnqzfqIy9tGLTYzZs3D9OmTWsxpkePHpBKpSgtLdVor6urQ1lZmU7n49RXogoKCpotdubm5k0Oa2RqaiqIIiCU9eQb9fegpqYGEydOxKJFi/Dmm2/C2toaEydO5DoVS6VSXm1fXdbFoMXOyckJTk5OrcYFBASgvLwc2dnZ8PPzAwAcPXoU9fX1Ol1Kv3TpEgBhj9ZK+MnDw4P7/eDBg1wncQAaXYkaxgmNUXQ96dOnD8LCwjBz5kycO3cOp06dwpw5czBx4kTuSuzdu3fh7e2Nc+fOAQBu3LiBFStWIDs7G0VFRdi3bx+mTp2KYcOGoW/fvoZcHUL0btSoUdzvz19xbfiQ7IZxQmMUxQ54dlXV29sbgYGBCA8Px5AhQ7Bp0yZuulKpRH5+PnfC0szMDL/88gtCQkLg7e2NefPmYdy4cfj5558NtQqEtJkRI0ZwR0mWlpYa09SvnZ2dMWLEiPZOrcMwiquxAODg4IDt27c3O93d3V1j+Gk3NzecOHGiPVIjxOAkEgmSkpIwbty4RtNEIhGAZ0OyC/nuGKPZsyOEtCwqKgq7d+9u1E3L2dkZu3fvFvzADkazZ0cIaV1UVBQiIiJoYIcmULEjhGfUAztUVVXRwA4N0GEsIUQQqNgRQgSBih0hRBDonF0r1N1ZdLkHzxgplUpUV1dDLpfz6nYioRLK9lR/L7UZcJ2KXSuePHkCgB4pSEhH9uTJE9jZ2bUYQ8+gaEV9fT3u3bsHGxsbrnMmH6lHd7lz5w6vR3cRCqFsT8YYnjx5gi5dujR6JOjzaM+uFWKxGK+++qqh02g3tra2vP5yCI0Qtmdre3RqdIGCECIIVOwIIYJAxY4AeDZo6bJly5ocuJQYH9qejdEFCkKIINCeHSFEEKjYEUIEgYodIUQQqNgZkREjRuCzzz4zdBqEGCUqdjx0/PhxiEQilJeXGzoVooOO9p9ZR8vnZVGxI4RHamtrDZ1Ch0XFzkgpFAosWrQIbm5uMDc3h5eXF7777jsUFRVh5MiRAIDOnTtDJBK1+iBy4NmN1NHR0bC2toarqyvWrl3b6H/2bdu2YcCAAbCxsYFUKsXkyZM1Hl6u3qM8cuQIBgwYACsrKwwePBj5+fn6Xn3emTZtGk6cOIHExESIRCKIRCLcuHEDM2bMgIeHBywtLdG7d28kJiY2mi8yMhJffvklunTpgt69ewMATp8+DV9fX1hYWGDAgAFITU2FSCTinp0MADk5ORg9ejQ6deoEFxcXTJkyBQ8fPmw2n6Kiovb6c7QNRozG8OHD2dy5cxljjL377rvMzc2N7dmzh924cYP98ssvbMeOHayuro7t3r2bAWD5+fmsuLiYlZeXt7rs999/n3Xv3p398ssv7PLly2zs2LHMxsaGez/GGPvuu+9YWloau3HjBjtz5gwLCAhgo0eP5qYfO3aMAWD+/v7s+PHjLDc3lw0dOpQNHjxY338K3ikvL2cBAQFs5syZrLi4mBUXF7OnT5+ypUuXsqysLHbz5k32/fffMysrK7Zz505uvpiYGNapUyc2ZcoUlpOTw3JyclhFRQVzcHBgf/nLX1hubi5LS0tjvXr1YgDYxYsXGWOMPX78mDk5ObG4uDiWl5fHLly4wIKDg9nIkSObzaeurs4Qfxq9oWJnRNTFLj8/nwFgMpmsyTh10Xn8+LFWy5XL5czU1JT9+OOPXFt5eTmzsrLSKHbPy8rKYgDYkydPNN73l19+4WIOHDjAALCamhqtchGyhv+ZNefjjz9m48aN417HxMQwFxcXplAouLaNGzeyV155ReNv/o9//EOj2K1YsYKFhIRoLPvOnTvcf5La5mNM6DDWCF26dIl7qIo+3Lx5E0qlEgMHDuTa7OzsuEMitezsbIwZMwbdunWDjY0N9/63b9/WiOvbty/3u6urKwBoHO4S7W3YsAF+fn5wcnJCp06dsGnTpkZ/bx8fH5iZmXGv8/Pz0bdvX1hYWHBtDbctAPz22284duwYOnXqxP14e3sDAG7cuNGGa2Q4NMSTEXr+ie/toaqqCqGhoQgNDUVKSgqcnJxw+/ZthIaGNjop3nBkXPUYgPX19e2aLx/s2LED8+fPx5o1axAQEAAbGxusXr0amZmZGnHW1tY6L7uyshJjxozBqlWrGk1T/wfFN1TsjJCPjw/q6+tx4sQJBAUFNZqu/l9epVJptbwePXrA1NQUWVlZ6NatGwCgoqIC165dw7BhwwAAV69exaNHj/DVV19xozafP39eH6tD/o+ZmZnGNjt16hQGDx6Mjz76iGvTZq+rd+/e+P7776FQKLiBALKysjRi3njjDezevRvu7u4wMWm6DDyfj7Gjw1gj5O7ujpiYGLz33ntITU1FYWEhjh8/jn/9618AgO7du0MkEmH//v148OABKisrW1yejY0NYmJisGDBAhw7dgy5ubmYMWMGxGIxt2fWrVs3mJmZYf369bh58yb27duHFStWtPm6Com7uzsyMzNRVFSEhw8fomfPnjh//jwOHz6Ma9eu4YsvvmhUtJoyefJk1NfXY9asWcjLy8Phw4cRHx8P4N972h9//DHKysowadIkZGVl4caNGzh8+DCmT5/OFbjn8zH2vXMqdkZq48aNGD9+PD766CN4e3tj5syZqKqqAgB07doVy5cvx+LFi+Hi4oI5c+a0uryEhAQEBATg7bffRlBQEP70pz+hT58+3HkfJycnbNmyBT/++CNee+01fPXVV9wXiOjH/PnzIZFI8Nprr8HJyQmhoaGIiorChAkT4O/vj0ePHmns5TXH1tYWP//8My5dugRfX1/8x3/8B5YuXQoA3Pbs0qULTp06BZVKhZCQEPj4+OCzzz6Dvb09N7z58/k8f67Q2NAQT6RJVVVV6Nq1K9asWYMZM2YYOh3yklJSUjB9+nRUVFQY5JxvR0Dn7AgA4OLFi7h69SoGDhyIiooK/O1vfwMAREREGDgz8iK2bt2KHj16oGvXrvjtt9+waNEivPvuu4ItdAAVO0G4ffs2XnvttWanX7lyBQAQHx+P/Px8mJmZwc/PDxkZGXB0dGyvNIkelZSUYOnSpSgpKYGrqyveeecdfPnll4ZOy6DoMFYA6urqWrzVp6UrcoTwBRU7Qogg0NVYQoggULEjhAgCFTtCiCBQsSOECAIVO9JhTJs2jRso0tTUFC4uLggODsbmzZt1ulVpy5YtsLe3b7tEm6EeSJN0TFTsSIcSFhaG4uJiFBUV4eDBgxg5ciTmzp2Lt99+G3V1dYZOjxgzQw6mR0hDMTExLCIiolH7kSNHGAD2j3/8gzHG2Jo1a9jrr7/OrKys2Kuvvspmz57daADRhj/Lli1jjDG2detW5ufnxzp16sRcXFzYpEmT2P3797n3KSsrY5MnT2aOjo7MwsKCeXl5sc2bN3PTb9++zd555x1mZ2fHOnfuzP785z+zwsJCxhhjy5Yta/S+x44da5O/E3kxtGdHOrxRo0ahX79+2LNnDwBALBbjm2++QW5uLv73f/8XR48excKFCwEAgwcPxrp162Bra4vi4mIUFxdj/vz5AAClUokVK1bgt99+Q2pqKoqKijSez/HFF1/gypUrOHjwIPLy8rBx40buDhKlUonQ0FDY2NggIyMDp06dQqdOnRAWFoba2lrMnz8f7777LrdnWlxcjMGDB7fvH4q0zNDVlhC15vbsGGNswoQJrE+fPk1O+/HHH9krr7zCvU5OTmZ2dnatvt/zw8qPGTOGTZ8+vcnYbdu2sd69e7P6+nquTaFQMEtLS3b48OFW8yeGR3t2xCgwxrix2H755RcEBgaia9eusLGxwZQpU/Do0SNUV1e3uIzWhpWfPXs2duzYAV9fXyxcuBCnT5/m5v3tt99QUFAAGxsbbhhzBwcHPH36lLfDmPMNFTtiFPLy8uDh4YGioiK8/fbb6Nu3L3bv3o3s7Gxs2LABQMvPTFUPK29ra4uUlBRkZWXhp59+0phv9OjRuHXrFj7//HPcu3cPgYGB3CFwZWUl/Pz8cOnSJY2fa9euYfLkyW289kQf6O5v0uEdPXoUly9fxueff47s7GzU19djzZo13CCT6hGa1ZoaTlzbYeWdnJwQExODmJgYDB06FAsWLEB8fDzeeOMN7Ny5E87OzrC1tW0yT74NY843tGdHOhSFQoGSkhLcvXsXFy5cwH//938jIiICb7/9NqZOnQovLy8olUpuePht27YhKSlJYxnu7u6orKzEkSNH8PDhQ1RXV2s1rPzSpUuxd+9eFBQUIDc3F/v370efPn0AANHR0XB0dERERAQyMjK4ofA//fRT/PHHH9z7/v7778jPz8fDhw+hVCrb549GtGPok4aEqMXExHDdNkxMTJiTkxMLCgpimzdvZiqViotLSEhgrq6uzNLSkoWGhrKtW7c2ek7uhx9+yF555RWNrifbt29n7u7uzNzcnAUEBLB9+/Y1epZqnz59mKWlJXNwcGARERHs5s2b3DKLi4vZ1KlTmaOjIzM3N2c9evRgM2fOZBUVFYwxxkpLS1lwcDDr1KkTdT3pgGiIJ0KIINBhLCFEEKjYEUIEgYodIUQQqNgRQgSBih0hRBCo2BFCBIGKHSFEEKjYEUIEgYodIUQQqNgRQgSBih0hRBCo2BFCBOH/Ayxk5j16zkA0AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T13:08:22.055965Z", + "iopub.status.busy": "2024-02-29T13:08:22.055681Z", + "iopub.status.idle": "2024-02-29T13:08:22.329266Z", + "shell.execute_reply": "2024-02-29T13:08:22.328326Z" + }, + "papermill": { + "duration": 0.298119, + "end_time": "2024-02-29T13:08:22.331388", + "exception": false, + "start_time": "2024-02-29T13:08:22.033269", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABciklEQVR4nO2dd3yT9fbH30nadC9WBxTKkiUbiqCCYBX8oeK4V0WUKYrI1SsiMhTEhQO5XFHhigJOwAkqikIVZMneexTK6qJ002Y9vz/SpBlP2qRNmqR8369XXzRPnvHNQ55Pzznf8z1HIUmShEAgEPgZSm8PQCAQCKqDEC+BQOCXCPESCAR+iRAvgUDglwjxEggEfokQL4FA4JcI8RIIBH6JEC+BQOCXBHh7ALWNwWDg4sWLREREoFAovD0cgUBggyRJFBYWkpCQgFLp2L665sTr4sWLJCYmensYAoGgCs6dO0eTJk0cvn/NiVdERARgvDGRkZFeHo1AILCloKCAxMRE87PqCJ8Qrw8++IB33nmHjIwMOnfuzPz580lOTq7yuOXLlzN06FCGDBnCypUrnbqWyVWMjIwU4iUQ+DBVhXW8HrBfsWIFEydOZObMmezevZvOnTszcOBAsrKyKj3uzJkzTJo0iZtvvrmWRioQCHwJr4vX3LlzGTt2LKNGjaJ9+/YsXLiQ0NBQFi9e7PAYvV7PsGHDmDVrFi1atKjF0QoEAl/Bq+Kl0WjYtWsXKSkp5m1KpZKUlBS2bt3q8LhXXnmFRo0aMWbMmNoYpkAg8EG8GvPKyclBr9cTGxtrtT02NpajR4/KHrNp0yY++eQT9u7d69Q1ysrKKCsrM78uKCio8hhJktDpdOj1eqeuIbj2UKlUBAQEiHQbL+ITAXtnKSws5NFHH2XRokU0aNDAqWNmz57NrFmznL6GRqPh0qVLlJSUVHeYgmuE0NBQ4uPjUavV3h7KNYlXxatBgwaoVCoyMzOttmdmZhIXF2e3/6lTpzhz5gx33XWXeZvBYAAgICCAY8eO0bJlS6tjpk6dysSJE82vTdOwchgMBtLS0lCpVCQkJKBWq8VfVoEdkiSh0WjIzs4mLS2N1q1bV5pMKfAMXhUvtVpN9+7dSU1N5Z577gGMApKamsqECRPs9m/bti0HDhyw2vbiiy9SWFjIf//7X1lRCgoKIigoyKnxaDQaDAYDiYmJhIaGuv6BBNcMISEhBAYGcvbsWTQaDcHBwd4eUrVZsP4UZy8X88a9HVEq/eePtdfdxokTJzJixAh69OhBcnIy8+bNo7i4mFGjRgEwfPhwGjduzOzZswkODub666+3Oj46OhrAbntNEH9FBc5QF74nJ7OKeGuNMb48vHcS7RP8J/fR6+L14IMPkp2dzYwZM8jIyKBLly6sWbPGHMRPT0+vE18SgcAXWbI5zfx7TlFZJXv6Hl4XL4AJEybIuokA69evr/TYpUuXun9AAsE1wJViDd/tPm9+fbnYv8RLmDR1hFtuuYV///vf3h5GnWH9+vUoFAry8vK8PRSP8dX2dEq1BvPry0UaL47GdYR4XWNcCw+loGo0OgOfbjkDQFykcbIhR4iXQOAbaDT+9TDWJj/vv0hWYRmNIoJ4KNk4S3/Zz2JeQryqQJIkSjQ6r/xUt5l5WVkZL7zwAomJiQQFBdGqVSs++eQTzpw5Q//+/QGIiYlBoVAwcuTIKs9XWFjIsGHDCAsLIz4+nv/85z92burnn39Ojx49iIiIIC4ujocffthqcb3J4ktNTaVHjx6EhobSp08fjh075vTneu2112jUqBERERE89thjTJkyhS5dupjfHzlyJPfccw+vv/46CQkJtGnTxqmxAfzyyy9cd911hISE0L9/f86cOeP0uPwNSZL4ZJMxUD+iTxLxUUbL63Kxf4m9TwTsfZmrWj3tZ/zmlWsffmUgoWrX/4uGDx/O1q1bee+99+jcuTNpaWnk5OSQmJjId999x/3338+xY8eIjIwkJCSkyvNNnDiRzZs38+OPPxIbG8uMGTPYvXu3lXBotVpeffVV2rRpQ1ZWFhMnTmTkyJH88ssvVueaPn067777Lg0bNmTcuHGMHj2azZs3VzmGL7/8ktdff50PP/yQG2+8keXLl/Puu+/SvHlzq/1SU1OJjIxk7dq1To/t3Llz3HfffTz11FM8/vjj7Ny5k+eee67KMfkrf5/O5dDFAoIDlTyc3JRdZ68A/md5CfGqYxw/fpyvv/6atWvXmhe8W1beqFevHgCNGjUy58hVRmFhIZ9++ilfffUVt956KwBLliwhISHBar/Ro0ebf2/RogXvvfcePXv2pKioiPDwcPN7r7/+Ov369QNgypQpDB48mNLS0iqTPOfPn8+YMWPM+X8zZszg999/p6ioyGq/sLAwPv74Y6slO1WNbcGCBbRs2ZJ3330XgDZt2nDgwAHeeuutKu+PP2Kyuu7v1oSYMDUNI4xJ3BfyriJJkt+sKhHiVQUhgSoOvzLQa9d2lb1796JSqcwCUVNOnz6NVqu1Kg4ZFRVldslM7Nq1i5dffpl9+/Zx5coV87Kt9PR02rdvb96vU6dO5t/j4+MByMrKomnTppWO49ixY4wfP95qW3JyMn/88YfVto4dO9qtNaxqbEeOHKFXr15Wx/Tu3bvS8fgraTnFpB41LscbfZPRam0TF4FapSSnSEN6bgnN6od5c4hOI8SrChQKRbVcN2/hjBvoboqLixk4cCADBw7kyy+/pGHDhqSnpzNw4EC7oHlgYKD5d9NfeJOYuIOwMOsHz5WxXQss2ZyGJMGAto1o2dBoEQcHqujUJIqdZ6+wLS3Xb8RLBOzrGB07dsRgMLBhwwbZ901WibPlflq0aEFgYCA7duwwb8vPz+f48ePm10ePHuXy5cu8+eab3HzzzbRt27bKSriu0qZNG6sxAHav5XBmbO3atWP79u1W2/7++++aD9rHyC/R8s1OY1LqmJusY4XJzY3hhO1pubU+ruoixKuOkZSUxIgRIxg9ejQrV64kLS2N9evX8/XXXwPQrFkzFAoFP//8M9nZ2XYxI1siIiIYMWIEzz//PH/++SeHDh1izJgxKJVKs+XUtGlT1Go18+fP5/Tp0/z444+8+uqrbv1c//rXv/jkk0/49NNPOXHiBK+99hr79++vMj7jzNjGjRvHiRMneP755zl27BhfffVVnVy58dX2dK5q9bSNi6BPy/pW7wnxEvgECxYs4B//+Afjx4+nbdu2jB07luLiYgAaN27MrFmzmDJlCrGxsQ6XZVkyd+5cevfuzZ133klKSgo33ngj7dq1MwfZGzZsyNKlS/nmm29o3749b775JnPmzHHrZxo2bBhTp05l0qRJdOvWjbS0NEaOHFlloN+ZsTVt2pTvvvuOlStX0rlzZxYuXMgbb7zh1vF7G62+Iil1zE3N7US/e7MYlApIzy3hUv5VL4zQdRRSdZOJ/JSCggKioqLIz8+36x5UWlpKWloazZs39+sSJ56muLiYxo0b8+6773q1FPdtt91GXFwcn3/+uVeu70/fl1V7L/DM8r00CA9i85T+BAXYTwbdNX8TBy7k89+HujCkS2MvjNJIZc+oJf4TiRZ4jT179nD06FGSk5PJz8/nlVdeAWDIkCG1NoaSkhIWLlzIwIEDUalULFu2jHXr1lnlcwnksUxKHd67maxwgdF1PHAhnx1ncr0qXs4i3MZrnPT0dMLDwx3+pKenAzBnzhw6d+5MSkoKxcXFbNy40elS3M7QoUMHh2P48ssvUSgU/PLLL/Tt25fu3bvz008/8d1331k1bxHIs/PsFfafz0cdoGRYL8cpKf4W9xKW1zVOQkJCpc1MEhISaNq0Kbt27fLoOH755Re0Wq3se7GxsYSEhLBu3TqPjqGu8vHG0wDc360x9cMdVxXumWQUr+OZReQWa6gX5tu1+YV4XeMEBATQqlUrbw+DZs2aeXsIdZKzl4v5/XB5UuqNzSvdt16YmtaNwjmRVcSOM7kM7GDfR8KXEG6jQFCHWbL5DJIEfa9rSOvYiCr39yfXUYiXQFBHKSjV8s3OcwA8dlPlVpcJIV4CgcDrrNh+jmKNnutiw7m5tXOTKybxOnQxn8JS+RikryDESyCog+j0BpZWkpTqiPioEJrWC8Ugwe70PM8N0A0I8RII6hhFZTpm/3qUC3lXqR+mdjlnq8J1vOyJ4bkNIV4CkpKSmDdvnvm1QqFg5cqVXhuPoHpo9QY+23qGW97505yU+njfFgS7WFopOck/4l4iVUJgx6VLl4iJifH2MAROIkkSvxzI4J3fjnLmcgkAzRuE8fzANtxxvevpDibLa9+5fEq1epfFr7YQ4iWwIy7O+/k9kiSh1+sJCBBf0cr4+/RlZv96lH3n8gBoEB7EMymteahnIoGq6jlWzeqH0igiiKzCMvaey+OGFvWrPsgLCLexKiQJNMXe+XFhzbwzTTKcxdJtPHPmDAqFgu+//57+/fsTGhpK586d2bp1q9UxmzZt4uabbyYkJITExESefvppcyULcL5Bx6+//kr37t0JCgpi06ZNLo/9WuFYRiGjl+7goY/+Zt+5PELVKv6d0poNz9/Cozc0q7ZwgfH/3x9SJsSftarQlsAbCVXv5wmmXQS1c1UtnWmSUROmT5/OnDlzaN26NdOnT2fo0KGcPHmSgIAATp06xaBBg3jttddYvHgx2dnZ5i7oS5YsAZxv0DFlyhTmzJlDixYtfM51NUgSeoNUI2GoKZfyrzL39+N8t/s8BgkClAqGJjfl6Vtbm2vRu4Nezevx8/5L7DgjxEvgQZxtklETJk2axODBgwGYNWsWHTp04OTJk7Rt25bZs2czbNgws5XXunVr3nvvPfr168eCBQsIDg52ukHHK6+8wm233ea2cbuTk1lFlGr1XBcbUetxoPTLJXy57SxLt5yhTGcsm/1/HeN4fmBbmjdwf9nm5OZGV3HX2Sto9QavCrYjhHhVRWCo0QLy1rWdwNkmGTXBUeOMtm3bsm/fPvbv38+XX35p3keSJAwGA2lpabRr187pBh09evRw25jdTanWWDo776qWuFoQr9xiDav3X+SHPRescq6Sm9dj6h1t6drUc5Zp60bhRIcGklei5dDFArokRnvsWtVFiFdVKBROu251mcoaZxQVFfHEE0/w9NNP2x3XtGlTl5pg2DbQ8EVqozHYtB8O8PWOc+gMxrinUgE3tmrAyD5JDGjbyOPtyZRKBT2a1WPdkUy2p10W4iXwDJZNMkwtxExNMvr27evx63fr1o3Dhw87rE5x4MABcxOMxERja/mdO3d6fFz+SlGZjq+2GeuodUiI5N6ujbm7cwKNImu3Wmuv5ibxyuXxvi1r9drOIMSrDmDZJKNevXo0atSImTNnWjXJ8CQvvPACN9xwAxMmTOCxxx4jLCyMw4cPs3btWt5//32rJhjjxo3j4MGDbm/QUZfQ6Cpawf004SaUSu80gbWccTQYJK+NwxG+F4UTVIuqmmR4kk6dOrFhwwaOHz/OzTffTNeuXZkxY4Z5wqA2GnTUJXTl7rhCgVcFo0NCJKFqFQWlOo5lFnptHI4QllcdISIiwipgXlxczKxZs3j88cerPPbMmTNWry17siQlJWHboyU6OtpuW8+ePfn9998dXmPo0KEMHTrU4XVuueUWu3Neq+j0xvsQqPSubRGgUtK9WQwbT+Sw40wu7eIdN8PwBsLyqiPs2bOHZcuWcerUKXbv3s2wYcOA2m2SIXAPJvEKUHnfTetV7jpu88FkVWF51SHmzJnDsWPHUKvVdO/enY0bN3LkyBHuuOMOh8dU1XRWUPuY3EaVD8SYTPle29NykSSpVmKoziLEq47QtWtX2SYZV69erbTBhsD3MKVH+EJiaKcmUahVSrILyzhzucQjCbHVRYhXHSckJMQnGmwInEerN1peAT5geQUHqujSNJrtablsPpnjU+LlfWn3QUTgWOAMnvqe6MstL18QL4CbWxlLSP91PNvLI7FGiJcFpizykpISL49E4MuYJMX0PbFcfeAOtOaAvW88nn2vawjAllOXzVahLyDcRgtUKhXR0dHmUi2hoaE+FaAUeBdJZ1zKpCmDy5eLycrKIjo6GpXKvescdSa30QdmGwGubxxFTGggV0q07EnPMyevehshXjaYCvFZ1poSCACyrlwFoDQkgMjgQKKjoz1SuNHX3EaVUsFNrRvy076L/HU8W4iXr6JQKIiPj6dRo0YO288Lrk0e+349AKNvas7ITq3cbnGZ0JrFyzfcRoC+rRsYxetENpMGuq9aSU0Q4uUAlUrlsS+nwD+5UGgsiVNq8Ox3w+Q2BvqI2wgVca8DF/LJLdZQL0zt5RH5SMD+gw8+ICkpieDgYHr16sX27dsd7vv999/To0cPoqOjCQsLo0uXLnz++ee1OFqBwD1kFZTKzlia8rx8IUnVRGxkMG3jIpAk2HQyx9vDAXxAvFasWMHEiROZOXMmu3fvpnPnzgwcONBhzKlevXpMnz6drVu3sn//fkaNGsWoUaP47bffannkgmsVd8zhfP73WZLfSOWbXeft3tP52GyjCZP15SspE16/O3PnzmXs2LGMGjWK9u3bs3DhQkJDQ1m8eLHs/rfccgv33nsv7dq1o2XLljzzzDN06tRJNGsQ+A2SJPHxxtOAsZGGLablQb7kNgL0bW0Ur40nsn0iF9Kr4qXRaNi1axcpKSnmbUqlkpSUFLvuNHJIkkRqairHjh1zWHSvrKyMgoICqx+BoCYoalhLdXtaLmfL+yvKaYDJ8lL5UMAeoEdSDMGBSjILynyiRI5X705OTg56vZ7Y2Fir7bGxsWRkZDg8Lj8/n/DwcNRqNYMHD2b+/PkOmzbMnj2bqKgo84+pkqdAYMnlojKeWb6HLac8H8+xdBUl5GJe5ZaXD8W8wLhUyNTD0RdcR9+SdieJiIhg79697Nixg9dff52JEyeyfv162X2nTp1Kfn6++efcuXO1O1iBX/Dqz4dZtfciDy/a5tHrFJfp+OXAJfNrOctL60MlcWwxuY5/Hfd+0N6rqRINGjRApVKRmZlptT0zM7PS5D+lUmlebNylSxeOHDnC7NmzueWWW+z2DQoKIijIff3sBHWTc+UJqM5Qk4D96gOXKNHoza/lYkd6H8zzMmEK2m8/k8tVjZ4QtffSibx6d0x1p1JTU83bDAYDqamp9O7d2+nzGAwGysrKPDFEwTVCbQWgv91pdBnDg4x2g9xVtT62PMiSlg3DaBwdgkZn4O+0y14di9elfeLEiSxatIhPP/2UI0eO8OSTT1JcXMyoUaMAGD58OFOnTjXvP3v2bNauXcvp06c5cuQI7777Lp9//jmPPPKItz6CoA7ginRVV1LO5BSz/UwuSoWxYSw4CNj7sOWlUCjoe51vVJnweob9gw8+SHZ2NjNmzCAjI4MuXbqwZs0acxA/PT0dpcV/YnFxMePHj+f8+fOEhITQtm1bvvjiCx588EFvfQSBoEr+t+EUs389CsDNrRsSHxUCyAfsfW1toy19Wzdk2fZz/i1eRUVF5sajJiIjXS/SP2HCBCZMmCD7nm0g/rXXXuO1115z+RoCgbuoTszLJFwA/+zRhJNZxvLbBtmAve+6jQB9WjVApVRwKruYC3lXaRwd4pVxuGyXpqWlMXjwYMLCwoiKiiImJoaYmBiio6OJifFc+3GBwFeoTp5XRHCFnZDSLtZ8jsryvHzV8ooKCTR30Pam9eWy5fXII48gSRKLFy8mNjZW1LsS1Ak8Ha9v2TCcvefyeHFwO4IDVRbWm+O1jb62PMiSvq0bsuvsFf46ns3Q5KZeGYPL4rVv3z527dpFmza+URZDIKhtqvP32lCujqYa8CajSt7y8m23EaDvdQ34z7rjbDqZg05v8IrQunzFnj17ikRPgcBFTEF4Uwdsk8dS+Wyj74pXpybRRIcGUliqY9/5PK+MwWXL6+OPP2bcuHFcuHCB66+/3q5+d6dOndw2OIGgtvB0lpejGUSDbEkcU/cg33UbVUoFN7ZqwOr9l9hwPIfuzWq/uqrL4pWdnc2pU6fMeVhg/Ctiakip1+srOVoguDYxiZdKYbK8jNvlRNMUsPe1qhK29GvdkNX7L/HX8Wwm3nZdrV/fZfEaPXo0Xbt2ZdmyZSJgL6g7eDhir5ds3MZKZhu1PlpVwpaby5NV95/PI69EQ3Ro7VZXdVm8zp49y48//igamQquWar6g30ut4ScojK6Nq1IHTLYVEc1B+xlk1R9s56XLfFRIVwXG87xzCI2nczhzk4JtXp9l6V9wIAB7Nu3zxNjEQjqBKOX7uD+BVvIKiw1bzNbXrZuo5zl5QcBexMVVSZqP9/LZcvrrrvu4tlnn+XAgQN07NjRLmB/9913u21wAoEvUpWkZBeVYZAgq6CMRhHBAJgWoqjs3EaZgH15qoTKh/O8TPS9riEfb0rjr+M55rh3beGyeI0bNw6AV155xe49EbAX+Drnckt4adVBHu/bgj4tG5i3uzPiZdKjMl3F0rmKGcSqA/am4L6vFSOUI7l5PYIClGQUlHIiq4jrYiNq7douS7vBYHD4I4RL4Ov8e8Ve1h/Ltis66Eq8vjLjQm+QzOkPZVq9xXbjvxVuY9UBe1/OsDcRHKiil5eqq7p0d7RaLQEBARw8eNBT4xEIPEpGfmnVO1WTL/4+S8eXf6OwVAdYW14mQatwG43IpkrYWGm+Tt/WRgt2gy+LV2BgIE2bNhUWluCawzI25UhStqXlWlVJLbWyvEziVX6O8pPIJqn6cBloOfqZqqum5Vp9Zk/jsl06ffp0pk2bRm5urifGIxD4JJYa4ygobStEVpaXwWa20Xxi+/P4cjFCOVo1Cic+KpgynYFtabWnCy4H7N9//31OnjxJQkICzZo1IywszOr93bt3u21wAkFtIZdvZf2+E+ewEy8Ly0uyFiRTsqps9yC9f7mNCoWCvq0bsmKnsUChyRLzNC6L1z333OOBYQgEvo2V2+hAU/Q2lQVLtZazjaYM+/JzmM9rf56Kkjj+IV5gTJkwiVdt4bJ4zZw50xPjEAh8GmcsL9uqqJaWl22GvUkBK4t5BfrBbKOJm1o1QKmAE1lFXMy7SkItVFetdhnoXbt2ceTIEQA6dOhA165d3TYogcDXsIp5OdzHseVlchtVNjEv+Qz78iRVP3EbAaJCA+mcGM2e9Dw2nsjmwZ6eL1DosnhlZWXx0EMPsX79eqKjowHIy8ujf//+LF++nIYNa8ffFQjcSVV5XlXFxMDebTRZXpIkmc+vNK9tNMW8HJ/H19c22tK3dUP2pOfx1/GcWhEvl+3Sf/3rXxQWFnLo0CFyc3PJzc3l4MGDFBQU8PTTT3tijAKBx6lSvJzwG+3cxnLLy1LU7DLsK61h7z9uI1Q0pN10MsdOyD2By5bXmjVrWLduHe3atTNva9++PR988AG33367WwcnEPgkTqZKlJZbXjqLB1lpm6Qqo16m7kH+5DYCdG4SRWRwAPlXtew7n0c3i6oanqBay4NsF2ODMYHVtg2aQFBXcM7ysnEbyy0vy+3OFCOscBv9y/IKUCm5qXXtNaStVkmcZ555hosXL5q3XbhwgWeffZZbb73VrYMTCHwFy5iXI3vI9m+3KUnV0oVypqqEr/dtrIzaLJHjsni9//77FBQUkJSURMuWLWnZsiXNmzenoKCA+fPne2KMAoHHqcywyikqY7sTmeN2bmP5UhlLUbOr5yVzHn9owOGIm8vjXnvP5ZFfovXotVyOeSUmJrJ7927WrVvH0aPGLsDt2rUjJSXF7YMTCLzN3LXHeS/1hNU2R0mqjpYH6SW5gL0T3YP8zG0EaBwdQsuGYZzKLmbzqRz+r2O8x65VrTwvhULBbbfdxm233ebu8QgEHsXVWnm2wlUZthNsJstLZ2F62Qbs5ZNUy8tA+6HlBcZZx1PZxfx1PNv3xCs1NZXU1FSysrLsgvSLFy92y8AEgtpELvbkCIWDqJcjy8u2iipUYr0ZJLMI+ttso4m+1zVkyeYz/HU826PVVV22S2fNmsXtt99OamoqOTk5XLlyxepHIKjrOHYbrV/buo0qiwOVDtxGy7QKf3QbAW5oXh91gJKL+aWcyi7y2HVctrwWLlzI0qVLefTRRz0xHoHAbzHYZtibA/bWi7LBMmBvfYyli+lvGfYmQtQqkpPqselkDhuO59CqkWdKQ7ss7RqNhj59+nhiLAKBX+AwVcJRwN5gb3mZj7FJr9DJpFX4I32v83y+l8vi9dhjj/HVV195YiwCgV/jKGBvdhutYl7y9bxMS4MAAv1seZAlpqVC29Iue6y6qstuY2lpKR999BHr1q2jU6dOdtn2c+fOddvgBAJfpLJguyV2lpfSMuZl/Ncu5lU+06hUVMxM+iNtYiOIjQwis6CMHWdyubm1+ws2uCxe+/fvp0uXLgB2jThqs2ebQOBOXOke5Ghfe7ex3PKSES9zhr3NOfytBLQjFAoFN7duyLe7zvPX8WzfEK8///zT7YMQCOoCtuKl1UvoDZJZvJQKmVQJO8vL/6qoOqLvdSbxymH6YPef37/lXSDwAo6MNLkqMGU6vV3bM3CcpKr1s7ZnlXFzqwYoFHAss9AjLeeEeAmuKRxFNpwpNlgVctnypVqDvNvoYG2j3o+XBtkSE6amU+MoADaecP+so//fIYGglnE25gVGy0tevOSrSmj9rHNQVZhmHf86keP2cwvxEgjchFw5uzJLy0th7zbaBez9sPlGZZirq57Idnt1VZfv0F9//YVOp7PbrtPp+Ouvv9wyKF9BozO4tOZNULdw9H/vyMWUdRt1enOel1LG8rJ9nnUyVpo/0yUxmoigAK6UaDl4Id+t53ZZvPr37y/bLTs/P5/+/fu7ZVC+QGGplhGLtzNvnfNVBQT+i5xOOfq75ZLbqDVULMyWsbxsT6bz40KEcgSqlPRpVR9wf7a9y+LlaJX45cuX7bpn+zN/Hstm6+nL/Df1BJ9uOePt4Qg8jJweuWpzy882GmQz7E1pXI7yvPw5u96WiriXe8XL6Tyv++67DzCauyNHjiQoKMj8nl6vZ//+/dVe8/jBBx/wzjvvkJGRQefOnZk/fz7Jycmy+y5atIjPPvvMnCDbvXt33njjDYf7V5e7OydwOruIeetO8PJPh4gODWRIl8ZuvYbAt5GzpKCSVAkZ9SrV6tHL9GGsKANtvX9dcxuhojT07vQ8Ckq1RAbb98CoDk7Le1RUFFFRUUiSREREhPl1VFQUcXFxPP7443zxxRcuD2DFihVMnDiRmTNnsnv3bjp37szAgQPJysqS3X/9+vUMHTqUP//8k61bt5KYmMjtt9/OhQsXXL52VTxza2tG9G6GJMFzX+9jQy22Mhd4Bke1uORwNdwpP9tooNwTtF7u46iqhKkQYR1xGwES64XSokEYeoPElpOX3XZepy2vJUuWAJCUlMSkSZPc5iLOnTuXsWPHMmrUKMBYcmf16tUsXryYKVOm2O3/5ZdfWr3++OOP+e6770hNTWX48OFuGZMJhULBzLs6kFui5ad9Fxn3+S6+GtuLrh5u6STwDRzmfjlQNTm3sVSrN88cquy1y26GUmvOsK87biPAuH4t0RoMdGsW7bZzunyHZs6c6Tbh0mg07Nq1y6r+vVKpJCUlha1btzp1jpKSErRaLfXq1ZN9v6ysjIKCAqsfV1AqFbz7z87c3LoBV7V6Ri3dwYnMQpfOIfAdHAmS3Myiy5aXjHqV6QyyGfaOOmbL5YTVBR7omciwXs1oFBHstnO6LF6ZmZk8+uijJCQkEBAQgEqlsvpxhZycHPR6PbGxsVbbY2NjycjIcOocL7zwAgkJCQ4bgMyePdvKxU1MTHRpjADqACULH+lOl8Ro8kq0DF+8nQt5V10+j8C/cDjb6GD/qpJU5dY22oqmqRhhXXIbPYXLC7NHjhxJeno6L730EvHx8V6tJPHmm2+yfPly1q9fT3CwvKJPnTqViRMnml8XFBRUS8DCggJYMrIn//zfVk5mFfHoJ9v4dlwf6oWpqz1+Qe3jUszLoZUmv7+822ggPMh+sbWjcZjdxjo02+gpXBavTZs2sXHjRnNZnJrQoEEDVCoVmZmZVtszMzOJi4ur9Ng5c+bw5ptvmuuKOSIoKMhqZrQmxISp+Wx0Mv9YsIXT2cWMWrKdL8feQHhQtfqYCHwcVxPCHVlepqx5OcvL9hh9HVqY7WlclvfExES3ZZ2r1Wq6d+9OamqqeZvBYCA1NZXevXs7PO7tt9/m1VdfZc2aNfTo0cMtY3GWhOgQPhvTi5jQQPadz2fc57vMdZsE/otsnpejVAmHAXsHC7MrqSphe4i2DpXE8TQui9e8efOYMmUKZ86cccsAJk6cyKJFi/j00085cuQITz75JMXFxebZx+HDhzN16lTz/m+99RYvvfQSixcvJikpiYyMDDIyMigq8lyXEltaNQpnyahkQtUqNp3MYeKKfW5ftyXwDA6jHDL/fa5bXvbbynR6cyDfKsPeQcC+IsNeuI1V4bK/8+CDD1JSUkLLli0JDQ21KwMtt3SoqvNlZ2czY8YMMjIy6NKlC2vWrDEH8dPT01Fa+P8LFixAo9Hwj3/8w+o8M2fO5OWXX3b141SbLonRfPRoD0Yt3c7qA5eIDg3ktXuuF9VkfRyXnAZ3BOwtLC+lXEkcu4C9KeYlvkdV4bJ4zZs3z+2DmDBhAhMmTJB9b/369Vav3WXxuYObWjdg3oNdmbBsN19uS6d+eBATb7vO28MSuAlXanxJkiQrjJaWV4ATbmNdKQNth64Mzm6Blu5b/+yyeI0YMcJtF68LDO4Uz5WS63lx5UHeSz1BvdBARt7Y3NvDuiaY9M0+zl8p4avHbvBIswpHbqOcSDnat0xrMAuSXFUJR25jnUqVuLAbVo6HnOMw9g9I6OKW01ZL3k+dOsWLL77I0KFDzct4fv31Vw4dOuSWQfkbj9zQzGxxvfzTYVbtdf9SJYE93+46z9+nc9l3Ps/pYxxXUpXZ5sLaRkfrIEstixFadcyWv0adWtuo08Afr8HHKZB9BELrwVXXwkqV4bJ4bdiwgY4dO7Jt2za+//57c6B83759zJw5020D8zf+NaAVI3o3A4zrINcfk1+bKXA/joSjprhyVkcTNmVa+Qx7R2Wg60wxwkv7YVF/+OsdkPTQ4T4Yvw1aDnDbJVy+Q1OmTOG1115j7dq1qNUVCZoDBgzg77//dtvA/A3TOsi7OyegM0g8+cVudqdf8fawrgk8VS/SYVUJF5YSWS3MtjL75KtKaGUqUPgVei2sf8soXJkHIaQe/HMp/HMJhNV366VcFq8DBw5w77332m1v1KgROTnur1PtTyiVCub8szN9r2vIVa2e0Ut3kF1Y5u1hCZxA1kV0QRQduo3aiu5BATKWl12Sqj/neWUegkUDYP0bYNBB2zvhqW3QwV4v3IHL4hUdHc2lS5fstu/Zs4fGjUW9K+M6yG4k1Q8lr0TL9jT3+fgCeTyVYefKeR2JV5nOUJFhL7cw28Fso18VI9Tr4K858L9+kLEfgqPh/k/gwS8gvJHHLuvyHXrooYd44YUXyMjIQKFQYDAY2Lx5M5MmTXJ7SRp/JVQdQIuG4QAUl9nX+xe4F3e4ja4E4eWQa74B5cUIzTGviu2O7CpT9yC/cRuzjsInt8Efr4JBC9fdYbS2Ov6jkoxg9+ByqsQbb7zBU089RWJiInq9nvbt26PX63n44Yd58cUXPTFGvyRUbaywUawR4uWvuFLDvjLLSz7D3nQu27WNpoC9j4uXQQ9b34c/Xgd9GQRFwR1vQeeHPC5aJlwWL7VazaJFi3jppZc4ePAgRUVFdO3aldatW3tifH5LmNp4a0s0Yt2jp3Fmre2B8/kYJMmFmhKO3Ua55FXH4uWgexDy3YP8ohhhzglY+SSc32F83eo2uPs9iEyo1WFUuxxC06ZNadq0qTvHUqcIDSq3vITb6HGqkq5SrZ673t8EQKMI5yuMyBUXdITeYcC+CsvLtgy0L1eVMOhh20JIfQV0paCOgEGzoesjtWZtWeKUeE2cOJFXX32VsLAwq9pYcsydO9ctA/N3hOVVe1RleFn+AXH0x8SVuJkzbdLUKiUavcG6Y7ZKzm20Ps5n1zZePgWrnoL08grHLfrD3fMh2vXaeO7CKfHas2cPWq3W/LsjxKLkCoTl5TtY6oPSwXdUzhV0RdBs3cbgQJN4VSwPsu7b6CdVJQwG2LEI1s4E3VVQh8Ptr0H3kV6xtixxSrz+/PNP2d8FjjEVKBQBe89T1QJqK2Fx4XlzpfWZbYZ9iFpFQakOScJc7002w97W8tL7kOV15QysmgBnNhpfJ90MQz6AmGZeHZYJUQLUQ4SWu43FZcJt9DhVWUjV0y6X8rxsRSgksKKfg+k74FwNex8I2EsS7FwMv78E2mIIDIXbXoEeYyq65foATomXqeGsM3z//ffVHkxdIqw8VaJEWF5ep7p1Ih1XUpW7hvVGdYAShcK4rynu6Uz3IK834Mg7Bz9OgNPrja+b9oF7PoB6LbwznkpwSryioqLMv0uSxA8//EBUVJS5BPOuXbvIy8tzSeTqOqFBwvLyBlmFpRRc1dKqUYR5m6Vb6TD9wYUyN/KpEtavlQoFwQEqrmr15j9g8mWgrQ80pUrUepKqJMGez2HNNNAUQkAIpMyE5Cd8ytqyxCnxMjWcBWOrsQceeICFCxeaW53p9XrGjx9PZGSkZ0bphwjLy32cySlmwfpTPNGvhXnlgiWWj3/y68Z+CJte6E+TmFDAWlhcy8Z3IVVCRumCApVc1eoplrG8HFWV0HujGGH+BfjpaTi5zvi6STLcswAatKq9MVQDl+/Q4sWLmTRpklWPRpVKxcSJE1m8eLFbB+fPmGJeRcLyqjHDPt7Gip3nGLpIvmqJnCDtO5dv/t0yX8uV5jGuZNjbnlepUBAUYHy8rposL5mqEra5ZLVajFCSYO9X8GFvo3CpguC2V2H0Gp8XLqhGwF6n03H06FHatGljtf3o0aMYHC3wugYxzTYKy6vmmBr8ZhY4X6HDtEYQrK0id7iNctjuq1BAcKApXaY8YG8V85IfT625jYUZ8NMzcHyN8XXj7kZrq2Gbyo/zIVwWr1GjRjFmzBhOnTpFcnIyANu2bePNN980d/wRVOR5lWiMNcw9UaZYYEQuBqXRWYiXhTK5stjalRr2tm6jleWlLXcbLb4CCgd+Y8XaRg+5jZIEB76BX56H0jxQqeGWqdDnaVD5V/KBy6OdM2cOcXFxvPvuu+bSOPHx8Tz//PM899xzbh+gv2LKsAfjlzdMNKb1GLKNLywsL0vXzBVrqiZJqtaWV7nbaCFI5oC9zXm0nlweVJQFPz8LR382vo7vDPcshNj27r9WLeDyE6VUKpk8eTKTJ0+moKAAQATqZQgOrJgqL9bohHjVMloHlpc7Cgw6U0lVoVCgLje1ysrH4kxVCZ2nihEe/B5WP2esIa8MhH4vwE3/BlVglYf6KjV6ooRoOUahUBCmDqCoTEdJmR4iqj5GUD3kJEbjMOblmYC97cJsBRAUoLLaZukJmvK8bC1Bt882FucYRevwSuPr2I5w7wKI6+ie83uRaonXt99+y9dff016ejoajcbqvd27d7tlYHWBULWKojIdRWJ9o0eRs4QsLS/LeSRPNTa3tdKUCqP1bb3N3pqyFVOteW2jGyyvwz8a3cSSHFCooO8kuHkSBKirPtYPcFne33vvPUaNGkVsbCx79uwhOTmZ+vXrc/r0ae644w5PjNFvqZhxFOkStY2V5WUhLI6z5p2v0SW31fZ4hUIhY3m5UlWiBpZXSS58Owa+ftQoXI3aw9hU6D+tzggXVEO8PvzwQz766CPmz5+PWq1m8uTJrF27lqeffpr8/PyqT3ANYa4sIdIlPIqs26hzLVVC9rwu7Ky3yRIyuo3Wj5fKhaaz1ba8jv4CH94AB78FhRJufg4eXw8JXat3Ph/GZfFKT0+nT58+AISEhFBYWAjAo48+yrJly9w7Oj/HlKhaIhJVPYvcbKOl22hledXotA7PYe82KggKrMTyMp9LfmG2yw04rl6BH8bB8qFQlAkNroMx6+DWGRDgfAFGf8Jl8YqLiyM319gRp2nTpuZejWlpaS5lL18LhIk69rWCXBDeUZKq43PY49htdMLFVMhYXlYdsx10DzIlqbpieZ1Ya8yS37fMeOE+T8MTG6FJd+fP4Ye4HLAfMGAAP/74I127dmXUqFE8++yzfPvtt+zcuVMszLbBtDi7RATs3U5Vfyg1Ovk8L9euUf19lRZ5XuZtznTMNlWVcCbPqzQffptuXFANUK+lMUu+aS/nB+7HuCxeH330kXkZ0FNPPUX9+vXZsmULd999N0888YTbB+jPVFhewm30JHIio3UQsHflHI4cR9lUCRuBVKCo1PKScxsNBsk8G1plPa9Tf8Cqf0HBeePZbngSBrwE6tDKj6tDuCReOp2ON954g9GjR9OkSRPA2MfxoYce8sjg/J0wc1kcYXl5EjkxcZTn5QqurW20iXkpjVUlLLFar2jumF2xSWuR0+FwbWNZobFI4K7ySi8xSTDkQ0i60fnB1hFcinkFBATw9ttvo9OJh9EZRBMO19DpDWQWlDq1b1XGlEZXvfWMzlxDPlXC+rUCYz0vS+SKEVpiKbKyVSVOb4AP+1QIV/Lj8OSWa1K4oBoB+1tvvZUNGzZ4Yix1DtGEwzUe+uhver2Ryu70KzU+l7XlVb1zuDIBZec2Kiq3vCylyXQdU0UJsMnzKiuC1ZPgs7shPx2im8KIn+D/3gF1mNNjrGu4HPO64447mDJlCgcOHKB79+6EhVnfvLvvvtttg/N3hOXlGjvPGkVrxfZzdGsa4/Rx8nleFffcudlGuRlERztXPduoUNhbXtY17Ct+lySj2OksVNa8MPvsFmOD1ytnjK97jDbWkw8S681cFq/x48cD8v0ZFQoFer14UE2EilSJWkF2eZDeNbdRtsCgo4C9zDb7MtAuWF7l/5pEVqkApe4q/PEq/L3AuEdkExgyH1oOqPyDXEO4LF6i4KDziIC956hKjhxl2Lt0jZqUxMG5hdmmY1Uo0JaPs6fqBCx8CXJPGXfoNtzYKzE4yqXx13VEnRYPEiaacFQLVyo/GPe3R+Mgw96l87pQVUIuw952YbbKMo5lYXqZDtWVlTA14EvGqn6BXAki4o1dqVvfVp3h13mcFq+rV6+SmprKnXfeCcDUqVMpK6soy6tSqXj11VcJDg52/yj9FNGEo2okSWLWT4eJDK7+31FPpUq4IqJyZaDtLC+Zel7m65zfRfx3j/NEwEnjxs4Pw6A3IMT52N+1htPfmE8//ZTVq1ebxev999+nQ4cOhISEAMYa9gkJCTz77LOeGakfYm48KwL2Dvni77Ms3XLG5eMs41xyMS9X3UZn4lgV+8oVI5QJ2NuWxJExvNRoUf3xKmz9L2rJQJYUzRuqccy7d2qVY77WcVq8vvzySyZPnmy17auvvqJFC2Mzyi+++IIPPvhAiJcFYaY69iLm5ZCXfzpc43PoZFTG0o2rvttYg1QJ5GJe1rON1ytO827gQgK2nAcgr9U93Hbw/wiKqF+t8V5rOJ3ndfLkSTp2rKi+GBwcjNLiT0lycjKHD9f8i1iXsLS8nF1fl5Ffyp3zN7JiR7onh+YTaPUGWavIVa3RVTGJVO08L0fbZWNe1q8tG3CYMLuNOg1Bf81mpXoGbZTnkUIbwAOfc7bff8kn3HPNN+oYTt+lvLw8qxhXdnY2SUlJ5tcGg8HqfUGF5QUVHWSq4teDlzh4oYAf9lzw1LB8BsuyNTXBMi1CjuqubXRYuFBmW2UNOEyolArIOACLBhC4eQ4BCgM/62+gZOxmaH+3WYTdXr++juK029ikSRMOHjxo16/RxP79+83rHQVGQgJVLjfhOHrJWB/tWshIqe4jmlusYeOJbPNrXRXiVRtVJWyvYWt5BaCj3s55sOM/YNAhhdRjQv4jrDbcwMHQBoBFORzRJs8pnLa8/u///o8ZM2ZQWmq/9uzq1avMmjWLwYMHuzyADz74gKSkJIKDg+nVqxfbt293uO+hQ4e4//77SUpKQqFQMG/ePJevV5uYmnCA8wUJj2QYOzI5Yy34OzLL+4Cqc7ge+mgrzyzfa34t5zZa3j7nZhudd1+dalBrYXldpzjHD+oZRG97Bww6aHsnZY9vYbXhhvLzGQ+udiHCaxSn79K0adPIzc2lTZs2vPPOO6xatYpVq1bx9ttv06ZNG65cucK0adNcuviKFSuYOHEiM2fOZPfu3XTu3JmBAweSlZUlu39JSQktWrTgzTffJC4uzqVreQtXsux1egPHMoyWl1wQuq6hqKbtdTyzyOp1VW5jdQP2rhwnl6SqVhoYr1rFT+rpdFSewRAUDfd9DA9+gTIi1uJY479ubb5xDeC02xgbG8uWLVt48sknmTJlivmvhUKh4LbbbuPDDz8kNja2irNYM3fuXMaOHWvutL1w4UJWr17N4sWLmTJlit3+PXv2pGfPngCy7/siYUEBUFjm1PrGM5eLzXGg6ro6/oQjy8tVdFVE5Kuf5+Voe9VrG2PLzqJaMonJgbsAWKvvRpcRS2mY0AwAhcJif8l6nB5pOFsHcSkzsHnz5qxZs4bc3FxOnjQm07Vq1Yp69eq5fGGNRsOuXbuYOrUin0WpVJKSksLWrVtdPp8jysrKrCYSTI1yawuT5eVM+7PD5fEuqP4DVxdwfbax5gH7mo7D9MdGiYExql+YnP4NSFoKpFBe1g7ne8PN7Iio8Bas1zZaV5WoshChAKjm8qB69eqRnJxcowvn5OSg1+vtrLXY2FiOHj1ao3NbMnv2bGbNmuW287mKKzGvo5cqhLW6rk5dwNXlQdoqLC9nrFhXZhvlhmeQoLniEnMCF9JdecK4T6vbeDDtHxwpM1aACJDpHmR5bfNso7C8nKLOS/zUqVPJz883/5w7d65Wrx/mQvuzIxbidS1bXq4iN9toKYBuz/Oy3WAw0O7sF/yqnkJ35QkKpRC+iH0ehn1DQWAj825Kq2KEFoeXq5fZbRQxL6fw2sLsBg0aoFKpyMzMtNqemZnp1mB8UFAQQUHea/3kShOOIxZu44msIvQGSUybO4G2qiRVT7qNuadh5VP0Tt8CCtiov54XtI/TNaojjygUVmVx5Po2QoUYmt1GMdvoFF67S2q1mu7du5OammreZjAYSE1NpXfv3t4alttxtgnHlWINGTYlkFcfuMTBC/m88O1+spwsj1wX2Hcuz6X93ZHn5UziqXlfSTIm4m1fBAtuhPQtaFUhTNOO4VHtVC7SwLyv5RIhlYMZCrPbWG4iypaAFtjh1ZI4EydOZMSIEfTo0YPk5GTmzZtHcXGxefZx+PDhNG7cmNmzZwPGIL9pCZJGo+HChQvs3buX8PBwWrVq5bXPURnmxrNVuI2WLqOJMznFPL1sDwAX86/y+Zhro6XVqexil/avKuZVbcvLwfaoskvw2WQ4s9G4Ielmlsc+z1cbiuz2tVycbWtQmRKYTS6uaeJBWNvO4VXxevDBB8nOzmbGjBlkZGTQpUsX1qxZYw7ip6enW62fvHjxIl27VrQtnzNnDnPmzKFfv36sX7++tofvFOaYVxUB+yPl+V0JUcFczDdaWVmFFdbW8cxC2eMEVc82OldJtepKESDxsOoPxh5cBoYSCAyFlFnQ8zHy158CjtudwyrL3ka9FBgF0tbyErONzuH1YoQTJkxgwoQJsu/ZClJSUpLfdeU2L86uIuZlsrw6NI4yi1dmQUWKh5997FqlqjwvdywPSiCHNwMX0Vd1AAxA094w5AOo39J4DQeXsFzfaGtQKRUKDJJkMdtoyrAXlpczCIn3MBUFCauwvEzilRBp3pZVaCFeHhibt3GXIFcV86r+bKMESDyg+pPfgl6gr+oApVIgvyU+AyN/MQsXyFh35S9NlpdSYR2kB8uu2bZuo3gsnUHcpcpww+pocynoSmJeWr2BE+VLXtrHV4hXtkWQ3t8sTkdcKdbw8o+HOHgh323n1LrBbZQjqCSTJYFv83bgIiIUV9llaM0dmjfZ1ughuwCWI+vOFLCXi2OZlkeJgH31EOLlCL0Wlv6fsXtLDUQsLKjqJNXT2cVo9AbCgwJIiA4xb88srHtu48wfD7F0yxnunL/Jbed0x/Igqz0kCfYuI+XPIfRX7aNMCuR17cP8UzOTNCle9njHbqPJ8pIRJLPlZaQiw16IlzN4Peblsxz4BtK3Gn+O/ARD3od6LVw+jTMLs4+WV5JoG2fdi8/yoasj2iU7q1pTTA+9I+vUldnGhlyBZUPh+K+ogb2GFjynfZJTUmPzPs6sbTRhsrzksuZNm0xWW8XaRmFTOIMQL0d0Hgraq/D7S3B2szGf57ZXoMcY+znvSnCm/dnhi8YHul18ZOW5RXUMV5cBOUIuVcLydjkVsJck7lZu5pXApXC8GFRq9rcez/17u6NH5fCwJZvTyCgotauQYfps5phXJW6j+XOI5UEuIcTLEQoF9BwDrW6FVROMOT2/TIIjP8Ld70NMM6dOE+pEkurBi8b4z/WNIytp+lA38MTnqMotrNJtLMpmjvQuKeptxtfxneGehRxJD0e/94Dd7pbCOKu8Bv8tbRrKnto02ygb8zK5jeaYV3nAXriNTiHs06qISYLhP8Id7xjzetL+ggV9YOcSpwJRFQuz5S0vSZI4eME00xhF+/hIEuuFyOxX/Y/gS3jCgjRZXo5OXanbeOgH+LAXKWxDK6l4V/sPeCwVYttXec8tP4uj2WST5SWXXW/aYrLS9KIYoUuIu+QMSiX0ehzGbTLm92iK4Od/w+f3Qv75Sg81B+y18k04LuRdJf+qlkCVgtax4agDlPz53C2M6G1t2dUVt9ETn6I6VSViKIBvRhp/Si5zlGYM0bzKfP19oAp0aqyWybGOXD2T5SXnNpqC+KIYYfUQ4uUK9VvCyNUwcDYEBMPpP+HD3rDnC4d/9k0Z9pIEpTr7v84mq+u62IiK4K5KSZOYUKv96oZ04ZEPYrJYHJ3aNg1soHIHvwdNNlpdChX0ncwwZnNYSrLar6r4o6VoOlrSY1qYLbuu0ew2lud5mRdmC/FyBiFerqJUQe/xRiusSU8oK4BVT8FXD0DBRbvdgwNU5tiG3BKhQ+XxLsvkVMAqZQKoM+plKQiVGZOuWJpyZaAljKJWpquweKMo4j+BH/A/9X9oqCiAhu1gbCoMmI5WJvxb1RC0uoodbBNQTZjdRtmAfcVYocKSE8uDnEPcperSoDWM/s04A6kKghO/w4c3wL7lVt96pVJBaKBpfaN93OtQ+Uzj9Y2jrLY3jrEWrzqiXU6T/EYq6w5nVr0jljEv67t09/ub6PHqOoo1OgYod7M2aDL3qjajlxR8oLsbntgACca1snL3t6p6XhoLy8tWm0xDqTxgb5OkKmYbXUKIV01QquDGZ+CJvyChG5Tmww9PwPKHobDiwQutJMvelGlua3k1trG8bB/MnKIy9p/Pc8enqFWcFeHswjIe+2ynU/s6mk08dLEARVk+9519jcXqOTRS5HHSkMB9mlm8o3sIAiqv81aV9WfZtch2DKZDK7W8zJuE21gdhHi5g0ZtYcxauHUGKAPh2C/wYS848C1IksP1jVmFpWQVlqFQGHO8LKkfpkZtUZHA8tEo0ejo+/afDPlgM6ez7cuw+DKemHcw5UfZnvoW5V5+C3qBu9mAQVLwP91gBmveYJ/kXPmkqlqfWbqNDpNUTQF7GT2yDdhXNJ0Vj6UziLvkLlQBcPNzRlckrhNcvQLfjYGvh5MQaBQYW7fR5DK2bBhurj5hQqlUWFlfls/G1zvOUaLRI0nW1Vf9AcvEVHfpmKWIAIRTwnTdhyxVv028Ipc0KY5/aGYyWzeMMtTOj7UKpbV0Gx1Zf7ERwQDUD7e38swxL5s8L7G20TlEkqq7ie0AY/+AjXPhr7fhyI8sVGzgBeVISjTdrHY95MBlNJEQHUxajrEwn+VDn2FRKsey5pc/4AnLy2SxSBLcqDzA24Ef0Vi6jEFSsEQ/iHd0D1BKFaXAZcblOGHYVAWiQrwcLX9tnxDJklE9adUw3O4926oSWlFVwiWEeHkCVSDc8gK0uQNWPklk5kEWqP9L+tZj0OJDCKsPVKRJXJ8QJXsaR5aX5UJk29LRvk5NxUvOGtLqJSgrRPXbS3ypXgLAOWJ5TvM426V21b+WwzEY/91x5op5W2WJsP3bNHLwjnXAXm8QVSVcQUi8J4nvBGP/5Jd6j6KTlDS9uMYYCzvyMwCHLpVbXo0dWV4W4mWx3TI5MjO/FL1B4o+jmeQUlVHXkXPPuun3w4I+qHYbhWup7nbul96pkXBB5W7j5pM5vLTyYKXjqgrzwmxz3phYmO0KwvLyNAFq/owfy4JLbfg0ZjH1ik/DimFo2v+TgtzbgHA6xFdteVmql6W7klFQyrojmTzx+S4ahKv54OFu9GpR30MfpubUdKWApXCHUMrkgBWMUv4GeSBFNeXh7EfZauhAqFIFVN0r0zwu2bE63nfdEes0DtuAvTOLzu3XNhr/X0UNe+cQEl8LhAUFcEBqwdKOn8JNz4JCifrwN/weNJkHow4SFRooe5yV22jVh9DC8ioo41LeVQByijQM+3gbvx/K8NAnqTmWj3R1hMwkEj0VR/lVPZVRAb8Z3+g+Cs3jG9lq6AC4p+9lZQJkO/TqXM+2qoR5baNwG51CiFctYFoiVKBVQcrLMGYteaFJxCryeKvsDVg5Hq7m2R1nmahq+bBYZpRn5JeaK1YoFUbL5Kvt6R75HO6gpjEvbWkJLwZ8zgr1qyQpM7kg1edRzRS4ax6SuqIemqtiIiekDgP2kv3+1RIvG8urohiheCydQdylWsCu/VmTHrzW5H/8TzcYCQXs/dJYqeLkOqvj4qKCzb9bPhqWD8pVrZ7M8qB9VIjRgquqprs3qUkNr26K4yj+dzOPBfyKUiGxXHcLg8reYqOhk72YOKmSBaVau4Xb2eUVbCs7ha1WVUeUTXletrOXIknVOYR41QLmxrMWaxv3XCpltm4Ye1KWQb2WUHABvrgffnwaSo2zkJYNSy0fTtsqCqfL+xyaRHLTyZxar0KxO/2KU81iXR1WUZmOIDRMCfiKb9SziCw+Q4YUw0jNZKboHqcQ4wJ22/Znzlxn66nLdJ71O/9NPWG1vefr67hcVFaJ0Ep2MS5bsXTlc1YkqYoMe1cQ4lUL2C4PKtHoOF2ev5XYeYBxkXevJ4077/7UaIWd+tPqHJbPpq2LYsoFM7mnAH8czXLrZ6iMojId9324hSEfbEajc77evzPP98Mvf8jP6umMC/gZlUJid8wgbi97i/WGLlb7VVUWR44p3+9HkrATL4D9F/IrzbC3fatmbqPN8iDhNjqFuEu1QEVBQqPldeRSAZIEjSKCaBgRBOpQuONNY7mdmCTIPwef3wM/TyQUo0vYKKIiydK2isKF8oC9qXYYwP/+Ok3y6+v446hzi5trQl6Jxvx7mUzZH0ucfcTVaJkUsILv1TNprbxAlhTNY5rn+O26lynAPuFTq5dctuoKrmrNv9tWutXrpUqtV9v3qtOhqCJJ1Yi56aywvJxCiFctEGrqml1ueZmTU20qSZB0E4zbDD3HGl/v/IQtUS9yg/Iwt7arSHQ0JTPaTqmbSk4DbE/LJauwjNFLnVvcXBMsy8HIPcIns4o4mWVcxuTMM95BkcaP6heZELCKAIWBlfo+3F72FusM3QlXy2f3VNVBSP4Yx4PRGRyLoTFgb72tenletlUlTJaXEC9nEHletUC4qZpq+V93Uw2v6+WWBQWFw+A50O4uWDWB6Px0lqtfY/OFI6CZD+ow/jyWDUCTmBDOXi4xH2q7PrK2qOxRK9XqSZm7AYCjrw6q1JoJRMeEgJU8pVpJgMJAjhTJdO1ofjMkVzkGnUFyeTKgMmvJIEkOZxvljq1eqoQRs9toTpUQNoUziLtUC5ibcJRZW17tHSwLAqBFP3hyM4fi7wPgxsvfwcKbyDjwh3mXEItW8lAhkt7EVg8sK2kUlGodyks7xVlWql/imYDvCVAY+Fnfi9vL3rYTLkcNZl2JtZmoTG8qE0MJe6usem6jabbRiFYkqbqEEK9awBTzKi7TUabTczzT6EJd72BZkJngSNZfN51HNVPIC2gIuaeJ/e4+pgd8QRAaom2SWy3dRkuKynT8b8Mp0i2sNHdiVURUgqsaPS//eIitpy5blT/WG6zjSDmFZQSgY4LqB1apX6SD8iy5UjhPaZ5mgvYZcrG/P47cw8rcPEdUlk5hMEiVupX2AfvK35fDtqqEaMDhGuIu1QKmmFeJVs+xjEJ0Bono0EC7goNyKBUKNho6MafVp+g6D0OBxNiAX/hFPZUeqlNW+9paYiZe+/kws389yl3vu69LtSWWmeISEgs2nGLpljMMXfS3VYtLW9dq3Nwv+F49k0mB36BW6PlN34Pby95hteEGh9eyTYkwb69GzKsyF1ZvkBx2BJIke0urOpaXwxr2IublFN73M64BTJaXJMHO8koEHRIiHdY9t8S0y5YLWu5UPUScJoE3Az+mpfISE8/9i/CAwfxHdz9lqB1Osf+0z1hbP99ids2dWH4MgwTncissPMuY3MmsIq6UaFGh53HVav4d8C1BCh15UhgztCP50dCHyiNoOFz6pNEbXE5/NemgWqW0qs0FJvFy0K4O9wbsDeUZ+6LprGsI8aoFLC2i7Wm5gOMyOLaYvsemRNQTym7cXvYWMwM/537VRsYF/MQA5W4maccRoJSvEFpZw1t3Y5Ak80MJMN2i8sLIJTtoqbjAnMD/0VV5EoB1+q5M1T5GNjFOnf+MA9e3OqsKTIITHRpIVqF1RQ695NjyAplUiWokyZpioEMX/U2gSmE+RuR5OYe4S7WAUqkwZ9nvOGMUr/YOChDa0rtFA5rEhNC/TUPeur8j26fdSkBYPZ7TPsmplEVkS1Fcp7zA9+qZ3Hx+IWo8Y13JYXr4LB9UgyRh+eyZ9lFi4DHVan5RT6Or8iQFUijPacbxmHaS08JVGTpH1QCdwHIZlolKLS/JPtjv7HIkSy7lV9RiM+XuJTevR4yDhfoCa4TlVUuEBgVQrNFzudiY0GmX4+WAjk2i2PTCAKtt65+/hUt5pbSMi6DLzwZmBX7KENUWep1bzCr1WiZpn+SQTQ9CS7aeusyy7em8dGd7Y5JsNfhs6xlmrDrE2//oxM2tG5i3SxJWlpdObyBJcYk5gf+jh/K4cfz6zkzRPkYG7ivdo9FVnlRaGbGRwUC+1bbKYl7gnlQJS7ZMGUC9MLW525CgaoTlVUuEWcwEhqlVNK8fVu1zRQYH0ibOWEGhUBnJM9oJjNP8m5LAGNopzxlTDlTfEYC85TB00d/8uO8ir60+XO0xzFh1CIDJ3+63srz0Bskcy1Ng4F7Nj/yqnkoP5XEKpRAma8cyUjvZrcIFNbS8IuUtL0futlwSRbUC9uWoA5QkRIcI4XIRIV61hGUCabv4SNn279XBlIqwxpDM591WsFqfTKBCz7OB37FK/RJtFY7L42Tku6eE9Le7zpt/N7mNTRWZLFe/xjPaxYQoNGzUX8/Asrf4Wt+fqoLy1UGnr369ithIe+vzRFZR5QvNqwzYOz+aoADxGFYHcddqCctF0866jM5gOdNXoIziKe2/maD5F1ekcDooz/KjejpPqVaikqkqqg5QUlha8xjZ3LXHzb9LBgM35PzAGvUUeimPUkIw07WjeVQ7lYs0qOQsNcN2ttAVYmUsr2WV1ET7fvcFVh+4ZLWtJl6jsLiqhxCvWsLS8nLULag6WGZjm7LMfzb05j7FXHaH9EGt0PN84NfGBc6K81bHbjyRQ8eXfyfLySYehaVa3lpzlMPlLdtsaaLIpuCjwdx5/l1CFWVs1bfnQeW7fKlPwRPWliW6aizMNiEXsK9NggPFY1gdxF2rJSyX7nRwMk3CGSyD45ZLZP6Y+QDru/yHZzTjyZPC6Kw8zc/qabDpP3ZWWPIbqUz6Zl+V15rz2zEWrD/F/7230eYdiaGqVNaoX6BD2V5KpCBmaEfwsHYa6ZKjzjnuRWcwVDtgLxfzqk0s67YJnEeIVy1hWrqjVilpHWtf0qW6WIbOOjWJNv+uUChAoWCV4SZuL3ubVH1XghQ6WPcy36pn0UJx0eo8lnErRxy+ZG9xxXOZzwLfZHbgJ4QrStluaMMdmtl8ph+IhNJjibG2XNXo+b//2oqqczTyuniJx7A6iFSJWsJUa6ttfIRbqwZYuo33dm2MQZLo1qw8b6rcKssihjHaSfzD8BczAj6jq/Ikv6in8o7uAZbo78Dg5N8w63FL/FO1gZcCPidScZVSKZC3dQ+xVD/Q6fO5k11nr3CxmhMQEV5e0C5iXtVDiFctYVpE3dGNwXqwdhuVSgX/7JFofm1KEDWi4Ft9Pzbpr+etwEX0U+3npcAvGaTawfPaJzgjxVd5LXW5hRBLLrMDP2aAai8AuwyteV77BKelBLd8pupwrHyxe3VwYpWWy7jiwYqYV/UQd62WeLhXU/41oBXj+8sv4aku/dsaY0rxMkHnAxfy7bZlUJ8R2hd4QTuWQimEnsrj/KqeykjVGppP+YmkKas5f8V+CY7eILEj7TL3Kjfye9BkBqj2UiYF8oZ2KP/UzPSqcAEczaieeBm9a3n1Gn9Ly5oMyWlEzKt6+IR4ffDBByQlJREcHEyvXr3Yvn17pft/8803tG3bluDgYDp27Mgvv/xSSyOtPo0ignnu9jZOVZJwhZl3tWfqHW35+onedu+NvrG5g6MUrND3Z1DZm2zSdyBEoeHlwM9Ypn6dREUmN731J6XaiqB+TlEZN0z7innSO/xHvYAoRQl7DS0YrHmdj/R3ecVNNDGgXLyrU88LrC1XWyxbz3kSYXlVD6/ftRUrVjBx4kRmzpzJ7t276dy5MwMHDiQrS76BxJYtWxg6dChjxoxhz5493HPPPdxzzz0cPHhQdv+6TkRwIE/0a0livVC79wZdH8d9XRs7PPYCDXlEO43p2tEUS0HcoDzCGvUUHlGtZcj8vwBj3taJdUv4PWgyt6l2oZFUvK19kPs1szgpNfHY53KGrVMH0DC8esubTKgqEa8mMfb3tCoig42RmEd6N3P6mJYN3TeBcy3hdfGaO3cuY8eOZdSoUbRv356FCxcSGhrK4sWLZff/73//y6BBg3j++edp164dr776Kt26deP999+v5ZH7B3d0rCqWpeBLfQqDNG/yt6EdYYoyXgtcwktXprPn7z/4deZAeu99gRhFEQcNSdyleZ0P9UPQ431XJz4qhMCAmgWsgsqtHrku1Y6WcL15X0fz77ZW0+djerFxcn/6t6k6ReTLx3rx6A3NGH+Le0MJ1wpeDdhrNBp27drF1KlTzduUSiUpKSls3bpV9pitW7cyceJEq20DBw5k5cqVsvuXlZVRVlZR7qSgQD7Bsq6S0q4RHw/vwed/n2XD8WyH+52TYhmqmc5w1VqmBCzjJtUhWHMvXVWglVTM193Lh/q70fnIHM/IPkkAtI+3ngBRKRVWS3Veu+d61h3JpGtiDKv2XTCXFnr/4a5sO51LjyTjzOzGyQPYdfYKPZvHsGzbORrHhNC0figRwQEUluoY3rsZ/+jehI6No1AoFNzTtTF5JVpiI4P4Ztd5Xlp5EI3eQFxUsGzGvhw3tmrAja08t+qgruPVb2JOTg56vZ7Y2Fir7bGxsRw9elT2mIyMDNn9MzLki9TNnj2bWbNmuWfAfohCoSClfSxN64eyJ/0KBaU6mtUPpVvTGH7Yc4FPRvRgzKfGDkMSSj7VD2S9oTNzAhfSU3mcI4amPKcdx2GZKhUNI4L4dlxv+r2zvtrju619LOuPZfFU/1aczCqid8v6bE/LZdVeYx7a8wPb0CgiiOe/3c9Hj3anf9tGlJTpiQwxfnWHJifSNj6CL/4+yw3N63NX5wSCApRWa0cfucHowj2T0pp1hzNp0TCMFg3DubNTxSRDXFQwgzvFm/cz8du/+7LxRDb3dG1sFVgPDlQRF2V8/UCPRPq0rE9uscZp4RLUHIVU262VLbh48SKNGzdmy5Yt9O5dEXCePHkyGzZsYNu2bXbHqNVqPv30U4YOHWre9uGHHzJr1iwyM+17FMpZXomJieTn5xMZ6b5lOv5AqVZPqVZPdKgaMBbUUygUSJJEbrEGpUJBTJiaUq2egpJSck7t4ayqKS1iYwhQKVCrlIQHBRATpsZgMC6ENuWZlen05ofbYJBkZ/F0eoMotCeokoKCAqKioqp8Rr1qeTVo0ACVSmUnOpmZmcTFxckeExcX59L+QUFBBAXVLKhbVwgOVFklRJpL1ygU1LcIfAcHqgiOCqNRt5to7+BctlUxLK0SRxUzhHAJ3IlXv01qtZru3buTmppq3mYwGEhNTbWyxCzp3bu31f4Aa9eudbi/QCCom3g9+jpx4kRGjBhBjx49SE5OZt68eRQXFzNq1CgAhg8fTuPGjZk9ezYAzzzzDP369ePdd99l8ODBLF++nJ07d/LRRx9582MIBIJaxuvi9eCDD5Kdnc2MGTPIyMigS5curFmzxhyUT09PR2nRP6tPnz589dVXvPjii0ybNo3WrVuzcuVKrr/+em99BIFA4AW8GrD3Bs4GAwUCgXdw9hkVEVSBQOCXCPESCAR+iRAvgUDgl3g9YF/bmEJ819oyIYHAXzA9m1WF46858SosNNZ9SkxMrGJPgUDgTQoLC4mKcly885qbbTQYDFy8eJGIiAgUCoV5udC5c+fE7KMN4t7II+6LY9xxbyRJorCwkISEBKs0KVuuOctLqVTSpIl9HarIyEjxRXSAuDfyiPvimJrem8osLhMiYC8QCPwSIV4CgcAvuebFKygoiJkzZ4rKEzKIeyOPuC+Oqc17c80F7AUCQd3gmre8BAKBfyLESyAQ+CVCvAQCgV8ixEsgEPgldV68cnNzGTZsGJGRkURHRzNmzBiKiooqPaa0tJSnnnqK+vXrEx4ezv33329XN1+hUNj9LF++3JMfpca4uzO5JEnMmDGD+Ph4QkJCSElJ4cSJE578CB7D3fdm5MiRdt+PQYMGefIjeARX7suhQ4e4//77SUpKQqFQMG/evBqfs1KkOs6gQYOkzp07S3///be0ceNGqVWrVtLQoUMrPWbcuHFSYmKilJqaKu3cuVO64YYbpD59+ljtA0hLliyRLl26ZP65evWqJz9KjVi+fLmkVqulxYsXS4cOHZLGjh0rRUdHS5mZmbL7b968WVKpVNLbb78tHT58WHrxxRelwMBA6cCBA+Z93nzzTSkqKkpauXKltG/fPunuu++Wmjdv7tP3QQ5P3JsRI0ZIgwYNsvp+5Obm1tZHcguu3pft27dLkyZNkpYtWybFxcVJ//nPf2p8zsqo0+J1+PBhCZB27Nhh3vbrr79KCoVCunDhguwxeXl5UmBgoPTNN9+Ytx05ckQCpK1bt5q3AdIPP/zgsbG7m+TkZOmpp54yv9br9VJCQoI0e/Zs2f0feOABafDgwVbbevXqJT3xxBOSJEmSwWCQ4uLipHfeecf8fl5enhQUFCQtW7bMA5/Ac7j73kiSUbyGDBnikfHWFq7eF0uaNWsmK141Oactddpt3Lp1K9HR0fTo0cO8LSUlBaVSKdsTEmDXrl1otVpSUlLM29q2bUvTpk3tung/9dRTNGjQgOTkZBYvXlxlCQ9vYepMbvmZnOlMbrk/GDuTm/ZPS0sjIyPDap+oqCh69erl8Jy+iCfujYn169fTqFEj2rRpw5NPPsnly5fd/wE8RHXuS22fs04vzM7IyKBRo0ZW2wICAqhXr57DDtsZGRmo1Wqio6Otttt25X7llVcYMGAAoaGh/P7774wfP56ioiKefvppt3+OmuKJzuSmf13pXu6LeKpr+6BBg7jvvvto3rw5p06dYtq0adxxxx1s3boVlUple0qfozr3pbbP6ZfiNWXKFN56661K9zly5IhHx/DSSy+Zf+/atSvFxcW88847PilegtrnoYceMv/esWNHOnXqRMuWLVm/fj233nqrF0dWd/BLt/G5557jyJEjlf60aNGCuLg4srKyrI7V6XTk5uZW2pFbo9GQl5dntb2yrtwAvXr14vz585SVldX487kbT3QmN/3ryjl9kdro2g7QokULGjRowMmTJ2s+6FqgOvelts/pl+LVsGFD2rZtW+mPWq2md+/e5OXlsWvXLvOxf/zxBwaDgV69esmeu3v37gQGBlp15T527Bjp6emVduXeu3cvMTExPrlY1xOdyZs3b05cXJzVPgUFBWzbts2vupfXVtf28+fPc/nyZeLj490zcA9TnftS6+d0OcTvZwwaNEjq2rWrtG3bNmnTpk1S69atrVIlzp8/L7Vp00batm2bedu4ceOkpk2bSn/88Ye0c+dOqXfv3lLv3r3N7//444/SokWLpAMHDkgnTpyQPvzwQyk0NFSaMWNGrX42V1i+fLkUFBQkLV26VDp8+LD0+OOPS9HR0VJGRoYkSZL06KOPSlOmTDHvv3nzZikgIECaM2eOdOTIEWnmzJmyqRLR0dHSqlWrpP3790tDhgzx21QJd96bwsJCadKkSdLWrVultLQ0ad26dVK3bt2k1q1bS6WlpV75jNXB1ftSVlYm7dmzR9qzZ48UHx8vTZo0SdqzZ4904sQJp8/pCnVevC5fviwNHTpUCg8PlyIjI6VRo0ZJhYWF5vfT0tIkQPrzzz/N265evSqNHz9eiomJkUJDQ6V7771XunTpkvn9X3/9VerSpYsUHh4uhYWFSZ07d5YWLlwo6fX62vxoLjN//nypadOmklqtlpKTk6W///7b/F6/fv2kESNGWO3/9ddfS9ddd52kVqulDh06SKtXr7Z632AwSC+99JIUGxsrBQUFSbfeeqt07Nix2vgobsed96akpES6/fbbpYYNG0qBgYFSs2bNpLFjx1brAfU2rtwX07Nk+9OvXz+nz+kKoiSOQCDwS/wy5iUQCARCvAQCgV8ixEsgEPglQrwEAoFfIsRLIBD4JUK8BAKBXyLESyAQ+CVCvAQCgV8ixEvgE8iVTfbX0smC2sEvS+II6iaDBg1iyZIlVtscLXTXarUEBgZabdNoNKjVapevW93jBN5FWF4CnyEoKIi4uDirn5iYGMDY8GTBggXcfffdhIWF8frrr/Pyyy/TpUsXPv74Y5o3b05wcDAA6enpDBkyhPDwcCIjI3nggQesyrA4Ok7gXwjxEvgNL7/8Mvfeey8HDhxg9OjRAJw8eZLvvvuO77//nr1792IwGBgyZAi5ubls2LCBtWvXcvr0aR588EGrc9keJ/A/hNso8Bl+/vlnwsPDrbZNmzaNadOmAfDwww8zatQoq/c1Gg2fffYZDRs2BIx1tQ4cOEBaWhqJiYkAfPbZZ3To0IEdO3bQs2dP2eME/ocQL4HP0L9/fxYsWGC1rV69eubfLRupmGjWrJmVAB05coTExESzcAG0b9+e6Ohojhw5YhYv2+ME/ocQL4HPEBYWRqtWrSp935ltzl5L4N+ImJegTtGuXTvOnTvHuXPnzNsOHz5MXl4e7du39+LIBO5GWF4Cn6GsrMyubVpAQAANGjRw+hwpKSl07NiRYcOGMW/ePHQ6HePHj6dfv36ybqfAfxGWl8BnWLNmDfHx8VY/N910k0vnUCgUrFq1ipiYGPr27UtKSgotWrRgxYoVHhq1wFuIMtACgcAvEZaXQCDwS4R4CQQCv0SIl0Ag8EuEeAkEAr9EiJdAIPBLhHgJBAK/RIiXQCDwS4R4CQQCv0SIl0Ag8EuEeAkEAr9EiJdAIPBLhHgJBAK/5P8BmlKOpkF2v/wAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.02152, + "end_time": "2024-02-29T13:08:22.374773", + "exception": false, + "start_time": "2024-02-29T13:08:22.353253", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1590.538902, + "end_time": "2024-02-29T13:08:25.117108", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/lct_gan/2/mlu-eval.ipynb", + "output_path": "eval/insurance/lct_gan/2/mlu-eval.ipynb", + "parameters": { + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/insurance/lct_gan/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "lct_gan" + }, + "start_time": "2024-02-29T12:41:54.578206", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/lct_gan/model.pt b/insurance/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..3637cb8155b22a6b2c7de414b541f1136ea7ad1f --- /dev/null +++ b/insurance/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:707ced24eb7135366f3aa72529a23efe2f8ab8103b2b6ab6bb57b1624efbb5ff +size 38580983 diff --git a/insurance/lct_gan/params.json b/insurance/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..eede52710afa04e125b27a42887d4bf655d0aeac --- /dev/null +++ b/insurance/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.77, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.75, "loss_balancer_r": 0.95, "fixed_role_model": "lct_gan", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": false, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "relu6", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "rrelu", "head_activation_final": "softsign", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/realtabformer/eval.csv b/insurance/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..0ca2aea61acad742b3077b77f42e0c0769b4d549 --- /dev/null +++ b/insurance/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.014167995679946173,0.02218004682639329,0.0011680017544638207,1.6954426765441895,0.22907589375972748,3.489088535308838,0.4298049509525299,1.9553140191419516e-06,2.115530014038086,0.024390142410993576,0.45029416680336,0.034176040440797806,0.16980427503585815,0.00019665222498588264,3.8109726905822754 diff --git a/insurance/realtabformer/history.csv b/insurance/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..f1b58830619bae0633dfc3a37deb6fe0f374275e --- /dev/null +++ b/insurance/realtabformer/history.csv @@ -0,0 +1,14 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.06551592258038,3.8791051520893234,0.03416722755192225,0.0,0.0,0.0,0.0,0.0,0.06551592258038,320,40,41.27148151397705,1.0317870378494263,0.12897337973117828,0.12673317359294742,0.0045693422085605565,0.18476937365267077,6.355656341838767e-05,0.0,0.0,0.0,0.0,0.0,0.0045693422085605565,80,10,8.436395406723022,0.8436395406723023,0.10545494258403779,0.08324137404561043 +1,0.0037559630061878126,1.925213829313543,3.748016030745149e-05,0.0,0.0,0.0,0.0,0.0,0.0037559630061878126,320,40,40.829859256744385,1.0207464814186096,0.1275933101773262,0.09227103746379725,0.0010763755642983596,0.15519529518205671,2.772719910471011e-06,0.0,0.0,0.0,0.0,0.0,0.0010763755642983596,80,10,8.419125080108643,0.8419125080108643,0.10523906350135803,0.09969702027738095 +2,0.0023816580101993167,3.9052163640380853,1.0410949786183142e-05,0.0,0.0,0.0,0.0,0.0,0.0023816580101993167,320,40,41.17148947715759,1.0292872369289399,0.12866090461611748,0.08443290112772957,0.010231703845784068,24.354415035247804,2.082776591478819e-05,0.0,0.0,0.0,0.0,0.0,0.010231703845784068,80,10,8.513915061950684,0.8513915061950683,0.10642393827438354,0.01369248509290628 +3,0.005280107025464531,12.423834150836047,1.5615434550020346e-05,0.0,0.0,0.0,0.0,0.0,0.005280107025464531,320,40,41.08780074119568,1.027195018529892,0.1283993773162365,0.06296185727987905,0.0026845919943298212,1.0221173237751728,4.954857529959611e-06,0.0,0.0,0.0,0.0,0.0,0.0026845919943298212,80,10,8.557486295700073,0.8557486295700073,0.10696857869625091,0.06754178307019174 +4,0.001628522769169649,0.9669525724625203,2.4910537018897618e-06,0.0,0.0,0.0,0.0,0.0,0.001628522769169649,320,40,41.08520555496216,1.027130138874054,0.12839126735925674,0.08523798966780305,0.0011392909364076331,0.8447293579599318,8.642555209048553e-07,0.0,0.0,0.0,0.0,0.0,0.0011392909364076331,80,10,8.54101276397705,0.8541012763977051,0.10676265954971313,0.07724661021493376 +5,0.0006698742679873248,0.5631417240535356,1.3532459000285823e-07,0.0,0.0,0.0,0.0,0.0,0.0006698742679873248,320,40,41.17172598838806,1.0292931497097015,0.1286616437137127,0.09377104000886902,0.00028418309084372596,0.9654686861199593,6.203523792436271e-08,0.0,0.0,0.0,0.0,0.0,0.00028418309084372596,80,10,8.475411891937256,0.8475411891937256,0.1059426486492157,0.08174715298227966 +6,0.00026526139699853955,0.0404354411696886,5.110972021091231e-08,0.0,0.0,0.0,0.0,0.0,0.00026526139699853955,320,40,41.210866928100586,1.0302716732025146,0.12878395915031432,0.1002270121127367,0.0003043986107513774,0.7276606579284817,4.158757311856221e-08,0.0,0.0,0.0,0.0,0.0,0.0003043986107513774,80,10,8.365734815597534,0.8365734815597534,0.10457168519496918,0.08551097614690661 +7,0.00033921911108336644,0.04215667733975863,2.0074933999580934e-07,0.0,0.0,0.0,0.0,0.0,0.00033921911108336644,320,40,41.27673935890198,1.0319184839725495,0.12898981049656869,0.09141667010262608,0.0003641421761130914,2.5711033316561953,8.640791372971357e-08,0.0,0.0,0.0,0.0,0.0,0.0003641421761130914,80,10,8.418156147003174,0.8418156147003174,0.10522695183753968,0.07711024282034487 +8,0.00027859737192557075,0.6936592234017018,1.7161798243723202e-08,0.0,0.0,0.0,0.0,0.0,0.00027859737192557075,320,40,40.9585645198822,1.0239641129970551,0.1279955141246319,0.09465919948415831,0.00026416685177537146,2.1159255215665325,2.42559791252539e-08,0.0,0.0,0.0,0.0,0.0,0.00026416685177537146,80,10,8.478416442871094,0.8478416442871094,0.10598020553588867,0.07725258702412248 +9,0.00025029900834852017,0.03145681113393835,3.047866507614738e-08,0.0,0.0,0.0,0.0,0.0,0.00025029900834852017,320,40,40.887590169906616,1.0221897542476654,0.12777371928095818,0.0979282318148762,0.00020534966315608472,2.1954285900741297,7.391519909029712e-09,0.0,0.0,0.0,0.0,0.0,0.00020534966315608472,80,10,8.422897815704346,0.8422897815704345,0.10528622269630432,0.08292618948034942 +10,0.00018911582246801119,0.10153866822858788,1.738624569006149e-08,0.0,0.0,0.0,0.0,0.0,0.00018911582246801119,320,40,41.04085445404053,1.0260213613510132,0.12825267016887665,0.09834078068379312,0.00036368721775943414,2.7509145542862825,1.876272959222547e-08,0.0,0.0,0.0,0.0,0.0,0.00036368721775943414,80,10,8.390719890594482,0.8390719890594482,0.10488399863243103,0.07442375177051871 +11,0.00028841259018008714,0.5210410886732475,1.396120100700081e-07,0.0,0.0,0.0,0.0,0.0,0.00028841259018008714,320,40,40.944926261901855,1.0236231565475464,0.1279528945684433,0.09693868652684615,0.0009965902085241397,2.239691164344549,6.967871311047701e-08,0.0,0.0,0.0,0.0,0.0,0.0009965902085241397,80,10,8.430182218551636,0.8430182218551636,0.10537727773189545,0.06215766463428736 +12,0.0009654337161919102,0.6124627925846198,8.020637164636666e-07,0.0,0.0,0.0,0.0,0.0,0.0009654337161919102,320,40,41.00464582443237,1.0251161456108093,0.12813951820135117,0.09093999108299614,0.0008928512179409154,1.408968701583035,4.1629629343731264e-08,0.0,0.0,0.0,0.0,0.0,0.0008928512179409154,80,10,8.480279445648193,0.8480279445648193,0.10600349307060242,0.06699345875531434 diff --git a/insurance/realtabformer/mlu-eval.ipynb b/insurance/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..dc9adb3d2387d94669108158165331c2cac30973 --- /dev/null +++ b/insurance/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2483 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:01.857264Z", + "iopub.status.busy": "2024-02-29T03:25:01.856891Z", + "iopub.status.idle": "2024-02-29T03:25:01.890331Z", + "shell.execute_reply": "2024-02-29T03:25:01.889589Z" + }, + "papermill": { + "duration": 0.048361, + "end_time": "2024-02-29T03:25:01.892459", + "exception": false, + "start_time": "2024-02-29T03:25:01.844098", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:01.917713Z", + "iopub.status.busy": "2024-02-29T03:25:01.917353Z", + "iopub.status.idle": "2024-02-29T03:25:01.924131Z", + "shell.execute_reply": "2024-02-29T03:25:01.923341Z" + }, + "papermill": { + "duration": 0.021362, + "end_time": "2024-02-29T03:25:01.926002", + "exception": false, + "start_time": "2024-02-29T03:25:01.904640", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:01.949739Z", + "iopub.status.busy": "2024-02-29T03:25:01.949480Z", + "iopub.status.idle": "2024-02-29T03:25:01.953512Z", + "shell.execute_reply": "2024-02-29T03:25:01.952662Z" + }, + "papermill": { + "duration": 0.01793, + "end_time": "2024-02-29T03:25:01.955392", + "exception": false, + "start_time": "2024-02-29T03:25:01.937462", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:01.978702Z", + "iopub.status.busy": "2024-02-29T03:25:01.978452Z", + "iopub.status.idle": "2024-02-29T03:25:01.982360Z", + "shell.execute_reply": "2024-02-29T03:25:01.981528Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017982, + "end_time": "2024-02-29T03:25:01.984356", + "exception": false, + "start_time": "2024-02-29T03:25:01.966374", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:02.007975Z", + "iopub.status.busy": "2024-02-29T03:25:02.007696Z", + "iopub.status.idle": "2024-02-29T03:25:02.013065Z", + "shell.execute_reply": "2024-02-29T03:25:02.012245Z" + }, + "papermill": { + "duration": 0.019231, + "end_time": "2024-02-29T03:25:02.014959", + "exception": false, + "start_time": "2024-02-29T03:25:01.995728", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3033c0c1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:02.039775Z", + "iopub.status.busy": "2024-02-29T03:25:02.039504Z", + "iopub.status.idle": "2024-02-29T03:25:02.044079Z", + "shell.execute_reply": "2024-02-29T03:25:02.043286Z" + }, + "papermill": { + "duration": 0.019104, + "end_time": "2024-02-29T03:25:02.045988", + "exception": false, + "start_time": "2024-02-29T03:25:02.026884", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"realtabformer\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/realtabformer/4\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010933, + "end_time": "2024-02-29T03:25:02.068115", + "exception": false, + "start_time": "2024-02-29T03:25:02.057182", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:02.092690Z", + "iopub.status.busy": "2024-02-29T03:25:02.091857Z", + "iopub.status.idle": "2024-02-29T03:25:02.101551Z", + "shell.execute_reply": "2024-02-29T03:25:02.100705Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024106, + "end_time": "2024-02-29T03:25:02.103499", + "exception": false, + "start_time": "2024-02-29T03:25:02.079393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/realtabformer/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:02.130001Z", + "iopub.status.busy": "2024-02-29T03:25:02.129041Z", + "iopub.status.idle": "2024-02-29T03:25:04.463979Z", + "shell.execute_reply": "2024-02-29T03:25:04.463041Z" + }, + "papermill": { + "duration": 2.350744, + "end_time": "2024-02-29T03:25:04.466209", + "exception": false, + "start_time": "2024-02-29T03:25:02.115465", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:04.493130Z", + "iopub.status.busy": "2024-02-29T03:25:04.492731Z", + "iopub.status.idle": "2024-02-29T03:25:04.504576Z", + "shell.execute_reply": "2024-02-29T03:25:04.503859Z" + }, + "papermill": { + "duration": 0.027796, + "end_time": "2024-02-29T03:25:04.506594", + "exception": false, + "start_time": "2024-02-29T03:25:04.478798", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:04.531337Z", + "iopub.status.busy": "2024-02-29T03:25:04.530963Z", + "iopub.status.idle": "2024-02-29T03:25:04.538581Z", + "shell.execute_reply": "2024-02-29T03:25:04.537836Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021716, + "end_time": "2024-02-29T03:25:04.540474", + "exception": false, + "start_time": "2024-02-29T03:25:04.518758", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:04.564099Z", + "iopub.status.busy": "2024-02-29T03:25:04.563831Z", + "iopub.status.idle": "2024-02-29T03:25:04.666778Z", + "shell.execute_reply": "2024-02-29T03:25:04.665966Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.117235, + "end_time": "2024-02-29T03:25:04.668965", + "exception": false, + "start_time": "2024-02-29T03:25:04.551730", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:04.694614Z", + "iopub.status.busy": "2024-02-29T03:25:04.694338Z", + "iopub.status.idle": "2024-02-29T03:25:09.370745Z", + "shell.execute_reply": "2024-02-29T03:25:09.369937Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.691753, + "end_time": "2024-02-29T03:25:09.373186", + "exception": false, + "start_time": "2024-02-29T03:25:04.681433", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 03:25:06.968899: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 03:25:06.968960: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 03:25:06.970732: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:09.399216Z", + "iopub.status.busy": "2024-02-29T03:25:09.398614Z", + "iopub.status.idle": "2024-02-29T03:25:09.405928Z", + "shell.execute_reply": "2024-02-29T03:25:09.405188Z" + }, + "papermill": { + "duration": 0.022272, + "end_time": "2024-02-29T03:25:09.407850", + "exception": false, + "start_time": "2024-02-29T03:25:09.385578", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:09.435072Z", + "iopub.status.busy": "2024-02-29T03:25:09.434254Z", + "iopub.status.idle": "2024-02-29T03:25:18.244653Z", + "shell.execute_reply": "2024-02-29T03:25:18.243385Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.826767, + "end_time": "2024-02-29T03:25:18.247128", + "exception": false, + "start_time": "2024-02-29T03:25:09.420361", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:18.763729Z", + "iopub.status.busy": "2024-02-29T03:25:18.763015Z", + "iopub.status.idle": "2024-02-29T03:25:18.834082Z", + "shell.execute_reply": "2024-02-29T03:25:18.833159Z" + }, + "papermill": { + "duration": 0.08788, + "end_time": "2024-02-29T03:25:18.836089", + "exception": false, + "start_time": "2024-02-29T03:25:18.748209", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T03:25:18.865541Z", + "iopub.status.busy": "2024-02-29T03:25:18.865251Z", + "iopub.status.idle": "2024-02-29T03:25:19.308543Z", + "shell.execute_reply": "2024-02-29T03:25:19.307591Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.460649, + "end_time": "2024-02-29T03:25:19.310570", + "exception": false, + "start_time": "2024-02-29T03:25:18.849921", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:19.340277Z", + "iopub.status.busy": "2024-02-29T03:25:19.339956Z", + "iopub.status.idle": "2024-02-29T03:25:19.344065Z", + "shell.execute_reply": "2024-02-29T03:25:19.343340Z" + }, + "papermill": { + "duration": 0.021386, + "end_time": "2024-02-29T03:25:19.345840", + "exception": false, + "start_time": "2024-02-29T03:25:19.324454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:19.373004Z", + "iopub.status.busy": "2024-02-29T03:25:19.372710Z", + "iopub.status.idle": "2024-02-29T03:25:19.379941Z", + "shell.execute_reply": "2024-02-29T03:25:19.379078Z" + }, + "papermill": { + "duration": 0.022893, + "end_time": "2024-02-29T03:25:19.381812", + "exception": false, + "start_time": "2024-02-29T03:25:19.358919", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10420892" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:19.409201Z", + "iopub.status.busy": "2024-02-29T03:25:19.408894Z", + "iopub.status.idle": "2024-02-29T03:25:19.503585Z", + "shell.execute_reply": "2024-02-29T03:25:19.502633Z" + }, + "papermill": { + "duration": 0.11087, + "end_time": "2024-02-29T03:25:19.505798", + "exception": false, + "start_time": "2024-02-29T03:25:19.394928", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 15200] --\n", + "├─Adapter: 1-1 [2, 1071, 15200] --\n", + "│ └─Embedding: 2-1 [2, 1071, 19, 800] (440,800)\n", + "│ └─TensorInductionPoint: 2-2 [19, 1] 19\n", + "│ └─Sequential: 2-3 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 820,224\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 15200] (recursive)\n", + "│ └─Embedding: 2-4 [2, 267, 19, 800] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [19, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,861,692\n", + "Trainable params: 10,420,892\n", + "Non-trainable params: 440,800\n", + "Total mult-adds (M): 43.07\n", + "========================================================================================================================\n", + "Input size (MB): 0.20\n", + "Forward/backward pass size (MB): 632.89\n", + "Params size (MB): 43.45\n", + "Estimated Total Size (MB): 676.54\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:25:19.536409Z", + "iopub.status.busy": "2024-02-29T03:25:19.536066Z", + "iopub.status.idle": "2024-02-29T03:37:37.107344Z", + "shell.execute_reply": "2024-02-29T03:37:37.106378Z" + }, + "papermill": { + "duration": 737.605486, + "end_time": "2024-02-29T03:37:37.126175", + "exception": false, + "start_time": "2024-02-29T03:25:19.520689", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.06551592258038, 'avg_role_model_std_loss': 3.8791051520893234, 'avg_role_model_mean_pred_loss': 0.03416722755192225, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.06551592258038, 'n_size': 320, 'n_batch': 40, 'duration': 41.27148151397705, 'duration_batch': 1.0317870378494263, 'duration_size': 0.12897337973117828, 'avg_pred_std': 0.12673317359294742}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0045693422085605565, 'avg_role_model_std_loss': 0.18476937365267077, 'avg_role_model_mean_pred_loss': 6.355656341838767e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0045693422085605565, 'n_size': 80, 'n_batch': 10, 'duration': 8.436395406723022, 'duration_batch': 0.8436395406723023, 'duration_size': 0.10545494258403779, 'avg_pred_std': 0.08324137404561043}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0037559630061878126, 'avg_role_model_std_loss': 1.925213829313543, 'avg_role_model_mean_pred_loss': 3.748016030745149e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037559630061878126, 'n_size': 320, 'n_batch': 40, 'duration': 40.829859256744385, 'duration_batch': 1.0207464814186096, 'duration_size': 0.1275933101773262, 'avg_pred_std': 0.09227103746379725}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010763755642983596, 'avg_role_model_std_loss': 0.15519529518205671, 'avg_role_model_mean_pred_loss': 2.772719910471011e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010763755642983596, 'n_size': 80, 'n_batch': 10, 'duration': 8.419125080108643, 'duration_batch': 0.8419125080108643, 'duration_size': 0.10523906350135803, 'avg_pred_std': 0.09969702027738095}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023816580101993167, 'avg_role_model_std_loss': 3.9052163640380853, 'avg_role_model_mean_pred_loss': 1.0410949786183142e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023816580101993167, 'n_size': 320, 'n_batch': 40, 'duration': 41.17148947715759, 'duration_batch': 1.0292872369289399, 'duration_size': 0.12866090461611748, 'avg_pred_std': 0.08443290112772957}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010231703845784068, 'avg_role_model_std_loss': 24.354415035247804, 'avg_role_model_mean_pred_loss': 2.082776591478819e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010231703845784068, 'n_size': 80, 'n_batch': 10, 'duration': 8.513915061950684, 'duration_batch': 0.8513915061950683, 'duration_size': 0.10642393827438354, 'avg_pred_std': 0.01369248509290628}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005280107025464531, 'avg_role_model_std_loss': 12.423834150836047, 'avg_role_model_mean_pred_loss': 1.5615434550020346e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005280107025464531, 'n_size': 320, 'n_batch': 40, 'duration': 41.08780074119568, 'duration_batch': 1.027195018529892, 'duration_size': 0.1283993773162365, 'avg_pred_std': 0.06296185727987905}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026845919943298212, 'avg_role_model_std_loss': 1.0221173237751728, 'avg_role_model_mean_pred_loss': 4.954857529959611e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026845919943298212, 'n_size': 80, 'n_batch': 10, 'duration': 8.557486295700073, 'duration_batch': 0.8557486295700073, 'duration_size': 0.10696857869625091, 'avg_pred_std': 0.06754178307019174}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001628522769169649, 'avg_role_model_std_loss': 0.9669525724625203, 'avg_role_model_mean_pred_loss': 2.4910537018897618e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001628522769169649, 'n_size': 320, 'n_batch': 40, 'duration': 41.08520555496216, 'duration_batch': 1.027130138874054, 'duration_size': 0.12839126735925674, 'avg_pred_std': 0.08523798966780305}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011392909364076331, 'avg_role_model_std_loss': 0.8447293579599318, 'avg_role_model_mean_pred_loss': 8.642555209048553e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011392909364076331, 'n_size': 80, 'n_batch': 10, 'duration': 8.54101276397705, 'duration_batch': 0.8541012763977051, 'duration_size': 0.10676265954971313, 'avg_pred_std': 0.07724661021493376}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006698742679873248, 'avg_role_model_std_loss': 0.5631417240535356, 'avg_role_model_mean_pred_loss': 1.3532459000285823e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006698742679873248, 'n_size': 320, 'n_batch': 40, 'duration': 41.17172598838806, 'duration_batch': 1.0292931497097015, 'duration_size': 0.1286616437137127, 'avg_pred_std': 0.09377104000886902}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00028418309084372596, 'avg_role_model_std_loss': 0.9654686861199593, 'avg_role_model_mean_pred_loss': 6.203523792436271e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00028418309084372596, 'n_size': 80, 'n_batch': 10, 'duration': 8.475411891937256, 'duration_batch': 0.8475411891937256, 'duration_size': 0.1059426486492157, 'avg_pred_std': 0.08174715298227966}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00026526139699853955, 'avg_role_model_std_loss': 0.0404354411696886, 'avg_role_model_mean_pred_loss': 5.110972021091231e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00026526139699853955, 'n_size': 320, 'n_batch': 40, 'duration': 41.210866928100586, 'duration_batch': 1.0302716732025146, 'duration_size': 0.12878395915031432, 'avg_pred_std': 0.1002270121127367}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0003043986107513774, 'avg_role_model_std_loss': 0.7276606579284817, 'avg_role_model_mean_pred_loss': 4.158757311856221e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003043986107513774, 'n_size': 80, 'n_batch': 10, 'duration': 8.365734815597534, 'duration_batch': 0.8365734815597534, 'duration_size': 0.10457168519496918, 'avg_pred_std': 0.08551097614690661}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00033921911108336644, 'avg_role_model_std_loss': 0.04215667733975863, 'avg_role_model_mean_pred_loss': 2.0074933999580934e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00033921911108336644, 'n_size': 320, 'n_batch': 40, 'duration': 41.27673935890198, 'duration_batch': 1.0319184839725495, 'duration_size': 0.12898981049656869, 'avg_pred_std': 0.09141667010262608}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0003641421761130914, 'avg_role_model_std_loss': 2.5711033316561953, 'avg_role_model_mean_pred_loss': 8.640791372971357e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003641421761130914, 'n_size': 80, 'n_batch': 10, 'duration': 8.418156147003174, 'duration_batch': 0.8418156147003174, 'duration_size': 0.10522695183753968, 'avg_pred_std': 0.07711024282034487}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00027859737192557075, 'avg_role_model_std_loss': 0.6936592234017018, 'avg_role_model_mean_pred_loss': 1.7161798243723202e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00027859737192557075, 'n_size': 320, 'n_batch': 40, 'duration': 40.9585645198822, 'duration_batch': 1.0239641129970551, 'duration_size': 0.1279955141246319, 'avg_pred_std': 0.09465919948415831}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00026416685177537146, 'avg_role_model_std_loss': 2.1159255215665325, 'avg_role_model_mean_pred_loss': 2.42559791252539e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00026416685177537146, 'n_size': 80, 'n_batch': 10, 'duration': 8.478416442871094, 'duration_batch': 0.8478416442871094, 'duration_size': 0.10598020553588867, 'avg_pred_std': 0.07725258702412248}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00025029900834852017, 'avg_role_model_std_loss': 0.03145681113393835, 'avg_role_model_mean_pred_loss': 3.047866507614738e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00025029900834852017, 'n_size': 320, 'n_batch': 40, 'duration': 40.887590169906616, 'duration_batch': 1.0221897542476654, 'duration_size': 0.12777371928095818, 'avg_pred_std': 0.0979282318148762}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00020534966315608472, 'avg_role_model_std_loss': 2.1954285900741297, 'avg_role_model_mean_pred_loss': 7.391519909029712e-09, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00020534966315608472, 'n_size': 80, 'n_batch': 10, 'duration': 8.422897815704346, 'duration_batch': 0.8422897815704345, 'duration_size': 0.10528622269630432, 'avg_pred_std': 0.08292618948034942}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00018911582246801119, 'avg_role_model_std_loss': 0.10153866822858788, 'avg_role_model_mean_pred_loss': 1.738624569006149e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00018911582246801119, 'n_size': 320, 'n_batch': 40, 'duration': 41.04085445404053, 'duration_batch': 1.0260213613510132, 'duration_size': 0.12825267016887665, 'avg_pred_std': 0.09834078068379312}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00036368721775943414, 'avg_role_model_std_loss': 2.7509145542862825, 'avg_role_model_mean_pred_loss': 1.876272959222547e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00036368721775943414, 'n_size': 80, 'n_batch': 10, 'duration': 8.390719890594482, 'duration_batch': 0.8390719890594482, 'duration_size': 0.10488399863243103, 'avg_pred_std': 0.07442375177051871}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00028841259018008714, 'avg_role_model_std_loss': 0.5210410886732475, 'avg_role_model_mean_pred_loss': 1.396120100700081e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00028841259018008714, 'n_size': 320, 'n_batch': 40, 'duration': 40.944926261901855, 'duration_batch': 1.0236231565475464, 'duration_size': 0.1279528945684433, 'avg_pred_std': 0.09693868652684615}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009965902085241397, 'avg_role_model_std_loss': 2.239691164344549, 'avg_role_model_mean_pred_loss': 6.967871311047701e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009965902085241397, 'n_size': 80, 'n_batch': 10, 'duration': 8.430182218551636, 'duration_batch': 0.8430182218551636, 'duration_size': 0.10537727773189545, 'avg_pred_std': 0.06215766463428736}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009654337161919102, 'avg_role_model_std_loss': 0.6124627925846198, 'avg_role_model_mean_pred_loss': 8.020637164636666e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009654337161919102, 'n_size': 320, 'n_batch': 40, 'duration': 41.00464582443237, 'duration_batch': 1.0251161456108093, 'duration_size': 0.12813951820135117, 'avg_pred_std': 0.09093999108299614}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008928512179409154, 'avg_role_model_std_loss': 1.408968701583035, 'avg_role_model_mean_pred_loss': 4.1629629343731264e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008928512179409154, 'n_size': 80, 'n_batch': 10, 'duration': 8.480279445648193, 'duration_batch': 0.8480279445648193, 'duration_size': 0.10600349307060242, 'avg_pred_std': 0.06699345875531434}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▄▂█▃▂▁▁▁▁▁▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▁▁▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▇█▁▅▆▇▇▆▆▇▆▅▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▄▃▁▃▄▅▄▄▅▅▅▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▄▂█▃▂▁▁▁▁▁▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▁▁▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▃▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▁▁█▁▁▁▁▂▂▂▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▃▂▃█▂▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▄▃▆█▇▅▁▃▅▃▂▃▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▁▆▅▅▆▇█▃▂▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▄▃▆█▇▅▁▃▅▃▂▃▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▁▆▅▅▆▇█▃▂▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▄▃▆█▇▅▁▃▅▃▂▃▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▁▆▅▅▆▇█▃▂▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00089\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00097\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.06699\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.09094\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00089\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00097\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 1.40897\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.61246\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.84803\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.02512\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.106\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.12814\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 8.48028\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 41.00465\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/realtabformer/4/wandb/offline-run-20240229_032521-mpqsh2g2\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_032521-mpqsh2g2/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.0011680080120791366, 'avg_g_mag_loss': 0.015933245898626762, 'avg_g_cos_loss': 0.024182093322725223, 'pred_duration': 2.114567995071411, 'grad_duration': 1.6933445930480957, 'total_duration': 3.807912588119507, 'pred_std': 0.16980430483818054, 'std_loss': 0.00019665305444505066, 'mean_pred_loss': 1.955350626303698e-06, 'pred_rmse': 0.03417613357305527, 'pred_mae': 0.02439017966389656, 'pred_mape': 0.45029589533805847, 'grad_rmse': 0.42980626225471497, 'grad_mae': 0.22907406091690063, 'grad_mape': 3.4890403747558594}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0011680080120791366, 'avg_g_mag_loss': 0.015933245898626762, 'avg_g_cos_loss': 0.024182093322725223, 'avg_pred_duration': 2.114567995071411, 'avg_grad_duration': 1.6933445930480957, 'avg_total_duration': 3.807912588119507, 'avg_pred_std': 0.16980430483818054, 'avg_std_loss': 0.00019665305444505066, 'avg_mean_pred_loss': 1.955350626303698e-06}, 'min_metrics': {'avg_loss': 0.0011680080120791366, 'avg_g_mag_loss': 0.015933245898626762, 'avg_g_cos_loss': 0.024182093322725223, 'pred_duration': 2.114567995071411, 'grad_duration': 1.6933445930480957, 'total_duration': 3.807912588119507, 'pred_std': 0.16980430483818054, 'std_loss': 0.00019665305444505066, 'mean_pred_loss': 1.955350626303698e-06, 'pred_rmse': 0.03417613357305527, 'pred_mae': 0.02439017966389656, 'pred_mape': 0.45029589533805847, 'grad_rmse': 0.42980626225471497, 'grad_mae': 0.22907406091690063, 'grad_mape': 3.4890403747558594}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0011680080120791366, 'avg_g_mag_loss': 0.015933245898626762, 'avg_g_cos_loss': 0.024182093322725223, 'pred_duration': 2.114567995071411, 'grad_duration': 1.6933445930480957, 'total_duration': 3.807912588119507, 'pred_std': 0.16980430483818054, 'std_loss': 0.00019665305444505066, 'mean_pred_loss': 1.955350626303698e-06, 'pred_rmse': 0.03417613357305527, 'pred_mae': 0.02439017966389656, 'pred_mape': 0.45029589533805847, 'grad_rmse': 0.42980626225471497, 'grad_mae': 0.22907406091690063, 'grad_mape': 3.4890403747558594}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:37:37.163535Z", + "iopub.status.busy": "2024-02-29T03:37:37.162754Z", + "iopub.status.idle": "2024-02-29T03:37:37.167538Z", + "shell.execute_reply": "2024-02-29T03:37:37.166628Z" + }, + "papermill": { + "duration": 0.025748, + "end_time": "2024-02-29T03:37:37.169465", + "exception": false, + "start_time": "2024-02-29T03:37:37.143717", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:37:37.203346Z", + "iopub.status.busy": "2024-02-29T03:37:37.203033Z", + "iopub.status.idle": "2024-02-29T03:37:37.493921Z", + "shell.execute_reply": "2024-02-29T03:37:37.493097Z" + }, + "papermill": { + "duration": 0.310748, + "end_time": "2024-02-29T03:37:37.496412", + "exception": false, + "start_time": "2024-02-29T03:37:37.185664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:37:37.533210Z", + "iopub.status.busy": "2024-02-29T03:37:37.532890Z", + "iopub.status.idle": "2024-02-29T03:37:37.801376Z", + "shell.execute_reply": "2024-02-29T03:37:37.800486Z" + }, + "papermill": { + "duration": 0.289623, + "end_time": "2024-02-29T03:37:37.803375", + "exception": false, + "start_time": "2024-02-29T03:37:37.513752", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2I0lEQVR4nO3dfVxUZf7/8deZYWYQFFBQECOxQjFvsDQI82bb2LA1i7ZW8ueq+TWtvloWaYarYNvukpW7VvrNtbZs9xtpbum25pdS0u0G1LxLXW9WDcVNhxvNQUC5mbl+fwwzOgrIIDDAfJ6Px3kwc+Y6Z65B5u11rnOu62hKKYUQQrQCOk9XQAghHCSQhBCthgSSEKLVkEASQrQaEkhCiFZDAkkI0WpIIAkhWg0fT1egKdhsNk6ePEmnTp3QNM3T1RFCXEYpxblz5wgPD0enq7sd1C4C6eTJk0RERHi6GkKIqzhx4gTXXXddna+3i0Dq1KkTYP+wAQEBHq6NEOJyJSUlREREOL+rdWkXgeQ4TAsICJBAEqIVu1qXinRqCyFaDQkkIUSrIYEkhGg12kUfkrg2VquVqqoqT1dDtGEGgwG9Xn/N+5FA8mJKKcxmM2fPnvV0VUQ7EBQURFhY2DVdCyiB5MUcYdStWzf8/PzkolLRKEopysvLKSwsBKB79+6N3pdXBVK11cZ3/7FwurSCn90c6tVfQKvV6gyj4OBgT1dHtHEdOnQAoLCwkG7dujX68M2rOrWrbYoH38xh2l93cK6i2tPV8ShHn5Gfn5+HayLaC8ff0rX0R3pVIPka9HQ02RuFxecqPFyb1sGbW4miaTXF35JXBRJASEcjAMWllR6uiRDicl4XSMEdTQCcLpUWkhCtjdcF0sUWkgSSaBxN01i7dq2nq9GkFixYwKBBgzxdDW8MJHsLqUgO2UQbtmLFCoKCgppsf7NmzSI7O7vJ9tdYXnXaHy4GkhyyCW9QWVmJ0Wi8armOHTvSsWPHFqhR/bywhSSHbHVRSlFeWd3ii7s3T87KymLYsGEEBQURHBzMvffey9GjRwEYOnQoc+bMcSlfVFSEwWDgyy+/BODUqVOMHj2aDh060KtXLzIzM4mMjGTx4sWN+r3t3buXn/70p3To0IHg4GCmTZtGaWmp8/XNmzcTGxuLv78/QUFB3HHHHRw/fhyA7777jjvvvJNOnToREBDA4MGD2b59e73vt3nzZiZPnozFYkHTNDRNY8GCBQBERkby4osvMnHiRAICApg2bRoAc+bMoXfv3vj5+XHDDTcwf/58l9Pzlx+yPfLIIyQlJfHqq6/SvXt3goODmT59erMPMfLaFpKcZbvS+SorN6d91uLvu/83ifgZG/6nWFZWRkpKCgMHDqS0tJS0tDQeeOABdu/ezfjx43n55Zd56aWXnKehV61aRXh4OMOHDwdg4sSJFBcXs3nzZgwGAykpKc6rjN1VVlZGYmIi8fHxfPvttxQWFvLoo48yY8YMVqxYQXV1NUlJSUydOpUPPviAyspKtm3b5qzb+PHjueWWW3jzzTfR6/Xs3r0bg8FQ73sOHTqUxYsXk5aWxqFDhwBcWjevvvoqaWlppKenO9d16tSJFStWEB4ezt69e5k6dSqdOnXiueeeq/N9Nm3aRPfu3dm0aRNHjhwhOTmZQYMGMXXq1Eb9rhrC+wKpkyOQpIXUVj344IMuz9955x26du3K/v37GTt2LE8//TRff/21M4AyMzMZN24cmqZx8OBBNm7cyLfffsuQIUMAePvtt4mKimpUXTIzM7lw4QJ/+ctf8Pf3B2DJkiWMGTOGhQsXYjAYsFgs3Hvvvdx4440A9O3b17l9fn4+s2fPJjo6GqBB9TAajQQGBqJpGmFhYVe8/tOf/pRnn33WZd28efOcjyMjI5k1axYrV66sN5A6d+7MkiVL0Ov1REdHM3r0aLKzsyWQmlKwv/2Q7bS0kK7QwaBn/28SPfK+7jh8+DBpaWls3bqV4uJibDYbYP9y9+/fn7vvvpv333+f4cOHk5eXR25uLn/6058AOHToED4+Ptx6663O/d1000107ty5UXU/cOAAMTExzjACuOOOO7DZbBw6dIgRI0bwyCOPkJiYyM9+9jMSEhIYO3asc7xXSkoKjz76KH/9619JSEjgl7/8pTO4GssRtJdatWoVr7/+OkePHqW0tJTq6uqrzq7ar18/lyEg3bt3Z+/evddUt6vxvj6kmhZSaUU1F6qsHq5N66JpGn5GnxZf3L3Cd8yYMZw5c4a33nqLrVu3snXrVsDegQv2w6C//e1vVFVVkZmZyYABAxgwYECT/74a6t133yU3N5ehQ4eyatUqevfuzZYtWwB7382//vUvRo8ezRdffMHNN9/MmjVrrun9Lg1HgNzcXMaPH8/Pf/5z1q1bx65du/j1r3/t/H3V5fJDR03TnOHfXLwukDqZfDD62D92kQwfaXNOnz7NoUOHmDdvHnfddRd9+/blxx9/dClz//33c+HCBbKyssjMzGT8+PHO1/r06UN1dTW7du1yrjty5MgV+2iovn378t1331FWVuZc980336DT6ejTp49z3S233EJqaio5OTn079+fzMxM52u9e/fmmWee4fPPP+cXv/gF77777lXf12g0YrU27D/UnJwcevbsya9//WuGDBlCVFSUs1O9tfG6QNI0ja6OU/9lctjW1nTu3Jng4GCWL1/OkSNH+OKLL0hJSXEp4+/vT1JSEvPnz+fAgQOMGzfO+Vp0dDQJCQlMmzaNbdu2sWvXLqZNm0aHDh0aNRZr/Pjx+Pr6MmnSJPbt28emTZt48sknmTBhAqGhoeTl5ZGamkpubi7Hjx/n888/5/Dhw/Tt25fz588zY8YMNm/ezPHjx/nmm2/49ttvXfqY6hIZGUlpaSnZ2dkUFxdTXl5eZ9moqCjy8/NZuXIlR48e5fXXX7/mVlhz8bpAAgh2nPqXFlKbo9PpWLlyJTt27KB///4888wzvPLKK1eUGz9+PN999x3Dhw/n+uuvd3ntL3/5C6GhoYwYMYIHHnjAecbJ19fX7fr4+fnx2WefcebMGW677TYeeugh7rrrLpYsWeJ8/eDBgzz44IP07t2badOmMX36dB577DH0ej2nT59m4sSJ9O7dm7Fjx3LPPffwwgsvXPV9hw4dyuOPP05ycjJdu3bl5ZdfrrPsfffdxzPPPMOMGTMYNGgQOTk5zJ8/3+3P2iJUIyxZskT17NlTmUwmFRsbq7Zu3Vpv+Q8//FD16dNHmUwm1b9/f/Xpp59eUWb//v1qzJgxKiAgQPn5+akhQ4ao48ePN6g+FotFAcpisTSo/OR3t6mec9apD7Y2bP/t0fnz59X+/fvV+fPnPV0Vjztx4oQC1MaNGz1dlTatvr+phn5H3W4hrVq1ipSUFNLT09m5cycxMTEkJibWeR1HTk4O48aNY8qUKezatYukpCSSkpLYt2+fs8zRo0cZNmwY0dHRbN68mT179jB//vxG/Y/VEHJxpHf74osv+OSTT8jLyyMnJ4eHH36YyMhIRowY4emqCXdTMDY2Vk2fPt353Gq1qvDwcJWRkVFr+bFjx6rRo0e7rIuLi1OPPfaY83lycrL61a9+5W5VnNxtIb30fwdUzznrVPrf9zX6Pds6b24hZWVlqX79+qkOHTqobt26qaSkJHXs2DGllFL/+7//q/z9/Wtdbr755har46hRo+qsx+9+97sWq4c7mqKF5NZ1SJWVlezYsYPU1FTnOp1OR0JCArm5ubVuk5ube0WnY2JionO0tM1m49NPP+W5554jMTGRXbt20atXL1JTU0lKSqp1nxUVFVRUXGzdlJSUuPMxLrlaW1pI3igxMZHExNqvt7rvvvuIi4ur9bWrXUHdlN5++23Onz9f62tdunRpsXq0NLcCqbi4GKvVSmhoqMv60NBQDh48WOs2ZrO51vJmsxmwz8FbWlrKSy+9xG9/+1sWLlxIVlYWv/jFL9i0aRMjR468Yp8ZGRkN6virixyyibp06tTpqvefbwk9evTwdBU8wuNn2RwXWt1///0888wzDBo0iOeff557772XZcuW1bpNamoqFovFuZw4ccKt93Se9pertYVoVdxqIYWEhKDX6ykoKHBZX1BQUOuYGoCwsLB6y4eEhODj48PNN9/sUqZv3758/fXXte7TZDJhMpncqbqLYDlkE6JVcquFZDQaGTx4sMtETjabjezsbOLj42vdJj4+/oqJnzZs2OAsbzQaue2225yjlh3+/e9/07NnT3eq12COQ7Yfy6uosjbvpfBCiIZze3BtSkoKkyZNYsiQIcTGxrJ48WLKysqYPHkyYJ/aoUePHmRkZAAwc+ZMRo4cyaJFixg9ejQrV65k+/btLF++3LnP2bNnk5yczIgRI7jzzjvJysriH//4B5s3b26aT3mZzn5GdBrYFJwpqyQ0oHkuLxBCuMftQEpOTqaoqIi0tDTMZjODBg0iKyvL2XGdn5+PTnex4TV06FAyMzOZN28ec+fOJSoqirVr19K/f39nmQceeIBly5aRkZHBU089RZ8+ffjoo48YNmxYE3zEK+l0Gl38TRSXVlBcWiGBJEQroSnl5nR9rVBJSQmBgYFYLJarTqngMGrxlxw0n+O9/4plZO+uzVzD1ufChQvk5eXRq1evZrsAtb3SNI01a9bUeVmKt6rvb6qh31GPn2XzlK6OidpkPJtog5p6kn+wT42raRpnz55t0v26w2sDyTnZf5kEkhCthdcGkmPmSJlb+xJKQWVZyy8yyX+TTfJfUVHBrFmz6NGjB/7+/sTFxbmcHDp+/Dhjxoyhc+fO+Pv7069fP9avX8+xY8e48847AfsUL5qm8cgjjzTq93EtvG4KW4cQOWS7UlU5/D685d937kkw+l+9XA2Z5L/uSf5nzJjB/v37WblyJeHh4axZs4ZRo0axd+9eoqKimD59OpWVlXz55Zf4+/uzf/9+OnbsSEREBB999BEPPvgghw4dIiAggA4dOjTqd3ItvDeQnDeMlEBqa2SS/9on+c/Pz+fdd98lPz+f8HD7fyyzZs0iKyuLd999l9///vfk5+fz4IMPOqf0veGGG5zbO8bIdevWrcn7pxrKawPJMUmbDB+5hMHP3lrxxPu6QSb5r93evXuxWq307t3bZX1FRQXBwcEAPPXUUzzxxBN8/vnnJCQk8OCDDzJw4MBGvV9z8No+pK4yfORKmmY/dGrpRSb5b5JJ/ktLS9Hr9ezYsYPdu3c7lwMHDvDaa68B8Oijj/L9998zYcIE9u7dy5AhQ3jjjTea7LNeK68NpJBL5tW22dr8pVheQyb5t6ttkv9bbrkFq9VKYWEhN910k8ty6aFdREQEjz/+OB9//DHPPvssb731lnOfQINvHtAcvDaQutScZbPaFJbzzXt7YNF0ZJJ/u9om+e/duzfjx49n4sSJfPzxx+Tl5bFt2zYyMjL49NNPAXj66af57LPPyMvLY+fOnWzatMn5fj179kTTNNatW0dRUZHLmcIW0zxzx7Usd2eMdBi44DPVc8469W9zSTPVrPVqyzNGbtiwQfXt21eZTCY1cOBAtXnzZgWoNWvWOMusX79eAWrEiBFXbH/y5El1zz33KJPJpHr27KkyMzNVt27d1LJlyxr0/pe/1549e9Sdd96pfH19VZcuXdTUqVPVuXPnlFJKmc1mlZSUpLp3766MRqPq2bOnSktLU1arVVVUVKiHH35YRUREKKPRqMLDw9WMGTMa/G/y+OOPq+DgYAWo9PR0pZRSlZWVKi0tTUVGRiqDwaC6d++uHnjgAbVnzx6llFIzZsxQN954ozKZTKpr165qwoQJqri42LnP3/zmNyosLExpmqYmTZrUoHo4NMWMkV47dATgrkWbOVpURubUOIbeGNKMNWx9ZOjIRf/5z3+IiIhg48aN3HXXXZ6uTpvVFENHvPYsG9j7kY4WlcnFkV7miy++oLS0lAEDBnDq1Cmee+45meS/lfDaPiS4pGNbzrR5laqqKubOnUu/fv144IEH6Nq1q/Miyffff5+OHTvWuvTr16/F6njPPffUWY/f//73LVaPlublLSSZW9sbyST/rZeXB5Jj+Igcsgk7meTfs7z7kK2TjPhvB+c0RCvRFH9LXh1IjhH/RV7Yqe04/CgvL/dwTUR74fhbupZDW+8+ZPPiEf96vZ6goCDnKHc/P79GXRgohFKK8vJyCgsLCQoKQq/XN3pfXh1Il45nU0p53RfSMZygsVNvCHGpoKCgOm+H1lBeHUiOEf8V1TbKKq10NHnXr0PTNLp37063bt2oqpLhM6LxDAbDNbWMHLzrG3gZP6MPfkY95ZVWis9VeF0gOej1+ib5YxLiWjWqU3vp0qVERkbi6+tLXFwc27Ztq7f86tWriY6OxtfXlwEDBrB+/XqX1x955BHnVJyOZdSoUY2pmttCZBoSIVoNtwNp1apVpKSkkJ6ezs6dO4mJiSExMbHOfoicnBzGjRvHlClT2LVrF0lJSSQlJbFv3z6XcqNGjeLUqVPO5YMPPmjcJ3LTxYsjve9MmxCtjduB9Ic//IGpU6cyefJkbr75ZpYtW4afnx/vvPNOreVfe+01Ro0axezZs+nbty8vvvgit956K0uWLHEpZzKZCAsLcy6NncHPXcHSQhKi1XArkCorK9mxYwcJCQkXd6DTkZCQQG5ubq3b5ObmupQH+6X7l5ffvHkz3bp1o0+fPjzxxBOcPn26znpUVFRQUlLisjSWHLIJ0Xq4FUjFxcVYrVbnbbMdQkNDMZvNtW5jNpuvWn7UqFH85S9/ITs7m4ULF/LPf/6Te+65p86Z6zIyMggMDHQuERER7nwMF11lPJsQrUarOK308MMPOx8PGDCAgQMHcuONN7J58+Za56dJTU11mSWwpKSk0aEU7BzxL31IQniaWy2kkJAQ9Ho9BQUFLusLCgrqvCAqLCzMrfJgvzVLSEgIR44cqfV1k8lEQECAy9JYcsgmROvhViAZjUYGDx5Mdna2c53NZiM7O5v4+Phat4mPj3cpD7Bhw4Y6y4N9Br/Tp087bxXTnOQsmxCth9tn2VJSUnjrrbd47733OHDgAE888QRlZWVMnjwZsN8VNDU11Vl+5syZZGVlsWjRIg4ePMiCBQvYvn07M2bMAOy3bpk9ezZbtmzh2LFjZGdnc//993PTTTfVOWdNU3KOZ5MWkhAe53YfUnJyMkVFRaSlpWE2mxk0aBBZWVnOjuv8/Hx0uos5N3ToUDIzM5k3bx5z584lKiqKtWvX0r9/f8B+lfCePXt47733OHv2LOHh4dx99928+OKLmEymJvqYdQvxt7/HuQvVXKiy4muQK5aF8BSvnuQf7COV+8zLotJq45vnf0qPoJa/n7kQ7V1Dv6NePR8S2AeYOgbZeuM0JEK0Jl4fSHBx1L83zxwpRGsggYTMrS1EayGBxMVAKpIzbUJ4lAQSl96fTVpIQniSBBJyfzYhWgsJJGT4iBCthQQSEkhCtBYSSFxy2l/6kITwKAkkLraQzpRXUm21ebg2QngvCSSgi78RnQZK2UNJCOEZEkiAXqfRxV8O24TwNAmkGsH+0rEthKdJINUI6STXIgnhaRJINWQ8mxCeJ4FUw3nIJiP+hfAYCaQazkM2aSEJ4TESSDXkam0hPE8CqUZXx4h/OWQTwmMkkGpcnMZWDtmE8BQJpBohl7SQ2sF9D4RokxoVSEuXLiUyMhJfX1/i4uLYtm1bveVXr15NdHQ0vr6+DBgwgPXr19dZ9vHHH0fTNBYvXtyYqjWao4VUZVWUnK9u0fcWQti5HUirVq0iJSWF9PR0du7cSUxMDImJiRQWFtZaPicnh3HjxjFlyhR27dpFUlISSUlJ7Nu374qya9asYcuWLYSHh7v/Sa6RyUdPJ1/7bepkKlshPMPtQPrDH/7A1KlTmTx5MjfffDPLli3Dz8+Pd955p9byr732GqNGjWL27Nn07duXF198kVtvvZUlS5a4lPvhhx948sknef/99zEYDI37NNeoq5xpE8Kj3AqkyspKduzYQUJCwsUd6HQkJCSQm5tb6za5ubku5QESExNdyttsNiZMmMDs2bPp16/fVetRUVFBSUmJy9IU5NS/EJ7lViAVFxdjtVqdt812CA0NxWw217qN2Wy+avmFCxfi4+PDU0891aB6ZGRkEBgY6FwiIiLc+Rh1clwcKSP+hfAMj59l27FjB6+99horVqxA07QGbZOamorFYnEuJ06caJK6yIh/ITzLrUAKCQlBr9dTUFDgsr6goICwsLBatwkLC6u3/FdffUVhYSHXX389Pj4++Pj4cPz4cZ599lkiIyNr3afJZCIgIMBlaQpyyCaEZ7kVSEajkcGDB5Odne1cZ7PZyM7OJj4+vtZt4uPjXcoDbNiwwVl+woQJ7Nmzh927dzuX8PBwZs+ezWeffebu57kmF6cgkUM2ITzBx90NUlJSmDRpEkOGDCE2NpbFixdTVlbG5MmTAZg4cSI9evQgIyMDgJkzZzJy5EgWLVrE6NGjWblyJdu3b2f58uUABAcHExwc7PIeBoOBsLAw+vTpc62fzy1yyCaEZ7kdSMnJyRQVFZGWlobZbGbQoEFkZWU5O67z8/PR6S42vIYOHUpmZibz5s1j7ty5REVFsXbtWvr37990n6KJdJVJ2oTwKE21g3ESJSUlBAYGYrFYrqk/6fjpMka+spkOBj0HXhzVhDUUwrs19Dvq8bNsrYmjU/t8lZXyShk+IkRLk0C6hJ9Rj6/B/iuRUf9CtDwJpEtomuZsJcl4NiFangTSZZzTkEggCdHiJJAuE9JRrkUSwlMkkC4jV2sL4TkSSJeRQBLCcySQLuM4ZJMR/0K0PAmkywTLWTYhPEYC6TJyyCaE50ggXaarTNImhMdIIF3GMeLfcr6Kymqbh2sjhHeRQLpMYAcDPjr7zJVyF1shWpYE0mV0Ok3uYiuEh0gg1cLZsS0tJCFalARSLRyn/ovPSSAJ0ZIkkGoh49mE8AwJpFp0lRH/QniEBFItnJ3aEkhCtCgJpFpcvFpbDtmEaEkSSLWQ4SNCeEajAmnp0qVERkbi6+tLXFwc27Ztq7f86tWriY6OxtfXlwEDBrB+/XqX1xcsWEB0dDT+/v507tyZhIQEtm7d2piqNQlpIQnhGW4H0qpVq0hJSSE9PZ2dO3cSExNDYmIihYWFtZbPyclh3LhxTJkyhV27dpGUlERSUhL79u1zlunduzdLlixh7969fP3110RGRnL33XdTVFTU+E92DRxn2c6UVWC1tfm7RAnRdig3xcbGqunTpzufW61WFR4erjIyMmotP3bsWDV69GiXdXFxceqxxx6r8z0sFosC1MaNGxtUJ0d5i8XSoPJXU1VtVZHPr1M956xTRecuNMk+hfBmDf2OutVCqqysZMeOHSQkJDjX6XQ6EhISyM3NrXWb3Nxcl/IAiYmJdZavrKxk+fLlBAYGEhMTU2uZiooKSkpKXJam5KPX0dlPRv0L0dLcCqTi4mKsVqvzttkOoaGhmM3mWrcxm80NKr9u3To6duyIr68vf/zjH9mwYQMhISG17jMjI4PAwEDnEhER4c7HaJBgfzn1L0RLazVn2e688052795NTk4Oo0aNYuzYsXX2S6WmpmKxWJzLiRMnmrw+cqZNiJbnViCFhISg1+spKChwWV9QUEBYWFit24SFhTWovL+/PzfddBO33347f/7zn/Hx8eHPf/5zrfs0mUwEBAS4LE0tpFPNVLYynk2IFuNWIBmNRgYPHkx2drZznc1mIzs7m/j4+Fq3iY+PdykPsGHDhjrLX7rfigrPhYFzsv8y6UMSoqX4uLtBSkoKkyZNYsiQIcTGxrJ48WLKysqYPHkyABMnTqRHjx5kZGQAMHPmTEaOHMmiRYsYPXo0K1euZPv27SxfvhyAsrIyfve733HffffRvXt3iouLWbp0KT/88AO//OUvm/CjuidERvwL0eLcDqTk5GSKiopIS0vDbDYzaNAgsrKynB3X+fn56HQXG15Dhw4lMzOTefPmMXfuXKKioli7di39+/cHQK/Xc/DgQd577z2Ki4sJDg7mtttu46uvvqJfv35N9DHdFyLj2YRocZpSqs1f+VdSUkJgYCAWi6XJ+pOyDxQw5b3tDLwukE9mDGuSfQrhrRr6HW01Z9laG5mkTYiWJ4FUh0snaWsHjUgh2gQJpDo4OrUrrTZKLlR7uDZCeAcJpDr4GvR0Mtn7/GXmSCFahgRSPYJlbm0hWpQEUj1k+IgQLUsCqR4hMtm/EC1KAqkejkO2IjlkE6JFSCDVQw7ZhGhZEkj1cIz4l4sjhWgZEkj16Coj/oVoURJI9QiWQzYhWpQEUj1kChIhWpYEUj0c49nKKq2cr7R6uDZCtH8SSPXoaPLB6GP/FclhmxDNTwKpHpqm0VX6kYRoMRJIVxEi49mEaDESSFchw0eEaDkSSFcRLHNrC9FiJJCu4uLwETlkE6K5SSBdhYxnE6LlNCqQli5dSmRkJL6+vsTFxbFt27Z6y69evZro6Gh8fX0ZMGAA69evd75WVVXFnDlzGDBgAP7+/oSHhzNx4kROnjzZmKo1OTlkE6LluB1Iq1atIiUlhfT0dHbu3ElMTAyJiYkUFhbWWj4nJ4dx48YxZcoUdu3aRVJSEklJSezbtw+A8vJydu7cyfz589m5cycff/wxhw4d4r777ru2T9ZEusohmxAtR7kpNjZWTZ8+3fncarWq8PBwlZGRUWv5sWPHqtGjR7usi4uLU4899lid77Ft2zYFqOPHjzeoThaLRQHKYrE0qLw7DplLVM8561TMC581+b6F8BYN/Y661UKqrKxkx44dJCQkONfpdDoSEhLIzc2tdZvc3FyX8gCJiYl1lgewWCxomkZQUFCtr1dUVFBSUuKyNBdHH9LZ8iqqrLZmex8hhJuHbMXFxVitVudtsx1CQ0Mxm821bmM2m90qf+HCBebMmcO4cePqvMNlRkYGgYGBziUiIsKdj+GWoA4G9DoNgDMyDYkQzapVnWWrqqpi7NixKKV488036yyXmpqKxWJxLidOnGi2Oul0Gl38a6aylVH/QjQrH3cKh4SEoNfrKSgocFlfUFBAWFhYrduEhYU1qLwjjI4fP84XX3xR7/2/TSYTJpPJnapfk5COJorOVchEbUI0M7daSEajkcGDB5Odne1cZ7PZyM7OJj4+vtZt4uPjXcoDbNiwwaW8I4wOHz7Mxo0bCQ4Odqdazc45nk1aSEI0K7daSAApKSlMmjSJIUOGEBsby+LFiykrK2Py5MkATJw4kR49epCRkQHAzJkzGTlyJIsWLWL06NGsXLmS7du3s3z5csAeRg899BA7d+5k3bp1WK1WZ/9Sly5dMBqNTfVZG00ujhSiZbgdSMnJyRQVFZGWlobZbGbQoEFkZWU5O67z8/PR6S42vIYOHUpmZibz5s1j7ty5REVFsXbtWvr37w/ADz/8wCeffALAoEGDXN5r06ZN/OQnP2nkR2s6IXJxpBAtQlNKKU9X4lqVlJQQGBiIxWKpt++psf70z6Nk/N9BfnFLD/6QPKjJ9y9Ee9fQ72irOsvWWjkm+y+SFpIQzUoCqQFkkjYhWoYEUgPIJG1CtAwJpAZwBlJZJTZbm+9yE6LVkkBqAMcUJFab4uz5Kg/XRoj2SwKpAQx6HUF+BkBO/QvRnCSQGkgujhSi+UkgNVCwv5xpE6K5SSA1UEinmhaSjGcTotlIIDVQV+eZNgkkIZqLBFIDOQ/ZzskhmxDNRQKpgZyHbNKpLUSzkUBqIOdZNpmkTYhmI4HUQDJJmxDNTwKpgS69DqkdzNgiRKskgdRAjkCqqLZRWlHt4doI0T5JIDVQB6Mef6MegNNycaQQzUICyQ3BMnxEiGYlgeQGmVtbiOYlgeSGix3bcsgmRHOQQHKDXBwpRPNqVCAtXbqUyMhIfH19iYuLY9u2bfWWX716NdHR0fj6+jJgwADWr1/v8vrHH3/M3XffTXBwMJqmsXv37sZUq9mF+MshmxDNye1AWrVqFSkpKaSnp7Nz505iYmJITEyksLCw1vI5OTmMGzeOKVOmsGvXLpKSkkhKSmLfvn3OMmVlZQwbNoyFCxc2/pO0gIsj/uWQTYhmodwUGxurpk+f7nxutVpVeHi4ysjIqLX82LFj1ejRo13WxcXFqccee+yKsnl5eQpQu3btcqtOFotFAcpisbi1nbs+3XNS9ZyzTj305jdNs8OT3ymV+z9KVVU0zf6EaKUa+h11q4VUWVnJjh07SEhIcK7T6XQkJCSQm5tb6za5ubku5QESExPrLN8QFRUVlJSUuCwtoUknabtQAu//ErKeh40Lrn1/QrQDbgVScXExVqvVedtsh9DQUMxmc63bmM1mt8o3REZGBoGBgc4lIiKi0ftyR5NO0vbPhVBa8zvYshT+/fm171OINq5NnmVLTU3FYrE4lxMnTrTI+zpO+5+rqOZClbXxOyo8AFvetD/uOcz+c+0TcK7xIS1Ee+BWIIWEhKDX6ykoKHBZX1BQQFhYWK3bhIWFuVW+IUwmEwEBAS5LSwjw9cGot//KTjd2GhKlYP1sUFaIvhd+9RGEDoDyYljzGNhsTVhjIdoWtwLJaDQyePBgsrOznetsNhvZ2dnEx8fXuk18fLxLeYANGzbUWb410zTNeY+2Rh+27fsIjn0FPr6Q+Hsw+MJD74DBD77fDDmvN12FhWhj3D5kS0lJ4a233uK9997jwIEDPPHEE5SVlTF58mQAJk6cSGpqqrP8zJkzycrKYtGiRRw8eJAFCxawfft2ZsyY4Sxz5swZdu/ezf79+wE4dOgQu3fvvqZ+puZyTbdDqjgHn8+zPx4+Czr3tD/u2hvuqbnk4YsX4T87mqCmQrQ9bgdScnIyr776KmlpaQwaNIjdu3eTlZXl7LjOz8/n1KlTzvJDhw4lMzOT5cuXExMTw9/+9jfWrl1L//79nWU++eQTbrnlFkaPHg3Aww8/zC233MKyZcuu9fM1Ocd4tkaN+P/nQjh3Cjr3gqFPur52ywTo9wDYquGjKfazcEJ4GU2ptj/bWElJCYGBgVgslmbvT5q1+jv+tuM/zE7sw/Q7b2r4hoUHYdkd9sD5f6uh991Xljl/FpYNB0s+DEyGXyxvsnoL4UkN/Y62ybNsntSoQzal4P9m28Ooz89rDyOADkHw4Nug6WHPKvhu5bVXWIg2RALJTY06ZPvXx5D3pb0je1RG/WWvj4Of1PTBffosnD7ayJoK0fZIILmpq7sj/itK4bOajuxhKdA58urbDE+xX59UWQp/+y+olrFzwjtIILkp2N/NQPryZTh30h5Ed8xs2DY6vb3/qENnOLUbvvhNo+oqRFsjgeSmkE5ujGcrOgS5S+2PRy20X3PUUIE94P6abXPegCMb3aypEG2PBJKbHJ3aP5ZXUm2t56pqxxXZtmroPQr6jHL/zaJHw21T7Y/XPA6ltU/xIkR7IYHkps5+RnSaPW/OlNfTStq/FvL+CXoTjHqp8W9494vQrR+UFdnHu8nQEtGOSSC5Sa/T6OKYhqSuidoqSuGzX9sfD3sGuvRq/BsaOtiHlvj42g/btvyPy8vFpRX84fNDfLAtH5utzV9SJrycj6cr0BaFdDRRXFrJ6bI6Ora/ehVKfoCgnjDs6Wt/w27R9ssF1j1jnzsp8g5KuvTn7S+/5+2v8yivtM88sGbnD7z80EAiQ/yv/T2F8ADvayF9t8p+TdA1qPfiyOLDkLPE/njUS/YWTlMYPBn6jgFbFZa/TmDUwvW8/sURyiut9O0egJ9Rz7ZjZxj12pe8+02etJZEm+RdgXQ2397KeG8M/H0GnP+xUbu5OOL/skM2Z0d2FUQlQp97rrXGTtU2xUc95mAmhMDzJ0ipfpsbu/qz7Fe3sv6pYXz29AjibwjmQpWNF/6xn4eXb+FYcVmTvb8QLcG7Ask3EGKS7Y93/RWWxMK+j+1B4oY6W0gHPoHvN9k7su95CTTtmqtssynW7TnJz/74Jc+uy+fJiv/Gio6H9F/y+V1mRvXvjqZpRHTx4/1H43gxqb+0lkSb5X2BdO8fYXIWhPSGskL422T44GGw/KfBu3EEUu73p9m4vwDL+SqoLIOsufYCd8yELjdcU1WVUmw+VMiYJV8zI3MXecVldPE3Mmr0L7ANnw2Afv0sOJPn3Ean05hwe09pLYk2y3tH+1dXwFeL4Ks/2A+xjB3hrnS4bYr9Sul6ZO0z8/j/XpyzSNPgpaC1JJ//kPN+Pah6IpeAToGN/jw7jv/Iy1kH2Zp3BoCOJh8eHd6LR4ffQEeTD1ir7Yed+TnQYzD812egN7jsw2ZTvL8tn4z1ByivtOJr0DFnVDST4iPR6a695SaEOxr6HfXeQHIoPACfPAX/qbnZ5XW3wZjXIfTmOjex2RSf7y/gy8NFbPn+NKr4CJ8Zn8OoWZlW+Qwb1W30Cw/k9hu6cPsNwdzWqwsBvoY69+dwyHyOVz47xMYD9il/jT46Jtzek//+yY0E17TKnM6egGXD4MJZ+6UFCQtq3eeJM+XM+WgPOUdPAxAb2UXOxIkWJ4HkDpsNtv8ZNr4AledAZ7B/yYc/e/XhHkpRseIBTMc3cajT7Txhe57vT5e7FNFp0L9HILffEMztN3RhSKRrQOWfLuePG//N2t0/oJS9/C8HRzAzIYrwoHrO0u3/BD6cAGgwbiX0Tqy130paS8LTJJAaw/KD/SzZoU/tz4Oj4L7XoefQurc58A9Y9SvQG+G/t0DwjRSUXGDL96drljPkXdZ/o9NgQE1AlVdaWfltPlVW+z/DzweEkfKzPtzUrWPD6vyPp2HHu/bHQdfDTQn2pdcIMHVyKSqtJeEpEkiNpZT9bNn62VBac7eUwY9Awgv2CdQuVVkOS2PBcsI+R/Zd82vdpdlyaUCd5thlLSiA4VEhzE7sw8Drgq7cQX0qy+Hv0+HgOrBechmCzgDX334xoEL7gaZhsykya1pLZdJaEi1EAulanf8RNqTDzvfszzuGwc9fgZvvu1gm+0X7VdmBETB9Gxj9GrTrU5bzbP3+DFu+P43lfBUTbu/J0JtCrq2+lWVw7Gv78JLDG+DHPNfXO3WHG++CqAS44SecOG+S1pJoMRJITeXY1/CPmXD6iP159L32YKo6D/9zu71VMvavrkHVGpw+Ckey7QGV9yVUn7/4mqaD625D3XgXWRX9mf2NRmmlwtegY3DPzvQJDSA6rBN9wjoRFdoRP6OMMBLXRgKpKVVdgC9fgW8W26cTMQXYW0WF/7K3On71UZNcBNlsqi7YLxFwBFTRQZeXrb5dyNUG8bGlN/9SkRxTYVRgvxpd0+D6Ln70Ce1UE1IB9AnrRGSwHz5677qMTTSeBFJzKPiX/RKBH7bbn+sM9o7sEDfuPtIanD1hD6YjG+H7f9rPLF5CoVGoD+OwrTsHqsI4qsL53tadoyqc0wQAGkYfHTd17ehsSTmWsABftNYczt6k6oJ9kLflP/al5Ad7f6flB/tja5X9+jWdAfQ+NT8N9uvwnI99QG+gGj3l1RqlVRrnqqCkEiyVYLmguC42ibg77623Ks0aSEuXLuWVV17BbDYTExPDG2+8QWxsbJ3lV69ezfz58zl27BhRUVEsXLiQn//8587XlVKkp6fz1ltvcfbsWe644w7efPNNoqKiGlSflrwNEjYrfPu2fSbI+OkQ91jzvl9zs1bBiW32cDr2FRT9GyosdRY/p3XkiC2cw1Z7QDmWfNUNK3oCOxiIDPGns5+Bzn5Ggi75GeRnvGK9n1EvAdYYNqv9pIulJmQuDR5H+JQVtUhVvr7haYZNfKHeMs0WSKtWrWLixIksW7aMuLg4Fi9ezOrVqzl06BDdunW7onxOTg4jRowgIyODe++9l8zMTBYuXMjOnTudN4tcuHAhGRkZvPfee/Tq1Yv58+ezd+9e9u/fj6/v1ad9bdFAau+Usv8hF/+7Zjl88efZfKD2P5cqfDimQjlqC6dABQGgARqqZrE/puYxNc/1OjDpdfj6aBh9NEx6DZOPDqNeQ6cpsFnRlBWUDZQNTVnRlM2+nkue17ymUzaX9RoKq+aDDb39p+aDTdNj03ywanpsOgOq5rlN80Hpah7rDKia50qzl1GaVvNcj0Jn74vT6VDoQdPVlLH/1HQ6e5ma8mgaaHp8VFXNUolBVaFXlRhslfjUPPaxVaJXVehtFehtVfjYKtDZ7M91tkp0tkoMlSUYywvQqeqr/nNW6Xw5ZwqlxBiGxRiKxdCNHw3dOOPTleLzcOZcOWfPlVNRWYEPVudiwIqPZsVANT5Y0WOlo0HRxVdHF1+NIJNGoAkCjBp+A++j28CEeuvRbIEUFxfHbbfdxpIl9ik2bDYbERERPPnkkzz//PNXlE9OTqasrIx169Y5191+++0MGjSIZcuWoZQiPDycZ599llmzZgFgsVgIDQ1lxYoVPPzww1fss6KigoqKiwNbS0pKiIiIkEBqblXn7Z3lLkH1b3uHf9WVlzKI5lWtdBTQmZMqmJMqhFMqmB9UMKdUcM26YM7SEWhYCzTIz8B1nTtwXZAfPTp3sD/u7EePoA706NyBwA5XH21Ql4YGklunTyorK9mxYwepqanOdTqdjoSEBHJzc2vdJjc3l5SUFJd1iYmJrF27FoC8vDzMZjMJCRcTNjAwkLi4OHJzc2sNpIyMDF54of4momgGhg4Q1t++XMpmsx8iOIKqrKimk1+r86cCqmyK81WK81XWS37anIsNe6sCnR5Np0fTdM7Hl67T9DWPdT7odDrQ9Oh89Gia/XBQ2axotiqUtdp+UsJaBbYqNFu1fVygzb5otio05Xhs/6mzVYOqRrM5WlxWdMoKStmfK0dL7dJW2SXPsZfTKSugqMaHas1AlWakCgNVmoFKDFRhoFIzUIGBKnyowEilsj++gL1MBT5cUAbK6cBZn26UGILR6w3odRo+eg2DXodep2HUa9yk0xFds16v02HQa/jodPjoNXx0Gj56HSEdjfQIqgmdzh3s4yQ9zK0aFBcXY7VaCQ0NdVkfGhrKwYMHa93GbDbXWt5sNjtfd6yrq8zlUlNTXULO0UISHqLTQVCEfbnprgZtogHGmqXxw5BFe+P5SGwEk8mEyWS6ekEhRJvi1oUkISEh6PV6CgoKXNYXFBQQFhZW6zZhYWH1lnf8dGefQoj2ya1AMhqNDB48mOzsbOc6m81GdnY28fHxtW4THx/vUh5gw4YNzvK9evUiLCzMpUxJSQlbt26tc59CiHZKuWnlypXKZDKpFStWqP3796tp06apoKAgZTablVJKTZgwQT3//PPO8t98843y8fFRr776qjpw4IBKT09XBoNB7d2711nmpZdeUkFBQervf/+72rNnj7r//vtVr1691Pnz5xtUJ4vFogBlsVjc/ThCiBbQ0O+o24GklFJvvPGGuv7665XRaFSxsbFqy5YtztdGjhypJk2a5FL+ww8/VL1791ZGo1H169dPffrppy6v22w2NX/+fBUaGqpMJpO666671KFDhxpcHwkkIVq3hn5HZeiIEKLZNfQ7KqMjhRCtRps87X85RyOvpKTEwzURQtTG8d282gFZuwikc+fso9Xl4kghWrdz584RGFj3pbDtog/JZrNx8uRJOnXqdNWR446ruk+cOCH9Tc1Afr/Nq63+fpVSnDt3jvDwcPvwnjq0ixaSTqfjuuuuc2ubgICANvUP2tbI77d5tcXfb30tIwfp1BZCtBoSSEKIVsPrAslkMpGeni6Dc5uJ/H6bV3v//baLTm0hRPvgdS0kIUTrJYEkhGg1JJCEEK2GBJIQotWQQBJCtBpeF0hLly4lMjISX19f4uLi2LZtm6er1C4sWLAATdNclujoaE9Xq8368ssvGTNmDOHh4Wia5rxLj4NSirS0NLp3706HDh1ISEjg8OHDnqlsE/KqQFq1ahUpKSmkp6ezc+dOYmJiSExMpLCw0NNVaxf69evHqVOnnMvXX3/t6Sq1WWVlZcTExLB06dJaX3/55Zd5/fXXWbZsGVu3bsXf35/ExEQuXLjQwjVtYs06TVwrExsbq6ZPn+58brVaVXh4uMrIyPBgrdqH9PR0FRMT4+lqtEuAWrNmjfO5zWZTYWFh6pVXXnGuO3v2rDKZTOqDDz7wQA2bjte0kBw3ubz0hpRXu8mlcM/hw4cJDw/nhhtuYPz48eTn53u6Su3S1W6u2pZ5TSDVd5PLum5IKRouLi6OFStWkJWVxZtvvkleXh7Dhw93zlUlmk5jbq7aVrSL6UeE591zzz3OxwMHDiQuLo6ePXvy4YcfMmXKFA/WTLQlXtNCasxNLkXjBQUF0bt3b44cOeLpqrQ77fnmql4TSI25yaVovNLSUo4ePUr37t09XZV2pz3fXNWrDtlSUlKYNGkSQ4YMITY2lsWLF1NWVsbkyZM9XbU2b9asWYwZM4aePXty8uRJ0tPT0ev1jBs3ztNVa5NKS0tdWpd5eXns3r2bLl26cP311/P000/z29/+lqioKHr16sX8+fMJDw8nKSnJc5VuCp4+zdfS6rvJpWi85ORk1b17d2U0GlWPHj1UcnKyOnLkiKer1WZt2rRJAVcsjpuwXuvNVVsrmQ9JCNFqeE0fkhCi9ZNAEkK0GhJIQohWQwJJCNFqSCAJIVoNCSQhRKshgSSEaDUkkIQQrYYEkhCi1ZBAEkK0GhJIQohW4/8DuzXTzIeIWroAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:37:37.841975Z", + "iopub.status.busy": "2024-02-29T03:37:37.841686Z", + "iopub.status.idle": "2024-02-29T03:38:24.155443Z", + "shell.execute_reply": "2024-02-29T03:38:24.154410Z" + }, + "papermill": { + "duration": 46.336522, + "end_time": "2024-02-29T03:38:24.157916", + "exception": false, + "start_time": "2024-02-29T03:37:37.821394", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:38:24.196181Z", + "iopub.status.busy": "2024-02-29T03:38:24.195314Z", + "iopub.status.idle": "2024-02-29T03:38:24.214970Z", + "shell.execute_reply": "2024-02-29T03:38:24.214178Z" + }, + "papermill": { + "duration": 0.040598, + "end_time": "2024-02-29T03:38:24.216824", + "exception": false, + "start_time": "2024-02-29T03:38:24.176226", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0141680.022180.0011681.6954430.2290763.4890890.4298050.0000022.115530.024390.4502940.0341760.1698040.0001973.810973
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.014168 0.02218 0.001168 1.695443 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.229076 3.489089 0.429805 0.000002 2.11553 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.02439 0.450294 0.034176 0.169804 0.000197 \n", + "\n", + " total_duration \n", + "realtabformer 3.810973 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:38:24.251404Z", + "iopub.status.busy": "2024-02-29T03:38:24.251124Z", + "iopub.status.idle": "2024-02-29T03:38:24.701499Z", + "shell.execute_reply": "2024-02-29T03:38:24.700698Z" + }, + "papermill": { + "duration": 0.470067, + "end_time": "2024-02-29T03:38:24.703615", + "exception": false, + "start_time": "2024-02-29T03:38:24.233548", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:38:24.742762Z", + "iopub.status.busy": "2024-02-29T03:38:24.742464Z", + "iopub.status.idle": "2024-02-29T03:39:15.523040Z", + "shell.execute_reply": "2024-02-29T03:39:15.522252Z" + }, + "papermill": { + "duration": 50.803335, + "end_time": "2024-02-29T03:39:15.525562", + "exception": false, + "start_time": "2024-02-29T03:38:24.722227", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:39:15.564540Z", + "iopub.status.busy": "2024-02-29T03:39:15.563736Z", + "iopub.status.idle": "2024-02-29T03:39:15.580811Z", + "shell.execute_reply": "2024-02-29T03:39:15.579915Z" + }, + "papermill": { + "duration": 0.038805, + "end_time": "2024-02-29T03:39:15.582884", + "exception": false, + "start_time": "2024-02-29T03:39:15.544079", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:39:15.618738Z", + "iopub.status.busy": "2024-02-29T03:39:15.618475Z", + "iopub.status.idle": "2024-02-29T03:39:15.623912Z", + "shell.execute_reply": "2024-02-29T03:39:15.623082Z" + }, + "papermill": { + "duration": 0.026041, + "end_time": "2024-02-29T03:39:15.625930", + "exception": false, + "start_time": "2024-02-29T03:39:15.599889", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.05701854543726574}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:39:15.661043Z", + "iopub.status.busy": "2024-02-29T03:39:15.660794Z", + "iopub.status.idle": "2024-02-29T03:39:15.995914Z", + "shell.execute_reply": "2024-02-29T03:39:15.995006Z" + }, + "papermill": { + "duration": 0.355249, + "end_time": "2024-02-29T03:39:15.998047", + "exception": false, + "start_time": "2024-02-29T03:39:15.642798", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGkUlEQVR4nO3deXxU9b3/8deZfbJN9g0SCIRNkUU2ERVUquJKrVhRC2gRb69eayn3p/RxLy5tpe2tVmtbqm0l2mqtWq1t3TfEjdUFFGUTQlgDIfvsc76/P85kIJCEzGSSmZDP8/GYR5LJmXO+GZI33+/3fBdNKaUQQogkYUp0AYQQ4mgSSkKIpCKhJIRIKhJKQoikIqEkhEgqEkpCiKQioSSESCoSSkKIpCKhJIRIKhJKol133303mqbF9ZwVFRVomsa6des6dfz//d//MWjQIMxmM2PGjIlrWURyklASUbnvvvv4xz/+0SPXev311/l//+//MWXKFJYvX859993XI9cViWVJdAFE73Lfffdx1VVXMXPmzG6/1ttvv43JZOJPf/oTNput268nkoPUlE4Szc3NiS5C3FVXV+N0OuMWSEopPB5PXM7VFW63O9FFSGoSSr1QS1/Ppk2buPbaa8nKyuKss84C4C9/+Qvjxo3D6XSSnZ3NNddcQ1VVVavXv/fee8yaNYvS0lLsdjslJSX84Ac/OOEfrKZpNDc38/jjj6NpGpqmMW/ePAAqKyv5z//8T4YNG4bT6SQnJ4dZs2axc+fONs/ldru5+eabycnJISMjgzlz5lBbW9vqWsuXL6e5uTlyrYqKCgCCwSA//vGPGTx4MHa7nYEDB/KjH/0In8/X6hoDBw7k0ksv5bXXXmP8+PE4nU4eeeQRVqxYgaZpPPPMM9xzzz3069eP9PR0rrrqKurr6/H5fNx+++3k5+eTlpbGDTfccNy5O/teT5s2jZEjR7J+/XrOOeccUlJS+NGPftTh+9zXSfOtF5s1axZDhgzhvvvuQynFT3/6U/73f/+Xq6++mvnz53Pw4EEefvhhzjnnHD755BMyMzMBePbZZ3G73Xzve98jJyeHNWvW8PDDD7N7926effbZdq/35z//mfnz5zNx4kQWLFgAwODBgwFYu3YtH374Iddccw39+/dn586dLFu2jGnTprFp0yZSUlJanevWW28lMzOTu+++m82bN7Ns2TIqKysjgfHnP/+ZRx99lDVr1vDHP/4RgDPPPBOA+fPn8/jjj3PVVVfxwx/+kNWrV7N06VK+/PJLXnjhhVbX2bx5M7Nnz+bmm2/mpptuYtiwYZHvLV26FKfTyZ133sm2bdt4+OGHsVqtmEwmamtrufvuu1m1ahUVFRWUlZWxZMmSyGs7+14D1NTUMGPGDK655hquv/56CgoKovyX7mOU6HXuuusuBajZs2dHntu5c6cym83qpz/9aatjN27cqCwWS6vn3W73cedcunSp0jRNVVZWHnedo6Wmpqq5c+ce9/q2zvnRRx8pQD3xxBOR55YvX64ANW7cOOX3+yPP/+IXv1CAevHFFyPPzZ07V6WmprY656effqoANX/+/FbPL1q0SAHq7bffjjw3YMAABahXX3211bHvvPOOAtTIkSNblWH27NlK0zQ1Y8aMVsdPnjxZDRgwIPJ1NO/11KlTFaB+//vfH/f+iLZJ860X+4//+I/I588//zy6rnP11Vdz6NChyKOwsJAhQ4bwzjvvRI51Op2Rz5ubmzl06BBnnnkmSik++eSTmMpy9DkDgQA1NTWUl5eTmZnJxx9/fNzxCxYswGq1Rr7+3ve+h8Vi4eWXX+7wOi3fX7hwYavnf/jDHwLw0ksvtXq+rKyMCy+8sM1zzZkzp1UZJk2ahFKKG2+8sdVxkyZNoqqqimAwCET3XgPY7XZuuOGGDn8ucYQ033qxsrKyyOdbt25FKcWQIUPaPPboP75du3axZMkS/vnPf7bqxwGor6+PqSwej4elS5eyfPly9uzZgzpqQdO2znlsOdPS0igqKmq3D6pFZWUlJpOJ8vLyVs8XFhaSmZlJZWVlq+ePfo+OVVpa2uprl8sFQElJyXHP67pOfX09OTk5Ub3XAP369ZO7h1GQUOrFjq6d6LqOpmm88sormM3m445NS0sDIBQK8Y1vfIPDhw9zxx13MHz4cFJTU9mzZw/z5s1D1/WYyvJf//VfLF++nNtvv53JkyfjcrnQNI1rrrkm5nN2pLODOo9+j47V1vvU0fMtQdvZ97ozZRDHk1A6SQwePBilFGVlZQwdOrTd4zZu3MiWLVt4/PHHmTNnTuT5N954o1PXaS8MnnvuOebOncv9998fec7r9VJXV9fm8Vu3buXcc8+NfN3U1MS+ffu4+OKLO7z+gAED0HWdrVu3MmLEiMjzBw4coK6ujgEDBnTq5+iKzr7XIjbSp3SSuPLKKzGbzdxzzz2tmk5g/A9fU1MDHKkFHH2MUoqHHnqoU9dJTU1tM2jMZvNx13344YcJhUJtnufRRx8lEAhEvl62bBnBYJAZM2Z0eP2W0HrwwQdbPf/AAw8AcMkll5zoR+iyzr7XIjZSUzpJDB48mJ/85CcsXryYnTt3MnPmTNLT09mxYwcvvPACCxYsYNGiRQwfPpzBgwezaNEi9uzZQ0ZGBn//+9+P61tqz7hx43jzzTd54IEHKC4upqysjEmTJnHppZfy5z//GZfLxSmnnMJHH33Em2++SU5OTpvn8fv9nH/++Vx99dVs3ryZ3/3ud5x11llcfvnlHV5/9OjRzJ07l0cffZS6ujqmTp3KmjVrePzxx5k5c2ar2ld36ex7LWIjoXQSufPOOxk6dCi/+tWvuOeeewCj0/aCCy6I/LFbrVb+9a9/cdttt7F06VIcDgff/OY3ufXWWxk9evQJr/HAAw+wYMEC/ud//gePx8PcuXOZNGkSDz30EGazmSeffBKv18uUKVN48803273z9Zvf/IYnn3ySJUuWEAgEmD17Nr/+9a871Vf0xz/+kUGDBlFRUcELL7xAYWEhixcv5q677ori3eqazrzXIjaaOrb+KYQQCSR9SkKIpCKhJIRIKhJKQoikIqEkhEgqEkpCiKQioSSESCq9epySruvs3buX9PT0uC9wL4ToOqUUjY2NFBcXYzJ1rg7Uq0Np7969x83oFkIkn6qqKvr379+pY3t1KKWnpwPGD5yRkZHg0gghjtXQ0EBJSUnkb7UzenUotTTZMjIyJJSESGLRdK9IR7cQIqlIKAkhkoqEkhAiqfTqPqXOUEoRDAbbXWxMtM9sNmOxWGS4hehRJ3Uo+f1+9u3bJzuSdkFKSgpFRUWy8L3oMSdtKOm6zo4dOzCbzRQXF2Oz2eR//CgopfD7/Rw8eJAdO3YwZMiQTg9+E6IrTtpQ8vv96LpOSUnJcbuzis5xOp1YrVYqKyvx+/04HI5EF0n0ASf9f33yv3vXyPsXm5CuONTkS3QxeqWE/8bt2bOH66+/npycHJxOJ6eddhrr1q1LdLGEiI5SeDa/hW/lQ9RtXcXyD3bw8sZ9iS5Vr5TQ5lttbS1Tpkzh3HPP5ZVXXiEvL4+tW7eSlZWVyGIJEb26Sg59voLddR7yDxwikP5NGs1p1LsDuFKsJ369iEhoKP385z+npKSE5cuXR57raJtlEX8DBw7k9ttv5/bbb090UXq3/Rup8xj72KXbTQzVKtnAqeyt90goRSmhzbd//vOfjB8/nlmzZpGfn8/YsWP5wx/+0O7xPp+PhoaGVg8hEk4p9JoduP1BalIGkeGwUqwfAGB/vTfBhet9EhpKX3/9NcuWLWPIkCG89tprfO973+O2227j8ccfb/P4pUuX4nK5Ig9ZtsTg9/sTXYS+zV2Dp7meIBYOZY/FbjWRq9eA0jnQIKEUrYSGkq7rnH766dx3332MHTuWBQsWcNNNN/H73/++zeMXL15MfX195FFVVRXTdf1Bvd1HMKR3+thAJ46NxbRp07j11lu59dZbcblc5Obm8r//+7+RLaIHDhzIj3/8Y+bMmUNGRgYLFiwA4P333+fss8/G6XRSUlLCbbfdRnNzc+S81dXVXHbZZTidTsrKynjyySdjKp84RuN+mn1Bmm25pGb3Q7M4SLXopAYOU9PsP25rb9GxhPYpFRUVccopp7R6bsSIEfz9739v83i73Y7dbu/ydX/7zrZ2v1eWm8rMsf0iXz+6cjuBUNu/VP2znMwaf6S29tgHO/D4W09n+cE3hsZUxscff5zvfve7rFmzhnXr1rFgwQJKS0u56aabAPjlL3/JkiVLIrvCbt++nYsuuoif/OQnPPbYYxw8eDASbC19dvPmzWPv3r288847WK1WbrvtNqqrq2MqnzhKczXegI7bmk1uugOshTgCO0n1HKY6mEuTL0i6Q/qVOiuhoTRlyhQ2b97c6rktW7YwYMCABJUoeZSUlPCrX/0KTdMYNmwYGzdu5Fe/+lUklM477zx++MMfRo6fP38+1113XaTDesiQIfz6179m6tSpLFu2jF27dvHKK6+wZs0aJkyYAMCf/vQnRowY0eM/20mn+RDeQAi3LZsBKVYw5WCqrWSEy09p/yyZSRClhIbSD37wA84880zuu+8+rr76atasWcOjjz7Ko48+2q3XveXc8na/Zzrm92fBOYPbPfbY37Ubp8TvzuEZZ5zR6pd58uTJ3H///ZGJxePHj291/GeffcaGDRtaNcmUUpHpNlu2bMFisTBu3LjI94cPH05mZmbcytxnNVWTk2ZjUL8yilxOMOUCcHquDkPyEly43iehoTRhwgReeOEFFi9ezL333ktZWRkPPvgg1113Xbde12bpfFdadx3bVampqa2+bmpq4uabb+a222477tjS0lK2bNnSU0XrW4I+8DWSk2onZ+QwsDoglG18z12T2LL1Ugmf+3bppZdy6aWXJroYSWf16tWtvl61ahVDhgzBbDa3efzpp5/Opk2bKC9vuxY4fPhwgsEg69evjzTfNm/eTF1dXVzL3ed46oyPVqcRSABOY/Cv8tTR7A0QCCmyUmWVhc5K+DQT0bZdu3axcOFCNm/ezF//+lcefvhhvv/977d7/B133MGHH37IrbfeyqeffsrWrVt58cUXufXWWwEYNmwYF110ETfffDOrV69m/fr1zJ8/H6fT2VM/0snJW4c/pFNH6pGbHPYM0EzUNnmoeOdzXv1if2LL2MtIKCWpOXPm4PF4mDhxIrfccgvf//73I7f+2zJq1CjeffddtmzZwtlnn83YsWNZsmQJxcXFkWOWL19OcXExU6dO5corr2TBggXk5+f3xI9z8vLWU+8J8H5VkJda5rqZTGBPw24xYw810ugNJLaMvUzCm2+ibVarlQcffJBly5Yd972dO3e2+ZoJEybw+uuvt3vOwsJC/v3vf7d67jvf+U6XytnneerwB3V8ljTS7Ef9OTlc2N112L1N1PhCBEI6VrPUATpD3iUhusJrhJLXkkG646hQsmdgNmmkYQxebfQGE1TA3kdCSYiu8NbjD+n4zOnH1ZQ0NDJNHgBpwkVBmm9JaMWKFYkugugsb/2R5tvRNSWHsTlqOsb68E0+qSl1ltSUhIhV0AehAP5gCL85lXR76+YbQIoyakpuv+ym01kSSkLEyteErhReZUE3WVvXlGxpAOTZg4wbkEVhhqxv3lnSfBMiVv5GFFBaVIA+MAun9aiBrTZjxH2+I0R+eY4xTEB0ioSSELHyNWHWNEoK8yk5do6bNcWYHKkUBNxgT0tMGXshiW8hYuVvMj7a2ggckwmsKSilaGqs42Cj7GzSWRJKQsTK14Q3EOJw0E5zW3fXbKl4gzrPfLiZZ9bFtiBhXyShJESs/I1UN/p4bVsTa3YcPv77tjSsZg1byN2llUj7GgklIWLlayIQ0gmYU0ixtbF6gy0Vi8lEimas091mbUocR0JJiFj5jVDymVNJsbVxzyh8By5NM/qTmv0SSp3Rt0JJKQj6e/4RxcLxTzzxBDk5Ofh8rTtGZ86cKZNnk4lS4ZqSImBOxdlmTcnoAE8L15RkAGXn9K0hAaEAvHd/z1/37B+CpXOLfM2aNYvbbruNf/7zn8yaNQswdiF56aWXOlwBQPSwoBf0IIGQjt/sJNXedvMNIAUjlI7dVEK0rW/VlHoBp9PJtdde22rX4L/85S+UlpYybdq0xBVMtOZrQqHwKCtKs5Bibb/55lBSU4pG36opma1GrSUR143CTTfdxIQJE9izZw/9+vWjoqKCefPmya4YySTgJqQrfCZj5c6UtmpK1hQAch0hxg/Mol+mrPLZGX0rlDSt082oRBo7diyjR4/miSee4IILLuCLL77gpZdeSnSxxNECHjQ0Bhfn4hiY3fYCblYjhHLtirPLc4/f/ka0qW+FUi8yf/58HnzwQfbs2cP06dNli/JkE3BjNmkMyM1hQHlu28eEQ8m4weI98rXokPQpJalrr72W3bt384c//IEbb7wx0cURxwoY6yS19Bu1yWQGix1dKRqbGqhpkqkmnSGhlKRcLhff+ta3SEtLY+bMmYkujjhWwIM3GKLGb+54UKQ1BY8/xNMfbOb5j/f0XPl6sYSG0t13342maa0ew4cPT2SRksqePXu47rrrsNvtiS6KOFbAzcFGH29saWDV1x1sOml1YDFrWENePIEQKooxa31VwvuUTj31VN58883I1xZLwouUcLW1taxYsYIVK1bwu9/9LtHFEW0JeAiGdAJWR9sDJ1tYU7CYTVh0n3G3LqjjsHZwvEh8KFksFgoLCxNdjKQyduxYamtr+fnPf86wYcMSXRzRloCbQEgRtDtaL+52LKsTs6bhDE818QZCEkonkPBQ2rp1K8XFxTgcDiZPnszSpUspLS1t81ifz9dq+kVDQ0NPFbNHtbevm0gifjdBXREwnaimFB7HpPkBYwBlZkpPFLD3Smif0qRJk6ioqODVV19l2bJl7Nixg7PPPpvGxsY2j1+6dCkulyvykNvkIiGUgoCHQEgnaHaeoKZkJFCqyQglT0BGdZ9IQkNpxowZzJo1i1GjRnHhhRfy8ssvU1dXxzPPPNPm8YsXL6a+vj7yqKo68cJZ0rHYNfL+tSEUAD1IMGTUlDpsjoVrSk7CoSRTTU4o4c23o2VmZjJ06FC2bdvW5vftdnun70RZrcbUDrfbjdMpg9Zi5XYb43Fa3k8BBNwoFH6loWuWE4SSUVMqTlGML80iJy35ZxQkWlKFUlNTE9u3b4/LEh1ms5nMzEyqq6sBSElJkbljUVBK4Xa7qa6uJjMzE7NZOmcjAh5Q0D8/l0mDctpe4K1FuKbULw36Hbu5gGhTQkNp0aJFXHbZZQwYMIC9e/dy1113YTabmT17dlzO33JXryWYRPQyMzPl7uixAm40TaMkP5uS9qaYtAjXlCIjwMUJJTSUdu/ezezZs6mpqSEvL4+zzjqLVatWkZcXn/9RNE2jqKiI/Px8AgHZyz1aVqtVakhtaQkYayduo4VrSiG/lya3DzDhSpGmcEcSGkpPP/10j1zHbDbLH5eIn/Cdt6agBbM3QIajg5CxGKFU5/bx1MqvKMjN5urxcte4IzL3TYhoBdzUewKs+LqJ17840PGxJhNYbFhMxqhunwwJOCEJJSGiFa4pBUwnGM3dwuLEYtKw6D68Adlm6UQklISIVsAYzR00OXBYO/EnZHVgNmuYdT9emZR7QhJKQkQr4CEYUgTNna0pOSI1paCuCOoSSh2RUBIiWn43Qd1ovjk6GqPUwuLAbNKwqSOTckX7JJSEiFZ4hYCA6QTz3lpYnWhopJiMYSnSr9SxpBrRLUTS03UIegmGdILmE8x7a2ExpkaVZ1nIKsrC3pl+qD5MQkmIaAS9oBSFLgenDyoiO6UTc9nCY5VOybOBTDU5IQklIaIR8ACQn+Uif2hB514TrikR9HZToU4uUo8UIhqBZuNjZ6aYtAhPNQn6PNR7Arj9HWw0ICSUhIhKwENIKeqCFurc/s69xuIA4Ov9h3js/R18XFnXfeU7CUgoCRGNgBuvP8SHuzw8s+7EiwwCkVCyhRd6kyEBHZNQEiIaAc+Rtbk7uwGA1QgluwqHUlBCqSMSSkJEI+AOr83dyeEAELn7ZiWEpoIyTukEJJSEiEa4phQ80S4mR7PYQdPCU0380nw7AQklIaLhd0dWCHBYOhlKmgYWe2RTSgmljkkoCRGN8AoBAXMUNSWITMo16z58QWm+dUQGTwoRjYCHUEjv/LIlLSwOLGaNU/JsqOxMlFKykUU7JJSEiEbATXaqndFlRRS5oti6y+rEYjIxZUAqFJ5gs4E+TkJJiM4KBSAUIDvVxsSh/SO3+jslMtXE1/FxQvqUhOi08Lw3NNORkOms8LAAv7eZek8Av/QrtUtCSYjOCm+t1BCyUusOoEezgmS4VvXhV7t57P0dVNXKPnDtkVASorMCbpRSrNvrpeLDnXiiubUfnmri0FoWepNhAe1JmlD62c9+hqZp3H777YkuihBtC0/GDZjCAdPZEd1wJJTC899kWED7kiKU1q5dyyOPPMKoUaMSXRQh2hfwhJfBdWCzmDCborilHw4lu0zKPaGEh1JTUxPXXXcdf/jDH8jKykp0cYRoX8DYMMAYoxTljsstk3Ixmm8+mf/WroSH0i233MIll1zC9OnTT3isz+ejoaGh1UOIHuN3EwzXlKIaOAlHli9RUlM6kYSOU3r66af5+OOPWbt2baeOX7p0Kffcc083l0qIdgTckf3eOj3vrUU4lKxK+pROJGE1paqqKr7//e/z5JNP4nB0bhDa4sWLqa+vjzyqqjq5yJYQ8RDwRPZ7i2reG0RCKdUKo4pSKctN7YYCnhwSVlNav3491dXVnH766ZHnQqEQK1eu5De/+Q0+nw+zufU/vN1ux26PctCaEPEScJNmt3BqYSFpeWnRvTa8fEma3cL5QzLAnt49ZTwJJCyUzj//fDZu3NjquRtuuIHhw4dzxx13HBdIQiRcwEO6w8q48n6QHmWohJcvIeA1pppIKLUrYaGUnp7OyJEjWz2XmppKTk7Occ8LkXBKHZlmYo1iIu7RLE5UwIPP04xPc5HhtMhKAW1I+N03IXqFoBeUjtsf5HDAQiAUQ0e1xY6u4K8fbuGxD3bgj+UcfUBSrRKwYsWKRBdBiLaFa0lbavy8s2o3V4wpZlDU/UoOTBo4IsMCdOzR3sXrA6SmJERnhCfjejFutEQ9eBLA6kBDw2FqGUApY5XaIqEkRGeEa0oebECMoRQeFpBiMnbIlV1N2hZTKH399dfxLocQyS28QoBXGTWlTu/5drRwKDlbpprI/m9tiimUysvLOffcc/nLX/6C1+uNd5mESD7+IxsGANgtMfzptEzK1Y70KYnjxRRKH3/8MaNGjWLhwoUUFhZy8803s2bNmniXTYjkEd7FJGhyYLeaMEWzQkAL6zFrKklNqU0xhdKYMWN46KGH2Lt3L4899hj79u3jrLPOYuTIkTzwwAMcPHgw3uUUIrECHoItu5jEescsXFPKdShO6+ciN01mJ7SlSx3dFouFK6+8kmeffZaf//znbNu2jUWLFlFSUsKcOXPYt29fvMopRGIF3FjNJoaVFnBaf1ds5wiHUr9UmH5Kgcx/a0eXQmndunX853/+J0VFRTzwwAMsWrSI7du388Ybb7B3716uuOKKeJVTiMQKuHFYzYwuK2bCwOzYzhEOJdnRpGMxDZ584IEHWL58OZs3b+biiy/miSee4OKLL8ZkMjKurKyMiooKBg4cGM+yCpE4LVNMbCmxnyO8A4oKePAFQoR0Rao9qcYvJ4WY3pFly5Zx4403Mm/ePIqKito8Jj8/nz/96U9dKpwQScPfjC8Ywh2wkBIMxTYSOzxnrq7JzePvbCE3I4XrzxgQ54L2fjGF0htvvEFpaWmkZtRCKUVVVRWlpaXYbDbmzp0bl0IKkVChIIQC7Knz8O/1B5lQbmXy4Jzoz2M2akoWs4ZZ9+MLRrGZZR8SU5/S4MGDOXTo0HHPHz58mLKysi4XSoikEmg2PugmQpot+qVwW5iMTSwtJhMW3SdL4rYjpndXqbY34Wtqaur0KpJC9Brh/iSfZizUFtMUkxYWBxaThkX34Q/qhKLZ0LKPiKr5tnDhQgA0TWPJkiWkpBzp9AuFQqxevZoxY8bEtYBCJJzfqCl5uzLvrYX1SCiBMdUkxSad3UeL6t345JNPAKOmtHHjRmw2W+R7NpuN0aNHs2jRoviWUIhEC9eU3HRh3lsLiwNN0yKjun0BnRTbCV7Tx0QVSu+88w5gLFv70EMPkZGR0S2FEiKpHLdsSReG94WHBaSYZKpJe2KqNy5fvjze5RAieQXc6Erh6cpaSi0sxrCAgS4zqdkuWeStDZ0OpSuvvJKKigoyMjK48sorOzz2+eef73LBhEgafjcKGNI/n8zszNhWCGgRrimNLrRDeUF8yneS6XQouVyuyCLnLleMc3+E6I0CbsyaxsgBRdAvv2vnatl0QKaatKvToXR0k02ab6JPCfcpYe3CFJMWkakm3shyuF1qDp6EYqqHejwe3G535OvKykoefPBBXn/99bgVTIikEd4Zty5o7fqAx3Cf0tY9B1m2Yjurvq6JQwFPLjGF0hVXXMETTzwBQF1dHRMnTuT+++/niiuuYNmyZXEtoBAJF3BzuNnP058e4rUv9nftXOGaki2yJK6sPnmsmFeePPvsswF47rnnKCwspLKykieeeIJf//rXcS2gEAmlhyDgJRhSBEyOrt8tC/cp2SLbLMmQgGPFFEput5v08LbFr7/+OldeeSUmk4kzzjiDysrKTp9n2bJljBo1ioyMDDIyMpg8eTKvvPJKLEUSonuEB04GdAia7F0bowSRNZVsGKHkk3W6jxPzxgH/+Mc/qKqq4rXXXuOCCy4AoLq6OqoBlf379+dnP/sZ69evZ926dZx33nlcccUVfPHFF7EUS4j4C3dy+zUbaKaud0qHm29WFQCly+DJNsQUSkuWLGHRokUMHDiQSZMmMXnyZMCoNY0dO7bT57nsssu4+OKLGTJkCEOHDuWnP/0paWlprFq1KpZiCRF/LaO5tThMMYFITcli1rDofmm+tSGmEd1XXXUVZ511Fvv27WP06NGR588//3y++c1vxlSQUCjEs88+S3NzcyTkjuXz+fD5jozvaGhoiOlaQnSa3willtHcTlsXQ8lkBrMVsymEOeDDF3CilIqMARQxhhJAYWEhhYWFrZ6bOHFi1OfZuHEjkydPxuv1kpaWxgsvvMApp5zS5rFLly7lnnvuiam8QsQksjNunGpKAFYn1oCfoTkWSM9AKZBMOiKmUGpubuZnP/sZb731FtXV1eh66866aHbQHTZsGJ9++in19fU899xzzJ07l3fffbfNYFq8eHFk+RQwakolJSWx/AhCdE54gbd+ednouZlkOK1dP6fFjtmkMX2IC7JlqsmxYgql+fPn8+677/Kd73yHoqKiLlU9bTYb5eXlAIwbN461a9fy0EMP8cgjjxx3rN1ux26XvbJEDwo338r7FVJe1sUpJi3CAygJyO7SbYkplF555RVeeuklpkyZEu/yoOt6q34jIRLK32R8tMVxj7aWqSZBY6qJxaRhMXdxqMFJJKZQysrKIjs7xr2vjrJ48WJmzJhBaWkpjY2NPPXUU6xYsYLXXnuty+cWIi78zYSUojFkx+4Pdb2jGyJ34N7+vJINpjQuG11MeX5a1897kogpnn/84x+zZMmSVvPfYlFdXc2cOXMYNmwY559/PmvXruW1117jG9/4RpfOK0Tc+Jtp9gV5dkMNT6/dFZ9zWo1QchAEZFT3sWKqKd1///1s376dgoICBg4ciNXauvPv448/7tR5ZF84kdSUAn+zMcXEnEJ6vGbzh2tKDi08qlsGULYSUyjNnDkzzsUQIgmF/KAHCeo6AWtKfJpuEOnotnNknW5xREyhdNddd8W7HEIkn/AuJn6s6CZL/NY9Cnd028Pz32SqSWsxd/nX1dXxxz/+kcWLF3P48GHAaLbt2bMnboUTIqHCd968mlGzicvASYisFBAJJakptRJTTWnDhg1Mnz4dl8vFzp07uemmm8jOzub5559n165dkbWWhOjVwjWluE0xadGyppIKgCYd3ceKqaa0cOFC5s2bx9atW1vtiHvxxRezcuXKuBVOiIRqCSXN+B2PW00p3KeUYgowpCCN/llxWGb3JBJTTWnt2rVtjrju168f+/d3cWU+IZJFuPmWn5PDmKxM8tLjNJsgXFPKsOpcelqRTHw7RkyhZLfb25yhv2XLFvLy8rpcKCGSQniKycCifAYOjNMUE4gMCUApCHqP7HAigBibb5dffjn33nsvgYBxS1PTNHbt2sUdd9zBt771rbgWUIiECTff4jrFBMBsMR4Yu5p4AyGUUvG9Ri8WUyjdf//9NDU1kZeXh8fjYerUqZSXl5Oens5Pf/rTeJdRiMTwN6GUojZow+0Pxjc4LA50pfjDii9ZtmK7bCBwlJiaby6XizfeeIMPPviAzz77jKamJk4//XSmT58e7/IJkTj+ZgK64tkNh/Fs2cFt55fH79wWByatCYfy0YwxgFL2fzNEHUq6rlNRUcHzzz/Pzp070TSNsrIyCgsLZQU9cfJQCgJuAiGdgDkFp80U39/tcL9SiilIDcYAShdxWKvpJBBV800pxeWXX878+fPZs2cPp512GqeeeiqVlZXMmzcv5qVwhUg6ATfoIQIhCJid8RsO0CLcue00yaTcY0VVU6qoqGDlypW89dZbnHvuua2+9/bbbzNz5kyeeOIJ5syZE9dCCtHjfI0A+E12lGaOf9MqPCzgyKRc6VNqEVVN6a9//Ss/+tGPjgskgPPOO48777yTJ598Mm6FEyJhwqHkNRkDG1PtMS9n37bwAEqnZtzBlprSEVGF0oYNG7jooova/f6MGTP47LPPulwoIRIuHEpuzQiluE0xadFSU6IllKSm1CKq+D98+DAFBe0vdF5QUEBtbW2XCyVEwh0dSgpSbfGuKRkd3Tl2naGZ6WSlSCd3i6je6VAohMXS/kvMZjPBYLDLhRIi4cJTTHKzcxiTkkmRy3GCF0QpvPrkAJeZAaOK4nvuXi6qUFJKMW/evHZ3FJEF/8VJI1xTKi0qoLQojlNMWrRMNQnvKyeOiCqU5s6de8Jj5M6bOCmEQwl7evecv2W+W9CLUgp/SMdukcGTEGUoLV++vLvKIURy8TWiUBwO2nD6gzit5vgOnrQaHegNDQ0sf2sbLqeFeVPK4nf+Xkw2mxLiWEE/BH2EdMWTn9TyyLtfE9LjPGE2XFOyKD9KD+CWIQEREkpCHCvcye3HSshkw241xX+zSIsDNBNWswlLyIcvoMc/+HopCSUhjuUz1grzhQdOpnTHRFlNA6sDi1nDpoztuz1SWwIklIQ4XmQ0t7GOUkq8R3O3sKagoZFmMgZQevwSSpDgUFq6dCkTJkwgPT2d/Px8Zs6cyebNmxNZJCEiodSyNndKvEdztwj3K6WajflvEkqGhIbSu+++yy233MKqVat44403CAQCXHDBBTQ3NyeyWKKv8xrNN7cpDeiG0dwtWkIpPClXmm+Gbnq3O+fVV19t9XVFRQX5+fmsX7+ec845J0GlEn2etx6AJs0IpbjPe2sRHhZQlAK+tPTuq5H1MgkNpWPV1xu/DNnZ2W1+3+fztRo13tbmBUJ0WTiU8vPzGRPKpNjVTQv7h2tKp+ZZOXWITDVpkTQd3bquc/vttzNlyhRGjhzZ5jFLly7F5XJFHiUlJT1cSnHSUyoSSoP6F3Pu8HxKc7ppX7ZwTYmAu3vO30slTSjdcsstfP755zz99NPtHrN48WLq6+sjj6qqqh4soegT/M2gB41b9vaM7r1Wy1STgMeYaiILvQFJ0ny79dZb+fe//83KlSvp379/u8fZ7fZ2JwMLERfhWpJuS+NQc4B0OziscV6fu0W4plR9uJan3tpKv0wns8ZL7T+hNSWlFLfeeisvvPACb7/9NmVlMvdHJFg4lHyWdJ5ctYtHV35Nt23JFq4pWUNelJK7by0SWlO65ZZbeOqpp3jxxRdJT0+PbPntcrlwOmXXUJEA4VDymo3VAVLtZkymbtqhJxxKNmXcvJFxSoaE1pSWLVtGfX0906ZNo6ioKPL429/+lshiib4sHEru8GjuuK/NfbRw882qhTDpQTyyUy6Q4JqS/AOIpNMyRgkjlNK6M5TMNjCZsSiFJeDFb0rDG9C7b1xUL5E0d9+ESArhUGrUeiCUNA2sTkyaRqrJaMK5/bKctISSEC2OGqNUr7ppa6VjhfuV0lsm5Upnd3IMCRAiKXjrjTFKJjN1ISfg7d6aEoA1FThISZrCmZaOzSL1BAklIVp4DhsfHZkMK3KRne4gN93Wvde0Gc3Eif0cUCpTTUBCSYgjPOE9C1OyGdnP1TPXtIWnmoRXuxTSpyTEEe5wKDmzeu6aNmMlAvzN6LrCF5Q+JQklIVqEm28+q4vqBi/enuh0Djff9h6q4eG3t/Hyxn3df80kJ6EkRItw8606lMqTq3fx3Prd3X/NcCjZQh50pWj2SU1JQkkIAD0EnjoA6pTRpMpwWrv/ulYjlBzK2Cm32SfjlCSUhABjOIDSwWyhLmSsRJHu6IH7QC01Jd0HSscTCKH38a2WJJSEAHDXGB+d2TSGm1AZjp6oKaWApmE1g135UAqa+/iobgklIQCaDxofU/No8BijqzN6oqZkMoHViYZGRmSqSd/uV5JQEgJahVKj16ip9EifEkSacBlmIwyb+ni/kgyeFAIioRR05kRCoUeabxAeq3SQknQdiyMdZ3fsyNuLSCgJoevgNsYo6c5czhkKDd4gDmsPNSTCNaWxBVbGylQTCSUh8NQaQwLMVmxpWYxL76aVJtsTDiWZamKQPiUhIv1JucYaRz3NZiy9i68RXVc9M5I8iUkoCXFUJ3d1o5fqBi+BUA9ud2Q3QulQzSF+/fZWnv94T89dOwlJKAlxVCh9tL2GJ1fv4st9Pbj7cjiU7KFmlIJGb6Dnrp2EJJSEaDpgfEzLp7bZD0BWSjevo3S0cCg5lAeUjtsf6tmaWpKRUBJ9W8ATmfOmpxZQ7zGGA7hSemg4ABhDAjQTFk2RqhkDKJu8fXeskoSS6Nsaw0uFOLNoCJrRlcJi0kjv7mVwj2YygS0VDY0si9coloSSEH1UY7jpll5Ardvoy8lMtXXPNt0dCTfhssxGKDX04X6lhIbSypUrueyyyyguLkbTNP7xj38ksjiiL2qpKaUXUes2+pMye2p6ydHCoZQRbr5JKCVIc3Mzo0eP5re//W0iiyH6skgnd0Gkkzs7tQc7uVs4MgAosPsZXphObpq958uQJBI6onvGjBnMmDEjkUUQfdlRndykF3KKGVxOK8WZzp4vi90IpQGpQQac2renmvSqaSY+nw+fzxf5uqGhB8eSiJNPw17jozMLrE6KXFDkSkAgQaT5hq8xMddPIr2qo3vp0qW4XK7Io6SkJNFFEr1ZfZXx0dU/seWASE0Jbz26rmjwBlCqb65A2atCafHixdTX10ceVVVViS6S6M3qwxsDuPpzuNnP1gON1LsT1MHszARA9zXyu7e/4k/v7eiz6yr1qlCy2+1kZGS0eggRk1AQGsJ33lwlbD3QyL837OOjrw8lpjzWFDBbMQE5ZmMTgbpEBWSC9apQEiJumvaDHgSrE1KyqQnfeUvYXS9Ni9SWci1GKLUMUehrEtrR3dTUxLZt2yJf79ixg08//ZTs7GxKS0sTWDJx0juq6YamUd1gDFpM6K14RyY0HSTb7Ab6bk0poaG0bt06zj333MjXCxcuBGDu3LlUVFQkqFSiT6hr6eQuwRsIRUZzF2Q4ElemcE0pSzNCSWpKCTBt2rQ+e4dBJJAegrpK4/OsgRwI15IyU6w4bQlcH9uZBUA6xgqUfbWmJH1Kou+p3w2hANhSIC2fffVGKBUmspYERvMNSNONUKr3BPrkxpS9avCk6J0CIZ2QrnAkyy4dtTuMj1lloGmRmlKhK8GhlJINgD1Qz7CCVFwpdkJKYSIBS/QmkISS6DZuf5B/b9jH3joPShn7qI0pcTGmJAuzKYF/aIfDoZRdBsD0EQXsLvRQkJHg+WZ2F5gtmEJBLi53RkKqr5Hmm+g2TquZkK5o6TZs8ARYueUQz66rwp2oran9zUcm4WYNBCDVbmFYYTqZPbnaZFtMJkjJMT5v2Ua8D5KakogrpVRkLSJN07jglAIsZhMOq4mtB5pYufUg++q9PLtuN7PG9yfF1sO/goe2glKQXnBkvlkyScmFxgOo5oM0pQ3EEwiRn57gZmUPk5qSiKuPd9Xx4bZDkbuqOWl2XE4rdouZkf1czJ5QSrrDQqM3wKHGBNzyPrTF+Jg7DIAPtx1izY7DybNYf2ouANX79/LH93bw2hcHElygnic1JRE3h5v9fLjtEEFd0T8rhdKclOOOyUq18c2x/QjpivyevtsV8ELtTuPzvGEEQjof76olEFIMzE0hvae26e5IihFKGaoOgMNNfkK6SmwfXA+TmpKIC11XvLFpP0FdUZabSkl2+0uA5KTZez6QAGq2GmOUUnMhNZeqw24CIUW6w0JesiyqFq4pOXyHsVtAV4qaZt8JXnRykZqSiItPqurYW+fFZjFx3oj8jte41kPGWKGmag4dOsCWPQeZ0D8VqwkiveKaZmxn7XAZU0FcJWDq4pCCfRuMj/kjAPj6YDMAg/JSe35N7vY4s8BiQwv66W9tZnswleoGX5/qV5JQEl1W5/bz0XZjdv05Q/LIaK8Z5G2AXavgwOcQ9KGUYvfuOrSAzh6vg4E5qe1fxJYC/cZBySQwx9DMch+Gul1G2BWehq4rvj5kDFIcnJcW/fm6i6ZBehHUVtLfWsd2Tyr7672M7OdKdMl6jISS6BKlFK9vOkAgpCjNTmFkvzaWk1EK9n0K2940lgwBY0uhjGKKslJ55+smdphtZJT0M9bH1jSjNuVvBvchox/I74Yd78G+z2D4JZHb+Z227zPjY1YZOFzsOtRMsy+E02amf9bxfV8JlV4ItZUUanVAP/bVexJdoh4loSS6pLrRx/56o9k2/ZSC45tBegg2vwz7Pze+dvWDgWdB5kAwmSgAXJZ9fLW/kVdrHFwzqATTsZ26eggOboav3zFqW589DYPOhZKJRoCdSNAHez8xPi8eAxDZlntYQXrydSKnFwOQqxtjlWqa/XgDoeQZEd/NJJROAspdizq4BZOvDkwWyCiGnPLYmjlRKshwcO2kUurcflzHbk2kh+Dz56FmG2gmGDStzSA5Z2geO2qaOdDgZcOeesaUZLY+j8kMBacYP9PW14yA2/62UYsaetGJ+5r2fmIEU0oO5AwBwGEzYzFpjChKwoUC0wsBsHlrOGNAGtkZ6ZiSpc+rB0go9VLBkM6Xlfuo3/QG1urP0RXYzCYyU6wUuhykpmVA2TlQNKZztYkuyE2zH78OkVJGDalmG5gtcOqVkDO4zden2i1MGZzL219V88G2Q5Tnp5HW1g61FhsMv9Toc9n2ptFx7W2AU78J1nY6gn1NUPmh8XnpGcaoaeDcYfmcUZaDw5qEN6AdLuPhrWdyjgdyihNdoh6VhP8iojNWrP2E2pW/x3zACKR6Rz92pI5mgypn7f4Q9fX1sPlV+PzvEIz/IMWtBxojE1nbVPmhUaPRTB0GUotR/V0UuRz4g3qkadUmTYP+42HkVUZNsHYnfPLnI1slHU0pI7yCPmMEd8HIVt922szJc9ftaJp2pM+sZfJwHyI1pd7o0FamNL7OZpOf7MJSMsZcgS27HzVNfjbuqac+GCAjZy/sWGlMq/jsKThtlnGLPQ7q3YFw57bOrPEl9Dt2n7QDm4xrAwyZfsJAAmNKyvkjCjjY6GNEUSemf+SWw9jvwMZnoPkQfPwEnHaV0XRtsesjqP7SCMYhF4LJxJYDjWQ4rIlfEeBEsgYanfO1lRxo8PL1wWZG9stIjgGe3UxCqZfQdcWeOg8lod2w6R84LTB67AS0EZcbzRogJdtCSXYKuq7QTKXgKiG04Rnc1btI//QpGHNtl4NJ1xWvbdqPP6jTL9NJ0bGDIOt3w1cvGZ+XTDBu43dSXrqdvPQoBjGmF8Dpc2Hjs9BUDR//GQpODd9S32EEMsDgc8HVD48/xJtfHsAX0LlqXH9KspPsrtvRsgYYH5uq+fDzr9nZbMFpMx/f33YSkuZbLxAI6fxrw17efv99atb8zehAzh+BduqVkUA6WsvdK5VRzFuOC/mkOsTB/Xvgs78au8J2wSdVteyp9WCzmLjw1MLWd8o8tUZzUQ9C7hAYdF7M1/EGQnxWVXfilUkdGTD2esgbCkqH/Rth6+tGIGkmI5BKJgLw3taD+AI6uen242t3ycaWGtmPboTF2DTzq46atScRqSklOW8gxD8/20vz3q8YUfMWWn6KcSdq+GWRTtv26AqCjiy+yLsEqv9FILSLIu1ptNGz2+8Y7sDuWjfvbzVuU08dmocr5aimRMADG58zxhOlF8CIy09Yvvb4gzp/WVVJo9cY0zT6RLUDix1GfstYd/vgV+BrMFZxLBwFaXkAfH2wiS/2NqBpcO6wvOOHHSSjglOgfjdleiUmbQD76r0cbvYbY7lOYlJTSmIef4i/f7ybpr1bOeXwm5xSmEr2wNGdCiQAs0ljxshCTikvY1PeJWyv06ncuR214W9G528UGrwBXtqwD10phhakc2rxUbfSQ0GjhtR8yFgOZORVbdbgOstmMTG2NBOAd7cc7PzgwcwSGPINI6DKz48EUm2zPzLbfmxpVvINlmxP3nDQTNg91QxNNabEbNhdl9gy9QAJpSTV6A3w7PoqfNXbGVn7BiOL0sgoORVGdC6QWmiaxtSheUwYOZRN+RdT1ajYtm0zoU+eiqopt3l/I25/iLx0O984epCkrsNX/zJqKRYbjLraaFJ10emlWQzOTyOkK178dC+Hm2O7g9jgDfCPT/fgDYQocjmYMjiny2XrMbZUyDOWWBlnMvrHPt9Tn7gF8nqIhFIScvuDPL2mCg5sYlTt65xWlEpq0XA4ZWbMk1LHDcjmnNNHsqXgEva7TezYsQ0++Qv4Gjv1+vEDsphSnsvlY4qxWcK/NnoIvnwRqr8yyjXyW5CWH1P5jqVpGheeWkBBhgOPP8TzH+/mYGP0s+W/PthMnTtAhtPKZaOLsZh72a986RkA5Hm2M8BhrGqw+uvDCS5U9+pl/0J9Q4rFxDj1OaMb32VkURrOfqcaAwS7OEt+RFEG0yeNZlu/y8nLyTGaW+uWGxNVj6Hrio276yP72WuaxsSy7COTbf1u465XSyCdckX089FOwG4xM3NsMTlpNhq9QZ5ZV9Xx2Kg2jCnJZOqwPGaN709qWwMyk116IeQOQVOKqXyMzayd9H1KmkqCjdd++9vf8n//93/s37+f0aNH8/DDDzNx4sQTvq6hoQGXy0V9fT0ZGUk4XSAKBxt9mDTIoQG2voZeuwtdKSylE6F8elxHZXsDIRzBBqNjuvkQVXUeDmWchnnQ2WhWJ4fdfrYeaKTRG2R4YTozTitqfYJD24zpHt6GE47Wjld5//nZXhq9Qb5zxoBITe3opXfB6CDfVt3E53vquXhUUdujwnsjTx2s/QOEgviLx2Mb9o1El6jTYvkbTXgo/e1vf2POnDn8/ve/Z9KkSTz44IM8++yzbN68mfz8jpsCvT2UlFLsrvWwYXc9e3bvYKS+lcmpe40NdcxWGDbDGHfTXYJ+vJte5rN176MrCJlsHEgbQY1zEM22XBw2C2cMymZMSSZa0GvsArL3kyM1q5RsowYXpyZbR0K6osETICtcSwiGdCo+3EmGw4rFrOH2h6hp8qOHf51H9Xdx/oiCbi9XjznwBWz6p/F50WgoP58DbkW9J8CQ/LTkHJlOLw2lSZMmMWHCBH7zm98AoOs6JSUl/Nd//Rd33nlnh6/tjaG0t6aB6tpaamtqOHxwL+bmalze3TiCDeSk2hicn4Y5fxgMPi+yY2p3CumKfTs24d30KqHmGpRS2C0m0lJSyM7JxWyxGh3ivsYjC7CZzMZUjwFTjNvxCVB12M1z63cf97zLaeXU4gxO6+/q+U0Jutvudca0GaXwm+y8XZvHPlMhzqwiBhUXUJSVSmaKlVSbJWmGPPS6UPL7/aSkpPDcc88xc+bMyPNz586lrq6OF198sdXxPp8Pn+9IZ2dDQwMlJSUn/oEP74Btb7L5QBN1bj8Q/pHDP7rxQTGpLDuy7d/W6gZqmlru+BjbBGktr0MxfkA2FpMGKLZVN3Gw8ei+DhW+hPG6iWVZxrGhINv213Kw6cidJLNJIzfNRr4rlbT+pxojoMOD5nqUUsbk2f0bjPcr1MZC+ik5xt2g4jHGhNEE0nVFdaOPRm8Af0jHaTVHNik4qR3+Gra+gd5cw546D/vqPITCv5Yhkw1dM6M0MwWuFIYXZYKmEVKKtTtqI6c4tlKVmWpjWP6Rhe5W7zwc+RM5dh/MYQXpZLa8x6O+fcI7rbGEUkL/Kzl06BChUIiCgtbV7IKCAr766qvjjl+6dCn33HNP9BcK+qD5EKqpAb2j/dndRP7FlLsR5Wl9G7pVevsskVvzmr8J/G3fGdKAoM+OxWJ0Umc4rYTQsKdkkJFTiCu3GHPWAGNaQYJqHUZBNWMUdu4Q4za/tw689cYoaYvdGIxoT54VGk0mjUKXI/nnsMVb9iCYMB/T4a8pObiZgrq9HD64jwa3jyZfEH/QjwIsvgC4QwBoSqGaj9yxO64WomyQeuQ/VdVYc/wxLdK8oIc72pUer5+qlV5Vv128eDELFy6MfN1SUzqhzBIYfQ25Q/yk+5WRO0f9dxH51OUENNA08t0BXCHjTdfCz2nhgzXAlGqHcBW5wBfEFdSP7HfW8irNeFgdViPATCbyrSnkm23dvpxIl5hMRn9RH92hNemZzJH/QGxAoVIUBjwQ8KCHAnj9fqPmawFQaEoxoKz94RQOqwmOuqNXMqAloI6PJkeqDVqWe4nTBO9jJTSUcnNzMZvNHDjQem+rAwcOUFhYeNzxdrsduz2G2oQtFbLLyI3ibywzitZJWjokTx1C9DmaZqxhbkvBBBw7Xl0DiqP43e/X/V2ZHUroOCWbzca4ceN46623Is/pus5bb73F5MmTE1gyIUSiJLz5tnDhQubOncv48eOZOHEiDz74IM3Nzdxwww2JLpoQIgESHkrf/va3OXjwIEuWLGH//v2MGTOGV1999bjObyFE35DwcUpd0RvHKQnRl8TyNypz34QQSUVCSQiRVCSUhBBJJeEd3V3R0h3W0NA31i4Wordp+duMpuu6V4dSY6OxQFmnRnULIRKmsbERl6tzI5J79d03XdfZu3cv6enpSbt0Q3drmWpTVVUldyBjIO9f15zo/VNK0djYSHFxMaZOLuPcq2tKJpOJ/v0TMKM+CWVkZMgfVRfI+9c1Hb1/na0htZCObiFEUpFQEkIkFQmlXs5ut3PXXXfFtnqCkPevi7rj/evVHd1CiJOP1JSEEElFQkkIkVQklIQQSUVCSQiRVCSUeqHDhw9z3XXXkZGRQWZmJt/97ndpamrq8DXTpk2LbGTQ8viP//iPHipxYv32t79l4MCBOBwOJk2axJo1azo8/tlnn2X48OE4HA5OO+00Xn755R4qaXKK5v2rqKg47vfM4Yhyxxklep2LLrpIjR49Wq1atUq99957qry8XM2ePbvD10ydOlXddNNNat++fZFHfX19D5U4cZ5++mlls9nUY489pr744gt10003qczMTHXgwIE2j//ggw+U2WxWv/jFL9SmTZvU//zP/yir1ao2btzYwyVPDtG+f8uXL1cZGRmtfs/2798f1TUllHqZTZs2KUCtXbs28twrr7yiNE1Te/bsafd1U6dOVd///vd7oITJZeLEieqWW26JfB0KhVRxcbFaunRpm8dfffXV6pJLLmn13KRJk9TNN9/creVMVtG+f8uXL1cul6tL15TmWy/z0UcfkZmZyfjx4yPPTZ8+HZPJxOrVqzt87ZNPPklubi4jR45k8eLFuN3u7i5uQvn9ftavX8/06dMjz5lMJqZPn85HH33U5ms++uijVscDXHjhhe0efzKL5f0DaGpqYsCAAZSUlHDFFVfwxRdfRHXdXj0hty/av38/+fn5rZ6zWCxkZ2ezf//+dl937bXXMmDAAIqLi9mwYQN33HEHmzdv5vnnn+/uIidMtDswg/H+tnV8R+/tySqW92/YsGE89thjjBo1ivr6en75y19y5pln8sUXX3R68ryEUpK48847+fnPf97hMV9++WXM51+wYEHk89NOO42ioiLOP/98tm/fzuDBg2M+rxBHmzx5cqs9G88880xGjBjBI488wo9//ONOnUNCKUn88Ic/ZN68eR0eM2jQIAoLC6murm71fDAY5PDhw23uKtyeSZMmAbBt27aTNpSi3YEZoLCwMKrjT2axvH/HslqtjB07lm3btnX6utKnlCTy8vIYPnx4hw+bzcbkyZOpq6tj/fr1kde+/fbb6LoeCZrO+PTTTwEoKiqK94+SNGLZgXny5Mmtjgd44403+uSOzfHYwToUCrFx48bofs+61E0uEuKiiy5SY8eOVatXr1bvv/++GjJkSKshAbt371bDhg1Tq1evVkoptW3bNnXvvfeqdevWqR07dqgXX3xRDRo0SJ1zzjmJ+hF6zNNPP63sdruqqKhQmzZtUgsWLFCZmZmR29Tf+c531J133hk5/oMPPlAWi0X98pe/VF9++aW66667+vyQgGjev3vuuUe99tpravv27Wr9+vXqmmuuUQ6HQ33xxRedvqaEUi9UU1OjZs+erdLS0lRGRoa64YYbVGNjY+T7O3bsUIB65513lFJK7dq1S51zzjkqOztb2e12VV5erv77v/+7T4xTUkqphx9+WJWWliqbzaYmTpyoVq1aFfne1KlT1dy5c1sd/8wzz6ihQ4cqm82mTj31VPXSSy/1cImTSzTv3+233x45tqCgQF188cXq448/jup6snSJECKpSJ+SECKpSCgJIZKKhJIQIqlIKAkhkoqEkhAiqUgoCSGSioSSECKpSCgJIZKKhJLoVSoqKsjMzIx8fffddzNmzJjI1/PmzWPmzJk9Xi4RPxJKok3z5s1rdx3vW265BU3TWq1q0B1hMHDgQB588MFWz337299my5Yt7b7moYceoqKiIvL1tGnTuP322+NaLtG9JJREu0pKSnj66afxeDyR57xeL0899RSlpaUJKZPT6TxukbujuVyuVjUp0ftIKIl2nX766ZSUlLRanfL555+ntLSUsWPHduncbdVgZs6cGal9TZs2jcrKSn7wgx9EdsWA45tvxzq6xjZv3jzeffddHnroocg5duzYQXl5Ob/85S9bve7TTz9F07So1v0R3UNCSXToxhtvZPny5ZGvH3vsMW644YZuv+7zzz9P//79uffee9m3bx/79u2L+hwPPfQQkydP5qabboqco7S09LifCWD58uWcc845lJeXx+tHEDGSUBIduv7663n//feprKyksrKSDz74gOuvv77br5udnY3ZbCY9PZ3CwsKYVn50uVzYbDZSUlIi5zCbzcybN4/NmzdH9i8LBAI89dRT3HjjjfH+MUQMZDlc0aG8vDwuueQSKioqUEpxySWXkJubm+hidUlxcTGXXHIJjz32GBMnTuRf//oXPp+PWbNmJbpoAqkpiU648cYbqaio4PHHH49bbcJkMnHsUl6BQCAu5+6M+fPnRzrxly9fzre//W1SUlJ67PqifRJK4oQuuugi/H4/gUCACy+8MC7nzMvLa9VPFAqF+Pzzz1sdY7PZCIVCXbpOe+e4+OKLSU1NZdmyZbz66qvSdEsi0nwTJ2Q2myPbO5nN5naPq6+vj2xI0CInJ4eSkpLjjj3vvPNYuHAhL730EoMHD+aBBx6grq6u1TEDBw5k5cqVXHPNNdjt9piajQMHDmT16tXs3LmTtLQ0srOzMZlMkb6lxYsXM2TIkD65MUCykpqS6JSMjAwyMjI6PGbFihWMHTu21eOee+5p89gbb7yRuXPnMmfOHKZOncqgQYM499xzWx1z7733snPnTgYPHkxeXl5M5V60aBFms5lTTjmFvLw8du3aFfned7/7Xfx+f4/cTRSdJ2t0iz7rvffe4/zzz6eqquq4XWBF4kgoiT7H5/Nx8OBB5s6dS2FhIU8++WSiiySOIs030ef89a9/ZcCAAdTV1fGLX/wi0cURx5CakhAiqUhNSQiRVCSUhBBJRUJJCJFUJJSEEElFQkkIkVQklIQQSUVCSQiRVCSUhBBJ5f8DXGvBgTPmhUIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:39:16.036478Z", + "iopub.status.busy": "2024-02-29T03:39:16.036182Z", + "iopub.status.idle": "2024-02-29T03:39:16.379067Z", + "shell.execute_reply": "2024-02-29T03:39:16.378175Z" + }, + "papermill": { + "duration": 0.364322, + "end_time": "2024-02-29T03:39:16.381233", + "exception": false, + "start_time": "2024-02-29T03:39:16.016911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDmElEQVR4nO2deXxU9bn/32f2TJKZ7BskLLLIDopQxBVpVSwVbdVaF3Dtgr1VLq1yb4tCq2irXrnqRWsL6E8pVevS1gWpCxQUZBFRkS0GCGtCQtbZ55zfH2dmIJBtJpPMhDzv12tek5mcOeeZYfLh+T7fZ1E0TdMQBEFIEgyJNkAQBOFERJQEQUgqRJQEQUgqRJQEQUgqRJQEQUgqRJQEQUgqRJQEQUgqRJQEQUgqTIk2oCOoqsrBgwdJT09HUZREmyMIwklomkZ9fT1FRUUYDO3zgbq1KB08eJDi4uJEmyEIQhuUl5fTu3fvdh3brUUpPT0d0N+ww+FIsDWCIJxMXV0dxcXFkb/V9tCtRSm8ZHM4HCJKgpDERBNekUC3IAhJhYiSIAhJhYiSIAhJRbeOKbUHTdMIBAIEg8FEmyJ0Q8xmM0ajMdFm9ChOa1Hy+XwcOnQIl8uVaFOEboqiKPTu3Zu0tLREm9JjOG1FSVVVysrKMBqNFBUVYbFYJMFSiApN06isrGT//v0MHDhQPKYu4rQVJZ/Ph6qqFBcXY7fbE22O0E3Jzc1lz549+P1+EaUuIuGB7gMHDnDjjTeSnZ1NSkoKI0aMYOPGjXE7f3tT2wWhOWL1roOqxtEGb5yt6Rkk1FM6duwYEydO5OKLL+add94hNzeXXbt2kZmZmUizBCF6NA33zg8wHPkSd+F4Xj2ch8Vk4OYJfRNtWbcjoaL0yCOPUFxczJIlSyLP9evXL4EWCUKM1Ozl6Jcfsb/GTd6Ro/jTr6LemEaty4/Tbk60dd2KhK5t/v73vzN27FiuueYa8vLyGDNmDM8991wiTTpt2bNnD4qisGXLli45z9q1axkxYgRms5lp06Z16JrdgsNfUOP2A5BuNTBI2QvAwVp3Iq3qliRUlL755hsWLVrEwIEDWbFiBT/96U/5j//4D55//vlmj/d6vdTV1TW5CbHx0UcfoSgKNTU1nXL+WbNmMXr0aMrKyli6dGmnXCNp0DTUqjJcvgBV9v44bGaK1CMAHK71JNi47kdCRUlVVc466yweeughxowZw5133skdd9zBM8880+zxCxYswOl0Rm49pW2Jz+dLtAlRU1payqRJk+jduzcZGRkxnaMr33c4yTYmXFW4G2sJYOJo1hisZgM5ahVoKkfqRJSiJaGiVFhYyNChQ5s8N2TIEPbt29fs8XPmzKG2tjZyKy8vj+m6voDa4i0QVNt9rL8dx8bCRRddxF133cXdd99NTk4Ol156KV9++SWXX345aWlp5Ofnc9NNN3H06NHIa959913OO+88MjIyyM7O5rvf/S6lpaXNnn/Pnj1cfPHFAGRmZqIoCjNmzIjqPNu3b+fcc8/FZrMxfPhwVq1aFTm3oihUVVVx6623oihKxFNatWoV48aNw2q1UlhYyH333ddECJp732GPbsWKFYwZM4aUlBQmTZpERUUF77zzDkOGDMHhcPCjH/2oSZKsqqosWLCAfv36kZKSwqhRo3j11Vcjvw+f95133uHss8/GarWyZs2amP69qD9MozdAoyWH1KxeKCYbqSaVVH81VY0+ZAh1dCQ00D1x4kR27NjR5LmdO3fSp0+fZo+3Wq1YrdYOX/fpD3e3+Lt+OalMG9Mr8viPq0vxB5v/UvXOTOGasce9tcVry3D7mpaz3PPtQTHZ+Pzzz/PTn/6UtWvXUlNTw6RJk7j99tv5n//5H9xuN/feey/XXnstH3zwAQCNjY3MmjWLkSNH0tDQwNy5c7nqqqvYsmXLKWkRxcXF/O1vf+P73/8+O3bswOFwkJKSEtV5fvnLX/LEE08wdOhQHn/8caZOnUpZWRnFxcUcOnSIwYMHM3/+fK677jqcTicHDhxgypQpzJgxgxdeeIHt27dzxx13YLPZeOCBB5p93wCHDh0C4IEHHuCpp57Cbrdz7bXXcu2112K1Wlm2bBkNDQ1cddVVPPnkk9x7772A7lW/+OKLPPPMMwwcOJDVq1dz4403kpuby4UXXhi53n333cejjz5K//79Y9/1bazA41dxmbPISbeBuQCbfw+p7moqAjk0eAOk2yTY3V4SKkr33HMP5557Lg899BDXXnstn376KX/84x/54x//mEizkoKBAwfy+9//HoDf/e53jBkzhoceeijy+8WLF1NcXMzOnTsZNGgQ3//+95u8fvHixeTm5rJt2zaGDx/e5HdGo5GsrCwA8vLymiyv2nueu+66K3LsokWLePfdd/nzn//Mr371KwoKClAUBafTSUFBAQD/93//R3FxMU899RSKonDmmWdy8OBB7r33XubOnRsRvBPfNxwXpd/97ndMnDgRgNtuu405c+ZQWlpK//79AfjBD37Ahx9+yL333ovX6+Whhx7iX//6FxMmTACgf//+rFmzhmeffbaJKM2fP59vf/vbbf+DtEbjUTz+IC5LFn3sZjBkYzi2lyFOHyW9M6WSIEoSKkrnnHMOr7/+OnPmzGH+/Pn069ePJ554ghtuuKFTrzvz4gEt/s5w0vfnzgvOaPHYk79rt06MXzrD2WefHfn5888/58MPP2y2/qq0tJRBgwaxa9cu5s6dy/r16zl69Ciqqi8d9+3bd4ootUZ7zxP+YwcwmUyMHTuWr7/+usXzfv3110yYMKHJH+jEiRNpaGhg//79lJSUnPK+T2TkyJGRn/Pz87Hb7RFBCj/36aefArB7925cLtcpYuPz+RgzZkyT58aOHduize2moYLsNAv9e/Wj0JkChhwAzspRYWBux8/fw0h4mcl3v/tdvvvd73bpNS2m9ofSOuvYtkhNTY383NDQwNSpU3nkkUdOOa6wsBCAqVOn0qdPH5577jmKiopQVZXhw4dHHSyO13li5cT3fSJm8/Hlj6IoTR6HnwsLaENDAwBvvfUWvXr1anLcycv/lq7XbgJe8NaTnWole/hgMNsgqHuhuKo6du4eSsJFSWibs846i7/97W/07dsXk+nUf7Kqqip27NjBc889x/nnnw/QZtDWYrEANGnpEs151q1bxwUXXABAIBBg06ZN3HXXXS1eb8iQIfztb39D07SIt7R27VrS09Pb3VC+vQwdOhSr1cq+ffuaLNU6BXeNfm9O0QUJIEWPTWnuGho9fvxBjcxUS+facRohhWHdgJkzZ1JdXc3111/Phg0bKC0tZcWKFdxyyy0Eg0EyMzPJzs7mj3/8I7t37+aDDz5g1qxZrZ6zT58+KIrCP//5TyorK2loaIjqPE8//TSvv/4627dvZ+bMmRw7doxbb721xev97Gc/o7y8nJ///Ods376dN998k/vvv59Zs2bFvT4xPT2d2bNnc8899/D8889TWlrK5s2befLJJ1vMgYsZTw2+oEoNqcc3OawOUAwca3Cz9MMveferw/G95mmOiFI3oKioiLVr1xIMBvnOd77DiBEjuPvuu8nIyMBgMGAwGFi+fDmbNm1i+PDh3HPPPfzhD39o9Zy9evVi3rx53HfffeTn53PXXXdFdZ6HH36Yhx9+mFGjRrFmzRr+/ve/k5OT0+r13n77bT799FNGjRrFT37yE2677TZ+/etfd+izaYnf/va3/OY3v2HBggUMGTKEyy67jLfeeiv+ZUyeWmrdftaUB3jrCz0oj8EA1jSsJiPWYD31Hn98r3mao2jdOImirq4Op9NJbW3tKdNMPB4PZWVl9OvXD5vNliALhe5Om9+jne9x4Ks1rAsMJHXIt7lsuL7byGcvEqjey7L6MVSlDuCuSQMwG3ueD9Da32hL9LxPSRDiiacGX0DFY3KQbjsh3md1YDQopNEIQL0nxmzxHoiIkiB0BE8tvqCK15hOmvUEUbI5UVDIMOgFubKEaz8iSoLQETy1+AIqXlMaaSd6SjZ9qZKOXvrS4BVPqb2IKAlCrAS8EPTjCwTxGVNJtzZdvgHYNd1Tcvlkmk57EVEShFjxNqBqGh7NhGowN/WULHr2fa41wNl9MilwyGZLe5HkSUGIFV89GlBSmI/aN5MU8wmDBSx6pnieLUjegGw9TUBoFyJKghAr3gaMikJxQR7FJ9e4me16caSmgd8FVpkb115EvgUhVnx6jV14qdYEgwHMdjRNo6G+hsp6mWzSXkSUBCFWvA14/EGqA1Yam9tds6TiCai8/PEOXt4YW0PCnoiIUhJy0UUXcffddyfajAjJZk/S4Kunot7Lit0NfFpWfervLWmYjQqWoKtDnUh7GiJKpyndsa93t8PbgD+o4jfasVuamZ5rScVkMGBX9D7dzXpTwimIKCUZM2bMYNWqVSxcuBBFUVAUhdLSUm677bZIv+nBgwezcOHCU143bdo0HnzwQYqKihg8eDAAH3/8MaNHj8ZmszF27FjeeOONU0Yktdb/uzl79uzZ01UfR3Lj00XJa0zFbmlmzyi0A5em6PGkRp+IUnvoWbtvmgbBBKT7G82ntqlsgYULF7Jz506GDx/O/PnzAb25f+/evXnllVfIzs7m448/5s4776SwsJBrr7028tr3338fh8PBypUrAb0YcurUqUyZMoVly5axd+/eU5ZhbfX/bs6e3FzppoimhTwlDb8xlZRmPSU9AJ4W8pQkgbJ99CxRCvrh3491/XXP/08wta/Jl9PpxGKxYLfbI/2tAebNmxf5uV+/fnzyySe8/PLLTUQpNTWVP/3pT5EGbs888wyKovDcc89hs9kYOnQoBw4c4I477oi85qmnnmqz/3dz9vR4Ah5QA/iDKj5jCqnW5pdvAHZ0UTp5qITQPD1LlLoxTz/9NIsXL2bfvn243W58Ph+jR49ucsyIESMiggSwY8cORo4c2aTlxrhx45q8pj39v4Vm8DagoeHWzGiKCbu55eWbTRNPKRp6ligZzbrXkojrdoDly5cze/ZsHnvsMSZMmEB6ejp/+MMfWL9+fZPjYuk33Z7+30Iz+F0EVQ2vQR9NZW/OUzLbAcixBRnbN5NeGSldaWG3pWeJkqK0exmVSCwWS5Pe2WvXruXcc8/lZz/7WeS5lgZNnsjgwYN58cUX8Xq9kYb5GzZsaHJMW/2/m7NHAPxuFBTOKMrB1jer+QZuZl2Ecqwa5w/IaXdcsacju29JSN++fVm/fj179uzh6NGjDBw4kI0bN7JixQp27tzJb37zm1PEpTl+9KMfoaoqd955J19//TUrVqzg0UcfBYg072+r/3dz9oSnhvRo/C6MBoU++dmcO6CFNsAhUULT9BiU0C5ElJKQ2bNnYzQaGTp0KLm5uVx66aVcffXVXHfddYwfP56qqqomXlNLOBwO/vGPf7BlyxZGjx7Nf//3fzN37lyASJyprf7fzdnT0lj1HoU/NCLc0sqS2WAEkxVV06hvqKOqQUpN2oP06O5hvPTSS9xyyy3U1tZGRnULLdPi92jXSjx71tNYMA7boEmkWluIhKx7hsaaSpYFLgRHb+64oH/zx52mdLse3Q888EAkIS98O/PMMxNp0mnHCy+8wJo1aygrK+ONN96I5CCJIHUQv4vKei8rd9ax7ptWhk6abZiMCuagB7c/SDf2AbqMhAe6hw0bxr/+9a/I45aCrUJsHD58mLlz53L48GEKCwu55pprePDBBxNtVvfH7yYQVPGbbc0nToYx2zEZDZhUr75bF1CxmVs5Xki8KJlMJknK60R+9atf8atf/SrRZpx++F34gxoBq61pc7eTMadgVBRSQqUmHn9QRKkNEh7o3rVrF0VFRfTv358bbrih1SCq1+ulrq6uyU0QEoLPRUDV8Bva8pRCeUyKXiAtCZRtk1BRGj9+PEuXLuXdd99l0aJFlJWVcf7551NfX9/s8QsWLMDpdEZuxcXFXWyxIBDqJunGH1QJGFPa8JT0BMpUgy5Kbr+IUlskVJQuv/xyrrnmGkaOHMmll17K22+/TU1NDS+//HKzx8+ZM4fa2trIrby87cZZElgUOkKz35+gH9QAgaDuKbW6HAt5SimEREk8pTZJeEzpRDIyMhg0aBC7d+9u9vdWqzWSmdwWZrNe2uFyuWSnSYiZcF8qo/EE4fG70NDwaQqqYmpDlHRPqciuMbYkk+y05K8oSDRJJUoNDQ2UlpZy0003dfhcRqORjIwMKioqALDb7ZEsZkFoD6qqUllZid1ub7or7HeDBr3zchjfP7v5Bm9hQp5SrzTodfJwAaFZEipKs2fPZurUqfTp04eDBw9y//33YzQauf766+Ny/vCuXliYBCFaDAYDJSUlTf9D87tQFIXivCyKWyoxCRPylCIZ4EKbJFSU9u/fz/XXX09VVRW5ubmcd955rFu3Lm5NxBRFobCwkLy8PPx+meUuRI/FYomU20QIC0xYcFoj5CkFfR4aXF7AgNPesa4RpzsJFaXly5d3yXWMRmPTmIAgdITQzltDwITR48dha0VkTLoo1bi8LFu9nfycLK4dK7vGrZHwPCVB6Hb4XdS6/Xz0TQPvfXWk9WMNBjBZMBn0rG6vpAS0iYiSIERLyFPyG9rI5g5jSsFkUDCpXjx+afvSFiJKghAtfj2bO2CwYTO340/IbMNoVDCqPjxSlNsmIkqCEC1+N4GgRsDYXk/JFvGUAqpGQBVRag0RJUGIFp+LgKov32yt5SiFMdkwGhQs2vGiXKFlRJQEIVpCHQL8hjbq3sKYU1BQsBv0tBSJK7VOUmV0C0LSo6oQ8BAIqgSMbdS9hTHppVEDMk1kFmZibU8cqgcjoiQI0RDwgKZR4LRxVv9CsuztqGUL5SoNzbWAlJq0iYiSIESD3w1AXqaTvEH57XtNyFOSiSbtQ/xIQYgGf6N+354SkzChUpOA102t24/LF+gEw04fRJQEIRr8boKaRk3ARI3L177XmPQpKN8cPsriNWVs3lvTefadBogoCUI0+F14fEE+3ufm5Y1tNxkEIqJkCTV6k5SA1hFREoRo8LuP9+Zu7wAAsy5KVi0kSgERpdYQURKEaPC7Qr2525kOAJHdNzNBFC0geUptIKIkCNEQ8pQCbU0xORGTFRQlVGrik+VbG4goCUI0+FyRDgE2UztFSVHAZI0MpRRRah0RJUGIhlCHAL8xCk8JIkW5RtWLNyDLt9aQ5ElBiAa/m2BQbX/bkjAmGyajwtBcC1pWBpqmySCLFhBREoRo8LvISrUyql8hhc4oRneZUzAZDEzskwoFbQwb6OGIKAlCewn6IegnK9XCuEG9I1v97SJSauLtHNtOIySmJAjtJVT3hmI4LjLtJZQW4PM0Uuv245O4UouIKAlCewmNVqoLmjnm8qNG00Ey5FV9vH0/i9eUUX5M5sC1hIiSILQXvwtN09h40MPSj/fgjmZrP1RqYlPCjd4kLaAlkkaUHn74YRRF4e677060KYLQPKFiXL8hJDDtzeiG46IUqn+TtICWSQpR2rBhA88++ywjR45MtCmC0DJ+d6gNrg2LyYDREMWWfkiUrFKU2yYJF6WGhgZuuOEGnnvuOTIzMxNtjiC0jF8fGKDnKEU5cTlclIu+fPNK/VuLJFyUZs6cyRVXXMHkyZPbPNbr9VJXV9fkJghdhs9FIOQpRZU4Ccfbl2jiKbVFQvOUli9fzubNm9mwYUO7jl+wYAHz5s3rZKsEoQX8rsi8t3bXvYUJiZJZk5hSWyTMUyovL+cXv/gFL730EjZb+5LQ5syZQ21tbeRWXt7OJluCEA/87si8t6jq3iAiSqlmGFmYSr+c1E4w8PQgYZ7Spk2bqKio4Kyzzoo8FwwGWb16NU899RRerxejsek/vNVqxWqNMmlNEOKF30Wa1cSwggLSctOie22ofUma1cQlAx1gTe8cG08DEiZKl1xyCV988UWT52655RbOPPNM7r333lMESRASjt9Nus3M2QN6QXqUohJqX4Lfo5eaiCi1SMJEKT09neHDhzd5LjU1lezs7FOeF4SEo2nHy0zMURTinogpBc3vxutuxKs4caSYpFNAMyR8900QugUBD2gqLl+Aar8JfzCGQLXJiqrBXz7eyeK1ZfhiOUcPIKm6BHz00UeJNkEQmifkJe2s8vHhuv1cObqI/lHHlWwYFLBF0gJUrNHu4vUAxFMShPYQKsb1oG+0RJ08CWC2oaBgM4QTKCVXqTlElAShPYQ8JTcWIEZRCqUF2A36hFyZatI8MYnSN998E287BCG5CXUI8Gi6p9TumW8nEhKllHCpicx/a5aYRGnAgAFcfPHFvPjii3g8nnjbJAjJh+/4wAAAqymGP51wUa5yPKYknEpMorR582ZGjhzJrFmzKCgo4Mc//jGffvppvG0ThOQhNMUkYLBhNRswRNMhIIz5pJ5K4ik1S0yiNHr0aBYuXMjBgwdZvHgxhw4d4rzzzmP48OE8/vjjVFZWxttOQUgsfjeB8BSTWHfMQp5Sjk1jRC8nOWlSndAcHQp0m0wmrr76al555RUeeeQRdu/ezezZsykuLubmm2/m0KFD8bJTEBKL34XZaGBwST4jejtjO0dIlHqlwuSh+VL/1gIdEqWNGzfys5/9jMLCQh5//HFmz55NaWkpK1eu5ODBg1x55ZXxslMQEovfhc1sZFS/Is7pmxXbOUKiJBNNWiem5MnHH3+cJUuWsGPHDqZMmcILL7zAlClTMBh0jevXrx9Lly6lb9++8bRVEBJHuMTEYo/9HKEJKJrfjdcfJKhqpFqTKn85KYjpE1m0aBG33norM2bMoLCwsNlj8vLy+POf/9wh4wQhafA14g0EcflN2APB2DKxQzVzNQ0unv9wJzkOOzd+q0+cDe3+xCRKK1eupKSkJOIZhdE0jfLyckpKSrBYLEyfPj0uRgpCQgkGIOjnQI2bf26q5JwBZiackR39eYy6p2QyKhhVH95AFMMsexAxxZTOOOMMjh49esrz1dXV9OvXr8NGCUJS4W/U71QDQcUSfSvcMAZ9iKXJYMCkeqUlbgvE9OlqWvND+BoaGtrdRVIQug2heJJX0Ru1xVRiEsZkw2RQMKlefAGVYDQDLXsIUS3fZs2aBYCiKMydOxe7/XjQLxgMsn79ekaPHh1XAwUh4fh0T8nTkbq3MObjogR6qYndIsHuE4nq0/jss88A3VP64osvsFgskd9ZLBZGjRrF7Nmz42uhICSakKfkogN1b2FMNhRFiWR1e/0qdksbr+lhRCVKH374IaC3rV24cCEOh6NTjBKEpOKUtiUdSO8LpQXYDVJq0hIx+Y1LliyJtx2CkLz4XaiahrsjvZTCmPS0gL5OI6lZTmny1gztFqWrr76apUuX4nA4uPrqq1s99rXXXuuwYYKQNPhcaMDA3nlkZGXE1iEgTMhTGlVghQH58bHvNKPdouR0OiNNzp3OGGt/BKE74ndhVBSG9ymEXnkdO1d46ICUmrRIu0XpxCWbLN+EHkUopoS5AyUmYSKlJp5IO9wOLQdPQ2LyQ91uNy6XK/J47969PPHEE7z33ntxM0wQkobQZNyagLnjCY+hmNKuA5Us+qiUdd9UxcHA04uYROnKK6/khRdeAKCmpoZx48bx2GOPceWVV7Jo0aK4GigICcfvorrRx/ItR1nx1eGOnSvkKVkiLXGl++TJxNx58vzzzwfg1VdfpaCggL179/LCCy/wv//7v3E1UBASihoEv4dAUMNvsHV8tywUU7JExixJSsDJxCRKLpeL9NDY4vfee4+rr74ag8HAt771Lfbu3dvu8yxatIiRI0ficDhwOBxMmDCBd955JxaTBKFzCCVO+lUIGKwdy1GCSE8lC7ooeaVP9ynEPDjgjTfeoLy8nBUrVvCd73wHgIqKiqgSKnv37s3DDz/Mpk2b2LhxI5MmTeLKK6/kq6++isUsQYg/oSC3T7GAYuh4UDq0fDNrftBUSZ5shphEae7cucyePZu+ffsyfvx4JkyYAOhe05gxY9p9nqlTpzJlyhQGDhzIoEGDePDBB0lLS2PdunWxmCUI8Secza3EocQEIp6SyahgUn2yfGuGmDK6f/CDH3Deeedx6NAhRo0aFXn+kksu4aqrrorJkGAwyCuvvEJjY2NE5E7G6/Xi9R7P76irq4vpWoLQbny6KIWzuVMsHRQlgxGMZoyGIEa/F68/BU3TIjmAQoyiBFBQUEBBQUGT58aNGxf1eb744gsmTJiAx+MhLS2N119/naFDhzZ77IIFC5g3b15M9gpCTEQm48bJUwIwp2D2+xiUbYJ0B5oGoknHiUmUGhsbefjhh3n//fepqKhAVZsG66KZoDt48GC2bNlCbW0tr776KtOnT2fVqlXNCtOcOXMi7VNA95SKi4tjeQuC0D5CDd565Wah5mTgSDF3/JwmK0aDwuSBTsiSUpOTiUmUbr/9dlatWsVNN91EYWFhh1xPi8XCgAEDADj77LPZsGEDCxcu5Nlnnz3lWKvVitUqs7KELiS0fBvQq4AB/TpYYhImlECJX6ZLN0dMovTOO+/w1ltvMXHixHjbg6qqTeJGgpBQfA36vSWOM9rCpSYBvdTEZFAwGTuYanAaEZMoZWZmkpUV4+yrE5gzZw6XX345JSUl1NfXs2zZMj766CNWrFjR4XMLQlzwNRLUNOqDVqy+YMcD3RDZgfvgy71sNaQxdVQRA/LSOn7e04SY5Pm3v/0tc+fObVL/FgsVFRXcfPPNDB48mEsuuYQNGzawYsUKvv3tb3fovIIQN3yNNHoDvLK1iuUb9sXnnGZdlGwEAMnqPpmYPKXHHnuM0tJS8vPz6du3L2Zz0+Df5s2b23UemQsnJDWaBr5GvcTEaCc9XtX8IU/JpoSyuiWBsgkxidK0adPibIYgJCFBH6gBAqqK32yPz9INIoFuK8f7dAvHiUmU7r///njbIQjJR2iKiQ8zqsEUv75HoUC3NVT/JqUmTYk55F9TU8Of/vQn5syZQ3V1NaAv2w4cOBA34wQhoYR23jyK7tnEJXESIp0CIqIknlITYvKUtm7dyuTJk3E6nezZs4c77riDrKwsXnvtNfbt2xfptSQI3ZqQpxS3EpMw4Z5Kmh8UCXSfTEye0qxZs5gxYwa7du1qMhF3ypQprF69Om7GCUJCCYuSon/H4+YphWJKdoOfgflp9M6MQ5vd04iYPKUNGzY0m3Hdq1cvDh/uYGc+QUgWQsu3vOxsRmdmkJsep2qCkKfkMKt8d0ShFL6dREyiZLVam63Q37lzJ7m5uR02ShCSglCJSd/CPPr2jVOJCURSAtA0CHiOTzgRgBiXb9/73veYP38+fr++pakoCvv27ePee+/l+9//flwNFISEEVq+xbXEBMBo0m/oU008/iCapsX3Gt2YmETpscceo6GhgdzcXNxuNxdeeCEDBgwgPT2dBx98MN42CkJi8DWgaRrHAhZcvkB8hcNkQ9U0nvvoaxZ9VCoDBE4gpuWb0+lk5cqVrF27ls8//5yGhgbOOussJk+eHG/7BCFx+BrxqxqvbK3GvbOM/7hkQPzObbJhUBqwaV4a0RMoZf6bTtSipKoqS5cu5bXXXmPPnj0oikK/fv0oKCiQDnrC6YOmgd+FP6jiN9pJsRji+90OxZXshgBV6AmUTuLQq+k0IKrlm6ZpfO973+P222/nwIEDjBgxgmHDhrF3715mzJgRcytcQUg6/C5Qg/iD4DemxC8dIEwouJ1ikKLck4nKU1q6dCmrV6/m/fff5+KLL27yuw8++IBp06bxwgsvcPPNN8fVSEHocrz1APgMVjTFGP+lVSgt4HhRrsSUwkTlKf3lL3/hv/7rv04RJIBJkyZx33338dJLL8XNOEFIGCFR8hj0xMZUa8zt7JsnlECZoug72OIpHScqUdq6dSuXXXZZi7+//PLL+fzzzztslCAknJAouRRdlOJWYhIm7CkRFiXxlMJEJf/V1dXk57fc6Dw/P59jx4512ChBSDgnipIGqZZ4e0p6oDvbqjIoI51MuwS5w0T1SQeDQUymll9iNBoJBAIdNkoQEk6oxCQnK5vR9gwKnbY2XhAloe6TfZxG+owsjO+5uzlRiZKmacyYMaPFiSLS8F84bQh5SiWF+ZQUxrHEJEy41CQ0V044TlSiNH369DaPkZ034bQgJEpY0zvn/OF6t4AHTdPwBVWsJkmehChFacmSJZ1lhyAkF956NDSqAxZSfAFSzMb4Jk+a9QB6XV0dS97fjTPFxIyJ/eJ3/m6MDJsShJMJ+CDgJahqvPTZMZ5d9Q1BNc4FsyFPyaT50FQ/LkkJiCCiJAgnEwpy+zATNFiwmg3xHxZpsoFiwGw0YAp68frV+AtfN0VESRBOxqv3CvOGEiftnVEoqyhgtmEyKlg0fXy3W7wlIMGitGDBAs455xzS09PJy8tj2rRp7NixI5EmCcIJ2dx6HyV7vLO5w5jtKCikGfQESrdPRAkSLEqrVq1i5syZrFu3jpUrV+L3+/nOd75DY2NjIs0SejohUQr35rbHO5s7TCiulGrU699ElHQ66b+A9vHuu+82ebx06VLy8vLYtGkTF1xwQYKsEno8Hn355jKkAZ2QzR0mLEqKDzRZvoVJqCidTG1tLQBZWVkJtkTo0Xj072GDootS3OvewoTSAgrt4E1L7zyPrJuRNKKkqip33303EydOZPjw4c0e4/V6m2SNNze8QBA6TEiU8vLyGB3MoMjZSY39Q57SsFwzwwZKqUmYpBGlmTNn8uWXX7JmzZoWj1mwYAHz5s3rQquEHoemRUSpf+8i+qdkdt61Qp4SflfnXaMbkhQpAXfddRf//Oc/+fDDD+ndu3eLx82ZM4fa2trIrby8vAutFHoEvkZQA/qWvdXRudcKl5r43XqpiTR6AxLsKWmaxs9//nNef/11PvroI/r1az3N3mq1tlgMLAhxIeQlqZY0jjb6SbeCzRzn/txhQp5SRfUxlr2/i14ZKVwztjj+1+lmJFSUZs6cybJly3jzzTdJT0+PTNd1Op2kpMiAPiEBhETJa0rnpXX7MCgKP580oHOG2IY8JXPQgya7bxESunxbtGgRtbW1XHTRRRQWFkZuf/3rXxNpltCTCYmSx6h3B0i1GjEYOmlCT0iULJq+eSN5SjoJX74JQlIREiVXKJs77r25TyS0fDMrQQzBAG4/MqaMJAl0C0LSEM5RQheltM4UJaMFDEZMBgWTqi/hpFe3iJIgNCUkSvVKF4iSooA5BYOikGrQl3Aun7STFlEShDAn5CjVap00WulkQnGl9HBRrgS7kyd5UhASjqdWz1EyGKkJpgCezvWUAMypQCXFaRopaelYTOIniCgJQhh3tX5vy2BwoZOsdBs56ZbOvaZFXyaO62WDEik1ARElQTiOOzSz0J7F8F7OrrmmJVRqEup2KUhMSRCO4wqJUmfWu52MRe9EgK8RVdXwBiSmJKIkCGFCyzev2UlFnQdPVwSdQ8u3g0erePKD3bz9xaHOv2aSI6IkCGFCy7eKYCovrd/Hq5v2d/41Q6JkCbpRNY1Gr3hKIkqCAKAGwV0DQI2mL6kcKebOv65ZFyWbpk/KbfRKnpKIkiCAng6gqWA0URPUO1Gk27pgHyjsKale0FTc/iBqDx+1JKIkCACuKv0+JYv60BLKYesKT8kOioLZCFbNi6ZBYw/P6hZREgSAxkr9PjWXOreeXe3oCk/JYABzCgoKjkipSc+OK4koCQI0EaV6j+6pdElMCSJLOIdRF8OGHh5XkuRJQYCIKAVSsiOi0CXLNwjlKlVSnK5isqWT0hkTebsRIkqCoKrg0nOU1JQcLhgEdZ4ANnMXLSRCntKYfDNjpNREREkQcB/TUwKMZixpmZyd3sVN1kKiJKUmOhJTEoRIPCmHzmnG3QYWvfUu3npUVeuaTPIkRkRJEE4IclfUe6io8+APdmEHSKsuSkerjvK/H+zitc0Huu7aSYiIkiCcIEqflFbx0vp9fH2oC6cvh0TJGmxE06De4++6aychIkqC0HBEv0/L41ijD4BMeyf3UTqRkCjZNDdoKi5fsGs9tSRDREno2fjdkZo3NTWfWreeDuC0d1E6AOgpAYoBk6KRqugJlA2enpurJKIk9GzqQ61CUjKpCxhRNQ2TQSG9s9vgnojBAJZUFBQyTR7dLBElQeih1IeWbun5HHPpsZyMVEvXz14LLeEyjboo1fXguFJCRWn16tVMnTqVoqIiFEXhjTfeSKQ5Qk8k7CmlF3LMpceTMrqqvOREQqLkCC3fRJQSRGNjI6NGjeLpp59OpBlCTyYS5M6PBLmzUrswyB3G5gAg3+rjzIJ0ctKsXW9DkpDQjO7LL7+cyy+/PJEmCD2ZE4LcpBcw1AjOFDNFGSldb4tVF6U+qQH6DOvZpSbdqszE6/Xi9Xojj+vqujCXRDj9qDuo36dkgjmFQicUOhMgSBBZvuGtT8z1k4huFehesGABTqczcisuLk60SUJ3prZcv3f2TqwdEPGU8NSiqhp1Hj+a1jM7UHYrUZozZw61tbWRW3l5eaJNEroztaHBAM7eVDf62HWknlpXggLMKRkAqN56/u+D7fz532U9tq9StxIlq9WKw+FochOEmAgGoC608+YsZteRev659RCffHM0MfaY7WA0YwCyjfoQgZpECWSC6VaiJAhxo+EwqAEwp4A9i6rQzlvCdr0UJeIt5Zh0UQqnKPQ0EhrobmhoYPfu3ZHHZWVlbNmyhaysLEpKShJomXDac8LSDUWhok5PWkzoVrwtAxoqyTK6gJ7rKSVUlDZu3MjFF18ceTxr1iwApk+fztKlSxNkldAjqAkHuYvx+IORbO58hy1xNoU8pUxFFyXxlBLARRdd1GN3GIQEogahZq/+c2ZfjoS8pAy7mRRLAvtjp2QCkI7egbKnekoSUxJ6HrX7IegHix3S8jhUq4tSQSK9JNCXb0CaqotSrdvfIwdTdqvkSaF74g+qBFUNW7JM6ThWpt9n9gNFiXhKBc4Ei5I9CwCrv5bB+ak47VaCmoaBBLToTSAiSkKn4fIF+OfWQxyscaNp+hy10cVORhdnYjQk8A+tOiRKWf0AmDwkn/0FbvIdCa43szrBaMIQDDBlQEpEpHoasnwTOo0Us5GgqhEOG9a5/azeeZRXNpbjStRoal/j8SLczL4ApFpNDC5IJ6Mru002h8EA9mz95/AY8R6IeEpCXNE0LdKLSFEUvjM0H5PRgM1sYNeRBlbvquRQrYdXNu7nmrG9sVu6+Ct4dBdoGqTnH683SybsOVB/BK2xkoa0vrj9QfLSE7ys7GLEUxLiyuZ9NXy8+2hkVzU7zYozxYzVZGR4LyfXn1NCus1EvcfP0foEbHkf3anf5wwG4OPdR/m0rDp5mvWn5gBQcfggf/p3GSu+OpJgg7oe8ZSEuFHd6OPj3UcJqBq9M+2UZNtPOSYz1cJVY3oRVDXyunq3y++BY3v0n3MH4w+qbN53DH9Qo2+OnfSuGtPdGnZdlBxaDQDVDT6CqpbYGFwXI56SEBdUVWPltsMEVI1+OakUZ7XcAiQ7zdr1ggRQtUvPUUrNgdQcyqtd+IMa6TYTucnSVC3kKdm81VhNoGoaVY3eNl50eiGekhAXPiuv4WCNB4vJwKQhea33uFaDeq5QQwVHjx5h54FKzumditkAkai4oujjrG1OvRTEWQyGDqYUHNqq3+cNAeCbykYA+uemdn1P7pZIyQSTBSXgo7e5kdJAKhV13h4VVxJREjpMjcvHJ6V6df0FA3NxtLQM8tTBvnVw5EsIeNE0jf37a1D8Kgc8Nvpmp7Z8EYsdep0NxePBGMMyy1UNNft0sSsYgapqfHNUT1I8Izct+vN1FooC6YVwbC+9zTWUulM5XOtheC9noi3rMkSUhA6haRrvbTuCP6hRkmVneK9m2sloGhzaArv/pbcMAX2kkKOIwsxUPvymgTKjBUdxL70/tqLo3pSvEVxH9TiQzwVl/4ZDn8OZV0S289vNoc/1+8x+YHOy72gjjd4gKRYjvTNPjX0llPQCOLaXAqUG6MWhWneiLepSRJSEDlFR7+Vwrb5smzw0/9RlkBqEHW/D4S/1x85e0Pc8yOgLBgP5gNN0iO2H63m3ysYP+xdjODmoqwahcgd886HubX2+HPpfDMXjdAFri4AXDn6m/1w0GiAylntwfnryBZHTiwDIUfVcpapGHx5/MHky4jsZEaXTAM11DK1yJwZvDRhM4CiC7AGxLXOiJN9h40fjS6hx+XCePJpIDcKXr0HVblAM0P+iZoXkgkG5lFU1cqTOw9YDtYwuzmh6HoMR8ofq72nXCl3gSj/QvahBl7Udazr4mS5M9mzIHgiAzWLEZFAYUpiEjQLTCwCweKr4Vp80shzpGJIl5tUFiCh1UwJBla/3HqJ220rMFV+iamAxGsiwmylw2khNc0C/C6BwdPu8iQ6Qk2Y9tQ+RpukeUtVuMJpg2NWQfUazr0+1mph4Rg4fbK9g7e6jDMhLI625CbUmC5z5XT3msvtfeuDaUwfDrgJzC4FgbwPs/Vj/ueRbetY0cPHgPL7VLxubOQk3oG1O/eapZUK2G7KLEm1Rl5KE/yJCe/how2ccW/0MxiO6INXaelGWOoqt2gA2HA5SW1sLO96FL/8GgfgnKe46Uh8pZG2WvR/rHo1iaFWQwozs7aTQacMXUCNLq2ZRFOg9Fob/QPcEj+2Bz/7f8VFJJ6JpungFvHoGd/7wJr9OsRiTZ9ftRBTleMwsXDzcgxBPqTtydBcT699jh8FHVkEJjtFXYsnqRVWDjy8O1FIb8OPIPghlq/Wyis+XwYhr9C32OFDr8oeC2yrXjC2m18lz0o5s068NMHBym4IEeknKJUPyqaz3MqSwHeUfOQNgzE3wxcvQeBQ2vwAjfqAvXcPs+wQqvtaFceClYDCw80g9Dps58R0B2iKzrx6cP7aXI3UevqlsZHgvR3IkeHYyIkrdBFXVOFDjpji4H7a9QYoJRo05B2XI9/RlDWDPMlGcZUdVNRRDCTiLCW59GVfFPtK3LIPRP+qwMKmqxopth/EFVHplpFB4chJk7X7Y/pb+c/E5+jZ+O8lNt5KbHkUSY3o+nDUdvngFGipg8/+D/GGhLfUyXZABzrgYnL1w+4L86+sjeP0qPzi7N8VZSbbrdiKZffT7hgo+/vIb9jSaSLEYT423nYbI8q0b4A+q/GPrQT5Ys4aqT/+qB5DzhqAMuzoiSCcS3r3SHEW8b7uUzyqCVB4+AJ//RZ8K2wE+Kz/GgWNuLCYDlw4raLpT5j6mLxfVAOQMhP6TYr6Oxx/k8/KatjuT2hww5kbIHQSaCoe/gF3v6YKkGHRBKh4HwL93VeL1q+SkW0/17pINS2pkHt0Qkz40c3try9rTCPGUkhyPP8jfPz9I48HtDKl6HyXPru9EnTk1ErRtCVWDgC2Tr3KvgIp/4A/uo1BZjjLq+pYDw62w/5iLNbv0beoLB+XitJ+wlPC74YtX9Xyi9HwY8r027WsJX0DlxXV7qffoOU2j2vIOTFYY/n2973bldvDW6V0cC0ZCWi4A31Q28NXBOhQFLh6ce2raQTKSPxRq99NP3YtB6cOhWg/VjT49l+s0RjylJMbtC/K3zftpOLiLodX/YmhBKll9R7VLkACMBoXLhxcwdEA/tuVeQWmNyt49pWhb/6oHf6OgzuPnra2HUDWNQfnpDCs6YSs9GNA9pMajejuQ4T9o1oNrLxaTgTElGQCs2lnZ/uTBjGIY+G1doAZcEhGkY42+SLX9mJLM5EuWbIncM0ExYHVXMChVL4nZur8msTZ1ASJKSUq9x88rm8rxVpQy/NhKhhem4SgeBkPaJ0hhFEXhwkG5nDN8ENvyplBer7F79w6Cny2Laim343A9Ll+Q3HQr3z4xSVJVYfs/dC/FZIGR1+pLqg5yVkkmZ+SlEVQ13txykOrG2HYQ6zx+3thyAI8/SKHTxsQzsjtsW5dhSYVcvcXK2QY9PvblgdrENcjrIkSUkhCXL8DyT8vhyDZGHnuPEYWppBaeCUOnxVyUenafLC44azg786/gsMtAWdlu+OxF8Na36/Vj+2QycUAO3xtdhMUU+tqoQfj6TajYrts1/PuQlheTfSejKAqXDssn32HD7Qvy2ub9VNZHXy3/TWUjNS4/jhQzU0cVYTJ2s698ybcAyHWX0semdzVY/011go3qXLrZv1DPwG4ycLb2JaPqVzG8MI2UXsP0BMEOVskPKXQwefwodvf6HrnZ2fpya+MSvVD1JFRV44v9tZF59oqiMK5f1vFiW59L3/UKC9LQK6OvR2sDq8nItDFFZKdZqPcEeHljeeu5Uc0wujiDCwfncs3Y3qQ2l5CZ7KQXQM5AFE3jQjZjMSqnfUxJ0ZJg8NrTTz/NH/7wBw4fPsyoUaN48sknGTduXJuvq6urw+l0Ultbi8ORhOUCUVBZ78WgQDZ1sGsF6rF9qJqGqWQcDJgc16xsjz+ILVCnB6Ybj1Je4+aoYwTG/uejmFOodvnYdaSeek+AMwvSuXxEYdMTHN2tl3t46trM1o6XvX///CD1ngA3fatPxFM7sfUu6AHy3RUNfHmglikjC5vPCu+OuGtgw3MQDOArGotl8LcTbVG7ieVvNOGi9Ne//pWbb76ZZ555hvHjx/PEE0/wyiuvsGPHDvLyWl8KdHdR0jSN/cfcbN1fy4H9ZQxXdzEh9aA+UMdohsGX63k3nUXAh2fb23y+cQ2qBkGDhSNpQ6hK6U+jJQebxcS3+mcxujgDJeDRp4Ac/Oy4Z2XP0j24OC3ZWiOoatS5/WSGvIRAUGXpx3tw2MyYjAouX5CqBh9q6Os8sreTS4bkd7pdXcaRr2Db3/WfC0fBgEs44tKodfsZmJeWnJnpdFNRGj9+POeccw5PPfUUAKqqUlxczM9//nPuu+++Vl/bHUXpYFUdFceOcayqiurKgxgbK3B69mML1JGdauGMvDSMeYPhjEmRiamdSVDVOFS2Dc+2dwk2VqFpGlaTgTS7nazsHIwmsx4Q99Yfb8BmMOqlHn0m6tvxCaC82sWrm/af8rwzxcywIgcjeju7fihBZ7N/o142o2n4DFY+OJbLIUMBKZmF9C/KpzAzlQy7mVSLKWlSHrqdKPl8Pux2O6+++irTpk2LPD99+nRqamp48803mxzv9Xrxeo8HO+vq6iguLm77DVeXwe5/seNIAzUuHxB6y6G3rt9pjO+XFRn7t6uijqqG8I6PPiZICb8OjbF9sjAZFEBjd0UDlfUnxjq00CX0143rl6kfGwyw+/AxKhuO7yQZDQo5aRbynKmk9R6mZ0CHkua6FE3Ti2cPb9U/r2AzjfTt2fpuUNFovWA0gaiqRkW9l3qPH19QJcVsjAwpOK2p/gZ2rURtrOJAjZtDNW6Coa9l0GBBVYxoipF8p50zCzNAUQhqGhvKjkVOcbJTlZFqYXDe8UZ36/dUR/5ETp6DOTg/nYzwZzzyujZ3WmMRpYT+V3L06FGCwSD5+U3d7Pz8fLZv337K8QsWLGDevHnRXyjghcajaA11qK3NZ3cR+RfTXPVo7qbb0E3U22uKbM0rvgbwNb8zpAABrxWTSQ9SO1LMBFGw2h04sgtw5hRhzOyjlxUkyOvQDVX0LOycgfo2v6cGPLV6lrTJqicjWpOnQ6PBoFDgtCV/DVu8yeoP59yOofobiit3kF9zkOrKQ9S5vDR4A/gCPjTA5PWDKwiAomlojcd37E7xQjQLpB7/T1Wrrzr1mDBpHlBDgXZNjde7akK38m/nzJnDrFmzIo/DnlKbZBTDqB+SM9BHuk/TdeeE/y4iPzpTAAUUhTyXH2dQ/9CV0HNK6GAFMKRaIeQi53sDOAPq8Xln4Vcp+s1sM+sCZjCQZ7aTZ7R0ejuRDmEw6PGiHjqhNekxGCP/gViAAk2jwO8Gvxs16Mfj8+merwlAQ9E0+vRrOZ3CZjbACTt6xX3CAnWqNNlSLRBu9xKnAu+TSago5eTkYDQaOXKk6WyrI0eOUFBQcMrxVqsVqzUGb8KSCln9yInibywjitVJWjokjw8h9DgURe9hbrFjAE7OV1eAoii++706P5TZKgnNU7JYLJx99tm8//77kedUVeX9999nwoQJCbRMEIREkfDl26xZs5g+fTpjx45l3LhxPPHEEzQ2NnLLLbck2jRBEBJAwkXpuuuuo7Kykrlz53L48GFGjx7Nu+++e0rwWxCEnkHC85Q6QnfMUxKEnkQsf6NS+yYIQlIhoiQIQlIhoiQIQlKR8EB3RwiHw+rqekbvYkHoboT/NqMJXXdrUaqv1xuUtSurWxCEhFFfX4/T2b6M5G69+6aqKgcPHiQ9PT1pWzd0NuFSm/LyctmBjAH5/DpGW5+fpmnU19dTVFSEoZ1tnLu1p2QwGOjdOwEV9UmIw+GQP6oOIJ9fx2jt82uvhxRGAt2CICQVIkqCICQVIkrdHKvVyv333x9b9wRBPr8O0hmfX7cOdAuCcPohnpIgCEmFiJIgCEmFiJIgCEmFiJIgCEmFiFI3pLq6mhtuuAGHw0FGRga33XYbDQ0Nrb7moosuigwyCN9+8pOfdJHFieXpp5+mb9++2Gw2xo8fz6efftrq8a+88gpnnnkmNpuNESNG8Pbbb3eRpclJNJ/f0qVLT/me2WxRTpzRhG7HZZddpo0aNUpbt26d9u9//1sbMGCAdv3117f6mgsvvFC74447tEOHDkVutbW1XWRx4li+fLlmsVi0xYsXa1999ZV2xx13aBkZGdqRI0eaPX7t2rWa0WjUfv/732vbtm3Tfv3rX2tms1n74osvutjy5CDaz2/JkiWaw+Fo8j07fPhwVNcUUepmbNu2TQO0DRs2RJ575513NEVRtAMHDrT4ugsvvFD7xS9+0QUWJhfjxo3TZs6cGXkcDAa1oqIibcGCBc0ef+2112pXXHFFk+fGjx+v/fjHP+5UO5OVaD+/JUuWaE6ns0PXlOVbN+OTTz4hIyODsWPHRp6bPHkyBoOB9evXt/ral156iZycHIYPH86cOXNwuVydbW5C8fl8bNq0icmTJ0eeMxgMTJ48mU8++aTZ13zyySdNjge49NJLWzz+dCaWzw+goaGBPn36UFxczJVXXslXX30V1XW7dUFuT+Tw4cPk5eU1ec5kMpGVlcXhw4dbfN2PfvQj+vTpQ1FREVu3buXee+9lx44dvPbaa51tcsKIdgIz6J9vc8e39tmersTy+Q0ePJjFixczcuRIamtrefTRRzn33HP56quv2l08L6KUJNx333088sgjrR7z9ddfx3z+O++8M/LziBEjKCws5JJLLqG0tJQzzjgj5vMKwolMmDChyczGc889lyFDhvDss8/y29/+tl3nEFFKEv7zP/+TGTNmtHpM//79KSgooKKiosnzgUCA6urqZqcKt8T48eMB2L1792krStFOYAYoKCiI6vjTmVg+v5Mxm82MGTOG3bt3t/u6ElNKEnJzcznzzDNbvVksFiZMmEBNTQ2bNm2KvPaDDz5AVdWI0LSHLVu2AFBYWBjvt5I0xDKBecKECU2OB1i5cmWPnNgcjwnWwWCQL774IrrvWYfC5EJCuOyyy7QxY8Zo69ev19asWaMNHDiwSUrA/v37tcGDB2vr16/XNE3Tdu/erc2fP1/buHGjVlZWpr355pta//79tQsuuCBRb6HLWL58uWa1WrWlS5dq27Zt0+68804tIyMjsk190003affdd1/k+LVr12omk0l79NFHta+//lq7//77e3xKQDSf37x587QVK1ZopaWl2qZNm7Qf/vCHms1m07766qt2X1NEqRtSVVWlXX/99VpaWprmcDi0W265Rauvr4/8vqysTAO0Dz/8UNM0Tdu3b592wQUXaFlZWZrVatUGDBig/fKXv+wReUqapmlPPvmkVlJSolksFm3cuHHaunXrIr+78MILtenTpzc5/uWXX9YGDRqkWSwWbdiwYdpbb73VxRYnF9F8fnfffXfk2Pz8fG3KlCna5s2bo7qetC4RBCGpkJiSIAhJhYiSIAhJhYiSIAhJhYiSIAhJhYiSIAhJhYiSIAhJhYiSIAhJhYiS0K1YunQpGRkZkccPPPAAo0ePjjyeMWMG06ZN63K7hPghoiQ0y4wZM1psmTtz5kwURWlSQNwZYtC3b1+eeOKJJs9dd9117Ny5s8XXLFy4kKVLl0YeX3TRRdx9991xtUvoXESUhBYpLi5m+fLluN3uyHMej4dly5ZRUlKSEJtSUlJO6Sd1Ik6ns4knJXQ/RJSEFjnrrLMoLi5u0gjutddeo6SkhDFjxnTo3M15MNOmTYt4XxdddBF79+7lnnvuiTSgh1OXbydzosc2Y8YMVq1axcKFCyPnKCsrY8CAATz66KNNXrdlyxYURYmqxYbQOYgoCa1y6623smTJksjjxYsXc8stt3T6dV977TV69+7N/PnzOXToEIcOHYr6HAsXLmTChAnccccdkXOUlJSc8p4AlixZwgUXXMCAAQPi9RaEGBFRElrlxhtvZM2aNezdu5e9e/eydu1abrzxxk6/blZWFkajkfT0dAoKCmJqsuZ0OrFYLNjt9sg5jEYjM2bMYMeOHZFRQX6/n2XLlnHrrbfG+20IMSCdJ4VWyc3N5YorrmDp0qVomsYVV1xBTk5Oos3qEEVFRVxxxRUsXryYcePG8Y9//AOv18s111yTaNMExFMS2sGtt97K0qVLef755+PmTRgMBk7umuP3++Ny7vZw++23R4L4S5Ys4brrrsNut3fZ9YWWEVES2uSyyy7D5/Ph9/u59NJL43LO3NzcJnGiYDDIl19+2eQYi8VCMBjs0HVaOseUKVNITU1l0aJFvPvuu7J0SyJk+Sa0idFojExSMRqNLR5XW1sb6f0dJjs7m+Li4lOOnTRpErNmzeKtt97ijDPO4PHHH6empqbJMX379mX16tX88Ic/xGq1xrRs7Nu3L+vXr2fPnj2kpaWRlZWFwWCIxJbmzJnDwIEDe2QP7mRFPCWhXTgcDhwOR6vHfPTRR4wZM6bJbd68ec0ee+uttzJ9+nRuvvlmLrzwQvr378/FF1/c5Jj58+ezZ88ezjjjDHJzc2Oye/bs2RiNRoYOHUpubi779u2L/O62227D5/N1yW6i0H6kHa7QY/n3v//NJZdcQnl5+SkDF4XEIaIk9Di8Xi+VlZVMnz6dgoICXnrppUSbJJyALN+EHsdf/vIX+vTpQ01NDb///e8TbY5wEuIpCYKQVIinJAhCUiGiJAhCUiGiJAhCUiGiJAhCUiGiJAhCUiGiJAhCUiGiJAhCUiGiJAhCUiGiJAhCUvH/AearL2BfN0wtAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:39:16.421404Z", + "iopub.status.busy": "2024-02-29T03:39:16.420676Z", + "iopub.status.idle": "2024-02-29T03:39:16.646712Z", + "shell.execute_reply": "2024-02-29T03:39:16.645882Z" + }, + "papermill": { + "duration": 0.248502, + "end_time": "2024-02-29T03:39:16.648701", + "exception": false, + "start_time": "2024-02-29T03:39:16.400199", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3rklEQVR4nO3deVwTd/4/8FfCEUAIiIB4IAFRsRWlxWrRKlq5xPLVetQqVkVXe9FtxaPa71a/1v0udavW1nXrV7dqvd0KulZFiSJK0XrgsYiIoqKWQzmECEgI4fP7w19miYGQSCDHvJ+Phw8zM++ZeU8mefOZI58RMMYYCCHEwgmNnQAhhLQHKnaEEF6gYkcI4QUqdoQQXqBiRwjhBSp2hBBeoGJHCOEFKnaEEF6wNnYCpq6hoQGFhYVwcnKCQCAwdjqEkEYYY3jy5Am6du0KoVB7242KXQsKCwvh5eVl7DQIIVo8ePAA3bt31xpDxa4FTk5OAJ69mWKx2MjZtB2FQoGUlBSEh4fDxsbG2OmQVuLL/pTJZPDy8uK+p9pQsWuB6tBVLBZbfLFzcHCAWCy26C8HX/Btf+pyiokuUBBCeIGKHSGEF6jYEUJ4gYodIYQXqNgRQniBih0hhBfo1hNCLEBNTQ1u3LjBDVc9leNM1m10dLsIR3uRWqy/vz8cHBzaO0Wjo2JHiAW4ceMGgoKCNMb/tYnYzMxMvPrqq22flImhYkeIBfD390dmZiY3nFtUgfifs7BmUgD6dHHRiOUjKnY8pethD18PecyNg4ODWmtNeK8MovSn6NtvAAK9OxkxM9NBxY6ndD3s4eshD7E8VOx44m5pNarl9dywUtwFe5PTuOF7pVVYczwP8aF+8HZzVIu7VlDJDXcQWcPHrUO75EyIIVGx44G7pdUYuSqtxTiRpx/WXwOAqv+MTLukEXdywQgqeMTsULHjgfKaKgjtCrAgrDe8XJs+//ZUXof0i1kYNjAA9iLbJmMelNdglfQmymuq4AMqdsb2fGu9sdsl1dz/1tbNf8351FKnYscDhdX30MFnHX7IayHQBki7qj2kgw9QWB2IIHQ2WH5Ef7q21ufvy2oxhi8tdSp2PNC1gzeq736C7yYHoqeHY5Mx9fX1yPg1A0PfGNpsS+D2oyp8uvcKuo70bst0iQ5ULbq1kwPh18Q+rX4qx6G0s3hrRDA6PHdTsUreoyp8tvdKs61DS0PFjgdEVnZoqO0GH3EfvNTJuckYhUKBu9Z30de1b7OdPTbUVqKhtgQiK7u2TJfowc/DEf26ae5ThUKBYnfgVe+OvOi8UxdU7AgxQ3JlLYR2Bbgry4XQTrNlV19fj8L6QuSU5zTbUr8re3YuV66sBdD0H0FLQsWOEDOkOg/7xXntcX8/+net0/l0Dtbsit369evxzTffoLi4GAMGDMC6deswaNCgJmOzs7OxdOlSZGZm4t69e/j222/x2WeftW/ChLSBls7D0jlYTWZV7Pbu3Yv4+Hhs2LABgwcPxtq1axEREYHc3Fx4eHhoxNfU1MDX1xeTJk3CvHnzjJCxaXiqUAKA2s3Bz6t+KsfFEsDz3mOtJ7SJaWjpPCydg9VkVsVuzZo1mDNnDmJjYwEAGzZswOHDh7F582YsXrxYI/61117Da6+9BgBNTueL2/+/SC1Oauk2BGtsz7vQ4vI6iMzqY0MIADMqdnV1dcjMzMSSJUu4cUKhEKGhoTh79qwRMzN94S97AgB6ejjC3saqyZjcokrM35eF1RMD0KdL8yer+XQTKrEsZlPsSktLoVQq0bmz+onUzp07q/Xe0VpyuRxyuZwblslkAJ4dFigUCoOtpz052Qow4ZUuWmNqa2sBAN4dRejjob2XE3N9HyxJfX09939T+0M1Ttu+amkZ5kCfvM2m2LWXhIQELF++XGN8SkqKRXd19KAKAKzx22+/oeCasbMhLVHtr19//RX3mr5PHAAglUpbvQxTVlNTo3Os2RQ7Nzc3WFlZ4eHDh2rjHz58CE9PT4OtZ8mSJYiPj+eGZTIZvLy8EB4eDrFYbLD1mJqr98uBrIt4/fXXMaCHq7HTIS3ILpRhVdZveOONN/ByV83PpUKhgFQqRVhYWLMXKFpahjlQHXnpwmyKna2tLYKCgnDixAmMGzcOANDQ0IATJ04gLi7OYOsRiUQQiTSvRtrY2Fj0neiq2xOsra0tejstha77S9vn1hL2uT55m02xA4D4+HjMmDEDAwcOxKBBg7B27VpUV1dzV2enT5+Obt26ISEhAcCzixrXr1/nXhcUFODKlStwdHSEn5+f0baDENL+zKrYTZ48GSUlJVi6dCmKi4sRGBiIo0ePchct7t+/D6HwP0+HLCwsxCuvvMINr1q1CqtWrUJISAjS0tLaO31CiBGZVbEDgLi4uGYPW58vYBKJBIyxdsjK/Dz/DIrcogrIi/OQc80eDWUu3Hh6BgWxFGZX7IhhNPcMiqk/qQ/TMyiIpaBix1ONH72nVCpx5rfzOJ5+FqHDgjHk9UGwsrLi4gixBFTseEr16L2kpCTMnz8f+fn5AIBDP++ERCLB6tWrMX78eOMmSYgBUbHjsaSkJEycOBFjxozBvHnzcOvWLfTq1QtSqRQTJ07Evn37qOARi0HFjqeUSiXmz5+PoKAgZGVl4dChQ9w0b29vBAUFYcGCBRg7dix3SEuIOaNix1Pp6enIz89Hfn4+7O3t1aY9evQI9+7d4+JGjBhhhAyJNi1120VddmmiYsdTBQUF3OuQkBBUVVXh3r178Pb2hqOjI44ePaoRR0yHbt12UZddjfFjK4mG4uJiAIC9vT1X2ADgwYMH3PinT59yccS0tNRtF3XZpYmKHU+Vl5cDAJ4+fdrkdNV4VRwxLa4dbPHuoB7NTld139TTvUOTTx/jI2HLIcQSqb4MhoojxNRRseOpU6dOGTSOEFNHh7E8dfPmTe61m5sbRowYgfLycri6uiItLQ2lpaUacYSYMyp2PKVUKrnXZWVl2LdvHzcsEAiajCPEnNFhLE/5+vpyr5/vGabxcOM4QswZFTueCg0NNWgcIaaOih1PPf+UttbGEWLqqNjxVFlZmUHjCDF1VOx46vfff+de29nZqU1rPNw4jhBzRsWOp3r0eHb3fe/evTUOVT09PdG7d2+1OELMHRU7nnrzzTcBPLuPrl+/fvjuu+8QFxeH7777Di+//DJ3f50qjhBzR/fZ8dSIESPg4eGBR48eITU1FYcPH+amqbp88vDwoO6diMUwu5bd+vXrIZFIYGdnh8GDB+P8+fNa43/++Wf4+/vDzs4OAQEBOHLkSDtlatqsrKzwww8/ANDsDEA1/MMPP1DHncRimFWx27t3L+Lj47Fs2TJcunQJAwYMQEREBB49etRk/JkzZzBlyhTMnj0bly9fxrhx4zBu3Dhcu3atnTM3bY1/MdHUMCGWQMDM6MGqgwcPxmuvvYa//e1vAICGhgZ4eXnhk08+weLFizXiJ0+ejOrqarUux19//XUEBgZiw4YNOq1TJpPB2dkZlZWVEIvFhtkQE6BUKtGlSxeUlJQgKioKvr6+uHnzJnr37o07d+7gyJEj8PDwQGFhIbXuzEBTzwGO/zkLayYFoE8XF7VYS3oWsD7fT7M5Z1dXV4fMzEwsWbKEGycUChEaGoqzZ882Oc/Zs2cRHx+vNi4iIgIHDhxodj1yuRxyuZwblslkAACFQgGFQtGKLTAtqampKCkpgb+/P7Kzs7nD+5SUFHh7e6NPnz7Izc3F8ePH6SKFGbh27RoGDx6sMf755wADwLlz5/DKK6+0Q1ZtT5/vpNkUu9LSUiiVSo3bJDp37qz2F62x4uLiJuO19b6bkJCA5cuXa4xPSUmxmL+GALBz504AQG5uLoKCghAREQFbW1vU1dXh0qVL3DNlf/zxR9TW1hozVaIDuVyO1atXc8OKBqC8FnC1A2yeO1mVn5+PoqKids6wbdTU1OgcazbFrr0sWbJErTUok8ng5eWF8PBwizqMVbWG/fz8UFJSgo0bN3LTvL294efnh1u3bsHX1xdRUVHGSpO8IIVCAalUirCwMNjY2Bg7nTajOvLShdkUOzc3N1hZWeHhw4dq4x8+fAhPT88m5/H09NQrHgBEIhFEIs2nMdnY2FjUh8bd3R0AcOvWLYwZMwbx8fHcc2NTUlK4W1Hc3d0tarv5xtI+t8/TZ9vMptjZ2toiKCgIJ06cwLhx4wA8u0Bx4sQJxMXFNTlPcHAwTpw4gc8++4wbJ5VKERwc3A4ZmzYPDw/u9fHjx9Xus2tc7BvHEWLOzKbYAUB8fDxmzJiBgQMHYtCgQVi7di2qq6sRGxsLAJg+fTq6deuGhIQEAMCnn36KkJAQrF69GmPGjMGePXtw8eJFtUM2vmr8A//nT/I2fu4EdQRALIVZFbvJkyejpKQES5cuRXFxMQIDA3H06FHuIsT9+/chFP7nbOyQIUOwa9cu/OlPf8IXX3yBXr164cCBA+jXr5+xNsFkdOrUCQAgFovRsWNH7qHYAODl5YXy8nLIZDIujhBzZ1bFDgDi4uKaPWxNS0vTGDdp0iRMmjSpjbMyP6oW25MnTzB8+HC1c3ZSqZQ7rKWWHbEUZlfsiGGoLlAEBgbi2rVrajde+/j4IDAwEJcvX+biCDF3VOx4qlu3bgCAK1euYMyYMZg3b16TLTtVHCHmjoodTw0bNgwSiQRubm5NtuyCgoJQVlaGYcOGGTFLQgyHih1PWVlZYfXq1Zg4cWKzLbt9+/bR72KJxaBix2Pjx4/Hvn37MH/+fI2W3b59+zB+/HgjZkeIYVGx47nx48dj7NixOHnyJJKTkzF69GiMHDmSWnTE4lCxI7CyskJISAiqq6sREhJChY5YJLPqvJMQQl4UFTtCCC9QsSOE8AIVO0IIL1CxI4TwAhU7QggvULEjhPACFTtCCC9QsSOE8AIVO0IIL1CxI4TwAhU7Qggv6F3s7ty50xZ5EEJIm9K72Pn5+WHkyJHYsWMHamtr2yInQggxOL2L3aVLl9C/f3/Ex8fD09MT77//Ps6fP98WuakpLy9HTEwMxGIxXFxcMHv2bFRVVWmdZ+PGjRgxYgTEYjEEAgEqKiraPE9CiGnSu9gFBgbiu+++Q2FhITZv3oyioiK88cYb6NevH9asWYOSkpK2yBMxMTHIzs6GVCrFoUOHcPr0acydO1frPDU1NYiMjMQXX3zRJjkRQswIa6Xa2lq2Zs0aJhKJmEAgYCKRiL333nussLCwtYvmXL9+nQFgFy5c4MYlJyczgUDACgoKWpz/5MmTDAB7/Pix3uuurKxkAFhlZaXe85qTuro6duDAAVZXV2fsVIgB8GV/6vP9fOGeii9evIjNmzdjz5496NChAxYsWIDZs2fj999/x/LlyzF27FiDHd6ePXsWLi4uGDhwIDcuNDQUQqEQ586dw9tvv22Q9QCAXC6HXC7nhmUyGQBAoVBAoVAYbD2mRrVtlryNfMKX/anP9uld7NasWYMtW7YgNzcXUVFR2LZtG6KioiAUPjsi9vHxwdatWyGRSPRddLOKi4vh4eGhNs7a2hqurq4oLi422HoAICEhAcuXL9cYn5KSAgcHB4OuyxRJpVJjp0AMyNL3Z01Njc6xehe7H374AbNmzcLMmTPRpUuXJmM8PDzw448/trisxYsXY+XKlVpjcnJy9E2xVZYsWYL4+HhuWCaTwcvLC+Hh4RCLxe2aS3tSKBSQSqUICwuDjY2NsdMhrcSX/ak68tKF3sVOKpWiR48eXEtOhTGGBw8eoEePHrC1tcWMGTNaXNb8+fMxc+ZMrTG+vr7w9PTEo0eP1MbX19ejvLwcnp6e+m6CViKRCCKRSGO8jY2NRX9oVPiynXxh6ftTn23Tu9j17NkTRUVFGoeV5eXl8PHxgVKp1HlZ7u7ucHd3bzEuODgYFRUVyMzMRFBQEAAgNTUVDQ0NGDx4sH4bQAjhJb1vPWGMNTm+qqoKdnZ2rU6oKX379kVkZCTmzJmD8+fPIyMjA3FxcXj33XfRtWtXAEBBQQH8/f3VLooUFxfjypUryMvLAwBkZWXhypUrKC8vb5M8CSGmS+eWneo8lkAgwNKlS9VO1iuVSpw7dw6BgYEGT1Bl586diIuLw6hRoyAUCjFhwgR8//333HSFQoHc3Fy1E5YbNmxQu9gwfPhwAMCWLVtaPHwmxFzV1dVh3bp1SE1NRV5eHj755BPY2toaOy2jE7DmmmrPGTlyJADg1KlTCA4OVnvzbG1tIZFIsGDBAvTq1attMjUSmUwGZ2dnVFZWWvwFiiNHjiAqKsqiz/FYukWLFmHNmjVqp5OsrKwQHx+Pv/71r0bMrG3o8/3UuWV38uRJAEBsbCy+++47i/7iE2KOFi1ahG+++UZjvFKp5MZbYsHTlc4tO76ilh0xB3V1dbCzs2v2nDrw7BRUbW2tRR3SGrxlN378eGzduhVisRjjx4/XGpuUlKR7poQQg/j++++1Fjrg2cXF77//HgsWLGinrEyLTldjnZ2dIRAIuNfa/hFC2t+uXbvUhiMiIvD1118jIiJCaxyf0GFsC+gwlpgDVWMEAKqrq2FjY8PtT4VCgQ4dOnDTLekrr8/3k7plJ8TCPH9OzpLO0bWGTufsXnnlFbW/HNpcunSpVQkRQlqne/fuWLZsGezs7PCPf/yjyY4t+EinYjdu3Lg2ToMQ0hqBgYG4cuUKAODhw4f46KOPmo3jK52K3bJly9o6D0JIK6SlpcHFxUWnOL6ic3aEWABnZ2f07NlTa0zPnj15fceETi07V1dX3Lx5E25ubujYsaPW83f0I3tCjCMvLw9+fn64ffu2xrSePXtyHWLwlU7F7ttvv4WTkxP3WteLFYSQ9pWXl4fKykqMHj0at27dQq9evZCcnMzrFp2KTsWucUec1FsIIabN2dkZp06dovsmn6P3OTsrKyuNXoMBoKysDFZWVgZJihBCDM1gnXfK5XK6eZEQYrJ07uJJ1VGmQCDAP/7xDzg6OnLTlEolTp8+DX9/f8NnSAghBqBzsfv2228BPGvZbdiwQe2QVdV554YNGwyfISGEGIDOxe7u3bsAnvVYnJSUhI4dO7ZZUoQQYmh6P11M1WMxIYSYE72L3axZs7RO37x58wsnQwghbUXvYvf48WO1YYVCgWvXrqGiogJvvvmmwRIj7UepVOLUqVM4ffo0OnTogJEjR9JtRMTyMANQKpVs7ty5bOXKlYZYXJPKysrY1KlTmZOTE3N2dmazZs1iT5480RofFxfHevfuzezs7JiXlxf75JNPWEVFhV7rraysZABYZWVlazfBJCUmJjKJRMIAcP8kEglLTEw0dmqkFerq6tiBAwdYXV2dsVNpU/p8Pw3SEYBQKER8fDx3xbYtxMTEIDs7G1KpFIcOHcLp06cxd+7cZuMLCwtRWFiIVatW4dq1a9i6dSuOHj2K2bNnt1mO5iYpKQkTJ05EQEAA0tPTsXv3bqSnpyMgIAATJ06k54kQy2KoCnv48GHm5uZmqMWpuX79OgPALly4wI1LTk5mAoGAFRQU6Lycf/7zn8zW1pYpFAqd57HUll19fT2TSCQsOjqaKZVKtZaAUqlk0dHRzMfHh9XX1xs7VfICqGWnSe9zdvHx8c8XSxQVFeHw4cNqv6E1pLNnz8LFxQUDBw7kxoWGhkIoFOLcuXN4++23dVqOqp96a+vmN1sul0Mul3PDMpkMwLNzkwqF4gW3wPScOnUK+fn52L59O8rLyxEdHc31mvHLL79g4cKFGD58OE6ePImQkBBjp0v0pPqsWtJntin6bJ/exe7y5ctqw0KhEO7u7li9enWLV2pfVHFxMTw8PNTGWVtbw9XVFcXFxToto7S0FCtWrNB66AsACQkJTXZjnZKSAgcHB92TNnGnT58GAEycOBEPHz7kxpeWlsLd3Z17v5OTk1FdXW2UHEnrSaVSY6fQpmpqanSONep9dosXL8bKlSu1xuTk5LR6PTKZDGPGjMFLL72E//mf/9Eau2TJErXWq0wmg5eXF8LDwy3q6WIdOnTAmjVruEI3cOBA+Pn5IS8vDxcvXuQ6exg9ejS17MyMUqlEWloapFIpwsLCMGLECIu9uq468tKF3sXOkObPn99il1G+vr7w9PTU6Gmlvr4e5eXl8PT01Dr/kydPEBkZCScnJ+zfv7/F7m5EIhFEIpHGeBsbG4vqKufVV1/lXvfo0QMXL17ExYsXueH79+9zcZa03ZYuKSkJ8fHxuHfvHgBgzZo18Pb2xpo1a1p8wL050uuz2fanEFtPdYHi4sWL3Lhjx461eIGisrKSvf766ywkJIRVV1e/0Lot9QLF0KFD1W43ae7f0KFDjZ0q0VFiYqLWfWmJtxO1+60nba1v376IjIzEnDlzcP78eWRkZCAuLg7vvvsuunbtCgAoKCiAv78/zp8/D+BZ8zY8PBzV1dX48ccfIZPJUFxcjOLiYiiVSmNujklQ/eU3VBwxLqVSidjYWK0xsbGxvP7sm0WxA4CdO3fC398fo0aNQlRUFN544w1s3LiRm65QKJCbm8udsLx06RLOnTuHrKws+Pn5oUuXLty/Bw8eGGszTIau5x8t6TylJTtx4kSL569kMhlOnDjRThmZHoOds/v999/x1VdfqRUgQ3J1dcWuXbuanS6RSNQ6Fh0xYkSzHY2SZ63l69ev6xRHTJ+uv0nfvHkzwsPD2zgb02Swll1ZWRl+/PFHQy2OtDFdCp0+ccS4zpw5Y9A4S2TUq7HEeIqKigwaR4yr8d0Kbm5uGDFiBMrLy+Hq6oq0tDSUlpZqxPENFTsCgUCgdsj//DAxfXV1ddzr0tJS7Nu3r8U4vjGbCxTEsBrfZPp8YWs8bKk3o1oaoVC3r7KucZZI55ZdSzckVlRUtDYX0o6cnZ1RVlamUxwxfZ6enigoKNApjq90LnYtfeidnZ0xffr0VidE2gfdemJZBgwYoFOxGzBgQDtkY5p0LnZbtmxpyzxIO3NxcTFoHDEubT35vEicJeLvATzPqa7OGSqOGBe11Fumc5nXtfsmeuCOeaAvh2V57733sGPHDtjZ2aGurg4NDQ3cNKFQCFtbW9TW1uK9994zYpbGpXOx27p1K7y9vfHKK6/QbQkWICAgQKcbTAMCAtohG9Jao0aNglgshkwmg4eHB4YNG4bHjx+jY8eOSE9Px6NHjyAWizFq1Chjp2o0Ohe7Dz/8ELt378bdu3cRGxuLadOmwdXVtS1zI21o8ODB+L//+z+d4ojps7KywpYtWzBhwgSUlJQgMTGRmyYQCAA8O+/O51uJdD5nt379ehQVFWHRokX45Zdf4OXlhXfeeQfHjh2jlp4ZOnv2rEHjiPGNHz8eiYmJ8PLyUhvfo0cPJCYmWmR/dvoQsBesVPfu3cPWrVuxbds21NfXIzs7G46OjobOz+hkMhmcnZ2551dYipdfflmn372+9NJLyM7OboeMiKEolUqcPHkSycnJGD16tEU/B1if7+cLX4cWCoXcz4r43EeWuaqsrOReC4VCjRPaquHGccQ8WFlZISQkBNXV1QgJCbHYQqcvvW49kcvl2L17N8LCwtC7d29kZWXhb3/7G+7fv2+RrTpLZmdnx71uXOieH24cR4g507ll99FHH2HPnj3w8vLCrFmzsHv3bri5ubVlbqQNubu74/bt29xw79694erqivLycty8eVMtjhBLoHOx27BhA3r06AFfX1+cOnUKp06dajKOniJvHuzt7dWGGxc4bXGEmCudi9306dO5S9jE/OnacQN18EAshV43FRPLoesfLvoDRywF/TaWp7p168a9fr6gNR5uHEeIOaNix1MTJkzgXj9/xbXxcOM4QsyZ2RS78vJyxMTEQCwWw8XFBbNnz0ZVVZXWed5//3307NkT9vb2cHd3x9ixY3Hjxo12yti0eXt7c6+fPn2qNq3xcOM4QsyZ2RS7mJgYZGdnQyqV4tChQzh9+jTmzp2rdZ6goCBs2bIFOTk53M/awsPD6SZoAMOGDYNEIkHnzp2bnN65c2f4+Phg2LBh7ZwZIW2EmYHr168zAOzChQvcuOTkZCYQCFhBQYHOy7l69SoDwPLy8nSep7KykgFglZWVeuVsDhITE5lAIGBRUVFs3LhxLCAggI0bN45FRUUxgUDAEhMTjZ0ieUF1dXXswIEDrK6uztiptCl9vp9m0W3p2bNn4eLigoEDB3LjQkNDIRQKce7cObz99tstLqO6uhpbtmyBj4+Pxg+lG5PL5ZDL5dyw6inrCoUCCoWiFVtheqKjo7Fnzx58/vnnyM/PBwBkZWXBx8cHe/bsQXR0tMVtM1+o9pul7z99ts8sil1xcTE8PDzUxllbW8PV1RXFxcVa5/373/+ORYsWobq6Gn369IFUKoWtrW2z8QkJCVi+fLnG+JSUFDg4OLzYBpgwkUiE1atX4/r161z/Zy+99BKsrKxw5MgRY6dHWkkqlRo7hTZVU1Ojc6xRi93ixYuxcuVKrTE5OTmtWkdMTAzCwsJQVFSEVatW4Z133kFGRkazv/lcsmQJ4uPjuWGZTAYvLy+Eh4dbVK8nz4uMjIRUKkVYWBhsbGyMnQ5pJYVCwYv9qTry0oVRi938+fMxc+ZMrTG+vr7w9PTUeJJ5fX09ysvLW3w0nLOzM5ydndGrVy+8/vrr6NixI/bv348pU6Y0GS8SiSASiTTG29jYWPSHRoUv28kXlr4/9dk2oxY7d3d3nX5oHhwcjIqKCmRmZiIoKAgAkJqaioaGBr160mWMgTGmdk6OEMIPZnHrSd++fREZGYk5c+bg/PnzyMjIQFxcHN5991107doVAFBQUAB/f3+cP38eAHDnzh0kJCQgMzMT9+/fx5kzZzBp0iTY29sjKirKmJtDCDECsyh2ALBz5074+/tj1KhRiIqKwhtvvIGNGzdy0xUKBXJzc7kTlnZ2dkhPT0dUVBT8/PwwefJkODk54cyZMxoXOwghls8srsYCgKurK3bt2tXsdIlEovYsjK5du9LVREIIx2xadoQQ0hpU7AghvEDFjhDCC1TsCCG8QMWOEMILVOwIIbxAxY4QwgtU7AghvEDFjhDCC1TsCCG8QMWOEMILVOwIIbxAxY4QwgtU7AghvEDFjhDCC1TsCCG8QMWOEMILVOwIIbxAxY4QwgtU7AghvGA2xa68vBwxMTEQi8VwcXHB7NmzUVVVpdO8jDGMHj0aAoEABw4caNtECSEmyWyKXUxMDLKzsyGVSnHo0CGcPn0ac+fO1WnetWvXQiAQtHGGhBBTZhaPUszJycHRo0dx4cIFDBw4EACwbt06REVFYdWqVdyDspty5coVrF69GhcvXkSXLl3aK2VCiIkxi5bd2bNn4eLiwhU6AAgNDYVQKMS5c+eana+mpgZTp07F+vXr4enp2R6pEkJMlFm07IqLi+Hh4aE2ztraGq6uriguLm52vnnz5mHIkCEYO3aszuuSy+WQy+XcsEwmAwAoFAooFAo9Mzcfqm2z5G3kE77sT322z6jFbvHixVi5cqXWmJycnBda9sGDB5GamorLly/rNV9CQgKWL1+uMT4lJQUODg4vlIs5kUqlxk6BGJCl78+amhqdYwWMMdaGuWhVUlKCsrIyrTG+vr7YsWMH5s+fj8ePH3Pj6+vrYWdnh59//hlvv/22xnyfffYZvv/+ewiF/zlSVyqVEAqFGDZsGNLS0ppcX1MtOy8vL5SWlkIsFuu5heZDoVBAKpUiLCwMNjY2xk6HtBJf9qdMJoObmxsqKytb/H4atWXn7u4Od3f3FuOCg4NRUVGBzMxMBAUFAQBSU1PR0NCAwYMHNznP4sWL8Yc//EFtXEBAAL799ltER0c3uy6RSASRSKQx3sbGxqI/NCp82U6+sPT9qc+2mcU5u759+yIyMhJz5szBhg0boFAoEBcXh3fffZe7EltQUIBRo0Zh27ZtGDRoEDw9PZu8KNGjRw/4+Pi09yYQQozMLK7GAsDOnTvh7++PUaNGISoqCm+88QY2btzITVcoFMjNzdXrGJ4Qwh9m0bIDAFdXV+zatavZ6RKJBC2dfjTi6UlCiJGZTcuOEEJag4odIYQXqNgRQniBih0hhBeo2BFiYZRKJU6dOoXTp0/j1KlTUCqVxk7JJFCxI8SCJCUlwc/PD2FhYVizZg3CwsLg5+eHpKQkY6dmdFTsCLEQSUlJmDhxIgICApCeno7du3cjPT0dAQEBmDhxIu8LHhU7QiyAUqnE/Pnz8dZbb+HAgQMYPHgw7O3tMXjwYBw4cABvvfUWFixYwOtDWip2hFiA9PR05Ofn44svvgBjTO2cHWMMS5Yswd27d5Genm7sVI2Gih0hFqCoqAgAcPv2bfj6+qqds/P19cWdO3fU4vjIbH4uRghpnuqRA9OmTdN43sqDBw8wbdo0tTg+omJHiAUYMmQIBAIBGGNwd3fHV199BZFIBLlcjqVLl+LRo0cQCAQYMmSIsVM1GjqMJcQCpKWlcR1dBAUF4dq1a9i9ezeuXbvG9QHJGGu201o+oJYdIRZg+/btAJ51UJucnMyNT0lJ4cZnZWVh+/btCA8PN0qOxkYtO0J33FsA1QPjs7KympyuGq/rg+UtERU7nktKSkLPnj3Vrt717NmT9zegmpvg4GCDxlkiKnY8lpSUhAkTJuDevXtq4+/du4cJEyZQwTMjurbG+dxqp2LHU0qlErNmzdIaM2vWLF5/OczJwYMHDRpniajY8VRqaioqKyu1xlRWViI1NbWdMiKtcffuXe718/fZNR5uHMc3VOx4atOmTQaNI8ZVX1/PvX78+DE++OADBAYG4oMPPtB43jJfGfUh2eZAJpPB2dlZp4fwmhMPDw+UlJS0GOfu7o5Hjx61Q0akNXx8fJCfnw8AEAqFaGho4KY1HpZIJBbVutPn+2k2Lbvy8nLExMRALBbDxcUFs2fPbvEy+ogRIyAQCNT+ffDBB+2UsWlrfAgrEAgQGhqKadOmITQ0VO2wp6VDXWIaGj8svnGhe364uYfK84HZ3FQcExODoqIiSKVSKBQKxMbGYu7cuVofrwgAc+bMwVdffcUNOzg4tHWqZqFxQbO2tsbx48e5YRsbGygUCo04YrpiY2Oxd+9eneL4yiyKXU5ODo4ePYoLFy5g4MCBAIB169YhKioKq1atQteuXZud18HBAZ6enu2VqtmwtbWFXC4HAK6wqTQetrW1bde8yIvR9Y8Sn/94mUWxO3v2LFxcXLhCBwChoaEQCoU4d+4c3n777Wbn3blzJ3bs2AFPT09ER0fjyy+/1Nq6k8vlXBEAnp0TAJ4VgOeLgjlzcXHBkydPdIqzpO22VI1b5i3FjRw5so2zaT/6fDbNotgVFxfDw8NDbZy1tTVcXV1RXFzc7HxTp06Ft7c3unbtin//+9/4/PPPkZubq/Vm2YSEBCxfvlxjfEpKikUdAtvb2+scd+TIkTbOhrSW6jewALjeT5oaTklJwdChQ9s9v7ZSU1Ojc6xRi93ixYuxcuVKrTE5OTkvvPy5c+dyrwMCAtClSxeMGjUKt2/fRs+ePZucZ8mSJYiPj+eGZTIZvLy8EB4eblFXY7dv346bN2+2GBcQEICoqKh2yIi0xqZNm3D16lVYW1ujrKwMZ86cgVQqRVhYGIYMGYJOnTqhvr4e3bt3t6j9qTry0oVRi938+fMxc+ZMrTG+vr7w9PTUuP2hvr4e5eXlep2PU12JysvLa7bYiUQiiEQijfE2NjawsbHReV2mTluL+Pk4S9puSyUUPruxor6+HjExMVi0aBFee+01dOjQATExMdz9dUKh0KL2pz7bYtRi5+7uDnd39xbjgoODUVFRgczMTK5vrtTUVDQ0NOh1Kf3KlSsA+N1bq4pEIkFGRgbs7OxQW1urMV3V8aNEImn/5Ijeunfvzr0+fvw4Dh06xA03PmXROI5vzOI+u759+yIyMhJz5szB+fPnkZGRgbi4OLz77rvcldiCggL4+/vj/PnzAJ71xb9ixQpkZmYiPz8fBw8exPTp0zF8+HD079/fmJtjEmbMmAEAqK2tRUREBIYOHQovLy8MHToUERER3EUaVRwxbb179+Ze19XVqU1rfMGtcRzvMDNRVlbGpkyZwhwdHZlYLGaxsbHsyZMn3PS7d+8yAOzkyZOMMcbu37/Phg8fzlxdXZlIJGJ+fn5s4cKFrLKyUq/1VlZWMgB6z2fq6uvrmbOzMwPABAIBA8D9EwqFDABzdnZm9fX1xk6V6EAulzNra2vm4ODArKys1PanlZUVc3BwYNbW1kwulxs7VYPS5/tpFldjAcDV1VXrDcQSiUTtCpSXlxdOnTrVHqmZJSsrK2zevBkTJkzQmKZ6Hzdv3gwrK6v2To28AFtbW8ybNw/ffPMNPDw8MGzYMDx+/BgdO3ZEeno6Hj16hIULF/L7vsk2L71mzlJbdiqJiYnM29tbrSUgkUhYYmKisVMjL2DhwoXM2tpabX9aW1uzhQsXGju1NqHP95M6AmiBpXYE0JhSqcTJkyeRnJyM0aNHY+TIkdSiM2N1dXVYt24dUlNT8eabb+KTTz6x2BadPt9PszmMJW3HysoKISEhqK6uRkhICBU6M2dra4s//vGP8PPzQ1RUlEXdatIaZnE1lhBCWouKHSGEF6jYEUJ4gc7ZtUB1/Uaf3+CZI4VCgZqaGshkMjrHYwH4sj9V30tdrrNSsWuBqhskLy8vI2dCCGnOkydP4OzsrDWGbj1pQUNDAwoLC+Hk5GTRHR+qend58OCBxd5iwyd82Z+MMTx58gRdu3blOkNoDrXsWiAUCnn142mxWGzRXw6+4cP+bKlFp0IXKAghvEDFjhDCC1TsCIBn/dctW7asyY5Lifmh/amJLlAQQniBWnaEEF6gYkcI4QUqdoQQXqBiZwby8/MhEAi4Bwa19XIyMjIQEBAAGxsbjBs3rlXrJMRUULEzQ2lpaRAIBKioqGiT5cfHxyMwMBB3797F1q1b22QdRNOIESPw2WefGTsNjqnl01pU7NrY8096Mge3b9/Gm2++ie7du8PFxeWFltGe280Y456Lynfm+HlrN23WOTxPhYSEsI8//ph9+umnrFOnTmzEiBEsKyuLRUZGsg4dOjAPDw82bdo0VlJSws2TnJzMhg4dypydnZmrqysbM2YMy8vL46arnpx2+fJl7nXjfzNmzNBrObt372bBwcFMJBKxl19+maWlpalNb/xvy5YtjDHG0tLS2GuvvcZsbW2Zp6cn+/zzz5lCodC63SdPnmQA2NGjR1lgYCCzs7NjI0eOZA8fPmRHjhxh/v7+zMnJiU2ZMoVVV1dzy1Iqlewvf/kLk0gkzM7OjvXv35/9/PPP3HTVco8cOcJeffVVZmNjwz1VzlzNmDFD473Py8tjs2bN4t6H3r17s7Vr12rMN3bsWPbnP/+ZdenShUkkEsYYYxkZGWzAgAFMJBKxoKAgtn//fu4zpKLtc9lUPnfv3m2vt6NNULEzsJCQEObo6MgWLlzIbty4wX777Tfm7u7OlixZwnJyctilS5dYWFgYGzlyJDfPvn37WGJiIrt16xa7fPkyi46OZgEBAUypVDLG1ItdfX09S0xMZABYbm4uKyoqYhUVFXotp3v37mzfvn3s+vXr7A9/+ANzcnJipaWlrL6+nhUVFTGxWMzWrl3LioqKWE1NDfv999+Zg4MD++ijj1hOTg7bv38/c3NzY8uWLWt2u2/cuMEVpddff539+uuv7NKlS8zPz4+FhISw8PBwdunSJXb69GnWqVMn9vXXX3PL+vOf/8z8/f3Z0aNH2e3bt9mWLVuYSCTiirJquf3792cpKSksLy+PlZWVtfWubVMVFRUsODiYzZkzhxUVFbGioiJWW1vLli5dyi5cuMDu3LnDduzYwRwcHNjevXu5+WbMmMEcHR3Ze++9x65du8auXbvGKisrmaurK5s2bRrLzs5mR44cYb1791Yrdo8fP9b6uWwqH3N/rCYVOwMLCQlhr7zyCje8YsUKFh4erhbz4MEDrlg1paSkhAFgWVlZjDH1YsfYf77sjx8/1ppLc8tpXFgUCgXr3r07W7lyJTfO2dmZa9ExxtgXX3zB+vTpwxoaGrhx69evZ46OjlwhfX67G+d5/PhxblxCQgIDwG7fvs2Ne//991lERARjjLHa2lrm4ODAzpw5o7as2bNnsylTpqgt98CBA1q339yEhISwTz/9VGvMxx9/zCZMmMANz5gxg3Xu3FntebA//PAD69SpE3v69Ck3btOmTWqfIV0+l7rkY07onF0bCAoK4l5fvXoVJ0+ehKOjI/fP398fwLNzYwBw69YtTJkyBb6+vhCLxZBIJACA+/fv67VeXZcTHBzMvba2tsbAgQORk5PT7HJzcnIQHBys1sXV0KFDUVVVhd9//73J7W6sf//+3OvOnTvDwcEBvr6+auMePXoEAMjLy0NNTQ3CwsLU3rNt27Zx75fKwIEDm83ZUqxfvx5BQUFwd3eHo6MjNm7cqLE/AwIC1J4elpubi/79+8POzo4bN2jQILV5dPlcWhrq4qkNdOjQgXtdVVWF6OhorFy5UiOuS5cuAIDo6Gh4e3tj06ZN6Nq1KxoaGtCvXz+9TzYbajkvqvF2N9a4p1yBQKDRc65AIEBDQwOAZ+8XABw+fBjdunVTi3v+d57Nrc9S7NmzBwsWLMDq1asRHBwMJycnfPPNNzh37pxa3Iu8D7p8Li0NFbs29uqrryIxMRESiQTW1ppvd1lZGXJzc7Fp0yYMGzYMAPDrr79qXabqr7hSqXyh5fz2228YPnw4AKC+vh6ZmZmIi4trdn19+/ZFYmIiGGNc6y4jIwNOTk4G7+vvpZdegkgkwv379xESEmLQZZs6W1tbtX2akZGBIUOG4KOPPuLG6dLq6tOnD3bs2AG5XM79gbhw4YJaTEufy6byMXd0GNvGPv74Y5SXl2PKlCm4cOECbt++jWPHjiE2NhZKpRIdO3ZEp06dsHHjRuTl5SE1NRXx8fFal+nt7Q2BQIBDhw6hpKQEVVVVei1n/fr12L9/P27cuIGPP/4Yjx8/xqxZs5pd30cffYQHDx7gk08+wY0bN/Cvf/0Ly5YtQ3x8fIu9w+rLyckJCxYswLx58/DTTz/h9u3buHTpEtatW4effvrJoOsyNRKJBOfOnUN+fj5KS0vRq1cvXLx4EceOHcPNmzfx5ZdfahStpkydOhUNDQ2YO3cucnJycOzYMaxatQoAuD9WLX0um8pH1fo2V1Ts2ljXrl2RkZEBpVKJ8PBwBAQE4LPPPoOLiwuEQiGEQiH27NmDzMxM9OvXD/PmzcM333yjdZndunXD8uXLsXjxYnTu3BlxcXF6Lefrr7/G119/jQEDBuDXX3/FwYMH4ebmpnV9R44cwfnz5zFgwAB88MEHmD17Nv70pz+16r1pzooVK/Dll18iISEBffv2RWRkJA4fPgwfH582WZ+pWLBgAaysrPDSSy/B3d0dERERGD9+PCZPnozBgwejrKxMrZXXHLFYjF9++QVXrlxBYGAg/vu//xtLly4FAO48Xkufy6by0fccsqmhLp4I4YGdO3ciNjYWlZWVsLe3N3Y6RkHn7AixQNu2bYOvry+6deuGq1ev4vPPP8c777zD20IHULEjxCIVFxdj6dKlKC4uRpcuXTBp0iT87//+r7HTMio6jCWE8AJdoCCE8AIVO0IIL1CxI4TwAhU7QggvULEjhPACFTtiMmbOnAmBQMB1FtC5c2eEhYVh8+bNev1UaevWrS/cw3JrzJw5k57ZYcKo2BGTEhkZiaKiIuTn5yM5ORkjR47Ep59+irfeeou6XietY9Te9AhpRNXF+PNOnDjBALBNmzYxxhhbvXo169evH3NwcGDdu3dnH374IXvy5Alj7D8dezb+p+pRedu2bSwoKIg5Ojqyzp07sylTprCHDx9y6ykvL2dTp05lbm5uzM7Ojvn5+bHNmzdz0+/fv88mTZrEnJ2dWceOHdl//dd/cV2VL1u2TGO95t5VvKWhlh0xeW+++SYGDBiApKQkAIBQKMT333+P7Oxs/PTTT0hNTcWiRYsAAEOGDMHatWshFotRVFSEoqIiLFiwAACgUCiwYsUKXL16FQcOHEB+fj5mzpzJrefLL7/E9evXkZycjJycHPzwww9cBwkKhQIRERFwcnJCeno6MjIy4OjoiMjISNTV1WHBggV45513uJZpUVERhgwZ0r5vFNHO2NWWEJXmWnaMMTZ58mTWt2/fJqf9/PPPrFOnTtzwli1bmLOzc4vru3DhAgPAtQqjo6NZbGxsk7Hbt2/X6JpeLpcze3t7duzYsRbzJ8ZHLTtiFlijjkOPHz+OUaNGoVu3bnBycsJ7772HsrIy1NTUaF1GZmYmoqOj0aNHDzg5OXGdg6q6Lvrwww+xZ88eBAYGYtGiRThz5gw379WrV5GXlwcnJyeuG3NXV1fU1tZabDfmloaKHTELOTk58PHxQX5+Pt566y30798fiYmJyMzMxPr16wFof2ZqdXU1IiIiIBaLsXPnTly4cAH79+9Xm2/06NG4d+8e5s2bh8LCQowaNYo7BK6qqkJQUBCuXLmi9u/mzZuYOnVqG289MQTq9YSYvNTUVGRlZWHevHnIzMxEQ0MDVq9ezXUy+c9//lMtvqnuxG/cuIGysjJ8/fXX8PLyAgBcvHhRY13u7u6YMWMGZsyYgWHDhmHhwoVYtWoVXn31VezduxceHh4Qi8VN5mlp3ZhbGmrZEZMil8tRXFyMgoICXLp0CX/5y18wduxYvPXWW5g+fTr8/PygUCiwbt063LlzB9u3b8eGDRvUliGRSFBVVYUTJ06gtLQUNTU16NGjB2xtbbn5Dh48iBUrVqjNt3TpUvzrX/9CXl4esrOzcejQIfTt2xcAEBMTAzc3N4wdOxbp6em4e/cu0tLS8Mc//pF7wppEIsG///1v5ObmorS0FAqFon3eNKIbY580JESl8VPora2tmbu7OwsNDWWbN2/mnk/LGGNr1qxhXbp0Yfb29iwiIoJt27ZN4zm6H3zwAevUqZParSe7du1iEomEiUQiFhwczA4ePKjxLNW+ffsye3t75urqysaOHcvu3LnDLbOoqIhNnz6dubm5MZFIxHx9fdmcOXNYZWUlY4yxR48esbCwMObo6Ei3npgg6s+OEMILdBhLCOEFKnaEEF6gYkcI4QUqdoQQXqBiRwjhBSp2hBBeoGJHCOEFKnaEEF6gYkcI4QUqdoQQXqBiRwjhBSp2hBBe+H8kgT34FR9otwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:39:16.689117Z", + "iopub.status.busy": "2024-02-29T03:39:16.688351Z", + "iopub.status.idle": "2024-02-29T03:39:16.978503Z", + "shell.execute_reply": "2024-02-29T03:39:16.977621Z" + }, + "papermill": { + "duration": 0.312537, + "end_time": "2024-02-29T03:39:16.980494", + "exception": false, + "start_time": "2024-02-29T03:39:16.667957", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEqCAYAAACBYUz6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABqjklEQVR4nO2dd3xTVfvAvzdpmw46KC0d0FJm2RsqCDIsFlTE8SriAlR8HeirFRm+yBCVISoOBBcCr4Jb8CdQhMqQDWVvKIUy2lIKnXRk3N8fIWnSJG1S0nRwvp/P/TS597nnnnubPHnOOc+QZFmWEQgEgjqIoro7IBAIBFWFUHACgaDOIhScQCCoswgFJxAI6ixCwQkEgjqLUHACgaDOIhScQCCoswgFJxAI6ixCwQkEgjqLUHACgaDOUq0KbvPmzQwdOpTw8HAkSWLFihXlyo8aNQpJkiy2du3aGWWmTZtmcbx169ZVfCcCgaAmUq0KrqCggE6dOjF//ny75D/++GPS0tKM2/nz5wkMDOThhx82k2vXrp2Z3JYtW6qi+wKBoIbjVp0XHzJkCEOGDLFb3t/fH39/f+P7FStWcO3aNUaPHm0m5+bmRmhoaKX7pdPpuHTpEr6+vkiSVOl2BAJB1SDLMnl5eYSHh6NQ2LbTqlXB3SzffPMNsbGxNGnSxGz/qVOnCA8Px9PTk169ejFz5kwiIyNttlNcXExxcbHx/cWLF2nbtm2V9VsgEDiH8+fP07hxY5vHa62Cu3TpEmvWrGHZsmVm+2NiYli8eDHR0dGkpaUxffp0+vbty+HDh/H19bXa1syZM5k+fbrF/vPnz+Pn51cl/RcIBJUnNzeXiIgIm99pA1JNyQcnSRK///47999/v13yM2fO5IMPPuDSpUt4eHjYlMvOzqZJkyZ8+OGHPPPMM1ZlylpwhoeXk5MjFJxAUAPJzc3F39+/wu9orbTgZFlm0aJFPPnkk+UqN4CAgABatWrF6dOnbcqoVCpUKpWzuykQCKqZWukHt2nTJk6fPm3TIjMlPz+f5ORkwsLCXNAzgUBQk6hWBZefn8/+/fvZv38/ACkpKezfv5/U1FQAJk2axFNPPWVx3jfffENMTAzt27e3ODZu3Dg2bdrE2bNn2bZtGw888ABKpZIRI0ZU6b0IBIKaR7UOUffs2cOAAQOM7+Pj4wEYOXIkixcvJi0tzajsDOTk5PDrr7/y8ccfW23zwoULjBgxgqysLIKDg+nTpw87duwgODjYqX2XZRmNRoNWq3VquwJBdePu7o5SqazubjiFGrPIUJOoaAKzpKSEtLQ0rl+/Xg29EwiqFkmSaNy4MfXq1avurtikTi8yVCc6nY6UlBSUSiXh4eF4eHgIZ2BBnUGWZTIzM7lw4QItW7as9ZacUHAOUlJSgk6nIyIiAm9v7+rujkDgdIKDgzl79ixqtfqmFNx3O86xPTmL9x7sgL+XuxN7aD9CwVWS8sJDBILajDNGJGqtjllrjpNfrKFBPQ/eHma5IOgKxLdUIBBUir2p18i5rrZ6bF9qNvnFGkBvyR25lOPKrhkRCk5wU5w9exZJkoyuPlXdztatW+nQoQPu7u52R70IzNm4cSOSJJGdnV3pNn5JusCDn2/jlR/2WT2++WQmAJIEOhmmrjxCdaxnCgUncCrO+PKUR3x8PJ07dyYlJYXFixdXyTUE5ZNXpGbWmuMAbDqZyfmrlt4E/5zSK7jXB7XC20PJnnPX+H3fRZf2E4SCu6UpKSmp7i44THJyMgMHDqRx48YEBARUqg1X3rfBX9LZuPIeXlq2l8HzNnO9RH8f8zckcyW/NHb7l6QLZvJXC0o4eFE/JH24ewQvD2wJwMw1x9FodS7qtR6h4G4h+vfvz9ixY3n11VcJCgoiLi6Ow4cPM2TIEOrVq0dISAhPPvkkV65cMZ6TkJBAnz59CAgIoEGDBtx7770kJydbbf/s2bNGx+369esjSRKjRo1yqJ3jx4/Tu3dvPD09ad++PZs2bTK2LUkSWVlZPP3000iSZLTgNm3aRM+ePVGpVISFhTFx4kQzpWLtvg2W5tq1a+nSpQteXl4MHDiQy5cvs2bNGtq0aYOfnx+PPfaYmb+jTqdj5syZNG3aFC8vLzp16sQvv/xiPG5od82aNXTr1g2VSmVXwtV33nmHhg0b4uvry7PPPsvEiRPp3Lmz8fioUaO4//77effddwkPDyc6OhqA//3vf3Tv3h1fX19CQ0N57LHHuHz5slnbq1evplWrVnh5eTFgwADOnj1bYX8MZF8vYdXBNI6n57H3XDbnsgpYtCUFgGGdwwG9gtPpSoeficcykGVoHepLiJ8no2+PAiAzr5jcIucr+/IQCs4JyLLM9RKNy7fKzGksWbIEDw8Ptm7dyqxZsxg4cCBdunRhz549JCQkkJGRwSOPPGKULygoID4+nj179pCYmIhCoeCBBx5Ap7P8JY6IiODXX38F4MSJE6SlpRkjTuxt54033uD1119n37599OrVi6FDh5KVlUVERARpaWn4+fkxb9480tLSGD58OBcvXuTuu++mR48eHDhwgAULFvDNN9/wzjvv2LzvhQsXGvdPmzaNzz77jG3btnH+/HkeeeQR5s2bx7Jly1i1ahV//fUXn376qVF+5syZLF26lIULF3LkyBFee+01nnjiCaMiNjBx4kRmzZrFsWPH6NixY7n/k++//553332X2bNnk5SURGRkJAsWLLCQS0xM5MSJE6xbt44///wTALVazYwZMzhw4AArVqzg7Nmzxh8V0Kf8evDBBxk6dCj79+83Kk97OXihdHHg8KUc3lt9jBKtjr4tg5j9UEd8VW5czC5kx5kso9yaw+kADGmvj/9WuZWqGZ2L5+GEm4gTKFRraTtlrcuve/TtOLw9HPsXtmzZkjlz5gB6q6FLly689957xuOLFi0iIiKCkydP0qpVKx566CGz8xctWkRwcDBHjx61iAVWKpUEBgYC0LBhQ7MhpL3tjB071ii7YMECEhIS+Oabbxg/fjyhoaFIkoS/v78xY/Pnn39OREQEn332mbH+xqVLl5gwYQJTpkwxuvOY3jdAWlqa8RncfvvtADzzzDNMmjSJ5ORkmjVrBsC//vUvNmzYwIQJEyguLua9995j/fr19OrVC4BmzZqxZcsWvvjiC/r162ds/+2332bQoEEV/0OATz/9lGeeecaYmXrKlCn89ddf5Ofnm8n5+Pjw9ddfm2XQefrpp42vmzVrxieffEKPHj3Iz8+nXr16LFiwgObNm/PBBx8AEB0dzaFDh5g9e7ZdfTtwPtv4+uc950nOLEAhwVv3tsXTXcm9ncJZviuVX5Iu0LtFELlFauP8290d9P8jSZJQ3FhsMLX0XIGw4G4xunXrZnx94MABNmzYQL169YyboUCPYfh46tQpRowYQbNmzfDz8yMqKgrAIka4Iuxtx6A4QJ96vnv37hw7dsxmu8eOHaNXr15mvlu33347+fn5XLhQOjdket+mmFpXISEheHt7G5WbYZ9hyHf69GmuX7/OoEGDzJ7Z0qVLLYbb3bt3t9nnspw4cYKePXua7Sv7HqBDhw4W6cGSkpIYOnQokZGR+Pr6GpWs4bkeO3aMmJgYs3NMn3FFHLiQbXydnFkAwKC2IbQK0SeafLi7Ppvu6sNp5BWpSTyWgVor06JhPVqGlCajVNz4/7hYvwkLzhl4uSs5+nZctVzXUXx8fIyv8/PzGTp0qNVfc0N6qaFDh9KkSRO++uorwsPD0el0tG/f3uFJbme1U1lM79sUd/dSD3tJkszeG/YZhtEGi2rVqlU0atTITK5sPkFb17sZyrZZUFBAXFwccXFxfP/99wQHB5OamkpcXJxTnqssy+w/b+m/dnuLIOPrLhEBNA/2ITmzgFUH00g8rv8xuLu9eU0UvYKTxRC1NiJJksNDxZpA165d+fXXX4mKisLNzbL/WVlZnDhxgq+++oq+ffsCVDhhbrAwTLOsONLOjh07uOOOOwDQaDQkJSUxduxYm9dr06YNv/76K7IsG624rVu34uvrW26u/srQtm1bVCoVqampZsPRmyU6Oprdu3ebpQbbvXt3hecdP36crKwsZs2aRUREBKDP0GNKmzZt+OOPP8z27dixw65+Xc4r5kp+MW4KiQBvD+PKaa9mDYwykiTxcPcIZq05zv92nOPUZf2PwJAO5vkXFQpAC1oxRBW4ipdeeomrV68yYsQIdu/eTXJyMmvXrmX06NFotVrq169PgwYN+PLLLzl9+jR///23MaWVLZo0aYIkSfz5559kZmaSn5/vUDvz58/n999/5/jx47z00ktcu3bNbJ6pLC+++CLnz5/n5Zdf5vjx46xcuZKpU6cSHx/v9HA6X19fxo0bx2uvvcaSJUtITk5m7969fPrppyxZsqTS7b788st88803LFmyhFOnTvHOO+9w8ODBCkOmIiMj8fDw4NNPP+XMmTP88ccfzJgxw0zm+eef59SpU7zxxhucOHGCZcuW2e0/eDw9F4DWYb50jggAIKieihYNzbOMPNilEUqFxJFLuZRodDQN8qF1qHmtBMMQ1dW+vkLB3cKEh4ezdetWtFotd911Fx06dODVV18lICAAhUKBQqHghx9+ICkpifbt2/Paa6/x/vvvl9tmo0aNmD59OhMnTiQkJISxY8c61M6sWbOYNWsWnTp1YsuWLfzxxx8EBQVZlTVcb/Xq1ezatYtOnTrx/PPP88wzzzB58uSbeja2mDFjBm+99RYzZ86kTZs2DB48mFWrVtG0adNKt/n4448zadIkxo0bR9euXUlJSWHUqFF4enqWe15wcDCLFy/m559/pm3btsyaNYu5c+eayURGRvLrr7+yYsUKOnXqxMKFC80WlcrjeFoeAJ0aB9AlMgCA21s0sFC8Df086deqNN/ikPahFjKlc3Cu1XAiH5wVyss1VVRUREpKCk2bNq3wAygQVJZBgwYRGhrK//73P5df2/AZ/3RXNquPXWXOvzpyb8cwlmw7x0NdG9HQz/Jzv+ZQGi98vxeAP1/uQ/tG/mbHO05bS26RhsTX+9E8+ObzzIl8cAJBLeH69essXLiQuLg4lEoly5cvZ/369axbt67a+iTLcDJDP5/WOSIAbw83Xujf3Kb8nW1CuL1FA3xV7rQLt1Q4CoVhiCoWGQSCOkW7du04d+6c1WNffPEFDz74IKtXr+bdd9+lqKiI6Ohofv31V2JjY13c01I0Oh2Fai3eHkq7LC4PNwXfP3ubzePCTUQgqKOsXr0atdp6WqGQkBC8vLxYv369i3tVPiUavWtMh0b+KBU3nx/OoOBcvYoqFJxAUMU0adKkurvgMOobQfGG1dObxaAjXb3IUK2rqJs3b2bo0KGEh4cjSRIrVqwoV94QyFx2S09PN5ObP38+UVFReHp6EhMTw65du6rwLgSCuofBguvkNAV3C7qJFBQU0KlTJ+bPn+/QeYZAbsPWsGFD47Eff/yR+Ph4pk6dyt69e+nUqRNxcXEWGRYEAoF1dDoZtVaviZyl4AzD3FtqiDpkyBCGDBni8HllA7lN+fDDDxkzZowxcHnhwoWsWrWKRYsWOZRFQSC4VSnSaJGBAG8Pwv2d4wol3YpD1MrSuXNnwsLCGDRoEFu3bjXuLykpISkpyWz1SaFQEBsby/bt26ujqwJBjaRIraVIbb1oebFaPzxtE+rrtJKYYhXVDsLCwli4cCHdu3enuLiYr7/+mv79+7Nz5066du3KlStX0Gq1hISEmJ0XEhLC8ePHbbZbXFxMcXFphtLc3NwquweBoLrR6WROZuijFAK8PQjxVaEySdxQeEPxlQ23uhkMQ1QRbF8O0dHRxkymAL179yY5OZmPPvropjy+Z86cyfTp0x0/UZbh+hVQ+YGbqmJ5gaAGYDoPln29BI1WRzMTXzeDBRcdajtCwFGMQ1QRbO8YPXv25PTp0wAEBQWhVCrJyMgwk8nIyDAmSLTGpEmTyMnJMW7nz5+37+I55yHnAuReqnT/awtRUVHMmzfP+N6eVW9BzURbxorKL9ZQrNFbbRqtjpIbmWCiQ28+pMqAspqGqLVewe3fv9+Yu8zDw4Nu3bqRmJhoPK7T6UhMTCw3yZ9KpcLPz89sswufGwHGRdlQnFfZW6iVpKWlVWqBSFD9WBsmXivQOyIbhqduCgk/Lw8LucpSXcH21TpEzc/PN1pfACkpKezfv5/AwEAiIyOZNGkSFy9eZOnSpQDMmzePpk2b0q5dO4qKivj666/5+++/+euvv4xtxMfHM3LkSLp3707Pnj2ZN28eBQUFxlVVZ6JRqChUBuCrzdZbcsGtS23xOk55FrGrkGUZrVZrNZedwDbWhonXrpcQ4qfieolewXm4Odf2uSVXUffs2UOXLl3o0qULoFdOXbp0YcqUKYDeSjBNaV1SUsLrr79Ohw4d6NevHwcOHGD9+vXceeedRpnhw4czd+5cpkyZQufOndm/fz8JCQkWCw/OICOvmFS1PxqdBIXX0F07ByUFrtsc/LDk5eXx+OOP4+PjQ1hYGB999BH9+/fn1VdfdfjeTYeohopXv/32GwMGDMDb25tOnTpZrFxv2bKFvn374uXlRUREBK+88goFBQXG4xVViKpsxSqBOdoyHxuFJKHW6sgr0lBoUHBK56qG0kUGpzZbIdX609e/f/9yswuUTcw3fvx4xo8fX2G7Y8eOLTcLrLMI8vFArdFxOd+L8MX9XP9r8eYl8LA/NXZ8fDxbt27ljz/+ICQkhClTprB3716z8nQ3w3//+1/mzp1Ly5Yt+e9//8uIESM4ffo0bm5uJCcnM3jwYN555x0WLVpEZmam8f/07bffAqUVoqKjo7l8+TLx8fGMGjWK1atXm11n4sSJzJ07l2bNmlG/fn2n9P1WoqyzbaCPPlvv1YKSKrPgjEPUW8nRt7ajclcSFeRDnofzrUNnk5eXx5IlS1i2bJnR4v32228JDw932jXGjRvHPffcA8D06dNp164dp0+fpnXr1sycOZPHH3/caC22bNmSTz75hH79+rFgwQI8PT0rrBBlwJGKVQJLTIeJjet74+2h5Ep+MblF+nk4CQk3pXOnWqorFlUoOCfg6+uP/PpJpKvJyECyLpxCPPDzdCfUz9zHSJZlzmZdR6uTaRbsY/xlqxTu3naLnjlzBrVabVatyd/f38zt5mYxrVBlWPi5fPkyrVu35sCBAxw8eJDvv//eKCPLMjqdjpSUFNq0aUNSUhLTpk3jwIEDXLt2zVjsJTU1lbZt2xrPc6RilcASgxUV6O1BoI9+IcHHw42CG5XrPdwUyE6eS1bcikPUOoMkIfmGgLoQqegaEYoiTmoCyNFCbrZMUD0FDX1VKBUK1BodeTr9B6kQT3xqYbEaW5StUAWYVaT697//zSuvvGJxXmRkpEMVoqqiYtWthMFNRGGSBinQx8Oo4LzcFVx38jVFuqS6gF84FOeg0hUS7a/lYrEneUVqMvOKuVagJtRfhZtJIZRijQ4fF/kHN2vWDHd3d3bv3k1kZCQAOTk5nDx50ljFqirp2rUrR48epUWLFlaPHzp0qMIKUQLnYLDgTPO8+Xu5cylHQquTUbkrq0DB6f+6OqNvrfeDq1G4eUA9/XycR0EaUYGeRDXwQeWmRKPTceFaIWezSlcNbcUCVgW+vr6MHDmSN954gw0bNnDkyBGeeeYZFAqF0+INy2PChAls27aNsWPHsn//fk6dOsXKlSuNi0H2VIgSOAfDKqrp9IhCIRHu74W/lzu+nu42zqw8dsWiHv4Nrp5x7nWd2poAfBqC0gN0aqT8DPy83GkZUo8wfy+L+bZCFyo40Gda6dWrF/feey+xsbHcfvvttGnTxiXFczp27MimTZs4efIkffv2NboDGRY57KkQJXAOpRac+f76Ph40aeDjlAy+ZTEOUW1ZcNmpsOIFmB8DGUeddl0xRHU2CgX4NYJrKZB/GbwboHBTEeyrokSjJaugdD6pSK01K1hc1fj6+ppN8hcUFDB9+nSee+65Cs89e/as2XvToUZUVJTF0CMgIMBiX48ePcycsssyYsQIRowYYfM6FbkVCezDOAfnQqd0w8yMzf/fX5NBUwRN+kDDNs67rtNaEpTi6Q8qX0CGnIvG3YYVKwNanYxaq6OwREv29RJjPGBVsW/fPpYvX24sWPz4448DMGzYsCq9rqBmYbDgFFVgqdmi3FCtM5vg6EqQFDBktlOjgYQFVxVIEvg1hszjUJwDRbng6YeXhxv1vT0o1ujQ6mSKNVqOp5fGsHq4KYgOcV4OLmvMnTuXEydOGON2//nnH44dO1ZuXGl+fn6V9UfgegxKRulKC864ilrmgFYDayboX3d/BkLbO/W6QsFVFe6e+mD8gsv6OFVVa5AURATqfddSr143WmySJIGsz4NfpNbiVUWuI126dCEpKclif2FhIfv376+SawpqHgYl41oLTv/XwoLb8w1kHgOv+jDgTadfVyi4qsQ3BAqvgrYYCq5AvdLaEZ7upbMD9b3d0WhlcovU5BZpqkzB2cLLy8um+4ag7lFqwbnumkprhZ8LrsCGd/WvB74F3oFOv66Yg6skdk12K9z0vnEAeWmgLa2N6elWGt3g5+WOn5deqeUVWa+fKRA4A50sGxWcrUWGqljIkawNUf9+B4pyIKQDdBvl9GuCUHAOY/DWv37dTldIr0B9SJWsM0uM6e1RquDqqdyMvkfXS7TGmpQCgbMxDXa35Q5iiBxRKpVWj1cGiyFq2gFIWqx/ffccUDjvWqaIIaqDKJVKAgICjGl8vL29K14UUAVD4VnIywKlL3jo5+Ei/d1RSFByox6Eh6SlWK0lKyefAG/nJRsUCAyUaLTImhIkSTKrQ2JAp9ORmZmJt7e3U/PsmdVkkGVYPR6Qof2/oElvp12nLELBVQJDskeHaq1ev67P4Zaeq5+Ls6IUc4vU5BZqyMtU0KCeqPEgcD7XS7RcLSjBXSmRct26g7dCoSAyMtKpq/mSabqkQ7/A+R36kc2gt512DWsIBVcJJEkiLCyMhg0bolbbOWdWUA+++xeoC2DgVGg71ELkZEYe8d8loXJT8vuLvc2ykAgEzmDKisNsTb7CYzGR3NapqVUZDw8PFAonJ7y8oeAU6uuw7i39zr6vg38jp16nLELB3QRKpdL+eQrPCOj+uP6fm/gmtL1L7xBsQodIFTqFO2euFdHp3Y30bBrId8/EOD35oODWJLdIzYrDmZRodNzZvrFLQvQMGObgOqZ8rV9wqx8Fvao+Ka345riSmOehQQu9b9ymORaHJUnimT6lv6q7Uq6y+lCaK3soqMOsO5JBiUZHy4b1iA5xXs1Te1BIEk2kdDqk3ijvGfee3le0qq9b5VcQlOLmAYNn61/vXAiZJy1Enritidn7xdvOuqBjgluB/zuoX8Uf2incZfHPBhQKiclu36GU1dB8IETf7ZrruuQqglJaxkKrIaDTQMIEi8Ixnu5Kvh3VgzZh+tKF+89ns/98djV01Jzf9l7gpWV7XZriSeA8rhaUsOXUFQDu7Rjm8uu3yd/BIOVetJISBs9yWfU5oeCqg7h39SmVkv+GE2ssDg9o3ZA1/+nLg131E7BLaoAVt2BjMqsOprH33LXq7oqgEiQcTkejk2nfyM+sir1L0JRwX9qnAOwJeQSCnZcmvyKqVcFt3ryZoUOHEh4eblel9N9++41BgwYRHByMn58fvXr1Yu3atWYy06ZNQ5Iks61169ZVeBeVoEHz0gnWtZNAXWRVbFTvKAD+PHiJy3nWZVxFTqF+tbhYI5yQayP/d+DG8LSj84oM2c3OhQSXnCdT9uPZs3dyIt11RdKrVcEVFBTQqVMn5s+fb5f85s2bGTRoEKtXryYpKYkBAwYwdOhQ9u3bZybXrl070tLSjFuNrJ3Z93XwDYNrZ2H7Z1ZFOjYOoGtkAGqtzPKd513bvzIYKi4JBVf7uJxbxI6ULADucfXwNC/DuKA2R/MoeXjz93EH/Edvkmp1ExkyZEi5aXrKMm/ePLP37733HitXruT//u//jMWjAdzc3GpE5fVyUdWDQTPgt2fhnw+g0wirPkEje0exN3U/3+88x7/7NcOzGnzjijVaitR6xSbCyGofqw6lIcvQNTKAxvXtr8TmFBKnQ0keF7zb8MtVfe2P/eddN81Rq+fgdDodeXl5BAaaZyE4deoU4eHhNGvWjMcff5zU1NRy2ykuLiY3N9dscwkd/gURt4H6OqyfalVkSPswwvw9uZxXzCeJp1zTrzLkFWmMr4WCq30Yh6edXDw8vbAH9uszSK+JiEe+oW5cuWhWqxXc3Llzyc/P55FHHjHui4mJYfHixSQkJLBgwQJSUlLo27cveXm2x/0zZ87E39/fuBmqOlU5kqTPYIoEh36Gc9stRDzcFEy7rx0AX2w+w5pDaVwv0XDOpHhNVZNbWBqtIRRc7eL05Tz2pmYjSXBPBxcOT3U6WP2G/nXnx0mrV5rIMiO3mPQc18wp11oFt2zZMqZPn85PP/1Ew4aledaGDBnCww8/TMeOHYmLi2P16tVkZ2fz008/2Wxr0qRJ5OTkGLfz51043xXeGbqN1L9e8wboLN0w4tqF8miPCLQ6mZeX76PL2+voP3cjm09muqSLuSYWXInW+al0BFXD+avXGbloNwB9WwbT0M91kQscWAaX9oKHL9w5lbKJS1w1TL0pBZefn18tQ7sffviBZ599lp9++onY2NhyZQMCAmjVqhWnT5+2KaNSqfDz8zPbXMrAt/RhW+mHYO8SqyLvPtCB+zuHo9HJFGt0yDK88J1ldt6qwDRHnVosMtQKLmYXMuKrHVzMLqRZsA9zH+7ouosX5cD6afrX/SeAb4hFaqZ9LhqmOqzgUlJSuOeee/Dx8cHf35/69etTv359AgICqF+/flX00Yzly5czevRoli9fzj333FOhfH5+PsnJyYSFud650W58gmDAf/WvE2fA9asWIkqFxNyHOxHXLsS4r6BEy84zWVXevdxCUwtOKLiaTlpOISO+3MGFa4U0DfJh+ZjbaOjrQutt0xwoyIQGLaHnvwEoa/fvT812SVccXkV94oknkGWZRYsWERISclMhH/n5+WaWVUpKCvv37ycwMJDIyEgmTZrExYsXWbp0KaAflo4cOZKPP/6YmJgY0tPTAX3KbX9/feD6uHHjGDp0KE2aNOHSpUtMnToVpVJpUY6uxtH9GdjzrT4//caZcPf7FiJuSgXzH+vKwk3JfPVPCjmFamYnHOfXF3pXaehNrrDgajwarY69qdn8ffwyK/dfJC2niMhAb5aNiSHElUPTzBP6METQRyy46fMalpT53By6mINWJ1dJDVZTHFZwBw4cICkpiejom/dG3rNnDwMGDDC+j4+PB2DkyJEsXryYtLQ0sxXQL7/8Eo1Gw0svvcRLL71k3G+QB7hw4QIjRowgKyuL4OBg+vTpw44dOwgODr7p/lYpSjf9gsPS+2D31/oUziHtLMTclArGDmzJw90j6Pf+BvamZrPuaAZ3tas6txixyFAzybmuJvF4Bn8fv8zmk5lmc6URgV4sf+42wvy9XNchWYaEifowxFZD9GGJNzC1/H08lBSUaDl1OY/WoVU7HeSwguvRowfnz593ioKrqJCvQWkZ2LhxY4Vt/vDDDzfZq2qkWT9oO0xfI3LNBBj5fzZj9kL8PHn69qZ8vjGZ99eeYGDrhriVLVXuJEwtOLHIUDPQ6WTu/uQfLmYXGvcFeLvTv1UwA1o35M42IdRTudjN9cRqffih0kMfjmiCqQXXsXEA289ksT81u+YpuK+//prnn3+eixcv0r59e2ONAgMdO7pwMrMuctc7cHItnP0Hjq6Adg/YFP13v+Ys25XKqcv5/Lb3Io/0qBr3FuEHV/O4rtYaldvz/ZozqG1DOkfUr/Ihn03URZAwSf+611h9OKIJpgquc+QNBXc+m0d7RlZptxxWcJmZmSQnJzN69GjjPkmSkGUZSZLQakW2iZsiIBL6vKafh1s7GVrGGWs4lMXfy52X+rfg3dXH+Gj9Se7rHF4lkQ5iiFrz0Jj8H16/qxXuVWS92832TyH7nD78sO/rFofNFFxEAOAah1+Hn8rTTz9Nly5d2L59O2fOnCElJcXsr8AJ3P4f8I+E3AuwdV65ok/2akK4vydpOUUs3X62Srpj5gdXZrL4x92pJIkMIy5HbTJV4FZdVpuBnIvwz4f614Nm6MMQy2D6w2hQcCcz8igo1ljIOhOHFdy5c+eYPXs2MTExREVF0aRJE7NN4ATcvSDuHf3rrR/DtXM2RT3dlbw6qBUAX2w6UyUWlqkFZzpZfDIjjwm/HmLirwedfk1B+Wh0+v+Dm0JyefJKC9ZN0YcbRtymDz+0gunnJsTPkzB/T3SyfjW1KnFYwQ0cOJADBw5URV8EprS5D6L6gqYI/ppcruiDXRoR6ONBVkEJO6rAL87MTcTEcriSpy87d71ETEu4Gs2N/4ObK8vTW+PcNjj8CyDp65vaULZls9C4apjq8Bzc0KFDee211zh06BAdOnSwWGS47777nNa5WxpDnOrCvnDsDzizEZr1tyrqplQwuH0oy3amsupgGn1bOtclxtTR19QPrkAotmrDYKm7O7n6lUPotDfqm6J3awrrZFO07NRG54gA1hxO50BNU3DPP/88AG+/bVnPUCwyOJmQdtDjWdj1BayZCM//A0p3q6L3dghj2c5UEo6kM+P+9k6ddDYL1TIZalwv0Su+8lx9BFWDRlcDLLikxZBxSB9mOPCtckXLKrhOLrLgHP4W6HQ6m5tQblXAgEngFaiPcNj9jU2xmGYNCKrnQfZ1NRucmFBQo9WZWWqmcykFxeL/XV0YfmiqyvexQq5fhb9n6F8PmAw+DcoVLxvi16GRPwoJ0nKKyMituswiDj0dtVqNm5sbhw8frqr+CMriVR/unKJ/veE9KLhiVUypkHioa2MA3lt9zGnFYUx94MDcgjOsgAn7zfUY5uDcq2sFdcN7UHgNGraF7k9XKP72MH1UzquxLQHwUbnR6kbpwn1VGJfqkIJzd3cnMjJSWGquputTENoRinMg0XJqwMDYgS0I8VNxNus68zfYzp7iCKYLDGA+1CgoqdolfoFtjKuo1WHBpR+GPTdGE4Nn6cMMK6B38yCOzxjMq7GtjPu6RAYAcOBCdhV0Uo/DT+e///0vb775JlevWma8EFQRCmVp8P3epXBpn1UxX093pt9IjrlwUzLH0m4+fZXpAgOYr6KK1dPqQ11dq6iyrA8jlHX6sMJm/ew+tawTunEltaZYcACfffYZmzdvJjw8nOjoaLp27Wq2CaqIyNugwyOArF+5sjGxH9culNg2Iai1Mo99tYPdZ2/uh6isBWc6RM03DFHFGNXllA5RXWzBHfkdzm0BN099WOFNYFhoOHghG52uaj5EDq+i3n///VXQDYFdDJoOx1fBhV1w8CfoNNxCRJIkZj/UgacXF3HgQg6Pf7WTuY904r5K5uPPKztENV1FrWIvdIFt1MYhqgstuJIC+OvGammf1/RhhTdBy4a+qNwUFJRoSb16naggHyd00hyHFdzUqdaLowhcgF843DFOX6lo3RRofTeofC3EGtRT8cNzvXj1x32sPZLBK8v3cf7qdV7s39xhr3fDELW+tzvXrqvNFxnEELXaKHX0daEFt2WePnzQP1IfTniTKBUSrUJ8OXQxh+PpuVWi4Cr9dJKSkvjuu+/47rvvLOqSCqqQXi9B/aaQnw6b59oU8/JQ8vnj3Xi2T1MA3l97gl+SLjh8OcMQtUE9FQBqTelQonQVVYxRXY3G6OjrIgvu2ll92CDowwjdnZNnLjpU/wN9vIqKQTus4C5fvszAgQPp0aMHr7zyCq+88grdunXjzjvvJDPTNUVQbmncVDB4pv719vmQlWxTVKmQmHxvW6OSSzzmuH+cIQ61gc+NzKzCgqsRqF3t6PvXZNAWQ9M79GGETqL1DQVXVdXuHVZwL7/8Mnl5eRw5coSrV69y9epVDh8+TG5uLq+88kpV9FFQllaDoUUs6NSlObjKwZDtd2/qNYejDgyZRIJ8DRacmIOrCRgtOFcMUZM3wLH/A0kJg2fbjDetDIaElzVGwSUkJPD555/Tpk0b4762bdsyf/581qxZ49TOCWwgSXr/I4UbnFoLJ/8qV7xjY3/cFBKX84rNMsDag2GIGmTFgjO4iYhVVNdjnIOr6iGqVq1PQw7QcwyEtHVq84YhakpWAYVVMCKoVKhW2QB70DsB63QiGaLLCGoJt72gf50wETQlNkU93ZW0Ddf/Uu510OfIsMgQ6HPDgrPiJiJwPWpXOfru/hoyj4N3A+g/0enNB/uqaODjgSzDqcvOt+IqlS7pP//5D5cuXTLuu3jxIq+99hp33nmnUzsnqIA7xoNPQ7iaDDsXlCvaNVJf0nGvg8kpSxcZ9BacTgbtjfkfY7C9Qy0KnIHRD64q5+DyM2HDjfneO6fowwargKpcaKiUo29ubi5RUVE0b96c5s2b07RpU3Jzc/n0008damvz5s0MHTqU8PBwJElixYoVFZ6zceNGunbtikqlokWLFhaFaQDmz59PVFQUnp6exMTEsGvXLof6VWvw9NP7xoG+FmVeuk1RQ1jMvlQHFdyNRYagGwoO9FZciUZnFtUgcC3GYPuqdPT9+219eGBYJ+jyZJVdxjAPdzytBii4iIgI9u7dy6pVq3j11Vd59dVXWb16NXv37qVx48YOtVVQUECnTp2YP3++XfKGotMDBgxg//79vPrqqzz77LOsXbvWKPPjjz8SHx/P1KlT2bt3L506dSIuLo7Ll52XYaNG0fFRaNQdSvJLq4lbwWDBHbmU61AgviHY3uAmAvrkhVWdalpQPlWeLuniXtj7P/3rIXP04YJVhHElNePmQwvLUqm6YpIkMWjQIAYNGnRTFx8yZAhDhgyxW37hwoU0bdqUDz74AIA2bdqwZcsWPvroI+Li4gD48MMPGTNmjLEozsKFC1m1ahWLFi1i4kTnzyFUOwqFPpPqVwPhwHJ9ZoeInhZijet7EeyrIjOvmEMXc+gRFWhX8wYLLtDH3IIzVZJikcH1aKoy4aVOp483RdaHB0be5vxrmBBdha4ilVJwiYmJJCYmcvnyZYuFhUWLFjmlY9bYvn07sbGxZvvi4uJ49dVXASgpKSEpKYlJk0pdJxQKBbGxsWzfvt1mu8XFxRQXFxvf5+Y6/5ekSmnUDbo8Afu+g9VvwJgNesVngiRJdI0MYO2RDPaeu2aXgtPpZPJvzLP5e7njrpRQa2XUWp0ItK9mqjTY/tBP+nBAd5/SKZAqpFWIL5IEV/JLyMwrJthXVfFJduKw+p8+fTp33XUXiYmJXLlyhWvXrpltVUl6ejohISFm+0JCQsjNzaWwsJArV66g1WqtyqSn256fmjlzJv7+/sYtIqJq6otWKXdOBZUfpO2H/d9ZFTEuNNg5D5dXrDFaZ76ebnjcWLFTa2QxRK1mDOmSnO4HV5ynDwMEfVigX+VimB3By0NJVAN9mJazrTiHLbiFCxeyePFinnyy6iYdXc2kSZOIj483vs/Nza19Sq5eQ/0y/to3Yf10vbe5V4CZSNcmBgWXbaxjWx6G4anKTYHKTYm7mwJKtJRodWWy+YoxqqupMj+4ze9DfoY+HLDXS85tuxyiQ3xJuVLA8fRc+rQMclq7Dqv/kpISevfu7bQOOEJoaCgZGRlm+zIyMvDz88PLy4ugoCCUSqVVmdDQUJvtqlQq/Pz8zLZaSc/nICgarl+BTbMtDndopHf4zbTT4dfgIuLnpfd7NFgLaq1OJLusZtRVEWx/5TRs/1z/evAsfVigi6iqeTiHn86zzz7LsmXLnNoJe+nVqxeJiYlm+9atW0evXr0A8PDwoFu3bmYyOp2OxMREo0ydRuleGqe68wu4fNzssKMOvwYnXz9PvaFvGKIWa3RGHzhB9VA6RHWiBbf2TX34X4tB0CrOee3aQZuwqvGFc3iIWlRUxJdffsn69evp2LGjRVTDhx9+aHdb+fn5nD5dmlo7JSWF/fv3ExgYSGRkJJMmTeLixYssXboU0Ff0+uyzzxg/fjxPP/00f//9Nz/99BOrVq0ythEfH8/IkSPp3r07PXv2ZN68eRQUFBhXVes8Le6E6HvgxCpYMx6eWmkWO9g1sj4HL+Sw99y1CnPElbXgVG56BVei0ZFfLFZRqxOjBeesVdSTa/Vhf4obP5IuLiYdfcMX7mRGHlqdjNJJQ2+HFdzBgwfp3LkzgEXxGUdzje3Zs4cBAwYY3xvmwUaOHMnixYtJS0sjNTXVeLxp06asWrWK1157jY8//pjGjRvz9ddfG11EAIYPH05mZiZTpkwhPT2dzp07k5CQYLHwUKeJexdOr4eUTXD8T2gz1HioS2QAi7fZ5/Br8IHz89QrOA8TBScC7asXjbGqlhMUgaa4NGnDbS/owwBdTGSgN1ENvIls4ENuoZr6Jm5JN4PDCm7Dhg1OuTBA//79y81uYS1KoX///hXmnxs7dixjx4692e7VXgKbQu+X4Z+5+mFHi1hj/q6yDr9l8+SbYlhk8DUMUQ0KTqs1S5UkDDjXY3D0dcoQdccCfbhfvRC4442bb68SKBUSG98YULGgg1RjWWxBldI3HnzDITsVtn1m3G1w+NXoZA5dzCm3CVtD1GK1sOCqG6eFauWm6VdOAWKn68P/6hBCwdVVPHzgrhuFef/5AHL02XwNDr9QceB96SJDmSGqWEWtdpwWbL9+mj7Mr3EP6GhZ46O2IxRcXab9QxDZGzSFpcVCsN/ht9SC0w9RVW764Wyx2twPztEkmoKbxyl1UVN3wsEf9K+HzLaIfqkL1L07EpQiSfoPrqSAI7/B2S2ApcOvLQwVtYwWnMFNRCvcRKqSkxl5HDifXa6M+mYdfXVa/So76MP8GnWrXDs1HIcV3ObNm9FoLD/cGo2GzZs3O6VTAicS1hG63XCRWTMBtBozh98L12w7/BqGqIZFBpW7YQ5OK5JdViF3fbSZYfO3crXAdhLTmw7V2vedPqxP5acP86ujOPx0BgwYYLWqfU5OjpnLh6AGMXAyeAZAxmFI+hZPdyXtjA6/toepZRcZDBZcSZlgezFArRrScmz/+NxUsH1htr70JOjD++o1rETvagcOKzhbMYxZWVn4+Di/rqHACXgH6pUcwN/vwPWrdLkxD7evnIiG3DJD1FILTuSDcwXlTW1qbmYVdeMsuJ6lD+vr+Vwle1c7sNsP7sEHHwT0q3CjRo1CpSqNU9NqtRw8eLDaYlQFdtBtNCQt1ltxf79D1ybjWLztbLkOv4Yhqr+XIVRLv8hgGWwvcBb2LthU2g/u8jHY9aX+9ZBZ+vC+OozdCs7f3x/Q/wN8fX3x8iot/Orh4cFtt93GmDFjnN9DgXNQuukXHBbfA0nfEtPqUcC2w68syxaLDGYWnMkig1hEdR72PstKBdvLsn4eVtZC63uh+cBK9LB2YbeC+/bbbwGIiopi3LhxYjhaG4nqA+0egCO/03DLFILrvU5mfonVDL8FJVpuGAkWc3DFGq1IeFlFmOo3e4aoDlW2P/6nPnxPqYK73qlcB2sZDg/gp06dKpRbbWbQDHDzQkrdznOB+pA3aw6/hjAtd6VkjGAwOPrmFWmMlbVA+ME5E0eHqHZbcOpCfdgewO2v6MP5bgEcVnAZGRk8+eSThIeH4+bmhlKpNNsENZyACH0YFzAi5yu8KLK6kmq6wGBYVDIoumvXbbsvCG4Ond1DVAeD7bd9qg/b82sEfV6rZO9qHw4H248aNYrU1FTeeustwsLCHM4gIqgB9H4Z9v2PetmpvOD2B/9LfdJiddwYpuVVOgltUHDl+WcJbg7ZZJAql+OAYwzVsmcVNfs8/HMjjdldM/RhfLcIDiu4LVu28M8//xhTJglqIe5eEPce/PgE/1au4uf8fly4VkhEoLdRxDBENSS7hNJQrezrarPmxADVeZiOUMudg9M5YMGte0sfrtfkdmj34E32sHZRqbqoYs6lDtD6XmjWH5WkZrLb9xbD1LJOvlA6BycsuOpHbW+wfco/cOR3fbjekNkuT2RZ3Tis4ObNm8fEiRM5e/ZsFXRH4DIkCQbPRoeSOOUecg+vNTtcNtkllA5RC2/URPUqJ5ecLXKuq7n743/4fOPpioVvQcwsuHLk7HL01Wpu1DdF7wcZ2uHmO1jLcFjBDR8+nI0bN9K8eXN8fX0JDAw02wS1iIatSWn+OAD9Uz4AbenQs2yySyi14Az4qG4oOAcM+kVbUzialsuchBOV7HTdxmwOrpyRktqeyvZJ38LlI/owPUMkyy2Gw3Nw8+bNq4JuCKoLz0FvcuX0CiK0F1BvX4h7n5cB60NUwxycAR+VG1fyHRuuGlb/BNaxd/bH6Adny02kIEsflgd65eZ9axofDiu4kSNHVkU/BNVEeEgo77o9zmTtAqRNs6HzcKjX0KKiFlix4Dwc/vg4t8xdHUS28doUnU42upPYTJe04R0oyoaQ9qXZZG5BKvVpS05OZvLkyYwYMYLLly8DsGbNGo4cOeLUzgmqHkmSuNjkIQ7qmuKmzjNmmbBuwZl/XOqp9ArOkSUnhzzvb0FMh6W2rDm1rtQKtvqDkXYQ9ugjjxgyWx+md4visILbtGkTHTp0YOfOnfz222/k5+cDcODAAaZOrVxeqfnz5xMVFYWnpycxMTHs2rXLpmz//v2RJMliu+eee4wyo0aNsjg+ePDgSvXtVqBLVCDT1Dcs833fwYUkq4sMNufgHMD0CylW4y2x54kYfODAyiqqId4UWe8SEtXHqf2rbTis4CZOnMg777zDunXr8PAoLe01cOBAduzY4XAHfvzxR+Lj45k6dSp79+6lU6dOxMXFGS3Dsvz222+kpaUZt8OHD6NUKnn44YfN5AYPHmwmt3z5cof7dqvQNbI+e+VWrJL66XesGU9eYTFQmq4cLBWct8GCc0BRmQ6pSsR8nAWyHY/EVMFZrKIe/hVSt4GbV2lNjlsYhxXcoUOHeOCBByz2N2zYkCtXrjjcgQ8//JAxY8YwevRo2rZty8KFC/H29mbRokVW5QMDAwkNDTVu69atw9vb20LBqVQqM7n69es73LdbhfaN/HFXSkwvfASduw9c3MNteesA8LXiJmLAx8NxC860oG9RiVBwZZHtmIUzHaKaWXAlBaW1N/q+Dv6Nq6CHtQuHFVxAQABpaWkW+/ft20ejRo0caqukpISkpCRiY2NLO6RQEBsby/bt2+1q45tvvuHRRx+1SACwceNGGjZsSHR0NC+88AJZWVk22yguLiY3N9dsu5XwdFfSNsyPy9TnWKvnARhTspR6XK9giHpzczsGfzpBKfZEMhgsOKVCMg+V/OdDyLsEAU304XgCxxXco48+yoQJE0hPT0eSJHQ6HVu3bmXcuHE89dRTDrV15coVtFqtRdX5kJAQ0tPTKzx/165dHD58mGeffdZs/+DBg1m6dCmJiYnMnj2bTZs2MWTIELRa61+omTNn4u/vb9wiIiIcuo+6gCHD72/uQ5EDmxNENi+7/W42RC3rJlKZRQZTNxGh4Cyx51mW1kQ1UW5Xz8C2T/Sv494Dd0/nd64W4rCCe++992jdujURERHk5+fTtm1b7rjjDnr37s3kya51Jvzmm2/o0KEDPXv2NNv/6KOPct9999GhQwfuv/9+/vzzT3bv3s3GjRuttjNp0iRycnKM2/nz513Q+5qFodLW7gsFFMe+C8BoZQL+BWeNMmWHqN6VcBPRmKTLKBQ55SwwW0W1IVOazdfk/7F2MmhLoNkAaH2PjTNvPRxWcB4eHnz11VckJyfz559/8t1333H8+HH+97//OZwuKSgoCKVSSUZGhtn+jIwMQkNDyz23oKCAH374gWeeeabC6zRr1oygoCBOn7YeHqRSqfDz8zPbbjUMxaCPXsolI+QOErVd8JC0eP092ThW8lCWdRNxfA6uRCMsuPKwJ+GlpmyqpNPr4cQqkJQweNYtF29aHpX2uoyMjOTuu+/mkUceoWXLlpVqw8PDg27dupGYmGjcp9PpSExMpFevXuWe+/PPP1NcXMwTTzxR4XUuXLhAVlYWYWFhlernrUCjAC8a+qrQ6GS2nL7CDM0TqFEinV4PJ/VxqgqFZDap7WNcRbX/OqZD1CKh4CwwfZY6Gw+2tCaqAjQlsGai/kDMv6Fh66ruYq3CrjFGfHw8M2bMwMfHh/j4+HJlP/zwQ4c6EB8fz8iRI+nevTs9e/Zk3rx5FBQUMHq03vv6qaeeolGjRsycOdPsvG+++Yb777+fBg0amO3Pz89n+vTpPPTQQ4SGhpKcnMz48eNp0aIFcXFxDvXtVkKSJLpG1ifhSDobT2RyVg7jJ7dhPK75DRImQvMB4KZC5aZErdX7yFVmkUEMUcvHdBXVloIrrYkq6QvIZJ0C7yDoN8ElfaxN2PUJ3bdvH2q12vjaFpVJfjl8+HAyMzOZMmUK6enpdO7cmYSEBOPCQ2pqKooyvj4nTpxgy5Yt/PXXXxbtKZVKDh48yJIlS8jOziY8PJy77rqLGTNmmFUCE1jStUkACUfS2XZa7+6zwu9RHi/ZBtdSYPt86BuvX0nVu8hVKlTLdIh6XVhwltgRq2Ww4BoqsvUlAAFip4JXQFX2rFZi1yd0w4YNVl87i7FjxzJ27Firx6wtDERHR9t0LvXy8mLt2rVWjwnKp+uNldSCG5aVu5c/9Hsbfn8ONs+FTo+aLTQYIhnKyzxbFo2JD1eRsOAs0MnWX5timIN7Xv09aPIgvAt0rniq5lZERD4LjBgcfg34ebpDx0egcU9QF8C6qWa+cPUqMURVa0yGqMKCs8C+IapMJ+k0QzQ35q6HzIHKFIC+BbDrE2oo+mwPv/32W6U7I6hePN2VtA3358D5bOBGmJYk6QO2vxoIh36ii3dXzhGJJGGspVrZRQah4CyxJ+GlWqNhmvtS/ZtOIyCipw1JgV1q39QJ1s/Pj8TERPbs2WM8npSURGJiorE4tKD2YnAXAZMwrUZdoeuTAIwt/goFOrzdlZXyRlCLRYZyMVVqtiy44DO/00VxmkLJC2KnuaRftRW7LDhD0WeACRMm8Mgjj7Bw4UKj35tWq+XFF1+8Jf3H6hpdI+vz7dazgHkmEQZOgSMraVGczHDlBtar7q7UopJaI9xEysM8XZIVBVeUS4sDcwH42ecxnvIt31/0VsfhgfuiRYsYN26cmVOvUqkkPj7eZoC8oPZgiGgA80wi1AuGAZMAeMPtR0I9ioyHHAnVMl1kEENUS8z84KzlItg8B1XxFZJ1YSTUu99V3aq1OKzgNBoNx48ft9h//PhxdFb/I4LaRLi/Jw199e40ZhYcQI9nuejehEApnzGaH6iMv3yJVgxR7cXihyPzJOxYAMAMzZMo3ITbU0U4vAw2evRonnnmGZKTk40xoDt37mTWrFlG51xB7UWSJB7o2ojvtp+jU0SZOVWlOz81eInX0sdzT/Eqsq4cc7h9jVb4wZWHzUgGWdY7XOs0XGrYj42pnelvb1X7WxiHFdzcuXMJDQ3lgw8+MKZNCgsL44033uD11193egcFrmfSkDaMuyvaakGTZL8erLnYgyHK3fhvegt4waExqlmolrDgLLBZVetkAiQngtKDPa3HQ2pO+SUDBUAlhqgKhYLx48dz8eJFsrOzyc7O5uLFi4wfP97hYHtBzcVWtSaVm5J3NU9QInmgOr+FwYrdDrVrNkQtY8H9vOe8RQHqWw2r+eDURXrrDeC2F8n20ieyrLDos+DmHH1v1cwbtzIebgouyMFsCnoMgMnu3+FpiN2yA40NP7hdKVd545eDPPj5Nud1thZiOiw1etTsmA/XzkK9ULhjXGmwvahQViGVSsn6yy+/8NNPP5GamkpJiXldzL179zqlY4KaiSFUa1ejpxhYtI7GeRd5RvEHcL9d5xcUa4yvTRcZzmYVOLObtRYLP7ici/owOYBBb4PKF41WX69EVCirGId/Aj755BNGjx5NSEgI+/bto2fPnjRo0IAzZ84wZMiQquijoAbRKcIfSYL2UaHk3jENgH8r/oDs1ArP3XD8Mmezrhvfm/rB2azveYthsciwfiqor+vD5To+ApRmZCm3qr0AqISC+/zzz/nyyy/59NNP8fDwYPz48axbt45XXnmFnJycquijoAbxQJfGHJ4Wx7DOjShuOZQdujZ4Smr4q+JszuN+PmD23nSIKoZbBko1XGBWEhz6GZDg7jnGRJbGlOXimVWIw08oNTWV3r17A/rMHXl5eQA8+eSTojTfLYIhD5ykkJimHolWluDoSjizqdzzyi62mg5RhQWnx2DBKdDR4eB7+jddn9JnDLmBoeiMGKJWjMMKLjQ0lKtXrwL6rL6GWqgpKSmikO8tyHE5ku+0N6qirZkAWk35J5hQpDap0G7yZb2VP0eGOx+u3EBAzjFQ+cOdU8xkDGUDhQVXMQ4/oYEDB/LHH38Aeqff1157jUGDBjF8+HCr9VIFdZ8PNQ9ToPCDzGOw5xubcmUVV4lWZ1lfgNKEjrcisgx+5POG24/6HQPeBJ8gMxmNVszB2YvDq6hffvmlMSTrpZdeokGDBmzbto377ruPf//7307voKDmYvh65VCPxV5P8lLBfNjwLrR/yOJLCdYTOBaqtfgqFWZOq2qtzqIG662CjMxrbr8SKOWT49sC/x6WRZUMPwruwtG3Qhx6QhqNhnfeecesZumjjz7KJ598wssvv4yHh4fTOyioHfyliqMkqB0U5XD250ks3JTMlJWH+X7nOaOMtaGnYaHBdIhqGu1wq+Fx5RhPKtcBsLftBFC6W8ioxSqq3Tik4Nzc3JgzZw4ajf3zLIJbgwOX8nn84kMARKb8zP8lrGHp9nP89/fDnLvh42Ztaq3YMA9n8l0t0erYcuoK9376D4cv3kIr87JMyLZpuEk6Vmt7kt7gNqtiRgtOzMFViMNP6M4772TTpvJXywS3Jrvl1vyl6ItCkvk04AeiAr0A2HwyE9ArLpuYKD+1VuaJb3Zy+GIuo751LBSsVnN0JT6XtlEku/Oe5vFy6qIaygYKC64iHFZwQ4YMYeLEiYwbN47ly5fzxx9/mG2VYf78+URFReHp6UlMTAy7du2yKbt48WIkSTLbPD09zWRkWWbKlCmEhYXh5eVFbGwsp06dqlTfBLYpKBMs/8Rtkdz1ny/A3ZtmhYd4M/IIAJtO6qt0mQ49DbkyDV9i0++yaThX9nXzSJk6S8l1oy/hF9qhXJCDbddF1YlQLXtxeJHhxRdfBKzXP5UkCa3WsQwRP/74I/Hx8SxcuJCYmBjmzZtHXFwcJ06coGHDhlbP8fPz48SJE2bXNWXOnDl88sknLFmyhKZNm/LWW28RFxfH0aNHLZShoPJEBnqbvf/Pna3AVwV9X4e/ZzAg9VO8mcn25CuUaHRmiwzuSoVZCUHZzIIr3W/rS14b0Gh1KCQJhT2W1taPIec8JT7hLMgaCth2lykdogoLriIc/gnQ6XQ2N0eVG+gV5ZgxYxg9ejRt27Zl4cKFeHt7l5sdWJIkQkNDjZuhhiroPxTz5s1j8uTJDBs2jI4dO7J06VIuXbrEihUrHO6fwDZKhcTpd4cwYXBr/hh7O8E3EmXSayzUj8L9egYvu6+koETLgo3JZucanFQN6YFMFVmJSeUtW6XzajolGh19Zm/gvvlbbMrMW3+Sl5btRXf1LGydB0BazGSK0D9Hm0VnTCvbC8qlWp9QSUkJSUlJxMbGGvcpFApiY2PZvn27zfPy8/Np0qQJERERDBs2jCNHjhiPpaSkkJ6ebtamv78/MTEx5bYpqBxuSgUv9G9Ox8YBpTvdPSFO74X/tGIVTaR0Plp/0uy8sla36Ze5LqyinkjPIz23iMMXc23KzFt/ilUH07i2YgJoiiCqL9lRdxuP62xod43O0ndQYB27h6iFhYUkJiZy7733AjBp0iSKi0vT5CiVSmbMmOHQEPDKlStotVozCwwgJCTEalp00Bd9XrRoER07diQnJ4e5c+fSu3dvjhw5QuPGjY0uLNbaNHVvMaW4uNjsXnJzbX8oBXYSfTc0H4gq+W8mu33HGPU4q2IGw83UgqsLCs5Uf8uybLNATy/FERqkJoCkoGDgu/znx/3GY7YLP98I1RIKrkLstuCWLFnCF198YXz/2WefsW3bNvbt28e+ffv47rvvWLBgQZV00pRevXrx1FNP0blzZ/r168dvv/1GcHCwWd8cZebMmWalESMiIpzY41sUSYLBs1DLSgYp99Jfsd/8cFl5ky9zuauttQSFiULT2tBUSrRMc1uif9PjWT457GGWbcXmIoMh+kMMUSvE7if0/fff89xzz5ntW7ZsGRs2bGDDhg28//77/PTTTw5dPCgoCKVSSUZGhtn+jIwMQkPtK4fm7u5Oly5dOH36NIDxPEfanDRpEjk5Ocbt/PnzDt2HwAbB0SzWxgHwltv/cMfSf9LwFTa34GrpxJsJpgab1oaiekK5nmjFBdSq+tB/Eum5RVblymJIlyQsuIqxW8GdPn2aDh06GN97enqiMPkF6dmzJ0ePHnXo4h4eHnTr1o3ExETjPp1OR2JiIr169bKrDa1Wy6FDhwgLCwOgadOmhIaGmrWZm5vLzp07bbapUqmM2YlFlmLn8onmQTJlP5or0hilTCg9UOa7abaKqqn9Fpz5ENXyuC4vk3i3nwG42HUceAdaylSwiiosuIqx+wllZ2ebzVNlZmYSFRVlfK/T6cyO20t8fDxfffUVS5Ys4dixY7zwwgsUFBQYK3Q99dRTTJo0ySj/9ttv89dff3HmzBn27t3LE088wblz53j22WcB/eT1q6++yjvvvMMff/zBoUOHeOqppwgPD+f+++93uH+CmyMPb+ZoHgXgP26/EYx5zQWDK0Rdm4OraIgqb5iBv3SdI7omXI3WP5+y+szWHJxaBNvbjd2LDI0bN+bw4cNER0dbPX7w4EEaN27scAeGDx9OZmYmU6ZMIT09nc6dO5OQkGBcJEhNTTWzFK9du8aYMWNIT0+nfv36dOvWjW3bttG2bVujzPjx4ykoKOC5554jOzubPn36kJCQIHzgqoEHujTil3138LhyPZ0VZ5jg/iPj1M9bzMGZraLWVt8QExTlDVEv7UexdykAU9UjmeLmxuXcIjYcv2wmZjOSQSdCtezFbgV39913M2XKFO655x4LRVFYWMj06dO55557KtWJsWPHMnbsWKvHNm7caPb+o48+4qOPPiq3PUmSePvtt3n77bcr1R+B8/jwkU409FMxbfMoVqim8C/lZr7X3EkybYBSa8TUqbUuDFFNx+Bm7h6yDGvGIyGzQtubPXJrFJLEHe9vMMuPB+UNUUWolr3Y/RPw5ptvcvXqVaKjo3n//fdZuXIlK1euZM6cOURHR3Pt2jXefPPNquyroBYiSRIeSgX75Rb8rLkDgGnuS2gRrI+CWLhJ7wBsK5KhJnMpu5DVh9Ks+quZzsGZHT70M5zfiezuw0y1vjJZkVprodzAdiSDSHhpP3ZbcCEhIWzbto0XXniBiRMnGh++JEkMGjSIzz//3ML3TCAAfcQDwBzNowxW7qaT4gzzWh+l/4XG/L7vIvd1CjdTArVFwd0++29kGWY92IFHe0aaHTO1rYxzcMV58NdbABTG/IeM9fqFhTd/P2S1feEHd/M49BPQtGlTEhISyMzMZMeOHezYsYPMzEwSEhJo1qxZVfVRUMtR3jBnMgngY82DAETufZ8XYoIB/Rc8v1htlC+pJW4iBgNra3KW5TGT18ah5j8fQH461I8iv2tpctiTGfnltl8WEaplP5V6QoGBgfTs2ZOePXsSGGi5vC0QmKI0sTSWaONI1oVBQSavevxOmL8naTlFbDNREjXdgpNlmbwitdl7S5nS11qdDFnJsH2+fkfcTLRKVYXXsTkHpxPB9vYifgIEVU6QT+mXWY0bZ7rrh2nue77kNl99KqVikzmomr7I8NbKw3SY9pfxvTU1ZKr0tDoZ1r4J2hJofidEDzEOM8vDdjYRkS7JXsQTElQ5YQHmq+7pDftAqyGg0zAydyEg1yo/uO92mBe5Lra2QGDy+stFX8DJBFC4weBZIEk2w7dstWFKaaiWsOAqQig4QZUTHuBl9l4hAXHvgtKDziV7GaRIMvsy15Y5OAPrj2Ww5+xVs30Gfe2OhlG5N+KkY56H4FZAabhVedgeohoWGcTXtyLEExJUOeH+5gpOKUnQoLk+bxz6OFWltjQKRlPDLThrvLv6mNl7Q567UcoEmivSwKch9JtgPG6PBWcqkl+sYcm2s6TnFBnPFZEMFSMUnKDK8fJQEuBdWh3KGMbU93WuKhoQqcjkrpyfjcdr+hDVGhZhVjoI5hr/cftNvyN2GniWxjgbFgrKw9SCm5NwnKl/HOFfC7cZ94mygRUjnpDAJZhacUYnWFU9/uenjyG+O2cZYehXUmvbENUaMjIT3H+knlTEfl1z6DTC7Lg9FtwXm87wwndJaLQ6Eo/pw7guXCs0HhcWXMUIBSdwCeEmCw2mgejnwoawW9cKT7mYSe7LAMxqNdRWPNP38i/lZkAfb0oZa8ueOTiANYfTSTiSjq+npU++UHAVIxScwCWEmVhwSpPVv0l3t+X7wLHoZIn7lNvpKR1j11lLx9lahU5H2LYpAPyk6ccBuYWFiD0WnIFZa45TT2Wp4MQQtWLEExK4BNOVVNM4zWBfFZ173MFy7UAAprsvITOnwNXds4tzWQWcvpxXseD+7/G+cpBc2cuYKqosjii4C9cK8fJQmu1TSNhXresWx+GygQJBZbA1RAWo5+nOu5qHuVe5nTaKVP6lTeQ7Brm6i+Wi08n0e39jxYKF2bB+GqBP9nkFf7PDGq2OL/85YzMMy+b1y5wgnHztQzwlgUuwNUQFULkpuIYfH2geBuB1t5/xx3p8ZnVwKbuQi9mFFQsCbJoD169Q5N+cJTfStZvyS9IF5iSc4P21J6ycbJv8IvN07+7CerMLoeAELiE8wHayUZWb/mP4vTaW47oI6kv5xnTe1U2JRkfvWX/Td86GioUvH4ddeqfeCzFvoS4zQJq5+hgTf7OeOaQi8orNFZyw4OxDPCWBSwjxK1VwVwtKzI6p3PXzS1qUTNc8BegLsrSWzEOiqoPrJZaFcqwiy5AwAXQaiL6bnEb9LUS+2Hym0v2wsODECqpdCAUncAmmYUWXy1SPMlhwANt17Vil7YlSkpnmvgTbEZmuwVY907LElGyHMxtB6QFx7zrd1SW/rAUnVlDtQjwlgcswzL11iaxvtt9UwQG8p36cItmd2xTHuEex02Z7Gq2OFfsucsne+bFKYM9Ul4oSRud/rX/T+2UIbMbE3w6aydiqUm8v10u0Zu+FD5x9CAUncBlbJwxk8ege9I8ONtuvcjN3gbhIMAs09wEw2f17KLHuNrJ421le/XE//d63Y36sCnlWuZpQXTr4hkOfeADOmRRwBvsde+1FBNrbh3hKApcR6u9J/+iGFsM+lbvlx3ChdigX5CDCpCy+/+A1DpzPtpDZclqfS64qC0VX1HIYWbzktlL/5q4ZoKpnVc5WZpDKIlIl2UeNUHDz588nKioKT09PYmJi2LVrl03Zr776ir59+1K/fn3q169PbGyshfyoUaOQJMlsGzx4cFXfhqCSeLorLfYV48E76icA+FfRb/z32z85fDGHpdvPGod7Zf3pqoKK9NIk92V4S8UcdWsH7R+yKeeIY689iFVU+6j2p/Tjjz8SHx/P1KlT2bt3L506dSIuLo7Lly9bld+4cSMjRoxgw4YNbN++nYiICO666y4uXrxoJjd48GDS0tKM2/Lly11xO4JKUHYOzkCCrgdbte1QSWrGqhdz76dbmLLyCH8cuATYNz9205Sjl3pKx7hPuR2tLPGV7/PmIRplsKiNWgl8TKIZxCqqfVS7gvvwww8ZM2YMo0ePpm3btixcuBBvb28WLVpkVf7777/nxRdfpHPnzrRu3Zqvv/4anU5HYmKimZxKpSI0NNS41a9f32p7gurHloIDiemap9DICgYrd9NHofchO5aeqz/qCgvOhoZTomW6+xIAlmsHcsatebntaJ0wjG7fqDQqwloWYYEl1argSkpKSEpKIjY21rhPoVAQGxvL9u3b7Wrj+vXrqNVqi+I3GzdupGHDhkRHR/PCCy+QlWU7gLu4uJjc3FyzTeA6yi4ymHJSjuB/Wn3Y1lS3pbihMVpVrrDgbBleI5R/00aRSrbso4/AqMBCc4YF1yy4dH4vu7CkHEmBgWpVcFeuXEGr1VrUUw0JCSE9Pd2uNiZMmEB4eLiZkhw8eDBLly4lMTGR2bNns2nTJoYMGYJWq7XaxsyZM/H39zduERERlb8pgcNYG261bFiPDjcslo80D5El+9JScZGnlOuM81kumYOzsi+APF6/EWnxgeZhruFXoQLr/s76m+5LuH+ps3T2dXU5kgID1T5EvRlmzZrFDz/8wO+//46nZ+k//9FHH+W+++6jQ4cO3H///fz555/s3r2bjRs3Wm1n0qRJ5OTkGLfz58+76A4EYH2oKUnQ0FdfjSuXeryvGQ7Aq26/8PuW/UDVKTiNVsfW01e4XqKxWtkq3u0X6kv5HNNFsEx7J6DP4FvVhJoouOI6kDPPFVSrggsKCkKpVJKRkWG2PyMjg9DQ0HLPnTt3LrNmzeKvv/6iY8eO5co2a9aMoKAgTp8+bfW4SqXCz8/PbBO4ls4RAWbvoxr40NCvtNzgT9r+HNZF4ScV8obbj0C5c/o3xWcbTvP41ztpO2WtRXX5NtI5HlfqrbG3NU+hRT+8drYbiDXKFu8RVEy1KjgPDw+6detmtkBgWDDo1auXzfPmzJnDjBkzSEhIoHv37hVe58KFC2RlZREWFuaUfgucz8/P92Lx6B7G90M6hBLsW2qx6FDoM+MCjyg3wcW9VbbI8N2Oc8bXu1JMq2Xpw8eUksyf2hi269qV9s8FCs7UghPYR7UPUePj4/nqq69YsmQJx44d44UXXqCgoIDRo0cD8NRTTzFp0iSj/OzZs3nrrbdYtGgRUVFRpKenk56eTn6+Pr1Ofn4+b7zxBjt27ODs2bMkJiYybNgwWrRoQVycZfoaQc3AXangjpbBtA71JaqBN0PahxmHqAaS5Gh+196OQpJhzQSUVM0wzbyaVelc172KHcQojlMoezBT/ZjZOYZ5wcISrUWsrbMI8/ekvknxHkHFVHvCy+HDh5OZmcmUKVNIT0+nc+fOJCQkGBceUlNTUZgEFi9YsICSkhL+9a9/mbUzdepUpk2bhlKp5ODBgyxZsoTs7GzCw8O56667mDFjBiqV+RdGULNQKCRWv9KXEq0OT3elhYIDmKUewV2KPfhc2EWP8PWsoL3T+2E672Z46UWRsWbEAs19XMQ83Cw5s4DzV6/bl1apknh7uDH/sa6M/HYXb97dpsquU5eodgUHMHbsWMaOHWv1WNmFgbNnz5bblpeXF2vXrnVSzwSuRqGQ8FTo57Ua+lkOyTII5DPNA0xw/4GhlxfyHu9TgHPnpkwtOMPL593+j0ZSFhfkIL7Q3mv1vIcX2ufaVBkMyr53iyCOTB+Mh03fQYEp4ikJaizWLDiAb7RDoH5T/DRXedlthdOvW9aCayxd5nnlnwDMUD9BMR5Wz7ucVzVDU8BswUUoN/sRT0pQYwmqZ13BleAOg2cB8LRyNU2lNPKKnOcXJptZcDKT3b5HJanZom3HWl0Pm+c5OdzUjIa+YoGhMggFJ6ixeLgpeKG/jRCoVnFs0HbCQ9Lyltv/eGvFYadd11RPhWRuZ7ByNxpZwXTNSKB6YkD9vcTiQmUQCk5Qo/nPnS2tH5AkZmiepERWMlC5n8Ijq512TYPLhxsauh+bDcBS7V2ckhs77RqO4oqojbqIUHCCGo2nu5I/xt5u9dgZOZxF2iEATJSWgKbYKdc0KLinlOsIKDhDluzLPI3tVEiuQOi3yiEUnKDGE2xjsQHgM839XJYDaKrIgB2fO+V6OhkakMOrbr8AMEfzKLn4OKXtyiL0W+UQCk5Q4/F2t/RmOnpJn/ElH29mqW9Uj9/0PuSmVeoa+89n8/dxfcigLMu84fYjflIhh3RR/KztV7mO28GTtzWxS05YcJVDKDhBjcfLwzKdkmk5v991fdinawHqAs4sf52cQsdXVO+fv5WnF+8h5UoBbXSn9eFgwDT1SHRV+DVpHmyfZSjm4CqHUHCCGo81vy+FSTI42SROtVnaKka//RkA649msGBjstWMILZYuu0M09yXoJBkftP2IUmOvsnel4/SJPX4hMGtbcoJ/VY5hIIT1ErK1jg4KDfnR01/AKa5L6GwqIRnl+5hdsJxtp+xney0LDk7vqOr4jQFsopZ6hHO7LJVlCaay9uKpVqK0HCVQSg4Qa2kbJ1QgPc1w8mVveioSOHd9/5r3H8pu/wIg6Rz1wDwoZCJ7j8A8KnmAS5T9WnuTatjlZehWFhwlUMoOEGt4J4O5qmurhZYuoRcwZ+Pb7hzvCYtxw99PVVtBdkod5/Vp0R62W0FDaVsUnQhRveTquDejqX3ojTVauVoMaHfKodQcIJawWePdaFNWGki0it51msSLNHexSldIxpIebzq9itQcdFlrU6mqZTG00q9s/Dbmqf04WBVxMDWDY2vTSvUl6fExCJD5RAKTlArkCSJoHqlQe5X8q079WpwY7rmKQCeUv5FS+kCAGqtdSvuXFYBRWp9uJeHpOVvbWc26Lo4uffmeJnUgTW14MrTYUK/VQ6h4AS1hi4mac0zbSg4gC26DqzVdsdN0jHVbQmf/32adlPXsvGEea3d1YfS6Pf+Rg5v/JmByv2UyEpmaJ6squ4b8TRZTDCdg5OQbFasF/qtcggFJ6g1vDighfH1b3svliMJ72gep1h2p4/yCO3zNlOi0TF68W4zmW+2pOCBmiluSwFYpL2bFLnq0tr7eeoTVvp4lDou+6hKX0uS7VRILUJ8q6xfdRmh4AS1Bk93JcO721fS8bwcwhfaewB4y/07VJRYlC71dFfwtHINTRUZXJYD+FRzv5N7bM7wHhHc0zHMrNB1PRMFp5Qk3JWWX8n4Qa0Y0UOUsqwMQsEJahXubvYP1hZo7uOSHEhj6Qr/vpGw0pSstHO87PY7ADPVI5yeGbgsnjfm3lTupV87X093nurVhBYN6zG0UziD2oZYnPfKnS1xs6L4BBVTI1KWCwT24kjix0I8mal+jE89PuNFt5X8qu1rdvy5kqX4KIvZq2vBCp31jCXl8WCXRtTzdGPp9nMVC4PRcvMwUVZ+nm68Pay0rsS0+9rRPtyP/GINc/866XCfBOaInwVBreKZPk3tDlAH+D9dL3bqWuMpqY1FYwC+/O57HlRuQSdLTFOPRK7gqxDTNJBmQeZxox8O70yPqEC7+2Kw4ExLDPp6mruj1FO5Mer2pkQ2qN7sJXWFGqHg5s+fT1RUFJ6ensTExLBr165y5X/++Wdat26Np6cnHTp0YPVq82SHsiwzZcoUwsLC8PLyIjY2llOnTlXlLQhchI/KjRn3t+fn53vx6wu9+OG523jr3rblBK1LTFc/hVaWuFe5E13yJjRqNb1OzgHgJ20/Dso2sgabsGzMbfw9rr/x/YBofVUtUzePf/drVm4bqhsKzrSAs6e79a+ghxiSOoVqf4o//vgj8fHxTJ06lb1799KpUyfi4uK4fPmyVflt27YxYsQInnnmGfbt28f999/P/fffz+HDpSmr58yZwyeffMLChQvZuXMnPj4+xMXFUVRUdUVBBK6lR1Qg3ZoEcluzBjzTpylfj+xBIxuV34/KUSzT3glAzu/x7F/5MR0UZ8mVvXhfM9yu6xnU2N63BvHWvW354JHOgLkDrukihtKKu4fB/83bw41d/72T/VMG2SxerbKh+ASOUe1P8cMPP2TMmDGMHj2atm3bsnDhQry9vVm0aJFV+Y8//pjBgwfzxhtv0KZNG2bMmEHXrl357DN9BglZlpk3bx6TJ09m2LBhdOzYkaVLl3Lp0iVWrFjhwjsTuJKmQT5snTiQO1oFWz3+geZhsmUf6uefpsuhdwCYp/kXWfhble/bMsjsvUEPBfp48EyfpgT66J2OTRWZn6eJy4fJuSN6RtKiYT2zcLOGvp4EeFuvzgWgEhacU6jWp1hSUkJSUhKxsbHGfQqFgtjYWLZvt15jcvv27WbyAHFxcUb5lJQU0tPTzWT8/f2JiYmx2WZxcTG5ublmm6B2svTpnhyadhcpM+/mnftLJ++z8WWu5hEAlJLMKV0jlmoHWW3jn/ED+HZUD0b0jARg3vDOti0tE5eP0bc3pV+rYGY92MGs5OHMBzuw7rU7rOa1s0Wov6ii5QyqdRX1ypUraLVaYxV7AyEhIRw/ftzqOenp6Vbl09PTjccN+2zJlGXmzJlMnz69UvcgqHkYJu6fuK0JF64VsvZIOs2D63Ek9wEOZm6knXSWqZqR3Ns5krfubYubUsHFa4Xc/ck/DGobQkSgN6BXTDMf7FDutXo3b8BtzQKJDvHFR+XGkqd7AtA5MoApK4/w+qBWADYVpC2aBdfjvQc6GC1FQeUQbiLApEmTiI+PN77Pzc0lIkI4VtYFJg5pzcQhpYkkr+f9Tc61DJZFtjGT8/dy58DUu8yGmfbgplTww3O9LPa3DvXjp39b7neEx2Iib+p8QTUruKCgIJRKJRkZGWb7MzIyCA0NtXpOaGhoufKGvxkZGYSFhZnJdO7c2WqbKpUKlcp2YRNB3cHbNxBvX+uuHaL2aN2jWufgPDw86NatG4mJicZ9Op2OxMREevWy/uvXq1cvM3mAdevWGeWbNm1KaGiomUxubi47d+602aZAIKijyNXMDz/8IKtUKnnx4sXy0aNH5eeee04OCAiQ09PTZVmW5SeffFKeOHGiUX7r1q2ym5ubPHfuXPnYsWPy1KlTZXd3d/nQoUNGmVmzZskBAQHyypUr5YMHD8rDhg2TmzZtKhcWFtrVp5ycHBmQc3JynHuzAoHAKdj7Ha32Objhw4eTmZnJlClTSE9Pp3PnziQkJBgXCVJTU1EoSg3N3r17s2zZMiZPnsybb75Jy5YtWbFiBe3bl66YjR8/noKCAp577jmys7Pp06cPCQkJeHqKlSmB4FZCkmUHSg7dIuTm5uLv709OTg5+fn4VnyAQCFyKvd9R4U0oEAjqLELBCQSCOotQcAKBoM5S7YsMNRHDtKQI2RIIaiaG72ZFSwhCwVkhLy8PQEQzCAQ1nLy8PPz9rSdMALGKahWdTselS5fw9fV1OIbQFobwr/Pnz9falVlxDzUDcQ96yy0vL4/w8HAzN7KyCAvOCgqFgsaNG1dJ235+frX2Q2lA3EPN4Fa/h/IsNwNikUEgENRZhIITCAR1FqHgXIRKpWLq1Km1OmuJuIeagbgH+xGLDAKBoM4iLDiBQFBnEQpOIBDUWYSCEwgEdRah4AQCQZ1FKDgncvXqVR5//HH8/PwICAjgmWeeIT8/v9xzvvzyS/r374+fnx+SJJGdne2Udl15D0VFRbz00ks0aNCAevXq8dBDD1nUzZAkyWL74YcfnNLn+fPnExUVhaenJzExMezatatc+Z9//pnWrVvj6elJhw4dWL16tdlxWZaZMmUKYWFheHl5ERsby6lTp5zSV1s4+x5GjRpl8bwHDx5clbfg0D0cOXKEhx56iKioKCRJYt68eTfdplWqNK/wLcbgwYPlTp06yTt27JD/+ecfuUWLFvKIESPKPeejjz6SZ86cKc+cOVMG5GvXrjmlXVfew/PPPy9HRETIiYmJ8p49e+TbbrtN7t27t5kMIH/77bdyWlqacbM3hXx5/PDDD7KHh4e8aNEi+ciRI/KYMWPkgIAAOSMjw6r81q1bZaVSKc+ZM0c+evSoPHnyZKsp7/39/eUVK1bIBw4ckO+77z6HUt7XhHsYOXKkPHjwYLPnffXq1Srpf2XuYdeuXfK4cePk5cuXy6GhofJHH310021aQyg4J3H06FEZkHfv3m3ct2bNGlmSJPnixYsVnr9hwwarCu5m23WEylwrOztbdnd3l3/++WfjvmPHjsmAvH37duM+QP7999+d2l9ZluWePXvKL730kvG9VquVw8PD5ZkzZ1qVf+SRR+R77rnHbF9MTIz873//W5ZlWdbpdHJoaKj8/vvvG49nZ2fLKpVKXr58udP7L8vOvwdZ1iu4YcOGVUl/reHoPZjSpEkTqwruZto0IIaoTmL79u0EBATQvXt3477Y2FgUCgU7d+6sce0661pJSUmo1WpiY2ON+1q3bk1kZCTbt283k33ppZcICgqiZ8+eLFq0qMJUNxVRUlJCUlKS2bUVCgWxsbEW1zawfft2M3mAuLg4o3xKSgrp6elmMv7+/sTExNhss6bdg4GNGzfSsGFDoqOjeeGFF8jKynJ6/6Fy9+CqNkWwvZNIT0+nYcOGZvvc3NwIDAwkPT29xrXrrGulp6fj4eFBQECA2f6QkBCzc95++20GDhyIt7c3f/31Fy+++CL5+fm88sorle7vlStX0Gq1xgJFptc+fvy4zf5akzf01fC3PBlnUhX3ADB48GAefPBBmjZtSnJyMm+++SZDhgxh+/btKJXKar8HV7UpFFwFTJw4kdmzZ5crc+zYMRf1pnLUhHt46623jK+7dOlCQUEB77///k0pOIFtHn30UePrDh060LFjR5o3b87GjRu58847q7FnrkUouAp4/fXXGTVqVLkyzZo1IzQ0lMuXL5vt12g0XL16ldDQ0Epf3xntVuU9hIaGUlJSQnZ2tpkVl5GRUW7/YmJimDFjBsXFxZWORwwKCkKpVFqs2JZ37dDQ0HLlDX8zMjIICwszk+ncuXOl+lkeVXEP1mjWrBlBQUGcPn3a6QquMvfgqjbFHFwFBAcH07p163I3Dw8PevXqRXZ2NklJScZz//77b3Q6HTExMZW+vjParcp76NatG+7u7iQmJhr3nThxgtTUVHr16mWzT/v376d+/fo3FWzt4eFBt27dzK6t0+lITEy0ee1evXqZyQOsW7fOKN+0aVNCQ0PNZHJzc9m5c2e591OT7sEaFy5cICsry0xpO4vK3IPL2rR7OUJQIYMHD5a7dOki79y5U96yZYvcsmVLMxeLCxcuyNHR0fLOnTuN+9LS0uR9+/bJX331lQzImzdvlvft2ydnZWXZ3W5138Pzzz8vR0ZGyn///be8Z88euVevXnKvXr2Mx//44w/5q6++kg8dOiSfOnVK/vzzz2Vvb295ypQpN93fH374QVapVPLixYvlo0ePys8995wcEBAgp6eny7Isy08++aQ8ceJEo/zWrVtlNzc3ee7cufKxY8fkqVOnWnUTCQgIkFeuXCkfPHhQHjZsWJW7iTjzHvLy8uRx48bJ27dvl1NSUuT169fLXbt2lVu2bCkXFRXViHsoLi6W9+3bJ+/bt08OCwuTx40bJ+/bt08+deqU3W3ag1BwTiQrK0seMWKEXK9ePdnPz08ePXq0nJeXZzyekpIiA/KGDRuM+6ZOnSoDFtu3335rd7vVfQ+FhYXyiy++KNevX1/29vaWH3jgATktLc14fM2aNXLnzp3levXqyT4+PnKnTp3khQsXylqt1il9/vTTT+XIyEjZw8ND7tmzp7xjxw7jsX79+skjR440k//pp5/kVq1ayR4eHnK7du3kVatWmR3X6XTyW2+9JYeEhMgqlUq+88475RMnTjilr664h+vXr8t33XWXHBwcLLu7u8tNmjSRx4wZ45BiqOp7MHyOym79+vWzu017EOmSBAJBnUXMwQkEgjqLUHACgaDOIhScQCCoswgFJxAI6ixCwQkEgjqLUHACgaDOIhScQCCoswgFJxAI6ixCwQlqDdbScLsiFbeg9iKyiQhqFYMHD+bbb78122crYF+tVuPu7m62r6SkBA8PD4evW9nzBNWLsOAEtQqVSkVoaKjZVr9+fUBf2GbBggXcd999+Pj48O677zJt2jQ6d+7M119/TdOmTfH09AQgNTWVYcOGUa9ePfz8/HjkkUfMUvPYOk9QuxAKTlCnmDZtGg888ACHDh3i6aefBuD06dP8+uuv/Pbbb+zfvx+dTsewYcO4evUqmzZtYt26dZw5c4bhw4ebtVX2PEHtQwxRBbWKP//8k3r16pnte/PNN3nzzTcBeOyxxxg9erTZ8ZKSEpYuXUpwcDCgz5126NAhUlJSiIiIAGDp0qW0a9eO3bt306NHD6vnCWofQsEJahUDBgxgwYIFZvsCAwONr00L5hho0qSJmZI6duwYERERRuUG0LZtWwICAjh27JhRwZU9T1D7EApOUKvw8fGhRYsW5R63Z5+91xLUbsQcnOCWo02bNpw/f57z588b9x09epTs7Gzatm1bjT0TOBthwQlqFcXFxRbl+9zc3AgKCrK7jdjYWDp06MDjjz/OvHnz0Gg0vPjii/Tr18/qEFdQexEWnKBWkZCQQFhYmNnWp08fh9qQJImVK1dSv3597rjjDmJjY2nWrBk//vhjFfVaUF2IlOUCgaDOIiw4gUBQZxEKTiAQ1FmEghMIBHUWoeAEAkGdRSg4gUBQZxEKTiAQ1FmEghMIBHUWoeAEAkGdRSg4gUBQZxEKTiAQ1FmEghMIBHUWoeAEAkGd5f8Bn2WQfgwoHZIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020752, + "end_time": "2024-02-29T03:39:17.021389", + "exception": false, + "start_time": "2024-02-29T03:39:17.000637", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 859.308783, + "end_time": "2024-02-29T03:39:19.761275", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/realtabformer/4/mlu-eval.ipynb", + "output_path": "eval/insurance/realtabformer/4/mlu-eval.ipynb", + "parameters": { + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "path": "eval/insurance/realtabformer/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "realtabformer" + }, + "start_time": "2024-02-29T03:25:00.452492", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/realtabformer/model.pt b/insurance/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..dccca359d1c4b542bce8e035a36c3fe1c87db7bf --- /dev/null +++ b/insurance/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d22b595a7793ec36a90bb1106eecd973db22fe0139551b119854e40360fd7e7 +size 43505805 diff --git a/insurance/realtabformer/params.json b/insurance/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..5be1d5532bc95560d1b743365ac78821b91a41c9 --- /dev/null +++ b/insurance/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "fixed_role_model": "realtabformer", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/eval.csv b/insurance/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..a6eb883baa90705d2d1f6cffff434f69ebdd5712 --- /dev/null +++ b/insurance/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,5.952381614060002e-08,0.609262997868067,0.01993643540660279,0.559147834777832,0.19419053196907043,0.9970712065696716,0.2823074758052826,1.8548176740296185e-05,0.8766729831695557,0.0972040519118309,0.7692358493804932,0.14119644463062286,0.053181055933237076,0.7849618196487427,1.4358208179473877 diff --git a/insurance/tab_ddpm_concat/history.csv b/insurance/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..c03a6946b411d933c40f099fb0d12d25c63cc6ba --- /dev/null +++ b/insurance/tab_ddpm_concat/history.csv @@ -0,0 +1,19 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.0265493107464863,9.705848431753656,0.0019637997826472465,0.0,0.0,0.0,0.0,0.0,0.0265493107464863,320,40,39.08715486526489,0.9771788716316223,0.12214735895395279,0.04609664692543447,0.012864274116873275,8.93672634124523,3.463389237516879e-05,0.0,0.0,0.0,0.0,0.0,0.012864274116873275,80,10,8.234524965286255,0.8234524965286255,0.10293156206607819,0.023089123656973243 +1,0.013430703204357996,10.238072396071818,0.0001760885078965657,0.0,0.0,0.0,0.0,0.0,0.013430703204357996,320,40,38.923088788986206,0.9730772197246551,0.12163465246558189,0.027457697270438074,0.01386686596670188,9.424022936335371,5.71949209714262e-05,0.0,0.0,0.0,0.0,0.0,0.01386686596670188,80,10,8.236119270324707,0.8236119270324707,0.10295149087905883,0.019944945629686118 +2,0.013098158335196786,6.953670260656827,7.627181049958409e-05,0.0,0.0,0.0,0.0,0.0,0.013098158335196786,320,40,38.896809816360474,0.9724202454090118,0.12155253067612648,0.03701225146651268,0.011231413613131735,4.642900250397725,1.232088975626766e-05,0.0,0.0,0.0,0.0,0.0,0.011231413613131735,80,10,8.272239923477173,0.8272239923477173,0.10340299904346466,0.031016640178859235 +3,0.013012661421089432,6.77741541211999,0.00014677781123761946,0.0,0.0,0.0,0.0,0.0,0.013012661421089432,320,40,39.03108096122742,0.9757770240306854,0.12197212800383568,0.040795679786242545,0.010680149483960122,5.439762359634369,8.51207419643174e-06,0.0,0.0,0.0,0.0,0.0,0.010680149483960122,80,10,8.236795425415039,0.8236795425415039,0.10295994281768799,0.02782872337847948 +4,0.012592662169481628,6.8064604322151805,0.00012719917820476213,0.0,0.0,0.0,0.0,0.0,0.012592662169481628,320,40,38.966336727142334,0.9741584181785583,0.12176980227231979,0.03671876427251845,0.012881963208201341,16.157494982505522,0.00010115250418607502,0.0,0.0,0.0,0.0,0.0,0.012881963208201341,80,10,8.331452369689941,0.8331452369689941,0.10414315462112426,0.012491705431602895 +5,0.013670370759791694,10.748200260194086,0.0001568969438597634,0.0,0.0,0.0,0.0,0.0,0.013670370759791694,320,40,38.94208788871765,0.9735521972179413,0.12169402465224266,0.029897483938839287,0.014085652580251917,22.363185199221174,0.00020219407759825003,0.0,0.0,0.0,0.0,0.0,0.014085652580251917,80,10,8.2787184715271,0.82787184715271,0.10348398089408875,0.009641142934560776 +6,0.014017040852922946,10.649183725507465,0.00013577813718335108,0.0,0.0,0.0,0.0,0.0,0.014017040852922946,320,40,38.94879508018494,0.9737198770046234,0.12171498462557792,0.028363983915187418,0.01068424858385697,3.8434145080467714,1.0552424407705985e-05,0.0,0.0,0.0,0.0,0.0,0.01068424858385697,80,10,8.305310726165771,0.8305310726165771,0.10381638407707214,0.03533868733793497 +7,0.011766438081394881,8.660977102358947,8.090821406305792e-05,0.0,0.0,0.0,0.0,0.0,0.011766438081394881,320,40,38.78416681289673,0.9696041703224182,0.12120052129030227,0.04158601735252887,0.012133054883452132,20.211999930033198,2.2262640635517526e-05,0.0,0.0,0.0,0.0,0.0,0.012133054883452132,80,10,8.369733810424805,0.8369733810424804,0.10462167263031005,0.010681234044022858 +8,0.012191647826693953,7.005204355998285,9.821474643096905e-05,0.0,0.0,0.0,0.0,0.0,0.012191647826693953,320,40,38.88823890686035,0.9722059726715088,0.1215257465839386,0.03872000898700208,0.014966235030442476,9.767283525761012,0.0001517352883070089,0.0,0.0,0.0,0.0,0.0,0.014966235030442476,80,10,8.22826075553894,0.8228260755538941,0.10285325944423676,0.01799462023191154 +9,0.012526353562134319,6.590273188782885,7.7691583878714e-05,0.0,0.0,0.0,0.0,0.0,0.012526353562134319,320,40,38.93899869918823,0.9734749674797059,0.12168437093496323,0.03674360387958586,0.012331876624375581,18.443907407086634,6.805989072731223e-05,0.0,0.0,0.0,0.0,0.0,0.012331876624375581,80,10,8.421772003173828,0.8421772003173829,0.10527215003967286,0.01039172657765448 +10,0.012064280622871593,9.317451603279006,3.690295125249321e-05,0.0,0.0,0.0,0.0,0.0,0.012064280622871593,320,40,39.005112171173096,0.9751278042793274,0.12189097553491593,0.0359303968725726,0.01261272220290266,10.194672084533522,5.446935015456234e-05,0.0,0.0,0.0,0.0,0.0,0.01261272220290266,80,10,8.333169937133789,0.8333169937133789,0.10416462421417236,0.01722581619396806 +11,0.012482693148194812,8.178162423045615,9.780007754767173e-05,0.0,0.0,0.0,0.0,0.0,0.012482693148194812,320,40,38.96896147727966,0.9742240369319916,0.12177800461649894,0.03824995262548327,0.012514100689440966,19.314230701327325,7.543949816977147e-05,0.0,0.0,0.0,0.0,0.0,0.012514100689440966,80,10,8.239241361618042,0.8239241361618042,0.10299051702022552,0.009454242698848248 +12,0.01332451379566919,10.310542043212262,0.0003665929893701819,0.0,0.0,0.0,0.0,0.0,0.01332451379566919,320,40,39.00809144973755,0.9752022862434387,0.12190028578042984,0.027350465022027492,0.010987071882118471,4.729085849918556,8.189743033426566e-06,0.0,0.0,0.0,0.0,0.0,0.010987071882118471,80,10,8.261511325836182,0.8261511325836182,0.10326889157295227,0.03069485481828451 +13,0.013592794616124592,7.457387926033698,0.000220215535729551,0.0,0.0,0.0,0.0,0.0,0.013592794616124592,320,40,38.93746519088745,0.9734366297721863,0.12167957872152328,0.03546805907972157,0.011548876191955059,6.165951245542237,1.5450504935188292e-05,0.0,0.0,0.0,0.0,0.0,0.011548876191955059,80,10,8.323935985565186,0.8323935985565185,0.10404919981956481,0.026838560402393342 +14,0.013447031378746033,8.19890535405798,0.00016373289685844838,0.0,0.0,0.0,0.0,0.0,0.013447031378746033,320,40,38.8496150970459,0.9712403774261474,0.12140504717826843,0.029747568373568355,0.011828925088047981,5.351523938098455,2.9337766557091526e-05,0.0,0.0,0.0,0.0,0.0,0.011828925088047981,80,10,8.288572311401367,0.8288572311401368,0.1036071538925171,0.0307698548771441 +15,0.01384369531297125,8.665561918970889,0.00016956335028766033,0.0,0.0,0.0,0.0,0.0,0.01384369531297125,320,40,38.970547676086426,0.9742636919021607,0.12178296148777008,0.03315324831055477,0.01150583740673028,6.560287872780464,1.5260961676233363e-05,0.0,0.0,0.0,0.0,0.0,0.01150583740673028,80,10,8.322679042816162,0.8322679042816162,0.10403348803520203,0.0259027692489326 +16,0.012172109389211982,7.0008499470219245,7.735751830111326e-05,0.0,0.0,0.0,0.0,0.0,0.012172109389211982,320,40,39.07010316848755,0.9767525792121887,0.12209407240152359,0.03525363316293806,0.012191956081369425,7.897130101547532,2.3637969795231585e-05,0.0,0.0,0.0,0.0,0.0,0.012191956081369425,80,10,8.288463592529297,0.8288463592529297,0.10360579490661621,0.022672764584422113 +17,0.012383807837613859,4.774037581340053,0.00011500121783720729,0.0,0.0,0.0,0.0,0.0,0.012383807837613859,320,40,38.94200682640076,0.9735501706600189,0.12169377133250237,0.04339534998871386,0.012275835702894256,9.627914267603774,4.899254280417153e-05,0.0,0.0,0.0,0.0,0.0,0.012275835702894256,80,10,8.323761701583862,0.8323761701583863,0.10404702126979828,0.018597377510741354 diff --git a/insurance/tab_ddpm_concat/mlu-eval.ipynb b/insurance/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7346374d54773a11bd06620b9d394b7738d08726 --- /dev/null +++ b/insurance/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2560 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.558870Z", + "iopub.status.busy": "2024-02-29T18:26:50.558154Z", + "iopub.status.idle": "2024-02-29T18:26:50.590106Z", + "shell.execute_reply": "2024-02-29T18:26:50.589458Z" + }, + "papermill": { + "duration": 0.046189, + "end_time": "2024-02-29T18:26:50.591955", + "exception": false, + "start_time": "2024-02-29T18:26:50.545766", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.617349Z", + "iopub.status.busy": "2024-02-29T18:26:50.616718Z", + "iopub.status.idle": "2024-02-29T18:26:50.624173Z", + "shell.execute_reply": "2024-02-29T18:26:50.623371Z" + }, + "papermill": { + "duration": 0.022172, + "end_time": "2024-02-29T18:26:50.626060", + "exception": false, + "start_time": "2024-02-29T18:26:50.603888", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.649180Z", + "iopub.status.busy": "2024-02-29T18:26:50.648911Z", + "iopub.status.idle": "2024-02-29T18:26:50.653522Z", + "shell.execute_reply": "2024-02-29T18:26:50.652094Z" + }, + "papermill": { + "duration": 0.019003, + "end_time": "2024-02-29T18:26:50.655938", + "exception": false, + "start_time": "2024-02-29T18:26:50.636935", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.679562Z", + "iopub.status.busy": "2024-02-29T18:26:50.679302Z", + "iopub.status.idle": "2024-02-29T18:26:50.683027Z", + "shell.execute_reply": "2024-02-29T18:26:50.682249Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.01756, + "end_time": "2024-02-29T18:26:50.684861", + "exception": false, + "start_time": "2024-02-29T18:26:50.667301", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.707664Z", + "iopub.status.busy": "2024-02-29T18:26:50.707390Z", + "iopub.status.idle": "2024-02-29T18:26:50.712716Z", + "shell.execute_reply": "2024-02-29T18:26:50.711943Z" + }, + "papermill": { + "duration": 0.018823, + "end_time": "2024-02-29T18:26:50.714583", + "exception": false, + "start_time": "2024-02-29T18:26:50.695760", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1800468a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.739163Z", + "iopub.status.busy": "2024-02-29T18:26:50.738835Z", + "iopub.status.idle": "2024-02-29T18:26:50.744203Z", + "shell.execute_reply": "2024-02-29T18:26:50.743458Z" + }, + "papermill": { + "duration": 0.019989, + "end_time": "2024-02-29T18:26:50.746255", + "exception": false, + "start_time": "2024-02-29T18:26:50.726266", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tab_ddpm_concat/4\"\n", + "param_index = 2\n", + "allow_same_prediction = True\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010925, + "end_time": "2024-02-29T18:26:50.768318", + "exception": false, + "start_time": "2024-02-29T18:26:50.757393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.791282Z", + "iopub.status.busy": "2024-02-29T18:26:50.791016Z", + "iopub.status.idle": "2024-02-29T18:26:50.799559Z", + "shell.execute_reply": "2024-02-29T18:26:50.798806Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022153, + "end_time": "2024-02-29T18:26:50.801373", + "exception": false, + "start_time": "2024-02-29T18:26:50.779220", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tab_ddpm_concat/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:50.825158Z", + "iopub.status.busy": "2024-02-29T18:26:50.824897Z", + "iopub.status.idle": "2024-02-29T18:26:52.939311Z", + "shell.execute_reply": "2024-02-29T18:26:52.938380Z" + }, + "papermill": { + "duration": 2.128694, + "end_time": "2024-02-29T18:26:52.941418", + "exception": false, + "start_time": "2024-02-29T18:26:50.812724", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:52.968131Z", + "iopub.status.busy": "2024-02-29T18:26:52.967214Z", + "iopub.status.idle": "2024-02-29T18:26:52.979057Z", + "shell.execute_reply": "2024-02-29T18:26:52.978209Z" + }, + "papermill": { + "duration": 0.027034, + "end_time": "2024-02-29T18:26:52.981011", + "exception": false, + "start_time": "2024-02-29T18:26:52.953977", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:53.005698Z", + "iopub.status.busy": "2024-02-29T18:26:53.005020Z", + "iopub.status.idle": "2024-02-29T18:26:53.011733Z", + "shell.execute_reply": "2024-02-29T18:26:53.010953Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021252, + "end_time": "2024-02-29T18:26:53.013739", + "exception": false, + "start_time": "2024-02-29T18:26:52.992487", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:53.037402Z", + "iopub.status.busy": "2024-02-29T18:26:53.037158Z", + "iopub.status.idle": "2024-02-29T18:26:53.136345Z", + "shell.execute_reply": "2024-02-29T18:26:53.135661Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.113075, + "end_time": "2024-02-29T18:26:53.138309", + "exception": false, + "start_time": "2024-02-29T18:26:53.025234", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:53.163865Z", + "iopub.status.busy": "2024-02-29T18:26:53.163568Z", + "iopub.status.idle": "2024-02-29T18:26:57.713583Z", + "shell.execute_reply": "2024-02-29T18:26:57.712809Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.565534, + "end_time": "2024-02-29T18:26:57.715908", + "exception": false, + "start_time": "2024-02-29T18:26:53.150374", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 18:26:55.362648: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 18:26:55.362701: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 18:26:55.364282: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:57.741084Z", + "iopub.status.busy": "2024-02-29T18:26:57.740440Z", + "iopub.status.idle": "2024-02-29T18:26:57.747224Z", + "shell.execute_reply": "2024-02-29T18:26:57.746357Z" + }, + "papermill": { + "duration": 0.021021, + "end_time": "2024-02-29T18:26:57.749183", + "exception": false, + "start_time": "2024-02-29T18:26:57.728162", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:26:57.774557Z", + "iopub.status.busy": "2024-02-29T18:26:57.774285Z", + "iopub.status.idle": "2024-02-29T18:27:05.763239Z", + "shell.execute_reply": "2024-02-29T18:27:05.762279Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.004862, + "end_time": "2024-02-29T18:27:05.766044", + "exception": false, + "start_time": "2024-02-29T18:26:57.761182", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.77,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.RReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:27:06.257693Z", + "iopub.status.busy": "2024-02-29T18:27:06.257389Z", + "iopub.status.idle": "2024-02-29T18:27:06.330588Z", + "shell.execute_reply": "2024-02-29T18:27:06.329733Z" + }, + "papermill": { + "duration": 0.088632, + "end_time": "2024-02-29T18:27:06.332688", + "exception": false, + "start_time": "2024-02-29T18:27:06.244056", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T18:27:06.361058Z", + "iopub.status.busy": "2024-02-29T18:27:06.360443Z", + "iopub.status.idle": "2024-02-29T18:27:06.783264Z", + "shell.execute_reply": "2024-02-29T18:27:06.782352Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.439025, + "end_time": "2024-02-29T18:27:06.785332", + "exception": false, + "start_time": "2024-02-29T18:27:06.346307", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:27:06.814758Z", + "iopub.status.busy": "2024-02-29T18:27:06.813947Z", + "iopub.status.idle": "2024-02-29T18:27:06.818524Z", + "shell.execute_reply": "2024-02-29T18:27:06.817759Z" + }, + "papermill": { + "duration": 0.021262, + "end_time": "2024-02-29T18:27:06.820462", + "exception": false, + "start_time": "2024-02-29T18:27:06.799200", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:27:06.848255Z", + "iopub.status.busy": "2024-02-29T18:27:06.847981Z", + "iopub.status.idle": "2024-02-29T18:27:06.854747Z", + "shell.execute_reply": "2024-02-29T18:27:06.853941Z" + }, + "papermill": { + "duration": 0.02231, + "end_time": "2024-02-29T18:27:06.856576", + "exception": false, + "start_time": "2024-02-29T18:27:06.834266", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9613953" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:27:06.882937Z", + "iopub.status.busy": "2024-02-29T18:27:06.882671Z", + "iopub.status.idle": "2024-02-29T18:27:06.968606Z", + "shell.execute_reply": "2024-02-29T18:27:06.967758Z" + }, + "papermill": { + "duration": 0.101194, + "end_time": "2024-02-29T18:27:06.970459", + "exception": false, + "start_time": "2024-02-29T18:27:06.869265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 12] --\n", + "├─Adapter: 1-1 [2, 1071, 12] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 13,312\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─RReLU: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,613,953\n", + "Trainable params: 9,613,953\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.08\n", + "========================================================================================================================\n", + "Input size (MB): 0.13\n", + "Forward/backward pass size (MB): 307.47\n", + "Params size (MB): 38.46\n", + "Estimated Total Size (MB): 346.05\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:27:07.000684Z", + "iopub.status.busy": "2024-02-29T18:27:07.000395Z", + "iopub.status.idle": "2024-02-29T18:42:55.818307Z", + "shell.execute_reply": "2024-02-29T18:42:55.817254Z" + }, + "papermill": { + "duration": 948.852675, + "end_time": "2024-02-29T18:42:55.837260", + "exception": false, + "start_time": "2024-02-29T18:27:06.984585", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0265493107464863, 'avg_role_model_std_loss': 9.705848431753656, 'avg_role_model_mean_pred_loss': 0.0019637997826472465, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0265493107464863, 'n_size': 320, 'n_batch': 40, 'duration': 39.08715486526489, 'duration_batch': 0.9771788716316223, 'duration_size': 0.12214735895395279, 'avg_pred_std': 0.04609664692543447}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012864274116873275, 'avg_role_model_std_loss': 8.93672634124523, 'avg_role_model_mean_pred_loss': 3.463389237516879e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012864274116873275, 'n_size': 80, 'n_batch': 10, 'duration': 8.234524965286255, 'duration_batch': 0.8234524965286255, 'duration_size': 0.10293156206607819, 'avg_pred_std': 0.023089123656973243}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013430703204357996, 'avg_role_model_std_loss': 10.238072396071818, 'avg_role_model_mean_pred_loss': 0.0001760885078965657, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013430703204357996, 'n_size': 320, 'n_batch': 40, 'duration': 38.923088788986206, 'duration_batch': 0.9730772197246551, 'duration_size': 0.12163465246558189, 'avg_pred_std': 0.027457697270438074}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01386686596670188, 'avg_role_model_std_loss': 9.424022936335371, 'avg_role_model_mean_pred_loss': 5.71949209714262e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01386686596670188, 'n_size': 80, 'n_batch': 10, 'duration': 8.236119270324707, 'duration_batch': 0.8236119270324707, 'duration_size': 0.10295149087905883, 'avg_pred_std': 0.019944945629686118}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013098158335196786, 'avg_role_model_std_loss': 6.953670260656827, 'avg_role_model_mean_pred_loss': 7.627181049958409e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013098158335196786, 'n_size': 320, 'n_batch': 40, 'duration': 38.896809816360474, 'duration_batch': 0.9724202454090118, 'duration_size': 0.12155253067612648, 'avg_pred_std': 0.03701225146651268}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011231413613131735, 'avg_role_model_std_loss': 4.642900250397725, 'avg_role_model_mean_pred_loss': 1.232088975626766e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011231413613131735, 'n_size': 80, 'n_batch': 10, 'duration': 8.272239923477173, 'duration_batch': 0.8272239923477173, 'duration_size': 0.10340299904346466, 'avg_pred_std': 0.031016640178859235}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013012661421089432, 'avg_role_model_std_loss': 6.77741541211999, 'avg_role_model_mean_pred_loss': 0.00014677781123761946, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013012661421089432, 'n_size': 320, 'n_batch': 40, 'duration': 39.03108096122742, 'duration_batch': 0.9757770240306854, 'duration_size': 0.12197212800383568, 'avg_pred_std': 0.040795679786242545}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010680149483960122, 'avg_role_model_std_loss': 5.439762359634369, 'avg_role_model_mean_pred_loss': 8.51207419643174e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010680149483960122, 'n_size': 80, 'n_batch': 10, 'duration': 8.236795425415039, 'duration_batch': 0.8236795425415039, 'duration_size': 0.10295994281768799, 'avg_pred_std': 0.02782872337847948}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012592662169481628, 'avg_role_model_std_loss': 6.8064604322151805, 'avg_role_model_mean_pred_loss': 0.00012719917820476213, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012592662169481628, 'n_size': 320, 'n_batch': 40, 'duration': 38.966336727142334, 'duration_batch': 0.9741584181785583, 'duration_size': 0.12176980227231979, 'avg_pred_std': 0.03671876427251845}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012881963208201341, 'avg_role_model_std_loss': 16.157494982505522, 'avg_role_model_mean_pred_loss': 0.00010115250418607502, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012881963208201341, 'n_size': 80, 'n_batch': 10, 'duration': 8.331452369689941, 'duration_batch': 0.8331452369689941, 'duration_size': 0.10414315462112426, 'avg_pred_std': 0.012491705431602895}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013670370759791694, 'avg_role_model_std_loss': 10.748200260194086, 'avg_role_model_mean_pred_loss': 0.0001568969438597634, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013670370759791694, 'n_size': 320, 'n_batch': 40, 'duration': 38.94208788871765, 'duration_batch': 0.9735521972179413, 'duration_size': 0.12169402465224266, 'avg_pred_std': 0.029897483938839287}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014085652580251917, 'avg_role_model_std_loss': 22.363185199221174, 'avg_role_model_mean_pred_loss': 0.00020219407759825003, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014085652580251917, 'n_size': 80, 'n_batch': 10, 'duration': 8.2787184715271, 'duration_batch': 0.82787184715271, 'duration_size': 0.10348398089408875, 'avg_pred_std': 0.009641142934560776}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014017040852922946, 'avg_role_model_std_loss': 10.649183725507465, 'avg_role_model_mean_pred_loss': 0.00013577813718335108, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014017040852922946, 'n_size': 320, 'n_batch': 40, 'duration': 38.94879508018494, 'duration_batch': 0.9737198770046234, 'duration_size': 0.12171498462557792, 'avg_pred_std': 0.028363983915187418}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01068424858385697, 'avg_role_model_std_loss': 3.8434145080467714, 'avg_role_model_mean_pred_loss': 1.0552424407705985e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01068424858385697, 'n_size': 80, 'n_batch': 10, 'duration': 8.305310726165771, 'duration_batch': 0.8305310726165771, 'duration_size': 0.10381638407707214, 'avg_pred_std': 0.03533868733793497}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.011766438081394881, 'avg_role_model_std_loss': 8.660977102358947, 'avg_role_model_mean_pred_loss': 8.090821406305792e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011766438081394881, 'n_size': 320, 'n_batch': 40, 'duration': 38.78416681289673, 'duration_batch': 0.9696041703224182, 'duration_size': 0.12120052129030227, 'avg_pred_std': 0.04158601735252887}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012133054883452132, 'avg_role_model_std_loss': 20.211999930033198, 'avg_role_model_mean_pred_loss': 2.2262640635517526e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012133054883452132, 'n_size': 80, 'n_batch': 10, 'duration': 8.369733810424805, 'duration_batch': 0.8369733810424804, 'duration_size': 0.10462167263031005, 'avg_pred_std': 0.010681234044022858}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012191647826693953, 'avg_role_model_std_loss': 7.005204355998285, 'avg_role_model_mean_pred_loss': 9.821474643096905e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012191647826693953, 'n_size': 320, 'n_batch': 40, 'duration': 38.88823890686035, 'duration_batch': 0.9722059726715088, 'duration_size': 0.1215257465839386, 'avg_pred_std': 0.03872000898700208}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014966235030442476, 'avg_role_model_std_loss': 9.767283525761012, 'avg_role_model_mean_pred_loss': 0.0001517352883070089, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014966235030442476, 'n_size': 80, 'n_batch': 10, 'duration': 8.22826075553894, 'duration_batch': 0.8228260755538941, 'duration_size': 0.10285325944423676, 'avg_pred_std': 0.01799462023191154}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012526353562134319, 'avg_role_model_std_loss': 6.590273188782885, 'avg_role_model_mean_pred_loss': 7.7691583878714e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012526353562134319, 'n_size': 320, 'n_batch': 40, 'duration': 38.93899869918823, 'duration_batch': 0.9734749674797059, 'duration_size': 0.12168437093496323, 'avg_pred_std': 0.03674360387958586}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012331876624375581, 'avg_role_model_std_loss': 18.443907407086634, 'avg_role_model_mean_pred_loss': 6.805989072731223e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012331876624375581, 'n_size': 80, 'n_batch': 10, 'duration': 8.421772003173828, 'duration_batch': 0.8421772003173829, 'duration_size': 0.10527215003967286, 'avg_pred_std': 0.01039172657765448}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012064280622871593, 'avg_role_model_std_loss': 9.317451603279006, 'avg_role_model_mean_pred_loss': 3.690295125249321e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012064280622871593, 'n_size': 320, 'n_batch': 40, 'duration': 39.005112171173096, 'duration_batch': 0.9751278042793274, 'duration_size': 0.12189097553491593, 'avg_pred_std': 0.0359303968725726}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01261272220290266, 'avg_role_model_std_loss': 10.194672084533522, 'avg_role_model_mean_pred_loss': 5.446935015456234e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01261272220290266, 'n_size': 80, 'n_batch': 10, 'duration': 8.333169937133789, 'duration_batch': 0.8333169937133789, 'duration_size': 0.10416462421417236, 'avg_pred_std': 0.01722581619396806}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012482693148194812, 'avg_role_model_std_loss': 8.178162423045615, 'avg_role_model_mean_pred_loss': 9.780007754767173e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012482693148194812, 'n_size': 320, 'n_batch': 40, 'duration': 38.96896147727966, 'duration_batch': 0.9742240369319916, 'duration_size': 0.12177800461649894, 'avg_pred_std': 0.03824995262548327}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012514100689440966, 'avg_role_model_std_loss': 19.314230701327325, 'avg_role_model_mean_pred_loss': 7.543949816977147e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012514100689440966, 'n_size': 80, 'n_batch': 10, 'duration': 8.239241361618042, 'duration_batch': 0.8239241361618042, 'duration_size': 0.10299051702022552, 'avg_pred_std': 0.009454242698848248}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01332451379566919, 'avg_role_model_std_loss': 10.310542043212262, 'avg_role_model_mean_pred_loss': 0.0003665929893701819, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01332451379566919, 'n_size': 320, 'n_batch': 40, 'duration': 39.00809144973755, 'duration_batch': 0.9752022862434387, 'duration_size': 0.12190028578042984, 'avg_pred_std': 0.027350465022027492}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010987071882118471, 'avg_role_model_std_loss': 4.729085849918556, 'avg_role_model_mean_pred_loss': 8.189743033426566e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010987071882118471, 'n_size': 80, 'n_batch': 10, 'duration': 8.261511325836182, 'duration_batch': 0.8261511325836182, 'duration_size': 0.10326889157295227, 'avg_pred_std': 0.03069485481828451}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013592794616124592, 'avg_role_model_std_loss': 7.457387926033698, 'avg_role_model_mean_pred_loss': 0.000220215535729551, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013592794616124592, 'n_size': 320, 'n_batch': 40, 'duration': 38.93746519088745, 'duration_batch': 0.9734366297721863, 'duration_size': 0.12167957872152328, 'avg_pred_std': 0.03546805907972157}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011548876191955059, 'avg_role_model_std_loss': 6.165951245542237, 'avg_role_model_mean_pred_loss': 1.5450504935188292e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011548876191955059, 'n_size': 80, 'n_batch': 10, 'duration': 8.323935985565186, 'duration_batch': 0.8323935985565185, 'duration_size': 0.10404919981956481, 'avg_pred_std': 0.026838560402393342}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013447031378746033, 'avg_role_model_std_loss': 8.19890535405798, 'avg_role_model_mean_pred_loss': 0.00016373289685844838, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013447031378746033, 'n_size': 320, 'n_batch': 40, 'duration': 38.8496150970459, 'duration_batch': 0.9712403774261474, 'duration_size': 0.12140504717826843, 'avg_pred_std': 0.029747568373568355}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011828925088047981, 'avg_role_model_std_loss': 5.351523938098455, 'avg_role_model_mean_pred_loss': 2.9337766557091526e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011828925088047981, 'n_size': 80, 'n_batch': 10, 'duration': 8.288572311401367, 'duration_batch': 0.8288572311401368, 'duration_size': 0.1036071538925171, 'avg_pred_std': 0.0307698548771441}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01384369531297125, 'avg_role_model_std_loss': 8.665561918970889, 'avg_role_model_mean_pred_loss': 0.00016956335028766033, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01384369531297125, 'n_size': 320, 'n_batch': 40, 'duration': 38.970547676086426, 'duration_batch': 0.9742636919021607, 'duration_size': 0.12178296148777008, 'avg_pred_std': 0.03315324831055477}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01150583740673028, 'avg_role_model_std_loss': 6.560287872780464, 'avg_role_model_mean_pred_loss': 1.5260961676233363e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01150583740673028, 'n_size': 80, 'n_batch': 10, 'duration': 8.322679042816162, 'duration_batch': 0.8322679042816162, 'duration_size': 0.10403348803520203, 'avg_pred_std': 0.0259027692489326}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012172109389211982, 'avg_role_model_std_loss': 7.0008499470219245, 'avg_role_model_mean_pred_loss': 7.735751830111326e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012172109389211982, 'n_size': 320, 'n_batch': 40, 'duration': 39.07010316848755, 'duration_batch': 0.9767525792121887, 'duration_size': 0.12209407240152359, 'avg_pred_std': 0.03525363316293806}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012191956081369425, 'avg_role_model_std_loss': 7.897130101547532, 'avg_role_model_mean_pred_loss': 2.3637969795231585e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012191956081369425, 'n_size': 80, 'n_batch': 10, 'duration': 8.288463592529297, 'duration_batch': 0.8288463592529297, 'duration_size': 0.10360579490661621, 'avg_pred_std': 0.022672764584422113}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012383807837613859, 'avg_role_model_std_loss': 4.774037581340053, 'avg_role_model_mean_pred_loss': 0.00011500121783720729, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012383807837613859, 'n_size': 320, 'n_batch': 40, 'duration': 38.94200682640076, 'duration_batch': 0.9735501706600189, 'duration_size': 0.12169377133250237, 'avg_pred_std': 0.04339534998871386}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012275835702894256, 'avg_role_model_std_loss': 9.627914267603774, 'avg_role_model_mean_pred_loss': 4.899254280417153e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012275835702894256, 'n_size': 80, 'n_batch': 10, 'duration': 8.323761701583862, 'duration_batch': 0.8323761701583863, 'duration_size': 0.10404702126979828, 'avg_pred_std': 0.018597377510741354}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▅▆▂▁▅▇▁▃█▄▄▄▂▂▃▂▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▂▂▁▂▂▁▁▁▁▁▂▂▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▅▄▇▆▂▁█▁▃▁▃▁▇▆▇▅▅▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▁▅▆▄▂▁▆▅▅▄▅▁▄▂▃▄▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▅▆▂▁▅▇▁▃█▄▄▄▂▂▃▂▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▂▂▁▂▂▁▁▁▁▁▂▂▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▂▃▁▁▄█▁▂▆▃▃▃▁▁▂▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▂▁▁▁▁▁▁▁▁▁▁▂▂▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▃▃▁▂▆█▁▇▃▇▃▇▁▂▂▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▇▇▄▃▃██▆▄▃▆▅▇▄▅▆▄▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▁▁▃▁▅▃▄▆▁█▅▁▂▄▃▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▄▄▇▅▅▅▁▃▅▆▅▆▅▃▅█▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▁▁▃▁▅▃▄▆▁█▅▁▂▄▃▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▄▄▇▅▅▅▁▃▅▆▅▆▅▃▅█▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▁▁▃▁▅▃▄▆▁█▅▁▂▄▃▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▄▄▇▅▅▅▁▃▅▆▅▆▅▃▅█▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.01228\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.01238\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.0186\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.0434\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.01228\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.01238\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 5e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.00012\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 9.62791\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 4.77404\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.83238\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.97355\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.10405\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.12169\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 8.32376\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 38.94201\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/tab_ddpm_concat/4/wandb/offline-run-20240229_182708-etiddam5\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_182708-etiddam5/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.01993635216760531, 'avg_g_mag_loss': 0.6526490935406888, 'avg_g_cos_loss': 4.4486220484027385e-08, 'pred_duration': 0.8737220764160156, 'grad_duration': 0.5640714168548584, 'total_duration': 1.437793493270874, 'pred_std': 0.053181204944849014, 'std_loss': 0.7849577069282532, 'mean_pred_loss': 1.8548063962953165e-05, 'pred_rmse': 0.141196146607399, 'pred_mae': 0.0972040668129921, 'pred_mape': 0.7692168354988098, 'grad_rmse': 0.28230687975883484, 'grad_mae': 0.19419056177139282, 'grad_mape': 0.9970712065696716}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.01993635216760531, 'avg_g_mag_loss': 0.6526490935406888, 'avg_g_cos_loss': 4.4486220484027385e-08, 'avg_pred_duration': 0.8737220764160156, 'avg_grad_duration': 0.5640714168548584, 'avg_total_duration': 1.437793493270874, 'avg_pred_std': 0.053181204944849014, 'avg_std_loss': 0.7849577069282532, 'avg_mean_pred_loss': 1.8548063962953165e-05}, 'min_metrics': {'avg_loss': 0.01993635216760531, 'avg_g_mag_loss': 0.6526490935406888, 'avg_g_cos_loss': 4.4486220484027385e-08, 'pred_duration': 0.8737220764160156, 'grad_duration': 0.5640714168548584, 'total_duration': 1.437793493270874, 'pred_std': 0.053181204944849014, 'std_loss': 0.7849577069282532, 'mean_pred_loss': 1.8548063962953165e-05, 'pred_rmse': 0.141196146607399, 'pred_mae': 0.0972040668129921, 'pred_mape': 0.7692168354988098, 'grad_rmse': 0.28230687975883484, 'grad_mae': 0.19419056177139282, 'grad_mape': 0.9970712065696716}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.01993635216760531, 'avg_g_mag_loss': 0.6526490935406888, 'avg_g_cos_loss': 4.4486220484027385e-08, 'pred_duration': 0.8737220764160156, 'grad_duration': 0.5640714168548584, 'total_duration': 1.437793493270874, 'pred_std': 0.053181204944849014, 'std_loss': 0.7849577069282532, 'mean_pred_loss': 1.8548063962953165e-05, 'pred_rmse': 0.141196146607399, 'pred_mae': 0.0972040668129921, 'pred_mape': 0.7692168354988098, 'grad_rmse': 0.28230687975883484, 'grad_mae': 0.19419056177139282, 'grad_mape': 0.9970712065696716}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:42:55.875215Z", + "iopub.status.busy": "2024-02-29T18:42:55.874907Z", + "iopub.status.idle": "2024-02-29T18:42:55.879007Z", + "shell.execute_reply": "2024-02-29T18:42:55.878119Z" + }, + "papermill": { + "duration": 0.025079, + "end_time": "2024-02-29T18:42:55.880936", + "exception": false, + "start_time": "2024-02-29T18:42:55.855857", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:42:55.915833Z", + "iopub.status.busy": "2024-02-29T18:42:55.915549Z", + "iopub.status.idle": "2024-02-29T18:42:55.991917Z", + "shell.execute_reply": "2024-02-29T18:42:55.990964Z" + }, + "papermill": { + "duration": 0.096139, + "end_time": "2024-02-29T18:42:55.994052", + "exception": false, + "start_time": "2024-02-29T18:42:55.897913", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:42:56.032185Z", + "iopub.status.busy": "2024-02-29T18:42:56.031896Z", + "iopub.status.idle": "2024-02-29T18:42:56.328976Z", + "shell.execute_reply": "2024-02-29T18:42:56.328057Z" + }, + "papermill": { + "duration": 0.319257, + "end_time": "2024-02-29T18:42:56.331162", + "exception": false, + "start_time": "2024-02-29T18:42:56.011905", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAESCAYAAAB3gfmJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABQC0lEQVR4nO2deVzU1f7/n8MyA8gqyKYIbrjihkGYSymJRhllaWZqZmo3bZEWs5uidb9Rpr/rzbztaXUzl9IWt0TUrERRFnfJBUVlE00QkG3m/P74OKMTiwwMDOJ5Ph6fB8PnvD/ncz7DzIuzvM/7rRJCCCQSiaQZYWXpBkgkEom5kcImkUiaHVLYJBJJs0MKm0QiaXZIYZNIJM0OKWwSiaTZIYVNIpE0O2ws3YCmhE6nIzMzEycnJ1QqlaWbI5FIbkAIwZUrV/D19cXKquY+mRS2G8jMzMTPz8/SzZBIJDVw9uxZ2rRpU6ONFLYbcHJyApQ3ztnZ2cKtkUgkN1JQUICfn5/he1oTUthuQD/8dHZ2lsImkTRRajNNJBcPJBJJs0MKm0QiaXZIYZNIJM0OOccmMQs6nY6ysjJLN0Nyi2Nra4u1tXW965HCJqk3ZWVlpKeno9PpLN0USTPA1dUVb2/vevmSSmGT1AshBFlZWVhbW+Pn53dTx0mJpDqEEBQXF5ObmwuAj49PneuSwlYHzv1VzOm8YjydNQR63dynpjlTUVFBcXExvr6+ODg4WLo5klsce3t7AHJzc/H09KzzsFT+e60D3+zJ4InP9/BtYoalm2JxtFotAGq12sItkTQX9P8gy8vL61yHFLY64GpvC0B+cd3f+OaG3FsrMRfm+CxJYasDbg5K7+SvYrkKKJE0RaSw1QEXB6XHdvmq7LFJJE0RKWx1QA5FJeZApVLxww8/WLoZZmXevHn07t3b0s2QwlYX3FrIoaikebB8+XJcXV3NVt/LL79MfHy82eqrK1LY6oChx3a1HJ1O5puWNH9qu6vE0dERd3f3Bm7NzZHCVgf0c2w6AVdKKyzcmqaFEILisgqLHEKY9k9m8+bNDBgwAFdXV9zd3bn//vs5efIkAP3792fWrFlG9hcuXMDW1padO3cCkJWVRWRkJPb29rRr144VK1YQEBDA4sWL6/TeHTx4kCFDhmBvb4+7uztTp06lsLDQUL5jxw5CQkJo0aIFrq6u3HXXXZw5cwaA/fv3c8899+Dk5ISzszPBwcHs27evxvvt2LGDSZMmkZ+fj0qlQqVSMW/ePAACAgJ46623mDBhAs7OzkydOhWAWbNmERgYiIODA+3bt2fOnDlGbhl/H4o++eSTREVFsXDhQnx8fHB3d2f69On1cuWoDdJBtw5obKxxUFtTXKblcnEZLtd6cBK4Wq6l29xfLHLvI29G4KCu/Ue6qKiI6OhoevbsSWFhIXPnzuWhhx4iNTWVcePGsWDBAt555x2D+8GqVavw9fVl4MCBAEyYMIG8vDx27NiBra0t0dHRBq95UykqKiIiIoKwsDD27t1Lbm4uTz/9NDNmzGD58uVUVFQQFRXFlClT+PbbbykrKyMxMdHQtnHjxtGnTx8+/PBDrK2tSU1Nxda25s9l//79Wbx4MXPnziUtLQ1Qelx6Fi5cyNy5c4mJiTGcc3JyYvny5fj6+nLw4EGmTJmCk5MTr776arX32b59Oz4+Pmzfvp0TJ04wZswYevfuzZQpU+r0XtUGKWx1xNXe9pqwleNv+Z63pA6MGjXK6PcvvviCVq1aceTIEUaPHs2LL77I77//bhCyFStWMHbsWFQqFceOHWPr1q3s3buXfv36AfDZZ5/RqVOnOrVlxYoVlJSU8NVXX9GiRQsAPvjgAx544AHeffddbG1tyc/P5/7776dDhw4AdO3a1XB9RkYGr7zyCl26dAGoVTvUajUuLi6oVCq8vb0rlQ8ZMoSXXnrJ6Nwbb7xheB0QEMDLL7/MypUraxQ2Nzc3PvjgA6ytrenSpQuRkZHEx8dLYWuKuDioycwvkS4ff8Pe1pojb0ZY7N6mcPz4cebOncuePXvIy8szbOLPyMigR48eDBs2jG+++YaBAweSnp5OQkICH3/8MQBpaWnY2NjQt29fQ30dO3bEzc2tTm0/evQovXr1MogawF133YVOpyMtLY1Bgwbx5JNPEhERwb333kt4eDijR4827KeMjo7m6aef5uuvvyY8PJxHH33UIIB1RS/YN7Jq1Sref/99Tp48SWFhIRUVFTeNNt29e3ejrVE+Pj4cPHiwXm27GXKOrY646X3Z5MqoESqVCge1jUUOUz3WH3jgAS5dusSnn37Knj172LNnD3B9onzcuHF89913lJeXs2LFCoKCgggKCjL7e1Zbli1bRkJCAv3792fVqlUEBgaye/duQJnbOnz4MJGRkWzbto1u3bqxbt26et3vRpEFSEhIYNy4cdx3332sX7+elJQU/vnPf950YeHvQ2KVStXgkWCksNURV4OwyR7brcjFixdJS0vjjTfeYOjQoXTt2pW//vrLyObBBx+kpKSEzZs3s2LFCsaNG2co69y5MxUVFaSkpBjOnThxolIdtaVr167s37+foqIiw7k//vgDKysrOnfubDjXp08fZs+eza5du+jRowcrVqwwlAUGBjJz5ky2bNnCww8/zLJly256X7VabdjvezN27dqFv78///znP+nXrx+dOnUyLF40NaSw1REXe8WXTQrbrYmbmxvu7u588sknnDhxgm3bthEdHW1k06JFC6KiopgzZw5Hjx5l7NixhrIuXboQHh7O1KlTSUxMJCUlhalTp2Jvb1+nvY7jxo3Dzs6OiRMncujQIbZv385zzz3H+PHj8fLyIj09ndmzZ5OQkMCZM2fYsmULx48fp2vXrly9epUZM2awY8cOzpw5wx9//MHevXuN5uCqIyAggMLCQuLj48nLy6O4uLha206dOpGRkcHKlSs5efIk77//fr17hQ2FFLY6oh+KSifdWxMrKytWrlxJUlISPXr0YObMmbz33nuV7MaNG8f+/fsZOHAgbdu2NSr76quv8PLyYtCgQTz00EOGFUI7OzuT2+Pg4MAvv/zCpUuXuOOOO3jkkUcYOnQoH3zwgaH82LFjjBo1isDAQKZOncr06dOZNm0a1tbWXLx4kQkTJhAYGMjo0aMZMWIE8+fPv+l9+/fvzzPPPMOYMWNo1aoVCxYsqNZ25MiRzJw5kxkzZtC7d2927drFnDlzTH7WxkAlTHX+acYUFBTg4uJCfn7+TSdEP9l5krc3HuOhPq3595jejdPAJkhJSQnp6em0a9euTl/o5sS5c+fw8/Nj69atDB061NLNuWWp7jNlyvdTrorWEVfDUFT22G5Xtm3bRmFhIUFBQWRlZfHqq68SEBDAoEGDLN202x45FK0jroahqJxju10pLy/n9ddfp3v37jz00EO0atXK4Kz7zTff4OjoWOXRvXv3RmvjiBEjqm3H22+/3WjtaGzqJGxLly4lICAAOzs7QkNDSUxMrNF+zZo1dOnSBTs7O4KCgti4caOhrLy8nFmzZhEUFESLFi3w9fVlwoQJZGZmGtUREBBg2PahP9555x0jmwMHDjBw4EDs7Ozw8/Orcb6gvrhei8mWL/3YblsiIiI4dOgQxcXF5OTksG7dOvz9/QFlPio1NbXK48bPf0Pz2WefVduOZ555ptHa0egIE1m5cqVQq9Xiiy++EIcPHxZTpkwRrq6uIicnp0r7P/74Q1hbW4sFCxaII0eOiDfeeEPY2tqKgwcPCiGEuHz5sggPDxerVq0Sx44dEwkJCSIkJEQEBwcb1ePv7y/efPNNkZWVZTgKCwsN5fn5+cLLy0uMGzdOHDp0SHz77bfC3t5efPzxx7V+tvz8fAGI/Pz8m9qmZRcI/1nrRe/5v9S6/ubI1atXxZEjR8TVq1ct3RRJM6G6z5Qp30+ThS0kJERMnz7d8LtWqxW+vr4iNja2SvvRo0eLyMhIo3OhoaFi2rRp1d4jMTFRAOLMmTOGc/7+/uLf//53tdf897//FW5ubqK0tNRwbtasWaJz587VXlNSUiLy8/MNx9mzZ2v9xuUUXBX+s9aLdq+tF1qt7qb2zRUpbBJzYw5hM2koWlZWRlJSEuHh4YZzVlZWhIeHk5CQUOU1CQkJRvagdOGrswcM0Qb+HifqnXfewd3dnT59+vDee+9RUXE9skZCQgKDBg0ySioSERFBWlpatU6TsbGxuLi4GA4/P79q2/R39BvfdQKulMgIHxJJU8IkYcvLy0Or1eLl5WV03svLi+zs7Cqvyc7ONsm+pKSEWbNmMXbsWKMl3eeff56VK1eyfft2pk2bxttvv2208ba6++jLqmL27Nnk5+cbjrNnz1bz5JXRR/gAuHxVroxKJE2JJuXuUV5ezujRoxFC8OGHHxqV3egV3rNnT9RqNdOmTSM2NhaNRlOn+2k0mjpfC0pSl+Kyq/wlI3xIJE0Kk3psHh4eWFtbk5OTY3Q+JyenyrAnAN7e3rWy14vamTNniIuLu6kDXmhoKBUVFZw+fbrG++jLGgL9cFT6skkkTQuThE2tVhMcHGwU01yn0xEfH09YWFiV14SFhVWKgR4XF2dkrxe148ePs3Xr1lqFFk5NTcXKygpPT0/DfXbu3GkUmTMuLo7OnTvXOZTMzdD7skmXD0ldaI7JXJoKJvuxRUdH8+mnn/Lll19y9OhR/vGPf1BUVMSkSZMAJaro7NmzDfYvvPACmzdvZtGiRRw7dox58+axb98+ZsyYASii9sgjj7Bv3z6++eYbtFot2dnZZGdnG8KhJCQksHjxYvbv38+pU6f45ptvmDlzJk888YRBtB5//HHUajWTJ0/m8OHDrFq1iv/85z+VNjabE0N+0SLZY5Pcmpg7mQsoIcdVKhWXL182a72mYPIc25gxY7hw4QJz584lOzub3r17s3nzZsNEfUZGBlZW1/Wyf//+rFixgjfeeIPXX3+dTp068cMPP9CjRw8Azp8/z08//QRQKW3X9u3bufvuu9FoNKxcuZJ58+ZRWlpKu3btmDlzppFoubi4sGXLFqZPn05wcDAeHh7MnTvXEKu9IZD5RSWSJkoDuaLckpjiJyOEEO9uOir8Z60XMT8eauCWNV0q+RzpdEKUFlrm0JnmT7hp0yZx1113CRcXF9GyZUsRGRkpTpw4IYQQIiwsTLz66qtG9rm5ucLGxkb8+uuvQgghMjMzxX333Sfs7OxEQECA+Oabb27qb3kjgFi3bp3h9wMHDoh77rlH2NnZiZYtW4opU6aIK1euGMq3b98u7rjjDuHg4CBcXFxE//79xenTp4UQQqSmpoq7775bODo6CicnJ9G3b1+xd+/eGu+/fft2ARgdMTExQgjFx/Oll14Svr6+wsHBQYSEhIjt27cbrj19+rS4//77haurq3BwcBDdunUTGzZsEOnp6ZXqnDhxYq3eDz3m8GNrUquitxr6oahcPLiB8mJ429cy9349E9Qtbm53DZnMpfpkLjNmzODIkSOsXLkSX19f1q1bx/Dhwzl48CCdOnVi+vTplJWVsXPnTlq0aMGRI0dwdHTEz8+P77//nlGjRpGWloazszP29vZ1ek/qgxS2eiCHorc2MplL1clcMjIyWLZsGRkZGfj6Kv+kXn75ZTZv3syyZct4++23ycjIYNSoUYZQ6e3btzdc37JlSwA8PT3NPn9XW6Sw1QNXexkevBK2DkrPyVL3NgGZzKVqDh48iFarJTAw0Oh8aWmpwWPh+eef5x//+AdbtmwhPDycUaNG0bNnzzrdryGQYYvqgVsLORSthEqlDActcchkLmZJ5lJYWIi1tTVJSUlG0UCOHj3Kf/7zHwCefvppTp06xfjx4zl48CD9+vVjyZIlZnvW+iKFrR4YemxyKHrLIZO5KFSVzKVPnz5otVpyc3Pp2LGj0XHjkNXPz49nnnmGtWvX8tJLL/Hpp58a6gRqnSSmIZDCVg9cbnDQ1elkhPVbCZnMRaGqZC6BgYGMGzeOCRMmsHbtWtLT00lMTCQ2NpYNGzYA8OKLL/LLL7+Qnp5OcnIy27dvN9zP398flUrF+vXruXDhAoWFhSa/H/XGpHXYZo6p7h6l5VrhP2u98J+1XvxVVHrzC5oht3LYori4ONG1a1eh0WhEz549xY4dOyq5YGzcuFEAYtCgQZWuz8zMFCNGjBAajUb4+/uLFStWCE9PT/HRRx/V6v5/v1dN7h7Z2dkiKipK+Pj4CLVaLfz9/cXcuXOFVqsVpaWl4rHHHhN+fn5CrVYLX19fMWPGjFr/TZ555hnh7u5u5O5RVlYm5s6dKwICAoStra3w8fERDz30kDhw4IAQQogZM2aIDh06CI1GI1q1aiXGjx8v8vLyDHW++eabwtvbW6hUKou4e8hkLjdgSrIIPd3nbqaoTMuOl+8mwKP2rgbNBZnM5ToymYt5kMlcmgCuDmqKyq7KebbbEJnMpeki59jqiavML3rbIpO5NF1kj62eGCJ8SF+2246IiAgiIiKqLBs5ciShoaFVlt1sR4A5+eyzz7h69WqVZXpH2uaIFLZ6IvOLSqrCyckJJycnSzeD1q1bW7oJFkEOReuJzC+qINegJObCHJ8lKWz15HYPNmltreR90HvrSyT1pbi4GKjfkF0OReuJfih6uy4e2NjY4ODgwIULF7C1tTWKxSeRmIIQguLiYnJzc3F1dTX806wLUtjqib7HdrtuhFepVPj4+JCens6ZM2cs3RxJM8DV1bXeeUqksNUTV31Mttt0KArK3sBOnTrJ4aik3tja2tarp6ZHCls9ud5ju72/1FZWVrf9zgNJ00FOiNQTt9t8KCqRNEWksNUTl2uLBwUl5WhlhA+JpEkgha2e6JMmCwEFt/E8m0TSlJDCVk/UNlY4apSpytt5AUEiaUrUSdiWLl1KQEAAdnZ2hIaGkpiYWKP9mjVr6NKlC3Z2dgQFBbFx40ZDWXl5ObNmzSIoKIgWLVrg6+vLhAkTyMy8Hjf/9OnTTJ48mXbt2mFvb0+HDh2IiYkxWoU7ffo0KpWq0qEPndyQuNjLBQSJpClhsrCtWrWK6OhoYmJiSE5OplevXkRERFSbdmzXrl2MHTuWyZMnk5KSQlRUFFFRURw6dAhQvIyTk5OZM2cOycnJrF27lrS0NEaOHGmo49ixY+h0Oj7++GMOHz7Mv//9bz766CNef/31SvfbunUrWVlZhiM4ONjURzSZ292XTSJpcpgU2lIIERISIqZPn274XavVCl9fXxEbG1ul/ejRo0VkZKTRudDQUDFt2rRq75GYmCgAcebMmWptFixYINq1a2f4XZ+oNSUlpZZPoiSFzc/PNxxnz541KYKunnGf7hb+s9aLtclnTbpOIpHUHlMi6JrUYysrKyMpKYnw8HDDOSsrK8LDw0lISKjymoSEBCN7UMK9VGcPkJ+fj0qlqjEnYX5+fpVhV0aOHImnpycDBgzgp59+qvF5YmNjcXFxMRx+fn412leHi+yxSSRNCpOELS8vD61Wi5eXl9F5Ly8vsrOzq7wmOzvbJPuSkhJmzZrF2LFjqw3/e+LECZYsWcK0adMM5xwdHVm0aBFr1qxhw4YNDBgwgKioqBrFbfbs2eTn5xuOs2fPVmtbE/psVbd7hA+JpKnQpHYelJeXM3r0aIQQfPjhh1XanD9/nuHDh/Poo48yZcoUw3kPDw+jLEN33HEHmZmZvPfee0bzdTei0WjQaDT1brfbtW1V+XLxQCJpEpjUY/Pw8MDa2pqcnByj8zk5OdVuWvX29q6VvV7Uzpw5Q1xcXJW9tczMTO655x769+/PJ598ctP2hoaGcuLEiZva1RfD4oF095BImgQmCZtarSY4OJj4+HjDOZ1OR3x8PGFhYVVeExYWZmQPEBcXZ2SvF7Xjx4+zdetW3N3dK9Vz/vx57r77boKDg1m2bFmtwuOkpqbi4+NT28erMy5yKCqRNClMHopGR0czceJE+vXrR0hICIsXL6aoqIhJkyYBMGHCBFq3bk1sbCwAL7zwAoMHD2bRokVERkaycuVK9u3bZ+hxlZeX88gjj5CcnMz69evRarWG+beWLVuiVqsNoubv78/ChQu5cOGCoT36nt+XX36JWq2mT58+AKxdu5YvvviCzz77rB5vT+2QQ1GJpIlRl2XXJUuWiLZt2wq1Wi1CQkLE7t27DWWDBw+ulCB19erVIjAwUKjVatG9e3exYcMGQ5neTaOqY/v27UIIIZYtW1atjZ7ly5eLrl27CgcHB+Hs7CxCQkLEmjVrTHouUxMm69mbflH4z1ovBi3YZtJ1Eomk9siEyXWkLgmTAU7kXiH8/+3E2c6GA/OqzlokkUjqhynfT7lX1Azog00WlFTICB8SSRNACpsZ0C8egIzwIZE0BaSwmQFb6+sRPm7XpC4SSVNCCpuZkL5sEknTQQqbmTDkF5W+bBKJxZHCZiZu9/yiEklTQgqbmZAx2SSSpoMUNjMh59gkkqaDFDYzoR+KyvDgEonlkcJmJuRQVCJpOkhhMxP63QdyKCqRWB4pbGbCVWaqkkiaDFLYzIRbCzkUlUiaClLYzISLXDyQSJoMUtjMhH7xoKCkggqtzsKtkUhub6SwmQnXGyN8lFRYsCUSiUQKm5mwsbbC6VqEDzkclUgsixQ2M6JPnCyTukgklkUKmxkxJHW5KntsEoklkcJmRuTuA4mkaSCFzYzI/KISSdNACpsZkflFJZKmQZ2EbenSpQQEBGBnZ0doaCiJiYk12q9Zs4YuXbpgZ2dHUFAQGzduNJSVl5cza9YsgoKCaNGiBb6+vkyYMIHMzEyjOi5dusS4ceNwdnbG1dWVyZMnU1hYaGRz4MABBg4ciJ2dHX5+fixYsKAuj1dnZOgiiaRpYLKwrVq1iujoaGJiYkhOTqZXr15ERESQm5tbpf2uXbsYO3YskydPJiUlhaioKKKiojh06BAAxcXFJCcnM2fOHJKTk1m7di1paWmMHDnSqJ5x48Zx+PBh4uLiWL9+PTt37mTq1KmG8oKCAoYNG4a/vz9JSUm89957zJs3z5BxvjGQQ1GJpIlgajbmkJAQMX36dMPvWq1W+Pr6itjY2CrtR48eLSIjI43OhYaGimnTplV7j8TERAGIM2fOCCGEOHLkiADE3r17DTabNm0SKpVKnD9/XgghxH//+1/h5uYmSktLDTazZs0SnTt3rvY+JSUlIj8/33CcPXu2Tpng9Xy376zwn7VePPHZ7jpdL5FIqseUTPAm9djKyspISkoiPDzccM7Kyorw8HASEhKqvCYhIcHIHiAiIqJae4D8/HxUKhWurq6GOlxdXenXr5/BJjw8HCsrK/bs2WOwGTRoEGq12ug+aWlp/PXXX1XeJzY2FhcXF8Ph5+dX8xtwEwwJXeRQVCKxKCYJW15eHlqtFi8vL6PzXl5eZGdnV3lNdna2SfYlJSXMmjWLsWPHGtLYZ2dn4+npaWRnY2NDy5YtDfVUdx99WVXMnj2b/Px8w3H27Nkq7WqLq8FBVy4eSCSWxMbSDbiR8vJyRo8ejRCCDz/8sMHvp9Fo0Gg0ZqvPEGxSzrFJJBbFpB6bh4cH1tbW5OTkGJ3PycnB29u7ymu8vb1rZa8XtTNnzhAXF2forenr+PviREVFBZcuXTLUU9199GWNgX4j/BUZ4UMisSgmCZtarSY4OJj4+HjDOZ1OR3x8PGFhYVVeExYWZmQPEBcXZ2SvF7Xjx4+zdetW3N3dK9Vx+fJlkpKSDOe2bduGTqcjNDTUYLNz507Ky6/3luLi4ujcuTNubm6mPGadcbkhwoecZ5NILIipKxMrV64UGo1GLF++XBw5ckRMnTpVuLq6iuzsbCGEEOPHjxevvfaawf6PP/4QNjY2YuHCheLo0aMiJiZG2NraioMHDwohhCgrKxMjR44Ubdq0EampqSIrK8tw3LjCOXz4cNGnTx+xZ88e8fvvv4tOnTqJsWPHGsovX74svLy8xPjx48WhQ4fEypUrhYODg/j4449r/WymrLpUR4+YzcJ/1npxIvdKneuQSCSVMeX7abKwCSHEkiVLRNu2bYVarRYhISFi9+7r7g2DBw8WEydONLJfvXq1CAwMFGq1WnTv3l1s2LDBUJaeni6AKo/t27cb7C5evCjGjh0rHB0dhbOzs5g0aZK4csVYPPbv3y8GDBggNBqNaN26tXjnnXdMei5zCNuAd+OF/6z1Yt/pS3WuQyKRVMaU76dKCCEs1VtsahQUFODi4kJ+fr7RHJ8pPLDkdw6ez+fzif0Y2tXr5hdIJJJaYcr3U+4VNTMywodEYnmksJkZmV9UIrE8UtjMjMwvKpFYHilsZsZNDkUlEosjhc3MuMihqERicaSwmRk5FJVILI8UNjPj1kIORSUSSyOFzcy42OuHorLHJpFYCilsZsbgx1Yke2wSiaWQwmZm9AldrpRWUC4jfEgkFkEKm5lxtrse4q5AroxKJBZBCpuZsbG2wumauMmkLhKJZZDC1gAY8ovKBQSJxCJIYWsA5EZ4icSySGFrAGR+UYnEskhhawDcDEld5FBUIrEEUtgaAJlfVCKxLFLYGgBXe5lfVCKxJFLYGgCZX1QisSxS2BoAORSVSCyLFLYGQC9scigqkVgGKWwNgByKSiSWpU7CtnTpUgICArCzsyM0NJTExMQa7desWUOXLl2ws7MjKCiIjRs3GpWvXbuWYcOG4e7ujkqlIjU11aj89OnTqFSqKo81a9YY7KoqX7lyZV0esV7oFw/ypbBJJBbBZGFbtWoV0dHRxMTEkJycTK9evYiIiCA3N7dK+127djF27FgmT55MSkoKUVFRREVFcejQIYNNUVERAwYM4N13362yDj8/P7KysoyO+fPn4+joyIgRI4xsly1bZmQXFRVl6iPWG1cZ4UMisSymZmMOCQkR06dPN/yu1WqFr6+viI2NrdJ+9OjRIjIy0uhcaGiomDZtWiVbfVb4lJSUm7ajd+/e4qmnnjI6B4h169bd/CGqwRyZ4IUQokKrE/6z1gv/WevFhSsl9apLIpEomPL9NKnHVlZWRlJSEuHh4YZzVlZWhIeHk5CQUOU1CQkJRvYAERER1drXhqSkJFJTU5k8eXKlsunTp+Ph4UFISAhffPEFooZE96WlpRQUFBgd5sDaSmUIXyTn2SSSxsfm5ibXycvLQ6vV4uXlZXTey8uLY8eOVXlNdnZ2lfbZ2dkmNvU6n3/+OV27dqV///5G5998802GDBmCg4MDW7Zs4dlnn6WwsJDnn3++ynpiY2OZP39+ndtRE64OagpKKuS2KonEApgkbE2Bq1evsmLFCubMmVOp7MZzffr0oaioiPfee69aYZs9ezbR0dGG3wsKCvDz8zNLO90cbMm4JHtsEoklMGko6uHhgbW1NTk5OUbnc3Jy8Pb2rvIab29vk+xvxnfffUdxcTETJky4qW1oaCjnzp2jtLS0ynKNRoOzs7PRYS70+UWlL5tE0viYJGxqtZrg4GDi4+MN53Q6HfHx8YSFhVV5TVhYmJE9QFxcXLX2N+Pzzz9n5MiRtGrV6qa2qampuLm5odFo6nSv+mBw+ZC7DySSRsfkoWh0dDQTJ06kX79+hISEsHjxYoqKipg0aRIAEyZMoHXr1sTGxgLwwgsvMHjwYBYtWkRkZCQrV65k3759fPLJJ4Y6L126REZGBpmZmQCkpaUBSm/vxp7diRMn2LlzZyU/OICff/6ZnJwc7rzzTuzs7IiLi+Ptt9/m5ZdfNvURzYKbDDYpkViOuiy7LlmyRLRt21ao1WoREhIidu/ebSgbPHiwmDhxopH96tWrRWBgoFCr1aJ79+5iw4YNRuXLli0TQKUjJibGyG727NnCz89PaLXaSm3atGmT6N27t3B0dBQtWrQQvXr1Eh999FGVttVhLncPIYRYtCVN+M9aL15fe6DedUkkEtO+nyohavCHuM0oKCjAxcWF/Pz8es+3ffF7Om+uP0JkTx+WPt7XTC2USG5fTPl+yr2iDYRbC7mtSiKxFFLYGghXe7kqKpFYCilsDYSLXDyQSCyGFLYG4npuUSlsEkljI4WtgdD7sRWWVlBWISN8SCSNiRS2BsLZ3haVSnkte20SSeMiha2BUCJ86HcfyAUEiaQxkcLWgFzPfSB7bBJJYyKFrQHRz7PJlVGJpHGRwtaAXE/qIoeiEkljIoWtAXGVvmwSiUWQwtaAGIaicvFAImlUpLA1IDK/qERiGaSwNSByKCqRWAYpbA2IQdjkUFQiaVSksDUgcigqkVgGKWwNiPRjk0gsgxS2BkT6sUkklkEKWwOiT+hSVKaVET4kkkZEClsD4mR3PcKHXECQSBoPKWwNiFGEDznPJpE0GlLYGhhDflEZk00iaTTqJGxLly4lICAAOzs7QkNDSUxMrNF+zZo1dOnSBTs7O4KCgiolPF67di3Dhg3D3d0dlUpFampqpTruvvtuVCqV0fHMM88Y2WRkZBAZGYmDgwOenp688sorVFRU1OURzYbLtQWEv4rkUFQiaSxMFrZVq1YRHR1NTEwMycnJ9OrVi4iICHJzc6u037VrF2PHjmXy5MmkpKQQFRVFVFQUhw4dMtgUFRUxYMAA3n333RrvPWXKFLKysgzHggULDGVarZbIyEjKysrYtWsXX375JcuXL2fu3LmmPqJZub5fVPbYJJJGw9RszCEhIWL69OmG37VarfD19RWxsbFV2o8ePVpERkYanQsNDRXTpk2rZJueni4AkZKSUqls8ODB4oUXXqi2XRs3bhRWVlYiOzvbcO7DDz8Uzs7OorS09CZPpWDOTPB6Xvg2WfjPWi8++fWk2eqUSG5HTPl+mtRjKysrIykpifDwcMM5KysrwsPDSUhIqPKahIQEI3uAiIiIau1r4ptvvsHDw4MePXowe/ZsiouLje4TFBSEl5eX0X0KCgo4fPhwlfWVlpZSUFBgdJgbvS+bzC/aQOi08L9H4OuHldcSCWBjinFeXh5ardZIPAC8vLw4duxYlddkZ2dXaZ+dnW1SQx9//HH8/f3x9fXlwIEDzJo1i7S0NNauXVvjffRlVREbG8v8+fNNaoepuMihaMOSsRtOxCmv03+FDkMs2x5Jk8AkYbMkU6dONbwOCgrCx8eHoUOHcvLkSTp06FCnOmfPnk10dLTh94KCAvz8/Ord1hvRr4pKd48G4sgP118fWCOFTQKYuHjg4eGBtbU1OTk5RudzcnLw9vau8hpvb2+T7GtLaGgoACdOnKjxPvqyqtBoNDg7Oxsd5kYORRsQnQ6O/HT996M/Q1lx9faS2waThE2tVhMcHEx8fLzhnE6nIz4+nrCwsCqvCQsLM7IHiIuLq9a+tuhdQnx8fAz3OXjwoNHqbFxcHM7OznTr1q1e96oPMiZbA3J2NxRmg8YZXNpC2RX4c5OlWyVpApg8FI2OjmbixIn069ePkJAQFi9eTFFREZMmTQJgwoQJtG7dmtjYWABeeOEFBg8ezKJFi4iMjGTlypXs27ePTz75xFDnpUuXyMjIIDMzE4C0tDRA6Wl5e3tz8uRJVqxYwX333Ye7uzsHDhxg5syZDBo0iJ49ewIwbNgwunXrxvjx41mwYAHZ2dm88cYbTJ8+HY1GU793qR7oe2wyaXIDcPgH5Wfn+8ClNfy2SBmO9hhl0WZJmgB1WXZdsmSJaNu2rVCr1SIkJETs3r3bUDZ48GAxceJEI/vVq1eLwMBAoVarRffu3cWGDRuMypctWyaASkdMTIwQQoiMjAwxaNAg0bJlS6HRaETHjh3FK6+8UmnZ9/Tp02LEiBHC3t5eeHh4iJdeekmUl5fX+rkawt0j/UKh8J+1XnSds8lsdUqEEFqtEAs7CxHjLMSxjULkHFVez28pRGGepVsnaQBM+X6qhBDCgrrapCgoKMDFxYX8/HyzzbflF5fT680tAKT9azgaG2uz1HvbcyYBlg1XhqGvnAAbDXw0ELIPQOQiuONpS7dQYmZM+X7KvaINjJOdjSHChxyOmpEjPyo/O49QRA2g52jl54E1lmmTpMkgha2BsbJSXfdlkwsI5kGnuy5s3aKun+8xClApiwp/nbFEyyRNBClsjYCbzH1gXs7thSuZoHYy9ltz9oV2g5TXB2Wv7XZGClsjcL3H1gx82c4lwdXLlm2D3im38wiwtTMuMwxHV4OcPr5tkcLWCDQbX7ZdS+CzIfDDs7Uy1+kEu09dZPepi+Zrg9Ew9MHK5V0fAGsN5KUpCwmS25JbZkvVrYxhKHorhwdP/w3iYpTXJ7YqHv5qhypNT14oZG3yOdYlnyczvwSAdc/2p09bt/q34/w+KDgPakfoOLRyuZ2L0pM78oPSa/PpVf97Sm45pLA1Ai72ttyhOoZdbilQt32tFqUgE76bBOJa9AxtKZzZBZ2uR225XFzGzwey+D7pHKlnL1eq4quEM+YRNr1TbuBwsLWv2qbnaEXYDn0P974JVtLFpq6UVegQiFvOTUkKWyMQdmUL8zRvUnZYA5H3gL0ZvuCNRUUZrJ4IRRfAqwd4dlUm5k/GU95+CDvSLvB90jm2HculTKtk4rK2UjE4sBWj+rahlZOG0R8nsOFAFv+M7IqHYz12gdw4DO0eVb1dx3vBzhWuZMHp36D93XW/523KyQuFfJ1whu+SzuFib8v65wbg1kJt6WbVGilsDc3R9dx74i0A1KJU+WIGP2nZNpnCln/CuUTQuMCYrxGZ+1EdXENu6iaG7x3CpRtCnnfzcebhvq15sHdrWjldF7BebVzYfy6fVXvPMv2ejnVvy/kkKDh3bRgaXr2djVoRvqTlik+bFLZaodUJth3L5auE0/x2PM9wvrC0gve3HSfmge4WbJ1pyMWDhuTUDvhuElZCyznhoZw7sNqiTTKJA6sh8dqe3oc/4bTOizFxdmiFCs+SdDRFWXg4apgysB2bXhjIxhcG8vTA9kaiBvDEnf4ArNiTgVZXj5VK/WpoYET1w1A9PccoP4/+BOVX637P24C/isr46NeTDFqwnSlf7eO343moVBDe1YvXRnQB4OuEM5y6UGjhltYe2WNrKM7uhW8fB20ZF9oM47GTkezUvIjVmT/g8llwNW/cN7OTfQh+el55PehVLrUZwpP//YPTF3Uc0HSkD8dZNugKHSOGYGNd8//HB3r58n8bj3L+8lW2Hcvl3m5eNdpXiRDXQxTd6JRbHX53gosf5J+FPzdD94dMv2cTJfdKCb+mXeDXPy+QU1BCgHsLOng60qGVI+1btaBtSwdsb/I3ATh0Pp+vEk7zY2ompdcSers62DLmDj+eCPXHr6WyOJSYfoltx3J5Z9MxPpnQr0GfzVxIYWsIsg/BN6OgvAja30PWoA84d2IfKaruBItDcOg7GDDT0q2snquXYfV4qLgKHYZQctcrPP35Xk5fLKaNmz2deo2E3YvoUpgI1tNvWp2drTVj+vnx8c5TfJVwum7Cdj4Z8jPAtgV0uvfm9lZWEPQI/P5vZTh6CwtbhVZHytnL7EjLZUfaBQ5nGoew33v6L6PfbaxU+Ls7XBM6Rzq0uiZ8Ho7Yq63ZdCiLrxLOkHTm+nXdfZ2Z2D+Akb18sbM1Xih4/b4u/PrnBbYcySHh5EXCOrg33MOaCSls5ubiSfj6ISjJhzYh8Ng3uFxRitZW9CfY+hDiwGpUTVXYdDpY9wxcOgUubdE99Bkz1xwkOeMyznY2LJ90B44lLWD3ImWordPWatXxiTv9+eS3U/x2PI9TFwpp38rRtHYdWaf8rM0wVE/PMYqwHd8CxZfAoaVp97QguQUl7PjzAr+mXeC34xcoKDFOI9mzjQt3B7aig6cjp/OKOXmhkFN5hZzMLeJquZaTF4o4eaEIMA6+qraxouxa78zWWsV9QT5MCAugb1tXVPpNzX+jo6cTj4e05evdZ/jXhiP8PGMAVlZV2zYVpLCZk/zz8FUUFOUqK4jjVoO6BR6OFdhaq/i5/A7mWn2BJvcI3/60kQED7jZ095sMv/8/JVijtQZGf0nsrzlsOpSN2tqKTyb0o6OnE2iDlcWEksuQmQJtbj488WvpwD2dPdl2LJf/7c5g7gMmBP8UAg7XYjX073h2Ba8gyDmoLNr0m1T7ay3A/rOX2XIku8pemauDLYM6teLuzq0YFNiq2tVlnU6QXVCiCN2FIk5eKDS8zsovoaxCh5ezhnGh/jwW4oenk12V9fydF8M78UPKeQ5nFrA25TyPBLep9/M2JFLYzEVRHnwdpQyXWraH8esMbh0tNDZ89VQoX+8+za9pfRim2kt+4goG7hL0betKVJ/WRAb54F4fVwhzcHIbbPuX8jpyIV+eacmnvykZvt57tCd3tr82BLG2gfaDlYn5E/G1EjaA8WH+bDuWy5qks7wcEYiDupYfv0z9MNRBceUwhZ6PQtxBZSGkiQqbEIL/F/cnS7adMDqv75Xd3cWTXm1csa5FL8nKSoWvqz2+rvYM7NTKqKywtILs/BL83Ws3B3cj7o4aZgzpSOymY7z3yzHuC/Ku/d/PAshVUXNQkg//exjy/gTn1jDhR3D0NDIJ6+DOf8cFM2CUMif1qCYBa5WO5IzLzP3xMCFvx/PkskR+SDlPUakFstdfPgvfTQYE9J1AnF0E839WRO2ViM482Lu1sb1+8/nJbbW+xeBOrWjb0oErJRX8mJpZ+7bpnXI7Dat2t0O19HgEUEHGLricYdq1jUCFVses7w8YRO2+IG/+3+he7HsjnJ9mDCB6WGf6tnWrlajdDEeNDR09HU0WNT0T+wfg19KenIJSPtl5qt7taUiksNWXsmJY8Rhk7QcHDxj/A7i2rdbcodsI0Ljgrs1j3xMOzLm/Gz3buKDVCXakXeDFVan0+9dWnv82hV8OZ5OWfYVLRWXo6uMmcTPKS5TFgquXwKc3B4L+yXPfJqMTMDbEj2fvrmK3hH4707m9irDXAisrFU/cqbw3XyWcoVYxToW47uZhyjBUj0trCBigvD74nenXNyDFZRVM/TqJ1fvOYaWC2IeD+O+4YB7u26Z+jswNhJ2tNa8N7wrAx7+eIvvadrmmSNPtS94KVJTB6glKb0DjDOPXQqvAmq+xtYNuIyHla9xO/sDkkUuYPKAdJy8U8lNqJj+mnuf0xWJ+2p/JT/uv92psrFS0bKGmlZMGD0flUF4r51o5avBw0hDg3gK1jYn/rzbPUubK7N3IjPiEp/53kJJyHYMDW/HWgz2qnlR2bQvuneDicTj1q/JMtWB0Pz8WbfmTo1kFJGf8RbD/TSb0M1OUnpaNvdJjqws9Rys7EA6sVlajq5kkb0wuFZXx1PK9pJ69jMbGig8e71u31eJG5r4gb4L93Ug68xcLt6Sx8NGmuRdXCltd0Wlh3TQlWa+NPTxuwobrnqMh5WtlQvu+hWCjoUMrR2beG8iL4Z3Yfy6fH1PP88eJPHKvlHK5uJwKnSD3Sim5V0prrNrDUc2EsADGhbat3Zxd8teKhz4qiu7/mPHfZ5FXWEY3H2eWjutbs49ax6GKsJ2Mr7WwuTqoGdnLlzVJ5/gq4czNhU2/hSpwGKhb1Ooeleg6Eja8BBeOQs4h8A6qWz1m4uylYiZ+kcipvCJcHWz5fGK/m78PTQSVSsUbkV156L+7+D75HE/2D6BHaxdLN6sSUtjqghCwfiYcXgtWtjDmf+BvQjpB/wHKXFzBecUVoesDhiKVSkVvP1d6+7kazpVV6LhUVMaFK6XkFZZyobDU8DqvsIy8K8q5nPwS8grL+H9xf7J0+wke7tuGyQPa0dGzGteKzFTlCw9UDJ7NpN+dOXnhEj4udiybdAeOmpt8PDoMgT0fwYltyntSy57QhLAA1iSdY+PBLN6I7FZpp4KBG4ehtXHKrQ57V2XT/NGflF6bBYXt0Pl8Ji3fy4UrpbR2tefLp0Kq//s0Ufq0dePB3r78mJrJ/204yoopodW6ilgKKWx1YWsMJH8JKisY9alRlItaYWWlhLHe9T4cWGUkbFWhtrHC28UOb5eal+bLtTo2Hszis9/SOXg+n28TM/g2MYMhXTx5emA7wtq7X/8ACgE/vwDaUkSnCF7KCicxPRsnjQ3LJt2Bl3Mt3AACBoC1WlmxvHgSPGq3DzSojQu9/VxJPXuZVXszmDGkU9WGWfvhr9NKjzgwolZ1V0vP0YqwHfwOwucrf4NG5vfjeTzzvyQKSyvo4u3El0+F1O59boK8EtGZTYeySTh1ka1H67ibpAGRiwd1waeX0lN74D9192jXR3r98xezRaS1tbbiwd6t+WnGXayaeif3dvNCpYJtx3J5/NM93L/kd9alnFMcNM/ugaxUsLHjA+eZ/HggGxsrFR+ND6aLdy0zdKlbQNs7ldcn42u2/RsTwpT9o9/syaDiWlSQSuh7a53urfswVE+nYUqstiuZcOb3+tVVB35MPc+k5YkUllZwZ/uWrH4m7JYVNYA2bg48PaAdAG9vPGpw+m0q1EnYli5dSkBAAHZ2doSGhpKYmFij/Zo1a+jSpQt2dnYEBQWxceNGo/K1a9cybNgw3N2VHoU+y7ueS5cu8dxzz9G5c2fs7e1p27Ytzz//PPn5xqtxKpWq0rFy5cq6PGLN9BgFzyVB3wl1r8OrB3h2A22Z0pMwIyqVitD27nw6oR/x0YMZf6c/drZWHM4sYOaq/QxcsI3jPy8C4KT3CBb9cQmAd0b15K6OHqbdrA5uHwD3BfnQsoWarPwSth7NrWwgxHU3j7qshv4dG831iLuNHIjg052neGFlKuVaQWRPH758KgRnO9tGbUND8I+7O+DhqCY9r4hv9jSt5DkmC9uqVauIjo4mJiaG5ORkevXqRUREBLm5VXw4gV27djF27FgmT55MSkoKUVFRREVFcejQIYNNUVERAwYM4N13362yjszMTDIzM1m4cCGHDh1i+fLlbN68mcmTJ1eyXbZsGVlZWYYjKirK1EesHW7+9btepYKgR5XXDfhFa9/KkbeiepDw2lBeiehMKycNoiCbgFylh/VCeojyc2inunmTd7jm9pH+m7JKXEvsbK0Z3U8JBPD17tOVDbIPwF/pYGMHneo5DNWjj/hx5CfFxeVmCAE5h2HnQlh2H3w6FFY9ARtfVbZq7V+lrAjnHYfSypEvdDrBv9Yf4f82HgVg0l0BLHmszy0XtLE6nOxsib63MwCLtx5vUjk9TE6YHBoayh133MEHH3wAgE6nw8/Pj+eee47XXnutkv2YMWMoKipi/fr1hnN33nknvXv35qOPPjKyPX36NO3atSMlJYXevXvX2I41a9bwxBNPUFRUhI2NMlWoUqlYt25dncWsIRIm18jlDFgcBKhg5mHF56qBKa3QcnLV63Q7/hGJus6MLothVN82LHy0Z90mgHU6WNRZ2UY28efrWaJqwdlLxQx6bztCwNbowcaT6FvnK9u7utwPj31jeruqaav23z2wvnKer/z+Rev+oxnSxdP4uctL4PTvSkSQP39R5g9ri8YZnHzA2Qetow/rsj14LeMOKrBh9oguTB3UvslNsteXCq2OyPd/Jy3nCpMHtGPO/SZslTMRU76fJi0elJWVkZSUxOzZsw3nrKysCA8PJyEhocprEhISiI6ONjoXERHBDz/8YMqtK6F/OL2o6Zk+fTpPP/007du355lnnmHSpEnVfphKS0spLb3uPlFQUFClXYPh2hba9lf84A59B3e90OC31FBBt8zvAXAaNIO3nYJ4JLhN3b9wVlbKcPTASmV7lQnC5tfSgaFdPNl6NJf/7T7DvJHXAhkaOeXWPypHUWkFW45k80NKJv0v92Wa9Xk8T//I5OPt6dHamZf6uzKYFKyO/wIntytRWfTY2CmBKgMjoEUrKMhS5umMfmZBWSGUFihHXhrWwCPABdsH8X74bR7q07T3VtYVG2sr/hnZlQlfJPJVwmmeuNOfdh6mz4cKIcwq+iYJW15eHlqtFi8v4xUQLy8vjh07VuU12dnZVdpnZ2eb2FTjdrz11ltMnTrV6Pybb77JkCFDcHBwYMuWLTz77LMUFhby/PPPV1lPbGws8+fPr3M7zELP0YqwHVjTKMLG4XVKmG8nX7reM5au1maY69EL28ltcK9p7+f4sAC2Hs3l+6RzvBLRmRYaG8g+qEQXsdbUeTW0XKvj9+N5/JB6ni2Hc7haruRryFbdxTTrnwm3SeUVq7XcdSGF3j+fNL7YyUe5b+AIRahrsY0rNy+Pn37bx579h3Eqy6WrVQZTbDYyzXYjVq1fr9Mz3CoMClQ25+9Iu8A7m47y8fib7x3+q6iM5Iy/SM74i6Qzf6HVCdY8099sbbrl3D0KCgqIjIykW7duzJs3z6hszpw5htd9+vShqKiI9957r1phmz17tlFvsqCgAD+/Rg4A2e1B2PiKEoEi5wh4NVxXHoA9Hys/73gKzCFqcH0BIfsAFOZW2idbEwM7ehDg7sDpi8X8kHqecaH+151yO90LGqda1yWEIDnjMj+mnmf9gSyjsOUB7g482Ls1UX3uhtVfY5N7mOmq7+BaJ2G/rj3x2r6ccLuL4UMjiOzpW6v9mYfO5/PF7+n8fCCTcq0AuuDv3pde/QOoSNdhc2Kz4iv45PomseOhofjnfV357XgevxzOYfepi9cDJqDMNZ64UEjSGUXEUs/k8VdeDu6qAjxU+XiSj52qnKLSEOUfmxkwqRYPDw+sra3JyTGO8ZSTk4O3t3eV13h7e5tkXxNXrlxh+PDhODk5sW7dOmxta/5ihoaG8tZbb1FaWopGU9kJVKPRVHm+UXFoqbgipG2Ag6vBa17D3evcPiVShrUa+j5pvnodW4F3T0XYTm6HXmNqfamyf9Sff204ytcJZ3j8Dj9UJjrlns4rYm3yOX5IzSTjUrHhvIejmvt7+hLVpzW92rhcH+oMfhV++afithMYQb7fPcTvL2HZH+lcyatg48pUFscf57khHXmgp2+l3Rc6nWB7Wi6f/ZZOwg05U0MCWjJ5YDvCu3opotj9PVi6U3Ev2b8Seo+t9ftyq9HJy4nH7vBj1Z5T/PTDt7TsbktO1jmuXMxEeyUHF+1leqvyCVfl05ICrO2Mp/a1ahdUtgvM1h6ThE2tVhMcHEx8fLxhgl6n0xEfH8+MGTOqvCYsLIz4+HhefPFFw7m4uDjCwkzw1EfpTUVERKDRaPjpp5+ws7u5D1Bqaipubm6WF6+b0fPRa8L2HQyZ23DOo/reWo9HFDEyJx2GXBO2bSYJG8CjwX4s3JLGsewrHEpJIOjiCWUY2nl4tdcIIdh35i8+2XmKrUdzDEnfHdTWRHT3JqpPa+7q4F71lrDuUUYuJC5A9L0weUA7vtx1ms9/T+fUhSJmrtrPf7YeZ/o9HYnq05oKreD75HN88Xs6p/KUeThrKxWRQT5MHtCOXjfsFgGUOdTBr8LWeUpSnMCIWyrYpam81M+Gsanz6VFwEhLAaNd0VQvB9i2V3n2LVlg7egECQxe6npjc74uOjmbixIn069ePkJAQFi9eTFFREZMmKbGuJkyYQOvWrYmNjQXghRdeYPDgwSxatIjIyEhWrlzJvn37+OSTTwx1Xrp0iYyMDDIzlU3faWlpgNLb8/b2pqCggGHDhlFcXMz//vc/CgoKDBP9rVq1wtramp9//pmcnBzuvPNO7OzsiIuL4+233+bll1+u3zvUGAQOV1bU8s/C2d3gb765BgNXcpT5NYDQqTXb1oWOQ+GPxYqw6XQmibOLgy0P9mrNqn1nubrj/12rL7zKYWiFVscvh3P45LdT7L8hf+ndnVvxUJ/W3NvNq85xwlzsbXl+aCcm3RXA17vP8OnOU5y+WMwr3x3gP/HHKSyt4HJxOQBOdjY8HtKWif0D8HWtIaJv2AzFLeTCUYifrzh1N0cOrKbl+pm0VBVSIBw4Zt0JlaMnDi19cPdsQytvP6ydPJV/qC08oYWH+aZCqkLUgSVLloi2bdsKtVotQkJCxO7duw1lgwcPFhMnTjSyX716tQgMDBRqtVp0795dbNiwwah82bJlAkWujY6YmBghhBDbt2+vshwQ6enpQgghNm3aJHr37i0cHR1FixYtRK9evcRHH30ktFptrZ8rPz9fACI/P78ub0v9WPesEDHOQvz0fMPUvz1Wqf/T8Iapv7xUiH/5KPfI3G/y5QfPXRYTZ78lRIyz0M1zFeLsPqPywpJysez3U2LAu/HCf9Z64T9rvej0z43ite8PiOM5V8z1FJXu+fGvJ0TwW1sM9xzwbrz44vdT4kpJee0rOv2H8r7EOAuRsadB2moxSq4IsfaZ68/3+XBRdvFMg9zKlO+nyX5szZlG92O7kVM74KsHlUS/Lx9XcmOai4oyWNwDCnNg1OdKkpOGYMUYxf8rfJ7pyWpKCriwoC+tdBdIbv0EfacsBZTY/18mnOZ/uzPIv6r0ltwcbBkfFsCEMP9GiVt2tUzL5sNZOGpsGdLFs25BH3+YDqn/U3acTP1ViUJ8q5O1H757Ci6eUPZND3oVBr3SYM/WYH5skgYkYCA4ekNhthIKqUuk+eo++pMiao7eSgifhqLDEEXYTm4zXdji59NKd4EzOk9m5kbyYWYBy/5I58fUTEOG+QB3ByYPbM8jfdtgr2487317tXX9/dDufVOZR805pERE6V/1nHSDIYRy77OJykJPm351X6UVQnmGuLnKlkAnXyUYhD6gZxNACltTwcpa6UklfKBE/DCnsO25tsOj31Pm7Qn+Hf32qozdUFZU+43rZ3bB3s8AeNvmWc5cEdz3/m+G4n7+bkwZ1P76auOtSAt3Rdx+eg62v60sXrg0sNNu+VVlq5t+F0XBuetlLm2hx8PKvmfvoNqLXNFF+PFZpU6AzvfBg0ub3KKIFLamRM/RirClbVbCbduZIYDf+WQlfLeVLQQ/Wf/6asK9g7ISeDlD2ZZUG+fa8qvw47XeS9+JdLa7j1+2nUClguHdvXl6YHuC/d0att2NRe8nIOV/SmSVza8pcfzMTUEWHP9FEbJTO6D8uvsLNvbQOliJ6pKfoSz2/LFYiYTcY5QidK06V193+k5YO1XZaWGtgWH/gpApTdI/TwpbU8K7J3h0hrw0OPoz9Hmi/nUmXlt97vEwODVwzCyVShmOJi1XhqO1EbZf34VLJxVv/3vf5Dm1Mx08HenVxpWAOmzNadJYWcH9/4aPBip/3z9/qX+cOZ1OEao/f1F6UVmpxuXObZR7dB6hDBVt7ZV/Jn/+Aoe+VwKdXjwOv76jHF5B13pyD4NbgFKHtkIp27kQEOARCI98YfFIxDUhFw9uwKKLB3p2LoRtb0G7wTCxnuGMCi/Av6+FRnp6G7QJNk8ba+LIT0piGPdO8Ny+mm0zU+HTISC08Ni30OW+hm9fU2DLG7BridK7fXaP6Zm3QIkm8tsiSF2hzMsaUCnzZ/otYV7da+5RlRRA2iZF5E7Gg+6GDGmt+ylD5qPrFTckgD7jYcS79Y+PVwfk4sGtTNCjirCl74SCTHD2rXtdycsVUWsd3DiiBkq+UZW10gu4nFF9xi5tOfw0QxG17g/fPqIGMPg1OLROeX9+WwhD59b+WiGUkPS/vKFswgdQOyo95cDhyi4WU5yv7ZwVh+peY6D4ktKTPPS9kvzm/D7lAMXP8v5/N9yKupmRwtbUcPMHvzuV/5CHvof+z9WtHm057P1ceR0yzXztuxl2LtDmDqX9J+KrT1K8631ls7u9G4ww31aaWwKNo9LrWTUO/nhfiRNX09yWntxjsOkV5Z8eKEPFe99UBM3GDG4vDi0heKJyXMlR9uweXqfUff+/oWW7+t+jkZChwZsiPc0QgPLoz8okbwtP80SgNYWbRdW98CfsuBZUdPi75t/edSvQJVIZKurKYX001DQjVHpF2dv60V2KqNnYwd2vK8PYbg+aR9T+jpOXskPlqU0w4YdbStRAClvTpPvDYGWj7L3MTKlbHfpFg36TGuaDXxP6ZMqnflUmnm9Ep1NcHrSl0PHe67kfbjdUKqXXZmN/fZP83xFCCWe1pJ+yWq6rgM6RMH0P3D1LyVErqRIpbE0Rh5bXw2F/MRx2vKOsZNWWrP2QkaCIY3A1Q8GGxLePsoOiNB/OJxmX7ftcGaaqHZXhTRN0FWg03PwVgQJlQaH40vWynCOw/H5Y+7SyOODWDh5fA2NXXF+tlFSLFLamyn0LlN0IFSWwIxY+CFHmPGqziL3nWm+t24Pg7NOw7awKK2sl6iwYD0cvZyiRLkDZduXayLHvmiJ3TodWXaA4T9kkX5IPm2fDRwOUnpyNPdzzBjy7W0kaLakVUtiaKi5tlBwCjy5XfJHyM2D1BPhqJOQerf66ootwcI3yOvSZRmlqleiHo/q0fPok02WF0DYM+lVOxHNbYqOGyGsRTZKWw5Jg2P1fZbW4y/0wIxEGvyKHnSYiha0po1IpMf9nJCobjK01yuTxh3fBpllV5yNN/lKZv/LpraxOWgr99qrzSXD1L2Wb2ImtyjOMXGKRhMVNloC7oPc45XXRBWjZAZ74XkliU527jKRG5KfrVkDdAob8UxG4Lvcr/833fARL+kLSl6BT4vmjrbju4hE6zbLzVy6tlSGW0CkBNDdfy2B292vgUU3m99uZYf9SxO3eN+HZBCUenaTOyJ0HN9Akdh7UhpPbYNNrytYrUHpn972nuHesngAO7jDziOWHL5tnK8MqKxtlRc+7J0zZ1rABBiXNFlO+n7LHdivSYQj84w+IiFU8wrNS4fN74ecXlfLgSZYXNbg+HNVVKLsRHvxAipqkUZDCdqtibQthz8JzSdc2y6vg6iVFQPo9ZenWKfj3V+bUQEkt6NPLsu2R3DbILVW3Oo6eSjysfk/B74sVMWmEjPK1Qu0AI95Rtk4NnmXp1khuI+Qc2w3cMnNsEsltiJxjk0gktzVS2CQSSbNDCptEIml21EnYli5dSkBAAHZ2doSGhpKYmFij/Zo1a+jSpQt2dnYEBQWxceNGo/K1a9cybNgw3N3dUalUpKamVqqjpKSE6dOn4+7ujqOjI6NGjSInJ8fIJiMjg8jISBwcHPD09OSVV16hoqKiUl0SiaR5Y7KwrVq1iujoaGJiYkhOTqZXr15ERESQm5tbpf2uXbsYO3YskydPJiUlhaioKKKiojh06JDBpqioiAEDBvDuu+9We9+ZM2fy888/s2bNGn799VcyMzN5+OGHDeVarZbIyEjKysrYtWsXX375JcuXL2fuXBOik0okkuaBqdmYQ0JCxPTp0w2/a7Va4evrK2JjY6u0Hz16tIiMjDQ6FxoaKqZNm1bJNj09XQAiJSXF6Pzly5eFra2tWLNmjeHc0aNHBSASEhKEEEJs3LhRWFlZiezsbIPNhx9+KJydnUVpaWmtns2imeAlEkmNmPL9NKnHVlZWRlJSEuHh1/exWVlZER4eTkJCQpXXJCQkGNkDREREVGtfFUlJSZSXlxvV06VLF9q2bWuoJyEhgaCgILy8rmdiioiIoKCggMOHD1dZb2lpKQUFBUaHRCK59THJQTcvLw+tVmskHgBeXl4cO3asymuys7OrtM/Ozq7Svro61Go1rq6u1dZT3X30ZVURGxvL/PnzK52XAieRND3030tRC9fb23rnwezZs4mOjjb8fv78ebp164afnwyAKJE0Va5cuYKLS83JxE0SNg8PD6ytrSutRubk5ODt7V3lNd7e3ibZV1dHWVkZly9fNuq13ViPt7d3pdVZ/X2ru5dGo0GjuZ4PwNHRkbNnz+Lk5ITqJiF/CgoK8PPz4+zZs7fVLgX53PK5LYUQgitXruDre/OUlCYJm1qtJjg4mPj4eKKiogDQ6XTEx8czY8aMKq8JCwsjPj6eF1980XAuLi6OsLCwWt83ODgYW1tb4uPjGTVqFABpaWlkZGQY6gkLC+P//u//yM3NxdPT03AfZ2dnunXrVqv7WFlZ0aZNm1q3C8DZ2dnif3BLIJ/79qKpPPfNemoGTF2ZWLlypdBoNGL58uXiyJEjYurUqcLV1dWwGjl+/Hjx2muvGez/+OMPYWNjIxYuXCiOHj0qYmJihK2trTh48KDB5uLFiyIlJUVs2LBBAGLlypUiJSVFZGVlGWyeeeYZ0bZtW7Ft2zaxb98+ERYWJsLCwgzlFRUVokePHmLYsGEiNTVVbN68WbRq1UrMnj3b1EesFbfrCqp8bvnctwImC5sQQixZskS0bdtWqNVqERISInbv3m0oGzx4sJg4caKR/erVq0VgYKBQq9Wie/fuYsOGDUbly5YtE0ClIyYmxmBz9epV8eyzzwo3Nzfh4OAgHnroISPhE0KI06dPixEjRgh7e3vh4eEhXnrpJVFeXl6XR7wpt+ofvL7I55bPfStQJ2GTCFFSUiJiYmJESUmJpZvSqMjnls99KyDDFkkkkmaH3AQvkUiaHVLYJBJJs0MKm0QiaXZIYZNIJM0OKWwSiaTZIYWtDpgaaPNWZ968eahUKqOjS5culm5Wg7Bz504eeOABfH19UalU/PDDD0blQgjmzp2Lj48P9vb2hIeHc/z4ccs01ozc7LmffPLJSp+B4cOHW6axtUAKm4mYGmizudC9e3eysrIMx++//27pJjUIRUVF9OrVi6VLl1ZZvmDBAt5//30++ugj9uzZQ4sWLYiIiKCkpKSRW2pebvbcAMOHDzf6DHz77beN2EITsbAf3S2HqYE2mwMxMTGiV69elm5GowOIdevWGX7X6XTC29tbvPfee4Zzly9fFhqNRnz77bcWaGHD8PfnFkKIiRMnigcffNAi7akLssdmAnUJtNlcOH78OL6+vrRv355x48aRkZFh6SY1Ounp6WRnZxv9/V1cXAgNDW32f3+AHTt24OnpSefOnfnHP/7BxYsXLd2kapHCZgI1Bdo0JXDmrUZoaCjLly9n8+bNfPjhh6SnpzNw4ECuXLli6aY1Kvq/8e329wdlGPrVV18RHx/Pu+++y6+//sqIESPQarWWblqV3NaBJiW1Y8SIEYbXPXv2JDQ0FH9/f1avXs3kyZMt2DJJY/HYY48ZXgcFBdGzZ086dOjAjh07GDp0qAVbVjWyx2YCdQm02RxxdXUlMDCQEydOWLopjYr+b3y7//0B2rdvj4eHR5P9DEhhM4EbA23q0QfaNCVw5q1OYWEhJ0+exMfHx9JNaVTatWuHt7e30d+/oKCAPXv23FZ/f4Bz585x8eLFJvsZkENRE4mOjmbixIn069ePkJAQFi9eTFFREZMmTbJ00xqMl19+mQceeAB/f38yMzOJiYnB2tqasWPHWrppZqewsNCoF5Kenk5qaiotW7akbdu2vPjii/zrX/+iU6dOtGvXjjlz5uDr62uIKH2rUtNzt2zZkvnz5zNq1Ci8vb05efIkr776Kh07diQiIsKCra4BSy/L3orUFGizOTJmzBjh4+Mj1Gq1aN26tRgzZow4ceKEpZvVIGzfvr3KoKf64Kk6nU7MmTNHeHl5CY1GI4YOHSrS0tIs22gzUNNzFxcXi2HDholWrVoJW1tb4e/vL6ZMmWKUw7epIeOxSSSSZoecY5NIJM0OKWwSiaTZIYVNIpE0O6SwSSSSZocUNolE0uyQwiaRSJodUtgkEkmzQwqbRCJpdkhhk0gkzQ4pbBKJpNkhhU0ikTQ7/j82gHfFNvw8sAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:42:56.370376Z", + "iopub.status.busy": "2024-02-29T18:42:56.370066Z", + "iopub.status.idle": "2024-02-29T18:43:43.371513Z", + "shell.execute_reply": "2024-02-29T18:43:43.370520Z" + }, + "papermill": { + "duration": 47.023831, + "end_time": "2024-02-29T18:43:43.374040", + "exception": false, + "start_time": "2024-02-29T18:42:56.350209", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:43:43.414397Z", + "iopub.status.busy": "2024-02-29T18:43:43.413556Z", + "iopub.status.idle": "2024-02-29T18:43:43.433770Z", + "shell.execute_reply": "2024-02-29T18:43:43.432945Z" + }, + "papermill": { + "duration": 0.042663, + "end_time": "2024-02-29T18:43:43.435796", + "exception": false, + "start_time": "2024-02-29T18:43:43.393133", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat5.952382e-080.6092630.0199360.5591480.1941910.9970710.2823070.0000190.8766730.0972040.7692360.1411960.0531810.7849621.435821
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 5.952382e-08 0.609263 0.019936 0.559148 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.194191 0.997071 0.282307 0.000019 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 0.876673 0.097204 0.769236 0.141196 0.053181 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.784962 1.435821 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:43:43.475408Z", + "iopub.status.busy": "2024-02-29T18:43:43.474824Z", + "iopub.status.idle": "2024-02-29T18:43:43.909232Z", + "shell.execute_reply": "2024-02-29T18:43:43.908268Z" + }, + "papermill": { + "duration": 0.456458, + "end_time": "2024-02-29T18:43:43.911370", + "exception": false, + "start_time": "2024-02-29T18:43:43.454912", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:43:43.951744Z", + "iopub.status.busy": "2024-02-29T18:43:43.950961Z", + "iopub.status.idle": "2024-02-29T18:44:32.670044Z", + "shell.execute_reply": "2024-02-29T18:44:32.669005Z" + }, + "papermill": { + "duration": 48.741699, + "end_time": "2024-02-29T18:44:32.672417", + "exception": false, + "start_time": "2024-02-29T18:43:43.930718", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:44:32.713356Z", + "iopub.status.busy": "2024-02-29T18:44:32.713056Z", + "iopub.status.idle": "2024-02-29T18:44:32.730203Z", + "shell.execute_reply": "2024-02-29T18:44:32.729517Z" + }, + "papermill": { + "duration": 0.038975, + "end_time": "2024-02-29T18:44:32.732107", + "exception": false, + "start_time": "2024-02-29T18:44:32.693132", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:44:32.769055Z", + "iopub.status.busy": "2024-02-29T18:44:32.768530Z", + "iopub.status.idle": "2024-02-29T18:44:32.773631Z", + "shell.execute_reply": "2024-02-29T18:44:32.772799Z" + }, + "papermill": { + "duration": 0.025757, + "end_time": "2024-02-29T18:44:32.775758", + "exception": false, + "start_time": "2024-02-29T18:44:32.750001", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.05795487974159697}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:44:32.813941Z", + "iopub.status.busy": "2024-02-29T18:44:32.813678Z", + "iopub.status.idle": "2024-02-29T18:44:33.119654Z", + "shell.execute_reply": "2024-02-29T18:44:33.118662Z" + }, + "papermill": { + "duration": 0.327687, + "end_time": "2024-02-29T18:44:33.121766", + "exception": false, + "start_time": "2024-02-29T18:44:32.794079", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDjklEQVR4nO2deXiTZdb/v0/2NE3SLd2gC3uVghQoCCoF5WURHRDHcUFpQcAZ4UVlnEuY9xVExmGYURZ1wBnfoYAiOs7guKAiPwVUVBCQHVkKtKULLbRN96z3748nedq0SZqkSZM253NdudI8ufPkJGm+Ofe5z30OxxhjIAiCCBFEwTaAIAiiNSRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRK3Yz09HTcc889AX0OjuPwwgsvdDjuhRdeAMdxAbWFCD9IlLqY7777Di+88AJqamqCbQrRDdi4cSO2bNkSbDO6FBKlLua7777DypUrSZQIjyBRIgiCCDIkSl3ICy+8gN/97ncAgD59+oDjOHAchytXriA/Px933nkn4uPjIZfLcfPNN2PTpk0uz/XFF19g2LBhUCgUuPnmm7Fz506v7TEYDHjmmWeg0+mgVqvxi1/8AlevXnU69ttvv0V2djYUCgX69euHv/3tb07HcRyHRYsWYfv27Rg0aBAUCgVGjBiBr7/+ut17wXEczp8/j0cffRRarRY6nQ7PP/88GGMoLi7G9OnTodFokJiYiFdeecXr1wcAn332GXJycqBWq6HRaJCdnY133nnHYcz777+PESNGQKlUIi4uDo8++ihKSkocxuTl5SEyMhIlJSWYMWMGIiMjodPp8Oyzz8JisTiMtVqt2LBhA4YMGQKFQgGdTocpU6bg8OHDwhhPPu/09HScPn0a+/fvF/5Xxo8f79P70K1gRJdx/Phx9vDDDzMAbN26deytt95ib731Fquvr2fZ2dksLy+PrVu3jr322mts0qRJDAB7/fXXHc6RlpbGBg4cyKKiotjSpUvZ2rVr2ZAhQ5hIJGJffPGFV/Y8+uijDAB75JFH2Ouvv85mzpzJhg4dygCwFStWCONOnDjBlEolS01NZatXr2arVq1iCQkJwtjWAGCZmZksLi6Ovfjii2zNmjUsLS2NKZVKdvLkSWHcihUrGAA2bNgw9vDDD7ONGzeyadOmMQBs7dq1bNCgQew3v/kN27hxI7vtttsYALZ//36vXl9+fj7jOI5lZmayl156if31r39l8+bNY4899pjDGAAsOzubrVu3ji1dupQplUqWnp7OqqurhXG5ublMoVCwwYMHs7lz57JNmzax+++/nwFgGzdudHjevLw8BoBNnTqVrV+/nr388sts+vTp7LXXXhPGePJ5f/DBB6x3794sIyND+F/x9jPujpAodTF/+ctfGAB2+fJlh+ONjY3txk6ePJn17dvX4VhaWhoDwP79738Lx/R6PUtKSmJZWVke23Hs2DEGgD355JMOxx955JF2ojRjxgymUChYYWGhcOzMmTNMLBY7FSUA7PDhw8KxwsJCplAo2H333Sccs4vSggULhGNms5n17t2bcRzH/vSnPwnHq6urmVKpZLm5uR6/vpqaGqZWq9no0aNZU1OTw31Wq5UxxpjRaGTx8fEsMzPTYcwnn3zCALDly5cLx3JzcxkA9uKLLzqcKysri40YMUK4/dVXXzEAbPHixe1ssj8vY55/3oMHD2Y5OTkevOKeA03fQgSlUin8rdfrcf36deTk5ODSpUvQ6/UOY5OTk3HfffcJtzUaDWbPno2ffvoJ5eXlHj3fp59+CgBYvHixw/Gnn37a4bbFYsHu3bsxY8YMpKamCsdvuukmTJ482em5x4wZgxEjRgi3U1NTMX36dOzevbvdVGfevHnC32KxGCNHjgRjDI8//rhwPCoqCoMGDcKlS5c8em0AsGfPHtTV1WHp0qVQKBQO99nTGA4fPoyKigo8+eSTDmOmTZuGjIwM7Nq1q915f/3rXzvcvuOOOxzs+ve//w2O47BixYp2j22dPuHN5x1ukCiFCAcOHMDEiROhUqkQFRUFnU6H3//+9wDQ7p+0f//+7fKDBg4cCAC4cuWKR89XWFgIkUiEfv36ORwfNGiQw+3Kyko0NTVhwIAB7c7RdqwdZ2MHDhyIxsZGVFZWOhxvLXQAoNVqoVAoEBcX1+54dXW16xfUhoKCAgBAZmamyzGFhYUAnL+OjIwM4X479vhQa6Kjox3sKigoQHJyMmJiYtza583nHW5Igm0Awf8j33XXXcjIyMDatWuRkpICmUyGTz/9FOvWrYPVag22iQFDLBZ7dAwAWJArN7uyy1vC+fP2BBKlLsZZBvTHH38Mg8GAjz76yMFz2Lt3r9NzXLx4EYwxh3OdP38eAL9i4wlpaWmwWq0oKChw8BTOnTvnME6n00GpVOLChQvtztF2rB1nY8+fP4+IiIh2nkagsHuAp06dQv/+/Z2OSUtLA8C/jjvvvNPhvnPnzgn3e/u8u3fvRlVVlUtvyZvPOxwz5mn61sWoVCoAcEietP8Ct/YE9Ho98vPznZ6jtLQUH3zwgXC7trYW27Ztw7Bhw5CYmOiRHVOnTgUAvPrqqw7H169f73BbLBZj8uTJ+M9//oOioiLh+NmzZ7F7926n5/7+++9x9OhR4XZxcTE+/PBDTJo0yW/eRkdMmjQJarUaq1evRnNzs8N99vd55MiRiI+PxxtvvAGDwSDc/9lnn+Hs2bOYNm2a1897//33gzGGlStXtrvP/rzefN4qlSrsEm3JU+pi7AHg//mf/8FDDz0EqVSKcePGQSaT4d5778UTTzyB+vp6vPnmm4iPj0dZWVm7cwwcOBCPP/44fvzxRyQkJGDz5s24du2aSxFzxrBhw/Dwww9j48aN0Ov1GDt2LL788ktcvHix3diVK1fi888/xx133IEnn3wSZrMZr732GgYPHowTJ060G5+ZmYnJkydj8eLFkMvl2Lhxo3CerkKj0WDdunWYN28esrOz8cgjjyA6OhrHjx9HY2Mjtm7dCqlUijVr1mDOnDnIycnBww8/jGvXrmHDhg1IT0/HM8884/XzTpgwAY899hheffVVXLhwAVOmTIHVasU333yDCRMmYNGiRZg0aZLHn/eIESOwadMm/OEPf0D//v0RHx/fzqvrcQRv4S98WbVqFevVqxcTiURCesBHH33Ehg4dyhQKBUtPT2dr1qxhmzdvbpc+kJaWxqZNm8Z2797Nhg4dyuRyOcvIyGDvv/++13Y0NTWxxYsXs9jYWKZSqdi9997LiouL26UEMMbY/v372YgRI5hMJmN9+/Zlb7zxhrCs3xoAbOHCheztt99mAwYMYHK5nGVlZbG9e/c6jLM/trKy0uF4bm4uU6lU7WzNyclhgwcP9vo1fvTRR2zs2LFMqVQyjUbDRo0axXbs2OEw5r333mNZWVlMLpezmJgYNmvWLHb16lWP7HL2HpjNZvaXv/yFZWRkMJlMxnQ6HZs6dSo7cuSIg12efN7l5eVs2rRpTK1WMwBhkR7AMUZ93wj/wXEcFi5ciNdffz3YphDdFIopEQQRUlBMqQfSUQKlUqmEVqvtImv8T2VlZbskzNbIZLIO84SI0IVEqQeSlJTk9v7c3NxuXQ4jOzu7XWJja3JycrBv376uM4jwKyRKPZA9e/a4vT85OTlgz90VIcrt27ejqanJ5f3R0dEBt4EIHBToJggipKBAN0EQIUW3nr5ZrVaUlpZCrVaHZTo+QYQ6jDHU1dUhOTkZIpFnPlC3FqXS0lKkpKQE2wyCIDqguLgYvXv39mhstxYltVoNgH/BGo0myNYQBNGW2tpapKSkCN9VT+jWomSfsmk0GhIlgghhvAmvUKCbIIiQgkSJIIiQgkSJIIiQolvHlDyBMQaz2ex2rxThHLFYDIlEQukWRJfSo0XJaDSirKwMjY2NwTal2xIREYGkpCTIZLJgm0KECT1WlKxWKy5fvgyxWIzk5GTIZDL6xfcCxhiMRiMqKytx+fJlDBgwwOPkN4LoDD1WlIxGI6xWK1JSUhARERFsc7olSqUSUqkUhYWFMBqN7fqnEUQg6PE/ffTr3jno/SO6mh7rKRGEPzl5VY9jV2uQkahGdjoVkAskJEoE0QF7zlzDqRK+a22ligL+gYZ88zAnPT29Xa83wpHCGw3C3/XN5iBaEh6QKBGEG6xWhgZDS45bnYFEKdCQKPUAjEZjsE3osTSbLbC2Ks7aYDB3ScnfcCYsRclotrq8mC1Wj8eaPBjrC+PHj8eiRYuwaNEiaLVaxMXF4fnnnxe+DOnp6Vi1ahVmz54NjUaDBQsWAAC+/fZb3HHHHVAqlUhJScHixYvR0NAy9aioqMC9994LpVKJPn36YPv27T7ZF07U2zwjuZT/qlisDM0m3z5XwjPCMtD9173tW1Pb6ROnwoysXsLtv39dAJPF+S9j72glHhjZUmRu84HLaDI6bmd55r8G+mTj1q1b8fjjj+PQoUM4fPgwFixYgNTUVMyfPx8A8PLLL2P58uVYsWIFAKCgoABTpkzBH/7wB2zevBmVlZWCsNnbeefl5aG0tBR79+6FVCrF4sWLUVFR4ZN94UKjbeqmVkjBmAlGsxXNJguUMnGQLeu5hKUodQdSUlKwbt06cByHQYMG4eTJk1i3bp0gSnfeeSd++9vfCuPnzZuHWbNm4emnnwYADBgwAK+++ipycnKwadMmFBUV4bPPPsOhQ4eQnZ0NAPjHP/6Bm266qctfW3eiwch7SiqZGCIOMJmtsND0LaCEpSgtnNDf5X2iNjtRFozr53Js210rc2/r0xmzHLj11lsdtsWMGTMGr7zyirCxeOTIkQ7jjx8/jhMnTjhMyRhjwnab8+fPQyKRYMSIEcL9GRkZiIqK8pvNPRGThUEs4hAhk2DmcM/KuRKdIyxFSSbxPJQWqLGdRaVSOdyur6/HE088gcWLF7cbm5qaivPnz3eVaT2KYSlRuKW3FuQcdR1hKUrdgYMHDzrc/uGHHzBgwACIxc5jGcOHD8eZM2fQv79zLzAjIwNmsxlHjhwRpm/nzp1DTU2NX+3uiXAc184rJgJHWK6+dQeKioqwZMkSnDt3Djt27MBrr72Gp556yuX45557Dt999x0WLVqEY8eO4cKFC/jwww+xaNEiAMCgQYMwZcoUPPHEEzh48CCOHDmCefPmQalUdtVL6vYcKazC2z8U4qei6mCb0qMhUQpRZs+ejaamJowaNQoLFy7EU089JSz9O2Po0KHYv38/zp8/jzvuuANZWVlYvny5Q4vu/Px8JCcnIycnBzNnzsSCBQsQHx/fFS+n2/LNhUp8eKwEJTVNaDRaUFlngL7JFGyzejQ0fQtRpFIp1q9fj02bNrW778qVK04fk52djS+++MLlORMTE/HJJ584HHvsscc6ZWdPp6S6CWX6ZgxO1kIm5n/DXaWIEP6BPCWCcIPBlgArl4ggldhFiZInAwmJEkG4wWDmUzDkUlErT4lEKZAEVZQsFguef/559OnTB0qlEv369cOqVavCfm/Rvn37aOd+iGAw2T0lsZDyYfBx+xDhGUGNKa1ZswabNm3C1q1bMXjwYBw+fBhz5syBVqt1mm9DEF2J2WKF2cr/QMolIkjJU+oSgipK3333HaZPn45p06YB4Dea7tixA4cOHQqmWQQBoMUj4jhelOQSEVRyMRQS2vcWSII6fRs7diy+/PJLIdv4+PHj+PbbbzF16lSn4w0GA2prax0uBBEojGYrxCIOMokIHMchOUqJBeP64f4RtN0kkATVU1q6dClqa2uRkZEBsVgMi8WCl156CbNmzXI6fvXq1Vi5cmUXW0mEK9EqGf77zv6wWMM7xtnVBNVT+uc//4nt27fjnXfewdGjR7F161a8/PLL2Lp1q9Pxy5Ytg16vFy7FxcVdbDERbnAcB4mYFqm7kqB6Sr/73e+wdOlSPPTQQwCAIUOGoLCwEKtXr0Zubm678XK5HHK5vKvNJAgAfOB7508lMFms+NXIFCHwTfiXoIpSY2Nju75iYrEYViutbhDBp7iqET8V1yBJq0B2egzEIg4l1U0A+HgTiVJgCKoo3XvvvXjppZeQmpqKwYMH46effsLatWsxd+7cYJpFEACAmkYTCirqhdscxwe9nZVCJvxHUEXptddew/PPP48nn3wSFRUVSE5OxhNPPIHly5cH0yyCAAAYbcIjE7fULZGKORjNLfcR/ieooqRWq7F+/fquy15mDLAEYYe3WNq+TKULtm3bhmeeeQalpaUO8bMZM2ZArVbjrbfeCpSVRBvs3pCkVYiB/9sCM23KDRjhVSXAYgK+eaXrn/eO3wISzzqrPvDAA1i8eDE++ugjPPDAAwD4LiS7du1yWwGA8D92UZK2qigqtXlNJEqBgyJ1IYZSqcQjjzwidCABgLfffhupqakYP3588AwLQ+zCI201fbOnB5hpMSZghJenJJbyXkswntcL5s+fj+zsbJSUlKBXr17YsmUL8vLyHBoJEIHHHjdqvcqmkIqovVKACS9R4jiPp1HBJCsrC7fccgu2bduGSZMm4fTp09i1a1ewzQo7WjylFlG6L4u2mASa8BKlbsS8efOwfv16lJSUYOLEiUhJSen4QYRfuXtIIiYPTgi2GWEHxZRClEceeQRXr17Fm2++SXlbQcK+xYS2mXQt9G6HKFqtFvfffz8iIyMxY8aMYJtD2DhaVI33DxfjVIk+2Kb0WEiUQpiSkhLMmjWL9vsFib0/V+DzU+WoaTQKx/RNJlytbqKOJgGEYkohSHV1Nfbt24d9+/Zh48aNwTYnbCmorEddsxnDU6OEY1KRPSWA8pQCBYlSCJKVlYXq6mqsWbMGgwYNCrY5YYuzlACxyJ48SXlKgYJEKQRx1deN6DoYYzCZeW9I0mbvG0C93wIJxZQIwgkWK4OVtc9Tsq/EUTXKwNHjRSnc2zV1lnB9/1rHjGStRck+faNtJgGjx4qSVMpv7WhsbAyyJd0b+/tnfz/DBaNQIYCDSNR6+iaCRMTRlp8A0mNjSmKxGFFRUaioqAAARERE0D+SFzDG0NjYiIqKCkRFRUEsDq/9XiZbe6W2iZODEtUYlKgOhklhQ48VJQBITEwEAEGYCO+JiooS3sdwIkYlw8IJ1MkkGPRoUeI4DklJSYiPj4fJRMlu3iKVSsPOQ7LDl74lzzoY9GhRsiMWi8P2y0X4F32jCfvOV0AqFuHuIUnBNqdHEhaiRBDeUlLThFMleujUcgxPjRaOm6xWXKpsoJpKAaTHrr4RRGeobjDiTGktiqscV2+FbSaU0R0wSJQIwgkmJ1tMgJbsbrOVhW0OV6AhUSIIJ9i3kUhEjsFu+943xiirO1CQKBGEE5x1MgEcPSeqFBAYSJQIwgktjSgdvyIirqWFH4lSYCBRIggnOGsaAPD5S1KxCGIRBwtVCggIlBJAEE4QuuOK2ydQ/iann8N+OMK/kCgRhBP+6+YE5AzUORUlEqTAQqJEEE6QikXtpm5E10CiRBBe8s2FSlQ3mnBrnxjEaxTBNqfHQT8FBOGE7y5ex95zFdA3tt/IfbW6CQUV9agzmINgWc+HRIkgnHCmrBbHimrQZLK0u8+eUEnJk4GBRIkgnGASUgLaB7UlQvMA2v8WCEiUCMIJZhcZ3QAgFlHzgEBCokQQbbBamZCtba8K0BqpqGVTLuF/SJSIsKXZZMGuE2U4UljtcNzYalrmbPrW0pCSRCkQkCgRYcux4hqcv1aHr89XoqHVSpo9ViTiOEGAWtNSvoRiSoEg6KJUUlKCRx99FLGxsVAqlRgyZAgOHz4cbLOIMKCiziD8XV7bLPwt7HuTOG+lNG6ADv99Z3+M6RsbeCPDkKAmT1ZXV+O2227DhAkT8Nlnn0Gn0+HChQuIjo7u+MEE0Un6xqlQUFEPALhRb0Q/HX9cq5Ri/ri+Ljfctm27JHCjAKi6BCRkAhqq3+0rQRWlNWvWICUlBfn5+cKxPn36BNEiIpzI7KVFbbMJBy9Voa65JUlSJOIQKffyq1FTBJx8n6/+VnYcyJ4HKKP8a3CYENTp20cffYSRI0figQceQHx8PLKysvDmm2+6HG8wGFBbW+twIYjOoFHwnX/rmj3Pzi680YDPT5XhaFGrAHnhd7wgAYDFBBQf8qeZYUVQRenSpUvYtGkTBgwYgN27d+M3v/kNFi9ejK1btzodv3r1ami1WuGSkpLSxRYTPQWTxYqaRiMSNApMHZKIsf1a4kPXapux71wFTpXonT62ptGEs2V1KKlu4g8064Gqy/zfAyfx15VnAQqE+0RQRclqtWL48OH44x//iKysLCxYsADz58/HG2+84XT8smXLoNfrhUtxcXEXW0z0FK7VNiP/wBXsOlGKjESNw8ba6/UG/FRUg4u2eFNb2q2+2QVJ2wtIGgZI5ICxEagtCeRL6LEEVZSSkpJw8803Oxy76aabUFRU5HS8XC6HRqNxuBCELzQZ+T1tEbL2sSOhaYCTHCUAkAhtlmzTtZpC/jo6HRCJ+WsA0F/1m73hRFBF6bbbbsO5c+ccjp0/fx5paWlBsogIFxptoqSQiVFQWY+TV/UwmPljrtor2RG33pDLGB/kBoCoVP5a25u/JlHyiaCuvj3zzDMYO3Ys/vjHP+JXv/oVDh06hL///e/4+9//HkyziDDAvvs/QirGnjPX0GS0IClKAXmkGCaz86YBduxZ3iYr4+NJhnqAEwEamxjZRam2hBctJ7lOhGuC6illZ2fjgw8+wI4dO5CZmYlVq1Zh/fr1mDVrVjDNIsIAg014FFIxImwtuO1TOqOnnpLFCtRX8AdVcYDY9huv0vFCZGoCjA2Begk9lqBXnrznnntwzz33BNsMIsww2r0hiQhKKS9K9imd2U3ZEv64LaZkZUD9Nf5gZELLALEUUEYDjVVAQwUgjwzES+ixBF2UCCIYOIiS3VMyOcaUXGVux0XK8eucfnwg/MxR/mBrUQJ4b6mxCmi4DsT0DcAr6LmQKBFhSVpsBBRSEeIiZahq4PfANRr5BMrxg+Ixum+s4EG1RSziBCFDQyV/HalzHKTSAZXnWqZ3hMeQKBFhSWYvLTJ7aQEAxVV8EqQ9pqSUiVtExx0WEx/oBngRao39duMNv9gbTpAoEWFP2+lbR5gsVnx9vhKixkrkWK0QSRWANKLNSW2bypuq25+AcAuJEhGW1DWbIBWLIJeIkBYTgbuHJCE6gt8Hd/DSDVgYw5BeWqhte+PacuKqHjGNV2GNZBBFxLRf9rdvxjU1AWYDn+VNeASJEhGWvPVDIQwmK3LHpiNGJUO0Sibcd6y4Bo1GCwbEq6F20tbN3s1EadLze3AjnNRVksgBWQS/3aSpGlAnBuiV9DyCXuSNILoaxhhMZn7ZX+akMUDrlTlncBwHiYiDwlwDC2OAMsb5Eymi+Oumms6aHFb4JEqXLl3ytx0E0WWYrQxWW5kRmVgEq5XhYkUdTl7Vw2SxCg0B5C5ECeDTBVo8JReiRHEln/BJlPr3748JEybg7bffRnNzc8cPIIgQwu4JcVxLguTHx8vw/85eg76ppdibq20mAGyekh5WqxtPyR5Xaq7xh9lhg0+idPToUQwdOhRLlixBYmIinnjiCRw6REWtiO5B6+kZx3EQiTgobDlJNbY23TKJCCInTQPsSGGCxGrgPS5XFSbJU/IJn0Rp2LBh2LBhA0pLS7F582aUlZXh9ttvR2ZmJtauXYvKykp/20kQfsO+t621J2Tf/6ZvMgJwP3UDgAjG72mziOWuV9YUfB6UkMtEeESnAt0SiQQzZ87E+++/jzVr1uDixYt49tlnkZKSgtmzZ6OsrMxfdhKE37B7Sq033CrbeEodidK0gZEYkRYNTZSbjiZyW70vQ11LqVyiQzolSocPH8aTTz6JpKQkrF27Fs8++ywKCgqwZ88elJaWYvr06f6ykyD8RoRMjCG9tBgQ37JR1p5AGRUhxewxaZg6xH03EhVrgEwsgsjuDTlDruYDV1YLYGr0i+3hgE95SmvXrkV+fj7OnTuHu+++G9u2bcPdd98Nka0iX58+fbBlyxakp6f701aC8AuxkXJMvNlxA619+mY0M8RGepDo2GxrWiFXux4jEgMyFV9vqbmW/5voEJ9EadOmTZg7dy7y8vKQlOT8FyU+Ph7/+Mc/OmUcQXQV9ulbk8mzriZF5ddgud4AlU6GeHcD5WpelAx1AKgXnCf4JEp79uxBamqq4BnZYYyhuLgYqampkMlkyM3N9YuRBOFPTBYrLFYGqVgkFGwbkKBGnFqO6/UGfFdwHX3iVEjSKl2eo+rGdTTVNiPKrOhYlFAGGKgdmKf4FFPq168frl+/3u54VVUVNZMkQp6TJXps2leAL06XC8d0ajkGJqhR3cA3pyzXu8+/k1v4TicmSQcF3OS2mBOJksf4JErMxUpCfX09FAonm4UIIoQwC91K2v/7Nxj46ZvKXYdcxiAz8ykBzWI3MSWgJeZkqPPe0DDFq+nbkiVLAPB7f5YvX46IiJZyDRaLBQcPHsSwYcP8aiBB+BuzUFmyJTmSMYYrNxpRUsPXVtIqnVcHAAAYGyCGBQAHoyjC9TigRZSayVPyFK9E6aeffgLAf4AnT56ETNays1omk+GWW27Bs88+618LCcLPmGx726StYqIcx+Hj46XCbbeiZKiDiONgFCvB0EGnEkWrXCXCI7wSpb179wIA5syZgw0bNlAzSKJb4sxTAoAkrQJXq5uglImFbSdOMdRCxAFGcSQ4SwdJka2nb1YrIKLCHB3h0zuUn59PgkR0W0wuupXcPiAOvaKVmHiT2/U0oLkWIo6DQRIpVBRwicyWQMmsgInaLXmCx57SzJkzsWXLFmg0GsycOdPt2J07d3baMIIIFGarzVNq47UkaZX41ciUjk9g0CNeI8f4gf0gGahzP1Yk4kvlGhv4i7tkSwKAF6Kk1WrB2Up+arVuUusJIsRJjlJCzHGIjpB1PNgZzbWQiESI1MQA7qZ5dmQqXpAM9QBpUod4LEr5+flO/yaI7sbw1OjOncAetFZ4GMKQq/lWS8b6zj1vmOBTTKmpqQmNjS0bDAsLC7F+/Xp88cUXfjOMIEIWQy2aTBYcKjXjaJEHtZLse96ohbdH+CRK06dPx7Zt2wAANTU1GDVqFF555RVMnz4dmzZt8quBBOFvjGYrXzHSF6wWwNiAZpMFB8vMOFvmQf6RzJb1TZ6SR/hcefKOO+4AAPzrX/9CYmIiCgsLsW3bNrz66qt+NZAg/M3bPxRiw5cXUGpLlPQKQy3AGERiCcwiBSyeiBuJklf4JEqNjY1Qq/mI3RdffIGZM2dCJBLh1ltvRWFhoV8NJAh/I6y+iTtIfHSGPTNboQE4TkgvcIvcJkoGEiVP8LlxwH/+8x8UFxdj9+7dmDRpEgCgoqKC8peIkEfIU/IlkdEW5OZsQW6LTeDcQjElr/BJlJYvX45nn30W6enpGD16NMaMGQOA95qysrL8aiBB+JuWDbk+eEq23f72ipMeeUqtp29UFrdDfKqn9Mtf/hK33347ysrKcMsttwjH77rrLtx3331+M44g/I2lVc83qZsWSi6xTd84pd1T8kKUrBbA3AxIXddpIjrRtjsxMRGJiY6tiEeNGtVpgwgikJhbTbckblooucTmKYltbZUsVgarlbltxwSxBJAqAFMzH1ciUXKLT6LU0NCAP/3pT/jyyy9RUVEBa5t5NXXQJUIV+9SN4yBUnfQKW7skWYQWs27tBalIBM6T08gieVEy1gPoYGtKmOOTKM2bNw/79+/HY489hqSkJGH7CUGEOhwHDExQg4H59n9rC3SLlFrEq7woaCiLBBquU1qAB/gkSp999hl27dqF2267zd/2EERAiZBJMG2ojwX8Tc2A2cD/LfdylZlW4DzGJ1GKjo5GTIyL/ukE0VOx73mTKgCJDEcKq2AwWTEsNQoRsg6+SpSr5DE+pQSsWrUKy5cvd9j/1ln+9Kc/geM4PP300347J0G0hTHmssZ8h9iL/9u8pCOF1Th4uQoNBkvHj6Wsbo/xyVN65ZVXUFBQgISEBKSnp0MqdSwdevToUa/O9+OPP+Jvf/sbhg4d6os5BOExV2404j8/lSA5SoEHs1O9e7AtyG0XJbFIBMDisKLnEhIlj/FJlGbMmOE3A+rr6zFr1iy8+eab+MMf/uB2rMFggMFgEG7X1lIxdsI77KVwfQtyt9pigpaUArNHCZQUU/IUn0RpxYoVfjNg4cKFmDZtGiZOnNihKK1evRorV67023MT4YerUrgeYY8p2Twle0Y4bcr1Lz5XMa+pqcH//d//YdmyZaiqqgLAT9tKSko8Pse7776Lo0ePYvXq1R6NX7ZsGfR6vXApLi72yXYifLFPtcS+7HtrduEpeTJ9swe6zUb+QrjEJ0/pxIkTmDhxIrRaLa5cuYL58+cjJiYGO3fuRFFRkVBryR3FxcV46qmnsGfPHo8bWMrlcsjlcl9MJggArTfjdmL65hBTQsfNAwBALOMzuy1mvoGAxMdSvGGAT57SkiVLkJeXhwsXLjgIyt13342vv/7ao3McOXIEFRUVGD58OCQSCSQSCfbv349XX30VEokEFosHKxoE4SUt7ZW8/NdnrNX0jS/bY58CehRT4riWKRylBbjFJ0/JvlrWll69eqG8vNzJI9pz11134eTJkw7H5syZg4yMDDz33HMQiz0oyE4QXmL3aryuEGCs5zfUcpwgSncM0OHWvrHuG1e2RqYCmmoo2N0BPomSXC53uvJ1/vx56HSe7etRq9XIzMx0OKZSqRAbG9vuOEH4C61Sij5xKsSpvAwD2L0kWSQg4n8wY1ReTsFoBc4jfJq+/eIXv8CLL74Ik8kEgF9eLSoqwnPPPYf777/frwYShD/J7KXFjKxeGNLbyzZhbYLcPiGz9VeiFTi3+Jw8+ctf/hI6nQ5NTU3IyclBeXk5xowZg5deeslnY/bt2+fzYwkioLQJcgNAcVUjymubkaBWIDU2ouNzCJ4SiZI7fBIlrVaLPXv24MCBAzh+/Djq6+sxfPhwTJw40d/2EURoYPeUWnW4vXKjAYevVGN4WrSXokTTN3d4LUpWqxVbtmzBzp07ceXKFXAchz59+iAxMRGM+VgOgiC6iA+PleBqdRPuuikeGYleTMUMti0mipZpn73tt0d1ugFKoPQQr2JKjDH84he/wLx581BSUoIhQ4Zg8ODBKCwsRF5eHpXCJUIeo9kKo9kKkbc/nm2yuYGWFTyP6nQDLQmU5Cm5xStPacuWLfj666/x5ZdfYsKECQ73ffXVV5gxYwa2bduG2bNn+9VIgvAX9pQAr6tO2jfjtgp028/h0TYToNX0rRGwWgFfssrDAK/elR07duD3v/99O0ECgDvvvBNLly7F9u3b/WYcQfgbe/KkV+2VLCZeSAAHT0nqTUY3AEhVfJ4TswIm/5X96Wl4JUonTpzAlClTXN4/depUHD9+vNNGEUSg8Cl50j51E0sdiv6LhSoBHsaURKKWx9MUziVeiVJVVRUSEhJc3p+QkIDq6upOG0UQgcKnnm/NNfy1QovWXQLs5/DYUwIoLcADvIopWSwWSCSuHyIWi2E2mzttFEEECpO9Zbc30zchcdIx4bJ3tBK/HNEbSpkXW6JkagCV5Cm5wStRYowhLy/P5U791gXYCCIU6RWlhMFkhVzihSg5SZwE+CYEETFeZtVQrlKHePWO5ubmdjiGVt6IUGb6sF7eP8jJypvP0PStQ7wSpfz8/EDZQRChi4vpW5PRgnPX6sABuCUlyrNzUQJlh/jctpsgwgYX07dmkwV7f66ATCLyXJQogbJDSJSIsKG22YS3vi+EQirG47f38exBVqvLCgEyW1zKZLF6vsWKYkodQimlRNhgsm0xMZo9zCsC+GkWswKcqKX0iA27KDEGGD3NVRKqT9Z5bkOYQaJEhA327SBedTIxtKoO0CaNQCLihD10Hgud3VOymKiBgAtIlIiwwWTP5vZm35ublTeO41pN4TxMoLQ3EAAo2O0CEiUibLBvBxF70zTAxcqbHbsoeewpcVyrCpQUV3IGiRIRNvjUXsnFypsdr0UJoGB3B9DqGxE2WITNuN54Su4TJycM0oExQKf2ohEBJVC6hUSJCBvkEhGSoxSIjfSiC0lTDX+tiHJ6d+9oD8rgtoUSKN1CokSEDelxKqTHqTx/AGMtFQKUUf4zhBIo3UKiRBCuMNTaGlCKALnzQHeZvgkVtQbo1HIkRymdjmkHxZTcQoFugnCFMHXTuixde/5aPb76uQIFlV5MxWj65hbylIiw4YdLN3Dyqh5De2sxum9sxw9oshUsVEa7HCITd2L1zUCi5AzylIiwoclkQb3B7HmlSA/iSZ1KCTDZGggQDpAoEWGDUArX0zwl+/TNjadkLxZn8EaUhAYCjBoIOIFEiQgb7BndHucp2advLtIBAAilcJtMFs8NEYkAqS2VgILd7SBRIsIGs7cbcoXpm2tPSSm1iZLRC1ECKIHSDSRKRNhg9qZpgKkJMDXzf7uJKQmi5I2nBNAKnBto9Y0IG0zetFeyT91kKr7fmwsiFRLce0sSFFIvOpoAlEDpBhIlImzQKqUwmK2Cd+OWxir+OsJ96oBULEL/eLXbMU6hBEqXkCgRYcPkwYmeD268zl93IEo+Q9M3l5AoEYQzGm/w1x6I0pXrDdA3mZAeq4I2wvVUzwFKoHQJBboJwhn26ZuqY1H68UoVvvq5AqX6Js/PL7dN+ahWdzvIUyLChm3fXwFjwH3De0GjcOPRWC0tgW4PPCW1QgqgCXXNXrSst4uSsZ5PovSkE0qYQKJEhAWMMVQ1GMEYhGL/LmnW88IklrisONkajYL/GtU1mzw3SKbmhchq4YVJ7kOwvIcS1Onb6tWrkZ2dDbVajfj4eMyYMQPnzp0LpklED8ViZWC2LW8dJk82tApye+DBaJS819XaU9I3mrD35wp8V3AdBrOTHCaRiNotuSCoorR//34sXLgQP/zwA/bs2QOTyYRJkyahoYGWSQn/0rrbiLSj5EkvgtwAoLZ5SrU2T8lkseJfR6/iWHENDl6qwsfHy2B1tgmY4kpOCer07fPPP3e4vWXLFsTHx+PIkSMYN25ckKwieiL2ZpESEQdRRxtyvRSlaBVfXre6wQSzxYofr1ShtskEqZgDY0BxVSPOlNUis1ebQnEKDVBb2tIxhQAQYjElvZ4v0h4TE+P0foPBAIPBINyuraUPk/AMk02UpBIPJgcNFfy1SufRudVyCZQyMZqMFly63oAjV/gg+eTBiag3mKFvMqGfLrL9AwVPif6PWxMyomS1WvH000/jtttuQ2ZmptMxq1evxsqVK7vYMqInIIhSRxUCrBagweYpeShKHMdh8uBEqGRi/FxeB7OVITUmAv3jI8G5i0nZg+gkSg6EjCgtXLgQp06dwrfffutyzLJly7BkyRLhdm1tLVJSUrrCPKKbI+I4xEbKECnv4F++sQqwmgGJzG11gLb0sTUk0KnlUMnFSI9VuRckoJUoUUypNSEhSosWLcInn3yCr7/+Gr1793Y5Ti6XQy73or8WQdhI0Cgwe0x6xwPrr/HXqnifcoc4jsOINMfwA2MMBZX1uFhRjwkZ8ZBLbHvv7NM3iik5ENTVN8YYFi1ahA8++ABfffUV+vTpE0xzCKJFlCLj/XraAxdv4GxZHS5fb7WybG9waaynsritCKooLVy4EG+//TbeeecdqNVqlJeXo7y8HE1NXqTrE4Q/aajkr/0oShzHoX88H+i+VNlKlKQqvn0TY4CRpnB2gipKmzZtgl6vx/jx45GUlCRc3nvvvWCaRfRATpXose37K/ju4nXXgxhr5Skl+PX502L58rdXqxvB7FmcIlFLXSWKKwkENaYkfDgEEWAaDGbcqDciWeumQqSxHjA28rEkD1fePCVRo4BUzKHBYMGNBiPiIm2xUYWWjyk16wGt63hqOEFVAoiwwJ7R7TZPqa6cv46IdVtt0hckYpHQQbe4qlUHE3tTAnvnFIJEiQgPhDwld9nctSX8taZXQGzoZROlcn1zy0F7/W97kwKCRIkID4yeZHTr7aKUHBAbEjQKAEBj684n5Cm1IyTylAgi0NgbUbrM6LZagboy/u8AeUq9o5X4dU4/oVccAPKUnECiRIQFLdtMXEzfGioBi4nP5FbFBcQGiVgESdueBXZPyVDHb3ERedkVpQdC0zciLFBIRVArJK5bIdnjSerkrq0CKVPxxeQY41fgCPKUiPBgSmaS+wG1gY0n2SmuasShy1WIipDirpsSeAFURPGF5ZprgAjnFTLCCfKUCIIxoKaI/zsqsBu8LVaGoqpGXK1utWuBgt0OkCgRRFM1n8AoEgPawIpSnJpPmqxuNApxLiHYbW9WEOaQKBE9HsYY3vqhEO/9WIQmo5OM7ppC/lqT7PekybaoZGIoZWIwBlQ3GPmD9imbva1TmEOiRPR4TBaG63UGlNY0Q+wsebLaJkpRaQG3heM4xNrK51bW26qo2svu2svwhjkkSkSPx95NRMRx7VMCGGvxlKIDL0pAyxTuer3dU7KJUnMNYPGid1wPhUSJ6PEYzXzsRiYRta8GWX+N34QrlgQsabItOttm3Bt2T0kWCUjkvEA20RSORIno8RhaiVI7blzkr6P7dFniYmykDEqZuKUCJcfRFK4VlKdE9HjsnpLcmShdv8Bfxw3oMnsSNQo8Ma6vo9cWEcu3WyJRIlEiej4uPaXmWr5cCccBMf26zB6nDQXsW1sa3BShCxNo+kb0eDiO72LbrpOJfeqmTmqpANnFCJ1zhekbiRJ5SkSPZ2CCGgMT1O3vqDzHX3fh1M3OufI6fHOhEikxEZg8OLGlJnjDDX4FThy+X03ylIjwxFDXkgoQf1OXP71YxKGu2YzKOtsKnFwDSJUAs7Y0LwhTSJSI8KTiZ34JXtvLq6aT/kJny1WqajDCYmX8HNPerKC+vMvtCSVIlIgez96fK/Dej0WOPdcqTvPX8YODYpNGIYFMIoLFylBl326itolS3bWg2BQqkCgRPZ5K2xYTs30DbH0lUFvG91yLzwiKTRzHCUmUwhQuMtFmH3lKBNGjaTLx20yEAm+lR/nruP58kbUgoRO2m9hESW0XpQq+CmaYQqJE9HjshfqVMjFgNgDlJ/k7kocH0SoIvd8EUVJG8yJptbTUCw9DSJSIHo3VyoQNuUqpGCg/xXshEbFAdHpQbUvQyJGkVSBezXc5Ace1FJmrKQ6eYUEmfJMhiLCg2WyBvRGzUgyg+Af+Rq8RXVuL2wnxGgUeGpXqeFCbwq8M6q8Gx6gQgESJCB6MAeZmwNTE5+cwK3+MWflia2IZn7vTicJr9Qa+FEiETAxRxUl+a4lMBSQN9der8C/2ypf64rBNogy/V0wED1MzcOMCcKOALxnSVMMLkDs4jo+1aHoBukFATF+vdvObLQwapRRqiRW4coA/mDom4BUmvcFksaLRYIE2QspndstUgLEB0BfxrzfMIFEiAo+xkZ82lRx1vqoklgIiCS9AnAgAB1jNfFCaWfkysY1VfIBargZSbwWShnnkRSRHKfH47X3ALn4FFNcBCi2QPMzfr9BnLl9vwEfHSqFTy/HI6FT+PYjtD5Qd58WbRInolljMfG6LsYH/UkfE8t5FkGMmYAyoOANc2MNP0QDetvgMQNsbiIgDpBGuxYUx/jXVXwOqLvPnMtTx5ys5Cgyc5FmwWl8C7uqP/N8D/iukvKQYlQxWxlBZZ4DJYuU7+AqidBHoPzH4n2MXQ6LUnam7Blw9xAdGrW3KqCqjgOQs/iKRd71tFjNw/vOW5XdVHNB3AhDbz/MvGcfxu/flkfzj+o4Hyo8DV77l6w4d2wEkDAb63el6l7+hDjjzH97jis8IyuZbd2hs1QvqDWaU1jQhLVbFC61Yyk9v9VcD3vYp1CBR6o5YzMCVb4DigxCWlmQRvHdkNfM7zZtqgIK9QNEP/K9twuCu+8Vt1gOndrbUKkq/nY/jdLayo1jCr5rFDwYuf80nQV47zcep0scBvYY7PkfDDZzY/Q+Im6qQ0qsXNAOndu75AwDHcUiNjcCZ0loU3mjkRUkiA3QZvKCXnyBRIkKc2jLg509aioHpBgEpo/n2QHbRMRuByp+Bou/5WMzZj4Frp4BBU/mYSiCpKQZO7+TjSFIlcPN0IKaPf59DquCnbolDgAu7+ffk4v/jX69uEP8aG66DVZyBQV+JBk6F+MG/4h8XgqTHqmyi1ABAxx9MGsqLUsVZ3hOUKoNqY1dCotRdsFqAwgNA4ff8VEQWAQycwn8J2yKR8f/UCYN5b+rKAT4m8+P/8V5T4tDAeE2lx4ALX/C2RsYDmfe3NFoMBJokYHguUHaMn9IZ6vlYkw2zxYobsl4oiM3BbTG6wNnRSdJiI8BxfHcTfaOJX4XTpgCROn6f3tUfgT7jgm1ml0Gi1B2or+C9nfoK/nZ8BjBgMi9M7hCJgbSx/FTg508AfQnw86d8cbNBU/mVLH9gtfBTRXswOT4DyLinawLKHMfHzRKH8qtV+iI+qC6LRIUoGWetgDZCxgeQQxSFVIyU6AgUVTXibHktbu0ba5v23sFPg6/+yL9Gf31eIQ6JUihjMQNF3/FxIauFd+EHTva+KFlEDDDsUT4ofvkb/st76E1gwKTOx5oaq4CzH/FTKADocweQdlvXrxiJxIBuIH+xUVRwA+BuIEkbmtO21gxPi0Z6nAoZia2EJ24g7w3WlgHnd/OeZxisxIXEz8df//pXpKenQ6FQYPTo0Th06FCwTQoujPEraof/wU+9rBb+y5Y9z/cqiSIRn98zcg6/G91s4L2vU//mA+PeYjHxth3+B/+lkciBzJl8UDtEvjjF1Y0A+FylUKdPnAoj0qKhal1HnOOAQdN4wb1+Abi8v2VhowcTdE/pvffew5IlS/DGG29g9OjRWL9+PSZPnoxz584hPj4+2OZ1LaZmPhen9KeWqZpMxXs0ukH++bKr4vg4TNH3fIzq+gU+HyZuAJAwhE/Wc5eU2FjFr3iV/sTnEAF8Z9mMaYEPonuBvtGE0ho+N6qPLnjlSXzBYNuvp5CK+bjSgEnAuc/4eKKhng98dzR178ZwjAVXekePHo3s7Gy8/vrrAACr1YqUlBT893//N5YuXer2sbW1tdBqtdDr9dBoNF1hrn+wmABTI79C1XiD72BRU8R7HPZtF2IpkDIK6D0qcKtG9RX80rq99xnAZ1ZHxvPiJY3gkzGtJj7FoKGSv7aj0PC5R/E3hYx3ZKe22YRvzl+HwWzBzOG9g22Ox1yvN2DXiTIopCJMyIiHLlLOt2S6ephfYWTMljJwE/8DEhnPx5pCKCG0Nb58R4MqSkajEREREfjXv/6FGTNmCMdzc3NRU1ODDz/80GG8wWCAwWAQbtfW1iIlJaXjF1x1mf9A2yK8dObdMYe3rO2xDs5lMbdPdGyNKo7fQpEwuOt+Desr+aTEip/5ZEN3cCIgKhVIuoX33rqoq6wdq5Xh7YOFYAywMuZwzcCQGhOBKZlJwnizxQpJCAe523Ktthn/PnoVBlNLrzqlVAyJmIOqqRTjRccRy2oAADWNRhRW8VNUmUyGm3rr+M+DE7VcAsnQB/kfJjf4IkpBnb5dv34dFosFCQkJDscTEhLw888/txu/evVqrFy50vsnMhtCr8kfJ+ID1xEx/NYLdRKfyRvIJXRXROr4VIF+dwFN1XyBsaaalt37YgkgU/OCqU4Kar4PxwE36o0u7282OW7w7U6CBAAJGgUezk7FgYLruFTZAKPZKnT4vYE43Lh5FmJlN4Dr52G4egn1pisQMQsYM7ZMp7uKjjZT+0jQY0resGzZMixZskS4bfeUOiQqBbjloTZTDNvfbo+1uq/tMV/PJRLZ9nvJQm7Kw/e0j+EvIQrHcfjlCH46JhJx4ACIOA4iDgAHyCVd67kFgmiVDPcMTYbZYoW+yQSD2cp3PAEQGykDZBogpg8i08xIGWoArEZIrc2AStSqBIwVYBbvntjbSVOASgkHVZTi4uIgFotx7Zpj94Zr164hMTGx3Xi5XA653Id9XDKV/7OKiaCREtNzg7ytkYhFiI10/f+ukktsq3XdK5DfEUH1bWUyGUaMGIEvv/xSOGa1WvHll19izJgxQbSMIIhgEfTp25IlS5Cbm4uRI0di1KhRWL9+PRoaGjBnzpxgm0YQRBAIuig9+OCDqKysxPLly1FeXo5hw4bh888/bxf8JggiPAh6nlJn6LZ5SgQRJvjyHe1e66UEQfR4SJQIgggpSJQIgggpgh7o7gz2cFhtbW2QLSEIwhn276Y3oetuLUp1dfw+LY+yugmCCBp1dXXQaj2rItGtV9+sVitKS0uhVqv5ndRhiH2rTXFxMa1A+gC9f52jo/ePMYa6ujokJydDJPIsWtStPSWRSITevbtPWYpAotFo6EvVCej96xzu3j9PPSQ7FOgmCCKkIFEiCCKkIFHq5sjlcqxYscK36gkEvX+dJBDvX7cOdBME0fMgT4kgiJCCRIkgiJCCRIkgiJCCRIkgiJCCRKkbUlVVhVmzZkGj0SAqKgqPP/446uvr3T5m/Pjx4DjO4fLrX/+6iywOLt52YH7//feRkZEBhUKBIUOG4NNPP+0iS0MTb96/LVu2tPs/Uyi87H7DiG7HlClT2C233MJ++OEH9s0337D+/fuzhx9+2O1jcnJy2Pz581lZWZlw0ev1XWRx8Hj33XeZTCZjmzdvZqdPn2bz589nUVFR7Nq1a07HHzhwgInFYvbnP/+ZnTlzhv3v//4vk0ql7OTJk11seWjg7fuXn5/PNBqNw/9ZeXm5V89JotTNOHPmDAPAfvzxR+HYZ599xjiOYyUlJS4fl5OTw5566qkusDC0GDVqFFu4cKFw22KxsOTkZLZ69Wqn43/1q1+xadOmORwbPXo0e+KJJwJqZ6ji7fuXn5/PtFptp56Tpm/djO+//x5RUVEYOXKkcGzixIkQiUQ4ePCg28du374dcXFxyMzMxLJly9DY2Bhoc4OK0WjEkSNHMHHiROGYSCTCxIkT8f333zt9zPfff+8wHgAmT57scnxPxpf3DwDq6+uRlpaGlJQUTJ8+HadPn/bqebv1htxwpLy8HPHx8Q7HJBIJYmJiUF5e7vJxjzzyCNLS0pCcnIwTJ07gueeew7lz57Bz585Amxw0vO3ADPDvr7Px7t7bnoov79+gQYOwefNmDB06FHq9Hi+//DLGjh2L06dPe7x5nkQpRFi6dCnWrFnjdszZs2d9Pv+CBQuEv4cMGYKkpCTcddddKCgoQL9+/Xw+L0G0ZsyYMQ49G8eOHYubbroJf/vb37Bq1SqPzkGiFCL89re/RV5entsxffv2RWJiIioqKhyOm81mVFVVOe0q7IrRo0cDAC5evNhjRcnbDswAkJiY6NX4nowv719bpFIpsrKycPHiRY+fl2JKIYJOp0NGRobbi0wmw5gxY1BTU4MjR44Ij/3qq69gtVoFofGEY8eOAQCSkpL8/VJCBl86MI8ZM8ZhPADs2bMnLDs2+6ODtcViwcmTJ737P+tUmJwIClOmTGFZWVns4MGD7Ntvv2UDBgxwSAm4evUqGzRoEDt48CBjjLGLFy+yF198kR0+fJhdvnyZffjhh6xv375s3LhxwXoJXca7777L5HI527JlCztz5gxbsGABi4qKEpapH3vsMbZ06VJh/IEDB5hEImEvv/wyO3v2LFuxYkXYpwR48/6tXLmS7d69mxUUFLAjR46whx56iCkUCnb69GmPn5NEqRty48YN9vDDD7PIyEim0WjYnDlzWF1dnXD/5cuXGQC2d+9exhhjRUVFbNy4cSwmJobJ5XLWv39/9rvf/S4s8pQYY+y1115jqampTCaTsVGjRrEffvhBuC8nJ4fl5uY6jP/nP//JBg4cyGQyGRs8eDDbtWtXF1scWnjz/j399NPC2ISEBHb33Xezo0ePevV8VLqEIIiQgmJKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKBEGEFCRKRLdiy5YtiIqKEm6/8MILGDZsmHA7Ly8PM2bM6HK7CP9BokQ4JS8vz2Ud74ULF4LjOIeqBoEQg/T0dKxfv97h2IMPPojz58+7fMyGDRuwZcsW4fb48ePx9NNP+9UuIrCQKBEuSUlJwbvvvoumpibhWHNzM9555x2kpqYGxSalUtmuyF1rtFqtgydFdD9IlAiXDB8+HCkpKQ7VKXfu3InU1FRkZWV16tzOPJgZM2YI3tf48eNRWFiIZ555RuiKAbSfvrWltceWl5eH/fv3Y8OGDcI5Ll++jP79++Pll192eNyxY8fAcZxXdX+IwECiRLhl7ty5yM/PF25v3rwZc+bMCfjz7ty5E71798aLL76IsrIylJWVeX2ODRs2YMyYMZg/f75wjtTU1HavCQDy8/Mxbtw49O/f318vgfAREiXCLY8++ii+/fZbFBYWorCwEAcOHMCjjz4a8OeNiYmBWCyGWq1GYmKiT5UftVotZDIZIiIihHOIxWLk5eXh3LlzQv8yk8mEd955B3PnzvX3yyB8gMrhEm7R6XSYNm0atmzZAsYYpk2bhri4uGCb1SmSk5Mxbdo0bN68GaNGjcLHH38Mg8GABx54INimESBPifCAuXPnYsuWLdi6davfvAmRSIS2pbxMJpNfzu0J8+bNE4L4+fn5ePDBBxEREdFlz0+4hkSJ6JApU6bAaDTCZDJh8uTJfjmnTqdziBNZLBacOnXKYYxMJoPFYunU87g6x9133w2VSoVNmzbh888/p6lbCEHTN6JDxGKx0N5JLBa7HKfX64WGBHZiY2ORkpLSbuydd96JJUuWYNeuXejXrx/Wrl2LmpoahzHp6en4+uuv8dBDD0Eul/s0bUxPT8fBgwdx5coVREZGIiYmBiKRSIgtLVu2DAMGDAjLxgChCnlKhEdoNBpoNBq3Y/bt24esrCyHy8qVK52OnTt3LnJzczF79mzk5OSgb9++mDBhgsOYF198EVeuXEG/fv2g0+l8svvZZ5+FWCzGzTffDJ1Oh6KiIuG+xx9/HEajsUtWEwnPoRrdRNjyzTff4K677kJxcXG7LrBE8CBRIsIOg8GAyspK5ObmIjExEdu3bw+2SUQraPpGhB07duxAWloaampq8Oc//znY5hBtIE+JIIiQgjwlgiBCChIlgiBCChIlgiBCChIlgiBCChIlgiBCChIlgiBCChIlgiBCChIlgiBCiv8P2IKlB2EsQyoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:44:33.160005Z", + "iopub.status.busy": "2024-02-29T18:44:33.159736Z", + "iopub.status.idle": "2024-02-29T18:44:33.460975Z", + "shell.execute_reply": "2024-02-29T18:44:33.460154Z" + }, + "papermill": { + "duration": 0.322423, + "end_time": "2024-02-29T18:44:33.462943", + "exception": false, + "start_time": "2024-02-29T18:44:33.140520", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAMElEQVR4nO2deXxTZfb/Pzd70yxd6AotLVD2XZYBkUX4KosKXxxBxZFN0BFnBhlcGEcUHUXHZWDG+TEzzkBFEVwGddSvIoMKyr4Isi+VLtCWbjTd0qzP74+be5u0SZqkSZM25/165ZXm5i4nafLJec5znnM4xhgDQRBEhCAJtwEEQRDOkCgRBBFRkCgRBBFRkCgRBBFRkCgRBBFRkCgRBBFRkCgRBBFRkCgRBBFRyMJtQFuw2+0oLi6GVqsFx3HhNocgiGYwxlBbW4v09HRIJL75QB1alIqLi5GRkRFuMwiCaIWioiJ069bNp307tChptVoA/AvW6XRhtoYgiObU1NQgIyND/K76QocWJWHIptPpSJQIIoLxJ7xCgW6CICIKEiWCICIKEiWCICKKDh1T6qwwxmC1WmGz2cJtCkF4RSqVQiaTBTUlh0QpwjCbzSgpKUFDQ0O4TSEIn1Cr1UhLS4NCoQjK+UiUIgi73Y7Lly9DKpUiPT0dCoWCkkKJiIUxBrPZjPLycly+fBk5OTk+J0h6g0QpgjCbzbDb7cjIyIBarQ63OQTRKjExMZDL5SgoKIDZbIZKpWrzOSnQHYEE49eGINqLYH9eyVMiCB9oMFthNNugkksRq6SvTSihd5cgWqHGaIHRws+EWm32MFvT+aFxAhE2FixYgFmzZgX1nBMnTsTy5cu97pOVlYV169b5fE6TtUmIbNSQLOSQKBFBwRcx6IgwxmB3ao1os0ePKj377LMYOnRou1+XRIkgvNBcg+yMgfq3hhYSpQ6C2Wr3eGse5/C2r8WHff1lwYIF2L17N9avXw+O48BxHPLy8rB48WJkZ2cjJiYGffr0wfr1690ev2bNGiQlJUGn0+Ghhx6C2Wz26br19fW4//77odFokJaWhtdee63FPmVlZbj99tsRExOD7OxsbNmypcU+HMdhw4YNmDZtGmJiYtCjRw98+OGHAHgRKiwoQKo+Bp9s/xAzp06GWq3GyJEjceHCBRw+fBgjRoyARqPBtGnTUF5e7vP7tnHjRgwYMABKpRJpaWl45JFHxOcKCwsxc+ZMaDQa6HQ6zJkzB9euXROfF7yYt99+G1lZWdDr9bj77rtRW1sr7mO32/HHP/4RvXr1glKpRGZmJl544QXx+SeeeAK9e/eGWq1Gjx498PTTT8NisQAAcnNzsWbNGpw4cUL8n+bm5vr82toCBbo7CH/95pLH57K7xGLWsK7i43/syYPFQ/CjW3wM7hrRVBhv497LMJpdl7M8+j+9/bJt/fr1uHDhAgYOHIjnnnsOABAfH49u3brhgw8+QGJiIvbt24elS5ciLS0Nc+bMEY/dtWsXVCoVvv32W+Tn52PhwoVITEx0+fJ44rHHHsPu3bvxySefIDk5Gb/73e9w7NgxlyHHggULUFxcjG+++QZyuRy//vWvUVZW1uJcTz/9NF566SWsX78eb7/9Nu6++26cPHkSPXo1vRevrv0Dnlv7Cgb07oGlSx7AvffeC61Wi/Xr10OtVmPOnDlYvXo1NmzY0KrtGzZswIoVK/DSSy9h2rRpMBgM2Lt3LwBeTARB2r17N6xWK5YtW4a5c+fi22+/Fc+Rl5eHjz/+GJ999hmuX7+OOXPm4KWXXhLfu1WrVuHNN9/En/70J4wbNw4lJSU4d+6ceLxWq0Vubi7S09Nx8uRJLFmyBFqtFo8//jjmzp2LU6dO4csvv8R///tfAIBer2/1dQUF1oExGAwMADMYDOE2JSgYjUZ25swZZjQaWzz3+lfnPd4+OnbFZd+/7Lrgcd/3Dxe67Lvh20st9gmECRMmsN/85jde91m2bBm78847xcfz589nCQkJrL6+vsmeDRuYRqNhNpvN67lqa2uZQqFg77//vritsrKSxcTEiHacP3+eAWCHDh0S9zl79iwDwP70pz+J2wCwhx56yOX8o0ePZr/85S9Zg8nCDv14jgFg697YwMprG5nZamNbt25lANiuXbvEY9auXcv69Onj1W6B9PR09tRTT7l97quvvmJSqZQVFjb9r06fPu3yWp555hmmVqtZTU2NuM9jjz3GRo8ezRhjrKamhimVSvbmm2/6ZA9jjL3yyivshhtuEB8/88wzbMiQIa0e5+1zG8h3lDylDsKySb08PidpthJl6fieHvdtvmpl0Y3ZbTHLK3/961+xceNGFBYWwmg0wmw2twicDhkyxCV7fcyYMairq0NRURG6d+/u8dx5eXkwm80YPXq0uC0hIQF9+vQRH589exYymQw33HCDuK1v376Ii4trcb4xY8a0eHz8+HE4+5tjR92ALholACAlJQUAMGjQIPH5lJQUt15Yc8rKylBcXIzJkye7ff7s2bPIyMhwKfXcv39/xMXF4ezZsxg5ciQAfhbRuaJjWlqaeP2zZ8/CZDJ5vAYAvPfee/jzn/+MvLw81NXVwWq1RkSxRIopdRAUMonHm0wq8XlfuQ/7BoNt27Zh5cqVWLx4Mb766iscP34cCxcu9DleFCmoFTJ00fALTeVyubhdWJPYfJvd3npMLiYmJii2OV+7+fVbu8b+/fsxb948TJ8+HZ999hl++OEHPPXUUxHx/yFRIoKCQqFwKbWyd+9ejB07Fg8//DCGDRuGXr16IS8vr8VxJ06cgNFoFB8fOHAAGo2m1YYQPXv2hFwux8GDB8Vt169fx4ULF8THffv2hdVqxdGjR8Vt58+fR3V1dYvzHThwoMXjfv36AfCvlKsvaLVaZGVlYdeuXW6f79evH4qKilBUVCRuO3PmDKqrq9G/f3+frpGTk4OYmBiP19i3bx+6d++Op556CiNGjEBOTg4KCgpc9mn+P20vaPhGBIWsrCwcPHgQ+fn50Gg0yMnJwebNm7Fjxw5kZ2fj7bffxuHDh5Gd7TpcNJvNWLx4MX7/+98jPz8fzzzzDB555JFW11NpNBosXrwYjz32GBITE5GcnIynnnrK5bg+ffpg6tSpePDBB7FhwwbIZDIsX77crRfxwQcfYMSIERg3bhy2bNmCQ4cO4V//+pfLPkazFZV1JsTIpW14p3ieffZZPPTQQ0hOTsa0adNQW1uLvXv34le/+hWmTJmCQYMGYd68eVi3bh2sVisefvhhTJgwASNGjPDp/CqVCk888QQef/xxKBQK3HjjjSgvL8fp06exePFi5OTkoLCwENu2bcPIkSPx+eef46OPPnI5R1ZWFi5fvozjx4+jW7du0Gq1UCqVbX7trUGeEhEUVq5cCalUiv79+yMpKQm33norZs+ejblz52L06NGorKzEww8/3OK4yZMnIycnB+PHj8fcuXNxxx134Nlnn/Xpmq+88gpuuukm3H777ZgyZQrGjRvnEj8CgE2bNiE9PR0TJkzA7NmzsXTpUiQnJ7c415o1a7Bt2zYMHjwYmzdvxtatW9G/f3/UNlpQY+SHNHbGYLUz2IKQpzR//nysW7cO/+///T8MGDAAt912Gy5evAiA98w++eQTxMfHY/z48ZgyZQp69OiB9957z69rPP300/jtb3+L1atXo1+/fpg7d64Yc7rjjjvw6KOP4pFHHsHQoUOxb98+PP300y7H33nnnZg6dSomTZqEpKQkbN26tc2v2xc4xjpuJlhNTQ30ej0MBkNEBOjaSmNjIy5fvozs7OyglIAgfIPjOHz00Udul7xU1ZtgsTHoY+Sw2hnqTVbEyKXQxchbnihK8fa5DeQ7Sp4SQXhByOiWcIAQWerAv+MdgrCKks1mw9NPPy1m/fbs2RPPP/88/dMJFBYWQqPReLwVFha2ix3CZ5HjODH1wpdPpzfbv/vuu9AZ3AkIa6D75ZdfxoYNG/DWW29hwIABOHLkCBYuXAi9Xo9f//rX4TSNCDPp6ek4fvy41+eDhbcfQcFT4rimWThf1uR6s71r164enyPCLEr79u3DzJkzMWPGDAB8tH/r1q04dOhQOM0iIgCZTIZevTwnjLYHzmIlAQfO4SP54smH2/aOTFiHb2PHjsWuXbvE3JITJ07g+++/x7Rp08JpFkEAcPWIBE9JwgESauYQUsLqKT355JOoqalB3759IZVKYbPZ8MILL2DevHlu9zeZTDCZTOLjmpqa9jKViEKa4km8IClkHJK0NCsaasLqKb3//vvYsmUL3n33XRw7dgxvvfUWXn31Vbz11ltu91+7di30er14ay3rlyDagkwqQbJWiSRN6BMGiSbCmqeUkZGBJ598EsuWLRO3/eEPf8A777zjUmJBwJ2nlJGRQXlKBBFGgp2nFNbhW0NDQ4vlBFKp1OOiRqVS2S5p7gThDsYYrjdYwBhDfKyCYkshIqyidPvtt+OFF15AZmYmBgwYgB9++AGvv/46Fi1aFE6zCAIAYLba0GC2QS6ViG2VhMqdjKEpm5IIKmGNKf3lL3/Bz3/+czz88MPo168fVq5ciQcffBDPP/98OM0iAiDSGgcEwx6rncHkVEKY4zjK6m4HwuopabVarFu3zq92N0TnxWw2Q6FQhNsMEeaUOCnAcfx2kqTQQWvfIh3GAKu5/W9+eAKBNg4Q+r698MILSE9PF6tG7tu3D0OHDoVKpcKIESPw8ccfg+M4lyzpU6dOYdq0adBoNEhJScEvfvELVFRUeLQnPz/f/7fecc85jdOEv8lRCh1UTynSsVmA71p26Qg5N/0WkPnmtbS1cYBOp8POnTsB8LM1t99+O6ZPn453330XBQUFLYZh1dXVuPnmm/HAAw/gT3/6E4xGI5544gnMmTMHX3/9tVt7kpKS/H4LnPOUBDgOAAMY+Uohg0SJaDN6vR4KhQJqtRqpqani9jVr1oh/Z2dnY//+/Xj//fddRCk2Nhb//Oc/xWHb3/72N3AchzfffBMqlQr9+/fH1atXsWTJEvGYN954A8OGDcOLL74obtu4cSMyMjJw4cIF9O7d2609/iIO35w3ikGlgE9LtAKJUqQjlfNeSziu20Z8aRwwaNAglzjS+fPnMXjwYJd8l1GjRrkcc+LECXzzzTfQaDQtrpmXl4fevf1rEeUJcfjm5CpJOA4cR4oUSkiUIh2O83kYFUkIjQNee+01jBkzBlqtFq+88opLTW2A95T8pa6uDrfffjtefvnlFs+lpaUFbHNz3A3f4tUd73/R0SBRIoKCt8YBAu4aBzSnT58+eOedd2AymcRE2cOHD7vsM3z4cPz73/9GVlYWZDL3H+FgFL3XU3XJsECzb0RQcG4cUFFRgZycHBw5cgQ7duzAhQsX8PTTT7cQF3fce++9sNvtWLp0Kc6ePYsdO3bg1VdfBdA0jFq2bBmqqqpwzz334PDhw8jLy8OOHTuwcOFCUYia2+NL66PmCDN3we5mQniHRIkICoE2DmiOTqfDp59+iuPHj2Po0KF46qmnsHr1agAQ40zp6enYu3cvbDYbbrnlFgwaNAjLly9HXFycuGypuT3BqlRZb7Kiqt4Mo9kalPMRLaHGAREELch1z5YtW7Bw4UIYDIagNXL0hRqjBQxArEIqNvysabTAaLYhViGFRkXDO6CTLcglCHds3rwZPXr0QNeuXXHixAkxB6k9BQkATFY77IxBrWjq80YZAaGHRImIOEpLS7F69WqUlpYiLS0Nd911F1544YV2t0NIkHSOKFFGd+ghUSIijscffxyPP/54WG1gjHlc+waQpxRKKNBNEK3gsvZNECVylUIGiVIEQh/48MOaNQ0Q/25/UyKeYH9eSZQiCLmcn81paGgIsyWE84Jb5zwlyllqifB5FT6/bYViShGEVCpFXFwcysrKAABqtZq+BGHCarPDYraA44DGRtfn9AoAsKOx+RNRBmMMDQ0NKCsrQ1xcHKRSaesH+QCJUoQhrGoXhIkIH8KwpIZ+GLwSFxfXpmoMzSFRijA4jkNaWhqSk5NhsVjCbQ5BeEUulwfNQxIgUYpQpFJp0P/ZRNsxNFjw7YUyyKUSTB8UvIoERBMkSgThhqvVRpy6akCSVonhmfHidovdjp/K6xGjoB+MUEGzbwThhuv1ZpwprkFRletMqNyx4Ndq87/qAOEbJEoE4QahrZJc6voVkUn5oLfVziifLESQKBGEGyw2XnBkEteZN6mkae2bzU6iFApIlAjCDaKnJHP9ijh7TlYSpZBAokQQbjA7REnRbPgm4ZqWnZAohQYSJYJwg9UxfGseU+I4DnKpBFIJB5uNRCkUUEoAQbhBGL4JgW1nfjmhJyQSyvIOFSRKBOGG/+mfggm9k9yKEglSaCFRIgg3yKWSFkM3on0gUSIIP/nuYjmuN1jws+wEJOuowUOwoZ8CgnDDvksV+OZ8GQwNLRdFX7luRF5ZHWpN1GYpFJAoEYQbzpTU4HhhNYyWll12hYRKSp4MDSRKBOEGi5gS0DKoLQS/LbT+LSSQKBGEG6weMroBQOpYlEueUmggUSKIZtjtTMzWFqoCOCOXNC3KJYIPiRIRtTRabPj8xxIcLbjust3sNCxzN3wTFuVaKaM7JJAoEVHL8aJqXLhWiz0XylHvNJMmxIokHCcKkDNN5UsophQKwi5KV69exX333YfExETExMRg0KBBOHLkSLjNIqKAslqT+HdpTVNnEnHdm4xz201mfE4SfnVzL4zpkRh6I6OQsCZPXr9+HTfeeCMmTZqEL774AklJSbh48SLi4+NbP5gg2kiPLrHIK6sDAFTWmdEzid+uj5FjyfgeHhfcyjxlelfmAVU/ASkDAR3V7w6UsIrSyy+/jIyMDGzatEnclp2dHUaLiGhiYFc9ahotOPhTFWobm5IkJRIOGqWfX43qQuDkB3z1t5ITwMgHgJi44BocJYR1+Paf//wHI0aMwF133YXk5GQMGzYMb775psf9TSYTampqXG4E0RZ0Kr6ra22j79nZBZX1+PJUCY4VOgXIC/Y19fq2WYCiQ8E0M6oIqyj99NNP2LBhA3JycrBjxw788pe/xK9//Wu89dZbbvdfu3Yt9Hq9eMvIyGhni4nOgsVmR3WDGSk6FaYNSsXYnk3xoWs1jfj2fBlOXTW4Pba6wYKzJbW4et3Ib2g0AFWX+b9738Lfl58FKBAeEGEVJbvdjuHDh+PFF1/EsGHDsHTpUixZsgR/+9vf3O6/atUqGAwG8VZUVNTOFhOdhWs1jdi0Nx+f/1iMvqk6l4W1FXUm/FBYjUuOeFNzWsy+CYKk7wqkDQVkSsDcANRcDeVL6LSEVZTS0tLQv39/l239+vVDYWGh2/2VSiV0Op3LjSACwWjm17SpFS1jR2LTADc5SgAgE9ssOYZr1QX8fXwWIJHy9wBguBI0e6OJsIrSjTfeiPPnz7tsu3DhArp37x4mi4hoocEhSiqFFHnldTh5xQCTld/mqb2SgNR5QS5jfJAbAOIy+Xt9N/6eRCkgwjr79uijj2Ls2LF48cUXMWfOHBw6dAj/+Mc/8I9//COcZhFRgLD6Xy2XYueZazCabUiLU0GpkcJidd80QEDI8rbYGR9PMtUBnATQOcRIEKWaq7xoucl1IjwTVk9p5MiR+Oijj7B161YMHDgQzz//PNatW4d58+aF0ywiCjA5hEcll0LtaMEtDOnMvnpKNjtQV8ZvjO0CSB2/8bFJvBBZjIC5PlQvodMS9sqTt912G2677bZwm0FEGWbBG5JJECPnRUkY0lm9lC3htztiSnYG1F3jN2pSmnaQyoGYeKChCqgvA5SaULyETkvYRYkgwoGLKAmeksU1puQpc7uLRomHJvTkA+FnjvEbnUUJ4L2lhiqgvgJI6BGCV9B5IVEiopLuiWqo5BJ00ShQVc+vgWsw8wmUE/skY3SPRNGDao5UwolChvpy/l6T5LpTbBJQfr5peEf4DIkSEZUM7KrHwK56AEBRFZ8EKcSUYhTSJtHxhs3CB7oBXoScER43VAbF3miCRImIepoP31rDYrNjz4VySBrKMcFuh0SuAuTqZid1LCo3Xm95AsIrJEpEVFLbaIFcKoFSJkH3BDWmD0pDvJpfB3fwp0rYGMOgrnpoHWvjmvPjFQMSGq7ArmGQqBNaTvsLi3EtRsBq4rO8CZ8gUSKikrcPFMBksWP+2CwkxCoQH6sQnzteVI0Gsw05yVpo3bR1E7qZxFgM/BpctZu6SjIloFDzy02M1wFtaoheSecj7EXeCKK9YYzBYuWn/RVuGgM4z8y5g+M4yCQcVNZq2BgDYhLcX0gVx98bq9tqclQRkCj99NNPwbaDINoNq53B7igzopBKYLczXCqrxckrBlhsdrEhgNKDKAF8ukCTp+RBlCiuFBABiVKvXr0wadIkvPPOO2hsbGz9AIKIIARPiOOaEiQ/PVGC/569BoOxqdibp2UmAByekgF2uxdPSYgrNVYHw+yoISBROnbsGAYPHowVK1YgNTUVDz74IA4doqJWRMfAeXjGcRwkEg4qR05StaNNt0ImgcRN0wABOSyQ2U28x+WpwiR5SgERkCgNHToU69evR3FxMTZu3IiSkhKMGzcOAwcOxOuvv47y8vJg20kQQUNY2+bsCQnr3wxGMwDvQzcAUDN+TZtNqvQ8s6bi86DEXCbCJ9oU6JbJZJg9ezY++OADvPzyy7h06RJWrlyJjIwM3H///SgpKQmWnQQRNARPyXnBbUwzT6k1UZrRW4MbusdDF+elo4nSUe/LVNtUKpdolTaJ0pEjR/Dwww8jLS0Nr7/+OlauXIm8vDzs3LkTxcXFmDlzZrDsJIigoVZIMairHjnJTQtlhQTKOLUc94/pjmmDvHcjiWX1UEglkAjekDuUWj5wZbcBloag2B4NBJSn9Prrr2PTpk04f/48pk+fjs2bN2P69OmQOCryZWdnIzc3F1lZWcG0lSCCQqJGiSn9XRfQCsM3s5UhUeNDomOjo2mFUut5H4kUUMTy9ZYaa/i/iVYJSJQ2bNiARYsWYcGCBUhLc/+LkpycjH/9619tMo4g2gth+Ga0+NbVpLD0GmwV9YhNUiDZ245KLS9KploA1AvOFwISpZ07dyIzM1P0jAQYYygqKkJmZiYUCgXmz58fFCMJIphYbHbY7AxyqUQs2JaTokUXrRIVdSbsy6tAdpdYpOljPJ6jqrICxppGxFlVrYsSSgATtQPzlYBiSj179kRFRUWL7VVVVdRMkoh4Tl41YMO3efjqdKm4LUmrRO8ULa7X880pSw3e8++UNr7TiUXWSgE3pSPmRKLkMwGJEvMwk1BXVweVys1iIYKIIKxit5KWH/96Ez98i/XWIZcxKKx8SkCj1EtMCWiKOZlq/Tc0SvFr+LZixQoA/Nqf1atXQ61uKtdgs9lw8OBBDB06NKgGEkSwsYqVJZuSIxljyK9swNVqvraSPsZ9dQAAgLkeUtgAcDBL1J73A5pEqZE8JV/xS5R++OEHAPw/8OTJk1AomlZWKxQKDBkyBCtXrgyuhQQRZCyOtW1yp5gox3H49ESx+NirKJlqIeE4mKUxYGilU4nKKVeJ8Am/ROmbb74BACxcuBDr16+nZpBEh8SdpwQAaXoVrlw3IkYhFZeduMVUAwkHmKUacLZWkiKdh292OyChwhytEdA7tGnTJhIkosNi8dCtZFxOF3SNj8GUfl7n04DGGkg4DiaZRqwo4BGFI4GS2QELtVvyBZ89pdmzZyM3Nxc6nQ6zZ8/2uu/27dvbbBhBhAqr3eEpNfNa0vQxmDMio/UTmAxI1ikxsXdPyHoned9XIuFL5Zrr+Zu3ZEsCgB+ipNfrwTlKfur1XlLrCSLCSY+LgZTjEK9WtL6zOxprIJNIoNElAN6GeQKKWF6QTHUAaVKr+CxKmzZtcvs3QXQ0hmfGt+0EQtBa5WMIQ6nlWy2Z69p23SghoJiS0WhEQ0PTAsOCggKsW7cOX331VdAMI4iIxVQDo8WGQ8VWHCv0oVaSsOaNWnj7RECiNHPmTGzevBkAUF1djVGjRuG1117DzJkzsWHDhqAaSBDBxmy18xUjA8FuA8z1aLTYcLDEirMlPuQfKRxZ3+Qp+UTAlSdvuukmAMCHH36I1NRUFBQUYPPmzfjzn/8cVAMJIti8c6AA63ddRLEjUdIvTDUAY5BIZbBKVLD5Im4kSn4RkCg1NDRAq+Ujdl999RVmz54NiUSCn/3sZygoKAiqgQQRbMTZN2kriY/uEDKzVTqA48T0Aq8oHaJkIlHyhYAbB3z88ccoKirCjh07cMsttwAAysrKKH+JiHjEPKVAEhkdQW7OEeS2OQTOKxRT8ouARGn16tVYuXIlsrKyMHr0aIwZMwYA7zUNGzYsqAYSRLBpWpAbgKfkWO0vVJz0yVNyHr5RWdxWCaie0s9//nOMGzcOJSUlGDJkiLh98uTJ+N///d+gGUcQwcbm1PNN7qWFkkccwzcuRvCU/BAluw2wNgJyz3WaiDa07U5NTUVqqmsr4lGjRrXZIIIIJVan4ZbMSwsljzg8JamjrZLNzmC3M6/tmCCVAXIVYGnk40okSl4JSJTq6+vx0ksvYdeuXSgrK4O92biaOugSkYowdOM4iFUn/cLRLkmh1mPez7pCLpGA8+U0Cg0vSuY6AK0sTYlyAhKlBx54ALt378YvfvELpKWlictPCCLS4Tigd4oWDCywz60j0C2J0SM51o+ChgoNUF9BaQE+EJAoffHFF/j8889x4403BtsegggpaoUMMwYHWMDf0ghYTfzfSj9nmWkGzmcCmn2Lj49HQoKH/ukB8tJLL4HjOCxfvjyo5yWIoCGseZOrAJkCRwuqsO9SBRrMPnRAoVwlnwlIlJ5//nmsXr3aZf1bWzh8+DD+/ve/Y/DgwUE5H0F4gjHmscZ8qwjF/x1e0tGC6zh4uQr1Jlvrx1JWt88ENHx77bXXkJeXh5SUFGRlZUEudy0deuzYMZ/PVVdXh3nz5uHNN9/EH/7wh0DMIQifya9swMc/XEV6nApzR2b6d7AjyC2IklQiAWBzmdHzCImSzwQkSrNmzQqaAcuWLcOMGTMwZcqUVkXJZDLBZDKJj2tqqBg74R9CKdzAgtxOS0zQlFJg9SmBkmJKvhKQKD3zzDNBufi2bdtw7NgxHD582Kf9165dizVr1gTl2kR04qkUrk8IMSWHpyRkhNOi3OAScBXz6upq/POf/8SqVatQVVUFgB+2Xb161afji4qK8Jvf/AZbtmzxuVfcqlWrYDAYxFtRUVGg5hNRijDUkgay7q3Rg6fky/BNCHRbzfyN8EhAntKPP/6IKVOmQK/XIz8/H0uWLEFCQgK2b9+OwsJCsdaSN44ePYqysjIMHz5c3Gaz2bBnzx688cYbMJlMkEpdS40qlUoolcpATCYIAM6LcdswfHOJKaH15gEAIFXwmd02K99AQBZgKd4oICBPacWKFViwYAEuXrzo4uVMnz4de/bs8ekckydPxsmTJ3H8+HHxNmLECMybNw/Hjx9vIUgEEQya2iv5+dFnzGn4xpftEYaAPsWUOK5pCEdpAV4JyFMSpvCb07VrV5SWlro5oiVarRYDBw502RYbG4vExMQW2wkiWAhejd8VAsx1/IJajhNF6aacJPysR6L3xpXOKGIBYzUFu1shIFFSKpVuZ74uXLiApCRa10NELvoYObK7xKJLrJ9hAMFLUmgACe/FJ8T6OQSjGTifCEiU7rjjDjz33HN4//33AfDTq4WFhXjiiSdw5513BmzMt99+G/CxBOELA7vqMbBrAC3CmgW5A0Lh6K9EM3BeCSim9Nprr6Gurg5JSUkwGo2YMGECevXqBa1WixdeeCHYNhJE+GkW5AaAoqoGHM6vQmGljysbRE+JRMkbAXlKer0eO3fuxN69e3HixAnU1dVh+PDhmDJlSrDtI4jIQPCUnDrc5lfW40j+dQzvHo/MRHXr56Dhm0/4LUp2ux25ubnYvn078vPzwXEcsrOzkZqaCsYCLAdBEO3EJ8ev4sp1Iyb3S0bfVD+GYibHEhNV09BPaPvtU51ugBIofcSv4RtjDHfccQceeOABXL16FYMGDcKAAQNQUFCABQsWUClcIuIxW+0wW+2Q+Pvj2SybG2iawfOpTjfQlEBJnpJX/PKUcnNzsWfPHuzatQuTJk1yee7rr7/GrFmzsHnzZtx///1BNZIggoWQEuB31UlhMa5ToFs4h0/LTACn4VsDYLcDgWSVRwF+vStbt27F7373uxaCBAA333wznnzySWzZsiVoxhFEsBGSJ/1qr2Sz8EICuHhKcn8yugFAHsvnOTE7YAlO2Z/OiF+i9OOPP2Lq1Kken582bRpOnDjRZqMIIlQElDwpDN2kcpei/1KxSoCPMSWJpOl4GsJ5xC9RqqqqQkpKisfnU1JScP369TYbRRChIqCeb43V/L1KD+cuAcI5fPaUAEoL8AG/Yko2mw0ymedDpFIprFYfSoMSRJiwCC27/Rm+iYmTrkmX3eJj8PMbuiFG4cc6TYUWQDl5Sl7wS5QYY1iwYIHHlfrOBdgIIhLpGhcDk8UOpcwPUXKTOAnwTQjUCX5m1VCuUqv49Y7Onz+/1X1o5o2IZGYO7er/QW5m3gKGhm+t4pcobdq0KVR2EETk4mH4ZjTbcP5aLTgAQzLifDsXJVC2SsBtuwkiavAwfGu02PDNuTIoZBLfRYkSKFuFRImIGmoaLXh7fwFUcikWj8v27SC73WOFAIUjLmWx2X1fYkUxpVahlFIiarA4lpiYrT7mFQH8MIvZAU7SVHrEgSBKjAFmX3OVxOqTtb7bEGWQKBFRg7AcxK9OJian6gDN0ghkEk5cQ+ez0Ameks1CDQQ8QKJERA0WIZvbn3VvXmbeOI5zGsL5mEApNBAAKNjtARIlImoQloNI/Wka4GHmTUAQJZ89JY5zqkBJcSV3kCgRUUNA7ZU8zLwJ+C1KAAW7W4Fm34iowSYuxvXHU/KeODmpTxIYA5K0fjQioARKr5AoEVGDUiZBepwKiRo/upAYq/l7VZzbp7vF+1AGtzmUQOkVEiUiasjqEousLrG+H8BYU4WAmLjgGUIJlF4hUSIIT5hqHA0oJYDSfaC7xGBEWY0JSVol0uNi3O7TAoopeYUC3QThCXHopvdYuvbCtTp8fa4MeeV+DMVo+OYV8pSIqOHAT5U4ecWAwd30GN0jsfUDjI6ChTHxHndRSNsw+2YiUXIHeUpE1GC02FBnsvpeKdKHeFKbUgIsjgYChAskSkTUIJbC9TVPSRi+efGUhGJxJn9ESWwgwKiBgBtIlIioQcjo9jlPSRi+eUgHACCWwjVabL4bIpEAckcqAQW7W0CiREQNVn8X5IrDN8+eUozcIUpmP0QJoARKL5AoEVGD1Z+mARYjYGnk//YSUxJFyR9PCaAZOC/Q7BsRNVj8aa8kDN0UsXy/Nw9oVDLcPiQNKrkfHU0ASqD0AokSETXoY+QwWe2id+OVhir+Xu09dUAulaBXstbrPm6hBEqPkCgRUcOtA1J937mhgr9vRZQChoZvHiFRIgh3NFTy9z6IUn5FPQxGC7ISY6FXex7quUAJlB6hQDdBuEMYvsW2LkqH86vw9bkyFBuMvp9f6RjyUa3uFpCnREQNm/fngzHgf4d3hU7lxaOx25oC3T54SlqVHIARtY1+tKwXRMlcxydR+tIJJUogUSKiAsYYqurNYAxisX+PNBp4YZLKPFacdEan4r9GtY0W3w1SaHkhstt4YVIGECzvpIR1+LZ27VqMHDkSWq0WycnJmDVrFs6fPx9Ok4hOis3OwBxL3lpNnqx3CnL74MHoYnivy9lTMjRY8M25MuzLq4DJ6iaHSSKhdkseCKso7d69G8uWLcOBAwewc+dOWCwW3HLLLaivp2lSIrg4dxuRt5Y86UeQGwC0Dk+pxuEpWWx2fHjsCo4XVePgT1X49EQJ7O4WAVNcyS1hHb59+eWXLo9zc3ORnJyMo0ePYvz48WGyiuiMCM0iZRIOktYW5PopSvGxfHnd6/UWWG12HM6vQo3RArmUA2NAUVUDzpTUYGDXZoXiVDqgpripYwoBIMJiSgYDX6Q9ISHB7fMmkwkmk0l8XFND/0zCNywOUZLLfBgc1Jfx97FJPp1bq5QhRiGF0WzDTxX1OJrPB8lvHZCKOpMVBqMFPZM0LQ8UPSX6HDsTMaJkt9uxfPly3HjjjRg4cKDbfdauXYs1a9a0s2VEZ0AUpdYqBNhtQL3DU/JRlDiOw60DUhGrkOJcaS2sdobMBDV6JWvAeYtJCUF0EiUXIkaUli1bhlOnTuH777/3uM+qVauwYsUK8XFNTQ0yMjLawzyigyPhOCRqFNAoW/nIN1QBdisgU3itDtCcbEdDgiStErFKKbISY70LEuAkShRTciYiROmRRx7BZ599hj179qBbt24e91MqlVAq/eivRRAOUnQq3D8mq/Ud667x97HJAeUOcRyHG7q7hh8YY8grr8OlsjpM6psMpcyx9k4YvlFMyYWwzr4xxvDII4/go48+wtdff43s7OxwmkMQTaKkSQ7qafdeqsTZklpcrnCaWRYaXJrrqCyuE2EVpWXLluGdd97Bu+++C61Wi9LSUpSWlsJo9CNdnyCCSX05fx9EUeI4Dr2S+UD3T+VOoiSP5ds3MQaYaQgnEFZR2rBhAwwGAyZOnIi0tDTx9t5774XTLKITcuqqAZv352PfpQrPOzHm5CmlBPX63RP58rdXrjeACVmcEklTXSWKK4mENaYk/nMIIsTUm6yorDMjXe+lQqS5DjA38LEkH2fefCVVp4JcyqHeZENlvRldNI7YqErPx5QaDYDeczw1mqAqAURUIGR0e81Tqi3l79WJXqtNBoJMKhE76BZVOXUwEZoSCJ1TCBIlIjoQ85S8ZXPXXOXvdV1DYkNXhyiVGhqbNgr1v4UmBQSJEhEdmH3J6DYIopQeEhtSdCoAQINz5xPylFoQEXlKBBFqhEaUHjO67XagtoT/O0SeUrf4GDw0oafYKw4AeUpuIFEiooKmZSYehm/15YDNwmdyx3YJiQ0yqQSy5j0LBE/JVMsvcZH42RWlE0LDNyIqUMkl0KpknlshCfEkbXr7VoFUxPLF5BjjZ+AI8pSI6GDqwDTvO9SENp4kUFTVgEOXqxCnlmNyvxReAFVxfGG5xmpA7b5CRjRBnhJBMAZUF/J/x4V2gbfNzlBY1YAr151WLVCw2wUSJYIwXucTGCVSQB9aUeqi5ZMmrzeYxTiXGOwWmhVEOSRKRKeHMYa3DxTgvcOFMJrdZHRXF/D3uvSgJ002J1YhRYxCCsaA6/VmfqMwZBPaOkU5JEpEp8diY6ioNaG4uhFSd8mT1x2iFNc95LZwHIdER/nc8jpHFVWh7K5QhjfKIVEiOj1CNxEJx7VMCWCsyVOKD70oAU1DuIo6wVNyiFJjNWDzo3dcJ4VEiej0mK187EYhk7SsBll3jV+EK5WFLGmyOUmOxbiVgqek0AAyJS+QRhrCkSgRnR6Tkyi1oPISfx+f3W6Ji4kaBWIU0qYKlBxHQzgnKE+J6PQInpLSnShVXOTvu+S0mz2pOhUeHN/D1WtTJ/LtlkiUSJSIzo9HT6mxhi9XwnFAQs92s8dtQwFhaUu9lyJ0UQIN34hOD8fxXWxbdDIRhm7atKYKkO2M2DlXHL6RKJGnRHR6eqdo0TtF2/KJ8vP8fTsO3QTOl9biu4vlyEhQ49YBqU01wesr+Rk4afR+NclTIqITU21TKkByv3a/vFTCobbRivJaxwycUgfIYwBmb2peEKWQKBHRSdk5fgpe39WvppPBIsmRq1RVb4bNzvgxptCsoK603e2JJEiUiE7PN+fK8N7hQteea2Wn+fvkAWGxSaeSQSGTwGZnqBKWm2gdolR7LSw2RQokSkSnp9yxxMQqLICtKwdqSviea8l9w2ITx3FiEqU4hNOkOuwjT4kgOjVGC7/MRCzwVnyMv+/Siy+yFiaSxOUmDlHSCqJUxlfBjFJIlIhOj1CoP0YhBawmoPQk/0T68DBaBbH3myhKMfG8SNptTfXCoxASJaJTY7czcUFujFwKlJ7ivRB1IhCfFVbbUnRKpOlVSNbyXU7AcU1F5qqLwmdYmIneZAgiKmi02iA0Yo6RAig6wD/oekP71uJ2Q7JOhbtHZbpu1GfwM4OGK+ExKgIgUSLCB2OAtRGwGPn8HGbntzE7X2xNquBzd9pQeK3OxJcCUSukkJSd5JeWKGKBtMHBehXBRah8aSiK2iTK6HvFRPiwNAKVF4HKPL5kiLGaFyBvcBwfa9F1BZL6AAk9/FrNb7Ux6GLk0MrsQP5efmPmmJBXmPQHi82OBpMNerWcz+xWxALmesBQyL/eKINEiQg95gZ+2HT1mPtZJakckMh4AeIkADjAbuWD0szOl4ltqOID1EotkPkzIG2oT15EelwMFo/LBrv0NVBUC6j0QPrQYL/CgLlcUY//HC9GklaJe0dn8u9BYi+g5AQv3iRKRIfEZuVzW8z1/Jdanch7F2GOmYAxoOwMcHEnP0QDeNuS+wL6boC6CyBXexYXxvjXVHcNqLrMn8tUy5/v6jGg9y2+BasNV8FdOcz/nfM/EeUlJcQqYGcM5bUmWGx2voOvKEqXgF5Twv9/bGdIlDoytdeAK4f4wKi9WRnVmDggfRh/kynb3zabFbjwZdP0e2wXoMckILGn718yjuNX7ys1/HE9JgKlJ4D87/m6Q8e3AikDgJ43e17lb6oFznzMe1zJfcOy+NYbOkf1gjqTFcXVRnRPjOWFVirnh7eGKyFv+xRpkCh1RGxWIP87oOggxKklhZr3juxWfqW5sRrI+wYoPMD/2qYMaL9f3EYDcGp7U62irHF8HKetlR2lMn7WLHkAcHkPnwR57TQfp8oaD3Qd7nqN+kr8uONfkBqrkNG1K3S9p7Xt+iGA4zhkJqpxprgGBZUNvCjJFEBSX17QS38kUSIinJoS4NxnTcXAkvoAGaP59kCC6FjNQPk5oHA/H4s5+ylw7RTQZxofUwkl1UXA6e18HEkeA/SfCSRkB/cachU/dEsdBFzcwb8nl/7Lv96kPvxrrK8AKzsDk6Ec9VwskgfM4Y+LQLISYx2iVA8gid+YNpgXpbKzvCcojwmrje0JiVJHwW4DCvYCBfv5oYhCDfSeyn8JmyNT8B/qlAG8N5W/l4/JHP4n7zWlDg6N11R8HLj4FW+rJhkYeGdTo8VQoEsDhs8HSo7zQzpTHR9rcmC12VGp6Iq8xAm4MSEpdHa0ke6JanAc393E0GDhZ+H0GYAmiV+nd+UwkD0+3Ga2GyRKHYG6Mt7bqSvjHyf3BXJu5YXJGxIp0H0sPxQ49xlguAqc+z++uFmfafxMVjCw2/ihohBMTu4L9L2tfQLKHMfHzVIH87NVhkI+qK7QoEySjrN2QK9W8AHkCEUllyIjXo3CqgacLa3Bz3okOoa9N/HD4CuH+dcYrP9XhEOiFMnYrEDhPj4uZLfxLnzvW/0vSqZOAIbexwfFL3/Hf3kPvQnk3NL2WFNDFXD2P/wQCgCybwK639j+M0YSKZDUm785KMyrBLhKpOkjc9jmzPDu8cjqEou+qU7C06U37w3WlAAXdvCeZxTMxEXEz8df//pXZGVlQaVSYfTo0Th06FC4TQovjPEzakf+xQ+97Db+yzbygcCrJEokfH7PiIX8anSrife+Tv2bD4z7i83C23bkX/yXRqYEBs7mg9oR8sUput4AgM9VinSyu8Tihu7xiHWuI85xQJ8ZvOBWXAQu726a2OjEhN1Teu+997BixQr87W9/w+jRo7Fu3TrceuutOH/+PJKTk8NtXvtiaeRzcYp/aBqqKWJ5jyapT3C+7LFd+DhM4X4+RlVxkc+H6ZIDpAzik/W8JSU2VPEzXsU/8DlEAN9Ztu+M0AfR/cDQYEFxNZ8blZ0UvvIkgWByrNdTyaV8XCnnFuD8F3w80VTHB75bG7p3YDjGwiu9o0ePxsiRI/HGG28AAOx2OzIyMvCrX/0KTz75pNdja2pqoNfrYTAYoNPp2sPc4GCzAJYGfoaqoZLvYFFdyHscwrILqRzIGAV0GxW6WaO6Mn5qXeh9BvCZ1ZpkXrzkaj4Z027hUwzqy/l7AZWOzz1K7hcx3pFATaMF312ogMlqw+zh3cJtjs9U1Jnw+Y8lUMklmNQ3GUkaJd+S6coRfoaRMUfKQD/+B0STzMeaIigh1JlAvqNhFSWz2Qy1Wo0PP/wQs2bNErfPnz8f1dXV+OSTT1z2N5lMMJlM4uOamhpkZGS0/oKrLvP/0OaIL535t83lLWu+rZVz2awtEx2die3CL6FIGdB+v4Z15XxSYtk5PtnQG5wEiMsE0obw3ls7dZUVsNsZ3jlYAMYAO2Mu9wwMmQlqTB2YJu5vtdkhi+Agd3Ou1TTi38euwGRp6lUXI5dCJuUQayzGRMkJJLJqAEB1gxkFVfwQVaFQoF+3JP7/wUmabqFk8Fz+h8kLgYhSWIdvFRUVsNlsSElJcdmekpKCc+fOtdh/7dq1WLNmjf8Xspoir8kfJ+ED1+oEfumFNo3P5A3lFLonNEl8qkDPyYDxOl9gzFjdtHpfKgMUWl4wtWlhzffhOKCyzuzx+UaL6wLfjiRIAJCiU+GekZnYm1eBn8rrYbbaxQ6/leiCyv7zkKioBCouwHTlJ9RZ8iFhNjBmbhpOtxetLaYOkLDHlPxh1apVWLFihfhY8JRaJS4DGHJ3syGG42+v25yea74t0HNJJI71XoqIG/LwPe0T+FuEwnEcfn4DPxyTSDhwACQcBwkHgAOUsvb13EJBfKwCtw1Oh9Vmh8Fogclq5zueAEjUKACFDkjIhqa7FRmDTYDdDLm9EYiVOJWAsQPM5t+F/R00haiUcFhFqUuXLpBKpbh2zbV7w7Vr15Camtpif6VSCaUygHVcitjgZxUTYSMjofMGeZ2RSSVI1Hj+vMcqZY7Zuo4VyG+NsPq2CoUCN9xwA3bt2iVus9vt2LVrF8aMGRNGywiCCBdhH76tWLEC8+fPx4gRIzBq1CisW7cO9fX1WLhwYbhNIwgiDIRdlObOnYvy8nKsXr0apaWlGDp0KL788ssWwW+CIKKDsOcptYUOm6dEEFFCIN/RjjVfShBEp4dEiSCIiIJEiSCIiCLsge62IITDampqwmwJQRDuEL6b/oSuO7Qo1dby67R8yuomCCJs1NbWQq/3rYpEh559s9vtKC4uhlar5VdSRyHCUpuioiKagQwAev/aRmvvH2MMtbW1SE9Ph0TiW7SoQ3tKEokE3bp1nLIUoUSn09GXqg3Q+9c2vL1/vnpIAhToJggioiBRIggioiBR6uAolUo888wzgVVPIOj9ayOheP86dKCbIIjOB3lKBEFEFCRKBEFEFCRKBEFEFCRKBEFEFCRKHZCqqirMmzcPOp0OcXFxWLx4Merq6rweM3HiRHAc53J76KGH2sni8OJvB+YPPvgAffv2hUqlwqBBg/B///d/7WRpZOLP+5ebm9vic6ZS+dn9hhEdjqlTp7IhQ4awAwcOsO+++4716tWL3XPPPV6PmTBhAluyZAkrKSkRbwaDoZ0sDh/btm1jCoWCbdy4kZ0+fZotWbKExcXFsWvXrrndf+/evUwqlbI//vGP7MyZM+z3v/89k8vl7OTJk+1seWTg7/u3adMmptPpXD5npaWlfl2TRKmDcebMGQaAHT58WNz2xRdfMI7j2NWrVz0eN2HCBPab3/ymHSyMLEaNGsWWLVsmPrbZbCw9PZ2tXbvW7f5z5sxhM2bMcNk2evRo9uCDD4bUzkjF3/dv06ZNTK/Xt+maNHzrYOzfvx9xcXEYMWKEuG3KlCmQSCQ4ePCg12O3bNmCLl26YODAgVi1ahUaGhpCbW5YMZvNOHr0KKZMmSJuk0gkmDJlCvbv3+/2mP3797vsDwC33nqrx/07M4G8fwBQV1eH7t27IyMjAzNnzsTp06f9um6HXpAbjZSWliI5Odllm0wmQ0JCAkpLSz0ed++996J79+5IT0/Hjz/+iCeeeALnz5/H9u3bQ21y2PC3AzPAv7/u9vf23nZWAnn/+vTpg40bN2Lw4MEwGAx49dVXMXbsWJw+fdrnxfMkShHCk08+iZdfftnrPmfPng34/EuXLhX/HjRoENLS0jB58mTk5eWhZ8+eAZ+XIJwZM2aMS8/GsWPHol+/fvj73/+O559/3qdzkChFCL/97W+xYMECr/v06NEDqampKCsrc9lutVpRVVXltquwJ0aPHg0AuHTpUqcVJX87MANAamqqX/t3ZgJ5/5ojl8sxbNgwXLp0yefrUkwpQkhKSkLfvn293hQKBcaMGYPq6mocPXpUPPbrr7+G3W4XhcYXjh8/DgBIS0sL9kuJGALpwDxmzBiX/QFg586dUdmxORgdrG02G06ePOnf56xNYXIiLEydOpUNGzaMHTx4kH3//fcsJyfHJSXgypUrrE+fPuzgwYOMMcYuXbrEnnvuOXbkyBF2+fJl9sknn7AePXqw8ePHh+sltBvbtm1jSqWS5ebmsjNnzrClS5eyuLg4cZr6F7/4BXvyySfF/ffu3ctkMhl79dVX2dmzZ9kzzzwT9SkB/rx/a9asYTt27GB5eXns6NGj7O6772YqlYqdPn3a52uSKHVAKisr2T333MM0Gg3T6XRs4cKFrLa2Vnz+8uXLDAD75ptvGGOMFRYWsvHjx7OEhASmVCpZr1692GOPPRYVeUqMMfaXv/yFZWZmMoVCwUaNGsUOHDggPjdhwgQ2f/58l/3ff/991rt3b6ZQKNiAAQPY559/3s4WRxb+vH/Lly8X901JSWHTp09nx44d8+t6VLqEIIiIgmJKBEFEFCRKBEFEFCRKBEFEFCRKBEFEFCRKBEFEFCRKBEFEFCRKBEFEFCRKRIciNzcXcXFx4uNnn30WQ4cOFR8vWLAAs2bNane7iOBBokS4ZcGCBR5L5i5btgwcx7ksIA6FGGRlZWHdunUu2+bOnYsLFy54PGb9+vXIzc0VH0+cOBHLly8Pql1EaCFRIjySkZGBbdu2wWg0itsaGxvx7rvvIjMzMyw2xcTEtKgn5Yxer3fxpIiOB4kS4ZHhw4cjIyPDpRDc9u3bkZmZiWHDhrXp3O48mFmzZone18SJE1FQUIBHH31ULEAPtBy+NcfZY1uwYAF2796N9evXi+e4fPkyevXqhVdffdXluOPHj4PjOL9KbBChgUSJ8MqiRYuwadMm8fHGjRuxcOHCkF93+/bt6NatG5577jmUlJSgpKTE73OsX78eY8aMwZIlS8RzZGZmtnhNALBp0yaMHz8evXr1CtZLIAKERInwyn333Yfvv/8eBQUFKCgowN69e3HfffeF/LoJCQmQSqXQarVITU0NqMiaXq+HQqGAWq0WzyGVSrFgwQKcP39ebBVksVjw7rvvYtGiRcF+GUQAUOVJwitJSUmYMWMGcnNzwRjDjBkz0KVLl3Cb1SbS09MxY8YMbNy4EaNGjcKnn34Kk8mEu+66K9ymESBPifCBRYsWITc3F2+99VbQvAmJRILmVXMsFktQzu0LDzzwgBjE37RpE+bOnQu1Wt1u1yc8Q6JEtMrUqVNhNpthsVhw6623BuWcSUlJLnEim82GU6dOueyjUChgs9nadB1P55g+fTpiY2OxYcMGfPnllzR0iyBo+Ea0ilQqFTupSKVSj/sZDAax9rdAYmIiMjIyWux78803Y8WKFfj888/Rs2dPvP7666iurnbZJysrC3v27MHdd98NpVIZ0LAxKysLBw8eRH5+PjQaDRISEiCRSMTY0qpVq5CTkxOVNbgjFfKUCJ/Q6XTQ6XRe9/n2228xbNgwl9uaNWvc7rto0SLMnz8f999/PyZMmIAePXpg0qRJLvs899xzyM/PR8+ePZGUlBSQ3StXroRUKkX//v2RlJSEwsJC8bnFixfDbDa3y2wi4TtUDpeIWr777jtMnjwZRUVFLRouEuGDRImIOkwmE8rLyzF//nykpqZiy5Yt4TaJcIKGb0TUsXXrVnTv3h3V1dX44x//GG5ziGaQp0QQRERBnhJBEBEFiRJBEBEFiRJBEBEFiRJBEBEFiRJBEBEFiRJBEBEFiRJBEBEFiRJBEBEFiRJBEBHF/wd7qD9T5r3nzgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:44:33.503100Z", + "iopub.status.busy": "2024-02-29T18:44:33.502818Z", + "iopub.status.idle": "2024-02-29T18:44:33.729512Z", + "shell.execute_reply": "2024-02-29T18:44:33.728656Z" + }, + "papermill": { + "duration": 0.248914, + "end_time": "2024-02-29T18:44:33.731368", + "exception": false, + "start_time": "2024-02-29T18:44:33.482454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2HklEQVR4nO3deVxU9f4/8BfDMkDDIrJq4IiYaKkkBOLPJGMRSS+puaIsebUsrEQlqQS91iUV1PKa3LqpWZLWdfl6zZAJ3FJChDRUxMQFE0ZElGHJYYDP7w/unOvINqMDw8x5Px8PHno+53POeZ85w5vP2T4fI8YYAyGEGDiBrgMghJDuQMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILJroOoKdrbm5GWVkZrKysYGRkpOtwCCEPYIyhpqYGffr0gUDQcduNkl0nysrK4OrqquswCCEduHHjBp588skO61Cy64SVlRWAlg/T2tpax9F0HYVCgczMTISEhMDU1FTX4ZDHxJfjKZPJ4Orqyv2edoSSXSeUp67W1tYGn+wsLS1hbW1t0L8cfMG346nOJSa6QUEI4QVKdoQQXqDTWIKGhgZs3LgR2dnZuHz5MhYuXAgzMzNdh0WIVlHLjufi4+NhaWmJJUuW4ODBg1iyZAksLS0RHx+v69AI0Sq9S3abNm2CWCyGubk5/Pz8cOrUqXbrfvHFF3j++efRq1cv9OrVC0FBQR3W55v4+HisXbsWD/ffyhjD2rVrKeERg6JXyW7Xrl2Ii4tDUlISCgoKMHz4cIwbNw4VFRVt1j9y5AhmzpyJw4cPIycnB66urggJCcHNmze7OfKep6GhAampqQCA0NBQLFy4ECEhIVi4cCFCQ0MBAKmpqWhoaNBlmIRoD9Mjvr6+7M033+Smm5qaWJ8+fVhycrJayzc2NjIrKyv21Vdfqb3N6upqBoBVV1drHG9PlpqaygAwBwcHZmxszABwP8bGxsze3p4BYKmpqboOlTyChoYGtm/fPtbQ0KDrULqUJr+fenODoqGhAfn5+UhISODKBAIBgoKCkJOTo9Y66uvroVAoYGdn124duVwOuVzOTctkMgAtzy0pFIpHjL7nOXr0KADg9u3bcHR0RFJSEiwtLVFfX4+VK1dyreWjR49i4cKFugyVPALld9WQvrNt0WT/9CbZVVZWoqmpCU5OTirlTk5OuHjxolrrePfdd9GnTx8EBQW1Wyc5ORkrV65sVZ6ZmQlLS0vNgu7B7t69CwAwNzfHZ599BhOTlq9Cr1698Nlnn2H27Nm4f/8+7t69i4MHD+oyVPIYJBKJrkPoUvX19WrX1Ztk97g+/vhj7Ny5E0eOHIG5uXm79RISEhAXF8dNK19HCQkJMag3KH744QccP34cAoEAYWFhYIxBIpEgODgYRkZG3EvVnp6eCAsL03G0RFMKhYI7nob8BoXyzEsdepPs7O3tYWxsjFu3bqmU37p1C87Ozh0um5KSgo8//hg//fQThg0b1mFdoVAIoVDYqtzU1NSgvjTKfamvr4dYLMaKFStgbm6Or776CitWrOD+YhrafvONoR8/TfZNb5KdmZkZvL29kZWVhZdffhlAS/dLWVlZiI2NbXe5NWvW4KOPPsKhQ4fg4+PTTdH2fP369eP+X1FRgTfeeKPTeoToM71JdgAQFxeHqKgo+Pj4wNfXFxs2bEBdXR1iYmIAAJGRkejbty+Sk5MBAKtXr0ZiYiLS09MhFoshlUoBACKRCCKRSGf70ROMGTNGq/UI6en0KtlNnz4dt2/fRmJiIqRSKby8vJCRkcHdtCgtLVXpwG/z5s1oaGjAK6+8orKepKQkrFixojtD73GGDRuGyMhIbN++Hb169YJ/QCCOlwPPuwA5R7Nw9+5dREZGdnraT4i+MGLsocfniQqZTAYbGxtUV1cb1A0Kpfj4eKxfvx6NjY1cmYmJCRYtWoQ1a9boMDLyOBQKBQ4ePIiwsDCDvmanye+nXr1BQbRvzZo1qKurw5Llq2A1YgKWLF+Furo6SnTE4OjVaSzpGmZmZoiYuwDfNzyLiLkjqccTYpCoZUcI4QVKdoQQXqBkRwjhBUp2hBBeoBsUhBiA+vp6lQ4xav+U42RhCXrZn4bIQvX1R09PT4Pq1EJdlOwIMQAXL16Et7d3q/K2HiDKz8/HiBEjuj6oHoaSHSEGwNPTE/n5+dx0cfk9xH1fiHVTh2KQi22runxEyY4QA2BpaanSWhNcvwPh8T8x+Jnh8OrXW4eR9Rx0g4IQwguU7AghvECnsYToqauVdaiTN7Y5r+R2Hfevssv9tjwhNEF/+ye6JL6ehpIdIXroamUdxqYc6bTe4n8Xdlrn8JIXeJHwKNkRooeULboN073g4di6I9q6P+U4cCQHE17wxxMWrYcZAIDLFbV4Z9eZdluHhoaSHSF6zMNRhGf62rQqVygUkDoAI/r1Muj+7DRByY4QPSRvug+B+U1clRVDYN66ZdfY2IiyxjIUVRW1e83uqqwWAvObkDfdB9A6YRoaSnaE6KGyuut4ov9GvHeq43qfZXzW4fwn+gNldV7whlOH9QyB3iW7TZs2Ye3atZBKpRg+fDg2btwIX1/fNuueP38eiYmJyM/Px/Xr17F+/Xq888473RswIV2gzxP9UHd1IT6Z7oUBbVyza2xsxImfT+D/jf5/7bbsSipq8fauM+gzlh8jyOlVstu1axfi4uKQlpYGPz8/bNiwAePGjUNxcTEcHR1b1a+vr4e7uzumTp2KRYsW6SBiQrqG0Ngczff7or/1IAzp3fY1u6smVzHYbnC71+ya71ej+f5tCI3bHzTekOjVQ8Xr1q3DvHnzEBMTgyFDhiAtLQ2WlpbYsmVLm/Wfe+45rF27FjNmzGhz4GtCCH/oTcuuoaEB+fn5SEhI4MoEAgGCgoKQk5Ojte3I5XLI5XJuWiaTAWj5S6lQKLS2nZ5GObpYY2OjQe+noejseCnLOjqWhnDMNYlbb5JdZWUlmpqauDFilZycnFT68XpcycnJWLlyZavyzMxMg+4D7EYtAJjgl19+wc1zuo6GdEZ5vH7++Wdc72C8d4lE8tjr6Mnq6+vVrqs3ya67JCQkIC4ujpuWyWRwdXVFSEiIQY4bq3S2tAooPI2RI0diuJudrsMhnThfJkNK4S8YPXo0nu7T+nupUCggkUgQHBzc7jW7ztahD5RnXurQm2Rnb28PY2Nj3Lp1S6X81q1bcHZ21tp2hEJhm9f3TE1NDfrhTOUdOxMTE4PeT0Oh7vHq6HtrCMdck7j15gaFmZkZvL29kZWVxZU1NzcjKysL/v7+OoyMEKIP9KZlBwBxcXGIioqCj48PfH19sWHDBtTV1SEmJgYAEBkZib59+yI5ORlAy02NCxcucP+/efMmzpw5A5FIBA8PD53tByGk++lVsps+fTpu376NxMRESKVSeHl5ISMjg7tpUVpaCoHgf43VsrIyPPvss9x0SkoKUlJSEBAQgCNHjnR3+IQQHdKrZAcAsbGxiI2NbXPewwlMLBaDMdYNUfV8HfV9BlD/Z8Tw6V2yI5pTt+8zgPo/I4aLkh0PdNb3GUD9nxHDR8mOR9rr+wyg/s+I4aNkR4ge+lPRBAA4d7O6zfl1f8px+jbgfP1uhy11PqFkR4geKvlvolq2p6NrrCb4+nJep+t6QsiPNMCPvSTEwIQ83fLW0ABHESxMjVvNLy6vxuJ/FyL1laEY5NJ+L8R8urtOyY4HOuvCG6BuvPWN3RNmmOHr1u58ZY8mAxyeaPc6Ld9QsuMBdbvwBqgbb2K4KNnxQGddeAPUjTcxfJTseKCzLrwB6sabGD696fWEEEIeByU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPCC3iW7TZs2QSwWw9zcHH5+fjh1quMXPr///nt4enrC3NwcQ4cOxcGDB7spUkJIT6JXr4vt2rULcXFxSEtLg5+fHzZs2IBx48ahuLgYjo6OreqfPHkSM2fORHJyMiZMmID09HS8/PLLKCgowDPPPKODPdCNzjp6BKizR2L4jJgeDb/l5+eH5557Dv/4xz8AtAyS7erqioULF2LZsmWt6k+fPh11dXU4cOAAVzZy5Eh4eXkhLS1NrW3KZDLY2Niguroa1tbW2tmRbrbzVGknnTxqhgbc6Xnq6+tx8eJFbrq4/B7ivi/EuqlDMcjFVqWup6cnLC0tuznCrqHJ76fetOwaGhqQn5+PhIQErkwgECAoKAg5OTltLpOTk4O4uDiVsnHjxmHfvn3tbkcul0Mul3PTMpkMQMuL8gqF4jH2QHfGPtUbH4UPgbvDE2129AgAl6TViN9bhDWTBuMp5446ezTGkzZmevtZGKpz587Bz8+vVfmsr1rXzc3NVRlPWZ9p8j3Um2RXWVmJpqYmbkBsJScnJ5W/aA+SSqVt1pdKpe1uJzk5GStXrmxVnpmZqdd/DUUAKiran99yhmqCisuFELb/8QAALmgxLqIdcrkcqamp3LSiGai6D9iZA6YPXZm/du0aysvLuznCrlFfX692Xb1Jdt0lISFBpTUok8ng6uqKkJAQvT2NVcfZ0iqg8DRGjhyJ4W52ug6HPCaFQgGJRILg4GCDHi1OeealDr1Jdvb29jA2NsatW7dUym/dugVnZ+c2l3F2dtaoPgAIhUIIha0v0Juamhr0l0bZYaeJiYlB7yffGPr3VpN905tHT8zMzODt7Y2srCyurLm5GVlZWfD3929zGX9/f5X6ACCRSNqtTwgxXHrTsgOAuLg4REVFwcfHB76+vtiwYQPq6uoQExMDAIiMjETfvn2RnJwMAHj77bcREBCA1NRUvPTSS9i5cydOnz6Nzz//XJe70SO0dfdOLr2MonMWaL5jy5Ub0p07wm96leymT5+O27dvIzExEVKpFF5eXsjIyOBuQpSWlkIg+F9jddSoUUhPT8cHH3yA9957DwMHDsS+fft49Yxdey5evAhvb+9W5Q/fvcvPz8eIESO6KSpCuo5ePWenC4bwnF1bHm7Z1f4pxw+Hc/DSWH+IHniomFp2+kmhUODgwYMICwsz6Gt2BvmcHdEuS0tLlRabQqHA3coK+Pv6GPQvB+EvvblBQQghj4OSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBc0TnZXrlzpijgIIaRLaZzsPDw8MHbsWHzzzTe4f/9+V8RECCFap3GyKygowLBhwxAXFwdnZ2e89tprnY4DQQghuqZxsvPy8sInn3yCsrIybNmyBeXl5Rg9ejSeeeYZrFu3Drdv3+6KOAkh5LE88g0KExMTTJ48Gd9//z1Wr16Ny5cvY8mSJXB1dUVkZKTB9IRKCDEMj5zsTp8+jTfeeAMuLi5Yt24dlixZgpKSEkgkEpSVlSE8PFybcRJCyGPRuCOAdevWYevWrSguLkZYWBi2b9+OsLAwrmul/v37Y9u2bRCLxdqOlRBCHpnGyW7z5s149dVXER0dDRcXlzbrODo64ssvv3zs4AghRFs0TnYSiQRubm4qnWQCAGMMN27cgJubG8zMzBAVFaW1IAkh5HFpfM1uwIABqKysbFVeVVWF/v37ayUoQgjRNo2TXXsdG9fW1sLc3PyxAyKEkK6g9mmscixVIyMjJCYmqnTV3dTUhNzcXHh5eWk9QKWqqiosXLgQ//nPfyAQCDBlyhR88sknEIlE7S7z+eefIz09HQUFBaipqcHdu3dha2vbZTESQnoutZPdr7/+CqClZVdYWAgzMzNunpmZGYYPH44lS5ZoP8L/ioiIQHl5OSQSCRQKBWJiYjB//nykp6e3u0x9fT1CQ0MRGhqKhISELouNENLzqZ3sDh8+DACIiYnBJ5980q2DzxQVFSEjIwN5eXnw8fEBAGzcuBFhYWFISUlBnz592lzunXfeAQAcOXKkmyIlhPRUGt+N3bp1a1fE0aGcnBzY2tpyiQ4AgoKCIBAIkJubi0mTJmltW3K5HHK5nJuWyWQAWgakUSgUWttOT6PcN0PeRz7hy/HUZP/USnaTJ0/Gtm3bYG1tjcmTJ3dYd8+ePWpvXF1SqRSOjo4qZSYmJrCzs4NUKtXqtpKTk7Fy5cpW5ZmZmbwYUlAikeg6BKJFhn486+vr1a6rVrKzsbGBkZER939tWbZsGVavXt1hnaKiIq1tTx0JCQnczRigpWXn6uqKkJAQgxo39mEKhQISiQTBwcE0lKIB4MvxVJ55qUOtZPfgqas2T2MXL16M6OjoDuu4u7vD2dkZFRUVKuWNjY2oqqqCs7Oz1uIBAKFQCKFQ2Krc1NTUoL80SnzZT74w9OOpyb7pdJBsBwcHODg4dFrP398f9+7dQ35+Pry9vQEA2dnZaG5uhp+fX1eHSQgxAGolu2effZY7je1MQUHBYwXUlsGDByM0NBTz5s1DWloaFAoFYmNjMWPGDO5O7M2bNxEYGIjt27fD19cXQMu1PqlUisuXLwMACgsLYWVlBTc3N9jZ2Wk9TkJIz6VWsnv55Ze7OIzO7dixA7GxsQgMDOQeKv7000+5+QqFAsXFxSoXLNPS0lRuNowZMwZAy6l4Z6fPhOir2tpazJo1C7/99hu+/PJLpKend/jwPV8Ysfbe/yIAWi6A2tjYoLq62uBvUBw8eBBhYWEGfY3H0Pn6+iIvL69V+XPPPWeQwydo8vtJQykSYiDaS3QAkJeXx13e4Su1TmPt7Oxw6dIl2Nvbo1evXh1ev6uqqtJacIQQ9dTW1rab6JTy8vJQW1vL21NatZLd+vXrYWVlxf1f3ZsVhJDuMWXKFLXrHTp0qIuj6Znoml0n6Jod0QcCgYDrfs3e3h4ffvghhEIh5HI5PvjgA64PSiMjIzQ3N+syVK3q0mt2xsbGrR7wBYA7d+7A2NhY09URQrTgwTZLeXk5Xn31VfTq1Quvvvqqykh/fG7baK3zTrlcrtLtEyFENxobGzuc5iu136BQPtNmZGSEf/3rXyoXOZuamnDs2DF4enpqP0JCSKesrKxQU1MDALCwsMCsWbPg7e2N6OholT4fldfe+UjtZLd+/XoALS27tLQ0lVNWMzMziMVipKWlaT9CQkin4uPjsXz5cm46PT29zY5t4+PjuzOsHkXjGxRjx47Fnj170KtXr66KqUehGxREHzQ0NLTZgcXDDO1yU5feoDh8+DBvEh0h+sLMzAxLly7tsM7SpUsNKtFpSuNeT1599dUO52/ZsuWRgyGEPLo1a9YAAFJSUlRuJAoEAixevJibz1caJ7u7d++qTCsUCpw7dw737t3Diy++qLXACCGaW7NmDT788ENs3LgR2dnZePHFF7Fw4UJet+iUNE52e/fubVXW3NyMBQsWYMCAAVoJihDy6MzMzPDWW2/Bw8ODrsE+QCsdAQgEAsTFxXF3bAkhpKfRWq8nJSUl9PAiIaTH0vg09sHBaICW5+7Ky8vxww8/ICoqSmuBEUKINmmc7H799VeVaYFAAAcHB6SmpnZ6p5YQQnRF42R3+PDhroiDEEK6FPVUTAjhBb1JdlVVVYiIiIC1tTVsbW0xd+5c1NbWdlh/4cKFGDRoECwsLODm5oa33noL1dXV3Rg1IaSn0JtkFxERgfPnz0MikeDAgQM4duwY5s+f3279srIylJWVISUlBefOncO2bduQkZGBuXPndmPUhJCeQqeDZKurqKgIGRkZyMvLg4+PDwBg48aNCAsLQ0pKCjd27IOeeeYZ7N69m5seMGAAPvroI8yePRuNjY0wMdGLXSeEaInWfuP/+OMP/O1vf8Pnn3+urVVycnJyYGtryyU6AAgKCoJAIEBubi4mTZqk1nqUPSN0lOjkcjnkcjk3LZPJALS8FqdQKB5xD3o+5b4Z8j7yCV+Opyb7p7Vkd+fOHXz55ZddkuykUikcHR1VykxMTGBnZwepVKrWOiorK7Fq1aoOT30BIDk5WWVgbaXMzExYWlqqH7Sekkgkug6BPKampiZcuHABd+/eRWFhIYYMGWKwQybU19erXVen53LLli3D6tWrO6xTVFT02NuRyWR46aWXMGTIEKxYsaLDugkJCSoPTstkMri6uiIkJMTg+7OTSCQIDg6mdyn12N69e7F06VKUlpZyZW5ubli7dq3aZ0D6RHnmpQ6dJrvFixcjOjq6wzru7u5wdnZuNchPY2Mjqqqq4Ozs3OHyNTU1CA0NhZWVFfbu3dvpL7JQKGyzE0RTU1NeJAG+7Kch2rNnD6ZPn96qvLS0FNOnT8fu3bsxefJkHUTWdTT5ruo02Tk4OMDBwaHTev7+/rh37x7y8/Ph7e0NAMjOzkZzczP8/PzaXU4mk2HcuHEQCoXYv38/zM3NtRY7IT1JU1MTYmJiOqwTExOD8PBwgz2l7Yzaya6zvwj37t173FjaNXjwYISGhmLevHlIS0uDQqFAbGwsZsyYwd2JvXnzJgIDA7F9+3b4+vpCJpMhJCQE9fX1+OabbyCTybgmr4ODA28PODFMWVlZnZ7SyWQyZGVlISQkpJui6lnUTnY2Njadzo+MjHzsgNqzY8cOxMbGIjAwEAKBAFOmTOFGPANarjkVFxdzFywLCgqQm5sLAPDw8FBZ19WrVyEWi7ssVkK627Zt29SuR8muE1u3bu3KODplZ2fX5mhJSmKxWKUr6hdeeIHXAwITfikoKNBqPUNET9YSYgCuX7/O/d/U1BSjR49Gc3MzBAIBfv75Z+55tAfr8Y3ayU7d7ptowB1Cul9DQwP3f4VC0W7vRA/W4xu1k922bdvQr18/PPvss3R6SAjRO2onuwULFuDbb7/F1atXERMTg9mzZ8POzq4rYyOEqMnR0VGtt4kefhOJT9Tu9WTTpk0oLy9HfHw8/vOf/8DV1RXTpk3DoUOHqKVHiI7169dPq/UMkUZdPAmFQsycORMSiQQXLlzA008/jTfeeANisbjDvuUIIV1LnYfzNalniB65PzuBQAAjIyMwxtDU1KTNmAghGrK1tdVqPUOkUbKTy+X49ttvERwcjKeeegqFhYX4xz/+gdLSUohEoq6KkRDSiTlz5gBo/11RZbmyHh+pfYPijTfewM6dO+Hq6opXX30V3377Lezt7bsyNkKImgIDA2FtbQ2ZTAZHR0d4enqisrIS9vb2uHjxIioqKmBtbY3AwEBdh6ozaie7tLQ0uLm5wd3dHUePHsXRo0fbrLdnzx6tBUcIUY+xsTG2bt2KKVOm4Pbt2yq9BBkZGQFoeQuKz++Eq53sIiMjuQ+NENLzTJ48Gbt378aiRYta9We3bt06g+veSVMaPVRMCOnZJk+ejPDwcBw+fBg//vgjxo8fj7Fjx/K6RadE78YSYmCMjY0REBCAuro6BAQEUKL7L70ZSpEQQh4HJTtCCC9QsiOE8AIlO0IIL1CyI4Twgt4ku6qqKkRERMDa2hq2traYO3dup50PvPbaaxgwYAAsLCzg4OCA8PBwXLx4sZsiJoT0JHqT7CIiInD+/HlIJBIcOHAAx44dw/z58ztcxtvbG1u3bkVRURHXFVVISAh1XEAID+nFc3ZFRUXIyMhAXl4efHx8AAAbN25EWFgYUlJSuOEUH/ZgMhSLxfjwww8xfPhwXLt2DQMGDOiW2AkhPYNeJLucnBzY2tpyiQ4AgoKCIBAIkJubi0mTJnW6jrq6OmzduhX9+/eHq6tru/Xkcjnkcjk3rRyLU6FQcIOWGCLlvhnyPvIJX46nJvunF8lOKpW26k7axMQEdnZ2nXZF/dlnnyE+Ph51dXUYNGgQJBIJzMzM2q2fnJyMlStXtirPzMyEpaXlo+2AHpFIJLoOgWiRoR9P5TjR6tBpslu2bBlWr17dYZ2ioqLH2kZERASCg4NRXl6OlJQUTJs2DSdOnIC5uXmb9RMSEhAXF8dNy2QyuLq6IiQkBNbW1o8VS0+mUCggkUgQHBzcbp9oRH/w5Xgqz7zUodNkt3jxYkRHR3dYx93dHc7Ozipd1gBAY2Mjqqqq4Ozs3OHyNjY2sLGxwcCBAzFy5Ej06tULe/fuxcyZM9usLxQKIRQKW5Wbmpoa9JdGiS/7yReGfjw12TedJjsHBwe1+sT39/fHvXv3kJ+fD29vbwBAdnY2mpub4efnp/b2GGNgjKlckyOE8INePHoyePBghIaGYt68eTh16hROnDiB2NhYzJgxg7sTe/PmTXh6euLUqVMAgCtXriA5ORn5+fkoLS3FyZMnMXXqVFhYWCAsLEyXu0MI0QG9SHYAsGPHDnh6eiIwMBBhYWEYPXo0Pv/8c26+QqFAcXExd8HS3Nwcx48fR1hYGDw8PDB9+nRYWVnh5MmTvB47kxC+0ou7sQBgZ2eH9PT0dueLxWKV8Wv79OmDgwcPdkdohBA9oDctO0IIeRyU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4Qwgt6k+yqqqoQEREBa2tr2NraYu7cuaitrVVrWcYYxo8fDyMjI+zbt69rAyWE9Eh6k+wiIiJw/vx5SCQSHDhwAMeOHcP8+fPVWnbDhg0wMjLq4ggJIT2ZXowuVlRUhIyMDOTl5cHHxwcAsHHjRoSFhSElJYUbO7YtZ86cQWpqKk6fPg0XF5fuCpkQ0sPoRbLLycmBra0tl+gAICgoCAKBALm5uZg0aVKby9XX12PWrFnYtGkTnJ2d1dqWXC6HXC7npmUyGYCWcWkVCsVj7EXPptw3Q95HPuHL8dRk//Qi2Uml0lYDW5uYmMDOzg5SqbTd5RYtWoRRo0YhPDxc7W0lJydj5cqVrcozMzNhaWmpftB6SiKR6DoEokWGfjzr6+vVrqvTZLds2TKsXr26wzpFRUWPtO79+/cjOzsbv/76q0bLJSQkIC4ujpuWyWRwdXVFSEgIrK2tHykWfaBQKCCRSBAcHAxTU1Ndh0MeE1+Op/LMSx06TXaLFy9GdHR0h3Xc3d3h7OyMiooKlfLGxkZUVVW1e3qanZ2NkpIS2NraqpRPmTIFzz//PI4cOdLmckKhEEKhsFW5qampQX9plPiyn3xh6MdTk33TabJzcHCAg4NDp/X8/f1x79495Ofnw9vbG0BLMmtuboafn1+byyxbtgx//etfVcqGDh2K9evXY+LEiY8fPCFEr+jFNbvBgwcjNDQU8+bNQ1paGhQKBWJjYzFjxgzuTuzNmzcRGBiI7du3w9fXF87Ozm22+tzc3NC/f//u3gVCiI7pzXN2O3bsgKenJwIDAxEWFobRo0fj888/5+YrFAoUFxdrdMGSEMIfetGyAwA7Ozukp6e3O18sFoMx1uE6OptPCDFcetOyI4SQx0HJjhDCC5TsCDEwTU1NOHr0KI4dO4ajR4+iqalJ1yH1CJTsCDEge/bsgYeHB4KDg7Fu3ToEBwfDw8MDe/bs0XVoOkfJjhADsWfPHrzyyisYOnQojh8/jm+//RbHjx/H0KFD8corr/A+4VGyI8QANDU1YfHixZgwYQL27dsHPz8/WFhYwM/PD/v27cOECROwZMkSXp/SUrIjxAAcP34c165dw3vvvQfGmMo1O8YYEhIScPXqVRw/flzXoeoMJTtCDEB5eTkAoKSkBO7u7irX7Nzd3XHlyhWVenykNw8VE0Lap+yYdvbs2a165b5x4wZmz56tUo+PKNkRYgBGjRoFIyMjMMbg4OCAv/3tbxAKhZDL5UhMTERFRQWMjIwwatQoXYeqM3QaS4gBOHLkCPc65IgRI5CRkYF169YhIyMDI0aMANDyumR7XZvxAbXsCDEAX3/9NYCWXn0yMjK48sLCQq68tLQUX3/9NUJCQnQSo65Ry44QA6AcVrS0tLTN+cpydYcfNUSU7AgxAL6+vlqtZ4go2RFiAEpKSrRazxBRsiPEABw9elSr9QwRJTtCDEBlZSUAwMjICMbGxirzTExMuGfvlPX4iJIdIQbAxKTlwQrGGDcui5KLiwv3WIqyHh/pTbKrqqpCREQErK2tYWtri7lz53Z6Z+mFF16AkZGRys/rr7/eTRET0n0eHET+jz/+wMCBA7mfP/74o816fKM3yS4iIgLnz5+HRCLBgQMHcOzYMcyfP7/T5ebNm4fy8nLuZ82aNd0QLSHdS/k6GNDSuvv999+5nwfHXnmwHt/oRZu2qKgIGRkZyMvLg4+PDwBg48aNCAsLQ0pKSqtm+4MsLS3bHUibEEOhHE9ZW/UMkV4ku5ycHNja2nKJDgCCgoIgEAiQm5uLSZMmtbvsjh078M0338DZ2RkTJ07E8uXLYWlp2W59uVwOuVzOTctkMgAtQzUqFAot7E3PpNw3Q95HQ3bjxg216xnSMdZkX/Qi2Uml0lbXGkxMTGBnZwepVNrucrNmzUK/fv3Qp08f/Pbbb3j33XdRXFzcYY+tycnJWLlyZavyzMzMDpOkoZBIJLoOgTyCf/7zn2rXs7e37+Jouo8m40TrNNktW7YMq1ev7rBOUVHRI6//wWt6Q4cOhYuLCwIDA1FSUoIBAwa0uUxCQgLi4uK4aZlMBldXV4SEhMDa2vqRY+npFAoFJBIJgoODYWpqqutwiIaUf6Ctra1RWlqKf/7znzh69CgCAgLw2muvwdXVFTU1NWhsbERYWJiOo9Ue5ZmXOnSa7BYvXozo6OgO67i7u8PZ2RkVFRUq5Y2NjaiqqtLoepyfnx8A4PLly+0mO6FQCKFQ2Krc1NSUF0mAL/tpaJTP1slkMsyZMwdLly6Fm5sb+vbtizlz5qCmpoarZ0jHV5N90Wmyc3BwgIODQ6f1/P39ce/ePeTn53MXWLOzs9Hc3MwlMHWcOXMGAL87MCSGKTAwEKdPn4axsTF+++03jBkzhpsnFothbGyMpqYmBAYG6jBK3dKLR08GDx6M0NBQzJs3D6dOncKJEycQGxuLGTNmcHdib968CU9PT5w6dQpAyzuAq1atQn5+Pq5du4b9+/cjMjISY8aMwbBhw3S5O4RoXXBwMICWgXfKysowbdo0REdHY9q0abh58yY30I6yHi8xPXHnzh02c+ZMJhKJmLW1NYuJiWE1NTXc/KtXrzIA7PDhw4wxxkpLS9mYMWOYnZ0dEwqFzMPDgy1dupRVV1drtN3q6moGQOPl9E1DQwPbt28fa2ho0HUo5BE0NjYyBwcHBqDdH0dHR9bY2KjrULVKk99PvbgbCwB2dnZIT09vd75YLFZ5eNLV1ZXXLz0TfjE2NkZaWhqmTJkCCwsL/Pnnn9w85fTmzZtbvTfLJ3pxGksI6dzkyZOxe/fuVo9pOTk5Yffu3Zg8ebKOIusZ9KZlRwjp3OTJkxEeHo7Dhw/jxx9/xPjx4zF27Fhet+iUKNkRYmCMjY0REBCAuro6BAQEUKL7LzqNJYTwAiU7QggvULIjhPACXbPrhPJxFk3ewdNHCoUC9fX1kMlkBvU6EV/x5Xgqfy8ffOysPZTsOqF8p9DV1VXHkRBC2lNTUwMbG5sO6xgxdVIijzU3N6OsrAxWVlbcoCWGSNm7y40bNwy6dxe+4MvxZIyhpqYGffr0gUDQ8VU5atl1QiAQ4Mknn9R1GN3G2traoH85+IYPx7OzFp0S3aAghPACJTtCCC9QsiMAWjotTUpKarPjUqJ/6Hi2RjcoCCG8QC07QggvULIjhPACJTtCCC9QstOy6OhovPzyy1pd5wsvvIB33nmnwzpisRgbNmzQ6nYJMSSU7DqgTpIh+mXFihXw8vLSdRht6mnft54Wz+OiZEeIAWloaNB1CD0WJbt2REdH4+jRo/jkk09gZGQEIyMjlJSUYO7cuejfvz8sLCwwaNAgfPLJJ20uv3LlSjg4OMDa2hqvv/662l/Curo6REZGQiQSwcXFBampqa3qVFRUYOLEibCwsED//v2xY8eOVnWMjIywefNmjB8/HhYWFnB3d8e///1vbv61a9dgZGSE7777Ds8//zwsLCzw3HPP4dKlS8jLy4OPjw9EIhHGjx+P27dvq/mpAVu2bMHTTz8NoVAIFxcXxMbGcvNKS0sRHh4OkUgEa2trTJs2Dbdu3eLmK1tdX3/9NcRiMWxsbDBjxgyuMwag5V3lNWvWwMPDA0KhEG5ubvjoo4+4+e+++y6eeuopWFpawt3dHcuXL4dCoQAAbNu2DStXrsTZs2e5Y7pt2za1960rPer3TXnZ5KOPPkKfPn0waNAgAMDJkyfh5eUFc3Nz+Pj4YN++fTAyMuLGTgaAc+fOYfz48RCJRHBycsKcOXNQWVnZbjzXrl3rro+ja3ThKGd67d69e8zf35/NmzePlZeXs/Lycnb//n2WmJjI8vLy2JUrV9g333zDLC0t2a5du7jloqKimEgkYtOnT2fnzp1jBw4cYA4ODuy9995Ta7sLFixgbm5u7KeffmK//fYbmzBhArOysmJvv/02V2f8+PFs+PDhLCcnh50+fZqNGjWKWVhYsPXr13N1ALDevXuzL774ghUXF7MPPviAGRsbswsXLjDG/jf0pKenJ8vIyGAXLlxgI0eOZN7e3uyFF15gP//8MysoKGAeHh7s9ddfVyv2zz77jJmbm7MNGzaw4uJidurUKS6mpqYm5uXlxUaPHs1Onz7NfvnlF+bt7c0CAgK45ZOSkphIJGKTJ09mhYWF7NixY8zZ2Vnls4uPj2e9evVi27ZtY5cvX2bHjx9nX3zxBTd/1apV7MSJE+zq1ats//79zMnJia1evZoxxlh9fT1bvHgxe/rpp7ljWl9fr9a+dbXH/b7NmTOHnTt3jp07d45VV1czOzs7Nnv2bHb+/Hl28OBB9tRTTzEA7Ndff2WMMXb37l3m4ODAEhISWFFRESsoKGDBwcFs7Nix7caj78MwUrLrQEBAgEqSacubb77JpkyZwk1HRUUxOzs7VldXx5Vt3ryZiUQi1tTU1OG6ampqmJmZGfvuu++4sjt37jALCwsujuLiYgaAnTp1iqtTVFTEALRKdg8nKT8/P7ZgwQLG2P+S3b/+9S9u/rfffssAsKysLK4sOTmZDRo0qMO4lfr06cPef//9NudlZmYyY2NjVlpaypWdP39eZV+SkpKYpaUlk8lkXJ2lS5cyPz8/xhhjMpmMCYVCleTWmbVr1zJvb29uOikpiQ0fPlzt5bvTo37fnJycmFwu58o2b97Mevfuzf7880+u7IsvvlBJdqtWrWIhISEq675x4wYDwIqLi9WOR59Qryca2rRpE7Zs2YLS0lL8+eefaGhoaHXBe/jw4bC0tOSm/f39UVtbixs3bqBfv37trrukpAQNDQ3w8/Pjyuzs7LhTEwAoKiqCiYkJvL29uTJPT0/Y2tq2Wp+/v3+r6QdPYwBg2LBh3P+dnJwAAEOHDlUpq6ioaDdmpYqKCpSVlSEwMLDN+UVFRXB1dVXpF3DIkCGwtbVFUVERnnvuOQAtd5WtrKy4Oi4uLtz2i4qKIJfL290GAOzatQuffvopSkpKUFtbi8bGRr3u9UOd79vQoUNhZmbGTRcXF2PYsGEwNzfnynx9fVWWOXv2LA4fPgyRSNRqmyUlJXjqqae0uyM9AF2z08DOnTuxZMkSzJ07F5mZmThz5gxiYmL0+qLwg73YKvvre7isubm50/VYWFhoPZ6Ht9/ZNnJychAREYGwsDAcOHAAv/76K95//329PT7qft+eeOIJjdddW1uLiRMn4syZMyo/v//+O8aMGaOtXehRKNl1wMzMDE1NTdz0iRMnMGrUKLzxxht49tln4eHhgZKSklbLnT17VmVE9l9++QUikajT3o4HDBgAU1NT5ObmcmV3797FpUuXuGlPT080NjYiPz+fKysuLsa9e/dare+XX35pNT148OAOY3hUVlZWEIvFyMrKanP+4MGDcePGDdy4cYMru3DhAu7du4chQ4aotY2BAwfCwsKi3W2cPHkS/fr1w/vvvw8fHx8MHDgQ169fV6nz8DHtSR71+/awQYMGobCwEHK5nCvLy8tTqTNixAicP38eYrEYHh4eKj/K5NmTP6tHQcmuA2KxGLm5ubh27RoqKysxcOBAnD59GocOHcKlS5ewfPnyVl8ioOX2/9y5c3HhwgUcPHgQSUlJiI2N7bQnVZFIhLlz52Lp0qXIzs7GuXPnEB0drbLcoEGDEBoaitdeew25ubnIz8/HX//61zZbPd9//z22bNmCS5cuISkpCadOnVK5O6ptK1asQGpqKj799FP8/vvvKCgowMaNGwEAQUFBGDp0KCIiIlBQUIBTp04hMjISAQEB8PHxUWv95ubmePfddxEfH4/t27ejpKQEv/zyC7788ksALcmwtLQUO3fuRElJCT799FPs3btXZR1isRhXr17FmTNnUFlZqZIQdO1Rv28PmzVrFpqbmzF//nwUFRXh0KFDSElJAfC/1vubb76JqqoqzJw5E3l5eSgpKcGhQ4cQExPDJbiH41Gnhd+j6fqiYU9WXFzMRo4cySwsLBgAdvHiRRYdHc1sbGyYra0tW7BgAVu2bJnKBe+oqCgWHh7OEhMTWe/evZlIJGLz5s1j9+/fV2ubNTU1bPbs2czS0pI5OTmxNWvWtLpQXF5ezl566SUmFAqZm5sb2759O+vXr1+rGxSbNm1iwcHBTCgUMrFYrHIXT3mDQnnBmjHGDh8+zACwu3fvcmVbt25lNjY2an9maWlpbNCgQczU1JS5uLiwhQsXcvOuX7/O/vKXv7AnnniCWVlZsalTpzKpVMrNb+vmwfr161m/fv246aamJvbhhx+yfv36MVNTU+bm5sb+/ve/c/OXLl3Kfe7Tp09n69evV4n//v37bMqUKczW1pYBYFu3blV737ra43zfHnbixAk2bNgwZmZmxry9vVl6ejq3TqVLly6xSZMmMVtbW2ZhYcE8PT3ZO++8w5qbm9uM5+rVq138CXQt6uLJQBkZGWHv3r1af3WN6KcdO3YgJiYG1dXVWru+qm/obiwhBmj79u1wd3dH3759cfbsWbz77ruYNm0abxMdQMmuW5WWlnZ4Mf7ChQtwc3Prxog009ZjCko//vgjnn/++W6MhnREKpUiMTERUqkULi4umDp1qsqbJnxEp7HdqLGxscNXbsRiMUxMeu7fn8uXL7c7r2/fvrxuNZCej5IdIYQX6NETQggvULIjhPACJTtCCC9QsiOE8AIlO9JjREdHcx1FmpqawsnJCcHBwdiyZYtGrypt27atzV5gulpXjD9CtIeSHelRQkNDUV5ejmvXruHHH3/E2LFj8fbbb2PChAlobGzUdXhEn+nyXTVCHtTee55ZWVkMANdpZ2pqKnvmmWeYpaUle/LJJ9mCBQtYTU0NY+x/7/c++JOUlMQYY2z79u3M29ubiUQi5uTkxGbOnMlu3brFbaeqqorNmjWL2dvbM3Nzc+bh4cG2bNnCzS8tLWVTp05lNjY2rFevXuwvf/kL975oUlJSq+0ePny4Sz4n8mioZUd6vBdffBHDhw/Hnj17AAACgQCffvopzp8/j6+++grZ2dmIj48HAIwaNQobNmyAtbU1ysvLUV5ejiVLlgAAFAoFVq1ahbNnz2Lfvn24du0aoqOjue0sX74cFy5cwI8//oiioiJs3rwZ9vb23LLjxo2DlZUVjh8/jhMnTkAkEiE0NBQNDQ1YsmQJpk2bxrVMy8vLMWrUqO79oEjHdJ1tCVFqr2XHGGPTp09ngwcPbnPe999/z3r37s1Nq9tTS15eHgPAtQonTpzIYmJi2qz79ddfs0GDBnE9gjDGmFwuZxYWFuzQoUOdxk90j1p2RC8wxri+2H766ScEBgaib9++sLKywpw5c3Dnzh3U19d3uI78/HxMnDgRbm5usLKyQkBAAICWd5YBYMGCBdi5cye8vLwQHx+PkydPcsuePXsWly9fhpWVFUQiEUQiEezs7HD//n21OtQkukfJjuiFoqIi9O/fH9euXcOECRMwbNgw7N69G/n5+di0aROAjsdMraurw7hx42BtbY0dO3YgLy+P69hTudz48eNx/fp1LFq0iBtPQ3kKXFtbC29v71bdmF+6dAmzZs3q4r0n2tBz3zon5L+ys7NRWFiIRYsWIT8/H83NzUhNTeV6cP7uu+9U6rfVnfjFixdx584dfPzxx1z3+KdPn261LQcHB0RFRSEqKgrPP/88li5dipSUFIwYMQK7du2Co6NjuwP4GFo35oaGWnakR5HL5ZBKpbh58yYKCgrw97//HeHh4ZgwYQIiIyPh4eEBhUKBjRs34sqVK/j666+Rlpamsg6xWIza2lpkZWWhsrIS9fX1cHNzg5mZGbfc/v37sWrVKpXlEhMT8X//93+4fPkyzp8/jwMHDnBjdkRERMDe3h7h4eE4fvw4rl69iiNHjuCtt97CH3/8wW33t99+Q3FxMSorK7nBuUkPoeuLhoQoRUVFcY9tmJiYMAcHBxYUFMS2bNmiMubuunXrmIuLC7OwsGDjxo1j27dvb9Wd/Ouvv8569+6t8uhJeno6E4vFTCgUMn9/f7Z///5WY6kOHjyYWVhYMDs7OxYeHs6uXLnCrbO8vJxFRkYye3t7JhQKmbu7O5s3bx6rrq5mjDFWUVHBgoODmUgkokdPeiDq4okQwgt0GksI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF74/+XDPxzUPy9hAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:44:33.772296Z", + "iopub.status.busy": "2024-02-29T18:44:33.772018Z", + "iopub.status.idle": "2024-02-29T18:44:34.042778Z", + "shell.execute_reply": "2024-02-29T18:44:34.041888Z" + }, + "papermill": { + "duration": 0.293553, + "end_time": "2024-02-29T18:44:34.044775", + "exception": false, + "start_time": "2024-02-29T18:44:33.751222", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAEmCAYAAAAEH9kkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABOl0lEQVR4nO2deVxU5f7HP8PAMOyoKIuyqSQupF4UwiyzKOzaVbpdt7yKRuktNY3MXcil8LrknrS51C9D7ZaZFfciV+2qiIkbKi4giIqDC8IAAgMzz+8PnCOHmWHmwMAc4Pt+vc4LznO+z3Oec4b58H227yNhjDEQBEEQBrGydAUIgiDEDgklQRCEEUgoCYIgjEBCSRAEYQQSSoIgCCOQUBIEQRiBhJIgCMIIJJQEQRBGsLZ0BVozGo0G+fn5cHJygkQisXR1CIKoBWMMJSUl8PLygpVV/T4jCWUTkp+fD29vb0tXgyCIerhx4wa6dOlSrw0JZRPi5OQEoOaDcHZ2tnBtCIKojVKphLe3N/c9rQ8SyiZE29x2dnYmoSQIkWJKtxgN5hAEQRiBhJIgCMIIJJQEQRBGoD5KEcIYQ3V1NdRqtaWrQhAtFqlUCmtra7NMzSOhFBkqlQq3b9/Gw4cPLV0Vgmjx2Nvbw9PTEzKZrFHlkFCKCI1Gg5ycHEilUnh5eUEmk9FEdYJoAIwxqFQq3L17Fzk5OQgICDA6qbw+SChFhEqlgkajgbe3N+zt7S1dHYuQX1QOCQBPVztLV4Vo4djZ2cHGxgbXr1+HSqWCXC5vcFk0mCNCGvOfryVTrdbgXmkl7pZWQq3RWLo6RCvAXN+ltvmNJEQJM/A7QVgaEkpCPJA6EiKFhJKwGJMmTUJkZCR3bg6dfO655zBr1qx6bfz8/LBu3Toz3I2wNKZ83uaAhJIwC2b/gyXv0uJ8+OGH6Nevn6WrIQpIKAmCsChVVVWWroJRSChFDmMMD1XVFjkYM82tmzRpEg4fPoz169dDIpFAIpEgOzsb0dHR8Pf3h52dHXr06IH169frzb9kyRJ07NgRbu1dsWz+e6hSqUxyKMvKyjBx4kQ4OjrC09MTa9as0bG5c+cO/vKXv8DOzg7+/v749ttvdWwkEgm2bNmCl19+GXZ2dujatSu+//577npubi4kEgl2796NZ555BnZ2dhg4cCCuXLmCP/74AwMGDICjoyNefvll3L1716R3BgBbt25F7969YWtrC09PT0yfPp27lpeXh5EjR8LR0RHOzs4YPXo0CgoKuOtab++bb76Bn58fXFxcMHbsWJSUlHA2Go0GK1euRPfu3WFrawsfHx989NFH3PW5c+fiiSeegL29Pbp27YrFixdzorV9+3YsWbIEZ8+e5T7T7du3G32mS5cuYfDgwZDL5ejVqxcOHDgAiUSCvXv38t7lrl27MGTIEMjlcnz77be4f/8+xo0bh86dO8Pe3h5BQUH47rvveGWb8nk3FTSPUuSUV6nRK/bfFrn3xaURsJcZ/xNZv349rly5gj59+mDp0qUAgHbt2qFLly7Ys2cPOnTogGPHjmHKlCnw9PTE6NGjubwpKSmQy+U4dOgQrmZn4403ouHq2h6frltl9L4ffPABDh8+jJ9++gmdOnXCggULcOrUKV5zcdKkScjPz8fBgwdhY2ODd999F3fu3NEpa/HixVixYgXWr1+Pb775BmPHjkVGRgZ69uzJ2cTFxWHdunXw8fHBG2+8gddffx1OTk5Yv3497O3tMXr0aMTGxmLLli1G675lyxbExMRgxYoVePnll1FcXIyjR48CqBE4rUgePnwY1dXVmDZtGsaMGYNDhw5xZWRnZ2Pv3r3Yv38/Hjx4gNGjR2PFihWcGM6fPx9ffPEF1q5di8GDB+P27du4dOkSl9/JyQnbt2+Hl5cXMjIy8NZbb8HJyQlz5szBmDFjcP78eSQlJeHAgQMAABcXl3qfSa1WIzIyEj4+PkhLS0NJSQnef/99vbbz5s3DmjVr0L9/f8jlclRUVCA4OBhz586Fs7MzfvnlF0yYMAHdunVDSEgIANM+76aChJJoNC4uLpDJZLC3t4eHhweXvmTJEu53f39/pKamYvfu3TyhlMlk2Lp1K+zt7dH9iUC88/58rP0oDpvW/BOQGm7wlJaW4quvvsL//d//4YUXXgAA7Nixgxep+sqVK/jtt99w4sQJDBw4EADw1Vdf8cRPy6hRo/Dmm28CAJYtW4bk5GRs3LgRn376KWcze/ZsREREAABmzpyJcePGISUlBU8//TQAIDo62iSvCwCWL1+O999/HzNnzuTStHVMSUlBRkYGcnJyuAj5X3/9NXr37o0//viDs9NoNNi+fTsXeHbChAlISUnBRx99hJKSEqxfvx6bNm1CVFQUAKBbt24YPHgwd79FixZxv/v5+WH27NlITEzEnDlzYGdnB0dHR1hbW/M+0/pITk5GdnY2Dh06xOX56KOP8OKLL+rYzpo1C3/96195abNnz+Z+nzFjBv79739j9+7dCAkJMenzbkpIKEWOnY0UF5dGWOzejWHz5s3YunUr8vLyUF5eDpVKpfPfv2/fvtwqJAagb/BAPCwrRd6NGwjo5m+w7OzsbKhUKoSGhnJp7du3R48ePbjzzMxMWFtbIzg4mEsLDAyEq6urTnlhYWE652fOnOGlPfnkk9zv7u7uAICgoCBemj5vtS537txBfn4+94WvS2ZmJry9vXnbiPTq1Quurq7IzMzkhNLPz48XndvT05O7f2ZmJiorKw3eAwB27dqFDRs2IDs7G6Wlpaiurm5UgOnLly/D29ubJ6xab7AuAwYM4J2r1Wp8/PHH2L17N27dugWVSoXKykrub8OUz7spIaEUORKJxKTmr9hITEzE7NmzsWbNGoSFhcHJyQmrVq1CWlqaiSWIb9jbxsaG+127Br9umsaEFUV2duZZnln73nXvb+weqampGD9+PJYsWYKIiAi4uLggMTGx2fr9HBwceOerVq3C+vXrsW7dOgQFBcHBwQGzZs2CSqVqlvoYQxSDOZs3b4afnx/kcjlCQ0Nx4sSJeu337NmDwMBAyOVyBAUF4ddff+VdZ4whNjYWnp6esLOzQ3h4OK5evcpdz83N5Q00dOvWDXFxcbwPRdvpXPc4fvy4eR++lSCTyXhh4Y4ePYpBgwbhnXfeQf/+/dG9e3dkZ2fr5Dt79izKy8sBAIwB506dhL2DI7oY2ZStW7dusLGx4QnvgwcPcOXKFe48MDAQ1dXVSE9P59IuX76MoqIinfLqfq7Hjx/X20Q3B05OTvDz80NKSore6z179sSNGzdw48YNLu3ixYsoKipCr169TLpHQEAA7OzsDN7j2LFj8PX1xcKFCzFgwAAEBATg+vXrPJu6n6kxevTogRs3bvAGnf744w+T8h49ehQjR47E3//+d/Tt2xddu3blfZamfN5NicWFcteuXYiJiUFcXBxOnTqFvn37IiIiwmAT5tixYxg3bhyio6Nx+vRpREZGIjIyEufPn+dsVq5ciQ0bNiAhIQFpaWlwcHBAREQEKioqANSMzGk0Gnz22We4cOEC1q5di4SEBCxYsEDnfgcOHMDt27e5o3YzjniMn58f0tLSkJubi3v37iEgIAAnT57Ev//9b1y5cgWLFy/W+6VRqVSIjo7GxYsXkfTbr9jyyQqMnfQmrCT1/2k6OjoiOjoaH3zwAf773//i/PnzmDRpEm9tb48ePTBs2DBMnToVaWlpSE9Px5tvvqnX29qzZw+2bt2KK1euIC4uDidOnOCNQpubDz/8EGvWrMGGDRtw9epVnDp1Chs3bgQAhIeHIygoCOPHj8epU6dw4sQJTJw4EUOGDNFpshpCLpdj7ty5mDNnDr7++mtkZ2fj+PHj+OqrrwDUCGleXh4SExORnZ2NDRs24Mcff+SV4efnh5ycHJw5cwb37t1DZWVlvfd88cUX0a1bN0RFReHcuXM4evQo1w9qLApWQEAAkpOTcezYMWRmZmLq1Kk8wTXl825SmIUJCQlh06ZN487VajXz8vJi8fHxeu1Hjx7Nhg8fzksLDQ1lU6dOZYwxptFomIeHB1u1ahV3vaioiNna2rLvvvvOYD1WrlzJ/P39ufOcnBwGgJ0+fbohj8UYY6y4uJgBYMXFxSbZl5eXs4sXL7Ly8vIG39NSXL58mT311FPMzs6OAWCXLl1ikyZNYi4uLszV1ZW9/fbbbN68eaxv375cnqioKDZy5EgWGxvLOnTowBwdHdlrr0exP7IUrFxVbfSeJSUl7O9//zuzt7dn7u7ubOXKlWzIkCFs5syZnM3t27fZ8OHDma2tLfPx8WFff/018/X1ZWvXruVsALDNmzezF198kdna2jI/Pz+2a9cu7rq+v4WDBw8yAOzBgwdc2rZt25iLi4vJ7ywhIYH16NGD2djYME9PTzZjxgzu2vXr19mIESOYg4MDc3JyYqNGjWIKhYK7HhcXx3uXjDG2du1a5uvry52r1Wq2fPly5uvry2xsbJiPjw/7+OOPuesffPAB997HjBnD1q5dy6t/RUUFe+2115irqysDwLZt22b0mTIzM9nTTz/NZDIZCwwMZD///DMDwJKSkhhjhr9X9+/fZyNHjmSOjo6sU6dObNGiRWzixIls5MiRnI0pn3dd6vtOCfl+WlQoKysrmVQqZT/++CMvfeLEiWzEiBF683h7e/P+yBljLDY2lj355JOMMcays7P1fhDPPvsse/fddw3WZeHChSw4OJg7136g3t7erGPHjuzpp59mP/30U73PU1FRwYqLi7njxo0bbUYoG0tVtZqdvfGAO0wRSnMBQOdvkDAPR44cYQBYVlaWRe5vLqG0aNP73r17UKvV3AiiFnd3dygUCr15FApFvfban0LKzMrKwsaNGzF16lQuzdHREWvWrMGePXvwyy+/YPDgwYiMjMS+ffsMPk98fDxcXFy4w9tIPxvxmIpqCqvWGvjxxx+RnJyM3NxcHDhwAFOmTMHTTz+Nbt26WbpqjaLlDaeamVu3bmHYsGEYNWoU3nrrLS7dzc0NMTEx3PnAgQORn5+PVatWYcSIEXrLmj9/Pi+PdoN1Qjh51/Pwp35BBq9fvHgRPj4+zVgjYTg6Ohq89ttvv+GZZ55pxtqYh2+//ZbnTNTG19cXFy5cQElJCebOnYu8vDy4ubkhPDy8WVfQNBUWFUo3NzdIpVJepy0AFBQUGJzk6uHhUa+99mdBQQE8PT15NnXn8OXn52Po0KEYNGgQPv/8c6P1DQ0NRXJyssHrtra2sLW1NVoOYRxPL0+deYy18fLyMtu9mIlLNYVQX907d+5s9vs1ByNGjODNY6yNdqrSxIkTMXHixOasVrNgUaGUyWQIDg5GSkoKF25Lo9EgJSXF4IhjWFgYUlJSeJFqkpOTuQnD/v7+8PDwQEpKCieMSqUSaWlpePvtt7k8t27dwtChQxEcHIxt27aZNHp25swZnvgSZqSOVllbW6N79+6WqYsZaMl1N4STkxNvgntbwuJN75iYGERFRWHAgAEICQnBunXrUFZWhsmTJwOo+Q/VuXNnxMfHA6hZOjZkyBCsWbMGw4cPR2JiIk6ePMl5hBKJBLNmzcLy5csREBAAf39/LF68GF5eXpwY37p1C8899xx8fX2xevVqXiADrUe6Y8cOyGQy9O/fHwDwww8/YOvWrfjyyy+b69W0MVg9ZwRhWSwulGPGjMHdu3cRGxsLhUKBfv36ISkpiRuMycvL43l7gwYNws6dO7Fo0SIsWLAAAQEB2Lt3L/r06cPZzJkzB2VlZZgyZQqKioowePBgJCUlcZsLJScnIysrC1lZWTprRWs3w5YtW4br16/D2toagYGB2LVrF/72t7815esgtJBSEiJCwpqig4YAUNPkd3FxQXFxsUlraCsqKpCTkwN/f/9G7RjXEimtqMK1e2XcebeOjnCwtfj/caKFU993Ssj30+IrcwiCIMQOCSVhEarVGuTcK0NxeU2gWGrWEGKGhJKwCLeLK1BSUYXr98vg5+eHzRs3cNf6erfDvp9+smDtCIIPdQIRFkGtMexDpqRfQt9uLXOuIdE6IY+SsAiaesYQ3Tq5Q2bhifuMMVRXV1u0DoR4IKEUO4wBqjLLHAImRJSUlGD8+PFwcHCAp6cn1q5da3AL2yq1BqWVhkWor3c77N9X0/TWxgX94YcfMHToUNjb26Nv375ITU3l5Tly5Ai38Ze3tzfeffddlJU9HkX/5ptvMGDAADg5OcHDwwOvv/46L5TfoUOHIJFI8NtvvyE4OBi2trY4cuSIyc9PtG6o6S12qh4CH5tvuZ4gFuQDMgfjdqhZOHD06FHs27cP7u7uiI2NNbjx04OHxqNW15XohQsXYvXq1QgICMDChQsxbtw4ZGVlwdraGtnZ2Rg2bBiWL1+OrVu34u7du5g+fTqmT5+Obdu2AajZEnXZsmXo0aMH7ty5g5iYGEyaNEkn6PO8efOwevVqdO3aFe3atTPp2YnWDwkl0WhKSkqwY8cO7Ny5k9ujZdu2bY1bj11HKWfPno3hw4cDqNm0rHfv3sjKykJgYCDi4+Mxfvx4znsNCAjAhg0bMGTIEGzZsgVyuRxvvPEGV1bXrl2xYcMGDBw4EKWlpbwAFkuXLtW7GRbRtiGhFDs29jWenaXubQLXrl1DVVUVbyMpFxcXgxs/1R/rWj+1N/bSrre/c+cOAgMDcfbsWZw7d463ZzdjDBqNBjk5OejZsyfS09Px4Ycf4uzZs3jw4AG3t0xeXh5vewVTI4gTbQsSSrEjkZjc/G05GJfKuk1vfRt7acWutLQUU6dOxbvvvqtTjo+PD8rKyhAREYGIiAh8++236NixI/Ly8hAREaGzeVXdTa8IAiChJMxA165dYWNjgz/++IOLEVlcXIwrV67g2WefbWCppg8k/elPf8LFixcNRuzJyMjA/fv3sWLFCi4+6MmTJxtYL6ItQqPeRKNxcnJCVFQUPvjgAxw8eBAXLlxAdHQ0rKysjG4qZQ7mzp2LY8eOYfr06Thz5gyuXr2Kn376iQvV5+PjA5lMho0bN+LatWvYt28fli1b1uT1IloPJJSEWfjkk08QFhaGV155BeHh4Xj66afRs2fPZgnu8eSTT+Lw4cO4cuUKnnnmGfTv3x+xsbHcYFLHjh2xfft27NmzB7169cKKFSuwevXqJq8X0Xqg6EFNSFuOHlRWVobOnTtjzZo1iI6O5l27V1KJ/OJy7vzJLq5Qllch9/7jeY8+7e3hai9rtvoSrRNzRQ+iPkrCLJw+fRqXLl1CSEgIiouLsXTpUgDAyJEjdY1NaI3Tf29CTJBQEmZj9erVuHz5MrfFx//+9z9kZmbi5Zdf5tkxPF70c/zyTf2FkVISIoKEkjAL/fv3R3p6uk56eXm5zkZbhWUq3FFW8NLq6iLpJCEmSCiJJsXOzk5n2s790krIi8oN5CAI8UGj3iKExtcA8ikJc2Cu7xIJpYjQrj55+PChhWtieeh/BWEOtN+l2iu7GgI1vUWEVCqFq6srF/7L3t6+WSZsNzeqShVY9eOlgxUVFTppqkorVFiTWhINgzGGhw8f4s6dO3B1dYVUKm1UeSSUIkO7r3jtWImtjbLKajx4WMWdy8rtUK5S435ZLaG0t0ER7cJINBJXV1fuO9UY6C9RZEgkEnh6eqJTp06oqqoynqEFsv9cPtYevMKdp7z/HH6/cgdLDl7k0qYP7Y5XA7voy04QJmFjY9NoT1ILCaVIkUqlZvuQxYZaYo1bJWruXC6Xo1piw0sr10hb/OokovVAgzlEsyPRszSn7uBNfXvqEERzQ0JJND96xqdYnelAJJSEmCChJJodU8bx1ZomrwZBmAwJJdHs6JvyRE1vQsyIQig3b94MPz8/yOVyhIaG4sSJE/Xa79mzB4GBgZDL5QgKCtLZSY8xhtjYWHh6esLOzg7h4eG4evUqdz03NxfR0dHw9/eHnZ0dunXrhri4OJ1tAc6dO4dnnnkGcrkc3t7eWLlypfkeug2jz6OsK4tqDQklIR4sLpS7du1CTEwM4uLicOrUKfTt2xcREREG5xEeO3YM48aNQ3R0NE6fPo3IyEhERkbi/PnznM3KlSuxYcMGJCQkIC0tDQ4ODoiIiEBFRU0ghkuXLkGj0eCzzz7DhQsXsHbtWiQkJGDBggVcGUqlEi+99BJ8fX2Rnp6OVatW4cMPP8Tnn3/etC+kDaBvDn3dpWbVJJSEmGAWJiQkhE2bNo07V6vVzMvLi8XHx+u1Hz16NBs+fDgvLTQ0lE2dOpUxxphGo2EeHh5s1apV3PWioiJma2vLvvvuO4P1WLlyJfP39+fOP/30U9auXTtWWVnJpc2dO5f16NHD5GcrLi5mAFhxcbHJedoCP566yXzn7ucOxhjbe5qftirpkoVrSbR2hHw/LepRqlQqpKenIzw8nEuzsrJCeHg4UlNT9eZJTU3l2QNAREQEZ5+TkwOFQsGzcXFxQWhoqMEygZrNsNq3b8+7z7PPPguZ7HGU7YiICFy+fBkPHjzQW0ZlZSWUSiXvIHQxZVUmeZSEmLCoUN67dw9qtRru7u68dHd3dygUCr15FApFvfban0LKzMrKwsaNGzF16lSj96l9j7rEx8fDxcWFO7Q7/hHGqTt2o9bQsDchHizeR2lpbt26hWHDhmHUqFF46623GlXW/PnzUVxczB03btwwUy1bF3pHvesM59D0IEJMNGoJY2lpKbcJvRZTNtHS4ubmBqlUioKCAl56QUGBwYXsHh4e9dprfxYUFMDT05Nn069fP16+/Px8DB06FIMGDdIZpDF0n9r3qIutrS1sbW31XiMeY9o8SlJKQjwI9ihzcnIwfPhwODg4wMXFBe3atUO7du3g6uqKdu3aCSpLu7dKSkoKl6bRaJCSkoKwsDC9ecLCwnj2AJCcnMzZ+/v7w8PDg2ejVCqRlpbGK/PWrVt47rnnEBwcjG3btsHKiv8qwsLC8Pvvv/MCUyQnJ6NHjx6Cn5Pgo3/Um3+upnmUhIgQ7FH+/e9/B2MMW7duhbu7e6PjJcbExCAqKgoDBgxASEgI1q1bh7KyMkyePBkAMHHiRHTu3Bnx8fEAgJkzZ2LIkCFYs2YNhg8fjsTERJw8eZLzCCUSCWbNmoXly5cjICAA/v7+WLx4Mby8vBAZGQngsUj6+vpi9erVuHv3Llcfrbf4+uuvY8mSJYiOjsbcuXNx/vx5rF+/HmvXrm3U8xKmrfWmeZSEmBAslGfPnkV6ejp69OhhlgqMGTMGd+/eRWxsLBQKBfr164ekpCRu4CQvL4/n7Q0aNAg7d+7EokWLsGDBAgQEBGDv3r3o06cPZzNnzhyUlZVhypQpKCoqwuDBg5GUlMRFo0lOTkZWVhaysrLQpQs/lBd79I11cXHBf/7zH0ybNg3BwcFwc3NDbGwspkyZYpbnbsuYNOqtJqEkxIOEMWFtnKFDh2LhwoU6U3QIXYRssN6W+C3jNt7+9hR3nrtiOL5Pv4nZe85yaX/9U2d8MrqfBWpHtBWEfD8Fe5Rffvkl/vGPf+DWrVvo06ePzl4UTz75pNAiiTaGKStzqOlNiAnBQnn37l1kZ2dzfYhATb8gYwwSiQRqtbqe3AQB6Bv3riuLNOGcEBOChfKNN95A//798d1335llMIcgAOgopYaEkhARgoXy+vXr2Ldvn86m9gRhOsZFkDxKQkwInkf5/PPP4+zZs8YNCUIAOhHOSSgJESHYo/zLX/6C9957DxkZGQgKCtIZzBkxYoTZKke0TvTNs6ibRh4lISYEC+U//vEPAMDSpUt1rtFgDmEuaNSbEBOChbLu2m6CEIo+CaQI54SYEdRHWVVVBWtra140cYIQiilNbxJKQkwIEkobGxv4+PhQ85owO3UHc6qp5UKICMGj3gsXLsSCBQtQWFjYFPUh2gB1RRHQFz2omSpDECYguI9y06ZNyMrKgpeXF3x9feHg4MC7furUKQM5CcJ0KB4lISYEC6U2VBlBNBS9fZR1zinCOSEmBAtlXFxcU9SDaEOY0qomj5IQEw3eCiI9PR2ZmZkAgN69e6N///5mqxTR+ihXqfGfiwo890Qn/Qa0rzchYgQL5Z07dzB27FgcOnQIrq6uAICioiIMHToUiYmJ6Nixo7nrSLQClu6/gO9O3EB/H1dMGuTHu5Z1pxT/dzyPl0ZLGAkxIXjUe8aMGSgpKcGFCxdQWFiIwsJCnD9/HkqlEu+++25T1JFoBew9nQ8AOJ1XpHMt/JPDuFxQwksjj5IQE4I9yqSkJBw4cAA9e/bk0nr16oXNmzfjpZdeMmvliNaDlcBofDThnBATgj1KjUajEwgDqJmMTssbCUNY1VJKUzYfIaEkxESDwqzNnDkT+fn5XNqtW7fw3nvv4YUXXjBr5YjWg5XAAM8klISYECyUmzZtglKphJ+fH7p164Zu3brB398fSqUSGzdubIo6Eq0AaW2P0oQJQrSvNyEmBPdRent749SpUzhw4AAuXboEAOjZsyftykjUi+A+SlrDSIiIBs2jlEgkePHFF/Hiiy+auz5EK6X23kqmOIs06k2IiQYJZUpKClJSUnDnzh2dAZytW7eapWJE60IqUCip6U2ICcFCuWTJEixduhQDBgyAp6cn7cLYRKiqNZBZC+5CFi1SgW1vGswhxIRgoUxISMD27dsxYcKEpqgPASDt2n2M+fw45gzrgXeeax27Xdb+f2raWm/G7RVPEJZGsMuiUqkwaNCgpqgL8YgFP2YAAFYmXbZwTcyHUI8SAMipJMSCYKF88803sXPnzqaoC/GIuvpQUdXyI8pb8fooTVNAinJOiAXBQllRUYFPPvkEQ4YMwYwZMxATE8M7hLJ582b4+flBLpcjNDQUJ06cqNd+z549CAwMhFwuR1BQEH799VfedcYYYmNj4enpCTs7O4SHh+Pq1as8m48++giDBg2Cvb09F9ijLhKJROdITEwU/HwNopaObD6YhcDFSfj9yt3muXcTYSWw6Q1QPyUhHgQL5blz59CvXz9YWVnh/PnzOH36NHecOXNGUFm7du1CTEwM4uLicOrUKfTt2xcRERG4c+eOXvtjx45h3LhxiI6OxunTpxEZGYnIyEjeZmcrV67Ehg0bkJCQgLS0NDg4OCAiIgIVFRWcjUqlwqhRo/D222/XW79t27bh9u3b3GGJoMWr/l3T/F64N6PZ721OhK7MAUgoCfEgeDDn4MGDZrv5J598grfeeguTJ08GUDNQ9Msvv2Dr1q2YN2+ejv369esxbNgwfPDBBwCAZcuWITk5GZs2bUJCQgIYY1i3bh0WLVqEkSNHAgC+/vpruLu7Y+/evRg7diyAmpF7ANi+fXu99XN1dYWHh4e5HrdRVFa17GaoldDRHJBQEuLBYvNPVCoV0tPTeSt6rKysEB4ejtTUVL15UlNTdVYARUREcPY5OTlQKBQ8GxcXF4SGhhossz6mTZsGNzc3hISEYOvWrUb71iorK6FUKnlHQ9B3l8rqFi6UDRjMIaEkxEKDI5w3lnv37kGtVsPd3Z2X7u7uzi2NrItCodBrr1AouOvaNEM2prJ06VI8//zzsLe3x3/+8x+88847KC0trTfmZnx8POetmpvK6pY9oCOt9S/ZlLXeAAklIR4sJpRiZ/Hixdzv/fv3R1lZGVatWlWvUM6fP583oKVUKuHt7W2W+rR4j1LgyhyAljES4sFiTW83NzdIpVIUFBTw0gsKCgz2C3p4eNRrr/0ppExTCQ0Nxc2bN1FZWWnQxtbWFs7OzryjIehr4rf0FX00mEO0ZAQL5e+//47q6mqd9Orqavz+++8mlyOTyRAcHIyUlBQuTaPRICUlBWFhYXrzhIWF8ewBIDk5mbP39/eHh4cHz0apVCItLc1gmaZy5swZtGvXDra2to0qp61C04OIlozgpvfQoUNx+/ZtdOrE302vuLgYQ4cOhVptel9aTEwMoqKiMGDAAISEhGDdunUoKyvjRsEnTpyIzp07Iz4+HgAwc+ZMDBkyBGvWrMHw4cORmJiIkydP4vPPPwdQM/dx1qxZWL58OQICAuDv74/FixfDy8uLN7UnLy8PhYWFyMvLg1qt5qY1de/eHY6Ojvj5559RUFCAp556CnK5HMnJyfj4448xe/Zsoa+LeERDPEpqehNiQbBQGlp/e//+fTg4OAgqa8yYMbh79y5iY2OhUCjQr18/JCUlcYMxeXl5sLJ67PQOGjQIO3fuxKJFi7BgwQIEBARg79696NOnD2czZ84clJWVYcqUKSgqKsLgwYORlJQEuVzO2cTGxmLHjh3cuXar3YMHD+K5556DjY0NNm/ejPfeew+MMXTv3p2bytQctEZ5ELoVBABoWnp/A9FqkDAT15P99a9/BQD89NNPGDZsGK8Jqlarce7cOfTo0QNJSUlNU9MWiFKphIuLC4qLiwX1Vz678iDyCh/qpOeuGG7O6jUrYz9PxfFrhQCAj17tg4U/njeSA/j13WfQy6th/bwEYQwh30+TPUoXFxcANR6lk5MT7OzsuGsymQxPPfVUs3lcRMujYUExyKMkxIHJQrlt2zYAgJ+fH2bPni24mU20XarUGhzNus+d0/QgoqUhuI8yLi6uKepB1MLUCdkthe9O5DUon5qiBxEiQfD0oIKCAkyYMAFeXl6wtraGVCrlHQRRl2t3y3jnpk8PMn9dCKIhCPYoJ02ahLy8PCxevJi2giAaBsWjJFoYgoXyyJEj+N///od+/fo1QXUIoOWvwjEXNOGcEAuCm97e3t4mR6gmCIC/Xw5AK3OIlodgoVy3bh3mzZuH3NzcJqgOAZBHqYWEkhALgpveY8aMwcOHD9GtWzfY29vDxsaGd72wsNBslSNq1ki3dL2QgO9SmvqPgISSEAuChXLdunVNUA3CEBKJpNW5mKZ23ZBQEmJBsFBGRUU1RT0IQgeacE6IhQbFo8zOzsaiRYswbtw4biOw3377DRcuXDBr5doqtT2u1jj5ylT5oyWMhFgQLJSHDx9GUFAQ0tLS8MMPP6C0tBQAcPbsWVq10wS05Wmq1WoSSkIcCBbKefPmYfny5UhOToZMJuPSn3/+eRw/ftyslSN0B0JaAzSYQ7Q0BAtlRkYGXn31VZ30Tp064d69e2apVFuHJw+tQCcbPI+Smt6ESBAslK6urrh9+7ZO+unTp9G5c2ezVIp4TCvQSR1MHfWmwRxCLAgWyrFjx2Lu3LlQKBSQSCTQaDQ4evQoZs+ejYkTJzZFHds0raGPsu4jmBzhnISSEAmChfLjjz9GYGAgvL29UVpail69euHZZ5/FoEGDsGjRoqaoY5ujtpC0hj7KunJnrEmtDfJLHiUhFgTPo5TJZPjiiy+wePFinD9/HqWlpejfvz8CAgKaon5tktrxKFuDR1kXY4M0UisJ1BpG8SgJ0SBYKLX4+PjAx8fHnHUh9NAadLLuMxgTSmsrCVSgeJSEeDBJKGNiYrBs2TI4ODggJiamXttPPvnELBUjamiN8T5NEcoaO1JKQhyYJJSnT59GVVUV97shWuOX2hLw+ihb4Ss1pekNUB8lIR5MEsqDBw/q/Z1oelqDTurOozQmlDVjjDTqTYiFBq31JojGYKzv0UZKHiUhLkzyKP/617+aXOAPP/zQ4MoQNdSWh9bQnVH3GYxNONc2vWllDiEWTPIoXVxcuMPZ2RkpKSk4efIkdz09PR0pKSlwcXFpsoq2JVpbH2VdYTQWFYgbzKGgGIRIMMmj3LZtG/f73LlzMXr0aCQkJHDb06rVarzzzjtwdnZumlq2YVqBTupQplLXe91aWvP/m5rehFgQ3Ee5detWzJ49m7eHt1QqRUxMDLZu3WrWyhGts+m9My2vXnutR0nxKAmxIFgoq6urcenSJZ30S5cuQdOAeW+bN2+Gn58f5HI5QkNDceLEiXrt9+zZg8DAQMjlcgQFBeHXX3/lXWeMITY2Fp6enrCzs0N4eDiuXr3Ks/noo48waNAg2Nvbw9XVVe998vLyMHz4cNjb26NTp0744IMPUF1dLfj5GkbjA/eqNQxJ52/jjrLCPFVqRmh6ECE2BAvl5MmTER0djU8++QRHjhzBkSNHsGbNGrz55puYPHmyoLJ27dqFmJgYxMXF4dSpU+jbty8iIiK4qOl1OXbsGMaNG4fo6GicPn0akZGRiIyMxPnz5zmblStXYsOGDUhISEBaWhocHBwQERGBiorHgqFSqTBq1Ci8/fbbeu+jVqsxfPhwqFQqHDt2DDt27MD27dsRGxsr6PnMQUMdym9Sc/GP/zuF8E8Om7dCzQD1URJiQ8IEbtKt0WiwevVqrF+/ngu35unpiZkzZ+L999/nNcmNERoaioEDB2LTpk1c2d7e3pgxYwbmzZunYz9mzBiUlZVh//79XNpTTz2Ffv36ISEhAYwxeHl54f3338fs2bMBAMXFxXB3d8f27dsxduxYXnnbt2/HrFmzUFRUxEv/7bff8MorryA/Px/u7u4AgISEBMydOxd3797lBSyuD6VSCRcXFxQXFwvqvx2wPBn3SlUAADdHW9wrrQQA5K4YbjRvSUUVxn5+HBfylVyaKfmakvhfM/HZ79dMtv+TjytO5RXhb8FdsHpU3yasGdGWEfL9FOxRWllZYc6cObh16xaKiopQVFSEW7duYc6cOYJEUqVSIT09HeHh4byyw8PDkZqaqjdPamoqzx4AIiIiOPucnBwoFAqejYuLC0JDQw2Waeg+QUFBnEhq76NUKuvdF6iyshJKpZJ3NITGjHp/nXqdJ5ItEe1gDkU4J8RCoyacOzs7N3ik+969e1Cr1TwxAgB3d3coFAq9eRQKRb322p9CyhRyn9r30Ed8fDxvKpW3t7fJ9zSE0JZ3ZbUI10cLfAhr6qMkREaDogd9//332L17N/Ly8qBSqXjXTp06ZZaKtUTmz5/PCxqiVCobLZatYNDb9L0fHqEdzKEljIRYEOxRbtiwAZMnT4a7uztOnz6NkJAQdOjQAdeuXcPLL79scjlubm6QSqUoKCjgpRcUFMDDw0NvHg8Pj3rttT+FlCnkPrXvoQ9bW1vOy26Mt81bmdPEMykZY5i+8xRidp9p0vsIwYabRylC75hokwgWyk8//RSff/45Nm7cCJlMhjlz5iA5ORnvvvsuiouLTS5HJpMhODgYKSkpXJpGo0FKSgrCwsL05gkLC+PZA0BycjJn7+/vDw8PD56NUqlEWlqawTIN3ScjI4M3+p6cnAxnZ2f06tXL5HLMgVCPUp/5618cN+idKZQV2H/uNn44dQullU00/UngM1hJtGHWmqAuBNEABAtlXl4eBg0aBACws7NDSUkJAGDChAn47rvvBJUVExODL774Ajt27EBmZibefvttlJWVcdOMJk6ciPnz53P2M2fORFJSEtasWYNLly7hww8/xMmTJzF9+nQANRObZ82aheXLl2Pfvn3IyMjAxIkT4eXlhcjISN4znDlzBnl5eVCr1Thz5gzOnDnD7VH+0ksvoVevXpgwYQLOnj2Lf//731i0aBGmTZsGW1tboa+sUZjDnzyWfR/nbun/JybGAROKR0mIDcF9lB4eHigsLISvry98fHxw/Phx9O3bFzk5OSbvrqdlzJgxuHv3LmJjY6FQKNCvXz8kJSVxAyd5eXmwsnqs5YMGDcLOnTuxaNEiLFiwAAEBAdi7dy/69OnD2cyZMwdlZWWYMmUKioqKMHjwYCQlJUEul3M2sbGx2LFjB3fev39/ADUh5J577jlIpVLs378fb7/9NsLCwuDg4ICoqCgsXbpU6OtqELXfo7lW5lSZ4J4J/fyaCmuKHkSIDMFC+fzzz2Pfvn3o378/Jk+ejPfeew/ff/89Tp48KSjKkJbp06dzHmFdDh06pJM2atQojBo1ymB5EokES5curVfUtm/fju3bt9dbL19fX51VP81FY+TBkK4aEsrmWCIptJ+VljASYkOwUH7++efcUsVp06ahQ4cOOHbsGEaMGIGpU6eavYJtHXPpWJUJq1wybhZjUHc389ywFgmHswXZawP3VtPKHEIkCOqjrK6uxvLly3lzCceOHYsNGzZgxowZJq9YIUzHXEJZbULT+/Uv00QxJUcbuFeM/adE20SQUFpbW2PlypXNGByibdKYfb0N2ZviUQKNa/brI/36A8F5KHAvITYEj3q/8MILOHy45QVaaKmYzaO00Ajya1uOCc7zeNSbhJIQB4L7KF9++WXMmzcPGRkZCA4OhoODA+/6iBEjzFa5tgpv1NtMZSrL9bcCxLjwx4qEkhAZgoXynXfeAaB//26JRAK1uv7o1YQwhI5KGzJf8GMGXg/1MZq/RqQtK582FBSDEBmCm94ajcbgQSJpfiwlWZXVapy5UQSNhuHmg4d4ZuV/8dWRnGa5NwXuJcQGbVcrQnjy0MxKqb33hpSriNx8FD+fy0f8r5dwo7Acy/ZfbJY6WFNQDEJkmNz0Li8vR0pKCl555RUANZFyKisruetSqRTLli3jrYAhGo9QnTSXrt4uqokIf/3+Q1RWN29LwdqKNhcjxIXJQrljxw788ssvnFBu2rQJvXv3hp2dHYCaPXO8vLzw3nvvNU1N2yjNvbmYdhxJOzWnokqN5tYra5pHSYgMk5ve3377LaZMmcJL27lzJw4ePIiDBw9i1apV2L17t9kr2BapLRBNLZOGdFhbhfIqdbMvJaTpQYTYMFkos7KyEBQUxJ3L5XJewIqQkBBcvNg8fVitnYe19r02Z/PzRuFDozbsUS+lhvMoNc3uUdJgDiE2TBbKoqIiXp/k3bt34efnx51rNBredcI8CPXm6mupx+0zvN+PFu3ttAMpFVXqZo8qREExCLFhslB26dKFty1sXc6dO4cuXbqYpVLEY8ypFUUPVTpp1+6W6bXVilS5Sm3WOpiCVBvhnCL3EiLBZKH885//jNjYWN7+2FrKy8uxZMkSDB9u2W1RWyPm9KpO5RXppK1Puapz/uzKg7hbUtM6qKimPkqCMHnUe8GCBdi9ezd69OiB6dOn44knngAAXL58GZs2bUJ1dTUWLFjQZBVtqwidSyh0lFxuw99ieMuhmpBoeY/6M8tV6gavN9eKrVAoKAYhNkwWSnd3dxw7dgxvv/025s2bx/VbSSQSvPjii/j00091tnglGk9TO1V2NvU3Kiqq1LC1MX2/9trM3nO2QfkozBohNgSt9fb390dSUhIKCwuRlZUFAOjevTvat2/fJJUj+E1vxpjZ51XW9SjrUlGlga11w4Ty7M2iBuWT0oRzQmQ0aF/v9u3bIyQkxNx1IfRQWysY0z+qfaWgBPP+dQ4xL/YQXL61Vf0eZXmVGk7yBv2ZcMEthKLto2SsputBG02IICwFrfUWOTyP0oDNP75Jx6m8Ivz9qzSz378xE84bKm/SWsJI/ZSEGCChFDm1++kMzWe8W2r6oMmK3y4h606JyfaNWcLY0F4CbR8lQP2UhDggoRQ5zASPUggJh7MRse5/JttXVKnNvj2EMaS1ugNIKAkxQELZgjDXfEYh4lOlZlBVN+/Eb+taTW8a0CHEAAmlyKktE4Z0sqmHOspVDdtMTujGaFpq91FSTEpCDJBQihxTnMjaU4aaIipbeVXD4lE2tC5SKwmXlzxKQgyQUIocXh+lIY+yEeJoSt7aW91WVKlNXoN9u1h3uaupSCU06ZwQDySUIofX9DbzsEpFlRolFVVG7Wr3UfZb+h+8tO53s9ajLhLQMkZCXDRsJjHRbNTWCVOcKyH9gk/Fp6DooXGhrL0VREWVxmDEIXNibSVBJQC1moSSsDyi8Cg3b94MPz8/yOVyhIaG4sSJE/Xa79mzB4GBgZDL5QgKCsKvv/7Ku84YQ2xsLDw9PWFnZ4fw8HBcvcqPklNYWIjx48fD2dkZrq6uiI6ORmlpKXc9NzcXEolE5zh+/Lj5HtwE6i5h1EdDW96miCTAb3prMdYkbkyTWSKptbc3eZSECLC4UO7atQsxMTGIi4vDqVOn0LdvX0RERODOnTt67Y8dO4Zx48YhOjoap0+fRmRkJCIjI3mxMleuXIkNGzYgISEBaWlpcHBwQEREBC9E3Pjx43HhwgUkJydj//79+P3333W2ugCAAwcO4Pbt29wRHBxs/pdQD8zA75amqp5+SrWGYdauM40q/3GoNYpJSVgeiwvlJ598grfeeguTJ09Gr169kJCQAHt7e2zdulWv/fr16zFs2DB88MEH6NmzJ5YtW4Y//elP2LRpE4Aar2vdunVYtGgRRo4ciSeffBJff/018vPzsXfvXgBAZmYmkpKS8OWXXyI0NBSDBw/Gxo0bkZiYiPz8fN79OnToAA8PD+6wsbFp0vdRH4YHc5p/LbSqHqH8Nu06fj6bb/C6cSS0HQQhKiwqlCqVCunp6QgPD+fSrKysEB4ejtTUVL15UlNTefYAEBERwdnn5ORAoVDwbFxcXBAaGsrZpKamwtXVFQMGDOBswsPDYWVlhbQ0/nrpESNGoFOnThg8eDD27dtX7/NUVlZCqVTyjkbDC4phvOndXJpZVc8k9BM5hY0qWyKpNZhDQkmIAIsK5b1796BWq3XiWLq7u0OhUOjNo1Ao6rXX/jRm06lTJ951a2trtG/fnrNxdHTEmjVrsGfPHvzyyy8YPHgwIiMj6xXL+Ph4uLi4cIe3t7exV2AUjQnTgyxBfR5lSUXDJqjXRhvViISSEAM06m0ANzc3xMTEcOcDBw5Efn4+Vq1ahREjRujNM3/+fF4epVLZaLGsLZTfp9/EW8921bFpiBfZ2A3Dqqr158+6U4LDV+42qmwJAO1yb2p6E2LAoh6lm5sbpFIpCgoKeOkFBQXw8PDQm8fDw6Nee+1PYzZ1B4uqq6tRWFho8L4AEBoaygUs1oetrS2cnZ15R2OpLRMf/ZqJOyX1T+I2VTMb650a8ig/fbSVRGPRepS0hJEQAxYVSplMhuDgYKSkpHBpGo0GKSkpCAsL05snLCyMZw8AycnJnL2/vz88PDx4NkqlEmlpaZxNWFgYioqKkJ6eztn897//hUajQWhoqMH6njlzBp6ensIftBHUFTRlub5mrXCXsrEBNgwFyjBH94BEQoM5hLiweNM7JiYGUVFRGDBgAEJCQrBu3TqUlZVh8uTJAICJEyeic+fOiI+PBwDMnDkTQ4YMwZo1azB8+HAkJibi5MmT+PzzzwHUfMlmzZqF5cuXIyAgAP7+/li8eDG8vLwQGRkJAOjZsyeGDRuGt956CwkJCaiqqsL06dMxduxYeHl5AQB27NgBmUyG/v37AwB++OEHbN26FV9++WUzvyE+5tpj+82vTzYqv6HpQebqU6QljISYsLhQjhkzBnfv3kVsbCwUCgX69euHpKQkbjAmLy8PVrXiEw4aNAg7d+7EokWLsGDBAgQEBGDv3r3o06cPZzNnzhyUlZVhypQpKCoqwuDBg5GUlAS5XM7ZfPvtt5g+fTpeeOEFWFlZ4bXXXsOGDRt4dVu2bBmuX78Oa2trBAYGYteuXfjb3/7WxG+kfvTpRu0+SlP7Kw9dblw/okGhNIOQ85YwklASIsDiQgkA06dPx/Tp0/VeO3TokE7aqFGjMGrUKIPlSSQSLF26FEuXLjVo0759e+zcudPg9aioKERFRRmutIXQJxy1tbG5RsYNN73NIJQ0PYgQGRafcE4Iw1jfYnP16RkazDFb05uEkhARJJQtjF8zbuuk1W5uVzdTEAlDHuX1+w8bXbYEEm4JIw3mEGKAhLKF8emhbFSpNQanzdS3Btuc6AuUAdSEbmssvKAYJJSECCChbIE8t+oQXtl4RG9/YHN5YIYEOdcMHiVQKyiGmJYjEW0WUQzmEMK4VVSOW0XlUGsYrKUSntdlavTxxtLUG45JKXoQISLIo2zBaOWxthdp6cEcc8FNOKfAvYQIIKFswWhbpbXFpPn6KHXvY67J8BLJ46a3ubboJYjGQE3vFoxWRGqLliVHvQvLVGYrn5YwEoLRqIHim0DhNaCdL9BeN4BMQyGhFBl7Tt4w2VbrbNXuo7SkR3m90DwDOZJagXspKAbBgxPD7BpBvH+t5mdhNvAgF1A/+mf9QhzwTEy9RQmBhFJEqDUMH3x/zmR7DWNgjPG8rqrm6qPU41HuO9OYqOaPqVmZU9MrRB5lG0SjBopvAPcfiaH2uP9IDDX17PUklQHt/AAbe7NWiYRSRAjtj9MwpjPPsLlGiVV6mvhHsu6ZrXzpo0n0NI+ylaKurhHDwmygMIcviiaJoX9N07pDN6C9P9C+W825SxfASmr26pJQigih4xYaputxGZoIbm70Nb2f7OyCrDul6OrmgGv3GrelrZQinLd81NVAcZ5uE7nwGvDguhExtH0kgF35R4dugHPnJhHD+iChFBFM4D6LjDEdwbLkPMqHqppVOb4d7BsllLVHvanpLXLU1UDR9RqvkOs3fPSz6DqgqWdbEE4MH3mFHR55he27Ac5ezS6G9UFCKSIa4lHWjbtmyZU5pZU1XwpHeeN2qpRAQksYxYS6CijK44ugVhSL8uoXQ2t5TTO5bhO5QzfAyevxnh8ih4SyBVPTR8lPM/eot71MynmKtdE34bxEK5S2jf+zsiahbF7UVTXN4doieL+WGLJ61vBby/U3kdt3bVFiWB8klCJCuEfJUHfsxtzzKO1l1vqFUk/Tu7Sips/JWd64PyuKR9lEVKseNZOv6XqHRTeMiKHdIxGs00Ru3xVw8mwVYlgfJJQiQngfpa4Hae6mt71Mfz9RfU1vp8YKJWoJJa3MEYZWDOs2ke9n14wys3paHDb2j8WwdhNZK4bNtWm8CCGhFBH3S4WtbNHUmUMJANVmnh5kSCj1e5TU9G4WqisfNZOzdT3D4ptGxNBBj2f4yDt08mjTYlgfJJQiIfliAd4SuOGXhunOmzR309uQ6NWdhqSq1qDsURPdqZGDOcDjeJRtNihGVYV+z7Dwmmli2KErXwS13qGjO4lhAyChFAlr/nNZcB6NhukIlrkHc+wMeZR17nOvtBJAjSfoat/IUe+2EhSjqqJmcrXOAEpOTTO5vq4YmaP+JnL7boBjJxJDM0NCKRIaogeM6TZNm6uPsm7Te0dqLgDARmoFq0Z/SWvv693C41FWVQAPcvRMrcmp8QyNimFX3SZy+64khs0MCaVIuFxQIjiPRu+Ec/MKpdxGv1BeUihxv7QSHRxtAQCfHb4GACivUvO3hWwgLWpf76ryRxOu9XiGyluoXwydajWT63iHDh1JDEUCCaUIKKusZ8JuPTTHYI6dAaGsqNJg29FczI7oYZZ9cmojkQBSqciEUvVQj2f46FDeqj+vrbPu/EKtKDq4kRi2AEgoRcCcf5keMag2GqbrQTaHR/mPId2QcDgbO1JzMXVIV+TeexxeLXqwf6PvKYGFljCqHvIFsHbAhhIjkZFsXXQ9Q60w2ncgMWzhkFCKAN8LWxAtlSGXuSOXeeAm64hKyIzmqwmxxvcgzT2Yo08ohwd54peMfNwoLMcfuYW4o6wZyHF3tsWCP/fE71fvNvq+2n5Os8ejVJXpWZf86LxEdytgHnIXXRHUCqN9exLDVgwJpaVhDO9Y74OjpIJL0jAJ8tEB1zXuuM7ckcvccZ15PPrpjgrU9Asy6Hpc5vbA9DW9ZdZW6OPlghuF5ci99xC592sCYIzs1xlSK4k5uigb51FWltY0k+sOntzPBkoV9efVimHdJnKHboBdOxLDNgoJpaVRV2GH+iX4SgrgJymAr6QATpJydME9dJHew9O4oJPlNmuP68wd0p97oks7f7xsVY3rj0S0Su1g1urZyXSXptlIJfBzq7lP7v0yXMhXAgB6eTqb5Z4SicT4EsbKEr5nWDuUlzExtGunf/Ckfdcaz5Ag6kBCaWmsZVhVPbZWAkMHKB8JpwK+VloBVcBfooCL5CE8JYXwlBQCNzOBm8CWWq30u8wFudJHHijnkXrgOnOHEsJFVF/T20ZqBb8ONRGkrxaUIvN2jVD29jKTUKImHqUDyuFVfgW4kMdvIhdeA0oL6i/Err2BqTX+JIaEYEQhlJs3b8aqVaugUCjQt29fbNy4ESEhIQbt9+zZg8WLFyM3NxcBAQH45z//iT//+c/cdcYY4uLi8MUXX6CoqAhPP/00tmzZgoCAAM6msLAQM2bMwM8//wwrKyu89tprWL9+PRwdHTmbc+fOYdq0afjjjz/QsWNHzJgxA3PmzGmal8AhwX244D5zwSn2BFCny9EFpTUC+sgDHd21EneuZ8JXUoAOkhJ0lBSjo6QYA3EFqKNxhcyR14TP0Xhw50VwhL55PYaEsp93OwBA6rX7j+ys4O8mXIgd8ZB7Fj+JAn4SBTz+tRavFl7D6/L7wC0Aewxktmuvp4n8SBTt2gmuC0EYwuJCuWvXLsTExCAhIQGhoaFYt24dIiIicPnyZXTq1EnH/tixYxg3bhzi4+PxyiuvYOfOnYiMjMSpU6fQp08fAMDKlSuxYcMG7NixA/7+/li8eDEiIiJw8eJFyOVyAMD48eNx+/ZtJCcno6qqCpMnT8aUKVOwc+dOAIBSqcRLL72E8PBwJCQkICMjA2+88QZcXV0xZcqU5ntBdSiGI86y7jjLugMAfPv1xftXzwIAnFEGn1pN+NoeaSdJEdpLStFekoX+yNItl9lznmcuc8d1TY2Auqi9ATBYSSRc6EtrqQRPuDvC380BOY8C9IZ17QBraU0zXVKnH88RDx+J4CMxtHpUN4kCHSVK3YesNaaitHKBs1cPPd6hP4kh0WxImLk2Y24goaGhGDhwIDZt2gQA0Gg08Pb2xowZMzBv3jwd+zFjxqCsrAz79+/n0p566in069cPCQkJYIzBy8sL77//PmbPng0AKC4uhru7O7Zv346xY8ciMzMTvXr1wh9//IEBAwYAAJKSkvDnP/8ZN2/ehJeXF7Zs2YKFCxdCoVBAJqtp286bNw979+7FpUuXTHo2pVIJFxcXFBcXw9nZcLPUb94vpr0sPXz0ah8s/PG8UTt7VMC3toBqvTgrRU0zvh5KmRx58ECOphOuMw9MeuV52HsEYE+ODeb8+w4YrPDxyz54vXsVUHgNOVcycOpMOncvN31iWIt7zPmxSGvcEfXKCzhf3gHv/qcYfQN88U10qKB3QhCmYOr3E7CwR6lSqZCeno758+dzaVZWVggPD0dqaqrePKmpqYiJ4W9DGRERgb179wIAcnJyoFAoEB4ezl13cXFBaGgoUlNTMXbsWKSmpsLV1ZUTSQAIDw+HlZUV0tLS8OqrryI1NRXPPvssJ5La+/zzn//EgwcP0K6drjdTWVmJyspK7lyprF8gzMH6A1dNsnsIOTKZLzKZr841OSrhI7nDE1DfR55fZ8l9OEoq0Au56CXNrcmQtA8AMArACDtbPGQytDtYAhysuewPwL9Oi/0uc37UzK/pO83lugA8UAL+jnmjewxFyY0iKHEa6dcfYNi63wW8EYIAxj/liwlP6f6tNxSLCuW9e/egVqvh7u7OS3d3dzfotSkUCr32CoWCu65Nq8+mbrPe2toa7du359n4+/vrlKG9pk8o4+PjsWTJEsMPbAB3Z1sUKCvxeqgPdqblCcp7p6RGmJ/s4oJzN4sF3xsAKmCLK8wbV5g3Lz28pztWjHwCb67/F0Z6V8Cp/AY6Vd3Csx1KuMjXtqwStnj0z8GhE9C+K5QOPvgsA5wY5jF3HTGsj45OtvBpX2P/UKXGJYXw5Z1E2+ZeSaVxIwFYvI+yNTF//nyet6tUKuHt7V1Pjhr++/5zKCxTwbu9PV550hPuznLYy6SQSa1w80E57GRSFD2sws0HD2Evk8JOZo2ihyp0cKiZTym3sUJ/n3a4pFCi6GEVFMUVOH3jAbq6OSLA3RFllWq82Msd90srcSz7PlTVGtjaWEEmtULO/TJUVmngam8DVbUGWXdKMdCvPUYN6ML1Ne5ZFAUbqZ4I1tq9VFRlNXspy2uaL84AXnuhFJm3S1BWWY3u7o64oihBUBcXqDUM+UUVsLWxgpeLHZQVVbCzkUJuY4Wih1Xw6WAPuY0Ufb1dkfzesyhQmvcPnmgbaP/RmguLCqWbmxukUikKCvhTPQoKCuDh4aE3j4eHR7322p8FBQXw9PTk2fTr14+zuXPnDq+M6upqFBYW8srRd5/a96iLra0tbG1tDT6vIRxsreHwKO7joG5uvGvaoBMAEOJf/7SW3l4u3O+vBXfRud7JWY7I/p0F10+vSAKA1KZmgEUPXTs6omvHxzMI/uTz2AN/Urdqeglwd0KAu5PJ9SSIpsKiG13IZDIEBwcjJSWFS9NoNEhJSUFYWJjePGFhYTx7AEhOTubs/f394eHhwbNRKpVIS0vjbMLCwlBUVIT09HTO5r///S80Gg1CQ0M5m99//x1VVVW8+/To0UNvs5sgiFYMszCJiYnM1taWbd++nV28eJFNmTKFubq6MoVCwRhjbMKECWzevHmc/dGjR5m1tTVbvXo1y8zMZHFxcczGxoZlZGRwNitWrGCurq7sp59+YufOnWMjR45k/v7+rLy8nLMZNmwY69+/P0tLS2NHjhxhAQEBbNy4cdz1oqIi5u7uziZMmMDOnz/PEhMTmb29Pfvss89Mfrbi4mIGgBUXFzfmFREE0QQI+X5aXCgZY2zjxo3Mx8eHyWQyFhISwo4fP85dGzJkCIuKiuLZ7969mz3xxBNMJpOx3r17s19++YV3XaPRsMWLFzN3d3dma2vLXnjhBXb58mWezf3799m4ceOYo6Mjc3Z2ZpMnT2YlJSU8m7Nnz7LBgwczW1tb1rlzZ7ZixQpBz0VCSRDiRcj30+LzKFszQuZpEQTRvAj5frbuzXgJgiDMAAklQRCEEUgoCYIgjEATzpsQbfdvcyxlJAhCGNrvpSnDNCSUTUhJSc3SO1NW5xAEYRlKSkrg4uJSrw2NejchGo0G+fn5cHJy0gk91lrQLtO8ceNGmx/Zp3dRQ0t5D4wxlJSUwMvLC1ZW9fdCkkfZhFhZWaFLFxPX67VwnJ2dRf2laE7oXdTQEt6DMU9SCw3mEARBGIGEkiAIwggklESjsLW1RVxcXIOiJrU26F3U0BrfAw3mEARBGIE8SoIgCCOQUBIEQRiBhJIgCMIIJJQEQRBGIKEkjLJ582b4+flBLpcjNDQUJ06cqNd+z549CAwMhFwuR1BQEH799ddmqmnTI+RdfPHFF3jmmWfQrl07tGvXDuHh4UbfXUtB6N+ElsTEREgkEkRGRjZtBc1N08UPJloDiYmJTCaTsa1bt7ILFy6wt956i7m6urKCggK99kePHmVSqZStXLmSXbx4kS1atEhnq46WitB38frrr7PNmzez06dPs8zMTDZp0iTm4uLCbt682cw1Ny9C34OWnJwc1rlzZ/bMM8+wkSNHNk9lzQQJJVEvISEhbNq0ady5Wq1mXl5eLD4+Xq/96NGj2fDhw3lpoaGhbOrUqU1az+ZA6LuoS3V1NXNycmI7duxoqio2Cw15D9XV1WzQoEHsyy+/ZFFRUS1OKKnpTRhEpVIhPT0d4eHhXJqVlRXCw8ORmpqqN09qairPHgAiIiIM2rcUGvIu6vLw4UNUVVWhffv6tx0WMw19D0uXLkWnTp0QHR3dHNU0OxQUgzDIvXv3oFar4e7uzkt3d3fHpUuX9OZRKBR67RUKRZPVszloyLuoy9y5c+Hl5aXzj6Ql0ZD3cOTIEXz11Vc4c+ZMM9SwaSChJIhmYMWKFUhMTMShQ4cgl8stXZ1mo6SkBBMmTMAXX3wBNzc3S1enwZBQEgZxc3ODVCpFQUEBL72goAAeHh5683h4eAiybyk05F1oWb16NVasWIEDBw7gySefbMpqNjlC30N2djZyc3Pxl7/8hUvTaDQAAGtra1y+fBndunVr2kqbAeqjJAwik8kQHByMlJQULk2j0SAlJQVhYWF684SFhfHsASA5OdmgfUuhIe8CAFauXIlly5YhKSkJAwYMaI6qNilC30NgYCAyMjJw5swZ7hgxYgSGDh2KM2fOtJzo/5YeTSLETWJiIrO1tWXbt29nFy9eZFOmTGGurq5MoVAwxhibMGECmzdvHmd/9OhRZm1tzVavXs0yMzNZXFxcq5oeJORdrFixgslkMvb999+z27dvc0dJSYmlHsEsCH0PdWmJo94klIRRNm7cyHx8fJhMJmMhISHs+PHj3LUhQ4awqKgonv3u3bvZE088wWQyGevduzf75ZdfmrnGTYeQd+Hr68sA6BxxcXHNX3EzI/RvojYtUSgpzBpBEIQRqI+SIAjCCCSUBEEQRiChJAiCMAIJJUEQhBFIKAmCIIxAQkkQBGEEEkqCIAgjkFASBEEYgYSSaFNMmjQJEolE5xg2bJilq0aIGIoeRLQ5hg0bhm3btvHSbG1t9dpWVVXBxsaGl6ZSqSCTyQTft6H5CMtDHiXR5rC1tYWHhwfvaNeuHQBAIpFgy5YtGDFiBBwcHPDRRx/hww8/RL9+/fDll1/C39+fiyeZl5eHkSNHwtHREc7Ozhg9ejQv/JihfETLg4SSIOrw4Ycf4tVXX0VGRgbeeOMNAEBWVhb+9a9/4YcffsCZM2eg0WgwcuRIFBYW4vDhw0hOTsa1a9cwZswYXll18xEtE2p6E22O/fv3w9HRkZe2YMECLFiwAADw+uuvY/LkybzrKpUKX3/9NTp27AigJsZmRkYGcnJyuJiKX3/9NXr37o0//vgDAwcO1JuPaJmQUBJtjqFDh2LLli28tNobfukLsOvr68sTu8zMTHh7e/MCz/bq1Quurq7IzMzkhLJuPqJlQkJJtDkcHBzQvXv3eq+bkmbqvYiWD/VREkQD6NmzJ27cuIEbN25waRcvXkRRURF69eplwZoRTQF5lESbo7KyUmf7XGtra0G7BIaHhyMoKAjjx4/HunXrUF1djXfeeQdDhgxpFXvjEHzIoyTaHElJSfD09OQdgwcPFlSGRCLBTz/9hHbt2uHZZ59FeHg4unbtil27djVRrQlLQltBEARBGIE8SoIgCCOQUBIEQRiBhJIgCMIIJJQEQRBGIKEkCIIwAgklQRCEEUgoCYIgjEBCSRAEYQQSSoIgCCOQUBIEQRiBhJIgCMIIJJQEQRBG+H+BvVAc11deGAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019946, + "end_time": "2024-02-29T18:44:34.084886", + "exception": false, + "start_time": "2024-02-29T18:44:34.064940", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1067.648958, + "end_time": "2024-02-29T18:44:36.826930", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tab_ddpm_concat/4/mlu-eval.ipynb", + "output_path": "eval/insurance/tab_ddpm_concat/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/insurance/tab_ddpm_concat/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T18:26:49.177972", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/model.pt b/insurance/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f6f04ce47ef2dc4e18acdc9dfefa95121729889 --- /dev/null +++ b/insurance/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11257d861ce29a41abfcdf4cdcbf17964078480c9fc41b54d90eec20c5e9e4e8 +size 38511671 diff --git a/insurance/tab_ddpm_concat/params.json b/insurance/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..ec8a95982691e544c0689d90d1107152b47aefe8 --- /dev/null +++ b/insurance/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.77, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.75, "loss_balancer_r": 0.95, "fixed_role_model": "tab_ddpm_concat", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": false, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "relu6", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "rrelu", "head_activation_final": "softsign", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tvae/eval.csv b/insurance/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..13636cdcc4db6bed59e5302c6fc6b21bc5e58ba9 --- /dev/null +++ b/insurance/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.13478357570810726,0.03636767101219685,0.000275009540043874,0.5683860778808594,0.018824299797415733,0.6825659275054932,0.0344335213303566,1.3921320984877639e-08,0.8837041854858398,0.01289679016917944,0.1385168433189392,0.016583411023020744,0.15040378272533417,0.0008385563851334155,1.4520902633666992 diff --git a/insurance/tvae/history.csv b/insurance/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..feb9efb9c1fccf4b72045a21725ceefe8c9a80b9 --- /dev/null +++ b/insurance/tvae/history.csv @@ -0,0 +1,23 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.05458167113538366,4.561985811768864,0.023550471702759836,0.0,0.0,0.0,0.0,0.0,0.05458167113538366,320,40,39.33485436439514,0.9833713591098785,0.12292141988873481,0.12334999229060487,0.01106511988909915,7.3775769050087545,0.00039846873109325995,0.0,0.0,0.0,0.0,0.0,0.01106511988909915,80,10,8.30074167251587,0.8300741672515869,0.10375927090644836,0.04176213040482253 +1,0.010921533098735382,3.7708608118317897,0.0006090835865870864,0.0,0.0,0.0,0.0,0.0,0.010921533098735382,320,40,38.92951965332031,0.9732379913330078,0.12165474891662598,0.07502402040408924,0.002461973318713717,0.2656627141033823,8.382165523856955e-06,0.0,0.0,0.0,0.0,0.0,0.002461973318713717,80,10,8.352109670639038,0.8352109670639039,0.10440137088298798,0.07963283583521844 +2,0.004752650485897902,4.5005246672456,7.41932757047259e-05,0.0,0.0,0.0,0.0,0.0,0.004752650485897902,320,40,39.09458088874817,0.9773645222187042,0.12217056527733802,0.0816895533236675,0.0009612412060960196,0.23112409779214432,2.9920761611550857e-06,0.0,0.0,0.0,0.0,0.0,0.0009612412060960196,80,10,8.301867723464966,0.8301867723464966,0.10377334654331208,0.08093988439068198 +3,0.0029934452861198222,1.4091149369219238,3.706777377407988e-05,0.0,0.0,0.0,0.0,0.0,0.0029934452861198222,320,40,39.057528257369995,0.9764382064342498,0.12205477580428123,0.08644149880856275,0.0017080451536457986,0.5054739748910834,2.0698145459556274e-06,0.0,0.0,0.0,0.0,0.0,0.0017080451536457986,80,10,8.39680528640747,0.839680528640747,0.10496006608009338,0.0637943553738296 +4,0.0022114409464847997,1.4571088086362807,9.073904502000795e-06,0.0,0.0,0.0,0.0,0.0,0.0022114409464847997,320,40,38.94040822982788,0.973510205745697,0.12168877571821213,0.08093992052599788,0.0034676186623983085,0.354912094264597,1.1135635656955855e-05,0.0,0.0,0.0,0.0,0.0,0.0034676186623983085,80,10,8.30032467842102,0.830032467842102,0.10375405848026276,0.10819654231891036 +5,0.0016322427756676916,0.8344269889868698,2.8054938205387956e-06,0.0,0.0,0.0,0.0,0.0,0.0016322427756676916,320,40,39.14137244224548,0.9785343110561371,0.12231678888201714,0.09135764897800983,0.0034494245337555185,2.7931900787574704,6.050434956339501e-06,0.0,0.0,0.0,0.0,0.0,0.0034494245337555185,80,10,8.366892337799072,0.8366892337799072,0.1045861542224884,0.055338869569823146 +6,0.002849590677578817,0.8129531741204119,4.8211906484207924e-05,0.0,0.0,0.0,0.0,0.0,0.002849590677578817,320,40,38.968292236328125,0.9742073059082031,0.1217759132385254,0.0901852805633098,0.0025212633569026365,0.6178526908131061,1.4414640320481454e-06,0.0,0.0,0.0,0.0,0.0,0.0025212633569026365,80,10,8.266654014587402,0.8266654014587402,0.10333317518234253,0.05773084256798029 +7,0.0034268524424987843,1.5629836895840525,1.5161425290398687e-05,0.0,0.0,0.0,0.0,0.0,0.0034268524424987843,320,40,39.01256036758423,0.9753140091896058,0.12191425114870072,0.0829970414401032,0.0014418774226214737,0.05386366389284376,2.4331560492640845e-06,0.0,0.0,0.0,0.0,0.0,0.0014418774226214737,80,10,8.32570481300354,0.832570481300354,0.10407131016254426,0.08880755109712482 +8,0.0016761758448410546,0.571136603817564,7.011435811053887e-06,0.0,0.0,0.0,0.0,0.0,0.0016761758448410546,320,40,38.852670669555664,0.9713167667388916,0.12141459584236144,0.09045831263065338,0.0006263804327318212,0.24181758030463243,6.295153740953907e-07,0.0,0.0,0.0,0.0,0.0,0.0006263804327318212,80,10,8.345160722732544,0.8345160722732544,0.1043145090341568,0.08191414531320333 +9,0.0008744017197386711,0.17949836104246067,4.63043962907906e-07,0.0,0.0,0.0,0.0,0.0,0.0008744017197386711,320,40,39.11712980270386,0.9779282450675965,0.12224103063344956,0.09466907754540443,0.0011390350133297033,0.004834387120854444,3.0137808032293377e-06,0.0,0.0,0.0,0.0,0.0,0.0011390350133297033,80,10,8.316069841384888,0.8316069841384888,0.1039508730173111,0.0990539627149701 +10,0.0004748740824652486,0.1777749692730623,2.3089836414527056e-08,0.0,0.0,0.0,0.0,0.0,0.0004748740824652486,320,40,39.06293201446533,0.9765733003616333,0.12207166254520416,0.09201494687004015,0.00032443252712255344,0.0010629200933180982,3.426863805611191e-07,0.0,0.0,0.0,0.0,0.0,0.00032443252712255344,80,10,8.351998329162598,0.8351998329162598,0.10439997911453247,0.0884638118557632 +11,0.00030916042924218347,0.04881817966124018,2.0088672352989394e-08,0.0,0.0,0.0,0.0,0.0,0.00030916042924218347,320,40,38.86811137199402,0.9717027842998505,0.12146284803748131,0.10129309091717005,0.00028257269877940416,1.0737754437432159,2.8357685949442768e-08,0.0,0.0,0.0,0.0,0.0,0.00028257269877940416,80,10,8.252684354782104,0.8252684354782105,0.10315855443477631,0.08038602282758803 +12,0.0013487103491570452,0.43372808683234754,1.0047366970687786e-06,0.0,0.0,0.0,0.0,0.0,0.0013487103491570452,320,40,39.09407997131348,0.9773519992828369,0.12216899991035461,0.0899976636399515,0.003439919964876026,0.015614798056776635,2.0251521429592856e-05,0.0,0.0,0.0,0.0,0.0,0.003439919964876026,80,10,8.342942476272583,0.8342942476272583,0.1042867809534073,0.11240037991665304 +13,0.0008618889094577753,0.1384840221481113,4.446653539750059e-07,0.0,0.0,0.0,0.0,0.0,0.0008618889094577753,320,40,38.886531829833984,0.9721632957458496,0.1215204119682312,0.09283134532161057,0.000532695987567422,0.6308531300281175,1.360833180625437e-06,0.0,0.0,0.0,0.0,0.0,0.000532695987567422,80,10,8.28925633430481,0.828925633430481,0.10361570417881012,0.0891546759288758 +14,0.00030356911156559363,0.3619111133062688,5.198837278813596e-08,0.0,0.0,0.0,0.0,0.0,0.00030356911156559363,320,40,38.988784074783325,0.9747196018695832,0.1218399502336979,0.09746413570828735,0.0005432431978988461,0.001004549844947178,2.781989758560144e-07,0.0,0.0,0.0,0.0,0.0,0.0005432431978988461,80,10,8.42578673362732,0.842578673362732,0.1053223341703415,0.09348368076607586 +15,0.00029625174347529536,0.0572095896306493,6.03426183574306e-08,0.0,0.0,0.0,0.0,0.0,0.00029625174347529536,320,40,39.27268958091736,0.981817239522934,0.12272715494036675,0.09890737304231152,0.00036838351952610536,0.7212186768025276,2.6941624464704718e-08,0.0,0.0,0.0,0.0,0.0,0.00036838351952610536,80,10,8.407346963882446,0.8407346963882446,0.10509183704853058,0.08277125156018883 +16,0.0005824315209792986,0.32841089839253074,9.46431564320671e-08,0.0,0.0,0.0,0.0,0.0,0.0005824315209792986,320,40,39.24979209899902,0.9812448024749756,0.12265560030937195,0.09515800991794095,0.0007761759276036173,1.2551941490234082,2.9720144345546373e-08,0.0,0.0,0.0,0.0,0.0,0.0007761759276036173,80,10,8.318324089050293,0.8318324089050293,0.10397905111312866,0.09093907248461619 +17,0.0012332158104982228,0.6590279597393532,8.36102873709775e-06,0.0,0.0,0.0,0.0,0.0,0.0012332158104982228,320,40,39.119892835617065,0.9779973208904267,0.12224966511130334,0.0890660552540794,0.001825959722918924,1.9564546512207017,1.1242688799484313e-06,0.0,0.0,0.0,0.0,0.0,0.001825959722918924,80,10,8.359159708023071,0.8359159708023072,0.1044894963502884,0.06533113070763648 +18,0.001502491006613127,0.4666562590015076,3.179587947280127e-06,0.0,0.0,0.0,0.0,0.0,0.001502491006613127,320,40,39.09075927734375,0.9772689819335938,0.12215862274169922,0.09002331190858967,0.0008729565364774316,0.20973070683976403,2.998759428507469e-06,0.0,0.0,0.0,0.0,0.0,0.0008729565364774316,80,10,8.317886352539062,0.8317886352539062,0.10397357940673828,0.08415974881500006 +19,0.001246437881127349,0.6120949116166994,2.0787087329172948e-06,0.0,0.0,0.0,0.0,0.0,0.001246437881127349,320,40,39.05946326255798,0.9764865815639496,0.1220608226954937,0.09171894917380996,0.002248370127927046,4.686978222953622,2.895269359082242e-06,0.0,0.0,0.0,0.0,0.0,0.002248370127927046,80,10,8.319932222366333,0.8319932222366333,0.10399915277957916,0.0715100662317127 +20,0.0029011325517785736,0.9372176351432528,1.1204305509332328e-05,0.0,0.0,0.0,0.0,0.0,0.0029011325517785736,320,40,38.93023109436035,0.9732557773590088,0.1216569721698761,0.08712862803367898,0.0013807336257741555,2.265311992234274,1.8232192309453056e-06,0.0,0.0,0.0,0.0,0.0,0.0013807336257741555,80,10,8.43259859085083,0.843259859085083,0.10540748238563538,0.07126395150553436 +21,0.0005831055974340416,0.7094658932399625,5.484142581780628e-08,0.0,0.0,0.0,0.0,0.0,0.0005831055974340416,320,40,38.932344913482666,0.9733086228370667,0.12166357785463333,0.09576541467686184,0.00040745751502981877,0.01960964320030456,8.632079813164495e-08,0.0,0.0,0.0,0.0,0.0,0.00040745751502981877,80,10,8.343457698822021,0.8343457698822021,0.10429322123527526,0.0816122055053711 diff --git a/insurance/tvae/mlu-eval.ipynb b/insurance/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..9fb47b49df5fb84967591679c13a372ab9ce28be --- /dev/null +++ b/insurance/tvae/mlu-eval.ipynb @@ -0,0 +1,2617 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.297106Z", + "iopub.status.busy": "2024-02-29T17:50:43.296820Z", + "iopub.status.idle": "2024-02-29T17:50:43.328516Z", + "shell.execute_reply": "2024-02-29T17:50:43.327846Z" + }, + "papermill": { + "duration": 0.045827, + "end_time": "2024-02-29T17:50:43.330441", + "exception": false, + "start_time": "2024-02-29T17:50:43.284614", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.355733Z", + "iopub.status.busy": "2024-02-29T17:50:43.354986Z", + "iopub.status.idle": "2024-02-29T17:50:43.361827Z", + "shell.execute_reply": "2024-02-29T17:50:43.360970Z" + }, + "papermill": { + "duration": 0.021501, + "end_time": "2024-02-29T17:50:43.363729", + "exception": false, + "start_time": "2024-02-29T17:50:43.342228", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.386989Z", + "iopub.status.busy": "2024-02-29T17:50:43.386508Z", + "iopub.status.idle": "2024-02-29T17:50:43.390341Z", + "shell.execute_reply": "2024-02-29T17:50:43.389543Z" + }, + "papermill": { + "duration": 0.017484, + "end_time": "2024-02-29T17:50:43.392283", + "exception": false, + "start_time": "2024-02-29T17:50:43.374799", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.415937Z", + "iopub.status.busy": "2024-02-29T17:50:43.415284Z", + "iopub.status.idle": "2024-02-29T17:50:43.419146Z", + "shell.execute_reply": "2024-02-29T17:50:43.418407Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017482, + "end_time": "2024-02-29T17:50:43.421010", + "exception": false, + "start_time": "2024-02-29T17:50:43.403528", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.444561Z", + "iopub.status.busy": "2024-02-29T17:50:43.444080Z", + "iopub.status.idle": "2024-02-29T17:50:43.449576Z", + "shell.execute_reply": "2024-02-29T17:50:43.448694Z" + }, + "papermill": { + "duration": 0.019709, + "end_time": "2024-02-29T17:50:43.451711", + "exception": false, + "start_time": "2024-02-29T17:50:43.432002", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "eb7d978f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.479388Z", + "iopub.status.busy": "2024-02-29T17:50:43.479092Z", + "iopub.status.idle": "2024-02-29T17:50:43.484500Z", + "shell.execute_reply": "2024-02-29T17:50:43.483600Z" + }, + "papermill": { + "duration": 0.021864, + "end_time": "2024-02-29T17:50:43.486766", + "exception": false, + "start_time": "2024-02-29T17:50:43.464902", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tvae/4\"\n", + "param_index = 2\n", + "allow_same_prediction = True\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011535, + "end_time": "2024-02-29T17:50:43.510177", + "exception": false, + "start_time": "2024-02-29T17:50:43.498642", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.535341Z", + "iopub.status.busy": "2024-02-29T17:50:43.534859Z", + "iopub.status.idle": "2024-02-29T17:50:43.544013Z", + "shell.execute_reply": "2024-02-29T17:50:43.543203Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023834, + "end_time": "2024-02-29T17:50:43.545888", + "exception": false, + "start_time": "2024-02-29T17:50:43.522054", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tvae/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:43.570861Z", + "iopub.status.busy": "2024-02-29T17:50:43.570608Z", + "iopub.status.idle": "2024-02-29T17:50:45.715341Z", + "shell.execute_reply": "2024-02-29T17:50:45.714481Z" + }, + "papermill": { + "duration": 2.159597, + "end_time": "2024-02-29T17:50:45.717481", + "exception": false, + "start_time": "2024-02-29T17:50:43.557884", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:45.744285Z", + "iopub.status.busy": "2024-02-29T17:50:45.743770Z", + "iopub.status.idle": "2024-02-29T17:50:45.755553Z", + "shell.execute_reply": "2024-02-29T17:50:45.754865Z" + }, + "papermill": { + "duration": 0.027328, + "end_time": "2024-02-29T17:50:45.757462", + "exception": false, + "start_time": "2024-02-29T17:50:45.730134", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:45.780934Z", + "iopub.status.busy": "2024-02-29T17:50:45.780680Z", + "iopub.status.idle": "2024-02-29T17:50:45.787778Z", + "shell.execute_reply": "2024-02-29T17:50:45.787084Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.020988, + "end_time": "2024-02-29T17:50:45.789578", + "exception": false, + "start_time": "2024-02-29T17:50:45.768590", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:45.813449Z", + "iopub.status.busy": "2024-02-29T17:50:45.812848Z", + "iopub.status.idle": "2024-02-29T17:50:45.911458Z", + "shell.execute_reply": "2024-02-29T17:50:45.910597Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.112764, + "end_time": "2024-02-29T17:50:45.913541", + "exception": false, + "start_time": "2024-02-29T17:50:45.800777", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:45.939348Z", + "iopub.status.busy": "2024-02-29T17:50:45.939072Z", + "iopub.status.idle": "2024-02-29T17:50:50.480136Z", + "shell.execute_reply": "2024-02-29T17:50:50.479366Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.556435, + "end_time": "2024-02-29T17:50:50.482677", + "exception": false, + "start_time": "2024-02-29T17:50:45.926242", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 17:50:48.176006: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 17:50:48.176066: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 17:50:48.177642: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:50.508054Z", + "iopub.status.busy": "2024-02-29T17:50:50.507215Z", + "iopub.status.idle": "2024-02-29T17:50:50.513947Z", + "shell.execute_reply": "2024-02-29T17:50:50.513275Z" + }, + "papermill": { + "duration": 0.021175, + "end_time": "2024-02-29T17:50:50.515900", + "exception": false, + "start_time": "2024-02-29T17:50:50.494725", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:50.541991Z", + "iopub.status.busy": "2024-02-29T17:50:50.541717Z", + "iopub.status.idle": "2024-02-29T17:50:58.595638Z", + "shell.execute_reply": "2024-02-29T17:50:58.594710Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.069751, + "end_time": "2024-02-29T17:50:58.598157", + "exception": false, + "start_time": "2024-02-29T17:50:50.528406", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.77,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.RReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:59.090944Z", + "iopub.status.busy": "2024-02-29T17:50:59.090399Z", + "iopub.status.idle": "2024-02-29T17:50:59.156842Z", + "shell.execute_reply": "2024-02-29T17:50:59.156038Z" + }, + "papermill": { + "duration": 0.081787, + "end_time": "2024-02-29T17:50:59.158740", + "exception": false, + "start_time": "2024-02-29T17:50:59.076953", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T17:50:59.187387Z", + "iopub.status.busy": "2024-02-29T17:50:59.187119Z", + "iopub.status.idle": "2024-02-29T17:50:59.608009Z", + "shell.execute_reply": "2024-02-29T17:50:59.607099Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.438163, + "end_time": "2024-02-29T17:50:59.610083", + "exception": false, + "start_time": "2024-02-29T17:50:59.171920", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:59.638580Z", + "iopub.status.busy": "2024-02-29T17:50:59.638286Z", + "iopub.status.idle": "2024-02-29T17:50:59.642446Z", + "shell.execute_reply": "2024-02-29T17:50:59.641637Z" + }, + "papermill": { + "duration": 0.020458, + "end_time": "2024-02-29T17:50:59.644327", + "exception": false, + "start_time": "2024-02-29T17:50:59.623869", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:59.670684Z", + "iopub.status.busy": "2024-02-29T17:50:59.670408Z", + "iopub.status.idle": "2024-02-29T17:50:59.676722Z", + "shell.execute_reply": "2024-02-29T17:50:59.675923Z" + }, + "papermill": { + "duration": 0.021573, + "end_time": "2024-02-29T17:50:59.678594", + "exception": false, + "start_time": "2024-02-29T17:50:59.657021", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9638529" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:59.705226Z", + "iopub.status.busy": "2024-02-29T17:50:59.704962Z", + "iopub.status.idle": "2024-02-29T17:50:59.791397Z", + "shell.execute_reply": "2024-02-29T17:50:59.790570Z" + }, + "papermill": { + "duration": 0.101763, + "end_time": "2024-02-29T17:50:59.793213", + "exception": false, + "start_time": "2024-02-29T17:50:59.691450", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 36] --\n", + "├─Adapter: 1-1 [2, 1071, 36] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 37,888\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 36] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─RReLU: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,638,529\n", + "Trainable params: 9,638,529\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.18\n", + "========================================================================================================================\n", + "Input size (MB): 0.39\n", + "Forward/backward pass size (MB): 307.47\n", + "Params size (MB): 38.55\n", + "Estimated Total Size (MB): 346.41\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:50:59.822160Z", + "iopub.status.busy": "2024-02-29T17:50:59.821866Z", + "iopub.status.idle": "2024-02-29T18:10:05.205760Z", + "shell.execute_reply": "2024-02-29T18:10:05.204793Z" + }, + "papermill": { + "duration": 1145.418287, + "end_time": "2024-02-29T18:10:05.225558", + "exception": false, + "start_time": "2024-02-29T17:50:59.807271", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.05458167113538366, 'avg_role_model_std_loss': 4.561985811768864, 'avg_role_model_mean_pred_loss': 0.023550471702759836, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.05458167113538366, 'n_size': 320, 'n_batch': 40, 'duration': 39.33485436439514, 'duration_batch': 0.9833713591098785, 'duration_size': 0.12292141988873481, 'avg_pred_std': 0.12334999229060487}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01106511988909915, 'avg_role_model_std_loss': 7.3775769050087545, 'avg_role_model_mean_pred_loss': 0.00039846873109325995, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01106511988909915, 'n_size': 80, 'n_batch': 10, 'duration': 8.30074167251587, 'duration_batch': 0.8300741672515869, 'duration_size': 0.10375927090644836, 'avg_pred_std': 0.04176213040482253}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010921533098735382, 'avg_role_model_std_loss': 3.7708608118317897, 'avg_role_model_mean_pred_loss': 0.0006090835865870864, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010921533098735382, 'n_size': 320, 'n_batch': 40, 'duration': 38.92951965332031, 'duration_batch': 0.9732379913330078, 'duration_size': 0.12165474891662598, 'avg_pred_std': 0.07502402040408924}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002461973318713717, 'avg_role_model_std_loss': 0.2656627141033823, 'avg_role_model_mean_pred_loss': 8.382165523856955e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002461973318713717, 'n_size': 80, 'n_batch': 10, 'duration': 8.352109670639038, 'duration_batch': 0.8352109670639039, 'duration_size': 0.10440137088298798, 'avg_pred_std': 0.07963283583521844}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004752650485897902, 'avg_role_model_std_loss': 4.5005246672456, 'avg_role_model_mean_pred_loss': 7.41932757047259e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004752650485897902, 'n_size': 320, 'n_batch': 40, 'duration': 39.09458088874817, 'duration_batch': 0.9773645222187042, 'duration_size': 0.12217056527733802, 'avg_pred_std': 0.0816895533236675}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009612412060960196, 'avg_role_model_std_loss': 0.23112409779214432, 'avg_role_model_mean_pred_loss': 2.9920761611550857e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009612412060960196, 'n_size': 80, 'n_batch': 10, 'duration': 8.301867723464966, 'duration_batch': 0.8301867723464966, 'duration_size': 0.10377334654331208, 'avg_pred_std': 0.08093988439068198}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029934452861198222, 'avg_role_model_std_loss': 1.4091149369219238, 'avg_role_model_mean_pred_loss': 3.706777377407988e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029934452861198222, 'n_size': 320, 'n_batch': 40, 'duration': 39.057528257369995, 'duration_batch': 0.9764382064342498, 'duration_size': 0.12205477580428123, 'avg_pred_std': 0.08644149880856275}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017080451536457986, 'avg_role_model_std_loss': 0.5054739748910834, 'avg_role_model_mean_pred_loss': 2.0698145459556274e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017080451536457986, 'n_size': 80, 'n_batch': 10, 'duration': 8.39680528640747, 'duration_batch': 0.839680528640747, 'duration_size': 0.10496006608009338, 'avg_pred_std': 0.0637943553738296}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0022114409464847997, 'avg_role_model_std_loss': 1.4571088086362807, 'avg_role_model_mean_pred_loss': 9.073904502000795e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022114409464847997, 'n_size': 320, 'n_batch': 40, 'duration': 38.94040822982788, 'duration_batch': 0.973510205745697, 'duration_size': 0.12168877571821213, 'avg_pred_std': 0.08093992052599788}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0034676186623983085, 'avg_role_model_std_loss': 0.354912094264597, 'avg_role_model_mean_pred_loss': 1.1135635656955855e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034676186623983085, 'n_size': 80, 'n_batch': 10, 'duration': 8.30032467842102, 'duration_batch': 0.830032467842102, 'duration_size': 0.10375405848026276, 'avg_pred_std': 0.10819654231891036}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016322427756676916, 'avg_role_model_std_loss': 0.8344269889868698, 'avg_role_model_mean_pred_loss': 2.8054938205387956e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016322427756676916, 'n_size': 320, 'n_batch': 40, 'duration': 39.14137244224548, 'duration_batch': 0.9785343110561371, 'duration_size': 0.12231678888201714, 'avg_pred_std': 0.09135764897800983}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0034494245337555185, 'avg_role_model_std_loss': 2.7931900787574704, 'avg_role_model_mean_pred_loss': 6.050434956339501e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034494245337555185, 'n_size': 80, 'n_batch': 10, 'duration': 8.366892337799072, 'duration_batch': 0.8366892337799072, 'duration_size': 0.1045861542224884, 'avg_pred_std': 0.055338869569823146}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002849590677578817, 'avg_role_model_std_loss': 0.8129531741204119, 'avg_role_model_mean_pred_loss': 4.8211906484207924e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002849590677578817, 'n_size': 320, 'n_batch': 40, 'duration': 38.968292236328125, 'duration_batch': 0.9742073059082031, 'duration_size': 0.1217759132385254, 'avg_pred_std': 0.0901852805633098}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025212633569026365, 'avg_role_model_std_loss': 0.6178526908131061, 'avg_role_model_mean_pred_loss': 1.4414640320481454e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025212633569026365, 'n_size': 80, 'n_batch': 10, 'duration': 8.266654014587402, 'duration_batch': 0.8266654014587402, 'duration_size': 0.10333317518234253, 'avg_pred_std': 0.05773084256798029}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0034268524424987843, 'avg_role_model_std_loss': 1.5629836895840525, 'avg_role_model_mean_pred_loss': 1.5161425290398687e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034268524424987843, 'n_size': 320, 'n_batch': 40, 'duration': 39.01256036758423, 'duration_batch': 0.9753140091896058, 'duration_size': 0.12191425114870072, 'avg_pred_std': 0.0829970414401032}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0014418774226214737, 'avg_role_model_std_loss': 0.05386366389284376, 'avg_role_model_mean_pred_loss': 2.4331560492640845e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014418774226214737, 'n_size': 80, 'n_batch': 10, 'duration': 8.32570481300354, 'duration_batch': 0.832570481300354, 'duration_size': 0.10407131016254426, 'avg_pred_std': 0.08880755109712482}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016761758448410546, 'avg_role_model_std_loss': 0.571136603817564, 'avg_role_model_mean_pred_loss': 7.011435811053887e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016761758448410546, 'n_size': 320, 'n_batch': 40, 'duration': 38.852670669555664, 'duration_batch': 0.9713167667388916, 'duration_size': 0.12141459584236144, 'avg_pred_std': 0.09045831263065338}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0006263804327318212, 'avg_role_model_std_loss': 0.24181758030463243, 'avg_role_model_mean_pred_loss': 6.295153740953907e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006263804327318212, 'n_size': 80, 'n_batch': 10, 'duration': 8.345160722732544, 'duration_batch': 0.8345160722732544, 'duration_size': 0.1043145090341568, 'avg_pred_std': 0.08191414531320333}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008744017197386711, 'avg_role_model_std_loss': 0.17949836104246067, 'avg_role_model_mean_pred_loss': 4.63043962907906e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008744017197386711, 'n_size': 320, 'n_batch': 40, 'duration': 39.11712980270386, 'duration_batch': 0.9779282450675965, 'duration_size': 0.12224103063344956, 'avg_pred_std': 0.09466907754540443}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011390350133297033, 'avg_role_model_std_loss': 0.004834387120854444, 'avg_role_model_mean_pred_loss': 3.0137808032293377e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011390350133297033, 'n_size': 80, 'n_batch': 10, 'duration': 8.316069841384888, 'duration_batch': 0.8316069841384888, 'duration_size': 0.1039508730173111, 'avg_pred_std': 0.0990539627149701}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004748740824652486, 'avg_role_model_std_loss': 0.1777749692730623, 'avg_role_model_mean_pred_loss': 2.3089836414527056e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004748740824652486, 'n_size': 320, 'n_batch': 40, 'duration': 39.06293201446533, 'duration_batch': 0.9765733003616333, 'duration_size': 0.12207166254520416, 'avg_pred_std': 0.09201494687004015}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00032443252712255344, 'avg_role_model_std_loss': 0.0010629200933180982, 'avg_role_model_mean_pred_loss': 3.426863805611191e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00032443252712255344, 'n_size': 80, 'n_batch': 10, 'duration': 8.351998329162598, 'duration_batch': 0.8351998329162598, 'duration_size': 0.10439997911453247, 'avg_pred_std': 0.0884638118557632}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00030916042924218347, 'avg_role_model_std_loss': 0.04881817966124018, 'avg_role_model_mean_pred_loss': 2.0088672352989394e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00030916042924218347, 'n_size': 320, 'n_batch': 40, 'duration': 38.86811137199402, 'duration_batch': 0.9717027842998505, 'duration_size': 0.12146284803748131, 'avg_pred_std': 0.10129309091717005}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00028257269877940416, 'avg_role_model_std_loss': 1.0737754437432159, 'avg_role_model_mean_pred_loss': 2.8357685949442768e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00028257269877940416, 'n_size': 80, 'n_batch': 10, 'duration': 8.252684354782104, 'duration_batch': 0.8252684354782105, 'duration_size': 0.10315855443477631, 'avg_pred_std': 0.08038602282758803}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013487103491570452, 'avg_role_model_std_loss': 0.43372808683234754, 'avg_role_model_mean_pred_loss': 1.0047366970687786e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013487103491570452, 'n_size': 320, 'n_batch': 40, 'duration': 39.09407997131348, 'duration_batch': 0.9773519992828369, 'duration_size': 0.12216899991035461, 'avg_pred_std': 0.0899976636399515}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003439919964876026, 'avg_role_model_std_loss': 0.015614798056776635, 'avg_role_model_mean_pred_loss': 2.0251521429592856e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003439919964876026, 'n_size': 80, 'n_batch': 10, 'duration': 8.342942476272583, 'duration_batch': 0.8342942476272583, 'duration_size': 0.1042867809534073, 'avg_pred_std': 0.11240037991665304}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008618889094577753, 'avg_role_model_std_loss': 0.1384840221481113, 'avg_role_model_mean_pred_loss': 4.446653539750059e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008618889094577753, 'n_size': 320, 'n_batch': 40, 'duration': 38.886531829833984, 'duration_batch': 0.9721632957458496, 'duration_size': 0.1215204119682312, 'avg_pred_std': 0.09283134532161057}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.000532695987567422, 'avg_role_model_std_loss': 0.6308531300281175, 'avg_role_model_mean_pred_loss': 1.360833180625437e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000532695987567422, 'n_size': 80, 'n_batch': 10, 'duration': 8.28925633430481, 'duration_batch': 0.828925633430481, 'duration_size': 0.10361570417881012, 'avg_pred_std': 0.0891546759288758}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00030356911156559363, 'avg_role_model_std_loss': 0.3619111133062688, 'avg_role_model_mean_pred_loss': 5.198837278813596e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00030356911156559363, 'n_size': 320, 'n_batch': 40, 'duration': 38.988784074783325, 'duration_batch': 0.9747196018695832, 'duration_size': 0.1218399502336979, 'avg_pred_std': 0.09746413570828735}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0005432431978988461, 'avg_role_model_std_loss': 0.001004549844947178, 'avg_role_model_mean_pred_loss': 2.781989758560144e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005432431978988461, 'n_size': 80, 'n_batch': 10, 'duration': 8.42578673362732, 'duration_batch': 0.842578673362732, 'duration_size': 0.1053223341703415, 'avg_pred_std': 0.09348368076607586}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00029625174347529536, 'avg_role_model_std_loss': 0.0572095896306493, 'avg_role_model_mean_pred_loss': 6.03426183574306e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00029625174347529536, 'n_size': 320, 'n_batch': 40, 'duration': 39.27268958091736, 'duration_batch': 0.981817239522934, 'duration_size': 0.12272715494036675, 'avg_pred_std': 0.09890737304231152}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00036838351952610536, 'avg_role_model_std_loss': 0.7212186768025276, 'avg_role_model_mean_pred_loss': 2.6941624464704718e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00036838351952610536, 'n_size': 80, 'n_batch': 10, 'duration': 8.407346963882446, 'duration_batch': 0.8407346963882446, 'duration_size': 0.10509183704853058, 'avg_pred_std': 0.08277125156018883}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005824315209792986, 'avg_role_model_std_loss': 0.32841089839253074, 'avg_role_model_mean_pred_loss': 9.46431564320671e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005824315209792986, 'n_size': 320, 'n_batch': 40, 'duration': 39.24979209899902, 'duration_batch': 0.9812448024749756, 'duration_size': 0.12265560030937195, 'avg_pred_std': 0.09515800991794095}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007761759276036173, 'avg_role_model_std_loss': 1.2551941490234082, 'avg_role_model_mean_pred_loss': 2.9720144345546373e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007761759276036173, 'n_size': 80, 'n_batch': 10, 'duration': 8.318324089050293, 'duration_batch': 0.8318324089050293, 'duration_size': 0.10397905111312866, 'avg_pred_std': 0.09093907248461619}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012332158104982228, 'avg_role_model_std_loss': 0.6590279597393532, 'avg_role_model_mean_pred_loss': 8.36102873709775e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012332158104982228, 'n_size': 320, 'n_batch': 40, 'duration': 39.119892835617065, 'duration_batch': 0.9779973208904267, 'duration_size': 0.12224966511130334, 'avg_pred_std': 0.0890660552540794}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001825959722918924, 'avg_role_model_std_loss': 1.9564546512207017, 'avg_role_model_mean_pred_loss': 1.1242688799484313e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001825959722918924, 'n_size': 80, 'n_batch': 10, 'duration': 8.359159708023071, 'duration_batch': 0.8359159708023072, 'duration_size': 0.1044894963502884, 'avg_pred_std': 0.06533113070763648}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001502491006613127, 'avg_role_model_std_loss': 0.4666562590015076, 'avg_role_model_mean_pred_loss': 3.179587947280127e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001502491006613127, 'n_size': 320, 'n_batch': 40, 'duration': 39.09075927734375, 'duration_batch': 0.9772689819335938, 'duration_size': 0.12215862274169922, 'avg_pred_std': 0.09002331190858967}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008729565364774316, 'avg_role_model_std_loss': 0.20973070683976403, 'avg_role_model_mean_pred_loss': 2.998759428507469e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008729565364774316, 'n_size': 80, 'n_batch': 10, 'duration': 8.317886352539062, 'duration_batch': 0.8317886352539062, 'duration_size': 0.10397357940673828, 'avg_pred_std': 0.08415974881500006}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001246437881127349, 'avg_role_model_std_loss': 0.6120949116166994, 'avg_role_model_mean_pred_loss': 2.0787087329172948e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001246437881127349, 'n_size': 320, 'n_batch': 40, 'duration': 39.05946326255798, 'duration_batch': 0.9764865815639496, 'duration_size': 0.1220608226954937, 'avg_pred_std': 0.09171894917380996}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002248370127927046, 'avg_role_model_std_loss': 4.686978222953622, 'avg_role_model_mean_pred_loss': 2.895269359082242e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002248370127927046, 'n_size': 80, 'n_batch': 10, 'duration': 8.319932222366333, 'duration_batch': 0.8319932222366333, 'duration_size': 0.10399915277957916, 'avg_pred_std': 0.0715100662317127}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029011325517785736, 'avg_role_model_std_loss': 0.9372176351432528, 'avg_role_model_mean_pred_loss': 1.1204305509332328e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029011325517785736, 'n_size': 320, 'n_batch': 40, 'duration': 38.93023109436035, 'duration_batch': 0.9732557773590088, 'duration_size': 0.1216569721698761, 'avg_pred_std': 0.08712862803367898}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0013807336257741555, 'avg_role_model_std_loss': 2.265311992234274, 'avg_role_model_mean_pred_loss': 1.8232192309453056e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013807336257741555, 'n_size': 80, 'n_batch': 10, 'duration': 8.43259859085083, 'duration_batch': 0.843259859085083, 'duration_size': 0.10540748238563538, 'avg_pred_std': 0.07126395150553436}\n", + "Epoch 21\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005831055974340416, 'avg_role_model_std_loss': 0.7094658932399625, 'avg_role_model_mean_pred_loss': 5.484142581780628e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005831055974340416, 'n_size': 320, 'n_batch': 40, 'duration': 38.932344913482666, 'duration_batch': 0.9733086228370667, 'duration_size': 0.12166357785463333, 'avg_pred_std': 0.09576541467686184}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00040745751502981877, 'avg_role_model_std_loss': 0.01960964320030456, 'avg_role_model_mean_pred_loss': 8.632079813164495e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00040745751502981877, 'n_size': 80, 'n_batch': 10, 'duration': 8.343457698822021, 'duration_batch': 0.8343457698822021, 'duration_size': 0.10429322123527526, 'avg_pred_std': 0.0816122055053711}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▂▁▂▃▃▂▂▁▂▁▁▃▁▁▁▁▂▁▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▅▅▃█▂▃▆▅▇▆▅█▆▆▅▆▃▅▄▄▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▁▂▃▂▃▃▂▃▄▃▅▃▄▄▄▄▃▃▃▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▂▁▂▃▃▂▂▁▂▁▁▃▁▁▁▁▂▁▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▁▁▁▁▄▂▁▁▁▁▂▁▂▁▂▂▃▁▅▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▇█▃▃▂▂▃▂▁▁▁▂▁▁▁▁▂▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▃▅▃▇▃▅▂▄▅▃▅▁▅▂█▇▄▅▄▄█▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▂▅▄▂▅▃▃▁▅▄▁▅▁▃▇▇▅▄▄▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▃▅▃▇▃▅▂▄▅▃▅▁▅▂█▇▄▅▄▄█▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▂▅▄▂▅▃▃▁▅▄▁▅▁▃▇▇▅▄▄▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▃▅▃▇▃▅▂▄▅▃▅▁▅▂█▇▄▅▄▄█▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▂▅▄▂▅▃▃▁▅▄▁▅▁▃▇▇▅▄▄▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00041\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00058\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.08161\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.09577\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00041\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00058\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.01961\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.70947\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.83435\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.97331\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.10429\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.12166\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 8.34346\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 38.93234\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/tvae/4/wandb/offline-run-20240229_175101-y38e2cwk\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_175101-y38e2cwk/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.00027497335946021655, 'avg_g_mag_loss': 0.044538616604244054, 'avg_g_cos_loss': 0.14042933068628, 'pred_duration': 0.8693411350250244, 'grad_duration': 0.5576837062835693, 'total_duration': 1.4270248413085938, 'pred_std': 0.15040387213230133, 'std_loss': 0.0008385280380025506, 'mean_pred_loss': 1.3921320984877639e-08, 'pred_rmse': 0.016582321375608444, 'pred_mae': 0.012896367348730564, 'pred_mape': 0.13851648569107056, 'grad_rmse': 0.034425459802150726, 'grad_mae': 0.018821561709046364, 'grad_mape': 0.6824872493743896}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.00027497335946021655, 'avg_g_mag_loss': 0.044538616604244054, 'avg_g_cos_loss': 0.14042933068628, 'avg_pred_duration': 0.8693411350250244, 'avg_grad_duration': 0.5576837062835693, 'avg_total_duration': 1.4270248413085938, 'avg_pred_std': 0.15040387213230133, 'avg_std_loss': 0.0008385280380025506, 'avg_mean_pred_loss': 1.3921320984877639e-08}, 'min_metrics': {'avg_loss': 0.00027497335946021655, 'avg_g_mag_loss': 0.044538616604244054, 'avg_g_cos_loss': 0.14042933068628, 'pred_duration': 0.8693411350250244, 'grad_duration': 0.5576837062835693, 'total_duration': 1.4270248413085938, 'pred_std': 0.15040387213230133, 'std_loss': 0.0008385280380025506, 'mean_pred_loss': 1.3921320984877639e-08, 'pred_rmse': 0.016582321375608444, 'pred_mae': 0.012896367348730564, 'pred_mape': 0.13851648569107056, 'grad_rmse': 0.034425459802150726, 'grad_mae': 0.018821561709046364, 'grad_mape': 0.6824872493743896}, 'model_metrics': {'tvae': {'avg_loss': 0.00027497335946021655, 'avg_g_mag_loss': 0.044538616604244054, 'avg_g_cos_loss': 0.14042933068628, 'pred_duration': 0.8693411350250244, 'grad_duration': 0.5576837062835693, 'total_duration': 1.4270248413085938, 'pred_std': 0.15040387213230133, 'std_loss': 0.0008385280380025506, 'mean_pred_loss': 1.3921320984877639e-08, 'pred_rmse': 0.016582321375608444, 'pred_mae': 0.012896367348730564, 'pred_mape': 0.13851648569107056, 'grad_rmse': 0.034425459802150726, 'grad_mae': 0.018821561709046364, 'grad_mape': 0.6824872493743896}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:05.264464Z", + "iopub.status.busy": "2024-02-29T18:10:05.264165Z", + "iopub.status.idle": "2024-02-29T18:10:05.268178Z", + "shell.execute_reply": "2024-02-29T18:10:05.267324Z" + }, + "papermill": { + "duration": 0.026093, + "end_time": "2024-02-29T18:10:05.270073", + "exception": false, + "start_time": "2024-02-29T18:10:05.243980", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:05.307164Z", + "iopub.status.busy": "2024-02-29T18:10:05.306497Z", + "iopub.status.idle": "2024-02-29T18:10:05.387248Z", + "shell.execute_reply": "2024-02-29T18:10:05.386472Z" + }, + "papermill": { + "duration": 0.101501, + "end_time": "2024-02-29T18:10:05.389452", + "exception": false, + "start_time": "2024-02-29T18:10:05.287951", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:05.429312Z", + "iopub.status.busy": "2024-02-29T18:10:05.428645Z", + "iopub.status.idle": "2024-02-29T18:10:05.699888Z", + "shell.execute_reply": "2024-02-29T18:10:05.699094Z" + }, + "papermill": { + "duration": 0.29332, + "end_time": "2024-02-29T18:10:05.701797", + "exception": false, + "start_time": "2024-02-29T18:10:05.408477", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5v0lEQVR4nO3deXxTVf7/8Ve6JN0XWrphoYAtZS2brcUF0UpBFIoIyPAF5Ie4DDhqFRUHqDPOdyqOOi7w1a+Ogs5YQWbEcZAvCgVcoKwtAgIVsFC2tpQl3bfk/P5IGxrolrY0gXyej8d9QG9Obs5tknfPPffcczVKKYUQQtgBJ1tXQAgh6kggCSHshgSSEMJuSCAJIeyGBJIQwm5IIAkh7IYEkhDCbrjYugLtwWg0cvr0aby9vdFoNLaujhDiMkopiouLCQsLw8mp8XbQdRFIp0+fJjw83NbVEEI048SJE9xwww2NPn5dBJK3tzdg2lkfHx8b10YIcbmioiLCw8PN39XGXBeBVHeY5uPjI4EkhB1rrktFOrWFEHZDAkkIYTckkIQQduO66EMSbWMwGKiurrZ1NcQ1zNXVFWdn5zZvRwLJgSmlyMvL4+LFi7auirgO+Pn5ERIS0qaxgBJIDqwujIKCgvDw8JBBpaJVlFKUlZVRUFAAQGhoaKu35VCBZDAqMnMvUFRezR29gnB2ctwvoMFgMIdRQECArasjrnHu7u4AFBQUEBQU1OrDN4fq1DYYFRPfy2DWx7soqaixdXVsqq7PyMPDw8Y1EdeLus9SW/ojHSqQtC5O6FxMu1xUIZ240PxANSFaqj0+Sw4VSADebq6ABJIQ9sjhAsnH3dRtVuzgh2xC2COHC6S6FpIEkmgtjUbDl19+aetqtKuXXnqJgQMH2roajhdIPm6mFlJRuRyyiWvX8uXL8fPza7ftPfvss6Snp7fb9lrLoU77A3i71R2ySSCJ619VVRVarbbZcl5eXnh5eXVAjZrmgC0kOWRrjFKKsqqaDl+svXnyunXruPXWW/Hz8yMgIIB7772Xo0ePAjBs2DCef/55i/Jnz57F1dWV77//HoAzZ84wZswY3N3d6d69O2lpaURERPDmm2+26ve2b98+7rzzTtzd3QkICOCRRx6hpKTE/PjmzZuJjY3F09MTPz8/brnlFo4fPw7ATz/9xIgRI/D29sbHx4chQ4awa9euJl9v8+bNzJw5E71ej0ajQaPR8NJLLwEQERHByy+/zPTp0/Hx8eGRRx4B4PnnnycqKgoPDw969OjBwoULLU7PX37I9tBDD5GUlMRrr71GaGgoAQEBzJkz56pfYuS4LaRKCaTLlVcb6LPomw5/3QN/TMRD2/KPYmlpKcnJyQwYMICSkhIWLVrE+PHj2bNnD1OnTuXVV1/llVdeMZ+GXrlyJWFhYdx2220ATJ8+ncLCQjZv3oyrqyvJycnmUcbWKi0tJTExkfj4eHbu3ElBQQEPP/wwc+fOZfny5dTU1JCUlMTs2bP57LPPqKqqYseOHea6TZ06lUGDBvHuu+/i7OzMnj17cHV1bfI1hw0bxptvvsmiRYvIzs4GsGjdvPbaayxatIiUlBTzOm9vb5YvX05YWBj79u1j9uzZeHt789xzzzX6Ops2bSI0NJRNmzZx5MgRJk+ezMCBA5k9e3arflct4YCBVHvaX/qQrlkTJkyw+Pmjjz6ic+fOHDhwgEmTJvHUU0/x448/mgMoLS2NKVOmoNFoOHToEBs2bGDnzp0MHToUgL/97W9ERka2qi5paWlUVFTwySef4OnpCcCSJUu47777WLx4Ma6uruj1eu6991569uwJQO/evc3Pz83NZd68eURHRwO0qB5arRZfX180Gg0hISFXPH7nnXfyzDPPWKxbsGCB+f8RERE8++yzrFixoslA8vf3Z8mSJTg7OxMdHc2YMWNIT0+XQGpPl/qQpIV0OXdXZw78MdEmr2uNw4cPs2jRIrZv305hYSFGoxEwfbn79evHyJEj+fTTT7ntttvIyckhIyOD//3f/wUgOzsbFxcXBg8ebN7ejTfeiL+/f6vqfvDgQWJiYsxhBHDLLbdgNBrJzs7m9ttv56GHHiIxMZG7776bhIQEJk2aZL7eKzk5mYcffpi///3vJCQkMHHiRHNwtVZd0Na3cuVK3n77bY4ePUpJSQk1NTXNzq7at29fi0tAQkND2bdvX5vq1hyH7UOSgZFX0mg0eGhdOnyxdoTvfffdx/nz5/nggw/Yvn0727dvB0wduGA6DPrnP/9JdXU1aWlp9O/fn/79+7f776ulli1bRkZGBsOGDWPlypVERUWxbds2wNR38/PPPzNmzBg2btxInz59WL16dZter344AmRkZDB16lTuuece1qxZQ1ZWFr///e/Nv6/GXH7oqNFozOF/tThcINW1kIqkhXRNOnfuHNnZ2SxYsIC77rqL3r17c+HCBYsy48aNo6KignXr1pGWlsbUqVPNj/Xq1YuamhqysrLM644cOXLFNlqqd+/e/PTTT5SWlprXbdmyBScnJ3r16mVeN2jQIObPn8/WrVvp168faWlp5seioqJ4+umn+fbbb7n//vtZtmxZs6+r1WoxGAwtquPWrVvp1q0bv//97xk6dCiRkZHmTnV744CBVHeWTVpI1yJ/f38CAgJ4//33OXLkCBs3biQ5OdmijKenJ0lJSSxcuJCDBw8yZcoU82PR0dEkJCTwyCOPsGPHDrKysnjkkUdwd3dv1bVYU6dOxc3NjRkzZrB//342bdrEE088wbRp0wgODiYnJ4f58+eTkZHB8ePH+fbbbzl8+DC9e/emvLycuXPnsnnzZo4fP86WLVvYuXOnRR9TYyIiIigpKSE9PZ3CwkLKysoaLRsZGUlubi4rVqzg6NGjvP32221uhV0tDhhI0od0LXNycmLFihXs3r2bfv368fTTT/OXv/zlinJTp07lp59+4rbbbqNr164Wj33yyScEBwdz++23M378ePMZJzc3N6vr4+HhwTfffMP58+e56aabeOCBB7jrrrtYsmSJ+fFDhw4xYcIEoqKieOSRR5gzZw6PPvoozs7OnDt3junTpxMVFcWkSZMYPXo0f/jDH5p93WHDhvHYY48xefJkOnfuzKuvvtpo2bFjx/L0008zd+5cBg4cyNatW1m4cKHV+9ohVCssWbJEdevWTel0OhUbG6u2b9/eZPnPP/9c9erVS+l0OtWvXz/19ddfWzw+Y8YMBVgsiYmJLa6PXq9XgNLr9c2WzT1Xqro9v0b1WrC2xdu/HpWXl6sDBw6o8vJyW1fF5k6cOKEAtWHDBltX5ZrW1Geqpd9Rq1tIK1euJDk5mZSUFDIzM4mJiSExMbHRcRxbt25lypQpzJo1i6ysLJKSkkhKSmL//v0W5UaNGsWZM2fMy2effWZt1VqkroVUUW2kqubqdtAJ+7Rx40a++uorcnJy2Lp1Kw8++CARERHcfvvttq6aw7M6kN544w1mz57NzJkz6dOnD++99x4eHh589NFHDZZ/6623GDVqFPPmzaN37968/PLLDB482NykraPT6QgJCTEvTZ2GrayspKioyGJpKS/dpZEO0o/kmKqrq3nxxRfp27cv48ePp3PnzuZBkp9++qn5MorLl759+3ZYHUePHt1oPf785z93WD06mlXjkKqqqti9ezfz5883r3NyciIhIYGMjIwGn5ORkXFFp2NiYuIVV0tv3ryZoKAg/P39ufPOO/nTn/7U6NSqqampLTrOboiLsxOeWmdKqwwUV9QQ4KVr1XbEtSsxMZHExIbHW40dO5a4uLgGH2tuBHV7+tvf/kZ5eXmDj3Xq1KnD6tHRrAqkwsJCDAYDwcHBFuuDg4M5dOhQg8/Jy8trsHxeXp7551GjRnH//ffTvXt3jh49yosvvsjo0aPJyMhocG7e+fPnW4Rc3X3DW8rbzdUcSELU5+3t3ez95ztCly5dbF0Fm7CLkdoPPvig+f/9+/dnwIAB9OzZk82bN3PXXXddUV6n06HTtb5l4+3mQl6RDI4Uwt5Y1YcUGBiIs7Mz+fn5Fuvz8/MbvKYGICQkxKryAD169CAwMJAjR45YU70WkylIhLBPVgWSVqtlyJAhFhM5GY1G0tPTiY+Pb/A58fHxV0z8tH79+kbLA5w8eZJz58616f5OTfFxr7t8RA7ZhLAnVp9lS05O5oMPPuDjjz/m4MGDPP7445SWljJz5kzANLVD/U7vJ598knXr1vH6669z6NAhXnrpJXbt2sXcuXMBKCkpYd68eWzbto1jx46Rnp7OuHHjuPHGGxvteGwrmcZWCPtkdR/S5MmTOXv2LIsWLSIvL4+BAweybt06c8d1bm4uTk6Xcm7YsGGkpaWxYMECXnzxRSIjI/nyyy/p168fAM7Ozuzdu5ePP/6YixcvEhYWxsiRI3n55Zfb1E/UFG+ZxlYIu9SqTu25c+eaWziX27x58xXrJk6cyMSJExss7+7uzjffdOykYHL5iGgLjUbD6tWrSUpKsnVVrjsOdy0b1J/GVlpI4trU3pP8g6kxodFouHjxYrtu1xoOGkjSQhLCHjlkIMndaxuhFFSVdvwik/y32yT/lZWVPPvss3Tp0gVPT0/i4uIsulGOHz/Offfdh7+/P56envTt25e1a9dy7NgxRowYAZimeNFoNDz00EOt+n20hV0MjOxo0ofUiOoy+HNYx7/ui6dB69l8uVoyyX/jk/zPnTuXAwcOsGLFCsLCwli9ejWjRo1i3759REZGMmfOHKqqqvj+++/x9PTkwIEDeHl5ER4ezr/+9S8mTJhAdnY2Pj4+uLu7t+p30hYOGUh145CkD+naJJP8NzzJf25uLsuWLSM3N5ewMNMflmeffZZ169axbNky/vznP5Obm8uECRPMU/r26NHD/Py6a+SCgoLavX+qpRwykKSF1AhXD1NrxRavawWZ5L9h+/btw2AwEBUVZbG+srLSfKH67373Ox5//HG+/fZbEhISmDBhAgMGDGjV610NDt+HpKzsv7iuaTSmQ6eOXmSS/3aZ5L+kpARnZ2d2797Nnj17zMvBgwd56623AHj44Yf59ddfmTZtGvv27WPo0KG888477bavbeWggWRqIVUbFJUySds1RSb5N2lokv9BgwZhMBgoKCjgxhtvtFjqH9qFh4fz2GOP8cUXX/DMM8/wwQcfmLcJtPjmAVeDQwaSl9bF/EdZzrRdW2SSf5OGJvmPiopi6tSpTJ8+nS+++IKcnBx27NhBamoqX3/9NQBPPfUU33zzDTk5OWRmZrJp0ybz63Xr1g2NRsOaNWs4e/asxZnCDnOVptftUNbMqV2nX8o61e35NepIQfFVrJn9upbn1F6/fr3q3bu30ul0asCAAWrz5s0KUKtXrzaXWbt2rQLU7bfffsXzT58+rUaPHq10Op3q1q2bSktLU0FBQeq9995r0etf/lp79+5VI0aMUG5ubqpTp05q9uzZqrjY9LnKy8tTSUlJKjQ0VGm1WtWtWze1aNEiZTAYVGVlpXrwwQdVeHi40mq1KiwsTM2dO7fF78ljjz2mAgICFKBSUlKUUkpVVVWpRYsWqYiICOXq6qpCQ0PV+PHj1d69e5VSSs2dO1f17NlT6XQ61blzZzVt2jRVWFho3uYf//hHFRISojQajZoxY0aL6lGnPebU1ih17XeiFBUV4evri16vb/ZunHVueWUjpy6Ws/q3wxjUtXUdmteyiooKcnJy6N69e6vutnE9OXnyJOHh4WzYsKHB+bdEyzT1mWrpd9Qhz7KBnGlzZBs3bqSkpIT+/ftz5swZnnvuOZnk3044ZB8S1L+eTQLJ0cgk//ZLWkjSqe1wZJJ/++XwgSRn2UR9Msm/bTnuIZu7HLIBMjBUtJv2+Cw5bCA5eqd23eFHWVmZjWsirhd1n6W2HNo68CGbY09B4uzsjJ+fn/kqdw8Pj1YNDBRCKUVZWRkFBQX4+fk1eC/FlnLgQKqbV9sxW0iA+XKC1k69IUR9fn5+Td7erCUcNpBkGlvT3NChoaEEBQVRXe24vwfRdq6urm1qGdVx2EBy9D6k+pydndvlwyREWzlwp3ZtC6lSWgZC2AuHDSQf6UMSwu44biDVjkMqqayRsThC2AmHDaS6PiSDUVFWZbsJqYQQlzhsILm7OuPsZBp3Ix3bQtgHhw0kjUYj17MJYWdaFUhLly4lIiICNzc34uLi2LFjR5PlV61aRXR0NG5ubvTv35+1a9c2Wvaxxx5Do9G0+qZ91pCxSELYF6sDaeXKlSQnJ5OSkkJmZiYxMTEkJiY2Otp369atTJkyhVmzZpGVlUVSUhJJSUns37//irKrV69m27Zt5ntKXW2XWkhyyCaEPbA6kN544w1mz57NzJkz6dOnD++99x4eHh589NFHDZZ/6623GDVqFPPmzaN37968/PLLDB48mCVLlliUO3XqFE888QSffvpph807I4MjhbAvVgVSVVUVu3fvJiEh4dIGnJxISEggIyOjwedkZGRYlAfTBFn1yxuNRqZNm8a8efNaNCtfZWUlRUVFFktrmC+wLZdDNiHsgVWBVFhYiMFgIDg42GJ9cHAweXl5DT4nLy+v2fKLFy/GxcWF3/3udy2qR2pqKr6+vuYlPDzcmt0wk2lshbAvNj/Ltnv3bt566y2WL1/e4ukv5s+fj16vNy8nTpxo1WvLNLZC2BerAikwMBBnZ2fy8/Mt1ufn5zc67UBISEiT5X/44QcKCgro2rUrLi4uuLi4cPz4cZ555hkiIiIa3KZOp8PHx8diaQ0f6UMSwq5YFUharZYhQ4aQnp5uXmc0GklPTyc+Pr7B58THx1uUB1i/fr25/LRp09i7d6/FvcjDwsKYN28e33zzjbX7Y5W6y0dkHJIQ9sHq6UeSk5OZMWMGQ4cOJTY2ljfffJPS0lJmzpwJwPTp0+nSpQupqakAPPnkkwwfPpzXX3+dMWPGsGLFCnbt2sX7778PQEBAAAEBARav4erqSkhIiMW90a8GOcsmhH2xOpAmT57M2bNnWbRoEXl5eQwcOJB169aZO65zc3NxcrrU8Bo2bBhpaWksWLCAF198kcjISL788kv69evXfnvRSt4yMFIIu+Kwt9IG+OHwWaZ9uINewd5887TctVSIq6Wl31Gbn2WzJbl0RAj74tCBJH1IQtgXBw+k2knaqmowGq/5I1chrnkOHkimFpJSUFwprSQhbM2hA8nN1Rmti+lXIP1IQtieQwcSyGhtIeyJwweSt1xgK4TdkEAy3w5JDtmEsDWHDyQfuWGkEHbD4QNJxiIJYT8kkCSQhLAbEkgyja0QdsPhA6muD0nuPCKE7Tl8IMk0tkLYDwkk6UMSwm44fCDJNLZC2A+HDyRpIQlhPxw+kGSSNiHsh8MHkrSQhLAfDh9IdS2ksioD1QajjWsjhGNz+EDycrt045USaSUJYVMOH0iuzk64uzoDctgmhK05fCBBvSlIpGNbCJuSQELGIglhLySQkDNtQtgLCSRkGlsh7IUEEnKBrRD2QgKJelOQlEsLSQhbalUgLV26lIiICNzc3IiLi2PHjh1Nll+1ahXR0dG4ubnRv39/1q5da/H4Sy+9RHR0NJ6envj7+5OQkMD27dtbU7VW8ZEWkhB2wepAWrlyJcnJyaSkpJCZmUlMTAyJiYkUFBQ0WH7r1q1MmTKFWbNmkZWVRVJSEklJSezfv99cJioqiiVLlrBv3z5+/PFHIiIiGDlyJGfPnm39nllBOrWFsBPKSrGxsWrOnDnmnw0GgwoLC1OpqakNlp80aZIaM2aMxbq4uDj16KOPNvoaer1eAWrDhg0tqlNdeb1e36Lyl1u+JUd1e36Nevwfu1r1fCFE01r6HbWqhVRVVcXu3btJSEgwr3NyciIhIYGMjIwGn5ORkWFRHiAxMbHR8lVVVbz//vv4+voSExPTYJnKykqKiooslrbwca+7N5u0kISwJasCqbCwEIPBQHBwsMX64OBg8vLyGnxOXl5ei8qvWbMGLy8v3Nzc+Otf/8r69esJDAxscJupqan4+vqal/DwcGt24wreOpmCRAh7YDdn2UaMGMGePXvYunUro0aNYtKkSY32S82fPx+9Xm9eTpw40abXlj4kIeyDVYEUGBiIs7Mz+fn5Fuvz8/MJCQlp8DkhISEtKu/p6cmNN97IzTffzIcffoiLiwsffvhhg9vU6XT4+PhYLG1x6dIRCSQhbMmqQNJqtQwZMoT09HTzOqPRSHp6OvHx8Q0+Jz4+3qI8wPr16xstX3+7lZWV1lSv1eTiWiHsg0vzRSwlJyczY8YMhg4dSmxsLG+++SalpaXMnDkTgOnTp9OlSxdSU1MBePLJJxk+fDivv/46Y8aMYcWKFezatYv3338fgNLSUv77v/+bsWPHEhoaSmFhIUuXLuXUqVNMnDixHXe1cXWXjlTVGKmsMaBzce6Q1xVCWLI6kCZPnszZs2dZtGgReXl5DBw4kHXr1pk7rnNzc3FyutTwGjZsGGlpaSxYsIAXX3yRyMhIvvzyS/r16weAs7Mzhw4d4uOPP6awsJCAgABuuukmfvjhB/r27dtOu9k0L92lX0NxRQ06LwkkIWxBo5RStq5EWxUVFeHr64ter291f1L/lG8orqxh07N30D3Qs51rKIRja+l31G7OstmauR+pXPqRhLAVCaRaMgWJELYngVRLpiARwvYkkGrVjUWSFpIQtiOBVEvGIglhexJItS4FkrSQhLAVCaRalzq1pYUkhK1IINXykbNsQticBFItGYckhO1JINWSKUiEsD0JpFrmQ7ZKaSEJYSsSSLVkGlshbE8CqZacZRPC9iSQatXvQ7oOJkAQ4pokgVSrrg+pxqioqDbauDZCOCYJpFoeWmecnTSAXD4ihK1IINXSaDTmmSOlH0kI25BAqkeuZxPCtiSQ6pHLR4SwLQmkeuTyESFsSwKpHpnGVgjbkkCqx0emsRXCpiSQ6pFpbIWwLQmkemQaWyFsSwKpHpmCRAjbkkCqRy6wFcK2JJDqqRuHJAMjhbANCaR6ZBySELbVqkBaunQpERERuLm5ERcXx44dO5osv2rVKqKjo3Fzc6N///6sXbvW/Fh1dTXPP/88/fv3x9PTk7CwMKZPn87p06dbU7U2kT4kIWzL6kBauXIlycnJpKSkkJmZSUxMDImJiRQUFDRYfuvWrUyZMoVZs2aRlZVFUlISSUlJ7N+/H4CysjIyMzNZuHAhmZmZfPHFF2RnZzN27Ni27VkrSB+SEDamrBQbG6vmzJlj/tlgMKiwsDCVmpraYPlJkyapMWPGWKyLi4tTjz76aKOvsWPHDgWo48ePt6hOer1eAUqv17eofGPyi8pVt+fXqO4vrFEGg7FN2xJCXNLS76hVLaSqqip2795NQkKCeZ2TkxMJCQlkZGQ0+JyMjAyL8gCJiYmNlgfQ6/VoNBr8/PwafLyyspKioiKLpT3UdWobFZRWyWGbEB3NqkAqLCzEYDAQHBxssT44OJi8vLwGn5OXl2dV+YqKCp5//nmmTJmCj49Pg2VSU1Px9fU1L+Hh4dbsRqN0Lk64OpsmaZN+JCE6nl2dZauurmbSpEkopXj33XcbLTd//nz0er15OXHiRLu8vkajkQtshbAhF2sKBwYG4uzsTH5+vsX6/Px8QkJCGnxOSEhIi8rXhdHx48fZuHFjo60jAJ1Oh06ns6bqLebj5sL50irp2BbCBqxqIWm1WoYMGUJ6erp5ndFoJD09nfj4+AafEx8fb1EeYP369Rbl68Lo8OHDbNiwgYCAAGuq1a68zYMjJZCE6GhWtZAAkpOTmTFjBkOHDiU2NpY333yT0tJSZs6cCcD06dPp0qULqampADz55JMMHz6c119/nTFjxrBixQp27drF+++/D5jC6IEHHiAzM5M1a9ZgMBjM/UudOnVCq9W21762iIxFEsJ2rA6kyZMnc/bsWRYtWkReXh4DBw5k3bp15o7r3NxcnJwuNbyGDRtGWloaCxYs4MUXXyQyMpIvv/ySfv36AXDq1Cm++uorAAYOHGjxWps2beKOO+5o5a61jlw+IoTtaJS69u+KWFRUhK+vL3q9vsm+p5aYt+onVu0+yXOjevHbO25spxoK4dha+h21q7Ns9sDch1QuLSQhOpoE0mW8ZRpbIWxGAukyMo2tELYjgXQZaSEJYTsSSJfxkbvXCmEzEkiXkSlIhLAdCaTLyO20hbAdCaTLyEhtIWxHAukydYFUUlmDwXjNjxkV4poigXSZuj4kgBJpJQnRoSSQLqN1ccLN1fRrkSv+hehYEkgNkEnahLANCaQGmO/PJi0kITqUBFIDpIUkhG1IIDXARy4fEcImJJAaIIMjhbANCaQGmPuQyqWFJERHkkBqgHkKkkppIQnRkSSQGuCtkz4kIWxBAqkBlw7ZpIUkREeSQGqA3JtNCNuQQGqATGMrhG1IIDVAprEVwjYkkBrgLdPYCmETEkgN8JFpbIWwCccMpJKCJh+uC6SKaiPVBmNH1EgIgaMFkv4kvD0Y3hkKhsZbP161h2wgHdtCdKRWBdLSpUuJiIjAzc2NuLg4duzY0WT5VatWER0djZubG/3792ft2rUWj3/xxReMHDmSgIAANBoNe/bsaU21mucdChV6qNRDbkajxZydNObBkSfOl12dugghrmB1IK1cuZLk5GRSUlLIzMwkJiaGxMRECgoaPgzaunUrU6ZMYdasWWRlZZGUlERSUhL79+83lyktLeXWW29l8eLFrd+TlnByhqhE0/+z/6/Jorf36gzAip0nrm6dhBCXKCvFxsaqOXPmmH82GAwqLCxMpaamNlh+0qRJasyYMRbr4uLi1KOPPnpF2ZycHAWorKwsq+qk1+sVoPR6ffOFD3ylVIqPUm/GKGU0Nlps+6/nVLfn16joBf+nLpZWWVUfIYSlln5HrWohVVVVsXv3bhISEszrnJycSEhIICOj4UOgjIwMi/IAiYmJjZZvicrKSoqKiiyWFusxApy1cCEHzmY3WuymCH+iQ7wprzaware0koToCFYFUmFhIQaDgeDgYIv1wcHB5OXlNficvLw8q8q3RGpqKr6+vuYlPDy85U/WeUH34ab//9L4YZtGo2HGsAgA/r7tOEa5JZIQV901eZZt/vz56PV683LihJUtmF6jTP820480bmAYPm4uHD9XxneHz7aytkKIlrIqkAIDA3F2diY/P99ifX5+PiEhIQ0+JyQkxKryLaHT6fDx8bFYrBJVG0gndkBpYaPFPLQuTBpqan19vPVYK2srhGgpqwJJq9UyZMgQ0tPTzeuMRiPp6enEx8c3+Jz4+HiL8gDr169vtHyH8L0BQgYACg5/22TR/7q5GxoNbM4+y7HC0o6pnxAOyupDtuTkZD744AM+/vhjDh48yOOPP05paSkzZ84EYPr06cyfP99c/sknn2TdunW8/vrrHDp0iJdeeoldu3Yxd+5cc5nz58+zZ88eDhw4AEB2djZ79uxpUz9Ts3qNNv2bvbbJYhGBntwRZRoC8Pdtx69efYQQ1p/2V0qpd955R3Xt2lVptVoVGxurtm3bZn5s+PDhasaMGRblP//8cxUVFaW0Wq3q27ev+vrrry0eX7ZsmQKuWFJSUlpUH6tO+9c5lWk6/f+nUKWqypssuvFQvur2/BrVL2WdKq2sbvlrCCGUUi3/jmqUUtf86aOioiJ8fX3R6/Ut709SCt7oDcVnYOq/IDKh0aJGo2LE65s5fq6MP4/vz2/iurZTzYVwDC39jl6TZ9nahUZzqXO7idP/AE5OGqbd3A2ATzKOcR1kuBB2yXEDCer1I60ztZiaMHFIOO6uzhzKK2ZHzvkOqJwQjsexA6n77eDqAUUnIW9fk0V9PVxJGtQFgE8ypHNbiKvBsQPJ1d10KQk0O0gSYHq86bBt3c955OkrrmbNhHBIjh1IcGnUdjP9SAC9Q32I7d4Jg1GRtl1aSUK0NwmkqFGABk5nQdGZZovPiI8AIG1HLpU1hqtbNyEcjASSVxB0GWL6/y/rmi0+sm8wwT46CkuqWLf/Kg7cFMIBSSBBvbNtzR+2uTo7MTXO1Jck17cJ0b4kkOBSIOV8B1XNT1n7YGw4rs4aMnMvsu+k/ipXTgjHIYEEENQH/LpCTQX8urn54t5u3NM/FDANlBRCtA8JJKgdtd2yi23rTK/t3P73T6e5UFp1lSomhGORQKpTd9j2yzdgbP5ebIO7+tGviw9VNUZW7pIpboVoDxJIdbrdAjofKC2A05nNFtdoNOZW0t8zjlNRLUMAhGgrCaQ6Llroeafp/y042wYwNiYMfw9XTl0sZ8zbP7DnxMWrVz8hHIAEUn297jH928JAcnN1ZunUwXT21nH0bCn3/88W/vLNIRkwKUQrSSDVF3k3aJyg4Ge40LJLQ4b1DOTbp25n3MAwjAqWbjrKuCVb+Pm0DAcQwloSSPV5dIKutXN9t2DUdh1/Ty1vPTiId6cOppOnlkN5xYxbsoW3Nhym2tB8B7kQwsTF1hWwO1Gj4PgW02Fb3KNWPXV0/1Bu6t6JBav3s+7nPP664Rc2HMzn9UkxRAV7t2wjOT/Awf9gmsVXY2qxaZxMQxM09X92Aq2nabhCcB+rd7O+imoD+0/pOa2vYHhkZ3w9XNu0PZtSCg59DQE3QlC0rWsjrOS4U9g2pvAILBkCTq7w3K/gZv32lFJ89dNpFv37Z/Tl1WidnXj67igeub0Hzk6ahp9UWQLrF8GuD62vc3A/GDAJ+j0Avl2ardvJC+VknbhI5vELZOVe4MCZIqoNpo9BkLeOVyb0587o4Ca3Y7e+fw02vmya5+qhNZeuUxQ21dLvqARSQ94ZAueOwMTl0Hd8qzeTX1TB/C/2sfFQAQD9u/gycegN3BkdxA3+HpcKHtsC//4tXDhm+nnAg6aR48p4aUHV/r/evxePw+H1YKyu3ZAGIm6F/hOhzzhw98NgVGTmXiDz+AXTv7kXOVtceUVdA7106FycOHWxHICJQ25g4X198HG7hlpLh76GFb+59LNHIMz6FgJ62q5O16CCogr2ndIzPKozLs7t06sjgdQW3y6Are/AgMlw//tt2pRSilW7T/Lyfw5QXFljXh8d4s2oKF9+U7qczj8vQ4MC33AY+w70HNHyFyg7Dwf+DftWmQ41617XWcuJwNt4/+JQPtf3oYpLweLipKFPmA+Du/ozqKsfg7v6c4O/O5U1Rl77JpsPt+SgFIT6uvHKhAEMr70NlF3L2w8fjoTqUhg8A87sgTM/gX8EzFpvmtVBNMloVPxj+3FeXZdNSWUNd0UH8c5vBuGhbXvPjgRSWxzbAsvvAXd/eDwDfELbvMk8fQWrs06RfjCfzNwLxHCY11zfo6eTaQ6m7X5jKBr+R4b1icBT18oPwMVcKrI+p2xnGp3KjppXF+HJTt+RnI2eRs/eg+jfxRc3V+dGN7Pz2HnmrfqJY+dMFxpPiQ3n92P64NXael1tpYXw/gjQ50L34fBfX0DZOfjwblMrMnQgPPQ16LxsXVO79Ut+MS/8ay+ZuRct1g8M9+PDGUMJ8NK1afsSSG1hqIG/9oWSPHD1hNuSIX4uuLq1fds1lZSv/xNu25egwUiB8uf56ofZZBwEgNbFibjunYiN6MTgbv7EhPu1KAgKiitYtuUY/8g4TnFlNb01uUz12M54l614VhZcKth9ONz0sGnMlXPj2y2rquHVddksr51ipYufO68+MIBbbgxs0+63u5oq+GQc5G6FTj3g4XTT2VKAc0dNoVR2zjTodcpK0wBYR1R02tS/lrfPNJd83yQI7kdFjZH/2XSEd787SrVB4aVz4blRvYgO8WH2J7vQl1fTPdCTj2fG0jXAo9mXafTlJZDaKG8/rHkKTu40/ezXFUb+N/S+z3S2qzVOZ8Hqx+HsQdPPAyZTdXcqO/IUGw7mk34onxPnyy2e4qSBqGBvBnfzZ3BXfwZ39aN7oCea2jocKyzl/R9+5Z+7T1JVYxpiEBnkxWPDezJ2YBiuGgVHN5k6y39ZV9sfBXiHwZCHYMgM8A5ptMoZR88x758/cfKCqV7Tbu7GC6OjW9+Ka09KwX+ehMyPTZf9PLwBOveyLHNyN3x8L1SXmfrmxr/X+vevo5Wdh6oS02evLdv48Q3Y8YFpNot6yr0j+FflTXxaPJiDqisJvUN4Oakvob7uABwpKGbGRzs5dbGcQC8tyx6Kpf8Nvq2qhgRSe1DK1DezPgWKT5vWRdwGo16BkH4t20aF3jSlyS/fwN6VYKwxdbbe96Yp3CxeTnG4oIQtRwrJzDWdBavrZK7P38OVQV39cXXWsP5APsbad3BwVz8ev+NG7ooOwqmhs3kXc2HXMsj8BMoKTeucXCD6XlOrKeLWBr+spZU1pP7fQf6xLReA8E7uPHJbDxL6BJs/vDax/X/h/54DNPCbzyFqZMPlDq+HtMmgDHDLU3D3HzqyltZRCo79CLs+Mg3/MFZDyACIedB0sqKlfWGVJbDtXdj6NlQWmdZ1jYf+D1D9Szoc2YCrujRLRalXNzwGTURT23Kq+xzkF1Xw0LKdHDxThIfWmXf/a0ir+hQlkNpTVSn8+Kbpza2pMI0BGvIQjPg9eF52CKMUFByAw9/C4Q1wYpsphOr0GQdj3rjyeY3IL6ogq/bsWObxC+w9pTe3hOrcGR3EY8N7clOEv7nl1KSaSjjwFez8m6l+dTpHQ78JpoAK6n1FOP14uJDn/7XXIiT7dfHh7t4h3N0nmN6h3i17/fZwdBP8Y4IpZO5+GW75HQCVNQa2HCnESWPquA/yrj3MzvrUdCYTYPSrVo8xu+rKzsOeNNi9HM4dNq9WGic0ta1ao8aZM4HDyPIfxTbXOE6WKPKLKskvqqC8ykBnbx1hXhruV+u558KneNVcAKDErzcX4l/As+9oMn49T8pXP1NecpG7nLJ4NPAn+pTuQGOod+a1U0/T2eUBk6FzFMUV1Tz2j91sOXIOFycNr0wYwANDbrBq9ySQroaLuaaxQj+vNv2s84U7XjD99Tq+xfSX+MgGKDpl+byASNNlKb3uabQV0lJVNUYOnCki8/gFzpZUMjYmjN6hbdjnvH2w80PY+7npDFUd/+4QPcbUirvhJnAydYIXV1Tz2Y5cvv05n925Fyzur3mDvzsjowMYG1ZMP+djuJw7bPprG3k3uPu1vo6XO3cUPhhhan3GTIGkd/m1sJQVO0/wz90nOV9vfqpALx19wnzoE+rDuOI0eh94C4UGzcRlbRrS0S6UgtxtsHsZ/Pwl1IZCjYsHe/xGsrxyBNvPu5NIBhOcf2CQ0xHzU4uUO2sNcXxhuI2dqhcaYLzTjzzt+k9u0Jhav8eMwbxRM5H/GG9GXXZRRo/OnqSO709cjwCoLDa14H9ebfoM1w+nG26Cgb+hKno8z605xpd7TEcK8xJ78ds7erb4D5AE0tV0bAuse77xm0u6uEP32yByJNyYAJ26X/06tVWF3jR84NDXptZH/Q+lZ2fTfFHR99XeXNPU6ii8WETmrgzOHNqGtmAvvckhWpOLm6baYtNGjQtVNwxD228sTr3HgE9Y2+r5twQo/AVjl6GsHfI3/rErj22/XrqbcLCPDi+dC78Wll52Q2LFH12WM91lPdW4sKzHX/HqdQcDw/2ICvZqtzE3TVIKivPg4Femw+e6/kTgiHNPPqq4g38bhlHKpUNhjcYUrIM8C7mP77m1NB3/6ks3mKjyugFc3dBeMAVWqa4zP4T9P9LdEskrqaGgqJKC4goulFWjdXHiseE9mTOiJzqXBs601oXTvlWmcFK1F4o761DRY/i85jbm/9QZI05Mu7kbL43t2/hg33quaiAtXbqUv/zlL+Tl5RETE8M777xDbGxso+VXrVrFwoULOXbsGJGRkSxevJh77rnH/LhSipSUFD744AMuXrzILbfcwrvvvktkZGSL6tPhgQRgNEDW3yH9ZVN/TKcetQF0N0TcYroJ5bWqssTU0jv0tenDWVnvQmGtl2nuqJI8yD9Qb1DmJcV4sN8QQY4KZojTYXo5nbR4PEcXzcngO6m8cRSdIwbQI8gL75YMwDQaIG0SHNlAkWsQ99f8iSPlplP5ThoY0SuIKbFduaOXaUBfWVUN2XnFHDhTxIHTRRw4U8QvZ/S8xhuMdt5JkfIgufpxipQHbi5O9OjsQWSQFzd29qRnZ08CPF1N48OUAhc3UyvP3R/c/Ezvb1OtA0M1nM+Bwl+g8BdqCn6hpuAQLueP4FJdbC5WpnT8xxBPmuFOflI9AQ2RQV7c3COAm3sEEBPuS7CPG671w9JoNJ1V/GmFqWVVVbs9Nz+49WmIfQS0V54Rq6oxYlSqySEfForzYd/npkPJggPm1aW6znxSejP/NNxOz96DeXvKoGa3edUCaeXKlUyfPp333nuPuLg43nzzTVatWkV2djZBQVd2uG3dupXbb7+d1NRU7r33XtLS0li8eDGZmZn062fqGF68eDGpqal8/PHHdO/enYULF7Jv3z4OHDiAm1vzp9ptEkh1qiug/EK7jFWyS4ZqUyfroa9NS13nfh03PwgbCKExpvE+oTEY/CLIPKHnu+yzHC4opjLvML2LfuAuzU4Gaw7jpLn0kTtqDGW9cQj5TkG4urqiddWi1WrR1S06LW5aHe46LRHnfyD65CrKlZaJVYvYr3oQ4uPG5JvCmXRTOF38mv8jYDAqcvML8fnnZALO7W79r8XJlWpXX2q0PlRpfalx9aFa6wuVJbgXHcW3/CTONDwNjUFpOKS6ssIwgi8NtxISFGQOoNjunejsbcWYn6oy001Oyy+a+v/a89C4jlKmQaZ70kwBVX7B/NAeY09+6jaDGbOebHITVy2Q4uLiuOmmm1iyZAkARqOR8PBwnnjiCV544YUryk+ePJnS0lLWrFljXnfzzTczcOBA3nvvPZRShIWF8cwzz/Dss88CoNfrCQ4OZvny5Tz44IPN1smmgeRIlDINXcjNMI0qD40xnZJuQT9CjcHIqYvlnMjNQWWvo/OpDfQs3oUrV7awmvNE9ROUR41jSmzX1l/eUH4BvvodFBxAoaHaoCivMVJRbaS82khljcLUlWzaNx1V+GpK8aUUF03LZnAoVTqOqjDTYgzjlPMNFHn3xOgfwQ2B/sT16ERc9wDrAsjWaipNreY9aajD36JRBi7e/kf87myfQLJqMElVVRW7d+9m/vz55nVOTk4kJCSQkZHR4HMyMjJITk62WJeYmMiXX34JQE5ODnl5eSQkJJgf9/X1JS4ujoyMjAYDqbKyksrKS30cRUVF1uyGaC2NBroMNi1WcnF2oluAJ90C+sGgfsCzpv6KIxuoyl5PTZmemupqamqqMdRUYaypwWioxmgw/YvRgNFo5Ej4A8y/70nCWtAaapK7P0z+u2m3AG3tUjfKprzKwL5Ten46cZE9Jy9yobTKdBmhMuKmyvE0FOOhSvAyluBpLMbTWIKnKgZnHZV+kRAYiU9QV0L8POjr68Zdvm7X1nWBjXHRQZ+x0GcsmpICjHs/x2/A5PbbvDWFCwsLMRgMBAdbXgkeHBzMoUOHGnxOXl5eg+Xz8vLMj9eta6zM5VJTU/nDH+x4LIloGZ039B2Ptu94Wjp+uttVrdAl7lpnYrt3IrZ7pw56xWuQVxBOw+a26yavyQna5s+fj16vNy8nTshdP4S4HlgVSIGBgTg7O5Ofn2+xPj8/n5CQhi8/CAkJabJ83b/WbFOn0+Hj42OxCCGufVYFklarZciQIaSnp5vXGY1G0tPTiY+Pb/A58fHxFuUB1q9fby7fvXt3QkJCLMoUFRWxffv2RrcphLhOKSutWLFC6XQ6tXz5cnXgwAH1yCOPKD8/P5WXl6eUUmratGnqhRdeMJffsmWLcnFxUa+99po6ePCgSklJUa6urmrfvn3mMq+88ory8/NT//73v9XevXvVuHHjVPfu3VV5eXmL6qTX6xWg9Hq9tbsjhOgALf2OWh1ISin1zjvvqK5duyqtVqtiY2PVtm3bzI8NHz5czZgxw6L8559/rqKiopRWq1V9+/ZVX3/9tcXjRqNRLVy4UAUHByudTqfuuusulZ2d3eL6SCAJYd9a+h2VS0eEEFddS7+j1+RZNiHE9ckOZtlqu7pGngyQFMI+1X03mzsguy4CqbjYdHFheHi4jWsihGhKcXExvr6Nzzp5XfQhGY1GTp8+jbd38xOEFRUVER4ezokTJ67p/qbrZT/g+tkX2Y/GKaUoLi4mLCwMJ6fGe4quixaSk5MTN9xg3Qx218uAyutlP+D62RfZj4Y11TKqI53aQgi7IYEkhLAbDhdIOp2OlJQUdLpraA6aBlwv+wHXz77IfrTdddGpLYS4PjhcC0kIYb8kkIQQdkMCSQhhNySQhBB2QwJJCGE3HC6Qli5dSkREBG5ubsTFxbFjxw5bV8kqL730EhqNxmKJjo62dbWa9f3333PfffcRFhaGRqMx33WmjlKKRYsWERoairu7OwkJCRw+fLjhjdlYc/vy0EMPXfEejRo1yjaVbUJqaio33XQT3t7eBAUFkZSURHZ2tkWZiooK5syZQ0BAAF5eXkyYMOGK6abbk0MF0sqVK0lOTiYlJYXMzExiYmJITEykoKDA1lWzSt++fTlz5ox5+fHHH21dpWaVlpYSExPD0qVLG3z81Vdf5e233+a9995j+/bteHp6kpiYSEVFRQfXtHnN7QvAqFGjLN6jzz77rANr2DLfffcdc+bMYdu2baxfv57q6mpGjhxJaWmpuczTTz/Nf/7zH1atWsV3333H6dOnuf/++69epa7qNHF2JjY2Vs2ZM8f8s8FgUGFhYSo1NdWGtbJOSkqKiomJsXU12gRQq1evNv9sNBpVSEiI+stf/mJed/HiRaXT6dRnn31mgxq23OX7opRSM2bMUOPGjbNJfdqioKBAAeq7775TSpneA1dXV7Vq1SpzmYMHDypAZWRkXJU6OEwLqe4ml/VvSNncTS7t1eHDhwkLC6NHjx5MnTqV3NxcW1epTZq7Wei1aPPmzQQFBdGrVy8ef/xxzp07Z+sqNUuv1wPQqZPpXnS7d++murra4n2Jjo6ma9euV+19cZhAauoml43dkNIexcXFsXz5ctatW8e7775LTk4Ot912m3lOqGtRa24Was9GjRrFJ598Qnp6OosXL+a7775j9OjRGAwGW1etUUajkaeeeopbbrmFfv36Aab3RavV4ufnZ1H2ar4v18X0I45k9OjR5v8PGDCAuLg4unXrxueff86sWbNsWDNRp/7t3/v378+AAQPo2bMnmzdv5q677rJhzRo3Z84c9u/fb/P+SIdpIbXmJpfXAj8/P6Kiojhy5Iitq9JqrblZ6LWkR48eBAYG2u17NHfuXNasWcOmTZss5hULCQmhqqqKixcvWpS/mu+LwwRSa25yeS0oKSnh6NGjhIaG2roqrXa93yz05MmTnDt3zu7eI6UUc+fOZfXq1WzcuJHu3btbPD5kyBBcXV0t3pfs7Gxyc3Ov3vtyVbrK7VRzN7m8FjzzzDNq8+bNKicnR23ZskUlJCSowMBAVVBQYOuqNam4uFhlZWWprKwsBag33nhDZWVlqePHjyul2n6z0I7U1L4UFxerZ599VmVkZKicnBy1YcMGNXjwYBUZGakqKipsXXULjz/+uPL19VWbN29WZ86cMS9lZWXmMo899pjq2rWr2rhxo9q1a5eKj49X8fHxV61ODhVISjV9k8trweTJk1VoaKjSarWqS5cuavLkyerIkSO2rlazNm3apIArlrqbirb1ZqEdqal9KSsrUyNHjlSdO3dWrq6uqlu3bmr27Nl2+UevoX0A1LJly8xlysvL1W9/+1vl7++vPDw81Pjx49WZM2euWp1kPiQhhN1wmD4kIYT9k0ASQtgNCSQhhN2QQBJC2A0JJCGE3ZBAEkLYDQkkIYTdkEASQtgNCSQhhN2QQBJC2A0JJCGE3fj/d6S3UxKs9ToAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:05.739775Z", + "iopub.status.busy": "2024-02-29T18:10:05.739483Z", + "iopub.status.idle": "2024-02-29T18:10:52.699221Z", + "shell.execute_reply": "2024-02-29T18:10:52.698208Z" + }, + "papermill": { + "duration": 46.981509, + "end_time": "2024-02-29T18:10:52.701858", + "exception": false, + "start_time": "2024-02-29T18:10:05.720349", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:52.742539Z", + "iopub.status.busy": "2024-02-29T18:10:52.742224Z", + "iopub.status.idle": "2024-02-29T18:10:52.761904Z", + "shell.execute_reply": "2024-02-29T18:10:52.761097Z" + }, + "papermill": { + "duration": 0.041848, + "end_time": "2024-02-29T18:10:52.763762", + "exception": false, + "start_time": "2024-02-29T18:10:52.721914", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.1347840.0363680.0002750.5683860.0188240.6825660.0344341.392132e-080.8837040.0128970.1385170.0165830.1504040.0008391.45209
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.134784 0.036368 0.000275 0.568386 0.018824 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.682566 0.034434 1.392132e-08 0.883704 0.012897 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.138517 0.016583 0.150404 0.000839 1.45209 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:52.802551Z", + "iopub.status.busy": "2024-02-29T18:10:52.801916Z", + "iopub.status.idle": "2024-02-29T18:10:53.233160Z", + "shell.execute_reply": "2024-02-29T18:10:53.232302Z" + }, + "papermill": { + "duration": 0.452737, + "end_time": "2024-02-29T18:10:53.235205", + "exception": false, + "start_time": "2024-02-29T18:10:52.782468", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:53.275648Z", + "iopub.status.busy": "2024-02-29T18:10:53.275331Z", + "iopub.status.idle": "2024-02-29T18:11:41.971985Z", + "shell.execute_reply": "2024-02-29T18:11:41.971200Z" + }, + "papermill": { + "duration": 48.719326, + "end_time": "2024-02-29T18:11:41.974270", + "exception": false, + "start_time": "2024-02-29T18:10:53.254944", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:11:42.014824Z", + "iopub.status.busy": "2024-02-29T18:11:42.014495Z", + "iopub.status.idle": "2024-02-29T18:11:42.030851Z", + "shell.execute_reply": "2024-02-29T18:11:42.030176Z" + }, + "papermill": { + "duration": 0.038526, + "end_time": "2024-02-29T18:11:42.032661", + "exception": false, + "start_time": "2024-02-29T18:11:41.994135", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:11:42.070440Z", + "iopub.status.busy": "2024-02-29T18:11:42.070179Z", + "iopub.status.idle": "2024-02-29T18:11:42.075287Z", + "shell.execute_reply": "2024-02-29T18:11:42.074351Z" + }, + "papermill": { + "duration": 0.026305, + "end_time": "2024-02-29T18:11:42.077266", + "exception": false, + "start_time": "2024-02-29T18:11:42.050961", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.05389225258396234}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:11:42.116430Z", + "iopub.status.busy": "2024-02-29T18:11:42.116168Z", + "iopub.status.idle": "2024-02-29T18:11:42.432730Z", + "shell.execute_reply": "2024-02-29T18:11:42.431890Z" + }, + "papermill": { + "duration": 0.338569, + "end_time": "2024-02-29T18:11:42.434727", + "exception": false, + "start_time": "2024-02-29T18:11:42.096158", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+FElEQVR4nO3deXyU1b348c/sM5mZTAhkhQSQsO/IIioCShWxKrV1bxX3e4tXLbWvSl+3KlqL9qrFelva26tQf3WrFq1X61YVcAMFRRCVNSFhCYGELDOTWZ/z+2OSkUACmckkzyT5vl+veck8c+Z5vjMm35xznrMYlFIKIYRIE0a9AxBCiKNJUhJCpBVJSkKItCJJSQiRViQpCSHSiiQlIURakaQkhEgrkpSEEGlFkpIQIq1IUhJCpBVJSiKlPvroI+69915qa2v1DkV0U5KUREp99NFHLFmyRJKSSJokJSFEWpGkJFLm3nvv5Wc/+xkAgwcPxmAwYDAYcLlczJ49+7jymqbRv39/fvCDH8SPPfzww5x++un07dsXh8PBqaeeyosvvtjq9f76179y6qmn4nA4yM7O5oorrqCioqJzPpzoMgZZukSkyubNm3nwwQd59tln+e1vf0u/fv0A2LVrF/fddx/79u0jPz8/Xn7t2rXMnDmTF154IZ6YioqKuOiiixg1ahShUIjnnnuOTz75hFdffZULLrgg/t4HHniAX/7yl1x22WXMnDmTQ4cO8fjjj+Nyufj888/Jysrq0s8uUkgJkUL/9V//pQBVWloaP7Zt2zYFqMcff7xF2R//+MfK5XIpv98fP3b0v5VSKhQKqTFjxqizzz47fqysrEyZTCb1wAMPtCi7ZcsWZTabjzsuuhdpvolON2zYMCZMmMDzzz8fPxaNRnnxxRe58MILcTgc8eNH//vIkSPU1dUxY8YMPvvss/jxVatWoWkal112GYcPH44/8vPzGTp0KO+9917XfDDRKcx6ByB6h8svv5xf/OIX7Nu3j/79+7N69Wqqqqq4/PLLW5R79dVX+dWvfsWmTZsIBoPx4waDIf7vHTt2oJRi6NChrV7LYrF0zocQXUKSkugSl19+OYsXL+aFF17gjjvu4G9/+xsej4e5c+fGy7z//vtcdNFFnHXWWfzhD3+goKAAi8XCihUreOaZZ+LlNE3DYDDw+uuvYzKZjruWy+Xqks8kOockJZFSR9dojjZ48GCmTp3K888/z6233sqqVauYP38+NpstXubvf/87drudN998s8XxFStWtDjXkCFDUEoxePBghg0b1jkfROhG+pRESjmdToBWB09efvnlrFu3jieffJLDhw8f13QzmUwYDAai0Wj8WFlZGS+//HKLcpdccgkmk4klS5agjrl5rJSiuro6NR9G6EKGBIiU+vTTT5k6dSrz5s3jiiuuwGKxcOGFF+J0Otm7dy/FxcW4XC4sFguVlZUt+n/effddzjnnHGbMmMFVV11FVVUVv//978nPz2fz5s0tEtCDDz7I4sWLOf3005k/fz5ut5vS0lJeeuklbr75Zu688049Pr5IBR3v/Ike6v7771f9+/dXRqPxuOEBZ5xxhgLUjTfe2Op7n3jiCTV06FBls9nUiBEj1IoVK9Q999yjWvtR/fvf/67OPPNM5XQ6ldPpVCNGjFALFy5U27Zt66yPJrqA1JSEEGlF+pSEEGlFkpIQIq1IUhJCpBVJSkKItCJJSQiRViQpCSHSSreeZqJpGvv378ftdrc5vUEIoR+lFA0NDRQWFmI0tq8O1K2T0v79+ykqKtI7DCHESVRUVDBgwIB2le3WScntdgOxD5yZmalzNEKIY9XX11NUVBT/XW2Pbp2UmptsmZmZkpSESGOJdK9IR7cQIq1IUhJCpBVJSkKItNKt+5TaQylFJBJpsXCYaB+TyYTZbJbhFqJL9eikFAqFOHDgAH6/X+9Quq2MjAwKCgqwWq16hyJ6iR6blDRNo7S0FJPJRGFhIVarVf7iJ0ApRSgU4tChQ5SWljJ06NB2D34ToiN6bFIKhUJomkZRUREZGRl6h9MtORwOLBYLe/bsIRQKYbfb9Q4pre2s8vJZ+REmFmUxNK/943JESz02KTWTv+4dI99f+zSGory+5QARTXGwLkBhlgOnrcf/enUK+YkTIgV2VDUQ0WIrS0c0xY4qr84RdV+SlIRIgdLDPgDsltjmmHuPyM2VZElS6uUGDRrEsmXL9A6j2+vnspGbaWPaKdkM6OMgP1P635IljV4hUuCMkn6cUdIPgEnFfXSOpnuTmlIPEAqF9A5BiJTplUkpFNHafESiWrvLhttRNhmzZs3i1ltv5dZbb8Xj8dCvXz9++ctfxneIHTRoEPfffz/XXHMNmZmZ3HzzzQB88MEHzJgxA4fDQVFREbfddhs+ny9+3qqqKi688EIcDgeDBw/m6aefTio+0VIwEiWqtdw+MRCO4g9FdIqoe+uVzbffv7ezzdcG93Myf2L/+PP/WbuLcLT1/ToH9HFw6eRvF5l78sNSGkMtp7P85DvDkorxL3/5CzfccAOffPIJGzZs4Oabb6a4uJibbroJgIcffpi7776be+65B4Bdu3Yxd+5cfvWrX/Hkk09y6NCheGJbsWIFAAsWLGD//v289957WCwWbrvtNqqqqpKKT3zro53VbN5bx+klfZkyKJuPdh1m/e4aJhZnMWt4rt7hdTu9Mil1B0VFRfz2t7/FYDAwfPhwtmzZwm9/+9t4Ujr77LP56U9/Gi9/4403cvXVV3PHHXcAMHToUH73u98xc+ZMli9fTnl5Oa+//jqffPIJU6ZMAeCJJ55g5MiRXf7ZepraxhCaUjia7ry5bZbYcX9Yz7C6rV6ZlBbOLmnzNeMxM1FuPmtIm2WPnbVy/RmDOxJWC6eddlqLaTHTp0/nkUceiU8snjx5covyX3zxBZs3b27RJFNKxafbbN++HbPZzKmnnhp/fcSIEWRlZaUs5t7qiC+WfDyOWDJy22O/Vg1Bab4lo1cmJau5/V1pnVW2o5xOZ4vnXq+XW265hdtuu+24ssXFxWzfvr2rQutVopqiPhBLSlkZxySlgNSUktErk1J3sH79+hbP161bx9ChQzGZTK2WnzRpEl999RUlJa3XAkeMGEEkEmHjxo3x5tu2bduora1Nady9jS8UQSkwGgy4mqaVuJqSUjAcu9nRlX+segL5ttJUeXk5ixYtYtu2bTz77LM8/vjj3H777W2W//nPf85HH33ErbfeyqZNm9ixYwf/+Mc/uPXWWwEYPnw4c+fO5ZZbbmH9+vVs3LiRG2+8EYfD0VUfqUfyBmJNNKfNFG9u28wmbJbYr5bUlhKne1Lat28fP/zhD+nbty8Oh4OxY8eyYcMGvcPS3TXXXENjYyNTp05l4cKF3H777fFb/60ZN24ca9asYfv27cyYMYOJEydy9913U1hYGC+zYsUKCgsLmTlzJpdccgk333wzublyd6gjfE39Rm67GYJeOPgVRIK4m2pNXulXSpiuzbcjR45wxhlnMHv2bF5//XVycnLYsWMHffrIiFiLxcKyZctYvnz5ca+VlZW1+p4pU6bw1ltvtXnO/Px8Xn311RbHfvSjH3Uozt4uw2ZmRL6bbKsGG1fEEpM7n5Kc71KY5cBhbb25Ldqma1J66KGHKCoqio+jARg8OHV3sITobP2zHPTPckDZh7GEBNBQyfSBNZAzXN/guildm2+vvPIKkydP5tJLLyU3N5eJEyfy5z//uc3ywWCQ+vr6Fg8h0sLhprublqY+ukPf6BdLN6drUtq9ezfLly9n6NChvPnmm/z7v/87t912G3/5y19aLb906VI8Hk/80VO37F69erXM3O8m/KEIWqABGipjB4bNBUDVlBEIRaRPKQm6JiVN05g0aRK//vWvmThxIjfffDM33XQTf/zjH1stv3jxYurq6uKPioqKLo5YiJae/7SC//fWxzQEw+DsB31LwGCkpvYIT7yzmX9uOaB3iN2OrkmpoKCAUaNGtTg2cuRIysvLWy1vs9niW3TLVt1Cb0opvIEIjlA1FqMR3PlgMoMrB7PRiCtUhV9qSgnTNSmdccYZbNu2rcWx7du3M3DgQJ0iEqL9QlGNiKZwhqqxmIzgyou94C7AYjLgDB3GH5b9BhOla1L6yU9+wrp16/j1r3/Nzp07eeaZZ/if//kfFi5cqGdYQrRL84oQmdEaTEYDuJrGfGX0w2o24gjXEgwfvxyOODFdk9KUKVN46aWXePbZZxkzZgz3338/y5Yt4+qrr9YzLCHaxR+KYtJCuLSmoQDNNaWMbExGAxnR2N1hqS0lRve5b9/97nf57ne/q3cYQiTMH4pii9THmm4Wx7fDATL6YsCAW6sDpdEYipJpt+gbbDei+zQTIbqrxlAUe6QBs8kAjqxvX7B7wGjGalTYol78IakpJUKSkhBJynSYGe6JkOmwgOOoqVEGA9gzyXZaGd8PnDLVJCG6N9+E6K4G9nUysMAEygH2rJYv2j0M6FPDgAFWkO2WEtK7akpKQSTU9Q/V+hrfrXnqqafo27cvwWCwxfH58+fL5Nl0FKiN/ffo5huArWkMXaCuK6PpEXpXTSkahvcf6frrzvgpmK3tKnrppZdy22238corr3DppZcCsV1IXnvttROuACC6XiAcxeo/EvvL3kpNSaEI+2qJhqKyWkACeldNqRtwOBxcddVVLVZO+Otf/0pxcTGzZs3SLzBxnBc2lLPuq93UNYZb9ikB2DM5WB/kvS928M43B/UJsJvqXTUlkyVWa9Hjugm46aabmDJlCvv27aN///6sXLmSBQsWtNhIQOgv0liHAYXJbAabu+WLdg9mowFrxEu93H1LSO9KSgZDu5tRepo4cSLjx4/nqaee4txzz2Xr1q289tpreocljqKUQmtsAMDiyDx+axtbJmaTAVvUS0AGTyakdyWlbuTGG29k2bJl7Nu3jzlz5vTYZVq6q0BYwxyJ7T5syWhlYrjVhdloxKA0IkF/F0fXvUmfUpq66qqr2Lt3L3/+85+5/vrr9Q5HHKMxHMUa9WE2GjDa3ccXMJkx2TIAYustiXaTpJSmPB4P3//+93G5XMyfP1/vcMQxAuEo1qg/Nprb2kpSAsx2FwCGsI+wTMptN0lKaWzfvn1cffXV2Gw2vUMRx2gMR7FGfJiNxuM7uZuYHW4MgCXaKP1KCZA+pTR05MgRVq9ezerVq/nDH/6gdziiFU6rmUHuKFlBC9hcrZYxWF3kuG2Ysw0Y5c5pu0lSSkMTJ07kyJEjPPTQQwwfLjtipKN8j538HDP4M9qsKWF1MiTHBYVWsMmvWnvJN5WG2trXTaQRpSDYtJuOtfWaUjxZhXxdE1MPIX1KQiQhGPSjRZq25D5BTUmhCDU2EIxIn1J79fikpBKYDCuOJ99f697fUsr60hr2+2h7xL7Vya5DPj7YWsbmvTIxt716bFKyWGI/KH6/DFzriObvr/n7FDGRxljTzWhvo+kGTQMoDVijfrn7loAe26dkMpnIysqiqqoKgIyMDJk7lgClFH6/n6qqKrKysjCZZJb70bRgbECkyX6Cbb6akpJJC+ELhroosu6vxyYlgPz8fIB4YhKJy8rKin+P4lvRYKzz2uQ4QU3JbItN1gUiQensbq8enZQMBgMFBQXk5uYSDof1DqfbsVgsUkNqg2pKMhabs+1CBgNGqwNoIBqQpNReuiale++9lyVLlrQ4Nnz4cL755puUXsdkMskvl0iZqKYg3AiA1dHGnbcmxqakpYWkb7O9dK8pjR49mn/961/x52az7iEJcULBSBRLNJaULPYT1JQAc1NSikrzrd10zwBms1n6LES3U+RWWE1WjG1MMWlmdbjo57Jyov5w0ZLuSWnHjh0UFhZit9uZPn06S5cupbi4uNWywWCwxYL69fX1XRWmEHEZVjNjciwQcH+7AWUbnM5Mhua6IV8mVbeXruOUpk2bxsqVK3njjTdYvnw5paWlzJgxg4aG1tefWbp0KR6PJ/6Qhc+ELpSC5j4ia8aJyzYnraY+KHFyBpVGQ3Zra2sZOHAgjz76KDfccMNxr7dWUyoqKqKuro7MTKkfi64RCQUwfvBobOb/yXaq2bsRtf1Nwn2HYR77fYzG3jVWrr6+Ho/Hk9DvqO7Nt6NlZWUxbNgwdu7c2errNptN1hYSuvuyrJLG0hr6eZwMPdmmEBYHG8uPUL2/jMlDQvR1yc/vyaTVNBOv18uuXbsoKCjQOxQh2hRp2jAAS8bxGwYcy5qByWjAogUIRmT1yfbQNSndeeedrFmzhrKyMj766CO+973vYTKZuPLKK/UMS4gTigS9ABhP1p8EYMnAbDRijgYlKbWTrs23vXv3cuWVV1JdXU1OTg5nnnkm69atIycnR8+whDihSNCPiZNMxm1mcXxbUwpHOj22nkDXpPTcc8/peXkhkhJp9GLi24GRJ2SJNd9AEQr4AU8nR9f9pVWfkhDdgda0kqS5PTUlowmDxQ5AROa/tYskJSES1DyP7WRTTJo19z1FAt5Oi6knSashAUJ0BwUODVPQgt154sm4zZwuN5qvBpNNFnprD0lKQiRodI4Z7Jngal//UEHfPhRQA33k1609pPkmRKKap5hY2jEkAMAc61MiHOiceHoYSUpCJECLat8uQ3KSybhxlgyUUoRk+ZJ2kfqkEAmoPFLPnt2HyLCaGG9tX0f3oaCBnaU1+OormDO0kwPsAaSmJEQCQk1TTJTJ2vbWSscwN919U7JSQLtIUhIiAcHGBKaYNLHYv01KabQoR9qSpCREAsJNScnQzqYbfJuUTJEAEU2S0slIUhIiAdFA4jUlqy0DA2DSZFJue0hSEiIBzfu3Gds5mhvA0DQp16wFCcpOuSclSUmIBESDsTFK5pNsGNCCxRHbKVeFCYZk/8GTkSEBQiQg2xImkmHB5Upg+WWTDY/TijOiYSUIJJDQeiFJSkIkoCTLCCoTcrLb/yajkVPy+8ZGdNuko/tkpPkmRCLCCU4xaWaWXU3aS5KSEAloXkuJBIYEAGCxo1BEQ5KUTkaSkhDtFI1qrPumgk/KaggYEtuV5JvqMOt217ClrLKTous5pE9JiHYKBnwYUGgaWNuz6uTRmppvkeYVBkSbpKYkRDsF/E0rR5ptGM3tm/fWzNQ02DIalObbyUhSEqKdQk1TTE66VXcrzLZYTUmTPqWTSpuk9OCDD2IwGLjjjjv0DkWIVoWSmPfWrDkpyUoBJ5cWSenTTz/lT3/6E+PGjdM7FCHaFA7Eli0xJFNTakpkWlj6lE5G96Tk9Xq5+uqr+fOf/0yfPn30DkeINkUCsYRiSqKmZHE0L18iS+KejO5JaeHChVxwwQXMmTPnpGWDwSD19fUtHkJ0FZchSJ8MC+7MxDeUtDmcZDks9LPLKgEno/sOuZ999hmffvppu8ovXbqUJUuWdHJUQrSuv1NBfiYMyEv4vW6nm5EFmZDgXbveKKma0u7duzt84YqKCm6//Xaefvpp7HZ7u96zePFi6urq4o+KiooOxyFEuzX3ByXRp0TTLrlEgqBJbelEkkpKJSUlzJ49m7/+9a8EAsm1kTdu3EhVVRWTJk3CbDZjNptZs2YNv/vd7zCbzUSjx687Y7PZyMzMbPEQoqtoie5icrSmwZMKhYrIHbgTSSopffbZZ4wbN45FixaRn5/PLbfcwieffJLQOc455xy2bNnCpk2b4o/Jkydz9dVXs2nTJkwmUzKhCdFpPt+9n0/KatjnS+LXxmhk4z4/63fXcPhIXeqD60GSSkoTJkzgscceY//+/Tz55JMcOHCAM888kzFjxvDoo49y6NChk57D7XYzZsyYFg+n00nfvn0ZM2ZMMmEJ0XmUgpCfqKYwJzrFpIlmsqGAcECGBZxIh+6+mc1mLrnkEl544QUeeughdu7cyZ133klRURHXXHMNBw4cSFWcQugr3Ei0qS/I5kh8SACAoalfKRyUpHQiHUpKGzZs4Mc//jEFBQU8+uij3HnnnezatYu3336b/fv3c/HFFyd0vtWrV7Ns2bKOhCREp4gGfUQ1RdRoxW6zJneSpr4oSUonltSQgEcffZQVK1awbds25s2bx1NPPcW8efMwGmM5bvDgwaxcuZJBgwalMlYhdNO831vY5MBmTu5vefMOKBFJSieUVFJavnw5119/PQsWLKCgoKDVMrm5uTzxxBMdCk6IdBHyx6aYYMnAYDAkdQ6jNVZTkpUCTiyppPT2229TXFwcrxk1U0pRUVFBcXExVquVa6+9NiVBCqG3ZHbGPZapOSnJmkonlFQ9dMiQIRw+fPi44zU1NQwePLjDQQmRbqxaI30yLGR6kp+f6XS6yHJYyDRHUhhZz5NUTamt/dC9Xm+7R2cL0Z30sUTok58JgwqTPscpBf2gLhM8MgbvRBJKSosWLQLAYDBw9913k5HxbVU2Go2yfv16JkyYkNIAhUgLzRsGWJIbDgDIjibtlFBS+vzzz4FYTWnLli1Yrd/eGrVarYwfP54777wztREKkQZUyIcBkpv31qxpnJIKN5JcV3nvkFBSeu+99wC47rrreOyxx2Tumeg1tlUcpL66GkdOmLG5yZ2jKmBgT1kNBmsjk09LbXw9SVJ9SitWrEh1HEKkNRXyEVUkvt/bUUzWDCKawhwOxKatJDm0oKdrd1K65JJLWLlyJZmZmVxyySUnLLtq1aoOByZE2tCi8RUjrY7k5r0BWJvW6Y5GNVQkgCGZ1QZ6gXYnJY/HEx805vEkvvKeEN1W2E8kqqEwYLMnX1Oy2exoBjNGFSEcaMQqSalV7U5KRzfZpPkmepWQn4imiJjs2K3JL9ZqMRmImmwYIxHCQR9Wd3YKg+w5kho82djYiN//7ajUPXv2sGzZMt56662UBSZE2gj7iEQ1wkYHDkvyY4wMBkN8WEAo4EtVdD1OUknp4osv5qmnngKgtraWqVOn8sgjj3DxxRezfPnylAYohN7CAS9RFZuMa7d2cK+N+PIlMlapLUmvPDljxgwAXnzxRfLz89mzZw9PPfUUv/vd71IaoBB604K+2C4mbg9WU8eSUqY7kyyHBXM0mKLoep6kGsh+vx+32w3AW2+9xSWXXILRaOS0005jz549KQ1QCL3ZtAAj8jNhQHGHb+NPGJwP9oNgk80D2pL0xgEvv/wyFRUVvPnmm5x77rkAVFVVyYBK0fN0ZBeTYzXvaiJTTdqUVFK6++67ufPOOxk0aBDTpk1j+vTpQKzWNHHixJQGKITeVHzeWwqSkrl5qyXZKbctSTXffvCDH3DmmWdy4MABxo8fHz9+zjnn8L3vfS9lwQmRDsorD3FwbzWaM8BpyS8SAMDmg0GCZTVkRqsYPiI18fU0SQ+6yM/PJz8/v8WxqVOndjggIdJNNOgnqkClYLCjZrIT0ZSsPnkCSSUln8/Hgw8+yDvvvENVVRXaMTt+pmIHXSHSglJEg7FVJ612d4dPZ7LFmoBK+pTalFRSuvHGG1mzZg0/+tGPKCgoSHrNYiHSXjRMNBwCwJaR/Ly3Zpam+W+SlNqWVFJ6/fXXee211zjjjDM6dPHly5ezfPlyysrKABg9ejR33303559/fofOK0TKhH1ENIVmMONwdLz5ZrUfVVOSlQJaldTdtz59+pCd3fF5OwMGDODBBx9k48aNbNiwgbPPPpuLL76YrVu3dvjcQqREyE84qhE22bF3YIpJM4stNqFXi0ZAk7W6W5NUUrr//vu5++67W8x/S8aFF17IvHnzGDp0KMOGDeOBBx7A5XKxbt26Dp1XiJQJ+YhEFWGjg4wOTMZtZrPZURiIakrGKrUhqW/5kUceYdeuXeTl5TFo0CAsFkuL1z/77LOEzxmNRnnhhRfw+XzxcU/HCgaDBIPfDs+vr69P+DpCJEKFvLjtZsIZWWRYO15TstvMOJwuMgg2jVWSwcbHSiopzZ8/P2UBbNmyhenTpxMIBHC5XLz00kuMGjWq1bJLly5lyZIlKbu2ECdjCHljU0wKB0IKmm+ZdgsTBxeAv1pqSm0wqLb2S+oioVCI8vJy6urqePHFF/nf//1f1qxZ02piaq2mVFRURF1dnUxvEZ1j2+uwfxMMOhMGz0jNOT97Cur2wZhLIGd4as6Zpurr6/F4PAn9jibdSK6treXFF19k165d/OxnPyM7O5vPPvuMvLw8+vfv3+7zWK1WSkpKADj11FP59NNPeeyxx/jTn/50XFmbzYbNZks2ZCESpoJeQGGwdXw4QJxstXRCSSWlzZs3M2fOHDweD2VlZdx0001kZ2ezatUqysvL42stJUPTtBa1ISH0dPDwYfaU1dBo9TOrg1NMmr1f5sVeXUNJfj19UnTOniSpu2+LFi1iwYIF7Nixo8WOuPPmzWPt2rXtPs/ixYtZu3YtZWVlbNmyhcWLF7N69WquvvrqZMISIuUiAS+agog5BZNxmwSxEtEU4WDH7l73VEnVlD799NNWm1f9+/ensrKy3eepqqrimmuu4cCBA3g8HsaNG8ebb77Jd77znWTCEiK1lEJrmmJiyUhdn6XRGvtDHglJUmpNUknJZrO1ejt++/bt5OTktPs8TzzxRDKXF6JrhP1EwrEBjjZHx+e9NTM2rcsUlZpSq5Jqvl100UXcd999hMNhILYgenl5OT//+c/5/ve/n9IAhdBNyEcoqhEx2nHaU3eDxWRt2v8tJB3drUkqKT3yyCN4vV5ycnJobGxk5syZlJSU4Ha7eeCBB1IdoxD6CHkJRxUhUwYZto6PUWpmaqopaZKUWpVU883j8fD222/z4Ycf8sUXX+D1epk0aRJz5sxJdXxC6Cfojc17M2fgsnV8ikkzsy2DKJKU2pLwN61pGitXrmTVqlWUlZVhMBgYPHgw+fn5KKVkGRPRc4R8uGxmfBkenClMSi6nm6jNjMMQStk5e5KEvmmlFBdddBH//Oc/GT9+PGPHjkUpxddff82CBQtYtWoVL7/8cieFKkQXC/kYludmWPEgSGFSGjUwFyo8sSdaFIypaxr2BAl90ytXrmTt2rW88847zJ49u8Vr7777LvPnz+epp57immuuSWmQQugi1BD7rzWFo7kBTLbYOkqqaaWAVI4W7wES6uh+9tln+cUvfnFcQgI4++yzueuuu3j66adTFpwQetKCXhQKrM7UnthoBHPT3TzZ1eQ4CSWlzZs3M3fu3DZfP//88/niiy86HJQQ6aDq0GE+Ka3hvbLUdkhXNQT4uNzP5n21Mv+tFQk132pqasjLy2vz9by8PI4cOdLhoITQnVJEA3VoCqKW1DavTAYD3qgZS1STmlIrEqopRaNRzOa285jJZCISkSU+RQ8Q9hMORwADNqcnpae2mo1EjDYiUYUKy6juYyV8923BggVtLh8is/tFjxFsiK/NncrR3AA2s4mI0Y4CIsFGLCd9R++SUFK69tprT1pG7ryJHiHYQCiiETR5yE3haG4Ai8mAZoolunDQL0npGAklpRUrVnRWHEKkl0B9bIqJ2YkzBRsGHM1gMGBomv8WDkjz7VhJzX0ToqdTwXpCUY2Q2ZnSKSbNmlcKiAR9KT93dydJSYhWaI31ZDksuDKzcdlTn5Tc7kxcNjNG6eg+Tuq/bSF6AFPYG5tiMqoETKn/2z1z9ECIeMAid6uPJTUlIVoTbJpiYuukXXIsTcvrSk3pOJKUhDiWUkQD9bEpJrbUrTjZgrU5KQVic+BEnDTfhDhW2M/+6gb21QXQ+gc5fVjqL/FlVZhQ+RH6OK0MDjd+m6SE1JSEOE6wgWBUI2S0Y7Z0ziiiKEb8mplQRJP5b8eQpCTEsQL1TQMnXZ0yHADAYTURNtqJRDUIy7CAo+malJYuXcqUKVNwu93k5uYyf/58tm3bpmdIQkCgllAkStCcibsThgMA2M0mwiYHEU1JTekYuialNWvWsHDhQtatW8fbb79NOBzm3HPPxeeTvxxCP6rxSKymZHZ1XlKyGok015RC8vN+NF07ut94440Wz1euXElubi4bN27krLPO0ikq0duFfEeIKghaMjut+Wa3NDXftNhKAbKy/bfS6u5bXV0dANnZ2a2+HgwGW6xE0NqGmEJ0VKC+GgCTIwtzJwycBHBYTERM9tiW4AGvTMo9Stp0dGuaxh133MEZZ5zBmDFjWi2zdOlSPB5P/FFUVNTFUYoeTylMoXr6Oq3k5eZ32mUsJiMOV6wmpsn23S2kTVJauHAhX375Jc8991ybZRYvXkxdXV38UVFR0YURil4h5MNlhmH5mcwaX9Kpl5o38RTG9vdg02QdsqOlRfPt1ltv5dVXX2Xt2rUMGDCgzXI2m63NBeaESIlAbey/Nnfnb30kU01apWtNSSnFrbfeyksvvcS7777L4MGD9QxHCGisJRzVUPbULoHbKklKrdK1prRw4UKeeeYZ/vGPf+B2u6msrARi24I7HA49QxO9VaCOLfvqOFDdyOQhAfIy7Z12qY8qGrGXH6F/dpg8pWJ7wQl9a0rLly+nrq6OWbNmUVBQEH88//zzeoYlerFo0xgln8mV0q26W9OorAQjGuFwGCLSr9RM15qSktnRIs0E66tRQNSSidPauX1KdrudqMFCWFOxJpyl82pl3Una3H0TIh001h0CwOLJwdDJzSm7xUjEZCMa1aRf6SiSlIRoFg4Q9McG5Do9uZ1+udiobkespiRjleIkKQnRrLGGQChK2OTA407trritybCaCZsyCEc1CDV0+vW6C0lKQjTzV9MYjtJozqKP09rpl8uwmgiZMghHlEzKPUpaDJ4UIi34a8jKsOJ15JPj7vxBuk6bGavDjT1iRAUbZFJuE0lKQjTzV9M/y0H/kqHg6vyk5LKZmTd5KGzbJX1KR5HmmxDN/LHVAcjo23XXtDb1XYW8XXfNNCc1JSEANI2gtxqiUayO7K5rSklSOo4kJSEAgnVU1njZWx8mXB5k1oiuuex7pX485UcY2C9MX00DozRe5BsQAsBXjT8UJWD20NfVdfMu/dgJRhShcEQGUDaRpCQEoLyV+EIR/NbsLrnz1izDZiFstBOOyrCAZpKUhAACtQcJRxV+a1/6ujp/jFKzDKvpqAGU0q8EkpSEAMBXsx8Aa2Yelk5al7s1GVYzIZNDktJRJCkJEQ0TqDsMgDO7oEsvnWEzETI5pfl2FElKQvgO0xgKEzY6yO7T+k46naVF8y0o899AhgQIAb4q+rlsRD0FZGU7u/TSTpsZqysLe9gEAdkyDCQpCQHeKvpkWOlTdAp4unahtUy7hYumjoDNmyEoSQmk+SYEeA/G/uvs/DWUWtW8SYEkJUBqSqK30zTqqsohGsGRkU/XDQY4is0d+284EFur29y7txGTpCR6N98h9lc3UB0Ed52F8V2ws9Kx1uyux7HXzyCPkdxgQ69PStJ8E72aqt+PNxjBa82lIEufbb0iUY0GlUEwokGgTpcY0omuSWnt2rVceOGFFBYWYjAYePnll/UMR/RCDYf2ENEUAXsu/bpgDaXWuGxmgmYXoYgm/UronJR8Ph/jx4/n97//vZ5hiF7Me6gCAFvfARiN+qz96LKbCZmcsaQkwwL07VM6//zzOf/88/UMQfRmkSC+I7FdmfvkDdQtDLfNEqsphaSmBN2sozsYDBIMfruTaH29/A8UyVP1+6n3hwmaXAzJ02k4ALGaUtDkaupTkp/pbtXRvXTpUjweT/xRVFSkd0iiG/NW7iYU1fA6Csnv4kGTR2vuU4pqioi/Rrc40kW3SkqLFy+mrq4u/qioqNA7JNGNORv3Ma6/hzGjx3bpygDHspqNuLJyyLSbiTbWQzSiWyzpoFs132w2GzZb7x7DIVIkGsborcRpM+M8Zbje0XDlGSNA5cYGTwZqwdlP75B0061qSkKkTN1e0KKx0dSOPnpHAwbDt3H08iacrjUlr9fLzp07489LS0vZtGkT2dnZFBcX6xiZ6OmO7N9JdVUDzuIhFBjSZBtIRx9U/QEMjUf0jkRXuialDRs2MHv27PjzRYsWAXDttdeycuVKnaISvUH13u0c8oY4EMqma5d1a92uQ16+2RlgiL+BEf0lKelm1qxZKKX0DEH0RiEf/sOxmyQ5A0fpHEyM1WTkiOYkEI5CL68pSZ+S6HV8B7bjC0bwW/tSXKjf+KSjeTIsNFo8BCMamu+w3uHoSpKS6HUOl38FgLFfCRnW9LgB7baZCduy0RSEfHWxZUx6KUlKonfRNOr3bQcgu3ikzsF8y2Aw4HK5CJmcBCJR8B3SOyTdSFISvYqvajc+n4+I0c6gwUP1DqcFj8OC35JNIKxJUhKitwgf+JIMmwmVMxxPRnoNxO3nsuG3ZuMPRnp1UkqPBrUQXUGLkuXdTVb/LEJjZ+gdzXHyMu14swuwN+zo1UlJakqi96jZHZvGYXNhzU6/wbkluS7OmzqWQo8DvFXQS4fLSE1JdLlwJEJ5eRmV+ysocUfJywCiQRpDYbZVNuC2m8m0W3BnerC7siGzEFx5YOzY39DqHevxaBrm3JEdPlencfYDkzmWPP014Oyrd0RdTpKS6DIqUM+eTas5uGMjWsiHEQj3cUCfDABCjWEaD9TTCFQ1vcdqMuC0mXF5svEMmYp7yGlJLazvrzvMzm++wKAUwyeMQYf9AdrHaCKckQe1FVjq90lSEqJTKEWo7GNKP32DmgY/AGarDU9uMc7cfMjMBLMNY0jD1cdHXWOEOn+QUGM9tnA9mYEDHPEfxBh8B3f1Fhh2HvRL7M7Zri/eR9MUqs9gMrPzOuNTpsRHuw5TWWpgsjFAcf1+KBind0hdTpJSD/b1gXp2VnkZ29/DwL4ZGPSYeBoJEtq8iu1ffUFDMILPnkfumNmMGjUes8XSoqgbGHtUrglFNKp9QfZVN1BXvpm+5m2x5WK3vMjh7AnU5p9OSd7J6zxebz3e3Z8AUDTmDH2+h3bKtFvYYc2lwbsV6vfpHY4uJCl1d5oG1TuIVm1ne/leBuVk4sjuD3ljqPUb2FnlZWeVl5JcF98ZlYfdYuq62EI+2Pw8VeWl1IVgf+5MzjjzHAqbmmsnYzUbKfA4KPA44JQ5EJ0JpWtRFes5sGUN+7/8mm0j5zNzdDEuW+s/ykopvvz4TYiGMXvyGVCS3jWPfI+dBmsuvmAEzVuFMdwIFn22ftKLJKXurLYctr1BxHuIbZUN1Aci7Km1MCK/HMrXMTZrBOHCiWyqDLGzyssRf4jvTxqAs41f4JQKB+CL58BbRWFuP8oGz2ZWyXBy3B0YG2SyQMk5aO5CMutewFe9n8Ytf+Vvh89n8qihjCn0HLcjydYdO9HKP8EIDJkyD0O6dnA36eu0YnS4aTD3oTEYwXmkDHLTZ+R5V0jv/0OidUpB6fuw6RmivsN8WRXiG8soyvLm4JlwUay/RSlcR77mrPpXuWKEGbfdTLU3xIsb9+ILdvJyq9EIfPl3lPcgWDMwTvoRZ04c17GEdBRT3kgGzvk3Rg8pop/Jz7B9q9j42Qae31DBziovUS12K/1IXQNV61/AgCLnlHFpNa2kLQaDgUKPg1p7EXWN4dgwhl5GklJ3o0Xhm1eh7AOUprEhOIC12ZdyMPd0zpk5i4KR02HsD2DSNbGVDAP15O58kcsGB3HbzdT4Qvxj0/7YHmOdEp8GX79CZfl2dlWHiI65DDKyU38dVy7u029k9KjRDOljYXT1W1j3rOGfn5fSEAhDoI4+u16m0NxATt9sTpk+P7a6YzcwqJ+TOscAjvhDsaTUy8YrSfOtO4mGYetLUL0LZTCwzjKNddEBmI0GLp7Qn1z3UTtyePrDqQvg61egeheZO1/hssHn8UyZG6vZiNYZP+hKwc5/UbtnC7urG/kmZy6RRicjOuv+u9WJccJVFGS8Sd+9n5NXt43BVV/h2TIIGmtBaZQU9sM04QoM9sxOCiL1Bvd1stqWT23ISKSxHnNdBWSl32DPziI1pe4iGoYtL0L1LjCZ2ZL9HdY1DsBggPPHFtA/q5XOUIsdxnwf8kaB0sgsfYMrB9TwvYn9O6fDe89H+EvXs73Kx87sWfQ/ZRTD89ypv87RjCYYfj7W8ZcysGggEwe4MfhrQGmQVYR58gIMnv6dG0OKeTIsjOyfTdHwiRgNBqj8Uu+QupTUlLqDSBC2vAC1FbHO3nGXMcCSj6d+H5MH9aEk19X2e40mGHkRmO2w7zM85f8CiwbF01BKcaAuQGFrCS1R+zcR3rmabyob2OU5DceAscwZmdc1t98NBsgZBv2Gxta3DtaD3ZMeGwIk6bzR+XBkKmzaCYe+hpI5YLbqHVaXkKSU7sIB2PI3qNsX+6Ecdzl4BpAN/PC0gVjN7ajsGgww9FwwWaF8Hex6FxUJ8H54JBvLazl7RC7ji7KSj7Hqa8Jf/5OvK+vZnTGOQN5ErhhXiMnYxX04BkOs/6oz+rD0kFUc+yz+Gtj/ORRP0zuiLiHNt3TWeAQ+/3+xhGSxU140n12hrPjL7UpIzQwGGDIbTpkZe7rnI/pVrgWl8d62Krbur0suxsotqK0vs72ynlJzCdU5p3HxhP44rF04HqqHCmuKryyjKav2QcW6WI25F5CklK6qd8FnT4HvMNjcbMu/iJd3aby2+QAH6hqTP+/A02HodwAYqW1nbnQ15oift7YeZP3u6vZv5KAUlK6Fr1/FAOQMm8KhwtlccmoR2c7e0czobDW+EG8d7kuZ34q3oR52vad3SF0iLZLS73//ewYNGoTdbmfatGl88skneoekn5Aftr8Jm/8GIT/hjFzWuM/nn6VRoppiSI6r5V22ZAyYDKMuxmCyMsJew7zAa+R6v+GjnYd55Yv9eE82jslXjdr0NI071saeF00hd/IlLDjjlJSNRRKx9ZWGFWSxq88MdlY1ENm7EQ58oXdYnU73PqXnn3+eRYsW8cc//pFp06axbNkyzjvvPLZt20ZubnrsNNElAvVwYBPs3QCRIFGl2GMfwXuB0dTXRwGYNjib6UP6pqbzOG8UuPIwfPUSp3CIjPqNfFP5BQe8o3ilbihXnjW25XU0DeoqCO39nNqyzVTW+fBFDIyZeSnuwacCYDZ1j3FA3cnZI3L5a20juwJjMe7/kqHaqziCDVA8PXYTowcyKJ03Xps2bRpTpkzhv//7vwHQNI2ioiL+4z/+g7vuuuuE762vr8fj8VBXV0dmZjcZh6JFIeyPzQvzHQbfISI1pUTqDmAzGTBgAFcOr/hGsiucA8TWbj5nZC4D+zo7J559G6HsA/x+H7sP+ejrslKQmwO2THxhxVcVh7GFa4lGQgTDGgqotRexL2cGM8YNZXh+J9/27+UOe4P8fUMFeQfXUuD7in4uG8WF+VgKx0LWQMjoG9t+PA0HhybzO6prTSkUCrFx40YWL14cP2Y0GpkzZw4ff/zxceWDwSDB4LedffX19e27UE0p7PzX8cdb5GMVP7bzkJcjviDq2HIqNgp60sBszE3//3cfbuBQQyj2fnXUuVTsvxOLs7CajICi7FADh+saWp4S4tMiJo0bh23gFOg3nL67azhc2cCkgX0YXZiJxdRJLW2jCYqmQsEEMg5uYfTBrai6/RD0QtBL0B8iUt9ABIgY7dQ4BxLKG0/RwCHMGpAlHdpdoJ/LxpWnDeStreeyvSKXSP2nDA41wJ6PYc/HfFNZjy+koYwWNJMVZTACBgwGA2aTiUkDm+5GGgx8daCehkDrzXOjwcCUQQkMoxh7GXTCoFRdk9Lhw4eJRqPk5bVc3yYvL49vvvnmuPJLly5lyZIliV8oEozVStpJ+RuI+EJtFwiY4isXqoAXLXCCuyJhG2hNZSMhwlEFGAgb7QQsHvyWPnituTRkFDFs5EhsrlifzNTB2ZyeqqZae5it0P9UDP1PxRAJgr8aAvXYQxEKh0aJ2vtgcWYzyWXvmgm9ooVMu4UfnDqAfUP6UlN/GkZrJVTvBO9BgtF6QpEoEAVa7hdnNBnA9+2UItVQh9ZGUjIYAG8C8yJV50xV6lY/XYsXL2bRokXx5/X19RQVFZ38jVlFMP6KY6q3Tf9u5Vhff5iMULRlQmgac2MwGDG6bbH3GQzkNIZxRbRYs+vo0zX9w+S0x9+bE1a4lQWD2Q6G2F8yowHsFhM2s7HF9TqtZtQeZltsCdrMQjKAgfpFIo7RP8vRNHq/H+SPAWDQ6ADBgI9oOADRUKxJjkIpFfvRc9viCSRvSJCsSOs9NgYD4EngJoq1E7oT0Dkp9evXD5PJxMGDB1scP3jwIPn5+ceVt9ls2GxJ3N2xOiF7cLuLZ2e1/9SZbmhvBdbV9BAilTwuO7jal0z6dYNxpboOCbBarZx66qm888478WOapvHOO+8wffp0HSMTQuhF9+bbokWLuPbaa5k8eTJTp05l2bJl+Hw+rrvuOr1DE0LoQPekdPnll3Po0CHuvvtuKisrmTBhAm+88cZxnd9CiN5B93FKHdEtxykJ0Ysk8zuaFtNMhBCimSQlIURakaQkhEgrund0d0Rzd1i7p5sIIbpU8+9mIl3X3TopNTTE5pG1a1S3EEI3DQ0NeDzt20GiW9990zSN/fv343a703or5s7UPNWmoqJC7kAmQb6/jjnZ96eUoqGhgcLCQozt3Ai0W9eUjEYjAwYM0DuMtJCZmSm/VB0g31/HnOj7a28NqZl0dAsh0ookJSFEWpGk1M3ZbDbuueee5FZPEPL9dVBnfH/duqNbCNHzSE1JCJFWJCkJIdKKJCUhRFqRpCSESCuSlLqhmpoarr76ajIzM8nKyuKGG27A6/We8D2zZs3C0LRZQfPj3/7t37ooYn0lugPzCy+8wIgRI7Db7YwdO5Z//vOfXRRpekrk+1u5cuVxP2d2e4I7OivR7cydO1eNHz9erVu3Tr3//vuqpKREXXnllSd8z8yZM9VNN92kDhw4EH/U1dV1UcT6ee6555TValVPPvmk2rp1q7rppptUVlaWOnjwYKvlP/zwQ2UymdRvfvMb9dVXX6n//M//VBaLRW3ZsqWLI08PiX5/K1asUJmZmS1+ziorKxO6piSlbuarr75SgPr000/jx15//XVlMBjUvn372nzfzJkz1e23394FEaaXqVOnqoULF8afR6NRVVhYqJYuXdpq+csuu0xdcMEFLY5NmzZN3XLLLZ0aZ7pK9PtbsWKF8ng8HbqmNN+6mY8//pisrCwmT54cPzZnzhyMRiPr168/4Xuffvpp+vXrx5gxY1i8eDF+v7+zw9VV8w7Mc+bMiR870Q7MEPt+jy4PcN5557VZvidL5vsD8Hq9DBw4kKKiIi6++GK2bt2a0HW79YTc3qiyspLc3NwWx8xmM9nZ2VRWVrb5vquuuoqBAwdSWFjI5s2b+fnPf862bdtYtWpVZ4esm0R3YIbY99ta+RN9tz1VMt/f8OHDefLJJxk3bhx1dXU8/PDDnH766WzdurXdk+clKaWJu+66i4ceeuiEZb7++uukz3/zzTfH/z127FgKCgo455xz2LVrF0OGDEn6vEIcbfr06S32bDz99NMZOXIkf/rTn7j//vvbdQ5JSmnipz/9KQsWLDhhmVNOOYX8/HyqqqpaHI9EItTU1LS6q3Bbpk2bBsDOnTt7bFJKdAdmgPz8/ITK92TJfH/HslgsTJw4kZ07d7b7utKnlCZycnIYMWLECR9Wq5Xp06dTW1vLxo0b4+9999130TQtnmjaY9OmTQAUFBSk+qOkjWR2YJ4+fXqL8gBvv/12r9yxORU7WEejUbZs2ZLYz1mHusmFLubOnasmTpyo1q9frz744AM1dOjQFkMC9u7dq4YPH67Wr1+vlFJq586d6r777lMbNmxQpaWl6h//+Ic65ZRT1FlnnaXXR+gyzz33nLLZbGrlypXqq6++UjfffLPKysqK36b+0Y9+pO666654+Q8//FCZzWb18MMPq6+//lrdc889vX5IQCLf35IlS9Sbb76pdu3apTZu3KiuuOIKZbfb1datW9t9TUlK3VB1dbW68sorlcvlUpmZmeq6665TDQ0N8ddLS0sVoN577z2llFLl5eXqrLPOUtnZ2cpms6mSkhL1s5/9rFeMU1JKqccff1wVFxcrq9Wqpk6dqtatWxd/bebMmeraa69tUf5vf/ubGjZsmLJarWr06NHqtdde6+KI00si398dd9wRL5uXl6fmzZunPvvss4SuJ0uXCCHSivQpCSHSiiQlIURakaQkhEgrkpSEEGlFkpIQIq1IUhJCpBVJSkKItCJJSQiRViQpiW5l5cqVZGVlxZ/fe++9TJgwIf58wYIFzJ8/v8vjEqkjSUm0asGCBW2u471w4UIMBkOLVQ06IxkMGjSIZcuWtTh2+eWXs3379jbf89hjj7Fy5cr481mzZnHHHXekNC7RuSQpiTYVFRXx3HPP0djYGD8WCAR45plnKC4u1iUmh8Nx3CJ3R/N4PC1qUqL7kaQk2jRp0iSKioparE65atUqiouLmThxYofO3VoNZv78+fHa16xZs9izZw8/+clP4rtiwPHNt2MdXWNbsGABa9as4bHHHoufo7S0lJKSEh5++OEW79u0aRMGgyGhdX9E55CkJE7o+uuvZ8WKFfHnTz75JNddd12nX3fVqlUMGDCA++67jwMHDnDgwIGEz/HYY48xffp0brrppvg5iouLj/tMACtWrOCss86ipKQkVR9BJEmSkjihH/7wh3zwwQfs2bOHPXv28OGHH/LDH/6w06+bnZ2NyWTC7XaTn5+f1MqPHo8Hq9VKRkZG/Bwmk4kFCxawbdu2+P5l4XCYZ555huuvvz7VH0MkQZbDFSeUk5PDBRdcwMqVK1FKccEFF9CvXz+9w+qQwsJCLrjgAp588kmmTp3K//3f/xEMBrn00kv1Dk0gNSXRDtdffz0rV67kL3/5S8pqE0ajkWOX8gqHwyk5d3vceOON8U78FStWcPnll5ORkdFl1xdtk6QkTmru3LmEQiHC4TDnnXdeSs6Zk5PTop8oGo3y5ZdftihjtVqJRqMduk5b55g3bx5Op5Ply5fzxhtvSNMtjUjzTZyUyWSKb+9kMpnaLFdXVxffkKBZ3759KSoqOq7s2WefzaJFi3jttdcYMmQIjz76KLW1tS3KDBo0iLVr13LFFVdgs9mSajYOGjSI9evXU1ZWhsvlIjs7G6PRGO9bWrx4MUOHDu2VGwOkK6kpiXbJzMwkMzPzhGVWr17NxIkTWzyWLFnSatnrr7+ea6+9lmuuuYaZM2dyyimnMHv27BZl7rvvPsrKyhgyZAg5OTlJxX3nnXdiMpkYNWoUOTk5lJeXx1+74YYbCIVCXXI3UbSfrNEteq3333+fc845h4qKiuN2gRX6kaQkep1gMMihQ4e49tpryc/P5+mnn9Y7JHEUab6JXufZZ59l4MCB1NbW8pvf/EbvcMQxpKYkhEgrUlMSQqQVSUpCiLQiSUkIkVYkKQkh0ookJSFEWpGkJIRIK5KUhBBpRZKSECKt/H9Fo9AF7h62WgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:11:42.474328Z", + "iopub.status.busy": "2024-02-29T18:11:42.474060Z", + "iopub.status.idle": "2024-02-29T18:11:42.757409Z", + "shell.execute_reply": "2024-02-29T18:11:42.756574Z" + }, + "papermill": { + "duration": 0.305409, + "end_time": "2024-02-29T18:11:42.759353", + "exception": false, + "start_time": "2024-02-29T18:11:42.453944", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8lUlEQVR4nO3deXxU5b348c/sSzIzISErBBIIIPtSBMEFUKqCG7VXraUKarW34m0pl1a5tz9abS21V7li66W9vQVqC6Vq0S62onUBRUFAWRQEEhKIQAgkZJmZzHrO749JRgIJzEwmOZPk+3695oWZOXPON2PyzfM853m+j05VVRUhhEgReq0DEEKIs0lSEkKkFElKQoiUIklJCJFSJCkJIVKKJCUhREqRpCSESCmSlIQQKcWodQAdoSgKx48fx+FwoNPptA5HCHEOVVVpbGykoKAAvT62NlC3TkrHjx+nsLBQ6zCEEBdRWVlJ//79Yzq2Wyclh8MBRL5hp9OpcTRCiHM1NDRQWFgY/V2NRbdOSi1dNqfTKUlJiBQWz/CKDHQLIVKKJCUhREqRpCSESCndekwpFqqqEgqFCIfDWofSoxkMBoxGo0zNEB3Wo5NSIBDgxIkTeL1erUPpFex2O/n5+ZjNZq1DEd1Yj01KiqJQXl6OwWCgoKAAs9ksf8U7iaqqBAIBTp06RXl5OUOGDIl5opwQ5+qxSSkQCKAoCoWFhdjtdq3D6fFsNhsmk4kjR44QCASwWq1ah9TlSqvdfHj0DOMLMxiSG/u8HNFaj01KLeQvdtfpzZ91UyDMP/aeIKSonKz3UZBhI83S43+9OkXv/SkSIokOVTcSUiJ7cIQUlUPVbo0j6r4kKQmRBOWnPQBYTQYAPjsjN1cSJe1LIZKgb7oFtz/E8HwnZdVu8py9b0wtWSQppaDp06czbtw4nn76aa1DETG6vKQvl5f0BWDCgD4aR9O9SfdNCJFSemVSCoSUdh+hsBLzscEYjo3X/Pnz2bRpEytWrECn06HT6ejfvz8rV65sddxHH32EXq/nyJEjACxfvpzRo0eTlpZGYWEhDz74IG5368HWd999lyuvvBKbzUZhYSHf+ta38Hg8cccoWvOHwoSV1htN+4JhvIGQRhF1b72y+/bsW6XtvlbcN4054/tFv/7fzWUEw23vbN6/j43bJn5eZG7VlnKaAq2Xs3zni0Pjim3FihUcPHiQUaNG8dhjjwHw1FNPsW7dOr75zW9Gj1u7di2XX345AwcOBCK345955hmKi4s5fPgwDz74IN/73vf4n//5HwDKysq4/vrr+fGPf8yqVas4deoUDz30EA899BCrV6+OK0bR2nulNez5rJ6pJVlcWpTJe2Wn2Xa4lvEDMpg+LEfr8LqdXtlSSmUulwuz2YzdbicvL4+8vDzmzp3Lli1bOHr0KBCZrb5+/Xrmzp0bfd/ChQuZMWMGRUVFXH311fz4xz/m+eefj76+bNky5s6dy8KFCxkyZAhTp07lmWee4bnnnsPn83X599mT1DUFUFQVW/OdN4fFFHneG9QyrG6rV7aUFswoafc1/TkrUR64anC7x567auXey4s7Ela7xo0bx/Dhw1m3bh2PPPIImzZtorq6mttuuy16zD//+U+WLVvGp59+SkNDA6FQCJ/Ph9frxW63s3v3bvbs2cPatWuj71FVNbocZ/jw4Z0Se29wxhNJPi5bJBk5rJFfq0a/dN8S0StbSmajvt2H0aCP+VhTDMcmy9y5c1m3bh0A69at4/rrrycrKwuAiooKbrzxRsaMGcOf/vQndu7cybPPPgtEltsAuN1uvvGNb7Br167oY/fu3Rw6dIjBg9tPvOLCwopKgy+SlDLs5yQln7SUEtErW0qpzmw2n1dq5atf/Srf//732blzJy+++CK//OUvo6/t3LkTRVF46qmnoks9zu66AUyYMIF9+/ZRUtJ+K1HEzxMIoaqg1+lIb15Wkt6clPzByM2OZP5x6g3k00pBRUVFbNu2jYqKCk6fPo2iKBQVFTF16lTuu+8+wuEwN998c/T4kpISgsEgP//5zzl8+DC/+93vWiUtgIcffpj33nuPhx56iF27dnHo0CH+/Oc/89BDD3X1t9ejuH2RLlqaxRCtQmExGrCYIr9a0lqKn+ZJ6dixY3zta18jKysLm83G6NGj2bFjh9ZhaWrx4sUYDAZGjBhBdnZ2dIB77ty57N69my996UvYbLbo8WPHjmX58uU88cQTjBo1irVr17Js2bJW5xwzZgybNm3i4MGDXHnllYwfP56lS5dSUFDQpd9bT+NpHjdyWI3gd8PJfRDy42huNbllXCluOlVV277f3QXOnDnD+PHjmTFjBt/85jfJzs6OjnHEMs7R0NCAy+Wivr7+vN1MfD4f5eXlFBcX98oyGlrojZ/5sbom9lTWkWlWmFyzIZKYHHm8n3Ej3qDC6P4uchy947Noy4V+R9uj6ZjSE088QWFhYat5MsXFnXMHS4jO0C/DRr8MG1RsiSQkgMYqpgyshexh2gbXTWnaffvLX/7CxIkTue2228jJyWH8+PH8+te/bvd4v99PQ0NDq4cQKeH0wci/puZu9alPtYulm9M0KR0+fJiVK1cyZMgQNm7cyDe/+U2+9a1v8dvf/rbN45ctW4bL5Yo+ZMtuoTVvIITia4TGqsgTQ68HQK2twBcIyZhSAjRNSoqiMGHCBH7yk58wfvx4HnjgAe6///7z7hy1WLJkCfX19dFHZWVlF0csRGt/3F7J7157n0Z/ENL6QlYJ6PTU1p3hN2/s4e97T2gdYrejaVLKz89nxIgRrZ4bPnx49G7TuSwWS3SLbtmqW2hNVVXcvhC2QA0mvR4ceWAwQno2Rr2e9EA1XmkpxU3TpHT55Zdz4MCBVs8dPHgwushUiFQWCCuEFJW0QE1kdn96buQFRz4mg460wGm8QdlvMF6aJqXvfOc7bN26lZ/85CeUlpaybt06/vd//5cFCxZoGZYQMWmpCOEM12LQ6yC9uSKAvS9mox5bsA5/8PxyOOLCNE1Kl156KS+99BJ/+MMfGDVqFD/60Y94+umnW61+FyJVeQNhDEqAdKV5KkBLS8meiUGvwx6O3B2W1lJ8NF/7duONN3LjjTdqHYYQcfMGwlhCDZGum8n2+XQAexY6dDiUelAVmgJhnFaTtsF2I5ovMxHnmz59OgsXLtQ6jKhUiydVNAXCWEONGA06sGV8/oLVBXojZr2KJezGG5CWUjwkKfVQLSVLROdx2owMc4Vw2kxgO2uzAJ0OrE4y08yM7QtpZoN2QXZDkpRSTFs1usvKyrjvvvsoLi7GZrMxbNgwVqxYcd775syZw+OPP05BQQHDhkWWOLz33nuMGzcOq9XKxIkTefnll9HpdOzatSv63o8//phZs2aRnp5Obm4ud911F6dPn243noqKiq76OFLawKw0Lss3UOCygTWj9YtWF/372Lm8v5kc2W4pLpqPKXUpVYWwBqUkDKbzy1S2o60a3X369KF///688MILZGVl8d577/HAAw+Qn5/P7bffHn3vG2+8gdPp5PXXXwciiyFvuukmZs+ezbp16zhy5Mh53bC6ujquvvpqvv71r/Pf//3fNDU18fDDD3P77bfz5ptvthlPdnZ2Ej6UHsJXF/n37O4bgKV5Dp2vviuj6RF6V1IKB+Gdp7r+ulf+OxjNMR16bo3uFo8++mj0v4uLi3n//fd5/vnnWyWltLQ0/u///g+zOXKtX/7yl+h0On79619jtVoZMWIEx44d4/7774++5xe/+AXjx4/nJz/5SfS5VatWUVhYyMGDBxk6dGib8YjIjiVm75lId6ONlpKKStBTRzgQxiZduJj1rqTUjT377LOsWrWKo0eP0tTURCAQYNy4ca2OGT16dDQhARw4cIAxY8a0KiMyadKkVu/ZvXs3b731Funp6edds6ysjKFD49uNpTd5YcdRBu87zMh8By7bORtQWp2cbPCz+/ghgspJbhwjdati1buSksEUabVocd0OWL9+PYsXL+app55iypQpOBwO/uu//ott27a1Oi4tLS3uc7vdbm666SaeeOKJ817Lz89POObeINRUjw4Vg9EIFkfrF60ujHod5pCbBrn7FpfelZR0upi7UVo6t0b3li1bmDp1Kg8++GD0ubKysoueZ9iwYfz+97/H7/djsVgA2L59e6tjJkyYwJ/+9CeKioowGtv+cWirZnhvp6oqSlMjACab8/wxQ4sTo0GHJezGJ5Mn4yJ331LQuTW6hwwZwo4dO9i4cSMHDx7k//2//3decmnLV7/6VRRF4YEHHmD//v1s3LiRJ598EiBaT3rBggXU1tZy5513sn37dsrKyti4cSP33HNPNBG1VTO8t/MFFYyhyO7CJnsbC8PN6Rj1enSqQsjv7eLoujdJSino3Brd1113Hbfeeit33HEHkydPpqamplWrqT1Op5O//vWv7Nq1i3HjxvGf//mfLF26FCA6zlRQUMCWLVsIh8Nce+21jB49moULF5KRkRHdGaW9muG9WVMwjDnswajXobc6zj/AYMRgsQNE6i2JmGlao7ujpEZ3/NauXcs999xDfX19q80HkqE3febH65p4//UXGdS0l/FTroUhM887Jvj+r9ix7xD7c2Zz1+zp5+0T2Bt0uxrdovM999xzDBo0iH79+rF79+7oHKRkJ6TepikYxhzyYNTrzx/kbma0OdABpnATvmC4VyalREhS6uGqqqpYunQpVVVV5Ofnc9ttt/H4449rHVa3l2Y2UuQIk+E3geX86RQAOnM62Q4Lxkwd+hgnzwpJSj3e9773Pb73ve9pHUaPk+eykpdtBK+93ZYS5jQGZ6dDgRks8qsWK2lPCpEIVQV/82465rZbStFkFfB0TUw9hCQlIRLg93tRQs3rKC/QUlJRCTQ14g/JXKVY9fik1I1vLnY7vemzfmdvOdvKaznuof0Z++Y0yk55ePeTCvZ8JgtzY9Vjk5LJFPlB8Xpl4lpXafmsWz77nizUFOm66a3tdN2geQKlDnPYK7O649BjR98MBgMZGRlUV1cDYLfbo7OYRXKpqorX66W6upqMjAwMhp6/Il7xRyZEGqwXmHvTnJQMSgCPX4ruxarHJiUgWmqjJTGJzpWRkdFrypuE/ZHBa4PtAi0loyWyWBcI+WWwO1Y9OinpdDry8/PJyckhGNSguFsvYjKZekULqYXanGRMlgtUZtDp0JttQCNhnySlWGmalH74wx+2Kl4GkZXtn376aVKvYzAYetUvjOhcYUWFYBMAZls7d96a6ZuTlhKQsc1Yad5SGjlyJP/85z+jX7dXPkOIVOEPhTGFI0nJZL1wDStjc1IKS/ctZppnAKPR2GvGIUTPUehQMRvM6NtZYtLCbEunb7qZC42Hi9Y0T0qHDh2ioKAAq9XKlClTWLZsGQMGDGjzWL/fj9/vj37d0NDQVWEKEWU3GxmVbQKf4/MNKNuRluZkSI4D8ixdFF33p+k8pcmTJ7NmzRpeffVVVq5cSXl5OVdeeSWNjW3Xn1m2bBkulyv6KCws7OKIhSCyxKRljMhsv/CxLUmreQxKXFxK1VOqq6tj4MCBLF++nPvuu++819tqKRUWFsZVq0WIjgoFfOjfXR5Z+X+xnWo+24l6cCPBrKEYR38Zvb53zZXr9vWUMjIyGDp0KKWlpW2+brFYorWmhdDKxxVVNJXX0teVxpCLbQphsrHz6BlqjlcwcXCArHT5+b2YlFpm4na7KSsrk100REoLNW8YgMl+8U1GzXYMeh0mxYc/JLXNY6FpUlq8eDGbNm2ioqKC9957jy996UsYDAbuvPNOLcMS4oJCfjcA+ouNJwGY7Bj1eoxhvySlGGnaffvss8+48847qampITs7myuuuIKtW7fKttAipYX8XgxcZDFuC5Pt85ZSMNTpsfUEmial9evXa3l5IRISanJj4POJkRdkinTfQCXg8wKuTo6u+0upMSUhugOluZKkMZaWkt6AzhTZ2SUk699iIklJiDi1rGO72BKTFi1jTyGfu9Ni6klSakqAEN1Bvk3B4DdhTbvwYtwWaekOFE8tBosUeouFJCUh4jQy2whWJ6THNj6Un9WHfGqhj/y6xUK6b0LEq2WJiSmGKQEAxubdgoO+zomnh5GkJEQclLDyeRmSiyzGjTLZUVWVgJQviYm0J4WIQ9WZBo4cPoXdbGCsObaB7lN+HaXltXgaKpk5pJMD7AGkpSREHALNS0xUg7n9rZXOYWy++6ZKpYCYSFISIg7+pjiWmDQzWT9PSilUlCNlSVISIg7B5qSki7HrBp8nJUPIR0iRpHQxkpSEiEPYF39LyWyxowMMiizKjYUkJSHi0LJ/mz7G2dwAuuZFuUbFj192yr0oSUpCxCHsj8xRMl5kw4BWTLbITrlqEH9A9h+8GJkSIEQcMk1BQnYT6elxlF82WHClmUkLKZjxA3EktF5IkpIQcSjJ0IPqhOzM2N+k1zMoLysyo9siA90XI903IeIRjHOJSQuj7GoSK0lKQsShpZYScUwJAMBkRUUlHJCkdDGSlISIUTissPXTSj6oqMWni29Xkk9rgmw9XMveiqpOiq7nkDElIWLk93nQoaIoYI6l6uTZmrtvoZYKA6Jd0lISIkY+b3PlSKMFvTG2dW8tDM2TLcN+6b5djCQlIWIUaF5ictGtuttgtERaSoqMKV1UyiSln/70p+h0OhYuXKh1KEK0KZDAurcWLUlJKgVcXEokpe3bt/OrX/2KMWPGaB2KEO0K+iJlS3SJtJSaE5kSlDGli9E8KbndbubOncuvf/1r+vTpo3U4QrQr5IskFEMCLSWTraV8iZTEvRjNk9KCBQu44YYbmDlz5kWP9fv9NDQ0tHoI0VXSdX762E04nPFvKGmxpZFhM9HXKlUCLkbzHXI//PBDtm/fHtPxy5Yt49FHH+3kqIRoW780FfKc0D837vc60hwMz3dCnHfteqOEWkqHDx/u8IUrKyv59re/zdq1a7FarTG9Z8mSJdTX10cflZWVHY5DiJi1jAclMKZE8y65hPygSGvpQhJKSiUlJcyYMYPf//73+HyJ9ZF37txJdXU1EyZMwGg0YjQa2bRpE8888wxGo5Fw+Py6MxaLBafT2eohRFdR4t3F5GzNkydVVNSQ3IG7kISS0ocffsiYMWNYtGgReXl5fOMb3+CDDz6I6xzXXHMNe/fuZdeuXdHHxIkTmTt3Lrt27cJgMCQSmhCd5qPDx/mgopZjngR+bfR6dh7zsu1wLafP1Cc/uB4koaQ0btw4VqxYwfHjx1m1ahUnTpzgiiuuYNSoUSxfvpxTp05d9BwOh4NRo0a1eqSlpZGVlcWoUaMSCUuIzqOqEPASVlSM8S4xaaYYLKhA0CfTAi6kQ3ffjEYjt956Ky+88AJPPPEEpaWlLF68mMLCQu6++25OnDiRrDiF0FawiXDzWJDFFv+UAABd87hS0C9J6UI6lJR27NjBgw8+SH5+PsuXL2fx4sWUlZXx+uuvc/z4cW655Za4zvf222/z9NNPdyQkITpF2O8hrKiE9WasFnNiJ2kei5KkdGEJTQlYvnw5q1ev5sCBA8yePZvnnnuO2bNno9dHclxxcTFr1qyhqKgombEKoZmW/d6CBhsWY2J/y1t2QAlJUrqghJLSypUruffee5k/fz75+fltHpOTk8NvfvObDgUnRKoIeCNLTDDZ0el0CZ1Db460lKRSwIUllJRef/11BgwYEG0ZtVBVlcrKSgYMGIDZbGbevHlJCVIIrSWyM+65DC1JSWoqXVBC7dDBgwdz+vTp856vra2luLi4w0EJkWrMShN97CacrsTXZ6alpZNhM+E0hpIYWc+TUEupvf3Q3W53zLOzhehO+phC9MlzQlFBwucYlN8X6p3gkjl4FxJXUlq0aBEAOp2OpUuXYrd/3pQNh8Ns27aNcePGJTVAIVJCy4YBpsSmAwCyo0mM4kpKH330ERBpKe3duxez+fNbo2azmbFjx7J48eLkRihEClADHnSQ2Lq3Fs3zlNRgE4kNlfcOcSWlt956C4B77rmHFStWyNoz0WscqDxJQ00Ntuwgo3MSO0e1T8eRilp05iYmXpbc+HqShMaUVq9enew4hEhpasBDWCX+/d7OYjDbCSkqxqAvsmwlwakFPV3MSenWW29lzZo1OJ1Obr311gseu2HDhg4HJkTKUMLRipFmW2Lr3gDMzXW6w2EFNeRDl0i1gV4g5qTkcrmik8Zcrvgr7wnRbQW9hMIKKjos1sRbShaLFUVnRK+GCPqaMEtSalPMSensLpt030SvEvASUlRCBitWc+LFWk0GHWGDBX0oRNDvwezITGKQPUdCkyebmprwej+flXrkyBGefvppXnvttaQFJkTKCHoIhRWCehs2U+JzjHQ6XXRaQMDnSVZ0PU5CSemWW27hueeeA6Curo5Jkybx1FNPccstt7By5cqkBiiE1oI+N2E1shjXau7gXhvR8iUyV6k9CVeevPLKKwF48cUXycvL48iRIzz33HM888wzSQ1QCK0pfk9kFxOHC7OhY0nJ6XCSYTNhDPuTFF3Pk1AH2ev14nA4AHjttde49dZb0ev1XHbZZRw5ciSpAQqhNYvi45I8J/Qf0OHb+OOK88B6EiyyeUB7Et444OWXX6ayspKNGzdy7bXXAlBdXS0TKkXP05FdTM7VsquJLDVpV0JJaenSpSxevJiioiImT57MlClTgEirafz48UkNUAitqdF1b0lISsaWrZZkp9z2JNR9+5d/+ReuuOIKTpw4wdixY6PPX3PNNXzpS19KWnBCpIKjVac4+VkNSpqPyxIvEgDAnpN+/BW1OMPVDLskOfH1NAlPusjLyyMvL6/Vc5MmTepwQEKkmrDfS1gFNQmTHRWDlZCiSvXJC0goKXk8Hn7605/yxhtvUF1djXLOjp/J2EFXiJSgqoT9kaqTZqujw6czWCJdQFXGlNqVUFL6+te/zqZNm7jrrrvIz89PuGbxypUrWblyJRUVFQCMHDmSpUuXMmvWrITOJ0TShYOEgwEALPbE1721MDWvf5Ok1L6EktI//vEPXnnlFS6//PIOXbx///789Kc/ZciQIaiqym9/+1tuueUWPvroI0aOHNmhcwuRFEEPIUVF0Rmx2TrefTNbz2opSaWANiWUlPr06UNmZsfX7dx0002tvn788cdZuXIlW7dulaQkUkPASzCsEDTYsXZgiUkLkyWyoFcJh0AJgcHU4XP2NAlNCfjRj37E0qVLW61/66hwOMz69evxeDzRKQZCaC7gIRRWCept2DuwGLeFxWJFRUdYUWWuUjsS+pSfeuopysrKyM3NpaioCJOpdbb/8MMPYz7X3r17mTJlCj6fj/T0dF566SVGjBjR5rF+vx+///Pp+Q0NDYmEL0TM1IAbh9VI0J6B3dzxlpLVYsSWlo4df/NcJZlsfK6EktKcOXOSFsCwYcPYtWsX9fX1vPjii8ybN49Nmza1mZiWLVvGo48+mrRrC3ExuoA7ssSkYCAkofvmtJoYX5wP3hppKbVDp7a3X5JGZs6cyeDBg/nVr3513mtttZQKCwupr6+X5S2icxz4BxzfBUVXQPGVyTnnh89B/TEYdStkD0vOOVNUQ0MDLpcrrt/RhDvJdXV1vPjii5SVlfHd736XzMxMPvzwQ3Jzc+nXr1+ip0VRlFaJ52wWiwWLxZLwuYWIl+p3Ayo6S8enA0TJVksXlFBS2rNnDzNnzsTlclFRUcH9999PZmYmGzZs4OjRo9FaSxezZMkSZs2axYABA2hsbGTdunW8/fbbbNy4MZGwhEi6k6dPc6Siliazl+kdXGLS4p0KN9aaWkryGuiTpHP2JAndfVu0aBHz58/n0KFDrXbEnT17Nps3b475PNXV1dx9990MGzaMa665hu3bt7Nx40a++MUvJhKWEEkX8rlRVAgZk7AYt5kfMyFFJehP3t3rniShltL27dvbHPPp168fVVVVMZ/nN7/5TSKXF6JrqCpK8xITkz15Y5Z6c+QPeSggSaktCbWULBZLm7fjDx48SHZ2doeDEiIlBL2EgiEALLaOr3troW+uyxSWllKbEkpKN998M4899hjBYBCIFEQ/evQoDz/8MF/+8peTGqAQmgl4CIQVQnoradbk3WAxmJv3fwvIQHdbEkpKTz31FG63m+zsbJqampg2bRolJSU4HA4ef/zxZMcohDYCboJhlYDBjt3S8TlKLQzNLSVFklKbEhpTcrlcvP7662zZsoXdu3fjdruZMGECM2fOTHZ8QmjH746sezPaSbd0fIlJC6PFThhJSu2J+5NWFIU1a9awYcMGKioq0Ol0FBcXk5eXh6qqCZcxESLlBDykW4x47C7SkpiU0tMchC1GbLpA0s7Zk8T1Sauqys0338zf//53xo4dy+jRo1FVlf379zN//nw2bNjAyy+/3EmhCtHFAh6G5joYOqAIkpiURgzMgUpX5AslDPrkdQ17grg+6TVr1rB582beeOMNZsyY0eq1N998kzlz5vDcc89x9913JzVIITQRaIz8a07ibG4AgyVSR0ltrhSQzNniPUBcA91/+MMf+I//+I/zEhLA1VdfzSOPPMLatWuTFpwQWlL8blRUMKcl98R6PRib7+bJribniSsp7dmzh+uvv77d12fNmsXu3bs7HJQQqaD61Gk+KK/lrYrkDkhXN/p4/6iXPcfqZP1bG+LqvtXW1pKbm9vu67m5uZw5c6bDQQmhOVUl7KtHUSFsSm73yqDT4Q4bMYUVaSm1Ia6WUjgcxmhsP48ZDAZCoVCHgxJCc0EvwWAI0GFJcyX11GajnpDeQiisogZlVve54r77Nn/+/HbLh7RXckSIbsff2FybO7mzuQEsRgMhvRUVCPmbkCrdrcWVlObNm3fRY+TOm+gR/I0EQgp+g4ucJM7mBjAZdCiGSKIL+r2SlM4RV1JavXp1Z8UhRGrxNUSWmBjTSEvChgFn0+l06JrXvwV90n07V0Jr34To6VR/A4GwQsCYltQlJi1aKgWE/J6kn7u7k6QkRBuUpgYybCbSnZmkW5OflBwOJ+kWI3oZ6D5P8j9tIXoAQ9AdWWIyogQMyf/bPW3kQAi5wCR3q88lLSUh2uJvXmJi6aRdckzN5XWlpXQeSUpCnEtVCfsaIktMLMmrONmKuSUp+SJr4ESUdN+EOFfQy/GaRo7V+1D6+Zk6NPmX+Lg6SODoGfqkmSkONn2epIS0lIQ4j78Rf1ghoLdiNHXOLKIweryKkUBIkfVv55CkJMS5fA3NEyfTO2U6AIDNbCCotxIKKxCUaQFn0zQpLVu2jEsvvRSHw0FOTg5z5szhwIEDWoYkBPjqCITC+I1OHJ0wHQDAajQQNNgIKaq0lM6haVLatGkTCxYsYOvWrbz++usEg0GuvfZaPB75yyG0ozadibSUjOmdl5TMekItLaWA/LyfTdOB7ldffbXV12vWrCEnJ4edO3dy1VVXaRSV6O0CnjOEVfCbnJ3WfbOamrtvSqRSgFS2/1xK3X2rr68HIDMzs83X/X5/q0oEbW2IKURH+RpqADDYMjB2wsRJAJvJQMhgjWwJ7nPLotyzpMxAt6IoLFy4kMsvv5xRo0a1ecyyZctwuVzRR2FhYRdHKXo8VcUQaCArzUxuTl6nXcZk0GNLj7TEFNm+u5WUSUoLFizg448/Zv369e0es2TJEurr66OPysrKLoxQ9AoBD+lGGJrnZPrYkk691Ozxgxjdz4VFkTpkZ0uJ7ttDDz3E3/72NzZv3kz//v3bPc5isbRbYE6IpPDVRf61ODp/6yNZatImTVtKqqry0EMP8dJLL/Hmm29SXFysZThCQFMdwbCCak1uCdw2SVJqk6YtpQULFrBu3Tr+/Oc/43A4qKqqAiLbgttsNi1DE72Vr569x+o5UdPExME+cp3WTrvUe5VNWI+eoV9mkFxVjewFJ7RtKa1cuZL6+nqmT59Ofn5+9PHHP/5Ry7BELxZunqPkMaQndavutjSpZvwhhWAwCCEZV2qhaUtJldXRIsX4G2pQgbDJSZq5c8eUrFYrYZ2JoKJGunCmzmuVdScpc/dNiFTQVH8KAJMrG10nd6esJj0hg4VwWJFxpbNIUhKiRdCH3xuZkJvmyun0y0VmddsiLSWZqxQlSUmIFk21+AJhggYbLkdyd8Vti91sJGiwEwwrEGjs9Ot1F5KUhGjhraEpGKbJmEGfNHOnX85uNhAw2AmGVFmUe5aUmDwpRErw1pJhN+O25ZHt6PxJumkWI2abA2tIj+pvlEW5zSQpCdHCW0O/DBv9SoZAeucnpXSLkdkTh8CBMhlTOot034Ro4Y1UB8Ce1XXXNDePXQXcXXfNFCctJSEAFAW/uwbCYcy2zK7rSklSOo8kJSEA/PVU1br5rCFI8Kif6Zd0zWXfKvfiOnqGgX2DZCkK6KXzIp+AEACeGryBMD6ji6z0rlt36cWKP6QSCIZkAmUzSUpCAKq7Ck8ghNec2SV33lrYLSaCeivBsEwLaCFJSQjAV3eSYFjFa84iK73z5yi1sJsNZ02glHElkKQkBACe2uMAmJ25mDqpLndb7GYjAYNNktJZJCkJEQ7iqz8NQFpmfpde2m4xEDCkSfftLJKUhPCcpikQJKi3kdmn7Z10Okur7ptf1r+BTAkQAjzV9E23EHblk5GZ1qWXTrMYMadnYA0awCdbhoEkJSHAXU0fu5k+hYPA1bWF1pxWEzdPugT27AG/JCWQ7psQ4D4Z+Tet82sotallkwJJSoC0lERvpyjUVx+FcAibPY+umwxwFosj8m/QF6nVbezd24hJUhK9m+cUx2saqfGDo97E2C7YWelcmw43YPvMS5FLT46/sdcnJem+iV5NbTiO2x/Cbc4hP0Obbb1CYYVG1Y4/pICvXpMYUommSWnz5s3cdNNNFBQUoNPpePnll7UMR/RCjaeOEFJUfNYc+nZBDaW2pFuM+I3pBEKKjCuhcVLyeDyMHTuWZ599VsswRC/mPlUJgCWrP3q9NrUf061GAoa0SFKSaQHajinNmjWLWbNmaRmC6M1CfjxnIrsy98kdqFkYDosp0lIKSEsJutlAt9/vx+//fCfRhgb5HygSpzYcp8EbxG9IZ3CuRtMBiLSU/Ib05jEl+ZnuVgPdy5Ytw+VyRR+FhYVahyS6MXfVYQJhBbetgLwunjR5tpYxpbCiEvLWahZHquhWSWnJkiXU19dHH5WVlVqHJLqxtKZjjOnnYtTI0V1aGeBcZqOe9IxsnFYj4aYGCIc0iyUVdKvum8ViwWLp3XM4RJKEg+jdVaRZjKQNGqZ1NNx5+SWg5kQmT/rqIK2v1iFpplu1lIRImvrPQAlHZlPb+mgdDeh0n8fRy7twmraU3G43paWl0a/Ly8vZtWsXmZmZDBgwQMPIRE935ngpNdWNpA0YTL4uRbaBtPVBbTiBrumM1pFoStOktGPHDmbMmBH9etGiRQDMmzePNWvWaBSV6A1qPjvIKXeAE4FMurasW9vKTrn5tNTHYG8jl/STpKSZ6dOno6qqliGI3ijgwXs6cpMke+AIjYOJMBv0nFHS8AXD0MtbSjKmJHodz4mDePwhvOYsBhRoNz/pbC67iSaTC39IQfGc1jocTUlSEr3O6aP7AND3LcFuTo0b0A6LkaAlE0WFgKc+Usakl5KkJHoXRaHh2EEAMgcM1ziYz+l0OtLT0wkY0vCFwuA5pXVImpGkJHoVT/VhPB4PIb2VouIhWofTistmwmvKxBdUJCkJ0VsET3yM3WJAzR6Gy55aE3H7plvwmjPx+kO9OimlRodaiK6ghMlwHyajXwaB0VdqHc15cp1W3Jn5WBsP9eqkJC0l0XvUHo4s47CkY85Mvcm5JTnpXDdpNAUuG7iroZdOl5GWkuhywVCIo0crqDpeSYkjTK4dCPtpCgQ5UNWIw2rEaTXhcLqwpmeCswDSc0Hfsb+hNYe24VIUjDnDO3yuTpPWFwzGSPL01kJaltYRdTlJSqLLqL4Gjux6m5OHdqIEPOiBYB8b9LEDEGgK0nSigSaguvk9ZoOONIuRdFcmrsGTcAy+LKHC+t7605R+uhudqjJs3Cg02B8gNnoDQXsu1FViajgmSUmITqGqBCrep3z7q9Q2egEwmi24cgaQlpMHTicYLegDCul9PNQ3haj3+gk0NWAJNuD0neCM9yR6/xs4avbC0Ougb3x3zsp2v4OiqKh9inFm5nbGd5kU75Wdpqpcx0S9jwENxyF/jNYhdTlJSj3Y/hMNlFa7Gd3PxcAsOzotFp6G/AT2bODgvt00+kN4rLnkjJrBiBFjMZpMrQ51AKPPyjWBkEKNx8+xmkbqj+4hy3ggUi5274uczhxHXd5USnIv3uZxuxtwH/4AgMJRl2vzOcTIaTVxyJxDo/sTaDimdTiakKTU3SkK1BwiXH2Qg0c/oyjbiS2zH+SOos6ro7TaTWm1m5KcdL44IherydB1sQU8sOePVB8tpz4Ax3OmcfkV11DQ3F27GLNRT77LRr7LBoNmQngalG9GrdzGib2bOP7xfg4Mn8O0kQNIt7T9o6yqKh+/vxHCQYyuPPqXpHbLI89lpdGcg8cfQnFXow82gUmbrZ+0IkmpO6s7CgdeJeQ+xYGqRhp8IY7Umbgk7ygc3crojEsIFoxnV1WA0mo3Z7wBvjyhP2nt/AInVdAHu9eDu5qCnL5UFM9geskwsh0dmBtkMEHJNSiOApz1L+CpOU7T3t/z/OlZTBwxhFEFrvN2JPnkUCnK0Q/QA4MvnY0uVQe4m2WlmdHbHDQa+9DkD5F2pgJyUmfmeVdI7f9Dom2qCuXvwK51hD2n+bg6wKemEVTkzsQ17ubIeIuqkn5mP1c1/I2vXGLEYTVS4w7w4s7P8Pg7udxqOAQf/wnVfRLMdvQT7uKK8WM6lpDOYsgdzsCZ/8rIwYX0NXgZemwDOz/cwR93VFJa7SasRG6ln6lvpHrbC+hQyR40JqWWlbRHp9NR4LJRZy2kvikYmcbQy0hS6m6UMHz6N6h4F1VR2OHvz+bM2ziZM5Vrpk0nf/gUGP0vMOHuSCVDXwM5pS9ye7Efh9VIrSfAn3cdj+wx1inxKbD/L1QdPUhZTYDwqNvBnpn866Tn4Jj6dUaOGMngPiZG1ryG+cgm/v5ROY2+IPjq6VP2MgXGRrKzMhk0ZU6kumM3UNQ3jXpbf854A5Gk1MvmK0n3rTsJB+GTl6CmDFWnY6tpMlvD/THqddwyrh85jrN25HD1gy/Mh/1/gZoynKV/4fbi61hX4cBs1KN0xg+6qkLpP6k7spfDNU18mn09oaY0Lums++/mNPTjvkq+fSNZn31Ebv0Biqv34dpbBE11oCqUFPTFMO4r6KzOTgoi+Yqz0njbkkddQE+oqQFjfSVkpN5kz84iLaXuIhyEvS9CTRkYjOzN/CJbm/qj08Gs0fn0y2hjMNRkhVFfhtwRoCo4y1/lzv61fGl8v84Z8D7yHt7ybRys9lCaOZ1+g0YwLNeR/OucTW+AYbMwj72NgYUDGd/fgc5bC6oCGYUYJ85H5+rXuTEkmctuYni/TAqHjUev00HVx1qH1KWkpdQdhPyw9wWoq4wM9o65nf6mPFwNx5hY1IeSnPT236s3wPCbwWiFYx/iOvpPMCkwYDKqqnKi3kdBWwktXsd3ESx9m0+rGilzXYat/2hmDs/tmtvvOh1kD4W+QyL1rf0NYHWlxoYACbpuZB6cmQS7SuHUfiiZCUaz1mF1CUlKqS7og73PQ/2xyA/lmDvA1Z9M4GuXDcRsjKGxq9PBkGvBYIajW6HsTdSQj3eCw9l5tI6rL8lhbGFG4jFW7ye4/+/sr2rgsH0MvtzxfGVMAQZ9F4/h6HSR8avOGMPSQsaAyPfirYXjH8GAyVpH1CWk+5bKms7AR7+LJCSTlaOFcygLZERfjikhtdDpYPAMGDQt8uWR9+hbtRlUhbcOVPPJ8frEYqzai/rJyxysaqDcWEJN9mXcMq4fNnMXzofqoYKKyj7TSCpqPFC5NdJi7gUkKaWqmjL48DnwnAaLgwN5N/NymcIre05wor4p8fMOnApDvgjAcOUg14ffxhjy8tonJ9l2uCb2jRxUFco3w/6/oQOyh17KqYIZ3PqFQjLTekc3o7PVegK8djqLCq8Zd2MDlL2ldUhdIiWS0rPPPktRURFWq5XJkyfzwQcfaB2SdgJeOLgR9jwPAS9Bew6bHLP4e3mYsKIyODu99V22RPSfCCNuQWcwc4m1ltm+V8hxf8p7paf5y+7juC82j8lTg7prLU2HNke+LryUnIm3Mv/yQUmbiyQi9ZWG5mdQ1udKSqsbCX22E07s1jqsTqf5mNIf//hHFi1axC9/+UsmT57M008/zXXXXceBAwfIyUmNnSa6hK8BTuyCz3ZAyE9YVTlivYS3fCNpaAgDMLk4kymDs5IzeJw7AtJz0e17iUGcwt6wk0+rdnPCPYK/1A/hzqtGt76OokB9JYHPPqKuYg9V9R48IR2jpt2Go/gLABgN3WMeUHdy9SU5/L6uiTLfaPTHP2aI8jds/kYYMCVyE6MH0qkab7w2efJkLr30Un7xi18AoCgKhYWF/Nu//RuPPPLIBd/b0NCAy+Wivr4ep7ObzENRwhD0RtaFeU6D5xSh2nJC9SewGHTo0EF6Nn/xDKcsmA1EajdfMzyHgVlpnRPPsZ1Q8S5er4fDpzxkpZvJz8kGixNPUGVf5WkswTrCoQD+oIIK1FkLOZZ9JVeOGcKwvE6+7d/LnXb7+dOOSnJPbibfs4++6RYGFORhKhgNGQPBnhXZfjwFJ4cm8juqaUspEAiwc+dOlixZEn1Or9czc+ZM3n///fOO9/v9+P2fD/Y1NDTEdqHacij95/nPt8rHavS50lNuznj8qOcep0ZmQU8YmImx+f//4dONnGoMRN6vnnUuNfLv+AEZmA16QKXiVCOn6xtbnxKiyyImjBmDZeCl0HcYWYdrOV3VyISBfRhZ4MRk6KSett4AhZMgfxz2k3sZefIT1Prj4HeD343fGyDU0EgICOmt1KYNJJA7lsKBg5neP0MGtLtA33QLd142kNc+uZaDlTmEGrZTHGiEI+/Dkff5tKoBT0BB1ZtQDGZUnR7QodPpMBoMTBjYfDdSp2PfiQYafW13z/U6HZcWxTGNYvTt0AmTUjVNSqdPnyYcDpOb27q+TW5uLp9++ul5xy9btoxHH300/guF/JFWSYxUbyMhT6D9A3yGaOVC1edG8V3grkjQAkrzsaEAwbAK6AjqrfhMLrymPrjNOTTaCxk6fDiW9MiYzKTiTKYmq6sWC6MZ+n0BXb8voAv5wVsDvgasgRAFQ8KErX0wpWUyId3aNQt6RStOq4l/+UJ/jg3OorbhMvTmKqgpBfdJ/OEGAqEwEAZa7xenN+jA8/mSIrWxHqWdpKTTAe441kWqnbNUqVv9dC1ZsoRFixZFv25oaKCwsPDib8wohLFfOad52/zfbTyX5Q1iD4RbJ4TmOTc6nR69wxJ5n05HdlOQ9JAS6Xadfbrm/zCkWaPvzQ6qOFQTOqMVdJG/ZHodWE0GLEZ9q+t1WssoFkZLpAStswA7MFC7SMQ5+mXYmmfv94W8UQAUjfTh93kIB30QDkS65Kioqhr50XNYogkkd7CfjFDbIzY6HeCK4yaKuROGE9A4KfXt2xeDwcDJkydbPX/y5Eny8vLOO95isWCxJHB3x5wGmcUxH56ZEfupnQ6ItQGb3vwQIplc6VZIjy2Z9O0G80o1nRJgNpv5whe+wBtvvBF9TlEU3njjDaZMmaJhZEIIrWjefVu0aBHz5s1j4sSJTJo0iaeffhqPx8M999yjdWhCCA1onpTuuOMOTp06xdKlS6mqqmLcuHG8+uqr5w1+CyF6B83nKXVEt5ynJEQvksjvaEosMxFCiBaSlIQQKUWSkhAipWg+0N0RLcNhMS83EUJ0qZbfzXiGrrt1UmpsjKwji2lWtxBCM42Njbhcse0g0a3vvimKwvHjx3E4HCm9FXNnallqU1lZKXcgEyCfX8dc7PNTVZXGxkYKCgrQx7gRaLduKen1evr37691GCnB6XTKL1UHyOfXMRf6/GJtIbWQgW4hREqRpCSESCmSlLo5i8XCD37wg8SqJwj5/DqoMz6/bj3QLYToeaSlJIRIKZKUhBApRZKSECKlSFISQqQUSUrdUG1tLXPnzsXpdJKRkcF9992H2+2+4HumT5+OrnmzgpbHv/7rv3ZRxNqKdwfmF154gUsuuQSr1cro0aP5+9//3kWRpqZ4Pr81a9ac93Nmtca5o7Mqup3rr79eHTt2rLp161b1nXfeUUtKStQ777zzgu+ZNm2aev/996snTpyIPurr67soYu2sX79eNZvN6qpVq9RPPvlEvf/++9WMjAz15MmTbR6/ZcsW1WAwqD/72c/Uffv2qd///vdVk8mk7t27t4sjTw3xfn6rV69WnU5nq5+zqqqquK4pSamb2bdvnwqo27dvjz73j3/8Q9XpdOqxY8fafd+0adPUb3/7210QYWqZNGmSumDBgujX4XBYLSgoUJctW9bm8bfffrt6ww03tHpu8uTJ6je+8Y1OjTNVxfv5rV69WnW5XB26pnTfupn333+fjIwMJk6cGH1u5syZ6PV6tm3bdsH3rl27lr59+zJq1CiWLFmC1+vt7HA11bID88yZM6PPXWgHZoh8vmcfD3Dddde1e3xPlsjnB+B2uxk4cCCFhYXccsstfPLJJ3Fdt1svyO2NqqqqyMnJafWc0WgkMzOTqqqqdt/31a9+lYEDB1JQUMCePXt4+OGHOXDgABs2bOjskDUT7w7MEPl82zr+Qp9tT5XI5zds2DBWrVrFmDFjqK+v58knn2Tq1Kl88sknMS+el6SUIh555BGeeOKJCx6zf//+hM//wAMPRP979OjR5Ofnc80111BWVsbgwYMTPq8QZ5syZUqrPRunTp3K8OHD+dWvfsWPfvSjmM4hSSlF/Pu//zvz58+/4DGDBg0iLy+P6urqVs+HQiFqa2vb3FW4PZMnTwagtLS0xyaleHdgBsjLy4vr+J4skc/vXCaTifHjx1NaWhrzdWVMKUVkZ2dzySWXXPBhNpuZMmUKdXV17Ny5M/reN998E0VRookmFrt27QIgPz8/2d9KykhkB+YpU6a0Oh7g9ddf75U7NidjB+twOMzevXvj+znr0DC50MT111+vjh8/Xt22bZv67rvvqkOGDGk1JeCzzz5Thw0bpm7btk1VVVUtLS1VH3vsMXXHjh1qeXm5+uc//1kdNGiQetVVV2n1LXSZ9evXqxaLRV2zZo26b98+9YEHHlAzMjKit6nvuusu9ZFHHokev2XLFtVoNKpPPvmkun//fvUHP/hBr58SEM/n9+ijj6obN25Uy8rK1J07d6pf+cpXVKvVqn7yyScxX1OSUjdUU1Oj3nnnnWp6errqdDrVe+65R21sbIy+Xl5ergLqW2+9paqqqh49elS96qqr1MzMTNVisaglJSXqd7/73V4xT0lVVfXnP/+5OmDAANVsNquTJk1St27dGn1t2rRp6rx581od//zzz6tDhw5VzWazOnLkSPWVV17p4ohTSzyf38KFC6PH5ubmqrNnz1Y//PDDuK4npUuEEClFxpSEEClFkpIQIqVIUhJCpBRJSkKIlCJJSQiRUiQpCSFSiiQlIURKkaQkupU1a9aQkZER/fqHP/wh48aNi349f/585syZ0+VxieSRpCTaNH/+/HZL5i5YsACdTtdqAXFnJIOioiKefvrpVs/dcccdHDx4sN33rFixgjVr1kS/nj59OgsXLkxqXKJzSVIS7SosLGT9+vU0NTVFn/P5fKxbt44BAwZoEpPNZjuvntTZXC5Xq5aU6H4kKYl2TZgwgcLCwlaF4DZs2MCAAQMYP358h87dVgtmzpw50dbX9OnTOXLkCN/5zneiBejh/O7buc5usc2fP59NmzaxYsWK6DnKy8spKSnhySefbPW+Xbt2odPp4iqxITqHJCVxQffeey+rV6+Ofr1q1SruueeeTr/uhg0b6N+/P4899hgnTpzgxIkTcZ9jxYoVTJkyhfvvvz96jgEDBpz3PQGsXr2aq666ipKSkmR9CyJBkpTEBX3ta1/j3Xff5ciRIxw5coQtW7bwta99rdOvm5mZicFgwOFwkJeXl1CRNZfLhdlsxm63R89hMBiYP38+Bw4ciG4VFAwGWbduHffee2+yvw2RAKk8KS4oOzubG264gTVr1qCqKjfccAN9+/bVOqwOKSgo4IYbbmDVqlVMmjSJv/71r/j9fm677TatQxNIS0nE4N5772XNmjX89re/TVprQq/Xc27VnGAwmJRzx+LrX/96dBB/9erV3HHHHdjt9i67vmifJCVxUddffz2BQIBgMMh1112XlHNmZ2e3GicKh8N8/PHHrY4xm82Ew+EOXae9c8yePZu0tDRWrlzJq6++Kl23FCLdN3FRBoMhupOKwWBo97j6+vpo7e8WWVlZFBYWnnfs1VdfzaJFi3jllVcYPHgwy5cvp66urtUxRUVFbN68ma985StYLJaEuo1FRUVs27aNiooK0tPTyczMRK/XR8eWlixZwpAhQ3plDe5UJS0lEROn04nT6bzgMW+//Tbjx49v9Xj00UfbPPbee+9l3rx53H333UybNo1BgwYxY8aMVsc89thjVFRUMHjwYLKzsxOKe/HixRgMBkaMGEF2djZHjx6NvnbfffcRCAS65G6iiJ2UwxW91jvvvMM111xDZWXleRsuCu1IUhK9jt/v59SpU8ybN4+8vDzWrl2rdUjiLNJ9E73OH/7wBwYOHEhdXR0/+9nPtA5HnENaSkKIlCItJSFESpGkJIRIKZKUhBApRZKSECKlSFISQqQUSUpCiJQiSUkIkVIkKQkhUookJSFESvn/JWQIbT7EEWAAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:11:42.801100Z", + "iopub.status.busy": "2024-02-29T18:11:42.800581Z", + "iopub.status.idle": "2024-02-29T18:11:43.019886Z", + "shell.execute_reply": "2024-02-29T18:11:43.019064Z" + }, + "papermill": { + "duration": 0.242308, + "end_time": "2024-02-29T18:11:43.021663", + "exception": false, + "start_time": "2024-02-29T18:11:42.779355", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyTklEQVR4nO3deVQUV9o/8G83SwPKIrIpoogYIRMFxSUwGjEsYieOGmM0EhfCq1HHTBSX6JyJjuPM8KqI2xg5ZqKOW3QS0HGMIh1ZJMgriOKgIkYFNUC7obRAaJru+/vDHzU0a7c2NF31fM7hpKv6Vvdzra4nt6pu3StijDEQQgjPiY0dACGEdAZKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQTA3dgBdnUajQVlZGWxtbSESiYwdDiGkEcYYnj9/jt69e0MsbrvtRsmuHWVlZfDw8DB2GISQNty/fx99+vRpswwlu3bY2toCePGPaWdnZ+RoOo5KpUJKSgrCw8NhYWFh7HDIKxLK/lQoFPDw8OCO07ZQsmtHw6mrnZ0d75OdjY0N7OzseH1wCIXQ9qcul5joBgUhRBAo2RFCBIGSHSFEECjZEUIEgZIdIUQQKNkRQgSBup4QwgM1NTW4ceMGt1z1ixLnC26jh9NFdLeWaJX18fGBjY1NZ4dodJTsCOGBGzduICAgoNn6jS2UzcvLw7Bhwzo+qC6Gkh0hPODj44O8vDxuuaj8GWK+LUD8tMEY1MuhWVkhomRHCA/Y2NhotdbEd59AkvkLfN/wg3+/nkaMrOugGxSEEEGgZEcIEQRKdoQQQaBrdgKla1cFoXZTMAXFj6tRraxv8b3bj6q5/5qbt36Yd5OYo79Ttw6Jr6sRMcaYsYPoyhQKBezt7VFZWWnSQzw1PTCuF+Rj+oTgdrc7ejodrw/255aFdHB0ZcWPqzEuLt0gn5W2PNhk96k+xye17ATgxoMnkO5K1FqnqVfC/bd/aHfblT/+B+L/K9Jad2rhVPi40h0+Y2r4H9fW6f7wdune/P1flDiZno13gwPRrUmn4ga3HlZhydH8VluHfEPJTgB+qriNbv13GPDz/CnZdRHeLt3xhrt9s/UqlQpyZ2BYvx6CGLxTF5TsBKB3t36oLv4U26b7Y0ALrQAAqK+vR9aPWfj16F+3eo3n9sMqfHY0H73H9evIcIkOlOpaiK1KUawogtiq+T6tr69HWX0ZCisKW92fxYoqiK1KoVTXAmieMPmGkp0ASMysoKl1R3+7QXi9Z8s/apVKhWLzYvg6+rbaEtDUVkJT+wgSM6uODJfooKz6Lrr134Hf57Rd7svkL9t8v1t/oKzaHwFwNWB0XZPJJbudO3di06ZNkMvl8PPzw44dOzBy5MgWy167dg1r1qxBXl4e7t69iy1btmDJkiWdGzAhHaC91jq11JszqWR39OhRxMTEICEhAaNGjcLWrVsxfvx4FBUVwcXFpVn5mpoaeHl5Ydq0aVi6dKkRIiakY7TXWqeWenMm1ak4Pj4e8+bNQ1RUFF5//XUkJCTAxsYGe/bsabH8iBEjsGnTJsyYMQMSSct3pAghwmAyya6urg55eXkIDQ3l1onFYoSGhiI7O9uIkRFCTIHJnMY+fvwYarUarq7aF1JdXV21ngR4VUqlEkqlkltWKBQAXpwWqFQqg31PZ6qvr+f+21odGta3VUddPod0jvb2hVD2pz5xm0yy6yyxsbFYt25ds/UpKSkm+9jU/SoAMMePP/6Iuy33POHIZDKDfA7pWLruC77vz5qaGp3Lmkyyc3JygpmZGR48eKC1/sGDB3BzczPY96xevRoxMTHcskKhgIeHB8LDw032cbFrZQrEFfwfRo8ejV/1brkOKpUKMpkMYWFhrV7Q1uVzSOdob18IZX82nHnpwmSSnaWlJQICAnD27FlMnjwZAKDRaHD27FksXrzYYN8jkUhavJlhYWFhsj3RG7oemJubt1uHtuqpz+eQjqXrvuD7/tQnbpNJdgAQExODOXPmYPjw4Rg5ciS2bt2K6upqREVFAQBmz54Nd3d3xMbGAnhxU+P69evc69LSUuTn56N79+7w9vY2Wj0IIZ3PpJLd9OnT8ejRI6xZswZyuRz+/v5ITk7mblrcu3cPYvF/bzCXlZVh6NCh3HJcXBzi4uIwduxYpKend3b4hBAjMqlkBwCLFy9u9bS1aQLz9PQEjWBFCAFMqJ8dIYS8Ckp2hBBBoGRHCBEESnaEEEGgZEcIEQSTuxtLCAF+UakBAFdLK1t8v/oXJS4+AtzuPm1zDgohoWRHiAm6/f8T1aqkgjZKmePArdx2P6ubRBhpQBi1JIRnwn/14nnwAS7dYW1h1uz9ovJKLPuuAJvfH4xBvVqfX0JIU2NSsiPEBDl2s8SMkX1bfb9h+KYBzt1anH1MiCjZCUB713cAusZD+I+SnQDodn0HoGs8hM/oVysA7V3fAegaD+E/SnYC0N71HYCu8RD+o07FhBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaC7sQJVU1OjNbl4UfkzKOW3UHjVGponDtx6Hx8fk50vl5DGKNkJ1I0bNxAQENBs/cx/aC/n5eVh2LBhnRQVIR2Hkp1A+fj4IC8vj1uu+kWJ79Oy8c64QHRv9LiYj4+PMcIjxOBM7prdzp074enpCSsrK4waNQo5OTltlv/222/h4+MDKysrDB48GKdOneqkSLs2GxsbDBs2jPsLHDkcQYMHIHDkcK31dApL+MKkkt3Ro0cRExODtWvX4tKlS/Dz88P48ePx8OHDFsufP38eH374IaKjo3H58mVMnjwZkydPxtWrVzs5ckKIsYmYCU2sOmrUKIwYMQJ/+9vfAAAajQYeHh749NNPsWrVqmblp0+fjurqapw8eZJb9+abb8Lf3x8JCQk6fadCoYC9vT0qKythZ2dnmIp0QSqVCqdOnYJUKoWFhYWxwyF6aumGU8y3BYifNhiDejloleXTTSd9jk+TuWZXV1eHvLw8rF69mlsnFosRGhqK7OzsFrfJzs5GTEyM1rrx48fj+PHjrX6PUqmEUqnklhUKBYAXyUClUr1CDbq2hrrxuY58dvXqVYwaNarZ+qY3nADgwoULGDp0aCdE1fH0+b2aTLJ7/Pgx1Go1XF1dtda7urpq/R+tMblc3mJ5uVze6vfExsZi3bp1zdanpKTw5v+GbZHJZMYOgbwEpVKJzZs3c8sqDVBRCzhaARZNLlaVlJSgvLy8kyPsGDU1NTqXNZlk11lWr16t1RpUKBTw8PBAeHg4709jZTIZwsLC6DSWB4SyPxvOvHRhMsnOyckJZmZmePDggdb6Bw8ewM3NrcVt3Nzc9CoPABKJBBJJ85F6LSwseP2jaSCUegoF3/enPnUzmbuxlpaWCAgIwNmzZ7l1Go0GZ8+eRWBgYIvbBAYGapUHXpymtVaeEMJfJtOyA4CYmBjMmTMHw4cPx8iRI7F161ZUV1cjKioKADB79my4u7sjNjYWAPDZZ59h7Nix2Lx5M9555x0cOXIEFy9exO7du41ZDUKIEZhUsps+fToePXqENWvWQC6Xw9/fH8nJydxNiHv37kEs/m9jNSgoCIcPH8Yf/vAH/P73v8fAgQNx/PhxvPHGG8aqAiHESEyqn50xUD87YoqEsj/1OT5N5podIYS8Ckp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBL2T3Z07dzoiDkII6VB6Jztvb2+MGzcOBw8eRG1tbUfERAghBqd3srt06RKGDBmCmJgYuLm54ZNPPkFOTk5HxKaloqICkZGRsLOzg4ODA6Kjo1FVVdXmNrt370ZwcDDs7OwgEonw7NmzDo/TFKnVamRkZODcuXPIyMiAWq02dkiEGJzeyc7f3x/btm1DWVkZ9uzZg/LycowePRpvvPEG4uPj8ejRo46IE5GRkbh27RpkMhlOnjyJc+fOYf78+W1uU1NTg4iICPz+97/vkJj4ICkpCQMGDEBYWBji4+MRFhaGAQMGICkpydihEWJY7BXV1tay+Ph4JpFImEgkYhKJhM2aNYuVlZW96kdzrl+/zgCw3Nxcbt3p06eZSCRipaWl7W6flpbGALCnT5/q/d2VlZUMAKusrNR7264uMTGRAWDW1tYMAPfXsJyYmGjsEMlLqqurY8ePH2d1dXXGDqVD6XN8mr9skrx48SL27NmDI0eOoFu3bli+fDmio6Px888/Y926dZg0aZLBTm+zs7Ph4OCA4cOHc+tCQ0MhFotx4cIFTJkyxSDfAwBKpRJKpZJbVigUAF5MOqxSqQz2PcamVquxYMECAMAvv/yi9V7D8sKFCyGVSmFmZtbp8ZFX0/Bb5dNvtiX61E/vZBcfH4+9e/eiqKgIUqkU+/fvh1QqhVj84oy4f//+2LdvHzw9PfX96FbJ5XK4uLhorTM3N4ejoyPkcrnBvgcAYmNjsW7dumbrU1JSYGNjY9DvMqYrV660e8nh4cOH2LhxI/z8/DopKmJoMpnM2CF0qJqaGp3L6p3sdu3ahY8//hhz585Fr169Wizj4uKCr7/+ut3PWrVqFTZs2NBmmcLCQn1DfCWrV69GTEwMt6xQKODh4YHw8HDY2dl1aiwd6ccff9Sp3PPnzyGVSjs4GmJoKpUKMpkMYWFhsLCwMHY4HabhzEsXeic7mUyGvn37ci25Bowx3L9/H3379oWlpSXmzJnT7mctW7YMc+fObbOMl5cX3Nzc8PDhQ6319fX1qKiogJubm75VaJNEIoFEImm23sLCglc/mrS0NO61hYUFRo8eDY1GA7FYjB9//JE7PUhLS+NVvYWGb7/bpvSpm97JbsCAASgvL292WllRUYH+/fvr1W3B2dkZzs7O7ZYLDAzEs2fPkJeXh4CAAABAamoqNBoNRo0apV8FCABodcPRaDRaya/xNTrqrkP4Qu+uJ4yxFtdXVVXBysrqlQNqia+vLyIiIjBv3jzk5OQgKysLixcvxowZM9C7d28AQGlpKXx8fLRuisjlcuTn5+PWrVsAgIKCAuTn56OioqJD4jQljTuEm5mZYcWKFdi1axdWrFihleyo4zjhC51bdg3XsUQiEdasWaN1sV6tVuPChQvw9/c3eIANDh06hMWLFyMkJARisRhTp07F9u3bufdVKhWKioq0LlgmJCRo3Wx46623AAB79+5t9/SZ75ydnVFaWsotb9q0iXttaWmpVY6Ylrq6OuzYsQOpqam4desWPv30U619KlQi1lpTrYlx48YBADIyMhAYGKj1j2dpaQlPT08sX74cAwcO7JhIjUShUMDe3h6VlZW8ukERFBSE7OzsdssFBgbi/PnznRARMYSVK1ciPj5e63KSmZkZYmJisHHjRiNG1jH0OT51btk1XNOJiorCtm3beHXgC5Gfn59OyY66nZiOlStXarXQG6jVam49HxOervS+Zrd3715KdDzg7e1t0HLEuOrq6hAXF9dmmbi4ONTV1XVSRF2PTi279957D/v27YOdnR3ee++9NsvSM5Wm4Ve/+pVByxHj2r59e6s3DxswxrB9+3YsX768k6LqWnRq2dnb20MkEnGv2/ojpiEzM9Og5YhxHT58WGt51KhRWLduXbOuWU3LCYlOLbu9e/e2+JqYrpKSEu61SCTSahU0Xm5cjnRd+fn53Ovnz59DIpHg1KlTWL16NZRKJWxtbZuVExoall2gGpKZlZUV3N3dtd5zd3fn+kzqeLOeGFnj/dS0v2vjZSHvT51adkOHDuVOY9tz6dKlVwqIdI6Gx/1qa2uhUqmwa9cuSCQSKJVK/PGPf+Q6Ezd9LJB0fX369MHatWthZWWFv//97y0ObCFEOiW7yZMnd3AYpLP17duXe61QKLBw4UJu2drausVypOsaMGAAbt++DQB48OABFi1a1Go5odIp2a1du7aj4yCdzMnJiXvd2nh2TcuRrisnJwc9e/bUqZxQ0TmKQLm6unKvm56qNl5uXI50XY6Oju3uK1dXVzg6OnZSRF2PTi07R0dH3Lx5E05OTujRo0eb1+/oIXvT0HhoLEtLS60H/hsvG3oILdJx5HI53Nzc8ODBg2bvubq6GnygW1OjU7LbsmULd+t6y5YtOt+sIF2fu7t7s4Ogvr4e7u7uWgMFENMgl8tRUVGBMWPG4P79+/Dw8EBmZqagW3QNdEp2jQfiFPpoIXzRMBhqaWkpXFxc8NZbb6GiogKOjo44d+4cl+iaDppKuj5HR0fk5+fj1KlTkEqlvB68Ux96D95pZmbW4uCdT548gYuLC805aiIa9l9Dy+67777j3jMzM+Nadk33MyGmSu9k11qnRKVSSWNmmaDS0lK88847CA8Px08//YSBAwciJSUF33//vbFDI8SgdE52DQNlikQi/P3vf0f37t2599RqNc6dOwcfHx/DR0g6ROPrdCKRCEOHDoWLiwvc3d21ZqQS+kVtwh86J7stW7YAeNGyS0hI0Bq6u2HwzoSEBMNHSDpEwzSKCxcuxOnTp7lRnIEX02EuWLAACQkJ7U63SIip0DnZFRcXA3gxYnFSUhJ69OjRYUGRjtcw3HpJSQlu3ryJjIwMnD59GhMmTMDYsWMxadIkrXKEmDq9OxWnpaVRouOBhof/k5OTMXXqVEgkEowYMQISiQRTp05FcnKyVjlCTJ3eNyg+/vjjNt/fs2fPSwdDOs+YMWPg6ekJJycnFBQUNDuNDQgIwJMnTzBmzBgjRkmI4eid7J4+faq1rFKpcPXqVTx79gxvv/22wQIjHcvMzAybN2/G+++/32xS8PLycpSUlOC7777TujZLiCnTO9kdO3as2TqNRoOFCxd26IgKFRUV+PTTT/Hvf/+bm0px27ZtWneFm5Zfu3YtUlJScO/ePTg7O2Py5MlYv349jajcCGOs2RMxYrFY0OOeEX4yyEAAYrEYMTEx3B3bjhAZGYlr165BJpPh5MmTOHfuHObPn99q+bKyMpSVlSEuLg5Xr17Fvn37kJycjOjo6A6L0ZSo1WosW7YMw4cPbzaySc+ePTF8+HAsX76cOokT/mAG8v333zMnJydDfZyW69evMwAsNzeXW3f69GkmEolYaWmpzp/zz3/+k1laWjKVSqXzNpWVlQwAq6ys1Cvmri4tLY0BYACYRCLhXjddTktLM3ao5CXU1dWx48ePs7q6OmOH0qH0OT71Po2NiYlpmixRXl6O77//XusZWkPKzs6Gg4MDhg8fzq0LDQ2FWCzGhQsXMGXKFJ0+p2EiXXPz1qutVCqhVCq5ZYVCAeDFtUmVSvWSNeh67t69y71uOr1e4+W7d+/yqt5C0bDP+L7v9Kmf3snu8uXLWstisRjOzs7YvHlzu3dqX5ZcLm/2jKa5uTkcHR117uH/+PFjrF+/vs1TXwCIjY1tcRjrlJQU2NjY6B50F5eens69Zk2uzzVeTk9Ph4ODQydFRQyt8dMwfFRTU6NzWb2TXVpamr6btGrVqlXYsGFDm2UKCwtf+XsUCgXeeecdvP766/jjH//YZtnVq1drtV4VCgU8PDwQHh7Oq8nBG0+R6OjoiFmzZqG2thZWVlY4cOAANy5hz549IZVKjRUmeQlqtRrp6emQyWQICwtDcHAwb++qN5x56ULvZGdIy5Yta3fIKC8vL7i5uTUbaqi+vh4VFRXtDi75/PlzREREwNbWFseOHWt3uBuJRNKsKwYAWFhY8GqonMYt9Orqamzbto1bblz/y5cv86refJeUlISYmBjuMkV8fDz69euH+Pj4die4N0X6/DaNmuycnZ11ehwpMDAQz549Q15eHgICAgAAqamp0Gg0zSYBbkyhUGD8+PGQSCQ4ceJEsynmhKzxaLaNr1E2XW5p1FvSNSUlJWHq1KnN1t+9exdTp05FYmIiLxOerkxiDgpfX19ERERg3rx5yMnJQVZWFhYvXowZM2agd+/eAF4MVeTj48NNKKJQKBAeHo7q6mp8/fXXUCgUkMvlkMvl1J0C0PmUnE+n7nymVqsRFRXVZpmoqChB//ZNItkBwKFDh+Dj44OQkBBIpVKMHj0au3fv5t5XqVQoKiriLlheunQJFy5cQEFBAby9vdGrVy/u7/79+8aqRpeh63BcNGyXaTh79my7168UCgXOnj3bSRF1PQY7jf3555/xpz/9SSsBGZKjoyMOHz7c6vuenp5adxGDg4PpKYA23Llzx6DliHHp+kz6nj17EB4e3sHRdE0Ga9k9efIEX3/9taE+jnSwpl2IXrUcMa7z588btBwfGfUGBTEeXVu91Do2DY17Kzg5OSE4OJibQCk9PR2PHz9uVk5oKNkJlIuLC54/f65TOdL1NX7q5fHjx1oTKLVWTmhM5gYFMayhQ4catBwxLrFYt0NZ13J8pHPLrr3+Oc+ePXvVWEgn0nUmOJoxzjS4ubnpNKl5e53w+UznZNfeGHD29vaYPXv2KwdEOkfTMexetRwxLj8/P52SnZ+fXydE0zXpnOz27t3bkXGQTtavXz/utYWFhdboEZaWlty1ncblSNfV1kg+L1OOj4R7Ai9wjYfQb3oANH5onIbaNw30REz7dE7zug7fRBPumIbg4GA4Ozu3OS+si4sLgoODOy8o8tJmzZqFgwcPwsrKCnV1ddBoNNx7YrEYlpaWqK2txaxZs4wYpXHpnOz27duHfv36YejQodT3igfMzMyQkJCAqVOntro/d+3axduhgfgmJCQEdnZ2UCgUcHFxwZgxY/D06VP06NEDmZmZePjwIezs7BASEmLsUI1H1+GPFy1axHr06MH8/f3Ztm3b2JMnT15mFGWTw9dh2RskJiayfv36aQ3L7unpyRITE40dGtFTYmIiA8BEIpHW/mxY5uM+1ef41GsOitraWnb48GEWGhrKbGxs2LRp01hycjLTaDQvHWxXx/dkxxhj9fX1TCaTsZiYGCaTyVh9fb2xQyIvKTExkfXt21cr2fXr14+XiY4x/Y5PEWMvd0569+5d7Nu3D/v370d9fT2uXbvW6rSGpkyhUMDe3p6bv4KvVCoVTp06BalUSoN1mji1Wo20tDScPn0aEyZMwLhx43h7OUKf4/Ol70OLxWKIRCIwxgQ9RhYhXY2ZmRnGjh2L6upqjB07lreJTl96dT1RKpX45ptvEBYWhtdeew0FBQX429/+hnv37vGyVUcI4Q+dW3aLFi3CkSNH4OHhgY8//hjffPNNs8mVCSGkq9I52SUkJKBv377w8vJCRkYGMjIyWiyXlJRksOAIIcRQdE52s2fPpuckCSEmS69OxYQQYqro2VhCiCBQsiOECILJJLuKigpERkbCzs4ODg4OiI6ORlVVVZvbfPLJJxgwYACsra3h7OyMSZMm4caNG50UMSGkKzGZZBcZGYlr165BJpPh5MmTOHfuHObPn9/mNgEBAdi7dy8KCwtx5swZMMYQHh5OnaAJESCTGMmvsLAQycnJyM3NxfDhwwEAO3bsgFQqRVxcHHr37t3ido2ToaenJ/785z/Dz88PJSUlGDBgQKfETgjpGkwi2WVnZ8PBwYFLdAAQGhoKsViMCxcuYMqUKe1+RnV1Nfbu3Yv+/fvDw8Oj1XJKpRJKpZJbbphlXaVSaY3myzcNdeNzHYVEKPtTn/qZRLKTy+XNpvQzNzeHo6Mj5HJ5m9t++eWXWLlyJaqrqzFo0CDIZLI2J5GJjY3FunXrmq1PSUmBjY3Ny1XAhMhkMmOHQAyI7/uzpqZG57JGTXarVq3Chg0b2ixTWFj4St8RGRmJsLAwlJeXIy4uDh988AGysrJgZWXVYvnVq1cjJiaGW1YoFPDw8EB4eDjvRz2RyWQICwujUU94QCj7s+HMSxdGTXbLli3D3Llz2yzj5eUFNze3ZjOZ19fXo6Kiot2p4ezt7WFvb4+BAwfizTffRI8ePXDs2DF8+OGHLZaXSCSQSCTN1ltYWPD6R9NAKPUUCr7vT33qZtRk5+zsDGdn53bLBQYG4tmzZ8jLy0NAQAAAIDU1FRqNBqNGjdL5+9iLwUq1rskRQoTBJLqe+Pr6IiIiAvPmzUNOTg6ysrKwePFizJgxg7sTW1paCh8fH+Tk5AAA7ty5g9jYWOTl5eHevXs4f/48pk2bBmtra0ilUmNWhxBiBCaR7ADg0KFD8PHxQUhICKRSKUaPHo3du3dz76tUKhQVFXEXLK2srJCZmQmpVApvb29Mnz4dtra2OH/+fLObHYQQ/jOJu7EA4OjoiMOHD7f6vqenp9YsWb1798apU6c6IzRCiAkwmZYdIYS8Ckp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBMJlkV1FRgcjISNjZ2cHBwQHR0dGoqqrSaVvGGCZMmACRSITjx493bKCEkC7JZJJdZGQkrl27BplMhpMnT+LcuXOYP3++Tttu3boVIpGogyMkhHRlJjGVYmFhIZKTk5Gbm4vhw4cDAHbs2AGpVIq4uDhuouyW5OfnY/Pmzbh48SJ69erVWSETQroYk2jZZWdnw8HBgUt0ABAaGgqxWIwLFy60ul1NTQ1mzpyJnTt3ws3NrTNCJYR0USbRspPL5XBxcdFaZ25uDkdHR8jl8la3W7p0KYKCgjBp0iSdv0upVEKpVHLLCoUCAKBSqaBSqfSM3HQ01I3PdRQSoexPfepn1GS3atUqbNiwoc0yhYWFL/XZJ06cQGpqKi5fvqzXdrGxsVi3bl2z9SkpKbCxsXmpWEyJTCYzdgjEgPi+P2tqanQuK2KMsQ6MpU2PHj3CkydP2izj5eWFgwcPYtmyZXj69Cm3vr6+HlZWVvj2228xZcqUZtstWbIE27dvh1j83zN1tVoNsViMMWPGID09vcXva6ll5+HhgcePH8POzk7PGpoOlUoFmUyGsLAwWFhYGDsc8oqEsj8VCgWcnJxQWVnZ7vFp1Jads7MznJ2d2y0XGBiIZ8+eIS8vDwEBAQCA1NRUaDQajBo1qsVtVq1ahf/5n//RWjd48GBs2bIFEydObPW7JBIJJBJJs/UWFha8/tE0EEo9hYLv+1OfupnENTtfX19ERERg3rx5SEhIgEqlwuLFizFjxgzuTmxpaSlCQkKwf/9+jBw5Em5ubi3elOjbty/69+/f2VUghBiZSdyNBYBDhw7Bx8cHISEhkEqlGD16NHbv3s29r1KpUFRUpNc5PCFEOEyiZQcAjo6OOHz4cKvve3p6or3Lj0a8PEkIMTKTadkRQsiroGRHCBEESnaEEEGgZEcIEQRKdoTwjFqtRkZGBs6dO4eMjAyo1Wpjh9QlULIjdHDwSFJSEry9vREWFob4+HiEhYXB29sbSUlJxg7N6CjZCRwdHPyRlJSE999/H4MHD0ZmZia++eYbZGZmYvDgwXj//fcFv0+N+mysKVAoFLC3t9fp2TtT03BwREREQKFQ4N69e+jbty/s7OyQnJyM7777Du+9956xwyQ6UKvV8Pb2xuDBg3H8+HGo1WqcOnUKUqkUZmZmmDx5Mq5evYqffvoJZmZmxg7XYPQ5PinZtYOvya7h4Hj69CkqKyubvW9vbw9HR0feHRx8lZ6ejnHjxiE7OxsjRoxAWloaTp8+jQkTJmDcuHHIyclBUFAQ0tLSEBwcbOxwDUaf45NOYwUqMzMTJSUlLSY6AKisrERxcTEyMzM7OTLyMsrLywEAt2/fhpeXl9ZlCS8vL9y5c0ernBBRshOo4uJig5YjxtUw5cBHH32E+/fva713//59fPTRR1rlhIiSnUAlJCQYtBwxrqCgIG5SKWdnZyQkJGDv3r1ISEjghlETiUQICgoyZphGRclOoHQdwVnfkZ6JcaSnp3MDXQQEBODq1av45ptvcPXqVW4MSMZYq4PWCoHJjHpCDKvx2P0WFhZYsmQJ+vfvj+LiYmzdulUwcxjwxYEDBwC8GKD29OnT3PqUlBRufUFBAQ4cOIDw8HCjxGhslOwIXF1dsWnTJm7Zw8Oj2XUf0rU1TBhfUFDQ4vsN63WdWJ6P6DRWoKysrLjXP//8M95++21ERkbi7bff1kp0jcuRriswMNCg5fiIWnYC5enpiRs3bnDLqamprZYjXZ+uj/gJ+VFAatkJ1Ny5cw1ajhjXiRMnDFqOj+gJinbw9QmKurq6FmdRa0qpVMLS0rITIiKvws3NDQ8ePADwootJ48O68bKrq2ubE8ubGnqCgrTL0tISK1asaLPMihUrKNGZiPr6eu7106dPsWDBAvj7+2PBggXN5lsWKrpmJ2AbN24EAGzevBkajYZbb2ZmhpiYGO590vXZ2tpyE847Ojpy+zM/P19rFj5bW1ujxNcVmEzLrqKiApGRkbCzs4ODgwOio6PbvY0eHBwMkUik9bdgwYJOitg0bNy4Eb/88gvi4uIglUoRFxeHmpoaSnQmpvFk8Y3/x9V0ubVJ5YXAZFp2kZGRKC8vh0wmg0qlQlRUFObPn9/m9IoAMG/ePPzpT3/ilm1sbDo6VJNjaWmJ3/3ud/D29oZUKuX1DPJ8FRUVhaNHj+pUTqhMItkVFhYiOTkZubm5GD58OABgx44dXEukd+/erW5rY2MDNze3zgqVEKNoeC7WUOX4yCSSXXZ2NhwcHLhEBwChoaEQi8W4cOECpkyZ0uq2hw4dwsGDB+Hm5oaJEyfiiy++aLN1p1QqoVQquWWFQgHgxWNTfH50ih4PM20//PCDzuXGjRvXwdF0Hn1+ryaR7ORyOVxcXLTWmZubw9HRsc3b6DNnzkS/fv3Qu3dv/Oc//8Hnn3+OoqKiNoenjo2Nxbp165qtT0lJEcQpsEwmM3YI5CU0PAMLtN31JCUlBb/+9a87Pb6OUlNTo3NZoya7VatWYcOGDW2WKSwsfOnPnz9/Pvd68ODB6NWrF0JCQnD79m0MGDCgxW1Wr16NmJgYblmhUMDDwwPh4eG86mfXlEqlgkwmQ1hYGF2zM0FfffUVrly5AnNzczx58gTnz5/n9mdQUBB69uyJ+vp69OnTB1Kp1NjhGkzDmZcujJrsli1b1m4PfS8vL7i5ueHhw4da6+vr61FRUaHX9biGO1G3bt1qNdlJJJIWO9taWFgIIgkIpZ58Ixa/6FhRX1+PyMhIrFy5EiNGjEC3bt0QGRnJ9a8Ti8W82r/61MWoyc7Z2ZkbWLAtgYGBePbsGfLy8rixuVJTU6HRaPS6lZ6fnw9A2KO1En7q06cP9/qHH37AyZMnuWVra+sWywmNSfSz8/X1RUREBObNm4ecnBxkZWVh8eLFmDFjBncntrS0FD4+PsjJyQHwYiz+9evXIy8vDyUlJThx4gRmz56Nt956C0OGDDFmdQgxuNdee417XVdXp/Ve4xtujcsJjUkkO+DFXVUfHx+EhIRAKpVi9OjRWj3DVSoVioqKuAuWlpaW+OGHHxAeHg4fHx8sW7YMU6dOxb///W9jVYGQDrNo0SKYm5vDxsamWfcSkUgEGxsbmJubY9GiRUaK0PhM4m4s8OIRmLY6EHt6emrdgfLw8EBGRkZnhEaI0VlaWmLp0qXYtGkTXFxcMGbMGDx9+hQ9evRAZmYmHj58KPhnnU0m2RFC2tbwiN+WLVuQmJjIrTc3N8eKFSsE/wigyZzGEkLat3HjRlRXV2s961xdXS34RAdQy44Q3qFnnVtGLTtCiCBQsiOECAIlO0KIINA1u3Y0dGfR5xk8U6RSqVBTUwOFQkHXeHhAKPuz4bjUZSodSnbteP78OYAX/fYIIV3T8+fPYW9v32YZml2sHRqNBmVlZbC1teX1wIcNo7vcv3+f16O7CIVQ9idjDM+fP0fv3r25wRBaQy27dojFYkE9PG1nZ8frg0NohLA/22vRNaAbFIQQQaBkRwgRBEp2BMCLQUvXrl3b4sClxPTQ/myOblAQQgSBWnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2PBQcHIwlS5YYOwyip66237paPK+Kkh0hPNJ0ZjHSCCO8MmfOHAZA68/d3Z19+eWXWuUuXbrERCIRKykpYYwxtnnzZvbGG28wGxsb1qdPH7Zw4UL2/PlzrW0yMzPZ6NGjmZWVFevTpw/79NNPWVVVVafVjc9a2m+3bt1iH3/8MfP09GRWVlbstddeY1u3bm223aRJk9if//xn1qtXL+bp6ckYYywrK4v5+fkxiUTCAgIC2LFjxxgAdvnyZW7bgoICFhERwbp168ZcXFzYRx99xB49etRqPMXFxZ31z9EhKNnxzLNnz1hgYCCbN28eKy8vZ+Xl5Wz58uVs9OjRWuWWLVumtW7Lli0sNTWVFRcXs7Nnz7JBgwaxhQsXcu/funWLdevWjW3ZsoXdvHmTZWVlsaFDh7K5c+d2Wt34rKX9Vltby9asWcNyc3PZnTt32MGDB5mNjQ07evQot92cOXNY9+7d2axZs9jVq1fZ1atXWWVlJXN0dGQfffQRu3btGjt16hR77bXXtJLd06dPmbOzM1u9ejUrLCxkly5dYmFhYWzcuHGtxlNfX2+MfxqDoWTHQ2PHjmWfffYZt3z58mUmEonY3bt3GWOMqdVq5u7uznbt2tXqZ3z77besZ8+e3HJ0dDSbP3++VpnMzEwmFovZL7/8YtgKCFTT/daS3/72t2zq1Knc8pw5c5irqytTKpXcul27drGePXtq7ZevvvpKK9mtX7+ehYeHa332/fv3GQBWVFSkczymhK7ZCYC/vz98fX25eXczMjLw8OFDTJs2jSvzww8/ICQkBO7u7rC1tcWsWbPw5MkTbtLxK1euYN++fejevTv3N378eGg0GhQXFxulXkKwc+dOBAQEwNnZGd27d8fu3btx7949rTKDBw/Wmg+2qKgIQ4YMgZWVFbdu5MiRWttcuXIFaWlpWvvTx8cHAHD79u0OrJHxULITiMjISC7ZHT58GBEREejZsycAoKSkBO+++y6GDBmCxMRE5OXlYefOnQD+e8G7qqoKn3zyCfLz87m/K1eu4KeffsKAAQOMUymeO3LkCJYvX47o6GikpKQgPz8fUVFRzW5CdOvWTe/PrqqqwsSJE7X2Z35+Pn766Se89dZbhqpCl0Lj2fGQpaUl1Gq11rqZM2fiD3/4A/Ly8vDdd98hISGBey8vLw8ajQabN2/mBkD85z//qbX9sGHDcP36dXh7e3d8BQSq6X7LyspCUFAQFi1axK3TpdU1aNAgHDx4EEqlkhsIIDc3V6vMsGHDkJiYCE9PT5ibt5wGWvodmTJq2fGQp6cnLly4gJKSEjx+/BgajQaenp4ICgpCdHQ01Go1fvOb33Dlvb29oVKpsGPHDty5cwcHDhzQSoYA8Pnnn+P8+fNYvHgx1wL417/+hcWLF3d29Xir6X4bOHAgLl68iDNnzuDmzZv44osvmiWtlsycORMajQbz589HYWEhzpw5g7i4OADgRtv+7W9/i4qKCnz44YfIzc3F7du3cebMGURFRXEJrqXfkUkz9kVDYnhFRUXszTffZNbW1lpdBr788ksGgM2ePbvZNvHx8axXr17M2tqajR8/nu3fv58BYE+fPuXK5OTksLCwMNa9e3fWrVs3NmTIEPaXv/ylk2rFf033240bN9jcuXOZvb09c3BwYAsXLmSrVq1ifn5+3DYNXU+aysrKYkOGDGGWlpYsICCAHT58mPvMBjdv3mRTpkxhDg4OzNramvn4+LAlS5YwjUbTYjym3vWEhngiRAAOHTqEqKgoVFZWwtra2tjhGAVdsyOEh/bv3w8vLy+4u7vjypUr+Pzzz/HBBx8INtEBlOwI4SW5XI41a9ZALpejV69emDZtGv7yl78YOyyjotNYQogg0N1YQoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI13G3LlzIRKJIBKJYGFhAVdXV4SFhWHPnj16Paq0b98+ODg4dFygrZg7dy4mT57c6d9LdEPJjnQpERERKC8vR0lJCU6fPo1x48bhs88+w7vvvov6+npjh0dMmXGfViPkv1p7zvPs2bMMAPvqq68YY20PIZ+WltZsOPG1a9cyxhjbv38/CwgIYN27d2eurq7sww8/ZA8ePOC+p6Kigs2cOZM5OTkxKysr5u3tzfbs2cO9f+/ePTZt2jRmb2/PevTowX7zm99wz4uuXbu22fempaV1yL8TeTnUsiNd3ttvvw0/Pz8kJSUBAMRiMbZv345r167hH//4B1JTU7Fy5UoAQFBQELZu3Qo7OzuUl5ejvLwcy5cvBwCoVCqsX78eV65cwfHjx1FSUoK5c+dy3/PFF1/g+vXrOH36NAoLC7Fr1y44OTlx244fPx62trbIzMxEVlYWunfvjoiICNTV1WH58uX44IMPuJZpeXk5goKCOvcfirTN2NmWkAattewYY2z69OnM19e3xfeaDiG/d+9eZm9v3+735ebmMgBcq3DixIksKiqqxbIHDhxggwYN4kYEYYwxpVLJrK2t2ZkzZ9qNnxgfteyISWCMcWOxtTeEfGvy8vIwceJE9O3bF7a2thg7diwAcMOcL1y4EEeOHIG/vz9WrlyJ8+fPc9teuXIFt27dgq2tLTeMuaOjI2pra3k7jDnfULIjJqGwsBD9+/fXaQj5llRXV2P8+PGws7PDoUOHkJubi2PHjmltN2HCBNy9exdLly5FWVkZQkJCuFPgqqoqBAQENBvG/ObNm5g5c2YH154YAo16Qrq81NRUFBQUYOnSpToNId/ScOI3btzAkydP8L//+7/w8PAAAFy8eLHZdzk7O2POnDmYM2cOxowZgxUrViAuLg7Dhg3D0aNH4eLiAjs7uxbj5Nsw5nxDLTvSpSiVSsjlcpSWluLSpUv461//ikmTJuHdd9/F7NmzdRpC3tPTE1VVVTh79iweP36Mmpoa9O3bF5aWltx2J06cwPr167W2W7NmDf71r3/h1q1buHbtGk6ePAlfX18ALyYscnJywqRJk5CZmYni4mKkp6fjd7/7HX7++Wfue//zn/+gqKgIjx8/hkql6px/NKIbY180JKRB41nozc3NmbOzMwsNDWV79uxharWaK6fLEPILFixgPXv21Op6cvjwYebp6ckkEgkLDAxkJ06caDaXqq+vL7O2tmaOjo5s0qRJ7M6dO9xnlpeXs9mzZzMnJycmkUiYl5cXmzdvHqusrGSMMfbw4UNu2HpQ15Muh8azI4QIAp3GEkIEgZIdIUQQKNkRQgSBkh0hRBAo2RFCBIGSHSFEECjZEUIEgZIdIUQQKNkRQgSBkh0hRBAo2RFCBIGSHSFEEP4f9JqX5k8v4BIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:11:43.063972Z", + "iopub.status.busy": "2024-02-29T18:11:43.063705Z", + "iopub.status.idle": "2024-02-29T18:11:43.256163Z", + "shell.execute_reply": "2024-02-29T18:11:43.255366Z" + }, + "papermill": { + "duration": 0.215855, + "end_time": "2024-02-29T18:11:43.258087", + "exception": false, + "start_time": "2024-02-29T18:11:43.042232", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEmCAYAAAADXQcxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbJUlEQVR4nO2dd3hTZdvAf9ndLVAoq1D2hjIsgoNVBD+U4UJElogDUV8rIiCC4ABliCLIKwpOBCf4OlCogDJEKHtvympLKXS3Wef7I02a1TYpaVOS53dduaAnzzm5s+7czz1lkiRJCAQCgY8g97YAAoFA4EmEUhMIBD6FUGoCgcCnEEpNIBD4FEKpCQQCn0IoNYFA4FMIpSYQCHwKodQEAoFPofS2AJWN0Wjk0qVLhIaGIpPJvC2OQCCwQ5IksrOzqVu3LnK5+3aX3ym1S5cuER0d7W0xBAJBGZw/f5769eu7fV6VUGqLFy9m7ty5pKSk0KFDBxYtWkRcXFyZ561atYphw4YxaNAg1qxZ49JjhYaGAqYXLCws7EbEFggEFUBWVhbR0dGW76q7eF2prV69moSEBJYuXUrXrl1ZuHAh/fr149ixY9SqVavE886ePcvEiRO544473Ho885YzLCxMKDWBoApTXveQ1wMFCxYsYNy4cYwZM4bWrVuzdOlSgoKCWL58eYnnGAwGhg8fzsyZM2ncuHElSisQCKo6XlVqWq2WpKQk4uPjLcfkcjnx8fFs3769xPNmzZpFrVq1GDt2bJmPUVhYSFZWls1NIBD4Ll5Vaunp6RgMBqKiomyOR0VFkZKS4vScLVu28Mknn7Bs2TKXHmP27NmEh4dbbiJIIBD4Nl73qblDdnY2I0aMYNmyZURGRrp0zpQpU0hISLD8bXZCCm4OJElCr9djMBi8LYrAQygUCpRKZYWlVHlVqUVGRqJQKEhNTbU5npqaSu3atR3Wnzp1irNnz3LvvfdajhmNRgCUSiXHjh2jSZMmNudoNBo0Gk0FSC+oaLRaLZcvXyYvL8/bogg8TFBQEHXq1EGtVnv82l5Vamq1ms6dO5OYmMjgwYMBk5JKTExkwoQJDutbtmzJgQMHbI5NmzaN7Oxs3nvvPWGB+RBGo5EzZ86gUCioW7cuarVaJEv7AJIkodVquXLlCmfOnKFZs2blSrAtDa9vPxMSEhg1ahRdunQhLi6OhQsXkpuby5gxYwAYOXIk9erVY/bs2QQEBNC2bVub8yMiIgAcjlc0OYV6dpy+ytaTVzmaksUTdzamZ4uSU1AE7qHVajEajURHRxMUFORtcQQeJDAwEJVKxblz59BqtQQEBHj0+l5XakOHDuXKlStMnz6dlJQUYmNjWbdunSV4kJyc7HFNXh4K9QZ2n7vOtlPpbD2Zzr4LmRiMxeMdJAmh1CqAqvDeCzxPRb6vMn8bvJKVlUV4eDiZmZmlJt8ajBKHL2WxtUiJ7TybQYHOaLOmYY0gWtYO5fdDqUSGqNk1rW9Fi+83FBQUcObMGRo1auTxX3KB9ynt/XX1O1oSXrfUqip/Hk1j3Oe7bI5Fhmjo3qQGtzeNpHvTGtSvFkSeVk/r6b+TnqPlak4hNUJEUEIg8CZCqZVAXKPqhAequCWmGt2bRHJb00iaR4U4OKuD1EoaVA8iOSOP46k5dBNKTSBg9OjRXL9+3eWabE8ilFoJhAeq2P1qXxTysiNuzaNCSM7I40RaNt2a1KgE6QRVmZ49exIbG8vChQu9LYpfIrywpeCKQgNoFmXqJnA8NbsixREIKhWdTudtEcqFUGoeoHlUCADHU3O8LInvIkkSeVq9V27uxNJGjx7N5s2bee+995DJZMhkMurXr8+HH35os27Pnj3I5XLOnTsHmBo7tGvXjuDgYKKjoxk/fjw5Obafpy1btnDHHXcQGBhIdHQ0zz33HLm5uS7JdfnyZQYMGEBgYCCNGjVi5cqVxMTE2FiTMpmMDz/8kIEDBxIcHMybb76JwWBg7NixNGrUiMDAQFq0aMF7771nc22DwUBCQgIRERHUqFGDSZMmufWaeRqx/fQAzWqZLLUTqdlIkiSSRCuAfJ2B1tN/98pjH57VjyC1a1+V9957j+PHj9O2bVtmzZoFwPz581m5ciVPP/20Zd1XX33FbbfdRsOGDQFTisP7779Po0aNOH36NOPHj2fSpEksWbIEMFXT9O/fnzfeeIPly5dz5coVJkyYwIQJE1ixYkWZco0cOZL09HQ2bdqESqUiISGBtLQ0h3WvvfYac+bMYeHChSiVSoxGI/Xr1+fbb7+lRo0abNu2jSeeeII6derw0EMPWZ7fp59+yvLly2nVqhXz58/nxx9/pHfv3i69Zp5GKDUP0LRWCHIZXMvTcSWnkFqhIgXBXwkPD0etVhMUFGQp9Rs+fDjz588nOTmZBg0aYDQaWbVqFdOmTbOc95///Mfy/5iYGN544w2eeuopi1KbPXs2w4cPt6xr1qwZ77//Pj169ODDDz8sNe3l6NGjbNiwgZ07d9KlSxcAPv74Y5o1a+aw9pFHHrEkvpuZOXOm5f+NGjVi+/btfPPNNxaltnDhQqZMmcJ9990HwNKlS/n9d+/8AIFQah4hQKWgQfUgzl7N40RqjlBqFUCgSsHhWf289tg3QmxsLK1atWLlypVMnjyZzZs3k5aWxoMPPmhZs2HDBmbPns3Ro0fJyspCr9dTUFBAXl4eQUFB7Nu3j/379/PVV19ZzpEkyVJO1qpVqxIf/9ixYyiVSjp16mQ51rRpU6pVq+aw1qz0rFm8eDHLly8nOTmZ/Px8tFotsbGxAGRmZnL58mW6du1qWa9UKunSpYvXtqDCp+YhmotgQYUik8kIUiu9cvOEO2H48OGsXLkSgJUrV9K/f39q1DBFys+ePcs999xD+/bt+f7770lKSmLx4sWAqVwMICcnhyeffJK9e/dabvv27ePEiRMOTRxuhODgYJu/V61axcSJExk7dix//PEHe/fuZcyYMRa5qiJCqXmIYqUmggX+jlqtdmiV9Mgjj3Dw4EGSkpL47rvvGD58uOW+pKQkjEYj8+fP59Zbb6V58+ZcunTJ5vxOnTpx+PBhmjZt6nArq9NFixYt0Ov17Nmzx3Ls5MmTXLt2rcznsnXrVrp378748ePp2LEjTZs25dSpU5b7w8PDqVOnDjt27LAc0+v1JCUllXntikIoNQ/RrCgCekJYan5PTEwMO3bs4OzZs6Snp2M0GomJiaF79+6MHTsWg8HAwIEDLeubNm2KTqdj0aJFnD59mi+++IKlS5faXPPll19m27ZtTJgwgb1793LixAnWrl3rtJuNPS1btiQ+Pp4nnniCf//9lz179vDEE08QGBhYphXarFkzdu3axe+//87x48d59dVX2blzp82a559/njlz5rBmzRqOHj3K+PHjuX79uusvmIcRSs1DWG8//aycVmDHxIkTUSgUtG7dmpo1a5KcnAyYtqD79u1jyJAhBAYGWtZ36NCBBQsW8Pbbb9O2bVu++uorZs+ebXPN9u3bs3nzZo4fP84dd9xBx44dmT59OnXr1nVJps8//5yoqCjuvPNOhgwZwrhx4wgNDS2zrvbJJ5/kvvvuY+jQoXTt2pWrV68yfvx4mzUvvvgiI0aMYNSoUXTr1o3Q0FCGDBniklwVgSho9xCFelPKgcEo8c+UPtQOF8GCG0EUtFcsFy5cIDo6mg0bNtCnT59Kf3xR0H4ToFEqaFgjiNNXcjmemi2UmqBK8eeff5KTk0O7du24fPkykyZNIiYmhjvvvNPbonkcsf30IC1EBFTgBf7++29CQkJKvIGp5Gnq1Km0adOGIUOGULNmTUsirq8hLDUP0iwqlN8OpnBCREAFlUiXLl3Yu3dvqWv69etHv37eyfOrbIRS8yCWGtA0YakJKo/AwECaNm3qbTGqDGL76UHMEdATqTkiAioQeAmh1DxITI1glHIZOYV6LmUWeFscgcAvEUrNg6iVchpFmspMRLBAIPAOQql5mOa1i9sQCQSCykcoNQ/TvJaoARUIvIlQah6muagBFdwAzrrRemN4yc2MSOnwMOZ5BSfScjAaJeQuzjkQCJxx+fJlp33PBCUjLDUPE1MjCLVCTp7WwMXr+d4WR3CTU7t2bTQa745dlCQJvV7vVRncQSg1D6NUyGlcU0RAPY4kgTbXOzc3cw6zs7MZPnw4wcHB1KlTh3fffZeePXvatOx2Fevt59mzZ5HJZPzwww/06tWLoKAgOnTowPbt223OKWtAyxdffEGXLl0IDQ2ldu3aPPLIIzbzCjZt2oRMJuO3336jc+fOaDQatmzZ4rbs3kJsPyuAZlGhHE3J5nhqDn1aRXlbHN9AlwdvudZmx+NMvQTq4LLXFZGQkMDWrVv56aefiIqKYvr06ezevdvSAvtGeeWVV5g3bx7NmjXjlVdeYdiwYZw8eRKlUunSgBadTsfrr79OixYtSEtLIyEhgdGjR/Prr7/aPM7kyZOZN28ejRs3vqm2wEKpVQAtokL4HyJY4I9kZ2fz2WefsXLlSktLnxUrVrjc98wVJk6cyIABAwDTUJQ2bdpw8uRJWrZs6dKAlscee8xyrcaNG/P+++9zyy23kJOTYymAB5g1axZ9+/b1mNyVhVBqFYBluLGoAfUcqiCTxeStx3aR06dPo9PpiIuLsxwLDw+nRYsWHhOnffv2lv/XqVMHgLS0NFq2bOnSgJakpCRee+019u3bx7Vr1zAajQAkJyfTunVry3nOhrDcDAilVgGYa0BPpuVgMEouT3oXlIJM5tYW0JexbhdkbsdtVkzmAS3PPfecw3kNGjQgNzfX0rHjq6++snTm7devn8MwFfshLDcLQqlVAA2qB6FRyinQGTmfkUdM5M354RC4T+PGjVGpVOzcuZMGDRoApjFyx48fr5SGjNYDWpxx4MABrl69ypw5c4iOjgZg165dFS5XZSKinxWAQi6jSc2iNkTCr+ZXhIaGMmrUKF566SU2btzIoUOHGDt2LHK53COj9sqirAEtDRo0QK1WW4a8/PTTT7z++usVLldlIpRaBWGpLEgT5VL+xoIFC+jWrRv33HMP8fHx3HbbbbRq1apSZi2UNaClZs2afPrpp3z77be0bt2aOXPmMG/evAqXqzIRg1cqiCWbTvLOumMMiq3Lew93rLDH8VV8afBKbm4u9erVY/78+YwdO9bb4lQJxOCVmxBR2O6/7Nmzh6NHjxIXF0dmZiazZs0CYNCgQV6WzD8QSq2CMEdAT13JQW8wolSInb4/MW/ePI4dO4ZaraZz5878/fffHDlyhLvvvrvEc3JyxA+gJxBKrYKoXy2QQJWCfJ2Bcxl5lsCBwPfp2LEjSUlJDsfz8/PLHJAiuHGEUqsg5HIZTWuFcOBiJidSs4VSE4gBKZWE2BNVIM3M06WEX63c+Fkcy2+oyPdVKLUKpLkYblxuzFnzeXl5XpZEUBGY39eKGKYstp8VSAurkXkC91AoFERERFha4gQFBVVK8qqgYpEkiby8PNLS0oiIiEChUHj8MYRSq0DM28/T6TnoDEZUIgLqFrVr1waw6fUl8A0iIiIs76+nEUqtAqkXEUiwWkGu1sDZ9FxL9w6Ba8hkMurUqUOtWrXQ6XTeFkfgIVQqVYVYaGaqhFJbvHgxc+fOJSUlhQ4dOrBo0SKb1i3W/PDDD7z11lucPHkSnU5Hs2bNePHFFxkxYkQlS102MpmMplGh7Dt/neOpOUKplROFQlGhXwKBb+H1/dDq1atJSEhgxowZ7N69mw4dOtCvX78StxzVq1fnlVdeYfv27ezfv58xY8YwZswYfv/990qW3DWa1xKF7QJBZeJ1pbZgwQLGjRvHmDFjaN26NUuXLiUoKIjly5c7Xd+zZ0+GDBlCq1ataNKkCc8//zzt27evsj3Um1umSwmlJhBUBl5ValqtlqSkJOLj4y3H5HI58fHxDsMknCFJEomJiRw7dqzEXlWFhYVkZWXZ3CoT88R2kasmEFQOXlVq6enpGAwGoqJsh5NERUWRkpJS4nmZmZmEhISgVqsZMGAAixYtKrGX+uzZswkPD7fczI3xKgtzC6Kz6bkU6g2V+tgCgT/i9e1neQgNDWXv3r3s3LmTN998k4SEBDZt2uR07ZQpU8jMzLTczp8/X6my1g4LIFSjRG+UOJOeW/YJAoHghvBq9DMyMhKFQkFqaqrN8dTU1FJzWORyuaWGLjY2liNHjjB79mx69uzpsFaj0Xh1GKxMJqNZVAi7k00R0Ja1K66Hm0Ag8LKlZm7LkpiYaDlmNBpJTEykW7duLl/HaDRSWFhYESJ6BEuwQERABYIKx+t5agkJCYwaNYouXboQFxfHwoULyc3NZcyYMQCMHDmSevXqMXv2bMDkI+vSpQtNmjShsLCQX3/9lS+++IIPP/zQm0+jVJqJGlCBoNLwulIbOnQoV65cYfr06aSkpBAbG8u6desswYPk5GTk8mKDMjc3l/Hjx3PhwgUCAwNp2bIlX375JUOHDvXWUygTy7wCEQEVCCocMaOgEkjLKiDurUTkMjg8qz8BKpEdLxCUxI1+R2/K6OfNRs1QDeGBKoySqb23QCCoOG5IqeXk5Hg1sfVmQSaTiS2oQFBJuK3Uzpw5w4ABAwgODiY8PJxq1apRrVo1IiIiqFatWkXI6BOIYIFAUDm4HSh49NFHkSSJ5cuXExUVJRr3uUhxYbuw1ASCisRtpbZv3z6SkpJo0aJFRcjjs4jCdoGgcnB7+3nLLbdUeqmRL2AubE/OyCNfK2pABYKKwm1L7eOPP+app57i4sWLtG3b1mFwQvv27T0mnC8RGaKherCajFwtJ9NyaFc/3NsiCQQ+idtK7cqVK5w6dcqS8Q+m6J4kSchkMgwGYYWURLNaIew4k8Hx1Gyh1ASCCsJtpfbYY4/RsWNHvv76axEocJPmUaEmpSb8agJBheG2Ujt37hw//fSTmDRdDjyZq3Y0JYtX1xwkoW8LujWpccPXEwh8BbcDBb1792bfvn0VIYvP48lctV/3X2bn2Wus3Xvxhq8lEPgSbltq9957Ly+88AIHDhygXbt2DoGCgQMHekw4X8Oc1nHhWj65hXqCNeXvJ5CZbxoZV6ATPkyBwBq3v1VPPfUUALNmzXK4TwQKSqd6sJrIEA3pOYWcSMshNjqi3NfKKtADUKg3ekg6gcA3cHv7aTQaS7wJhVY2Zr/ajW5Bs4SlJhA4xS2lptPpUCqVHDx4sKLk8Xk81QU3q8Ck1ISlJhDY4pZSU6lUNGjQQFhkN0CzKM/UgGbli+2nQOAMt7efr7zyClOnTiUjI6Mi5PF5PG2pie2nQGCL24GCDz74gJMnT1K3bl0aNmxIcHCwzf27d+/2mHC+SLOibh2XMgvI0+oJUpcvAmr2qQlLTSCwxe1v1ODBgytADP8hIkhNtSAV1/J0nEnPpU1d98uldAYjuUVF8WJAskBgi9tKbcaMGRUhh1/RuGYISeeucfpK+ZRadlE6B0CBTlhqAoE15c7+TEpK4siRIwC0adOGjh07ekwoX6dRZDBJ566Ve2K7eesJUCh8agKBDW4rtbS0NB5++GE2bdpEREQEANevX6dXr16sWrWKmjVrelpGn6NxTZMf8nQ5h7CYgwQgfGoCgT1uRz+fffZZsrOzOXToEBkZGWRkZHDw4EGysrJ47rnnKkJGn6NxpEmpld9SK95+FuqN+NmUQ4GgVNy21NatW8eGDRto1aqV5Vjr1q1ZvHgxd911l0eF81UaRZoioKfTcy196NzB2lIDk2ITs0QFAhPlKpOyL2IHU2Ku0Si2Qq7QsEYQMpnJ4Z+eo3X7fGufGogtqEBgTblaDz3//PNcunTJcuzixYu88MIL9OnTx6PC+SoBKgX1IgKB8m1BHSw1ESwQCCy4rdQ++OADsrKyiImJoUmTJjRp0oRGjRqRlZXFokWLKkJGn6RxzaItaDmCBdY+NRCWmkBgjds+tejoaHbv3s2GDRs4evQoAK1atSI+Pt7jwvkyjSOD+ev4Fc9YaiIBVyCwUK48NZlMRt++fenbt6+n5fEbzGkdp664r9Qy7XxqIgFXICimXEotMTGRxMRE0tLSHIIDy5cv94hgvk4jS1pHebafwlITCErCbaU2c+ZMZs2aRZcuXahTp46YJlVOzD615Iw89AYjSoXr7s2sAjufmrDUBAILbiu1pUuX8umnnzJixIiKkMdvqBMWQIBKToHOyIVr+cREBpd9UhH2llqBsNQEAgtuRz+1Wi3du3evCFn8CrlcRkyNonIpN7eg5kBBYFHCrbDUBIJi3FZqjz/+OCtXrqwIWfyO4hpQ94IF5pSOWmEaQKR0CATWuL39LCgo4KOPPmLDhg20b9/eobpgwYIFHhPO12lsVS7lKlq9kfyiZNuaIRrOXc0TgQKBwAq3ldr+/fuJjY0FcBjAIoIG7mGJgLphqWVb5ahFhpgsNZHSIRAU47ZS27hxY0XI4ZdYtp9u+NTMkc8QjZIgTZFPTVhqAoEFt31qAs9h3n6mZhWSW6gvY7UJc+QzPFCFRikCBQKBPUKpeZHwIBU1gtWA64Xt5mqC0AAlGqXp7RMpHQJBMUKpeRmzX83VYIE5nSMsUGXpoSYsNYGgGKHUvIzZr+ZqsMCczhEWoLJYaiKlQyAoxm2l9tdff6HXO/p/9Ho9f/31l0eE8ieKu+C6FiwottSUaFRF2083+6npDaIFuMB3cVup9erVy+l09szMTHr16uURofyJRm7OKzAHCsICVASYAwVuWGr5WgN3vrORcZ/vclNSgeDmwG2lVlJP/atXrzpMa3eVxYsXExMTQ0BAAF27duXff/8tce2yZcu44447qFatGtWqVSM+Pr7U9VWdJlZVBa5YT9Y+NbOl5k5Kx/lreVzKLGDbqavlkFYgqPq4nKd23333AaYE29GjR6PRaCz3GQwG9u/fX66a0NWrV5OQkMDSpUvp2rUrCxcupF+/fhw7doxatWo5rN+0aRPDhg2je/fuBAQE8Pbbb3PXXXdx6NAh6tWr5/bje5sGNYKQyyCnUM+VnEJqhQaUur7Yp6a0pHS4k3yrN5gUZ77OUK6hLwJBVcdlSy08PJzw8HAkSSI0NNTyd3h4OLVr1+aJJ57gyy+/dFuABQsWMG7cOMaMGUPr1q1ZunQpQUFBJfZl++qrrxg/fjyxsbG0bNmSjz/+GKPRSGJiotP1hYWFZGVl2dyqEhqlgvrVggDXakBto5/uW2r6ov53kiQCDALfxGVLbcWKFQDExMQwceLEcm81rdFqtSQlJTFlyhTLMblcTnx8PNu3b3fpGnl5eeh0OqpXr+70/tmzZzNz5swblrUiaRQZTHJGHmfSc7m1cY1S11on38qLrCx3lJPOULzFzdcaxGg9gc/htk9txowZHlFoAOnp6RgMBqKiomyOR0VFkZKS4tI1Xn75ZerWrVvijIQpU6aQmZlpuZ0/f/6G5fY07kxsz7QKFFiSb93afhavzRdTqAQ+iNtKLTU1lREjRlC3bl2USiUKhcLmVpnMmTOHVatW8eOPPxIQ4NwXpdFoCAsLs7lVNdyZ2G6u/QwLVFrlqbmunAzGYkstTyuUmj+w9WQ6y7ec8Zs0HrcL2kePHk1ycjKvvvrqDbfzjoyMRKFQkJqaanM8NTWV2rVrl3ruvHnzmDNnjqUF0s1M8bg8F5SalaVmdvq7U1Ggs1Jq7ua3CW5Opq05yJn0XG5vFknzqFDAlMXwxi9HiKkRxIhuMd4V0MO4rdS2bNnC33//bWk/dCOo1Wo6d+5MYmIigwcPBrA4/SdMmFDiee+88w5vvvkmv//+O126dLlhObyNOVctOSMPncGIqoR5BQU6g8V/FhaoIldrstrcChRYbT+FpeYfmNtVWbeBv3Atn0+2nCFUo/Q5peb29jM6OtqjZmxCQgLLli3js88+48iRIzz99NPk5uYyZswYAEaOHGkTSHj77bd59dVXWb58OTExMaSkpJCSkkJOjvtTmaoKtcMCCFQp0BslLlzLL3FddtHWUyaDUI2yXF06bAIFwlLzC8zvudYqoKQr+nHL0ep9blvqtlJbuHAhkydP5uzZsx4RYOjQocybN4/p06cTGxvL3r17WbdunSV4kJyczOXLly3rP/zwQ7RaLQ888AB16tSx3ObNm+cRebyBXC6zDF4pLVhgTucI0SiRy2VWKR1uBAqsRhrmC0vNLzD7UQutrHSzGvPF1B63t59Dhw4lLy+PJk2aEBQU5NDO21kJVVlMmDChxO3mpk2bbP72lDKtajSuGcyRy1mlBgus/WmAxVLTGowYjBIKedn+TetAQb7OtR5ugpsbs1VmbalZW2cFOt9K7XFbqS1cuLACxBCYI6ClTWwvjnyalVqxoa3VGwlUl/3BtM1T861faIFzzD9k1krN6reNfJ2BiEqWqSJxW6mNGjWqIuTweywtiErp1lGceGt626yVWqHe4JJSsw0UCEvN15EkCb0TpWbtRvM1N0S5+qmdOnWKadOmMWzYMNLS0gD47bffOHTokEeF8ycsLYhKtdRst59KhRxl0ZbT1QRckdLhX+it3m+twdpS892AkdtKbfPmzbRr144dO3bwww8/WKKO+/btY8aMGR4X0F8wp3WkZReSU8K8Aks1QWCxH9PdBFyDqCjwK6x9qLbbT+sfN99yQ7it1CZPnswbb7zB+vXrUavVluO9e/fmn3/+8ahw/kR4oIrIkKJ5BSVYa9Zdb81YWnq7GMHSi4oCv0Jn9SNW0vbT1yx2t5XagQMHGDJkiMPxWrVqkZ6e7hGh/JXGZXTBte56a6a4/tO1D6Z1oMDXPswCR/QG59tP4VOzIiIiwiZvzMyePXtuyn5mVQnLEJYSLTVbnxqAxl1LTVQU+BXWlnlhCdtPX3NDuK3UHn74YV5++WVSUlKQyWQYjUa2bt3KxIkTGTlyZEXI6Dc0qll6Ybt9SgdY+dRc9ItYf8h97Rda4Ih1srXN9tNqjd8rtbfeeouWLVsSHR1NTk4OrVu35s4776R79+5MmzatImT0GxpbxuWVsP20WGpW20+Vufutax9Mm4oCH/swCxyx2X6WGCjwrc+B23lqarWaZcuW8eqrr3Lw4EFycnLo2LEjzZo1qwj5/ArrcXnOWm1bd7014+6YPL1BWGr+hG1KR/H7bV1R4GufA7eVmpkGDRrQoEEDT8ri9zSoHoxcBrlaA1eyC6kVZtsjzhz9DA90Fv10P1AgLDXfR+9C9NPXPgcuKbWEhARef/11goODSUhIKHXtggULPCKYP6JWyomuHsS5q3mcupLrqNRKsdRczTUSBe3+ha7E7WfxGr9Uanv27EGn01n+XxJiMtGN0zgymHNXTfMKujUpnldQoDNYPpQ2PjU3k29tAgU+9mEWOGJwoaKgwMd+3FxSahs3bnT6f4HnaRQZwsZjVxxaEJmDBHIZBKuL3za3k29FRYFfoSsp+unDllq5aj8FFUfjEtI6zFvP0AAVcqsWQ+4m31oHCkSemu9j/X4XltB6KN/HyqRcstTMg4xd4Ycffii3MALrtA5bpZaZXzxwxRpL91sXLTWdXS2gq33YBFWX/+27xE/7LvHu0FhCNLafj5Ly1Iz+XiZlPbg4LCyMxMREdu3aZbk/KSmJxMREwsPDK0xQf8E8hMU8r8CMfYcOM5buty7+2hqMtut8bevhb1zL1fLs13tYfziVDYdTHe4vsUwKP89TMw8yBtOczYceeoilS5daRuIZDAbGjx9fJcfP3WxEhWkIUivI0xpIzsijSZGSc1YiBcWWWkE5UjrAFAG1/3UX3Dx8v/uC5f9qpaON4oql5mtRcLd9asuXL2fixIk2Mz4VCgUJCQksX77co8L5IzKZzFIDat2tw3repzUaNy0160AB+N6vtL9xLU9b6v2uVBT4mrXutlLT6/UcPXrU4fjRo0cxGn3L4egtGjkplyruemu3/byBlA4QwYKbmfc2nOCTLWcsfzsbClVSk0jJh5Wa2/uOMWPGMHbsWE6dOkVcXBwAO3bsYM6cOZaxdoIbw+xXs46AluRTK679dL9MCnzvA+0vXM7M590Nx22OSThqNZf6qfnYD5vbSm3evHnUrl2b+fPnW1oQ1alTh5deeokXX3zR4wL6I86GsFgaRAba+9TctdRslZ+YU3BzonUx2l1y59viNb72w+a2UpPL5UyaNIlJkyaRlZUFIAIEHsZZrpqzDh3gWvLtuoMprDt4mdcHt3UIFAif2s2JszQcp9tPP/Sp3VDYSyizisHsU7uSXciRy1m0qhPmtO4TrPupOf9gJp3LYMLK3eiNEne1qe3EUvOtD7S/IHdSkuhszrp1RUFhie28jRiNkk1S981MuZTad999xzfffENycjJarW30Zffu3R4RzJ8JDVDRpWE1dp27xoNLt/PBIx3LTOlwZqmlZRfw9Je7bUakmX+5lXIZeqPkc+F8f8GZApOcmGr2tZ/mllb2awtdnBt7M+B29PP9999nzJgxREVFsWfPHuLi4qhRowanT5/m7rvvrggZ/ZKPR3Wha6Pq5BTqeezTnRxLzQYcLTVL8q2dUtMZjDzz1W7Ssgstx/TG4hmQoUXbWLH9vDkxGp2pNUfs3Q3mCKj96b60BXVbqS1ZsoSPPvqIRYsWoVarmTRpEuvXr+e5554jMzOzImT0SyKC1HwxtisPdK6PUSqObpZUJmWvnN785Qg7z14jVKOkRVQoYKomMOephRZZfGL7eXNidGKVOfep2f7Ymf1q9pFSv1ZqycnJdO/eHYDAwECys00WxIgRI/j66689K52fo1bKmftAeyb1bwGYOnTUDNHYrNE4sdR+3HOBT7edBWDB0Fga1ggCTJaa+ZfbrBx96cPsTxhctNTs8xLNSs3BUvOhHze3lVrt2rXJyMgATN1vzbM+z5w543RPL7gxZDIZ43s25bunuvHJ6FuoYafUAsw+tSLldPhSFlN+OADAs72b0rd1FEqFyQFsMEqWQEGoxmSp+dKH2VdxpsCc6TRneWr2eYnm7af9d9WX3BBuK7XevXvz008/AaZE3BdeeIG+ffsydOhQp/NAb1okCdY+Ayc2eFsSALrEVKdXi1oOx82WWoHeyPU8LU9+uYsCnZEezWvyn/jmACjkpjUGo2T5gph9asJSq9pM/HYft7y5get25VAubz+NJWw/7db6klJzO/r50UcfWcqhnnnmGWrUqMG2bdsYOHAgTz75pMcF9BoHv4c9X5pube+HfrMhNMrbUjlgTukwGCWe/XoP5zPyia4eyHsPx1pymZTyYkvNvP00+9SEpVZ1WbzxJN8lmQrWv0u6wON3NLbc57pSK2n7KXxqgKnu84033iAlJcVy7OGHH+b999/n2WefRa1We1xAr9G8P9z6DMjkJgW3+BbYtQKqWH2rOfkW4O8T6WiUcpY+2pmIoOL3wqzc9EbJKlBg+j3L86EPsy+x/8J15v5+zPK3fV6asy2pM+ePfaCgUPjUbFEqlbzzzjvo9X5QWqMJgf5vwbiNUCcWCjLh5//Airsh7Yi3pbOgVti+hXPub0eburZ97RQyK0vNaA4UmCw1X6v78xVOpNq2czf7Rc04s8qc+bRLSumwX+u3lhpAnz592Lx5c0XIUjWpGwuPJ0L/OaAOgfP/wNLbIXEW6PK9LR1yuYzAImttdPcYhnSs77BGUfSF0BuKLTVzuZVI6ahamPPPDHZKxxVLzRn264RPzQl33303kydP5sCBA3Tu3Jng4GCb+wcOHOgx4aoMCiXc+jS0uhd+fQmO/Qp/z4eDP8A970KTXl4Vb/LdLTl3NY/Jd7d0er/Ssv00WrYdIlBQ9dhyIp0nv9jFW/e1c1BG9rWeTn1qTq5ZUqDAwafmQz9ubiu18ePHA87ne8pkMgwG33lxHAivD8O+hiM/m5TbtTPwxWBoPxTuehNCanpFrFHdY0q93/yFsM5lMwcKfOkX+mZn1Ip/MRglnl+1l9cHt7W5zxWl5kyrOWw/Lcm3tvjS8BW3t59Go7HEm08rNGta3QPP7ICuTwEy2L8aPugCuz937uzwMmZLzVqBhYrtZ5XD2s9lsHPwK2T2Ss3J+U60msP20+CF6KckwYHv4LN7QV96p15PIEbklZeAMLj7bRiXCLXbQcF1+OlZ+HQAXDlW5umViTlPzVapFaV0CEutymCtZuwMLAdLzWn006mlVtL203ZdhVnsGWfgy/vh+7Fw5i/Y/VnFPI4VLm8/8/PzSUxM5J577gFgypQpFBYWF0srFApef/11AgICPC9lVaZeZxi3CXZ8CBvfgnNb4cPb4I4EuD0BVN5/PcwBUtvtZ5FPTVhqVQZrpWQ/9avcPrUStp/2GtDjnwODDrYtgs1vg74AFBro8RJ0GuXZx3GCy0rts88+45dffrEotQ8++IA2bdoQGBgImGYU1K1blxdeeKFiJK3KKJTQ/VloPQh+mQgnfje9mQe/NwUSGt3pXfHsLDXrKe/5OoOlHY2g6mCfNOug1Jy4wFxJvi2sjC4d5/+F/z0PaYdNfzfqYfoe1GjiuccoBZe3n1999RVPPPGEzbGVK1eyceNGNm7cyNy5c/nmm288LuBNRUQDeGQ1PPgZhNSGqydNfoQfn4bcq14TS2kXKFAq5JbeWdZVBoKqg8FQhlJz0XfravTTI9vP/Ovw8wvwyV0mhRZUA4Z8BCPXVppCAzeU2smTJ2nXrp3l74CAAOTy4tPj4uI4fPiwZ6W7GZHJoM1gmPAv3PI4IIN9K02BhL0rvRJIUBXtP3MLTUnTKqvcNhBbUG/x7a7zjP8qyalCsc9Tsw8U2N8PpRe0q4pyFSskT02STOlNi+Ng13JAgo6PwoRd0GGo6TtRibis1K5fv27jQ7ty5QoxMTGWv41Go839fk9AOAyYD2PXQ602kJ8Ba542WW7pJytVFHOboYxcU+RJIZehUsgsv/4iWOAdXvpuP78eSOHLf8453FdWnpqz6oHSCtqDitwN9paaWd+U+zNw7RysfAi+GwM5qVCjGYz+BQYthqDq5bvmDeKyUqtfvz4HDx4s8f79+/dTv75jNntZLF68mJiYGAICAujatSv//vtviWsPHTrE/fffT0xMDDKZjIULF7r9eJVO9C3w5GaInwnKQDj7N3zYDTa9DfrK+REwtwA3KzWVQo5MJiOoyFoTE6W8i/l9saYsn5rBmU/NybXNllpwkbtBW5R2ZVaA5s+A29a6QQdb34clt8KJP0Chhp5T4OmtEHO7e9fyMC4rtf/7v/9j+vTpFBQUONyXn5/PzJkzGTBggFsPvnr1ahISEpgxYwa7d++mQ4cO9OvXj7S0NKfr8/LyaNy4MXPmzKF27dpuPZZXUajg9v/A+O3QpA8YtLDpLVO51dmtFf7w5gHI14vmHJjrCAOKPujCUvMuztIzylNR4MxUM9f6mn2o9p1vgzTmgJEbybcXkuCjXrD+VdDlQcwd8PQ26DkZlJqyz69gXFZqU6dOJSMjgxYtWjB37lzWrl3L2rVreeedd2jRogXXrl1j6tSpbj34ggULGDduHGPGjKF169YsXbqUoKAgli9f7nT9Lbfcwty5c3n44YfRaFx78QoLC8nKyrK5eY3qjeDR7+H+TyC4FqQfh0//D9ZOgLyMCntYc/G6+TOvLPKFBqmdtwIXVC72Vhk4KjV7t5SzGQXOLDVzakiwxn77abo/2J3PQEGWqZLm4z6QegACq8GgJTDqfxDZrOzzKwmXUzqioqLYtm0bTz/9NJMnT7bs6WUyGX379mXJkiVERbneb0yr1ZKUlMSUKVMsx+RyOfHx8Wzfvt2Np1A6s2fPZubMmR673g0jk0G7B6BpH9gwE5JWwJ4v4Nhv0O8taP+Qxx2r9rNCzZZaoGX7KZSaN3HFUlPKbe0PZ4ECZ5i3n0GW7aetTy1Q7UK+oiTBkf/Bb5Mg2zTAnA7D4K43IDjSJTkqE7dqPxs1asS6devIyMjg5EmTs7tp06ZUr+6+QzA9PR2DweCgCKOiojh69Kjb1yuJKVOmkJCQYPk7KyuL6Ohoj12/3ARWg3sXQoeHTTk9V47Cj0+YIqUDFng0BB5uN4HKnOJh3pKI6Kd3sU+7cHbMfiSn0zKpUioKzIGCQrvoZ3BZLojr503W2fHfTH9Xb2LKOWvcw/n6KkC55n5Wr16duLg4T8tSIWg0Gpe3ql6hwa3w5N+w7X34ay6c3gRLupmyr7s/D8obb7wZajcr1JziYbbUhE/Nu7hiqdnjdPtZytzPIHufmmTvU7P7DBj08O9/4c83QZcLchXc/gLc8WKVqJIpDa/VfkZGRqJQKEhNTbU5npqaenMFATyBUg13TjQ5Wxv3BEMh/PkG/PcOOHfjW3G1Um6Tl2Z2OgeWI/K1O/ka6TkidceT2JcygQtKzcUyKZ0l+uncp2aOfmr1xuLHvLQHPu4Nv081KbQG3eCpLdD7lSqv0MCLSk2tVtO5c2cSExMtx4xGI4mJiXTr1s1bYnmXGk1gxBq4bxkERZq2pCv6m7an+ddu6NLW80KVZkvNzejnzrMZ3LdkG7e+lVj2YoHLOPOPOQse2JzjYkG7eRsbaOdTs6R0WE1lL8i5Dr9NhmW94fI+U67lve/D6F+hlvNefVURr3bpSEhIYNmyZXz22WccOXKEp59+mtzcXMaMGQPAyJEjbQIJWq2WvXv3snfvXrRaLRcvXmTv3r0W/55PIJOZggUTdkLHEaZjSZ/CB3Gm9i3lrEiw9qup5OULFGw5kQ44/8LZ98IXuE55tp8utlOzXCdYY7v9NFt65rSevvJdBCzrbmrMIBmh3UOmioDOo0B+czXzKZdPzVMMHTqUK1euMH36dFJSUoiNjWXdunWW4EFycrJNKdalS5fo2LGj5e958+Yxb948evTowaZNmypb/IolqDoM+sAUZfr5P6b0j+/Hmkqt7lkA1WLculyYlV/NHP10N6XDPI4PTF+O/24+xZ3Na/LZ9rP8cSiVzS/1dJhLKigbpz8SZVlqTisKnOSpWaKf9oEC0/Fquiss07xLX9lOyMb0uRqwwBSdv0nxqlIDmDBhAhMmTHB6n72iiomJ8b+ByTG3mfwZW98zBRJOJcLiW6Hny9Btgimx1wXCrCw1c3qA+VfaVUtNoyzeqizfeob5648zf/1xy7HVu84zvmdTftp3iaWbTrFkeCdiIoOdXUpghX3xOjgPBNjc73JKhzn6aWupIRkYrVjHc0e/RSPLRycpyO78NNXvngaqQDekr3rcXHalv6LUQI9J8PR2U/a2Ph82vAb/7QHnd7p0Cevtp32emqs+NfOMUYCDFzMd7jd/z577eg+HLxdPiheUTnkstbKUnv11bAIFl/cxbP9jvKb6HI0xn/2yFgzQvsXFTpNueoUGQqndXEQ2NWVvD/4QAqtD2iH4pC/8nGAa4VcK1gm4DhUFLltqxR8X+46q4Lj98eea0rV7L/L7oZSyF+LYENJ0rCxLzfFYaf3UgjQKgijgsbxP4KNe1Mk9QpYUxM8NJvFC8ByOS9E+k9ojlNrNhkwGsY+YnLixwwEJdn0CH9wCh34sMZBgvf1UlbOiQG2l1JydY/9Fk9tnjPoJqVkFPL9qL09+keSSRaU3SpzPyHM4VhrOhxnbHpMkybIuOv0v/tBMYqhuDUgGDlePp0/hPPbUGkKA2rdauwuldrMSXAMGF9XdVW9iavvy7WhYOdTUDsYO60CBJU9NXULiZQlY+9SyCxytMHt9at8DzF9IzXJs+lAaf59I5453NtocK49Pzf6Q3ihRi2ssVi2kw19PUl+WziVqwvDvWNv0Da4QgVxWvnzFqoxQajc7je40Je32eNmU9X3id1M7mK3vm7LCi7BJ6bCvKCjH9jO7QOdwv72l4K+WWm5h8etZ3rCWfZmU/XXKTL41GpD+/ZgNmokMUPyLJFOwVH8vQ5gPzfpa9VOTWXLYfKWxgVBqvoAqAHpNNfWyanibqR3M+ldhWU9Tmxjskm8tlprp7S/PtiOn0NFSc9h++qdOI193477E8vjULKQchE/uQv37S4TJ8tlrbML5B39jjn4YmQZT2Z1ZJ8pkxRa42H4Kqh41W8Con2HgBxAQASkHTG1ifp1ENXnxlshSUaAqe/u57VQ6x1KyAVtLLMfJ9tN+/2PfA8xfsLHUypmCVB6fmkKfD+unw3/vhIu7kNQhvKobzX3amajqtQccy6TkVpaa2H4KqiZyOXQaYQoktB8KSPDvf+n0S3/6yXcCkstdOg5ezGT4xzsY9/kuwFZn5To5x/5rJvdTn5onlENZPjV7ZdlDvo+hOx805TNKBmg1kKtjtvKF4S4kmZyAImvMKJly1yzbTyBQVX6LvSoilJqvElIT7vsIRvwI1Rqhyk3hv+p3WaZaQHWDqbNwUBm1n6t2JiNJkJyRh95gLLNCy97P47eWmgdSWZz9aFhjzqipyXUWqd7nM/XbhBVcgvBoGLYKhn6BNtBUmaOSy20i11qrdBy5rHgIj/CpCW4OmvSG8dsp6JaATlLQV5HEM4cege1LCFSYlJCzfLICnYG1ey9Z/r6W5xgYsEdEP01Yp7uUN1BwMi2n1PuNRgOPKBJJ1EzkXsU/GCQZe+sPh/H/QIu7geItqkIus1Vq+mJLTS6zausutp+CmwZVIKq+0/k/7Wx2GZujNubD71Oo8+0A2spOU6AzOmx31h1MsUnbyMjVlvkFtb/fXwckV3jScephHjowjrdUnxAmy2OfsTEDtW+wpfELoAmxLDMnSCsVMpRymSVwY63UsLLUfGX76fXaT0HloJDLSNHE8GDBdD5qfYi+l5agTN3PWvUBPjX0pzDvdgJDwi3rV+88b3P+1ZzCMp3ejttPz8l/M2EbKPDghXX5pvrfre9Rz6gnRwpgnv4hPjfchRE5d9stNwcbzNPD1Eo5BTojhXqjVaAAq+2nb3Ra8dOPnX8SFqBCQs6RevfDMzsxtrkfhUxirPI31P/tBkd/BeDc1Vy2n76KTAYNawQBcNUFS81+gb/61CpiGxd26W9T/uHf88Go53i1HvQtnMunhv4Yi77GDsm3huLtJ4C66FdGa+UflYs8NcHNjLlUSqmQQWgU8geX87hhMsnGmiiyL8KqYbD6UX7buhuAO5rVpHWdMKBo+1mGVisp+plbqKfvgs289tMhjz6fqkqeztqndmOmWg0yeVe1mBZ/jIRrZyG0Lgz9iu+bvc1latistX8kcwKvuX+eWlncqUOyin4G+Nj2Uyg1PyK8KAFXZdWjbpeqM3dp3yGj43iQKeDI/xix+0FGKX5naOe6VA82JWtedTJw1x57v5xZqX2/+wIn0nL4dNtZDz2Tqk2+B3xqMow8pNhIomYiQxRbkWRy6Po0TPgXWt3jUudbcy81c16iuSLEJlAgl4kyKcHNi7n+09x6CEw96gvQcKHzy/DkX2TWiCWYfGaqPuPuHSNoyVkAMnILKSuWZ3+veduTlV965NSUD/cPBy6U3mmkKuBKMq21T82aveevk5JZUGYOWhPZRVap3+Ad1TIiZLkcNMZweMCPcPcc0ITyv32X+HjLGUfZ7N4Bs+JTWiw1x+2nTCYsNcFNzINdounUIIKeLWpZjtk0iqzdlikR85imG0OBPBj5pd0M3z+KKcqvyMnOLHv76VAmZfoy5ZTwJTcz/OMdbD15lcFLKn5a/Y0wY+1Bus3+k+t5Jqu1pFImm+1n0ZKjKVkMXryVW2cn0nPeJqf+Kw1aXlB+y2/qyXSVHyVP0vC6bjiDtK+TF9nesu7Zr/c4fVxHn1px9BOsfGo2gYLy56ldz9NWyaatQqn5EX1bR/HD+NtoZNWN1jqcfzWnkPVHr/CloS8Xhm+C1oORSwaeVP7C1LNjiLy8udTrlxT9zHVSJ2pNZpElV1a9ozcxGiU+236OlKwCVv6bzIVreXSc9Qdv/XrEYW2ek+e7/3yxFZqckcemY1ds7u8mP8Rv6sk8r/wRtczABkNH+ha+wyeGARhQsOlYWpky2r96OoulZnoj1FbbT2ufWnlqgD/ffpbYWesZ/vGOMnPqKhuh1Pwc60aRP+65iM4g0aF+OE2bNIeHPuNwz4+5IEVSy5DGLdue5APVe9TkWonXs1ZMlkBBKT6m9YdTS7yvKjFqxb+W/0sSzP/jOFkFej7667TDWme95oI0Cpu/v/zH1B6qGlnMUy3la/WbNJankCpF8JT2Pzyum8hFalrWL954ikvX892S2dx80mKpFSm1Qr3RogDlMlnx9tNFn1pmno55vx8DYNupq9z93l/M/+NYlYmeCqXm55g/0LuTr/HGLyar48EuxRPs5S370bfwHT6XDcQoU3CPYgeJmok8qliPDLv2OJJk0zLH3Hooz2r7uXTzKYtPacuJdEtdqZkz6bkefHae4++iSVpgep6lzT61Tr41ShK7zmY4bA2Pp2TxgGIziZqJPKD4C6Mk4zN9X+IL57HOGIfJhrLl7NUyXhu7B7EECpykdBS3HnK/rfuHm0+RVaCnWa0QerWoic4gsejPk9z17l8uWZQVjVBqfo7ZUlv2d7HjeWBsXcv/qwerySeAGQUP81fPb9lrbEyYLJ83VCv4Xv0aLWTJlrUymcxmMK/CiaU257ej/HzgMgD7Llx3kGf62oOeeWIViFGCqznOo8GSJNlUYny76wIPLN1u4wdrLLvEEsMM5qn+S3VZDkeMDbhf+xoz9GPIJqjEx83KL30bbz9hyhIoUDhuP42WQIF7eWopmQWs2Gr6rEz5v5YsH30LSx/tRO2wAJIz8hi9Yifjv0oiJdO9RpmeRCg1P8d6cjvAqG4NbbrkVgsq7r+VrG7CfdpZzNCNIlsKpJP8JD+rX+Fl5dcEUIhcJrNpmWOOftr3Xlv212ne/OWw0y4eZfnfqgJGSeJqrnNL7cK1fJvX4IfdFyz/V6PjOcUP/KaeTBfpEPmSmrd0w7hX+wZ7pGZlPm6hvnSlYz/p3VImZR/9tPKpWVcU6AyS09kT1ryXeIJCvZFbYqrRq0UtZDIZ/dvWYcOLPXj89kYo5DJ+PZBCn/mbWL7ljFfmwQql5ueYW3qbmTqglc3fKoXc0jU3PUeLETmfGUxb0nWGW1DJDDyt/B9/qCfRJOsfmw+xWWfl2UU/D1zMZNnfZ9hwxNGfpr0JhiJLkikZ2Rlf/GPbSt2s3+JkR/hVPYUE1XdoZHq2EEtf7Tt8ZLgXvYvVijono/RKu19vKMlSM9hUFARY/bCVZq2dvpLDN7tM5XOT+re0qe0N0SiZdk9r/jfhdjo2iCBXa2DWz4cZ+MFW9iSX7IOtCIRS83OsLbXIEI3NHAIzNYoScK39SCnU4CndC4zTJnBJqk4D+RWGn3iB4P89SSS2+WYlBQrMqRHWFN4E9YcStgrk8c92IkkSX+045xA4CDZm8bbyI77RvE5T+SWuSOFM0D7LE8bJXJBq4Q6ZZeT72VtZ5u2nuaJAU6TcMvJ0lh8UU+dbueUHqLT6z/nrj2MwSvRpWYtbYqo7XdO6bhjfP9Wdt4a0IzxQxeHLWdz34TZe+fEAmS50evEEQqn5OeZwPkCd8ACna8xVBSt3JDvct97Yhb6Fc/lEfzdG5AQc+5FEzYs8rPgTioIGJW0pT11xdHwX6o0s3niSu97d7FTpeQN7ZWGfm7XhSBrfJl3glR+t/YESg+VbWHr9KYYqNwHwlb4PfQrn8bOxG/py6O7MMl4Pezl1Ra+/wm77+X7RFhJMPjWZC7lqBy5k8sv+y8hkMLFfi1LlkMtlPNK1AYkv9uC+TvWQJPhqRzJ9FmyqlECC6NLh5wRZbT8jQ9RO15iVWknkEsjr+hFcaTyIAWfn0E5+ljmqjzl3/F9I+6jMhofWaPVG5halC0z8dj/j7mhE18Y1yjirYrGPCjoberLjdIbl/w1lKbyhXM4dioMgwTFjfaboHme31NyyRlsOrXa9DEvNvHVPyypg0vf7LblwKrvtpzXmngMBKgV5WkOJEdB3fj8KwODYerQqqgcui8gQDQseiuWhLtFMW3OQM+m51Ap1/sPpSYRS83Os/SkBKsetJ0CNEI1L1zqras5g7euMUvzBi8pvaJi7H5bewbMM4AMGU0jpyhFsfWobjqSy4Ugqf73UiwY1So4KVjT2+VuLN55yWKNRyVGh5wnFzzyn/BGNTEeBpOLrwGG8dT0enQe+apn5ulKVodnRH/dWos1x+4oCa2TYzoB1lqu27WQ6f59IR6WQ8UJ8c4f7y+LWxjX49bk72Hk2g9Z1XVOIN4LYfvo55pQOsB2BZ02NMiw1MwqFDAMKlhvupm/hXI6E3QZGHc8q17BO/TK3yQ+UeY1CJ5bCibRslx6/onAlKbVp/gF+UU/hJdU3aGQ6/jK04y7tO3ytecAjCg3gep6OKT/YvoYJfZvTtp5JUaRlFbDzbIbDeeYtZemWmvOqAkmSeLvIcn4krkG5f1zUSjm3NY0s17nuIpSan2MdKHAWJICyt59mlFb90y4RyecN51B432ekSNVoJE/lK/VsFqiWUJ2sEq/hbKtaVtSvoiktKTWMHN5Sfsxjx5+mufwi6VIYz2mfYaRuMslSFMdTPVdCdCY9l++tUkTAtMV7+/72yGWw40wG09Y45vlduGaqRLjsJHfMnFYTWMK8it8PpbLv/HWC1Aom9C477aQqIJSan2O95dSoSrDUSvC12WM/k0ACMhv1J75wLp/q78IoybhPsYVEzUQeVGzC1Q7+ZeVOVTTOlZrEQPk2EjUTeUT5JwBf63vRp3AePxlvw1lFwI2SnJHncEwhhzZ1w5nUvyUAp4uCL8/2bsqXY7tSLyKQ5/qYlFFtZ4GgIjEtgQKrHxW9wcjcIl/a47c3omaoa24IbyN8an6O9fazJJ+aq5aaPZIE2QV6cgjiNf1ofjTczmzVJ7SWn2Ou6iMeUPzFVN1YTkn1Sr2O/bTyyqbAznqMlqXyhnIFPRT7AThhrMeBjjOZsqNy/H4v9m3O/PXHgeI5EE/e2Zi9yddZdygFlULGkz2aEKJRsnVyb8t5T/VogqKogsAcjDFbas7aD/2w+yKnruRSLUjF43c2rpTn5gmEpebnBLrgU3NVqdkP4DXalQztk5pyr/YN3tQ9Qp6koav8KL+pJ/OC8js0lJyuoNNXje2nEj1PKX7iD/XL9FDsp1BSMV/3AAO0b5Feo1OlyfPQLcW1uZZuGzIZcx9sz8AOdXm5f0tCNI72Snigion9WtAiKtRyTG5nqZmfa4HOwLsbTIrzmV5NbapMqjrCUvNzbH1qJQUKXNt22CeHGiXILrA9ZkDBMsM9/GbsyizlCnor9vK88gfukW9nmv4xthvbOFxXVwGWmtEoMWrFv9QIVrPw4Y429128ns+5q7l0b2JybOfrDHSSHedN1Se0kpsy6rca2jBN/xhnpDpAybWgpaGQy9xutzQ4ti4RQcUKptAqGhoaoOL9YR2dnWZDSEDx197sMbAfbP3lP+e4nFlAnfAAHr21oVsyehthqfk5tpbajW0/t51Kt/lbkiRyCpwn3l6QavKY7iXGa58jTYqgifwyX6vfZJ5qKdXsAgmvrjnIsRTPRkBPXcnh7xPprNl7yUGx3DbnTx5ZtoN/z2RA/nVaJr3Gd+qZtJKfJ0MKIUH7FMN1Uy0KDeD8NUd/V1k4S7Ewo1HKmf9gB5rUNPW+e31QGx7sXJ+ZA9vanFeedj+hVkrNEiiwSr7NKtCxeONJAF6Ib16iW6KqIiw1PyfQhUCBs1QAZ9hHKSWw2X46IuNX461sKWzHS8rVDFck8oDiL3rLd/Om7lG+N94ByDBK0G/hX2x5uRf1q3nGb/W/fcWDmrV6I4FqBZcz863mKEikbv8aLr1H05xUkMG3+jt5S/8I13DMtTrtpDqiNNrWC+NUmvNz3r6/HQ91iUYmkzG4Yz1ytfoSt3/lGWsXqim+lsyJT+3jv05zLU9Hk5rB3NepdH9nVURYan6OdaCgtJF2ynKMuzNKElkFZdf7ZRHMq/rHeEA7g6PGaKrLcpivXspK1Zs0kl22rPv1wOVSruI6OYV63v/zpOVvs7Xz4NLt/HfzaerLrrBcNZd7j0+FnFSuBzXkYe00XtI/5VShARx105KUIXM6aUqjlDP0lgYWZaOQy0r1Z5XHUrPefpp9cmalduFavmX+wUv9WliK4W8mbj6JBR7FemtRmnvH2o/jKhm5Wnaddb1Dw26pOfdo32SO7mHyJTXdFYdZp57Ms4ofUKNzWux+Mi3HrS92vtZA+9d+tzlWoDfwxfazXL6WwzjFz/yhnkRvxV70MhX0mMyXsSv5x9ja5cdwBeueZta4++NRWI5yq2CrLrzmLr1mi/1/+y6RpzXQoX44/drUdvvaVQGh1Pwc6+BAaUM0zO2H3OHvE+msO5Ti1jl6lCw1DOQu7dv8ZWiHRqbjRdV3/Kqewq6/f2HEJzsszuwtJ9KJX7CZ+5Zsc+na+VoDC9Yfc1Am9y3Zxnc/reV/6mm8olpJkKyQf4ytmN3wY+g1hRyD57w0Zsf8uYxcS8nTCCtHvLMec6VRVo81Z1j7Ts3NBsyNDcyvzct2rYVuJoRS83OsP7iljW4rj1K7Ec5LUYzUTeZZ7QSuSGE0lV/iM2Yw4MxbDJn/M19sP8vavRcBOHy55AoFa6b8sN+mwy9ACHk8kftfflTPoLX8HNekEF7SPcHD2ml8clTF/gvXPdp7P0BpdsibFFqnBhG8Prit5X5X9ciTdzYmLEDJk3c2uSF5zOP8rH2rdzSLpHsllTRVBCJQILBQWjVSRFD5EnBvDBn/M3Znc2F7Jiu/5hHlRh5WbiK+YDev/+9RQroMc+tqa/ZesvpLop98JzNVn1FbZtoif2+4nTd1j5Jh5Tcb+MFW7mlfB1c4Mqs/+ToD1YPVLFh/nL9PXGFP8nWbNQEquU2C6y2NbPuSyV3cfk75v1ZM6t+yVD+oK5jnKVi7ISb1a3lD1/Q2wlITWKhfLbDE+8rjU7PG1aJ4e/q3qU0WIUzVj+OBwukcN9YjUpbFe+ol9NszngYy96dR1SWdZaoF/Fe9kNqya5wxRjFcO4UXdeNtFJqZn/eXHaB4tndTAtUKS/pLQt/mLHKSM2YfIY6za7bozvbzRhUaFFvqHaIjUMhlDItrQLv64Td8XW8ilJqAL8bGMfGu5tzVOqrENRGBN2ap9WrpXpdXMzVDNUwrajG+S2rJAO1s5uoeolBScafiAH+oJzFesYZmk9dyObPkEXLZBToUGHhM8RvrNS/RV5GEVlKwSD+Y/tq32WpsVy75zCT0dWzJExXmWGtpPa9BJoMuDU1Kzfyj0a2SesfNGtSGDtERPFFU/tQ8KpQDr93FW0PalnFm1UdsPwXc0awmdzSrWeqaG7XUqpXz/GpBKsKs/Hk6lCw2DOZn4628qfyE2xWHmKT6hkGKbUxacJYPpz7jtERozvJVrFHPo538LAA7jc2ZqnucE1L9csllTe+WtZw61VVlpEO0iAolvOh1+emZ21m79yIju8XcsDyuMLJbjMNjBal9Qx1UCUtt8eLFxMTEEBAQQNeuXfn3339LXf/tt9/SsmVLAgICaNeuHb/++mslSeq/3GgSZnl9cmGBKqdBinNSbR7VTeU/2vFclUJpIb/AF7LpbHhnGI8sWsf5oo4W+dnXYd0UZqU+Szv5WTKlICbrHuch7fQbVmhKualp4rsPxZbrfOs+/w1qBPFsn2YWJScoP15XaqtXryYhIYEZM2awe/duOnToQL9+/UhLc97LfNu2bQwbNoyxY8eyZ88eBg8ezODBgzl4sOrPi7yZqV8tqExr7bamJW+dwgLKZwWYhsGU9DGVscZ4O30K57Fa3xOAwYY/eC/9Cd6Z9yZrVi0jY14n+GcJCpnEWkN3+hTOZ5WhN5IHPvrvPNCe5+NdV0R3t63N0kc7W/62DxIIPINMKi05qRLo2rUrt9xyCx988AEARqOR6Ohonn32WSZPnuywfujQoeTm5vLzzz9bjt16663ExsaydOlSh/WFhYUUFhZPQcrKyiI6OprMzEzCwiq+tbAv8eDSbex0kky7ZHgnaoZqaFs3nFbT1zk9d/NLPekxd5Nbjzfujkb8J745aqWcZq/8Vub6rrIjvKX6mCZyW8d+srEm0/SP8Zexg1uPb0+IRmnxibWICmXdf+4oM5drwPt/c+hSFnGNqvPNk92QJIl2r/1BnlbP9il9nPrd/J2srCzCw8PL/R31qlLTarUEBQXx3XffMXjwYMvxUaNGcf36ddauXetwToMGDUhISOA///mP5diMGTNYs2YN+/btc1j/2muvMXPmTIfjQqm5T/LVPO6cuxEwFWN/9lgc3ZrYWmcXruXxwZ8neaRrA/ZfyMRglFDIZTx6a0PytHq0eiMrtp7lvcQTAEwb0IoR3RqyN/k6/57J4KO/ThPbIIKPR3WxSRI1GCXGfb6LP4+WPo1IjY6nFT8xXrkWORLLDAN4Xz+EAkruNBKiUXJvh7qcupLDv2cy6N2yFgcvZpKWXcj/tavNrwdS6N+mNkuGd+LAxUzOXs1lUKxr2/G0rAK+2pHMsLgGliaN206mU6A30LtlyYEZf+amVmqXLl2iXr16bNu2jW7dulmOT5o0ic2bN7Njxw6Hc9RqNZ999hnDhhXnKC1ZsoSZM2eSmuoY3heWmuc5k55LnfCAG+rekFWgI6dAT90I2zQSrd6ISiEr1QKSJAlJMhVfq5VyVAo5RqOpkvLI5Syu5+lIu3QaNRK339KRiCA113K1pGYXUC8i0DLAVy7jps2a92VuVKn5RrijFDQaDRrNzdGG+GahUWTwDV8jLEDltFDblY4gplmVEGwV5TQnrbatV5Rj1cw2I75asJpq5cyVE9xceDVQEBkZiUKhcLCwUlNTqV3beTFt7dq13VovEAj8C68qNbVaTefOnUlMLJ5TaDQaSUxMtNmOWtOtWzeb9QDr168vcb1AIPAvvL79TEhIYNSoUXTp0oW4uDgWLlxIbm4uY8aMAWDkyJHUq1eP2bNnA/D888/To0cP5s+fz4ABA1i1ahW7du3io48+8ubTEAgEVQSvK7WhQ4dy5coVpk+fTkpKCrGxsaxbt46oKFNkKDk5Gbm82KDs3r07K1euZNq0aUydOpVmzZqxZs0a2ra9+cs7BALBjeP1PLXK5kYjKwKBoGK50e+o1ysKBAKBwJMIpSYQCHwKr/vUKhvzbjsry7VuqQKBoHIxfzfL6xnzO6WWnW2a+hMdHV3GSoFA4E2ys7MJD3e/YaXfBQqMRiOXLl0iNDS0QktkzOVY58+fvykDEkJ+7+LP8kuSRHZ2NnXr1rXJfHAVv7PU5HI59evfeGNAVwkLC7spP5RmhPzexV/lL4+FZkYECgQCgU8hlJpAIPAphFKrIDQaDTNmzLhpO4QI+b2LkL/8+F2gQCAQ+DbCUhMIBD6FUGoCgcCnEEpNIBD4FEKpCQQCn0IoNRfJyMhg+PDhhIWFERERwdixY8nJySn1nIKCAp555hlq1KhBSEgI999/v0MrclO/fdvbqlWrbNZs2rSJTp06odFoaNq0KZ9++mmVkH/fvn0MGzaM6OhoAgMDadWqFe+9956D7M6eY0pKSqmP7ekB15IkMX36dOrUqUNgYCDx8fGcOHHihl+jypBfp9Px8ssv065dO4KDg6lbty4jR47k0qVLNteIiYlxeJ3nzJlTLvk9/RwARo8e7SBf//79bdZ45D2QBC7Rv39/qUOHDtI///wj/f3331LTpk2lYcOGlXrOU089JUVHR0uJiYnSrl27pFtvvVXq3r27zRpAWrFihXT58mXLLT8/33L/6dOnpaCgICkhIUE6fPiwtGjRIkmhUEjr1q3zuvyffPKJ9Nxzz0mbNm2STp06JX3xxRdSYGCgtGjRIsuajRs3SoB07Ngxm+doMBhKfNxVq1ZJarVaWr58uXTo0CFp3LhxUkREhJSamup0/datWyWFQiG988470uHDh6Vp06ZJKpVKOnDggGXNnDlzpPDwcGnNmjXSvn37pIEDB0qNGjWyea3L8xpVhvzXr1+X4uPjpdWrV0tHjx6Vtm/fLsXFxUmdO3e2uU7Dhg2lWbNm2bzOOTk5bstfEc9BkiRp1KhRUv/+/W3ky8jIsLmOJ94DodRc4PDhwxIg7dy503Lst99+k2QymXTx4kWn51y/fl1SqVTSt99+azl25MgRCZC2b99uOQZIP/74Y4mPPWnSJKlNmzY2x4YOHSr169evSshvz/jx46VevXpZ/jYrtWvXrrksb1xcnPTMM89Y/jYYDFLdunWl2bNnO13/0EMPSQMGDLA51rVrV+nJJ5+UJEmSjEajVLt2bWnu3Lk2z0+j0Uhff/21JEnle40qS35n/PvvvxIgnTt3znKsYcOG0rvvvuuWrCVREc9h1KhR0qBBg0p8TE+9B2L76QLbt28nIiKCLl26WI7Fx8cjl8udziYFSEpKQqfTER8fbznWsmVLGjRowPbt223WPvPMM0RGRhIXF8fy5cttWq5s377d5hoA/fr1c7iGN+W3JjMzk+rVqzscj42NpU6dOvTt25etW7eWeL5WqyUpKcnmceVyOfHx8SU+blmv0ZkzZ0hJSbFZEx4eTteuXS1ryvMaVZb8zsjMzEQmkxEREWFzfM6cOdSoUYOOHTsyd+5c9Hq9y7JXxnPYtGkTtWrVokWLFjz99NNcvXrV5hqeeA/8rqC9PKSkpFCrVi2bY0qlkurVq5foG0pJSUGtVjt86KKiomzOmTVrFr179yYoKIg//viD8ePHk5OTw3PPPWe5jnleg/U1srKyyM/PJzDQdhhwZctvzbZt21i9ejW//PKL5VidOnVYunQpXbp0obCwkI8//piePXuyY8cOOnXq5HCN9PR0DAaD0+d89OjREmV1tt4sp/nfsta4+xo5oyLkt6egoICXX36ZYcOG2RSLP/fcc3Tq1Inq1auzbds2pkyZwuXLl1mwYIHL8lfkc+jfvz/33XcfjRo14tSpU0ydOpW7776b7du3o1AoPPYe+LVSmzx5Mm+//Xapa44cOVKhMrz66quW/3fs2JHc3Fzmzp1rUWplnTt//vxS11S0/GYOHjzIoEGDmDFjBnfddZfleIsWLWjRooXl7+7du3Pq1Cneffddvvjii0qRzZfQ6XQ89NBDSJLEhx9+aHNfQkKC5f/t27dHrVbz5JNPMnv27CpRbvXwww9b/t+uXTvat29PkyZN2LRpE3369PHY4/i1UnvxxRcZPXp0qWsaN25M7dq1SUtLszmu1+vJyMgodeiyVqvl+vXrNtZOWYOXu3btyuuvv05hYSEajabE4c1hYWG8/PLLPP74416X//Dhw/Tp04cnnniCadOmlSoPQFxcHFu2bHF6X0UMuDb/m5qaSp06dWzWxMbGWta4+xpVlvxmzArt3Llz/Pnnn2W29OnatSt6vZ6zZ8/a/LB48zlY07hxYyIjIzl58iR9+vTx2HsgAgUuYHZg7tq1y3Ls999/d8nR/t1331mOHT16tExH+xtvvCFVq1bN8vekSZOktm3b2qwZNmxYuQIFFSH/wYMHpVq1akkvvfSSy/LEx8dLQ4YMKfH+uLg4acKECZa/DQaDVK9evVKd1Pfcc4/NsW7dujkECubNm2e5PzMz02mgwJ3XqLLklyRJ0mq10uDBg6U2bdpIaWlpLsnx5ZdfSnK53CHC6K3nYM/58+clmUwmrV27VpIkz70HQqm5SP/+/aWOHTtKO3bskLZs2SI1a9bMJtR84cIFqUWLFtKOHTssx5566impQYMG0p9//int2rVL6tatm9StWzfL/T/99JO0bNky6cCBA9KJEyekJUuWSEFBQdL06dMta8wpHS+99JJ05MgRafHixeVO6fC0/AcOHJBq1qwpPfroozZheusv3bvvviutWbNGOnHihHTgwAHp+eefl+RyubRhw4YSZV21apWk0WikTz/9VDp8+LD0xBNPSBEREVJKSookSZI0YsQIafLkyZb1W7dulZRKpTRv3jzpyJEj0owZM5ymdEREREhr166V9u/fLw0aNMhpSkdpr5GreFp+rVYrDRw4UKpfv760d+9em9e6sLBQkiRJ2rZtm/Tuu+9Ke/fulU6dOiV9+eWXUs2aNaWRI0e6LX9FPIfs7Gxp4sSJ0vbt26UzZ85IGzZskDp16iQ1a9ZMKigosFzHE++BUGoucvXqVWnYsGFSSEiIFBYWJo0ZM0bKzs623H/mzBkJkDZu3Gg5lp+fL40fP16qVq2aFBQUJA0ZMkS6fPmy5f7ffvtNio2NlUJCQqTg4GCpQ4cO0tKlSx1yuDZu3CjFxsZKarVaaty4sbRixYoqIf+MGTMkwOHWsGFDy5q3335batKkiRQQECBVr15d6tmzp/Tnn3+WKe+iRYukBg0aSGq1WoqLi5P++ecfy309evSQRo0aZbP+m2++kZo3by6p1WqpTZs20i+//GJzv9FolF599VUpKipK0mg0Up8+faRjx4659Rq5gyflN783zm7m9yspKUnq2rWrFB4eLgUEBEitWrWS3nrrLRuF4c3nkJeXJ911111SzZo1JZVKJTVs2FAaN26cRUma8cR7IFoPCQQCn0LkqQkEAp9CKDWBQOBTCKUmEAh8CqHUBAKBTyGUmkAg8CmEUhMIBD6FUGoCgcCnEEpNIBD4FEKpCQQCn0IoNUGVwFn/emc97AWCsvDr1kOCqkX//v1ZsWKFzbGS+oDpdDpUKpXNMa1Wi1qtdvtxy3ueoGoiLDVBlcHcP876Vq1aNcA0devDDz9k4MCBBAcH8+abb/Laa68RGxvLxx9/TKNGjQgICAAgOTmZQYMGERISQlhYGA899JBNr6+SzhP4BkKpCW4aXnvtNYYMGcKBAwd47LHHADh58iTff/89P/zwA3v37sVoNDJo0CAyMjLYvHkz69ev5/Tp0wwdOtTmWvbnCXwHsf0UVBl+/vlnQkJCbI5NnTqVqVOnAvDII48wZswYm/u1Wi2ff/45NWvWBGD9+vUcOHCAM2fOEB0dDcDnn39OmzZt2LlzJ7fccovT8wS+g1BqgipDr169HPruW0+msp4yZKZhw4Y2iunIkSNER0dbFBpA69atiYiI4MiRIxalZn+ewHcQSk1QZQgODqZp06al3u/KMVcfS+CbCJ+awKdo1aoV58+f5/z585Zjhw8f5vr167Ru3dqLkgkqC2GpCaoMhYWFDvMdlUolkZGRLl8jPj6edu3aMXz4cBYuXIher2f8+PH06NHD6fZV4HsIS01QZVi3bh116tSxud1+++1uXUMmk7F27VqqVavGnXfeSXx8PI0bN2b16tUVJLWgqiFmFAgEAp9CWGoCgcCnEEpNIBD4FEKpCQQCn0IoNYFA4FMIpSYQCHwKodQEAoFPIZSaQCDwKYRSEwgEPoVQagKBwKcQSk0gEPgUQqkJBAKf4v8B0zdF2Wzkb30AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021173, + "end_time": "2024-02-29T18:11:43.300089", + "exception": false, + "start_time": "2024-02-29T18:11:43.278916", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1264.119122, + "end_time": "2024-02-29T18:11:46.042253", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "output_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/insurance/tvae/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tvae" + }, + "start_time": "2024-02-29T17:50:41.923131", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tvae/model.pt b/insurance/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..d532271fbaee243484595ca7c317aec7fec279bc --- /dev/null +++ b/insurance/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eada8e849eb7c6638d89cc73f312038c150117726abd4743d18461204c5e8d3 +size 38609591 diff --git a/insurance/tvae/params.json b/insurance/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..ad830093a24a9a9581e14d1f6982746d792bf68c --- /dev/null +++ b/insurance/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.77, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.75, "loss_balancer_r": 0.95, "fixed_role_model": "tvae", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": false, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "relu6", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "rrelu", "head_activation_final": "softsign", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/lct_gan/eval.csv b/treatment/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..1b6986248e08ebadba2c80adaa2161e2152f2012 --- /dev/null +++ b/treatment/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0,4.3308387261260425e-08,0.0023591548839308565,4.458594799041748,0.011746696196496487,0.16063837707042694,0.01545888464897871,4.918354079563869e-06,2.3799874782562256,0.03671034052968025,0.06946055591106415,0.048571132123470306,0.07008553296327591,0.010716128163039684,6.838582277297974 diff --git a/treatment/lct_gan/history.csv b/treatment/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..000af0aa609fd8126f66e6836def940faaff436f --- /dev/null +++ b/treatment/lct_gan/history.csv @@ -0,0 +1,29 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.22352998348069378,80.13195664776967,0.09520609854183135,0.0,0.0,0.0,0.0,0.0,0.22352998348069378,320,80,103.12237620353699,1.2890297025442123,0.3222574256360531,0.0988283173581749,0.01245800144970417,1.6089594815347597,0.00019736949793127678,0.0,0.0,0.0,0.0,0.0,0.01245800144970417,80,20,20.080093383789062,1.004004669189453,0.25100116729736327,0.07783476307522505 +1,0.008753520530444803,0.34713386677235575,0.00011480308697381734,0.0,0.0,0.0,0.0,0.0,0.008753520530444803,320,80,103.45079112052917,1.2931348890066148,0.3232837222516537,0.19368809863808564,0.007529928936855867,3.185711348353652,0.00010870498384889516,0.0,0.0,0.0,0.0,0.0,0.007529928936855867,80,20,20.23315191268921,1.0116575956344604,0.2529143989086151,0.043344746553339066 +2,0.006802400949891307,0.3522556849198281,0.0001498469549909341,0.0,0.0,0.0,0.0,0.0,0.006802400949891307,320,80,103.24245595932007,1.290530699491501,0.3226326748728752,0.18977850895607845,0.007473361069423845,4.433246247235365,0.00010783077031044641,0.0,0.0,0.0,0.0,0.0,0.007473361069423845,80,20,20.317150354385376,1.0158575177192688,0.2539643794298172,0.043154743919149044 +3,0.008915021323991823,0.718201418556464,8.08643664615523e-05,0.0,0.0,0.0,0.0,0.0,0.008915021323991823,320,80,103.76086711883545,1.2970108389854431,0.3242527097463608,0.18378724994836376,0.015541119105182587,4.1963203363062345,0.0004786531295351892,0.0,0.0,0.0,0.0,0.0,0.015541119105182587,80,20,20.26967740058899,1.0134838700294495,0.2533709675073624,0.03725434660445899 +4,0.006931817024451448,0.35298403866354533,7.142922729861737e-05,0.0,0.0,0.0,0.0,0.0,0.006931817024451448,320,80,103.68316864967346,1.2960396081209182,0.32400990203022956,0.1754610677191522,0.006783264396653976,2.084030251805075,0.00010564910719947917,0.0,0.0,0.0,0.0,0.0,0.006783264396653976,80,20,20.532756090164185,1.0266378045082092,0.2566594511270523,0.046784830396063626 +5,0.005099834372958867,0.0875181610394841,9.274947589943961e-05,0.0,0.0,0.0,0.0,0.0,0.005099834372958867,320,80,103.51723384857178,1.2939654231071471,0.3234913557767868,0.19479809664189815,0.006807428642059676,6.448283100366529,8.50121301514406e-05,0.0,0.0,0.0,0.0,0.0,0.006807428642059676,80,20,20.114439249038696,1.005721962451935,0.2514304906129837,0.04373221881105564 +6,0.00480109396030457,0.9528829011607514,5.0811379982423e-05,0.0,0.0,0.0,0.0,0.0,0.00480109396030457,320,80,103.59416389465332,1.2949270486831665,0.32373176217079164,0.178886199297267,0.006987621737061999,4.279015872643868,9.801611965674085e-05,0.0,0.0,0.0,0.0,0.0,0.006987621737061999,80,20,20.27844500541687,1.0139222502708436,0.2534805625677109,0.04849170843372121 +7,0.00464072308905088,0.1356045210827208,6.834771834407436e-05,0.0,0.0,0.0,0.0,0.0,0.00464072308905088,320,80,103.63158965110779,1.2953948706388474,0.32384871765971185,0.19017753867083229,0.006893738606595434,3.8557679186087626,8.831684561805275e-05,0.0,0.0,0.0,0.0,0.0,0.006893738606595434,80,20,20.594661712646484,1.0297330856323241,0.25743327140808103,0.04350378216477111 +8,0.004308880444841634,0.06846157661166216,5.0194533013741347e-05,0.0,0.0,0.0,0.0,0.0,0.004308880444841634,320,80,103.90516233444214,1.2988145291805266,0.32470363229513166,0.18733386998064816,0.006929469533497467,3.2670128452096834,7.968755999527843e-05,0.0,0.0,0.0,0.0,0.0,0.006929469533497467,80,20,20.433101177215576,1.021655058860779,0.2554137647151947,0.04181941950228065 +9,0.004508837422326906,0.08966287379656705,8.609927907704219e-05,0.0,0.0,0.0,0.0,0.0,0.004508837422326906,320,80,103.64076137542725,1.2955095171928406,0.32387737929821014,0.18831826079403982,0.0070835084756254215,4.297422426286471,0.00011362928610676448,0.0,0.0,0.0,0.0,0.0,0.0070835084756254215,80,20,20.304687023162842,1.015234351158142,0.2538085877895355,0.0519675396499224 +10,0.004514601970731747,0.1381884859496653,8.4598984369378e-05,0.0,0.0,0.0,0.0,0.0,0.004514601970731747,320,80,103.90474796295166,1.2988093495368958,0.32470233738422394,0.17621430779545336,0.005505984234332573,5.328735202133521,3.370500902892815e-05,0.0,0.0,0.0,0.0,0.0,0.005505984234332573,80,20,20.54746174812317,1.0273730874061584,0.2568432718515396,0.051376725709997115 +11,0.004419548849091371,0.07128540304256603,6.623427232033962e-05,0.0,0.0,0.0,0.0,0.0,0.004419548849091371,320,80,103.64773535728455,1.2955966919660569,0.3238991729915142,0.18890725779347123,0.005045573477400467,4.834901636225572,2.2054046640818116e-05,0.0,0.0,0.0,0.0,0.0,0.005045573477400467,80,20,20.591118812561035,1.0295559406280517,0.2573889851570129,0.051232723612338306 +12,0.004236863098185495,0.04387387494761443,8.839865267179564e-05,0.0,0.0,0.0,0.0,0.0,0.004236863098185495,320,80,103.94386696815491,1.2992983371019364,0.3248245842754841,0.1935716205276549,0.008129652022034861,5.03432033594964,0.00015811533519221043,0.0,0.0,0.0,0.0,0.0,0.008129652022034861,80,20,20.881214380264282,1.044060719013214,0.2610151797533035,0.04673133364703972 +13,0.004276435652536747,0.0654336524629164,4.581930034836763e-05,0.0,0.0,0.0,0.0,0.0,0.004276435652536747,320,80,103.00667309761047,1.287583413720131,0.32189585343003274,0.18084844152908772,0.007877110847039149,2.788165700972968,0.0001813338503336759,0.0,0.0,0.0,0.0,0.0,0.007877110847039149,80,20,20.325989723205566,1.0162994861602783,0.2540748715400696,0.04980120111722499 +14,0.0038373295942619734,0.0791764844770995,4.599695211216274e-05,0.0,0.0,0.0,0.0,0.0,0.0038373295942619734,320,80,103.24337792396545,1.290542224049568,0.322635556012392,0.18313483651727439,0.0050060375331668185,3.8634611237153877,2.9480706338880223e-05,0.0,0.0,0.0,0.0,0.0,0.0050060375331668185,80,20,20.198811054229736,1.0099405527114869,0.2524851381778717,0.058232192660216245 +15,0.003900589924239739,0.10312648875277333,2.1983545730452913e-05,0.0,0.0,0.0,0.0,0.0,0.003900589924239739,320,80,103.35287404060364,1.2919109255075454,0.32297773137688635,0.19720614301040768,0.009117326361592858,1.760603382944828,0.00027292398581290066,0.0,0.0,0.0,0.0,0.0,0.009117326361592858,80,20,20.001904249191284,1.0000952124595641,0.25002380311489103,0.048848784435540436 +16,0.0018584069126518442,0.04251637269783757,1.0266101569923053e-05,0.0,0.0,0.0,0.0,0.0,0.0018584069126518442,320,80,102.94779467582703,1.286847433447838,0.3217118583619595,0.18552915730979294,0.007703973473689984,1.3023191384279245,0.00017489787961899594,0.0,0.0,0.0,0.0,0.0,0.007703973473689984,80,20,20.37928342819214,1.018964171409607,0.25474104285240173,0.05299922423437238 +17,0.0008170823710770492,0.018880133842297652,5.023515140430146e-07,0.0,0.0,0.0,0.0,0.0,0.0008170823710770492,320,80,102.69966387748718,1.2837457984685898,0.32093644961714746,0.1925133554963395,0.00931795308351866,2.071352872970965,0.0002587945756321958,0.0,0.0,0.0,0.0,0.0,0.00931795308351866,80,20,19.96341824531555,0.9981709122657776,0.2495427280664444,0.053129641944542526 +18,0.0003576292982074847,0.012536979420885785,4.498594921749366e-08,0.0,0.0,0.0,0.0,0.0,0.0003576292982074847,320,80,100.82452273368835,1.2603065341711044,0.3150766335427761,0.19631691183894873,0.00876514861229225,1.4645986258908124,0.0002133372895583463,0.0,0.0,0.0,0.0,0.0,0.00876514861229225,80,20,19.451266765594482,0.9725633382797241,0.24314083456993102,0.05004617176018655 +19,0.0002510753680212474,0.008272775144363465,3.3253448800756014e-08,0.0,0.0,0.0,0.0,0.0,0.0002510753680212474,320,80,102.30520558357239,1.2788150697946548,0.3197037674486637,0.19034422542899848,0.00618170693560387,1.5110018466951716,7.781338554512241e-05,0.0,0.0,0.0,0.0,0.0,0.00618170693560387,80,20,19.740620136260986,0.9870310068130493,0.24675775170326233,0.055138330021873114 +20,0.00028350398017664704,0.019687367545015277,2.482109546082703e-07,0.0,0.0,0.0,0.0,0.0,0.00028350398017664704,320,80,99.86743569374084,1.2483429461717606,0.31208573654294014,0.1864254915737547,0.007587061779486248,1.3006179411045196,0.00014909935981792798,0.0,0.0,0.0,0.0,0.0,0.007587061779486248,80,20,20.21121335029602,1.0105606675148011,0.2526401668787003,0.05435547353699803 +21,0.00025932476952164054,0.007903821782640907,9.45318477345975e-10,0.0,0.0,0.0,0.0,0.0,0.00025932476952164054,320,80,103.72837948799133,1.2966047435998918,0.32415118589997294,0.18518321572337298,0.006430168935912662,1.9212478918598208,9.073310130256474e-05,0.0,0.0,0.0,0.0,0.0,0.006430168935912662,80,20,19.81023097038269,0.9905115485191345,0.24762788712978362,0.05623150994069874 +22,0.00033484522286357786,0.012371280277519502,1.6720436467560026e-08,0.0,0.0,0.0,0.0,0.0,0.00033484522286357786,320,80,102.39797282218933,1.2799746602773667,0.3199936650693417,0.1927722441148944,0.0066237156010174655,1.626585018528567,9.447487505163111e-05,0.0,0.0,0.0,0.0,0.0,0.0066237156010174655,80,20,20.115633487701416,1.0057816743850707,0.2514454185962677,0.054200840881094337 +23,0.00015932412852635026,0.025226200853674642,3.639718875007858e-08,0.0,0.0,0.0,0.0,0.0,0.00015932412852635026,320,80,103.57392120361328,1.294674015045166,0.3236685037612915,0.1952298643416725,0.0065823450138850605,1.5178716897570212,9.93379512048212e-05,0.0,0.0,0.0,0.0,0.0,0.0065823450138850605,80,20,21.15859341621399,1.0579296708106996,0.2644824177026749,0.0543066727463156 +24,0.00012041164002258853,0.006688666018078895,1.577604157902094e-08,0.0,0.0,0.0,0.0,0.0,0.00012041164002258853,320,80,107.04027843475342,1.3380034804344176,0.3345008701086044,0.18686838666908442,0.007150729962449987,1.6130354540884582,0.00011762036998774761,0.0,0.0,0.0,0.0,0.0,0.007150729962449987,80,20,21.25710916519165,1.0628554582595826,0.26571386456489565,0.050873439060524106 +25,8.769563716697349e-05,0.005887569199180521,3.494414894220867e-09,0.0,0.0,0.0,0.0,0.0,8.769563716697349e-05,320,80,106.57941937446594,1.3322427421808243,0.33306068554520607,0.18599217470618895,0.00832268671510974,1.4815574481464182,0.00019259734704257792,0.0,0.0,0.0,0.0,0.0,0.00832268671510974,80,20,20.612091302871704,1.0306045651435851,0.2576511412858963,0.05257055321708322 +26,8.882110219019524e-05,0.002085926350794054,2.3958830939162234e-09,0.0,0.0,0.0,0.0,0.0,8.882110219019524e-05,320,80,106.64562821388245,1.3330703526735306,0.33326758816838264,0.20409600271377712,0.006672654673457146,1.4586342717834213,9.671619951117094e-05,0.0,0.0,0.0,0.0,0.0,0.006672654673457146,80,20,20.21643877029419,1.0108219385147095,0.25270548462867737,0.0529700854793191 +27,7.043356762892472e-05,0.005430679229713497,4.15760022837944e-10,0.0,0.0,0.0,0.0,0.0,7.043356762892472e-05,320,80,102.27982902526855,1.278497862815857,0.31962446570396424,0.2028546938439831,0.006019308035320137,1.3263217964900833,7.28448786667002e-05,0.0,0.0,0.0,0.0,0.0,0.006019308035320137,80,20,20.296292066574097,1.0148146033287049,0.2537036508321762,0.05590685121715069 diff --git a/treatment/lct_gan/mlu-eval.ipynb b/treatment/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..18ec2cd22dafa9b9925e144020e906e1ffc6cd75 --- /dev/null +++ b/treatment/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2670 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:40.905323Z", + "iopub.status.busy": "2024-03-01T06:37:40.904929Z", + "iopub.status.idle": "2024-03-01T06:37:40.938985Z", + "shell.execute_reply": "2024-03-01T06:37:40.938267Z" + }, + "papermill": { + "duration": 0.049669, + "end_time": "2024-03-01T06:37:40.941076", + "exception": false, + "start_time": "2024-03-01T06:37:40.891407", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:40.968919Z", + "iopub.status.busy": "2024-03-01T06:37:40.968442Z", + "iopub.status.idle": "2024-03-01T06:37:40.975769Z", + "shell.execute_reply": "2024-03-01T06:37:40.974924Z" + }, + "papermill": { + "duration": 0.023557, + "end_time": "2024-03-01T06:37:40.977775", + "exception": false, + "start_time": "2024-03-01T06:37:40.954218", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:41.001740Z", + "iopub.status.busy": "2024-03-01T06:37:41.001463Z", + "iopub.status.idle": "2024-03-01T06:37:41.005366Z", + "shell.execute_reply": "2024-03-01T06:37:41.004588Z" + }, + "papermill": { + "duration": 0.018202, + "end_time": "2024-03-01T06:37:41.007313", + "exception": false, + "start_time": "2024-03-01T06:37:40.989111", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:41.030702Z", + "iopub.status.busy": "2024-03-01T06:37:41.030401Z", + "iopub.status.idle": "2024-03-01T06:37:41.034415Z", + "shell.execute_reply": "2024-03-01T06:37:41.033610Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017916, + "end_time": "2024-03-01T06:37:41.036404", + "exception": false, + "start_time": "2024-03-01T06:37:41.018488", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:41.060140Z", + "iopub.status.busy": "2024-03-01T06:37:41.059891Z", + "iopub.status.idle": "2024-03-01T06:37:41.065405Z", + "shell.execute_reply": "2024-03-01T06:37:41.064582Z" + }, + "papermill": { + "duration": 0.01973, + "end_time": "2024-03-01T06:37:41.067481", + "exception": false, + "start_time": "2024-03-01T06:37:41.047751", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1d904d96", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:41.092350Z", + "iopub.status.busy": "2024-03-01T06:37:41.092062Z", + "iopub.status.idle": "2024-03-01T06:37:41.097169Z", + "shell.execute_reply": "2024-03-01T06:37:41.096307Z" + }, + "papermill": { + "duration": 0.019934, + "end_time": "2024-03-01T06:37:41.099194", + "exception": false, + "start_time": "2024-03-01T06:37:41.079260", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"lct_gan\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/lct_gan/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011024, + "end_time": "2024-03-01T06:37:41.121282", + "exception": false, + "start_time": "2024-03-01T06:37:41.110258", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:41.145649Z", + "iopub.status.busy": "2024-03-01T06:37:41.144969Z", + "iopub.status.idle": "2024-03-01T06:37:41.154625Z", + "shell.execute_reply": "2024-03-01T06:37:41.153823Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023987, + "end_time": "2024-03-01T06:37:41.156560", + "exception": false, + "start_time": "2024-03-01T06:37:41.132573", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/lct_gan/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:41.180579Z", + "iopub.status.busy": "2024-03-01T06:37:41.180271Z", + "iopub.status.idle": "2024-03-01T06:37:43.405343Z", + "shell.execute_reply": "2024-03-01T06:37:43.404398Z" + }, + "papermill": { + "duration": 2.23974, + "end_time": "2024-03-01T06:37:43.407601", + "exception": false, + "start_time": "2024-03-01T06:37:41.167861", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:43.434451Z", + "iopub.status.busy": "2024-03-01T06:37:43.434028Z", + "iopub.status.idle": "2024-03-01T06:37:43.461510Z", + "shell.execute_reply": "2024-03-01T06:37:43.460714Z" + }, + "papermill": { + "duration": 0.043451, + "end_time": "2024-03-01T06:37:43.463747", + "exception": false, + "start_time": "2024-03-01T06:37:43.420296", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:43.489093Z", + "iopub.status.busy": "2024-03-01T06:37:43.488805Z", + "iopub.status.idle": "2024-03-01T06:37:43.509803Z", + "shell.execute_reply": "2024-03-01T06:37:43.509067Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.035791, + "end_time": "2024-03-01T06:37:43.511912", + "exception": false, + "start_time": "2024-03-01T06:37:43.476121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:43.539665Z", + "iopub.status.busy": "2024-03-01T06:37:43.538932Z", + "iopub.status.idle": "2024-03-01T06:37:44.020315Z", + "shell.execute_reply": "2024-03-01T06:37:44.019458Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.498436, + "end_time": "2024-03-01T06:37:44.022641", + "exception": false, + "start_time": "2024-03-01T06:37:43.524205", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:44.047511Z", + "iopub.status.busy": "2024-03-01T06:37:44.047200Z", + "iopub.status.idle": "2024-03-01T06:37:57.447527Z", + "shell.execute_reply": "2024-03-01T06:37:57.446595Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 13.41538, + "end_time": "2024-03-01T06:37:57.450103", + "exception": false, + "start_time": "2024-03-01T06:37:44.034723", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-01 06:37:48.725607: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-01 06:37:48.725704: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-01 06:37:48.855733: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:57.477690Z", + "iopub.status.busy": "2024-03-01T06:37:57.476436Z", + "iopub.status.idle": "2024-03-01T06:37:57.490600Z", + "shell.execute_reply": "2024-03-01T06:37:57.489899Z" + }, + "papermill": { + "duration": 0.029971, + "end_time": "2024-03-01T06:37:57.492902", + "exception": false, + "start_time": "2024-03-01T06:37:57.462931", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:37:57.518368Z", + "iopub.status.busy": "2024-03-01T06:37:57.518061Z", + "iopub.status.idle": "2024-03-01T06:38:22.498118Z", + "shell.execute_reply": "2024-03-01T06:38:22.497159Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 24.995876, + "end_time": "2024-03-01T06:38:22.500742", + "exception": false, + "start_time": "2024-03-01T06:37:57.504866", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:47:42.874049Z", + "iopub.status.busy": "2024-03-01T06:47:42.873701Z", + "iopub.status.idle": "2024-03-01T06:57:03.489733Z", + "shell.execute_reply": "2024-03-01T06:57:03.488592Z" + }, + "papermill": { + "duration": 560.649636, + "end_time": "2024-03-01T06:57:03.504726", + "exception": false, + "start_time": "2024-03-01T06:47:42.855090", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-01T06:57:03.535116Z", + "iopub.status.busy": "2024-03-01T06:57:03.534120Z", + "iopub.status.idle": "2024-03-01T06:57:04.098122Z", + "shell.execute_reply": "2024-03-01T06:57:04.097203Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.581203, + "end_time": "2024-03-01T06:57:04.100206", + "exception": false, + "start_time": "2024-03-01T06:57:03.519003", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:57:04.131337Z", + "iopub.status.busy": "2024-03-01T06:57:04.131012Z", + "iopub.status.idle": "2024-03-01T06:57:04.135399Z", + "shell.execute_reply": "2024-03-01T06:57:04.134498Z" + }, + "papermill": { + "duration": 0.022449, + "end_time": "2024-03-01T06:57:04.137334", + "exception": false, + "start_time": "2024-03-01T06:57:04.114885", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:57:04.165908Z", + "iopub.status.busy": "2024-03-01T06:57:04.165622Z", + "iopub.status.idle": "2024-03-01T06:57:04.172471Z", + "shell.execute_reply": "2024-03-01T06:57:04.171692Z" + }, + "papermill": { + "duration": 0.023524, + "end_time": "2024-03-01T06:57:04.174595", + "exception": false, + "start_time": "2024-03-01T06:57:04.151071", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18680833" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:57:04.203425Z", + "iopub.status.busy": "2024-03-01T06:57:04.202969Z", + "iopub.status.idle": "2024-03-01T06:57:04.337294Z", + "shell.execute_reply": "2024-03-01T06:57:04.336388Z" + }, + "papermill": { + "duration": 0.151558, + "end_time": "2024-03-01T06:57:04.339656", + "exception": false, + "start_time": "2024-03-01T06:57:04.188098", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 75] --\n", + "├─Adapter: 1-1 [2, 2648, 75] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 77,824\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 75] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,680,833\n", + "Trainable params: 18,680,833\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.97\n", + "========================================================================================================================\n", + "Input size (MB): 1.99\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.72\n", + "Estimated Total Size (MB): 1156.19\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T06:57:04.372051Z", + "iopub.status.busy": "2024-03-01T06:57:04.371747Z", + "iopub.status.idle": "2024-03-01T07:59:06.110637Z", + "shell.execute_reply": "2024-03-01T07:59:06.109597Z" + }, + "papermill": { + "duration": 3721.757757, + "end_time": "2024-03-01T07:59:06.112909", + "exception": false, + "start_time": "2024-03-01T06:57:04.355152", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.22352998348069378, 'avg_role_model_std_loss': 80.13195664776967, 'avg_role_model_mean_pred_loss': 0.09520609854183135, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.22352998348069378, 'n_size': 320, 'n_batch': 80, 'duration': 103.12237620353699, 'duration_batch': 1.2890297025442123, 'duration_size': 0.3222574256360531, 'avg_pred_std': 0.0988283173581749}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01245800144970417, 'avg_role_model_std_loss': 1.6089594815347597, 'avg_role_model_mean_pred_loss': 0.00019736949793127678, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01245800144970417, 'n_size': 80, 'n_batch': 20, 'duration': 20.080093383789062, 'duration_batch': 1.004004669189453, 'duration_size': 0.25100116729736327, 'avg_pred_std': 0.07783476307522505}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008753520530444803, 'avg_role_model_std_loss': 0.34713386677235575, 'avg_role_model_mean_pred_loss': 0.00011480308697381734, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008753520530444803, 'n_size': 320, 'n_batch': 80, 'duration': 103.45079112052917, 'duration_batch': 1.2931348890066148, 'duration_size': 0.3232837222516537, 'avg_pred_std': 0.19368809863808564}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007529928936855867, 'avg_role_model_std_loss': 3.185711348353652, 'avg_role_model_mean_pred_loss': 0.00010870498384889516, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007529928936855867, 'n_size': 80, 'n_batch': 20, 'duration': 20.23315191268921, 'duration_batch': 1.0116575956344604, 'duration_size': 0.2529143989086151, 'avg_pred_std': 0.043344746553339066}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006802400949891307, 'avg_role_model_std_loss': 0.3522556849198281, 'avg_role_model_mean_pred_loss': 0.0001498469549909341, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006802400949891307, 'n_size': 320, 'n_batch': 80, 'duration': 103.24245595932007, 'duration_batch': 1.290530699491501, 'duration_size': 0.3226326748728752, 'avg_pred_std': 0.18977850895607845}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007473361069423845, 'avg_role_model_std_loss': 4.433246247235365, 'avg_role_model_mean_pred_loss': 0.00010783077031044641, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007473361069423845, 'n_size': 80, 'n_batch': 20, 'duration': 20.317150354385376, 'duration_batch': 1.0158575177192688, 'duration_size': 0.2539643794298172, 'avg_pred_std': 0.043154743919149044}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008915021323991823, 'avg_role_model_std_loss': 0.718201418556464, 'avg_role_model_mean_pred_loss': 8.08643664615523e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008915021323991823, 'n_size': 320, 'n_batch': 80, 'duration': 103.76086711883545, 'duration_batch': 1.2970108389854431, 'duration_size': 0.3242527097463608, 'avg_pred_std': 0.18378724994836376}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.015541119105182587, 'avg_role_model_std_loss': 4.1963203363062345, 'avg_role_model_mean_pred_loss': 0.0004786531295351892, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015541119105182587, 'n_size': 80, 'n_batch': 20, 'duration': 20.26967740058899, 'duration_batch': 1.0134838700294495, 'duration_size': 0.2533709675073624, 'avg_pred_std': 0.03725434660445899}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006931817024451448, 'avg_role_model_std_loss': 0.35298403866354533, 'avg_role_model_mean_pred_loss': 7.142922729861737e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006931817024451448, 'n_size': 320, 'n_batch': 80, 'duration': 103.68316864967346, 'duration_batch': 1.2960396081209182, 'duration_size': 0.32400990203022956, 'avg_pred_std': 0.1754610677191522}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006783264396653976, 'avg_role_model_std_loss': 2.084030251805075, 'avg_role_model_mean_pred_loss': 0.00010564910719947917, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006783264396653976, 'n_size': 80, 'n_batch': 20, 'duration': 20.532756090164185, 'duration_batch': 1.0266378045082092, 'duration_size': 0.2566594511270523, 'avg_pred_std': 0.046784830396063626}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005099834372958867, 'avg_role_model_std_loss': 0.0875181610394841, 'avg_role_model_mean_pred_loss': 9.274947589943961e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005099834372958867, 'n_size': 320, 'n_batch': 80, 'duration': 103.51723384857178, 'duration_batch': 1.2939654231071471, 'duration_size': 0.3234913557767868, 'avg_pred_std': 0.19479809664189815}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006807428642059676, 'avg_role_model_std_loss': 6.448283100366529, 'avg_role_model_mean_pred_loss': 8.50121301514406e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006807428642059676, 'n_size': 80, 'n_batch': 20, 'duration': 20.114439249038696, 'duration_batch': 1.005721962451935, 'duration_size': 0.2514304906129837, 'avg_pred_std': 0.04373221881105564}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00480109396030457, 'avg_role_model_std_loss': 0.9528829011607514, 'avg_role_model_mean_pred_loss': 5.0811379982423e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00480109396030457, 'n_size': 320, 'n_batch': 80, 'duration': 103.59416389465332, 'duration_batch': 1.2949270486831665, 'duration_size': 0.32373176217079164, 'avg_pred_std': 0.178886199297267}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006987621737061999, 'avg_role_model_std_loss': 4.279015872643868, 'avg_role_model_mean_pred_loss': 9.801611965674085e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006987621737061999, 'n_size': 80, 'n_batch': 20, 'duration': 20.27844500541687, 'duration_batch': 1.0139222502708436, 'duration_size': 0.2534805625677109, 'avg_pred_std': 0.04849170843372121}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00464072308905088, 'avg_role_model_std_loss': 0.1356045210827208, 'avg_role_model_mean_pred_loss': 6.834771834407436e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00464072308905088, 'n_size': 320, 'n_batch': 80, 'duration': 103.63158965110779, 'duration_batch': 1.2953948706388474, 'duration_size': 0.32384871765971185, 'avg_pred_std': 0.19017753867083229}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006893738606595434, 'avg_role_model_std_loss': 3.8557679186087626, 'avg_role_model_mean_pred_loss': 8.831684561805275e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006893738606595434, 'n_size': 80, 'n_batch': 20, 'duration': 20.594661712646484, 'duration_batch': 1.0297330856323241, 'duration_size': 0.25743327140808103, 'avg_pred_std': 0.04350378216477111}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004308880444841634, 'avg_role_model_std_loss': 0.06846157661166216, 'avg_role_model_mean_pred_loss': 5.0194533013741347e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004308880444841634, 'n_size': 320, 'n_batch': 80, 'duration': 103.90516233444214, 'duration_batch': 1.2988145291805266, 'duration_size': 0.32470363229513166, 'avg_pred_std': 0.18733386998064816}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006929469533497467, 'avg_role_model_std_loss': 3.2670128452096834, 'avg_role_model_mean_pred_loss': 7.968755999527843e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006929469533497467, 'n_size': 80, 'n_batch': 20, 'duration': 20.433101177215576, 'duration_batch': 1.021655058860779, 'duration_size': 0.2554137647151947, 'avg_pred_std': 0.04181941950228065}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004508837422326906, 'avg_role_model_std_loss': 0.08966287379656705, 'avg_role_model_mean_pred_loss': 8.609927907704219e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004508837422326906, 'n_size': 320, 'n_batch': 80, 'duration': 103.64076137542725, 'duration_batch': 1.2955095171928406, 'duration_size': 0.32387737929821014, 'avg_pred_std': 0.18831826079403982}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0070835084756254215, 'avg_role_model_std_loss': 4.297422426286471, 'avg_role_model_mean_pred_loss': 0.00011362928610676448, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0070835084756254215, 'n_size': 80, 'n_batch': 20, 'duration': 20.304687023162842, 'duration_batch': 1.015234351158142, 'duration_size': 0.2538085877895355, 'avg_pred_std': 0.0519675396499224}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004514601970731747, 'avg_role_model_std_loss': 0.1381884859496653, 'avg_role_model_mean_pred_loss': 8.4598984369378e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004514601970731747, 'n_size': 320, 'n_batch': 80, 'duration': 103.90474796295166, 'duration_batch': 1.2988093495368958, 'duration_size': 0.32470233738422394, 'avg_pred_std': 0.17621430779545336}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005505984234332573, 'avg_role_model_std_loss': 5.328735202133521, 'avg_role_model_mean_pred_loss': 3.370500902892815e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005505984234332573, 'n_size': 80, 'n_batch': 20, 'duration': 20.54746174812317, 'duration_batch': 1.0273730874061584, 'duration_size': 0.2568432718515396, 'avg_pred_std': 0.051376725709997115}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004419548849091371, 'avg_role_model_std_loss': 0.07128540304256603, 'avg_role_model_mean_pred_loss': 6.623427232033962e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004419548849091371, 'n_size': 320, 'n_batch': 80, 'duration': 103.64773535728455, 'duration_batch': 1.2955966919660569, 'duration_size': 0.3238991729915142, 'avg_pred_std': 0.18890725779347123}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005045573477400467, 'avg_role_model_std_loss': 4.834901636225572, 'avg_role_model_mean_pred_loss': 2.2054046640818116e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005045573477400467, 'n_size': 80, 'n_batch': 20, 'duration': 20.591118812561035, 'duration_batch': 1.0295559406280517, 'duration_size': 0.2573889851570129, 'avg_pred_std': 0.051232723612338306}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004236863098185495, 'avg_role_model_std_loss': 0.04387387494761443, 'avg_role_model_mean_pred_loss': 8.839865267179564e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004236863098185495, 'n_size': 320, 'n_batch': 80, 'duration': 103.94386696815491, 'duration_batch': 1.2992983371019364, 'duration_size': 0.3248245842754841, 'avg_pred_std': 0.1935716205276549}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008129652022034861, 'avg_role_model_std_loss': 5.03432033594964, 'avg_role_model_mean_pred_loss': 0.00015811533519221043, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008129652022034861, 'n_size': 80, 'n_batch': 20, 'duration': 20.881214380264282, 'duration_batch': 1.044060719013214, 'duration_size': 0.2610151797533035, 'avg_pred_std': 0.04673133364703972}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004276435652536747, 'avg_role_model_std_loss': 0.0654336524629164, 'avg_role_model_mean_pred_loss': 4.581930034836763e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004276435652536747, 'n_size': 320, 'n_batch': 80, 'duration': 103.00667309761047, 'duration_batch': 1.287583413720131, 'duration_size': 0.32189585343003274, 'avg_pred_std': 0.18084844152908772}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007877110847039149, 'avg_role_model_std_loss': 2.788165700972968, 'avg_role_model_mean_pred_loss': 0.0001813338503336759, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007877110847039149, 'n_size': 80, 'n_batch': 20, 'duration': 20.325989723205566, 'duration_batch': 1.0162994861602783, 'duration_size': 0.2540748715400696, 'avg_pred_std': 0.04980120111722499}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038373295942619734, 'avg_role_model_std_loss': 0.0791764844770995, 'avg_role_model_mean_pred_loss': 4.599695211216274e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038373295942619734, 'n_size': 320, 'n_batch': 80, 'duration': 103.24337792396545, 'duration_batch': 1.290542224049568, 'duration_size': 0.322635556012392, 'avg_pred_std': 0.18313483651727439}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0050060375331668185, 'avg_role_model_std_loss': 3.8634611237153877, 'avg_role_model_mean_pred_loss': 2.9480706338880223e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0050060375331668185, 'n_size': 80, 'n_batch': 20, 'duration': 20.198811054229736, 'duration_batch': 1.0099405527114869, 'duration_size': 0.2524851381778717, 'avg_pred_std': 0.058232192660216245}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003900589924239739, 'avg_role_model_std_loss': 0.10312648875277333, 'avg_role_model_mean_pred_loss': 2.1983545730452913e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003900589924239739, 'n_size': 320, 'n_batch': 80, 'duration': 103.35287404060364, 'duration_batch': 1.2919109255075454, 'duration_size': 0.32297773137688635, 'avg_pred_std': 0.19720614301040768}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009117326361592858, 'avg_role_model_std_loss': 1.760603382944828, 'avg_role_model_mean_pred_loss': 0.00027292398581290066, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009117326361592858, 'n_size': 80, 'n_batch': 20, 'duration': 20.001904249191284, 'duration_batch': 1.0000952124595641, 'duration_size': 0.25002380311489103, 'avg_pred_std': 0.048848784435540436}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018584069126518442, 'avg_role_model_std_loss': 0.04251637269783757, 'avg_role_model_mean_pred_loss': 1.0266101569923053e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018584069126518442, 'n_size': 320, 'n_batch': 80, 'duration': 102.94779467582703, 'duration_batch': 1.286847433447838, 'duration_size': 0.3217118583619595, 'avg_pred_std': 0.18552915730979294}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007703973473689984, 'avg_role_model_std_loss': 1.3023191384279245, 'avg_role_model_mean_pred_loss': 0.00017489787961899594, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007703973473689984, 'n_size': 80, 'n_batch': 20, 'duration': 20.37928342819214, 'duration_batch': 1.018964171409607, 'duration_size': 0.25474104285240173, 'avg_pred_std': 0.05299922423437238}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008170823710770492, 'avg_role_model_std_loss': 0.018880133842297652, 'avg_role_model_mean_pred_loss': 5.023515140430146e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008170823710770492, 'n_size': 320, 'n_batch': 80, 'duration': 102.69966387748718, 'duration_batch': 1.2837457984685898, 'duration_size': 0.32093644961714746, 'avg_pred_std': 0.1925133554963395}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00931795308351866, 'avg_role_model_std_loss': 2.071352872970965, 'avg_role_model_mean_pred_loss': 0.0002587945756321958, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00931795308351866, 'n_size': 80, 'n_batch': 20, 'duration': 19.96341824531555, 'duration_batch': 0.9981709122657776, 'duration_size': 0.2495427280664444, 'avg_pred_std': 0.053129641944542526}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0003576292982074847, 'avg_role_model_std_loss': 0.012536979420885785, 'avg_role_model_mean_pred_loss': 4.498594921749366e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003576292982074847, 'n_size': 320, 'n_batch': 80, 'duration': 100.82452273368835, 'duration_batch': 1.2603065341711044, 'duration_size': 0.3150766335427761, 'avg_pred_std': 0.19631691183894873}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00876514861229225, 'avg_role_model_std_loss': 1.4645986258908124, 'avg_role_model_mean_pred_loss': 0.0002133372895583463, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00876514861229225, 'n_size': 80, 'n_batch': 20, 'duration': 19.451266765594482, 'duration_batch': 0.9725633382797241, 'duration_size': 0.24314083456993102, 'avg_pred_std': 0.05004617176018655}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0002510753680212474, 'avg_role_model_std_loss': 0.008272775144363465, 'avg_role_model_mean_pred_loss': 3.3253448800756014e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0002510753680212474, 'n_size': 320, 'n_batch': 80, 'duration': 102.30520558357239, 'duration_batch': 1.2788150697946548, 'duration_size': 0.3197037674486637, 'avg_pred_std': 0.19034422542899848}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00618170693560387, 'avg_role_model_std_loss': 1.5110018466951716, 'avg_role_model_mean_pred_loss': 7.781338554512241e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00618170693560387, 'n_size': 80, 'n_batch': 20, 'duration': 19.740620136260986, 'duration_batch': 0.9870310068130493, 'duration_size': 0.24675775170326233, 'avg_pred_std': 0.055138330021873114}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00028350398017664704, 'avg_role_model_std_loss': 0.019687367545015277, 'avg_role_model_mean_pred_loss': 2.482109546082703e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00028350398017664704, 'n_size': 320, 'n_batch': 80, 'duration': 99.86743569374084, 'duration_batch': 1.2483429461717606, 'duration_size': 0.31208573654294014, 'avg_pred_std': 0.1864254915737547}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007587061779486248, 'avg_role_model_std_loss': 1.3006179411045196, 'avg_role_model_mean_pred_loss': 0.00014909935981792798, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007587061779486248, 'n_size': 80, 'n_batch': 20, 'duration': 20.21121335029602, 'duration_batch': 1.0105606675148011, 'duration_size': 0.2526401668787003, 'avg_pred_std': 0.05435547353699803}\n", + "Epoch 21\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00025932476952164054, 'avg_role_model_std_loss': 0.007903821782640907, 'avg_role_model_mean_pred_loss': 9.45318477345975e-10, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00025932476952164054, 'n_size': 320, 'n_batch': 80, 'duration': 103.72837948799133, 'duration_batch': 1.2966047435998918, 'duration_size': 0.32415118589997294, 'avg_pred_std': 0.18518321572337298}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006430168935912662, 'avg_role_model_std_loss': 1.9212478918598208, 'avg_role_model_mean_pred_loss': 9.073310130256474e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006430168935912662, 'n_size': 80, 'n_batch': 20, 'duration': 19.81023097038269, 'duration_batch': 0.9905115485191345, 'duration_size': 0.24762788712978362, 'avg_pred_std': 0.05623150994069874}\n", + "Epoch 22\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00033484522286357786, 'avg_role_model_std_loss': 0.012371280277519502, 'avg_role_model_mean_pred_loss': 1.6720436467560026e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00033484522286357786, 'n_size': 320, 'n_batch': 80, 'duration': 102.39797282218933, 'duration_batch': 1.2799746602773667, 'duration_size': 0.3199936650693417, 'avg_pred_std': 0.1927722441148944}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0066237156010174655, 'avg_role_model_std_loss': 1.626585018528567, 'avg_role_model_mean_pred_loss': 9.447487505163111e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0066237156010174655, 'n_size': 80, 'n_batch': 20, 'duration': 20.115633487701416, 'duration_batch': 1.0057816743850707, 'duration_size': 0.2514454185962677, 'avg_pred_std': 0.054200840881094337}\n", + "Epoch 23\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00015932412852635026, 'avg_role_model_std_loss': 0.025226200853674642, 'avg_role_model_mean_pred_loss': 3.639718875007858e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00015932412852635026, 'n_size': 320, 'n_batch': 80, 'duration': 103.57392120361328, 'duration_batch': 1.294674015045166, 'duration_size': 0.3236685037612915, 'avg_pred_std': 0.1952298643416725}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0065823450138850605, 'avg_role_model_std_loss': 1.5178716897570212, 'avg_role_model_mean_pred_loss': 9.93379512048212e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0065823450138850605, 'n_size': 80, 'n_batch': 20, 'duration': 21.15859341621399, 'duration_batch': 1.0579296708106996, 'duration_size': 0.2644824177026749, 'avg_pred_std': 0.0543066727463156}\n", + "Epoch 24\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00012041164002258853, 'avg_role_model_std_loss': 0.006688666018078895, 'avg_role_model_mean_pred_loss': 1.577604157902094e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00012041164002258853, 'n_size': 320, 'n_batch': 80, 'duration': 107.04027843475342, 'duration_batch': 1.3380034804344176, 'duration_size': 0.3345008701086044, 'avg_pred_std': 0.18686838666908442}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007150729962449987, 'avg_role_model_std_loss': 1.6130354540884582, 'avg_role_model_mean_pred_loss': 0.00011762036998774761, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007150729962449987, 'n_size': 80, 'n_batch': 20, 'duration': 21.25710916519165, 'duration_batch': 1.0628554582595826, 'duration_size': 0.26571386456489565, 'avg_pred_std': 0.050873439060524106}\n", + "Epoch 25\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 8.769563716697349e-05, 'avg_role_model_std_loss': 0.005887569199180521, 'avg_role_model_mean_pred_loss': 3.494414894220867e-09, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 8.769563716697349e-05, 'n_size': 320, 'n_batch': 80, 'duration': 106.57941937446594, 'duration_batch': 1.3322427421808243, 'duration_size': 0.33306068554520607, 'avg_pred_std': 0.18599217470618895}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00832268671510974, 'avg_role_model_std_loss': 1.4815574481464182, 'avg_role_model_mean_pred_loss': 0.00019259734704257792, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00832268671510974, 'n_size': 80, 'n_batch': 20, 'duration': 20.612091302871704, 'duration_batch': 1.0306045651435851, 'duration_size': 0.2576511412858963, 'avg_pred_std': 0.05257055321708322}\n", + "Epoch 26\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 8.882110219019524e-05, 'avg_role_model_std_loss': 0.002085926350794054, 'avg_role_model_mean_pred_loss': 2.3958830939162234e-09, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 8.882110219019524e-05, 'n_size': 320, 'n_batch': 80, 'duration': 106.64562821388245, 'duration_batch': 1.3330703526735306, 'duration_size': 0.33326758816838264, 'avg_pred_std': 0.20409600271377712}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006672654673457146, 'avg_role_model_std_loss': 1.4586342717834213, 'avg_role_model_mean_pred_loss': 9.671619951117094e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006672654673457146, 'n_size': 80, 'n_batch': 20, 'duration': 20.21643877029419, 'duration_batch': 1.0108219385147095, 'duration_size': 0.25270548462867737, 'avg_pred_std': 0.0529700854793191}\n", + "Epoch 27\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 7.043356762892472e-05, 'avg_role_model_std_loss': 0.005430679229713497, 'avg_role_model_mean_pred_loss': 4.15760022837944e-10, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 7.043356762892472e-05, 'n_size': 320, 'n_batch': 80, 'duration': 102.27982902526855, 'duration_batch': 1.278497862815857, 'duration_size': 0.31962446570396424, 'avg_pred_std': 0.2028546938439831}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006019308035320137, 'avg_role_model_std_loss': 1.3263217964900833, 'avg_role_model_mean_pred_loss': 7.28448786667002e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006019308035320137, 'n_size': 80, 'n_batch': 20, 'duration': 20.296292066574097, 'duration_batch': 1.0148146033287049, 'duration_size': 0.2537036508321762, 'avg_pred_std': 0.05590685121715069}\n", + "Epoch 28\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 5.8602433459142846e-05, 'avg_role_model_std_loss': 0.002511359712229444, 'avg_role_model_mean_pred_loss': 2.106261276456074e-09, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 5.8602433459142846e-05, 'n_size': 320, 'n_batch': 80, 'duration': 101.94170665740967, 'duration_batch': 1.2742713332176208, 'duration_size': 0.3185678333044052, 'avg_pred_std': 0.18126765387132765}\n", + "Time out: 3605.992097377777/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.0023591548596047225, 'avg_g_mag_loss': 4.580079086250558e-08, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.3744425773620605, 'grad_duration': 4.456961393356323, 'total_duration': 6.831403970718384, 'pred_std': 0.07008553296327591, 'std_loss': 0.010716128163039684, 'mean_pred_loss': 4.918354079563869e-06, 'pred_rmse': 0.04857112839818001, 'pred_mae': 0.03671034052968025, 'pred_mape': 0.06946056336164474, 'grad_rmse': 0.015458883717656136, 'grad_mae': 0.011746696196496487, 'grad_mape': 0.16063837707042694}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0023591548596047225, 'avg_g_mag_loss': 4.580079086250558e-08, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 2.3744425773620605, 'avg_grad_duration': 4.456961393356323, 'avg_total_duration': 6.831403970718384, 'avg_pred_std': 0.07008553296327591, 'avg_std_loss': 0.010716128163039684, 'avg_mean_pred_loss': 4.918354079563869e-06}, 'min_metrics': {'avg_loss': 0.0023591548596047225, 'avg_g_mag_loss': 4.580079086250558e-08, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.3744425773620605, 'grad_duration': 4.456961393356323, 'total_duration': 6.831403970718384, 'pred_std': 0.07008553296327591, 'std_loss': 0.010716128163039684, 'mean_pred_loss': 4.918354079563869e-06, 'pred_rmse': 0.04857112839818001, 'pred_mae': 0.03671034052968025, 'pred_mape': 0.06946056336164474, 'grad_rmse': 0.015458883717656136, 'grad_mae': 0.011746696196496487, 'grad_mape': 0.16063837707042694}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0023591548596047225, 'avg_g_mag_loss': 4.580079086250558e-08, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.3744425773620605, 'grad_duration': 4.456961393356323, 'total_duration': 6.831403970718384, 'pred_std': 0.07008553296327591, 'std_loss': 0.010716128163039684, 'mean_pred_loss': 4.918354079563869e-06, 'pred_rmse': 0.04857112839818001, 'pred_mae': 0.03671034052968025, 'pred_mape': 0.06946056336164474, 'grad_rmse': 0.015458883717656136, 'grad_mae': 0.011746696196496487, 'grad_mape': 0.16063837707042694}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T07:59:06.154815Z", + "iopub.status.busy": "2024-03-01T07:59:06.154402Z", + "iopub.status.idle": "2024-03-01T07:59:06.159150Z", + "shell.execute_reply": "2024-03-01T07:59:06.158318Z" + }, + "papermill": { + "duration": 0.027765, + "end_time": "2024-03-01T07:59:06.160986", + "exception": false, + "start_time": "2024-03-01T07:59:06.133221", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T07:59:06.204309Z", + "iopub.status.busy": "2024-03-01T07:59:06.203737Z", + "iopub.status.idle": "2024-03-01T07:59:06.329694Z", + "shell.execute_reply": "2024-03-01T07:59:06.328627Z" + }, + "papermill": { + "duration": 0.149027, + "end_time": "2024-03-01T07:59:06.333013", + "exception": false, + "start_time": "2024-03-01T07:59:06.183986", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T07:59:06.377391Z", + "iopub.status.busy": "2024-03-01T07:59:06.376577Z", + "iopub.status.idle": "2024-03-01T07:59:06.693165Z", + "shell.execute_reply": "2024-03-01T07:59:06.692176Z" + }, + "papermill": { + "duration": 0.340956, + "end_time": "2024-03-01T07:59:06.695412", + "exception": false, + "start_time": "2024-03-01T07:59:06.354456", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtt0lEQVR4nO3de3xU1b338c/MZC4JuRLIDQMJQuQeEEhOoGCVHAPiBUorcHgEORWqh/RU4620knjpY0A4lqo80tIjaI9cpC32KVq8hEQfJVzkrkAKGggYkhAgCbnPZT1/TDLJQAKZEJgN83u/Xvs1M3vW7FkzZL6stfbea+uUUgohhNAAvbcrIIQQzSSQhBCaIYEkhNAMCSQhhGZIIAkhNEMCSQihGRJIQgjN8PN2BbqCw+GguLiYoKAgdDqdt6sjhLiIUooLFy4QExODXt9+O+imCKTi4mJiY2O9XQ0hxBWcPHmSW265pd3nb4pACgoKApwfNjg42Mu1EUJcrKqqitjYWNdvtT03RSA1d9OCg4MlkITQsCsNqcigthBCMySQhBCaIYEkhNCMm2IMSVwdu92O1Wr1djXEDcxoNGIwGK56OxJIPkwpRUlJCRUVFd6uirgJhIaGEhUVdVXHAkog+bDmMIqIiCAgIEAOKhWdopSitraWsrIyAKKjozu9LZ8KJLtDsev4Oeqsdsb164GfwXeH0Ox2uyuMwsPDvV0dcYPz9/cHoKysjIiIiE5333zqF6mUYsYftjN39S6qG2zero5XNY8ZBQQEeLkm4mbR/Ld0NeORPhVIfgY9pqZWUb3V4eXaaIN000RX6Yq/JZ8KJACz0fmR66x2L9dECHExnwskf6Ozb1vXKIEkhNb4XiCZmgJJWkiik3Q6He+//763q9Glnn/+eYYPH+7tavhgIDW1kOolkMQNbM2aNYSGhnbZ9p566ilycnK6bHud5VO7/QEs0mUTPqSxsRGTyXTFcoGBgQQGBl6HGl2ez7aQpMt2KaUUtY226754evHkLVu28IMf/IDQ0FDCw8O59957+fbbbwEYM2YMzz77rFv5M2fOYDQa+fzzzwE4ffo0kydPxt/fn/j4eNauXUtcXBzLly/v1Pd28OBB7rrrLvz9/QkPD2f+/PlUV1e7ns/LyyMpKYlu3boRGhrK2LFjOXHiBAD79+/nzjvvJCgoiODgYEaOHMlXX3112ffLy8tj7ty5VFZWotPp0Ol0PP/88wDExcXx0ksvMXv2bIKDg5k/fz4Azz77LAkJCQQEBNC3b18WLVrktnv+4i7bww8/zJQpU1i2bBnR0dGEh4ezYMGCa36Kkc+1kGQMqX11VjuDMj+67u976MU0Akwd/1OsqakhIyODYcOGUV1dTWZmJlOnTmXfvn3MmjWLV155hcWLF7t2Q2/YsIGYmBjGjRsHwOzZsykvLycvLw+j0UhGRobrKGNP1dTUkJaWRkpKCrt27aKsrIxHHnmE9PR01qxZg81mY8qUKcybN49169bR2NjIzp07XXWbNWsWI0aM4M0338RgMLBv3z6MRuNl33PMmDEsX76czMxMCgoKANxaN8uWLSMzM5OsrCzXuqCgINasWUNMTAwHDx5k3rx5BAUF8cwzz7T7Prm5uURHR5Obm8uxY8eYPn06w4cPZ968eZ36rjrC9wJJxpBueNOmTXN7/NZbb9GzZ08OHTrEgw8+yOOPP84XX3zhCqC1a9cyc+ZMdDodR44c4dNPP2XXrl2MGjUKgD/+8Y/079+/U3VZu3Yt9fX1vPPOO3Tr1g2AN954g/vuu48lS5ZgNBqprKzk3nvv5dZbbwVg4MCBrtcXFRXx9NNPM2DAAIAO1cNkMhESEoJOpyMqKuqS5++66y6efPJJt3XPPfec635cXBxPPfUU69evv2wghYWF8cYbb2AwGBgwYACTJ08mJydHAqkryRhS+/yNBg69mOaV9/XE0aNHyczMZMeOHZSXl+NwOA9yLSoqYsiQIdx99928++67jBs3jsLCQvLz8/n9738PQEFBAX5+ftx+++2u7fXr14+wsLBO1f3w4cMkJia6wghg7NixOBwOCgoKGD9+PA8//DBpaWn867/+K6mpqTz44IOu870yMjJ45JFH+NOf/kRqaio/+clPXMHVWc1B29qGDRt47bXX+Pbbb6mursZms11xdtXBgwe7nQISHR3NwYMHr6puV+J7Y0gmOTCyPTqdjgCT33VfPD3C97777uPcuXOsWrWKHTt2sGPHDsA5gAvObtCf//xnrFYra9euZejQoQwdOrTLv6+OWr16Nfn5+YwZM4YNGzaQkJDA9u3bAefYzTfffMPkyZPZunUrgwYNYtOmTVf1fq3DESA/P59Zs2Zxzz33sHnzZvbu3cuvf/1r1/fVnou7jjqdzhX+14rvBZIMat/Qzp49S0FBAc899xwTJkxg4MCBnD9/3q3MAw88QH19PVu2bGHt2rXMmjXL9dxtt92GzWZj7969rnXHjh27ZBsdNXDgQPbv309NTY1r3Zdffoler+e2225zrRsxYgQLFy5k27ZtDBkyhLVr17qeS0hI4IknnuDjjz/mRz/6EatXr77i+5pMJuz2jv0Nb9u2jT59+vDrX/+aUaNG0b9/f9egutb4bCDVS5fthhQWFkZ4eDh/+MMfOHbsGFu3biUjI8OtTLdu3ZgyZQqLFi3i8OHDzJw50/XcgAEDSE1NZf78+ezcuZO9e/cyf/58/P39O3Uu1qxZs7BYLMyZM4evv/6a3Nxcfv7zn/PQQw8RGRlJYWEhCxcuJD8/nxMnTvDxxx9z9OhRBg4cSF1dHenp6eTl5XHixAm+/PJLdu3a5TbG1J64uDiqq6vJycmhvLyc2tradsv279+foqIi1q9fz7fffstrr7121a2wa8XnAskie9luaHq9nvXr17N7926GDBnCE088wdKlSy8pN2vWLPbv38+4cePo3bu323PvvPMOkZGRjB8/nqlTp7r2OFksFo/rExAQwEcffcS5c+cYPXo0P/7xj5kwYQJvvPGG6/kjR44wbdo0EhISmD9/PgsWLOBnP/sZBoOBs2fPMnv2bBISEnjwwQeZNGkSL7zwwhXfd8yYMTz66KNMnz6dnj178sorr7Rb9v777+eJJ54gPT2d4cOHs23bNhYtWuTxZ70edMrTg0A0qKqqipCQECorK684ULf6y0Je+Psh7k+M4bWZI65TDbWnvr6ewsJC4uPjO/VDvJmcOnWK2NhYPv30UyZMmODt6tywLvc31dHfqO/uZZMWks/aunUr1dXVDB06lNOnT/PMM88QFxfH+PHjvV01n+dzXTY5DklYrVZ+9atfMXjwYKZOnUrPnj1dB0m+++67rtMoLl4GDx583eo4adKkduvx8ssvX7d6XG++20KSQW2flZaWRlpa28db3X///SQnJ7f53JWOoO5Kf/zjH6mrq2vzue7du1+3elxvnWohrVixgri4OCwWC8nJyezcubPdsqtWrWLcuHGEhYURFhZGamrqJeWVUmRmZhIdHY2/vz+pqakcPXq0M1W7Ijl1RFxOUFAQ/fr1a3Pp06fPdatHr1692q2HBFIrGzZsICMjg6ysLPbs2UNiYiJpaWntnguUl5fHzJkzyc3NJT8/n9jYWO6++26+//57V5lXXnmF1157jZUrV7Jjxw66detGWloa9fX1nf9k7ZDjkITQMOWhpKQktWDBAtdju92uYmJiVHZ2dodeb7PZVFBQkHr77beVUko5HA4VFRWlli5d6ipTUVGhzGazWrduXYe2WVlZqQBVWVl5xbIHTlaoPs9uVikvf9qhbd+s6urq1KFDh1RdXZ23qyJuEpf7m+rob9SjFlJjYyO7d+8mNTXVtU6v15Oamkp+fn6HtlFbW4vVanU1OwsLCykpKXHbZkhICMnJye1us6GhgaqqKrelo+TUESG0y6NAKi8vx263ExkZ6bY+MjKSkpKSDm3j2WefJSYmxhVAza/zZJvZ2dmEhIS4ltjY2A5/BtntL4R2Xdfd/osXL2b9+vVs2rTpqg7GW7hwIZWVla7l5MmTHX5ty25/Bw7HDX9MqBA3FY8CqUePHhgMBkpLS93Wl5aWtjkvS2vLli1j8eLFfPzxxwwbNsy1vvl1nmzTbDYTHBzstnRU8142gAabXJtNeO5mnORfKzwKJJPJxMiRI90mA3c4HOTk5JCSktLu61555RVeeukltmzZcslcLfHx8URFRblts6qqih07dlx2m51l8WsJJOm2iRtVV0/yD8494jqdjoqKii7dric8PjAyIyODOXPmMGrUKJKSkli+fDk1NTXMnTsXcE4P2qtXL7KzswFYsmQJmZmZrnmLm8eFmo861el0PP744/zmN7+hf//+xMfHs2jRImJiYpgyZUrXfdImer0Os5+eBptDAkkIrenM7r3XX39d9e7dW5lMJpWUlKS2b9/ueu6OO+5Qc+bMcT3u06ePAi5ZsrKyXGUcDodatGiRioyMVGazWU2YMEEVFBR0uD6e7PZXSqnEFz5SfZ7drI6WXujwe9xs2txF63Ao1VB9/ReHw6O6/+Mf/1Bjx45VISEhqnv37mry5Mnq2LFjSimlUlJS1DPPPONWvqysTPn5+anPPvtMKaVUcXGxuueee5TFYlFxcXHq3XffVX369FG//e1vO/T+gNq0aZPr8YEDB9Sdd96pLBaL6t69u5o3b566cKHlbys3N1eNHj1aBQQEqJCQEDVmzBh1/PhxpZRS+/btUz/84Q9VYGCgCgoKUrfffrvatWvXZd8/Nze33d9TfX29evLJJ1VMTIwKCAhQSUlJKjc31/Xa48ePq3vvvVeFhoaqgIAANWjQIPXBBx+owsLCS7bZ+nfcEV2x279Tp46kp6eTnp7e5nN5eXluj48fP37F7el0Ol588UVefPHFzlTHY/5GAxVY5Xy2i1lr4eWY6/++vyoGU7crl2sik/y3P8l/eno6hw4dYv369cTExLBp0yYmTpzIwYMH6d+/PwsWLKCxsZHPP/+cbt26cejQIQIDA4mNjeUvf/kL06ZNo6CggODgYPz9/Tv1nVwNnzuXDVp2/Usg3Zhkkv+2J/kvKipi9erVFBUVERPj/I/lqaeeYsuWLaxevZqXX36ZoqIipk2b5prSt2/fvq7XNx8bGBER0eXjUx3l04EkY0gXMQY4WyveeF8PyCT/bTt48CB2u52EhAS39Q0NDYSHhwPwn//5nzz22GN8/PHHpKamMm3aNLe93t7mc9OPAPgbm47WljP+3el0zq7T9V5kkv8umeS/uroag8HA7t272bdvn2s5fPgwv/vd7wB45JFH+O6773jooYc4ePAgo0aN4vXXX++yz3q1fDOQ5Iz/G5ZM8u/U1iT/I0aMwG63U1ZWdskMAa27drGxsTz66KP89a9/5cknn2TVqlWubQIdvnjAteCbgSRjSDcsmeTfqa1J/hMSEpg1axazZ8/mr3/9K4WFhezcuZPs7Gw++OADAB5//HE++ugjCgsL2bNnD7m5ua7369OnDzqdjs2bN3PmzBm3y4FfNx7t19MoT3f7L3h3t+rz7Ga1+ovvrnHNtOtGPtv/k08+UQMHDlRms1kNGzZM5eXlXbIr/sMPP1SAGj9+/CWvLy4uVpMmTVJms1n16dNHrV27VkVERKiVK1d26P0vfq/L7fYvKSlRU6ZMUdHR0cpkMqk+ffqozMxMZbfbVUNDg5oxY4aKjY1VJpNJxcTEqPT09A7/mzz66KMqPDzcbbd/Y2OjyszMVHFxccpoNKro6Gg1depUdeDAAaWUUunp6erWW29VZrNZ9ezZUz300EOqvLzctc0XX3xRRUVFKZ1O55Xd/j43yT/A0xv3s3H3KZ6dOIDHfnh1Vwm9Uckk/y1kkv+uIZP8d5KMIfk2meRfu2QMSfgcmeRfu3yyhSQT/fs2meRfu3wykKTLJtoTFBREUFCQt6tBr169vF0Fr/DpLpsEkvOKL0J0ha74W/LpQKr34S5bc/ejtrbWyzURN4vmv6Wr6dr6ZJfNIl02DAYDoaGhrrPcAwICOnVgoBBKKWpraykrKyM0NBSDwXDlF7XDNwPJz9kw9PW9bM2nE3R26g0hWgsNDb3iVNZX4pOB1DKo7dtzaut0OqKjo4mIiMBqtXq7OuIGZjQar6pl1Mw3A0mOQ3JjMBi65I9JiKvlk4PachySENrkk4EkxyEJoU2+GUhyHJIQmuTTgdRoc2CXq9cKoRm+GUitrl4rA9tCaIdPBpLZr+VjS7dNCO3wyUDS6XQt40iyp00IzfDJQIKWbpt02YTQDt8NJNnTJoTm+GwgWeTabEJojg8HkrSQhNAanw0kOZ9NCO3x3UByDWr79hn/QmiJzwaSdNmE0B6fDSQ5DkkI7ZFAkhaSEJrhu4EkB0YKoTk+G0gySZsQ2uOzgSRdNiG0p1OBtGLFCuLi4rBYLCQnJ7Nz5852y37zzTdMmzaNuLg4dDody5cvv6TM888/j06nc1sGDBjQmap1mL+p6UhtCSQhNMPjQNqwYQMZGRlkZWWxZ88eEhMTSUtLa/dSOrW1tfTt25fFixdf9hIpgwcP5vTp067liy++8LRqHpEDI4XQHo8D6dVXX2XevHnMnTuXQYMGsXLlSgICAnjrrbfaLD969GiWLl3KjBkzMJvN7W7Xz8+PqKgo19KjRw9Pq+YRGUMSQns8CqTGxkZ2795Nampqywb0elJTU8nPz7+qihw9epSYmBj69u3LrFmzKCoqardsQ0MDVVVVbounZKJ/IbTHo0AqLy/HbrcTGRnptj4yMpKSkpJOVyI5OZk1a9awZcsW3nzzTQoLCxk3bhwXLlxos3x2djYhISGuJTY21uP3bBnUllNHhNAKTexlmzRpEj/5yU8YNmwYaWlpfPjhh1RUVPDee++1WX7hwoVUVla6lpMnT3r8ns1dtnrpsgmhGR5dubZHjx4YDAZKS0vd1peWll71Nb1bCw0NJSEhgWPHjrX5vNlsvux4VEfIuWxCaI9HLSSTycTIkSPJyclxrXM4HOTk5JCSktJllaqurubbb78lOjq6y7Z5MTkOSQjt8aiFBJCRkcGcOXMYNWoUSUlJLF++nJqaGubOnQvA7Nmz6dWrF9nZ2YBzIPzQoUOu+99//z379u0jMDCQfv36AfDUU09x33330adPH4qLi8nKysJgMDBz5syu+pyXkFNHhNAejwNp+vTpnDlzhszMTEpKShg+fDhbtmxxDXQXFRWh17c0vIqLixkxYoTr8bJly1i2bBl33HEHeXl5AJw6dYqZM2dy9uxZevbsyQ9+8AO2b99Oz549r/LjtU+OQxJCe3RKqRv+0q1VVVWEhIRQWVlJcHBwh15TWWsl8cWPATj6vydhNGhifF+Im1JHf6M++yu0mFo+urSShNAGnw0kk0GPXue8LwPbQmiDzwZS66vX1jfKwZFCaIHPBhLI6SNCaI1PB5IcHCmEtvh0IMlE/0Joi28HkhwcKYSm+HQgSZdNCG2RQEK6bEJohU8Hkr9R5tUWQkt8PJBkDEkILfHtQJJBbSE0xacDSQa1hdAWnw6kluOQ5NQRIbRAAglpIQmhFb4dSDKGJISm+HQgyXFIQmiLTweSdNmE0BbfDiSZfkQITfHtQJIDI4XQFJ8OJBlDEkJbfDyQ5Fw2IbTEpwNJdvsLoS2+HUjSZRNCUySQcHbZboLrZQpxw/PpQLI0ddkcChrtcj6bEN7m04HU3EICqLdKIAnhbT4dSEaDHr+my9fKwLYQ3ufTgQQysC2Elvh8IFnk9BEhNMPnA0lOsBVCOySQms9nky6bEF7n84EkXTYhtMPnA0muzSaEdkggyV42ITTD5wPJInMiCaEZnQqkFStWEBcXh8ViITk5mZ07d7Zb9ptvvmHatGnExcWh0+lYvnz5VW+zK8leNiG0w+NA2rBhAxkZGWRlZbFnzx4SExNJS0ujrKyszfK1tbX07duXxYsXExUV1SXb7EquQW25NpsQXudxIL366qvMmzePuXPnMmjQIFauXElAQABvvfVWm+VHjx7N0qVLmTFjBmazuUu22ZWkhSSEdngUSI2NjezevZvU1NSWDej1pKamkp+f36kKdGabDQ0NVFVVuS2dJfNqC6EdHgVSeXk5drudyMhIt/WRkZGUlJR0qgKd2WZ2djYhISGuJTY2tlPvDTJrpBBackPuZVu4cCGVlZWu5eTJk53elkW6bEJohp8nhXv06IHBYKC0tNRtfWlpabsD1tdim2azud3xKE/JcUhCaIdHLSSTycTIkSPJyclxrXM4HOTk5JCSktKpClyLbXrC3yRHaguhFR61kAAyMjKYM2cOo0aNIikpieXLl1NTU8PcuXMBmD17Nr169SI7OxtwDlofOnTIdf/7779n3759BAYG0q9fvw5t81qSQW0htMPjQJo+fTpnzpwhMzOTkpIShg8fzpYtW1yD0kVFRej1LQ2v4uJiRowY4Xq8bNkyli1bxh133EFeXl6HtnktyRiSENqhUzfB5TaqqqoICQmhsrKS4OBgj16747uzTP/Ddm7t2Y2cJ394bSoohI/r6G/0htzL1pVadvvLkdpCeJsEknTZhNAMnw8ki+z2F0IzJJDk6rVCaIbPB1LzGBJAg03GkYTwJp8PJItfy1cg3TYhvMvnA8nPoMdkkKO1hdACnw8kAItM9C+EJkggIVOQCKEVEkjI+WxCaIUEEq2PRZK9bEJ4kwQSLV02GUMSwrskkJDTR4TQCgkkWo0hyXFIQniVBBKtrs0mLSQhvEoCCemyCaEVEki0OjBSumxCeJUEEnIckhBaIYGEdNmE0AoJJFoNakuXTQivkkBCWkhCaIUEEjKGJIRWSCAhVx4RQiskkJCLRQqhFRJItBpDkkFtIbxKAgmZoE0IrZBAQvayCaEVEkjIGJIQWiGBRKsJ2mQMSQivkkCi5dpsDTYHDodcvVYIb5FAwv3qtfU2aSUJ4S0SSIDFryWQpNsmhPdIIAF6vQ6zn1wsUghvk0BqIsciCeF9EkhN/OXabEJ4nQRSEzk4Ugjv61QgrVixgri4OCwWC8nJyezcufOy5Tdu3MiAAQOwWCwMHTqUDz/80O35hx9+GJ1O57ZMnDixM1XrNDk4Ugjv8ziQNmzYQEZGBllZWezZs4fExETS0tIoKytrs/y2bduYOXMmP/3pT9m7dy9TpkxhypQpfP31127lJk6cyOnTp13LunXrOveJOknGkITwPo8D6dVXX2XevHnMnTuXQYMGsXLlSgICAnjrrbfaLP+73/2OiRMn8vTTTzNw4EBeeuklbr/9dt544w23cmazmaioKNcSFhbWuU/USTJJmxDe51EgNTY2snv3blJTU1s2oNeTmppKfn5+m6/Jz893Kw+QlpZ2Sfm8vDwiIiK47bbbeOyxxzh79my79WhoaKCqqsptuVoWmYJECK/zKJDKy8ux2+1ERka6rY+MjKSkpKTN15SUlFyx/MSJE3nnnXfIyclhyZIlfPbZZ0yaNAm7ve1wyM7OJiQkxLXExsZ68jHa5C9XrxXC6/y8XQGAGTNmuO4PHTqUYcOGceutt5KXl8eECRMuKb9w4UIyMjJcj6uqqq46lPyNcmCkEN7mUQupR48eGAwGSktL3daXlpYSFRXV5muioqI8Kg/Qt29fevTowbFjx9p83mw2Exwc7LZcLdcYknTZhPAajwLJZDIxcuRIcnJyXOscDgc5OTmkpKS0+ZqUlBS38gCffPJJu+UBTp06xdmzZ4mOjvakeldFdvsL4X0e72XLyMhg1apVvP322xw+fJjHHnuMmpoa5s6dC8Ds2bNZuHChq/wvfvELtmzZwn/9139x5MgRnn/+eb766ivS09MBqK6u5umnn2b79u0cP36cnJwcHnjgAfr160daWloXfcwrk0ASwvs8HkOaPn06Z86cITMzk5KSEoYPH86WLVtcA9dFRUXo9S05N2bMGNauXctzzz3Hr371K/r378/777/PkCFDADAYDBw4cIC3336biooKYmJiuPvuu3nppZcwm81d9DGvrGWSNjl1RAhv0SmlbvgZyaqqqggJCaGysrLT40lvbztO1v/9hslDo1kx6/YurqEQvq2jv1E5l62JnMsmhPdJIDWxyLzaQnidBFITaSEJ4X0SSE3kXDYhvE8CqYm/yflVSCAJ4T0SSE3kOCQhvE8CqYm/nO0vhNdJIDVpmaBNDowUwlskkJo0t5Aa7Q5sdgklIbxBAqlJ8xgSQL1NAkkIb5BAamL206PTOe/LOJIQ3iGB1ESn07kuqS27/oXwDgmkVmQaWyG8SwKpFdn1L4R3SSC1YpF5tYXwKgmkVqTLJoR3SSC1IhP9C+FdEkityPlsQniXBFIrMieSEN4lgdSKnM8mhHdJILUik7QJ4V0SSK1Y5DgkIbxKAqkV2e0vhHdJILXSfC6bBJIQ3iGB1IprXu3OdtmOfwFrp8N3eV1XKSF8iARSK1e12//op/A/0+CfW5yhVPj/urh2Qtz8fC+QqsvA0XbgdPrAyCMfwvqZYKsH/+7O23Uz4NRXV1tbIXyKbwWSw+Fsvay6C07uuuRp/85cvfabTfDeQ2BvhEEPwOMHIP4OaKyG//kRlBzsqtoLcdPzrUAq/yecPQan98F/p8L7/+FsMTXx+Dik/Rvgz/8ODhsM/QlMewvMQTBjLcQmQ30lvDMFzvyz6z+LEDch3wqkiAHw890wfJbz8b534fWRkP9/wG71bAxpzzuw6WegHDDif8HU34PBz/mcORD+7T2IGga15fDOA3D++LX5TELcRHRKKeXtSlytqqoqQkJCqKysJDg4uGMvOrkLPnzK2VoC6DmQf45cxN3vQ2SwmX/8Yjzdu5nafu3OVc7XAoz6KdyzjLLqRvK/O8vh0xe4tWc3kuK709tSh27NZDhzBEL7wL9vgeCYq/24ojVrPez9E/zzIwiOhp4Dnf/xRAyCwEhcE6W3VlMOp/c7l5IDztvas2AJAUso+IdeetvjNrj1TjB1u56f7tpTqu3vqIt19Dfqu4EEzsHtvX+CT1+AunMAbLYns9H+Q2zoiQ4NICEqhNsigxkQHUJEiD+6E1/A1t8A8N2ts/ljt3lsLzzHd2dqLtl8RJCZ1FgHvzydQXDdSVSPBHRz/wHdenTJ574hNNbA4b/DgffgwmnwM4PB7Lz1s4CfqenWDFGJznG4oMgrb9da52ylfvFb53bbYgl1BlPEAAgIh9JvnOFT9X3nPoufBfreCQMmQ8JECOx5+fJ2K5z7Ds4VOv/Nw+IhoPt1CQA3tgaoKGqqy0VLRRF06wnRwyFmOEQnOu8HR3dpFSSQPFF7DnJfRn313+hUx06sXWG7n6W26YDzj0ung8ExwQztFcI/S6s5cKoCq9351d6iO8N7pheI0Z3je2Mfynsk0TPQRI9AEyaDHlDO/6kAlB3sNucgucPq/KO2NzbdWp1dRL0BdPqmW4P7LTRtS7nfKofzvsPm3L7D1rJ9R9NjgB4JEDXU2d2MGgoht3j+A3I44MQXsH89HPqbc4C/o3R6iBsHQ6bBwPucP+DWGmth92r48ndQXepcF9wLRj/iDKkzh6HssPPHdrl/y/B+zs8YnQjRwyAoBhqqoK4C6ivcb+vOwYltUHGidUWd44QD7oHb7nE+PnMYyo601KH8qPM7bs0cAt3joHtfZ0B17+v8jqHVv3Wj+32HveXfQKdzfkfomtbpnGXqKqDufFOdz7s/ri69/HfRlsDIpu8m0dmqDwhvtfQA/7CWIYoOkEDqjJKDkJsNFUXY7HbqrTbqGq00NNpotNlAKRQ61trv4r/t9zAgKpiUW8NJ6RtOcnw4IQFG16bqrXb2naxgV+E5dh4/x7kT37BGl0VPXVUXfOLryD+sJaB6DnA+tgQ7B+/NzbdBztbDue9g/zrnYH9lUcs2wuIhcSbEJjl/aLZ65//a9oaW+w0X4Ogn8H2rQyX0Rug3wRlO8XfAgQ2w7TWoOeN8PiQWfvCEcwzPz+xeb2u9cyfGmSNQdsj5n07EIOcPLGqIs86eUMq5nSMfOJfmrv6VGLs5Q6f2LFwo9uw9u5KxG4T3ddal9RLaG6qKoXif8zOd3u/8zjoSYJZQZ0CN/QWMnHPZohJIXazeaufAqUoOnKrgljB/kuPDCWtvjKkNNruDb//5DTVfraOs4gKnK+uprHe2SpTSodChwPm/n8HoXPRGMJjQ+7XcGvQGjHqFUe/AD4WfXmHUKfyaF4MOo58eo8GAyc+An0GPyc+A0WDA6KfHrjNgw4BV+WFVeqzKgBU9DQ4DymYltOYY3S8UEH6hgLCa79ArW4c+n0Pn51bWZgykvM9kzvX/MQ1RozD6GdDpWjUEFShU0y0opTD7GQhtKCa08O9YjmxCX/ZN228W2hvGPQmJ/+bs8nlD5Sko+Acc2ew8Ql9vhJ4J7mNYPQc4Q1PftO/IWufcuXGu0Bne55tuq4pB79f0725qWowtt7rmlm9TK1dd1Po1GJ3jXP5hTWNeYe6Pg6IhMKLjLd3GWij92hlSpV87/wOoPduy1J13Lz9pKSTPv+wmJZBuAGcuNLC36Dx7T1aw58R5Dpyq1NR5dCas9Nd9zyD9cQbpTnCrrphAXR2B1LXcUo9e5/wTsisdnzuG8Rf7eD5xjKSBqwuLgYZiphi3c4/uS2LVaYr10awzP8inxjuw4odDOQPNoRQGvY5hvUJIig8nuW93+vbohu56jdXYGpyB0txlvtnZbc6uYHNAhfaBkF6Xfck1DaQVK1awdOlSSkpKSExM5PXXXycpKand8hs3bmTRokUcP36c/v37s2TJEu655x7X80opsrKyWLVqFRUVFYwdO5Y333yT/v37d6g+N2ogXcxmd1Be3Ui91U6d1U691U691UG9zU5D0zqrTWF1OLDZFVa7A5tDYbM7sNoVNoeDBquDulavr2u0U9tod23ToNdjMugwGvTOxa/lsZ9Bj/4yv+HmH79DKewOhd0BOOz4Oeow2Wqow0ilCqLR7nDWramOVoeDRpvDtUNHh67p1nmBzubcaLA5qK63tRHKip5Uco4g7HTsR98zyExSfHeS47uTHB9O/4hA9Jf7cOKaumaBtGHDBmbPns3KlStJTk5m+fLlbNy4kYKCAiIiIi4pv23bNsaPH092djb33nsva9euZcmSJezZs4chQ4YAsGTJErKzs3n77beJj49n0aJFHDx4kEOHDmGxWLrsw4obg83uoKbRTnWDjZoGGxfqnbcAep0Ovc4ZZHod6PXO2+oGO7tPnGfHd2fZe7KCRpv7GEiQxY/e3QO4JcyfW8IuvvUnyGJsqyqii1yzQEpOTmb06NG88cYbADgcDmJjY/n5z3/OL3/5y0vKT58+nZqaGjZv3uxa9y//8i8MHz6clStXopQiJiaGJ598kqeech7bU1lZSWRkJGvWrGHGjBld9mGFb6i32tl/soKdhefYUXiO3SfOX7Er3M1kIMhiJNDiR6DZj6Cm20CzH4EWP8x+Bgx6MOj1GHS6lvt6Z0g2u3RsrOU9XDvKaGkpNq93tjzB7lCoi+4D+Bn0zvFBvR6jQeccG2xaZ9DrnC1N2m+Buu43l2l+f13zfuIra+4CN2/DeV9HXI8AbgkLuOxrO/ob7fh+O6CxsZHdu3ezcOFC1zq9Xk9qair5+fltviY/P5+MjAy3dWlpabz//vsAFBYWUlJSQmpqquv5kJAQkpOTyc/PbzOQGhoaaGhocD2uqrrB9lyJa8piNJDcN5zkvuH8HGi0OfiuvJrvz9dx6nwdp87XNt0675+vtVLTaKem0Q7yp+Sx5yYP5JFxfbtkWx4FUnl5OXa7nchI9wPXIiMjOXLkSJuvKSkpabN8SUmJ6/nmde2VuVh2djYvvPCCJ1UXPszkp2dAVDADotr+n7m6wUb5hQaqm7qH1Q02qhusVNfbqKp3rmu0OVrGzpTC4VDYHM5be1Mrprml0dwSoVVrBVpaT84HNN9DKdXU9XR2P5tbPHodGHTOva82h8Jqc44ZNtod2JrG6BrtjqaWVKuWWav7DqVcLTXVVAmFe5mOjP277x1tqTdAWEDX7en0KJC0YuHChW6trqqqKmJjY71YI3Eja+6aCe/z6OTaHj16YDAYKC0tdVtfWlpKVFRUm6+Jioq6bPnmW0+2aTabCQ4OdluEEDc+jwLJZDIxcuRIcnJyXOscDgc5OTmkpKS0+ZqUlBS38gCffPKJq3x8fDxRUVFuZaqqqtixY0e72xRC3KSUh9avX6/MZrNas2aNOnTokJo/f74KDQ1VJSUlSimlHnroIfXLX/7SVf7LL79Ufn5+atmyZerw4cMqKytLGY1GdfDgQVeZxYsXq9DQUPW3v/1NHThwQD3wwAMqPj5e1dXVdahOlZWVClCVlZWefhwhxHXQ0d+ox4GklFKvv/666t27tzKZTCopKUlt377d9dwdd9yh5syZ41b+vffeUwkJCcpkMqnBgwerDz74wO15h8OhFi1apCIjI5XZbFYTJkxQBQUFHa6PBJIQ2tbR36icOiKEuOY6+hv1rRkjhRCaJoEkhNCMm+Lgi+ZepxyxLYQ2Nf82rzRCdFME0oULFwDk4EghNO7ChQuEhIS0+/xNMajtcDgoLi4mKCjoinPgNB/VffLkSRkA94B8b50n352zZXThwgViYmLQ69sfKbopWkh6vZ5bbrnFo9fIEd6dI99b5/n6d3e5llEzGdQWQmiGBJIQQjN8LpDMZjNZWVmYzeYrFxYu8r11nnx3HXdTDGoLIW4OPtdCEkJolwSSEEIzJJCEEJohgSSE0AwJJCGEZvhcIK1YsYK4uDgsFgvJycns3LnT21XSlM8//5z77ruPmJgYdDqd63JVzZRSZGZmEh0djb+/P6mpqRw9etQ7ldWQ7OxsRo8eTVBQEBEREUyZMoWCggK3MvX19SxYsIDw8HACAwOZNm3aJXPJ+zqfCqQNGzaQkZFBVlYWe/bsITExkbS0NMrKyrxdNc2oqakhMTGRFStWtPn8K6+8wmuvvcbKlSvZsWMH3bp1Iy0tjfr6+utcU2357LPPWLBgAdu3b+eTTz7BarVy9913U1NT4yrzxBNP8Pe//52NGzfy2WefUVxczI9+9CMv1lqDrum8lRqTlJSkFixY4Hpst9tVTEyMys7O9mKttAtQmzZtcj12OBwqKipKLV261LWuoqJCmc1mtW7dOi/UULvKysoUoD777DOllPN7MhqNauPGja4yhw8fVoDKz8/3VjU1x2daSM1X3W19hdwrXXVXuLvSVYZFi8rKSgC6d+8OwO7du7FarW7f3YABA+jdu7d8d634TCBd7qq77V0hV7jrzFWGfZHD4eDxxx9n7NixDBkyBHB+dyaTidDQULey8t25uymmHxFCSxYsWMDXX3/NF1984e2q3HB8poXUmavuCneducqwr0lPT2fz5s3k5ua6zdEVFRVFY2MjFRUVbuXlu3PnM4HUmavuCndyleH2KaVIT09n06ZNbN26lfj4eLfnR44cidFodPvuCgoKKCoq8vnvzo23R9WvpytddVcodeHCBbV37161d+9eBahXX31V7d27V504cUIpdfVXGb5ZPfbYYyokJETl5eWp06dPu5ba2lpXmUcffVT17t1bbd26VX311VcqJSVFpaSkeLHW2uNTgaTU5a+6K5TKzc1VwCVL89WIr/Yqwzertr4zQK1evdpVpq6uTv3Hf/yHCgsLUwEBAWrq1Knq9OnT3qu0Bsl8SEIIzfCZMSQhhPZJIAkhNEMCSQihGRJIQgjNkEASQmiGBJIQQjMkkIQQmiGBJITQDAkkIYRmSCAJITRDAkkIoRn/H+ZIluxd5QF5AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T07:59:06.744179Z", + "iopub.status.busy": "2024-03-01T07:59:06.743845Z", + "iopub.status.idle": "2024-03-01T08:01:04.891031Z", + "shell.execute_reply": "2024-03-01T08:01:04.890005Z" + }, + "papermill": { + "duration": 118.173831, + "end_time": "2024-03-01T08:01:04.893531", + "exception": false, + "start_time": "2024-03-01T07:59:06.719700", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:01:04.936305Z", + "iopub.status.busy": "2024-03-01T08:01:04.936009Z", + "iopub.status.idle": "2024-03-01T08:01:04.956174Z", + "shell.execute_reply": "2024-03-01T08:01:04.955302Z" + }, + "papermill": { + "duration": 0.043782, + "end_time": "2024-03-01T08:01:04.958202", + "exception": false, + "start_time": "2024-03-01T08:01:04.914420", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.04.330839e-080.0023594.4585950.0117470.1606380.0154590.0000052.3799870.036710.0694610.0485710.0700860.0107166.838582
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.0 4.330839e-08 0.002359 4.458595 0.011747 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.160638 0.015459 0.000005 2.379987 0.03671 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.069461 0.048571 0.070086 0.010716 6.838582 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:01:04.999016Z", + "iopub.status.busy": "2024-03-01T08:01:04.998742Z", + "iopub.status.idle": "2024-03-01T08:01:05.590268Z", + "shell.execute_reply": "2024-03-01T08:01:05.589336Z" + }, + "papermill": { + "duration": 0.614328, + "end_time": "2024-03-01T08:01:05.592373", + "exception": false, + "start_time": "2024-03-01T08:01:04.978045", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:01:05.635590Z", + "iopub.status.busy": "2024-03-01T08:01:05.635268Z", + "iopub.status.idle": "2024-03-01T08:03:10.580103Z", + "shell.execute_reply": "2024-03-01T08:03:10.579304Z" + }, + "papermill": { + "duration": 124.968914, + "end_time": "2024-03-01T08:03:10.582484", + "exception": false, + "start_time": "2024-03-01T08:01:05.613570", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:03:10.625502Z", + "iopub.status.busy": "2024-03-01T08:03:10.625187Z", + "iopub.status.idle": "2024-03-01T08:03:10.641540Z", + "shell.execute_reply": "2024-03-01T08:03:10.640870Z" + }, + "papermill": { + "duration": 0.04006, + "end_time": "2024-03-01T08:03:10.643439", + "exception": false, + "start_time": "2024-03-01T08:03:10.603379", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:03:10.683584Z", + "iopub.status.busy": "2024-03-01T08:03:10.683221Z", + "iopub.status.idle": "2024-03-01T08:03:10.688358Z", + "shell.execute_reply": "2024-03-01T08:03:10.687496Z" + }, + "papermill": { + "duration": 0.027411, + "end_time": "2024-03-01T08:03:10.690288", + "exception": false, + "start_time": "2024-03-01T08:03:10.662877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.5627981924771664}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:03:10.732702Z", + "iopub.status.busy": "2024-03-01T08:03:10.731901Z", + "iopub.status.idle": "2024-03-01T08:03:11.120708Z", + "shell.execute_reply": "2024-03-01T08:03:11.119784Z" + }, + "papermill": { + "duration": 0.412224, + "end_time": "2024-03-01T08:03:11.122843", + "exception": false, + "start_time": "2024-03-01T08:03:10.710619", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDs0lEQVR4nO2deZhT9fX/3zf7nsy+DzMsguw7RUQW+bqgKLW1tlhlpGj7FetC+T5KfSqKC9SnKrS1tPpVRp+qWP3h8lURqRWpWhZBkEXZZhiGmWH2JJPJnvv5/XGTzGSSgUkmyU1mzut58iS59+bm3JnknfM5n885h2OMMRAEQYiIRGwDCIIgSIgIghAdEiKCIESHhIggCNEhISIIQnRIiAiCEB0SIoIgRIeEiCAI0SEhIghCdEiIiAtSWVkJjuNw5swZsU0hBjAkRETcef3117FhwwaxzSDSCBIiIu6QEBHRQkJEEITokBARUbNt2zbMmTMHer0eBoMB06ZNw+uvvw4AmDt3Lj788EPU1NSA4zhwHIeysrI+n5vneTz66KMoLCyERqPBvHnzcOzYMZSVlaGioiJ4XFtbG1atWoVx48ZBp9PBYDDg2muvxaFDh0LOt3PnTnAch3/84x948sknUVxcDJVKhSuvvBKnTp2Kx5+DiAMysQ0g0ovKykosW7YMY8aMwerVq2EymfDNN9/g448/xpIlS/Dwww/DYrHg3LlzeO655wAAOp2uz+dfvXo1nn76aSxatAhXX301Dh06hKuvvhpOpzPkuKqqKrz77ru4+eabUV5ejsbGRvztb3/DnDlzcOzYMRQWFoYcv379ekgkEqxatQoWiwVPP/00br31VuzZs6f/fxSi/zCCuACbN29mAFh1dTUzm81Mr9ezGTNmMIfDEXIcz/PBx9dddx0bMmRI1O91/vx5JpPJ2OLFi0O2P/roowwAW7p0aXCb0+lkPp8v5Ljq6mqmVCrZ2rVrg9s+++wzBoBdeumlzOVyBbdv3LiRAWCHDx+O2k4i/tDQjOgzO3bsQEdHBx566CGoVKqQfRzH9fv8n376KbxeL+6+++6Q7b/+9a/DjlUqlZBIhI+vz+dDa2srdDodRo4ciQMHDoQdf8cdd0ChUASfz549G4DgWRHiQ0JE9JnTp08DAMaOHZuQ89fU1AAAhg8fHrI9MzMTGRkZIdt4nsdzzz2HESNGQKlUIjs7Gzk5Ofj2229hsVjCzl1aWhryPHC+9vb2eF4CESMkRERa8tRTT2HlypW44oor8Pe//x3bt2/Hjh07MGbMGPA8H3a8VCqNeB5GlZJTAgpWE31m2LBhAIAjR46EeS3diXWYNmTIEADAqVOnUF5eHtze2toa5rm8/fbbmDdvHl566aWQ7WazGdnZ2TG9PyEe5BERfeaqq66CXq/HunXrwmaxunsWWq024vDoYlx55ZWQyWTYtGlTyPY///nPYcdKpdIwb+att95CXV1d1O9LiA95RESfMRgMeO6557B8+XJMmzYNS5YsQUZGBg4dOgS73Y5XXnkFADBlyhS8+eabWLlyJaZNmwadTodFixZd9Px5eXm477778Mwzz+CGG27ANddcg0OHDmHbtm3Izs4O8bSuv/56rF27FnfccQcuu+wyHD58GK+99hqGDh2asOsnEojIs3ZEitN9+j7A+++/zy677DKmVquZwWBg06dPZ2+88UZwv81mY0uWLGEmk4kBiGoq3+v1st/97ncsPz+fqdVqNn/+fPbdd9+xrKws9qtf/Sp4nNPpZL/5zW9YQUEBU6vVbNasWew///kPmzNnDpszZ07wuMD0/VtvvRXyPtXV1QwA27x5c7R/EiIBcIxRtI5IbcxmMzIyMvDEE0/g4YcfFtscIgFQjIhIKRwOR9i2QALt3Llzk2sMkTQoRkQkhebmZvh8vl73KxQKZGZm4s0330RlZSUWLlwInU6HL774Am+88QauuuoqzJo1K4kWE8mEhIhICtOmTQsuWIzEnDlzsHPnTowfPx4ymQxPP/00rFZrMID9xBNPJNFaItlQjIhICl9++WXEYVeAjIwMTJkyJYkWEakECRFBEKJDwWqCIEQnrWNEPM+jvr4eer0+LtnfBEHEF8YYOjo6UFhYGKyWEIm0FqL6+nqUlJSIbQZBEBehtrYWxcXFve5PayHS6/UAhIs0GAwiW0MQRE+sVitKSkqC39XeSGshCgzHDAYDCRFBpDAXC51QsJogCNEhISIIQnRIiAiCEJ20jhH1BcYYvF7vBfOciMhIpVLIZDJaGkEknAEtRG63Gw0NDbDb7WKbkrZoNBoUFBSEdMAgiHgzYIWI53lUV1dDKpWisLAQCoWCftmjgDEGt9uN5uZmVFdXY8SIERdckEYQ/UF0Iaqrq8ODDz6Ibdu2wW63Y/jw4di8eTOmTp3ar/O63W7wPI+SkhJoNJo4WTu4UKvVkMvlqKmpgdvtDutlls74eAaphH6YUgVRhai9vR2zZs3CvHnzsG3bNuTk5ODkyZNhPaz6A/2K94+B8Pdze3kwMChlQkuhU002fHmqBT+ZWgK1Qgo0HweavwcMxUDRZIA856QjqhD9/ve/R0lJCTZv3hzc1r2NDEHEg89PNKOpw4kbJxZBI5fiy1MtaOt04/MTTbgmqwn4/iPhwMZjgKcTKL9CXIMHIaL+3L3//vuYOnUqbr75ZuTm5mLSpEl48cUXez3e5XLBarWG3AjiQjRYHDhSZ0GT1QWz3Q2JhMPVY/IBAKfPnYfj2HbhQJM/Z/HsbsBhFsfYQYyoQlRVVYVNmzZhxIgR2L59O/77v/8b9957b7AtTU/WrVsHo9EYvFHCa/8pKysL1oQeiHx1qhUAMLrQgOIMIVaYb1RhaI4WBdZvcb7NChgKgAlLAFMpwPuAhkNimjwoEVWIeJ7H5MmT8dRTT2HSpEm46667cOedd+Kvf/1rxONXr14Ni8USvNXW1ibZYiKdaLI6cbbNDgnH4QdDs0L2TShQI7fze7R2usCXXg5IJEDhJGFn41GA6gUmFVGFqKCgAKNHjw7Zdumll+Ls2bMRj1cqlcEEV0p07cLtdottQkpy6JzQbXZEng5GtTxkX4mvFirOiw7OgFpOGKohewQgkQFOC2BvS7a5gxpRhWjWrFk4fvx4yLYTJ04Ee6AnCreX7/Xm9fF9PtbTh2NjYe7cubjnnntwzz33wGg0Ijs7G7/73e+CLZbLysrw+OOP4/bbb4fBYMBdd90FAPjiiy8we/ZsqNVqlJSU4N5770VnZ2fwvE1NTVi0aBHUajXKy8vx2muvxWRfOuDx8TjR2AEAGF9sDNsvbf4eGRoFmrXDUd3qX/AqlQPGIuFx+5kkWUoAIs+aPfDAA7jsssvw1FNP4Sc/+Qn27t2LF154AS+88EJC3/f5z071uq88W4vFk4qCz1/YdRoeX2Q3vThDjZundsWpXv6yGg53aCrJA/91SUw2vvLKK/jFL36BvXv34uuvv8Zdd92F0tJS3HnnnQCAP/zhD3jkkUewZs0aAMDp06dxzTXX4IknnsDLL7+M5ubmoJgFZiUrKipQX1+Pzz77DHK5HPfeey+amppisi/VqWruhNvLw6CWo8ikDt3p7gTazyBLq0Be4SSUZWu79mWUAe01gPkMUEzF/JOFqEI0bdo0vPPOO1i9ejXWrl2L8vJybNiwAbfeequYZqUEJSUleO6558BxHEaOHInDhw/jueeeCwrR/Pnz8Zvf/CZ4/PLly3Hrrbfi/vvvBwCMGDECf/zjHzFnzhxs2rQJZ8+exbZt27B3715MmzYNAPDSSy/h0ksvTfq1JYNhOVosmlAAL8/CV9S3nAAYD1NeKeZMGhW6z+j/YbHWJ8dQAkAKrKy+/vrrcf311yf1PVfMG97rvp6Lbe+6Ylivx/b8fC+bFb81UD/4wQ9CvkAzZ87EM888E0ze7bny/NChQ/j2229DhluMsWCqy4kTJyCTyUJa9owaNQomkyluNqcSMqkEw3N7qQrYelq4z47greryhH+syybclLrEGUkEEV2IxEAh63toLFHH9hetVhvy3Gaz4Ze//CXuvffesGNLS0tx4sSJZJmW2vi8XfGfrGHw+ng021zgGYQhnEwBaLKAzhag4zyg7P1Hi4gfg1KI0oE9e/aEPN+9ezdGjBgBqVQa8fjJkyfj2LFjGD488hdn1KhR8Hq92L9/f3Bodvz4cZjN5rjanQrsO9MGj4/HmAIjjJrQ2TJYagGfB1BoAV0evq+3YsexxtB4ny5PECLbeSCbhCgZpH8i0QDl7NmzWLlyJY4fP4433ngDf/rTn3Dffff1evyDDz6Ir776Cvfccw8OHjyIkydP4r333sM999wDABg5ciSuueYa/PKXv8SePXuwf/9+LF++HGq1utdzpiOMMRw8a8aeqja02yMsa2jzD8syhwIch3yjkMjb1OECz/snJXR5wn1ncxIsJgASopTl9ttvh8PhwPTp07FixQrcd999wWn6SIwfPx6ff/45Tpw4gdmzZ2PSpEl45JFHUFhYGDxm8+bNKCwsxJw5c3DTTTfhrrvuQm5ubjIuJ2m0drphc3khl3Iozoggsq1Vwn2WEPvL1CigkEng9vJoCwiXxr/40d6aBIsJgIZmKYtcLseGDRuwadOmsH1nzpyJ+Jpp06bhk08+6fWc+fn5+OCDD0K23Xbbbf2yM9U41+4AABSa1JBJe/zOOsyCuHASIEOYWJBIOGRpFWiwONFqcyNbpwQ0mcLx9nZhhTVl4ycc8oiIAcW5dmFxYiCvLIRAkNpQCMi7aitl65QAgFabS9igMgESKcB7hVXWRMIhISIGDIyxoEcUcVgWEKKM0JX7WTqhDG5Lp39oJpEAan9NLBqeJQUamqUgO3fuFNuEtKS10w2H2we5lEOeoUc1ScYAc43w2BQqRAGPqKXD1bUxMIVvbwvGk4jEQUJEDBjMdg8UMgkKjKrwMrCdLYDbDkhlgKEoZFeOXom5I3OCggSgW5yIPKJkQEJEDBiG5+owLGcYXJGSjQPekLFEEKNuqORSTCrtUZ44MDSjGFFSoBgRMaDgOA4qeYRFn4H4kKmPlR1U/ox9EqKkQB4RMSBgLEJyawCeB8z+GlcZkYWovdONBosTBrVMmHHrLkQ0hZ9wyCMiBgQ1rXa8/EU1dp2IsBra1gh4XYBMCejyI77+eGMHth89j6P1/jroSoMgPrxXKBtCJBQSImJAcN7qhMXhgd3tDd8ZHJaVClPzEcjQCFP4FrtH2CCRAkp/9j4NzxIOCRExIDhvcQJA+LQ90Ou0fXdM/uRYs6NbfhrFiZIGCRGR9jDGcN4qCFGBscdCRt4nZNwDQvXFXgjUtO50+eDy+qtsBoXIHEdriUiQEBFpj9XhhcPtg1TCIdu/SrprZ51Qg0ihAbTZvZ5DJZdCoxBm24LDM/KIksbgEiLGAK87+bcoWtO8+uqryMrKgsvlCtm+ePHiAZegGi+aOgRvKEunCE90be82LLvIzFfX8MwvRIEYkcsWN1uJyAyu6XufB/j3M8l/39m/ESr/9YGbb74Z9957L95//33cfPPNAITuGx9++OEFM+sHM83+ZNWc7iujAwTiQ71M23dHr5IDcKLDGRAif7sqF3UUTjSDyyNKA9RqNZYsWRLsvAEAf//731FaWoq5c+eKZ1gKo1HIkGdQhQeqve6uIvh9WMg4qdSEH04qwsh8vwAp/PWq3eQRJZrB5RFJ5YJ3Isb7RsGdd96JadOmoa6uDkVFRaisrERFRUXvC/YGORNLTJhYYgrfYakVgtUqQ1fKxgUIC3QHhmZuuxBnkg6ur0syGVx/WY7r8xBJTCZNmoQJEybg1VdfxVVXXYWjR4/iww8/FNus9CM4LCuLbWW0XC10fuW9gLujT2JGxMbgEqI0Yvny5diwYQPq6uqwYMEClJSUXPxFgxCPj4eE48Kz7YHQQHUfz3Wy0Qa724upZZmCeCl1QmVHl42EKIFQjChFWbJkCc6dO4cXX3wRy5YtE9uclOX7hg48/9kp/PNYY+gOj0NI7QD6FKgGAJ4xbD96Hv8+2dK1ligQJ3J1xMliIhKiCtGjjz4KjuNCbqNGjbr4CwcBRqMRP/rRj6DT6bB48WKxzUlZWjpd8PEsvKecuVZYNqHJ6or1XASlTBrM3O9w+lNFgnEiClgnEtGHZmPGjME///nP4HOZTHSTUoa6ujrceuutUCojTEsTALqqKmb3nLqPYtq+O3qVDE6PDx1Or3DOQKdXmsJPKKJ/62UyGfLzI2dED1ba29uxc+dO7Ny5E3/5y1/ENidlYYyhxSbkhoWtqI62/pAfvUqG5g4XrI6ea4nII0okogvRyZMnUVhYCJVKhZkzZ2LdunUoLS2NeKzL5QpZcWy1DsxfqUmTJqG9vR2///3vMXLkSLHNSVlsLi+cHh8kHIdMbTchctmE0rAcJ2TcR4HBn3MWNjSjGFFCEVWIZsyYgcrKSowcORINDQ147LHHMHv2bBw5cgR6ffi4ft26dXjsscdEsDS59Na3jAil1e8NZWjloakdgSJo2hwhxywKDCrhKxFcXU2LGpOCqMHqa6+9FjfffDPGjx+Pq6++Gh999BHMZjP+8Y9/RDx+9erVsFgswVttbW2SLSZSiRZbb/EhvxBFOSwDAJ3S7xG5/B6RQivcU3G0hCL60Kw7JpMJl1xyCU6dOhVxv1KpjDpwy6JIOCXCSeW/X4ZWgZH5epT0bKYYKPsR5bAMEPqh/XBSUbAsSFCIfB4hZSQNFsSmIym1jshms+H06dMoKCjo97nkcuGDZLfb+32uwUzg7xf4e6YSw3J0WDiuAOOKjV0b3Z1CfAgAjMVRn1OrlKEsW4uMQMxJquhK7fCQV5QoRPWIVq1ahUWLFmHIkCGor6/HmjVrIJVK8bOf/azf55ZKpTCZTGhqagIAaDQaytWKAsYY7HY7mpqaYDKZIJVG6IyRigSGZbro40MR4ThArgV8FkHkaHV1QhBViM6dO4ef/exnaG1tRU5ODi6//HLs3r0bOTk5cTl/YFlAQIyI6DGZTCm5vMLp8cHp8cGolof+wJj9wzJj9MOyAMfPd8Di8ODSAr1QGkShFYqjUZwoYYgqRFu2bEno+TmOQ0FBAXJzc+HxeBL6XgMRuVyesp7QmdZObDt8HqWZGvxoSrchWLA+dexCtLe6FS02N/IMyi4hAkiIEkhKBasThVQqTdkvFBEbLR1dU/dB3Pau+JAp9iRhrVKGFpsbNpo5SxopFawmiL4Sceo+MFumze4SjxjQKoXf505XIPGVhCjRkBARaUlAiLK6C5E59mn77uj9QmRzBRY1+oWIZs0SBgkRkXYEklIBIKt7akcgPmTsX+0mbVCI/B6RnDyiRENCRKQdAW/IoJYHy3bA4wQ6/e2m++kRdQ3NKEaULEiIiLQjYsa9tU6oP6TO6CrdESM6EqKkMyhmzYiBRb5BhRnlmV2rnwFBiICYVlP3JFOrwE2Ti4KeUTDxldI8EgYJEZF25BtVyDf2aB1kCQhRUb/Pr5BJMCSr26ybzJ/m4fMKWfiyzH6/BxEKDc2I9IfnuzwiQ/89oogEvCIP5S4mAhIiIq1wuH0409LZtdgQEILUPo/guVygv300nG62YW91WzAwDrk/b81NQpQISIiItKLObMc739ThvYN1XRut54R7Q3Fs/csicKTOgi9PteC8xSlsCAgReUQJgYSISCuaOwIzZt1XVMcvPhRA7V8WEJw5k/u7wJIQJQQSIiKtiJjaEYwPxU+IAjNmdncgzYM8okRCQkSkFc3+9kG5er8QuTqETqwcBxgK4/Y+GoXfI3IHPCKKESUSEiIibXB5fbD42/wEPSJrvXCvzQFk8ev/FuYRBWNEjri9B9EFCRGRNgS6duhVMqj9Hgss/kB1HBYydifgEdldPTwiGpolBBIiIm1ojtTVNRgfit+wDAC0Cn+aB8WIkgKtrCbShrJsLa4akxec0QLvAzoahcdxDFQDgtf1o8nFUCukYIyBo1mzhEJCRKQNRrUcRnW3jh2dLQDvFWJDcS5qL5NKUJrVrfh+YGjm81K+WQKgoRmRvnT4A9X6grgtZOwVqQKQBNoKkVcUb0iIiLTA5vLiYK0Z9eZus1Yd54V7fWK6jJxutmFPVSuaOpz+tkI0PEsUJEREWtBgduCz75uw83hz18bA1H2cA9UBjtZb8dXp1q40DwVN4ScKEiIiLeiaMfPHZnyero4dCfKINME0j55ricgjijcpI0Tr168Hx3G4//77xTaFSEEaOwSvJM/gr0NkawQYL1RPVBoS8p4apX8tEa2uTjgpIUT79u3D3/72N4wfP15sU4gUhDGGRqvgEQWFKBgfSlygOmwtEXlECUN0IbLZbLj11lvx4osvIiOD+ooT4VgdXjjcPkglXNfQLBgfKkjY+2r9HpHDTRn4iUZ0IVqxYgWuu+46LFiw4KLHulwuWK3WkBsx8AkMy7J1Ssik/o9sd48oQWgUPRstUrA6UYi6oHHLli04cOAA9u3b16fj161bh8ceeyzBVhGpRqM1EB/yp3Z4nIC9VXicQCEKDM3sbq9/dXUgRkTdPOKNaEJUW1uL++67Dzt27IBKpbr4CwCsXr0aK1euDD63Wq0oKelfMz0i9ZlRnoWhOTooZX5vyOb3hlTGLi8lAej8aR6BoDVl4CcO0YRo//79aGpqwuTJk4PbfD4fdu3ahT//+c9wuVyQSqUhr1EqlVAq41fqgUgPFDIJikzqrg3WBuE+gfEhAJBKuMhpHhQjijuiCdGVV16Jw4cPh2y74447MGrUKDz44INhIkQQQTr8QpTAYVlEAt6X1yUk3EroMxovRBMivV6PsWPHhmzTarXIysoK204MXk4323C21Y7huTqUZPqFIIlCVNVsQ3OHC+XZWuTqVQAnEdYveeyAUp/w9x8sUPY9kdKcbrLhaL0VcqlEECJ3J+C0CmuHErSiujvHGqw42WiDQiZBrkEFyFXCgkY3CVE8SSkh2rlzp9gmEClGgz/Pq9Dkn9AIxIc0WXEtDdsbgZkzR/dFjW47xYnijOjriAiiN+xuL9o6hfKwhYFgdXBYlnhvCECwJG346mqaOYsnJEREylJvFryhLJ0CqkBVxqAQJSbjvifd1xIBoEWNCYKEiEhZArWHCo1+b4ixpHtEXYmvPT0iWtQYT2ISoqqqqnjbQRBhNFj8QhQYljktQnyGkwC6vKTYEEx8Dev4Sh5RPIlJiIYPH4558+bh73//O5xOZ7xtIgjwPIPLywPoFqgO5JfpcgBpcuZZAjEiu9sHxhgtakwQMQnRgQMHMH78eKxcuRL5+fn45S9/ib1798bbNmIQI5FwuH1mGZZdXg6jWi5sDNaoTk58CAB0Shl+PKUYt84oFTaQR5QQYhKiiRMnYuPGjaivr8fLL7+MhoYGXH755Rg7diyeffZZNDc3X/wkBNEHjGo5uEC9oQTXqI6EVMKhJFODLJ1SsIMSXxNCv4LVMpkMN910E9566y38/ve/x6lTp7Bq1SqUlJTg9ttvR0NDQ7zsJAYZPM9CN3QPVCeoRnWfoOn7hNAvIfr6669x9913o6CgAM8++yxWrVqF06dPY8eOHaivr8eNN94YLzuJQYTd7cVfd53G+4fq4QsIkr1N6CcmlQGa7KTaU9Vsw+6qVqEcSfehGWMXfiHRZ2KK+D377LPYvHkzjh8/joULF+LVV1/FwoULIZEIulZeXo7KykqUlZXF01ZikFDb5oDLw8Pq8EAqCQzL/PEhXR4gSe6qk+8aOnCisQMKmQR5Rf60DsYLya/yvpWwIS5MTEK0adMmLFu2DBUVFSgoiJx4mJubi5deeqlfxhGDk7NtwoxUaWa3EhzB+FDyh2WaYMlYn+CRyRSCd+axkxDFiZiEaMeOHSgtLQ16QAEYY6itrUVpaSkUCgWWLl0aFyOJwQNjDDWtQiA4VIiSu5CxO+FriTRdQoTMpNszEInJxx02bBhaWlrCtre1taG8vLzfRhGDF4vDgw6nF1IJ17WQkfcBHY3CYxEC1RpFz9XVNIUfb2ISItZLkM5ms/W57CtBRCIwLCswqqAIlIbtbAF4rzAkUie/04smmPjao78ZLWqMG1ENzQL1ojmOwyOPPAKNpst19vl82LNnDyZOnBhXA4nBReT4UGAhY+J6mF0IrdKf+Nqz4ys1WowbUQnRN998A0DwiA4fPgyFQhHcp1AoMGHCBKxatSq+FhKDikKTGna3D0OytF0bk9A66EJoeqR5cNTfLO5EJUSfffYZAKG29MaNG2EwJKbVLzF4mVyagcmlPYZf1m4ekQhoFUKaR0CQaFFj/Ilp1mzz5s3xtoMgIuPzCDEiIOFdO3pD4k/zCELB6rjTZyG66aabUFlZCYPBgJtuuumCx27durXfhhGDj9o2O7J1ymDGOwDA1iQsHlRoAGWKeOAK/7CRahLFjT4LkdFoDCYfGo3GhBlEDE48Ph7vfFMHnjHcMat7xn23jh0iBKoDVDXb0NThQlmWFvnkEcWdPgtR9+EYDc2IeFNvdsDHM+hVMhhU3T6WYvUw68Hx8x34/nwH5FIJ8nNo+j7exLSOyOFwwG7v+ifU1NRgw4YN+OSTT+JmGDG4CEzbl2Rqusp+AF1dO0QWoq4Cad6uGJHXDfi8Ilo1cIhJiG688Ua8+uqrAACz2Yzp06fjmWeewY033ohNmzbF1UBicBBx/ZDHAdhbhcciBaoDBNYSdbp8gMzfaBEgryhOxFyhcfbs2QCAt99+G/n5+aipqcGrr76KP/7xj3E1kBj4ONw+NHe4APQQosC0vTqjK0AsEoGpe4fHK8SqKE4UV2ISIrvdDr1eKIfwySef4KabboJEIsEPfvAD1NTU9Pk8mzZtwvjx42EwGGAwGDBz5kxs27YtFpOINKa23Q7GgGy9Muh5AACsdcK9mIXQ/HQlvvbMNyOPKB7EXDz/3XffRW1tLbZv346rrroKANDU1BTVIsfi4mKsX78e+/fvx9dff4358+fjxhtvxNGjR2Mxi0hTzrZGGJYBXR6RsSjJFoWj6R4jAmhRY5yJSYgeeeQRrFq1CmVlZZgxYwZmzpwJQPCOJk2a1OfzLFq0CAsXLsSIESNwySWX4Mknn4ROp8Pu3btjMYtIU6YPzcR/jc7Dpfndeskz1s0jSgEhUgZaT/NCGVsFzZzFk5hWVv/4xz/G5ZdfjoaGBkyYMCG4/corr8QPf/jDmAzx+Xx466230NnZGRS2nrhcLrhcruBzq9Ua03sRqYVBJcfYoh5r0zpb/KVh5YA2VxzDuqGRS4NpHhwHysCPMzE3h8rPz0d+fmiRqunTp0d9nsOHD2PmzJlwOp3Q6XR45513MHr06IjHrlu3Do899lhM9hJpRsAb0hckvTRsJCjNI7HEJESdnZ1Yv349Pv30UzQ1NYHn+ZD90XSCHTlyJA4ePAiLxYK3334bS5cuxeeffx5RjFavXh0sRQIIHlFJSUksl0CkCPvOtEEq4XBJnh66SIHqFIgPRYQ8orgSkxAtX74cn3/+OW677TYUFBSELkCLEoVCgeHDhwMApkyZgn379mHjxo3429/+FnasUqmEUqmM+b2I1ILnGb4+0w6nx4d8g6qHEPkD1SkQHwoQSPMYkqVBAdUkiisxCdG2bdvw4YcfYtasWfG2BzzPh8SBiIFLU4cLTo9P6I5h6FbZ0+PolnEv/tR9gBONHfiuoQNyKYcCPU3fx5OYhCgjIwOZmf0vGr569Wpce+21KC0tRUdHB15//XXs3LkT27dv7/e5idTnTLci+cG2QUBKLWTsjqb7WqJMmr6PJzFFAR9//HE88sgjIflmsdDU1ITbb78dI0eOxJVXXol9+/Zh+/bt+K//+q9+nZdIDwLrh8qyeoiNpVa4NxYn2aILo4mUb0aNFuNCTB7RM888g9OnTyMvLw9lZWWQy+Uh+w8cONCn81Dfs8GL0+NDg8UJACjN6rGQ0XxWuDel1kREiEcUiBExHvB26wBLxERMQrR48eI4m0EMNmrb7OAZQ6ZW0VV7CBAqMgZqVJtKxTGuF7T+Rot2T89Giw4Son4SkxCtWbMm3nYQgwyrv3fZkJ7ekLVO6GOm1AMqkyi29UbAI7JTo8W4E/OCRrPZjLfffhunT5/G//zP/yAzMxMHDhxAXl4eiopSZ8qVSE2mDMnAuCIjvD3WoIUMy0SsyBiJrgx8H3ieQSJXAw4zBazjQExC9O2332LBggUwGo04c+YM7rzzTmRmZmLr1q04e/ZssFYRQVwIhUwCRc/5ErM/UJ1iwzIAUPvTPLRKmT/Nwx9kd1Pt6v4S06zZypUrUVFRgZMnT4Z0dl24cCF27doVN+OIgYnXx0fe4fN2y7hPPSEKpHlkahXCIl5K84gbMXlE+/bti7jyuaioCOfPn++3UcTAZus3dfD4eMwflYsCY7cgb0e90FpaoQU0aRBzoZpEcSMmIVIqlREz30+cOIGcnJx+G0UMXBxuH+rNDjDWFfwN0n5GuE/B+FCAqmYbGq1Cmkch1SSKGzENzW644QasXbsWHo8HAMBxHM6ePYsHH3wQP/rRj+JqIDGwqGqxgTEgR68MnbYHuoQoozzpdvWVE4027K5qRZ3ZQR5RHIlJiJ555hnYbDbk5OTA4XBgzpw5GD58OPR6PZ588sl420gMIKqahcDu0Jweq6k9zq74UGbqClFgLZHN5e3WaJGEqL/ENDQzGo3YsWMHvvzySxw6dAg2mw2TJ0/GggUL4m0fMYDw+HjU+PPLhufoQneaa4RUCU0WoErdBp66YDcPLwWr40jUQsTzPCorK7F161acOXMGHMehvLwc+fn5YIz1qyQIMbA509IJj09oopij71HOpa1auE9hbwjoEiKb0wvI/WJKHlG/iWpoxhjDDTfcgOXLl6Ourg7jxo3DmDFjUFNTg4qKipjLxBKDg+ONHQCAkfn68B+sdr8QpXB8CAB0/i60Nhc1WownUXlElZWV2LVrFz799FPMmzcvZN+//vUvLF68GK+++ipuv/32uBpJDAxGFxjAgcPI7kXyAcDeJqxQlkhTciFjd7o3WmRSJThOIiS+euyAtO8dbIhQovKI3njjDfz2t78NEyEAmD9/Ph566CG89tprcTOOGFgMzdHhuvEFyNWrQne0nhLujcVCImkKo1UIq6p5xmD38BQnihNRCdG3336La665ptf91157LQ4dOtRvo4hBRstJ4T77EnHt6ANSCYebp5ag4rIyqOVSmsKPE1EJUVtbG/Ly8nrdn5eXh/b29n4bRQwsOpwefHGyBWa7O3yn295VCC1reHINi5EikxoZWgUkEo4aLcaJqITI5/NBJus9rCSVSuH1UtCOCOVovRX7zrThk2ON4TvbTgvT9rocQG1Kum39hhotxoWogtWMMVRUVPTaSYOK3hM94XmGI3UWAMC4nk0UgbQalgWoMztQ09qJXL0Kw6mtUFyISoiWLl160WNoxozoTk2bHR1OL1RyKUbk9ljE6PMAbf4eeFkjkm9cjNS1O7Cnqg2jCw0YrqFgdTyISog2b96cKDuIAUrAG7q0QA+ZtEckoPWUIEYqI6DPj/Dq1CSQ5tHp8gJG8ojigfi9fIkBi8XhwelmGwCE97YHgMajwn3upSmbbR8JvVJI1hUWNQYaLVJxtP5AQkQkjIO1ZjAGDMnSIFvXI67ocXYNy/LGJN+4fhAx8ZU6vvYLEiIiYSikEihkEkwuzQjf2XJcKJKvzQZ0uck3rh8E0jxcHh5uacAjsoloUfojqhCtW7cO06ZNg16vR25uLhYvXozjx4+LaRIRR2YOy8Ly2eXhnTqArmFZmnlDAKCUSaGQCV+dTub39LwuId5FxISoQvT5559jxYoV2L17N3bs2AGPx4OrrroKnZ003h4oKGXS8ARXexvQXiPEhXJHi2NYPwlm4XulgMQ/50NxopiJuZ1QPPj4449DnldWViI3Nxf79+/HFVdcIZJVRH85124HY0BxhjpyWZgGfxpQRnl6LmIEcO3YfChkEuhVciFO5LQIQpSm1yM2ogpRTywWYao3MzNy4XSXyxWyaDJS3WxCXBhj2Hm8Gc0dLswblYuJJabQA3gfcP5b4XHhxGSbFzdyDd0Sd7sLERETKROs5nke999/P2bNmoWxY8dGPGbdunUwGo3BW0lJavVGJ4Dqlk40d7igkEkwqme5DwBoOSHMMCm0aZNbdlGCM2cUsI6VlBGiFStW4MiRI9iyZUuvx6xevRoWiyV4q62tTaKFxMVgjGFvdRsAYEKxCSq5tOcBQO0e4XHhRKH+UJpitrvx1ekW7DvTBij8K8bJI4qZlBia3XPPPfjggw+wa9cuFBcX93qcUqnsNc+NEJ/aNgcaLE7IpRwmDzGFH2CpBawNQnC3aErS7YsnNpcXe6raYFTLMa2IOr72F1GFiDGGX//613jnnXewc+dOlJendplQ4sLsqW4FIKyiDutZBgBn/d5Q/riu4UyaYlB3ra5mCi04gIZm/UBUIVqxYgVef/11vPfee9Dr9cEusUajEWq1+iKvJlKJ2jY7zrU7IJVwmDIkwgJGa4OQW8ZxQMn05BsYZ3QKGSQcBx/PYIcKWoA8on4gaoxo06ZNsFgsmDt3LgoKCoK3N998U0yziBjJ0MgxrsgoTGn3pGqncJ83Jj3aSV8EiYQLrrDu4P3hAhKimBF9aEYMDEoyNbhtZhm8PB++s/W00MVVIgXKZifdtkShV8lgdXhg5RXIBwQhYiytEnhThZSZNSPSH6mEg1LWYybM5wFO7hAeF04eUAv+DH7Pz+L1F/znvUKqBxE1JEREvzhSZ8GBs+3g+V682zP/BhztgFIPlA8cbwgADGphQGF1A5DR8Kw/pMT0PZGe2N1e7DrZDJeHh0omxejCHn29Wk8DtXuFx5dc3fVlHSCMLzbh0nwD9CoZYNUJ3pDbBmizxDYt7SAhImLm3ydb4PLwyDUow1dRdzQCx94TYiaFk4Ds9CkF21d0ShkQ0FaFFrC3kkcUIyREREzUttlxrF7I9Zs/KldorROgvQY4+o7gIRiLgOELRLIyiSioUmN/ICEiosbl9QVbA40vNqLAqBY8H1sTUH9AyK5nDDAUAON+AkgH7sds35k2tNrcuIJTQQMA7g6xTUpLBu4nhEgMvA+7D3wD/flTKJU6cIVTBey1Cdnn3QuD5Y8DRlyV8i2k+8uxeivaOt2YmK8WhMhFq6tjgYSI6BteN3BuLxxn9kFVVYdSBowuNEBu7rZ4USoTagyVzABMg6MygkkjR1unGxZeJawlcpFHFAskRMTFaasGjm8DnBaoAYwpK0CzLB+GklJAZQCUBqElkNIwoIdhkTBpFAA60c776xOREMXE4PrUENERKNtRtVN4rDICQ+dAlzMKujQu4RFPTP7k1zaPfwjq6qDV1TFAQkREhueBEx8Hy7rWqUZAOvIq5GcaLvLCwYVJIwhRi7vb6mqPo2sWjegTtLKaCIf3Ad+9L4gQx8FaMh/v2MfjzQONaLBQa+XuCEMzwOxi4APNFml4FjUkREQoPg9wZCvQ9B0gkYKNvhEfmwvh4YVi+PndazUT0CtlkEo48Ix19TgjIYoaGpoRXfi8ggi1VQlB5zE34agzG3XtjZBLOSwYnRe5K8cgRiLhsHRmGbRKKWTHvgWcLYCLmjpEC3lEhADPA9+91yVC434Ch74MX5xqASA0SzSqI9QZImDUyCGTSoTEXoAqNcYACREhzPIc/xBoPiHUDBr7YyBjCPZUt8Lh9iFbr8TEkghVF4lQAkJEQ7OooaEZAdR8BZw/AnASYMwPgcxyWBwefHtO6DN3xYhsSCU0JOuNVpsLX9e0w2RxYwZAQhQD5BENdppPANW7hMeXXBXMktcpZZg7MgeXFugxJCu9C90nGh/PcKzeilNWCRgYCVEMkEc0mHFagO8/EB4XTxXKdfiRSjiMLzZhfLFJHNvSiAytAhwHWHkVPD4GhdNCixqjhDyiwQpjwPcfCaU6DIXAsCvFtihtkUslMKrlcMl0cLh9whIID623igYSosFKw0GhoL1UBly6CJAIH4VOlxd/312DI3UWam4QBVk6JRgng43511k5zaLak26QEA1GPE6g6nPhcfnckPY+B2vNaO5w4Wi9hdYMRUG2VlhhbWH+eJrTIqI16QcJ0WCk5gth6KDNBoomBze7vTwOnTMDAKYMSf/eY8kkUycIURvvbwzqMItnTBpCQjTYcFqAugPC42HzhXVDfk42dcDl4WFUyzEsh2bKoiFXLwzJnDK9MHNGQ7OoEFWIdu3ahUWLFqGwsBAcx+Hdd98V05zBwdk9QlJrxhAga1jIrqN1QmrC2CIjDcuiJEMjx93zhmHBpJHgwNHQLEpEFaLOzk5MmDABzz//vJhmDB5cHcGyHhhyWciuVpsLdWYHJBwX3haIuCgc528uqTIKG2hoFhWiriO69tprce2114ppwuCidq9QL8dYDJiGhOw66u/IUZatEdrkELER6GTrsgr5exKKfvSFtPrEuVwuuFxdLX2tVspy7jNed5c3VDozbLHdkCwNzA4PxpI3FDONVic+P27G2EYbRufphI4eAQ+JuCBpJdfr1q2D0WgM3kpKBkeB9rjQdFRYvKjOCIsNAcCQLC1umFCIoTk6EYwbGMgkHOrMLrR4VMIaLBqe9Zm0EqLVq1fDYrEEb7W1tWKblB4wBtTtFx4XTabUgwSRoVFAIZPALtHD7vHRzFkUpNXQTKlUQqkcWP3Tk4KlFrA1C6uo88eF7LI6PThyzoIxhUYYNVRvqD9IJBzyDSo4mozocLZDa28V26S0Ia08IiJGAt5Q3lhArg7ZdbTOij3VbdjxXaMIhg08ijLUcMhN6HB6AHub2OakDaJ6RDabDadOnQo+r66uxsGDB5GZmYnS0lIRLRtAuDuFUh8AUDg5ZBfPMxytF9a7jC2iIHU8KDKpcVRugtXqBetsAQ2C+4aoQvT1119j3rx5wecrV64EACxduhSVlZUiWTXAOH8YYLzQh16fF7LrbJsdHU4vVHIphlOQOi7kG1VwKzLg9vFwdbRC5fMOuqaTsSDqX2ju3LmU4Z1IGOuasi+YGLb7iN8bGlWgF2ouE/1GLpWgICcbRosWPM8DjnZAlyO2WSkPSfVAxnxWiFNI5UDu6JBddrcXVc2dAICxhbTWJZ4snlwMsHLA2gDYW0mI+gD9DA5kAt5Q3hhApgjZ9V2DFT6eId+oQo6eZiLjjiZLuKeZsz5BQjRQ8TiA5uPC44IJYbt9PKCQScgbShSabDg9PjgtNBvZF2hoNlBpPCrklelyAX1B2O7p5ZmYWGKitY0J4qtGKbhaMwo8Z1AW/jtA9IA8ooEIY0D9N8Ljgom9rqRWyCSQU5A6IehzigEANnOzkFpDXBD6FA5ErPVAZ4swbZw3JmSX0+NDvdlBs5UJZkhBDtxSLWxODzrbGsQ2J+UhIRqIBLyhnFGAXBWy61iDFW/uq8VHh8+LYNjgwaCSQ2HKBwNQe7ZKbHNSHhKigYa7E2g6JjwumhKyizGGI3XC2qHiDHXPVxJxJqtAyA5oaagR2ZLUh4RooFH/jVAK1lAo3LrRYHGi1eaGXMphZL5eJAMHD0Ul5eAAuNrrYXF4xDYnpSEhGkjwvq5hWfHUsN0Ha80AgEvy9FDJpWH7ifiizR4Cg1oOjceM47VNYpuT0tD0/UCi6RjgsgEKrRAf6obV6cHJRhsAYGKpSQTjBiEKLQoKipDX2QKTqVNsa1Ia8ogGCjwP1PxHeFw8NaRNEAAcqjWDZwwlmZpg6xsi8WQUDkOWVgmplYr4XQgSooFC8/dCOoFcFTFIfa5d6MU+mbyh5GL0lzO21MLr42nZRC/Q0GwgwPNAzZfC4+JpgCw0d4zjOPx0WglqWu0YkqURwcBBjEkQooa6Gnxk/Q5XTShHSSb9D3pCHtFAoOGgsIBRrgKKwoPUgCBGZdlaapyYbFRGQJsNp8sNuaUGe6upamMkSIjSHY8TqN4lPC6bHbaAsdHqhMfHi2AYESR7BApMamQ7a3C2zY7qFgpc94SEKN05/amQaa/NBgonhexyenx495s6bP6yGi02yncSjawRUMmkuFTZDI55setEM3w8xYq6Q0KUzrScAhq+FZJaL7k6bKZsT3Ub7G4fFFIJMjSKXk5CJBxDIaAyoMQgRZHnLNo63TRE6wEJUbriaAe+/0B4XDwVMIU2G2i1uXDwrBkAMHdkLqQSig2JBscB+eMgk0gwS1cPANhb3YbzFqfIhqUOJETpiMcBHH5buNfnA+VzQ3bzPMOOY43gGcPQHC3KsrWimEl0w99ProA/j3EmNxgYWjtpuByAhCjdcNuBQ28Is2RKHTD2R2FdIvZUt6HB4oRSLsHckbkiGUqEoM4Aci4BBw5XKE/gxolFGEPVMYOQEKUTtmbgwKtARyOg0ADjfwqoQvuRnW21Y0+1UCd5/qhcGNXUvTVlGHI5AEDRdgLl8vbgZpvLC+8gn9kkIUoHeB6o3QscqBRiQyojMPHWiN0hCkwqDMvRYXyxEaPyqWliSqHPE4ZojAHHtwFeNzpdXrz9dS3ePVgPh9sntoWikRJC9Pzzz6OsrAwqlQozZszA3r17xTYpNeB5oPEY8PVLwKlPAZ8XyCgDplQI0/V+GGPBX1S5VILrxxdgHg3JUpPhVwpJyZ0twLF3Yel0oNPtQ22bHa/tqcHZVrvYFoqC6EL05ptvYuXKlVizZg0OHDiACRMm4Oqrr0ZT0yAtm8AYYGsCqnYCe/4KHHuva9X0yGuBCT8VhmV+Wm0u/N+3DdhxrKtbBMdxkNAsWWoiVwtxPYkMaD2NwuqtuGW0GhkaOTqcXvy/A+fw3sE61A2ycr4cE/lqZ8yYgWnTpuHPf/4zAIDneZSUlODXv/41HnrooQu+1mq1wmg0wmKxwGBIo2EIY4DPDXidQvDZ0Q442gBbI2A5J2wLEEjbKJ4Kr0QJu8cHc6cHjR1OVDd3os4sJLNKOA4//0EpsnTUoywtaD8DHNkqFNbnOHhMQ3HInoVvzBo4JFr4JErkm9T46bSSYFqOw+2DUiZJqx+Zvn5HRU16dbvd2L9/P1avXh3cJpFIsGDBAvznP/8JO97lcsHl6prytFqtfXujtmrg1D/Dt4doMLvgtrNtdjRaneDA/FtZcDfAMLbICLVMcDDPtdtR7xeIruO7zj02TwWNXDi2weJAvdkZPA8A+DgZrOpitGmH44qZM5GfKfwDv65qxX9Ohzbs4zhgWI4OPxiaRSKUTmSUAVOXCZ/LlpOQt5/GVJzGaKkXDRYnmju9yLBqwXkzAYkMDMDuE80AOEglgsfLgYOEAzhOgkydApfkdVXd3FPV+4JJo0aOUd0qdO470w6+x0rvmvwFcCmzkKtX4uox+cHtb31dC6eXx48nF0OtiF9xPVGFqKWlBT6fD3l5eSHb8/Ly8P3334cdv27dOjz22GPRv5HXJQxv+oHP0Qmv/QIL0FxSgBf+MbzLDp/L0euhjCkASACJFG6pFq0yAxwyI5xyE6zKPHQqssE44V/DS7pmvTQKKaQSDnqVDLl6FQpNKgzP1UGvopmxtERtAsb9GOhsFYramc9CY2/BMIUMZVkMXp4HXB0AAB/PQ+WxRDyNICEKQNv1+WS2VvQ21GG8HLB1P7YtTIjMNic63S4oZaHRm7ZON+xuX/ef17iQVmVAVq9ejZUrVwafW61WlJSUXPyFphIhthKSee5/3MdtJocXCpcXHAcwdO3n/G6yTK8CJMI/zeTyQuHyAVz3U0mC51YYtEKcRypHttsHtcvb9dbdz80hZPp9bKER44qMlEE/0NBmAeWzu577PJC6OyHlfQDvAXweyADMmMTD6eXh9vrg4xl4xoPnGXw8g0ouAbql8ZSW9f6jqZBxgLbLey4uc6JngKZIlw9IlVDKQ4XouvEF4HlAKYtvqWFRhSg7OxtSqRSNjaFteRsbG5Gfnx92vFKphFIZw/BDoQUyy2M1EwCQYQQy+nisQQ/0NWKlVcqgVfbt35BOsQGiH0jlgrfUAwkAjf92MQoz+/520RxbnJGYWkqizpopFApMmTIFn376aXAbz/P49NNPMXPmTBEtIwgimYg+NFu5ciWWLl2KqVOnYvr06diwYQM6Oztxxx13iG0aQRBJQnQhuuWWW9Dc3IxHHnkE58+fx8SJE/Hxxx+HBbAJghi4iL6OqD+k7Toighgk9PU7KvrKaoIgCBIigiBEh4SIIAjRET1Y3R8C4a0+p3oQBJFUAt/Ni4Wi01qIOjqE5e99Wl1NEIRodHR0wGjsvSJlWs+a8TyP+vp66PX6hKQ9BFJIamtrB/SsHF3nwCKVrpMxho6ODhQWFkIi6T0SlNYekUQiQXFxccLfx2AwiP4PTQZ0nQOLVLnOC3lCAShYTRCE6JAQEQQhOiREF0CpVGLNmjWxZfynEXSdA4t0vM60DlYTBDEwII+IIAjRISEiCEJ0SIgIghAdEiKCIERn0AtRNF1mX3zxRcyePRsZGRnIyMjAggUL0qYrbazddLds2QKO47B48eLEGhgnor1Os9mMFStWoKCgAEqlEpdccgk++uijJFkbO9Fe54YNGzBy5Eio1WqUlJTggQcegNN5ga40yYYNYrZs2cIUCgV7+eWX2dGjR9mdd97JTCYTa2xsjHj8kiVL2PPPP8+++eYb9t1337GKigpmNBrZuXPnkmx5dER7nQGqq6tZUVERmz17NrvxxhuTY2w/iPY6XS4Xmzp1Klu4cCH74osvWHV1Ndu5cyc7ePBgki2Pjmiv87XXXmNKpZK99tprrLq6mm3fvp0VFBSwBx54IMmW986gFqLp06ezFStWBJ/7fD5WWFjI1q1b16fXe71eptfr2SuvvJIoE+NCLNfp9XrZZZddxv73f/+XLV26NC2EKNrr3LRpExs6dChzu93JMjEuRHudK1asYPPnzw/ZtnLlSjZr1qyE2hkNg3ZoFugyu2DBguC2C3WZjYTdbofH40FmZhT9WJJMrNe5du1a5Obm4he/+EUyzOw3sVzn+++/j5kzZ2LFihXIy8vD2LFj8dRTT8Hn8yXL7KiJ5Tovu+wy7N+/Pzh8q6qqwkcffYSFCxcmxea+kNZJr/0h2i6zkXjwwQdRWFgY8qFINWK5zi+++AIvvfQSDh48mAQL40Ms11lVVYV//etfuPXWW/HRRx/h1KlTuPvuu+HxeLBmzZpkmB01sVznkiVL0NLSgssvvxyMMXi9XvzqV7/Cb3/722SY3CcGrUfUX9avX48tW7bgnXfegUqlEtucuNHR0YHbbrsNL774IrKzs8U2J6HwPI/c3Fy88MILmDJlCm655RY8/PDD+Otf/yq2aXFl586deOqpp/CXv/wFBw4cwNatW/Hhhx/i8ccfF9u0IIPWI4q2y2x3/vCHP2D9+vX45z//ifHjxyfSzH4T7XWePn0aZ86cwaJFi4LbeJ4HAMhkMhw/fhzDhg1LrNExEMv/s6CgAHK5HFJpV/vkSy+9FOfPn4fb7YZCoYj4OjGJ5Tp/97vf4bbbbsPy5csBAOPGjUNnZyfuuusuPPzwwxesE5QsxLdAJGLtMvv000/j8ccfx8cff4ypU6cmw9R+Ee11jho1CocPH8bBgweDtxtuuAHz5s3DwYMHU7YaZiz/z1mzZuHUqVNBoQWAEydOoKCgICVFCIjtOu12e5jYBMSXpUqqqdjRcjHZsmULUyqVrLKykh07dozdddddzGQysfPnzzPGGLvtttvYQw89FDx+/fr1TKFQsLfffps1NDQEbx0dHWJdQp+I9jp7ki6zZtFe59mzZ5ler2f33HMPO378OPvggw9Ybm4ue+KJJ8S6hD4R7XWuWbOG6fV69sYbb7Cqqir2ySefsGHDhrGf/OQnYl1CGINaiBhj7E9/+hMrLS1lCoWCTZ8+ne3evTu4b86cOWzp0qXB50OGDGEAwm5r1qxJvuFREs119iRdhIix6K/zq6++YjNmzGBKpZINHTqUPfnkk8zr9SbZ6uiJ5jo9Hg979NFH2bBhw5hKpWIlJSXs7rvvZu3t7ck3vBeoDAhBEKIzaGNEBEGkDiREBEGIDgkRQRCiQ0JEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOiQEBFpRWVlJUwmU/D5o48+iokTJwafV1RUpE1ZW6ILEiIiIhUVFeA4Dr/61a/C9q1YsQIcx6GioiLk+HgLQFlZGTZs2BCy7ZZbbsGJEyd6fc3GjRtRWVkZfD537lzcf//9cbWLiD8kRESvlJSUYMuWLXA4HMFtTqcTr7/+OkpLS0WxSa1WIzc3t9f9RqMxxGMi0gMSIqJXJk+ejJKSEmzdujW4bevWrSgtLcWkSZP6de5InsrixYuDXtbcuXNRU1ODBx54ABzHgeM4AOFDs55098wqKirw+eefY+PGjcFzVFdXY/jw4fjDH/4Q8rqDBw+C4zicOnWqX9dFxAYJEXFBli1bhs2bNwefv/zyy7jjjjsS/r5bt25FcXEx1q5di4aGBjQ0NER9jo0bN2LmzJm48847g+coLS0NuyYA2Lx5M6644goMHz48XpdARAEJEXFBfv7zn+OLL75ATU0Nampq8OWXX+LnP/95wt83MzMTUqkUer0e+fn5F62aGQmj0QiFQgGNRhM8h1QqRUVFBY4fPx4sJu/xePD6669j2bJl8b4Moo8M2lKxRN/IycnBddddh8rKSjDGcN1116V9LevCwkJcd911ePnllzF9+nT83//9H1wuF26++WaxTRu0kEdEXJRly5ahsrISr7zySty8BolEElam1OPxxOXcfWH58uXBQPzmzZtxyy23QKPRJO39iVBIiIiLcs0118DtdsPj8eDqq6+OyzlzcnJC4j4+nw9HjhwJOUahUPS7x1hv51i4cCG0Wi02bdqEjz/+mIZlIkNDM+KiSKVSfPfdd8HHvWGxWMJ6oWVlZUUsuD9//nysXLkSH374IYYNG4Znn30WZrM55JiysjLs2rULP/3pT6FUKmMaEpaVlWHPnj04c+YMdDodMjMzIZFIgrGi1atXY8SIERdsmEAkHvKIiD5hMBhgMBgueMzOnTsxadKkkNtjjz0W8dhly5Zh6dKluP322zFnzhwMHToU8+bNCzlm7dq1OHPmDIYNG4acnJyY7F61ahWkUilGjx6NnJwcnD17NrjvF7/4Bdxud1JmAYkLQzWriUHLv//9b1x55ZWora0N65xKJBcSImLQ4XK50NzcjKVLlyI/Px+vvfaa2CYNemhoRgw63njjDQwZMgRmsxlPP/202OYQII+IIIgUgDwigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhIggCNEhISIIQnRIiAiCEJ3/D6MvEAKyHO++AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:03:11.166723Z", + "iopub.status.busy": "2024-03-01T08:03:11.166412Z", + "iopub.status.idle": "2024-03-01T08:03:11.463758Z", + "shell.execute_reply": "2024-03-01T08:03:11.462881Z" + }, + "papermill": { + "duration": 0.321978, + "end_time": "2024-03-01T08:03:11.465833", + "exception": false, + "start_time": "2024-03-01T08:03:11.143855", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCNklEQVR4nO2deXxU9bn/P2f2zGRmsieTZLKwi+wiFHABpFJQhFqXVqtEXG6vtPcqP1qlvgqiV6G9aqWtF6sVorcqVYvWuiGXCigqIBgUkASyk32dfT/n98eZJZMFMpOZOTOZ5/165ZXJOSdnnkPCJ8/3+T4Lw3EcB4IgCAERCW0AQRAECRFBEIJDQkQQhOCQEBEEITgkRARBCA4JEUEQgkNCRBCE4JAQEQQhOBKhDRgJLMuiubkZarUaDMMIbQ5BEP3gOA4mkwn5+fkQiYb2exJaiJqbm6HX64U2gyCIi9DY2IjCwsIhzye0EKnVagD8Q2o0GoGtIQiiP0ajEXq93v9/dSgSWoh8yzGNRkNCRBBxzMVCJxSsJghCcEiICIIQHBIigiAEJ6FjRMOB4zi43W54PB6hTUlqpFIpxGKx0GYQccqoFiKn04mWlhZYrVahTUl6GIZBYWEhUlNThTaFiENGrRCxLIva2lqIxWLk5+dDJpNR0qNAcByHjo4OnD9/HuPHjyfPiBiA4ELU1NSEhx56CB9++CGsVivGjRuHnTt3Yvbs2SO6r9PpBMuy0Ov1UCqVEbKWCJfs7GzU1dXB5XLFhRB5WA5iEf1hihcEFaKenh4sWLAAixYtwocffojs7GycPXsW6enpEXuPC6WVE7FDSG/U6WbBgYNcwgvguXYzDp3rxC2z9UiRiYGOSqDjDKApBApmAeQ5xxxBhei3v/0t9Ho9du7c6T9WWloqoEXEaORAVQfaTXasnFEApVSMQ+c60W1x4kBVO36Q2Q6c+YC/sO004LIApVcJa3ASIqi78O6772L27Nm4+eabkZOTg5kzZ+LFF18c8nqHwwGj0Rj0QRAXosVgw8kmA9qNDvRanRCJGCy9NA8AUH2+FbbTe/gL07w1iw1fArZeYYxNYgQVopqaGmzfvh3jx4/Hnj178O///u/4j//4D7z88suDXr9lyxZotVr/x2gteF24cCEeeOABoc0YFXx+rgsAMDlfg8J0PlaYp1VgTLYKOuM3aO02AhodMP02IK0IYD1AywkhTU5KBBUilmUxa9YsPPnkk5g5cybuu+8+3HvvvXj++ecHvX7Dhg0wGAz+j8bGxhhbHF/s378fDMOgt7dXaFPiknajHQ3dVogYBt8bkxl0brouBTmWM+iyOMAWXQGIRED+TP5k2ymAxv3FFEGFSKfTYfLkyUHHLrnkEjQ0NAx6vVwu9xe4UqErcTFOnDcAAMbnpkKbIg06p/c0QsG4YWI0aGT4pRqyxgMiCWA3ANbuWJub1AgqRAsWLEBlZWXQsaqqKhQXF0f1fZ1udsgPt4cd9rWuYVw7UhwOBx566CHo9XrI5XKMGzcOL730Eurq6rBo0SIAQHp6OhiGQVlZ2UXvZzKZcPvtt0OlUkGn0+H3v//9gKXg//7v/2L27NlQq9XIy8vDbbfdhvb2dv95nye2b98+zJ49G0qlEvPnzx/wsxQSl4dFVZsJADCtUDvgvLjjDNKVMnSoxqG2y5vwKpYC2gL+dU9djCwlAIF3zR588EHMnz8fTz75JG655RYcOXIEL7zwAl544YWovu9zn5wb8lxplgqrZhb4v37hYDVcnsHd9ML0FNw8OxCn2nGoFjZncCnJg9+fMCJb77zzTnzxxRf4wx/+gOnTp6O2thadnZ3Q6/X4+9//jh/96EeorKyERqNBSkrKRe+3bt06HDp0CO+++y5yc3OxceNGHD9+HDNmzPBf43K58Pjjj2PixIlob2/HunXrUFZWhg8++CDoXo888giefvppZGdn42c/+xnWrFmDQ4cOjeh5I0VNhwVONwtNihQFaf3+XZwWoKcOmSoZcvNnoiRLFTiXXgL01AO9dUDhZbE0OakRVIguv/xyvP3229iwYQMee+wxlJaW4tlnn8Xtt98upFlxQ1VVFd544w3s3bsXS5YsAQCMGTPGfz4jIwMAkJOTg7S0tIvez2Qy4eWXX8Zrr72Ga665BgCwc+dO5OfnB123Zs0a/+sxY8bgD3/4Ay6//HKYzeagEo0nnngCV199NQDg4YcfxnXXXQe73Q6FQhHeA0eQsdkqrJiug5vlBuYwdVYBHIu03CJcPXNS8Dmt9w+LsTk2hhIA4iCz+vrrr8f1118f0/dcu2jckOf6J9ved9XYIa/t//u9ZkFkc6AqKiogFov9/9lHSk1NDVwuF+bMmeM/ptVqMXHixKDrjh07hkcffRQnTpxAT08PWJZfYjY0NATF9KZNm+Z/rdPpAADt7e0oKiqKiL0jQSIWYVzOEF0Bu6r5z1mDeKupufwP1mHmP+RUGxcLBBciIZBJhh8ai9a1w2E4S61IY7FYsHTpUixduhSvvvoqsrOz0dDQgKVLl8LpdAZdK5UGAsA+r8MnWnGLxx2I/2SOhdvDosPsAMuBX8JJZIAyE7B0AqZWQD70Hy0iclD9QxwzdepUsCyLAwcODHpeJpMBwLBbnIwZMwZSqRRHjx71HzMYDKiqqvJ/febMGXR1dWHr1q248sorMWnSpKBAdSJwtK4bn1d3wmB1DTxpaAQ8LkCmAlJzcabVhF1HGvH5uc7ANam5/Gdza2wMJkiI4pmSkhKsXr0aa9aswTvvvIPa2lrs378fb7zxBgCguLgYDMPgvffeQ0dHB8xm8wXvp1arsXr1avzyl7/EJ598glOnTuHuu++GSCTyezRFRUWQyWT44x//iJqaGrz77rt4/PHHo/6skYLjOFQ09OJwTTd6rM6BF3R7l2UZYwCGQZ6Wj2e1mxxgWe+mhE+ILB0xsJgASIjinu3bt+Omm27C/fffj0mTJuHee++FxWIBABQUFGDz5s14+OGHkZubi5///OcXvd8zzzyDefPm4frrr8eSJUuwYMECXHLJJf4Ac3Z2NsrLy/Hmm29i8uTJ2Lp1K5566qmoPmMk6bI4YXa4IRUzKEwfZGnbVcN/zuRjfxlKGWQSEZxuFt0+4VJ6kx+tXTGwmAAAhuMSN4XUaDRCq9XCYDAMSG602+2ora1FaWlpXOzixCsWiwUFBQV4+umncffdd0ftfWL186ho7MUnZ9pRnKnEjbP6zdGy9QJfbgcYEbDgPwEpb8euIw1oMdixfKoOE/PUfDLj4T/zyY1Xradq/BFwof+jfUnKYHUy8/XXX+PMmTOYM2cODAYDHnvsMQDAypUrBbYsMpzv4ZMTfXVlQfiC1Jp8vwgBQFaqHC0GO7rMDgBqQJEGiMQA6+azrFPSom120kNCNIrov73en9OnTwMAnnrqKVRWVkImk+Gyyy7Dp59+iqysrFiZGTU4jsP5HhsADL4s8wlRenDmfmYqH/TvtHiXZiIRkJLO75xZu0iIYgAJ0SgiPz8fFRUVFzxfVFSEY8eOxc6oGNJlccLm9EAqZpCr6bf84zigt55/nRYsRFmpcgBAp8kROOjbwrd2++NJRPQgIRpFSCQSjBuXvHkvvVYXZBIRdFrFwDawlk7AaQXEEkBTEHQqWy3HwonZfkECACj5rHUKWMcGEiJi1DAuJxVjs8fCMVixsc8b0up5MeqDQirGzKJ+7YlTvF/bDVGwlOgPbd8TowqGYaCQDtKc3xcfShtmZweFt2KfhCgmkEdEjAo4bpDiVh8sC/R6e1ylDy5EPRYnWgx2aFIk/I5bXyHiONrCjzLkERGjgvouK3Z8VouDVYNkQ5vbALcDkMiB1LxBv7+yzYQ9p1pxqtnbB12u4cWHdfNtQ4ioQkJEjApajXYYbC5Yne6BJ/3LsiJ+a34Q0pX8Fr6/Pk0kBuTe6n1ankUdEiJiVNBqsAPAwG17YMht+76kKflOAr22PvVpFCeKGSREcUi8TfGIN3v6w3EcWo28EOm0/RIZWQ9fcQ/w3ReHwNfT2uLwwOH2djPwC1FvBK0lBoOEaJTSv3fQaMZoc8Pm9EAsYpDlzZIOnGziexDJlIBq6OxxhVQMpYzfbfMvz8gjihnJJUQcB7idsf8Ioa64rKwMBw4cwLZt28AwDBiGQXV1Ne6++26UlpYiJSUFEydOxLZt2wZ836pVq/DEE08gPz/f33Xx888/x4wZM6BQKDB79my88847YBgmKAP75MmTWLZsGVJTU5Gbm4s77rgDnZ2dQ9pTV1c34h9FJGk38d5QZqoMEnG/X+mePsuyi+x8BZZnXiHyxYgcF26vQoyc5Nq+97iAT5+O/fte+f/4zn/DYNu2baiqqsKUKVP8Banp6ekoLCzEm2++iczMTHz++ee47777oNPpcMstt/i/d9++fdBoNNi7dy8AvvJ5xYoVWL58OV577TXU19cPWGL19vZi8eLFuOeee/D73/8eNpsNDz30EG655Rb861//GtSe7OzsCPyjRI4OM1+akd03M9qHLz40xLZ9X9QKKQA7THafEHmrxR00UTjaJJcQJQBarRYymQxKpRJ5eYGt5s2bN/tfl5aW4osvvsAbb7wRJEQqlQp/+ctf/J0bn3/+eTAMgxdffBEKhQKTJ09GU1MT7r33Xv/3/OlPf8LMmTPx5JNP+o/t2LEDer0eVVVVmDBhwqD2xBNKmQS5GsXAQLXbGWiCP4xExplFaZis0yBL7RU0mbdftZM8omiTXEIklvLeiRDvO0Kee+457NixAw0NDbDZbHA6nUEjgAC+taxPhACgsrIS06ZNC+r/07dxPgCcOHECn3zySdB0Dh/V1dWYMGFk45BiwQx9Gmbo0waeMDTywWqFJlCycQEGBLp9SzOnlY8ziZPrv0ssSa5/WYYZ9hIpnti1axfWr1+Pp59+GvPmzYNarcZ///d/4/Dhw0HXqVSqIe4wNGazGStWrMBvf/vbAed8kzkSFv+yrCS8zGhpCt8cjXUDTtOwxIwIj+QSogRBJpMFNcQ/dOgQ5s+fj/vvv99/rLq6+qL3mThxIv7617/C4XBALueXG30b5wPArFmz8Pe//x0lJSWQSAb/dehvTzzh8rAQMczAansgOFA9zHudbTPD6nRjdkkGL17yVL6zo8NMQhRFBN01e/TRR/07Mb6PSZMmXfwbRzklJSU4fPgw6urq0NnZifHjx+Orr77Cnj17UFVVhd/85jcDBGUwbrvtNrAsi/vuuw/fffcd9uzZ4+8/7avLWrt2Lbq7u/GTn/wER48eRXV1Nfbs2YO77rrLLz797YmnkUFnWkx47pNz+L/TbcEnXDa+tAMYVqAaAFiOw55Trfj0bGcgl8gXJ3KYImQxMRiCb99feumlaGlp8X989tlnQpskOOvXr4dYLMbkyZORnZ2NpUuX4sYbb8Stt96KuXPnoqurK8g7GgqNRoN//vOfqKiowIwZM/DII49g48aNAOCPG+Xn5+PQoUPweDy49tprMXXqVDzwwANIS0uDyFsO0d+ehoaG6D18iHRaHPCw3MCZcr2NfNqEMjMQ67kIconYX7lvsntLRfxxIgpYRxPBl2YSiSRud2OEYsKECfjiiy+Cju3cuRM7d+4MOrZlyxb/6/Ly8kHvNX/+fJw4ccL/9auvvgqpVBo0jXX8+PHYvXt3SPbEC76uiln9t+5D2Lbvi1ohgd3lgcnu5u/pm/RKW/hRRXCP6OzZs8jPz8eYMWNw++23x9Vf29HAK6+8gs8++wy1tbV45513/DlCQkyRjTQcx6HTzGeQD8ioDrX/kBe1gv/bbLT1zyUijyiaCOoRzZ07F+Xl5Zg4cSJaWlqwefNmXHnllTh58iTU6oHutMPhgMMR6CtsNNJfqYvR2tqKjRs3orW1FTqdDjfffDOeeOIJoc2KCGaHG3aXByKGQYaqjxA5zHxrWIbhK+5DQOOtORuwNKMYUVQRVIiWLVvmfz1t2jTMnTsXxcXFeOONNwadsbVly5agxD7i4vzqV7/Cr371K6HNiApdXm8oXSUNLu3wNUFTZfM1ZiGg8XpE/uxqSmqMCYIvzfqSlpaGCRMm4Ny5c4Oe37BhAwwGg/+jsbExxhYS8USneaj4kFeIQlyWAUCq3OsRObwekcybm0XN0aKK4MHqvpjNZlRXV+OOO+4Y9LxcLvfnwwyXBB5kO6qIxs8hXSXDxDw19P2HKfrafoS4LAP4eWg/nFngbwviFyKPiy8ZScCE2ERAUCFav349VqxYgeLiYjQ3N2PTpk0Qi8X4yU9+MuJ7S6X8L5LVah0VgdlEx9eWRCwepLF9mIzNTsXY7H6lKU4LHx8CAG3hwG+6CCq5BCp5n/8WYhlf2uFxAy4LCVGUEFSIzp8/j5/85Cfo6upCdnY2rrjiCnz55ZcRqe4Wi8VIS0tDe3s7AECpVA7dXJ2IKizLoqOjA0qlcsjs7YjhW5alhh4fGhSGAaQqwGPgRY6yq6OCoEK0a9euqN7fl5/kEyNCOEQiEYqKiiL2x8Du8sDu8kCbIg2+Z693WaYNfVnmo7LVBIPNhUt0ar41iEzFN0ejOFHUiKsYUaRhGAY6nQ45OTlwuVxCm5PUyGQyf6Z2JKjrsuDDb1tRlKHEjy7rswTz96cOX4iO1Hah0+xErkYeECKAhCiKjGoh8iEWiyMamyCEp9MU2Lr347QG4kNp+rDvrZJL0Gl2wkw7ZzEjrrbvCWK4DLp179stU2UFxCMMfMFqi8NX+EpCFG1IiIiExCdEmX2FqDf8bfu+qL1CZHb4khq9QuQiIYoWJEREwuErSgWAzL6lHb74kDb8ZRkQ8IjMPo9ISh5RtCEhIhIOnzekSZH623bAZQcs3nHTI/SIAkszihHFChIiIuEYtOLe2MT3H0pJD7TuCJNUEqKYkxS7ZsToIk+jwNzSDKSr+gkREFY2dX8yVDLcOKsgkGHtK3ylMo+oQUJEJBx5WgXytP1GBxl8QlQw4vvLJCIUZ/bZdZP0KfNwmgFJxojfgwiGlmZE4sOyAY9IM3KPaFB8XpHLGp37JzkkRERCYXN6UNdpCSQbAnyQ2uPiPZcLzLcPheoOM47UdvsD45B669acJETRgISISCiaeq14++sm/KOiKXDQeJ7/rCkMb37ZIJxsMuDQuU60Guz8AZ8QkUcUFUiIiISiw+TbMeubUR25+JCPFG9agH/nTOptJUNCFBVIiIiEYtDSDn98KHJC5Nsxszp9ZR7kEUUTEiIioejwjg/KUXuFyGHiJ7EyDKDJj9j7KGVej8jp84goRhRNSIiIhMHh9sDgHfPj94iMzfxnVTYgCa2N8IUY4BH5Y0S2iL0HEYCEiEgYfFM71AoJUrweCwzeQHUEEhn74vOIrI5+HhEtzaICCRGRMHQMNtXVHx+K3LIMAFQyb5kHxYhiAmVWEwlDSZYK116a69/RAusBTG386wgGqgHe6/rRrEKkyMTgOA4M7ZpFFRIiImHQpkihTdEGDlg6AdbNx4Yi3NReIhahKLNP833f0szjpnqzKEBLMyJxMXkD1WpdxBIZh0QsA0Tev9vkFUUcEiIiITA73Kho7EVzb59dK1Mr/1mdF5X3rO4w43BNF9pNdu9YIVqeRQsSIiIhaOm14ZMz7dhf2RE46Nu6j3Cg2sepZiM+r+4KlHnIaAs/WpAQEQlBYMfMG5vxuAITO6LkESn9ZR79c4nII4o0cSNEW7duBcMweOCBB4Q2hYhD2ky8V5Kr8fYhMrcBHMt3T5RrovKeSrk3l4iyq6NOXAjR0aNH8ec//xnTpk0T2hQiDuE4Dm1G3iPyC5E/PhS9QPWAXCLyiKKG4EJkNptx++2348UXX0R6Os0VJwZitLlhc3ogFjGBpZk/PqSL2vuqvB6RzUkV+NFGcCFau3YtrrvuOixZsuSi1zocDhiNxqAPYvTjW5ZlpcohEXt/Zft6RFFCKes/aJGC1dFC0ITGXbt24fjx4zh69Oiwrt+yZQs2b94cZauIeKPN6IsPeUs7XHbA2sW/jqIQ+ZZmVqfbm13tixHRNI9II5gQNTY24j//8z+xd+9eKBSKi38DgA0bNmDdunX+r41GI/T6kQ3TI+KfuaWZGJOdCrnE6w2Zvd6QQhvwUqJAqrfMwxe0pgr86CGYEB07dgzt7e2YNWuW/5jH48HBgwfxpz/9CQ6HA2KxOOh75HI55PLItXogEgOZRISCtJTAAWML/zmK8SEAEIuYwcs8KEYUcQQTomuuuQbffvtt0LG77roLkyZNwkMPPTRAhAjCj8krRFFclg2Kz/tyO/iCWxH9jkYKwYRIrVZjypQpQcdUKhUyMzMHHCeSl+oOMxq6rBiXkwp9hlcIYihENR1mdJgcKM1SIUetABgRn7/ksgJyddTfP1mg6nsirqluN+NUsxFSsYgXIqcFsBv53KEoZVT35XSLEWfbzJBJRMjRKACpgk9odJIQRZK4EqL9+/cLbQIRZ7R467zy07wbGr74kDIzoq1hh8K3c2brm9TotFKcKMIInkdEEENhdbrRbeHbw+b7gtX+ZVn0vSEA/pa0A7OraecskpAQEXFLcy/vDWWmyqDwdWX0C1F0Ku770zeXCAAlNUYJEiIibvH1HsrXer0hjou5RxQofO3vEVFSYyQJS4hqamoibQdBDKDF4BUi37LMbuDjM4wISM2NiQ3+wtcBE1/JI4okYQnRuHHjsGjRIvz1r3+F3W6PtE0EAZbl4HCzAPoEqn31ZanZgDg2+yy+GJHV6QHHcZTUGCXCEqLjx49j2rRpWLduHfLy8vBv//ZvOHLkSKRtI5IYkYjBnfNKsOaKUmhTpPxBf4/q2MSHACBVLsFNlxXi9rlF/AHyiKJCWEI0Y8YMbNu2Dc3NzdixYwdaWlpwxRVXYMqUKXjmmWfQ0dFx8ZsQxDDQpkjB+PoNRblH9WCIRQz0GUpkpsp5O6jwNSqMKFgtkUhw44034s0338Rvf/tbnDt3DuvXr4der8edd96JlpaWSNlJJBksywUf6BuojlKP6mFB2/dRYURC9NVXX+H++++HTqfDM888g/Xr16O6uhp79+5Fc3MzVq5cGSk7iSTC6nTj+YPVePdEMzw+QbJ28/PExBJAmRVTe2o6zPiypotvR9J3acZxF/5GYtiEFfF75plnsHPnTlRWVmL58uV45ZVXsHz5cohEvK6VlpaivLwcJSUlkbSVSBIau21wuFgYbS6IRb5lmTc+lJoLiGKbdfJdiwlVbSbIJCLkFnjLOjiWL36VDq+FDXFhwhKi7du3Y82aNSgrK4NON3jhYU5ODl566aURGUckJw3d/I5UUUafFhz++FDsl2VKf8tYD++RSWS8d+aykhBFiLCEaO/evSgqKvJ7QD44jkNjYyOKioogk8mwevXqiBhJJA8cx6G+iw8EBwtRbBMZ+zIwl0gZECJkxNye0UhYPu7YsWPR2dk54Hh3dzdKS0tHbBSRvBhsLpjsbohFTCCRkfUApjb+tQCBaqWsf3Y1beFHmrCEiBsiSGc2m4fd9pUgBsO3LNNpFZD5WsNaOgHWzS+JUmI/6UXpL3ztN9+MkhojRkhLM1+/aIZhsHHjRiiVAdfZ4/Hg8OHDmDFjRkQNJJKLweNDvkTG6M0wuxAqubfwtf/EVxq0GDFCEqKvv/4aAO8Rffvtt5DJZP5zMpkM06dPx/r16yNrIZFU5KelwOr0oDhTFTgYg9FBF0LZr8yDoflmESckIfrkk08A8L2lt23bBo0mOqN+ieRlVlE6ZhX1W34Z+3hEAqCS8WUePkGipMbIE9au2c6dOyNtB0EMjsfFx4iAqE/tGAqRt8zDDwWrI86whejGG29EeXk5NBoNbrzxxgteu3v37hEbRiQfjd1WZKXK/RXvAABzO588KFMC8jjxwGXeZSP1JIoYwxYirVbrLz7UarVRM4hITlweFm9/3QSW43DXgr4V930mdggQqPZR02FGu8mBkkwV8sgjijjDFqK+yzFamhGRprnXBg/LQa2QQKPo82sp1AyzflS2mnCm1QSpWIS8bNq+jzRh5RHZbDZYrYEfQn19PZ599ll8/PHHETOMSC582/b6DGWg7QcQmNohsBAFGqS5AzEitxPwuAW0avQQlhCtXLkSr7zyCgCgt7cXc+bMwdNPP42VK1di+/btETWQSA4GzR9y2QBrF/9aoEC1D18ukcXhASTeQYsAeUURIuwOjVdeeSUA4K233kJeXh7q6+vxyiuv4A9/+MOw77N9+3ZMmzYNGo0GGo0G8+bNw4cffhiOSUQCY3N60GFyAOgnRL5t+5T0QIBYIHxb9zaXm49VUZwoooQlRFarFWo13w7h448/xo033giRSITvfe97qK+vH/Z9CgsLsXXrVhw7dgxfffUVFi9ejJUrV+LUqVPhmEUkKI09VnAckKWW+z0PAICxif8sZCM0L4HC1/71ZuQRRYKwm+e/8847aGxsxJ49e3DttdcCANrb20NKclyxYgWWL1+O8ePHY8KECXjiiSeQmpqKL7/8MhyziASloWuQZRkQ8Ii0BTG2aCDKvjEigJIaI0xYQrRx40asX78eJSUlmDt3LubNmweA945mzpwZliEejwe7du2CxWLx349IDuaMycD3J+fikrw+s+Q5ro9HFAdCJPeNnmb5NrYy2jmLJGFlVt9000244oor0NLSgunTp/uPX3PNNfjhD38Y0r2+/fZbzJs3D3a7HampqXj77bcxefLkQa91OBxwOBz+r41GYzjmE3GGRiHFlIJ+uWmWTm9rWCmgyhHGsD4opWJ/mQfDgCrwI0zYw6Hy8vKQlxfcpGrOnDkh32fixImoqKiAwWDAW2+9hdWrV+PAgQODitGWLVuwefPmcE0mEgmfN6TWxbw17GBQmUd0YbihmgtdAIvFgq1bt2Lfvn1ob28Hy7JB50cyCXbJkiUYO3Ys/vznPw84N5hHpNfrYTAYqAA3QTla1w2xiMGEXDVS+waqz7wPtHwDFM8DxiwUzL4hOf8VcHYvkDMJuDS0VUAyYTQaodVqL/p/NCyP6J577sGBAwdwxx13QKfTBSegjRCWZYPEpi9yuRxyuTxi70UIC8ty+KquB3aXB3kaRbAQ+QLVcRAf8uEr8yjOVEJHPYkiSlhC9OGHH+L999/HggULRvTmGzZswLJly1BUVASTyYTXXnsN+/fvx549e0Z0XyIxaDc5YHd5+OkYmj6dPV22PhX3wm/d+6hqM+G7FhOkYgY6NW3fR5KwhCg9PR0ZGSNvGt7e3u4fxKjVajFt2jTs2bMH3//+90d8byL+qevTJN8/NgiIq0TGvij75hJl0PZ9JAlLiB5//HFs3LgRL7/8clC72FChcUPJjS9/qCSzn9gYGvnP2sIYW3RhgnKJpL5WIN5BiwJ2BhgNhCVETz/9NKqrq5Gbm4uSkhJIpdKg88ePH4+IccToxe7yoMVgBwAUZfb7Y9bbwH9O08fYqgsT5BH5YkQcC7j7TIAlwiIsIVq1alWEzSCSjcZuK1iOQ4ZKFug9BPAdGX09qtOKhDFuCFTeQYtWV/9BizYSohESlhBt2rQp0nYQSYbRO7usuL83ZGzi55jJ1YAiTRDbhsLnEVlp0GLECTuhsbe3F2+99Raqq6vxy1/+EhkZGTh+/Dhyc3NRUBA/W65EfHJZcTqmFmjh7peDFrQsi7O4S6AC3wOW5SCSpgC2XgpYR4CwhOibb77BkiVLoNVqUVdXh3vvvRcZGRnYvXs3Ghoa/L2KCOJCyCQiyPqXO/Z6A9VxtiwDgBRvmYdKLvGWeXgD1k7qXT1SwsqdX7duHcrKynD27Nmgya7Lly/HwYMHI2YcMTpxe9jBT3jcfSru40+IfGUeGSoZn8RLZR4RIyyP6OjRo4OWYBQUFKC1tXXERhGjm91fN8HlYbF4Ug502j5BXlMzP1papgKUCRBzoZ5EESMsIZLL5YNWvldVVSE7O3vERhGjF5vTg+ZeGzguEPz101PHf47D+JCPmg4z2ox8mUc+9SSKGGEtzW644QY89thjcLlcAACGYdDQ0ICHHnoIP/rRjyJqIDG6qOk0g+OAbLU8eNseCAhRemnM7RouVW1mfFnThaZeG3lEESQsIXr66adhNpuRnZ0Nm82Gq6++GuPGjYNarcYTTzwRaRuJUURNBx/YHZPdL5vaZQ/EhzLiV4h8uURmh7vPoEUSopES1tJMq9Vi7969OHToEE6cOAGz2YxZs2ZhyZIlkbaPGEW4PCzqvfVl47JTg0/21vOlEspMQBG/AzxT/dM83BSsjiAhCxHLsigvL8fu3btRV1cHhmFQWlqKvLw8cBwX0ZYgxOiirtMCl4cfopit7tfOpbuW/xzH3hAQECKz3Q1IvWJKHtGICWlpxnEcbrjhBtxzzz1oamrC1KlTcemll6K+vh5lZWUht4klkovKNhMAYGKeeuAfrB6vEMVxfAgAUr1TaM0OGrQYSULyiMrLy3Hw4EHs27cPixYtCjr3r3/9C6tWrcIrr7yCO++8M6JGEqODyToNGDCY2LdJPgBYu/kMZZE4LhMZ+9J30CInloNhRHzhq8sKiKlLaLiE5BG9/vrr+PWvfz1AhABg8eLFePjhh/Hqq69GzDhidDEmOxXXTdMhR60IPtF1jv+sLeQLSeMYlYzPqmY5DlYXS3GiCBGSEH3zzTf4wQ9+MOT5ZcuW4cSJEyM2ikgyOs/yn7MmCGvHMBCLGNw8W4+y+SVIkYppCz9ChCRE3d3dyM3NHfJ8bm4uenp6RmwUMbow2V347Gwneq3OgSed1kAjtMxxsTUsTArSUpCukkEkYmjQYoQISYg8Hg8kkqHDSmKxGG43Be2IYE41G3G0rhsfn24beLK7mt+2T80GUtJibtuIoUGLESGkYDXHcSgrKxtyksZQ0zeI5IVlOZxsMgAApvYfoggk1LLMR1OvDfVdFuSoFRhHgxYjQkhCtHr16oteQztmRF/qu60w2d1QSMUYn9MvidHjArq9M/Ayx8feuDBp6rHhcE03JudrME5JwepIEJIQ7dy5M1p2EKMUnzd0iU4NibhfJKDrHC9GCi2gzhvku+MTX5mHxeEGtOQRRQLhZ/kSoxaDzYXqDjMADJxtDwBtp/jPOZfEbbX9YKjlfLEun9ToG7RIzdFGAgkRETUqGnvBcUBxphJZqf3iii57YFmWe2nsjRsBgxa+0sTXEUFCREQNmVgEmUSEWUXpA092VvJN8lVZQGpO7I0bAb4yD4eLhVPs84jMAlqU+AgqRFu2bMHll18OtVqNnJwcrFq1CpWVlUKaRESQeWMzcc+VpQMndQCBZVmCeUMAIJeIIZPw/3UsnNfTczv4eBcRFoIK0YEDB7B27Vp8+eWX2Lt3L1wuF6699lpYLLTeHi3IJeKBBa7WbqCnno8L5UwWxrAR4q/Cd4sBkXfPh+JEYRP2OKFI8NFHHwV9XV5ejpycHBw7dgxXXXWVQFYRI+V8jxUcBxSmpwzeFqbFWwaUXpqYSYwAlk3Jg0wigloh5eNEdgMvRAn6PEIjqBD1x2Dgt3ozMgZvnO5wOIKSJgfrm00IC8dx2F/ZgQ6TA4sm5WCGPi34AtYDtH7Dv86fEWvzIkaOpk/hbl8hIsIiboLVLMvigQcewIIFCzBlypRBr9myZQu0Wq3/Q6+Pr9noBFDbaUGHyQGZRIRJ/dt9AEBnFb/DJFMlTG3ZRfHvnFHAOlziRojWrl2LkydPYteuXUNes2HDBhgMBv9HY2NjDC0kLgbHcThS2w0AmF6YBoVU3P8CoPEw/zp/Bt9/KEHptTrxeXUnjtZ1AzJvxjh5RGETF0uzn//853jvvfdw8OBBFBYWDnmdXC4fss6NEJ7GbhtaDHZIxQxmFacNvMDQCBhb+OBuwWUxty+SmB1uHK7phjZFissLaOLrSBFUiDiOwy9+8Qu8/fbb2L9/P0pL47tNKHFhDtd2AeCzqAfMLAOABq83lDc1sJxJUDQpgexqTqYCA9DSbAQIKkRr167Fa6+9hn/84x9Qq9X+KbFarRYpKSkX+W4inmjstuJ8jw1iEYPLigdJYDS28LVlDAPo58TewAiTKpNAxDDwsBysUEAFkEc0AgSNEW3fvh0GgwELFy6ETqfzf/ztb38T0iwiTNKVUkwt0PJb2v2p2c9/zr00McZJXwSRiPFnWJtYb7iAhChsBF+aEaMDfYYSd8wrgZtlB57squanuIrEQMmVMbctWqgVEhhtLhhZGfIAXog4LqEKeOOFuNk1IxIfsYiBXNJvJ8zjAs7u5V/nzxpVCX8ar+dncHsb/rNuvtSDCBkSImJEnGwy4HhDD1h2CO+27lPA1gPI1UDp6PGGAECTwi8ojE4AElqejYS42L4nEhOr042DZzvgcLFQSMSYnN9vrldXNdB4hH89YWngP+soYVphGi7J00CtkADGVN4bcpoBVabQpiUcJERE2Hx6thMOF4scjXxgFrWpDTj9Dz5mkj8TyEqcVrDDJVUuAXzaKlMB1i7yiMKEhIgIi8ZuK04387V+iyfl8KN1fPTUA6fe5j0EbQEwbolAVsYQGXVqHAkkRETIONwe/2igaYVa6LQpvOdjbgeaj/PV9RwHaHTA1FsA8ej9NTta140usxNXMQooAcBpEtqkhGT0/oYQ0YH14MvjX0Pdeg5FYhuusiuAI2a++rxvY7C8qcD4a+N+hPRIOd1sRLfFiRl5KbwQOSi7OhxIiIjh4XYC54/AVncUipomFHHA5HwNpL19khfFEr7HkH4ukJYcnRHSlFJ0W5wwsAo+l8hBHlE4kBARF6e7Fqj8ELAbkALg0hIdOiR50OiLAIUGkGv4kUByzahehg1GmlIGwIIe1tufiIQoLJLrt4YIDV/bjpr9/GuFFhhzNVKzJyE1gVt4RJI0b/Frt8u7BHWYKLs6DEiIiMFhWaDqI39b1ybFeIgnXou8DM1FvjG5SFPyQtTp7JNd7bIFdtGIYUGZ1cRAWA/w3bu8CDEMjPrFeNs6DX873oYWA41W7gu/NAN6HRxY37BFWp6FDAkREYzHBZzcDbR/B4jE4CavxEe9+XCxfDP8vL69mgmo5RKIRQxYjgvMOCMhChlamhEBPG5ehLpr+KDzpTfilD0LTT1tkIoZLJmcO/hUjiRGJGKwel4JVHIxJKe/AeydgIOGOoQKeUQED8sC3/0jIEJTb4FNXYLPznUC4IclalMG6TNEQKuUQiIW8YW9AHVqDAMSIoLf5al8H+io4nsGTbkJSC/G4dou2JweZKnlmKEfpOsiEYxPiGhpFjK0NCOA+s+B1pMAIwIu/SGQUQqDzYVvzvNz5q4anwWxiJZkQ9FlduCr+h6kGZyYC5AQhQF5RMlORxVQe5B/PeFaf5V8qlyChROzcYlOjeLMxG50H208LIfTzUacM4rAgSMhCgPyiJIZuwE48x7/unA2367Di1jEYFphGqYVpgljWwKRrpKBYQAjq4DLw0FmN1BSY4iQR5SscBxw5gO+VYcmHxh7jdAWJSxSsQjaFCkcklTYnB4+BcJF+VahQEKUrLRU8A3txRLgkhWAiP9VsDjc+OuX9TjZZKDhBiGQmSoHx0hg5rx5VvZeQe1JNEiIkhGXHag5wL8uXRg03qeisRcdJgdONRsoZygEslR8hrWB88bT7AYBrUk8SIiSkfrP+KWDKgsomOU/7HSzOHG+FwBwWXHizx6LJRmpvBB1s97BoLZe4YxJQAQVooMHD2LFihXIz88HwzB45513hDQnObAbgKbj/Ouxi/m8IS9n201wuFhoU6QYm007ZaGQo+aXZHaJmt85o6VZSAgqRBaLBdOnT8dzzz0npBnJRcNhvqg1vRjIHBt06lQTX5owpUBLy7IQSVdKcf+isVgycyIYMLQ0CxFBt++XLVuGZcuWCWlCcuEw+dt6oHh+0KkuswNNvTaIGGbgWCDiojCMd7ikQssfoKVZSFCMKJloPML3y9EWAmnFQadOeSdylGQp+TE5RHj4Jtk6jHz9HjEsEuo3zuFwwOEIjPQ1GqnKedi4nQFvqGjegGS74kwlem0uTCFvKGzajHYcqOzFlDYzJuem8hM9fB4ScUESyiPasmULtFqt/0OvT44G7RGh/RSfvJiSPiA2BADFmSrcMD0fY7JTBTBudCARMWjqdaDTpeBzsGh5NmwSSog2bNgAg8Hg/2hsbBTapMSA44CmY/zrgllUehAl0pUyyCQiWEVqWF0e2jkLgYRamsnlcsjlo2t+ekwwNALmDj6LOm9q0Cmj3YWT5w24NF8LrZL6DY0EkYhBnkYBW7sWJnsPVNYuoU1KGAT1iMxmMyoqKlBRUQEAqK2tRUVFBRoaGoQ0a/Th84ZypwDSlKBTp5qMOFzbjb3ftQlg2OijID0FNmkaTHYXYO0W2pyEQVCP6KuvvsKiRYv8X69btw4AsHr1apSXlwtk1SjDaeFbfQBA/qygUyzL4VQzn+8ypYCC1JGgIC0Fp6RpMBrd4CydoEXw8BBUiBYuXEiFldGm9VuAY/k59OrcoFMN3VaY7G4opGKMoyB1RMjTKuCUpcPpYeEwdUHhcSfd0MlwSKhgNREiHBfYstfNGHD6pNcbmqRT8z2XiREjFYugy86CVq0Cy7KArUdokxICkurRTG8DH6cQS4GcyUGnrE43ajosAIAp+ZTrEklWzSoEuFLA2AJYu4DUbKFNinvoz+BoxucN5V4KSGRBp75rMcLDcsjTKpCtpp3IiKPM5D/TztmwICEarbhsQEcl/1o3fcBpDwvIJCLyhqKFMgt2lwd2A+1GDgdamo1W2k7xdWWpOYBaN+D0nNIMzNCnUW5jlPi8TQymsRc6Vx1KBv4dIPpBHtFohOOA5q/517oZQ2ZSyyQiSClIHRXU2YUAAHNvB19aQ1wQ+i0cjRibAUsnv22ce2nQKbvLg+ZeG6VNRJliXTacYhXMdhcs3S1CmxP3kBCNRnzeUPYkQKoIOnW6xYi/HW3EB9+2CmBY8qBRSCFLywMHoLGhRmhz4h4SotGG0wK0n+ZfF1wWdIrjOJxs4nOHCtNT+n8nEWEydUUAgM6WeoEtiX9IiEYbzV/zrWA1+fxHH1oMdnSZnZCKGUzMUwtkYPJQoC8FA8DR0wyDzSW0OXENCdFogvUElmWFswecrmjsBQBMyFVDIRUPOE9EFlVWMTQpUihdvahsbBfanLiGtu9HE+2nAYcZkKn4+FAfjHYXzraZAQAzitIEMC4Jkamg0xUg19KJtDSL0NbENeQRjRZYFqj/gn9dODtoTBAAnGjsBctx0Gco/aNviOiTnj8WmSo5xEZq4nchSIhGCx1n+HICqWLQIPX5Hn4W+yzyhmKL1tvO2NAIt4eltIkhoKXZaIBlgfpD/OvCywFJcO0YwzD48eV61HdZUZypFMDAJCaNF6KWpnp8YPwO104vhT6Dfgb9IY9oNNBSwScwShVAwcAgNcCLUUmWigYnxhqFFlBlwe5wQmqox5Fa6to4GCREiY7LDtQe5F+XXDkggbHNaIfLQ/O1BCVrPHRpKciy16Oh24raTgpc94eEKNGp3sdX2quygPyZQafsLg/e+boJOw/VotNM9U6CkTkeCokYl8g7wHBuHKzqgIelWFFfSIgSmc5zQMs3fFHrhKUDdsoO13bD6vRAJhYhXSkb4iZE1NHkAwoN9BoxClwN6LY4aYnWDxKiRMXWA5x5j39dOBtIKwo63WV2oKKhFwCwcGIOxCKKDQkGwwB5UyERibAgtRkAcKS2G60Gu8CGxQ8kRImIywZ8+xb/WZ0HlC4MOs2yHPaebgPLcRiTrUJJlkoQM4k+eOfJ6dhWTE1zggOHLgstl32QECUaTitw4nV+l0yeCkz50YApEYdru9FisEMuFWHhxByBDCWCSEkHsieAAYOr5FVYOaMAl1J3TD8kRImEuQM4/gpgagNkSmDajwFF8Dyyhi4rDtfyfZIXT8qBNoWmt8YNxVcAAGTdVSiVBqZ7mB1uuJN8Z5OEKBFgWaDxCHC8nI8NKbTAjNsHnQ6hS1NgbHYqphVqMSmPhibGFepcfonGcUDlh4DbCYvDjbe+asQ7Fc2wOT1CWygYcSFEzz33HEpKSqBQKDB37lwcOXJEaJPiA5YF2k4DX70EnNsHeNxAeglwWRm/Xe+F4zj/X1SpWITrp+mwiJZk8cm4a/iiZEsncPodGCw2WJweNHZb8erhejR0WYW2UBAEF6K//e1vWLduHTZt2oTjx49j+vTpWLp0Kdrbk7RtAscB5nagZj9w+Hng9D8CWdMTlwHTf8wvy7x0mR345zct2Hs6MC2CYRiIaJcsPpGm8HE9kQToqkZ+7W7cOjkF6UopTHY3/n78PP5R0YSmJGvny3ACP+3cuXNx+eWX409/+hMAgGVZ6PV6/OIXv8DDDz98we81Go3QarUwGAzQaBJoGcJxgMcJuO188NnWA9i6AXMbYDjPH/PhK9sonA23SA6ry4NeiwttJjtqOyxo6uWLWUUMg59+rwiZqTSjLCHoqQNO7uYb6zMMXGljcMKaia97lbCJVPCI5MhLS8GPL9f7y3JsTg/kElFC/ZEZ7v9RQYtenU4njh07hg0bNviPiUQiLFmyBF988cWA6x0OBxyOwJan0Wgc3ht11wLn/m/g8SAN5i54rKHbijajHQw471HOfxrgMKVAixQJ72Ce77Gi2SsQgesD956Sq4BSyl/bYrChudfuvw8AeBgJjCmF6FaNw1Xz5iEvg/8BflXThS+qgwf2MQwwNjsV3xuTSSKUSKSXALPX8L+XnWch7anGbFRjstiNFoMdHRY30o0qMO4MQCQBB+DLqg4ADMQi3uNlwEDEAAwjQkaqDBNyA103D9cMnTCpVUoxqU+HzqN1PWD7ZXrX5y2BQ56JHLUcSy/N8x9/86tG2N0sbppViBRZ5JrrCSpEnZ2d8Hg8yM3NDTqem5uLM2fODLh+y5Yt2Lx5c+hv5Hbwy5sR4LFZ4LZeIAHNIQZY/gfDOqzwOGxDXspxMgAiQCSGU6xCl0QDm0QLuzQNRnkuLLIscAz/o2FFgV0vpUwMsYiBWiFBjlqB/DQFxuWkQq2gnbGEJCUNmHoTYOnim9r1NkBp7cRYmQQlmRzcLAs4TAAAD8tC4TIMehteQmSAKvD7yZm7MNRSh2OlgLnvtd0DhKjXbIfF6YBcEhy96bY4YXV6+v55jQgJ1QZkw4YNWLdunf9ro9EIvV5/8W9M0/OxlaDKc+/rYR5Ls7khc7jBMACHwHnG6yZL1ApAxP/Q0hxuyBwegOl7K5H/3jKNio/ziKXIcnqQ4nAH3rrvvRkEbb9PyddiaoGWKuhHG6pMoPTKwNceF8ROC8SsB2BdgMcFCYC5M1nY3Sycbg88LAeWY8GyHDwsB4VUBPQp4ykqGfqPpkzCAKqA91xYYkf/AE1Bah4glkMuDRai66bpwLKAXBLZVsOCClFWVhbEYjHa2oLH8ra1tSEvL2/A9XK5HHJ5GMsPmQrIKA3XTABAuhZIH+a1GjUw3IiVSi6BSj68H0MixQaIESCW8t5SP0QAlN6Pi5GfMfy3C+XawvTo9FISdNdMJpPhsssuw759+/zHWJbFvn37MG/ePAEtIwgilgi+NFu3bh1Wr16N2bNnY86cOXj22WdhsVhw1113CW0aQRAxQnAhuvXWW9HR0YGNGzeitbUVM2bMwEcffTQggE0QxOhF8DyikZCweUQEkSQM9/+o4JnVBEEQJEQEQQgOCRFBEIIjeLB6JPjCW8Mu9SAIIqb4/m9eLBSd0EJkMvHp78PKriYIQjBMJhO02qE7Uib0rhnLsmhuboZarY5K2YOvhKSxsXFU78rRc44u4uk5OY6DyWRCfn4+RKKhI0EJ7RGJRCIUFhZG/X00Go3gP9BYQM85uoiX57yQJ+SDgtUEQQgOCRFBEIJDQnQB5HI5Nm3aFF7FfwJBzzm6SMTnTOhgNUEQowPyiAiCEBwSIoIgBIeEiCAIwSEhIghCcJJeiEKZMvviiy/iyiuvRHp6OtLT07FkyZKEmUob7jTdXbt2gWEYrFq1KroGRohQn7O3txdr166FTqeDXC7HhAkT8MEHH8TI2vAJ9TmfffZZTJw4ESkpKdDr9XjwwQdht19gKk2s4ZKYXbt2cTKZjNuxYwd36tQp7t577+XS0tK4tra2Qa+/7bbbuOeee477+uuvue+++44rKyvjtFotd/78+RhbHhqhPqeP2tparqCggLvyyiu5lStXxsbYERDqczocDm727Nnc8uXLuc8++4yrra3l9u/fz1VUVMTY8tAI9TlfffVVTi6Xc6+++ipXW1vL7dmzh9PpdNyDDz4YY8uHJqmFaM6cOdzatWv9X3s8Hi4/P5/bsmXLsL7f7XZzarWae/nll6NlYkQI5zndbjc3f/587i9/+Qu3evXqhBCiUJ9z+/bt3JgxYzin0xkrEyNCqM+5du1abvHixUHH1q1bxy1YsCCqdoZC0i7NfFNmlyxZ4j92oSmzg2G1WuFyuZCREcI8lhgT7nM+9thjyMnJwd133x0LM0dMOM/57rvvYt68eVi7di1yc3MxZcoUPPnkk/B4PLEyO2TCec758+fj2LFj/uVbTU0NPvjgAyxfvjwmNg+HhC56HQmhTpkdjIceegj5+flBvxTxRjjP+dlnn+Gll15CRUVFDCyMDOE8Z01NDf71r3/h9ttvxwcffIBz587h/vvvh8vlwqZNm2JhdsiE85y33XYbOjs7ccUVV4DjOLjdbvzsZz/Dr3/961iYPCyS1iMaKVu3bsWuXbvw9ttvQ6FQCG1OxDCZTLjjjjvw4osvIisrS2hzogrLssjJycELL7yAyy67DLfeeiseeeQRPP/880KbFlH279+PJ598Ev/zP/+D48ePY/fu3Xj//ffx+OOPC22an6T1iEKdMtuXp556Clu3bsX//d//Ydq0adE0c8SE+pzV1dWoq6vDihUr/MdYlgUASCQSVFZWYuzYsdE1OgzC+XnqdDpIpVKIxYHxyZdccglaW1vhdDohk8kG/T4hCec5f/Ob3+COO+7APffcAwCYOnUqLBYL7rvvPjzyyCMX7BMUK4S3QCDCnTL7u9/9Do8//jg++ugjzJ49OxamjohQn3PSpEn49ttvUVFR4f+44YYbsGjRIlRUVMRtN8xwfp4LFizAuXPn/EILAFVVVdDpdHEpQkB4z2m1WgeIjU98uXgpNRU6Wi4ku3bt4uRyOVdeXs6dPn2au++++7i0tDSutbWV4ziOu+OOO7iHH37Yf/3WrVs5mUzGvfXWW1xLS4v/w2QyCfUIwyLU5+xPouyahfqcDQ0NnFqt5n7+859zlZWV3Hvvvcfl5ORw//Vf/yXUIwyLUJ9z06ZNnFqt5l5//XWupqaG+/jjj7mxY8dyt9xyi1CPMICkFiKO47g//vGPXFFRESeTybg5c+ZwX375pf/c1Vdfza1evdr/dXFxMQdgwMemTZtib3iIhPKc/UkUIeK40J/z888/5+bOncvJ5XJuzJgx3BNPPMG53e4YWx06oTyny+XiHn30UW7s2LGcQqHg9Ho9d//993M9PT2xN3wIqA0IQRCCk7QxIoIg4gcSIoIgBIeEiCAIwSEhIghCcEiICIIQHBIigiAEh4SIIAjBISEiEory8nKkpaX5v3700UcxY8YM/9dlZWUJ002SCEBCRAxKWVkZGIbBz372swHn1q5dC4ZhUFZWFnR9pAWgpKQEzz77bNCxW2+9FVVVVUN+z7Zt21BeXu7/euHChXjggQciahcReUiIiCHR6/XYtWsXbDab/5jdbsdrr72GoqIiQWxKSUlBTk7OkOe1Wm2Qx0QkBiRExJDMmjULer0eu3fv9h/bvXs3ioqKMHPmzBHdezBPZdWqVX4va+HChaivr8eDDz4IhmHAMAyAgUuz/vT1zMrKynDgwAFs27bNf4/a2lqMGzcOTz31VND3VVRUgGEYnDt3bkTPRYQHCRFxQdasWYOdO3f6v96xYwfuuuuuqL/v7t27UVhYiMceewwtLS1oaWkJ+R7btm3DvHnzcO+99/rvUVRUNOCZAGDnzp246qqrMG7cuEg9AhECJETEBfnpT3+Kzz77DPX19aivr8ehQ4fw05/+NOrvm5GRAbFYDLVajby8vIs2qxsMrVYLmUwGpVLpv4dYLEZZWRkqKyv9PZxdLhdee+01rFmzJtKPQQyTpO3QSAyP7OxsXHfddSgvLwfHcbjuuusSvoVsfn4+rrvuOuzYsQNz5szBP//5TzgcDtx8881Cm5a0kEdEXJQ1a9agvLwcL7/8csS8BpFINKA7oMvlisi9h8M999zjD8Tv3LkTt956K5RKZczenwiGhIi4KD/4wQ/gdDrhcrmwdOnSiNwzOzs7KO7j8Xhw8uTJoGtkMtmIR/sMdY/ly5dDpVJh+/bt+Oijj2hZJjC0NCMuilgsxnfffed/PRQGg2HACKLMzMxB+1wvXrwY69atw/vvv4+xY8fimWeeQW9vb9A1JSUlOHjwIH784x9DLpeHtSQsKSnB4cOHUVdXh9TUVGRkZEAkEvljRRs2bMD48eMv2KeciD7kERHDQqPRQKPRXPCa/fv3Y+bMmUEfmzdvHvTaNWvWYPXq1bjzzjtx9dVXY8yYMVi0aFHQNY899hjq6uowduxYZGdnh2X3+vXrIRaLMXnyZGRnZ6OhocF/7u6774bT6YzJLiBxYahVLJG0fPrpp7jmmmvQ2Ng4YGAhEVtIiIikw+FwoKOjA6tXr0ZeXh5effVVoU1KemhpRiQdr7/+OoqLi9Hb24vf/e53QptDgDwigiDiAPKICIIQHBIigiAEh4SIIAjBISEiCEJwSIgIghAcEiKCIASHhIggCMEhISIIQnBIiAiCEJz/D9aQrUCUpaDHAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:03:11.509477Z", + "iopub.status.busy": "2024-03-01T08:03:11.509173Z", + "iopub.status.idle": "2024-03-01T08:03:11.709396Z", + "shell.execute_reply": "2024-03-01T08:03:11.708588Z" + }, + "papermill": { + "duration": 0.22411, + "end_time": "2024-03-01T08:03:11.711393", + "exception": false, + "start_time": "2024-03-01T08:03:11.487283", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAj9ElEQVR4nO3df1RUdf4/8OfwawD54Q9+iIjij1U0f5B4RCwzSyRbTZdMN00QjT2r8l2LKGP3hEc9SX1S1Hb9yOaCZLXqmmSmpiKpUWoq/ghM8IgipoCiAgI2DDPv7x99uOs4AwwJ3Lkzz8c5HJz7Y+Z1ufjk3vd93/dVCSEEiIgUxk7uAoiIfguGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDnIX0NH0ej1u3LgBd3d3qFQqucshoocIIXDv3j306NEDdnZNH1/ZXHjduHEDAQEBcpdBRC24du0aevbs2eR8mwsvd3d3AL/+YDw8PGSupv1otVocOHAAEydOhKOjo9zlUBuwlX1aXV2NgIAA6f9qU2wuvBpPFT08PKw+vFxdXeHh4WHVv+i2xNb2aUvNOmywJyJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpks11lbBWdXV1KCgokF7X3NfgaF4RunidgpuL2mDZoKAguLq6dnSJRG2K4WUlCgoKEBISYjT9f0wsm5ubixEjRrR/UUTtiOFlJYKCgpCbmyu9LiytRPz2PKS8NBQD/TobLUukdAwvK+Hq6mpwNGV39TbUOfcxaMhwBPfuJmNlRO3DIhrs169fj8DAQDg7OyM0NBQnTpxoctmnn34aKpXK6Ov3v/99B1ZMRHKTPby2bduG+Ph4LF26FKdPn8bw4cMRERGBmzdvmlw+MzMTpaWl0ld+fj7s7e3x0ksvdXDlRCQn2cMrJSUFsbGxiImJweDBg5GamgpXV1ekp6ebXL5r167o3r279JWVlQVXV1eGF5GNkbXNq76+Hrm5uUhMTJSm2dnZYcKECTh27JhZ75GWloY//vGP6NSpk8n5Go0GGo1Gel1dXQ3g1zv0tVrtI1Rv2RoaGqTv1rydtqRxP1r7/jR3+2QNr4qKCuh0Ovj6+hpM9/X1Neiz1JQTJ04gPz8faWlpTS6TnJyMZcuWGU0/cOCAVfd1ulYDAA44fvw4rufLXQ21paysLLlLaFd1dXVmLafoq41paWkYOnQoRo0a1eQyiYmJiI+Pl143DnQ2ceJEqx7P61zJHSDvFEaPHo3hvbrKXQ61Aa1Wi6ysLISHh1v1eF6NZ0ctkTW8vLy8YG9vj/LycoPp5eXl6N69e7Pr1tbWYuvWrVi+fHmzy6nVaqjVaqPpjo6OVv0L4ODgIH235u20Rdb+u2vutsnaYO/k5ISQkBBkZ2dL0/R6PbKzsxEWFtbsutu3b4dGo8Err7zS3mUSkQWS/bQxPj4e0dHRGDlyJEaNGoW1a9eitrYWMTExAICoqCj4+/sjOTnZYL20tDRMmzYN3bqxAyaRLZI9vGbOnIlbt24hKSkJZWVlCA4Oxr59+6RG/JKSEqPHHxUWFuK7777DgQMH5CiZiCyA7OEFAHFxcYiLizM57/Dhw0bTBg4cCCFEO1dFRJZM9k6qRES/BcOLiBSJ4UVEisTwIiJFYngRkSJZxNVGIjL08DMJgKafS2CrzyRgeBFZoKaeSQAYP5fAVp9JwPAiskAPP5MAaPq5BLb6TAKGF5EFeviZBACfS/AwNtgTkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTnNhJZiCsVtajVNDQ5v+hWrfTdwcH0f91Oagf08erULvVZGoYXkQW4UlGL8asOm7XsG5/nNTv/UMLTNhFgDC8iC9B4xLV2ZjD6+7iZXua+BrsPH8Pkp8PQyUVtNP/SzRq8tu1ss0dv1oThRWRB+vu4YYi/p8l5Wq0WZd7AiN5d4Ojo2MGVWR422BORIjG8iEiReNqoYM1dnTLnyhRgW1enyLowvBTK3KtTLV2ZAmzn6hRZF4aXQrV0daqlK1OA7V2dIuvC8FK4pq5O8coUWTs22BORIjG8iEiRZA+v9evXIzAwEM7OzggNDcWJEyeaXb6yshKLFi2Cn58f1Go1BgwYgL1793ZQtURkKWRt89q2bRvi4+ORmpqK0NBQrF27FhERESgsLISPj4/R8vX19QgPD4ePjw8+//xz+Pv74+rVq+jcuXPHF09EspI1vFJSUhAbG4uYmBgAQGpqKvbs2YP09HS8/fbbRsunp6fjzp07OHr0qNQIHRgY2JElE5GFkC286uvrkZubi8TERGmanZ0dJkyYgGPHjplcZ9euXQgLC8OiRYvw5ZdfwtvbG7NmzcKSJUtgb29vch2NRgONRiO9rq6uBvDr1TitVtuGW9SxGhoapO+mtqNxWnPb2NJ7UMcxZ1+0tE+tZX+aW7ts4VVRUQGdTgdfX1+D6b6+vigoKDC5zuXLl/HNN99g9uzZ2Lt3Ly5duoSFCxdCq9Vi6dKlJtdJTk7GsmXLjKYfOHAArq6uj74hMrlWAwAO+O6773DV9CAEAICsrKxHfg9qf63ZF03tU2vZn3V1dWYtp6h+Xnq9Hj4+Pvjoo49gb2+PkJAQXL9+HR988EGT4ZWYmIj4+HjpdXV1NQICAjBx4kR4eHh0VOlt7vyNaqzKO44nn3wSj/Uw3g6tVousrCyEh4c32c+rpfegjmPOvmhpn1rL/mw8O2qJbOHl5eUFe3t7lJeXG0wvLy9H9+7dTa7j5+cHR0dHg1PEQYMGoaysDPX19XBycjJaR61WQ6027mHu6Oio6M6bjfcrOjg4NLsdzW2nue9B7a81+6KpfWot+9Pc2mXrKuHk5ISQkBBkZ2dL0/R6PbKzsxEWFmZynSeeeAKXLl2CXq+Xpl28eBF+fn4mg4uIrJes/bzi4+OxceNGfPzxx7hw4QIWLFiA2tpa6epjVFSUQYP+ggULcOfOHSxevBgXL17Enj17sHLlSixatEiuTSAimcja5jVz5kzcunULSUlJKCsrQ3BwMPbt2yc14peUlMDO7r/5GhAQgP379+P111/HsGHD4O/vj8WLF2PJkiVybQIRyUT2Bvu4uDjExcWZnHf48GGjaWFhYTh+/Hg7V0VElk7224OIiH4L2Y+8iAjQ6H6BnfN1XKkuhJ2z6U5aDQ0NuNFwAxfuXDA5Ou6V6hrYOV+HRvcLANMP8bAmDC8iC3Cj9io69fk7/tr8uAQAgP/d979NzuvUB7hRG4wQ+Da5jLVgeBFZgB6deqP2yv/DupnB6NfEcxsbGhrw/Xff44knnzB55FV0swaLt51Fj/G927tci8DwIrIAantn6H/xRx+PgRjcrennNl5xuIJBXQeZ7Mip/6UK+l9uQW3v3N7lWoRWh9fly5fRt2/f9qiFWqGlNpKW2kcA22sjIevS6vDq378/xo0bh/nz52P69OlwdraNlLc05raRNNc+AthWGwlZl1aH1+nTp7Fp0ybEx8cjLi4OM2fOxPz58zFq1Kj2qI+a0FIbSUvtI4DttZGQdWl1eAUHB2PdunVYvXo1du3ahYyMDDz55JMYMGAA5s2bhzlz5sDb27s9aqUHtNRG0lL7CGB7bSRkXX5zJ1UHBwdERkZi+/bteP/993Hp0iUkJCQgICAAUVFRKC0tbcs6iYgM/ObwOnXqFBYuXAg/Pz+kpKQgISEBRUVFyMrKwo0bNzB16tS2rJOIyECrTxtTUlKwadMmFBYW4vnnn8fmzZvx/PPPSzdQ9+nTBxkZGRxbnojaVavDa8OGDZg3bx7mzp0LPz8/k8v4+PggLS3tkYsjImpKq8MrKysLvXr1MhiqBgCEELh27Rp69eoFJycnREdHt1mRREQPa3WbV79+/VBRUWE0/c6dO+jTp0+bFEVE1JJWh5cQwuT0mpoadlglog5j9mlj4xN4VCoVkpKSDB4bptPp8MMPPyA4OLjNCyQiMsXs8Dpz5gyAX4+88vLyDB544eTkhOHDhyMhIaHtKyQiMsHs8Dp06BAAICYmBuvWrVP0Mw+JSPlafbVx06ZN7VEHEVGrmBVekZGRyMjIgIeHByIjI5tdNjMzs00KIyJqjlnh5enpCZVKJf2biEhuZoXXg6eKPG0kIkvAR58RkSKZdeT1+OOPS6eNLTl9+vQjFUREZA6zwmvatGntXAYRUeuYFV5Lly5t7zqIiFqFbV5EpEhmHXl17doVFy9ehJeXF7p06dJs+9edO3farDgioqaYFV5r1qyBu7u79G9zG++JiNqLWeH14MCCc+fOba9aiIjM1uo2L3t7e9y8edNo+u3bt2Fvb98mRRERtaTNBiPUaDQGw+QQEbUns0eV+PDDDwH8Ohjhv/71L7i5/fcpzTqdDt9++y2CgoLavkIiIhPMDq81a9YA+PXIKzU11eAU0cnJCYGBgUhNTW37ComITDA7vK5cuQIAGD9+PDIzM9GlS5d2K4qIqCWtHoywcURVktd9rQ4AkH+9yuT82vsanLoFdL96F51c1CaXuXSzpt3qo9ZpaX8CLe9TW9ufrQ6vefPmNTs/PT39NxdD5iv6v1/UtzPzmlnKAZ9cOtnie3VSt/rXgNqYefsTMGef2sr+bPVW3r171+C1VqtFfn4+Kisr8cwzz7RZYdS8iY91BwD083GDi6NxF5XC0iq88XkeVk8fioF+TQ8g2UntgD5endqtTjJPS/sTMG+f2tL+bHV4ffHFF0bT9Ho9FixYgH79+rVJUdSyrp2c8MdRvZqc39DQAADo590JQ/w5+q2la2l/AtynD2uTG7Pt7OwQHx8vXZEkImpvbTaqRFFRkfSXgYiovbX6tLHxydmNhBAoLS3Fnj17DO6BJCJqT60+8jpz5ozB148//ggAWL16NdauXfubili/fj0CAwPh7OyM0NBQnDhxosllMzIyoFKpDL6cnZ1/0+cSkXLJ3s9r27ZtiI+PR2pqKkJDQ7F27VpERESgsLAQPj4+Jtfx8PBAYWGh9JpD9BDZHtlHUk1JSUFsbCxiYmIwePBgpKamwtXVtdn+YiqVCt27d5e+fH19O7BiIrIEsvZmq6+vR25uLhITE6VpdnZ2mDBhAo4dO9bkejU1Nejduzf0ej1GjBiBlStX4rHHHjO5rEajgUajkV5XV1cD+LV/mlarbaMtsTyNF08aGhqsejttia3sU3O3TdbwqqiogE6nMzpy8vX1RUFBgcl1Bg4ciPT0dAwbNgxVVVVYtWoVxowZg/Pnz6Nnz55GyycnJ2PZsmVG0w8cOABXV9e22RALdK0GABxw/PhxXM+XuxpqC7ayT+vq6sxaTnH3EYSFhSEsLEx6PWbMGAwaNAj//Oc/sWLFCqPlExMTDa6QVldXIyAgABMnToSHh0eH1CyHcyV3gLxTGD16NIb36ip3OdQGbGWfNp4dtaTNwuvnn3/G8uXL8dFHH5m9jpeXF+zt7VFeXm4wvby8HN27dzfrPRwdHfH444/j0qVLJuer1Wqo1cY3sTo6OsLR0dHsWpXGwcFB+m7N22lLbGWfmrttbdZgf/v2baSlpbVqHScnJ4SEhCA7O1uaptfrkZ2dbXB01RydToe8vDz4+fm16rOJSNlkP22Mj49HdHQ0Ro4ciVGjRmHt2rWora1FTEwMACAqKgr+/v5ITk4GACxfvhyjR49G//79UVlZiQ8++ABXr17Fq6++KudmEFEHkz28Zs6ciVu3biEpKQllZWUIDg7Gvn37pEb8kpIS2Nn99wDx7t27iI2NRVlZGbp06YKQkBAcPXoUgwcPlmsTiEgGsocXAMTFxSEuLs7kvMOHDxu8XrNmDW8AJyLzwysyMrLZ+ZWVlY9aCxGR2cwOL0/P5scP8vT0RFRU1CMXRERkDrPDa9OmTe1ZBxFRq8h+byMR0W9h9pFXSw/eaMQHcBBRRzA7vDIyMtC7d288/vjjEEK0Z01ERC0yO7wWLFiALVu24MqVK4iJicErr7yCrl2t9/4qIrJsZrd5rV+/HqWlpXjrrbfw1VdfISAgADNmzMD+/ft5JEZEHa5VDfZqtRovv/wysrKy8NNPP+Gxxx7DwoULERgYiJoa23paLxHJ6zdfbbSzs4NKpYIQAjqdri1rIiJqUavCS6PRYMuWLQgPD8eAAQOQl5eHf/zjHygpKYGbm1t71UhEZMTsBvuFCxdi69atCAgIwLx587BlyxZ4eXm1Z21ERE0yO7xSU1PRq1cv9O3bF0eOHMGRI0dMLpeZmdlmxRERNcXs8IqKiuIjxojIYrSqkyoRkaXgvY1EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKZBHhtX79egQGBsLZ2RmhoaE4ceKEWett3boVKpUK06ZNa98CicjiyB5e27ZtQ3x8PJYuXYrTp09j+PDhiIiIwM2bN5tdr7i4GAkJCRg7dmwHVUpElkT28EpJSUFsbCxiYmIwePBgpKamwtXVFenp6U2uo9PpMHv2bCxbtgx9+/btwGqJyFI4yPnh9fX1yM3NRWJiojTNzs4OEyZMwLFjx5pcb/ny5fDx8cH8+fORk5PT7GdoNBpoNBrpdXV1NQBAq9VCq9U+4hZYroaGBum7NW+nLbGVfWrutskaXhUVFdDpdPD19TWY7uvri4KCApPrfPfdd0hLS8PZs2fN+ozk5GQsW7bMaPqBAwfg6ura6pqV4loNADjg+PHjuJ4vdzXUFmxln9bV1Zm1nKzh1Vr37t3DnDlzsHHjRnh5eZm1TmJiIuLj46XX1dXVCAgIwMSJE+Hh4dFepcruXMkdIO8URo8ejeG9uspdDrUBW9mnjWdHLZE1vLy8vGBvb4/y8nKD6eXl5ejevbvR8kVFRSguLsaUKVOkaXq9HgDg4OCAwsJC9OvXz2AdtVoNtVpt9F6Ojo5wdHRsi82wSA4ODtJ3a95OW2Ir+9TcbZO1wd7JyQkhISHIzs6Wpun1emRnZyMsLMxo+aCgIOTl5eHs2bPS1wsvvIDx48fj7NmzCAgI6MjyiUhGsp82xsfHIzo6GiNHjsSoUaOwdu1a1NbWIiYmBgAQFRUFf39/JCcnw9nZGUOGDDFYv3PnzgBgNJ2IrJvs4TVz5kzcunULSUlJKCsrQ3BwMPbt2yc14peUlMDOTvYeHURkYWQPLwCIi4tDXFycyXmHDx9udt2MjIy2L4iILB4PaYhIkRheRKRIDC8iUiSGFxEpEsOLiBTJIq420qOrq6szuB+0sLQSmrJLuJDvAv3tzgbLBgUFWfV9nWQbGF5WoqCgACEhIUbTZ31svGxubi5GjBjRAVURtR+Gl5UICgpCbm6u9LrmvgZ7Dh3D78eHwc1FbbQskdIxvKyEq6urwdGUVqvF3YqbCBs10qpv4iXbxQZ7IlIkhhcRKRLDi4gUieFFRIrEBnsiC/Rwvz2g6b57ttpvj+FFZIGa6rcHGPfds9V+ewwvIgv0cL89oOm+e7bab4/hRWSBHu63B7Dv3sPYYE9EisTwIiJFYngRkSIxvIgUQKfT4ciRI/j2229x5MgR6HQ6uUuSHcOLyMJlZmaif//+CA8PR0pKCsLDw9G/f39kZmbKXZqsGF5EFiwzMxPTp0/H0KFDkZOTgy1btiAnJwdDhw7F9OnTbTrAGF5EFkqn0+GNN97A5MmTsXPnToSGhsLFxQWhoaHYuXMnJk+ejISEBJs9hWR4EVmonJwcFBcX469//avRU+Pt7OyQmJiIK1euICcnR6YK5cXwIrJQpaWlAIAhQ4aYnN84vXE5W8PwIrJQfn5+AID8/HyT8xunNy5naxheRBZq7NixCAwMxMqVK6HVag26Smi1WiQnJ6NPnz4YO3as3KXKgvc2Elkoe3t7rF69Gi+++CI8PT1x//59AEBKSgpcXFxw//597NixA/b29jJXKg8eeRFZOJVKZXKaqem2hOFFZKEe7CpRVVWFrKwsxMfHIysrC5WVlewqIXcBRGTag10lHB0dMW7cODz11FMYN24cHB0d2VVC7gKIyDR2lWgew4vIQrGrRPMYXkQW6sGuEnq93mCeXq+3+a4SDC8iC9XYVWL37t2YNm0ajh8/jvv37+P48eOYNm0adu/ejVWrVtlsVwn28yKyYJGRkfj888/xxhtv4KmnnpKm9+nTB59//jkiIyNlrE5eDC8iCxcZGYmpU6fi0KFD+PrrrzFp0iSMHz/eZo+4GjG8iBTA3t4e48aNQ21tLcaNG2fzwQWwzYuIFIrhRUSKxPAiIkWyuTYvIQQAoLq6WuZK2pdWq0VdXR2qq6v5dGUrYSv7tPH/ZuP/1abYXHjdu3cPABAQECBzJUTUnHv37sHT07PJ+SrRUrxZGb1ejxs3bsDd3d2qhxSprq5GQEAArl27Bg8PD7nLoTZgK/tUCIF79+6hR48eRmP3P8jmjrzs7OzQs2dPucvoMB4eHlb9i26LbGGfNnfE1YgN9kSkSAwvIlIkhpeVUqvVWLp0KdRqtdylUBvhPjVkcw32RGQdeORFRIrE8CIiRWJ4EZEiMbxk9vTTT+O1116TuwwixWF4KcThw4ehUqlQWVkpdynUCpb2x8nS6nkUDC8iC1dfXy93CRaJ4WVBNBoNlixZgoCAAKjVavTv3x9paWkoLi7G+PHjAQBdunSBSqXC3LlzW3y/e/fuYfbs2ejUqRP8/PywZs0ao7+8n3zyCUaOHAl3d3d0794ds2bNws2bN6X5jUd82dnZGDlyJFxdXTFmzBgUFha29eZbnblz5+LIkSNYt24dVCoVVCoVioqKMH/+fPTp0wcuLi4YOHAg1q1bZ7TetGnT8O6776JHjx4YOHAgAODo0aMIDg6Gs7MzRo4ciZ07d0KlUuHs2bPSuvn5+Zg0aRLc3Nzg6+uLOXPmoKKiosl6iouLO+rH0fYEyWrcuHFi8eLFQgghZsyYIQICAkRmZqYoKioSBw8eFFu3bhUNDQ1ix44dAoAoLCwUpaWlorKyssX3fvXVV0Xv3r3FwYMHRV5envjDH/4g3N3dpc8TQoi0tDSxd+9eUVRUJI4dOybCwsLEpEmTpPmHDh0SAERoaKg4fPiwOH/+vBg7dqwYM2ZMW/8orE5lZaUICwsTsbGxorS0VJSWlopffvlFJCUliZMnT4rLly+LTz/9VLi6uopt27ZJ60VHRws3NzcxZ84ckZ+fL/Lz80VVVZXo2rWreOWVV8T58+fF3r17xYABAwQAcebMGSGEEHfv3hXe3t4iMTFRXLhwQZw+fVqEh4eL8ePHN1lPQ0ODHD+aNsHwklljeBUWFgoAIisry+RyjSFy9+5ds963urpaODo6iu3bt0vTKisrhaurq0F4PezkyZMCgLh3757B5x48eFBaZs+ePQKAuH//vlm12LIH/zg1ZdGiReLFF1+UXkdHRwtfX1+h0WikaRs2bBDdunUz+Jlv3LjRILxWrFghJk6caPDe165dk/7omVuPUvC00UKcPXtWeshCW7h8+TK0Wi1GjRolTfP09JROQRrl5uZiypQp6NWrF9zd3aXPLykpMVhu2LBh0r8bn9D84OklmW/9+vUICQmBt7c33Nzc8NFHHxn9vIcOHQonJyfpdWFhIYYNGwZnZ2dp2oP7FgDOnTuHQ4cOwc3NTfoKCgoCABQVFbXjFsnD5obEsVQuLi4d/pm1tbWIiIhAREQEPvvsM3h7e6OkpAQRERFGjcQPjtzZOA7aw09xppZt3boVCQkJWL16NcLCwuDu7o4PPvgAP/zwg8FynTp1avV719TUYMqUKXj//feN5jX+wbEmDC8LMXToUOj1ehw5cgQTJkwwmt/4V1in05n1fn379oWjoyNOnjyJXr16AQCqqqpw8eJF6eGlBQUFuH37Nt577z1pZNlTp061xebQ/3FycjLYZ99//z3GjBmDhQsXStPMOSoaOHAgPv30U2g0GunG7JMnTxosM2LECOzYsQOBgYFwcDD9X/vhepSMp40WIjAwENHR0Zg3bx527tyJK1eu4PDhw/jPf/4DAOjduzdUKhV2796NW7duoaamptn3c3d3R3R0NN58800cOnQI58+fx/z582FnZycdOfXq1QtOTk74+9//jsuXL2PXrl1YsWJFu2+rLQkMDMQPP/yA4uJiVFRU4He/+x1OnTqF/fv34+LFi3jnnXeMQsiUWbNmQa/X409/+hMuXLiA/fv3Y9WqVQD+eyS8aNEi3LlzBy+//DJOnjyJoqIi7N+/HzExMVJgPVyPko+eGV4WZMOGDZg+fToWLlyIoKAgxMbGora2FgDg7++PZcuW4e2334avry/i4uJafL+UlBSEhYVh8uTJmDBhAp544gkMGjRIajfx9vZGRkYGtm/fjsGDB+O9996T/kNQ20hISIC9vT0GDx4Mb29vREREIDIyEjNnzkRoaChu375tcBTWFA8PD3z11Vc4e/YsgoOD8be//Q1JSUkAIO3PHj164Pvvv4dOp8PEiRMxdOhQvPbaa+jcubM0nPLD9Tzc1qYkHBLHhtTW1sLf3x+rV6/G/Pnz5S6HHtFnn32GmJgYVFVVydJmKje2eVmxM2fOoKCgAKNGjUJVVRWWL18OAJg6darMldFvsXnzZvTt2xf+/v44d+4clixZghkzZthkcAEML8UqKSnB4MGDm5z/008/AQBWrVqFwsJCODk5ISQkBDk5OfDy8uqoMqkNlZWVISkpCWVlZfDz88NLL72Ed999V+6yZMPTRoVqaGho9taO5q44EVkDhhcRKRKvNhKRIjG8iEiRGF5EpEgMLyJSJIYXtau5c+dKA985OjrC19cX4eHhSE9Pb9WtKRkZGejcuXP7FdqExoEByfIwvKjdPffccygtLUVxcTG+/vprjB8/HosXL8bkyZPR0NAgd3mkVHIOJkbWLzo6WkydOtVoenZ2tgAgNm7cKIQQYvXq1WLIkCHC1dVV9OzZUyxYsMBoQMQHv5YuXSqEEGLz5s0iJCREuLm5CV9fX/Hyyy+L8vJy6XPu3LkjZs2aJby8vISzs7Po37+/SE9Pl+aXlJSIl156SXh6eoouXbqIF154QVy5ckUIIcTSpUuNPvfQoUPt8nOi1uORF8nimWeewfDhw5GZmQkAsLOzw4cffojz58/j448/xjfffIO33noLADBmzBisXbsWHh4eKC0tRWlpKRISEgAAWq0WK1aswLlz57Bz504UFxcbjO//zjvv4KeffsLXX3+NCxcuYMOGDdIdBlqtFhEREXB3d0dOTg6+//57uLm54bnnnkN9fT0SEhIwY8YM6cixtLQUY8aM6dgfFDVN7vQk69bUkZcQQsycOVMMGjTI5Lzt27eLbt26Sa83bdokPD09W/y8h4exnjJlioiJiTG57CeffCIGDhwo9Hq9NE2j0QgXFxexf//+FusnefHIi2QjhJDGojp48CCeffZZ+Pv7w93dHXPmzMHt27dRV1fX7Hu0NIz1ggULsHXrVgQHB+Ott97C0aNHpXXPnTuHS5cuwd3dXRo2uWvXrvjll1+scthka8PwItlcuHABffr0QXFxMSZPnoxhw4Zhx44dyM3Nxfr16wE0/8zCxmGsPTw88Nlnn+HkyZP44osvDNabNGkSrl69itdffx03btzAs88+K51y1tTUICQkBGfPnjX4unjxImbNmtXOW0+Pinfukiy++eYb5OXl4fXXX0dubi70ej1Wr14tDZrXOIJsI1PDF5s7jLW3tzeio6MRHR2NsWPH4s0338SqVaswYsQIbNu2DT4+PvDw8DBZpzUNm2xteORF7U6j0aCsrAzXr1/H6dOnsXLlSkydOhWTJ09GVFQU+vfvD61WKw1H/cknnyA1NdXgPQIDA1FTU4Ps7GxUVFSgrq7OrGGsk5KS8OWXX+LSpUs4f/48du/ejUGDBgEAZs+eDS8vL0ydOhU5OTnS0Nt/+ctf8PPPP0uf++OPP6KwsBAVFRXQarUd80Ojlsnd6EbWLTo6Wupm4ODgILy9vcWECRNEenq60Ol00nIpKSnCz89PuLi4iIiICLF582aj51T++c9/Ft26dTPoKvHvf/9bBAYGCrVaLcLCwsSuXbuMnmU4aNAg4eLiIrp27SqmTp0qLl++LL1naWmpiIqKEl5eXkKtVou+ffuK2NhYUVVVJYQQ4ubNmyI8PFy4ubmxq4SF4ZA4RKRIPG0kIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEj/H0LISLgLQuYZAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T08:03:11.755669Z", + "iopub.status.busy": "2024-03-01T08:03:11.755336Z", + "iopub.status.idle": "2024-03-01T08:03:12.047269Z", + "shell.execute_reply": "2024-03-01T08:03:12.046369Z" + }, + "papermill": { + "duration": 0.316437, + "end_time": "2024-03-01T08:03:12.049444", + "exception": false, + "start_time": "2024-03-01T08:03:11.733007", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPCUlEQVR4nO3deVxU1f/H8dfMwLDJZiiLoogbaoqmQliKJgZmpaWFZm6Ztm9kilqiqWFm5tclLb+Z2eJSWZpbKbmkIhbgvismLoCo7LLNnN8f/Jxvk6AMDlxmPM/H4z6UO+feec/IfLxz77nnqIQQAkmSJCukVjqAJElSdZEFTpIkqyULnCRJVksWOEmSrJYscJIkWS1Z4CRJslqywEmSZLVkgZMkyWrZKB2gNtLr9Vy8eBFnZ2dUKpXScSRJ+hchBLm5ufj4+KBWV3ycJgtcOS5evIivr6/SMSRJuo3U1FQaNmxY4eOywJXD2dkZKHvzXFxcFE4jSdK/5eTk4Ovra/isVkQWuHLc+Frq4uIiC5wk1WK3O4UkLzJIkmS1ZIGTJMlqyQInSZLVkufgrJROp6OkpETpGFItpdFosLGxsfpuULLAWaG8vDzOnz+PHMtUuhVHR0e8vb3RarVKR6k2ssBZGZ1Ox/nz53F0dKRevXpW/z+0ZDohBMXFxVy+fJmUlBSaN29+y86yNaY4H7ROZt2lLHBWpqSkBCEE9erVw8HBQek4Ui3l4OCAra0tf//9N8XFxdjb2ysbSK+DxT3Bozn0ngku3mbZbS0o27BgwQL8/Pywt7cnODiYvXv3Vth29erVdOrUCTc3N5ycnGjfvj1ff/21UZvhw4ejUqmMloiIiOp+GbWKPHKTbqdWHLXdcGAVXD6KSNkBtub7j1nxV7hy5UqioqKIiYkhKSmJwMBAwsPDycjIKLd93bp1mThxIvHx8Rw4cIARI0YwYsQIfv31V6N2ERERXLp0ybAsX768Jl6OJEmmKi1CbJ0OwOzrfcgoNd/RpOIFbvbs2YwaNYoRI0bQunVrFi1ahKOjI0uWLCm3fffu3XniiSdo1aoVTZs25Y033qBdu3bs3LnTqJ2dnR1eXl6Gxd3dvSZejiRJpkpciio7lXThxo+aR6hXx85su1a0wBUXF5OYmEhYWJhhnVqtJiwsjPj4+NtuL4QgLi6O48eP061bN6PHtm3bRv369WnZsiUvvfQSV65cqXA/RUVF5OTkGC1SzerevTtvvvmm0jGsxrZt21CpVGRlZSkd5daK8mD7TAD+U9qflr71zXp6RdECl5mZiU6nw9PT02i9p6cnaWlpFW6XnZ1NnTp10Gq19OnTh3nz5tGrVy/D4xERESxbtoy4uDg+/PBDtm/fTu/evdHpdOXuLzY2FldXV8MiRxKpvSzmgytVzp6FUJDJZdsGrNKFEujrZtbdW+RVVGdnZ/bt20deXh5xcXFERUXh7+9P9+7dARg4cKChbdu2bWnXrh1NmzZl27Zt9OzZ86b9jR8/nqioKMPPN0YqkCQlFRcXW3UfNQquwu65ACxSD6QUG9qbucApegTn4eGBRqMhPT3daH16ejpeXl4VbqdWq2nWrBnt27fn7bffZsCAAcTGxlbY3t/fHw8PD06dOlXu43Z2doaRQ6xtBBEhBAXFpYosVe1oXFRUxLhx4/D19cXOzo5mzZrxxRdfcPbsWXr06AGAu7s7KpWK4cOH33Z/ubm5DB48GCcnJ7y9vfnkk09u+kr89ddf06lTJ5ydnfHy8uKZZ54xutB148gxLi6OTp064ejoSJcuXTh+/HilX9e0adOoX78+zs7OPP/880RHR9O+fXvD48OHD6dfv35Mnz4dHx8fWrZsWalsABs2bKBFixY4ODjQo0cPzp49W+lcitk5G4py0NVvy5LsDgAENnQz61MoegSn1Wrp2LEjcXFx9OvXDygbTTcuLo5XX3210vvR6/UUFRVV+Pj58+e5cuUK3t7m6VtjSa6X6Gg96dfbN6wGR94Px1Fr+q/Y0KFDiY+PZ+7cuQQGBpKSkkJmZia+vr78+OOP9O/fn+PHj+Pi4lKpvn5RUVHs2rWLtWvX4unpyaRJk0hKSjIqLiUlJUydOpWWLVuSkZFBVFQUw4cPZ8OGDUb7mjhxIh9//DH16tXjxRdf5LnnnmPXrl23zfDtt98yffp0Pv30Ux544AFWrFjBxx9/TJMmTYzaxcXF4eLiwubNmyudLTU1lSeffJJXXnmF0aNH89dff/H222/fNpOisi9AwucAJPi/gjinpvE9jrg7mfeIVfGvqFFRUQwbNoxOnToRFBTEnDlzyM/PZ8SIEUDZL3uDBg0MR2ixsbF06tSJpk2bUlRUxIYNG/j6669ZuHAhUHab0pQpU+jfvz9eXl6cPn2asWPH0qxZM8LDwxV7nVLlnDhxglWrVrF582bDxSd/f3/D43Xr1gWgfv36uLm53XZ/ubm5fPXVV3z33XeG0xNffvklPj4+Ru2ee+45w9/9/f2ZO3cunTt3Ji8vjzp16hgemz59OqGhoQBER0fTp08fCgsLb9tRdt68eYwcOdLwez1p0iR+++038vLyjNo5OTnx3//+1+ir6e2yLVy4kKZNm/Lxxx8D0LJlSw4ePMiHH3542/dHMds/BF0RhT7BvLjHHdDxaDvzH4AoXuAiIyO5fPkykyZNIi0tjfbt27Np0ybDhYdz584ZdUjMz8/n5Zdf5vz58zg4OBAQEMA333xDZGQkUHYT8YEDB/jqq6/IysrCx8eHhx9+mKlTp2JnZ77Lz5bCwVbDkfeVKewOthqTt9m3bx8ajcZQRO7UmTNnKCkpISgoyLDO1dXV8PXvhsTERCZPnsz+/fu5du0aer0eKPv9a926taFdu3btDH+/8Y0gIyODRo0a3TLH8ePHefnll43WBQUF8fvvvxuta9u27U3n3W6X7ejRowQHBxttExIScss8iso8BcnfABCTP4CcQh0dGrnxRs8WZn8qxQscwKuvvlrhV9Jt27YZ/Txt2jSmTZtW4b4cHBxu6vR7N1OpVFX6mqgUJW4vy8/PJzw8nPDwcL799lvq1avHuXPnCA8Pp7i42Kitra2t4e83ujPcKDjm4ORkfC+mKdksxtZpIHQcce7CyvQGuDvasuCZ+9DamP+SgOIdfSXpn9q2bYter2f79u3lPn7j6KaiLj//5u/vj62tLX/++adhXXZ2NidOnDD8fOzYMa5cucKMGTPo2rUrAQEBFd5JU1UtW7Y0ygDc9HN5KpOtVatWN93euGfPnjsPXR0u7oPDPyFQEZX5OGoVzBnYAR+36vmPTRY4qVbx8/Nj2LBhPPfcc/z888+kpKSwbds2Vq1aBUDjxo1RqVSsW7eOy5cv33QO69+cnZ0ZNmwY77zzDlu3buXw4cOMHDkStVptOAJr1KgRWq2WefPmcebMGdauXcvUqVPN+rpee+01vvjiC7766itOnjzJtGnTOHDgwG07tVYm24svvsjJkyd55513OH78ON999x1Lly41a36ziXsfgDW6LhwTjZj0aGtCW9SrtqeTBU6qdRYuXMiAAQN4+eWXCQgIYNSoUeTn5wPQoEEDpkyZQnR0NJ6enpW62j579mxCQkJ49NFHCQsL44EHHqBVq1aGCwP16tVj6dKlfP/997Ru3ZoZM2Ywa9Yss76mwYMHM378eMaMGcN9991HSkoKw4cPv+3Ficpka9SoET/++CM///wzgYGBLFq0iA8++MCs+c0i5Q84HUeJ0PBx6QCGhjRm+ANNbr/dHVAJOSriTXJycnB1dSU7O9vi+sQVFhaSkpJCkyZNlB8Cp5bKz8+nQYMGfPzxx4wcOVKxHL169cLLy+um0XBqSo3+rghB8ec90V5KZFlpL7b4j2XJsE7YaKp2jFXZz6jlnH2WpCpKTk7m2LFjBAUFkZ2dzfvvl31N6tu3b41lKCgoYNGiRYSHh6PRaFi+fDlbtmwx6u9mzYoOr8PuUiIFwo71boNZ/EyHKhc3U8gCJ1m0f3fj+LcjR44AMGvWLI4fP27oXP7HH3/g4eFhthxt2rTh77//Lvexzz77jCeffJINGzYwffp0CgsLadmyJT/++KPRQBPWSl9aSuaad2kArFA/wqznwnGxt73tduYgC5xk0Xx8fNi3b98tH2/UqBGJiYnVmmPDhg0VTvLj6emJg4MDW7ZsqdYMtdWmFfN4pOQs2cKJ+56JwbeuY409tyxwkkWzsbGhWbNmSsegcePGSkeolfQlRbQ7uQBUkNp6NO1bVO9FhX+TV1HvQEZOAV9/MYfDRw8rHUWSaqXsXYtpqLpMunAnoO+YGn9+WeDuwOmvXmZIagxpa2LkFH2S9G9FeTjtmQ3ACoeB2NjXuc0G5icL3B1o2et5ALpf38LWnX8onEaSapmEhWgLr3BW78lxnycUiSAL3B2oG/Agp+/pjkYlUG2dTmFJ5W4fkiSrV3AVdpUNZjm79Cn86rsqEkMWuDvUsP909Kjood/DmvW/KB1HkmqHnZ9AUQ7HRGN+0d9PgLcyHeZlgbtDdj73cr5RWYfRRskfkZ5TqHCiu5Ofnx9z5swx/KxSqfj5558Vy3NXy7mI2Fs2mOWMkqfp5HcPEW0qHqG7OskCZwa+T0yhBBtCVIf46YdvlY4jAZcuXaJ3795Kx7gr6bd9iKq0kL36lpx0DmHhsx2rZSikypAFzgxU7n5ktR4CwP1n53Mg9ZrCiSQvLy/FBzgVQlBaWqpohhp35TQkld1bO0cM4vNhnfAw4zynppIFzkzqPTKRIpUD7dVn+PWH/9aebiNCQHG+MouJ70FlJoeprH9+RT179iwqlYrVq1fTo0cPHB0dCQwMvGnu3Z07d9K1a1ccHBzw9fXl9ddfN4xiApWfmGbjxo107NgROzu7myYkt3apP05EjY44XQeeeSqSNj7KXFy4Qd7JYC516lEc9CJ2CZ/wxLUlbNgfSZ/2tx7GukaUFMAHPrdvVx0mXASt0+3b/b/KTA5zJyZOnMisWbNo3rw5EydOZNCgQZw6dQobGxtOnz5NREQE06ZNY8mSJVy+fNkw0vSXX34JVH5imujoaGbNmoW/vz/u7u5myW4JjifvpOXFjeiFitQOYxjeTqHfu3+QBc6MnHu8xfXEJTQrvcjKdZ/Rs8372FdhXoK7UWUnh7kTY8aMoU+fPgBMmTKFNm3acOrUKQICAoiNjWXw4MGGo8XmzZszd+5cQkNDWbhwIfb29pWemOb99983moj8bpCRU8iVte8CsLfOQwzt10fhRGVqRYFbsGABH330EWlpaQQGBjJv3jyjSUL+afXq1XzwwQecOnWKkpISmjdvzttvv82QIUMMbYQQxMTEsHjxYrKysnjggQdYuHAhzZs3r94XYu+KTbco+D2G4SXLWbrjGV7s2aZ6n/N2bB3LjqSUeu5KquzkMHeiogljAgIC2L9/PwcOHODbb/93kUgIgV6vJyUlhVatWlV6YppOnTqZLbOl+PK7bxgnkilFw73PzkCtvvVIxTVF8XNwK1euJCoqipiYGJKSkggMDCQ8PLzCMfHr1q3LxIkTiY+P58CBA4wYMYIRI0YYTTQzc+ZM5s6dy6JFi0hISMDJyYnw8HAKC6u/C4dtyAtct/ekgeoK17YvIkPpbiMqVdnXRCWW2wzHXdNuNWFMXl4eL7zwAvv27TMs+/fv5+TJkzRt2tQw+YuLiwvffvstf/75Jz/99BPATZO//HviGGuXeiWfsIuLAMhvM5g63uafHauqFC9ws2fPZtSoUYwYMYLWrVuzaNEiHB0dWbJkSbntu3fvzhNPPEGrVq1o2rQpb7zxBu3atTOczBVCMGfOHN5991369u1Lu3btWLZsGRcvXqyZflG2DtiHTQBgtOon5m5Irv7ntAKVmRymOt13330cOXKEZs2a3bRotdoamZjGUv21eTkd1ScpUtnhGjFR6ThGFC1wxcXFJCYmGg36p1arCQsLu+kKV3mEEMTFxXH8+HG6desGQEpKCmlpaUb7dHV1JTg4uMJ9FhUVkZOTY7TcCVWHZyl08eceVS73HFrMwfPZd7S/u0FlJoepTuPGjWP37t28+uqr7Nu3j5MnT7JmzRrDnA81MTGNJSotKaHt8f8AkNp8KDgr06G3IooWuMzMTHQ6nWGS5xs8PT1JS0urcLvs7Gzq1KmDVqulT58+zJs3z3BS98Z2puwzNjYWV1dXw+Lr63snLws0Ntg//B4AozTrmbN2V+3pNlKL3W5ymOrUrl07tm/fzokTJ+jatSsdOnRg0qRJhoscNTExjSU6tuVLmolz5OBEo8cmKB3nJrXiIoOpnJ2d2bdvH3l5ecTFxREVFYW/vz/du3ev0v7Gjx9PVFSU4eecnJw7L3Kt+1Fc/xPqZBwg5OIyNhzsQJ923ne2Tyvn7OxsdJI/Pz+fKVOmMHr06Ntue/bsWaOf//kfip+f303/wbi5ud20rnPnzvz2228VPsegQYMYNGhQhc/TvXv3u+o/Ml1JEfX++hiAxIbD6OFcV+FEN1P0CM7DwwONRkN6errR+vT0dLy8Kj7UVavVNGvWjPbt2/P2228zYMAAYmNjAQzbmbJPOzs7XFxcjJY7plajfTgGgCGazXyxfoccbeQ2kpOTWb58OadPnyYpKYnBgwcDNTs5jFR5u1Z9jKcujcvCjeaPva10nHIpWuBuTAASFxdnWKfX64mLiyMkJKTS+9Hr9RQVFQHQpEkTvLy8jPaZk5NDQkKCSfs0i6Y90TV6EDtVKZH53/HFzpSafX4LNGvWLAIDAwkLCyM/P58//viDo0ePUqdOnQoXqeYdOHORVifKrpyea/sqDT3NN4GPOSn+FTUqKophw4bRqVMngoKCmDNnDvn5+YwYMQKAoUOH0qBBA8MRWmxsLJ06daJp06YUFRWxYcMGvv76axYuXAiUXf5/8803mTZtGs2bN6dJkya89957+Pj40K9fv5p9cSoVml6T4YswBmi203frdp7q2JD6LnK+0vJ06NCh3Mlhrl+/fsuJZaSalVdUyp7l0xityuayrQ/39Xtd6UgVUrzARUZGcvnyZSZNmkRaWhrt27dn06ZNhosE586dQ63+34Fmfn4+L7/8MufPn8fBwYGAgAC++eYbIiMjDW3Gjh1Lfn4+o0ePJisriwcffJBNmzYpMxGyb2dEy0fQHN/Ay/oVzPqtEzMHBNZ8Dgvm4OBQKyaWkcrMWB3P2OKfQAVOEZNQ2Sg7qMGtyJnty2H2me3TjyAWdkGFoG/xVKa/Mox7G1TPTcg3Ziv38/PDwcGhWp5Dsg7Xr1/n7NmzJs1s//eVfDZ+8gIv2vxCgXsAjq/Fg7rmz3RV9jOqeEffu4Jna1SBAwEYo1nJ++uOVNvVNo2m7N7Xf/eul6R/KygoAIzv8LidTbuTGK7ZBIBj7/cVKW6mUPwr6l2j+3jEwR/oyiEW/r2DTYf86N3W/N1GbGxscHR05PLly9ja2hp9vZckKOvaUlBQQEZGBm5ubob/FG9HpxfUT56LvaqEq3Xvo27zh6s56Z2TBa6muDdG1ek52PsZY21W8tqGTvQIqG/20UZUKhXe3t6kpKTw999/m3XfknVxc3O7ZXesf0tM+ovHdFvKzr09Oq3W3WtcHlngalK3MYjkb2hfcprW2X/w5S4/Xure1OxPo9Vqad68ufyaKlXI1ta20kduN6i3f4CNSs8JlxBa+D9QTcnMSxa4mlSnPqr7X4I/ZjHGZhX9twbTv2MD6jub/+quWq1W5qqxZJVyUxLplPs7AOqwGIXTVJ48QVPTHngd4eBOc/UFHi7dyuzfama0DEm6E9nrJwGw1TaUpm3vVzhN5ckCV9PsXVE9+BYAb9r8yE9/neHwRTnaiFSLnd1Fw8ydlAgNmZ3frpHRXcxFFjglBI0GZ28aqjJ5Rr2F93+pvm4jknRHhOD6prKjt+/1Pej5QA3f7niHZIFTgq0DhI4D4DWbnzmUcoFfD6ffZiNJUsCJX3FI+4vrQsvOBs9R10mrdCKTyAKnlA7PQt2m1FXlMlKzkQ82HKWoVI42ItUiej0ibgoAS3XhtDHj/Bg1RRY4pWhs4aGy4Z1H264n92oa3+w5p3AoSfqHQz+gyjhCjnBksf5xHm7tefttahlZ4JTU+gnwakcdrvOyzVr++8cZSnR6pVNJEpQWc/3X9wFYVPoYbz4WRHNPZ4VDmU4WOCWp1dCzrE/RUJvNkH2BX/YrNMWfJP2DLnEpDvmpXBau5ASOZMj9jZWOVCWywCmtWU9o/CB2lPCGzY98tv2MvKIqKar0ei5Zmz4A4AvNU7z7RCeL6hryT7LAKU2lgv/vGf6UZjulGcfYduKywqGku1n88g+4R1zjnL4eLl1Gmv1+6Zp0RwUuLy/PrNPt3bV8g6DlI2hUgiib7/ls+2mlE0l3qdQLF2n391IAPi59ime6WPZAoyYXuJSUFPr06YOTkxOurq64u7vj7u6Om5sb7u7u1ZHx7vDQewhU9NHsJT/lL/alZimdSLrLCCHYt2IyrqoCzqgbMzH6PdwcLavf27+ZfLP9s88+ixCCJUuW4OnpabHfzWsdz9ao2kXCgRW8Y7OSz3d04dPBHZVOJd1FNsYnE5ZTNhS5Y+/3qe/iqHSkO2byEdz+/fv58ssviYyMpHv37oSGhhotVbFgwQL8/Pywt7cnODiYvXv3Vth28eLFdO3a1XDkGBYWdlP74cOHo1KpjJaIiIgqZatRPcYj1LZ00xwk63AcZzPzlU4k3SWu5heT/9sHOKiKueQSiFcn65iq0eQC17lzZ1JTU80WYOXKlURFRRETE0NSUhKBgYGEh4eTkZFRbvtt27YxaNAgtm7dSnx8PL6+vjz88MNcuHDBqF1ERASXLl0yLMuXLzdb5mrj7oeq43AAxtqs4PMd8lycVDMWrf6NfqJsOKR6T3xgEYNZVobJk86cPn2aF198kWeffZZ77733pvHc27VrZ1KA4OBgOnfuzPz584GyOU59fX157bXXiI6Ovu32Op0Od3d35s+fz9ChQ4GyI7isrCx+/vlnk7LcYPZJZ0yRm47uP4FoSq/zsu5tpowdRz3n2jtrkWT54k9fIWPps/TV7Ca7YXdcn1+jdKTbqrZJZy5fvszp06cZMWIEnTt3pn379nTo0MHwpymKi4tJTEwkLCzsf4HUasLCwoiPj6/UPgoKCigpKaFu3bpG67dt20b9+vVp2bIlL730EleuXKlwH0VFRbXnarCzJ+r7XwbgTdUKlu2SR3FS9dHrBd+tXUdfzW4AXPtMVTiReZlc4J577jk6dOhAfHw8Z86cISUlxehPU2RmZqLT6QxzoN7g6elJWlpapfYxbtw4fHx8jIpkREQEy5YtIy4ujg8//JDt27fTu3dvdLryb2aPjY3F1dXVsPj6+pr0OsxN9cDrFNu60kJ9gWt7via/qFTRPJL1+u1IOk9cXQJAUcAT4G3aN7BaT5jI0dFRnDx50tTNynXhwgUBiN27dxutf+edd0RQUNBtt4+NjRXu7u5i//79t2x3+vRpAYgtW7aU+3hhYaHIzs42LKmpqQIQ2dnZlX8xZqbb8YkQMS4idZK/+H7PKcVySNbtvbmfCxHjInQx7kJkWs7vWXZ2dqU+oyYfwT300EPs37/fLMXVw8MDjUZDerrxWGjp6em3ne1n1qxZzJgxg99+++225/38/f3x8PDg1KlT5T5uZ2eHi4uL0aI0dfBocm3r0VCViSrxS6XjSFbo8IUsHrv8OQCFbQfDPeafAElpJveDe+yxx3jrrbc4ePAgbdu2vekiw+OPP17pfWm1Wjp27EhcXBz9+vUDyi4yxMXF8eqrr1a43cyZM5k+fTq//vornTp1uu3znD9/nitXruDtbf55SKuN1pGM+97AOeFdemR8hSgci8pe+cIrWY/t67/lZfUJilVaHHuNVzpO9TD10FClUlW4qNVqkw81V6xYIezs7MTSpUvFkSNHxOjRo4Wbm5tIS0sTQggxZMgQER0dbWg/Y8YModVqxQ8//CAuXbpkWHJzc4UQQuTm5ooxY8aI+Ph4kZKSIrZs2SLuu+8+0bx5c1FYWFipTJU9/K1uhYXXRcqkFkLEuIjL695XNItkXbYdSxNH3ru37Hdr9Til45is2r6i6vX6CpeKTuLfSmRkJLNmzWLSpEm0b9+effv2sWnTJsOFh3PnznHp0iVD+4ULF1JcXMyAAQPw9vY2LLNmzQJAo9Fw4MABHn/8cVq0aMHIkSPp2LEjf/zxB3Z2ltXdws7OnvUeI8r+vncB/1m7R+FEkjUoLNGxY/VCWqnPUaipg0f4WKUjVRuT+sGVlJTg4ODAvn37uPfee6szl6IU7Qf3L1/uPE3Qb0/QRv03n5f2Ycikr3HQWu7oDpLy5vx6mCd29aOxOoOi0InY9bC8Alct/eBsbW1p1KhRlY7UpKoZFOzHR6WRAAzT/Mbxk8cUTiRZsjOX87i2879lxc3OA7sHXlE6UrUy+SvqxIkTmTBhAlevXq2OPNK/2Ntq2KYPJEEfgJ2qBLudHykdSbJQQgim/vQXr6hXA6B9aBxonRROVb1Mvoo6f/58Tp06hY+PD40bN8bJyfgNSkpKMls46QYVH5YMZLXdZFpcWoM+4wQJufdwX2M37Gzk11Wpctbuv0jA38upb5tFqUsjbP7/vmdrZnKBu9GdQ6pZSaIFm3X30UuTxNnVExl0djjPPdCESY+1VjqaZAGyr5cw55c/+dlmLQA2Pd8FG8se660yTC5wMTEx1ZFDuoVH2nqx4WAas0qfpqc6Gb+037hX1Y01+7RMeCQAG40ceV66tU82n+Cpoh9xtSlAX68V6rYDlI5UI6r8yUhMTOSbb77hm2++ITk52ZyZpH+ZO7ADCRN6ovJswxp9FwDG2qzkSn4xe1PkuVDp1gpLdPz+5wFGaDYBoA6LAfXdcWrD5CO4jIwMBg4cyLZt23BzcwMgKyuLHj16sGLFCurVq2fujHc9G40aTxd7OjRyY3b6AB5V76Gb5iAhusNsONSILs08lI4o1WJ/nMxklPgBB1UxomEQqhYWMPirmZh8BPfaa6+Rm5vL4cOHuXr1KlevXuXQoUPk5OTw+uuvV0dG6f+193UjVXiyXPcQUHYUt+lgGjq9nGZQqthfyYkM1GwFQBU22WoGs6wMkwvcpk2b+PTTT2nVqpVhXevWrVmwYAEbN240azjJWHvfskl95pX2o0RtTwf1KTpc382fZ+XXVOlm+UWlJP59lcCTC7BV6bjmEwp+Dygdq0ZV6Vatf99gD2WdgPV6vVlCSeVrVr8OznY2XMad9NZlt3C9Y7OSjQfOK5xMqo3e/fkQ7y5aQYTYBYDzI+8rnKjmVWm4pDfeeIOLFy8a1l24cIG33nqLnj17mjWcZEyjVvHRU+14K6wFDfqMo0RbNihmSfJKMnILlY4n1SIlOj1r9l1gjM0q1CpBsstD2DRsr3SsGmdygZs/fz45OTn4+fnRtGlTmjZtSpMmTcjJyWHevHnVkVH6h4h7vXkjrDkqB3c0Xd8C4GVWMmv9QUy4rViyckl/X6MDx+mpSUaPBq9+1jUUeWWZfBXV19eXpKQktmzZwrFjZfdFtmrVymjIcKlmqINfoHj3pzS8noHDwa958moxq1/qIueqldh+PINxtisAUHccgre/9Q6OcSsmFzgAlUpFr1696NWrl7nzSKbQOqJ9KBrWR/GazU90OxfKtYIS6jpZfw916dbyD28kSH0cndoOTeg4peMopkoFLi4ujri4ODIyMm66sLBkyRKzBJMq6b6hsHseHtdSeE6zkWsF4bLA3eUycgqIzFkKaijq+DyOLj5KR1KMyefgpkyZwsMPP0xcXByZmZlcu3bNaJFqmMYWHnoXgNE268i9mn6bDSRrl7Lta1qr/yZf5YhjjzFKx1GUyUdwixYtYunSpQwZMqQ68khV0eZJzvw8DX9dCm5JC6DlbKUTSUrRleB/cA4ASQ2H0tWx7q3bWzmTj+CKi4vp0qVLdWSRqkqtZnXdkQD4nvwGsi8oHEhSij5xGfVKLnJZuODYreKJm+4WJhe4559/nu+++86sIRYsWICfnx/29vYEBwezd+/eCtsuXryYrl274u7ujru7O2FhYTe1F0IwadIkvL29cXBwICwsjJMnT5o1c22TWvcBEvQBaPRF6LbNUDqOpITiAkq3lv3bL1YNoJ1/A4UDKc/kAldYWMjs2bMJDQ3ltddeIyoqymgx1cqVK4mKiiImJoakpCQCAwMJDw8nIyOj3Pbbtm1j0KBBbN26lfj4eHx9fXn44Ye5cOF/Ry0zZ85k7ty5LFq0iISEBJycnAgPD6ew0Ho7w6blFDGzpGxoc5H8DRu3/aFwIqnG7f0M7fUMUvX1uNbqGWzlMFqmTToD0KNHj4p3plLx+++/mxQgODiYzp07M3/+fKDsVjBfX19ee+01oqOjb7u9TqfD3d2d+fPnM3ToUIQQ+Pj48PbbbzNmTNkJ1uzsbDw9PVm6dCkDBw687T5r06QzlTU37iSzN5/gv7YfEaZJZr3ufoLGrqWes2XNJCZV0fVriP8EoirM5q3il+g9+E0ebnPrydMtWWU/oyZfZNi6desdBfun4uJiEhMTGT/+f5POqtVqwsLCiI+Pr9Q+CgoKKCkpoW7dspOpKSkppKWlGXU8dnV1JTg4mPj4+EoVOEs08sEm3FNHS1D9T9B/1YM+mj1sS9xB9+6yr+Ld4Pjq6bQszOa4viGbbboR20IOWwZ3MOClOWRmZqLT6QxzoN7g6elJWlpapfYxbtw4fHx8DAXtxnam7LOoqIicnByjxdI42dkwOLgxLk06cKRuWVHz/mumwqmkmlB49QK+J5cB8FFpJN1aemJve3cMaHk7Fv0lfcaMGaxYsYKffvoJe3v7Ku8nNjYWV1dXw+Lr62vGlDUvt8tYSoSGlnl7IUWei7N2F9a+jyNFJOqbc8AxhGEhfkpHqjUULXAeHh5oNBrS0407p6anp+PldevzB7NmzWLGjBn89ttvtGvXzrD+xnam7HP8+PFkZ2cbltTU1Kq8nFqjbdsOrNKXnSst+jWGq3lFFJfKoays0tUUGp/9HoBzHcaw991eBPvfo3Co2kPRAqfVaunYsSNxcXGGdXq9nri4OEJCQircbubMmUydOpVNmzbRqVMno8eaNGmCl5eX0T5zcnJISEiocJ92dna4uLgYLZasjp0Nv3uO4LrQYpeWyLsffsTor/9SOpZkZkWlOg5/F40NOnbo29G11xNKR6p1TC5wO3bsoLS09Kb1paWl7Nixw+QAUVFRLF68mK+++oqjR4/y0ksvkZ+fz4gRZQM6Dh061OgixIcffsh7773HkiVL8PPzIy0tjbS0NPLy8oCyK7lvvvkm06ZNY+3atRw8eJChQ4fi4+NzV0152LpFc77UlY29/4ZqOTuOp8vhlKzMjKU/0uryrwDsafIqHnXkFfN/M/kqao8ePbh06RL169c3Wp+dnU2PHj3Q6XQm7S8yMpLLly8zadIk0tLSaN++PZs2bTJcJDh37hxq9f/q8MKFCykuLmbAAONpz2JiYpg8eTIAY8eOJT8/n9GjR5OVlcWDDz7Ipk2b7ug8naUJaXoPL/7+KIM1W2ipPk9f9S4u5/aivsvd8x5Ysz1nrtDl74WoNYJ1uvvl1fIKmNwPTq1Wk56eftPsWSdOnKBTp04WeQXy3yyxH9y/FZboCHhvEy9p1jLOdgWp+nocfep3Hm7bSOlokhnM/HwpYy++QalQ83XHVQx/LOyuGgfQ7P3gnnzySaDsK+Dw4cOxs/vf4bBOp+PAgQPyHtVaxN5WwyNtvVh6KJyR/Iqv+jLLf5pDcLMPcXW4eU4NyXLkFZbQ48JCUEFOq0hGPC6P3ipS6QLn6uoKlN3n6ezsjIODg+ExrVbL/fffz6hRo8yfUKqyjwYEkv5wS+yORUPcOEbofmDlruGMDmt3+42lWuvg9h8IUR2jGFvce7+rdJxardIF7ssvvwTAz8+PMWPG4OTkVG2hJPNwsrPBv14dqDuSvPj51CtIpXT3pxSGLpAdQS2VXk+DxFkA7PeJpLNrQ4UD1W4mX0WNiYmRxc3SaGyxf/g9AJ7V/cyGhMMKB5Kq6sKub2lUfIoc4YDrw2OVjlPrmVzg0tPTGTJkCD4+PtjY2KDRaIwWqXayafcUV+o0x0V1neLts9HrZZcRS5NXcB3x+3QAtrhF0ryxvGB0OyZ3Exk+fDjnzp3jvffew9vb+666cmPR1Goce78P3w+iX/E6/kg6QGinQKVTSZUkhGD9VzOJFJe4iivdh8fIz14lmFzgdu7cyR9//EH79u2rIY5UnRxa9ybVORDf3P0UxcVCpxVKR5Iq6Yc9JwhN+xJUkBv8Jo3d7+6hyCvL5K+ovr6+ske8pVKpqPNI2QTADxX8yi9b/5D/lrVciU7PM4v3cGrdJ3iprpFj503jXq8oHctimFzg5syZQ3R0NGfPnq2GOFJ1c28VytE692Oj0sPv0/h8xxmlI0m3sP7AJQ6dPsdLNmsBqBMxCWzkLVmVZXKBi4yMZNu2bTRt2hRnZ2fq1q1rtEi1n9Mj7wPwmGYPazdtUjiNdCvbT1xmtM063FT5lHoEoA6MVDqSRTH5HNycOXOqIYZUkxq1DmaNrgt9Nbt5x2YlIL/y1EalOj0Hjh1nuqbsPyGbnu+BWvZUMIXJBW7YsGHVkUOqYdcfGEtJfH+6a/aTsHUtwT0eVzqS9C9J57IYUvIDjjZFiAadUAX0UTqSxanSeHCnT5/m3XffZdCgQYbZrzZu3Mjhw7IDqaUYGNGDg559AbDfPpXMXOudccxS/bUvmWc0ZeMaqsJiQHYLMZnJBW779u20bduWhIQEVq9ebRiHbf/+/cTExJg9oFR92gyaSiFaAjnB8m8+l1dUa5lmh+eiVenIqP8ANOmmdByLZHKBi46OZtq0aWzevBmtVmtY/9BDD7Fnzx6zhpOql517Q/ICnweg16XP+THxnMKJpBsuHPuLsJLtADg+MkXhNJbL5AJ38OBBnnji5qGR69evT2ZmpllCSTXHI2IshTbOBKhTSfzlcy5kXVc6kgSUbHkftUqwx6Erdfw6Kx3HYplc4Nzc3Lh06dJN65OTk2nQoIFZQkk1yMEd265vAvCSWEn0qr/kfaoKupJXxKE9v+GXuZ1SoSa94xilI1k0kwvcwIEDGTduHGlpaahUKvR6Pbt27WLMmDEMHTq0OjJK1UwT8hKljvVppL6M398/cOBCttKR7kp5RaV0mRFH/oaykV9+0IfywP0VT74k3Z7JBe6DDz4gICAAX19f8vLyaN26Nd26daNLly68+64cfM8iaZ2w6V429M7rNj9x+kL6bTaQqsOe01e4X7+PYPUxioQtF9q9LieSuUMmFzitVsvixYs5ffo069at45tvvuHYsWN8/fXXVRouacGCBfj5+WFvb09wcDB79+6tsO3hw4fp378/fn5+qFSqcjsdT548GZVKZbQEBASYnOuuc98wrmp9qKfK5p5DS5ROc1c6nZHz/x2vwa7LC7z91EMKJ7J8VZ4XtVGjRjzyyCM8/fTTNG/evEr7WLlyJVFRUcTExJCUlERgYCDh4eGGvnX/VlBQgL+/PzNmzLjlxNBt2rTh0qVLhmXnzp1VyndXsdFyqEXZHQ0dzy9jz6FTCge6u+j0gpzE77lXfZZijRM8GKV0JKtQqTsZoqKimDp1Kk5OTkRF3fqNnz17dqWffPbs2YwaNcowB+qiRYtYv349S5YsITo6+qb2nTt3pnPnsitK5T1+g42NzS0LoFS+NuEjOXV4Mc3EWfatmMzR3lMZ8UATpWPdFZZsP07/7K9ADYVBr6B1krPTm0OlClxycjIlJSWGv1fElAH4iouLSUxMNJrUWa1WExYWRnx8fKX3U56TJ0/i4+ODvb09ISEhxMbG0qhRxaOfFhUVUVRUZPjZGqY+rIp7nB1w6v8B/PAMwzWbCP0lnIh7vfB2dbj9xlKVCSFI37EEf3Uahdq6uHR/XelIVqNSBW7r1q3l/v1OZGZmotPpDBM83+Dp6cmxY8eqvN/g4GCWLl1Ky5YtuXTpElOmTKFr164cOnQIZ2fncreJjY1lyhTZmRLAvs0j/LWqBZ3UJ3jd5ieS/g6jTztZ4KrT5atZPK9fBSrQdH8H7Mr/PZVMV+VzcLVV7969eeqpp2jXrh3h4eFs2LCBrKwsVq1aVeE248ePJzs727CkpqbWYOJaRqVir/9rAERqtnLy6D5l89wFUjbMwUt1jTRVfWyDRiodx6pU6gjuxqTPlbF69epKtfPw8ECj0ZCebtwlIT093aznz9zc3GjRogWnTlV80tzOzs5oIuu73cvDh5L26Sq8Mv7A/9B/2N8lhEBfN6VjWafCbNqkfAHA8YBX8JKDWZpVpY7gXF1dDYuLiwtxcXH89ddfhscTExOJi4szTA5dGVqtlo4dOxIXF2dYp9friYuLIyTEfJ0b8/LyOH36NN7e3mbb593gWkjZRZzHNfF8+eMahdNYsd3zqKPP5YS+AXWCBiudxupU6gjuxqTPAOPGjePpp59m0aJFhn5vOp2Ol19+GRcXF5OePCoqimHDhtGpUyeCgoKYM2cO+fn5hquqQ4cOpUGDBsTGxgJlFyaOHDli+PuFCxfYt28fderUoVmzZgCMGTOGxx57jMaNG3Px4kViYmLQaDQMGjTIpGx3O3vfDqzVhfC4Jp4BWV8ixFA5i5O55WUg4hegAj7WPc0nDdyVTmR9hIk8PDzEsWPHblp/7NgxUbduXVN3J+bNmycaNWoktFqtCAoKEnv27DE8FhoaKoYNG2b4OSUlRQA3LaGhoYY2kZGRwtvbW2i1WtGgQQMRGRkpTp06ZVKm7OxsAYjs7GyTX481+WnzdlE8yV2IGBdxNvFXpeNYlaISnTi25EUhYlxE8nsdRM9ZW5WOZFEq+xk1eUTf0tJSjh07RsuWLY3WHzt2DL1eb3KBffXVV3n11VfLfWzbtm1GP/v5+d12zLIVK+RUeObSL6wb2/f3JjR3HcS9Dx16yUEXzeSDbzcx4exKUMEC9WBeD2uhdCSrZHKBGzFiBCNHjuT06dMEBQUBkJCQwIwZMwxfLSXr4fjwBAp/+JXG+QfJO7ieOu0eVTqSxUu9WkDbU5+i1eg4X/d+5rzwBk52Jn8UpUow+V2dNWsWXl5efPzxx4Zhk7y9vXnnnXd4++23zR5QUlane1vzw7rHearoR1J/iKbAJYSOfrKX/Z2I276Noeqy2wcb9o8FWdyqjcn94NRqNWPHjuXChQtkZWWRlZXFhQsXGDt2bJVutpdqN5VKhX2PKHKEI63Uqfz41RwuykExq6xUp6fJgdmoVYJLDcKhwX1KR7Jqd9TR18XFxeQrp5LleTS4DWdalnVAfUG3ghXxpxVOZLkSd/1KqPgTHWrueex9peNYvSodG//www+sWrWKc+fOUVxcbPRYUlKSWYJJtYdKpaL9gPEUfvwdjYsycDz8LfSOVTqW5RECl10fAHCwXh/ae8lhvKqbyUdwc+fOZcSIEXh6epKcnExQUBD33HMPZ86coXfv3tWRUaoNtE4UdSk7x/pk7nfk5GQpm8cCnUlYS6uiAxQJW+55ZJLSce4KJhe4Tz/9lM8//5x58+ah1WoZO3Ysmzdv5vXXXyc7Ww51bc1cHxjFRZUn9VVZfDd3IntTriodyXLo9ah/nwpA/D1P4NtEdgupCSYXuHPnztGlSxcAHBwcyM3NBWDIkCEsX77cvOmk2sVGy1bvsmkGB5Ws5vnPNiscyHKc2fEtfsUnyRUO+D/xntJx7homFzgvLy+uXi37n7tRo0aGuVBTUlLkxMF3AZvApziq98VVVcBLNr/wny0nlY5U++lKcdg5A4B4z0E08q14bELJvEwucA899BBr164Fyjr9vvXWW/Tq1YvIyMhy50uVrEu3AC/mq8ru6x2u+ZVvtyRQojP9Dpa7ScqWz/EuPc8V4ULrJ8fffgPJbFTCxMMuvV6PXq/HxqbsAuyKFSvYvXs3zZs354UXXjCa7d5S5eTk4OrqSnZ2tuwGU46c68XkLniIBnkH+aa0J61G/ZeOjesqHatWEsUFXJ3Rlnv0mazzeZ1HR09VOpJVqOxn1KQjuNLSUqZNm0ZaWpph3cCBA5k7dy6vvfaaVRQ36fZcHLQ0GPAhAJGabfwU94eygWqxMxv/wz36TC4ID+57Uk4kU9NMKnA2NjbMnDmT0tLS6sojWQq/B8j37Y6tSkfnlEXsOXNF6US1jr4gi3r7FgCQ2OQFfDzkcEg1zeRzcD179mT79u3VkUWyME6PlPXE76vZzW9x8orqv51aMwMXkctp0YAH+5c/Yo5UvUy+k6F3795ER0dz8OBBOnbsiJOTk9Hjjz/+uNnCSbWcdyDZ/o/heuYXup1fxPXip3DQyvuRAfQ56fgeLxso9ljrN+jj7KhworuTyRcZ1OqKD/pUKhU6ne6OQylNXmSoPJF5Ct38ztigZ8eDy+gW1lfpSLXC99Oe4anS9RwQTfGL3oOLgzw/bU7VcpEB/ncVtbzFGoqbZBqVRzP2ezwGgMOOaZxMuzvnlP0n3dWz9C3ZBMBXjsNkcVOQ1U0bKNU8/wFTKcSWzuoTLP7yM3IKS5SOpKiEJe+gVen4Q3cvvfo8rXScu1qlC9z169dZt26d4efx48cTFRVlWN555x0KCwtNDrBgwQL8/Pywt7cnODiYvXv3Vtj28OHD9O/fHz8/P1QqFXPmzLnjfUp3zt2rMSUdRwEw4voyPtxwROFEyjmQFM/9uWUXXBZrhxBxr5zNTUmVLnBfffUVn332meHn+fPns3v3bpKTk0lOTuabb75h4cKFJj35ypUriYqKIiYmhqSkJAIDAwkPDycjI6Pc9gUFBfj7+zNjxowK5041dZ+SeTj3fOf/B8U8R+5fK5WOoxjV1umoVYINuiCu12undBypsrPYPPjgg2Lt2rWGn+vUqSNOnz5t+Pnrr78W999/v0kz4wQFBYlXXnnF8LNOpxM+Pj4iNjb2tts2btxYfPLJJ2bd5w1yVq2qmT/5hbIZuN5rJkqKrisdp8alHdohRIyLKJ3kKh6K/kwM/SJB6UhWq7Kf0UofwZ06dYq2bdsafra3tze6ohoUFGSYs7QyiouLSUxMJCwszLBOrVYTFhZGfHx8pfdjjn0WFRWRk5NjtEimGznmQy4LVxqrM0j8aa7ScWpUfmEJqT+MA+AHXSinRQNiHmutcCqp0gUuKyuLoqIiw8+XL1/Gz8/P8LNerzd6/HYyMzPR6XR4enoarff09DS6FcwUVd1nbGwsrq6uhsXX17dKz3+3s3dyJaXVSwD4H13AlWvXFE5Uc377ZQWdxGGKhA3LtJEcnhKOf706Sse661W6wDVs2JBDhw5V+PiBAwdo2LChWULVtPHjx5OdnW1YUlNTlY5kse578i3SVPWpRxYbv5hCYYn1dx3S6/QEHPkEgHNNn+H76KflNIC1RKUL3COPPMKkSZPKvVJ6/fp1pkyZQp8+fSr9xB4eHmg0GtLT043Wp6enV3gBobr2aWdnZ5hAR06kc2dstPaoHpoIwGO5K1n1x0GFE1Wvv85e5ZX3JtNKnCZf2NOo77s4amVxqy0qXeAmTJjA1atXadmyJR999BFr1qxhzZo1zJw5k5YtW3Lt2jUmTJhQ6SfWarV07NiRuLg4wzq9Xk9cXBwhISGmvYpq3KdkOs8HhnDNqSmuqgLyfv9Y6TjVavavRxhjswqAHR6R2Ll63mYLqUaZcuXizJkzIjw8XKjVaqFSqYRKpRJqtVqEh4cbXVGtrBUrVgg7OzuxdOlSceTIETF69Gjh5uYm0tLShBBCDBkyRERHRxvaFxUVieTkZJGcnCy8vb3FmDFjRHJysjh58mSl91kZ8irqnUvb+6MQMS6iYJKH2JF4QOk41UKn04t333tHiBgXkTmpgdh12PTPgFQ1lf2MmlTgbrhy5YpISEgQCQkJ4sqVK1UKeMO8efNEo0aNhFarFUFBQWLPnj2Gx0JDQ8WwYcMMP6ekpAjgpiU0NLTS+6wMWeDMQK8Xx6ffL0SMi1g28UmRVVCsdCKzO3ouXVyY5CdEjIs4t26m0nHuKpX9jJp8s/3dQN5sbx5Hdm+g9W+DKBEaon2+4OMXrGtI+73fvU/QiY+5oqnHPdGHwNZe6Uh3jWq72V6SKsuzXU+269phq9LR9fzn7D6VqXQk8ynMofWpxQDsb/aSLG61lCxwUrWp66RlZmkkAI+r45n6xSqFE5mP2D2POvocTul9cA8ZqnQcqQKywEnVRqVS8f6Lz7BOdz9qlWCMzSp2n7b8o7hz5/6mdNc8AOaISNo0vEfhRFJFZIGTqlXHxnVpPGA6pUJNT00ys//7ldKR7tjeZROw1V1nv96fVM8wtDbyY1RbyX8Zqdq1DezEKl0oAGNtV7LHgo/i/vNDHI/9/2CWM0sj6dNODodUm8kCJ9WIbqM+olDYEqQ+zvJvv1A6TpUIIfDZNwc7VSk7dW34Yso7jO7WVOlY0i3IAifViIaNm3O80UAAXij9lk0HLyicyHTr4rbypKZsDtiPSiOxt5UT7NR2ssBJNabpk5PIEQ60Vv/NhuWfUlRqOTfiCyGw3f4BGpVgo64zHULCbr+RpDhZ4KQaU8e9Pp+XPgpAlM33bNx/TuFElXco4XciNH+iEypmlT7Nu31aKR1JqgRZ4KQaFfDEWC4LF/zU6fy52jIGxSzR6SndMhmAPc7hbJz6PDYa+dGxBPJfSapRj3ZqwWqnQQC8brOa/LzaP3rymYR1dCg9QJGwwf+pqbJbiAWR/1JSjXty1LucFx54qrLY+tU0pePcmhC47voAgN+cHsW7cQuFA0mmkAVOqnH13F3Y2XA0AF0zvubAybPKBrqFPzd+hVf+UfKEPfsaP6d0HMlEssBJiug/PIozKl9cVQUc+aF2HsVdupZL3T0zANjpEUlUvwcUTiSZShY4SRG2trY4REwB4PHCtWzas0/ZQOX4fO50mqovcVXUodfzU+U8CxZIFjhJMd5BT5Kkb4ajqoiMdbXrKG77kVRG6csmsF5Q2heNg6vCiaSqkAVOUo5KxV9NXwdgkOZ3EpOTFA5URgjBjm9n4KO6ykVRl1aPRSkdSaoiWeAkRY0aOpQdurbYqnScW/2u0nEAWPL7AV6x+RmAOaX9eaxTE2UDSVUmC5ykKJVKZRgUs696NycP7FE4EeRunUNdVR6n9d4E9X0FOxt5z6mlqhUFbsGCBfj5+WFvb09wcDB79+69Zfvvv/+egIAA7O3tadu2LRs2bDB6fPjw4ahUKqMlIiKiOl+CdAe+GD+Kdbpg1CpBwcYYRbOs2bWf5zVlv0+f2wxmQJA8erNkihe4lStXEhUVRUxMDElJSQQGBhIeHk5GRka57Xfv3s2gQYMYOXIkycnJ9OvXj379+nHo0CGjdhEREVy6dMmwLF++vCZejlQFni723Dt4JqVCTeD1PZxJ2qJIjuyCEjI3xlJHVcgBfRPGvvWOIjkk81G8wM2ePZtRo0YxYsQIWrduzaJFi3B0dGTJkiXltv/Pf/5DREQE77zzDq1atWLq1Kncd999zJ8/36idnZ0dXl5ehsXd3b0mXo5URX4B7Ulw7Q1A/vpJZBcU13iGiUvX86xmMwA5D0zgHmc5kYylU7TAFRcXk5iYSFjY/4aeUavVhIWFER8fX+428fHxRu0BwsPDb2q/bds26tevT8uWLXnppZe4cuVKhTmKiorIyckxWqSa5/bIexQJW9rqDvPLj8tq/Pm7XVyCnaqUYw4dePDhp2r8+SXzU7TAZWZmotPp8PT0NFrv6elJWlpaudukpaXdtn1ERATLli0jLi6ODz/8kO3bt9O7d290uvLHH4uNjcXV1dWw+Pr63uErk6qiTUArzjUbDECHk3MpLS2tsec+eegv+mt2AODRdzqoVDX23FL1UfwranUYOHAgjz/+OG3btqVfv36sW7eOP//8k23btpXbfvz48WRnZxuW1NTUmg0sGfg8OpEc4UAb9d9Mjp1eI8+p0wtOrYxGoxJs0nXGI0DekmUtFC1wHh4eaDQa0tPTjdanp6fj5eVV7jZeXl4mtQfw9/fHw8ODU6dOlfu4nZ0dLi4uRoukDCf3+iwu7QPAyJJvKSwsrPbnTPhjM73/fzDLc+1lp15romiB02q1dOzYkbi4OMM6vV5PXFwcISEh5W4TEhJi1B5g8+bNFbYHOH/+PFeuXMHbW86AZAkGvTGDTOFCE3U6SWvmVetz6fQCp51lt4klu0cw6sne1fp8Ug0TCluxYoWws7MTS5cuFUeOHBGjR48Wbm5uIi0tTQghxJAhQ0R0dLSh/a5du4SNjY2YNWuWOHr0qIiJiRG2trbi4MGDQgghcnNzxZgxY0R8fLxISUkRW7ZsEffdd59o3ry5KCwsrFSm7OxsAYjs7Gzzv2CpUv5c8YEQMS4iPcZPXM/Pqbbn+fSLxULEuIjCSXVF5vmT1fY8knlV9jOqeIETQoh58+aJRo0aCa1WK4KCgsSePXsMj4WGhophw4YZtV+1apVo0aKF0Gq1ok2bNmL9+vWGxwoKCsTDDz8s6tWrJ2xtbUXjxo3FqFGjDAWzMmSBU17R9QJxMaapEDEuYvPn46vlOa7lFYrk99oLEeMilkx8qlqeQ6oelf2MqoQQQtljyNonJycHV1dXsrOz5fk4BW3+7hN6nZhMlnDi0vA9tGrSyKz7j1v9X3oeeJt8Ycfs1t/zXmSoWfcvVZ/Kfkat8iqqZB269n+FE/oGuKnyifvCvDfin0nPovG+jwFYafM44wd0Nev+pdpBFjip1rK301IcOhGA5zSb2Bi/z2z7XjR3Os3UF8sGsxw5Tc6SZaXkv6pUq9370DMk3xgUc715BsXMycvlDZsfAfjWdgC+PhV3MZIsmyxwUu2mUrGvxf8Gxdz15593vMs5seNpoLrCRVGXF97+4I73J9VessBJtd7Ap59lh64tWpWO9LWT72hfmVeuGAazXOc2FK2D0x3nk2ovWeCkWs9Bq+FP/1cA6Kfexa+/x91mi4od/GE696hyOa33ZvgrE80VUaqlZIGTLMLrQyNZrwtCrRKot1btHtWjp87Q+eK3AJSGTkCr1ZozolQLyQInWQRbjZpjrd6gVKjppUnk8J7fTN7HX9+8axjMsmWPZ6shpVTbyAInWYy3BvbhR31ZZ1z17++DCX3Uz54+xtOirCieafc2qOWv/t1A/itLFkOtVhE6ehZFwpZWxQc5uvOnSm974efJ2KlKOGrfnn795dHb3UIWOMmiePk248/6TwKg3TatwkFM/+nowb+4P2cTAI69p8jBLO8issBJFidgwGTycKCp7jR7fvnilm2FEOSsj0GjEhx07krjwO41E1KqFWSBkyyOh6cPx5sMA8B3/+xbDoq5c/tvBBfuRC9U1Os7taYiSrWELHCSRWrTfzxXcaGRuMRfP5c/KKYQAuLeB2B/3XC8mnWoyYhSLSALnGSR7Ou4kdLqJQBaHPuUvLzcm9rs376GrppDFAsNdR+dXMMJpdpAFjjJYrXr9xZpqnrU5yr7fpxp9Jhep8f+j7IOwfu9nqRx01ZKRJQUJgucZLFs7Ry42P5NANqe+YLsq5cNj+3b/A0BuhMUCDtaDpiiUEJJabLASRatfZ8X+Vvti6sqn0M/lA2npCst5Z69HwKwv+FgXOo1UDKipKBaUeAWLFiAn58f9vb2BAcHs3fv3lu2//777wkICMDe3p62bduyYcMGo8eFEEyaNAlvb28cHBwICwvj5MmT1fkSJIWobWzIDhkHQIcLy7l86Rz71y+isf48WdTh3qfMOxKwZFkUL3ArV64kKiqKmJgYkpKSCAwMJDw8nIyMjHLb7969m0GDBjFy5EiSk5Pp168f/fr149ChQ4Y2M2fOZO7cuSxatIiEhAScnJwIDw+vkTk2pZrXtudgTti0wFFVxJlVE/DZNweAQ01G4ux2j7LhJGVV//w3txYUFCReeeUVw886nU74+PiI2NjYcts//fTTok+fPkbrgoODxQsvvCCEEEKv1wsvLy/x0UcfGR7PysoSdnZ2Yvny5ZXKJGfVsjyH/lgrRIyLYUmP8RP5edU33aCkrMp+RhU9gisuLiYxMZGwsDDDOrVaTVhYGPHx8eVuEx8fb9QeIDw83NA+JSWFtLQ0ozaurq4EBwdXuM+ioiJycnKMFsmytHnwMQ7Z/a+f27m2r+Po5KxgIqk2ULTAZWZmotPp8PT0NFrv6elJWlpaudukpaXdsv2NP03ZZ2xsLK6urobF19e3Sq9HUpZd76kUCw1nNY3p0PdVpeNItYDi5+Bqg/Hjx5OdnW1YUlNTlY4kVUHz9l3Jfj6e+m9sRWNjq3QcqRawUfLJPTw80Gg0pKenG61PT0/Hy6v8mY68vLxu2f7Gn+np6Xh7exu1ad++fbn7tLOzw87OrqovQ6pF6vm2VDqCVIsoegSn1Wrp2LEjcXH/G2Nfr9cTFxdHSEhIuduEhIQYtQfYvHmzoX2TJk3w8vIyapOTk0NCQkKF+5QkyUrV0EWPCq1YsULY2dmJpUuXiiNHjojRo0cLNzc3kZaWJoQQYsiQISI6OtrQfteuXcLGxkbMmjVLHD16VMTExAhbW1tx8OBBQ5sZM2YINzc3sWbNGnHgwAHRt29f0aRJE3H9+vVKZZJXUSWpdqvsZ1TRr6gAkZGRXL58mUmTJpGWlkb79u3ZtGmT4SLBuXPnUP9jeOkuXbrw3Xff8e677zJhwgSaN2/Ozz//zL333mtoM3bsWPLz8xk9ejRZWVk8+OCDbNq0CXt7+xp/fZIkKUclhAkD298lcnJycHV1JTs7GxcXF6XjSJL0L5X9jMqrqJIkWS1Z4CRJslqywEmSZLUUv8hQG904LSlv2ZKk2unGZ/N2lxBkgStHbm7Z8Nfyli1Jqt1yc3NxdXWt8HF5FbUcer2eixcv4uzsjKqWzaGZk5ODr68vqamp8gpvOeT7c2vW8v4IIcjNzcXHx8eoG9m/ySO4cqjVaho2bKh0jFtycXGx6F/Q6ibfn1uzhvfnVkduN8iLDJIkWS1Z4CRJslqywFkYOzs7YmJi5OgnFZDvz63dbe+PvMggSZLVkkdwkiRZLVngJEmyWrLASZJktWSBkyTJaskCV8tdvXqVwYMH4+LigpubGyNHjiQvL++W23z++ed0794dFxcXVCoVWVlZNRO2hixYsAA/Pz/s7e0JDg5m7969t2z//fffExAQgL29PW3btmXDhg01lFQZprw/hw8fpn///vj5+aFSqZgzZ07NBa0BssDVcoMHD+bw4cNs3ryZdevWsWPHDkaPHn3LbQoKCoiIiGDChAk1lLLmrFy5kqioKGJiYkhKSiIwMJDw8HAyMjLKbb97924GDRrEyJEjSU5Opl+/fvTr149Dhw7VcPKaYer7U1BQgL+/PzNmzKhwoieLVr0jp0t34siRIwIQf/75p2Hdxo0bhUqlEhcuXLjt9lu3bhWAuHbtWjWmrFlBQUHilVdeMfys0+mEj4+PiI2NLbf9008/Lfr06WO0Ljg4WLzwwgvVmlMppr4//9S4cWPxySefVGO6mieP4Gqx+Ph43Nzc6NSpk2FdWFgYarWahIQEBZMpo7i4mMTERMLCwgzr1Go1YWFhxMfHl7tNfHy8UXuA8PDwCttbsqq8P9ZOFrhaLC0tjfr16xuts7GxoW7duqSlpSmUSjmZmZnodDrDhEQ3eHp6Vvh+pKWlmdTeklXl/bF2ssApIDo6GpVKdcvl2LFjSseUJIsnh0tSwNtvv83w4cNv2cbf3x8vL6+bTg6XlpZy9epV6zwhfBseHh5oNBrS09ON1qenp1f4fnh5eZnU3pJV5f2xdvIITgH16tUjICDglotWqyUkJISsrCwSExMN2/7+++/o9XqCg4MVfAXK0Gq1dOzYkbi4OMM6vV5PXFwcISEh5W4TEhJi1B5g8+bNFba3ZFV5f6ye0lc5pFuLiIgQHTp0EAkJCWLnzp2iefPmYtCgQYbHz58/L1q2bCkSEhIM6y5duiSSk5PF4sWLBSB27NghkpOTxZUrV5R4CWa1YsUKYWdnJ5YuXSqOHDkiRo8eLdzc3ERaWpoQQoghQ4aI6OhoQ/tdu3YJGxsbMWvWLHH06FERExMjbG1txcGDB5V6CdXK1PenqKhIJCcni+TkZOHt7S3GjBkjkpOTxcmTJ5V6CWYlC1wtd+XKFTFo0CBRp04d4eLiIkaMGCFyc3MNj6ekpAhAbN261bAuJiZGADctX375Zc2/gGowb9480ahRI6HVakVQUJDYs2eP4bHQ0FAxbNgwo/arVq0SLVq0EFqtVrRp00asX7++hhPXLFPenxu/P/9eQkNDaz54NZDDJUmSZLXkOThJkqyWLHCSJFktWeAkSbJassBJkmS1ZIGTJMlqyQInSZLVkgVOkiSrJQucJElWSxY4yWIMHz683JFXIiIilI4m1VJyNBHJokRERPDll18aratolvaSkhJsbW2N1hUXF6PVak1+3qpuJylLHsFJFsXOzg4vLy+jxd3dHQCVSsXChQt5/PHHcXJyYvr06UyePJn27dvz3//+lyZNmmBvbw/AuXPn6Nu3L3Xq1MHFxYWnn37aaJihiraTLIsscJJVmTx5Mk888QQHDx7kueeeA+DUqVP8+OOPrF69mn379qHX6+nbty9Xr15l+/btbN68mTNnzhAZGWm0r39vJ1ke+RVVsijr1q2jTp06RusmTJhgmEHsmWeeYcSIEUaPFxcXs2zZMurVqweUjQd38OBBUlJS8PX1BWDZsmW0adOGP//8k86dO5e7nWR5ZIGTLEqPHj1YuHCh0bq6desa/v7PCXpuaNy4sVGROnr0KL6+vobiBtC6dWvc3Nw4evSoocD9ezvJ8sgCJ1kUJycnmjVrdsvHK7Ouss8lWTZ5Dk6667Rq1YrU1FRSU1MN644cOUJWVhatW7dWMJlkbvIITrIoRUVFN02BZ2Njg4eHR6X3ERYWRtu2bRk8eDBz5syhtLSUl19+mdDQ0HK/4kqWSx7BSRZl06ZNeHt7Gy0PPvigSftQqVSsWbMGd3d3unXrRlhYGP7+/qxcubKaUktKkUOWS5JkteQRnCRJVksWOEmSrJYscJIkWS1Z4CRJslqywEmSZLVkgZMkyWrJAidJktWSBU6SJKslC5wkSVZLFjhJkqyWLHCSJFktWeAkSbJa/wcJVktjgVwHMQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.022292, + "end_time": "2024-03-01T08:03:12.093896", + "exception": false, + "start_time": "2024-03-01T08:03:12.071604", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5135.39536, + "end_time": "2024-03-01T08:03:14.838135", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/lct_gan/42/mlu-eval.ipynb", + "output_path": "eval/treatment/lct_gan/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/lct_gan/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "lct_gan" + }, + "start_time": "2024-03-01T06:37:39.442775", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/lct_gan/model.pt b/treatment/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e3448292379f7384a04c99c12efcdae93131a0e --- /dev/null +++ b/treatment/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa44e8b1c64400dcb7cbae969a7e28bf2478f52136ab2aacaa8dd8cf8014335 +size 74778241 diff --git a/treatment/lct_gan/params.json b/treatment/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..4410842506b52d79da1affeab41f2ec12dc54451 --- /dev/null +++ b/treatment/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "lct_gan", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/realtabformer/eval.csv b/treatment/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..956ad67a930ea8254dda10f5994bc1283c5af77d --- /dev/null +++ b/treatment/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.0,,0.0015596782029655273,2.3078653812408447,0.30178436636924744,5.190309524536133,0.41850993037223816,5.543264705920592e-06,10.77073621749878,0.02881322056055069,0.054603252559900284,0.039492759853601456,0.07855503261089325,8.974096999736503e-05,13.078601598739624 diff --git a/treatment/realtabformer/history.csv b/treatment/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..6a4d6e7e5d77d0c17f03a3b692d64f6482191d2c --- /dev/null +++ b/treatment/realtabformer/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03644084440955699,6.612403465312778,0.0045574880142667225,0.0,0.0,0.0,0.0,0.0,0.03644084440955699,320,160,166.3736925125122,1.0398355782032014,0.5199177891016007,0.07979056920851804,0.007586553805595031,3.274879661123663,0.00010986064107032512,0.0,0.0,0.0,0.0,0.0,0.007586553805595031,80,40,34.049020767211914,0.8512255191802979,0.42561275959014894,0.04524085290104267 +1,0.010942080118709896,3.6911100070230938,0.00037560693868070696,0.0,0.0,0.0,0.0,0.0,0.010942080118709896,320,160,166.92550325393677,1.0432843953371047,0.5216421976685524,0.14872968647239873,0.026652724126324755,7.903173122670358,0.0026581332727844825,0.0,0.0,0.0,0.0,0.0,0.026652724126324755,80,40,33.64800572395325,0.8412001430988312,0.4206000715494156,0.02281103633413295 +2,0.010455712701009312,3.9791094253597508,0.0003505937737199728,0.0,0.0,0.0,0.0,0.0,0.010455712701009312,320,160,166.77985906600952,1.0423741191625595,0.5211870595812798,0.15552815834055309,0.012862359535210999,5.169579194596042,0.000530584767799982,0.0,0.0,0.0,0.0,0.0,0.012862359535210999,80,40,33.91816258430481,0.8479540646076202,0.4239770323038101,0.026617402605188543 +3,0.007001378159810656,2.796910241492641,0.00011505681204335635,0.0,0.0,0.0,0.0,0.0,0.007001378159810656,320,160,166.83584594726562,1.04272403717041,0.521362018585205,0.13855754688775052,0.004590427323637414,6.018633032932655,3.031255504705524e-05,0.0,0.0,0.0,0.0,0.0,0.004590427323637414,80,40,33.972458839416504,0.8493114709854126,0.4246557354927063,0.04482766297978742 +4,0.0072793409375947245,2.296859505424894,0.0001573822781525827,0.0,0.0,0.0,0.0,0.0,0.0072793409375947245,320,160,167.01165390014648,1.0438228368759155,0.5219114184379577,0.16224303948229135,0.004423623792263243,6.739300958566085,6.933249757632432e-05,0.0,0.0,0.0,0.0,0.0,0.004423623792263243,80,40,33.651365756988525,0.8412841439247132,0.4206420719623566,0.026270963538991055 +5,0.006381618711196779,2.8839405857504543,0.00012163765120135503,0.0,0.0,0.0,0.0,0.0,0.006381618711196779,320,160,170.09552717208862,1.063097044825554,0.531548522412777,0.14938008590495427,0.0050863039046817,7.422684189675602,4.899889018377124e-05,0.0,0.0,0.0,0.0,0.0,0.0050863039046817,80,40,37.50921392440796,0.937730348110199,0.4688651740550995,0.029683142538488028 +6,0.006045459082537263,2.934065179698594,8.634548304524475e-05,0.0,0.0,0.0,0.0,0.0,0.006045459082537263,320,160,173.5575668811798,1.0847347930073739,0.5423673965036869,0.128542572739525,0.004006205599580426,6.11909287295653,2.419293156137453e-05,0.0,0.0,0.0,0.0,0.0,0.004006205599580426,80,40,36.28923416137695,0.9072308540344238,0.4536154270172119,0.02928218668603222 +7,0.005807553204306259,3.233931762049849,6.728753389705139e-05,0.0,0.0,0.0,0.0,0.0,0.005807553204306259,320,160,173.0330581665039,1.0814566135406494,0.5407283067703247,0.13020459838949136,0.004725078083447442,5.915451907179363,4.036447823969336e-05,0.0,0.0,0.0,0.0,0.0,0.004725078083447442,80,40,35.27379822731018,0.8818449556827546,0.4409224778413773,0.03375284938047116 +8,0.005597162095671138,2.4687882317225887,9.451023661321612e-05,0.0,0.0,0.0,0.0,0.0,0.005597162095671138,320,160,171.33222126960754,1.070826382935047,0.5354131914675235,0.16350435888944048,0.005341960320765793,4.580611580479113,5.5816291564272924e-05,0.0,0.0,0.0,0.0,0.0,0.005341960320765793,80,40,35.73819422721863,0.8934548556804657,0.44672742784023284,0.028984847072570118 +9,0.005475803721810735,2.9333888558279755,5.944834627596224e-05,0.0,0.0,0.0,0.0,0.0,0.005475803721810735,320,160,181.32838702201843,1.1333024188876153,0.5666512094438076,0.14286351032374114,0.00505017776886234,5.1855854878382335,4.6662756986093344e-05,0.0,0.0,0.0,0.0,0.0,0.00505017776886234,80,40,39.708540201187134,0.9927135050296784,0.4963567525148392,0.028717920677536313 +10,0.005619597199086002,2.6044400273167847,0.0001110976432221262,0.0,0.0,0.0,0.0,0.0,0.005619597199086002,320,160,191.31377625465393,1.1957111015915871,0.5978555507957936,0.1449692299744129,0.003595894821683032,4.273874850746305,3.0524735783286906e-05,0.0,0.0,0.0,0.0,0.0,0.003595894821683032,80,40,40.169761657714844,1.004244041442871,0.5021220207214355,0.038896191439198445 +11,0.005651603202302624,2.1689565299521996,8.184791047085491e-05,0.0,0.0,0.0,0.0,0.0,0.005651603202302624,320,160,191.00746726989746,1.1937966704368592,0.5968983352184296,0.13332524879906488,0.003731331395374582,4.425117476265046,2.9901788849429067e-05,0.0,0.0,0.0,0.0,0.0,0.003731331395374582,80,40,40.135857343673706,1.0033964335918426,0.5016982167959213,0.04076760089155869 +12,0.005675629577149266,1.5336641537275533,8.70407526315881e-05,0.0,0.0,0.0,0.0,0.0,0.005675629577149266,320,160,191.66054034233093,1.1978783771395682,0.5989391885697841,0.16240290804776122,0.0038316375943395543,4.709503752670197,2.2613220733549987e-05,0.0,0.0,0.0,0.0,0.0,0.0038316375943395543,80,40,40.268686056137085,1.0067171514034272,0.5033585757017136,0.028456049150554462 +13,0.00614657213177452,2.04597273792412,7.693676020577578e-05,0.0,0.0,0.0,0.0,0.0,0.00614657213177452,320,160,189.26595377922058,1.1829122111201287,0.5914561055600643,0.15553884600512902,0.0042953793235938065,4.08081223766667,3.2513099143405276e-05,0.0,0.0,0.0,0.0,0.0,0.0042953793235938065,80,40,40.096407890319824,1.0024101972579955,0.5012050986289978,0.02533484929444967 +14,0.005231396525572052,1.73261989282967,7.520394053998295e-05,0.0,0.0,0.0,0.0,0.0,0.005231396525572052,320,160,191.00612378120422,1.1937882736325265,0.5968941368162632,0.1496530410122432,0.003655807935683697,4.01203602381832,2.1951282996873765e-05,0.0,0.0,0.0,0.0,0.0,0.003655807935683697,80,40,40.18678307533264,1.004669576883316,0.502334788441658,0.03820230678429652 +15,0.005476373057177852,2.6803855865461252,8.069490082747689e-05,0.0,0.0,0.0,0.0,0.0,0.005476373057177852,320,160,191.20353507995605,1.1950220942497254,0.5975110471248627,0.12890588956015564,0.0038699784756772715,5.125400327792415,2.3996304280442248e-05,0.0,0.0,0.0,0.0,0.0,0.0038699784756772715,80,40,39.97018790245056,0.9992546975612641,0.49962734878063203,0.03100545472552767 diff --git a/treatment/realtabformer/mlu-eval.ipynb b/treatment/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..04ac833709cb3fda4ea1e9c4f862e1503d0e2d5c --- /dev/null +++ b/treatment/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2628 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.440606Z", + "iopub.status.busy": "2024-02-28T22:46:20.440338Z", + "iopub.status.idle": "2024-02-28T22:46:20.472910Z", + "shell.execute_reply": "2024-02-28T22:46:20.472197Z" + }, + "papermill": { + "duration": 0.047001, + "end_time": "2024-02-28T22:46:20.474926", + "exception": false, + "start_time": "2024-02-28T22:46:20.427925", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.499779Z", + "iopub.status.busy": "2024-02-28T22:46:20.499428Z", + "iopub.status.idle": "2024-02-28T22:46:20.506046Z", + "shell.execute_reply": "2024-02-28T22:46:20.505208Z" + }, + "papermill": { + "duration": 0.021123, + "end_time": "2024-02-28T22:46:20.507872", + "exception": false, + "start_time": "2024-02-28T22:46:20.486749", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.531214Z", + "iopub.status.busy": "2024-02-28T22:46:20.530952Z", + "iopub.status.idle": "2024-02-28T22:46:20.534804Z", + "shell.execute_reply": "2024-02-28T22:46:20.533980Z" + }, + "papermill": { + "duration": 0.017658, + "end_time": "2024-02-28T22:46:20.536612", + "exception": false, + "start_time": "2024-02-28T22:46:20.518954", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.560086Z", + "iopub.status.busy": "2024-02-28T22:46:20.559839Z", + "iopub.status.idle": "2024-02-28T22:46:20.564596Z", + "shell.execute_reply": "2024-02-28T22:46:20.563904Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018513, + "end_time": "2024-02-28T22:46:20.566370", + "exception": false, + "start_time": "2024-02-28T22:46:20.547857", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.589981Z", + "iopub.status.busy": "2024-02-28T22:46:20.589728Z", + "iopub.status.idle": "2024-02-28T22:46:20.594905Z", + "shell.execute_reply": "2024-02-28T22:46:20.594056Z" + }, + "papermill": { + "duration": 0.0195, + "end_time": "2024-02-28T22:46:20.596881", + "exception": false, + "start_time": "2024-02-28T22:46:20.577381", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f92cb7be", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.622481Z", + "iopub.status.busy": "2024-02-28T22:46:20.621857Z", + "iopub.status.idle": "2024-02-28T22:46:20.627221Z", + "shell.execute_reply": "2024-02-28T22:46:20.626424Z" + }, + "papermill": { + "duration": 0.019988, + "end_time": "2024-02-28T22:46:20.628985", + "exception": false, + "start_time": "2024-02-28T22:46:20.608997", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"realtabformer\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/realtabformer/0\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01156, + "end_time": "2024-02-28T22:46:20.651680", + "exception": false, + "start_time": "2024-02-28T22:46:20.640120", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.675074Z", + "iopub.status.busy": "2024-02-28T22:46:20.674834Z", + "iopub.status.idle": "2024-02-28T22:46:20.683731Z", + "shell.execute_reply": "2024-02-28T22:46:20.682904Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022929, + "end_time": "2024-02-28T22:46:20.685686", + "exception": false, + "start_time": "2024-02-28T22:46:20.662757", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/realtabformer/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:20.709305Z", + "iopub.status.busy": "2024-02-28T22:46:20.709045Z", + "iopub.status.idle": "2024-02-28T22:46:22.946964Z", + "shell.execute_reply": "2024-02-28T22:46:22.946035Z" + }, + "papermill": { + "duration": 2.251874, + "end_time": "2024-02-28T22:46:22.948957", + "exception": false, + "start_time": "2024-02-28T22:46:20.697083", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:22.975664Z", + "iopub.status.busy": "2024-02-28T22:46:22.975252Z", + "iopub.status.idle": "2024-02-28T22:46:22.989343Z", + "shell.execute_reply": "2024-02-28T22:46:22.988506Z" + }, + "papermill": { + "duration": 0.029358, + "end_time": "2024-02-28T22:46:22.991230", + "exception": false, + "start_time": "2024-02-28T22:46:22.961872", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:23.015334Z", + "iopub.status.busy": "2024-02-28T22:46:23.015056Z", + "iopub.status.idle": "2024-02-28T22:46:23.022432Z", + "shell.execute_reply": "2024-02-28T22:46:23.021606Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021828, + "end_time": "2024-02-28T22:46:23.024422", + "exception": false, + "start_time": "2024-02-28T22:46:23.002594", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:23.048190Z", + "iopub.status.busy": "2024-02-28T22:46:23.047929Z", + "iopub.status.idle": "2024-02-28T22:46:23.149543Z", + "shell.execute_reply": "2024-02-28T22:46:23.148838Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.115831, + "end_time": "2024-02-28T22:46:23.151607", + "exception": false, + "start_time": "2024-02-28T22:46:23.035776", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:23.178030Z", + "iopub.status.busy": "2024-02-28T22:46:23.177758Z", + "iopub.status.idle": "2024-02-28T22:46:27.841309Z", + "shell.execute_reply": "2024-02-28T22:46:27.840345Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.679411, + "end_time": "2024-02-28T22:46:27.843795", + "exception": false, + "start_time": "2024-02-28T22:46:23.164384", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-28 22:46:25.432696: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-28 22:46:25.432746: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-28 22:46:25.434571: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:27.869201Z", + "iopub.status.busy": "2024-02-28T22:46:27.868627Z", + "iopub.status.idle": "2024-02-28T22:46:27.874790Z", + "shell.execute_reply": "2024-02-28T22:46:27.873989Z" + }, + "papermill": { + "duration": 0.020612, + "end_time": "2024-02-28T22:46:27.876625", + "exception": false, + "start_time": "2024-02-28T22:46:27.856013", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T22:46:27.902920Z", + "iopub.status.busy": "2024-02-28T22:46:27.902257Z", + "iopub.status.idle": "2024-02-28T22:46:50.204577Z", + "shell.execute_reply": "2024-02-28T22:46:50.203353Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.318331, + "end_time": "2024-02-28T22:46:50.207281", + "exception": false, + "start_time": "2024-02-28T22:46:27.888950", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T23:00:41.335465Z", + "iopub.status.busy": "2024-02-28T23:00:41.335120Z", + "iopub.status.idle": "2024-02-28T23:14:31.088947Z", + "shell.execute_reply": "2024-02-28T23:14:31.087910Z" + }, + "papermill": { + "duration": 829.78602, + "end_time": "2024-02-28T23:14:31.105834", + "exception": false, + "start_time": "2024-02-28T23:00:41.319814", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-28T23:14:31.136884Z", + "iopub.status.busy": "2024-02-28T23:14:31.136577Z", + "iopub.status.idle": "2024-02-28T23:14:31.630269Z", + "shell.execute_reply": "2024-02-28T23:14:31.629305Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.511492, + "end_time": "2024-02-28T23:14:31.632432", + "exception": false, + "start_time": "2024-02-28T23:14:31.120940", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T23:14:31.665912Z", + "iopub.status.busy": "2024-02-28T23:14:31.665608Z", + "iopub.status.idle": "2024-02-28T23:14:31.669779Z", + "shell.execute_reply": "2024-02-28T23:14:31.668807Z" + }, + "papermill": { + "duration": 0.023076, + "end_time": "2024-02-28T23:14:31.671760", + "exception": false, + "start_time": "2024-02-28T23:14:31.648684", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T23:14:31.700201Z", + "iopub.status.busy": "2024-02-28T23:14:31.699929Z", + "iopub.status.idle": "2024-02-28T23:14:31.706585Z", + "shell.execute_reply": "2024-02-28T23:14:31.705699Z" + }, + "papermill": { + "duration": 0.023171, + "end_time": "2024-02-28T23:14:31.708552", + "exception": false, + "start_time": "2024-02-28T23:14:31.685381", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19390534" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T23:14:31.736946Z", + "iopub.status.busy": "2024-02-28T23:14:31.736683Z", + "iopub.status.idle": "2024-02-28T23:14:31.862055Z", + "shell.execute_reply": "2024-02-28T23:14:31.861176Z" + }, + "papermill": { + "duration": 0.141988, + "end_time": "2024-02-28T23:14:31.864203", + "exception": false, + "start_time": "2024-02-28T23:14:31.722215", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 52992] --\n", + "├─Adapter: 1-1 [2, 2648, 52992] --\n", + "│ └─Embedding: 2-1 [2, 2648, 69, 768] (215,808)\n", + "│ └─TensorInductionPoint: 2-2 [69, 1] 69\n", + "│ └─Sequential: 2-3 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 787,456\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 52992] (recursive)\n", + "│ └─Embedding: 2-4 [2, 661, 69, 768] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [69, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-7 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 19,606,342\n", + "Trainable params: 19,390,534\n", + "Non-trainable params: 215,808\n", + "Total mult-adds (M): 77.67\n", + "========================================================================================================================\n", + "Input size (MB): 1.83\n", + "Forward/backward pass size (MB): 3885.09\n", + "Params size (MB): 78.43\n", + "Estimated Total Size (MB): 3965.34\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-28T23:14:31.896841Z", + "iopub.status.busy": "2024-02-28T23:14:31.896573Z", + "iopub.status.idle": "2024-02-29T00:19:32.662416Z", + "shell.execute_reply": "2024-02-29T00:19:32.661320Z" + }, + "papermill": { + "duration": 3900.785913, + "end_time": "2024-02-29T00:19:32.665710", + "exception": false, + "start_time": "2024-02-28T23:14:31.879797", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03644084440955699, 'avg_role_model_std_loss': 6.612403465312778, 'avg_role_model_mean_pred_loss': 0.0045574880142667225, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03644084440955699, 'n_size': 320, 'n_batch': 160, 'duration': 166.3736925125122, 'duration_batch': 1.0398355782032014, 'duration_size': 0.5199177891016007, 'avg_pred_std': 0.07979056920851804}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007586553805595031, 'avg_role_model_std_loss': 3.274879661123663, 'avg_role_model_mean_pred_loss': 0.00010986064107032512, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007586553805595031, 'n_size': 80, 'n_batch': 40, 'duration': 34.049020767211914, 'duration_batch': 0.8512255191802979, 'duration_size': 0.42561275959014894, 'avg_pred_std': 0.04524085290104267}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010942080118709896, 'avg_role_model_std_loss': 3.6911100070230938, 'avg_role_model_mean_pred_loss': 0.00037560693868070696, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010942080118709896, 'n_size': 320, 'n_batch': 160, 'duration': 166.92550325393677, 'duration_batch': 1.0432843953371047, 'duration_size': 0.5216421976685524, 'avg_pred_std': 0.14872968647239873}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.026652724126324755, 'avg_role_model_std_loss': 7.903173122670358, 'avg_role_model_mean_pred_loss': 0.0026581332727844825, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.026652724126324755, 'n_size': 80, 'n_batch': 40, 'duration': 33.64800572395325, 'duration_batch': 0.8412001430988312, 'duration_size': 0.4206000715494156, 'avg_pred_std': 0.02281103633413295}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010455712701009312, 'avg_role_model_std_loss': 3.9791094253597508, 'avg_role_model_mean_pred_loss': 0.0003505937737199728, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010455712701009312, 'n_size': 320, 'n_batch': 160, 'duration': 166.77985906600952, 'duration_batch': 1.0423741191625595, 'duration_size': 0.5211870595812798, 'avg_pred_std': 0.15552815834055309}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012862359535210999, 'avg_role_model_std_loss': 5.169579194596042, 'avg_role_model_mean_pred_loss': 0.000530584767799982, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012862359535210999, 'n_size': 80, 'n_batch': 40, 'duration': 33.91816258430481, 'duration_batch': 0.8479540646076202, 'duration_size': 0.4239770323038101, 'avg_pred_std': 0.026617402605188543}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007001378159810656, 'avg_role_model_std_loss': 2.796910241492641, 'avg_role_model_mean_pred_loss': 0.00011505681204335635, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007001378159810656, 'n_size': 320, 'n_batch': 160, 'duration': 166.83584594726562, 'duration_batch': 1.04272403717041, 'duration_size': 0.521362018585205, 'avg_pred_std': 0.13855754688775052}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004590427323637414, 'avg_role_model_std_loss': 6.018633032932655, 'avg_role_model_mean_pred_loss': 3.031255504705524e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004590427323637414, 'n_size': 80, 'n_batch': 40, 'duration': 33.972458839416504, 'duration_batch': 0.8493114709854126, 'duration_size': 0.4246557354927063, 'avg_pred_std': 0.04482766297978742}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0072793409375947245, 'avg_role_model_std_loss': 2.296859505424894, 'avg_role_model_mean_pred_loss': 0.0001573822781525827, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0072793409375947245, 'n_size': 320, 'n_batch': 160, 'duration': 167.01165390014648, 'duration_batch': 1.0438228368759155, 'duration_size': 0.5219114184379577, 'avg_pred_std': 0.16224303948229135}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004423623792263243, 'avg_role_model_std_loss': 6.739300958566085, 'avg_role_model_mean_pred_loss': 6.933249757632432e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004423623792263243, 'n_size': 80, 'n_batch': 40, 'duration': 33.651365756988525, 'duration_batch': 0.8412841439247132, 'duration_size': 0.4206420719623566, 'avg_pred_std': 0.026270963538991055}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006381618711196779, 'avg_role_model_std_loss': 2.8839405857504543, 'avg_role_model_mean_pred_loss': 0.00012163765120135503, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006381618711196779, 'n_size': 320, 'n_batch': 160, 'duration': 170.09552717208862, 'duration_batch': 1.063097044825554, 'duration_size': 0.531548522412777, 'avg_pred_std': 0.14938008590495427}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0050863039046817, 'avg_role_model_std_loss': 7.422684189675602, 'avg_role_model_mean_pred_loss': 4.899889018377124e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0050863039046817, 'n_size': 80, 'n_batch': 40, 'duration': 37.50921392440796, 'duration_batch': 0.937730348110199, 'duration_size': 0.4688651740550995, 'avg_pred_std': 0.029683142538488028}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006045459082537263, 'avg_role_model_std_loss': 2.934065179698594, 'avg_role_model_mean_pred_loss': 8.634548304524475e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006045459082537263, 'n_size': 320, 'n_batch': 160, 'duration': 173.5575668811798, 'duration_batch': 1.0847347930073739, 'duration_size': 0.5423673965036869, 'avg_pred_std': 0.128542572739525}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004006205599580426, 'avg_role_model_std_loss': 6.11909287295653, 'avg_role_model_mean_pred_loss': 2.419293156137453e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004006205599580426, 'n_size': 80, 'n_batch': 40, 'duration': 36.28923416137695, 'duration_batch': 0.9072308540344238, 'duration_size': 0.4536154270172119, 'avg_pred_std': 0.02928218668603222}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005807553204306259, 'avg_role_model_std_loss': 3.233931762049849, 'avg_role_model_mean_pred_loss': 6.728753389705139e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005807553204306259, 'n_size': 320, 'n_batch': 160, 'duration': 173.0330581665039, 'duration_batch': 1.0814566135406494, 'duration_size': 0.5407283067703247, 'avg_pred_std': 0.13020459838949136}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004725078083447442, 'avg_role_model_std_loss': 5.915451907179363, 'avg_role_model_mean_pred_loss': 4.036447823969336e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004725078083447442, 'n_size': 80, 'n_batch': 40, 'duration': 35.27379822731018, 'duration_batch': 0.8818449556827546, 'duration_size': 0.4409224778413773, 'avg_pred_std': 0.03375284938047116}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005597162095671138, 'avg_role_model_std_loss': 2.4687882317225887, 'avg_role_model_mean_pred_loss': 9.451023661321612e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005597162095671138, 'n_size': 320, 'n_batch': 160, 'duration': 171.33222126960754, 'duration_batch': 1.070826382935047, 'duration_size': 0.5354131914675235, 'avg_pred_std': 0.16350435888944048}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005341960320765793, 'avg_role_model_std_loss': 4.580611580479113, 'avg_role_model_mean_pred_loss': 5.5816291564272924e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005341960320765793, 'n_size': 80, 'n_batch': 40, 'duration': 35.73819422721863, 'duration_batch': 0.8934548556804657, 'duration_size': 0.44672742784023284, 'avg_pred_std': 0.028984847072570118}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005475803721810735, 'avg_role_model_std_loss': 2.9333888558279755, 'avg_role_model_mean_pred_loss': 5.944834627596224e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005475803721810735, 'n_size': 320, 'n_batch': 160, 'duration': 181.32838702201843, 'duration_batch': 1.1333024188876153, 'duration_size': 0.5666512094438076, 'avg_pred_std': 0.14286351032374114}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00505017776886234, 'avg_role_model_std_loss': 5.1855854878382335, 'avg_role_model_mean_pred_loss': 4.6662756986093344e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00505017776886234, 'n_size': 80, 'n_batch': 40, 'duration': 39.708540201187134, 'duration_batch': 0.9927135050296784, 'duration_size': 0.4963567525148392, 'avg_pred_std': 0.028717920677536313}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005619597199086002, 'avg_role_model_std_loss': 2.6044400273167847, 'avg_role_model_mean_pred_loss': 0.0001110976432221262, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005619597199086002, 'n_size': 320, 'n_batch': 160, 'duration': 191.31377625465393, 'duration_batch': 1.1957111015915871, 'duration_size': 0.5978555507957936, 'avg_pred_std': 0.1449692299744129}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003595894821683032, 'avg_role_model_std_loss': 4.273874850746305, 'avg_role_model_mean_pred_loss': 3.0524735783286906e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003595894821683032, 'n_size': 80, 'n_batch': 40, 'duration': 40.169761657714844, 'duration_batch': 1.004244041442871, 'duration_size': 0.5021220207214355, 'avg_pred_std': 0.038896191439198445}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005651603202302624, 'avg_role_model_std_loss': 2.1689565299521996, 'avg_role_model_mean_pred_loss': 8.184791047085491e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005651603202302624, 'n_size': 320, 'n_batch': 160, 'duration': 191.00746726989746, 'duration_batch': 1.1937966704368592, 'duration_size': 0.5968983352184296, 'avg_pred_std': 0.13332524879906488}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003731331395374582, 'avg_role_model_std_loss': 4.425117476265046, 'avg_role_model_mean_pred_loss': 2.9901788849429067e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003731331395374582, 'n_size': 80, 'n_batch': 40, 'duration': 40.135857343673706, 'duration_batch': 1.0033964335918426, 'duration_size': 0.5016982167959213, 'avg_pred_std': 0.04076760089155869}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005675629577149266, 'avg_role_model_std_loss': 1.5336641537275533, 'avg_role_model_mean_pred_loss': 8.70407526315881e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005675629577149266, 'n_size': 320, 'n_batch': 160, 'duration': 191.66054034233093, 'duration_batch': 1.1978783771395682, 'duration_size': 0.5989391885697841, 'avg_pred_std': 0.16240290804776122}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0038316375943395543, 'avg_role_model_std_loss': 4.709503752670197, 'avg_role_model_mean_pred_loss': 2.2613220733549987e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038316375943395543, 'n_size': 80, 'n_batch': 40, 'duration': 40.268686056137085, 'duration_batch': 1.0067171514034272, 'duration_size': 0.5033585757017136, 'avg_pred_std': 0.028456049150554462}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00614657213177452, 'avg_role_model_std_loss': 2.04597273792412, 'avg_role_model_mean_pred_loss': 7.693676020577578e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00614657213177452, 'n_size': 320, 'n_batch': 160, 'duration': 189.26595377922058, 'duration_batch': 1.1829122111201287, 'duration_size': 0.5914561055600643, 'avg_pred_std': 0.15553884600512902}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0042953793235938065, 'avg_role_model_std_loss': 4.08081223766667, 'avg_role_model_mean_pred_loss': 3.2513099143405276e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0042953793235938065, 'n_size': 80, 'n_batch': 40, 'duration': 40.096407890319824, 'duration_batch': 1.0024101972579955, 'duration_size': 0.5012050986289978, 'avg_pred_std': 0.02533484929444967}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005231396525572052, 'avg_role_model_std_loss': 1.73261989282967, 'avg_role_model_mean_pred_loss': 7.520394053998295e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005231396525572052, 'n_size': 320, 'n_batch': 160, 'duration': 191.00612378120422, 'duration_batch': 1.1937882736325265, 'duration_size': 0.5968941368162632, 'avg_pred_std': 0.1496530410122432}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003655807935683697, 'avg_role_model_std_loss': 4.01203602381832, 'avg_role_model_mean_pred_loss': 2.1951282996873765e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003655807935683697, 'n_size': 80, 'n_batch': 40, 'duration': 40.18678307533264, 'duration_batch': 1.004669576883316, 'duration_size': 0.502334788441658, 'avg_pred_std': 0.03820230678429652}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005476373057177852, 'avg_role_model_std_loss': 2.6803855865461252, 'avg_role_model_mean_pred_loss': 8.069490082747689e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005476373057177852, 'n_size': 320, 'n_batch': 160, 'duration': 191.20353507995605, 'duration_batch': 1.1950220942497254, 'duration_size': 0.5975110471248627, 'avg_pred_std': 0.12890588956015564}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0038699784756772715, 'avg_role_model_std_loss': 5.125400327792415, 'avg_role_model_mean_pred_loss': 2.3996304280442248e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038699784756772715, 'n_size': 80, 'n_batch': 40, 'duration': 39.97018790245056, 'duration_batch': 0.9992546975612641, 'duration_size': 0.49962734878063203, 'avg_pred_std': 0.03100545472552767}\n", + "Epoch 16\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005867927873640611, 'avg_role_model_std_loss': 1.9389087880430025, 'avg_role_model_mean_pred_loss': 0.00011942612780763556, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005867927873640611, 'n_size': 320, 'n_batch': 160, 'duration': 190.98737239837646, 'duration_batch': 1.193671077489853, 'duration_size': 0.5968355387449265, 'avg_pred_std': 0.14752135770559108}\n", + "Time out: 3686.0073795318604/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▂█▄▁▁▁▁▁▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▁▂█▂▃▃▄▃▃▆▇▃▂▆▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁▇▇▆█▇▅▅█▆▆▅█▇▇▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▂█▄▁▁▁▁▁▂▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▁█▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▁█▄▅▆▇▅▅▃▄▃▃▃▂▂▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▄▄▃▂▃▃▃▂▃▂▂▁▂▁▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▁▁▁▁▁▅▄▃▃▇██████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▁▁▁▁▁▂▃▃▂▅███▇██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▁▁▁▁▁▅▄▃▃▇██████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▁▁▁▁▁▂▃▃▂▅███▇██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▁▁▁▁▁▅▄▃▃▇██████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▁▁▁▁▁▂▃▃▂▅███▇██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00387\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00548\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03101\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.12891\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00387\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00548\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 2e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 8e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 5.1254\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 2.68039\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.99925\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.19502\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.49963\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.59751\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 39.97019\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 191.20354\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/realtabformer/0/wandb/offline-run-20240228_231433-dmwrmfcg\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240228_231433-dmwrmfcg/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.00155967819215616, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0, 'pred_duration': 10.782897710800171, 'grad_duration': 2.320918083190918, 'total_duration': 13.103815793991089, 'pred_std': 0.07855503261089325, 'std_loss': 8.974096999736503e-05, 'mean_pred_loss': 5.543264705920592e-06, 'pred_rmse': 0.039492763578891754, 'pred_mae': 0.02881322056055069, 'pred_mape': 0.054603252559900284, 'grad_rmse': 0.41850996017456055, 'grad_mae': 0.3017843961715698, 'grad_mape': 5.190309524536133}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.00155967819215616, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 10.782897710800171, 'avg_grad_duration': 2.320918083190918, 'avg_total_duration': 13.103815793991089, 'avg_pred_std': 0.07855503261089325, 'avg_std_loss': 8.974096999736503e-05, 'avg_mean_pred_loss': 5.543264705920592e-06}, 'min_metrics': {'avg_loss': 0.00155967819215616, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0, 'pred_duration': 10.782897710800171, 'grad_duration': 2.320918083190918, 'total_duration': 13.103815793991089, 'pred_std': 0.07855503261089325, 'std_loss': 8.974096999736503e-05, 'mean_pred_loss': 5.543264705920592e-06, 'pred_rmse': 0.039492763578891754, 'pred_mae': 0.02881322056055069, 'pred_mape': 0.054603252559900284, 'grad_rmse': 0.41850996017456055, 'grad_mae': 0.3017843961715698, 'grad_mape': 5.190309524536133}, 'model_metrics': {'realtabformer': {'avg_loss': 0.00155967819215616, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0, 'pred_duration': 10.782897710800171, 'grad_duration': 2.320918083190918, 'total_duration': 13.103815793991089, 'pred_std': 0.07855503261089325, 'std_loss': 8.974096999736503e-05, 'mean_pred_loss': 5.543264705920592e-06, 'pred_rmse': 0.039492763578891754, 'pred_mae': 0.02881322056055069, 'pred_mape': 0.054603252559900284, 'grad_rmse': 0.41850996017456055, 'grad_mae': 0.3017843961715698, 'grad_mape': 5.190309524536133}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:19:32.715578Z", + "iopub.status.busy": "2024-02-29T00:19:32.714767Z", + "iopub.status.idle": "2024-02-29T00:19:32.720024Z", + "shell.execute_reply": "2024-02-29T00:19:32.718888Z" + }, + "papermill": { + "duration": 0.030478, + "end_time": "2024-02-29T00:19:32.722108", + "exception": false, + "start_time": "2024-02-29T00:19:32.691630", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:19:32.764230Z", + "iopub.status.busy": "2024-02-29T00:19:32.763105Z", + "iopub.status.idle": "2024-02-29T00:19:33.290989Z", + "shell.execute_reply": "2024-02-29T00:19:33.290124Z" + }, + "papermill": { + "duration": 0.552001, + "end_time": "2024-02-29T00:19:33.293673", + "exception": false, + "start_time": "2024-02-29T00:19:32.741672", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:19:33.339937Z", + "iopub.status.busy": "2024-02-29T00:19:33.339496Z", + "iopub.status.idle": "2024-02-29T00:19:33.651734Z", + "shell.execute_reply": "2024-02-29T00:19:33.650693Z" + }, + "papermill": { + "duration": 0.339022, + "end_time": "2024-02-29T00:19:33.653918", + "exception": false, + "start_time": "2024-02-29T00:19:33.314896", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEsUlEQVR4nO3deVxTV94/8E8SSMK+b0EWFxCriNuQYhe1MqK1tbROVeqo9edo29G2Su1YOiq2fZ5iV63L1LHT1uk8pVpnrNNaxxaptmNBUNCqVakrWCEsIglrAsn5/XHJhUACuWEJyPf9et1XknvPvfdwTb6ec+6554gYYwyEENJPiO2dAUIIEYKCFiGkX6GgRQjpVyhoEUL6FQpahJB+hYIWIaRfoaBFCOlXHOydgd5iMBhQXFwMNzc3iEQie2eHENIGYwzV1dVQKBQQiy2XpwZM0CouLkZISIi9s0EI6cSNGzcwaNAgi9sHTNByc3MDwF0Qd3d3O+eGENKWRqNBSEgI/1u1ZMAELWOV0N3dnYIWIX1YZ8031BBPCOlXKGgRQvoVClqEkH5lwLRpka7R6/VobGy0dzZIP+bo6AiJRNLl41DQIh1ijEGlUqGqqsreWSF3AE9PTwQGBnapryQFLdIhY8Dy9/eHs7MzdcwlNmGMoa6uDmVlZQCAoKAgm49FQasNdX0jLpRoIAKgHOJj7+zYlV6v5wOWj8/Avhak65ycnAAAZWVl8Pf3t7mqSA3xbZwquo15O4/jla/O2zsrdmdsw3J2drZzTsidwvhd6kr7KAWtNnxdZQCAihqtnXPSd1CVkHSX7vguUdBqwxi0Kmt1MBhozg9C+hoKWm14u0gBAE0GBk0D3eInpK+hoNWG1EEMdzl3f6KiRmfn3JD+SiQSYf/+/fbORrfasGEDxowZY+9sUNAyx1hFvEXtWqQf27VrFzw9PbvteKtXr0ZmZma3Hc9WNgWt7du3Izw8HHK5HEqlErm5uR2m37t3L6KioiCXyxEdHY2DBw+abN+wYQOioqLg4uICLy8vxMfHIycnxyRNeHg4RCKRybJx40Zbst8pH1euinirlkpa5M6n01n3PXd1de0TXV8EB609e/YgOTkZqampyM/PR0xMDBISEvhOY21lZWUhKSkJS5YswalTp5CYmIjExEScO3eOTxMZGYlt27bh7NmzOHbsGMLDwzFt2jSUl5ebHOvVV19FSUkJvzz77LNCs28VHxcqaZnDGEOdrskui9CJ0A8dOoR7770Xnp6e8PHxwUMPPYQrV64AACZOnIg1a9aYpC8vL4ejoyN++OEHAEBJSQlmzpwJJycnDB48GOnp6QgPD8fmzZttunZnz57FAw88ACcnJ/j4+GDZsmWoqanhtx89ehSxsbFwcXGBp6cn7rnnHhQWFgIAfvrpJ0yZMgVubm5wd3fH+PHjcfLkyQ7Pd/ToUSxevBhqtZr/T37Dhg0AuALAa6+9hoULF8Ld3R3Lli0DAKxZswaRkZFwdnbGkCFDsG7dOpOuCW2rh08++SQSExPx9ttvIygoCD4+Pli+fHmPP+4luHPpu+++i6VLl2Lx4sUAgB07duDrr7/GRx99hJdeeqld+vfeew/Tp0/Hiy++CAB47bXXkJGRgW3btmHHjh0AgCeeeKLdOT788EOcOXMGU6dO5de7ubkhMDBQaJYFM5a0qE3LVH2jHnet/8Yu5z7/agKcpdZ/XWtra5GcnIzRo0ejpqYG69evx6OPPorTp09j/vz5ePPNN7Fx40b+FvyePXugUChw3333AQAWLlyIiooKHD16FI6OjkhOTrb4H7M1eUlISEBcXBxOnDiBsrIy/OEPf8CKFSuwa9cuNDU1ITExEUuXLsVnn30GnU6H3NxcPm/z58/H2LFj8f7770MikeD06dNwdHTs8JwTJ07E5s2bsX79ehQUFADgSkpGb7/9NtavX4/U1FR+nZubG3bt2gWFQoGzZ89i6dKlcHNzw5/+9CeL5zly5AiCgoJw5MgRXL58GXPnzsWYMWOwdOlSm66VNQQFLZ1Oh7y8PKSkpPDrxGIx4uPjkZ2dbXaf7OxsJCcnm6xLSEiw2Eip0+mwc+dOeHh4ICYmxmTbxo0b8dprryE0NBRPPPEEVq1aBQcH83+CVquFVttSUtJoNNb8iQAAH+qr1e/Nnj3b5PNHH30EPz8/nD9/HnPmzMHKlStx7NgxPkilp6cjKSkJIpEIFy9exOHDh3HixAlMmDABAPC3v/0NERERNuUlPT0dDQ0N+OSTT+Di4gIA2LZtGx5++GG88cYbcHR0hFqtxkMPPYShQ4cCAEaMGMHvX1RUhBdffBFRUVEAYFU+pFIpPDw8IBKJzP5H/8ADD+CFF14wWbd27Vr+fXh4OFavXo3du3d3GLS8vLywbds2SCQSREVFYebMmcjMzOw7QauiogJ6vR4BAQEm6wMCAnDx4kWz+6hUKrPpVSqVyboDBw5g3rx5qKurQ1BQEDIyMuDr68tvf+655zBu3Dh4e3sjKysLKSkpKCkpwbvvvmv2vGlpaXjllVeE/Hk8P2ObFpW0TDg5SnD+1QS7nVuIS5cuYf369cjJyUFFRQUMBgMALgCMGjUK06ZNw6effor77rsP165dQ3Z2Nv76178CAAoKCuDg4IBx48bxxxs2bBi8vLxsyvuFCxcQExPDBywAuOeee2AwGFBQUID7778fTz75JBISEvDb3/4W8fHxmDNnDv98XnJyMv7whz/gH//4B+Lj4/H444/zwc1WxmDc2p49e7BlyxZcuXIFNTU1aGpq6nSU35EjR5o8jhMUFISzZ892KW+d6TN3D6dMmYLTp08jKysL06dPx5w5c0yK48nJyZg8eTJGjx6Np59+Gu+88w62bt1qUppqLSUlBWq1ml9u3LhhdV6MJa1btVTSak0kEsFZ6mCXRWhP6ocffhiVlZX44IMPkJOTw9/YMTY6z58/H//85z/R2NiI9PR0REdHIzo6utuvmbU+/vhjZGdnY+LEidizZw8iIyNx/PhxAFxb0s8//4yZM2fiu+++w1133YUvvviiS+drHUABrkY0f/58PPjggzhw4ABOnTqFP//5z5020retpopEIv4/iJ4iKGj5+vpCIpGgtLTUZH1paanFtqbAwECr0ru4uGDYsGG4++678eGHH8LBwQEffvihxbwolUo0NTXh+vXrZrfLZDJ+PHih48L7uFBJqz+7desWCgoKsHbtWkydOhUjRozA7du3TdI88sgjaGhowKFDh5Ceno758+fz24YPH46mpiacOnWKX3f58uV2x7DWiBEj8NNPP6G2tpZf9+OPP0IsFmP48OH8urFjxyIlJQVZWVkYNWoU0tPT+W2RkZFYtWoVvv32Wzz22GP4+OOPOz2vVCqFXq+3Ko9ZWVkICwvDn//8Z0yYMAERERH8jYC+RlDQkkqlGD9+vElfDYPBgMzMTMTFxZndJy4url3fjoyMDIvpWx/XUikKAE6fPg2xWAx/f38Bf4F1qE2rf/Py8oKPjw927tyJy5cv47vvvmvXruri4oLExESsW7cOFy5cQFJSEr8tKioK8fHxWLZsGXJzc3Hq1CksW7YMTk5ONj07N3/+fMjlcixatAjnzp3DkSNH8Oyzz2LBggUICAjAtWvXkJKSguzsbBQWFuLbb7/FpUuXMGLECNTX12PFihU4evQoCgsL8eOPP+LEiRMmbV6WhIeHo6amBpmZmaioqEBdXZ3FtBERESgqKsLu3btx5coVbNmypculuR7DBNq9ezeTyWRs165d7Pz582zZsmXM09OTqVQqxhhjCxYsYC+99BKf/scff2QODg7s7bffZhcuXGCpqanM0dGRnT17ljHGWE1NDUtJSWHZ2dns+vXr7OTJk2zx4sVMJpOxc+fOMcYYy8rKYps2bWKnT59mV65cYf/3f//H/Pz82MKFC63Ot1qtZgCYWq3uNO3tWi0LW3OAha05wLSNeiGX545SX1/Pzp8/z+rr6+2dFcEyMjLYiBEjmEwmY6NHj2ZHjx5lANgXX3zBpzl48CADwO6///52+xcXF7MZM2YwmUzGwsLCWHp6OvP392c7duyw6vxtz3XmzBk2ZcoUJpfLmbe3N1u6dCmrrq5mjDGmUqlYYmIiCwoKYlKplIWFhbH169czvV7PtFotmzdvHgsJCWFSqZQpFAq2YsUKq/9Nnn76aebj48MAsNTUVMYYY2FhYWzTpk3t0r744ovMx8eHubq6srlz57JNmzYxDw8PfntqaiqLiYnhPy9atIg98sgjJsd4/vnn2aRJkyzmp6PvlLW/UcFBizHGtm7dykJDQ5lUKmWxsbHs+PHj/LZJkyaxRYsWmaT//PPPWWRkJJNKpWzkyJHs66+/NvkjHn30UaZQKJhUKmVBQUFs1qxZLDc3l0+Tl5fHlEol8/DwYHK5nI0YMYK9/vrrrKGhweo8Cwlaer2BDU35moWtOcBKqvrfD7a79Oeg1d1u3LjBALDDhw/bOyv9WncELRFjAnvt9VMajQYeHh5Qq9VWtW/F/u9hlFVrceDZezEq2KMXctj3NDQ04Nq1axg8eDDkcrm9s9OrvvvuO9TU1CA6OholJSX405/+hJs3b+KXX37ptI8Usayj75S1v9E+c/ewr6F2rYGtsbERL7/8MkaOHIlHH30Ufn5+fEfTTz/9FK6urmaXkSNH9loeZ8yYYTEfr7/+eq/lo7fRcMsW+FJfrQEtISEBCQnm+6TNmjULSqXS7LbeLIX97W9/Q319vdlt3t7evZaP3kZBywJf6qtFLHBzc4Obm5u9s4Hg4GB7Z8EuqHpoAfXVIqRvoqBlQUubFgUtQvoSCloWtIypRdVDQvoSCloWUEM8IX0TBS0LaCBAQvomCloW8AMB1uoEj5pJyJ04sUVfQUHLAmOXB12TAdXaJjvnhhDhuntiC4AbxlkkEqGqqqpbjysEBS0L5I4SuMq4bmzUrkVI30FBqwP8HURq1+IwBuhq7bPQxBbdNrGFVqvF6tWrERwcDBcXFyiVShw9epTft7CwEA8//DC8vLzg4uKCkSNH4uDBg7h+/TqmTJkCgBv+RyQS4cknn7TpenQF9YjvgI+LFIW36qivllFjHfC6wj7nfrkYkLp0nq4ZTWxheWKLFStW4Pz589i9ezcUCgW++OILTJ8+HWfPnkVERASWL18OnU6HH374AS4uLjh//jxcXV0REhKCf/3rX5g9ezYKCgrg7u4OJycnm65JV1DQ6gANu9x/0cQW5ie2KCoqwscff4yioiIoFNx/QKtXr8ahQ4fw8ccf4/XXX0dRURFmz57NDz89ZMgQfn/jM43+/v7d3l5mLQpaHaC+Wm04OnMlHnudWwCa2MK8s2fPQq/XIzIy0mS9VqvlJ2J97rnn8Mwzz+Dbb79FfHw8Zs+ejdGjR9t0vp5AbVodoL5abYhEXBXNHgtNbNEtE1vU1NRAIpEgLy8Pp0+f5pcLFy7gvffeAwD84Q9/wNWrV7FgwQKcPXsWEyZMwNatW7vtb+0qClodaN1Xi/QfNLEFx9zEFmPHjoVer0dZWRmGDRtmsrSuRoaEhODpp5/Gvn378MILL+CDDz7gjwnA6gkzegIFrQ4Y+2pVVFNJqz+hiS045ia2iIyMxPz587Fw4ULs27cP165dQ25uLtLS0vD1118DAFauXIlvvvkG165dQ35+Po4cOcKfLywsDCKRCAcOHEB5ebnJHdBe0yMDQfdBQsaIN/rxcjkLW3OATX3naA/mrO/qz2PE08QWHHMTW+h0OrZ+/XoWHh7OHB0dWVBQEHv00UfZmTNnGGOMrVixgg0dOpTJZDLm5+fHFixYwCoqKvhjvvrqqywwMJCJRKJ280F0hsaIF0DoGPEA8EtpNaZt+gFezo44tX5aD+ew7xnIY8S39euvvyIkJASHDx/G1KlT7Z2dfqs7xoinu4cdMA4EeLuuEU16AxwkVJseKMxNbBEeHo7777/f3lkb8Gz6FW7fvh3h4eGQy+VQKpXIzc3tMP3evXsRFRUFuVyO6OhoHDx40GT7hg0bEBUVBRcXF3h5eSE+Pp6/22NUWVmJ+fPnw93dHZ6enliyZEmP16c9naUQNzdhVNZRY/xAQhNb9F2CS1p79uxBcnIyduzYAaVSic2bNyMhIQEFBQVmZ3vOyspCUlIS0tLS8NBDDyE9PR2JiYnIz8/HqFGjAHB3RrZt24YhQ4agvr4emzZtwrRp03D58mX4+fkB4BozS0pKkJGRgcbGRixevBjLli0zucPS3SRiEbxdpKio0eFWjQ7+bgO7ijSQ0MQWfZigVjTGWGxsLFu+fDn/Wa/XM4VCwdLS0symnzNnDps5c6bJOqVSyZ566imL5zA2yBknxjx//jwDwE6cOMGn+c9//sNEIhG7efOmVfm2pSGeMcamvfs9C1tzgP33l3JB+90J+nNDPOmbuqMhXlD1UKfTIS8vD/Hx8fw6sViM+Ph4ZGdnm90nOzvbJD3A/S9mKb1Op8POnTvh4eGBmJgY/hienp78IxUAEB8fD7FY3K4aaaTVaqHRaEwWW9Cwy6DxxEi36Y7vkqCgVVFRAb1ej4CAAJP1AQEBUKlUZvdRqVRWpT9w4ABcXV0hl8uxadMmZGRkwNfXlz9G26qng4MDvL29LZ43LS0NHh4e/BISEiLkT+UZ+2qVD8C+WsaqTl1dnZ1zQu4Uxu9SV6rRfebu4ZQpU3D69GlUVFTggw8+wJw5c5CTk2O2ncwaKSkpJh0KNRqNTYGrpaQ18BriJRIJPD09+dENnJ2dbepcSQhjDHV1dSgrK4OnpyckEonNxxIUtHx9fSGRSFBaWmqyvrS01OQRgNYCAwOtSu/i4sI/TnD33XcjIiICH374IVJSUhAYGNhuWJCmpiZUVlZaPK9MJoNMJhPy55nFT9o6QJ8/NF5fW4dlIaQ1T09Pi79ZawkKWlKpFOPHj0dmZiYSExMBAAaDAZmZmVixYoXZfeLi4pCZmYmVK1fy6zIyMhAXF9fhuQwGA7RaLX+Mqqoq5OXlYfz48QC4fjQGg8HiXZzuMtAnbRWJRAgKCoK/vz8aGxvtnR3Sjzk6OnaphMUT2vq/e/duJpPJ2K5du9j58+fZsmXLmKenJ1OpVIwxxhYsWMBeeuklPv2PP/7IHBwc2Ntvv80uXLjAUlNTmaOjIzt79ixjjLGamhqWkpLCsrOz2fXr19nJkyfZ4sWLmUwmY+fOneOPM336dDZ27FiWk5PDjh07xiIiIlhSUpLV+bb17uG3P6tY2JoDbNa2Y4L2I4QIY+1v1KZnD7du3cpCQ0OZVCplsbGx7Pjx4/y2SZMmtXse6fPPP2eRkZFMKpWykSNHsq+//prfVl9fzx599FGmUCiYVCplQUFBbNasWSw3N9fkGLdu3WJJSUnM1dWVubu7s8WLF/PPblnD1qCVV1jJwtYcYPdszBS0HyFEGHr2sA1bnj0EgKJbdbj/rSNwcpTgwmvTezCHhAxs1v5G6WG6ThjvHtY36lGno6nECLE3ClqdcJE5wMmRazysqB6YjfGE9CUUtKzQMoLpwOz2QEhfQkHLCvysPAO02wMhfQkFLSv4utCkrYT0FRS0rDCQH+UhpK+hoGUFY/WwgkpahNgdBS0rDPRHeQjpSyhoWYF/aJruHhJidxS0rNAy/yGVtAixNwpaVqDRSwnpOyhoWcEYtCprddAbBsSjmoT0WRS0rODtzAUtAwOqaCoxQuyKgpYVHCRieDlzY1pb7KvFGPDd/wJn9vZizggZeChoWanTvlplF4Af3gQOrAQMht7LGCEDDAUtK3XaV6uqkHvV1QCam72UK0IGHgpaVup0ggv1ry3vKwp6IUeEDEwUtKzkaxyexlJJq3XQKv+lF3JEyMBEQctKPp31iqeSFiG9goKWlXw6K2m1bseikhYhPYaClpV8XKhNi5C+wKagtX37doSHh0Mul0OpVCI3N7fD9Hv37kVUVBTkcjmio6Nx8OBBfltjYyPWrFmD6OhouLi4QKFQYOHChSguLjY5Rnh4OEQikcmyceNGW7JvE9+OxtQy6AFNq/zW3QJqb/VSzggZWAQHrT179iA5ORmpqanIz89HTEwMEhISLE6bnpWVhaSkJCxZsgSnTp1CYmIiEhMTce7cOQBAXV0d8vPzsW7dOuTn52Pfvn0oKCjArFmz2h3r1VdfRUlJCb88++yzQrNvsw6HXK5WAUwPiB0A90HcOiptEdIzhE6oGBsby5YvX85/1uv1TKFQsLS0NLPp58yZw2bOnGmyTqlUsqeeesriOXJzcxkAVlhYyK8LCwtjmzZtEppdnq2TtfL71+tY2JoDLGzNAVavazLdWHicsVR3xjaNYuwfj3HvT3xkc14JGYis/Y0KKmnpdDrk5eUhPj6eXycWixEfH4/s7Gyz+2RnZ5ukB4CEhASL6QFArVZDJBLB09PTZP3GjRvh4+ODsWPH4q233kJTk+V5CLVaLTQajcnSFW4yB0gl3OVqV0XUNLdnuQ8CfCO59xXUGE9IT3AQkriiogJ6vR4BAQEm6wMCAnDx4kWz+6hUKrPpVSqV2fQNDQ1Ys2YNkpKSTGaZfe655zBu3Dh4e3sjKysLKSkpKCkpwbvvvmv2OGlpaXjllVeE/HkdEolE8HWVoljdgIpqLYI9nVo2GhvhPShoEdLTBAWtntbY2Ig5c+aAMYb333/fZFtycjL/fvTo0ZBKpXjqqaeQlpYGmUzW7lgpKSkm+2g0GoSEhHQpfz6uMhSrG9r31VI3d3fwGAT4DefeU7cHQnqEoKDl6+sLiUSC0tJSk/WlpaUIDAw0u09gYKBV6Y0Bq7CwEN99951JKcscpVKJpqYmXL9+HcOHD2+3XSaTmQ1mXWGxrxZf0goGfJvzoi4CdLWA1KVb80DIQCeoTUsqlWL8+PHIzMzk1xkMBmRmZiIuLs7sPnFxcSbpASAjI8MkvTFgXbp0CYcPH4aPj0+neTl9+jTEYjH8/f2F/Ald0tJXq23QusG9eoQALj6Ac3P+Ky71Wt4IGSgEVw+Tk5OxaNEiTJgwAbGxsdi8eTNqa2uxePFiAMDChQsRHByMtLQ0AMDzzz+PSZMm4Z133sHMmTOxe/dunDx5Ejt37gTABazf/e53yM/Px4EDB6DX6/n2Lm9vb0ilUmRnZyMnJwdTpkyBm5sbsrOzsWrVKvz+97+Hl5dXd12LTvF9tdp2MDX2hncPbk44HCjK4tq1FGN6LX+EDASCg9bcuXNRXl6O9evXQ6VSYcyYMTh06BDf2F5UVASxuKUAN3HiRKSnp2Pt2rV4+eWXERERgf3792PUqFEAgJs3b+LLL78EAIwZM8bkXEeOHMHkyZMhk8mwe/dubNiwAVqtFoMHD8aqVatM2qx6g9lJW3V1XGdSgGvTAgC/SC5olVNfLUK6m00N8StWrMCKFSvMbjt69Gi7dY8//jgef/xxs+nDw8PBWMfjro8bNw7Hjx8XnM/uZqwemgwEaOwJL3UF5B7ce2O7FnUwJaTb0bOHAvAlrdZtWnx71iBAJOLe+zV3e6A7iIR0OwpaAviaG3K5dR8tPmFzSavyCqBv7KXcETIwUNASwBi0Kmt1MBinEmvbCA9wAczRBTA0AZXXejmXhNzZKGgJ4N08TnyTgUHT0FyCat3dwUgkAnwjuPfUrkVIt6KgJYDUQQx3OXfvgu9gyveGDzZNbOwZT4/zENKtKGgJ1G6CC3NtWkBLSYsa4wnpVhS0BDLpq8VYB0GLuj0Q0hMoaAlkMuxy/W2gqZ7b4G6peniJC26EkG5BQUsgk4emjY3wLv6AQ5uHs72HcCOZ0uSthHQrCloCmfTVstQIDwASRy5wAfQ4DyHdiIKWQL6te8Vbas/iE9OAgIR0NwpaAplM2mquj1Zr/ICAVNIipLtQ0BLIx6VVSctcb/jWfKmvFiHdjYKWQD4mbVqdVA/5B6eppEVId6GgJZCxTUvT0ARmbZtWXQVQV9kLuSPkzkdBSyB3uSMcxCJIoAeqS7iVloKW1KWlvYuqiIR0CwpaAonFIni7SOGPKoiYARA7cv20LPGlKiIh3YmClg18XWVQiCq4D+4KQNzBZaRuD4R0qz4172F/4eMqhZfIOC58J3MpUmM8Id2KgpYNfF1l8OeDloXuDnxienCakO5E1UMb+LhIW6qHlhrhjYwdTKtucDP3EEK6xKagtX37doSHh0Mul0OpVCI3N7fD9Hv37kVUVBTkcjmio6Nx8OBBfltjYyPWrFmD6OhouLi4QKFQYOHChSguLjY5RmVlJebPnw93d3d4enpiyZIlqKmpsSX7XebjKoNC1NyFwVLHUiMXX8DJGwADbtHkrYR0leCgtWfPHiQnJyM1NRX5+fmIiYlBQkICysrKzKbPyspCUlISlixZglOnTiExMRGJiYk4d+4cAKCurg75+flYt24d8vPzsW/fPhQUFGDWrFkmx5k/fz5+/vlnZGRk4MCBA/jhhx+wbNkyG/7krvNxbV3S6qRNC2j1OA81xhPSZUyg2NhYtnz5cv6zXq9nCoWCpaWlmU0/Z84cNnPmTJN1SqWSPfXUUxbPkZubywCwwsJCxhhj58+fZwDYiRMn+DT/+c9/mEgkYjdv3rQq32q1mgFgarXaqvQdybygYpXrFYylujOm+rnzHf79LJc287Uun5uQO5W1v1FBJS2dToe8vDzEx8fz68RiMeLj45GdnW12n+zsbJP0AJCQkGAxPQCo1WqIRCJ4enryx/D09MSECRP4NPHx8RCLxcjJyTF7DK1WC41GY7J0Fz+pHl6i5qppZw3xAD04TUg3EhS0KioqoNfrERAQYLI+ICAAKpXK7D4qlUpQ+oaGBqxZswZJSUlwd3fnj+Hvb9qB08HBAd7e3haPk5aWBg8PD34JCbGiGmclf3BVw2rmBCZz73wHenCakG7Tp+4eNjY2Ys6cOWCM4f333+/SsVJSUqBWq/nlxo0b3ZRLwKuRa78rZj6o1jZ1voOxr9atK4DeivSEEIsE9dPy9fWFRCJBaWmpyfrS0lIEBgaa3ScwMNCq9MaAVVhYiO+++44vZRmP0bahv6mpCZWVlRbPK5PJIJPJzG7rKmktd2ezmPlAWqODu9yx4x3cBwGOzkBjHXD7OuA7rEfyRchAIKikJZVKMX78eGRmZvLrDAYDMjMzERcXZ3afuLg4k/QAkJGRYZLeGLAuXbqEw4cPw8fHp90xqqqqkJeXx6/77rvvYDAYoFQqhfwJ3aN5dIdi5tsylVhHxGKavJWQbiK4epicnIwPPvgAf//733HhwgU888wzqK2txeLFiwEACxcuREpKCp/++eefx6FDh/DOO+/g4sWL2LBhA06ePIkVK1YA4ALW7373O5w8eRKffvop9Ho9VCoVVCoVdDpuQtQRI0Zg+vTpWLp0KXJzc/Hjjz9ixYoVmDdvHhQKRXdcB2Gax4YvZj4tk7Z2hh6cJqRbCH6MZ+7cuSgvL8f69euhUqkwZswYHDp0iG9sLyoqgrjVA8QTJ05Eeno61q5di5dffhkRERHYv38/Ro0aBQC4efMmvvzySwDAmDFjTM515MgRTJ48GQDw6aefYsWKFZg6dSrEYjFmz56NLVu22PI3d13zMMvFzAeKWitKWgA1xhPSTUSMDYxJ+TQaDTw8PKBWq03ay2yydTxw6zLm6dZi4tREPDc1ovN9zv8b+HwhoBgHLDvStfMTcgey9jfap+4e9gutZpW+yXysa9MCWpW0aPJWQrqCgpZQdbeApgYAQCnztr5Ny3sIIJIAumpAU9x5ekKIWRS0hGouZTXIfKGDIzfBhTUcpC2Tt9IdREJsRkFLqOag1ejKPb5zq9bKkhZAD04T0g0oaAllnOuw+ZlDq9u0gFZDL1NJixBbUdASqrm7g4MX9yzj7bpGNOkN1u1LJS1CuoyCllDN1UOZTyjEIm5VZZ3ADqbUV4sQm1HQEqq5N7zYMwTeLtzErbeE9oqvLQPqb/dE7gi541HQEqrVrNI+LtwD2VYHLZkr9/A0QFVEQmxEQUsIfSNQ0zx+l/sg+LpxJS2ruz0A9OA0IV1EQUuI6hKAGQCJFHDx40tagoIWjWJKSJdQ0BKiuT3LOKu0j2tzm5aQvlrUGE9Il1DQEoJvz+K6O/i6Gtu0qKRFSG+hoCVEcx8t41yHPkLvHgItD05XFQGN9d2ZO0IGBApaQvC94bk7gD7NJa0KIdVDF1/AyQsA40Z8IIQIQkFLiFbdHQC0tGkJqR6KRDQgICFdQEFLCLVpSctXaD8tIz8aepkQW1HQEsLYpmUMWs39tOob9ajTCZgajEpahNiMgpa1tDVAQxX3vrkh3lnqACdHCQCgotqGIWooaBEiGAUtaxkb4WUegLxl/Gpju1aFtRNcAC19tW5dpslbCRGIgpa12lQNjXxcbWjX8ggBHJwAvQ6oKuyuHBIyINgUtLZv347w8HDI5XIolUrk5uZ2mH7v3r2IioqCXC5HdHQ0Dh48aLJ93759mDZtGnx8fCASiXD69Ol2x5g8eTJEIpHJ8vTTT9uSfduoTQf/M/J1seEOoljcMss0NcYTIojgoLVnzx4kJycjNTUV+fn5iImJQUJCQrtp642ysrKQlJSEJUuW4NSpU0hMTERiYiLOnTvHp6mtrcW9996LN954o8NzL126FCUlJfzy5ptvCs2+7dp0dzCy6VEeoFVjPAUtQoQQHLTeffddLF26FIsXL8Zdd92FHTt2wNnZGR999JHZ9O+99x6mT5+OF198ESNGjMBrr72GcePGYdu2bXyaBQsWYP369YiPj+/w3M7OzggMDOSXLs9fKIQxaLmblrT4DqZCSloAjWJKiI0EBS2dToe8vDyT4CIWixEfH4/s7Gyz+2RnZ7cLRgkJCRbTd+TTTz+Fr68vRo0ahZSUFNTV1VlMq9VqodFoTJYu0Zg+d2hk06M8AI0XT4iNHIQkrqiogF6vR0BAgMn6gIAAXLx40ew+KpXKbHqVSiUoo0888QTCwsKgUChw5swZrFmzBgUFBdi3b5/Z9GlpaXjllVcEnaNDFqqHfm7NDfFC7h4CpiUtxrie8oSQTgkKWva0bNky/n10dDSCgoIwdepUXLlyBUOHDm2XPiUlBcnJyfxnjUaDkJCQdumswpjFhnh+TC0h/bQAwHtoy+St1SXccDeEkE4Jqh76+vpCIpGgtLTUZH1paSkCAwPN7hMYGCgovbWUSiUA4PLly2a3y2QyuLu7myw2q60A9FoAIsDNNLi0NMQLLGk5SAHvwdx7uoNIiNUEBS2pVIrx48cjMzOTX2cwGJCZmYm4uDiz+8TFxZmkB4CMjAyL6a1l7BYRFBTUpeNYxdhHyzWACzatGINWZa0OegMTdlx6nIcQwQRXD5OTk7Fo0SJMmDABsbGx2Lx5M2pra7F48WIAwMKFCxEcHIy0tDQAwPPPP49JkybhnXfewcyZM7F7926cPHkSO3fu5I9ZWVmJoqIiFBcXAwAKCriSh/Eu4ZUrV5Ceno4HH3wQPj4+OHPmDFatWoX7778fo0eP7vJF6FSbIWla83bmgpaBAVV1Ov5uolX8IoGCryloESKA4KA1d+5clJeXY/369VCpVBgzZgwOHTrEN7YXFRVBLG4pwE2cOBHp6elYu3YtXn75ZURERGD//v0YNWoUn+bLL7/kgx4AzJs3DwCQmpqKDRs2QCqV4vDhw3yADAkJwezZs7F27Vqb/3BB+Eb44HabHCRieDk74nZdI27VCgxavjSKKSFCiRhjAus0/ZNGo4GHhwfUarXw9q1v/gxkbwPiVgAJ/9tuc/y73+NyWQ3Slyoxcaiv9ce9mQd88ABX7VxNpS0ysFn7G6VnD61hobuDUZf7atWUAvVVNmaOkIGFgpY1jG1a7u2rhwDg62bDBBcAIHNruRtJ7VqEWIWCljU6KWkZH5quEFrSAmgUU0IEoqDVmSYdUN3ce99S9dDVxl7xAD04TYhAFLQ6U10CgAESGeBsvpGdHwiwSyUtqh4SYg0KWp1p3d1BbP5y+bjY2KYFUEmLEIEoaHWmk0Z4APC1dUwtAPCL4l5vFwINauH7EzLAUNDqDD/MsuWHrW0actnI1Y97eBoMuH7MhgwSMrBQ0OpMB73hjYxtWjXaJjQ06oWfY8hk7vXqUeH7EjLAUNDqjNryc4dGbjIHSB24S2lTFZGCFiFWo6DVmU76aAGASCRq6atVbUNj/OD7AJGY62BqDJKEELMoaHXGOMyyu+WgBXSxr5aTF6AYy72/9r3w/QkZQChodaRB03JHr4M2LaCLfbWAlirilSO27U/IAEFBqyPG7g5yD+45wQ609NXqYtC6epQb3pkQYhYFrY7wjfCdjy3P99WypYMpAIQouVmna8uAsgu2HYOQAYCCVkeMfbQ66FhqZPOkrUYOMiBsIvee7iISYhEFrY50MMxyW/ysPLaWtADq+kCIFShodcSK7g5GxpJWebUWNg8Gawxa149xo0sQQtrpN/Me2oWAoOXb3OXhoqoasa9nYlyoJ8aGemFcqBeigz3gJJV0fr6AUdxIEnUVwM2TLdVFQgiPglZHBAStqEA3PBDljx9+KUd5tRbf/FyKb37m5nt0EIswIsgdY0M9MS7UC2NDPRHq7QxR21mlxWJgyCTg3L+4KiIFLULaoaBlicFg1QgPRg4SMT568jdoaNTj3E018otu41RRFfKLbqNUo8XZm2qcvanGJ9mFALhx5cc2l8aMwUzuKOGqiMagNeXlnvv7COmnbGrT2r59O8LDwyGXy6FUKpGbm9th+r179yIqKgpyuRzR0dE4ePCgyfZ9+/Zh2rRp8PHxgUgk4idiba2hoQHLly+Hj48PXF1dMXv27HYzV3erugpArwMgEjRlvdxRggnh3lh2/1C8//vxOJ4yFVkvPYBtT4zFknsHY2yoJ6QSMW7V6nD4Qhne+qYAT3yQg5lb/ovqhsaWdq1fT3KdWwkhJgQHrT179iA5ORmpqanIz89HTEwMEhISUFZWZjZ9VlYWkpKSsGTJEpw6dQqJiYlITEzEuXPn+DS1tbW499578cYbb1g876pVq/DVV19h7969+P7771FcXIzHHntMaPatZ+zu4BYESBxtPoxIJILC0wkPjVZg3UN34Ys/3oOzr0zDvj9OxLqH7sLM0UHwcHLElfJavPVNAeAZCngPAZiehqohxBwmUGxsLFu+fDn/Wa/XM4VCwdLS0symnzNnDps5c6bJOqVSyZ566ql2aa9du8YAsFOnTpmsr6qqYo6Ojmzv3r38ugsXLjAALDs726p8q9VqBoCp1Wqr0rOf9zOW6s7YB1OtS98Fxy6Vs7A1B1j4SwfYyeu3GPtqJXfur1/s8XMT0ldY+xsVVNLS6XTIy8tDfHw8v04sFiM+Ph7Z2dlm98nOzjZJDwAJCQkW05uTl5eHxsZGk+NERUUhNDTU4nG0Wi00Go3JIogVQ9J0l3uG+eLx8YPAGPDSv86iMXwSt4H6axHSjqCgVVFRAb1ej4CAAJP1AQEBUKlUZvdRqVSC0ls6hlQqhaenp9XHSUtLg4eHB7+EhHT+KI4J451DKxrhu8OfZ46Ar6sUl8pq8OGvIQBE3LjxmuJeOT8h/cUd27k0JSUFarWaX27cuCHsAMYhaax47rA7eDpLsf7hkQCAd/9bhgb/0dyGqzRUDSGtCQpavr6+kEgk7e7alZaWIjAw0Ow+gYGBgtJbOoZOp0NVVZXVx5HJZHB3dzdZBBHQR6u7PDw6CFOG+0GnN+BgTfOEF1dpqBpCWhMUtKRSKcaPH4/MzEx+ncFgQGZmJuLi4szuExcXZ5IeADIyMiymN2f8+PFwdHQ0OU5BQQGKiooEHUcQvk2rd6qHAHen8X8ejYazVIK9VcO4lTRUDSEmBHcuTU5OxqJFizBhwgTExsZi8+bNqK2txeLFiwEACxcuRHBwMNLS0gAAzz//PCZNmoR33nkHM2fOxO7du3Hy5Ens3LmTP2ZlZSWKiopQXMy13xQUcHMABgYGIjAwEB4eHliyZAmSk5Ph7e0Nd3d3PPvss4iLi8Pdd9/d5YvQTpMWqDHOKt071UOjYE8nvJgwHBu/qkc9k8KpphQovwj4j+jVfBDSZ9lya3Lr1q0sNDSUSaVSFhsby44fP85vmzRpElu0aJFJ+s8//5xFRkYyqVTKRo4cyb7++muT7R9//DED0G5JTU3l09TX17M//vGPzMvLizk7O7NHH32UlZSUWJ1nQV0ebl3luhy85s+YwWD1ObpLk97AHtl2jH2/9h4uH9l/6fU8ENLbrP2NihgbGHUPjUYDDw8PqNXqztu3rh8Dds3k5iN8Lr93MtjGRZUG+7etwUsO6SgLmgz/p/5tl3wQ0lus/Y3esXcPu8SKuQ57WlSgOxTjZgAAXEqOQ11TZ7e8ENKXUNAyR9273R0smfPQDFTBHS5owGdf7LNrXgjpKyhomWOH7g7myKWOaAq/HwDQUJCJnKu37JofQvoCClrm9HJv+I74Rk8DANwjPoeUL86ioVFv5xwRYl8UtMwRMDZ8j2seqmas+DLKysux/chl++aHEDujoGVOH6keAgC8wgCvwXCAAUrxBbx/9AoKVNX2zhUhdkNBq60GNaBtHhGiD1QPAfClrSTfq2gyMKz51xnoDQOipwoh7VDQasv4+I6TFyBztW9ejIZOAQBMcjgPV5kDTt+owj+yr9s3T4TYCQWttvhG+D5QNTQKvw+ACI6VBdgw2QsA8NY3BSiuqrdvvgixA5rYoq1BE4Df/8veuTDl7A0oxgDFp/CY1xV8FhaGvMLbWLf/HP62aEL7WX2a6Q0MpZoG/Hq7Hr/ersOvt+txo5J7dXQQY/a4YEwfFQiZgxXTmxHSR1DQasvZGxgW33m63jZkMlB8CuKrR7Hxsbfx4Jb/IvNiGXafuIEIf1eTgPRrFfdaXFWPRr3ltq8ffimHt4sUj08YhCdiQxHm49J7fw8hNqJnD/uLq0eBTx4BXAOBFy5i0+FLeC/zUqe7OYi5iTVCvJ0wyNMZg7ycEOLtjMJbddh9oggl6gY+7X0RvpivDEP8CH84SKjlgPQua3+jVNLqL0LuBhzk3JA55QX445QIfP9LOc7eVCPIQ45BXk4Y5OWMEC/n5vdccApwl0MiNl99XD5lKI4UlOPTnEJ8/0s5/nupAv+9VIEAdxnm/SYU82JDEOTh1Mt/KCEdo5JWf/JJIjeS6fQ3gLufBmMMBgaLQUmIolt1+OxEET4/cQO3anUAuONOjfLH/LvDcN8wX4i74TyEWGLtb5SCVn9ybDNwOBWInAE8sbtHTqFrMuCbn1X4v+OFyLlWya8P9XbGE8pQ/G78IPi6ynrk3GRgo6DVxh0RtIpPAzsnAVI3YM21Lk0ia43LZdX4NKcI/8z7FdUNTfx6fzcZhvi5YIifK4b4umConyuG+LlgkJdzt5T6yMBEQauNOyJoGQzAW0OB+krg/30DhPbAUNNm1Ov0+OpMMT7NKcJPN6osppNKxAjzcTYJaEP8XDHUzwWeztJeySsx73atDqd/rcLZX9XwcHJE4phgeDj37H96QlHQauOOCFoAsPdJ4OcvgMkpwOSXev306vpGXKuoxdXyGlwtr8XVCu71WkUttE0Gi/t5OTsiyMMJQR5yBHjIEeQuR6AHtwR5yBHo4QRXGd0X6g7aJj3OF2tw+kYVvxTeMh1EUu4oxiMxwVgQF4ZRwR52yqkpClpt3DFBK28X8NXzQGgc8P8O2Ts3PIOB4WZVPa5W1OJKWQ0fzK6W10Klaej8AABcZQ58EAtwb3n1d5PBz00Gf3c5fF2l1Bm2FcYYrlXU4qdfq3C6iAtQ50s0ZvvnDfF1wehBHrioqsbFVg/djw31xIK7w/BgdBDkjva7thS02rhjgtbt68B7MYDYAVhzHZC52TtHnarVNqGosg4qdQNUmgaUqBtQqm5AiaYBKnU9VOoGaFq1mXXG09kRfq4y+LvLml/lJp/9moOcm9yxR9vYDAaG6oYmqOsboWlohE5vgMHA0GRg0LdauM8G6A1Ak8HQbpuBce8NjDum3vi5eZ2eMX69oTl9k4HhSnktfrpRBXV9Y7u8ebtIMSbEk19iBnny1UHGGPIKb+Mfxwtx8GwJH+C8nB0x5zch+L0yDCHezj123SyhoNXGHRO0AC5o3b4OJO0Bhk+3d266Ra22CSpNczBrDm7GIFdWrUVFtRbl1Vro9JaroOY4SyVwlTnAVe4AN5kD3OSO/GdXmQPcml+Nn52lDqjTNUFT3whNc0BS1zXygUld3/y+vhHV2qY+MSWlzEGMUcEeiBnkiTGhnhgb4olBXk4WH+9qrbxai89P3sCnxwtR3NzRWCQCpgz3x4K4MEyK8LOpq0uT3oDyGi2Kq7h/xwei/OEk7bgU16NBa/v27XjrrbegUqkQExODrVu3IjY21mL6vXv3Yt26dbh+/ToiIiLwxhtv4MEHH+S3M8aQmpqKDz74AFVVVbjnnnvw/vvvIyIigk8THh6OwsJCk+OmpaXhpZesa9e5o4LWV89z1UTlM8CMjfbOTa9hjEFd34jyai3KmoNYWXUDyvn3WpRrGuBVXYDBjZdQydxQxPxxg/mjDvIey5fcUQx3uSNkjmI4iMWQiEWQiESQiEVwkIggFongIOY+GxfusxgSMdcfTtycXiwyvufWi0Qtx+K2NacXi6DwkGNMiBeigtzg2MUnGJr0BhwpKMcn2dfx30sV/PpQb2fMV4ZizoQQeLlI+bStA1KJuh4lrV+rGlBW3YDWoyd9s/J+DA/suFbQYz3i9+zZg+TkZOzYsQNKpRKbN29GQkICCgoK4O/v3y59VlYWkpKSkJaWhoceegjp6elITExEfn4+Ro0aBQB48803sWXLFvz973/H4MGDsW7dOiQkJOD8+fOQy1u+bK+++iqWLl3Kf3Zz6/tVox4xZAoXtK4etXdOepVIJIKnsxSezlJEBLT6t6+9xXW6vZwJqDMBlAJtbow1yn1Q5zwI1c6DcFuqQIU0CGXiIBSLA/Cr3hPVWoYabRNqdXo4O0rg4eQIdycHeDg5Nr9v8yp35NPcCW1sDhIxfntXAH57VwCuVdTi0+OF+PzkDRRV1iHtPxfxTsYvGBHohrJqLUo1pgHJ4jHFIr5tsskgrITcEcElLaVSid/85jfYtm0bAMBgMCAkJATPPvus2VLP3LlzUVtbiwMHDvDr7r77bowZMwY7duwAYwwKhQIvvPACVq9eDQBQq9UICAjArl27MG/ePABcSWvlypVYuXKlTX/oHVXSqqsE3hwCgAEvFABugT17Pm0NcOU74JdvgPrbwLgFQOR0rh5hD/om4GYecCUTuHwYuJkPbn7fZo7O3Ggd2mquGl1/u+PjiR0BzxDAKxzwHgKMXcCNqjHA1ev0+OqnYnxy/DrO3dSYbDMGJIUnd+c3qPkGCrdwn31dZYKqlj1SPdTpdHB2dsY///lPJCYm8usXLVqEqqoq/Pvf7ScUDQ0NRXJyskmwSU1Nxf79+/HTTz/h6tWrGDp0KE6dOoUxY8bwaSZNmoQxY8bgvffeA8AFrYaGBjQ2NiI0NBRPPPEEVq1aBQcH84VFrVYLrVbLf9ZoNAgJCbkzghYA/HUSUHIaeHQnEDO3+49fdQP45RBQ8B/g+n8Bvc50e+Bo4P4XgaiHAHEvPFytvtkcpDK5UlWD2nS7/0hg2FRuCY0DHFr12q+vAqoKuQDGL82fq4oAQ5uGbJEEuOd5YNIawLHnqpX9BWMM525qcLOqng9MQgOSNXqkelhRUQG9Xo+AgACT9QEBAbh48aLZfVQqldn0KpWK325cZykNADz33HMYN24cvL29kZWVhZSUFJSUlODdd981e960tDS88sorQv68/mXIZC5oXT3SPUHLYACK87kg9cshoPSc6XavwcDwGYBYApz4CFCdAT5fAPjfBdy/GrgrkdvWXQx64EYOUHCQC1Rl5023yz25EV2HxQNDHwDcFZaP5eTJLUEx5s+jKW4JZpe+BS58CRx7F7j4NZD4F67UNoCJRCJED/JA9KC+0Z+r3/TmS05O5t+PHj0aUqkUTz31FNLS0iCTtX8WLiUlxWQfY0nrjjFkMvDjZq7Klvkq4OwDOHlz44E5+3DDRTv7AHIPy9U4XS1w5Qjwy3+AX74FastatonEQIiSqwYOnwH4RrYc555VwPG/ALk7uWDyz/8H+G4E7lsNjJoNSGz8WumbgMJjwPkvgYsHgJpS0/wEjweGTuUCVfC47gmSYglXNfQMAQbfx1V9L3wFHEgGKgqAD38L3P1H4IG1gCONeNEXCPp2+fr6QiKRoLS01GR9aWkpAgPNt6sEBgZ2mN74WlpaiqCgIJM0rauLbSmVSjQ1NeH69esYPnx4u+0ymcxsMLtjhMYBUlfukZ7/vmM5nUjSEsCcvVsCW7UKuPYDoG+pQkPqxlWvhs8Ahv0WcPExf0wXH2DqOmDis0DOX4Hj24GKX4AvlgHfbwTuewEYPde6ZyObdFw+zu/nSjb1LQ9pQ+7BPRwemcAFaWdva65M1414GAi7Bzj0EnBmD5C9jSuBPrIdCIvrnTwQiwQFLalUivHjxyMzM5Nv0zIYDMjMzMSKFSvM7hMXF4fMzEyTNq2MjAzExXH/+IMHD0ZgYCAyMzP5IKXRaJCTk4NnnnnGYl5Onz4NsVhs9o7lgOAoBxZ8wTWQ11UCdbe4H3xd81JfCehqAKYH6iq4xRzPMC5IRU7nfqgOAp4RdPIEJq8B7n4GOPEBkLUNqLwK/Hs58P0bwL3JwJgnTNuXAKCxgcv3hS+56l/r9iknb2DEQ8CIR4DB9wvLT3dy9gYe2wmMfAw4sBKovAJ8PANQPgVMXQ9I+9Eor9oaoOQn7uZF8SmuxDhsKletdvKyd+4EE3z3cM+ePVi0aBH++te/IjY2Fps3b8bnn3+OixcvIiAgAAsXLkRwcDDS0tIAcF0eJk2ahI0bN2LmzJnYvXs3Xn/9dZMuD2+88QY2btxo0uXhzJkzfJeH7Oxs5OTkYMqUKXBzc0N2djZWrVqFGTNm4O9//7tV+b6j7h5aq0nbEsDqbpm+d5Bz1Sy/qO67C6itAU5+BGRtAWrLuXXuwcC9q7hq47UfuED1yzdcQDVyDeAa9O96hAuctlYve0p9FfDNn4HT/8d99goHZm3jqpN9jb6Rq7LfzOeC1M18oPwCwMx0ORBJuCaAiN9ypVn/u+x3Rxg93Ll027ZtfOfSMWPGYMuWLVAqlQCAyZMnIzw8HLt27eLT7927F2vXruU7l7755ptmO5fu3LkTVVVVuPfee/GXv/wFkZGRAID8/Hz88Y9/xMWLF6HVajF48GAsWLAAycnJVlcBB2TQshddHZD/d278rxqV+TTuwcCIWcBds7gfTnc24veUS4eBr55rmYF8whLgt6/Y71EqxoDb11oFqDyuRNVk5llPNwXXDhg8jusCcikDKG9z88x9EBfAIqZxpdzumEKPMW4eUalrp//G9BhPGxS07KCxATj1Dy54aX7lqqJ3zeKqfsHje6erRHdr0AAZ67jOvQDgEQrMeo+raglhMHClzQY196qr414b67gbJPy6Wgvra7igY64PmswDCB7LXePg8YBiHOAe1D7d7etc8LqUwZWCm1pNSSeRcqXeyAQuiPkMbZ//ulvcf0rVpc2vKu7mifG1ppTb1lQPrMgDfId1eEkoaLVBQcuOmnTcnUn3YLtWP7rV1aPAl89y/bwAYNxC7uaFVsMFIn5p9VnbZj264acnkXJ95owBKng810FW6H8IjfXA9WNcl49fvuH6tbXmPYS7g2wMRLVlgMH6h9zx5EEg/J4Ok1DQaoOCFul22hrg8AbuJoStxI5c9VLqCkiduQZ+qQvg2PwqdW7e5sL19De+lzoDnqFAQHT336xgDLh1mQtel74FCrPad8AFAIgAF19uhihXf+7JDNeA9q+uAVx+O0FBqw0KWqTHXP+R6+rRpOW6acjcuVe5ByBv9V7m0Wp98zYHed8vfWqrgavfc3egjUHILRBw8evWIb9pCjFCekv4PUD4V/bORc+RuXHdUPqIftgSSggZyChoEUL6FQpahJB+hYIWIaRfoaBFCOlXKGgRQvoVClqEkH5lwPTTMvah1Wg0naQkhNiD8bfZWX/3ARO0qqu5GXXvqNFLCbkDVVdXw8PD8tDOA+YxHoPBgOLiYri5uXU6iaVxaOYbN27QIz+t0HWxjK6NeUKuC2MM1dXVUCgUEHfwwPeAKWmJxWIMGjRI0D7u7u70BTSDrotldG3Ms/a6dFTCMqKGeEJIv0JBixDSr1DQMkMmkyE1NfXOns3HBnRdLKNrY15PXJcB0xBPCLkzUEmLENKvUNAihPQrFLQIIf0KBS1CSL9CQYsQ0q9Q0Gpj+/btCA8Ph1wuh1KpRG5urr2zZHcbNmyASCQyWaKiouydrV73ww8/4OGHH4ZCoYBIJML+/ftNtjPGsH79egQFBcHJyQnx8fG4dOmSfTLbyzq7Nk8++WS779D06dNtOhcFrVb27NmD5ORkpKamIj8/HzExMUhISEBZWZm9s2Z3I0eORElJCb8cO3bM3lnqdbW1tYiJicH27dvNbn/zzTexZcsW7NixAzk5OXBxcUFCQgIaGsxMU3+H6ezaAMD06dNNvkOfffaZbSdjhBcbG8uWL1/Of9br9UyhULC0tDQ75sr+UlNTWUxMjL2z0acAYF988QX/2WAwsMDAQPbWW2/x66qqqphMJmOfffaZHXJoP22vDWOMLVq0iD3yyCPdcnwqaTXT6XTIy8tDfHw8v04sFiM+Ph7Z2dl2zFnfcOnSJSgUCgwZMgTz589HUVGRvbPUp1y7dg0qlcrk++Ph4QGlUknfn2ZHjx6Fv78/hg8fjmeeeQa3bt2y6TgUtJpVVFRAr9cjICDAZH1AQABUKpWdctU3KJVK7Nq1C4cOHcL777+Pa9eu4b777uPHKCPgvyP0/TFv+vTp+OSTT5CZmYk33ngD33//PWbMmAG9Xi/4WANmaBpiuxkzZvDvR48eDaVSibCwMHz++edYsmSJHXNG+ot58+bx76OjozF69GgMHToUR48exdSpUwUdi0pazXx9fSGRSFBaWmqyvrS0FIGBgXbKVd/k6emJyMhIXL582d5Z6TOM3xH6/lhnyJAh8PX1tek7REGrmVQqxfjx45GZmcmvMxgMyMzMRFxcnB1z1vfU1NTgypUrCAoKsndW+ozBgwcjMDDQ5Puj0WiQk5ND3x8zfv31V9y6dcum7xBVD1tJTk7GokWLMGHCBMTGxmLz5s2ora3F4sWL7Z01u1q9ejUefvhhhIWFobi4GKmpqZBIJEhKSrJ31npVTU2NScng2rVrOH36NLy9vREaGoqVK1fif/7nfxAREYHBgwdj3bp1UCgUSExMtF+me0lH18bb2xuvvPIKZs+ejcDAQFy5cgV/+tOfMGzYMCQkJAg/Wbfcg7yDbN26lYWGhjKpVMpiY2PZ8ePH7Z0lu5s7dy4LCgpiUqmUBQcHs7lz57LLly/bO1u97siRIwxAu2XRokWMMa7bw7p161hAQACTyWRs6tSprKCgwL6Z7iUdXZu6ujo2bdo05ufnxxwdHVlYWBhbunQpU6lUNp2LxtMihPQr1KZFCOlXKGgRQvoVClqEkH6FghYhpF+hoEUI6VcoaBFC+hUKWoSQfoWCFiGkX6GgRQjpVyhoEUL6FQpahJB+5f8DJjf+MZ0qHFMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:19:33.697225Z", + "iopub.status.busy": "2024-02-29T00:19:33.696845Z", + "iopub.status.idle": "2024-02-29T00:23:08.566035Z", + "shell.execute_reply": "2024-02-29T00:23:08.565136Z" + }, + "papermill": { + "duration": 214.893592, + "end_time": "2024-02-29T00:23:08.568713", + "exception": false, + "start_time": "2024-02-29T00:19:33.675121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:23:08.613751Z", + "iopub.status.busy": "2024-02-29T00:23:08.613365Z", + "iopub.status.idle": "2024-02-29T00:23:08.635830Z", + "shell.execute_reply": "2024-02-29T00:23:08.634883Z" + }, + "papermill": { + "duration": 0.047388, + "end_time": "2024-02-29T00:23:08.637850", + "exception": false, + "start_time": "2024-02-29T00:23:08.590462", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0NaN0.001562.3078650.3017845.190310.418510.00000610.7707360.0288130.0546030.0394930.0785550.0000913.078602
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.0 NaN 0.00156 2.307865 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.301784 5.19031 0.41851 0.000006 10.770736 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.028813 0.054603 0.039493 0.078555 0.00009 \n", + "\n", + " total_duration \n", + "realtabformer 13.078602 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:23:08.680347Z", + "iopub.status.busy": "2024-02-29T00:23:08.679709Z", + "iopub.status.idle": "2024-02-29T00:23:09.119104Z", + "shell.execute_reply": "2024-02-29T00:23:09.118206Z" + }, + "papermill": { + "duration": 0.463642, + "end_time": "2024-02-29T00:23:09.121769", + "exception": false, + "start_time": "2024-02-29T00:23:08.658127", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:23:09.165012Z", + "iopub.status.busy": "2024-02-29T00:23:09.164604Z", + "iopub.status.idle": "2024-02-29T00:26:57.965756Z", + "shell.execute_reply": "2024-02-29T00:26:57.964688Z" + }, + "papermill": { + "duration": 228.842612, + "end_time": "2024-02-29T00:26:57.985045", + "exception": false, + "start_time": "2024-02-29T00:23:09.142433", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:26:58.029341Z", + "iopub.status.busy": "2024-02-29T00:26:58.028945Z", + "iopub.status.idle": "2024-02-29T00:26:58.047413Z", + "shell.execute_reply": "2024-02-29T00:26:58.046452Z" + }, + "papermill": { + "duration": 0.043717, + "end_time": "2024-02-29T00:26:58.050007", + "exception": false, + "start_time": "2024-02-29T00:26:58.006290", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:26:58.091713Z", + "iopub.status.busy": "2024-02-29T00:26:58.091065Z", + "iopub.status.idle": "2024-02-29T00:26:58.097147Z", + "shell.execute_reply": "2024-02-29T00:26:58.096144Z" + }, + "papermill": { + "duration": 0.029509, + "end_time": "2024-02-29T00:26:58.099411", + "exception": false, + "start_time": "2024-02-29T00:26:58.069902", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.5586999652529121}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:26:58.146772Z", + "iopub.status.busy": "2024-02-29T00:26:58.146411Z", + "iopub.status.idle": "2024-02-29T00:26:58.538071Z", + "shell.execute_reply": "2024-02-29T00:26:58.537194Z" + }, + "papermill": { + "duration": 0.419424, + "end_time": "2024-02-29T00:26:58.540385", + "exception": false, + "start_time": "2024-02-29T00:26:58.120961", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHzUlEQVR4nO3dd3hUZdo/8O+ZmkmmpfdOFynSpAkqq2JFV+xCRMRVWFdZrlfx/S2Khbj7ri6su4trS8DGrrvgunZRsFOkKEqHNJJAeqbX8/z+OJlJJnVmMsmZydyf6xqSOefkzH2GzJ3nec5TOMYYAyGEiEgidgCEEEKJiBAiOkpEhBDRUSIihIiOEhEhRHSUiAghoqNERAgRHSUiQojoKBERQkRHiYj06PHHHwfHcSE9Z2lpKTiOw/fff+/X8f/3f/+HgoICSKVSTJgwIaSxkPBBiYgEZN26dXjnnXcG5bU++eQT/M///A9mzpyJkpISrFu3blBelww+mdgBkMiybt063HjjjViwYMGAv9bnn38OiUSCV155BQqFYsBfj4iHSkRDhNlsFjuEkKurq4NKpQpZEmKMwWq1huRc/WGxWMQOIexQIopAnrabw4cP47bbbkN8fDxmzZoFAHj99dcxadIkqFQqJCQk4JZbbkFVVZXPz3/11VdYuHAhcnJyoFQqkZ2djYceeqjPDynHcTCbzdi0aRM4jgPHcSgqKgIAVFRU4P7778fIkSOhUqmQmJiIhQsXory8vNtzWSwW3HvvvUhMTIRWq8WiRYvQ3Nzs81olJSUwm83e1yotLQUAuFwuPPnkkygsLIRSqUReXh4effRR2O12n9fIy8vD1VdfjY8//hiTJ0+GSqXC3//+d+zcuRMcx+Gf//wn1q5di8zMTGg0Gtx4441obW2F3W7Hgw8+iJSUFKjVatx1111dzu3vez137lyMHTsW+/btw0UXXYTY2Fg8+uijvb7P0YiqZhFs4cKFGD58ONatWwfGGJ5++mn87ne/w0033YSlS5eivr4ezz//PC666CIcOHAAer0eAPD222/DYrHgvvvuQ2JiIvbs2YPnn38eZ86cwdtvv93j67322mtYunQppk6dimXLlgEACgsLAQB79+7Ft99+i1tuuQVZWVkoLy/Hxo0bMXfuXBw+fBixsbE+51qxYgX0ej0ef/xxHDt2DBs3bkRFRYU3Sbz22mt48cUXsWfPHrz88ssAgBkzZgAAli5dik2bNuHGG2/Eb3/7W+zevRvFxcU4cuQItm3b5vM6x44dw6233op7770X99xzD0aOHOndV1xcDJVKhUceeQQnT57E888/D7lcDolEgubmZjz++OPYtWsXSktLkZ+fjzVr1nh/1t/3GgAaGxsxf/583HLLLbjjjjuQmpoa4P90FGAk4jz22GMMALv11lu928rLy5lUKmVPP/20z7GHDh1iMpnMZ7vFYulyzuLiYsZxHKuoqOjyOh3FxcWxxYsXd/n57s753XffMQBs8+bN3m0lJSUMAJs0aRJzOBze7X/4wx8YAPaf//zHu23x4sUsLi7O55wHDx5kANjSpUt9tq9atYoBYJ9//rl3W25uLgPAPvroI59jd+zYwQCwsWPH+sRw6623Mo7j2Pz5832Onz59OsvNzfU+D+S9njNnDgPAXnjhhS7vD2lHVbMI9qtf/cr7/datW8HzPG666SY0NDR4H2lpaRg+fDh27NjhPValUnm/N5vNaGhowIwZM8AYw4EDB4KKpeM5nU4nGhsbMWzYMOj1euzfv7/L8cuWLYNcLvc+v++++yCTyfDBBx/0+jqe/StXrvTZ/tvf/hYA8P777/tsz8/Px+WXX97tuRYtWuQTw7Rp08AYw5IlS3yOmzZtGqqqquByuQAE9l4DgFKpxF133dXrdUU7qppFsPz8fO/3J06cAGMMw4cP7/bYjh+4yspKrFmzBu+++65PuwwAtLa2BhWL1WpFcXExSkpKUF1dDdZh4s/uztk5TrVajfT09B7blDwqKiogkUgwbNgwn+1paWnQ6/WoqKjw2d7xPeosJyfH57lOpwMAZGdnd9nO8zxaW1uRmJgY0HsNAJmZmXTXrw+UiCJYx1IIz/PgOA4ffvghpFJpl2PVajUAwO124xe/+AWamprw8MMPY9SoUYiLi0N1dTWKiorA83xQsfz6179GSUkJHnzwQUyfPh06nQ4cx+GWW24J+py98bejZcf3qLPu3qfetnuSq7/vtT8xEAEloiGisLAQjDHk5+djxIgRPR536NAhHD9+HJs2bcKiRYu82z/99FO/XqenBPCvf/0LixcvxrPPPuvdZrPZ0NLS0u3xJ06cwMUXX+x9bjKZUFtbiyuvvLLX18/NzQXP8zhx4gRGjx7t3X7u3Dm0tLQgNzfXr+voD3/fa+I/aiMaIm644QZIpVKsXbvWp1oECH/JGxsbAbT/te94DGMMGzZs8Ot14uLiuk0uUqm0y+s+//zzcLvd3Z7nxRdfhNPp9D7fuHEjXC4X5s+f3+vrexLV+vXrfbY/99xzAICrrrqqr0voN3/fa+I/KhENEYWFhXjqqaewevVqlJeXY8GCBdBoNCgrK8O2bduwbNkyrFq1CqNGjUJhYSFWrVqF6upqaLVa/Pvf/+7SVtSTSZMmYfv27XjuueeQkZGB/Px8TJs2DVdffTVee+016HQ6jBkzBt999x22b9+OxMTEbs/jcDhw6aWX4qabbsKxY8fwt7/9DbNmzcK1117b6+uPHz8eixcvxosvvoiWlhbMmTMHe/bswaZNm7BgwQKfUtZA8fe9Jv6jRDSEPPLIIxgxYgT+9Kc/Ye3atQCEhtfLLrvM+wGXy+X473//iwceeADFxcWIiYnB9ddfjxUrVmD8+PF9vsZzzz2HZcuW4f/9v/8Hq9WKxYsXY9q0adiwYQOkUineeOMN2Gw2zJw5E9u3b+/xjtVf/vIXvPHGG1izZg2cTiduvfVW/PnPf/ar7efll19GQUEBSktLsW3bNqSlpWH16tV47LHHAni3+sef95r4j2Ody5aEEDLIqI2IECI6SkSEENFRIiKEiI4SESFEdJSICCGio0RECBFdRPcj4nkeNTU10Gg0IZ/knRDSf4wxGI1GZGRkQCLpudwT0Ymopqamy0hpQkj4qaqqQlZWVo/7IzoRaTQaAMJFarVakaMhhHRmMBiQnZ3t/az2JKITkac6ptVqKREREsb6ajqhxmpCiOgoERFCRCdqIsrLy/MuFdPxsXz5cjHDIoQMMlHbiPbu3eszcdZPP/2EX/ziF1i4cGHIXoMxBpfL1eMEXaRnUqkUMpmMukaQASdqIkpOTvZ5/swzz6CwsBBz5swJyfkdDgdqa2tpZc1+iI2NRXp6Ok3+TgZU2Nw1czgceP3117Fy5coe/wLb7XafFTcNBkOP5+N5HmVlZZBKpcjIyIBCoaC/7AFgjMHhcKC+vh5lZWUYPnx4rx3SCOmPsElE77zzDlpaWrxLGHenuLjYOxteXxwOB3ieR3Z2dpdVRol/VCoV5HI5Kioq4HA4EBMTI3ZIA87NM3x1oh6JcUqcn6UTO5yoETZ/4l555RXMnz8fGRkZPR6zevVqtLa2eh+d1xnvDv0V759oe/++OlGPA5UtiLLLFl1YlIgqKiqwfft2bN26tdfjlEollErlIEVFoo3J7sKPZ4TFILUx8j6OJqEUFnm/pKQEKSkpg7IUDCE9OXSmFW6eIVOvQnaCUJ1vtTjhcod+gUjiS/RExPM8SkpKsHjxYshkYVFAiyp5eXld1giLVifqjACAsZlC29C2A2fw6jdlqGyiu64DTfREtH37dlRWVmLJkiVih0KiWJPZgUaTA1IJh4LkOADt1TNKRANP9CLIZZdd1mW1TBIYh8NB/Xz6yZNsMvQqxMil3u9/PNOKs602MUOLCqKXiMTgcPE9Pjq3B/R2rNOPY4Mxd+5crFixAitWrIBOp0NSUhJ+97vfeRN2Xl4ennzySSxatAharRbLli0DAHz99deYPXs2VCoVsrOz8cADD8BsNnvPW1dXh2uuuQYqlQr5+fl44403gopvKLI4XJBKOGTHq7zb0nVCd4V6o53aiQaY6CUiMfx1x8ke9+UnxWHBxEzv8xe/PAWnu/sSW1a8Cgsnt0/M9uo3ZbA6fIeSPPSLEUHFuGnTJtx9993Ys2cPvv/+eyxbtgw5OTm45557AAB//OMfsWbNGu/qpqdOncIVV1yBp556Cq+++irq6+u9yaykpAQAUFRUhJqaGuzYsQNyuRwPPPAA6urqgopvqJlRmISpeQlw8e3/1zqVHCqFFFaHGw0mB9J0Q78flViiMhFFguzsbPzpT38Cx3EYOXIkDh06hD/96U/eRHTJJZfgt7/9rff4pUuX4vbbb8eDDz4IABg+fDj+/Oc/Y86cOdi4cSMqKyvx4YcfYs+ePZgyZQoAoe/W6NGjB/3awpVMKoFM2v6c4zgkq5WobLKgwWSnRDSAojIRLb94WI/7JJ1GgSy7qLDHYzuPGFkyM78/Yfm48MILfYakTJ8+Hc8++6x38O7kyZN9jv/hhx/w448/+lS3GGPeoS7Hjx+HTCbDpEmTvPtHjRoFvV4fspgjFWOsx+E/CWoFKpssaDI7Bjmq6BKViUgh879pbKCO7a+4uDif5yaTCffeey8eeOCBLsfm5OTg+PHjgxVaxNlxrA7VLTZcmJ+A4am+U5rmJcaBA5CTQMOEBlJUJqJIsHv3bp/nu3btwvDhwyGVSrs9/oILLsDhw4cxbFj3pb1Ro0bB5XJh37593qrZsWPH0NLSEtK4I1Ftqw0NRnu3+/KT4pCfFNftPhI6UXnXLBJUVlZi5cqVOHbsGN566y08//zz+M1vftPj8Q8//DC+/fZbrFixAgcPHsSJEyfwn//8BytWrAAAjBw5EldccQXuvfde7N69G/v27cPSpUuhUql6PGc0cLl5NJqEaleKltqAxEKJKEwtWrQIVqsVU6dOxfLly/Gb3/zGe5u+O+PGjcMXX3yB48ePY/bs2Zg4cSLWrFnjM4i4pKQEGRkZmDNnDm644QYsW7YMKSkpg3E5YavB5ICbZ1AppNDGdF9BsDhcqG21wuakyfUGClXNwpRcLsf69euxcePGLvvKy8u7/ZkpU6bgk08+6fGcaWlpeO+993y23Xnnnf2KM9KdMwidFVO1yh4brLcdqEadwY5rJ2SgMFk9mOFFDSoRkajmTUSanqtlOpUw1KPV6hyUmKIRJSIS1eraGql7ax+iRDTwqGoWhnbu3Cl2CFGBMYYktRIuN48Ubc/zXHkSkYES0YChRESiFsdxuGJsWp/HUYlo4FHVjJA+dCwR0UwRA4MSEYlaVofbr8SiVgoVB6ebwR7kjAqkd1Q1I1HrnYPVaDI7cM24DOQk9jyEQyaVYFp+AmIU0i7jC0loUCIiUYnnGRqMdrh4Bk0PHRk7mjEsaRCiil5UNSNRqcnigItnUMgk0MfSih1ioxIRiUp1BqH/ULK65x7VHdmcbjRbHFBIJUhU05JWoSZ6iai6uhp33HEHEhMToVKpcP755+P7778XOywyxNWb2hJRL/2HOvqhqgVb9lRhX0XzQIYVtUQtETU3N2PmzJm4+OKL8eGHHyI5ORknTpxAfHy8mGGRKFDXNrQjRdNLIjLUAE2nAU061DFCG5HJ7hqM8KKOqIno97//PbKzs71zKgNAfn7oZjnsgjHALUKnNKm863SOPdi8eTMeeugh1NTU+Kxqu2DBAmg0Grz22msDFWXUYIy1l4h6SkS1PwDHPhR+ZwAkJU4FkE2JaICImojeffddXH755Vi4cCG++OILZGZm4v777/fOy9yZ3W6H3d4+gZXBYAjsBd1O4Ktn+xNycGb/FpD5t9zPwoUL8cADD+Ddd9/FwoULAQirb7z//vu9jqwn/nPzDBOy9Kg32ZEY100iMp4Djn8sJCFNGmA8C83ZXYh1qmC0Rfe0KQNF1Dai06dPY+PGjRg+fDg+/vhj3HfffXjggQewadOmbo8vLi6GTqfzPrKzs7s9LpKpVCrcdtttPqXE119/HTk5OZg7d654gQ0hMqkEM4Yl4boJmZB2nqScMeDUZwDvBpKGA5OKgJTRUEo5ZLXuh8PFw+6ieYlCTdQSEc/zmDx5MtatWwcAmDhxIn766Se88MILWLx4cZfjV69ejZUrV3qfGwyGwJKRVC6UTgabNLDbw/fccw+mTJmC6upqZGZmorS0FEVFRX7d3SH91FIBNFcAEikw/BdClTp3JqR1R5Bkr0SZ2wKL3Q2lrPspe0lwRE1E6enpGDNmjM+20aNH49///ne3xyuVSp92k4BxnN9VJDFNnDgR48ePx+bNm3HZZZfh559/xvvvvy92WENGdYsVepUcccpufv2r9wlf08cDMTrhe3UyoE2HQtqMeGsFLM6RoNspoSVqIpo5cyaOHTvms+348ePIzc0VKaLwsXTpUqxfvx7V1dWYN2/ekKyGioExhncOVMPh4nHHhbm+jdU2A9DQtvhmxgW+P5g4HBm6Mui1zX71xCaBEbWN6KGHHsKuXbuwbt06nDx5Em+++SZefPFFLF++XMywwsJtt92GM2fO4KWXXsKSJUvEDmfIaDI74HDxkEs5JMZ1Kh3X/gAwHtDnCKWgjhILkaqNQb60Hlo5VZFDTdRENGXKFGzbtg1vvfUWxo4diyeffBLr16/H7bffLmZYYUGn0+GXv/wl1Go1FixYIHY4Q0Zta1v/IW0MJB0bqhkD6g4L36eP7/qD6lRAEQu4XYDp7CBEGl1EL2NeffXVuPrqq8UOIyxVV1fj9ttv71+7GPHhmaM6vfPy0aZzgKUJkMiEu2WdcRwccemwG4/CWVuGBF3WIEQbPURPRKSr5uZm7Ny5Ezt37sTf/vY3scMZUjwlorTOc1TXHRG+JhYCsu4Tf5UrAXXVBshcx5AwavZAhhl1KBGFoYkTJ6K5uRm///3vMXLkSLHDGTLsLrd3McW0jiUixtoTUcqYbn5SIIsXSkEyU+2AxRitKBGFoZ7WLSP9U9tiA88YdCo5NDEd+nYZagBbq9DfK7Gwx59XxKcD4MDsRsBhBhS0FHWoUCIiUSNVG4MrxqaB7zw9bP1R4WvS8F47n8aqYmGTaSBxG8BMdeASBnBcZJQZ8omIJjvvn6H0/qkUUoxO1/puZAxoOCF8n9R7NThOIYVFnoAYlwEOwzkoKRGFjOjzEQ0UuVz4y2axWESOJLJ53j/P+znkmBsAa7NwtyyhoNdDZVIJnKpEAICj5dxgRBc1hmyJSCqVQq/Xo66uDgAQGxtLY7UCwBiDxWJBXV0d9Ho9pNLIHltV22pFTYsVOQlxvr2pG9tKQ/G5/g3/iUsBmgCngfoShdKQTUQAkJYmLJ7nSUYkcHq93vs+RrIjtQb8UNWKcVlOXDo6tX1Hw3Hha3d9h7oxoqAQ8ZZYxLpaAJ4HJEO2UjGohnQi4jgO6enpSElJgdNJq3QGSi6XR3xJCBBKd6frzQCA/KQOd7rsRsDQdis+cZhf5xpTkANUq4VpQuwGQKUPcbTRaUgnIg+pVDokPlAkOA0mB4w2F+RSDtkJHdYv8zRSazMApca/k0kkQIwesDQKbUuUiEKCypVkyDtdbwIAZCfEQi7t8CvvvVs2wu9zWR1uNLM4mB0uwNoUyjCjGiUiMuQdrxMSUUGSun2jyy5Mggb43T4EAEfPGrCj0oXqZqtQIiIhQYmIDGn1RjsajHZIJRyGp3ZIRE2nhXae2AQgNtHv86kUUthkOjjdPGChRBQqlIjIkNZscUAhkyA/KQ4x8g7thJ5qWeIwv1dYAYBYuQw2uRYunlGJKISiorGaRK8RqRrkJ8XB5uww4T3vBhrbZmIMoH0IAGIUElhleqFEZGuhW/ghQu8gGfLkUonvINeWSqGNSBELaDMDOpdKLoVDGgcnLwHjXcItfNJvlIjIkGW0ObsfK+cpDSUOC7g0o5JLAY6DTaqGm2fCqH3Sb5SIyJDkcvN4bVcFXttVAaOtQ2dWxjr0pg6sWgYI480UMgnsMjWcbkYlohChNiIyJJU3mmF38lBIeag7LhtkqhNW65DKgPi8oM59YUEC9PJMyGwnqEQUIqKWiB5//HFwHOfzGDVqlJghkSHi6FkjAGBkmsZ3sLOnNBSfH/DClx6TchNQmJUpdI6kRBQSopeIzjvvPGzfvt37XCYTPSQS4WxON8raxpaNTOs0dCPAQa49immb14gSUUiI/qmXyWRDYnQ3CR+n6k1w8QyJagWS1R2m/LC2CFUzjvN7kGt3THYXzA4ltE43VDZqIwoF0RurT5w4gYyMDBQUFOD2229HZWVlj8fa7XYYDAafByGdHfNUy1I7Vcs8d8t0Wf2ab/pAZTP+9bNJWJrIbhAawEm/iJqIpk2bhtLSUnz00UfYuHEjysrKMHv2bBiNxm6PLy4uhk6n8z5oGWbSmc3pRlWTFUB31TJPb+r+VcuEvkSxcPJomw6k+99X4j9RE9H8+fOxcOFCjBs3Dpdffjk++OADtLS04J///Ge3x69evRqtra3eR1VV1SBHTMKdTMLhmvHpmFaQAH1shxkXXXahIyPQ7/ahGLkU4CSwStqmFKF2on4TvY2oI71ejxEjRuDkyZPd7lcqlbTqKemVTCpBQbIaBclq3x1NZcK69rEJwqMfYhXCmDUzpwZgpr5EISB6G1FHJpMJp06dQnp6utihkKGm6ZTwtZd1y/yl8iaithIRVc36TdREtGrVKnzxxRcoLy/Ht99+i+uvvx5SqRS33nqrmGGRCFVvtOPrEw2obbX67mCsvaE6IQSJqG0Uv5m1lc7tpn6fM9qJWjU7c+YMbr31VjQ2NiI5ORmzZs3Crl27kJycLGZYJEJVNJqxt7wJTRYHrh2vat9hrAUcFmGVDn1Ov1/HUyKycHFwMwYpVc36TdREtGXLFjFfngwx1S1CSShTr/Ld0dhWLYvPAyT9n7tcIZVgWkEC4k1ZQONRqpqFQFg1VhMSLMZYL4mow2j7EOA4DjMKk4BWO9DEAQ6qmvVXWDVWExKsBpNDGOQqkyCl4wKKdhNgbFsMMQTtQz48K3/YTdSpsZ8oEZEhwVMaStfFQCLp0Jvac7dMkwYo1d38ZHAMNidqrFLY3bzQLcBhDtm5oxElIjIknG21AQDSdT20D4WoWubx9YkG/OP7ajTY20bwUztRv1AiIkNCk9kBAEjVdqiW8TzQXC58H4L+Qx15buHbJG2JjxJRv1BjNRkSbpmSjWaLw3duamONMLRDHgOoQzvDg2dFEGGYhwlwUCLqD0pEZEiQSDgkqjsN//GUhvS5IV9po70vEfWuDgWqmpGhq6lM+JqQH/JTt/eu9lTN6BZ+f1CJiES87041otXqwLgsPTI8fYhcdsBQI3wf5NzUvfEmIlAbUShQiYhEvFP1JhypNcLi6LCIYkulcFtdFS88QixGIXx0jIgRNlAi6hcqEZGIxvMMzW13zJLUHeYfGsBqGQBoY+SYVpAALS8DzoAaq/uJEhGJaAabEy6eQSbhoO14x8zTUD0A1TJAuGs2ozAJcGmERORyCNVBGc2XFQyqmpGI1thWGoqPU7T3qLa1ApZGYZJ8fe7ABiBTti9LRL2rg0aJiEQ0T0fGxLgO1TJPaUiTLvQhGiAtFgeqW6xwytpu4dPg16BR1YxENE8iSvBJRBXC1wGqlnm8f6gWdQY77lAqkAxQiagfqEREIprTzUPCce2JiLH2SfLjB7Za1j7Mw1MiokQUrKBKRKdPn0ZBQUGoYyEkYFePy4DLzbdvsDYLt9IlUkCbOaCv7U1EHN3C76+gSkTDhg3DxRdfjNdffx02my3UMRESEJlUApm07VfZUxrSpAe9tr2/YhSeRNTWqZFKREELKhHt378f48aNw8qVK5GWloZ7770Xe/bsCXVshATOk4hCMDd1XzwlIgtrKxFRIgpaUIlowoQJ2LBhA2pqavDqq6+itrYWs2bNwtixY/Hcc8+hvr4+4HM+88wz4DgODz74YDAhkSj0c00rtuypxP7KZmEDY0Br26Kbg5iIzGjrO0R3zYLWr8ZqmUyGG264AW+//TZ+//vf4+TJk1i1ahWys7OxaNEi1NbW+nWevXv34u9//zvGjRvXn3BIlKk32lHbaoPJ5hI22FoAm0FoH9JlDfjre0bgm3iqmvVXvxLR999/j/vvvx/p6el47rnnsGrVKpw6dQqffvopampqcN111/V5DpPJhNtvvx0vvfQS4uNDPyaIDF2tVicAQKdqawsaxPYhAEhSKzGtIAEj89rmOnJahMnYSMCCSkTPPfcczj//fMyYMQM1NTXYvHkzKioq8NRTTyE/Px+zZ89GaWkp9u/f3+e5li9fjquuugrz5s3r81i73Q6DweDzINHLk4j0sZ0SkT57UF4/IU6BGYVJGJmZKvTiZgxwUqkoGEHdvt+4cSOWLFmCoqKiHpeHTklJwSuvvNLrebZs2YL9+/dj7969fr1ucXEx1q5dG3C8ZOhhjKHV0paIVG19iAaxodqHRALIY4WqmcPcvroH8VtQiejTTz9FTk4OJJ1mvWOMoaqqCjk5OVAoFFi8eHGP56iqqsJvfvMbfPrpp4iJ8a8b/urVq7Fy5Urvc4PBgOzswfnrR8KLye6Ci2eQcBw0MTLA2iK0D3ESQDvw7UMeTWYHrE430uRxkHoSEQlYUImosLAQtbW1SElJ8dne1NSE/Px8uN3uHn6y3b59+1BXV4cLLrjAu83tduPLL7/EX/7yF9jtdkilvqtyKpVKKJU0upkALW2lIa1KJgx29ZSGtOnC0tKD5K09lXC4eCzTqxAH0J2zIAWViFgPi8mZTCa/SzeXXnopDh065LPtrrvuwqhRo/Dwww93SUKEdOTiGbQqOeJjO1XLdINbQlbJpXC4eNglbYmIpowNSkCJyFMt4jgOa9asQWxsrHef2+3G7t27MWHCBL/OpdFoMHbsWJ9tcXFxSExM7LKdkM7yk+Jw96z89j+KIrUPqRRStFqd7cM8qGoWlIAS0YEDBwAIJaJDhw5BoWgvAisUCowfPx6rVq0KbYSE9ILjuLb2oVahfUiEEhEAWL3DPKhEFIyAEtGOHTsACFWoDRs2QKvVhjSYnTt3hvR8JEp4elNr0ga1fQjosL4ZjTfrl6DaiEpKSkIdByEB2fRtOVRyKeafnwbNIPcf6si7vhmoatYffieiG264AaWlpdBqtbjhhht6PXbr1q39DoyQnticbu+EaEqZFGjxjC8b4Glhu+GpmpmYZ7yZUejYyHGDHksk8zsR6XQ6oT7e9j0hYjHYhFv3KoUUCpdJmIOI4wZlfFlnmfEqXFiQiNQ4DmgB4HYBbgdNoh8gvxNRx+oYVc2ImIxtg1y1MfL29iF1qigf/ky9CpmeRR2PywG3U6ieUSIKSFBjzaxWKywWi/d5RUUF1q9fj08++SRkgRHSE08i0sTIOlTLwqCHvWdoB905C1hQiei6667D5s2bAQAtLS2YOnUqnn32WVx33XXYuHFjSAMkpDND22BXIRF5OjIO8viyNjzP0Giyo6bFCijihI3UqTFgQc/QOHv2bADAv/71L6SlpaGiogKbN2/Gn//855AGSEhnnhKRXuYQ1i8DRCsR2VxubP6uAv/YWwVe3paI6M5ZwIJKRBaLBRqNUAz95JNPcMMNN0AikeDCCy9ERUVFSAMkpLMYuUQY3uGsEzaokwG5SpxYZFLvDTKHlDo1BivoyfPfeecdVFVV4eOPP8Zll10GAKirqwt5J0dCOrt0dCrunpWPXEmDsEGkahkASCSc0IUAgJ06NQYtqES0Zs0arFq1Cnl5eZg2bRqmT58OQCgdTZw4MaQBEtKjVpHmH+pEJRc+RjYJJaJgBdWz+sYbb8SsWbNQW1uL8ePHe7dfeumluP7660MWHCE9cloBs6dENPj9hzpSKaRotnQc+EpVs0AFveR0Wloa0tLSfLZNnTq13wER0puqJgs+/vksRkhqcRFjQGwioFSLGlOMd1khaiMKVlCJyGw245lnnsFnn32Guro68J0mDD99+nRIgiOks1arE0abC1K+ClBA9GoZ0HFZobYSkdMqTKIv6dfaFFElqES0dOlSfPHFF7jzzjuRnp7uHfpByEDz3LpPcJxrS0Tid2QsSFZDEyNHekIMUNNhEn2au9pvQSWiDz/8EO+//z5mzpwZ6ngI6ZXR5oSUt0PjagSgGvT5h7ozLEWNYSlt1UNFnNChkSbRD0hQZcf4+HgkJCSEOhZC+mSwuaCxn4NSKgFU8UBMmHUXUVCnxmAElYiefPJJrFmzxme8GSGDwWhzQmuvhVIuCYtqGQC43DwaTXbUGWyAoq1kRA3WAQmqavbss8/i1KlTSE1NRV5eHuRy31U1/VlYkZBAMcZgtLmQZauBQi8Ji4ZqAKgz2vGPvVXQqeRYkkTjzYIRVCJasGBBiMMgpG8ON48MNYdE1gSFVCfKRGjd8c5b7XRT1SxIQSWixx57LNRxENInpUyKGwt4wKIX+g+FSfuQZ7pYh4uHSxYrfKioahaQoDs6tLS04OWXX8bq1avR1NQEQKiSVVdX+32OjRs3Yty4cdBqtdBqtZg+fTo+/PDDYEMi0aC5XPgaHx6lIQBQyiSQtHVhsUvbltiiElFAgioR/fjjj5g3bx50Oh3Ky8txzz33ICEhAVu3bkVlZaV3rqK+ZGVl4ZlnnsHw4cPBGMOmTZtw3XXX4cCBAzjvvPOCCY0MYYwxcC1tszvE54kaS0ccxyFGLoHF4YYNnhVfKREFIqgS0cqVK1FUVIQTJ074rOx65ZVX4ssvv/T7PNdccw2uvPJKDB8+HCNGjMDTTz8NtVqNXbt2BRMWGeK+/rkM+4+cwFmDPWwaqj081TMabxacoEpEe/fuxd///vcu2zMzM3H27NmgAnG73Xj77bdhNpu9o/k7s9vtsNvt3ucGgyGo1yKRydVYDruLhysuRbT5h3oS03mYh9sJuOw0d7WfgkpESqWy2yRw/PhxJCcnB3SuQ4cOYfr06bDZbFCr1di2bRvGjBnT7bHFxcVYu3ZtMCGTIcBTLZMm5IkbSDfGpGuRkxCLeI0akNIk+oEKqmp27bXX4oknnoDTKcwdzHEcKisr8fDDD+OXv/xlQOcaOXIkDh48iN27d+O+++7D4sWLcfjw4W6PXb16NVpbW72PqqqqYMInkYgxyA3C/EPKlAKRg+lqbKYOFxYkIlmjpEn0gxBUInr22WdhMpmQnJwMq9WKOXPmYNiwYdBoNHj66acDOpdCocCwYcMwadIkFBcXY/z48diwYUO3xyqVSu8dNs+DRAe7sQEShxGMkyAuOU/scHpHfYkCFlTVTKfT4dNPP8U333yDH374ASaTCRdccAHmzZvX74B4nvdpByIEAKxnjwtfYzOhVMb0cfTgc7h4GG1OcByHBEpEAQs4EfE8j9LSUmzduhXl5eXgOA75+flIS0sTbq8GMCXI6tWrMX/+fOTk5MBoNOLNN9/Ezp078fHHHwcaFhniHPWnAABOXZ64gfTgZJ0JH/98FrmJsbhB0zbezG4UN6gIElAiYozh2muvxQcffIDx48fj/PPPB2MMR44cQVFREbZu3Yp33nnH7/PV1dVh0aJFqK2thU6nw7hx4/Dxxx/jF7/4RaDXQYYytxMKYxW0MTI4UkeIHU23PLfvaZhHcAJKRKWlpfjyyy/x2Wef4eKLL/bZ9/nnn2PBggXYvHkzFi1a5Nf5XnnllUBenkSr5grolRLoC7KBiaPEjqZb3vFmDneHEfiUiPwVUGP1W2+9hUcffbRLEgKASy65BI888gjeeOONkAVHCACgqW3q4YRCIExnA/UkIptPiYjumvkroET0448/4oorruhx//z58/HDDz/0OyhCvBgDmk6BMQYkFoodTY9iFMJHyelmcMpovFmgAkpETU1NSE1N7XF/amoqmpub+x0UIV6WJsDagv1njHj1Zx4NpvC8o6qQSiCVCKU1q2eYh9MiTKJP+hRQInK73ZDJem5WkkqlcLlc/Q6KEK+G42CMoUGWilYH5x1KEW44jmuvnjGlUIX0TKJP+hTwXbOioiIold13W6f+PyTkGo7D4ebRGJMHqYRDnCI8ExEATMjRw80zxCjlgDxWqJrRJPp+CSgRLV68uM9j/L1jRkifbAbAUAO7m6FJlQu1UhbWS1dNyeuwoIRS3Z6ISJ8CSkQlJSUDFQchXTWcAACYlalwuuKgiQl6YeLBp1ADOEd3zvwUQf+zJOo0CMM6WmLzAAOgiZH3frzIbE43THYX5FIJdNSpMSCUiEh4cliAFmG0fX1MLmAAtGFeItpf2Yzdp5swPluHSygRBSS8/2dJ9Ko/CjAeUKcgTp+MTKcZierwntvHc9fM4nADahpvFghKRCQ8nftZ+Jp6HiblJGBSbvivLBynFD5OFhrmEbCgV/EgZMBYW4DWM0JfnJTuZ+sMR7FtXQssdhcNfA0QJSISfuraZujU54BXaODmmbjx+ClOIZSIzI6O482MQsdG0itKRCS8MNZeLUsZg0azA89/fgKv7aoQNy4/xCrbF1p0SNsSkdslTKJPekWJiIQXYy1gbgAkMiB5FIw2JxgDJOHbj9FLIZVALhUCtbg5QN425owarPtEjdUkvNQcEL6mjALkMTDaWgCEfx8iQBhvdkFuPKQcB7lUIgztcNoAuwFQB7a6TbShRETCh9PW3j6UPgEAYLQJg6gjpVf1jMKk9idKLWCqp97VfqCqGQkf534S2lTikgBdFgDAaBOWrAr3zozd8gx2papZnyLwf5cMSYy1V8syLvDOxNhq9SSi8K+aAYDd5YbR5oJcIoFOQZ0a/SVqiai4uBhTpkyBRqNBSkoKFixYgGPHjokZEhFL02mhkVoqB1LP8272JCKdKjIS0YHKFrz2XQX2lDdRiSgAoiaiL774AsuXL8euXbvw6aefwul04rLLLoPZTJ3AIgEfyv49lbuErxkTvHebeJ4hNzEOmXoVtBGSiDx9iSwOV4dE1HV5duJL1KrZRx995PO8tLQUKSkp2LdvHy666CKRoiL+MNld2Lb/DOaMSEFOYmz/TtZaLQxw5SRA1lTvZomEwxVj0/oZ6eDy9CUy291CYzVAJSI/hFVjdWtrKwAgIaH7cUV2ux0Gg8HnQQafm2d474caNJgc+OpkPRhjYIzh0JlWVDVZAj9h5XfC19TzgJjIXkZcreymROS0AW6niFGFv7BJRDzP48EHH8TMmTMxduzYbo8pLi6GTqfzPrKzswc5SgIAB6uaUdtqQ4xciqvOTwfHcThca8D2I+ew41hdYFW21jPCBGgcB+RM99lld7kjZniHh3e8mcMNJlUIbV4AlYr6EDaJaPny5fjpp5+wZcuWHo9ZvXo1WltbvY+qqqpBjJAAwgKCu043AQAuGpEEfawCAFCYrIZKIUWjyYHjdX5+6BgDTu8Uvk8bB8Ql+uz+9lQjnv/8BHafbgxV+AMutq2NyM0z2FyMGqz9FBaJaMWKFXjvvfewY8cOZGVl9XicUqmEVqv1eZDBdbCqBQ4XjxStEmPS29//GLkUE7L1wjGVLf6drPEU0FIlDOfIm9llt8EqDO9QhfGE+Z1JJZw3XrNPgzUlot6ImogYY1ixYgW2bduGzz//HPn5+WKGQ/rgdPP44UwLAGGi+M4T2Z+fqYNUwqG21YY6g633k7mdwIlPhO+zJgExui6HGCLs1r3HxGw9Zg5LEpY+okTkF1Hvmi1fvhxvvvkm/vOf/0Cj0eDs2bMAAJ1OB5VKJWZopBsnzplgdbihU8kxLFndZX+cUoaC5DicOGfC0bNGpGhjej5ZxTeArVVonM6d1WU3YyziOjN6TCvoUMWkROQXURPRxo0bAQBz58712V5SUoKioqLBD4j0anS6BvpYOZxuHpIehsOPStPgxDkTjp8zYvbwpO6X/2mpau83NGweIFN0OcTicMPpZuA4REwfom5RXyK/iJqIGE0YFVE4jkOGvveSam5iHBQyCWIVMlgcbu/0qV5OK3Dkv0JDdep5QPLIbs/jKQ2plTLvUs6RwuHi0Wp1QsIBiYq2REQDX3tFY82IXxhjfi1uKJdKsGRmfvcNzG4X8NNWoUqm0gMjLu/xPJE2tKOjo2cN+OxIHQqS43BdYVuDvq1V3KDCXFjcNSPhzc0zbPq2HJ8fPQeb093n8d0nISfw8zahB7VMAYz9JSDreVUOtVKG0eka5CXF9Sd0UXg6NZrsrvZGeIeFOjX2gkpEpE9lDSY0W5xwus2YOyLF759zuHgAgMLRAhz+D2A8K9yqP+8GQN37ebITYpGd0M+hIyJRt01ZYrK5AFmMkHhdDmEJ7U59pYiAEhHp0881QkPr6HRtj43UnX155AxOnzqBufENyHOVAbxbGMw69peAPmcgwxWdRilUJy0ON1w8gyxGJ0yQZmuhRNQDSkSkVya7C+UNwvixMRmdOpDyPNBSARiqAVOdsHSO0wo4zSioa4K82Qq7SQGkaoCEAmDkfL/GkjHG0GJxQquSR1xDNQDEyIW5q51uBpPdBX2Mvi0RUTtRTygRkV4drTWAZwwZ+hgkxLXdZnfZgep9wJm9QttHN/SxCpwySHBKmo1hEy6DNN7/UpDR7kLpt+WQSzncP3eY36WwcMFxHNRKGZotThhtLug97USUiHpEiYj0iDHmrZadl9H2Yao7Cpz4uD0ByVVCaUeTJkx7IVcB8ljEKTU48m01rA43xiEJPQ/c6arF3H7rPtKSkIc6Ro5mi9O3wZoSUY8oEZEe1bba0GR2QC7lMDxRDvz8DlB3RNipigfyZgEpowFJ17tkHIDchFgcPWtEZaMFWfH+Nzw3WxwAgPi4rh0dI8WYdC1yEmKRrFECVkpEfaFERHqkiZFhan4CJNZGKH94HbA0CpOX5VwI5M4EpL3/+uQkComoosmCGQG8rjcRxUZwIurYnsYoEfWFEhHpkSZGjpm6JqD2PeH2s1INnHe9d4WNvuS03X4/Z7DB6nD7PYq+0SQkooQILhH58PYlMgt9iaSR10lzoFEiIt1jDKjaDZzaITzXZwNjFgjJyE+aGDnGZekQH6eAH52yvRpMwhLNSeqeOzyGO5ebR7PFCYebR6aO+hL1hRIR6YLxbhzc8S/k245CFysHlzkZGHZpt21Bfbl0dGpAx5vtLlgcbnAckKiO3BJRs8WJ13dVIEYuxX1zC4VSEfUl6hEN8SC+eDca9rwNW/leHK8zw5F3MTDisqCSUDA4DphemIjxWXph2eYI5RkjZ3O6hWExMXphh7VFtJjCGZWISDueB468i4ayH8A4Cdh510GZP73vn+uDweZEZaMF2QmxfQ5ijVXIcGFB5JcYhBkIpLA43DBYnYhRxQs7rE3iBhamIvdPDgktxoDjH8JYeQjNVjdOJF+GkWOnhOTU2w+fw6eHz+FUfXRNheFJuq1WJxDbtjKNtVnEiMIXJSIiqNoDVvsDKpptOJ44DxmF53snxu+v3LZ1zyob+15qqLzBDIPNOSTmqvJJRKq2RGShElF3KBERoKkMOL0DTWYHDsVMhkGdj2kF3a8tF4ycBGEqjzPNFrjcfI/HOVw83jlYjVe+KoPZ0fd0I+Gu2xKRrUUYAEx8UCKKdk4bcPR98DyP7+05OKs+D5Pz4kM6T3SSWoE4pRRON0Nta8+T6p8z2MCY0JFS3Xlmxwik7ZiIFGqh/xBj1GDdDUpE0e7kp4DdCElsAobPXIB0vQpT80JXGgKEQaCeUlF5o7nH4862rfyRrhsaCydk6FWYOSwJE3PihduB1GDdI1ET0ZdffolrrrkGGRkZ4DgO77zzjpjhRJ/mcuDsT8KHZPTVKEhLwM1TsiEbgNvmnnaiil7aiWparACANF3kdmTsKCFOgan5Ccj3zDIZS+1EPRE1EZnNZowfPx5//etfxQwjOvE8nEc/wal6E4yJ53uHbfgzL3UwPImowWTvdrpZN89wpllIRIEMkI0ongZrKhF1IWpFfP78+Zg/f76YIUStxpO7UXn8JEy8HF81FeB2Nz+gHQhjFTIsmJiJNG2MsPBgJ2cNNjhcPFQKKZIjeGhHZ61WJxpMdiTFKaGjElGPIqpF0G63w263e58bDLRWVKAYY9hfVgfLtx9C5ubRlDYN8yfmD0ov5vxeJsL33NrPjo+N2DmIuvPl8XqcrDNhzshkXKBv66hpaRQ3qDAUUYmouLgYa9euFTuMiOVw8fjwp1qYT+9BvtsCrT4BE+ddgRjl4I/p6rw80aTceOhj5YhTRNSvZJ8S4xQ4CaDJ5AAykoSNDrPwUETeCiUDJaLumq1evRqtra3eR1VVldghRQyb041/7z+DsjoDsk0/oDA5DmOmXT7oSehUvQlv7anENyd9SwUKmQSj07XISRxa7UOeyd2aLA5hBL7nzpm5XsSowk9E/flRKpVQKodO+8Fg4jhAIZUg03EaE1Ik0GjjgfTxosRytm3mx0m58VAppH4v3hiJEj2JyCzMsYS4JGGYh6keiM8TL7AwE1ElIhI8pUyKBRPScbWuXFjuJntanzMsDoSCpDgka5RwuHjsPFaH2lYrSr4px8GqliExrKMzzzAZq8MNq8Pdvp6buU7EqMKPqInIZDLh4MGDOHjwIACgrKwMBw8eRGVlpZhhDSlmu8v7vbThKFQugzDBffoEUeLhOA6XjEqBhONw9KwRW/ZUodXqRHWzdUiWihQyibeHdaPZDsR5EhFVzToSNRF9//33mDhxIiZOnAgAWLlyJSZOnIg1a9aIGdaQYXW4sfm7Cmw/fA5Olxuo+FbYkTVFaK8QSYZehcvOS4VCJvz65STE4tLR/q8gG2l8qmdxycJGc70w3IMAELmNaO7cuUOyOB4u9pY3weZ0o9Zgg6TpJGBuEBJQ5iSxQ8PodC0Kk9WwudzQKGVDsjTkMS5Lh9HpWmTGqwC5RKgSu11CfyKarRFAhDVWE/8ZbE78UNUCAJhdmAhpxdvCjszJwtLPYUAhk3hLRUNZQXKneb7VaUDrGWGFXEpEAKixesjadaoRLp4hK16FXMk5wHhW+EucFZrJzkg/aDOEr8ZaceMII5SIhqBGkx2Ha4Ve57OGJYLztA1lTAQUQ6ufTqQ402zB9+VNaLU42xORoVrcoMIIVc2GoG9ONYIxYFiKGumsXqgGSKTCLXsiiu9ONeJMsxUqhRS6hLZEZKqndc7aUIloiLE63KhpsYLjgBmFiUD518KO9AmAUiNqbNEsRSu0y50z2AClVhjewXihykwoEQ01KoUURTPycPW4dCS6zgEtlUJpKIdKQ2LK0AmJqLrFJnRz12UKO1ppmBJAiWhIipFLMSxFA5R/I2xIG9e+7DERRWa8MOtkg7FtPiZ9nrCjuVy0mMIJJaIhgjGG8gZze7+s5grhl1wiBXIuFDU2IszH5Fm59kyzFUjIF3a0nhHaiaIcJaIh4uhZI7YdqMY7B6vBeB449ZmwI308oNKLGhsRZOqFUlF1i1UYhR+jFVb0aKEhTZSIhgA3z/DdKWFajUx9LLi6w4DxnNCLOm+WyNERD88UuOda29qJPKPvm8vECypMUCIaAn4404JWqxNqpQwTM2KA0zuEHTkzaPKtMJKXFIuFk7Nw4yRhfnAkFApfG05E/bgzSkQRzuZ0Y/dpYQ7k6YWJkJd/CdhNwooR1Is6rChlUmR1nAo3sVDo7W5tifrb+JSIItzuMmFga5JagTExTUDNAWHHiCtEmW+I+IfnmdCRMXG4sKHusLgBiYwSUQRrMju8A1svylVBcvRdYUfGRCA+V7zASI8YY/j6RANe+uo0ms0OIGWMsOPcT8KI/ChFiSiCuXmG+DgFChMVyD37CeCwAOpkYNilYodGesBxHBrNdlgcbvxU0wokDhN6vDssQP1RscMTDSWiCJasUeK2KVm4gtsj9EeRKYEx19PYpTB3XobQufTnGgMcPIQSLABU7Y7aRmtKRBHI6eaFb1wOSH/eCkXLKUAiA86/kea3iQAFSXHQx8phdbhxsKpFSEQyBWCqA+qOiB2eKCgRRRiDzYnN31Vg3+ETYAdeAxpPCknovOsBfY7Y4RE/SCQcphcKfzC+r2iCiSmA7Lbe76c+B5w2EaMTByWiCGK0OfHO9xVQn/se3L4SuA1nhYnwx98CJA0TOzwSgJGpGqRqY2B38vj08FmwrMlCb2u7ETj+YdRV0cIiEf31r39FXl4eYmJiMG3aNOzZs0fskMJOTWMrPtv+IXJOvYHhpr04Ly0WssR8YMrdgD5b7PBIgDiOw+XnpUIm4XC21Q6DQwKMvhrgJEDdUeDk9qhKRqJ3NPnHP/6BlStX4oUXXsC0adOwfv16XH755Th27BhSUobuyg5+cTlgrD2J00f2wVR9GKm8E7EKKUZkZ0A58hIg7XxhqACJSIlqJa4YmwZ9rAK6WDmALGDUlcCR94Az3wOWRmD4ZULn1CGOYyIvozFt2jRMmTIFf/nLXwAAPM8jOzsbv/71r/HII4/0+rMGgwE6nQ6tra3QarWDEW5oMQa4HYDLBua0wmluhsPYBDVvAAw1gLkeP55phtnuBgDoE1NRMGEOlFkTRF0OiAycveVNsFb9iJGtX0EtA5QKKaSJBeAShwOaNGE6F3ksIAmLykyf/P2Miloicjgc2LdvH1avXu3dJpFIMG/ePHz33Xddjrfb7bDb7d7nBoPBvxdqKhOKut3x5mHWZdupehOazHYwABxj7Ucw4a7VBbnxkLWVSMoaTKgz2juds+28DJiYo4NCKvzyVDSacbbVCgnvBAcejAE8Y+Dbfmxybjzkbcdq45NhlGaiYPQFSM8ZQSWgIczucmNveRPszgz8LLkcuU27EG+tBFfWBJl0H2IVMoxJ1wrVN6kcP501w+riwDgJAN/fC4VcignZ8d7nP9UYYHZ4Okz6HiuTcJiUG4+AnX+TMINACIiaiBoaGuB2u5GamuqzPTU1FUePdu3cVVxcjLVr1wb+Qi67sKZXgHiLES7PmuXdsUm9f5l4mwm8zd7jocwhB2RS4XunFbzTBt6zj5PAxcXAIY+DS6lDa0ohktJyAG0GcpUa5FHyiQpKmRS/vCALJ+tMqG5RoSL2KlRYGpFoOQ2N/RySuVbhDxHjAZcdLqsRLiff7bmYTAKY3e3PTa3g7d333GZSDjAF0aubdf/awRC9jSgQq1evxsqVK73PDQYDsrP9aKjVZwt3loAOJYoOH+4etiWaHYh1uNsX//N+Ff7htDHCNo5DstUJtYsH13YOn9zBcZDFKb1JK8nhgsbJA1IFIIsBJDLI5VKo5FJvScj7o31fHRlCUrUxSNW2rzvndOfD5pwIm5OHhAMQKwecZsDtRPpIE9wuJ8B3TSJSDoBW6X2eVmCDw919K4xEAkAbxFp3IZzZQdRElJSUBKlUinPnzvlsP3fuHNLS0rocr1QqoVQqu2zvkyKufUa8ACQEMLuqVgP4W0hVxwHqvg8jBHKpBHKpBJqOeaJtEYTUABqxk4OoeQ0mUVu8FAoFJk2ahM8++8y7jed5fPbZZ5g+fbqIkRFCBpPoVbOVK1di8eLFmDx5MqZOnYr169fDbDbjrrvuEjs0QsggET0R3Xzzzaivr8eaNWtw9uxZTJgwAR999FGXBmxCyNAlej+i/oj4fkSEDHH+fkYjo1cUIWRIo0RECBEdJSJCiOhEb6zuD0/zlt9DPQghg8rz2eyrKTqiE5HRaAQA/3pXE0JEYzQaodP13EM4ou+a8TyPmpoaaDSa9mEYIeYZRlJVVRUVd+ai7XoBuuaBvGbGGIxGIzIyMiDpZcaAiC4RSSQSZGVlDcprabXaqPklBaLvegG65oHSW0nIgxqrCSGio0RECBEdJaI+KJVKPPbYY8GN+o9A0Xa9AF1zOIjoxmpCyNBAJSJCiOgoERFCREeJiBAiOkpEhBDRUSJCYCvNvvTSS5g9ezbi4+MRHx+PefPmRdzKtMGurLtlyxZwHIcFCxYMbIADINBrbmlpwfLly5Geng6lUokRI0bggw8+GKRoQyPQa16/fj1GjhwJlUqF7OxsPPTQQ7DZbIMTLItyW7ZsYQqFgr366qvs559/Zvfccw/T6/Xs3Llz3R5/2223sb/+9a/swIED7MiRI6yoqIjpdDp25syZQY48OIFer0dZWRnLzMxks2fPZtddd93gBBsigV6z3W5nkydPZldeeSX7+uuvWVlZGdu5cyc7ePDgIEcevECv+Y033mBKpZK98cYbrKysjH388ccsPT2dPfTQQ4MSb9QnoqlTp7Lly5d7n7vdbpaRkcGKi4v9+nmXy8U0Gg3btGnTQIUYUsFcr8vlYjNmzGAvv/wyW7x4ccQlokCveePGjaygoIA5HI7BCjHkAr3m5cuXs0suucRn28qVK9nMmTMHNE6PqK6aeVaanTdvnndbbyvNdsdiscDpdCIhIfzXJw/2ep944gmkpKTg7rvvHowwQyqYa3733Xcxffp0LF++HKmpqRg7dizWrVsHt9vd7fHhJphrnjFjBvbt2+etvp0+fRoffPABrrzyykGJOaIHvfZXoCvNdufhhx9GRkaGz396uArmer/++mu88sorOHjw4CBEGHrBXPPp06fx+eef4/bbb8cHH3yAkydP4v7774fT6cRjjz02GGH3SzDXfNttt6GhoQGzZs0CYwwulwu/+tWv8Oijjw5GyNRY3R/PPPMMtmzZgm3btiEmJoiVMsOc0WjEnXfeiZdeeglJSUlihzNoeJ5HSkoKXnzxRUyaNAk333wz/vd//xcvvPCC2KENmJ07d2LdunX429/+hv3792Pr1q14//338eSTTw7K60d1iSjQlWY7+uMf/4hnnnkG27dvx7hx4wYyzJAJ9HpPnTqF8vJyXHPNNd5tPC+sdy6TyXDs2DEUFhYObND9FMz/cXp6OuRyOaRSqXfb6NGjcfbsWTgcDigUigGNub+Cuebf/e53uPPOO7F06VIAwPnnnw+z2Yxly5bhf//3f3udSygUorpEFOxKs3/4wx/w5JNP4qOPPsLkyZMHI9SQCPR6R40ahUOHDuHgwYPex7XXXouLL74YBw8ejIiZMYP5P545cyZOnjzpTboAcPz4caSnp4d9EgKCu2aLxdIl2XgSMRuM4aiD0iQexrZs2cKUSiUrLS1lhw8fZsuWLWN6vZ6dPXuWMcbYnXfeyR555BHv8c888wxTKBTsX//6F6utrfU+jEajWJcQkECvt7NIvGsW6DVXVlYyjUbDVqxYwY4dO8bee+89lpKSwp566imxLiFggV7zY489xjQaDXvrrbfY6dOn2SeffMIKCwvZTTfdNCjxRn0iYoyx559/nuXk5DCFQsGmTp3Kdu3a5d03Z84ctnjxYu/z3NxcBqDL47HHHhv8wIMUyPV2FomJiLHAr/nbb79l06ZNY0qlkhUUFLCnn36auVyuQY66fwK5ZqfTyR5//HFWWFjIYmJiWHZ2Nrv//vtZc3PzoMRK04AQQkQX1W1EhJDwQImIECI6SkSEENFRIiKEiI4SESFEdJSICCGio0RECBEdJSJCiOgoEZGIUlpaCr1e733++OOPY8KECd7nRUVFETmVbbSjRES6VVRUBI7j8Ktf/arLvuXLl4PjOBQVFfkcH+oEkJeXh/Xr1/tsu/nmm3H8+PEef2bDhg0oLS31Pp87dy4efPDBkMZFQo8SEelRdnY2tmzZAqvV6t1ms9nw5ptvIicnR5SYVCoVUlJSetyv0+l8SkwkMlAiIj264IILkJ2dja1bt3q3bd26FTk5OZg4cWK/zt1dSWXBggXeUtbcuXNRUVGBhx56CBzHgeM4AF2rZp11LJkVFRXhiy++wIYNG7znKCsrw7Bhw/DHP/7R5+cOHjwIjuNw8uTJfl0XCQ4lItKrJUuWoKSkxPv81VdfxV133TXgr7t161ZkZWXhiSeeQG1tLWprawM+x4YNGzB9+nTcc8893nPk5OR0uSYAKCkpwUUXXYRhw4aF6hJIACgRkV7dcccd+Prrr1FRUYGKigp88803uOOOOwb8dRMSEiCVSqHRaJCWltbnjJnd0el0UCgUiI2N9Z5DKpWiqKgIx44d804U73Q68eabb2LJkiWhvgzip6ieKpb0LTk5GVdddRVKS0vBGMNVV10V8fNXZ2Rk4KqrrsKrr76KqVOn4r///S/sdjsWLlwodmhRi0pEpE9LlixBaWkpNm3aFLJSg0Qi6TIFqdPpDMm5/bF06VJvQ3xJSQluvvlmxMbGDtrrE1+UiEifrrjiCjgcDjidTlx++eUhOWdycrJPu4/b7cZPP/3kc4xCoej3WmI9nePKK69EXFwcNm7ciI8++oiqZSKjqhnpk1QqxZEjR7zf96S1tbXL+meJiYndTrJ/ySWXYOXKlXj//fdRWFiI5557Di0tLT7H5OXl4csvv8Qtt9wCpVIZVJUwLy8Pu3fvRnl5OdRqNRISEiCRSLxtRatXr8bw4cN7XSyBDDwqERG/aLVaaLXaXo/ZuXMnJk6c6PNYu3Ztt8cuWbIEixcvxqJFizBnzhwUFBTg4osv9jnmiSeeQHl5OQoLC5GcnBxU3KtWrYJUKsWYMWOQnJyMyspK7767774bDodjUO4Ckt7RnNUkan311Ve49NJLUVVV1WVVVDK4KBGRqGO321FfX4/FixcjLS0Nb7zxhtghRT2qmpGo89ZbbyE3NxctLS34wx/+IHY4BFQiIoSEASoREUJER4mIECI6SkSEENFRIiKEiI4SESFEdJSICCGio0RECBEdJSJCiOj+P33WYsHAgXbkAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:26:58.586175Z", + "iopub.status.busy": "2024-02-29T00:26:58.585801Z", + "iopub.status.idle": "2024-02-29T00:26:59.001223Z", + "shell.execute_reply": "2024-02-29T00:26:59.000300Z" + }, + "papermill": { + "duration": 0.441211, + "end_time": "2024-02-29T00:26:59.003522", + "exception": false, + "start_time": "2024-02-29T00:26:58.562311", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABE00lEQVR4nO2dd3xT99X/P1fbsiV5L7zNCBsCgbDCKBkkoaFpkzQTB5I8eQLtE/jRBNqnkA1NmxTaprShAZJm0DYhPGkGkMEIEAgzhbCNF3gvDWvrfn9/XEu2sGVLsuwrWef9el1k3Xt1dS6WPj7f8z3fczjGGANBEISISMQ2gCAIgoSIIAjRISEiCEJ0SIgIghAdEiKCIESHhIggCNEhISIIQnRIiAiCEB2Z2Ab0BJ7nUVlZCY1GA47jxDaHIIirYIzBaDQiMzMTEolvvyeihaiyshLZ2dlim0EQRDdUVFQgKyvL5/GIFiKNRgNAuEmtViuyNQRBXI3BYEB2drbnu+qLiBYi93BMq9WSEBFEGNNd6ISC1QRBiI6oQpSXlweO4zpsixYtEtMsgiD6GFGHZocPH4bL5fI8P3XqFG688UbcddddIlpFEERfI6oQpaSkeD1fs2YNCgsLMX36dJEsIghCDMImRmS32/H2229jwYIFlBNEEFFG2Myabdu2Dc3NzSgqKvJ5js1mg81m8zw3GAx9YBlBEL1N2HhEb7zxBubMmYPMzEyf56xevRo6nc6zUTIjEWpcPMPuc7U4eVkvtilRRVgIUVlZGb744gs88sgjXZ63YsUK6PV6z1ZRUdFHFhLRwtcX6nC8vBldrEYgeoGwGJpt2rQJqampuO2227o8T6lUQqlU9pFVRLRhsjnxn1ZPSKuSi2xNdCG67vM8j02bNmH+/PmQycJCF4ko5eRlPVw8w4D4GGQnqgEAerMDThcvsmX9H9GF6IsvvkB5eTkWLFggtilElHOh1ggAGDFABwD48PhlbNxfgvJGs5hmRQWiuyA33XQTqLUaITaNLXY0mOyQSjgUpMQCaBuelTeaUZASJ6Z5/R7RPSKCCAfcXk9mfAxUcqnnZwCo1ltFsytaICEiCABmuxNSCYfshBjPvgydCgBQZ7RRnKiXEX1oRhDhwOTCZEzIS4STbwsT6GLkiFFIYbG7UG+yI71VmIjQQx4RQbQik0o8wzJAqKGTEieki9SbbL5eRoQAEiIi6ulqsiQxTgFACGYTvQcNzYioZ9e5WlxptuL6/EQMSvMuaZqXFAsOQE5rXhHRO5AQEVFPld6KemPnQ6/85FjkJ8f2sUXRBw3NiKjG6eLRYBKGXalaCkaLBQkREdXUm+xw8QwxCim0qs4HCGa7E1V6C6wOV6fHiZ5DQkRENTUGIVkxTav0WZDvw+NXsOXbClxptvSlaVEFCRER1XiESON7WKaLEZZ66C2OPrEpGiEhIqKa2tYgdVfxIRKi3odmzYiohTGG5DglnC4eqVrfda7cQmQgIeo1SIiIqIXjONwyIr3b88gj6n1oaEYQ3dDeI6KSNb0DCRERtVjsLr+EJU4pDBwcLgabk1bh9wY0NCOilm0nrqCxxY65ozKRk+R7CYdMKsHE/ESoFFJQy73egYSIiEp4nqHeaIOTZ9D4SGRsz+SByX1gVfRCQzMiKmk02+HkGRQyCeLV1LFDbMgjIqKSWoOQP5QS5zujuj1WhwtNZjsUUgmS4qilVagR3SO6cuUKHnjgASQlJSEmJgYjR47EkSNHxDaL6OfUtRY6S+kif6g931U0Y8u3FTha1tSbZkUtonpETU1NmDJlCmbOnInPPvsMKSkpuHDhAhISEsQ0i4gCaluXdqRquhAiQyXQeAnQZCBOJcSITDZnX5gXdYgqRL/5zW+QnZ2NTZs2efbl5+eLaBERDTDG2jwiX0JU9R1w7jOgdXo/OWkCgGwSol5C1KHZRx99hPHjx+Ouu+5Camoqxo4diw0bNvg832azwWAweG0EESgunmFMVjwKUmKRFNuJEBlrgPM7BBHSCJnXmuqDUNvrYbSSEPUGogrRpUuXsH79egwaNAg7duzAf//3f+PnP/853nzzzU7PX716NXQ6nWfLzs7uY4uJ/oBMKsHkgcm4Y8wASCVXBaoZA4q/BHgXkDwIGFcEpA6FUsohS38MdicPm5PqEoUajomYs65QKDB+/HgcOHDAs+/nP/85Dh8+jG+++abD+TabDTZbW0lPg8GA7Oxs6PV6aLXaPrGZ6Oc0lQIn3gMkUmDifwEqHWCqAw7/DYdKm3A44z7cP20YEmIVYlsaERgMBuh0um6/o6J6RBkZGRg2bJjXvqFDh6K8vLzT85VKJbRarddGEIFypdmCFl+xnitHhceM0YIIAUBcCqDNgEIKJFjKYKZKjSFH1GD1lClTcO7cOa9958+fR25urkgWEf0dxhi2Hb8Cu5PHA9fnegerrQag/qLwc+a13i9MGoRMXQnitU1+ZWITgSHq/+iSJUswefJkvPTSS7j77rvx7bff4vXXX8frr78upllEP6axxQ67k4dcyiHp6uFV1XcA44H4HMELak9SIdK0KkBaB8hpwVmoEXVodt111+HDDz/Ee++9hxEjRuD555/H2rVrcf/994tpFtGPqdK35g9pVZC0D1QzBtSeFn7OGN3xhXFpgEINuJyAqboPLI0uRPcxb7/9dtx+++1im0FECe4a1RlX97E31QDmRkAiE2bLrobjYI/NgM14Fo6qEiTqsvrA2uhB9CUeBNGXuD2i9KtrVNeeER6TCgFZ50mOFc5EnLpiQPHFc50eJ4KHhIiIGmxOl6eZYnp7j4ixNiFKHdbJKwVkCYIXJDNV9ZqN0QoJERE1VDVbwTMGXYwcGlW70h+GSsCqB6RywSPygSIhAwAHZjMC9pbeNziKED1GRBB9RZpWhVtGpIO/Ooe37qzwmDxIECMfqGPUsMo0kLgMYKZacIm0LjJUkBARUUOMQoqhGVclwTIG1F8Qfk4e0uXrYxVSmOWJUDkNsBtqoCQhChk0NCOim5Z6wNIkzJYlFnR5qkwqgSMmCQBgb67pC+uiBhIiIiqo0ltwtKwRdUab94GGVm8oIReQ+bF+LDYVAOAwUC5RKCEhIqKCM1UG7D1fj/9cbvY+UH9eeOwsd6gTBhcUIi9JDbWzGeCptVCoICEi+j2MMVyqE2a58pNj2w7YjIChdSo+aaBf1xpWkIOMhDioOB6wUT2sUEFCRPR76k12GK1OyKUcshPb9S9zB6m1mYBS49/FJBJAFS/8bKH61aGChIjo91yqMwEAshPVkEvbfeQ9s2WD/b6Wxe5CE4tFi90JWBpDaWZUQ0JE9HvO1wpCVJAc17bTaQOay4Sf/YwPAcDZagN2lTtxpclCHlEIISEi+jV1RhvqjTZIJRwGpbUTosZLQjlYdSKgTvL7ejEKKawyHRwuHjCTEIUKEiKiX9NktkMhkyA/ORYqubTtgHtYljQQgTS0V8tlsMq1cPKMPKIQQpnVRL9mcJoG+cmxsLYv78q7gIbWSowBxIcAQKWQwCKLFzwia7MwhS+hv+c9hf4HiX6PXCrxXuTaXC7EiBRqQDsgoGvFyKWwS2Ph4CVgvJOm8EMECRHRbzFaHei0SY3bG0oaGLA3EyOXAhwHqzQOLp4Jq/aJHkNCRPRLnC4efz9Yhr8fLIPR6mg7wFi7bOrAhmWAsN5MIZPAJouDw8XIIwoRFCMi+iWlDS2wOXgopDzilO0+5qZaoVuHVAYk5AV17esLEhEvHwCZ9QJ5RCFCVI/omWeeAcdxXts111wjpklEP+FstREAMCRdA679rJjbG0rI77L2UFeMy01EYdYAITmShCgkiO4RDR8+HF988YXnuUwmuklEhGN1uFDSurZsSPpVSzcCXOTqE1VrXSMSopAg+rdeJpMhPT1dbDOIfkRxnQlOniEpToGUuHaF8C3NwtCM4/xe5NoZJpsTLXYltA4XYqwUIwoFogerL1y4gMzMTBQUFOD+++/32W4aAGw2GwwGg9dGEFdzzj0sS7tqWOaeLdNlAYrYTl7pH8fLm/D+9yahNZHNIATAiR4hqhBNnDgRmzdvxvbt27F+/XqUlJRg2rRpMBqNnZ6/evVq6HQ6z5adnd3HFhPhjtXhQkWjBUBnwzJ3NnXPhmVCLpEaDh5CcqSt888r4T+iCtGcOXNw1113YdSoUbj55pvx6aeform5Gf/85z87PX/FihXQ6/WeraKioo8tJsIdmYTD3NEZmFiQiHh1u4qLTpuQyAj0OD6kkksBTgKLpLWkCMWJeozoMaL2xMfHY/Dgwbh48WKnx5VKJZTKzpvfEQQg5PkUpMShICXO+0BjidDXXp0obD1ArRDWrLVwcQBaKJcoBIgeI2qPyWRCcXExMjIyxDaF6G80FguPXfQt85cYjxC1ekQ0NOsxogrRsmXLsGfPHpSWluLAgQP40Y9+BKlUinvvvVdMs4gIpc5ow74L9ajSW7wPMNYWqE4MgRC1ruJvYa3euc3U42tGO6IOzS5fvox7770XDQ0NSElJwdSpU3Hw4EGkpKSIaRYRoZQ1tOBwaSMazXb8cHRM2wFjFWA3C1064nN6/D5uj8jMxcLFGKQ0NOsxogrRli1bxHx7op9xpVnwhAbEx3gfaGgdliXkARIpeopCKsHEgkQkmLKAhrM0NAsBYRWsJohgYYx1IUTtVtuHAI7jMLkwGdDbgEYOsNPQrKeEVbCaIIKl3mQXFrnKJEjVtJtZtZkAY2szxBDEh7xwd/6wmSipsYeQEBH9Arc3lKFTQSJpl03tni3TpAPKuE5eGRwGqwOVFilsLl5IC7C3hOza0QgJEdEvqNZbAQAZOh/xoRANy9zsu1CPfxy5gnpb6wp+ihP1CBIiol/Q2GIHAKRp2w3LeB5oKhV+DkH+UHvcU/hWSavwkRD1CApWE/2Cn16XjSaz3bs2tbFSWNohVwFxoa3w4O4IIizzMAF2EqKeQEJE9AskEg5JcVct/3F7Q/G5Ie+00ZZLRNnVoYCGZkT/pbFEeEzMD/ml27Kr3UMzmsLvCeQRERHPN8UN0FvsGJUVj0x3DpHTBhgqhZ+DrE3dFR4hAsWIQgF5RETEU1xnwpkqI8z2dk0Um8uFafWYBGELMSqF8NUxQiXsICHqEeQRERENzzM0tc6YJce1qz/Ui8MyANCq5JhYkAgtLwMug4LVPYSEiIhoDFYHnDyDTMJB237GzB2o7oVhGSDMmk0uTAacGkGInHZhOCijelnBQEMzIqJpaPWGEmIVbRnVVj1gbhCK5Mfn9q4BMmVbWyLKrg4aEiIionEnMibFthuWub0hTYaQQ9RLNJvtuNJsgUPWOoVPi1+DhoZmRETjFqJELyEqEx57aVjm5pOTVag12PCAUoEUgDyiHhAVQuRyueBwOLo/kYg4eKcdsVIeOgVgtVqFVfD6WkCiBmKzAKu1R9dXKBSQ+EiGbFvmoQZcICHqAUEJ0aVLl1BQUBBqW0IOYwzV1dVobm4W2xSil8hTALnpDLyhBiVGTlhfphoBqAA02Npmz4JEIpEgPz8fCoWiwzGPEHE0hd9TghKigQMHYvr06Vi4cCF+8pOfQKXqvXF4T3CLUGpqKtRqtXezPaJ/YrcIU+kSOaDuWf4Qz/OorKxEVVUVcnJyOnx+VAq3ELUmNZJHFDRBCdGxY8ewadMmLF26FIsXL8Y999yDhQsXYsKECaG2L2hcLpdHhJKSksQ2h+grmBWAXKg9pOz5H8iUlBRUVlbC6XRCLpd7HXN7RGbW+j4kREET1KzZmDFjsG7dOlRWVmLjxo2oqqrC1KlTMWLECLz66quoq6sL+Jpr1qwBx3F48skngzGpA+6YkFqtDsn1iPDDYneiscWGFptT2MEY4BKC15B2HEoFg3tI5nK5OhxrW+bRmjtEs2ZB06Ppe5lMhjvvvBP/+te/8Jvf/AYXL17EsmXLkJ2djYceeghVVVV+Xefw4cP461//ilGjRvXEnE6h4Vj/xckzOFwMvLtMK3MJLaA5ri23p4d09flxr8A38TQ06yk9EqIjR47giSeeQEZGBl599VUsW7YMxcXF+Pzzz1FZWYk77rij22uYTCbcf//92LBhAxISQr8miOi/uHhBgKTuREZnqzckkQNc76fIJccpMbEgEUPyWmsdOcxCsJwImKB+W6+++ipGjhyJyZMno7KyEm+99RbKysrwwgsvID8/H9OmTcPmzZtx7Nixbq+1aNEi3HbbbZg9e3a359psNhgMBq+N8I/S0lJwHIcTJ070yXX279+PkSNHQi6XY968eT16T184rxYi97BMFpphWXckxiowuTAZQwakCV4YY4CDvKJgCCpYvX79eixYsABFRUU+20OnpqbijTfe6PI6W7ZswbFjx3D48GG/3nf16tV49tlnA7aX6Mju3bsxc+ZMNDU1IT4+PuTXX7p0KcaMGYPPPvsMcXGhK1rvhjHW5hFxVwlRiOJDfiORAHK1MDSzt7R19yD8JiiP6PPPP8fTTz/dQYQYYygvLwcgBPnmz5/v8xoVFRX4n//5H7zzzjt+T/+vWLECer3es1VUVARjfsRht9vFNiFgiouLMWvWLGRlZQUtdF3dN9+ufY9UwgG8szU+hKDiQ4wxOJ3OgF/X2CIs83DJY4UdFCcKiqCEqLCwEPX19R32NzY2Ij/fv7ILR48eRW1tLa699lrIZDLIZDLs2bMHf/jDHyCTyTqdpVAqldBqtV5bf2TGjBlYvHgxnnzySSQnJ+Pmm2/GqVOnMGfOHMTFxSEtLQ0PPvig1+9g+/btmDp1KuLj45GUlITbb78dxcXFnV6/tLQUM2fOBAAkJCSA4zgUFRUFdJ2zZ89i8uTJUKlUGDFiBPbs2eO5NsdxaGhowIIFC8BxHDZv3gwA2LNnDyZMmAClUomMjAwsX77c68vf2X3v3r0bHMdhx44dGDt2LGJiYjBr1ixUVdfgy893YNp1Y6DT6XDffffBbDZ74kM8z2P16tXIz89HTEwMRo8ejffff9/zXu7rfvbZZxg3bhyUSiX27dsX8O/qvW/L8c/DFe1yiWjmLChYEHAcx2pqajrsLy0tZWq12q9rGAwGdvLkSa9t/Pjx7IEHHmAnT5706xp6vZ4BYHq9vsMxi8XCTp8+zSwWS4djNofL5+Zwuvw+1+7HucEwffp0FhcXx37xi1+ws2fPsoMHD7KUlBS2YsUKdubMGXbs2DF24403spkzZ3pe8/7777MPPviAXbhwgR0/fpzNnTuXjRw5krlcgg0lJSUMADt+/DhzOp3sgw8+YADYuXPnWFVVFWtubg7oOllZWez9999np0+fZo888gjTaDSsvr6eOZ1OVlVVxbRaLVu7di2rqqpiZrOZXb58manVavbEE0+wM2fOsA8//JAlJyezVatW+bzvs2fPsl27djEA7Prrr2f79u1jx44dYwMHDmTTbriBzZg1m+3ef4jt3buXJSUlsjXP/JIxi/BZeOGFF9g111zDtm/fzoqLi9mmTZuYUqlku3fvZowxz3VHjRrFdu7cyS5evMgaGho6/C66+hwxxtgbX19ir+48xxqObGXsq5cYK9kX1O+8v9LVd7Q9AcWIli5dCkCY0ly5cqVXjo7L5cKhQ4cwZswYv66l0WgwYsQIr32xsbFISkrqsD/UvLbros9j+cmxmDd2gOf563uL4XB13sUzKyEGd43P9jzfuL8EFru3J7fkxsFB2Tho0CC8/PLLAIAXXngBY8eOxUsvvdT2Xhs3Ijs7G+fPn8fgwYPx4x//2Ov1GzduREpKCk6fPt3h/1MqlSIxMRGAEMtrP3Ty9zqLFy/2nLt+/Xps374db7zxBp566imkp6eD4zjodDqkpwszSn/+85+RnZ2NP/3pT+A4Dtdccw0qKyvx9NNPY+XKlZ71XO3vG4AnBeSFF17AlClTAAALFy7EihUrUFxcjPz8fHAch5/ccTt2fX0AT/9KAZvNhpdeeglffPEFJk2aBAAoKCjAvn378Ne//hXTp0/3XP+5557DjTfe6NfvpDNiFFLoLY62ZR40NAuKgITo+PHjAITx9MmTJ73W3ygUCowePRrLli0LrYVRyrhx4zw/f/fdd9i1a1enQd/i4mIMHjwYFy5cwMqVK3Ho0CHU19eDb51GLi8vD0jY/b2O+wsOCPlk48ePx5kzZ3xe98yZM5g0aZJXXs6UKVNgMplw+fJl5OTkdLjv9rTPMUtLS4NarW5b78g7kZaShG+PHgOkClw8dwZms7mDwNjtdowdO9Zr3/jx47v67+gWd1KjhYZmPSIgIdq1axcA4OGHH8a6detCHqPZvXt3SK/ni0UzfXf9lFyVv/bYDb4b812d67ZgSujKksbGxnp+NplMmDt3Ln7zm990OM89YTB37lzk5uZiw4YNyMzMBM/zGDFiRMCB7lBdJ1ja33d72i+v4DjOe7mFyw6O48DzDOAkMJkEMfjkk08wYMAAr+sold4VFH29n7+oOggReUTBENT0/aZNm0JtR5+ikPkfo++tcwPh2muvxQcffIC8vDzIZB1/ZQ0NDTh37hw2bNiAadOmAUC3gdfOli4Ecp2DBw/ihhtuAAA4nU4cPXoUixcv9vl+Q4cOxQcffADGmMcr2r9/PzQaDbKysrq0tTMYhBkrXYwMUnciY2sS47Bhw6BUKlFeXu41DOsNPP3NQEOznuC3EN15553YvHkztFot7rzzzi7P3bp1a48NI9pYtGgRNmzYgHvvvRdPPfUUEhMTcfHiRWzZsgV/+9vfkJCQgKSkJLz++uvIyMhAeXk5li9f3uU1c3NzwXEcPv74Y9x6662IiYkJ6DqvvfYaBg0ahKFDh+L3v/89mpqasGDBAp/v98QTT2Dt2rX42c9+hsWLF+PcuXNYtWoVli5d6rPejy/41vwhh4sXRM2dPwRB4DQaDZYtW4YlS5aA53lMnToVer0e+/fvh1ar7TKtJFDcQzMTc683MwqJjbS0KCD8/gTodDrPXzKdTtflRoSWzMxM7N+/Hy6XCzfddBNGjhyJJ598EvHx8ZBIJJBIJNiyZQuOHj2KESNGYMmSJfjtb3/b5TUHDBiAZ599FsuXL0daWhoWL14c0HXWrFmDNWvWYPTo0di3bx8++ugjJCcnd/l+n376Kb799luMHj0ajz/+OBYuXIj//d//Dfj/g0EQIo4DJIwX8ofcO1p5/vnn8etf/xqrV6/G0KFDccstt+CTTz7xO73EXwYkxOD6giTkZaQKO1zOdsJI+AvHGOt8SigCMBgM0Ol00Ov1HeJVVqsVJSUlyM/PD9t6SURwWB0u6C0OyCQckpQ8YGkWkhhjfQth0O8VyOdo7+8AlwOY+F+AOjHktkQiXX1H2xNUUMNisQjJY62UlZVh7dq12LlzZzCXI4iAcGdVSyWceMs6OsO9tINmzgImKCG644478NZbbwEAmpubMWHCBLzyyiu44447sH79+pAaSBBX47Xq3imuEPE8Q4PJhspmC6BonYGzkRAFSlBCdOzYMc+syvvvv4/09HSUlZXhrbfewh/+8IeQGkgQV+MRIjBhjRkgmhBZnS689U0Z/nG4AjytNwuaoITIbDZDoxHc0J07d+LOO++ERCLB9ddfj7KyspAaSBBXI+E4SDgOUtbamUUqE1bAi4BKJvXEyO1SSmoMlqB+ewMHDsS2bdtQUVGBHTt24KabbgIA1NbW9tuFqET4oI2RI0WjhJIT1xsCAImEg1ImTOHbKKkxaIISopUrV2LZsmXIy8vDxIkTPen+O3fu7JBCTxC9RpgEqmPkwtfIKiEhCpagMqt/8pOfYOrUqaiqqsLo0aM9+3/wgx/gRz/6UciMIwif8Lzo8SE3MQopmsztF77S0CxQgu70mp6e7llZ7Sac2gkR/RO70wW9xQkV54CGAZDIAIlUVJtUnrZCFCMKlqCEqKWlBWvWrMGXX36J2tpazwptN5cuXQqJcQRxNS5e6NrB8XZhRUcf1afuira2Qq0ekcMieGwiBdAjkaCE6JFHHsGePXvw4IMPIiMjg1r2EH2Gq/VvnpQ5WsvCii9EBSlx0KjkyEhUAZXtiuhT7Wq/CUqIPvvsM3zyySeeQlVEaJkxYwbGjBmDtWvXim0KgPCyx8UYwHhImROAJCyEaGBqHAamttaKUsQKCY1URD8ggvIdExISPBX+iPAkEgvu+4OLZ5Ayp+CFS6Six4c6oKCkxmAISoief/55rFy50mu9GREaioqKsGfPHqxbtw4cx4HjOBQXF2PhwoWeQvBDhgzBunXrOrxu3rx5ePHFF5GZmYkhQ4YAAA4cOIAxY8ZApVJh/Pjx2LZtW4e+ZF0V5u/MntLS0r767+gAzxikvEMoYBcG3hAAOF08Gkw21BqsgKLVM6KAdUAENTR75ZVXUFxcjLS0NOTl5XlXywP8aqwoCowJq6P7Gqnc7/o069atw/nz5zFixAg899xzAAQPNCsrC//617+QlJSEAwcO4LHHHkNGRgbuvvtuz2u//PJLaLVafP755wCElc9z587FrbfeinfffRdlZWV48sknvd6vubkZs2bNwiOPPILf//73sFgsePrpp3H33Xfjq6++6tSelJSUEPynBA5r7WWmYHbhvzMMAtUAUGu04R+HK6CLkWNBMq03C4aghKi3Onf2Oi4H8PUrff++0/6f318anU4HhUIBtVrtlR7RvrFkfn4+vvnmG/zzn//0EqLY2Fj87W9/81Rf/Mtf/gKO47BhwwaoVCoMGzYMV65cwaOPPup5zZ/+9KduC/N3Zo8YMAByCSBlLnCQAlJlt6/pCzx1qx0uGpoFSVBCtGrVqpC8+fr167F+/XqPqz98+HCsXLkSc+bMCcn1+xOvvfYaNm7ciPLyclgsFtjt9g4dU0aOHOnV0ODcuXMYNWqUVx2dq3O9/CnMHy5IOA6JSgC8NCzyh9y4y8XanTycMrXwpaKhWUAEndDY3NyM999/H8XFxfjFL36BxMREHDt2DGlpaR0KlvsiKysLa9aswaBBg8AYw5tvvok77rgDx48fx/Dhw4M1zTdSueCd9DVBdB5tz5YtW7Bs2TK88sormDRpEjQaDX7729/i0KFDXucFUwjen8L8YYXLJjyGybAMAJQyCSQcB54x2KRuISKPKBCCEqL//Oc/mD17NnQ6HUpLS/Hoo48iMTERW7duRXl5uadWUXfMnTvX6/mLL76I9evX4+DBg70jRBwXVh9gXygUCq+i9vv378fkyZPxxBNPePb56uLaniFDhuDtt9+GzWbzdK84fPiw1zndFebvzB6xYIyB86wvC49hGSB0FVHJJTDbXbAiBrEACVGABDVrtnTpUhQVFeHChQtebv+tt96KvXv3BmWIy+XCli1b0NLS4tUzKxrJy8vDoUOHUFpaivr6egwaNAhHjhzBjh07cP78efz617/uICidcd9994HneTz22GM4c+YMduzYgd/97ncA4ElCXbRoERobG3Hvvffi8OHDKC4uxo4dO/Dwww97xOdqe67OpO8rTFY7Wiw2OHgWNjNmbtzDM1pvFhxBCdHhw4fxX//1Xx32DxgwANXV1QFd6+TJk4iLi4NSqcTjjz+ODz/8EMOGDev0XJvNBoPB4LX1R5YtWwapVIphw4YhJSUFN998M+68807cc889mDhxIhoaGry8I19otVr8+9//xokTJzBmzBj86le/wsqVKwHA8weku8L8ndlTXl7eezffBcxpA2MMTCJe/SFfqK5e5uFyAE6biBZFGMH0s05JSWHHjh1jjDEWFxfHiouLGWOM7dy5k2VlZQV0LZvNxi5cuMCOHDnCli9fzpKTk9n333/f6bmrVq1iECZPvLbO+mp317M8Wnn77beZXC5nZrNZbFMCxtBUx4y15cxmau6z9/T3c3TycjP7prie1RqsjO35LWNfvcRYS0MfWRm+6PV6n9/R9gT1Z+WHP/whnnvuOTgcQk4Ox3EoLy/H008/3aF3encoFAoMHDgQ48aNw+rVqzF69OgOyXpuVqxYAb1e79kqKiqCMT+qeOutt7Bv3z6UlJRg27ZtnhyhmJgYsU0LjHbxIYk8vIZlADBigA7XFyQhRaOkIvpBEJQQvfLKKzCZTEhJSYHFYsH06dMxcOBAaDQavPjiiz0yiOd52Gydu7RKpRJardZrI7qmuroaDzzwAIYOHYolS5bgrrvuwuuvvy62WQHD806hfxnHQSILn0B1p1AuUcAENWum0+nw+eefY//+/fjuu+9gMplw7bXXYvbs2QFdZ8WKFZgzZw5ycnJgNBrx7rvvYvfu3dixY0cwZhGd8NRTT+Gpp54S24wewzuEP068RB5wZ9i+wO7kYbQ6wHEcEkmIAiZgIeJ5Hps3b8bWrVtRWloKjuOQn5+P9PR0r77m/lBbW4uHHnoIVVVV0Ol0GDVqFHbs2IEbb7wxULOIfg5zWoXHMJq2b8/FWhN2fF+N3CQ17tS0JofajOIaFUEEJESMMfzwhz/Ep59+itGjR2PkyJFgjOHMmTMoKirC1q1bsW3bNr+v98YbbwRqb8CwyG1kS7hhPDiXA1IJB9bHwzJ/Pz/u6Xta5hEcAQnR5s2bsXfvXnz55ZeYOXOm17GvvvoK8+bNw1tvvYWHHnoopEYGg3shrtlsjrzALOGN0w4ZB8iUCkDdt+3D3eVUpNKul5N41pvZXe1W4JMQ+UtAQvTee+/hl7/8ZQcRAoBZs2Zh+fLleOedd8JCiKRSKeLj41FbWwsAUKvVVEkyUrEZAYcDkMsAHxMZvQHP86irq4NarfaZde7GLURWL4+IZs38JSAh+s9//oOXX37Z5/E5c+aEVadX92pxtxgREYrNCDAekKsBaX2fvrVEIkFOTk63f8RUCiGA7nAxOGRqyAHyiAIgICFqbGxEWlqaz+NpaWloamrqsVGhguM4ZGRkIDU11ZPzREQY5ibg5E58X9WCs5k/wqzhA5AQ23d5RAqFwq9ZOoVUAqmEg4tnsHAqQYgcZiqi7ycBCZHL5erSRZVKpXA6nT02KtRIpdJux/hEmFJTCuZqQT2vRq1NCk2cGipl0EUjeg2O4xAjl8Jkc8LKlNByVEQ/EAKeNSsqKvKs5L4aX4mIBBE09edhd/FoUOVBKuEQqwjfPyhjcuLh4hlUSrkwjLS3UBF9PwlIiObPn9/tOeEQqCb6CVYDYKiEzcXQGJOLOKUsrCccrstr11BCGdcmRES3BCREmzZt6i07CKIj9RcAAC3KNDicsdCowm9I5hNFHIAamjnzkwj6zRJRR/15AECzOg8wABpVzypd9jZWhwsmmxNyqQQ6SmoMCBIiIjyxm4Fmoe5RnSoXMADaMPeIjpU34dClRozO1mEWCVFAhPdvlohe6s4KuUNxqYiNT8EARwuS4sJznZkbd1Kj2e4C4mi9WSCQEBHhSc33wmPacIzLScS43PDvLBzbmlZgpmUeAUOZVkT4YWkG9JeFZgepnZcNDkfUrakFZpuTFr4GCAkREX7UnhYe43PAKzRw8ZFRQSFWIXhELfb2682MQmIj0SUkRER4wVjbsCx1GBpa7PjjVxfw94Nl4trlB2plW6NFu7RViFxOKqLvByRERHhhrAJa6oVOrinXwGh1gDFAEr55jB4UUgnkUsFQs4sD5K0lSyhg3S0UrCbCi8rjwmPqNYBcBaO1GUD45xABwnqza3MTIOU4yKUSYWmHwwrYDEBcitjmhTUkRET44LC2xYcyxgAAjFZhEXWkZFVPLkxue6LUAqY6yq72AxqaEeFDzSkhphKbDOiyAABGq1C+JdyTGTvFvdiVhmbdEoG/XaJfwljbsCzzWmHqHoDe4hai8B+aAYDN6YLR6oRcIoFOQUmN/iKqR7R69Wpcd9110Gg0SE1Nxbx583Du3DkxTSLEovGSEKSWyoG04Z7dbiHSxUSGEB0vb8bfvynDt6WN5BEFgKhCtGfPHixatAgHDx7E559/DofDgZtuugktLZQEFgnwoczvKT8oPGaO8cw28TxDblIsBsTHQBshQuTOJTLbne2EyCCiRZGBqEOz7du3ez3fvHkzUlNTcfToUdxwww0iWUX4g8nmxIfHLmP64FTkJKl7djH9FWGBKycBsiZ4dkskHG4Zkd5DS/sWdy5Ri80lBKsB8oj8IKyC1Xq9HgCQmNj5uiKbzQaDweC1EX2Pi2f4+LtK1Jvs+PpiHRhjYIzh5GU9KhrNgV+w/BvhMW04oIrsNuJxyk48IocVcFHN9K4IGyHieR5PPvkkpkyZghEjRnR6zurVq6HT6TxbdnZ2H1tJAMCJiiZU6a1QyaW4bWQGOI7D6SoDvjhTg13nagMbsukvCwXQOA7ImeR1yOZ0RczyDjee9WZ2F5hUIcS8APKKuiFshGjRokU4deoUtmzZ4vOcFStWQK/Xe7aKioo+tJAAhAaCBy81AgBuGJyMeLXQUaMwJQ4xCikaTHacr/XzS8cYcGm38HP6KCA2yevwgeIG/PGrCzh0qSFU5vc66tYYkYtnsDoZBaz9JCyEaPHixfj444+xa9cuZGVl+TxPqVRCq9V6bUTfcqKiGXYnj1StEsMy2v7/VXIpxmTHC+eUN/t3sYZioLlCWM6RN6XDYYNFWN4RE8YF869GKuE89rZ4BaxJiLpCVCFijGHx4sX48MMP8dVXXyE/P19Mc4hucLh4fHe5GYBQKP7qQvYjB+gglXCo0ltRa7B2fTGXA7iwU/g5axyg0nU4xRBhU/duxmbHY8rAZKjkUhIiPxF11mzRokV499138X//93/QaDSorq4GAOh0OupXH4ZcqDHBYndBFyPHwJS4DsdjlTIUpMTiQo0JZ6uNSNV20ae+bD9g1QvB6dypHQ4zxiIumdHNxIJ2Q0wSIr8QVYjWr18PAJgxY4bX/k2bNqGoqKjvDSK6ZGiGBvFqORwuHhIfy+GvSdfgQo0J52uMmDYoufP2P80VbXlDA2cDso6dW812FxwuBo5DxOQQdQrlEvmFqELEqGBURMFxHDLju/ZUc5NioZBJoFbIYLa7POVTPTgswJl/C4HqtOFAypBOr+P2huKUMkgjoQZIO+xOHnqLAxIOSFK0ChEtfO0SWmtG+AVjzK/mhnKpBAum5HceYHY5gVNbhSFZTDww+Gaf14m0pR3tOVttwJdnalGQEos7ClsD+la9uEaFOWExa0aENy6e4c0DpfjqbA2sDle353cuQg7g+w+FDGqZAhjxY0DmuytHnFKGoRka5CXH9sR0UXAnNZpszrYgvN1MSY1dQB4R0S0l9SY0mR1wuFowY3Cq36+zO3kAgMLeDJz+P8BYLUzVD78TiOv6OtmJamQn9nDpiEjEtZYsMVmdgEwlCK/TLrTQvipXihAgISK65ftKIdA6NEPrM0h9NXvPXMal4guYkVCPPGcJwLuExawjfgzE5/SmuaKjUQrDSbPdBSfPIFPphAJp1mYSIh+QEBFdYrI5UVovrB8blnlVAinPA81lgOEKYKoVWuc4LICjBQW1jZA3WWAzKYA0DZBYAAyZ49daMsYYms0OaGPkEReoBgCVXKhd7XAxmGxOxKviW4WI4kS+ICEiuuRslQE8Y8iMVyExtnWa3WkDrhwFLh8WYh+dEK9WoNggQbE0GwPH3ARpgv9ekNHmxOYDpZBLOTwxY6DfXli4wHEc4pQyNJkdMFqdiHfHiUiIfEJCRPiEMeYZlg3PbP0y1Z4FLuxoEyB5jODtaNKFshfyGECuRqxSgzMHrsBid2EUkuF74U5Hmlvapu4jTYTcxKnkaDI7vAPWJEQ+ISEifFKlt6KxxQ65lMOgJDnw/Tag9oxwMCYByJsKpA4FJB1nyTgAuYlqnK02orzBjKwE/wPPTWY7ACAhtmOiY6QwLEOLnEQ1UjRKwEJC1B0kRIRPNCoZJuQnQmJpgPK7twFzg1C8LOd6IHcKIO3645OTJAhRWaMZkwN4X48QqSNYiNrH0xgJUXeQEBE+0ajkmKJrBKo+FqaflXHA8B95Omx0R07r9HuNwQqL3eX3KvoGkyBEiRHsEXnhySVqEXKJpJGXpNnbkBARncMYUHEIKN4lPI/PBobNE8TITzQqOUZl6ZAQq4AfSdke6k1Ci+bkON8Jj+GO08WjyeyA3cVjgI5yibqDhIjoAONdOLHrfeRbz0KnloMbMB4Y+INOY0Hd8YOhaQGd32Jzwmx3geOApLjI9YiazA68fbAMKrkU/z2jUPCKKJfIJ7TEg/CGd6H+23/BWnoY52tbYM+bCQy+KSgRCgaOAyYVJmF0VrzQtjlCca+RszpcwrIYVbxwwNIsmk3hDHlERBs8D5z5CPUl34FxErDhd0CZP6n713WDwepAeYMZ2YnqbhexqhUyXF8Q+R6DUIFACrPdBYPFAVVMgnDA0iiuYWFK5P7JIUILY8D5z2AsP4kmiwsXUm7CkBHXheTSX5yuweena1BcF12lMNyiq7c4AHVrZxpLk4gWhS8kRIRAxbdgVd+hrMmK80mzkVk40lMYv6fktvY9K2/ovtVQaX0LDFZHv6hV5SVEMa1CZCaPqDNIiAigsQS4tAuNLXacVI2HIS4fEws67y0XDDmJQimPy01mOF28z/PsTh7bTlzBG1+XoMXefbmRcKdTj8jaLCwAJrwgIYp2HFbg7CfgeR5HbDmojhuO8XkJIa0TnRynQKxSCoeLoUrvu6h+jcEKxoREyrirKztGINr2QqSIE/KHGKOAdSeQEEU7Fz8HbEZI1IkYNGUeMuJjMCEvdN4QICwCdXtFpQ0tPs+rbu38kaHrH40TMuNjMGVgMsbmJAjTgRSw9omoQrR3717MnTsXmZmZ4DgO27ZtE9Oc6KOpFKg+JXxJht6OgvRE3HNdNmS9MG3ujhOVdREnqmy2AADSdZGbyNiexFgFJuQnIt9dZVJNcSJfiCpELS0tGD16NF577TUxzYhOeB6OsztRXGeCMWmkZ9mGP3Wpg8EtRPUmW6flZl08w+UmQYgCWSAbUbgD1uQRdUDUgficOXMwZ84cMU2IWhouHkL5+Ysw8XJ83ViA+118ryYQqhUyzBs7AOlaldB48CqqDVbYnTxiFFKkRPDSjqvRWxyoN9mQHKuEjjwin0RURNBms8Fms3meGwzUKypQGGM4VlIL84HPIHPxaEyfiDlj8/skizm/i0L47qn97AR1xNYg6oy95+twsdaE6UNScG18a6KmuUFco8KQiBKi1atX49lnnxXbjIjF7uTx2akqtFz6FvkuM7TxiRg7+xaolH2/puvq9kTjchMQr5YjVhFRH8luSYpV4CKARpMdyEwWdtpbhE0ReR1KeouImjVbsWIF9Hq9Z6uoqBDbpIjB6nDhg2OXUVJrQLbpOxSmxGLYxJv7XISK60x479ty7L/o7RUoZBIMzdAiJ6l/xYfcxd0azXZhBb575qylTkSrwo+I+vOjVCqhVPaf+EFfwnGAQirBAPsljEmVQKNNADJGi2JLdWvlx3G5CYhRSP1u3hiJJLmFqEWosYTYZGGZh6kOSMgTz7AwI6I8IiJ4lDIp5o3JwO26UqHdTfbEbiss9gYFybFI0Shhd/LYfa4WVXoLNu0vxYmK5n6xrONq3MtkLHYXLHZXWz+3lloRrQo/RBUik8mEEydO4MSJEwCAkpISnDhxAuXl5WKa1a9osTk9P0vrzyLGaRAK3GeMEcUejuMw65pUSDgOZ6uN2PJtBfQWB640WfqlV6SQSTwZ1g0tNiDWLUQ0NGuPqEJ05MgRjB07FmPHjgUALF26FGPHjsXKlSvFNKvfYLG78NY3ZfjidA0cThdQdkA4kHWdEK8Qicz4GNw0PA0KmfDxy0lU4wdD/e8gG2l4Dc9iU4SdLXXCcg8CgMgxohkzZvRLdzxcOFzaCKvDhSqDFZLGi0BLvSBAA8aJbRqGZmhRmBIHq9MFjVLWL70hN6OydBiaocWAhBhALhGGxC6nkE9E1RoBRFiwmvAfg9WB7yqaAQDTCpMgLfuXcGDAeKH1cxigkEk8XlF/piDlqjrfcemA/rLQIZeECAAFq/stB4sb4OQZshJikCupAYzVwl/irNAUOyN6gDZTeDRWiWtHGEFC1A9pMNlwukrIOp86MAmcOzaUORZQ9K88nUjhcpMZR0oboTc72oTIcEVco8IIGpr1Q/YXN4AxYGBqHDJYnTAMkEiFKXtCFL4pbsDlJgtiFFLoEluFyFRHfc5aIY+on2Gxu1DZbAHHAZMLk4DSfcKBjDGAUiOqbdFMqlaIy9UYrIBSKyzvYLwwZCZIiPobMQopiibn4fZRGUhy1gDN5YI3lEPekJhk6gQhutJsFdLcdQOEA3papgSQEPVLVHIpBqZqgNL9wo70UW1tjwlRGJAgVJ2sN7bWY4rPEw40lYpmUzhBQtRPYIyhtL6lLS+rqUz4kEukQM71otpGCPWY3J1rLzdZgMR84YD+shAninJIiPoJZ6uN+PD4FWw7cQWM54HiL4UDGaOBmHhRbSMEBsQLXtGVZouwCl+lFTp6NNOSJhKifoCLZ/imWCirMSBeDa72NGCsEbKo86aKbB3hxl0Ct0bfGidyr75vKhHPqDCBhKgf8N3lZugtDsQpZRibqQIu7RIO5Eym4lthRF6yGneNz8JPxgn1wZFYKDzWX4j6dWckRBGO1eHCoUtCDeRJhUmQl+4FbCahYwRlUYcVSpkUWe1L4SYVCtnuluaon8YnIYpwDpUIC1uT4xQYpmoEKo8LBwbfIkq9IcI/eJ4JiYxJg4QdtafFNUhkSIgimMYWu2dh6w25MZCc/Ug4kDkWSMgVzzDCJ4wx7LtQjw1fX0JTix1IHSYcqDklrMiPUkiIIhgXz5AQq0BhkgK51TsBuxmISwEG/kBs0wgfcByHhhYbzHYXTlXqgaSBQsa73QzUnRXbPNEgIYpgUjRK3HddFm7hvhXyUWRKYNiPaO1SmDM8U0gu/b7SADsPwYMFgIpDURu0JiGKQBwuXvjBaYf0+61QNBcDEhkw8idU3yYCKEiORbxaDovdhRMVzYIQyRSAqRaoPSO2eaJAQhRhGKwOvPVNGY6evgB2/O9Aw0VBhIb/CIjPEds8wg8kEg6TCoU/GEfKGmFiCiC7Nfu9+CvAYRXROnEgIYogjFYHth0pQ1zNEXBHN8FlqBYK4Y/+KZA8UGzziAAYkqZBmlYFm4PH56erwbLGC9nWNiNw/rOoG6KFhRC99tpryMvLg0qlwsSJE/Htt9+KbVLYUdmgx5dffIac4ncwyHQYw9PVkCXlA9ctBOKzxTaPCBCO43Dz8DTIJByq9TYY7BJg6O0AJwFqzwIXv4gqMRI90eQf//gHli5dir/85S+YOHEi1q5di5tvvhnnzp1Damr/7ezgF047jFUXcenMUZiunEYa74BaIcXg7Ewoh8wC0kcKSwWIiCQpTolbRqQjXq2ATi0HkAVccytw5mPg8hHA3AAMuklITu3ncEzkNhoTJ07Eddddhz/96U8AAJ7nkZ2djZ/97GdYvnx5l681GAzQ6XTQ6/XQarV9YW5oYQxw2QGnFcxhgaOlCXZjI+J4A2CoBFrq8J/LTWixuQAA8UlpKBgzHcqsMaK2AyJ6j8OljbBU/AdD9F8jTgYoFVJIkwrAJQ0CNOlCORe5GpCExWCmW/z9jorqEdntdhw9ehQrVqzw7JNIJJg9eza++eabDufbbDbYbDbPc4PB4N8bNZYIrm5neHSYddhXXGdCY4sNDADHWNsZTJi1ujY3AbJWj6Sk3oRao+2qa7ZelwFjc3RQSIUPT1lDC6r1Fkh4BzjwYAzgGQPf+rLxuQmQt56rTUiBUToABUOvRUbOYPKA+jE2pwuHSxthc2Tie8nNyG08iARLObiSRsikR6FWyDAsQysM36RynKpugcXJgXESAN6fC4VcijHZCZ7npyoNaLG7Eya9z5VJOIzLTUDAjLxbqCAQAkQVovr6erhcLqSlpXntT0tLw9mzHZO7Vq9ejWeffTbwN3LahJ5eAcKbjXC6e5Z3hlXq+cvEW03grTafpzK7HJBJhZ8dFvAOK3j3MU4CJ6eCXR4Lp1IHfWohktNzAG0mcpUa5JH4RAVKmRQ/vjYLF2tNuNIcgzL1bSgzNyDJfAkaWw1SOL3wh4jxgNMGp8UIp4Pv9FpMJgFaXG3PTXrwts4zt5mUA0xBZHWzzt87GESPEQXCihUrsHTpUs9zg8GA7Gw/ArXx2cLMEtDOo2j35faxL6nFDrXd1db8z/Mo/MNpVcI+jkOKxYE4Jw+u9Rpe2sFxkMUqPaKVbHdC4+ABqQKQqQCJDHK5FDFyqccT8ry0+7sj+hFpWhXStG195xyufFgdY2F18JBwANRywNECuBzIGGKCy+kA+I4iIuUAaJWe5+kFVthdnUdhJBIA2iB63YWwsoOoQpScnAypVIqamhqv/TU1NUhPT+9wvlKphFKp7LC/WxSxbRXxAiAxgOqqWg3gr5MaFwvEdX8aQUAulUAulUDTXidamyCkBRDETgli5NWXiBrxUigUGDduHL788kvPPp7n8eWXX2LSpEkiWkYQRF8i+tBs6dKlmD9/PsaPH48JEyZg7dq1aGlpwcMPPyy2aQRB9BGiC9E999yDuro6rFy5EtXV1RgzZgy2b9/eIYBNEET/RfQ8op4Q8XlEBNHP8fc7GhlZUQRB9GtIiAiCEB0SIoIgREf0YHVPcIe3/F7qQRBEn+L+bnYXio5oITIajQDgX3Y1QRCiYTQaodP5zhCO6FkznudRWVkJjUbTtgwjxLiXkVRUVETFzFy03S9A99yb98wYg9FoRGZmJiRdVAyIaI9IIpEgKyurT95Lq9VGzYcUiL77Beiee4uuPCE3FKwmCEJ0SIgIghAdEqJuUCqVWLVqVXCr/iOQaLtfgO45HIjoYDVBEP0D8ogIghAdEiKCIESHhIggCNEhISIIQnRIiBBYp9kNGzZg2rRpSEhIQEJCAmbPnh1xnWmD7ay7ZcsWcByHefPm9a6BvUCg99zc3IxFixYhIyMDSqUSgwcPxqefftpH1oaGQO957dq1GDJkCGJiYpCdnY0lS5bAarX2jbEsytmyZQtTKBRs48aN7Pvvv2ePPvooi4+PZzU1NZ2ef99997HXXnuNHT9+nJ05c4YVFRUxnU7HLl++3MeWB0eg9+umpKSEDRgwgE2bNo3dcccdfWNsiAj0nm02Gxs/fjy79dZb2b59+1hJSQnbvXs3O3HiRB9bHjyB3vM777zDlEole+edd1hJSQnbsWMHy8jIYEuWLOkTe6NeiCZMmMAWLVrkee5yuVhmZiZbvXq1X693Op1Mo9GwN998s7dMDCnB3K/T6WSTJ09mf/vb39j8+fMjTogCvef169ezgoICZrfb+8rEkBPoPS9atIjNmjXLa9/SpUvZlClTetVON1E9NHN3mp09e7ZnX1edZjvDbDbD4XAgMTH8+5MHe7/PPfccUlNTsXDhwr4wM6QEc88fffQRJk2ahEWLFiEtLQ0jRozASy+9BJfL1en54UYw9zx58mQcPXrUM3y7dOkSPv30U9x66619YnNEL3rtKYF2mu2Mp59+GpmZmV6/9HAlmPvdt28f3njjDZw4caIPLAw9wdzzpUuX8NVXX+H+++/Hp59+iosXL+KJJ56Aw+HAqlWr+sLsHhHMPd93332or6/H1KlTwRiD0+nE448/jl/+8pd9YTIFq3vCmjVrsGXLFnz44YdQqYLolBnmGI1GPPjgg9iwYQOSk5PFNqfP4HkeqampeP311zFu3Djcc889+NWvfoW//OUvYpvWa+zevRsvvfQS/vznP+PYsWPYunUrPvnkEzz//PN98v5R7REF2mm2Pb/73e+wZs0afPHFFxg1alRvmhkyAr3f4uJilJaWYu7cuZ59PC/0O5fJZDh37hwKCwt71+geEszvOCMjA3K5HFKp1LNv6NChqK6uht1uh0Kh6FWbe0ow9/zrX/8aDz74IB555BEAwMiRI9HS0oLHHnsMv/rVr7qsJRQKotojCrbT7Msvv4znn38e27dvx/jx4/vC1JAQ6P1ec801OHnyJE6cOOHZfvjDH2LmzJk4ceJERFTGDOZ3PGXKFFy8eNEjugBw/vx5ZGRkhL0IAcHds9ls7iA2biFmfbEctU9C4mHMli1bmFKpZJs3b2anT59mjz32GIuPj2fV1dWMMcYefPBBtnz5cs/5a9asYQqFgr3//vusqqrKsxmNRrFuISACvd+ricRZs0Dvuby8nGk0GrZ48WJ27tw59vHHH7PU1FT2wgsviHULARPoPa9atYppNBr23nvvsUuXLrGdO3eywsJCdvfdd/eJvVEvRIwx9sc//pHl5OQwhULBJkyYwA4ePOg5Nn36dDZ//nzP89zcXAagw7Zq1aq+NzxIArnfq4lEIWIs8Hs+cOAAmzhxIlMqlaygoIC9+OKLzOl09rHVPSOQe3Y4HOyZZ55hhYWFTKVSsezsbPbEE0+wpqamPrGVyoAQBCE6UR0jIggiPCAhIghCdEiICIIQHRIigiBEh4SIIAjRISEiCEJ0SIgIghAdEiIioti8eTPi4+M9z5955hmMGTPG87yoqCgiK0hGOyRERKcUFRWB4zg8/vjjHY4tWrQIHMehqKjI6/xQC0BeXh7Wrl3rte+ee+7B+fPnfb5m3bp12Lx5s+f5jBkz8OSTT4bULiL0kBARPsnOzsaWLVtgsVg8+6xWK959913k5OSIYlNMTAxSU1N9HtfpdF4eExEZkBARPrn22muRnZ2NrVu3evZt3boVOTk5GDt2bI+u3ZmnMm/ePI+XNWPGDJSVlWHJkiXgOA4cxwHoODS7mvaeWVFREfbs2YN169Z5rlFSUoKBAwfid7/7ndfrTpw4AY7jcPHixR7dFxEcJERElyxYsACbNm3yPN+4cSMefvjhXn/frVu3IisrC8899xyqqqpQVVUV8DXWrVuHSZMm4dFHH/VcIycnp8M9AcCmTZtwww03YODAgaG6BSIASIiILnnggQewb98+lJWVoaysDPv378cDDzzQ6++bmJgIqVQKjUaD9PT0bgvVdYZOp4NCoYBarfZcQyqVoqioCOfOnfPUZ3Y4HHj33XexYMGCUN8G4SdRXaGR6J6UlBTcdttt2Lx5MxhjuO222yK+bGxmZiZuu+02bNy4ERMmTMC///1v2Gw23HXXXWKbFrWQR0R0y4IFC7B582a8+eabIfMaJBJJh8p/DocjJNf2h0ceecQTiN+0aRPuueceqNXqPnt/whsSIqJbbrnlFtjtdjgcDtx8880huWZKSopX3MflcuHUqVNe5ygUih638PF1jVtvvRWxsbFYv349tm/fTsMykaGhGdEtUqkUZ86c8fzsC71e36HtUFJSUqe1rWfNmoWlS5fik08+QWFhIV599VU0Nzd7nZOXl4e9e/fipz/9KZRKZVBDwry8PBw6dAilpaWIi4tDYmIiJBKJJ1a0YsUKDBo0qMsa5UTvQx4R4RdarRZarbbLc3bv3o2xY8d6bc8++2yn5y5YsADz58/HQw89hOnTp6OgoAAzZ870Oue5555DaWkpCgsLkZKSEpTdy5Ytg1QqxbBhw5CSkoLy8nLPsYULF8Jut/fJLCDRNVQqlohavv76a/zgBz9ARUVFh2aERN9CQkREHTabDXV1dZg/fz7S09PxzjvviG1S1ENDMyLqeO+995Cbm4vm5ma8/PLLYptDgDwigiDCAPKICIIQHRIigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhIggCNEhISIIQnRIiAiCEJ3/D2sWYCCKbdV7AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:26:59.050438Z", + "iopub.status.busy": "2024-02-29T00:26:59.049575Z", + "iopub.status.idle": "2024-02-29T00:26:59.292474Z", + "shell.execute_reply": "2024-02-29T00:26:59.291535Z" + }, + "papermill": { + "duration": 0.269364, + "end_time": "2024-02-29T00:26:59.294728", + "exception": false, + "start_time": "2024-02-29T00:26:59.025364", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2nUlEQVR4nO3deVhT17o/8C8JEEAGUWZEgqLiBCj+RLTWkcFWr5baUm0d0NJ7VHqsORwttoWH6pG2WkoHKrfe4tBBPXWqRy2CKK0D2Iq1RQUUBFEZZJBZQ0jW7w8vu8YECAiG7Lyf58kDe+21V9ZiJy97WHstA8YYAyGE8JBA2xUghJCeQgGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbhtquQG+kUChQUlICCwsLGBgYaLs6hJDHMMZQX18PJycnCARtH6dRgFOjpKQELi4u2q4GIaQDt27dwoABA9pcTwFODQsLCwAP/3iWlpZark3PkclkSElJQUBAAIyMjLRdHfKE9Gl/1tXVwcXFhfuutoUCnBqtp6WWlpa8D3BmZmawtLTk/RdCH+jj/uzoEhLdZCCE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RNxFCdFRTUxNyc3O55Yb7UpzLLoC1zQWYm4qU8np4eMDMzOxpV1HrKMARoqNyc3Ph4+Ojkv6RmrxZWVkYO3Zsz1eql6EAR4iO8vDwQFZWFrecV1oDyQ/ZiHtpNIY59lXJq48owBGio8zMzJSOygQ3qyA6fR/DR3nB27W/FmvWe9BNBkIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFnX01SP07CLRO0zLvvjiC+bq6spEIhEbP348O3/+fLv57927x1auXMkcHByYsbExGzJkCDt69Ci3Pjo6mgFQeg0bNqxTdaqtrWUAWG1tbZfa1FtlZWWp/G3aemVlZWm7uqSTfi+qZK7rjrDfiyq1XZUep+l3VKtHcHv37oVEIkFiYiJ8fX0RHx+PwMBA5OXlwc7OTiV/c3Mz/P39YWdnh3379sHZ2Rk3b95E3759lfKNHDkSJ06c4JYNDelAFaBnF4n+0eo3Py4uDmFhYQgNDQUAJCYm4ujRo0hKSsLbb7+tkj8pKQnV1dU4d+4cN2uQWCxWyWdoaAgHB4cerbsuomcXib7RWoBrbm5GVlYWIiMjuTSBQICZM2ciIyND7TaHDx+Gn58fVq1ahR9//BG2trZYuHAh1q1bB6FQyOW7fv06nJycYGJiAj8/P8TGxmLgwIFt1kUqlUIqlXLLdXV1AB5OwyaTyZ60qb1WS0sL95PP7dQX+rQ/NW2f1gJcZWUl5HI57O3tldLt7e2VLoQ/6saNGzh58iReffVVHDt2DPn5+Vi5ciVkMhmio6MBAL6+vtixYweGDRuG0tJSxMTEYPLkybh8+XKbk8TGxsYiJiZGJT0lJYXXF9pvNQCAITIzM3HnsrZrQ56UPu3PpqYmjfLp1MUphUIBOzs7fPXVVxAKhfDx8cGdO3ewefNmLsDNmjWLy+/p6QlfX1+4urri3//+N5YvX6623MjISEgkEm65ddbsgIAAXk/8/EdxNZB9ARMmTIDXwH7arg55Qvq0P1vPsjqitQBnY2MDoVCI8vJypfTy8vI2r585OjrCyMhI6XR0+PDhKCsrQ3NzM4yNjVW26du3L4YOHYr8/Pw26yISiSASiVTSjYyMeD1DeOvNF0NDQ163U1/o0/7UtH1a6+hrbGwMHx8fpKWlcWkKhQJpaWnw8/NTu82kSZOQn58PhULBpV27dg2Ojo5qgxsANDQ0oKCgAI6Ojt3bAEJIr6fVJxkkEgm2bduGnTt3IicnBytWrEBjYyN3V3Xx4sVKNyFWrFiB6upqrF69GteuXcPRo0exadMmrFq1issTERGBn3/+GUVFRTh37hxeeOEFCIVCLFiw4Km3jxCiXVq9BhcSEoKKigpERUWhrKwM3t7eSE5O5m48FBcXQyD4Kwa7uLjg+PHjWLNmDTw9PeHs7IzVq1dj3bp1XJ7bt29jwYIFqKqqgq2tLZ555hlkZmbC1tb2qbePEKJdWr/JEB4ejvDwcLXr0tPTVdL8/PyQmZnZZnl79uzprqoRQnQcPWxPCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDe0no3EUKI5gorG9EobVG7rqCikfvZ3hiIfUSGcLPp0yP1620owBGiIworGzFtS3qH+f6xL7vDPKcipupFkKMAR4iOaD1yiw/xhruduer6+1IcSc/A7Kl+6GOqOngEAOTfbcBbey+1eRTINxTgCNEx7nbmGOVspZIuk8lQZguMdbXm/WgimqKbDIQQ3qIjOB5r74I0QBelCf9RgOMpTS9IA3RRmvAXBTie6uiCNEAXpQn/UYDjubYuSAN0UZrwH91kIITwFgU4QghvUYAjhPAWBThCCG9pPcAlJCRALBbDxMQEvr6++PXXX9vNX1NTg1WrVsHR0REikQhDhw7FsWPHnqhMQgg/aTXA7d27FxKJBNHR0bh48SK8vLwQGBiIu3fvqs3f3NwMf39/FBUVYd++fcjLy8O2bdvg7Ozc5TIJIfyl1QAXFxeHsLAwhIaGYsSIEUhMTISZmRmSkpLU5k9KSkJ1dTUOHTqESZMmQSwWY8qUKfDy8upymYQQ/tJaP7jm5mZkZWUpTewsEAgwc+ZMZGRkqN3m8OHD8PPzw6pVq/Djjz/C1tYWCxcuxLp16yAUCrtUJgBIpVJIpVJuua6uDsDDfmIymexJm6oVLS0t3M+22tCa3l4bNSmHPB0d7Qt92p+a1l1rAa6yshJyuZyb5LmVvb09cnNz1W5z48YNnDx5Eq+++iqOHTuG/Px8rFy5EjKZDNHR0V0qEwBiY2MRExOjkp6SkgIzM7MutE77bjUAgCHOnDmDm+ofZOCkpqZ2SzmkZ2m6L/RhfzY1NWmUT6eeZFAoFLCzs8NXX30FoVAIHx8f3LlzB5s3b0Z0dHSXy42MjIREIuGW6+rq4OLigoCAAFhaWnZH1Z+6KyV12JKdiWeeeQYjndS3QSaTITU1Ff7+/m0+yaBJOeTp6Ghf6NP+bD3L6ojWApyNjQ2EQiHKy8uV0svLy+Hg4KB2G0dHRxgZGUEoFHJpw4cPR1lZGZqbm7tUJgCIRCKIRKrPYhoZGensI0yto4MYGhp22Ib22tmZckjP0nRf6MP+1LTuWrvJYGxsDB8fH6SlpXFpCoUCaWlp8PPzU7vNpEmTkJ+fD4VCwaVdu3YNjo6OMDY27lKZhBD+0updVIlEgm3btmHnzp3IycnBihUr0NjYiNDQUADA4sWLlW4YrFixAtXV1Vi9ejWuXbuGo0ePYtOmTVi1apXGZRJC9IdWr8GFhISgoqICUVFRKCsrg7e3N5KTk7mbBMXFxRAI/orBLi4uOH78ONasWQNPT084Oztj9erVWLduncZlEkL0h9ZvMoSHhyM8PFztuvT0dJU0Pz8/ZGZmdrlMQoj+0PqjWoQQ0lMowBFCeEvrp6iEEM1I5Q8gMLmDwro8CExUe+m2tLSgpKUEOdU5bU4iVFjXAIHJHUjlDwCoH+mZTyjAEaIjShpvoo/b51jfweA4XyZ/2e76Pm5ASaM3fMD/G28U4AjREU59XNFY+CY+DfHGYDUTCbW0tODsmbOY9MykNo/gCu42YPXeS3Ca5trT1e0VKMDxVEenMwCd0ugakdAEigfOcLMchhH91c9sX2hYiOH9hrfZ01/xoBaKBxUQCU16urq9AgU4ntL0dAagUxrCXxTgeKqj0xmATmkI/1GA46mOTmcAOqUh/Ef94AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbvSLAJSQkQCwWw8TEBL6+vvj117afEN+xYwcMDAyUXiYmyo8RLV26VCVPUFBQTzeDENLLaP1Z1L1790IikSAxMRG+vr6Ij49HYGAg8vLyYGdnp3YbS0tL5OXlccsGBgYqeYKCgrB9+3ZuWd3EzoQQfuv0EdyNGze6tQJxcXEICwtDaGgoRowYgcTERJiZmSEpKanNbQwMDODg4MC91E0JKBKJlPJYW1t3a70JIb1fp4/g3N3dMWXKFCxfvhzz589XOT3sjObmZmRlZSlN7iwQCDBz5kxkZGS0uV1DQwNcXV2hUCgwduxYbNq0CSNHjlTKk56eDjs7O1hbW2P69OnYuHEj+vfvr7Y8qVQKqVTKLdfV1QF4ONqGTCbrcvu0qaWlhfvZVhta09troyblkKejo32hT/tT07p3OsBdvHgR27dvh0QiQXh4OEJCQrB8+XKMHz++05WsrKyEXC5XOQKzt7dHbm6u2m2GDRuGpKQkeHp6ora2Flu2bMHEiRNx5coVDBgwAMDD09Pg4GC4ubmhoKAA69evx6xZs5CRkQGhUKhSZmxsLGJiYlTSU1JSYGZm1ul29Qa3GgDAEGfOnMFN9cPBcVJTU7ulHNKzNN0X+rA/m5qaNMpnwBhjXXmDlpYWHD58GDt27EBycjKGDh2KZcuWYdGiRbC1tdWojJKSEjg7O+PcuXPw8/Pj0teuXYuff/4Z58+f77AMmUyG4cOHY8GCBdiwYYPaPDdu3MDgwYNx4sQJzJgxQ2W9uiM4FxcXVFZWwtLSUqO29DZXSuowb2smDq2YgJFO6tsgk8mQmpoKf3//NseD06Qc8nR0tC/0aX/W1dXBxsYGtbW17X5Hu3yTwdDQEMHBwXj++efx5ZdfIjIyEhEREVi/fj1efvllfPjhh3B0dGy3DBsbGwiFQpSXlyull5eXw8HBQaN6GBkZYcyYMcjPz28zz6BBg2BjY4P8/Hy1AU4kEqm9CWFkZNTmB6W3ax2h19DQsMM2tNfOzpRDepam+0If9qemde9yN5ELFy5g5cqVcHR0RFxcHCIiIlBQUIDU1FSUlJRg7ty5HZZhbGwMHx8fpKWlcWkKhQJpaWlKR3TtkcvlyM7ObjeY3r59G1VVVR0GXEIIv3T6CC4uLg7bt29HXl4ennvuOezatQvPPfccBIKHsdLNzQ07duyAWCzWqDyJRIIlS5Zg3LhxGD9+POLj49HY2IjQ0FAAwOLFi+Hs7IzY2FgAwPvvv48JEybA3d0dNTU12Lx5M27evInXX38dwMMbEDExMXjxxRfh4OCAgoICrF27Fu7u7ggMDOxscwkhOqzTAW7r1q1YtmwZli5d2uYRkZ2dHb7++muNygsJCUFFRQWioqJQVlYGb29vJCcnczceiouLueAJAPfu3UNYWBjKyspgbW0NHx8fnDt3DiNGjAAACIVC/Pnnn9i5cydqamrg5OSEgIAAbNiwgfrCEaJnOh3gUlNTMXDgQKWgAwCMMdy6dQsDBw6EsbExlixZonGZ4eHhCA8PV7suPT1dafmTTz7BJ5980mZZpqamOH78uMbvTQjhr05fgxs8eDAqKytV0qurq+Hm5tYtlSKEkO7Q6QDXVq+ShoaGJ+r0Swgh3U3jU1SJRALg4WNSUVFRSh1g5XI5zp8/D29v726vIOma+zI5AODyndo28zTel+JCBeBw8x76mKq/Ppl/t6FH6kfI06BxgPv9998BPDyCy87OhrGxMbfO2NgYXl5eiIiI6P4aki4p+L/A9PaB7A5yGuKb/N86LK+PSOvjMhDSaRp/ak+dOgUACA0NxaeffqqzPfz1RcDIhx2lB9uZw9RI9fE0AMgrrcU/9mXj4/mjMczRqs2y+ogM4WbTp0fqSUhP6vS/5UeHICK9V78+xnhl/MB287Q+eD3Ytg9GObcd4AjRVRoFuODgYOzYsQOWlpYIDg5uN++BAwe6pWKEEPKkNApwVlZW3KCSVlb0n54Qohs0CnCPnpbSKSohRFf0ijkZCCGkJ2h0BDdmzBi18x6oc/HixSeqECGEdBeNAty8efN6uBqEkI501HmbOm6r0ijARUdH93Q9CCEd0KzzNnXcfpR+tJIQHuio8zZ13FalUYDr168frl27BhsbG1hbW7d7Pa66urrbKkcI+UtHnbep47YqjQLcJ598AgsLC+53TW84EEKINmkU4B4dvHLp0qU9VRdCCOlWne4HJxQKcffuXZX0qqoqtXOOEkKItnTbgJdSqVRpCCVCCNE2je+ifvbZZwAeDnj5v//7vzA3/2tabLlcjl9++QUeHh5dqkRCQgI2b96MsrIyeHl54fPPP8f48ePV5t2xYwc341YrkUiEBw8ecMuMMURHR2Pbtm2oqanBpEmTsHXrVgwZMqRL9SOE6CaNA1zrRC+MMSQmJiqdjhobG0MsFiMxMbHTFdi7dy8kEgkSExPh6+uL+Ph4BAYGIi8vD3Z2dmq3sbS0RF5eHrf8+E2Pjz76CJ999hl27twJNzc3vPfeewgMDMTVq1dpWHVC9AnrpKlTp7Lq6urObtam8ePHs1WrVnHLcrmcOTk5sdjYWLX5t2/fzqysrNosT6FQMAcHB7Z582YuraamholEIrZ7926N6lRbW8sAsNraWs0aoaN+L6pkruuOsN+LKrVdFdIN9Gl/avod7fQ1uFOnTsHa2rpbgmtzczOysrIwc+ZMLk0gEGDmzJnIyMhoc7uGhga4urrCxcUFc+fOxZUrV7h1hYWFKCsrUyrTysoKvr6+7ZZJCOGfTj/JsGzZsnbXJyUlaVxWZWUl5HI5N8lzK3t7e+Tm5qrdZtiwYUhKSoKnpydqa2uxZcsWTJw4EVeuXMGAAQNQVlbGlfF4ma3rHieVSiGVSrnluro6AIBMJoNMJtO4PbqmtWNoS0sLr9upL/Rpf2ravk4HuHv37qm80eXLl1FTU4Pp06d3trhO8/Pzg5+fH7c8ceJEDB8+HP/zP/+DDRs2dKnM2NhYxMTEqKSnpKQozR7GN7caAMAQmZmZuHNZ27UhT0qf9mdTU5NG+Tod4A4ePKiSplAosGLFCgwePLhTZdnY2EAoFKK8vFwpvby8HA4ODhqVYWRkhDFjxiA/Px8AuO3Ky8vh6OioVGZb0xpGRkZy0yICD4/gXFxcEBAQwOvJdf4orgayL2DChAnwGthP29UhT0if9mfrWVZHuuVhe4FAAIlEgqlTp2Lt2rUab2dsbAwfHx+kpaVxQzIpFAqkpaUhPDxcozLkcjmys7Px3HPPAQDc3Nzg4OCAtLQ0LqDV1dXh/PnzWLFihdoyRCIRRCLV4WWMjIxgZGSkcXt0jaGhIfeTz+3UF/q0PzVtX7eNJlJQUMBdA+gMiUSCJUuWYNy4cRg/fjzi4+PR2NjI9XVbvHgxnJ2dERsbCwB4//33MWHCBLi7u6OmpgabN2/GzZs38frrrwN42GXkrbfewsaNGzFkyBCum4iTkxONa0eInul0gHv0VA542C+utLQUR48eVXpmVVMhISGoqKhAVFQUysrK4O3tjeTkZO4mQXFxMQSCv2723rt3D2FhYSgrK4O1tTV8fHxw7tw5jBgxgsuzdu1aNDY24o033kBNTQ2eeeYZJCcnUx84QvSMAWNtPHvVhmnTpiktCwQC2NraYvr06Vi2bBl3mKzL6urqYGVlhdraWl5fg7t0swrztmbi0IoJ8Hbtr+3qkCekT/tT0+9op6NR6wz3hBDS29GsWoQQ3qIARwjhLQpwhBDeogBHCOGtbgtwt2/fxhtvvNFdxRFCyBPrtgBXVVWFr7/+uruKI4SQJ0anqIQQ3qIARwjhLQpwhBDe0vhJhuDg4HbX19TUPGldCCGkW2kc4KysrDpcv3jx4ieuECGEdBeNA9z27dt7sh6EENLt6BocIYS3ND6C62iymVadmXSGEEJ6ksYBbseOHXB1dcWYMWPQySHkCCFEKzQOcCtWrMDu3btRWFiI0NBQvPbaa+jXj98TWxBCdJvG1+ASEhJQWlqKtWvX4j//+Q9cXFzw8ssv4/jx43RERwjplTp1k0EkEmHBggVITU3F1atXMXLkSKxcuRJisRgNDQ09VUdCCOmSLt9FFQgEMDAwAGMMcrn8iSqRkJAAsVgMExMT+Pr64tdff9Vouz179sDAwEBltqylS5fCwMBA6RUUFPREdSSE6J5OBTipVIrdu3fD398fQ4cORXZ2Nr744gsUFxfD3Ny8SxXYu3cvJBIJoqOjcfHiRXh5eSEwMBB3795td7uioiJERERg8uTJatcHBQWhtLSUe+3evbtL9SOE6C6NA9zKlSvh6OiIDz74ALNnz8atW7fwww8/4LnnnlOa1q+z4uLiEBYWhtDQUIwYMQKJiYkwMzNrt7uJXC7Hq6++ipiYGAwaNEhtHpFIBAcHB+5lbW3d5ToSQnSTxndRExMTMXDgQAwaNAg///wzfv75Z7X5Dhw4oPGbNzc3IysrC5GRkVyaQCDAzJkzkZGR0eZ277//Puzs7LB8+XKcPn1abZ709HTY2dnB2toa06dPx8aNG9G/P7+nUiOEKNM4wC1evBgGBgbd+uaVlZWQy+XcJM+t7O3tkZubq3abM2fO4Ouvv8alS5faLDcoKAjBwcFwc3NDQUEB1q9fj1mzZiEjIwNCoVAlv1QqhVQq5Zbr6uoAADKZDDKZrAst0w0tLS3cTz63U1/o0/7UtH2d6uirbfX19Vi0aBG2bdsGGxubNvO98sor3O+jR4+Gp6cnBg8ejPT0dMyYMUMlf2xsLGJiYlTSU1JSYGZm1j2V74VuNQCAITIzM3HnsrZrQ56UPu3PpqYmjfJpdRp6GxsbCIVClJeXK6WXl5fDwcFBJX9BQQGKioowZ84cLk2hUAAADA0NkZeXh8GDB6tsN2jQINjY2CA/P19tgIuMjIREIuGW6+rq4OLigoCAAF7PbP9HcTWQfQETJkyA10DqtK3r9Gl/tp5ldUSrAc7Y2Bg+Pj5IS0vjunooFAqkpaUhPDxcJb+Hhweys7OV0t59913U19fj008/hYuLi9r3uX37NqqqquDo6Kh2vUgkgkgkUkk3MjKCkZFRJ1ulOwwNDbmffG6nvtCn/alp+7Qa4ABAIpFgyZIlGDduHMaPH4/4+Hg0NjYiNDQUwMNrf87OzoiNjYWJiQlGjRqltH3fvn0BgEtvaGhATEwMXnzxRTg4OKCgoABr166Fu7s7AgMDn2rbCCHapfUAFxISgoqKCkRFRaGsrAze3t5ITk7mbjwUFxd3qhuKUCjEn3/+iZ07d6KmpgZOTk4ICAjAhg0b1B6lEUL4S+sBDgDCw8PVnpICD7t7tOfxmx+mpqY4fvx4N9WMEKLLaMBLQghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgNNTcrkcFzLOoPHqz7iQcQZyuVzbVSKk21GA00MHDhyAu7s7whbMQ+V/NiNswTy4u7t3ak5bQnQBBTg9c+DAAcyfPx+jR4/GzgPJcFnzA3YeSMbo0aMxf/58CnKEV3rFkOXk6aivr8ebb76JyZMn45133sGRk2dRd+ESrtt645133kFtbS3+/ve/w8XFBSNHjuT1nLBEP1CA0yO7d+9GSUkJSkpKMGHCBC594+lvsPGRfOPHj0dWVhbGjh379CtJSDfqFaeoCQkJEIvFMDExga+vL3799VeNttuzZw8MDAy4OVVbMcYQFRUFR0dHmJqaYubMmbh+/XoP1Fy3GBsbc78/++yz+DLxK7z17iZ8mfgVnn32WW7dv/71L3h4eGijioR0K60HuL1790IikSA6OhoXL16El5cXAgMDcffu3Xa3KyoqQkREBCZPnqyy7qOPPsJnn32GxMREnD9/Hn369EFgYCAePHjQU83QCa0TYz/zzDM4ceIEhgx2g6CpEkMGu+HEiROYNGkSAMDX15dOTwkvaD3AxcXFISwsDKGhoRgxYgQSExNhZmaGpKSkNreRy+V49dVXERMTg0GDBimtY4whPj4e7777LubOnQtPT0/s2rULJSUlOHToUA+3RjdUVVVhyJAh8Pf3R1xcHPz9/TFkyBBUVVVpu2qEdCutXoNrbm5GVlYWIiMjuTSBQICZM2ciIyOjze3ef/992NnZYfny5Th9+rTSusLCQpSVlWHmzJlcmpWVFXx9fZGRkYFXXnlFpTypVAqpVMot19XVAQBkMhlkMlmX29fblJSUAABycnJgZ2eHL774AmZmZmhqasL777+Pmzdvcvn41G590dLSwv3k+/7TtH1aDXCVlZWQy+XcLPat7O3tkZubq3abM2fO4Ouvv8alS5fUri8rK+PKeLzM1nWPi42NRUxMjEp6SkoKr07VCgsLAQDOzs6QyWRKk23b29vD2dkZd+7cQWFhIY4dO6atapIuutUAAIbIzMzEncvark3Pampq0iifTt1Fra+vx6JFi7Bt2zbY2Nh0W7mRkZGQSCTccl1dHVxcXBAQEABLS8tuex9tMzExAQCIxWKkpKTgl19+QWpqKvz9/fHss8/C398fd+7cga+vL6ZPn67l2pLO+qO4Gsi+gAkTJsBrYD9tV6dHtZ5ldUSrAc7GxgZCoRDl5eVK6eXl5XBwcFDJX1BQgKKiIsyZM4dLUygUAABDQ0Pk5eVx25WXl8PR0VGpTG9vb7X1EIlEEIlEKulGRkYwMjLqdLt6q+rqagDAuXPnEBISAn9/fzQ3N+PatWtISEjgLgtUV1fzqt36wtDQkPvJ9/2nafu0GuCMjY3h4+ODtLQ0rquHQqFAWlqa0ulTKw8PD2RnZyulvfvuu6ivr8enn34KFxcXGBkZwcHBAWlpaVxAq6urw/nz57FixYqeblKv1hrwFy5ciL179+LIkSPcOkNDQyxYsADff/+90j8GQnSZ1k9RJRIJlixZgnHjxmH8+PGIj49HY2MjQkNDAQCLFy+Gs7MzYmNjYWJiglGjRilt37dvXwBQSn/rrbewceNGDBkyBG5ubnjvvffg5OSk0l9O30yePBm2trb47rvv8PzzzyMgIADXr1/HkCFDkJKSgu+//x52dnZqu94Qoou0HuBCQkJQUVGBqKgolJWVwdvbG8nJydxNguLiYggEnevNsnbtWjQ2NuKNN95ATU0NnnnmGSQnJ3PXoPSZgYEB93PMmDGws7ODs7MzUlNTtVwzQnoAIypqa2sZAFZbW6vtqnSrU6dOMQAsNjaWicViBoB7ubm5sU2bNjEA7NSpU9quKumC34sqmeu6I+z3okptV6XHafod1XpHX/L0lJaWAgDCw8ORn5+P1NRUSCQSpKam4vr169x1z9Z8hOg6CnB6pPXmweXL6jtJtabTTQbCF1q/BkeensmTJ0MsFuPNN99ERUUF9+RCXFwcXF1dYWtrCzc3N7rJQHiDjuD0iFAoxEsvvYQLFy7g/v37mD9/PqZPn4758+fj/v37uHDhAubPnw+hUKjtqhLSLegITo/I5XL88MMPsLe3R3l5Ofbt26e03t7eHvv27UNsbCwFOcILFOD0yOnTp1FUVATgYf9BgUCAxsZG9OnTBwqFgnui5PTp05g6dar2KkpIN6EAp0du3boF4GEfuJqaGi69dSQVAwMDMMa4fIToOroGp0fOnz8P4OGYeeq0prfmI0TXUYDTI/X19dzvjz+s/Ojyo/kI0WUU4PTI8ePHud8fn+j50eVH8xGiyyjA6ZFHBwlsHWZK3bKmgwkS0ttRgNMj6sa8e5J8hPR2FOD0yKNTA3ZHPkJ6O+omokceHy7K1tYW5ubmaGhoQEVFRZv5CNFVFOD0SOtYcK0qKiqUAltb+QjRVXSKqkdcXV27NR8hvR0dwemRKVOmYNOmTRrlI71fU1OT0vSaeaU1kJblI+eyKRRVfZXyenh48GoKTE1RgNMjmj5ATw/a64bc3Fz4+PiopC/cqZo3KysLY8eOfQq16l0owOmRtia+7mo+ol0eHh7IysrilhvuS3H0VAaen+YHc1ORSl591CuuwSUkJEAsFsPExAS+vr749ddf28x74MABjBs3Dn379kWfPn3g7e2Nb775RinP0qVLYWBgoPQKCgrq6Wb0eo/PP/uk+Yh2mZmZYezYsdzLb/w4TBw9GH7jxymljx07Vi9PT4FecAS3d+9eSCQSJCYmwtfXF/Hx8QgMDEReXh7s7OxU8vfr1w/vvPMOPDw8YGxsjCNHjiA0NBR2dnYIDAzk8gUFBWH79u3cMnVeBSorK7nf1U0bePToUZV8hOgyrR/BxcXFISwsDKGhoRgxYgQSExNhZmaGpKQktfmnTp2KF154AcOHD8fgwYOxevVqeHp64syZM0r5RCIRHBwcuJe1tfXTaE6vVlxczP3eOm3gpEmTMGbMGKWuIY/mI0SXafUIrrm5GVlZWYiMjOTSBAIBZs6ciYyMjA63Z4zh5MmTyMvLw4cffqi0Lj09HXZ2drC2tsb06dOxceNG9O/fX205UqmUGxMNAOrq6gAAMpkMMpmsK03rlVqfNx0wYACys7OVnlgQi8UYMGAAbt++DYVCwat264vWfaYP+07TNmo1wFVWVkIul3OTPLeyt7dXuv39uNraWjg7O0MqlUIoFOLLL7+Ev78/tz4oKAjBwcFwc3NDQUEB1q9fj1mzZiEjI0PtHcLY2FjExMSopKekpPDq2kXrh+L27dvw8fFBQEAARCIRpFIpLl68yF2wlslkOHbsmDarSjpJLpfj6tWruHfvHrKzszFixAhe3w3XdEAIrV+D6woLCwtcunQJDQ0NSEtLg0QiwaBBg7hhtl955RUu7+jRo+Hp6YnBgwcjPT0dM2bMUCkvMjISEomEW66rq4OLiwsCAgJgaWnZ4+15WkxNTbl5GK5evap0B87U1JT7PSwsDNOmTXvq9SNdc/DgQaxbt44bjh54eET+4Ycf4oUXXtBexXpQ61lWR7Qa4GxsbCAUClXu2pWXl8PBwaHN7QQCAdzd3QEA3t7eyMnJQWxsbJvzCAwaNAg2NjbIz89XG+BEIpHamxBGRkYqA0PqshkzZsDOzg53795tM4+dnR1mzJjB6//+fHLgwAG88sormD17Nr755hvcvn0bAwYMwEcffYRXXnkF+/btQ3BwsLar2e00/V5q9SaDsbExfHx8kJaWxqUpFAqkpaXBz89P43IUCoXSNbTH3b59G1VVVXo/obFQKMTWrVu5uRceZ2BggK1bt1Jw0xFyuRz/+Mc/MHv2bBw6dAi+vr4wNTWFr68vDh06hNmzZyMiIkJlcFO9wrRsz549TCQSsR07drCrV6+yN954g/Xt25eVlZUxxhhbtGgRe/vtt7n8mzZtYikpKaygoIBdvXqVbdmyhRkaGrJt27Yxxhirr69nERERLCMjgxUWFrITJ06wsWPHsiFDhrAHDx5oVKfa2loGgNXW1nZ/g3uB/fv3M7FYzABwLzc3N7Z//35tV410wqlTpxgAlpGRwRhjrLm5mR06dIg1Nzczxhg7d+4cA8BOnTqlxVr2DE2/o1q/BhcSEoKKigpERUWhrKwM3t7eSE5O5m48FBcXQyD460CzsbERK1euxO3bt2FqagoPDw98++23CAkJAfDwKOXPP//Ezp07UVNTAycnJwQEBGDDhg3UF+7/BAcHY+7cuTh16hR++uknzJo1C9OmTaMjNx1TWloKABg1apTa9a3prfn00lMKuDqF70dwrR7/j090Cx3Bdfwd1XpHX0JI10yePBlisRibNm1SO8dGbGws3NzcMHnyZC3VUPu0fopKCOkaoVCIjz/+GPPnz8fcuXPh7++P69ev4+bNm0hNTcXRo0exb98+vb70QAGOEB0WHByMiIgIfPLJJzhy5AiXbmhoiIiICF52EekMCnCE6LADBw5gy5YteP7557kjuCFDhiA1NRVbtmzBhAkT9DrIGTCmpkOUnqurq4OVlRVqa2t59STD41ofyXruued41aFZX8jlcri7u2P06NE4dOgQ5HI5tz+FQiHmzZuHy5cv4/r167w7TdX0O0o3GQjRUadPn0ZRURHWr1+v1JUKePi0T2RkJAoLC3H69Gkt1VD7KMARoqOoH1zHKMARoqNaHz28fPmy2vWt6fr8iCIFOEJ0FPWD6xgFOEJ0VGs/uCNHjmDevHnIzMzE/fv3kZmZiXnz5uHIkSPYsmUL724wdAZ1EyFEhwUHB2Pfvn34xz/+oTRCs5ubG2+HSuoMCnCE6DgaPKFtFOAI4QGhUIgpU6agsbERU6ZMoeD2f+gaHCGEtyjAEUJ4iwIcIYS36BqcGq2P52o6c4+ukslkaGpqQl1dHT2LygP6tD9bv5sdPUpPAU6N+vp6AICLi4uWa0IIaU99fT2srKzaXE+jiaihUChQUlICCwsLGBgYaLs6PaZ1/tdbt27xetQUfaFP+5Mxhvr6ejg5OakMNPAoOoJTQyAQYMCAAdquxlNjaWnJ+y+EPtGX/dnekVsruslACOEtCnCEEN6iAKfHRCIRoqOjab5YnqD9qYpuMhBCeIuO4AghvEUBjhDCWxTgCCG8RQGulyoqKoKBgQEuXbr0VMo5e/YsRo8eDSMjI8ybN++J3pOQ3oICnI5IT0+HgYEBampqeqR8iUQCb29vFBYWYseOHT3yHkS9qVOn4q233tJ2NTi9rT5PggJcD2hubtZ2FTqtoKAA06dPx4ABA9C3b98ulfE0280YQ0tLy1N7v95OFz9zTwUjT2zKlCls1apVbPXq1ax///5s6tSpLDs7mwUFBbE+ffowOzs79tprr7GKigpum59++olNmjSJWVlZsX79+rHnn3+e5efnc+sLCwsZAPb7779zvz/6WrJkSafK2b17N/Pz82MikYiNHDmSpaenK61/9LV9+3bGGGPp6ens//2//8eMjY2Zg4MDW7duHZPJZO22+9SpUwwAS05OZt7e3szExIRNmzaNlZeXs2PHjjEPDw9mYWHBFixYwBobG7my5HI527RpExOLxczExIR5enqyH374gVvfWu6xY8fY2LFjmZGRETt16lR37katWLJkicrfPz8/ny1btoz7WwwdOpTFx8erbDd37ly2ceNG5ujoyMRiMWOMsbNnzzIvLy8mEomYj48PO3jwIPc5atXeZ1NdfQoLC5/Wn6PbUYDrBlOmTGHm5ubsn//8J8vNzWWZmZnM1taWRUZGspycHHbx4kXm7+/Ppk2bxm2zb98+tn//fnb9+nX2+++/szlz5rDRo0czuVzOGFMOcC0tLWz//v0MAMvLy2OlpaWspqamU+UMGDCA7du3j129epW9/vrrzMLCglVWVrKWlhZWWlrKLC0tWXx8PCstLWVNTU3s9u3bzMzMjK1cuZLl5OSwgwcPMhsbGxYdHd1mu3Nzc7lANGHCBHbmzBl28eJF5u7uzqZMmcICAgLYxYsX2S+//ML69+/PPvjgA66sjRs3Mg8PD5acnMwKCgrY9u3bmUgk4gJxa7menp4sJSWF5efns6qqqp7etT2upqaG+fn5sbCwMFZaWspKS0vZgwcPWFRUFPvtt9/YjRs32LfffsvMzMzY3r17ue2WLFnCzM3N2aJFi9jly5fZ5cuXWW1tLevXrx977bXX2JUrV9ixY8fY0KFDlQLcvXv32v1sqqtPS0uLNv403YICXDeYMmUKGzNmDLe8YcMGFhAQoJTn1q1bXIBSp6KiggFg2dnZjDHlAMfYX1/we/futVuXtsp5NJjIZDI2YMAA9uGHH3JpVlZW3JEbY4ytX7+eDRs2jCkUCi4tISGBmZubc8Hz8XY/Ws8TJ05wabGxsQwAKygo4NL++7//mwUGBjLGGHvw4AEzMzNj586dUypr+fLlbMGCBUrlHjp0qN3266IpU6aw1atXt5tn1apV7MUXX+SWlyxZwuzt7ZlUKuXStm7dyvr378/u37/PpW3btk3pc6TJZ1OT+ugKugbXTXx8fLjf//jjD5w6dQrm5ubcy8PDA8DDa10AcP36dSxYsACDBg2CpaUlxGIxAKC4uLhT76tpOX5+ftzvhoaGGDduHHJyctosNycnB35+fkrDRU2aNAkNDQ24ffu22nY/ytPTk/vd3t4eZmZmGDRokFLa3bt3AQD5+floamqCv7+/0t9s165d3N+r1bhx49qsM58kJCTAx8cHtra2MDc3x1dffaWyT0ePHg1jY2NuOS8vD56enjAxMeHSxo8fr7SNJp9NPqHhkrpJnz59uN8bGhowZ84cfPjhhyr5HB0dAQBz5syBq6srtm3bBicnJygUCowaNarTF4u7q5yuerTdj3p0RFkDAwOVEWYNDAy42dgbGhoAAEePHoWzs7NSvsefq2zr/fhkz549iIiIwMcffww/Pz9YWFhg8+bNOH/+vFK+rvwtNPls8gkFuB4wduxY7N+/H2KxGIaGqn/iqqoq5OXlYdu2bZg8eTIA4MyZM+2W2fqfWi6Xd6mczMxMbmLglpYWZGVlITw8vM33Gz58OPbv3w/GGHcUd/bsWVhYWHT7WHkjRoyASCRCcXExpkyZ0q1l6wJjY2Ol/Xr27FlMnDgRK1eu5NI0OboaNmwYvv32W0ilUu4fw2+//aaUp6PPprr66DI6Re0Bq1atQnV1NRYsWIDffvsNBQUFOH78OEJDQyGXy2FtbY3+/fvjq6++Qn5+Pk6ePAmJRNJuma6urjAwMMCRI0dQUVGBhoaGTpWTkJCAgwcPIjc3F6tWrcK9e/ewbNmyNt9v5cqVuHXrFt58803k5ubixx9/RHR0NCQSSbsjqHaFhYUFIiIisGbNGuzcuRMFBQW4ePEiPv/8c+zcubNb36s3EovFOH/+PIqKilBZWYkhQ4bgwoULOH78OK5du4b33ntPJVCps3DhQigUCrzxxhvIycnB8ePHsWXLFgDg/kl19NlUV5/WI21dRAGuBzg5OeHs2bOQy+UICAjA6NGj8dZbb6Fv374QCAQQCATYs2cPsrKyMGrUKKxZswabN29ut0xnZ2fExMTg7bffhr29PcLDwztVzgcffIAPPvgAXl5eOHPmDA4fPgwbG5t23+/YsWP49ddf4eXlhb/97W9Yvnw53n333Sf627Rlw4YNeO+99xAbG4vhw4cjKCgIR48ehZubW4+8X28SEREBoVCIESNGwNbWFoGBgQgODkZISAh8fX1RVVWldDTXFktLS/znP//BpUuX4O3tjXfeeQdRUVEAwF2X6+izqa4+nb0u3JvQcEmE8Nh3332H0NBQ1NbWwtTUVNvVeeroGhwhPLJr1y4MGjQIzs7O+OOPP7Bu3Tq8/PLLehncAApwhPBKWVkZoqKiUFZWBkdHR7z00kv417/+pe1qaQ2dohJCeItuMhBCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhzRqqVLl8LAwIB7IN/e3h7+/v5ISkrq1CNCO3bs6PJIxE9i6dKlNIdFL0YBjmhdUFAQSktLUVRUhJ9++gnTpk3D6tWrMXv2bBqWnDwZrY5GR/Re69Dbj0tLS2MA2LZt2xhjjH388cds1KhRzMzMjA0YMICtWLGC1dfXM8b+Ggzz0VfryMO7du1iPj4+zNzcnNnb27MFCxaw8vJy7n2qq6vZwoULmY2NDTMxMWHu7u4sKSmJW19cXMxeeuklZmVlxaytrdl//dd/cUN4R0dHq7wvH4ZR5xM6giO90vTp0+Hl5YUDBw4AAAQCAT777DNcuXIFO3fuxMmTJ7F27VoAwMSJExEfHw9LS0uUlpaitLQUERERAACZTIYNGzbgjz/+wKFDh1BUVISlS5dy7/Pee+/h6tWr+Omnn5CTk4OtW7dygxDIZDIEBgbCwsICp0+fxtmzZ2Fubo6goCA0NzcjIiICL7/8MncEWlpaiokTJz7dPxRpn7YjLNFvbR3BMcZYSEgIGz58uNp1P/zwA+vfvz+3vH37dmZlZdXh+/32228MAHf0N2fOHBYaGqo27zfffKMybLtUKmWmpqbs+PHjHdafaB8dwZFeiz0y2OaJEycwY8YMODs7w8LCAosWLUJVVRWampraLSMrKwtz5szBwIEDYWFhwQ2o2ToE0IoVK7Bnzx54e3tj7dq1OHfuHLftH3/8gfz8fFhYWHDDe/fr1w8PHjzg5fDefEQBjvRaOTk5cHNzQ1FREWbPng1PT0/s378fWVlZSEhIAND+fKCNjY0IDAyEpaUlvvvuO/z22284ePCg0nazZs3CzZs3sWbNGpSUlGDGjBnc6W1DQwN8fHxw6dIlpde1a9ewcOHCHm496Q40mgjplU6ePIns7GysWbMGWVlZUCgU+Pjjj7lBGf/9738r5Vc3zHZubi6qqqrwwQcfwMXFBQBw4cIFlfeytbXFkiVLsGTJEkyePBn//Oc/sWXLFowdOxZ79+6FnZ0dLC0t1daTT8N78xEdwRGtk0qlKCsrw507d3Dx4kVs2rQJc+fOxezZs7F48WK4u7tDJpPh888/x40bN/DNN98gMTFRqQyxWIyGhgakpaWhsrISTU1NGDhwIIyNjbntDh8+jA0bNihtFxUVhR9//BH5+fm4cuUKjhw5guHDhwMAXn31VdjY2GDu3Lk4ffo0CgsLkZ6ejr///e/czGJisRh//vkn8vLyUFlZCZlM9nT+aEQz2r4ISPTbozOpGxoaMltbWzZz5kyWlJTEzb/KGGNxcXHM0dGRmZqassDAQLZr1y6VeWL/9re/sf79+yt1E/n++++ZWCxmIpGI+fn5scOHD6vMEzp8+HBmamrK+vXrx+bOnctu3LjBlVlaWsoWL17MbGxsmEgkYoMGDWJhYWGstraWMcbY3bt3mb+/PzM3N6duIr0QjQdHCOEtOkUlhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4Qghv/X/kNno9Fs419gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T00:26:59.341610Z", + "iopub.status.busy": "2024-02-29T00:26:59.341255Z", + "iopub.status.idle": "2024-02-29T00:26:59.632728Z", + "shell.execute_reply": "2024-02-29T00:26:59.631685Z" + }, + "papermill": { + "duration": 0.317667, + "end_time": "2024-02-29T00:26:59.634858", + "exception": false, + "start_time": "2024-02-29T00:26:59.317191", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNYklEQVR4nO3dd3hTZRvA4d9J0r2hdEELlFEom0IRUNkC4gAHiIhMcTCUigqILFFQUVFREJWhgAMZ+rEUCoJsgbLKLoWW0l3opiM53x9pQ0Nb2pS0Sdr3vq5ctifnnDyNycN7nvMOSZZlGUEQBAujMHUAgiAIFSGSlyAIFkkkL0EQLJJIXoIgWCSRvARBsEgieQmCYJFE8hIEwSKJ5CUIgkVSmTqAqqbRaLhx4wZOTk5IkmTqcARBuIssy6Snp+Pj44NCUXr7qsYlrxs3buDr62vqMARBKEN0dDT16tUr9fkal7ycnJwA7Rvj7Oxs4mgEQbhbWloavr6+uu9qaWpc8iq8VHR2dhbJSxDMWFllHZMW7OfPn0/Hjh1xcnLCw8ODgQMHcuHChTKPW7duHc2aNcPW1pZWrVqxdevWKohWEARzYtLktWfPHsaPH8+hQ4fYsWMHeXl5PPLII2RmZpZ6zIEDBxg6dChjxowhLCyMgQMHMnDgQM6cOVOFkQuCYGqSOU2Jk5iYiIeHB3v27OHhhx8ucZ8hQ4aQmZnJ5s2bddseeOAB2rZty9KlS8t8jbS0NFxcXEhNTRWXjYJghsr7HTWrmldqaioAtWrVKnWfgwcPEhISoretb9++bNq0qcT9c3JyyMnJ0f2elpZ2/4FWAVmWyc/PR61WmzoUQTAqpVKJSqW6765KZpO8NBoNb7zxBl27dqVly5al7hcXF4enp6feNk9PT+Li4krcf/78+cyZM8eosVa23NxcYmNjycrKMnUoglAp7O3t8fb2xtrausLnMJvkNX78eM6cOcO+ffuMet5p06bptdQKb8OWV26+BmtV1ZUGNRoNkZGRKJVKfHx8sLa2Fp1phWpDlmVyc3NJTEwkMjKSJk2a3LMj6r2YRfKaMGECmzdvZu/evffslAbg5eVFfHy83rb4+Hi8vLxK3N/GxgYbGxuDYwqLuskHW85Ry8GaZS92MPj4isrNzUWj0eDr64u9vX2Vva4gVBU7OzusrKy4du0aubm52NraVug8Jr3bKMsyEyZMYOPGjezatYuGDRuWeUznzp0JDQ3V27Zjxw46d+5s1NgcbFQcvXaTXecTSM7IKfsAI6vov0aCYAmM8fk26Tdk/PjxrF69mrVr1+Lk5ERcXBxxcXFkZ2fr9nnxxReZNm2a7vfXX3+d7du38+mnn3L+/Hlmz57N0aNHmTBhglFja+rpROt6LuRrZP44ccOo5xYE4f6ZNHktWbKE1NRUunfvjre3t+7x66+/6vaJiooiNjZW93uXLl1Yu3Yty5Yto02bNvz+++9s2rTpnkX+inq6vfYSdv3x60Y/tyAI98fkl40lPUaOHKnb559//mHlypV6xz377LNcuHCBnJwczpw5w6OPPlop8T3RxgcrpUT4jTTOxVpGF4vq5urVq0iSxIkTJ6rkPPv376dVq1ZYWVkxcODA+3rNmuqff/5BkiRu3bpVqa8jCiv34OZgTa9m2m4Z64+J1pc5qOwvRkhICG3btiUyMrLYP5qCeRHJqwzPBGkvHTediCFPrTFxNJYlNzfX1CEYLCIigp49e1KvXj1cXV0rdI6q/LsLOzMbmyX8vxPJqwzdAurg7mhNUkYuey4kmiQGWZbJys03ycOQ0WPdu3dnwoQJvPHGG7i7u9O3b1/OnDlD//79cXR0xNPTk+HDh5OUlKQ7Zvv27Tz44IO4urpSu3ZtHnvsMSIiIko8/9WrV+nRowcAbm5uSJKkKzGU9zznz5+nS5cu2Nra0rJlS/bs2aM7tyRJJCcnM3r0aCRJ0rW89uzZQ3BwMDY2Nnh7ezN16lS9hFHS313YQvzrr79o164ddnZ29OzZk4SEBLZt20bz5s1xdnbm+eef1+uMrNFomD9/Pg0bNsTOzk5X1y1UeN5t27YRFBSEjY1NufpGzps3Dw8PD5ycnBg7dixTp06lbdu2uudHjhzJwIED+eCDD/Dx8SEgIACAn376iQ4dOuDk5ISXlxfPP/88CQkJeufeunUrTZs2xc7Ojh49enD16tUy4zEGs+jnZbZunMBKncuTbevyw75I1h+/Tu9Az7KPM7LsPDWBM/+q8tcFODu3L/bW5f+YrFq1ildffZX9+/dz69YtevbsydixY/n888/Jzs7mnXfeYfDgwezatQuAzMxMQkJCaN26NRkZGcycOZNBgwZx4sSJYrfTfX19Wb9+PU8//TQXLlzA2dkZOzs7g87z1ltvsWjRIgIDA/nss894/PHHiYyMxNfXl9jYWAICApg7dy5DhgzBxcWFmJgYHn30UUaOHMmPP/7I+fPneemll7C1tWX27Nkl/t2A7ibT7NmzWbx4Mfb29gwePJjBgwdjY2PD2rVrycjIYNCgQXz11Ve88847gHZEyOrVq1m6dClNmjRh7969vPDCC9SpU4du3brpXm/q1KksXLgQf39/3Nzc7vn/ZM2aNXzwwQd88803dO3alV9++YVPP/20WNek0NBQnJ2d2bFjh25bXl4e77//PgEBASQkJBASEsLIkSN1M7lER0fz1FNPMX78eMaNG8fRo0d58803y/6gGIFIXqUJ3wTrRkLtxjwzaDs/7Itk57l4bmbm4uZQ8SEN1V2TJk34+OOPAe2/9u3atePDDz/UPb98+XJ8fX25ePEiTZs25emnn9Y7fvny5dSpU4ezZ88Wu4OsVCp14149PDz0LuvKe54JEybo9l2yZAnbt2/nhx9+4O2338bLywtJknBxcdF1ev7mm2/w9fVl8eLFSJJEs2bNuHHjBu+88w4zZ87UJcaifzfcSV7z5s2ja9euAIwZM4Zp06YRERGBv78/AM888wy7d+/mnXfeIScnhw8//JCdO3fq+i36+/uzb98+vv32W73kNXfuXPr06VP2/xDgq6++YsyYMYwaNQqAmTNn8vfff5ORkaG3n4ODA99//73ekJ3Ro0frfvb39+fLL7+kY8eOZGRk4OjoyJIlS2jUqBGffvopAAEBAZw+fZqPPvqoXLHdD5G8SuPfHRzqQPIlml/9iUDvDpyNTeN/p27wYucGVRqKnZWSs3P7VulrFn1tQwQFBel+PnnyJLt378bR0bHYfhERETRt2pRLly4xc+ZMDh8+TFJSEhqNtq4YFRVlUPeX8p6naGdmlUpFhw4dOHfuXKnnPXfuHJ07d9YbotW1a1cyMjK4fv06fn5+xf7uolq3bq372dPTE3t7e13iKtx25MgRAC5fvkxWVlaxpJSbm0u7du30tnXoUP5RHxcuXOC1117T2xYcHKxr/RZq1apVsbGGx44dY/bs2Zw8eZKbN2/qva+BgYGcO3eOTp066R1j7A7jpRHJqzR2rtBnLmx6BfZ8zIhOG3gnFn4/dr3Kk5ckSQZdupmSg4OD7ueMjAwef/zxEv8V9vb2BuDxxx+nfv36fPfdd/j4+KDRaGjZsqXBBWNjnaeiiv7dRVlZWel+liRJ7/fCbYUJobAltGXLFurWrau3391D3Ep7vftx9zkzMzPp27cvffv2Zc2aNdSpU4eoqCj69u1rFgV9UbC/lzbPgV9nyMtiYMI3qBQSp66ncjE+3dSRWYT27dsTHh5OgwYNaNy4sd7DwcGB5ORkLly4wIwZM+jVqxfNmzfn5s2b9zxnYcug6FRBhpzn0KFDup/z8/M5duwYzZs3L/X1mjdvzsGDB/VuXOzfvx8nJ6cyx+EaKjAwEBsbG6Kiooq9X/ezaExAQAD//fef3ra7fy/J+fPnSU5OZsGCBTz00EM0a9asWLG+efPmupZjoaLvcWUSyeteJAke/QQkBTYX/+Q1v2hA9Pkqr/Hjx5OSksLQoUP577//iIiI4K+//mLUqFGo1Wrc3NyoXbs2y5Yt4/Lly+zatavYXG13q1+/PpIksXnzZhITE8nIyDDoPF9//TUbN27k/PnzjB8/nps3b+rVde722muvER0dzcSJEzl//jx//PEHs2bNIiQkxOjjT52cnJgyZQqTJ09m1apVREREcPz4cb766itWrVpV4fNOnDiRH374gVWrVnHp0iXmzZvHqVOnypytxM/PD2tra7766iuuXLnCn3/+yfvvv6+3zyuvvMKlS5d46623uHDhAmvXrq2y/nEieZXFqxUEjwNgXOYSrMhnY1gM+aLPV5l8fHzYv38/arWaRx55hFatWvHGG2/g6uqKQqFAoVDwyy+/cOzYMVq2bMnkyZP55JNP7nnOunXrMmfOHKZOnYqnpycTJkww6DwLFixgwYIFtGnThn379vHnn3/i7u5+z9fbunUrR44coU2bNrzyyiuMGTOGGTNm3Nd7U5r333+f9957j/nz59O8eXP69evHli1byjVpQWmGDRvGtGnTmDJlCu3btycyMpKRI0eWOZtDnTp1WLlyJevWrSMwMJAFCxawcOFCvX38/PxYv349mzZtok2bNixdulTvBk1lMqtpoKtChaaBzr4FiztCZgJfSMP4PHsAK0Z1pEeAh9Hju337NpGRkTRs2LDCU4UIQln69OmDl5cXP/30k0le/16f8/J+R0XLqzwKi/fAq9IGvEkWl46CxcjKyuKzzz4jPDyc8+fPM2vWLHbu3MmIESNMHdp9EcmrvAqK99aabN61Ws3fZ+NJzcoj/XYeo1Yc4bej0aaOUKihWrRogaOjY4mPNWvWIEkSW7du5eGHHyYoKIj//e9/rF+/nt69e5s69PtiGfffzUFB8V7+9mEeUx7m59yT/O9UIFm5+ey+kMjuC4kM7lDxO0KCUFFbt24lLy+vxOc8PT2xs7Nj586dVRyVvrx8DTKgUkoojDStuUhehvBqhRQ8Dg4vZa5qJVOPPUDnJt66p2/nqbE1sFOnINyv+vXrmzqEMkUmZXI7X42/uwOOtlZlH1AO4rLRUN2nobF3p5EilqAbP3Mx/s4Qi8sJGfc4UBBqLt1dQSMuJiOSl6HsXFE8Mg+AiaqNnDobrntKJC9BKJlckL6MuQ6WSF4V0eY5UmoH4SDlMF21WrdZ9LwXhFJUQocskbwqQpJwGPQ5aiQeUx6mq+I0AJdEy0sQSlSYu4y5BKlIXhVkU68NR9yfAWCuaiVW5IvLRkEog7hsNBOO/d4jUXamkSKWMcqtXEvO5HaeuuwDhRI1aNCARYsW6X6XJIlNmzaZLB7BeO6M4xEFe7PQspEfR5tMBmCS1UY85WSuJGaaOKrqIzY2lv79+5s6DMEoCgr24rLRPEiSRP9hk8H3AezJ4V2r1VxKEEV7Y/Hy8io2j1VVq6wFLmqayhhALZLX/ZIkGLAQDQoeUx4m58Kuso8xlCxDbqZpHgaO209PT2fYsGE4ODjg7e3N559/Tvfu3XnjjTcM/rOLXjYWLpCxYcMGevTogb29PW3atOHgwYN6x+zbt4+HHnoIOzs7fH19mTRpEpmZd1rDZS0oUdEFLoR7K/wYGbPmJXrYG4NXK877DiEw+mcevvwR5A8BlRHnuc/Lgg99jHc+Q0y/Adbln7UzJCSE/fv38+eff+Lp6cnMmTM5fvy43ko19+Pdd99l4cKFNGnShHfffZehQ4dy+fJlVCoVERER9OvXj3nz5rF8+XISExOZMGECEyZMYMWKFUDZC0oUMmSBC6H8jHnZKJKXkaQ+8DaJUVvwyouGQ9/Ag2+YOqQql56ezqpVq1i7di29evUCYMWKFfj4GC/xTpkyhQEDBgAwZ84cWrRoweXLl2nWrBnz589n2LBhulZekyZN+PLLL+nWrRtLlizB1ta2zAUlChmywIVQtjvtd+NlL5G8jKSRrw/z857nM+ulyHs+Rmr1DLgYaZpgK3ttC8gUrOzLveuVK1fIy8sjODhYt83FxUW3BqAxFF3QonAe/ISEBJo1a8bJkyc5deoUa9as0e0jyzIajYbIyEiaN29e5oIShQxZ4EIoB9n4BXuRvIykjpMNO6178J9mFx3zLsJf78Lgik/dq0eSDLp0q87uXtAC0FvA4uWXX2bSpEnFjvPz8zNoQYnKWOCippJlWRTszZkkSTTxdGZm3ihkFHB2E0TsNnVYVcrf3x8rKyu9xR1SU1O5ePFilbx++/btOXv2bLHFKxo3boy1tXW5FpQQKpfopGqmmno6ck6uT5iXtuc9W9+CfNMvEVVVnJycGDFiBG+99Ra7d+8mPDycMWPGoFAoylzswRjeeecdDhw4wIQJEzhx4gSXLl3ijz/+YMKECUD5FpQQjK9oq0v08zJTjT2cAPjJ9gXdgrUc+sbEUVWtzz77jM6dO/PYY4/Ru3dvunbtSvPmzatkPv7WrVuzZ88eLl68yEMPPUS7du2YOXOm7oZBeRaUECqB3jWj8bKXWIDDiP69lMjwH47QqI4Dob1itQvWWjnAhP/ApW7ZJ6D6LcCRmZlJ3bp1+fTTTxkzZoypwxFMQK2RCb+RCkBLHxcUCkkswGFumhS0vK4mZ5HV/BnwfQDyMuHvd00cWdUJCwvj559/1q05OGzYMACefPJJE0cmmIpe+0hcNponT2cb6te2R62R2Xw6DgYsBEkB4RtrVPF+4cKFtGnTht69e5OZmcm///7LuXPnSl0komj/KqF6Ez3szZQkSTzX0Y+Ptp9n7eEoBnfoCh1fgiPfwra34ZX9xu15b4batWvHsWPHim3Pzs7mxIkTVR+QYHL6BXvRSdVsPduhHp/tuMCJ6FucvZFGYI/pEL4Bki7W2J73AHZ2djRu3NjUYQimoBvXaNw7zuKy0cjcHW14JNALgJ+PROktWMuejyE1plznqWH3UYRqrKSpvIzx+RbJqxI838kPgE1hMWTl5kPr58pdvC/sQZ6VlVXpcQpC1Si++Ebh57voiAlDicvGStDZvzb1a9tzLTmLdUevM6JLA23x/tuHtcX79iOgUY8Sj1Uqlbi6uup6ftvb21dJB09BqCy5eWrk/FxkSSI7O5usrCwSEhJwdXVFqaz4OqcieVUChULixc4NeH/zWT7Yco4ALyce8G9V7uK9l5f2slMMXRGqgzy1hoS0HBQSqLLsAHB1ddV9zitKdFKtJGqNzPg1x9keHoezrYr1r3ahibMaFneAzEToPafM4r1arS51GXdBsBSRSZmMXfUfrnZWrH+tK1ZWVvdscZX3OypaXpVEqZBY9Fxbhn1/mGPXbjJyxX9seK0Lnn3mwqZXtcX7Vs/es+e9Uqm8r2a1IJgDWZFLTLqaXFRGHTVi0oL93r17efzxx/Hx8SnXSjGFU/Te/YiLi6uagA1ka6Xk+xc74O/uQMytbEau+I/0gKdrZM97oebSFFzcKY1cuzVp8srMzKRNmzZ8/fXXBh134cIFYmNjdQ8PD49KivD+uTlYs2p0MO6O1pyLTeO1tSfI6/dxjex5L9RMak1B8lJUo+TVv39/5s2bx6BBgww6zsPDAy8vL91DoTDvHh++texZPrIjdlZK/r2UxNT9IHccq31y29s1atocoebJr47Jq6Latm2Lt7c3ffr0Yf/+/aYOp1xa13Plm2HtUSok1h+/zjcM0U6bU9jzXhCqKd1lY01OXt7e3ixdupT169ezfv16fH196d69O8ePHy/1mJycHNLS0vQeptKjmQfzBrYE4JO98Rxq9Lr2iT0fE3X1EtvPxJosNkGoLNXystFQAQEBvPzyywQFBdGlSxeWL19Oly5d+Pzzz0s9Zv78+bi4uOgevr6+VRhxcUOD/ZjUUzvG74WjDbnl3h7yMjm7ciKvrD7O/stJJo1PEIxNl7yqU8HeGIKDg7l8+XKpz0+bNo3U1FTdIzo6ugqjK9nkPk15un098jUSI+OHoEZBPw7SRXGGnefiTR2eIBhVara2r6JKKZKXnhMnTuiWwCqJjY0Nzs7Oeg9TkySJBU+34qEm7pzI8+XHfO36gHNVK9l/3kRLnAlCJVl/7DoAwQ1rGfW8Ju2kmpGRoddqioyM5MSJE9SqVQs/Pz+mTZtGTEwMP/74IwCLFi2iYcOGtGjRgtu3b/P999+za9cu/v77b1P9CRVmpVTwzbD2TFgbxv68cbx46xiNs2/Q49Z6opK74le7/OslCoK5ik7JYtcF7TC34Q/UN+q5TZq8jh49So8edwYoh4SEADBixAhWrlxJbGwsUVFRuudzc3N58803iYmJwd7entatW7Nz5069c1gSJ1srVo0uWKD1xPuw6VUmqTaw7dQo/Hp0Mm1wgmAEqw9fQ5bhoSbu+Ncx7oy59zW2MSMjQ7fgZyFzuCy7l6oa22gwjYbYRT3wTjvBEftuBL/9p6kjEoT7cjtPTef5odzMymPZ8CAeaVG+gdiVtgBHZGQkAwYMwMHBARcXF9zc3HBzc8PV1RU3NzdDTycUUijI6vMRalkiOGsPuRd3mToiQbgvW07FcjMrDx8XW3o2cQUjzwFh8GXjCy+8gCzLLF++HE9PTzHXlBH5t+zEbxv7M0SzlfzNU7CedKjaz3kvVF8/HroGwLAH6qP65wO4fhQGfAqegUY5v8HJ6+TJkxw7doyAgACjBCDcIUkS4QHjSTy7jzppEXB4CXR93dRhCYLBTl2/xcnoW1grFTzfMAt+/AY0+ZAabbTkZfBlY8eOHc2ir1R19UBgIxbkPa/95Z+Pyj3nvSCYkx8Paltdj7b0xG3Pu9rEFfAoNO1rtNcwuOX1/fff88orrxATE0PLli2LzUHdunVrowVXE3Vt7M4kHmKoZhcd8i5qp815dqWpwxKEcruZmcv/Tmr7K07yOgN79oLKFvrNN+rrGJy8EhMTiYiIYNSoUbptkiQhyzKSJKFWq40aYE3jYmdFO79azLw2ks02M1CEb4SgkeDf3dShCUK5rDsWTU6+hiAvFQ2Pf6jd+GAIuDUw6usYfNk4evRo2rVrx8GDB7ly5QqRkZF6/xXuX/cAD87KDdjt/IR2w9a3xLQ5gkXQaGRWH9L2zZxXaxtSeqw2aVVC7dbglte1a9f4888/xQKilahb0zp88tcFpt96gkNO+5GSLorivWAR9lxMJColiza2cTS7+pN2Y/+Pwcp40z8XMrjl1bNnT06ePGn0QIQ7Wvg4U8fJhvhcWy61fku7URTvBQvw06FrgMwipzVIlVCkL8rgltfjjz/O5MmTOX36NK1atSpWsH/iiSeMFlxNJUkSDzZ2Z2NYDNsU3Wnq2wmiD8PfM+DZFaUeJ8syu84n0NzbGR9XuyqMWBC04xh3X0jgMcUhGqYfq5QifVEGJ69XXnkFgLlz5xZ7ThTsjae5txMbw+BiQiY8uhCWdYPwDRA0otTi/Q/7Ipm35Rzdmta5M2ZSEKrI6kPXsJezmWu/FtRUSpG+KIMvGzUaTakPkbiMJ8BLO6brQnw6eLeGwjnvSyneRyVnsfDvCwBEJGZUWZyCANpxjL8ejWaiaiO11MmVVqQvyqDklZeXh0ql4syZM5UVj1AgwNMJ0C7YmZOvhh7vgr27ds77w0v09pVlmakbTnE7TztIPiEthxq2lrBgYptPxeKeHclY1Tbthkoq0hdlUPKysrLCz89PtLCqgKezDS52Vqg1MhEJmWDnCo+8r33yruL9b0ejORCRjI1K+78zV63hVpZYaVuoOj8diGSOahUq1NC0f6UV6Ysy+LLx3XffZfr06aSkpFRGPEIBSZJ0ra8L8QWLhrR+Dnw7FSxYOwOA+LTbzNtyDoApjwRQy0E7kDsu7XbVBy3USCejb+Eb+xddleHIKlvov6BKXtfggv3ixYu5fPkyPj4+1K9fHwcHB73n77WSj2CYpl6OHLmawoW4ghqWQqFXvJfbj+C9/Y6k386nTT0XRnVtwIawGFIyc4lPu01zbzOar0yotn7dd44ZVqsBkCq5SF+Uwclr4MCBlRCGUBJd0T6uyHJthcX7I8vI2BTC7sRZqBRWfPRMa1RKBZ7ONpyL1da9BKGypWTm0vDc13gpbpLj5IdNFXakNjh5zZo1qzLiEEpQeNl4Mf6uu4c93kVzZgNO6RGMUm7DtnsIzQoSnaeTtkgaLy4bhSqwY88eRkpbAbB+7JNKL9IXVeE57I8dO8a5c9paS4sWLWjXrp3RghK0CpNXzK1s0m7n4Wxb0CHYzpXfXF/iuaz5TLbaiCJouu4YT2cbQNS8hMonazQ0PjoHK0lNjEd36gb0q9LXN7hgn5CQQM+ePenYsSOTJk1i0qRJBAUF0atXLxITEysjxhrLxd4KL2ftv2SX4tN122NTs5l2pQVHNU2x4zY2oXdawx7OhS0vcdkoVK6M4+sI0pzmtmyF29OfVfnrG5y8Jk6cSHp6OuHh4aSkpJCSksKZM2dIS0tj0qRJlRFjjdbUq+COY9ydS8edZ+ORUfCz+ySQFNqe91f+AcCzIHklpIuWl1CJctKxCX0PgNVWT2Pv2ajKQzA4eW3fvp1vvvmG5s2b67YFBgby9ddfs23bNqMGJ0AzXfK6U7T/+6x2Ve2mbboU6Xn/NuTn6lpqouYlVKo9H2OdHc81jQcHPIeZJIQKDQ+6ezA2aDuw3r0MmnD/mur6emkvG1Oz8zgYkQygXUpK1/P+Ahxeoqt5JabnkK8W/z+ESpB4AQ59A8Ds/BH4edY2SRgVmhLn9ddf58aNO8vSx8TEMHnyZHr16mXU4ISiLa90ZFnmnwsJ5GtkGns40tDdQdvzvk/BIPl/PqK2JgmlQkIjQ1RKlukCF6onWYatU0CTzwm7zuzWtKNRHYeyj6sEBievxYsXk5aWRoMGDWjUqBGNGjWiYcOGpKWl8dVXX1VGjDVaYw9HFBLczMojMSNHd8n4SKDnnZ3aDNX1vFfueI/O/tp/CZftFTPbCkYWvhEitXPSfyRpp4I39krY5WVwVwlfX1+OHz/Ozp07OX/+PADNmzend+/eRg9OAFsrJQ1qO3AlKZMzMansuaC9o9unaPK6q+f9zL5P88hliXXHrvNKt0Y0cDfNv4xCNZOTDn9pu+Xkd3mDIzudABl/S2l5gXbcXZ8+fZg4cSITJ04UiauSFda9Vuy/SkZOPh5ONrSp56q/U5Fpc5oem0vvpq6oNTKLdl6s4miFamvPx1AwJ/21ZmNQa2TsrZW6m0RVrUKdVENDQwkNDSUhIaFYkX758uVGCUy4o6mXE9vD4/j3UhKgbXUpFCWsVN7jXTizAZIuMC94HzsvtuSPkzd4rUdjXQIUhAopUqSn30dcvqn93vvXcUCSSvgsVgGDW15z5szhkUceITQ0lKSkJG7evKn3EIyvsGhfSO+SsagixXuvsC94LkCJLMPPR6IqOUKhWitSpKdpfwjox5XETAD83U1T74IKtLyWLl3KypUrGT58eGXEI5SgaKvJ0UZF50b3uDXdZigcXwXRhxl3ezm/MIKEdNHbXrgPRYr0hdPdFM7W28hExXqoQMsrNzeXLl26VEYsQika1LbHumCiwe4BdbBRKUvfubB4Lynwj/+LzopwtpyKFTOrChWTkwF/vav9+cHJuulurhQkL1MV66ECyWvs2LGsXbu2MmIRSqFSKggsmJurbwuvsg8oUryfq1qJFflcTRZ9voQK2PsxpN/Qm5NelmUiCi8bTZi8DL5svH37NsuWLWPnzp20bt26WG/7zz6r+gGaNcFHT7cmLOomj7X2Lt8BPd4l5+TvNMmJYaRyO2FRQdpOrYJQXokX4ODX2p/7fQRW2uX0UjJzSc3WTjNuUTWvU6dO0bZtW4BiC3GY6q5DTRDg5USAlwF3DO1cUfV9H/4czxuq9Xx16RloX6/yAhSqlxKK9IWuJGlbXXVd7bCzvkcJo5IZnLx2795dGXEIlUDZ9nlu7v8et+QwHrj8OdALWZY5dCWFNr4u2FtXeDo3oborLNIrbYotHGsO9S6oYCdVwUIoFMiPLkQtS3TP+5fM86FsDIth6HeHmPVHuKmjE8xV0SL9QyFQq6He04X1LlPeaQSRvKq9Wo06sMmqv/aXrW+x55x2ybStp2O5nSeWsBNKUEKRvijR8hKqzDH/10iSnXFIi8A/QrvKS2aumn0FPfYFQaeUIn1R5tBBFUTyqhEC/f1YkD8UgLHqX/FEu+bm9vA4U4YlmAmNpqAPoCzD1rdKLNIXys3XcK1gqqVGHhbW8tq7dy/5+fnFtufn57N3716jBCUYV4cGbqxXP8QxTRMcpBxm22r76e04G0+emLCwRgv57QSdF4Ty39WUgiL9nhKL9IWiUrJMPiC7kMHJq0ePHiWulp2amkqPHj2MEpRgXE08nHC0sWZm3ijUskR/DtDP/gKp2XkcupJs6vAEE9p/OYn4tBzGfb+H21umajeWUKQvVFjvauhuugHZhQxOXrIslxh0cnJysdWzBfOgVEi09XMlXG7AT+o+AMyx0va8335GXDoK8DLrsc2OJ8OubolF+kLmcqcRDOjn9dRTTwHajqgjR47ExsZG95xarebUqVNizKMZ61C/Fv9eSuJz9bO86Hwcz6xrjFRuZ2P408x9siXKkqbYEWqERlIMY1XahWMnpQ6lzZ7rTOrVuMRGirncaQQDWl4uLi64uLggyzJOTk66311cXPDy8mLcuHGsXr3aoBffu3cvjz/+OD4+PkiSxKZNm8o85p9//qF9+/bY2NjQuHFjVq5cadBr1lQPN3UHILChH4qCaXMmW61HmRHL8SgxlVGNJcvMUa1EhZoIt4fYpWnP5zsvMmPTGdSa4oP5C3vXm2rq56LK3fJasWIFAA0aNGDKlClGuUTMzMykTZs2jB49Wteyu5fIyEgGDBjAK6+8wpo1awgNDWXs2LF4e3vTt2/f+46nOmvn58afE7ri42oH9sFwbCX214/wrtUatp0OomODWqYOUTCBXpoDPKgMR6O0odHwr5h7QWLWn+GsORxFUkYOXzzXDlurO0OA7kyFY/qWlySbyVwpkiSxceNGBg4cWOo+77zzDlu2bNEbU/ncc89x69Yttm/fXq7XSUtLw8XFhdTUVJydne83bMsVexL52+5IaJhoNZcvp08yeQFWqDoHI5IJrK3g9qL2eJJCQtBkPB6fDWg7ML/xywly1RqCG9TiuxEdcLGzIiUzl/bv7wDg7Ny+lTa8rLzfUYML9vHx8QwfPhwfHx9UKhVKpVLvUZkOHjxYbL78vn37cvDgwVKPycnJIS0tTe8hAN5tUAeNBmBSzrecjhYdVmuK3RcSGPrdIQ6vmoonKURp6pDU5lXd84+28mbV6GCcbFQcuZrC4KUHiUu9rat3+bjYmsW4WIMjGDlyJFFRUbz33nt4e3tX6b/WcXFxeHrqT4Hs6elJWloa2dnZ2NkV7w08f/585syZU1UhWhRV7xmkn/idJsSwe+cXMHqeqUMSqsDWU7E0kmLoeXMdoF04dopK/7vTuVFtfn25MyNXHOFCfDpPfbOfx9r4ANDIw/T1LqhA8tq3bx///vuvblocczdt2jRCQkJ0v6elpeHr62vCiMyInRuX27xNu+PT6RT1HXLqq0gudU0dlVCJNBqZ3ecTWFRQpP9XCmKXpj1TStg30MeZ9a92YcTyI1xJytStA+pvJvPCGXzZ6Ovra7Iphb28vIiPj9fbFh8fj7Ozc4mtLgAbGxucnZ31HsIdTR55ieNyE+y5Tfr/ppk6HKGSnY5JpVP2Xh5UhpOLFZ8qR99zf99a9vz+ahfa+LrqtpnDnUaoQPJatGgRU6dO5erVq5UQzr117tyZ0NBQvW07duygc+fOVR5LdeFoa83/6oagliWcL/+hncNJqLb2hl9lhpW2S9MGhyHESGVPK17LwZqfX+pE3xaeONqoeLCJe2WHWS4GXzYOGTKErKwsGjVqhL29fbFpoEsaOlSajIwMLl++rPs9MjKSEydOUKtWLfz8/Jg2bRoxMTH8+OOPALzyyissXryYt99+m9GjR7Nr1y5+++03tmzZYuifIRQR2P4hVkf1ZoRqB2yZAq/uB6VV2QcKJpWRk4+DtdKgurP3iS/xlrRF+k32T0OudmxrWaewt1bx7fAO5Ks1qJTmMZ+Dwclr0aJFRnvxo0eP6o2HLKxNjRgxgpUrVxIbG0tU1J01Bxs2bMiWLVuYPHkyX3zxBfXq1eP7778XfbzuU59AT3puGMwA+TDuSRfg0BLoOsnUYQn3cCYmlce+2sdT7ery2ZC25TomKfIUT2ZvAklbpM+VbJDlbINe11wSF5hRP6+qIvp5lWz4D4fxuvI7n1gtA2tHmPAfOPuYOiyhFBN/DuN/J28AcHXBgLIPkGXiFvfFK/kwO9XtGJv3Fu38XIlOySYpI4dtrz9Ec2/z+D5UWj8vgIiICGbMmMHQoUNJSEgAYNu2bYSHi6mFLVXfFl78rn6Y86rmkJsBf88wdUjCPdioDPzqhm/EK/kwObIVv7mPB7TTd1kyg5PXnj17aNWqFYcPH2bDhg1kZGg7rp08eZJZs2YZPUChajzSwhMkBW9mDkeWFHBmva54H5Wcxcr9keTmi7m/zIW1AcnryIUo8rdNB2CJ+nFatGgLQNHcZYmDKwxOXlOnTmXevHns2LEDa2tr3faePXty6NAhowYnVB0PJ1s61HcjXG7A+XrPajdumQLqPD7+6zyz/3eW7/69YtogBR3rctaeDl9J5vhP01FlxhKlqcMGu2do4VNwKSbL6Kcwy2Jw8jp9+jSDBg0qtt3Dw4OkJDHExJL1a6ld0Pbj3GfA3h0KivdXk7UzCfx2NNpkffwEfeW9bNy2ew9jlNrpbmbnj6Brc19dK8vS/08anLxcXV2JjY0ttj0sLIy6dUXvbEvWr6W2z88/UXmkPfSeduOej1Df0haGryVncSSy5K4w6bfzSEi/XSVxCuW7bIy9lUWfa59iJan5h/bs0rSnT6DnneRl4dnL4OT13HPP8c477xAXF4ckSWg0Gvbv38+UKVN48cUXKyNGoYrUdbWjdT0XZBk2S92hXjDkZvBq7grdPuuOXedIZAq/HIm6s3AD8PSSA/T45B/Sb+dVfeA1UNGWV2mt4aNbV9BVcYZcrPB/YTFfPNeWHgEeSGizl1yk7VW4zZIYnLw+/PBDmjVrhq+vLxkZGQQGBvLwww/TpUsXZswQd6gsXWHra1t4PAxYiCwpeEJ5kM4K7Z3kLadiGbniCFM3nGbdsWgAbmbmcjE+g8xcNQnpOSaLvSaxUd2ZwSVPXTx55WalEXxxIQBXm43Dr3ELnmxbV9uhtUjLy5JbXwYnL2tra7777jsiIiLYvHkzq1ev5vz58/z000+VPiWOUPn6tdAmr4MRyVxU+PNjXi8A3rdeRRN3G7Lz1GTlaher3X0+EYArSRm643PyxB3JqlD0svF2fvHFg69tnI0nKcTgQcOB7+o9V9jGsuTEBRXoYV/Iz88PPz8/Y8YimAH/Oo4EeDpxIT6dsauOciv/WQYoD9NYus4cz308n9RRt6+64NNfuCgDQE4JXyTB+IquOXA7T42zbZHhXIkXaHhpJQBHm7/Dk7b6s0AUDiey8NxVvuQVEhLC+++/j4ODg970MiX57LPPjBKYYDr9WnpxIT6dqJQswJGE4Gm4/zeNztHLeLpxJ85lOnI2Nk1X8yqcGhggR/QFqxJFE49ea1eWydgUgiNqdmna07n/C8WOvdPyKlLzsrySV/mSV1hYGHl5ebqfSyOmEa4e+rX04ovQSwB0bOBG8/79IHYj0vUjfOq6jnVt5/DW76fQFHz4r+i1vETyqgpFE897f5yhvZ8bA1p70yhhB44x+8iRrfjH/016lrAwbOHXNP12vkV3PC5X8tq9e3eJPwvVUzMvJ92l4+TeTZEUShiwEJZ1hzPr8XQaANhSWCfWa3nlicvGqlD0Tu8/FxL550Iiv+w7yx67qVih7Uk/4OGSp4qyK1hQI+aWYYOyzY35DBEXzIYkSawaHczG17rQpXHB3E3ebaDDGADanvkAFfloNDJ5ag1RyVm6Y0XLq2qUVK8anrcOq6w4ojR1WJL/BEH13Uo8tq2vK5N7N+WhJu54OdvSzs+VBrXNY3ZUQ5Sr5VWeZckKbdiwocLBCObDy8UWL5e7Ljl6vgvhG3FOj2Ck8i/Oyi8SlZJFfpFWgEheVePuJRUbSTGMLehJPyf/RdQKm1Knr1EpFbzeu0llh1jpytXyKrrArLOzM6GhoRw9elT3/LFjxwgNDcXFxaXSAhXMgJ0b9NEuZvKGaj3OeUl69S4QdxsrmyzLnIi+RWpWbtGtzFGtxEpSs1PdjlBNkN5ai9VVuVpehQvOgnbtxMGDB7N06VJdvy61Ws1rr70m5seqCdo8z81/v8Mt5QQvpH3HmcTmek9bcgHYEuy5mMjIFf/pbXtUcZgHleGoFdbMydGOcqkJycvgmtfy5cuZMmWKXodUpVJJSEgIy5cvN2pwghlSKDjbbhZqWeLB2/8g3TXnvbhsrFzbz8Tp/W7Pbd4rmJM+ue1rRMvapQFtrap/OdvgvzA/P5/z588X237+/Hk0GvHBrQmyardgtVq7+O+j1z9DRT5ONtpG/M3M3HsdKtwnlVK/O9JE1Ua8pRRwrU9q0Hjd9prQ8jK4h/2oUaMYM2YMERERBAcHA3D48GEWLFjAqFGjjB6gYH6UCvg0/1metDqCb34UI5V/cch9KGdi0vh27xUeaeFJUP1apg6zWlIp7rQ3ihbp6f8RVjZ37hjaieRV3MKFC/Hy8uLTTz/VTY3j7e3NW2+9xZtvvmn0AAXzI0kSaTiy0n4kb2R+wRuq9XzoPJAzMdrnP99xidVjO5k2yGrqzrAgmdmqVboife+A/lgX6bclLhtLOkCh4O233yYmJoZbt25x69YtYmJiePvtt8XA7BpCWdBF+/f8hzmuaYyjdJthad/pnr/70kYwHlVB8npUcZiHlGfIka2Yk68t0hcdrF101onq6r7Ss1iBumZSFCSvG2k5vJc3CjUKWiT/rZs2R6UQyauyKBWSXpF+ifpxXZG+aPJS1oD/BxWaVeL333/nt99+Iyoqitxc/QLt8ePHjRKYYL4Kyy4aGcJpyFab/jyes4U5qpU8mjtfry4jGJdSIemK9IU96QsVnde+JiQvgz9lX375JaNGjcLT05OwsDCCg4OpXbs2V65coX///pURo2BmWtV1oamnI+6O1jTzckLZawa3rd1oqohhpPIvlOKysVLE3Mrm6vkwvZ70OdxZBKdo8qoBucvw5PXNN9+wbNkyvvrqK6ytrXn77bfZsWMHkyZNIjU1tTJiFMyMk60Vf0/uxtEZfdj+xsM8GhzI6ebaqZLeUK2nljoZjUbmt6PRvLbmGNdvZpVxRqEsm8Ji6LoglMGJX2ElqTll/wCTx79OXVc7vniuLQCKIhmrJszwYnDyioqKokuXLgDY2dmRnp4OwPDhw/n555+NG51gMWIbPqUr3ne4+BmPfvkvb/9+iq2n49gUFmPq8CzOkcgUxv14lJhb2WTk5PPO+lN6Rfr/mr9Dy7ou7J/akyfbFl/4RrS8SuDl5UVKinYFGT8/P91ajZGRkWJZrBrMxkqlLd7LEk8qD+CWcGcNz5hbYlUhQw3+9iB/n41n8q8nSL+dhzI/ixlFivS3He89i7GoeZWgZ8+e/Pnnn4C2w+rkyZPp06cPQ4YMKXE9R6FmsFEpCJcbsqag5/2SWr/wZk9/AGJTLXveKFO6kphBbOptJqo24lOkSF/WorM14bLR4LuNy5Yt0w0DGj9+PLVr1+bAgQM88cQTvPzyy0YPULAMhbfpF+Y/yzDHY7hmRPBo9iY+pTWxouVVYTl5Gjb8tZtZdxXpy1q3UVkDkpdBLa/8/HzmzZtHXNydwaHPPfccX375JRMnTsTa2voeRwvVWWGnyDQcudlVuwRewzNf4cFNblj4jJ2mlJ6TR9+oT/Wmu4GyV8wWl413UalUfPzxx+Tn51dWPIKFKvplym35HNTriCIvk3et1pCeky8Woy2nfLUGdZGZBguL9HmSta4nPZS9YnYNaHgZXvPq1asXe/bsqYxYBAtWNHmpVEp4dCFICp5UHuABxVliU8WlY1k0Gpn+X/xLq9l/AdrpbgqL9Neaj9P1pIeyk1dNGB5kcM2rf//+TJ06ldOnTxMUFISDg/7c10888UQpRwrVWdEph62VCvBpCx1Gw3/fM1e1gujEoTT1dDJdgBYgKSOHSwl3FjMpWqSPbfkKT8nJbCjodlJawf6tvgH8fCSK13tZ/jTPZTE4eb322mtAyeszSpKEWi2mAa6JihaIdYms5wwyjq+jKTHc+G8ZtJxrougsQ9F/ABpJMYwpUqR/ydaeYQ843ElepbS8xvdozPgejSs/WDNg8GWjRqMp9SESV81lb3PnMkU3MNvOTdfzvlPUMtbtPkK/RXu5lpxZ0ilqvDv9JLXT3VhLakILivQ2KgVORVbFLuuysSYQ74BgFO6ONswY0Jz3n2yhN4unW9eRHNc0xk7OxnrXLM7HpdPtk38sfs3AylBYpy9puhsblRIn2zsXSmXdbawJyn3ZmJ2dTWhoKI899hgA06ZNIycnR/e8Uqnk/fffx9a2+Aq9Qs0w9iH/YtuaerrwnDSWn+VpPKk8wM/qnhzSBPLTwWtM7d/MBFGaL1mW9Yr0S9SPE1VkupuiLa+7lz6ricqdvletWsW3336r+33x4sUcOHCAsLAwwsLCWL16NUuWLKmUIAXLpVBI2NYP0vW8n6tagYp8UrPFXPd308j6Rfp1Ns/onrNRKXCwvtOiFas0GZC81qxZw7hx4/S2rV27lt27d7N7924++eQTfvvtN6MHKFi+DvXdWJj/LMmyE00VMYxQ/sXNTNHv626K5It6RfohXZrqnrOxUugN+alf277K4zM35U5ely9fplWrVrrfbW1tURSZdC44OJizZ88aNzqhWuhQ3400HFmQPxSAyar1HA8/Z+KoTKvYJAayjPPud/WK9F0bu+uetimYYn33lO6sf7UL9dxE8ip38rp165ZejSsxMZEGDRroftdoNHrPC0KhNr6uKBUSv6vvzHn/rtUa1h+7TvP3tnMgIsnUIVapfLWGxxfv45Wfjum2ZZ3cgG30Xr0ifQsf7RTrjjYqHAru5jZ0dyCovlvVB22Gyp286tWrx5kzZ0p9/tSpU9SrV88oQQnVi4ONiubeTsgoeC9vFJqCaXPWrf+Z7Dw10zecNnWIVSr8RhpnYtLYHl4wRjgng7wtUwFYWlCkXz2mE7ZWSo7N6M2et7rr9QETtMr9jjz66KPMnDmT27eLD/PIzs5mzpw5DBgwoEJBfP311zRo0ABbW1s6derEkSNHSt135cqVSJKk9xB3OM1fkJ+2tRAuN9QtWFtYvK9pig6aVmtk1Hs+xiUvgWhNHb4pmJP+wSbaS8bajjbUdrQxSZzmrtzJa/r06aSkpBAQEMAnn3zCH3/8wR9//MHHH39MQEAAN2/eZPr06QYH8OuvvxISEsKsWbM4fvw4bdq0oW/fviQkJJR6jLOzM7GxsbrHtWvXDH5doWp1D/DQ/Xx38T4j507n5gMRSfT+bA9HIlNMEWaVsCrSisqPPw8HvwZg9l1z0gv3Vu7k5enpyYEDB2jevDlTp05l0KBBDBo0iGnTphEYGMi+ffvw9PQs+0R3+eyzz3jppZcYNWoUgYGBLF26FHt7e5YvX17qMZIk4eXlpXtU5HWFqtU9oA6vdm8EUKx4r8iIIzVLe/fx+e8Oczkhg8HfHuSF7w+TmVP9WmZ31rWUUWx/B6WcryvSC+Vn0IV0w4YN2b59O4mJiRw6dIhDhw6RmJjI9u3b8fcv3kGxLLm5uRw7dozevXvfCUihoHfv3hw8eLDU4zIyMqhfvz6+vr48+eSThIeHl7pvTk4OaWlpeg+h6kmSpLt0BPhd/TAn5CY4SreZbrWG19Ye42J8ut4x+y4n8UXopaoO1ehkWda7u1g4fKq/4ghW1/boivRlzY4q6KvQu1WrVi2Cg4MJDg6mVq1aFX7xpKQk1Gp1sZaTp6en3oSHRQUEBLB8+XL++OMPVq9ejUajoUuXLly/fr3E/efPn4+Li4vu4evrW+F4hftTdDyejIJ3c0eiQWKg8gDqK/8y6Ov9xY65O6FZGlmWef67wwz/4YheAtMuHPsTcKdI36GBuItoCItL9Z07d+bFF1+kbdu2dOvWjQ0bNlCnTh293v9FTZs2jdTUVN0jOjq6iiMWCgXVd8OuyLjHcLkhR2oPBLTF+5zc4l1tFBY+q15Ceg4HrySz73ISkUmZ/LAvkpTMXCaoNuEjpeiK9B5ONng4icK8ISq0YraxuLu7o1QqiY+P19seHx+Pl5dXuc5hZWVFu3btuHz5conP29jYYGMjPhTmwMFGRdjMPuy7lMSczeGsHfsAmw560CRpp654/4Na/461pc9mXHRW1FdWH+NifAYPu6XwvXILcGdOem8XWzycxV1zQ5i05WVtbU1QUBChoaG6bRqNhtDQUDp37lyuc6jVak6fPo23t3dlhSkYka2Vkt6Bnvz7dk98a9njV9dHr3jvwc27jrDs7KUpcql4MT4DkHkpY6muJ/1OTXsAXOyt8asles0bwuSXjSEhIXz33XesWrWKc+fO8eqrr5KZmcmoUaMAePHFF5k2bZpu/7lz5/L3339z5coVjh8/zgsvvMC1a9cYO3asqf4E4T481toHm6AXdD3vp1ut0Xvewq8auXsUUH/Fkbumu9H+gRqNzDNB9XioiTtTHmla/ERCMSa9bAQYMmQIiYmJzJw5k7i4ONq2bcv27dt1RfyoqCi9MZQ3b97kpZdeIi4uDjc3N4KCgjhw4ACBgYGm+hOE+6BUSMx7qg0D/hvF/6xnMFB5gF8Kps0BS293wZmYVN3PJRXpC+WpNdhaKflpTKcqj9FSSXINW+Y6LS0NFxcXUlNTcXZ2NnU4QoH9l5OIWPkKL6p2cEFTjwG5H5KPin4tvFg63PL6P32+4yLhN1LZee5OZ+u3Vb/wmupPojV16J37CfU9axVcSmoHr//+ahdThWtWyvsdNflloyAAdG3szv76r5AsOxGguM4IpXYFHYWFfkK/CL2kl7j8pRuMvatI/1T7O2OB88Tsggaz0I+GUB25u3sWK96rq8WXWn9O+sIifdFpndUaMbmgoUTyEsxGPTd7vWlzplutIS7N8qZZursS019xhIeVp4sV6Yv2qG9dz7UKI6weRPISzEaAl6PetDkDlQfwvXVnzitZllnyTwR/h5c8+sJc5BSZovleRXpJktgZ8jCv92rCNDGfv8FE8hLMRvemHjwTVE9v2pyJOd+CWjtoOyz6Fh9tP8+4IpP4maOiyevunvRFSUBjDycm92mqt7iGUD4ieQlmQ6GQWPhsG+DOtDkBiuvE/LUIQG+GCY0Z18Jy8rVT/JRUpC/K0vuwmZpIXoLZ+e7FDnrT5jgf/pSkG1dxLtI6Sb9tXlPl/HjwKm+tO4lGI5Odq6a0In0bX1fdMTliBaD7IpKXYHYa1XEAtNPmhGka4yRlc+2XN4vMgwU3s8xr6bSZf4Sz7th1/r2cRHxaTqlF+gGt7ozZVYqm130RyUswO4XTHssomFFQvA9K28ncr77T7XMr2zyXTsvJU5OQnFxqkd7BRsWm8V0Z+2BDnmjrY6owqwWRvASz42JnxfY3HgLumvPe6s6c9+bU8spT37n8s7VS4nPq61KL9H617Gnr68qMxwKxLTI9kGA4kbwEs9TMy5k6BfNbFS3eF/a8v2VGySuryBz8btlXaR2tbXXdXaQf0MqbB4usxSjcH5G8BLNla6X9eBYt3r+h2oAHN7mVZT6XjZm5hTcPZLz2z0RVMCd9YZG+ZV1nNk98kK+Htddb9Vq4PyJ5CWbri+fa6X4uWryfbrWGm2aUvAoTaX/FEeokHChWpPd3d6RlXRcTRlg9ieQlmK32fm5Ezn+UJcPa6xXvByoP4Bp/2NTh6Xy49Rx29+hJ72InOqBWBpG8BLMmSRL9W3kjSdri/Rp1LwC6XlzAb4cjTByd1r7LSUy8qyd90XGL4kqxcojkJViEHZO7AbAwf7CueH/hz8+ITyu+gntVOHQlmQ+3niMnX63Xk/5X9/H8/dYjbH39Qd2++WY8GsCSieQlWITGHo70CKhDql7xfj1jF/+P9Nt5/BUex6UqXCbtuWWHWLb3Ct/vvaLrSb9L3ZazTl2pX9uBxh5Oun3z1aInfWUw+TTQglBe9Ws7AIn8rn6Y55W7aKe4zJjs5bSafWfhiqsLBpR+gkpgd3mLrif97PwRtLS+03fL0UZFRk4+DzetU6Ux1RSi5SVYjBFdGgAUK953ks6ZJB47bvNY7JfAnSJ90fGXO0IeZtnwIAa0EitbVQaRvASL0dDdgbD3+vDruAf0ivdFe95XpYmqTXjIyboifUN3B0L63Fn5x9vFjkdaeIm+XZVEJC/Borg5WNPJvzabxnfVK96PUP5dpXGUNN3N7indxcKxVUgkL8EitfV1ZelLvfko/zlAW7yvw022nY6t9NfOycvXK9IX9qQXqpZIXoLFquNkzTp1tyI979fywdbKrX/dysolZPb7ekV6kGhZVyyjV9VE8hIsVj03e73i/SDlfureOk5yhvEX7TgTk8rrv4TRee7/mK4q7En/GFGyJ18815Zfx3U2+msK9yaSl2CxCqeUubt4fy0p9V6HVcjQZYf448QNJqg2UVfSFumX5D/BqtHBPNm2Lg42otdRVRPJS6gWihbv0/75GsCoaz6m5+TjL93gpSJF+tvY4O/uYLTXEAwjkpdg0T5+ujV2VkpScdQV74Miv2X4F3/QavZfrDsafd+v8cXOSxSdk76wSN/OzxVvF3F30VRE8hIs2uCOvpyZ0xdAr3j/VNK3ZOWqeev3U/f9Gp/vvEg/xX96Rfq3+jZj42tdUSnFV8hUxDsvWDylQuKp9nWxtbLivbyRuuK9sXre6093oy3S56vFYGtTE8lLqBY+G9yWU7Mf4bzUSFe8n2O1EhX5yHLFE01uvqZYkR7Axkp8dUxN/B8Qqg0rpYJtrz+kK943U0QzQvk3wR+GEhZ10+DzybLMhh27ixfp6zgw/IH6xg5fMJBIXkK10sTTiZMLhuj1vCc9jkm/hBl8rp1n4/E5MEuvSP96rybserO76BphBkTyEqqlu3veX7+ZbfA51q9Zolekv7rgMSYXGXgtmJZIXkK1tCOku17xPhjDivf52enFivSCeRHJS6iWGtVx5Izsr1e8T8vMKvfxO759S69I37qeWP3H3IjkJVRLkiTxz5TuLMwfTIrsSDNFNF98+FaZdx7P3kij57Tv6HXzNwDm5g+nfzt//pzw4D2PE6qeSF5CtdXA3YGTC4YQ23EqoC3eT/9p5z2PGbH8sF5P+h2aIGY/0aIqwhUMJJKXUO0F9HtVV7wPvvQ5mnuMeQzK2qdXpLdSKsS6i2ZKJC+h2lOpVHrF+2U//Vji5WPkjYRiRfr1r3ap6nCFchLJS6gRZr88TFe87x7xCXP/OKn3fGZOPlu/eVNXpF8pDeL8+/1oXc/VBNEK5SGSl1AjdGhQi6Wq53XFe8V/3+k9P/mbdbqe9HPzh7PlzUd084UJ5kkkL6HG2BgyQG/B2qyU6wBo1BpeSPlaV6R/c+Ib+LjamTJUoRzMInl9/fXXNGjQAFtbWzp16sSRI0fuuf+6deto1qwZtra2tGrViq1bt1ZRpIIl83C2ZchLU3XF+78+H4csyxzaurKgSK/itzoTaOYt+nRZApMnr19//ZWQkBBmzZrF8ePHadOmDX379iUhIaHE/Q8cOMDQoUMZM2YMYWFhDBw4kIEDB3LmzJkqjlywREEN3IntOk9XvN/420rqH50HaBeO/XrCMyaOUCgvSb6f+UKMoFOnTnTs2JHFixcDoNFo8PX1ZeLEiUydOrXY/kOGDCEzM5PNmzfrtj3wwAO0bduWpUuXlvl6aWlpuLi4kJqairOzWPGlJtJoZNbMfIbhqp3kykqsJTXRmjpkvbSfAF8xDMjUyvsdNWnLKzc3l2PHjtG7d2/dNoVCQe/evTl48GCJxxw8eFBvf4C+ffuWun9OTg5paWl6D6FmUygkgkZ+RorsiLWkBuBc22kicVkYkyavpKQk1Go1np76HxpPT0/i4uJKPCYuLs6g/efPn4+Li4vu4evra5zgBYsW2Lg+Ee2mAXDCvguPDBpt4ogEQ5m85lXZpk2bRmpqqu4RHX3/CzII1UPHgRNIHrGHNpM3gCSZOhzBQCadUc3d3R2lUkl8fLze9vj4eLy8vEo8xsvLy6D9bWxssLGxMU7AQrVTu2FbU4cgVJBJW17W1tYEBQURGhqq26bRaAgNDaVz55JXIO7cubPe/gA7duwodX9BEKonk89lGxISwogRI+jQoQPBwcEsWrSIzMxMRo0aBcCLL75I3bp1mT9/PgCvv/463bp149NPP2XAgAH88ssvHD16lGXLlpnyzxAEoYqZPHkNGTKExMREZs6cSVxcHG3btmX79u26onxUVBQKxZ0GYpcuXVi7di0zZsxg+vTpNGnShE2bNtGyZUtT/QmCIJiAyft5VTXRz0sQzJtF9PMSBEGoKJG8BEGwSCJ5CYJgkUxesK9qhSU+MUxIEMxT4XezrHJ8jUte6enpAGKYkCCYufT0dFxcSp+eqMbdbdRoNNy4cQMnJyekajAkJC0tDV9fX6Kjo8Xd0woQ79/9qYz3T5Zl0tPT8fHx0esmdbca1/JSKBTUq1fP1GEYnbOzs/jy3Qfx/t0fY79/92pxFRIFe0EQLJJIXoIgWCSRvCycjY0Ns2bNEjNnVJB4/+6PKd+/GlewFwShehAtL0EQLJJIXoIgWCSRvARBsEgieQmCYJFE8rIwKSkpDBs2DGdnZ1xdXRkzZgwZGRn3PGbZsmV0794dZ2dnJEni1q1bVROsmRArst8fQ96/8PBwnn76aRo0aIAkSSxatKjS4hLJy8IMGzaM8PBwduzYwebNm9m7dy/jxo275zFZWVn069eP6dOnV1GU5kOsyH5/DH3/srKy8Pf3Z8GCBaUuimM0smAxzp49KwPyf//9p9u2bds2WZIkOSYmpszjd+/eLQPyzZs3KzFK8xIcHCyPHz9e97tarZZ9fHzk+fPnl7j/4MGD5QEDBuht69Spk/zyyy9XapzmytD3r6j69evLn3/+eaXFJlpeFuTgwYO4urrSoUMH3bbevXujUCg4fPiwCSMzT1WxInt1VpH3ryqJ5GVB4uLi8PDw0NumUqmoVatWqSuG12RVsSJ7dVaR968qieRlBqZOnYokSfd8nD9/3tRhCoJZqXFT4pijN998k5EjR95zH39/f7y8vIoVSvPz80lJSan84qgFqooV2auzirx/VUm0vMxAnTp1aNas2T0f1tbWdO7cmVu3bnHs2DHdsbt27UKj0dCpUycT/gXmSazIfn8q8v5VqUq7FSBUin79+snt2rWTDx8+LO/bt09u0qSJPHToUN3z169flwMCAuTDhw/rtsXGxsphYWHyd999JwPy3r175bCwMDk5OdkUf0KV+uWXX2QbGxt55cqV8tmzZ+Vx48bJrq6uclxcnCzLsjx8+HB56tSpuv33798vq1QqeeHChfK5c+fkWbNmyVZWVvLp06dN9SeYlKHvX05OjhwWFiaHhYXJ3t7e8pQpU+SwsDD50qVLRo9NJC8Lk5ycLA8dOlR2dHSUnZ2d5VGjRsnp6em65yMjI2VA3r17t27brFmzZKDYY8WKFVX/B5jAV199Jfv5+cnW1tZycHCwfOjQId1z3bp1k0eMGKG3/2+//SY3bdpUtra2llu0aCFv2bKliiM2L4a8f4Wfv7sf3bp1M3pcYkocQRAskqh5CYJgkUTyEgTBIonkJQiCRRLJSxAEiySSlyAIFkkkL0EQLJJIXoIgWCSRvARBsEgieQlmYeTIkSXOptGvXz9ThyaYKTGrhGA2+vXrx4oVK/S2lbYSc15eHlZWVnrbcnNzsba2Nvh1K3qcYFqi5SWYDRsbG7y8vPQebm5uAEiSxJIlS3jiiSdwcHDggw8+YPbs2bRt25bvv/+ehg0bYmtrC0BUVBRPPvkkjo6OODs7M3jwYL1pXUo7TrAsInkJFmP27NkMGjSI06dPM3r0aAAuX77M+vXr2bBhAydOnECj0fDkk0+SkpLCnj172LFjB1euXGHIkCF657r7OMHyiMtGwWxs3rwZR0dHvW3Tp0/XrXr0/PPPM2rUKL3nc3Nz+fHHH6lTpw6gnXvr9OnTREZG4uvrC8CPP/5IixYt+O+//+jYsWOJxwmWRyQvwWz06NGDJUuW6G2rVauW7ueiC48Uql+/vl4COnfuHL6+vrrEBRAYGIirqyvnzp3TJa+7jxMsj0hegtlwcHCgcePG93y+PNvK+1qCZRM1L6Faad68OdHR0URHR+u2nT17llu3bhEYGGjCyARjEy0vwWzk5OQUW1JLpVLh7u5e7nP07t2bVq1aMWzYMBYtWkR+fj6vvfYa3bp1K/GyU7BcouUlmI3t27fj7e2t93jwwQcNOockSfzxxx+4ubnx8MMP07t3b/z9/fn1118rKWrBVMQ00IIgWCTR8hIEwSKJ5CUIgkUSyUsQBIskkpcgCBZJJC9BECySSF6CIFgkkbwEQbBIInkJgmCRRPISBMEiieQlCIJFEslLEASLJJKXIAgW6f8giFzV35pLXQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.025323, + "end_time": "2024-02-29T00:26:59.683588", + "exception": false, + "start_time": "2024-02-29T00:26:59.658265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 6043.402197, + "end_time": "2024-02-29T00:27:02.431753", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/realtabformer/0/mlu-eval.ipynb", + "output_path": "eval/treatment/realtabformer/0/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "path": "eval/treatment/realtabformer/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "realtabformer" + }, + "start_time": "2024-02-28T22:46:19.029556", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/realtabformer/model.pt b/treatment/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..073c9ee0b69c351664ca8ad04d2149abc2677d21 --- /dev/null +++ b/treatment/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2575e1d5e72b45904d2984f2ebd0d340d2dcad8b7d40a0cab678e9c61c484c17 +size 78481207 diff --git a/treatment/realtabformer/params.json b/treatment/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..3c694d43b2f23956ddc0a5e9ec37d0428cd2b9a0 --- /dev/null +++ b/treatment/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "realtabformer", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/eval.csv b/treatment/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..f07b6253df7383fa958fc52b29d51b35c5c692fd --- /dev/null +++ b/treatment/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.0,0.001862262442891467,0.0014934718466911203,4.408010721206665,0.027836401015520096,0.4924542009830475,0.038891058415174484,7.311713034141576e-06,2.313991069793701,0.027990560978651047,0.05307392030954361,0.038645464926958084,0.07223384827375412,0.0061825597658753395,6.722001791000366 diff --git a/treatment/tab_ddpm_concat/history.csv b/treatment/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..a29f41b117bb3994671dc481b872a8cb8e800114 --- /dev/null +++ b/treatment/tab_ddpm_concat/history.csv @@ -0,0 +1,31 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.11455535657005385,15.63152334621248,0.03261089399050121,0.0,0.0,0.0,0.0,0.0,0.11455535657005385,320,80,99.41325306892395,1.2426656633615494,0.31066641584038734,0.09075278166747011,0.007890956761548296,4.913448315569985,9.587779401849516e-05,0.0,0.0,0.0,0.0,0.0,0.007890956761548296,80,20,19.218095064163208,0.9609047532081604,0.2402261883020401,0.04594327691011131 +1,0.0068825013451714765,0.5873005052374232,8.441511653969323e-05,0.0,0.0,0.0,0.0,0.0,0.0068825013451714765,320,80,98.16534066200256,1.227066758275032,0.306766689568758,0.17083694040193223,0.010920317904083276,5.493061433892126,0.00042163531188073035,0.0,0.0,0.0,0.0,0.0,0.010920317904083276,80,20,18.999029397964478,0.9499514698982239,0.23748786747455597,0.0285534585127607 +2,0.007368110892275581,0.6256727599678242,0.00019790053640412092,0.0,0.0,0.0,0.0,0.0,0.007368110892275581,320,80,98.2070198059082,1.2275877475738526,0.30689693689346315,0.18491398493060843,0.009714379241086136,6.2370832271572,0.00026508491033682134,0.0,0.0,0.0,0.0,0.0,0.009714379241086136,80,20,18.896817922592163,0.9448408961296082,0.23621022403240205,0.02282829804462381 +3,0.007530722331830475,0.4640916418485176,0.00011972417069327918,0.0,0.0,0.0,0.0,0.0,0.007530722331830475,320,80,98.34642100334167,1.229330262541771,0.30733256563544276,0.1856370047375094,0.006328437828778987,4.5384834828913885,6.255692790020362e-05,0.0,0.0,0.0,0.0,0.0,0.006328437828778987,80,20,19.502550840377808,0.9751275420188904,0.2437818855047226,0.05491759981960058 +4,0.006046878009510692,0.3354074442230967,8.722203440768646e-05,0.0,0.0,0.0,0.0,0.0,0.006046878009510692,320,80,98.46714925765991,1.2308393657207488,0.3077098414301872,0.1968570870347321,0.003924696132889949,5.36268491241317,4.107249934470758e-06,0.0,0.0,0.0,0.0,0.0,0.003924696132889949,80,20,18.971222400665283,0.9485611200332642,0.23714028000831605,0.043903833779040724 +5,0.005646668715053238,0.22879745735647247,0.00010420996314914471,0.0,0.0,0.0,0.0,0.0,0.005646668715053238,320,80,98.24562430381775,1.2280703037977219,0.30701757594943047,0.17895883410237728,0.005248893459065584,6.201540100930288,4.5616924159114224e-05,0.0,0.0,0.0,0.0,0.0,0.005248893459065584,80,20,19.050581216812134,0.9525290608406067,0.23813226521015168,0.04073070023441687 +6,0.005301459577640344,0.8160801359513838,6.89963721392106e-05,0.0,0.0,0.0,0.0,0.0,0.005301459577640344,320,80,98.10252213478088,1.226281526684761,0.3065703816711903,0.1658085669245338,0.003359271524823271,4.790561649674601,3.5786874698511918e-06,0.0,0.0,0.0,0.0,0.0,0.003359271524823271,80,20,18.99564027786255,0.9497820138931274,0.23744550347328186,0.03619469592813403 +7,0.005624368396092904,0.35675310027072554,0.00011016933555625902,0.0,0.0,0.0,0.0,0.0,0.005624368396092904,320,80,98.15186715126038,1.2268983393907547,0.30672458484768866,0.18299917249241843,0.0058501261519268155,5.208834768453289,7.912151082312135e-05,0.0,0.0,0.0,0.0,0.0,0.0058501261519268155,80,20,18.967280626296997,0.9483640313148498,0.23709100782871245,0.03817387481685728 +8,0.005031787550251465,0.25296989842264284,6.438524839464101e-05,0.0,0.0,0.0,0.0,0.0,0.005031787550251465,320,80,98.21737504005432,1.227717188000679,0.30692929700016974,0.1820793646154925,0.008046689907496329,3.4085470171728955,0.0001837004517647312,0.0,0.0,0.0,0.0,0.0,0.008046689907496329,80,20,19.01164937019348,0.9505824685096741,0.23764561712741852,0.03427380793727934 +9,0.0050649845143198036,0.41440914815246116,6.793519232238903e-05,0.0,0.0,0.0,0.0,0.0,0.0050649845143198036,320,80,98.00610780715942,1.2250763475894928,0.3062690868973732,0.18072777504567056,0.004020300185220549,3.4647489232461566,1.5592471368332078e-05,0.0,0.0,0.0,0.0,0.0,0.004020300185220549,80,20,18.98149037361145,0.9490745186805725,0.23726862967014312,0.03623254182748496 +10,0.00501764012269632,0.13707398006754373,6.615641782753076e-05,0.0,0.0,0.0,0.0,0.0,0.00501764012269632,320,80,97.98977613449097,1.224872201681137,0.30621805042028427,0.19283153250580654,0.0033454992568294982,2.84586438119004,4.5407900678307024e-06,0.0,0.0,0.0,0.0,0.0,0.0033454992568294982,80,20,19.105846405029297,0.9552923202514648,0.2388230800628662,0.04278229686897248 +11,0.006034849434036005,0.16983350860522534,0.00010824183813607309,0.0,0.0,0.0,0.0,0.0,0.006034849434036005,320,80,98.26459741592407,1.2283074676990509,0.3070768669247627,0.19247694574878552,0.013785485102562233,4.251315702055581,0.0005580191170141191,0.0,0.0,0.0,0.0,0.0,0.013785485102562233,80,20,19.08789825439453,0.9543949127197265,0.23859872817993164,0.024024457717314363 +12,0.005237574208877049,0.43369768181680685,4.29253954864622e-05,0.0,0.0,0.0,0.0,0.0,0.005237574208877049,320,80,98.5061149597168,1.23132643699646,0.307831609249115,0.18032529047923163,0.009730586926161777,3.2115105665361625,0.00032237912933010817,0.0,0.0,0.0,0.0,0.0,0.009730586926161777,80,20,18.963314533233643,0.9481657266616821,0.23704143166542052,0.033251468231901525 +13,0.004799769270812248,0.14246705247829966,2.979202441775533e-05,0.0,0.0,0.0,0.0,0.0,0.004799769270812248,320,80,98.31657290458679,1.228957161307335,0.3072392903268337,0.19850680916570126,0.004224570611222589,3.5081634806441344,4.4415249958229544e-05,0.0,0.0,0.0,0.0,0.0,0.004224570611222589,80,20,18.933969020843506,0.9466984510421753,0.23667461276054383,0.03777870242483914 +14,0.0048562844836851585,0.34933257212423,3.709438940599349e-05,0.0,0.0,0.0,0.0,0.0,0.0048562844836851585,320,80,98.43161106109619,1.2303951382637024,0.3075987845659256,0.18646608913550153,0.0036178016431222203,3.4112093091501157,6.294249446359146e-06,0.0,0.0,0.0,0.0,0.0,0.0036178016431222203,80,20,19.08294177055359,0.9541470885276795,0.23853677213191987,0.04057027366943657 +15,0.004255864443257451,0.1656667333722055,6.187442170108235e-05,0.0,0.0,0.0,0.0,0.0,0.004255864443257451,320,80,98.33895921707153,1.2292369902133942,0.30730924755334854,0.18711729196365923,0.0035861808126355756,3.0565624250225483,1.7166080699237973e-05,0.0,0.0,0.0,0.0,0.0,0.0035861808126355756,80,20,19.049601793289185,0.9524800896644592,0.2381200224161148,0.0467706841416657 +16,0.00863765765352582,0.3068369619014121,0.0002027381960549475,0.0,0.0,0.0,0.0,0.0,0.00863765765352582,320,80,98.02224159240723,1.2252780199050903,0.3063195049762726,0.20716268247924746,0.005656487263331655,3.9132582969115903,3.6503343883431685e-05,0.0,0.0,0.0,0.0,0.0,0.005656487263331655,80,20,19.1426043510437,0.9571302175521851,0.23928255438804627,0.03317992691881955 +17,0.004622109833326249,0.13771271949620206,3.9370777925482045e-05,0.0,0.0,0.0,0.0,0.0,0.004622109833326249,320,80,98.46045637130737,1.2307557046413422,0.30768892616033555,0.18359890060964973,0.003573411981051322,2.8382400677964967,5.809271045431608e-06,0.0,0.0,0.0,0.0,0.0,0.003573411981051322,80,20,19.13273310661316,0.956636655330658,0.2391591638326645,0.04146898430772126 +18,0.004134476073704718,0.17380634212921872,2.713264719204356e-05,0.0,0.0,0.0,0.0,0.0,0.004134476073704718,320,80,98.26009583473206,1.2282511979341506,0.30706279948353765,0.18229138196911662,0.003516078496249975,2.6227631476780346,7.24400641837486e-06,0.0,0.0,0.0,0.0,0.0,0.003516078496249975,80,20,19.15674638748169,0.9578373193740845,0.2394593298435211,0.05090378848835826 +19,0.004667191044973151,0.12221738814653094,6.367694598248957e-05,0.0,0.0,0.0,0.0,0.0,0.004667191044973151,320,80,98.04523253440857,1.2255654066801072,0.3063913516700268,0.18032254496356473,0.003785224206512794,2.3461711474920777,6.868326291198379e-06,0.0,0.0,0.0,0.0,0.0,0.003785224206512794,80,20,19.05215072631836,0.952607536315918,0.2381518840789795,0.054367284569889304 +20,0.0038501727190123347,0.19964894671832384,3.2192668145688413e-05,0.0,0.0,0.0,0.0,0.0,0.0038501727190123347,320,80,98.21941018104553,1.2277426272630692,0.3069356568157673,0.18122183008817955,0.004117234158547945,2.0348365343492447,1.597860968834408e-05,0.0,0.0,0.0,0.0,0.0,0.004117234158547945,80,20,19.195982217788696,0.9597991108894348,0.2399497777223587,0.046505967248231174 +21,0.004357830920935157,0.14809448825766366,7.662803581378607e-05,0.0,0.0,0.0,0.0,0.0,0.004357830920935157,320,80,98.2528235912323,1.2281602948904038,0.30704007372260095,0.18343111716676502,0.008492800514795817,2.4765617495399965,0.0002142982311968633,0.0,0.0,0.0,0.0,0.0,0.008492800514795817,80,20,19.087544918060303,0.9543772459030151,0.23859431147575377,0.048690214613452555 +22,0.004575736820424936,0.1652553465189321,5.503663022018094e-05,0.0,0.0,0.0,0.0,0.0,0.004575736820424936,320,80,98.41540288925171,1.2301925361156463,0.30754813402891157,0.17578000344801695,0.006678492884748266,2.3104023507566267,0.00011631947952993465,0.0,0.0,0.0,0.0,0.0,0.006678492884748266,80,20,18.901337385177612,0.9450668692588806,0.23626671731472015,0.03939043255522847 +23,0.0038681995100887435,0.09681270272101301,2.6917657260010597e-05,0.0,0.0,0.0,0.0,0.0,0.0038681995100887435,320,80,98.39344000816345,1.229918000102043,0.3074795000255108,0.18267457764595746,0.0037568479468291114,2.3940217966547266,2.3744597207904506e-05,0.0,0.0,0.0,0.0,0.0,0.0037568479468291114,80,20,18.862091779708862,0.9431045889854431,0.23577614724636078,0.04538590800948441 +24,0.0037726155537711747,0.08587725751030462,3.8970149140249434e-05,0.0,0.0,0.0,0.0,0.0,0.0037726155537711747,320,80,98.2309718132019,1.2278871476650237,0.30697178691625593,0.1927345296833664,0.003963983212452149,1.9307850714754582,1.0076468615227707e-05,0.0,0.0,0.0,0.0,0.0,0.003963983212452149,80,20,19.073886156082153,0.9536943078041077,0.23842357695102692,0.05581457819789648 +25,0.0045225964966448375,0.28626666026474223,5.253834299471811e-05,0.0,0.0,0.0,0.0,0.0,0.0045225964966448375,320,80,98.45195150375366,1.2306493937969207,0.3076623484492302,0.1844044709519949,0.003984417723040678,1.8695672683701106,1.225423164505357e-05,0.0,0.0,0.0,0.0,0.0,0.003984417723040678,80,20,18.98694658279419,0.9493473291397094,0.23733683228492736,0.0516578221693635 +26,0.0034221824243786613,0.21675637563972483,2.862178868469961e-05,0.0,0.0,0.0,0.0,0.0,0.0034221824243786613,320,80,98.409903049469,1.2301237881183624,0.3075309470295906,0.17961671216180547,0.004005031670385506,1.6670396929865092,9.724470967875654e-06,0.0,0.0,0.0,0.0,0.0,0.004005031670385506,80,20,18.86527991294861,0.9432639956474305,0.23581599891185762,0.05642150053754449 +27,0.0036453872757192586,0.08814689166802622,2.907900363656457e-05,0.0,0.0,0.0,0.0,0.0,0.0036453872757192586,320,80,98.16981959342957,1.2271227449178697,0.3067806862294674,0.18638201602734625,0.0039380389488542274,1.8897666597908027,2.370818725645485e-05,0.0,0.0,0.0,0.0,0.0,0.0039380389488542274,80,20,18.981795072555542,0.9490897536277771,0.23727243840694429,0.04444735175929963 +28,0.0038631531702776555,0.20194541933975962,2.7849157054867935e-05,0.0,0.0,0.0,0.0,0.0,0.0038631531702776555,320,80,98.47567534446716,1.2309459418058395,0.3077364854514599,0.19445467893965543,0.003990629422332859,1.7048710367631883,1.0106287987253522e-05,0.0,0.0,0.0,0.0,0.0,0.003990629422332859,80,20,19.01001262664795,0.9505006313323975,0.23762515783309937,0.05604702327400446 +29,0.0031632644509954843,0.05915382719257707,1.8334584522583518e-05,0.0,0.0,0.0,0.0,0.0,0.0031632644509954843,320,80,98.40432929992676,1.2300541162490846,0.30751352906227114,0.19318407390965148,0.0054594465718764695,1.8558304001606303,5.508621057241925e-05,0.0,0.0,0.0,0.0,0.0,0.0054594465718764695,80,20,19.13079309463501,0.9565396547317505,0.23913491368293763,0.04586299415677786 diff --git a/treatment/tab_ddpm_concat/mlu-eval.ipynb b/treatment/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..491981f654597d9bbeed1935a98b5973463bb774 --- /dev/null +++ b/treatment/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2791 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.235974Z", + "iopub.status.busy": "2024-02-29T18:47:07.235697Z", + "iopub.status.idle": "2024-02-29T18:47:07.267659Z", + "shell.execute_reply": "2024-02-29T18:47:07.266940Z" + }, + "papermill": { + "duration": 0.046233, + "end_time": "2024-02-29T18:47:07.269623", + "exception": false, + "start_time": "2024-02-29T18:47:07.223390", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.294401Z", + "iopub.status.busy": "2024-02-29T18:47:07.294068Z", + "iopub.status.idle": "2024-02-29T18:47:07.300586Z", + "shell.execute_reply": "2024-02-29T18:47:07.299788Z" + }, + "papermill": { + "duration": 0.021168, + "end_time": "2024-02-29T18:47:07.302592", + "exception": false, + "start_time": "2024-02-29T18:47:07.281424", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.326524Z", + "iopub.status.busy": "2024-02-29T18:47:07.325823Z", + "iopub.status.idle": "2024-02-29T18:47:07.329970Z", + "shell.execute_reply": "2024-02-29T18:47:07.329278Z" + }, + "papermill": { + "duration": 0.018317, + "end_time": "2024-02-29T18:47:07.331787", + "exception": false, + "start_time": "2024-02-29T18:47:07.313470", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.354768Z", + "iopub.status.busy": "2024-02-29T18:47:07.354504Z", + "iopub.status.idle": "2024-02-29T18:47:07.358470Z", + "shell.execute_reply": "2024-02-29T18:47:07.357633Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017716, + "end_time": "2024-02-29T18:47:07.360326", + "exception": false, + "start_time": "2024-02-29T18:47:07.342610", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.386276Z", + "iopub.status.busy": "2024-02-29T18:47:07.385987Z", + "iopub.status.idle": "2024-02-29T18:47:07.391076Z", + "shell.execute_reply": "2024-02-29T18:47:07.390240Z" + }, + "papermill": { + "duration": 0.018777, + "end_time": "2024-02-29T18:47:07.392912", + "exception": false, + "start_time": "2024-02-29T18:47:07.374135", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ceb1b868", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.418344Z", + "iopub.status.busy": "2024-02-29T18:47:07.417620Z", + "iopub.status.idle": "2024-02-29T18:47:07.422202Z", + "shell.execute_reply": "2024-02-29T18:47:07.421429Z" + }, + "papermill": { + "duration": 0.019358, + "end_time": "2024-02-29T18:47:07.424024", + "exception": false, + "start_time": "2024-02-29T18:47:07.404666", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tab_ddpm_concat/4\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010839, + "end_time": "2024-02-29T18:47:07.445880", + "exception": false, + "start_time": "2024-02-29T18:47:07.435041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.469069Z", + "iopub.status.busy": "2024-02-29T18:47:07.468575Z", + "iopub.status.idle": "2024-02-29T18:47:07.477729Z", + "shell.execute_reply": "2024-02-29T18:47:07.476975Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022765, + "end_time": "2024-02-29T18:47:07.479585", + "exception": false, + "start_time": "2024-02-29T18:47:07.456820", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tab_ddpm_concat/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:07.502648Z", + "iopub.status.busy": "2024-02-29T18:47:07.502401Z", + "iopub.status.idle": "2024-02-29T18:47:09.654705Z", + "shell.execute_reply": "2024-02-29T18:47:09.653836Z" + }, + "papermill": { + "duration": 2.166219, + "end_time": "2024-02-29T18:47:09.656852", + "exception": false, + "start_time": "2024-02-29T18:47:07.490633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:09.683127Z", + "iopub.status.busy": "2024-02-29T18:47:09.682691Z", + "iopub.status.idle": "2024-02-29T18:47:09.696671Z", + "shell.execute_reply": "2024-02-29T18:47:09.695959Z" + }, + "papermill": { + "duration": 0.029281, + "end_time": "2024-02-29T18:47:09.698692", + "exception": false, + "start_time": "2024-02-29T18:47:09.669411", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:09.722267Z", + "iopub.status.busy": "2024-02-29T18:47:09.721956Z", + "iopub.status.idle": "2024-02-29T18:47:09.729567Z", + "shell.execute_reply": "2024-02-29T18:47:09.728861Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021785, + "end_time": "2024-02-29T18:47:09.731623", + "exception": false, + "start_time": "2024-02-29T18:47:09.709838", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:09.755626Z", + "iopub.status.busy": "2024-02-29T18:47:09.755350Z", + "iopub.status.idle": "2024-02-29T18:47:09.857963Z", + "shell.execute_reply": "2024-02-29T18:47:09.857261Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.117017, + "end_time": "2024-02-29T18:47:09.860215", + "exception": false, + "start_time": "2024-02-29T18:47:09.743198", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:09.886794Z", + "iopub.status.busy": "2024-02-29T18:47:09.886511Z", + "iopub.status.idle": "2024-02-29T18:47:14.578522Z", + "shell.execute_reply": "2024-02-29T18:47:14.577761Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.708234, + "end_time": "2024-02-29T18:47:14.580855", + "exception": false, + "start_time": "2024-02-29T18:47:09.872621", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 18:47:12.130701: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 18:47:12.130753: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 18:47:12.132303: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:14.605735Z", + "iopub.status.busy": "2024-02-29T18:47:14.605175Z", + "iopub.status.idle": "2024-02-29T18:47:14.611399Z", + "shell.execute_reply": "2024-02-29T18:47:14.610691Z" + }, + "papermill": { + "duration": 0.020704, + "end_time": "2024-02-29T18:47:14.613314", + "exception": false, + "start_time": "2024-02-29T18:47:14.592610", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:14.639418Z", + "iopub.status.busy": "2024-02-29T18:47:14.639136Z", + "iopub.status.idle": "2024-02-29T18:47:36.331427Z", + "shell.execute_reply": "2024-02-29T18:47:36.330276Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 21.708306, + "end_time": "2024-02-29T18:47:36.333831", + "exception": false, + "start_time": "2024-02-29T18:47:14.625525", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:36.834314Z", + "iopub.status.busy": "2024-02-29T18:47:36.833501Z", + "iopub.status.idle": "2024-02-29T18:47:36.903073Z", + "shell.execute_reply": "2024-02-29T18:47:36.902209Z" + }, + "papermill": { + "duration": 0.085571, + "end_time": "2024-02-29T18:47:36.905093", + "exception": false, + "start_time": "2024-02-29T18:47:36.819522", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T18:47:36.934254Z", + "iopub.status.busy": "2024-02-29T18:47:36.933597Z", + "iopub.status.idle": "2024-02-29T18:47:37.457784Z", + "shell.execute_reply": "2024-02-29T18:47:37.456850Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.541056, + "end_time": "2024-02-29T18:47:37.459802", + "exception": false, + "start_time": "2024-02-29T18:47:36.918746", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:37.487651Z", + "iopub.status.busy": "2024-02-29T18:47:37.487352Z", + "iopub.status.idle": "2024-02-29T18:47:37.491528Z", + "shell.execute_reply": "2024-02-29T18:47:37.490508Z" + }, + "papermill": { + "duration": 0.020462, + "end_time": "2024-02-29T18:47:37.493577", + "exception": false, + "start_time": "2024-02-29T18:47:37.473115", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:37.522322Z", + "iopub.status.busy": "2024-02-29T18:47:37.521721Z", + "iopub.status.idle": "2024-02-29T18:47:37.529535Z", + "shell.execute_reply": "2024-02-29T18:47:37.528734Z" + }, + "papermill": { + "duration": 0.024102, + "end_time": "2024-02-29T18:47:37.531356", + "exception": false, + "start_time": "2024-02-29T18:47:37.507254", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18616321" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:37.558888Z", + "iopub.status.busy": "2024-02-29T18:47:37.558598Z", + "iopub.status.idle": "2024-02-29T18:47:37.653174Z", + "shell.execute_reply": "2024-02-29T18:47:37.652339Z" + }, + "papermill": { + "duration": 0.110627, + "end_time": "2024-02-29T18:47:37.655052", + "exception": false, + "start_time": "2024-02-29T18:47:37.544425", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 12] --\n", + "├─Adapter: 1-1 [2, 2648, 12] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 13,312\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,616,321\n", + "Trainable params: 18,616,321\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.71\n", + "========================================================================================================================\n", + "Input size (MB): 0.32\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.47\n", + "Estimated Total Size (MB): 1154.27\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:47:37.685436Z", + "iopub.status.busy": "2024-02-29T18:47:37.685162Z", + "iopub.status.idle": "2024-02-29T19:50:58.249276Z", + "shell.execute_reply": "2024-02-29T19:50:58.248269Z" + }, + "papermill": { + "duration": 3800.600566, + "end_time": "2024-02-29T19:50:58.270214", + "exception": false, + "start_time": "2024-02-29T18:47:37.669648", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.11455535657005385, 'avg_role_model_std_loss': 15.63152334621248, 'avg_role_model_mean_pred_loss': 0.03261089399050121, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.11455535657005385, 'n_size': 320, 'n_batch': 80, 'duration': 99.41325306892395, 'duration_batch': 1.2426656633615494, 'duration_size': 0.31066641584038734, 'avg_pred_std': 0.09075278166747011}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007890956761548296, 'avg_role_model_std_loss': 4.913448315569985, 'avg_role_model_mean_pred_loss': 9.587779401849516e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007890956761548296, 'n_size': 80, 'n_batch': 20, 'duration': 19.218095064163208, 'duration_batch': 0.9609047532081604, 'duration_size': 0.2402261883020401, 'avg_pred_std': 0.04594327691011131}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0068825013451714765, 'avg_role_model_std_loss': 0.5873005052374232, 'avg_role_model_mean_pred_loss': 8.441511653969323e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0068825013451714765, 'n_size': 320, 'n_batch': 80, 'duration': 98.16534066200256, 'duration_batch': 1.227066758275032, 'duration_size': 0.306766689568758, 'avg_pred_std': 0.17083694040193223}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010920317904083276, 'avg_role_model_std_loss': 5.493061433892126, 'avg_role_model_mean_pred_loss': 0.00042163531188073035, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010920317904083276, 'n_size': 80, 'n_batch': 20, 'duration': 18.999029397964478, 'duration_batch': 0.9499514698982239, 'duration_size': 0.23748786747455597, 'avg_pred_std': 0.0285534585127607}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007368110892275581, 'avg_role_model_std_loss': 0.6256727599678242, 'avg_role_model_mean_pred_loss': 0.00019790053640412092, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007368110892275581, 'n_size': 320, 'n_batch': 80, 'duration': 98.2070198059082, 'duration_batch': 1.2275877475738526, 'duration_size': 0.30689693689346315, 'avg_pred_std': 0.18491398493060843}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009714379241086136, 'avg_role_model_std_loss': 6.2370832271572, 'avg_role_model_mean_pred_loss': 0.00026508491033682134, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009714379241086136, 'n_size': 80, 'n_batch': 20, 'duration': 18.896817922592163, 'duration_batch': 0.9448408961296082, 'duration_size': 0.23621022403240205, 'avg_pred_std': 0.02282829804462381}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007530722331830475, 'avg_role_model_std_loss': 0.4640916418485176, 'avg_role_model_mean_pred_loss': 0.00011972417069327918, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007530722331830475, 'n_size': 320, 'n_batch': 80, 'duration': 98.34642100334167, 'duration_batch': 1.229330262541771, 'duration_size': 0.30733256563544276, 'avg_pred_std': 0.1856370047375094}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006328437828778987, 'avg_role_model_std_loss': 4.5384834828913885, 'avg_role_model_mean_pred_loss': 6.255692790020362e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006328437828778987, 'n_size': 80, 'n_batch': 20, 'duration': 19.502550840377808, 'duration_batch': 0.9751275420188904, 'duration_size': 0.2437818855047226, 'avg_pred_std': 0.05491759981960058}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006046878009510692, 'avg_role_model_std_loss': 0.3354074442230967, 'avg_role_model_mean_pred_loss': 8.722203440768646e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006046878009510692, 'n_size': 320, 'n_batch': 80, 'duration': 98.46714925765991, 'duration_batch': 1.2308393657207488, 'duration_size': 0.3077098414301872, 'avg_pred_std': 0.1968570870347321}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003924696132889949, 'avg_role_model_std_loss': 5.36268491241317, 'avg_role_model_mean_pred_loss': 4.107249934470758e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003924696132889949, 'n_size': 80, 'n_batch': 20, 'duration': 18.971222400665283, 'duration_batch': 0.9485611200332642, 'duration_size': 0.23714028000831605, 'avg_pred_std': 0.043903833779040724}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005646668715053238, 'avg_role_model_std_loss': 0.22879745735647247, 'avg_role_model_mean_pred_loss': 0.00010420996314914471, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005646668715053238, 'n_size': 320, 'n_batch': 80, 'duration': 98.24562430381775, 'duration_batch': 1.2280703037977219, 'duration_size': 0.30701757594943047, 'avg_pred_std': 0.17895883410237728}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005248893459065584, 'avg_role_model_std_loss': 6.201540100930288, 'avg_role_model_mean_pred_loss': 4.5616924159114224e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005248893459065584, 'n_size': 80, 'n_batch': 20, 'duration': 19.050581216812134, 'duration_batch': 0.9525290608406067, 'duration_size': 0.23813226521015168, 'avg_pred_std': 0.04073070023441687}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005301459577640344, 'avg_role_model_std_loss': 0.8160801359513838, 'avg_role_model_mean_pred_loss': 6.89963721392106e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005301459577640344, 'n_size': 320, 'n_batch': 80, 'duration': 98.10252213478088, 'duration_batch': 1.226281526684761, 'duration_size': 0.3065703816711903, 'avg_pred_std': 0.1658085669245338}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003359271524823271, 'avg_role_model_std_loss': 4.790561649674601, 'avg_role_model_mean_pred_loss': 3.5786874698511918e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003359271524823271, 'n_size': 80, 'n_batch': 20, 'duration': 18.99564027786255, 'duration_batch': 0.9497820138931274, 'duration_size': 0.23744550347328186, 'avg_pred_std': 0.03619469592813403}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005624368396092904, 'avg_role_model_std_loss': 0.35675310027072554, 'avg_role_model_mean_pred_loss': 0.00011016933555625902, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005624368396092904, 'n_size': 320, 'n_batch': 80, 'duration': 98.15186715126038, 'duration_batch': 1.2268983393907547, 'duration_size': 0.30672458484768866, 'avg_pred_std': 0.18299917249241843}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0058501261519268155, 'avg_role_model_std_loss': 5.208834768453289, 'avg_role_model_mean_pred_loss': 7.912151082312135e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0058501261519268155, 'n_size': 80, 'n_batch': 20, 'duration': 18.967280626296997, 'duration_batch': 0.9483640313148498, 'duration_size': 0.23709100782871245, 'avg_pred_std': 0.03817387481685728}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005031787550251465, 'avg_role_model_std_loss': 0.25296989842264284, 'avg_role_model_mean_pred_loss': 6.438524839464101e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005031787550251465, 'n_size': 320, 'n_batch': 80, 'duration': 98.21737504005432, 'duration_batch': 1.227717188000679, 'duration_size': 0.30692929700016974, 'avg_pred_std': 0.1820793646154925}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008046689907496329, 'avg_role_model_std_loss': 3.4085470171728955, 'avg_role_model_mean_pred_loss': 0.0001837004517647312, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008046689907496329, 'n_size': 80, 'n_batch': 20, 'duration': 19.01164937019348, 'duration_batch': 0.9505824685096741, 'duration_size': 0.23764561712741852, 'avg_pred_std': 0.03427380793727934}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0050649845143198036, 'avg_role_model_std_loss': 0.41440914815246116, 'avg_role_model_mean_pred_loss': 6.793519232238903e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0050649845143198036, 'n_size': 320, 'n_batch': 80, 'duration': 98.00610780715942, 'duration_batch': 1.2250763475894928, 'duration_size': 0.3062690868973732, 'avg_pred_std': 0.18072777504567056}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004020300185220549, 'avg_role_model_std_loss': 3.4647489232461566, 'avg_role_model_mean_pred_loss': 1.5592471368332078e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004020300185220549, 'n_size': 80, 'n_batch': 20, 'duration': 18.98149037361145, 'duration_batch': 0.9490745186805725, 'duration_size': 0.23726862967014312, 'avg_pred_std': 0.03623254182748496}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00501764012269632, 'avg_role_model_std_loss': 0.13707398006754373, 'avg_role_model_mean_pred_loss': 6.615641782753076e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00501764012269632, 'n_size': 320, 'n_batch': 80, 'duration': 97.98977613449097, 'duration_batch': 1.224872201681137, 'duration_size': 0.30621805042028427, 'avg_pred_std': 0.19283153250580654}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0033454992568294982, 'avg_role_model_std_loss': 2.84586438119004, 'avg_role_model_mean_pred_loss': 4.5407900678307024e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033454992568294982, 'n_size': 80, 'n_batch': 20, 'duration': 19.105846405029297, 'duration_batch': 0.9552923202514648, 'duration_size': 0.2388230800628662, 'avg_pred_std': 0.04278229686897248}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006034849434036005, 'avg_role_model_std_loss': 0.16983350860522534, 'avg_role_model_mean_pred_loss': 0.00010824183813607309, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006034849434036005, 'n_size': 320, 'n_batch': 80, 'duration': 98.26459741592407, 'duration_batch': 1.2283074676990509, 'duration_size': 0.3070768669247627, 'avg_pred_std': 0.19247694574878552}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013785485102562233, 'avg_role_model_std_loss': 4.251315702055581, 'avg_role_model_mean_pred_loss': 0.0005580191170141191, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013785485102562233, 'n_size': 80, 'n_batch': 20, 'duration': 19.08789825439453, 'duration_batch': 0.9543949127197265, 'duration_size': 0.23859872817993164, 'avg_pred_std': 0.024024457717314363}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005237574208877049, 'avg_role_model_std_loss': 0.43369768181680685, 'avg_role_model_mean_pred_loss': 4.29253954864622e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005237574208877049, 'n_size': 320, 'n_batch': 80, 'duration': 98.5061149597168, 'duration_batch': 1.23132643699646, 'duration_size': 0.307831609249115, 'avg_pred_std': 0.18032529047923163}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009730586926161777, 'avg_role_model_std_loss': 3.2115105665361625, 'avg_role_model_mean_pred_loss': 0.00032237912933010817, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009730586926161777, 'n_size': 80, 'n_batch': 20, 'duration': 18.963314533233643, 'duration_batch': 0.9481657266616821, 'duration_size': 0.23704143166542052, 'avg_pred_std': 0.033251468231901525}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004799769270812248, 'avg_role_model_std_loss': 0.14246705247829966, 'avg_role_model_mean_pred_loss': 2.979202441775533e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004799769270812248, 'n_size': 320, 'n_batch': 80, 'duration': 98.31657290458679, 'duration_batch': 1.228957161307335, 'duration_size': 0.3072392903268337, 'avg_pred_std': 0.19850680916570126}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004224570611222589, 'avg_role_model_std_loss': 3.5081634806441344, 'avg_role_model_mean_pred_loss': 4.4415249958229544e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004224570611222589, 'n_size': 80, 'n_batch': 20, 'duration': 18.933969020843506, 'duration_batch': 0.9466984510421753, 'duration_size': 0.23667461276054383, 'avg_pred_std': 0.03777870242483914}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0048562844836851585, 'avg_role_model_std_loss': 0.34933257212423, 'avg_role_model_mean_pred_loss': 3.709438940599349e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0048562844836851585, 'n_size': 320, 'n_batch': 80, 'duration': 98.43161106109619, 'duration_batch': 1.2303951382637024, 'duration_size': 0.3075987845659256, 'avg_pred_std': 0.18646608913550153}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036178016431222203, 'avg_role_model_std_loss': 3.4112093091501157, 'avg_role_model_mean_pred_loss': 6.294249446359146e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036178016431222203, 'n_size': 80, 'n_batch': 20, 'duration': 19.08294177055359, 'duration_batch': 0.9541470885276795, 'duration_size': 0.23853677213191987, 'avg_pred_std': 0.04057027366943657}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004255864443257451, 'avg_role_model_std_loss': 0.1656667333722055, 'avg_role_model_mean_pred_loss': 6.187442170108235e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004255864443257451, 'n_size': 320, 'n_batch': 80, 'duration': 98.33895921707153, 'duration_batch': 1.2292369902133942, 'duration_size': 0.30730924755334854, 'avg_pred_std': 0.18711729196365923}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0035861808126355756, 'avg_role_model_std_loss': 3.0565624250225483, 'avg_role_model_mean_pred_loss': 1.7166080699237973e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035861808126355756, 'n_size': 80, 'n_batch': 20, 'duration': 19.049601793289185, 'duration_batch': 0.9524800896644592, 'duration_size': 0.2381200224161148, 'avg_pred_std': 0.0467706841416657}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00863765765352582, 'avg_role_model_std_loss': 0.3068369619014121, 'avg_role_model_mean_pred_loss': 0.0002027381960549475, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00863765765352582, 'n_size': 320, 'n_batch': 80, 'duration': 98.02224159240723, 'duration_batch': 1.2252780199050903, 'duration_size': 0.3063195049762726, 'avg_pred_std': 0.20716268247924746}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005656487263331655, 'avg_role_model_std_loss': 3.9132582969115903, 'avg_role_model_mean_pred_loss': 3.6503343883431685e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005656487263331655, 'n_size': 80, 'n_batch': 20, 'duration': 19.1426043510437, 'duration_batch': 0.9571302175521851, 'duration_size': 0.23928255438804627, 'avg_pred_std': 0.03317992691881955}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004622109833326249, 'avg_role_model_std_loss': 0.13771271949620206, 'avg_role_model_mean_pred_loss': 3.9370777925482045e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004622109833326249, 'n_size': 320, 'n_batch': 80, 'duration': 98.46045637130737, 'duration_batch': 1.2307557046413422, 'duration_size': 0.30768892616033555, 'avg_pred_std': 0.18359890060964973}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003573411981051322, 'avg_role_model_std_loss': 2.8382400677964967, 'avg_role_model_mean_pred_loss': 5.809271045431608e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003573411981051322, 'n_size': 80, 'n_batch': 20, 'duration': 19.13273310661316, 'duration_batch': 0.956636655330658, 'duration_size': 0.2391591638326645, 'avg_pred_std': 0.04146898430772126}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004134476073704718, 'avg_role_model_std_loss': 0.17380634212921872, 'avg_role_model_mean_pred_loss': 2.713264719204356e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004134476073704718, 'n_size': 320, 'n_batch': 80, 'duration': 98.26009583473206, 'duration_batch': 1.2282511979341506, 'duration_size': 0.30706279948353765, 'avg_pred_std': 0.18229138196911662}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003516078496249975, 'avg_role_model_std_loss': 2.6227631476780346, 'avg_role_model_mean_pred_loss': 7.24400641837486e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003516078496249975, 'n_size': 80, 'n_batch': 20, 'duration': 19.15674638748169, 'duration_batch': 0.9578373193740845, 'duration_size': 0.2394593298435211, 'avg_pred_std': 0.05090378848835826}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004667191044973151, 'avg_role_model_std_loss': 0.12221738814653094, 'avg_role_model_mean_pred_loss': 6.367694598248957e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004667191044973151, 'n_size': 320, 'n_batch': 80, 'duration': 98.04523253440857, 'duration_batch': 1.2255654066801072, 'duration_size': 0.3063913516700268, 'avg_pred_std': 0.18032254496356473}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003785224206512794, 'avg_role_model_std_loss': 2.3461711474920777, 'avg_role_model_mean_pred_loss': 6.868326291198379e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003785224206512794, 'n_size': 80, 'n_batch': 20, 'duration': 19.05215072631836, 'duration_batch': 0.952607536315918, 'duration_size': 0.2381518840789795, 'avg_pred_std': 0.054367284569889304}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038501727190123347, 'avg_role_model_std_loss': 0.19964894671832384, 'avg_role_model_mean_pred_loss': 3.2192668145688413e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038501727190123347, 'n_size': 320, 'n_batch': 80, 'duration': 98.21941018104553, 'duration_batch': 1.2277426272630692, 'duration_size': 0.3069356568157673, 'avg_pred_std': 0.18122183008817955}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004117234158547945, 'avg_role_model_std_loss': 2.0348365343492447, 'avg_role_model_mean_pred_loss': 1.597860968834408e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004117234158547945, 'n_size': 80, 'n_batch': 20, 'duration': 19.195982217788696, 'duration_batch': 0.9597991108894348, 'duration_size': 0.2399497777223587, 'avg_pred_std': 0.046505967248231174}\n", + "Epoch 21\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004357830920935157, 'avg_role_model_std_loss': 0.14809448825766366, 'avg_role_model_mean_pred_loss': 7.662803581378607e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004357830920935157, 'n_size': 320, 'n_batch': 80, 'duration': 98.2528235912323, 'duration_batch': 1.2281602948904038, 'duration_size': 0.30704007372260095, 'avg_pred_std': 0.18343111716676502}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008492800514795817, 'avg_role_model_std_loss': 2.4765617495399965, 'avg_role_model_mean_pred_loss': 0.0002142982311968633, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008492800514795817, 'n_size': 80, 'n_batch': 20, 'duration': 19.087544918060303, 'duration_batch': 0.9543772459030151, 'duration_size': 0.23859431147575377, 'avg_pred_std': 0.048690214613452555}\n", + "Epoch 22\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004575736820424936, 'avg_role_model_std_loss': 0.1652553465189321, 'avg_role_model_mean_pred_loss': 5.503663022018094e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004575736820424936, 'n_size': 320, 'n_batch': 80, 'duration': 98.41540288925171, 'duration_batch': 1.2301925361156463, 'duration_size': 0.30754813402891157, 'avg_pred_std': 0.17578000344801695}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006678492884748266, 'avg_role_model_std_loss': 2.3104023507566267, 'avg_role_model_mean_pred_loss': 0.00011631947952993465, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006678492884748266, 'n_size': 80, 'n_batch': 20, 'duration': 18.901337385177612, 'duration_batch': 0.9450668692588806, 'duration_size': 0.23626671731472015, 'avg_pred_std': 0.03939043255522847}\n", + "Epoch 23\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038681995100887435, 'avg_role_model_std_loss': 0.09681270272101301, 'avg_role_model_mean_pred_loss': 2.6917657260010597e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038681995100887435, 'n_size': 320, 'n_batch': 80, 'duration': 98.39344000816345, 'duration_batch': 1.229918000102043, 'duration_size': 0.3074795000255108, 'avg_pred_std': 0.18267457764595746}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0037568479468291114, 'avg_role_model_std_loss': 2.3940217966547266, 'avg_role_model_mean_pred_loss': 2.3744597207904506e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037568479468291114, 'n_size': 80, 'n_batch': 20, 'duration': 18.862091779708862, 'duration_batch': 0.9431045889854431, 'duration_size': 0.23577614724636078, 'avg_pred_std': 0.04538590800948441}\n", + "Epoch 24\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0037726155537711747, 'avg_role_model_std_loss': 0.08587725751030462, 'avg_role_model_mean_pred_loss': 3.8970149140249434e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037726155537711747, 'n_size': 320, 'n_batch': 80, 'duration': 98.2309718132019, 'duration_batch': 1.2278871476650237, 'duration_size': 0.30697178691625593, 'avg_pred_std': 0.1927345296833664}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003963983212452149, 'avg_role_model_std_loss': 1.9307850714754582, 'avg_role_model_mean_pred_loss': 1.0076468615227707e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003963983212452149, 'n_size': 80, 'n_batch': 20, 'duration': 19.073886156082153, 'duration_batch': 0.9536943078041077, 'duration_size': 0.23842357695102692, 'avg_pred_std': 0.05581457819789648}\n", + "Epoch 25\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0045225964966448375, 'avg_role_model_std_loss': 0.28626666026474223, 'avg_role_model_mean_pred_loss': 5.253834299471811e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0045225964966448375, 'n_size': 320, 'n_batch': 80, 'duration': 98.45195150375366, 'duration_batch': 1.2306493937969207, 'duration_size': 0.3076623484492302, 'avg_pred_std': 0.1844044709519949}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003984417723040678, 'avg_role_model_std_loss': 1.8695672683701106, 'avg_role_model_mean_pred_loss': 1.225423164505357e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003984417723040678, 'n_size': 80, 'n_batch': 20, 'duration': 18.98694658279419, 'duration_batch': 0.9493473291397094, 'duration_size': 0.23733683228492736, 'avg_pred_std': 0.0516578221693635}\n", + "Epoch 26\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0034221824243786613, 'avg_role_model_std_loss': 0.21675637563972483, 'avg_role_model_mean_pred_loss': 2.862178868469961e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034221824243786613, 'n_size': 320, 'n_batch': 80, 'duration': 98.409903049469, 'duration_batch': 1.2301237881183624, 'duration_size': 0.3075309470295906, 'avg_pred_std': 0.17961671216180547}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004005031670385506, 'avg_role_model_std_loss': 1.6670396929865092, 'avg_role_model_mean_pred_loss': 9.724470967875654e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004005031670385506, 'n_size': 80, 'n_batch': 20, 'duration': 18.86527991294861, 'duration_batch': 0.9432639956474305, 'duration_size': 0.23581599891185762, 'avg_pred_std': 0.05642150053754449}\n", + "Epoch 27\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0036453872757192586, 'avg_role_model_std_loss': 0.08814689166802622, 'avg_role_model_mean_pred_loss': 2.907900363656457e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036453872757192586, 'n_size': 320, 'n_batch': 80, 'duration': 98.16981959342957, 'duration_batch': 1.2271227449178697, 'duration_size': 0.3067806862294674, 'avg_pred_std': 0.18638201602734625}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0039380389488542274, 'avg_role_model_std_loss': 1.8897666597908027, 'avg_role_model_mean_pred_loss': 2.370818725645485e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0039380389488542274, 'n_size': 80, 'n_batch': 20, 'duration': 18.981795072555542, 'duration_batch': 0.9490897536277771, 'duration_size': 0.23727243840694429, 'avg_pred_std': 0.04444735175929963}\n", + "Epoch 28\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038631531702776555, 'avg_role_model_std_loss': 0.20194541933975962, 'avg_role_model_mean_pred_loss': 2.7849157054867935e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038631531702776555, 'n_size': 320, 'n_batch': 80, 'duration': 98.47567534446716, 'duration_batch': 1.2309459418058395, 'duration_size': 0.3077364854514599, 'avg_pred_std': 0.19445467893965543}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003990629422332859, 'avg_role_model_std_loss': 1.7048710367631883, 'avg_role_model_mean_pred_loss': 1.0106287987253522e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003990629422332859, 'n_size': 80, 'n_batch': 20, 'duration': 19.01001262664795, 'duration_batch': 0.9505006313323975, 'duration_size': 0.23762515783309937, 'avg_pred_std': 0.05604702327400446}\n", + "Epoch 29\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0031632644509954843, 'avg_role_model_std_loss': 0.05915382719257707, 'avg_role_model_mean_pred_loss': 1.8334584522583518e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031632644509954843, 'n_size': 320, 'n_batch': 80, 'duration': 98.40432929992676, 'duration_batch': 1.2300541162490846, 'duration_size': 0.30751352906227114, 'avg_pred_std': 0.19318407390965148}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0054594465718764695, 'avg_role_model_std_loss': 1.8558304001606303, 'avg_role_model_mean_pred_loss': 5.508621057241925e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0054594465718764695, 'n_size': 80, 'n_batch': 20, 'duration': 19.13079309463501, 'duration_batch': 0.9565396547317505, 'duration_size': 0.23913491368293763, 'avg_pred_std': 0.04586299415677786}\n", + "Epoch 30\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0030803180492512184, 'avg_role_model_std_loss': 0.12737443418910246, 'avg_role_model_mean_pred_loss': 1.2523905468642128e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030803180492512184, 'n_size': 320, 'n_batch': 80, 'duration': 98.34276390075684, 'duration_batch': 1.2292845487594604, 'duration_size': 0.3073211371898651, 'avg_pred_std': 0.1860601048101671}\n", + "Time out: 3688.284231185913/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▄▆▅▃▁▂▁▃▄▁▁█▅▂▁▁▃▁▁▁▂▄▃▁▁▁▁▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▆▂▁█▅▅▄▄▃▄▅▁▃▄▅▆▃▅▇█▆▆▄▆█▇█▆█▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁▆▇▇▇▆▆▇▆▆▇▇▆▇▇▇█▇▇▆▆▇▆▇▇▇▆▇▇▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▄▆▅▃▁▂▁▃▄▁▁█▅▂▁▁▃▁▁▁▂▄▃▁▁▁▁▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▂▆▄▂▁▂▁▂▃▁▁█▅▂▁▁▁▁▁▁▁▄▂▁▁▁▁▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▆▇█▅▇█▆▆▄▄▃▅▃▄▄▃▄▃▂▂▂▂▂▂▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▅▂▁█▂▃▂▂▃▂▄▃▂▂▃▃▄▄▄▃▅▃▁▁▃▂▁▂▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▂▂▃▃▂▂▂▂▁▁▂▄▃▃▃▁▃▂▁▂▂▃▃▂▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▅▂▁█▂▃▂▂▃▂▄▃▂▂▃▃▄▄▄▃▅▃▁▁▃▂▁▂▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▂▂▃▃▂▂▂▂▁▁▂▄▃▃▃▁▃▂▁▂▂▃▃▂▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▅▂▁█▂▃▂▂▃▂▄▃▂▂▃▃▄▄▄▃▅▃▁▁▃▂▁▂▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▂▂▃▃▂▂▂▂▁▁▂▄▃▃▃▁▃▂▁▂▂▃▃▂▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00546\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00316\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.04586\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.19318\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00546\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00316\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 6e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 2e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 1.85583\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.05915\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.95654\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.23005\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.23913\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.30751\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 19.13079\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 98.40433\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/tab_ddpm_concat/4/wandb/offline-run-20240229_184739-p2d3oj7p\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_184739-p2d3oj7p/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.0014934718416216562, 'avg_g_mag_loss': 0.00030894312128962575, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.306877851486206, 'grad_duration': 4.418365001678467, 'total_duration': 6.725242853164673, 'pred_std': 0.07223384827375412, 'std_loss': 0.0061825597658753395, 'mean_pred_loss': 7.311713943636278e-06, 'pred_rmse': 0.038645464926958084, 'pred_mae': 0.027990560978651047, 'pred_mape': 0.05307391658425331, 'grad_rmse': 0.038891054689884186, 'grad_mae': 0.027836401015520096, 'grad_mape': 0.4924542307853699}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0014934718416216562, 'avg_g_mag_loss': 0.00030894312128962575, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 2.306877851486206, 'avg_grad_duration': 4.418365001678467, 'avg_total_duration': 6.725242853164673, 'avg_pred_std': 0.07223384827375412, 'avg_std_loss': 0.0061825597658753395, 'avg_mean_pred_loss': 7.311713943636278e-06}, 'min_metrics': {'avg_loss': 0.0014934718416216562, 'avg_g_mag_loss': 0.00030894312128962575, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.306877851486206, 'grad_duration': 4.418365001678467, 'total_duration': 6.725242853164673, 'pred_std': 0.07223384827375412, 'std_loss': 0.0061825597658753395, 'mean_pred_loss': 7.311713943636278e-06, 'pred_rmse': 0.038645464926958084, 'pred_mae': 0.027990560978651047, 'pred_mape': 0.05307391658425331, 'grad_rmse': 0.038891054689884186, 'grad_mae': 0.027836401015520096, 'grad_mape': 0.4924542307853699}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.0014934718416216562, 'avg_g_mag_loss': 0.00030894312128962575, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.306877851486206, 'grad_duration': 4.418365001678467, 'total_duration': 6.725242853164673, 'pred_std': 0.07223384827375412, 'std_loss': 0.0061825597658753395, 'mean_pred_loss': 7.311713943636278e-06, 'pred_rmse': 0.038645464926958084, 'pred_mae': 0.027990560978651047, 'pred_mape': 0.05307391658425331, 'grad_rmse': 0.038891054689884186, 'grad_mae': 0.027836401015520096, 'grad_mape': 0.4924542307853699}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:50:58.313245Z", + "iopub.status.busy": "2024-02-29T19:50:58.312915Z", + "iopub.status.idle": "2024-02-29T19:50:58.317099Z", + "shell.execute_reply": "2024-02-29T19:50:58.316244Z" + }, + "papermill": { + "duration": 0.028486, + "end_time": "2024-02-29T19:50:58.318974", + "exception": false, + "start_time": "2024-02-29T19:50:58.290488", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:50:58.359418Z", + "iopub.status.busy": "2024-02-29T19:50:58.359125Z", + "iopub.status.idle": "2024-02-29T19:50:58.801731Z", + "shell.execute_reply": "2024-02-29T19:50:58.800700Z" + }, + "papermill": { + "duration": 0.465834, + "end_time": "2024-02-29T19:50:58.804169", + "exception": false, + "start_time": "2024-02-29T19:50:58.338335", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:50:58.847737Z", + "iopub.status.busy": "2024-02-29T19:50:58.847426Z", + "iopub.status.idle": "2024-02-29T19:50:59.130006Z", + "shell.execute_reply": "2024-02-29T19:50:59.129155Z" + }, + "papermill": { + "duration": 0.306704, + "end_time": "2024-02-29T19:50:59.132095", + "exception": false, + "start_time": "2024-02-29T19:50:58.825391", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASoAAAEXCAYAAAAa3glcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5VElEQVR4nO3deXxU1d348c/MZJbsCQGyQCAg+5bIFoMKVfMYLS5BLEgpIFVxSawYQRYVWtsa6vbgQqXUCj9b0iD1gVqkUQyLCwEkCbJHBCRhySYyIXsyc35/TDIwkkAmBDLDfN+v130lc++59557yXw595xzz9EopRRCCOHCtO2dASGEuBQJVEIIlyeBSgjh8iRQCSFcngQqIYTLk0AlhHB5EqiEEC5PApUQwuVJoBJCuDwJVEIIl9eqQLVkyRKioqIwmUzExsayY8eOZtPu27eP8ePHExUVhUajYfHixRekSU1NZcSIEfj7+9O5c2cSExPJy8trTdaEENcgpwPVqlWrSElJYeHCheTk5BAdHU1CQgLFxcVNpq+srKRnz54sWrSIsLCwJtNs2bKFpKQktm3bxoYNG6irq+P222+noqLC2ewJIa5BGmdfSo6NjWXEiBG8/fbbAFitViIjI3nyySeZO3fuRfeNiopi5syZzJw586LpSkpK6Ny5M1u2bGH06NGXzJPVauXkyZP4+/uj0WhafC1CiKtDKcXZs2eJiIhAq3X+Qc7LmcS1tbVkZ2czb948+zqtVkt8fDxZWVlOn7w5ZrMZgA4dOjS5vaamhpqaGvvnEydOMGDAgDY7vxDiyigoKKBr165O7+dUoCotLcVisRAaGuqwPjQ0lIMHDzp98qZYrVZmzpzJjTfeyKBBg5pMk5qayu9+97sL1hcUFBAQENAm+RBCtJ2ysjIiIyPx9/dv1f5OBaqrISkpib179/Lll182m2bevHmkpKTYPzfehICAAAlUQriw1lbNOBWoOnbsiE6no6ioyGF9UVFRsxXlzkhOTmbdunV8/vnnFy0eGo1GjEbjZZ9PCOEenKrVMhgMDBs2jMzMTPs6q9VKZmYmcXFxrc6EUork5GTWrFnDxo0b6dGjR6uPJYS49jj96JeSksK0adMYPnw4I0eOZPHixVRUVDB9+nQApk6dSpcuXUhNTQVsFfD79++3/37ixAl27dqFn58fvXr1AmyPe2lpafz73//G39+fwsJCAAIDA/H29m6TCxVCuC+nuycAvP3227zyyisUFhYSExPDm2++SWxsLAA/+9nPiIqKYsWKFQB8//33TZaQxowZw+bNm22ZaOa5dfny5Tz44IOXzE9ZWRmBgYGYzWapo2oFi8VCXV1de2dDuDG9Xo9Op2t2++V+R1sVqFyNBKrWUUpRWFjImTNn2jsr4hoQFBREWFhYkwWPy/2Oulyrn7h6GoNU586d8fHxkc6yolWUUlRWVtrfTgkPD2/zc3hUoLJYFTuOnqa63sLNvTripfPcd7ItFos9SIWEhLR3doSba6xLLi4upnPnzhd9DGwNj/qmKqWY9NdtTF/+NeU19e2dnXbVWCfl4+PTzjkR14rGv6UrUd/pUYHKS6fFS2t7vKmus7ZzblyDPO6JtnIl/5Y8KlABmPS2ImlVnaWdcyKEaCkPDFS2S66WQCWE2/DAQGUrUUmgEq2l0WhYu3Zte2ejTf32t78lJiamvbPRLA8OVFJHJdzXihUrCAoKarPjzZo1y+HVOFfjUd0T4LxHv3opUYlrX21tLQaD4ZLp/Pz88PPzuwo5ah3PK1F5NZSoaiVQ/ZRSisra+qu+OPtyREZGBjfddBNBQUGEhIRw1113cfjwYQBGjRrFnDlzHNKXlJSg1+v5/PPPATh16hRjx47F29ubHj16kJaWRlRUVJPj+bfEnj17uPXWW/H29iYkJIQZM2ZQXl5u375582ZGjhyJr68vQUFB3HjjjRw7dgyAb775hltuuQV/f38CAgIYNmwYO3fuvOj5Nm/ezPTp0zGbzWg0GjQaDb/97W8B2yi6v//975k6dSoBAQHMmDEDgDlz5tCnTx98fHzo2bMnL7zwgkM3gp8++j344IMkJiby6quvEh4eTkhICElJSe32qpUHlqgaApWUqC5QVWdhwIJPrvp597+YgI+h5X+KFRUVpKSkMGTIEMrLy1mwYAHjxo1j165dTJ48mZdffplFixbZm8tXrVpFREQEN998M2B7cb60tJTNmzej1+tJSUlpdsz/luQlISGBuLg4vv76a4qLi3n44YdJTk5mxYoV1NfXk5iYyCOPPMI///lPamtr2bFjhz1vkydP5vrrr+edd95Bp9Oxa9cu9Hr9Rc85atQoFi9ezIIFC+yToJxfGnr11VdZsGABCxcutK/z9/dnxYoVREREsGfPHh555BH8/f159tlnmz3Ppk2bCA8PZ9OmTXz33XdMnDiRmJgYHnnkkVbdq8vhuYFK6qjc1vjx4x0+v/fee3Tq1In9+/czYcIEZs6cyZdffmkPTGlpaUyaNAmNRsPBgwf57LPP+Prrrxk+fDgA7777Lr17925VXtLS0qiurub999/H19cXsL20f/fdd/OnP/0JvV6P2Wzmrrvu4rrrrgOgf//+9v3z8/OZPXs2/fr1A2hRPgwGA4GBgWg0mibHgbv11lt55plnHNY9//zz9t+joqKYNWsW6enpFw1UwcHBvP322+h0Ovr168fYsWPJzMyUQHU1SPeE5nnrdex/MaFdzuuMQ4cOsWDBArZv305paSlWq+0/nfz8fAYNGsTtt9/OypUrufnmmzl69ChZWVn85S9/ASAvLw8vLy+GDh1qP16vXr0IDg5uVd4PHDhAdHS0PUgB3HjjjVitVvLy8hg9ejQPPvggCQkJ/M///A/x8fFMmDDB/j5cSkoKDz/8MH//+9+Jj4/nF7/4hT2gtVZjAD7fqlWrePPNNzl8+DDl5eXU19df8uXggQMHOrwKEx4ezp49ey4rb63leXVUUqJqlkajwcfgddUXZ3s033333Zw+fZq//vWvbN++ne3btwO2imOwPU7961//oq6ujrS0NAYPHszgwYPb/H611PLly8nKymLUqFGsWrWKPn36sG3bNsBWN7Rv3z7Gjh3Lxo0bGTBgAGvWrLms850fNAGysrKYPHkyP//5z1m3bh25ubk899xz9vvVnJ8+gmo0Gvt/ClebBwYq2yVLz3T39MMPP5CXl8fzzz/PbbfdRv/+/fnxxx8d0tx7771UV1eTkZFBWloakydPtm/r27cv9fX15Obm2td99913Fxyjpfr3788333zjMAflV199hVarpW/fvvZ1119/PfPmzWPr1q0MGjSItLQ0+7Y+ffrw9NNP8+mnn3LfffexfPnyS57XYDBgsbTsb3jr1q10796d5557juHDh9O7d297Zb678LxA1dDqVyOByi0FBwcTEhLCsmXL+O6779i4caPDRB9gK1EkJibywgsvcODAASZNmmTf1q9fP+Lj45kxYwY7duwgNzeXGTNm4O3t3ap31SZPnozJZGLatGns3buXTZs28eSTTzJlyhRCQ0M5evQo8+bNIysri2PHjvHpp59y6NAh+vfvT1VVFcnJyWzevJljx47x1Vdf8fXXXzvUYTUnKiqK8vJyMjMzKS0tpbKystm0vXv3Jj8/n/T0dA4fPsybb7552aW2q83jApW3QXqmuzOtVkt6ejrZ2dkMGjSIp59+mldeeeWCdJMnT+abb77h5ptvplu3bg7b3n//fUJDQxk9ejTjxo2zt4CZTCan8+Pj48Mnn3zC6dOnGTFiBPfffz+33XabfYJeHx8fDh48yPjx4+nTpw8zZswgKSmJRx99FJ1Oxw8//MDUqVPp06cPEyZM4M4772xyKrifGjVqFI899hgTJ06kU6dOvPzyy82mveeee3j66adJTk4mJiaGrVu38sILLzh9re3J40b4XLLpO175JI+JwyP50/1DrlIOXU91dTVHjx6lR48erfqCXkuOHz9OZGQkn332Gbfddlt7Z8dtXexvSkb4dJLRS3qme7qNGzdSXl7O4MGDOXXqFM8++yxRUVGMHj26vbMmmuFxj372YV6kZ7rHqqurY/78+QwcOJBx48bRqVMne+fPlStX2l8n+ekycODAq5bHO++8s9l8vPTSS1ctH67C40pU53qmS/cET5WQkEBCQtP9xe655x77jEo/dake423p3XffpaqqqsltHTp0uGr5cBUeGKikw6donr+/P/7+/u2dDbp06dLeWXApHvfo19gLWronCOE+PC5QSc90IdyPBwYqafUTwt14XKAyekmrnxDuxuMClYyZLoT7aVWgWrJkCVFRUZhMJmJjY9mxY0ezafft28f48eOJiopCo9E0O4qiM8e8HPZXaKR7gmila3FyB1fndKBatWoVKSkpLFy4kJycHKKjo0lISGh2hMTKykp69uzJokWLmhzkqzXHvBymhp7ptfVWrFa3f3tIeKi2ntwBbEMcazQazpw506bHbQtOB6rXX3+dRx55hOnTpzNgwACWLl2Kj48P7733XpPpR4wYwSuvvMIDDzyA0Whsk2PW1NRQVlbmsLSU6bxB2mqkVCWEW3AqUNXW1pKdnU18fPy5A2i1xMfHk5WV1aoMtOaYqampBAYG2pfIyMgWn+/8QCVjUv2EUlBbcfUXmdyhzSZ3qKmpYdasWXTp0gVfX19iY2PZvHmzfd9jx45x9913ExwcjK+vLwMHDmT9+vV8//333HLLLYBtKB2NRsODDz7YqvtxJTjVM720tBSLxUJoaKjD+tDQUA4ePNiqDLTmmPPmzXMYg6isrKzFwUqn1aDXaaizKKlQ/6m6Sngp4uqfd/5JMPheOl0Dmdyh+ckdkpOT2b9/P+np6URERLBmzRruuOMO9uzZQ+/evUlKSqK2tpbPP/8cX19f9u/fj5+fH5GRkXz44YeMHz+evLw8AgIC8Pb2btU9uRLc8hUao9HY7GNkS5j0Ouos9RKo3JRM7tD05A75+fksX76c/Px8IiJs/+HMmjWLjIwMli9fzksvvUR+fj7jx4+3D83cs2dP+/6N7xB27ty5zeu/LpdTgapjx47odDqKiooc1hcVFTVbUd4ex7wUk17H2ep66Z3+U3ofW+mmPc7rBJncoWl79uzBYrHQp08fh/U1NTWEhIQA8Jvf/IbHH3+cTz/9lPj4eMaPH8+QIa4/LptTdVQGg4Fhw4Y5TP1stVrJzMwkLi6uVRm4Ese8FOmd3gyNxvYIdrUXmdyhTSZ3KC8vR6fTkZ2dza5du+zLgQMHeOONNwB4+OGHOXLkCFOmTGHPnj0MHz6ct956q82u9YpRTkpPT1dGo1GtWLFC7d+/X82YMUMFBQWpwsJCpZRSU6ZMUXPnzrWnr6mpUbm5uSo3N1eFh4erWbNmqdzcXHXo0KEWH/NSzGazApTZbG5R+vjXNqvuc9aprw6VOHHl15aqqiq1f/9+VVVV1d5ZcUppaakC1Oeff25f98UXXyhArVmzRimlVHl5ufL19VUfffSRGjBggFq0aJE97YEDBxSgdu7caV936NAhBaj//d//bVEezj/XsmXLVHBwsCovL7dv//jjj5VWq2327/eGG25QTz75ZJPbHnjgAXX33XdfMg8rV65Ufn5+Duvy8vIuuDeXMnfuXDV48GCllFJfffWVAlRpaWmL9z/fxf6mnP2O/pTTgUoppd566y3VrVs3ZTAY1MiRI9W2bdvs28aMGaOmTZtm/3z06FEFXLCMGTOmxce8FGdvwl1vfqG6z1mnMg+0LBBei9w1UFksFhUSEqJ+9atfqUOHDqnMzEw1YsQIh+ChlFKTJ09W0dHRSqPRqGPHjjkcIz4+Xg0dOlRt375d5eTkqFtuuUV5e3urxYsXtygP55+roqJChYeHq/Hjx6s9e/aojRs3qp49e9q/A0eOHFFz585VW7duVd9//7365JNPVEhIiPrzn/+sKisrVVJSktq0aZP6/vvv1Zdffqmuu+469eyzz14yD41B5bPPPlMlJSWqoqLCft1RUVHqww8/VEeOHFHbt29XL730klq3bp1SSqmnnnpKZWRkqCNHjqjs7GwVGxurJkyYoJRS6vjx40qj0agVK1ao4uJidfbs2Rbdj0YuF6hcjbM34RfvbFXd56xTH+8+eYVz5rrcNVAppdSGDRtU//79ldFoVEOGDFGbN2++IFCtX79eAWr06NEX7H/y5El15513KqPRqLp3767S0tJU586d1dKlS1t0/p+ea/fu3eqWW25RJpNJdejQQT3yyCP2L3lhYaFKTExU4eHhymAwqO7du6sFCxYoi8Wiampq1AMPPKAiIyOVwWBQERERKjk5ucX/Jo899pgKCQlRgFq4cKFSSqna2lq1YMECFRUVpfR6vQoPD1fjxo1Tu3fvVkoplZycrK677jplNBpVp06d1JQpUxxKUC+++KIKCwtTGo3GocDRElcyUHnc5A4AU/62nS8OlfL6hGjuG9r1KuTQ9cjkDufI5A5tQyZ3aGMyJpVnk8kd3I/HjZ4A503wIP2oPJJM7uB+PLNE5SXjpnsymdzB/XhkoGoc6kXGTRc/JZM7uCaPfvSTMangGmhLES7iSv4teWagkkc/+2NMZWVlO+dEXCsa/5auxCOyRz76GWW2ZHQ6HUFBQfZRA3x8fOxv9AvhDKUUlZWVFBcXExQUhE6nu/ROTvLIQCWPfjaNL31fiZFUhecJCgq6YgMJeGSg8pYJHgDb2N/h4eF07tyZurq69s6OcGN6vf6KlKQaeWSgkmndHel0uiv6RybE5fLMynT7tO6e/egnhLvw0EAl41EJ4U48M1DJbMlCuBXPDFT2SUglUAnhDjwzUHnJ6AlCuBPPDFTS6ieEW/HQQCWtfkK4E48OVLUWKxarvJQrhKvz0EB17rLl8U8I1+eZgcrrXC9sCVRCuD6PDFRarQZD41AvHv5ishDuwCMDFciYVEK4E88NVDImlRBuw+MDVY30ThfC5XlsoPKWuf2EcButClRLliwhKioKk8lEbGwsO3bsuGj61atX069fP0wmE4MHD2b9+vUO28vLy0lOTqZr1654e3szYMAAli5d2pqstZj0ThfCfTgdqFatWkVKSgoLFy4kJyeH6OhoEhISmh3OduvWrUyaNImHHnqI3NxcEhMTSUxMZO/evfY0KSkpZGRk8I9//IMDBw4wc+ZMkpOT+eijj1p/ZZdglBKVEO5DOWnkyJEqKSnJ/tlisaiIiAiVmpraZPoJEyaosWPHOqyLjY1Vjz76qP3zwIED1YsvvuiQZujQoeq5555rUZ7MZrMClNlsbullqCl/2666z1mnVu8saPE+QojWac139HxOlahqa2vJzs4mPj7evk6r1RIfH09WVlaT+2RlZTmkB9tMteenHzVqFB999BEnTpxAKcWmTZv49ttvuf3225s8Zk1NDWVlZQ6Ls6R7ghDuw6lAVVpaisViITQ01GF9aGgohYWFTe5TWFh4yfRvvfUWAwYMoGvXrhgMBu644w6WLFnC6NGjmzxmamoqgYGB9iUyMtKZywDOzZYsgUoI1+cSrX5vvfUW27Zt46OPPiI7O5vXXnuNpKQkPvvssybTz5s3D7PZbF8KCgqcPmfjazQ10jNdCJfn1Cw0HTt2RKfTUVRU5LC+qKio2fm8wsLCLpq+qqqK+fPns2bNGsaOHQvAkCFD2LVrF6+++uoFj40ARqMRo9HoTNYvIK1+QrgPp0pUBoOBYcOGkZmZaV9ntVrJzMwkLi6uyX3i4uIc0gNs2LDBnr6uro66ujq0Wses6HQ6rNYrV9qRnulCuA+n5/VLSUlh2rRpDB8+nJEjR7J48WIqKiqYPn06AFOnTqVLly6kpqYC8NRTTzFmzBhee+01xo4dS3p6Ojt37mTZsmUABAQEMGbMGGbPno23tzfdu3dny5YtvP/++7z++utteKmO7N0TpGe6EC7P6UA1ceJESkpKWLBgAYWFhcTExJCRkWGvMM/Pz3coHY0aNYq0tDSef/555s+fT+/evVm7di2DBg2yp0lPT2fevHlMnjyZ06dP0717d/74xz/y2GOPtcElNk16pgvhPjRKKbcf4rKsrIzAwEDMZjMBAQEt2mf5V0f53X/2c9eQcN7+5dArnEMhPFtrvqPnc4lWv/ZgkhKVEG7DgwOV7dJl9AQhXJ/nBiqZLVkIt+G5gUpmSxbCbXhuoJLZkoVwG54bqKRnuhBuw4MDlZSohHAXEqikRCWEy/PYQOUtgUoIt+Gxgaqxjqreqqi3yOOfEK7MgwPVedO6y5hUQrg0jw1URq9zly6Pf0K4No8NVBqNxh6spHe6EK7NYwMVnBs3Xd73E8K1eXSgkt7pQrgHzw5U0jtdCLfg4YFKSlRCuAOPDlSN46ZXSYlKCJfm0YFKZksWwj14dKCS2ZKFcA8eHajsrX7SM10Il+bZgapx3HQpUQnh0jw8UMm46UK4AwlUyLjpQrg6CVRIPyohXJ2HByrpniCEO/DwQCUlKiHcQasC1ZIlS4iKisJkMhEbG8uOHTsumn716tX069cPk8nE4MGDWb9+/QVpDhw4wD333ENgYCC+vr6MGDGC/Pz81mSvxewdPqWOSgiX5nSgWrVqFSkpKSxcuJCcnByio6NJSEiguLi4yfRbt25l0qRJPPTQQ+Tm5pKYmEhiYiJ79+61pzl8+DA33XQT/fr1Y/PmzezevZsXXngBk8nU+itrAXuJSlr9hHBpGqWUcmaH2NhYRowYwdtvvw2A1WolMjKSJ598krlz516QfuLEiVRUVLBu3Tr7uhtuuIGYmBiWLl0KwAMPPIBer+fvf/97qy6irKyMwMBAzGYzAQEBLd7v37tO8FT6Lm7sFcLKh29o1bmFEJfW2u9oI6dKVLW1tWRnZxMfH3/uAFot8fHxZGVlNblPVlaWQ3qAhIQEe3qr1crHH39Mnz59SEhIoHPnzsTGxrJ27dpm81FTU0NZWZnD0hpGGY9KCLfgVKAqLS3FYrEQGhrqsD40NJTCwsIm9yksLLxo+uLiYsrLy1m0aBF33HEHn376KePGjeO+++5jy5YtTR4zNTWVwMBA+xIZGenMZdhJq58Q7qHdW/2sVltp5t577+Xpp58mJiaGuXPnctddd9kfDX9q3rx5mM1m+1JQUNCqc8skpEK4By9nEnfs2BGdTkdRUZHD+qKiIsLCwprcJyws7KLpO3bsiJeXFwMGDHBI079/f7788ssmj2k0GjEajc5kvUnSPUEI9+BUicpgMDBs2DAyMzPt66xWK5mZmcTFxTW5T1xcnEN6gA0bNtjTGwwGRowYQV5enkOab7/9lu7duzuTPafJbMlCuAenSlQAKSkpTJs2jeHDhzNy5EgWL15MRUUF06dPB2Dq1Kl06dKF1NRUAJ566inGjBnDa6+9xtixY0lPT2fnzp0sW7bMfszZs2czceJERo8ezS233EJGRgb/+c9/2Lx5c9tcZTOkjkoIN6Fa4a233lLdunVTBoNBjRw5Um3bts2+bcyYMWratGkO6T/44APVp08fZTAY1MCBA9XHH398wTH/9re/qV69eimTyaSio6PV2rVrW5wfs9msAGU2m526jkJzleo+Z53qOe/C/Agh2k5rv6ONnO5H5Ypa20fDXFlH9IufAnDoj3ei17V724IQ16Sr2o/qWmPUn7t8meBBCNfl2YHKS4tGY/td6qmEcF0eHag0Go193PQa6aIghMvy6EAF0vInhDuQQCWdPoVweRKoZLZkIVyeBCrpnS6Ey5NAJXVUQrg8CVQyW7IQLk8ClZSohHB5EqikjkoIl+fxgUqGehHC9Xl8oDJKPyohXJ7HByqpoxLC9UmgkhKVEC5PApW9e4KUqIRwVR4fqLwNDY9+MluyEC7L4wOV/dFPSlRCuCwJVDJbshAuz+MDlVFa/YRweR4fqKRnuhCuTwKVfTwqefQTwlV5fKBqfIWmRkpUQrgsjw9U0jNdCNcngUov41EJ4eokUHlJZboQrk4CVcOjX1WdhWtgdnshrkmtClRLliwhKioKk8lEbGwsO3bsuGj61atX069fP0wmE4MHD2b9+vXNpn3sscfQaDQsXry4NVlzmslgK1EpBbUWefwTwhU5HahWrVpFSkoKCxcuJCcnh+joaBISEiguLm4y/datW5k0aRIPPfQQubm5JCYmkpiYyN69ey9Iu2bNGrZt20ZERITzV9JKjY9+IL3ThXBVTgeq119/nUceeYTp06czYMAAli5dio+PD++9916T6d944w3uuOMOZs+eTf/+/fn973/P0KFDefvttx3SnThxgieffJKVK1ei1+svmoeamhrKysocltbS6zRoNQ3HlXoqIVySU4GqtraW7Oxs4uPjzx1AqyU+Pp6srKwm98nKynJID5CQkOCQ3mq1MmXKFGbPns3AgQMvmY/U1FQCAwPtS2RkpDOX4UCj0ciYVEK4OKcCVWlpKRaLhdDQUIf1oaGhFBYWNrlPYWHhJdP/6U9/wsvLi9/85jctyse8efMwm832paCgwJnLuIDMliyEa/Nq7wxkZ2fzxhtvkJOTg0ajadE+RqMRo9HYZnmQCR6EcG1Olag6duyITqejqKjIYX1RURFhYWFN7hMWFnbR9F988QXFxcV069YNLy8vvLy8OHbsGM888wxRUVHOZK/VZAQFIVybU4HKYDAwbNgwMjMz7eusViuZmZnExcU1uU9cXJxDeoANGzbY00+ZMoXdu3eza9cu+xIREcHs2bP55JNPnL2eVpHZkoVwbU4/+qWkpDBt2jSGDx/OyJEjWbx4MRUVFUyfPh2AqVOn0qVLF1JTUwF46qmnGDNmDK+99hpjx44lPT2dnTt3smzZMgBCQkIICQlxOIderycsLIy+ffte7vW1iLzvJ4RrczpQTZw4kZKSEhYsWEBhYSExMTFkZGTYK8zz8/PRas8V1EaNGkVaWhrPP/888+fPp3fv3qxdu5ZBgwa13VVcJhmTSgjXplHXwHsjZWVlBAYGYjabCQgIcHr/h1Z8TebBYv40fjATR3S7AjkUwrNd7nfU49/1A5nbTwhXJ4EKafUTwtVJoEJKVEK4OglUyGzJQrg6CVScmy25SmZLFsIlSaDiXImqRkpUQrgkCVRIHZUQrk4CFdIzXQhXJ4EKMErPdCFcmgQqzg3zIuNRCeGaJFAhdVRCuDoJVEgdlRCuTgIV50pUNTIelRAuSQIVMluyEK5OAhXn9UyXQCWES5JABRilRCWES5NAhWOr3zUwjqAQ1xwJVJxr9QOpUBfCFUmg4lyJCqBG+lIJ4XIkUAF6nRad1jb5qVSoC+F6JFA1kNmShXBdEqga2Huny5hUQrgcCVQNznVRkDoqIVyNBKoG8r6fEK5LAlUDkwz1IoTLkkDVoLEyvUYClRAuRwJVAxmTSgjX1apAtWTJEqKiojCZTMTGxrJjx46Lpl+9ejX9+vXDZDIxePBg1q9fb99WV1fHnDlzGDx4ML6+vkRERDB16lROnjzZmqy1mtRRCeG6nA5Uq1atIiUlhYULF5KTk0N0dDQJCQkUFxc3mX7r1q1MmjSJhx56iNzcXBITE0lMTGTv3r0AVFZWkpOTwwsvvEBOTg7/93//R15eHvfcc8/lXZmTZNx0IVyYctLIkSNVUlKS/bPFYlEREREqNTW1yfQTJkxQY8eOdVgXGxurHn300WbPsWPHDgWoY8eONbm9urpamc1m+1JQUKAAZTabnb0cu5RVu1T3OevUO5u/a/UxhBBNM5vNl/UddapEVVtbS3Z2NvHx8fZ1Wq2W+Ph4srKymtwnKyvLIT1AQkJCs+kBzGYzGo2GoKCgJrenpqYSGBhoXyIjI525jCbJbMlCuC6nAlVpaSkWi4XQ0FCH9aGhoRQWFja5T2FhoVPpq6urmTNnDpMmTSIgIKDJNPPmzcNsNtuXgoICZy6jSfZRPqVnuhAux6u9M3C+uro6JkyYgFKKd955p9l0RqMRo9HYpue2j5surX5CuBynAlXHjh3R6XQUFRU5rC8qKiIsLKzJfcLCwlqUvjFIHTt2jI0bNzZbmrpSpNVPCNfl1KOfwWBg2LBhZGZm2tdZrVYyMzOJi4trcp+4uDiH9AAbNmxwSN8YpA4dOsRnn31GSEiIM9lqEyZp9RPCZTn96JeSksK0adMYPnw4I0eOZPHixVRUVDB9+nQApk6dSpcuXUhNTQXgqaeeYsyYMbz22muMHTuW9PR0du7cybJlywBbkLr//vvJyclh3bp1WCwWe/1Vhw4dMBgMbXWtFyWv0AjhupwOVBMnTqSkpIQFCxZQWFhITEwMGRkZ9grz/Px8tNpzBbVRo0aRlpbG888/z/z58+nduzdr165l0KBBAJw4cYKPPvoIgJiYGIdzbdq0iZ/97GetvDTnSM90IVyXRin3n82grKyMwMBAzGZzq+u21u0+SXJaLrE9OrDq0aYfY4UQrXO531F516/Bue4JUqISwtVIoGpgktEThHBZEqgayGzJLVT1IxQfbO9cCA8jgaqBzJbcQisnwJ9jYcdf2zsnwoNIoGogrX4tULgXjjcM6bN+FuxKa9/8CI8hgaqB9ExvgW/+aftpCrT9/HcS7FvbbtkRnkMCVQN7ZXq9FavV7XtstD1LPez+wPZ74jswdCooK3z4EHz7SfvmTVzzJFA18D5/WnfponChwxuhohh8QqD37XDXYhh0P1jrYdUUOLKlvXMormESqBqYdPC017/4xPAsmv/OhiObwVLX3tlyHd801EcN/gXo9KDVwbil0HcsWGrgn5Og4OJDUgvRWtIzHaDiB9sjzJFNjutNgdDnDug3Fq67DYx+bZNhd1P1I7za1xaQZmyBiJhz2+pr4J8P2EpcxkCY9pHjdiGQnumX70QOLBsDRzZRqYyk1k3i7IBJ4NMRqs2wexV8MBVe7glpE+FEdnvn+Orbt8YWpDoPgPBox21eRpi4ErqNghoz/H0clOS1Tz7FNcuzA1XO+/DeHWAugA7XMd1rEX+x3M3xm1+GWd/C9AyIS4bgHrYv6rcZsPznsP+j9s751fVNuu1n9CTQaC7cbvCBX66CiKFQdRpWT4e66qubR3FN88xAVVcNHz1pWyw1tnqWGZs4aYwCGnqna3XQPQ4S/gi/yYUnttkeA+urbSWsbc2PQHpN+eEwFGwHjRaGTGg+nSkAfvkB+HaC4n2w8fdXJXtl1XX8efN3HDhVdlXO1yJlJ219zkSb8bxAdSYflt9hK02hgVtfgIn/AFPguReTf9qXSqOBzv1tjzjDHwIUZMyFjHlgddEWwpI8yF15+SWbxr5T190K/k2P4mrn1wnuedv2e9bbtgaJK6im3sIj/28nL2fkMWFpFruPn7mi52uR4oPw5xtg6Y220vq3n4D7VwO3O88KVMUH4S9j4GQueHeAX30Io2dBw/hZFxs33VxVx8GSSnYOfI4jMc/aVm77M/l/mcDyzft5e+Mh/vPNyfbvMFrxA3z8DPw5Dv79hK30V1/bumNZrY6PfS3R9w4YZhtEkTWP2yrirwCrVTFr9W62Hz0NwNmaeqb8bUf7lqzKTsHK+211mwD5WZA2Ad650dYHzVLffnlzc57V6ldfC//vLltL1cS/Q1A3h82/WLqVr7//kfuGdiHI28DxHys5/mMVx3+spKza8Y/sbu1WXtUvxaipZ6e1D4/UpvAjAQSYvEi8vgsThkcyqEvglbjc5q9txzLY8rKtUhtAowNlgQH3wvj3QOfkOIlHv7DdL2MgzMoDvXfL9qutgKU3w+nDMPA+uP+9puu2LkPq+gP85fMjeGk1LJk8lKVbDpObf4YQXwOrHo2jV+er3EJbXWarvyzaAyG9YML7tiC/8z2oLbelCeoGo34D1/+q5ffyGnG5rX6eFagAKkrB4Ad60wWbpr23gy3fljS7a7CPnkBvPd4GL7z1Wq5X+0n54bf4WsspMXTlCebzdVmQPf2A8AAmDO/KvTFdCPa9QkMqKwV56+HT5+H0Edu6sMGQ8BJYam39myy1MGQiJC61lx5bZO0TsGslDJ0G97zpXL5OZMO7/2MLlPf99eL1W05a8dVRfvuf/QC8PiGa+4Z2xVxVxy//uo19J8sIDTDywaNxdA/xbbNzXpSlDlb+wta9xbcTPPwZBEfZtlX9CF//zVanWVlqW+fbCe76X+h/99XJnwuQQEXbjPAJsOlgMUu3HCbEz0DXYB+6BnvTNdibyGAfugR742NookRSkgf/uB/M+SiDP0eGPM0bZWPI2FdCrcX2CGnQabmpd0cig70JDTQR6m8iLNBEaICR0AATfkYvNK0pcRTuhU/mwdHPbZ99O8NtL0DMZFtjAMDBj209x5XFFnDufqNlpZvaCni1j600MD2D6oiR7Pz+R778rpTS8hr6hwcQ3TWQgRGBeBt0TR9jy8uw6Y9gDIDHv7qgBNsaGXsLeXxlNkrB7IS+JN3Sy77tdEUtk5ZtI6/oLF2CvPngsTi6BF3hkotStoD+TRrofeHBddBl6IXp6qog9x+w9U1bPSnADU9A/O/A6+rMC9CeJFDRdoGq1c4WwQdTbK1jABHXczb+Ff6vsBMf7Cxg38nm6010WOhuKKPeN4KO/kY6+hntPzv5GQjxM1JTb+GH8lp+qKjlh/Iazp49S3zJCu6t/BAvrNShZ0f4JE4NfoJu4aH07ORLiK/hXPDb+yF8+LDt3bzYx+CORZcMVpbcNHT/fhyzqSuPd3iXnflnqG3i1SKtBvqE+jOkayBDugYxuEsg3Tr4EOSjR2O1wPI7bSMudL8Rpv3nXABthexjp/nlX7dTU2/ll7Hd+OPPo9Bse8f2eNX9Rrj1OYr1ETzwl20cKa2ge4gPHzwaR2jAhaXnNrPxD/D5K7bH7Enp0Of2i6e31EHm72DrW7bPXYbDL1ZA0OXP9n1V1dfCt/+1Bd/b/wCd+l40uQQqXCBQga3iOXs5fPY7Wx2RRgsjH4Vbn2NvqZXsYz9SVFZNYVk1JeYqQn7M5YbKTdymttFJU8YBazfeqk/kv9aRqIu0ccRqDpCq/ys9tbaZev5rGcEf63/FcdXJIV2AyYuuwT5otbas3Va9gWeq3gAgzTCed/VT0Gg16LQatBrbz8bftRp4tngON7CH1+vu503LfQCEBZgY1SuErsE+7D9Zxu7jZyg+W9NkPr31OsIDTUT7nSa1+AlM1ip29X2aI30fpt6qsJy32D5b8TV6cV0nP3p28qWTn9GhlHm4pJzx72zlTGUdt/cN5p3+e9B98art/cNGWi8Y/muKrn+S+//+HQWnq+jV2Y/0GTfQ0c+5CWutVkVZdR2B3vrmS7s7l8O6mbbf734Thk1r+QkOroe1j9kq3r2DYdxfoE+CU3lsF6d226oDdn9g6zMHcONT8D8vXnQ3CVS4SKBqdLYIPpkPe/9l++wfAXf+yVYfcSLHVrrZtwbOnmxy9zN+17E14kG2mkZTUlHPD+W1mPQ6Iky1TDjzLsNL1wJQberE8bg/UNQlniMl5RwuqeBIaQVHSso5caaqyRbxX+k28Af9cgBerfsFb1vGNZmHcH7gK+Nv0GoU87v9g379BzPquo5c18n3gi9tobmab46fYc9xM98cP8OBU2WUlju2Mv5Ct5lX9MuoVToerJvDTmtfatFf9Db6m84Fres6+ZH+dT7HT1eQ1OkbUnSr0Z753pYwuAeMSrZ98Q83zB9p8MMc8yj37RrK4TINJr2Wbh18Lnic7xrsg49RR/7pSvJ/qOTYD5Uc+6GCY6crKThdSU29lQ6+BqK7BhIdGWRbugbRwdcAeRmQPslWSh0zh7K42RScrqTgdBUFpyspKa/hbHU9Z6vrKK+p52x1PeUNn+usius6+XJDhwqmFCwkxNzQ5+rGp2zdZXQXvzcASilOnKli74ky9p00s+9kGWVVddzQM4Sf9e1ETGQQXro2atSvPA17VttKT4W7z/tHCofoByDmV9CxV/P7I4EKcLFA1ei7TPg4BX783vbZp+O5ylSw1dv0uwsG3Wd7LeXrd2Hb0nMtdh2ug5ufsVVCH9pgO9bZU7ZtQ6fZ/gfzDmry1NV1Fo6WVlBorgYN9lKSTqMh4sDfiMp+CYCyziOoCO5HeWBfzgb24WxAL2p1fnTf/w599v4vqtsoNL/+r9OXXl1nodBczUlzFafOVHPqTCW37H6GgWZbXZoVLSVe4RQZIikydqekYSmoD+D709WcOFNDvdJiRdOwaInWHuZ542r6qIYGA9/O8LM5tnvR+MU+sgU+W2jrfgJYvEN4o/4+3j0bRy1e1KMDLiwd6akniHICNBUEUU6QppwgKgjSnCVEc5YQygjRmOmoKSOEMjpqy/DGVpL8zBjPrNoZnKlqXdcDPfXM91rJdC/bUDmHvQfzbYdb8VZVeKtKvK2VmFQlRkslRksFlrpqqmrrqa6rx2q1okGhAbSoht9ti04LfgYtPnodPgYtXhpsj91a2wvlSqfHqtFhUTrqNTosFisWSz1WSz1WiwVlqUNZLSirhZDqY3gp2wv6SmdA0/fntpbLnre0uCVZAhUuGqjAVoH6+avw1RtgrQO9D/S9EwaNt73k/NOWx2qzrYtB1pJz/Y98QqDyB9vvHXraHjF63Hx5+dryCmz6Q9PbArtBTRlUn7F13hw65fLO1ajiB/goGb7/0nb81jL420oeNzze9EviSsH+tZD54rlW0PNY0WLR6LAoLfXo0CoLPpqmH18vZZ3lBmbWPUF9w/SYIb4GunbwoVsHH8ICjPib9PibvPAzejn8DvBt0VkOnDrLgVNlHCgs44bqr3hZ/xcCNFWtysuVttcaxWrLGP7LTXTsHE6/cH8GhAfQLyyAgREBl2zVlkCFCweqRqeP2r403W4AQwuazGvKYeffbBWuFSW2itpRyfCzeW3X/6b4IJzMgeL9ULTf9rOxxAa2FqxnDtpejWlLSkF5EZR+27AcsrWcln5rC8jK6rg08vKG4b+2lTJ9Qy59Hksd5Pw/W8tjeVELMqaxjZbhHXzeEmTrSuDbseFnJ8q9OnCgzMiu0zo0Rn+6dfAhsmFpDELO3xJFYVk133+7l+DsN6GuiiqtD9VaHyo13lTiQ6XGh3JMGE0+dOvgS7eOfnQJ9kGv0zU0jGhsPzVaLFb4rrSCbwrM5B4v47viCgB0Gis6LOixoMOCF1b0Ggt+eoVRr8eg98JgMGA06DHoDZgMeoxGA8UE88WPHTlYeJbymgtLjk/e2otnbpfK9Ety+UDVWrWVcOAjCB0EYYOu/PkqT0PxASg5aHsc7Tr8yp/zUqwNAUujaV2LodVq62Jhrbcdx1p/3mKxHdc72Nap1Zk+Zm6ktLyGXfln0Htp8Td5EWDyws9oK+H5GHQt7hqjlOL4j1W2UuCpsxwsLONg4VmeTejLnYPDL7qvBCqu4UAlxDWiXcajWrJkCVFRUZhMJmJjY9mx4+IjO65evZp+/fphMpkYPHgw69evd9iulGLBggWEh4fj7e1NfHw8hw4dak3WhBDXIKcD1apVq0hJSWHhwoXk5OQQHR1NQkICxcXFTabfunUrkyZN4qGHHiI3N5fExEQSExPZu/fcMBgvv/wyb775JkuXLmX79u34+vqSkJBAdbWMaSSEAJSTRo4cqZKSkuyfLRaLioiIUKmpqU2mnzBhgho7dqzDutjYWPXoo48qpZSyWq0qLCxMvfLKK/btZ86cUUajUf3zn/9sUZ7MZrMClNlsdvZyhBBXweV+R50qUdXW1pKdnU18fLx9nVarJT4+nqysrCb3ycrKckgPkJCQYE9/9OhRCgsLHdIEBgYSGxvb7DFramooKytzWIQQ1y6nAlVpaSkWi4XQ0FCH9aGhoRQWFja5T2Fh4UXTN/505pipqakEBgbal8hIN3tPSgjhFLdsj503bx5ms9m+FBQUtHeWhBBXkFOBqmPHjuh0OoqKHDvQFRUVERbW9DC1YWFhF03f+NOZYxqNRgICAhwWIcS1y6mutAaDgWHDhpGZmUliYiIAVquVzMxMkpOTm9wnLi6OzMxMZs6caV+3YcMG4uLiAOjRowdhYWFkZmYSExMD2PpcbN++nccff7xF+VINXcGkrkoI19T43VSt7bbpbO17enq6MhqNasWKFWr//v1qxowZKigoSBUWFiqllJoyZYqaO3euPf1XX32lvLy81KuvvqoOHDigFi5cqPR6vdqzZ489zaJFi1RQUJD697//rXbv3q3uvfde1aNHD1VVVdWiPBUUFChAFllkcfGloKDA2ZCjlFLK6ZeTJk6cSElJCQsWLKCwsJCYmBgyMjLsleH5+floz3sVYdSoUaSlpfH8888zf/58evfuzdq1axk06NwrIc8++ywVFRXMmDGDM2fOcNNNN5GRkYHJ1LIBzyIiIigoKMDf3/+SrwOUlZURGRlJQUGBPDI2Q+7Rpck9urTz75G/vz9nz54lIiKiVce6Jl6hcYa8bnNpco8uTe7RpbXlPXLLVj8hhGeRQCWEcHkeF6iMRiMLFy7EaHRuDG1PIvfo0uQeXVpb3iOPq6MSQrgfjytRCSHcjwQqIYTLk0AlhHB5EqiEEC5PApUQwuV5XKBydrz3a9nnn3/O3XffTUREBBqNhrVr1zpsVx4+ln1qaiojRozA39+fzp07k5iYSF5enkOa6upqkpKSCAkJwc/Pj/Hjx18wEsi17J133mHIkCH2UUzi4uL473/PTVrbVvfHowKVs+O9X+sqKiqIjo5myZIlTW739LHst2zZQlJSEtu2bWPDhg3U1dVx++23U1FRYU/z9NNP85///IfVq1ezZcsWTp48yX333deOub66unbtyqJFi8jOzmbnzp3ceuut3Hvvvezbtw9ow/vTqleZ3ZSz4717EkCtWbPG/rktxrK/1hQXFytAbdmyRSllux96vV6tXr3anubAgQMKUFlZWe2VzXYXHBys3n333Ta9Px5TomrNeO+erDVj2V/rzGYzAB06dAAgOzuburo6h3vUr18/unXr5pH3yGKxkJ6eTkVFBXFxcW16f1o3B7Ubuth47wcPHmynXLmu1oxlfy2zWq3MnDmTG2+80T5EUWFhIQaDgaCgIIe0nnaP9uzZQ1xcHNXV1fj5+bFmzRoGDBjArl272uz+eEygEuJyJCUlsXfvXr788sv2zorL6du3L7t27cJsNvOvf/2LadOmsWXLljY9h8c8+rVmvHdP1pqx7K9VycnJrFu3jk2bNtG1a1f7+rCwMGprazlz5oxDek+7RwaDgV69ejFs2DBSU1OJjo7mjTfeaNP74zGB6vzx3hs1jvfeOH67OOf8sewbNY5l7yn3SylFcnIya9asYePGjfTo0cNh+7Bhw9Dr9Q73KC8vj/z8fI+5R02xWq3U1NS07f1p4wp/l3ap8d49zdmzZ1Vubq7Kzc1VgHr99ddVbm6uOnbsmFLq8seyd3ePP/64CgwMVJs3b1anTp2yL5WVlfY0jz32mOrWrZvauHGj2rlzp4qLi1NxcXHtmOura+7cuWrLli3q6NGjavfu3Wru3LlKo9GoTz/9VCnVdvfHowKVUkq99dZbqlu3bspgMKiRI0eqbdu2tXeW2s2mTZuaHIB/2rRpSilbF4UXXnhBhYaGKqPRqG677TaVl5fXvpm+ipq6N4Bavny5PU1VVZV64oknVHBwsPLx8VHjxo1Tp06dar9MX2W//vWvVffu3ZXBYFCdOnVSt912mz1IKdV290fGoxJCuDyPqaMSQrgvCVRCCJcngUoI4fIkUAkhXJ4EKiGEy5NAJYRweRKohBAuTwKVEMLlSaASQrg8CVRCCJcngUoI4fL+P9UvPyH+hCI7AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:50:59.176543Z", + "iopub.status.busy": "2024-02-29T19:50:59.175725Z", + "iopub.status.idle": "2024-02-29T19:52:50.180880Z", + "shell.execute_reply": "2024-02-29T19:52:50.180058Z" + }, + "papermill": { + "duration": 111.03017, + "end_time": "2024-02-29T19:52:50.183447", + "exception": false, + "start_time": "2024-02-29T19:50:59.153277", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:52:50.228883Z", + "iopub.status.busy": "2024-02-29T19:52:50.228536Z", + "iopub.status.idle": "2024-02-29T19:52:50.249483Z", + "shell.execute_reply": "2024-02-29T19:52:50.248633Z" + }, + "papermill": { + "duration": 0.0465, + "end_time": "2024-02-29T19:52:50.251410", + "exception": false, + "start_time": "2024-02-29T19:52:50.204910", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.00.0018620.0014934.4080110.0278360.4924540.0388910.0000072.3139910.0279910.0530740.0386450.0722340.0061836.722002
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.0 0.001862 0.001493 4.408011 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.027836 0.492454 0.038891 0.000007 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 2.313991 0.027991 0.053074 0.038645 0.072234 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.006183 6.722002 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:52:50.293058Z", + "iopub.status.busy": "2024-02-29T19:52:50.292758Z", + "iopub.status.idle": "2024-02-29T19:52:50.872173Z", + "shell.execute_reply": "2024-02-29T19:52:50.871317Z" + }, + "papermill": { + "duration": 0.602698, + "end_time": "2024-02-29T19:52:50.874204", + "exception": false, + "start_time": "2024-02-29T19:52:50.271506", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:52:50.918494Z", + "iopub.status.busy": "2024-02-29T19:52:50.918192Z", + "iopub.status.idle": "2024-02-29T19:54:51.003175Z", + "shell.execute_reply": "2024-02-29T19:54:51.002074Z" + }, + "papermill": { + "duration": 120.109987, + "end_time": "2024-02-29T19:54:51.005785", + "exception": false, + "start_time": "2024-02-29T19:52:50.895798", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:54:51.051959Z", + "iopub.status.busy": "2024-02-29T19:54:51.051653Z", + "iopub.status.idle": "2024-02-29T19:54:51.068651Z", + "shell.execute_reply": "2024-02-29T19:54:51.067935Z" + }, + "papermill": { + "duration": 0.04231, + "end_time": "2024-02-29T19:54:51.070562", + "exception": false, + "start_time": "2024-02-29T19:54:51.028252", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:54:51.112234Z", + "iopub.status.busy": "2024-02-29T19:54:51.111943Z", + "iopub.status.idle": "2024-02-29T19:54:51.116886Z", + "shell.execute_reply": "2024-02-29T19:54:51.116037Z" + }, + "papermill": { + "duration": 0.02829, + "end_time": "2024-02-29T19:54:51.118854", + "exception": false, + "start_time": "2024-02-29T19:54:51.090564", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.5590865004779701}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:54:51.162295Z", + "iopub.status.busy": "2024-02-29T19:54:51.161999Z", + "iopub.status.idle": "2024-02-29T19:54:51.541724Z", + "shell.execute_reply": "2024-02-29T19:54:51.540700Z" + }, + "papermill": { + "duration": 0.404237, + "end_time": "2024-02-29T19:54:51.543881", + "exception": false, + "start_time": "2024-02-29T19:54:51.139644", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKXElEQVR4nO2deZxT1fn/Pzd7ZibJ7PsMAzPAsIjsFhFZpCigglprBZURUVuhVPnal9K+Ki5VtHUBN2xtAW1V+rOF1oqIVAVUCgoIIlBg9hlm3zLJZM89vz9ukkmYNZlMbjJ53q9XXknOPbl57iTzyTnPec7zcIwxBoIgCBGRiG0AQRAECRFBEKJDQkQQhOiQEBEEITokRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDglRhJGXl4frr79+UN+D4zg8/vjjffZ7/PHHwXHcoNpCRAckRCHm0KFDePzxx9HW1ia2KUQE8Prrr2P79u1imzHokBCFmEOHDuGJJ54gISL6BQkRQRBEiCAhCiGPP/44fvnLXwIAhg8fDo7jwHEcysvLsW3bNsybNw+pqalQKpUYO3YstmzZ0uO5PvnkE0ycOBEqlQpjx47Fzp07/bbHarXioYceQkpKCjQaDW688UZUV1d32/fLL7/EtGnToFKpkJ+fjz/84Q/d9uM4DmvWrME777yD0aNHQ6VSYcqUKTh48GCXvwXHcTh//jzuuOMO6HQ6pKSk4De/+Q0YY6iqqsKSJUug1WqRnp6OF154we/rA4A9e/Zg9uzZ0Gg00Gq1mDZtGt59912fPu+//z6mTJkCtVqN5ORk3HHHHbh48aJPn6KiIsTFxeHixYtYunQp4uLikJKSgocffhhOp9OnL8/z2Lx5My677DKoVCqkpKTguuuuw9GjRz19+vN55+Xl4fTp0zhw4IDnuzJnzpyA/g5hDyNCxsmTJ9ntt9/OALCXXnqJ/eUvf2F/+ctfmNFoZNOmTWNFRUXspZdeYq+88gpbsGABA8BeffVVn3MMGzaMjRo1isXHx7NHH32Uvfjii+yyyy5jEomEffLJJ37Zc8cddzAAbNmyZezVV19lN998M5swYQIDwDZs2ODp99133zG1Ws1yc3PZxo0b2VNPPcXS0tI8fb0BwMaPH8+Sk5PZk08+yZ577jk2bNgwplar2alTpzz9NmzYwACwiRMnsttvv529/vrrbPHixQwAe/HFF9no0aPZz372M/b666+zmTNnMgDswIEDfl3ftm3bGMdxbPz48ezpp59mr732Glu1ahW78847ffoAYNOmTWMvvfQSe/TRR5larWZ5eXmstbXV02/FihVMpVKxcePGsZUrV7ItW7awW265hQFgr7/+us/7FhUVMQBs4cKFbNOmTez5559nS5YsYa+88oqnT38+7127drHs7GxWWFjo+a74+xlHCiREIeb3v/89A8DKysp82k0mU5e+1157LRsxYoRP27BhwxgA9o9//MPTptfrWUZGBps0aVK/7Thx4gQDwB544AGf9mXLlnURoqVLlzKVSsUqKio8bWfOnGFSqbRbIQLAjh496mmrqKhgKpWK3XTTTZ42txDdd999njaHw8Gys7MZx3Hs2Wef9bS3trYytVrNVqxY0e/ra2trYxqNhl1xxRXMbDb7HON5njHGmM1mY6mpqWz8+PE+fT788EMGgD322GOethUrVjAA7Mknn/Q516RJk9iUKVM8zz/77DMGgK1du7aLTe73Zaz/n/e4cePY7Nmz+3HFkQ1NzcIEtVrteazX69HU1ITZs2ejtLQUer3ep29mZiZuuukmz3OtVou77roL3377Lerq6vr1fh999BEAYO3atT7tDz74oM9zp9OJvXv3YunSpcjNzfW0jxkzBtdee223554xYwamTJnieZ6bm4slS5Zg7969XaYxq1at8jyWSqWYOnUqGGO45557PO3x8fEYPXo0SktL+3VtALBv3z4YDAY8+uijUKlUPsfcIQdHjx5FQ0MDHnjgAZ8+ixcvRmFhIXbv3t3lvD/96U99ns+aNcvHrn/84x/gOA4bNmzo8lrvUAd/Pu9ogIQoTPjqq68wf/58xMbGIj4+HikpKfjVr34FAF2+mAUFBV3id0aNGgUAKC8v79f7VVRUQCKRID8/36d99OjRPs8bGxthNpsxcuTILue4tK+b7vqOGjUKJpMJjY2NPu3e4gYAOp0OKpUKycnJXdpbW1t7vqBLKCkpAQCMHz++xz4VFRUAur+OwsJCz3E3bn+PNwkJCT52lZSUIDMzE4mJib3a58/nHQ3IxDaAEL6811xzDQoLC/Hiiy8iJycHCoUCH330EV566SXwPC+2iYOGVCrtVxsAMJGzGvdkl79E8+fdEyREIaa7SOR///vfsFqt+OCDD3xGCJ9//nm35yguLgZjzOdc58+fByCstPSHYcOGged5lJSU+IwIzp0759MvJSUFarUaFy5c6HKOS/u66a7v+fPnERMT02VEMVi4R3rff/89CgoKuu0zbNgwAMJ1zJs3z+fYuXPnPMf9fd+9e/eipaWlx1GRP593tESu09QsxMTGxgKAT0Cj+5fW+xdfr9dj27Zt3Z6jpqYGu3bt8jxvb2/H22+/jYkTJyI9Pb1fdixcuBAA8PLLL/u0b9q0yee5VCrFtddei3/+85+orKz0tJ89exZ79+7t9tz//e9/cfz4cc/zqqoq/Otf/8KCBQuCNqroiwULFkCj0WDjxo2wWCw+x9x/56lTpyI1NRVvvPEGrFar5/iePXtw9uxZLF682O/3veWWW8AYwxNPPNHlmPt9/fm8Y2NjoyL4lUZEIcbtxP31r3+Nn/zkJ5DL5bj66quhUChwww034P7774fRaMSbb76J1NRU1NbWdjnHqFGjcM899+Cbb75BWloatm7divr6+h6FqzsmTpyI22+/Ha+//jr0ej2uvPJKfPrppyguLu7S94knnsDHH3+MWbNm4YEHHoDD4cArr7yCcePG4bvvvuvSf/z48bj22muxdu1aKJVKvP76657zhAqtVouXXnoJq1atwrRp07Bs2TIkJCTg5MmTMJlMeOuttyCXy/Hcc8/h7rvvxuzZs3H77bejvr4emzdvRl5eHh566CG/33fu3Lm488478fLLL+PChQu47rrrwPM8vvjiC8ydOxdr1qzBggUL+v15T5kyBVu2bMFvf/tbFBQUIDU1tcvobUgg3oJd9PLUU0+xrKwsJpFIPEv5H3zwAZswYQJTqVQsLy+PPffcc2zr1q1dlvqHDRvGFi9ezPbu3csmTJjAlEolKywsZO+//77fdpjNZrZ27VqWlJTEYmNj2Q033MCqqqq6LN8zxtiBAwfYlClTmEKhYCNGjGBvvPGGZwneGwBs9erV7K9//SsbOXIkUyqVbNKkSezzzz/36ed+bWNjo0/7ihUrWGxsbBdbZ8+ezcaNG+f3NX7wwQfsyiuvZGq1mmm1WjZ9+nT23nvv+fT529/+xiZNmsSUSiVLTExky5cvZ9XV1f2yq7u/gcPhYL///e9ZYWEhUygULCUlhS1cuJAdO3bMx67+fN51dXVs8eLFTKPRMABDdimfY4zqmhHBg+M4rF69Gq+++qrYphARBPmICIIQHfIRDUH6CmpUq9XQ6XQhsib4NDY2dgmM9EahUPQZx0OEFyREQ5CMjIxej69YsSKiU0tMmzatS7ChN7Nnz8b+/ftDZxAxYEiIhiD79u3r9XhmZuagvXcoXI7vvPMOzGZzj8cTEhIG3QYiuJCzmiAI0SFnNUEQohPRUzOe51FTUwONRhM1ofAEEUkwxmAwGJCZmQmJpOdxT0QLUU1NDXJycsQ2gyCIPqiqqkJ2dnaPxyNaiDQaDQDhIrVarcjWEARxKe3t7cjJyfH8r/ZERAuRezqm1WpJiAgijOnLdULOaoIgRIeEiCAI0SEhIghCdCLaR9QfGGNwOBy97k0iukcqlUImk1FoBDHoDGkhstlsqK2thclkEtuUiCUmJgYZGRlQKBRim0IMYUQXoosXL+KRRx7Bnj17YDKZUFBQgG3btmHq1KkDOi/P8ygrK4NUKkVmZiYUCgX9svsBYww2mw2NjY0oKyvDyJEjew1II4iBIKoQtba2YubMmZg7dy727NmDlJQUXLhwISibFm02G3ieR05ODmJiYoJgbfShVqshl8tRUVEBm83WpT7YkIYxoPIw0HQO0GYDI2YDUrnYVg1ZRBWi5557Djk5OT65locPHx7U96Bf8YERDX8/m4OHxeGERunlDyv9HKg8IjxurwXsJmDsjeIZOcQR9Vv2wQcfYOrUqbj11luRmpqKSZMm4c033+yxv9VqRXt7u8+NIAbC/+ra8eYXpfjXiZrOxvYaoOpr4XHWFIDjgPrTgL5aHCOjAFGFqLS0FFu2bMHIkSOxd+9e/OxnP8PatWvx1ltvddt/48aN0Ol0nhvtMyMGQq3ejI+/r4PNwWNClq5zNFT+pTA1SxsHjFoApLmqxV483vPJiAEhqhDxPI/JkyfjmWeewaRJk3Dffffh3nvvxRtvvNFt//Xr10Ov13tuVVVVIbZ46JGXl9elllk0wBjDZ/9rAGPA6HQNJmS7UucaG4HmEmEUlHeV0JY1WbhvPAc4bOIYPMQRVYgyMjIwduxYn7YxY8b4FPLzRqlUevaV0f4yYiBUt5rR0G6FQibBnNEp4DgODieP4lOHcKa2HSypAIhx5b3WZADqeIB3AG09p6glAkdUIZo5c2aXssXnz58PqNRvNGOz0a+0v5ysbgMAjMnQIEYhrNk4nE7UXzgOvdmOOvXIzs4cByQKJazRXBJiS6MDUYXooYcewuHDh/HMM8+guLgY7777Lv74xz9i9erVg/q+Ngff483h5Pvd196PvoEwZ84crFmzBmvWrIFOp0NycjJ+85vfePJB5+Xl4amnnsJdd90FrVaL++67DwDw5ZdfYtasWVCr1cjJycHatWvR0dHhOW9DQwNuuOEGqNVqDB8+HO+8805A9kU6FrsTJQ3C3+WyrHhPu8pYjXSVAw6JEifNSb4vSsgT7vXkDhgMRF2+nzZtGnbt2oX169fjySefxPDhw7Fp0yYsX758UN/3tc+7llV2Mzw5FksnZXme//FgCezO7tN6ZyeocevUTof51q/KYLb5biV56IejArLxrbfewj333IOvv/4aR48exX333Yfc3Fzce++9AIDnn38ejz32GDZs2AAAKCkpwXXXXYff/va32Lp1KxobGz1i5g6PKCoqQk1NDT7//HPI5XKsXbsWDQ0NAdkXybSb7dCqZZBJOKRolJ0HGs4iVavEKZaF6kYz7E4ecqnrt1rrKjhgagbsFkAeRTFVIUD0yOrrr78e119/vdhmhB05OTl46aWXwHEcRo8ejVOnTuGll17yCNG8efPwf//3f57+q1atwvLly/Hggw8CAEaOHImXX34Zs2fPxpYtW1BZWYk9e/bg66+/xrRp0wAAf/7znzFmzJiQX5vYpGpVKLoyD1bvEStjQHMx4pQy2BJGwe5kqGwxIT8lTjiujBP8ROY2wFALJAY33i3aEV2IxGD13IIej0ku2QVy39X5Pfa9dMfIypnB+3L+4Ac/8NmSMmPGDLzwwguezbuXboE5efIkvvvuO5/pFmPMs9Xl/PnzkMlkmDJliud4YWEh4uPjg2ZzJMFxHFRyaWeDoRawdYCTKpCSXoCL1QaUNnZ0ChEgjIrMbUKcEQlRUIlKIVLI+u8aG6y+AyU2NtbnudFoxP3334+1a9d26Zubm4vz58+HyrSwxupwQi6RQHLpL06za7qeOAL5qTqcqDagtNEInk/t7BuXDtSfAYz1oTU6CohKIYoEjhw54vP88OHDGDlyJKRSabf9J0+ejDNnzqCgoPvRXmFhIRwOB44dO+aZmp07dw5tbW1BtTvcOVreihNVbfjBiERMGeZVltotREkFyEpQQ6uWI1WjhM3JQyVx/c1jk4X7jqbQGh0FDP2NRBFKZWUl1q1bh3PnzuG9997DK6+8gl/84hc99n/kkUdw6NAhrFmzBidOnMCFCxfwr3/9C2vWrAEAjB49Gtdddx3uv/9+HDlyBMeOHcOqVaugVqtDdUlhQXWrCTYHD6XMS9At7YChXphrJ+VDKuGwcmYebrg803f6Fpsi3JtbAacjtIYPcUiIwpS77roLZrMZ06dPx+rVq/GLX/zCs0zfHRMmTMCBAwdw/vx5zJo1C5MmTcJjjz3mU15627ZtyMzMxOzZs3HzzTfjvvvuQ2pqaiguJyywOXjU6a0AgJxEr4wMLa7YIE0GoBCmvN2mjFFqhNUyxgurZ0TQoKlZmCKXy7Fp0yZs2bKly7Hy8vJuXzNt2jR88sknPZ4zPT0dH374oU/bnXfeOSA7I4maNjN4xqBVy6FTe6X0aCkT7pN8FyYYY9Cb7dCq5IKfiOOEUVFbFdDRCGjSQmj90IZGRETUUNUqZOrMSfCajvJ857aNBN+VsHeOVGLbV+WoN1g6G2NcgY7mlsE0NeogISKihupWMwAgO8FrWmasEwIUZUphauaFRiVMGOr0XkKkdiXtM7cOqq3RBk3NwpD9+/eLbcKQw+pwor5dEJTsRK8RkXtaljAMuCQJXLpWhdLGDhKiEEBCREQFjAE/GJGE1g4btCov/1BruXDv3kvmRYZOEKxaEqJBh4SIiApUcil+MOKSjawOG9B+UXic0DVSOlWrBMcBerMdJptD2KWvihcO2i2A3QzIoyv8YbAgHxERveirAN4JqHSdIx0vVHIpkmKFMkqeUZFMIew7A2hUFERIiIghj5NnKG4woMN6SRBiq9s/lNd146CLdNf0jPxEgwtNzYghT6PBin+frIVKLsVPZ4/oDFZ0O6p72cA6IiUWSpkEud4BkOoEIZbI3DZ4RkcZJETEkOdim7Bsnxmv6hQhq0HYM8ZxQHzPGUHzU+J8d+ADgNKVothKVWSCBU3NiCFPjUeIvBzL7tWyuDRA4WcBTpVLiCwkRMGChIgY0jDGPEKU1Z0QdbNsfykWuxNVLSbozXahQakR7mlEFDRIiIghTZvJDpPNCZmEQ6o7LSxjnULUjwRn/zlbj78fq0Zxg0FoULpKD1n0wrmIARNdQsSYEDsS6psfX9a3334bSUlJsFqtPu1Lly6Nqg2qwcLtH0rTqSBz55/uaAKsRkAqE+ra90FKnCBgjQbXZ+KemjntgMPaw6sIf4guZ7XTDnzxQujfd9b/CfEn/eDWW2/F2rVr8cEHH+DWW28FIFTf2L17d68764nu6XVapssVxKgP3An2PUIklQuBjHazMD2jRPoDJrpGRBGAWq3GsmXLPJU3AOCvf/0rcnNzMWfOHPEMi1CuGJGEa8elY1SaprPRO36oHyTFCkLUarKD512jW3JYB5XoGhFJ5cLoRIz39YN7770X06ZNw8WLF5GVlYXt27ejqKio+2RdRK/oLs09xDuBNlcl4X4KkVYtg1zKwe4U8hMlxCqEJXxDPWDVB9/oKCS6hIjj+j1FEpNJkybh8ssvx9tvv40FCxbg9OnT2L17t9hmDQ301cIUXREDxPUvOyXHcUiIVaCh3YoWk00QIpXbYU0jomAQXUIUQaxatQqbNm3CxYsXMX/+fOTk5PT9IsKHYxXCFoxRaXHQuHfcey/b+zHCTIxxCVGHDfkpoKDGICOqj+jxxx8Hx3E+t8LCQjFNChuWLVuG6upqvPnmm1i5cqXY5kQcjDEcr2jFwfONaDPZOw94hMi/umRjM7W4ZkwqRiS7yjh5YokMAzeWEH9ENG7cOPznP//xPJfJRDcpLNDpdLjllluwe/duLF26VGxzIo5Wkx1GqwMyCYd0nWtVy24WCikC/fYPuRmW5FtHzp1kH7aOgRlKAAgDIZLJZEhPTxfbjLDk4sWLWL58OZRKZd+dCR+qWoT81Bnx6s769W2VQkxXbHLnqleg0IgoqIi+fH/hwgVkZmZixIgRWL58OSorK3vsa7Va0d7e7nMbirS2tmLXrl3Yv38/Vq9eLbY5EUlFSzeJ8lv8W7a/lJo2M07X6GGxOztHRBTUGBREFaIrrrgC27dvx8cff4wtW7agrKwMs2bNgsHQ/a/Mxo0bodPpPLeh6sCdNGkSioqK8Nxzz2H06NFimxNxOJy8Z0Q0PNlrSuXH/rLu2Hu6Dp+crhcCG2XKzrAMmp4NGFGnZgsXLvQ8njBhAq644goMGzYM/+///T/cc889XfqvX78e69at8zxvb28fkmLUU90yon9Ut5phc/CIU8o8UdGw6IVEZpwEiM8N6LzxMXK0mezQm+3IAYTpmalFmJ7FJPb1cqIXRPcReRMfH49Ro0ahuLi42+NKpZL8JUSf6M12yCQchifHdgaBuoMYNenCaCYA4tUKAKbOVThFrCBENCIaMGElREajESUlJUHd3Mlod/SAiMS/3+U58RibqYXNwXc2trqKKAY4GgIAXYwwFfOkA1G4EqbZjAGfkxAQ1Uf08MMP48CBAygvL8ehQ4dw0003QSqV4vbbbx/wueVy4UtjMpkGfK5oxv33c/89IwW5VIJYpet3ljGvaq49Z2PsC/dWkTazTWhwJ9GnlbMBI+qIqLq6Grfffjuam5uRkpKCq666CocPH0ZKSsqAzy2VShEfH4+GhgYAQExMDO3V8gPGGEwmExoaGhAfHw+pVCq2Sf3C7uQ7l+vdmFuFrRgSKaAL3KcY7xYikx2MMXCeERFNzQaKqEK0Y8eOQT2/Oz7JLUaE/8THx0dUnNeu4xdhdfKYPybVUyDR4x/SZvq9AdkbnVoOjgNsDh5muxMxNDULGmHlIwo2HMchIyMDqampsNvtfb+A8EEul0fMSAgAzDYnavRmIWZR6fXVbhu4fwgAZFIJfjg2DXFKGRRSSefUjEZEA2ZIC5EbqVQaUf9QRGCUNXWAMSBZo+wsK82Yl6M6cP+Qm3GZus4nCvIRBQvRI6sJIliUNQkjk3zvIEZTszBikcgAbVZw39AtRA6rEGFNBAwJETEkcDh5lDcLQjTCuw6Ze1qmy+pXWti+0JvtOFPTjuIGoxCPJHGdk/xEA4KEiBgSXGwToqljlVKkab0CFoM4LQOAWr0Ze0/X4dvKViGfEfmJggIJETEkKG10jYaS4zrDNBgD9FXC4wE6qt24E6wZLA6hQe4qzmijeLWBEBXOamLoMzw5FlYHj5FpXtMyc6sgEBIZoMkIyvtoVMK/jMHiAM8zSNy78O00IhoIJETEkCAvORZ5yZckL3OPhrQZQfEPAUCcQgYJx4FnDB02BzQ0IgoKNDUjhi76auFe13cRxf4ikXCI8xoVQeESIjsJ0UAgISIinv/VtaO+3dJZc8yNR4iCmyrGPT1rt9gBuTtlLK2aDQQSIiKisTt57Dtdj3ePVKLN7BXLYzUKKTo4LujxQ1pvh7WCpmbBgHxERERT02aGg2fQqGRIiPHaR+YeDcUmB70k9MSceBSma5CsUQIdrgKLNDUbECRERERT6c5NnXhJdoVBmpYB6KwKAgB2iiMKBjQ1IyIatxANS4rxPeBeMQuio7pb3FMzhwXg+d77Ej1CQkRELBa7U0hkDyAnwUuIHDbA6Er9MghCZHfyOFPTjq/LWsBkKsEPxRhNzwYACRERsdTpLWAMSIiR+6b9aL8IMF6oXabS9XyCAbD3dB2+Km6C1QlA7sp7REIUMCRERMRSozcDANJ1at8Dg+gfAoQ0tCq5kFbGaHV4bfMgP1GgkLOaiFim5SUiNzFGSFLmTXuNcB/stB9exCmlsNidMFocSFbEAh1NNCIaACRERMQil0qQnXCJk5oxwOAWosxBe+84lQxNRhuNiIIETc2IoYW5FbBbhI2ucamD9jaxCuE33Gh1dJafJiEKGBIiIiK5UG/A5+caPKWlPbRfFO41aULVjkHCvd+sw3tERFOzgCEhIiKSkkYjTlS2obrV7HugvVa4H8RpGQDEKb1HRLTNY6CQj4iISBpc8UM+Uc6A14hocIUoLzkWN03KQnyMHHAX8aScRAETNiOiZ599FhzH4cEHHxTbFCLMsTl4tHQI1VZTNV5pYZ32zkDGQR4RaVVy5CXHIj5GQVkag0BYCNE333yDP/zhD5gwYYLYphARQJPRCsaE6ZFPIKOxXghkVMQOWiBjt3hyEnUIq3aE34guREajEcuXL8ebb76JhIQEsc0hIgD3tCzVO0k+4BU/lClsuxhkzta240hpM8xwTQ+dDiorFCCiC9Hq1auxePFizJ8/v8++VqsV7e3tPjci+mhotwAAUjS9CFEIOFTSjEMlzWi1sM5UtJQgLSBEFaIdO3bg+PHj2LhxY7/6b9y4ETqdznPLyRmcEH4ivHFX0EjVXOqodglRkBLl90WcUggP6LA5vZbwzb28gugJ0VbNqqqq8Itf/AL79u2DStW/xFXr16/HunXrPM/b29tJjKKQW6Zkw2h1+G7tsBoBi96VkTE0I6I4pRyABQZ3LJGlnWKJAkQ0ITp27BgaGhowefJkT5vT6cTBgwfx6quvwmq1dqlXr1QqoVQqLz0VEYXEKS/56hpc8UMxSUIF1hAQ6xoRGS0UXT1QRBOia665BqdOnfJpu/vuu1FYWIhHHnmkiwgRRK+444dCNBoCOpPo+0ZX09QsEEQTIo1Gg/Hjx/u0xcbGIikpqUs7Qbg5Wt6C6lYzxmfpUJDqVUwxRBHV3rhDBzpsTkDtzklEI6JAEH3VjCD84WKbGWVNHTDZHJ2N3jvuBzmi2hv3xleTzXtqRj6iQAirLR779+8X2wQizGk2ChHVibGKzkZTs5AeVioDYlNCZkuKRombJ2cJ/iqDK5SEnNUBEVZCRBC9YXPw0LtqlyXFejmkPfvLMgBJ6Ab5KrkUw5JcIyGr656EKCBoakZEDK0mYTQUo5BCrfBazAhxIGO3uPNW09QsIGhEREQM3U7LgJCkhu2J8/UGtJnsGK1TQAcIIyLGQrLFZChBIyIiYnDvuE+K8xIih03IFw2ELKLam+MVrfiquAmNVtcIjXcCTlvI7Yh0AhKi0tLSYNtBEH3CMwaFTIJEb/+QsU7Yca/UCOWDQkyMawnf5OQAqavkNQU1+k1AQlRQUIC5c+fir3/9KywWS7BtIohuuXpUCh6Yk4/LsrxSfHjih0I/GgKAWJevqsPqpJSxAyAgITp+/DgmTJiAdevWIT09Hffffz++/vrrYNtGEF3gOA5SiZf/JUQZGXsiRuEVXU0pYwMmICGaOHEiNm/ejJqaGmzduhW1tbW46qqrMH78eLz44otobGwMtp0E0T2G0EdUexPr2YHvAOS0hB8oA3JWy2Qy3HzzzXj//ffx3HPPobi4GA8//DBycnJw1113oba2Nlh2ElHOuToD3jpUjkPFTZ2NVqOw453jAE26KHbFeKKrnV6ZGkmI/GVAQnT06FE88MADyMjIwIsvvoiHH34YJSUl2LdvH2pqarBkyZJg2UlEOU1GK1o6bMI/vBv3sn0Id9xfimdEZHVQLNEACCiO6MUXX8S2bdtw7tw5LFq0CG+//TYWLVoEiSuqdfjw4di+fTvy8vKCaSsRxTS7lu4TvZfuDeLFD7lJjFXglsnZgiC1uFwStPHVbwISoi1btmDlypUoKipCRkb3qxWpqan485//PCDjCMJNi1HIU53ss7VD3BUzAFDKpMhNck3JjJQKJFACEqJ9+/YhNzfXMwJywxhDVVUVcnNzoVAosGLFiqAYSUQ3DiePNtceM8+ISKQd973iKStEIyJ/CchHlJ+fj6ampi7tLS0tGD58+ICNIghvWk12MAYo5RJP3I5YO+67o7jBgCOlzWh1uAIayVntNwGNiFgPtZuMRmO/808TRH/xbO2IVYBz7+HyTpQfwh333XGiSo+qFhPiR8YgARCmZrTfzC/8EiJ34nqO4/DYY48hJibGc8zpdOLIkSOYOHFiUA0kCAkn1DDzqdoR4oodveEepRmZa9rIOwGHFZDTj3J/8UuIvv32WwDCiOjUqVNQKDpXMBQKBS6//HI8/PDDwbWQiHpGpmkwMk3j22gIg9QfLjwpY+0cIFMIU0a7iYTID/wSos8//xyAkOR+8+bN0GpDv8mQIIQa966l8rAQImFEZLK5kug7bILDOiZRZMsih4B8RNu2bQu2HQTRLYwxMAZIvPeXGeo6a9wrxf8x7Nxv5tr4am4jh7Wf9FuIbr75Zmzfvh1arRY333xzr3137tw5YMMIAhAc1e8eqUSaVoUfT3MV0/TeXxYGDmF3Ev0OmwOIo/1mgdBvIdLpdJ4VC51O10dvgggOLR02OHgGp/dKbTikhvUiRumVCiSBduAHQr+FyHs6RlMzIlQ0ey3de/BOlh8G6NRy3DI5WxCk+kqhkUZEfhFQAIbZbIbJ1PmHrqiowKZNm/DJJ58EzTCCALpJD2tp79xxHyYjIrlUgtykGCTHKan0dIAEJERLlizB22+/DQBoa2vD9OnT8cILL2DJkiXYsmVLUA0koptm1x4zT3pY92goLlW0Hfe9QqWnAyLgDI2zZs0CAPz9739Heno6Kioq8Pbbb+Pll1/u93m2bNmCCRMmQKvVQqvVYsaMGdizZ08gJhFDEJ5naDW59pi5p2b6auFemy2SVd1T3GDEkdJmNNtd3g7age8XAQmRyWSCRiMEmH3yySe4+eabIZFI8IMf/AAVFRX9Pk92djaeffZZHDt2DEePHsW8efOwZMkSnD59OhCziCFGm9kOJy8kzNeqXP/gbiHSiZf6oztO1+hxqKQZDWbXvxQ5q/0i4OT5//znP1FVVYW9e/diwYIFAICGhga/ghxvuOEGLFq0CCNHjsSoUaPw9NNPIy4uDocPHw7ELGIIMipNg+HJscKKrcMGGBuEA7rwGhG5Y4mMvGvk5t5vRvSLgAIaH3vsMSxbtgwPPfQQrrnmGsyYMQOAMDqaNGlSQIY4nU68//776Ojo8JzvUqxWK6xWq+d5e3t7QO9FRAaJsQosnuC1Mmao8SodFF4hJJ79ZrxrBz7jAYelM2sj0SsBCdGPfvQjXHXVVaitrcXll1/uab/mmmtw0003+XWuU6dOYcaMGbBYLIiLi8OuXbswduzYbvtu3LgRTzzxRCAmE0MBvctRHWbTMqCzvlmHgxOc6A6rMD0jIeoXAedPSE9Px6RJk3ySo02fPh2FhYV+nWf06NE4ceIEjhw5gp/97GdYsWIFzpw5023f9evXQ6/Xe25VVVWBmk9EAEarwzfljHvFTJcjjkG94B4RmXzqm5HDur8ENCLq6OjAs88+i08//RQNDQ3ged7nuD+VYBUKBQoKCgAAU6ZMwTfffIPNmzfjD3/4Q5e+SqUSSmUYLtkSQYfnGbZ9WQaJhMNdM4ZBo5R5rZiF8YjI5gDUMYC5lZbw/SAgIVq1ahUOHDiAO++8ExkZGZ3JqoIAz/M+fiAiOmkz2+HgGeQcEKeUCfXtHVYhI2NcqtjmdcEzIrI5AS2ljPWXgIRoz5492L17N2bOnDmgN1+/fj0WLlyI3NxcGAwGvPvuu9i/fz/27t07oPMSkU9LR2cgI8dxgN41DddmARKpiJZ1T5xShh9NyUaMQgpWfR4cQNs8/CAgIUpISEBi4sBzrTQ0NHgKMep0OkyYMAF79+7FD3/4wwGfm4hsmoyu8kHuQMY21x6u+FyRLOodmVSCnETXSIhKT/tNQEL01FNP4bHHHsNbb73lky7WX6jcENET7j1myXEKIR4nzIXIByo97TcBCdELL7yAkpISpKWlIS8vD3K53Of48ePHg2IcEb107jFTCBU7bB2ARBY+pYO6oaTRiEaDFSOZFEkACZEfBCRES5cuDbIZBNGJ02uPWVKcEmg5JxzQZQnO6jDlbG07LtQboc2UCUJEzup+E9CnumHDhmDbQRAeHDyPybkJaDPbhD1mETIt82Rq5Km+mb8E/PPS1taGv//97ygpKcEvf/lLJCYm4vjx40hLS0NWVvjFeRCRg1ImxVUjk4UnPv6hYeIZ1Q9iXEv4hkv3m4VBOttwJyAh+u677zB//nzodDqUl5fj3nvvRWJiInbu3InKykpPriKCGDAdTcLqk1QWNhkZe8JdVsjg2W/GBDFSBL6gEy0EtMVj3bp1KCoqwoULF3wquy5atAgHDx4MmnFEdNJosHZu73CPhrTZYe0fAjpHRB02dCZto+lZvwhIiL755hvcf//9XdqzsrJQV1c3YKOI6GbP97V482ApKltMQItru1BCeE/LgM5UICabg1LG+klAQqRUKrtNwXH+/HmkpKQM2CgienHyDK0drqyMagnQ5kq0l5gvolX9I0bZuc2DuXfd036zfhGQEN1444148sknYbcLXxiO41BZWYlHHnkEt9xyS1ANJMIHxhjO1xvwTXkLOqyOQXmPVpMNPBOyMsZZ6oSqrorYsNxfdilxCmGbx/Ircr2CGmlE1B8CEqIXXngBRqMRKSkpMJvNmD17NgoKCqDRaPD0008H20YiTDhc2oLd39XiywtN+Ns3VXDywc9A2OQKZEyOU4BrLRMaE4dHxMqTRMIhJzEGSXFKcLTNwy8C8v7pdDrs27cPX331FU6ePAmj0YjJkydj/vz5wbaPCBP0Jju+KW8BAKgVUlwzJhVSSfDFodEgCFGKRtnpH0ocEfT3GXQ8OYlIiPqD30LE8zy2b9+OnTt3ory8HBzHYfjw4UhPTwdjLKgpQYjw4XSNHk6eIScxBrdMzhq0z9ktRGkKB2BsFEZCCXmD8l6DQWmjEQ0GK0baZLTNww/8mpoxxnDjjTdi1apVuHjxIi677DKMGzcOFRUVKCoq8jtNLBEZMMZwts4AALgsSzdoIsQY6xQip6usdFxa5wpUBHCuzoD/ljSjzkLVPPzBrxHR9u3bcfDgQXz66aeYO3euz7HPPvsMS5cuxdtvv4277rorqEYS4tJotKLdbIdCJsGIFEEUOqwOnKxug8XuxLzCtKC911Ujk9FgsCLedEJoSB4ZtHOHAk+mRk90NQlRf/BrRPTee+/hV7/6VRcRAoB58+bh0UcfxTvvvBM044jwwGLjER8jR1a8GnKp8JUx2Zw4UtqCMzXtcDj5Ps7QPziOw7hMHebmx0PmXrZPHhWUc4eKzmoeLiGiOKJ+4ZcQfffdd7juuut6PL5w4UKcPHlywEYR4UVuUgzunjncp7RPcpwCcUoZ7E6Gi21BjpVpKQV4B6COB2IjKy7NHdTY7nQJkcMC8MER6qGMX0LU0tKCtLSeh+FpaWlobW0dsFFEeOIeDQHC6GVYkrAyVNkSnOlHeVMHLraZYW9wpf1IHhURy/bexCov2fjKGOCgoMa+8EuInE4nZLKe3UpSqRQOx+AEuhHiwPPMt6SPF1kJQvRwbZslKO914Hwj3v+6HMaLZ4WGCJuWAd7bPHhA7tqHSQ7rPvHLWc0YQ1FRUY8lfaj6xtDjfIMBn55twJgMTRendKZOEKL6dgscTh4yacBl8mB38mg12aC11CA2xgko4sKybFBfuEdEZrsTvDIGErvFFV0dWVPMUOOXEK1YsaLPPrRiNrRoaLfC5ujexxEfI4daIYXZ5kSj0YoMXeBVTZuMVjAGZNlKodBIgJRCQBK4sImFWi7Fj6ZkI1YpA3cuBjC30H6zfuCXEG3btm2w7CDClAZXXE+qRtXlGMdxyNCpUNNmGfDes/p2KyS8AznOKgBqILX7suPhDsdxXtU83DvwaWrWF+Gd4IUQFcYYGgyC/ydV2/10/Npx6VDKJAMOcqzTW5BgLodGzgCVDtBlD+h8YQGVnu43JEREjxisDljtPKQSDkmx3QuRSh6cYof17RakmEoQp5MBaWMjbrXMm7KmDtTpLRhpkyIZoBFRP4i8STgRMtpceYF0anm/Nrj2tLrWF1aHE0ZDG+LNlUJ56bTxAZ0nXLhQb8Dh0mbUm10iTdHVfSKqEG3cuBHTpk2DRqNBamoqli5dinPnzolpEuFFq0kochgfI++135cXmvCnL0pR0mgM6H1kEgluy25DfnIM5AnZQGxyQOcJF9y5q42gdLH9RVQhOnDgAFavXo3Dhw9j3759sNvtWLBgATo6aE4dDsQqZRieHIvM+N5Xwyx2JwwWB2r1gcUTSTkg2XAWaVoVkDkpoHOEE57c1e7oapqa9YmoPqKPP/7Y5/n27duRmpqKY8eO4eqrrxbJKsJNQWocClLj+uyXrlPh1EU96gIUIrSWAeY2IeF86pjAzhFGdKnmQSOiPgkrZ7VerwcAJCYmdnvcarX6BE12lzebCD1pWmFpv8FgBc8zSPxMmHbu2H4kmC1IGDUJcmnv08BIwFPfzOlV34x3ApLgOPaHImHjrOZ5Hg8++CBmzpyJ8eO7d1Zu3LgROp3Oc8vJyQmxldEDzzNY7M5+9U2KVUAhk8Dm4NHcYfPrfYyt9WipPIOSJhP4zMmBmBp2uCu+6h1ygHPnJSJ3Q2+EjRCtXr0a33//PXbs2NFjn/Xr10Ov13tuVVVVIbQwumi32LFlfwn+9EVpn6thEgmHVI3gmK1v9296pj/3JQDAkZAPpTb8E+T3B3c1D5uTwSFz564OzJEfLYTF1GzNmjX48MMPcfDgQWRn9xzIplQqe9znRgSXVpOwdK+US/sVrJihU6O61Yw6vQXjs3T9exOrAdaLQtoYae6MgG0NNxRSiWebh+SsBrAbASsJUW+IKkSMMfz85z/Hrl27sH//fgwfPlxMcwgvPEv36v75bDLiVUjTqqDrY6nfh4pDMJqtMCjTMSwrAhPk94DPNg9lHGAAYDOIalO4I6oQrV69Gu+++y7+9a9/QaPReKrE6nQ6qNWBb6AkBo7eNSJKiFH0q39+ShzyU/peYfNgaoGj+jhMVieqUqdiRnzXvWxDAqVGuKcRUa+I6iPasmUL9Ho95syZg4yMDM/tb3/7m5hmEeh/MGPAlB2E0WJDqyobSBgGjSryV8u8qWw24VBJExqsrt968hH1iuhTMyI8cfuIEmL7NyJyY3fysDp4YatGT7SUAg1nYbLxqIqfjtwhOBoqbTLi28o2qLUypAI0IuqDsHBWE+GFw8nDYBGEqL8+IkCoffafMw3IT43F9RMyu+/ktAPn9wIAMsddheW50+FwDr0fJI3KFdTIXIsr5CPqFRIiogsOnuHy7Hi0W+ye4Lz+oFXJwTPWe4T1hX1CFLVSAwy/GrG9pB6OZNzR1W1OlxDRiKhXhua3gBgQKrkUcwv9j+lJ06rAcYDB4oDR6ug6Pas5AdSeFFJ8FC4WtnQMUeIuFSK7GXA6ACn9y3VH2AQ0EpGPQiZBUpzwj9dlVFR/Bjjv2ls4bCa+bNbgH8eqUdE8NCOONUphSqu3ycA80dU0KuoJEiKiC+0We7+3d1xKumvfmUeInA6g9ABw5l9CaZ2My4G8q1DaZERli6nHfNiRjjuJvoMBdpk7ZSwJUU/QOJHowt7v61DdasaiyzIwOl3T/xcyhsxYhjJbEzou1gnfrrpTgMW1OTl7KpB/DfQWB5qNNnAcOgP/hhgyqQQxCilMNicsEjUUMJCfqBdIiIgutLmW7rXqPr4eDhvQfAFoLgE6GgFzK/LMZrTX6SGVcGB8grA9RKkB8ucJKWABlDQKWRYy49VBSzUbjiyZmAWVXIK4ku8BawONiHqBhIjwwebgYXRV5OgxqtrpAKq/BqqOAHZfX1CMXIqE+HioNQlwpmRAljIKSBkNeKX3KG4Q/iH7k+sokknXueKjVO7oalrC7wkSIsKHNrMQUa1WSLsfrbTXAv/7EOhoEp6rEwSh0WUD6kRwKi0Ke8kpZLI5UNMm1Pka6kLkQeG6ThoR9QgJEeGDe1rWbSBj3Sng3B4hyZciVphupY71qxDi2VoDGBNGC9ohtq3jUhraLShuNCKlg8NIgEZEvUBCRPjQ2uHeY+Y1LWMMKDsIVBwSniePBEYvAhQ9O5rbLXaUNnYgPyXWZx9ZUqwCuYkxGJXmhxM8Qmky2nCktAWFaokgRBbKKNoTJESED21m9657l3jwPHB+D1D7nfB82Axg+Ow+6465V95MtkRcmd9ZlSMvORZ5ybFRsc/QE9TIu3xFVoMg6hFcs22woDgiwofhybGYkK0TKnfwTuDsB4IIcRxQuAgYMadf/0gTsuMBAN9f1HcbKzTQyrCRQJxrv1mrQyn8zXgHpYztARoRET6MStMI0yanAzi9C2i6ICR9H7tEcEr3k4LUOOjUcujNdhw434hYpRRWB4+rCpIhl0bH7597RGR1cnAoYiCzdwDWdiFZGuFDdHwjCP/gnV4iJAPG3+KXCAGAVMJhzugUAMKo6EhpC05UtuFCffSsHClkEijlwr+YWeoSH4teRIvCFxIiwoPZ5kRDuxn20/8GmosFEZpwK5CUH9D5RqTEYcG4NMS4QgFmj07BmIyh76T2xr0yaOJc2zzIYd0tNDUjPJQ3d+DsF7sw2n4W47ISgHE3AQl5AzrnuEwdxmX2M5n+EESnlqPRYIUBsUgDhKkZ0QUSIsKDtfokMgynoNIogTHXA8kFYpsU8VxVkIxZI5OhaTYAxadoatYDJESEgLEBMWX7YAbgyJkBpI0T26IhgSfVrsk1KiQh6hbyERGeZXqrzYY2VQ6kI2aLbdHQQ+USIpqadQsJEQFUfAVmbIDBKUdJ0mzE+5kwn+gZi92J/5Y047Ny1+Zgu0XIWkD4QEIU7ZhagMrDsDsZiuOvhEMW41fCfKJ3JByHw6XNOFlrhl3iEniannWBhCjaKf0c4J0wqLPRrB4BrUoOWZQEHIYChUziKUBglrpCFyxt4hkUppCzOppprQAazwMcB/noH2KWUQWJZOhvvQg1WrUcJpsTHRINtGgGzK1imxR2iPrTd/DgQdxwww3IzMwEx3H45z//KaY50Uf5l8J9xkRokzMxNS8Rk3MTxLVpCOIOajRwrhERCVEXRBWijo4OXH755XjttdfENCM6aasC2iqFfWTDrhTbmiGNO+WuHq7oahKiLog6NVu4cCEWLlzY7/5WqxVWq9XzvL2dlkIDxp1bKP0yQKVFWVMH4pQyJMUqaHoWZOLVgpO6hSch6omI8kpu3LgROp3Oc8vJyRHbpMjEUCfUn+ckQO4P4OQZPjhRg78eroApwDJCRM/Eu3I7NTpcieQseiF2i/AQUUK0fv166PV6z62qqkpskyKT6m+E+9QxgDoBbSYbeMagkEkQ60eJaaJ/pOtUWHFlHm6/aqxQ6ZUxWsK/hIhaNVMqlVAqh26Z4pBg6wAazgqPs6cCAFpN7qyMiqhIWBZq5FIJEt1BouoEwCiUXkJMoriGhRERNSIigkDtSWFaoM0AtJkAgFaTEOnrSQ9LDB5q16ok+Yl8iKgRETFAeB6o+VZ4nDXF09ziSpifQFs7Bo2SRiMu1BsxxqzGMKCzHBMBQGQhMhqNKC4u9jwvKyvDiRMnkJiYiNzcXBEtG6I0XxASc8nVQMoYT3OTUViJTI4jIRosGg1WnK1thzYmxiVEjWKbFFaIKkRHjx7F3LlzPc/XrVsHAFixYgW2b98uklVDmIvHhPvMiYLTFADPM7QYhRFRUiz53wYLd9XcRl4rNJiaqKKHF6IK0Zw5c6KirExY0NEkbOngOCBzkqeZAVh4WQaajVboaLProOH2v9XaY8E4gLNbhMqvyuhKndsT5KyOFty+oaSCztw4EJLcF6TG4YoRSRTIOIgkxiog4TiYnRysctffn6ZnHkiIogGHDahzFUjMmiyuLVGKTCpBQqwwKtJL3ELULKJF4QUJUTTQcEYQI3UCkDDc59D5egOKG4ywUET1oJMSJ/jgWhAvNNCIyAMJ0VCHMaDmuPA4c1IX5+hXxU3498kaNLRbu3kxEUySNUpwHGCQxQsNHQ2i2hNOUBzRUMdQCxjqhRpl6Zf5HLLYnWhzRVWnaGjFbLCZkK3DxJx4yG0pwOGDgLFBCC6V0LYaGhENdS66RkOphYAixudQfbuQR1mnlkNNe8wGHaVMKpTbVsULsVy8UxAjgoRoSGMzde4ry+zqpK53TcfSdapQWkVwHKDJEB4basS1JUygqdlQpuZbgHcAmjTPvjJv6lwjojQtTctCxfcX9fj+oh5TWRwKAKC9FsgS2yrxoRHRUMXpAC4eFR7nXNFtBG+DR4hoRBQqTDYnavUWVDvihQZDraj2hAskREOV+u+FqZlKC6QUdjlssNhhsDgg4TikakiIQkWGaxpcZtWBgQGmZsBuFtkq8aGp2VCE54Gqr4XH2dO6XZWJU8pw98w8NBmtUMjo9yhUpGlVkHAc2pwKWGISoLa1CfnDU0aJbZqo0DdwKNJwRvillSmBjMu77cJxHOJjFChIpb1OoUQhk3hCJZpk6UJjW4WIFoUHJERDDd4JlH8hPM79gSBGRFiRnaAGAFTwKUJDa7l4xoQJJERDjbrvAHObEDOUNbXbLnqzHR9+V4PvqttCahohkJckVPM4Z4kHA4TMCJborkhDQjSUsJuBsoPC49wrAVn3ic7Kmjpwod6I/9UaQmgc4SYzXgWtWo7M5CQ44lzxRM0XxDVKZMhZPZQoOyislMUm97rLvrTRCADIT40NlWWEFzKpBCtn5gmFCirHAMZaofS3V/reaINGREOFtsrOnEMjF/S4f8lid6K6VVguHpEcFyrriEvwVEtJHinct1UKFVaiFBKioYDNBJz5QNhpnzEBSBjWY9fz9QY4eYbkOAUlyxcZxhjqHTEwqVIBxgN134ttkmiQEEU6vBM4+wFgNQh1sgp+2Gv3MzWCU3RspjYU1hG9cOB8I949UonTGCE01J4QfkyiEBKiSIYx4NweoKVMSIY/dmmPDmpA2NJRq7dAwnEoTCchEpvhyYKP7qgpHXZODphagObiPl41NCEhilR4J3D230DdKaGG/bibhc2tfZCdoMbo9DjEKmmdQmxyE2OQrFHCwktxQebyFZUdjMpREQlRJGJpB07uAOpPCyI05nogKb/Pl6VqVbh1ag7mj+lbsIjBh+M4TM8Tyk4fMI+AFTIhP5F70SGKICGKJBgTSkYf3SqsskjlwPhbgLRxvb6M55lP2SaZlD72cGFUWhyy4tWwQIn/8uOEz6nks6hLrB8W38jXXnsNeXl5UKlUuOKKK/D111+LbVJ4wfNCgrNj24D/fSQELsalAlNXAskFfbyU4ePTdfj0bAOcfPQN+cMdjuMwb0wq5FIO3zrzUexIBJx24Lu/AeZWsc0LGaIL0d/+9jesW7cOGzZswPHjx3H55Zfj2muvRUNDlKfQ5HlAfxEo/hQ4sgU4/U8h97RMARRcA0wpElbJesFodeDf39XgXJ0Bp2va0WigBPnhSHKcEvPHpiFGKUPC1B8Ln6tFDxx7S5h+R4HPiGMil1q94oorMG3aNLz66qsAAJ7nkZOTg5///Od49NFHe31te3s7dDod9Ho9tNoIXAXiecBhEW5WA2BpE1ZOjPVA+0WhBJAbuVqIls6aAii6j4hmjEFvtqPRYEV5swnn6tphdzLIJBwWXpZOO+3DHIvdCZVcKvgAv/8HqipLoZZLodalQpU5FtKEHEGklNqISbjf3/9RUZdObDYbjh07hvXr13vaJBIJ5s+fj//+979d+lutVlitnb/q7e393CjYUgYU/6f7Yx4dZt20ebX3t63PczI4eYbvqlog4e1eh5jnFfFqOYYnxwmjn8R8/KNagzZJNli1HKyqXkio5XpddkIMFk/I8Dzffqjcx6x0nQrXjEml5GcRgEruEheVFhdH/Aj/Ld+NzMaTkDaUAhdKIeGEyrwSjkN6ohaZiRpAIoPJzvB9TbsrC6cQsc1c9+CADJ0auQlqgONgsTtxslrvec/4GDlGpwX4A3XZj4XEe0FAVCFqamqC0+lEWprvKk5aWhr+97//dem/ceNGPPHEE/6/kcMq7HAOEzjGYLFYPM+dnBx2qRoWmRYWmQ5JKZkYPmUiEJsCSCSorr0A3sYAOLqcy+roLIwokXDQquRQyCTIjFdhVJoGWfHqzu0ERMSgVCiQNuEaVDdNBdd8DuqOamisDVA4jZAwJ3i7FbAJwsVsDjg79D2fTKrurOBid4I3tHUe4xVArKXbl/UJ4wN7XTdEVDDJ+vXrsW7dOs/z9vZ25OTk9P3C+Bzg8p9ckrfZ67Gnvbs2r/b+tvVxTg5AzmgrIFOCyVQAJ/F05zgOarkU8Np+8ZPpOZ1n8vzQceA4COVpvFh5lW8lVyIySY5TYvaoFGBUChjLh9XBw+rgYbM7wds6ECPjhf9e3g653Y7c0dauviTXc41KAqiEctdSniG3oHNWoZRzQGyAOat6cBEEgqhClJycDKlUivr6ep/2+vp6pKend+mvVCqhVAbwR1PEAonh8w/KAcj24zOk5PbRDcdxUMmlwtRNLQfg+31QAMhK6t+55ACykoNt4cARddVMoVBgypQp+PTTTz1tPM/j008/xYwZM0S0jCCIUCL61GzdunVYsWIFpk6diunTp2PTpk3o6OjA3XffLbZpBEGECNGF6LbbbkNjYyMee+wx1NXVYeLEifj444+7OLAJghi6iB5HNBAiPo6IIIY4/f0fFT2ymiAIgoSIIAjRISEiCEJ0RHdWDwS3e6vfWz0Igggp7v/NvlzRES1EBoNQl6tf0dUEQYiGwWCATqfr8XhEr5rxPI+amhpoNJpB20/l3kZSVVUVFStz0Xa9AF3zYF4zYwwGgwGZmZmQSHr2BEX0iEgikSA7Ozsk76XVaqPmSwpE3/UCdM2DRW8jITfkrCYIQnRIiAiCEB0Soj5QKpXYsGFDYLv+I5Bou16ArjkciGhnNUEQQwMaEREEITokRARBiA4JEUEQokNCRBCE6JAQwb9Ks2+++SZmzZqFhIQEJCQkYP78+RFXmTbQyro7duwAx3FYunTp4Bo4CPh7zW1tbVi9ejUyMjKgVCoxatQofPTRRyGyNjj4e82bNm3C6NGjoVarkZOTg4ceesin2sygwqKcHTt2MIVCwbZu3cpOnz7N7r33XhYfH8/q6+u77b9s2TL22muvsW+//ZadPXuWFRUVMZ1Ox6qrq0NseWD4e71uysrKWFZWFps1axZbsmRJaIwNEv5es9VqZVOnTmWLFi1iX375JSsrK2P79+9nJ06cCLHlgePvNb/zzjtMqVSyd955h5WVlbG9e/eyjIwM9tBDD4XE3qgXounTp7PVq1d7njudTpaZmck2btzYr9c7HA6m0WjYW2+9NVgmBpVArtfhcLArr7yS/elPf2IrVqyIOCHy95q3bNnCRowYwWw2W6hMDDr+XvPq1avZvHnzfNrWrVvHZs6cOah2uonqqZm70uz8+fM9bb1Vmu0Ok8kEu92OxMTe69CHA4Fe75NPPonU1FTcc889oTAzqARyzR988AFmzJiB1atXIy0tDePHj8czzzwDp9PZbf9wI5BrvvLKK3Hs2DHP9K20tBQfffQRFi1aFBKbI3rT60Dxt9JsdzzyyCPIzMz0+dDDlUCu98svv8Sf//xnnDhxIgQWBp9Arrm0tBSfffYZli9fjo8++gjFxcV44IEHYLfbsWHDhlCYPSACueZly5ahqakJV111FRhjcDgc+OlPf4pf/epXoTCZnNUD4dlnn8WOHTuwa9cuqFRDrwiiwWDAnXfeiTfffBPJyWFYlW+Q4Hkeqamp+OMf/4gpU6bgtttuw69//Wu88cYbYps2aOzfvx/PPPMMXn/9dRw/fhw7d+7E7t278dRTT4Xk/aN6RORvpVlvnn/+eTz77LP4z3/+gwkTJgymmUHD3+stKSlBeXk5brjhBk8bzwv1zmUyGc6dO4f8/PzBNXqABPIZZ2RkQC6XQyqVetrGjBmDuro62Gw2KBSKbl8XLgRyzb/5zW9w5513YtWqVQCAyy67DB0dHbjvvvvw61//utdcQsEgqkdEgVaa/d3vfoennnoKH3/8MaZOnRoKU4OCv9dbWFiIU6dO4cSJE57bjTfeiLlz5+LEiRMRkRkzkM945syZKC4u9oguAJw/fx4ZGRlhL0JAYNdsMpm6iI1biFkotqOGxCUexuzYsYMplUq2fft2dubMGXbfffex+Ph4VldXxxhj7M4772SPPvqop/+zzz7LFAoF+/vf/85qa2s9N4PBINYl+IW/13spkbhq5u81V1ZWMo1Gw9asWcPOnTvHPvzwQ5aamsp++9vfinUJfuPvNW/YsIFpNBr23nvvsdLSUvbJJ5+w/Px89uMf/zgk9ka9EDHG2CuvvMJyc3OZQqFg06dPZ4cPH/Ycmz17NluxYoXn+bBhwxiALrcNGzaE3vAA8ed6LyUShYgx/6/50KFD7IorrmBKpZKNGDGCPf3008zhcITY6oHhzzXb7Xb2+OOPs/z8fKZSqVhOTg574IEHWGtra0hspTQgBEGITlT7iAiCCA9IiAiCEB0SIoIgRIeEiCAI0SEhIghCdEiICIIQHRIigiBEh4SIIAjRISEiIort27cjPj7e8/zxxx/HxIkTPc+LiooiMpVttENCRHRLUVEROI7DT3/60y7HVq9eDY7jUFRU5NM/2AKQl5eHTZs2+bTddtttOH/+fI+v2bx5M7Zv3+55PmfOHDz44INBtYsIPiRERI/k5ORgx44dMJvNnjaLxYJ3330Xubm5otikVquRmpra43GdTuczYiIiAxIiokcmT56MnJwc7Ny509O2c+dO5ObmYtKkSQM6d3cjlaVLl3pGWXPmzEFFRQUeeughcBwHjuMAdJ2aXYr3yKyoqAgHDhzA5s2bPecoKytDQUEBnn/+eZ/XnThxAhzHobi4eEDXRQQGCRHRKytXrsS2bds8z7du3Yq777570N93586dyM7OxpNPPona2lrU1tb6fY7NmzdjxowZuPfeez3nyM3N7XJNALBt2zZcffXVKCgoCNYlEH5AQkT0yh133IEvv/wSFRUVqKiowFdffYU77rhj0N83MTERUqkUGo0G6enpfWbM7A6dTgeFQoGYmBjPOaRSKYqKinDu3DlPoni73Y53330XK1euDPZlEP0kqlPFEn2TkpKCxYsXY/v27WCMYfHixRGfvzozMxOLFy/G1q1bMX36dPz73/+G1WrFrbfeKrZpUQuNiIg+WblyJbZv34633noraKMGiUTSJQWp3W4Pyrn7w6pVqzyO+G3btuG2225DTExMyN6f8IWEiOiT6667DjabDXa7Hddee21QzpmSkuLj93E6nfj+++99+igUigHXEuvpHIsWLUJsbCy2bNmCjz/+mKZlIkNTM6JPpFIpzp4963ncE3q9vkv9s6SkpG6T7M+bNw/r1q3D7t27kZ+fjxdffBFtbW0+ffLy8nDw4EH85Cc/gVKpDGhKmJeXhyNHjqC8vBxxcXFITEyERCLx+IrWr1+PkSNH9losgRh8aERE9AutVgutVttrn/3792PSpEk+tyeeeKLbvitXrsSKFStw1113Yfbs2RgxYgTmzp3r0+fJJ59EeXk58vPzkZKSEpDdDz/8MKRSKcaOHYuUlBRUVlZ6jt1zzz2w2WwhWQUkeodyVhNRyxdffIFrrrkGVVVVXaqiEqGFhIiIOqxWKxobG7FixQqkp6fjnXfeEdukqIemZkTU8d5772HYsGFoa2vD7373O7HNIUAjIoIgwgAaEREEITokRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDgkRQRCiQ0JEEITo/H+nTfceIolbcQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:54:51.589817Z", + "iopub.status.busy": "2024-02-29T19:54:51.589257Z", + "iopub.status.idle": "2024-02-29T19:54:51.897193Z", + "shell.execute_reply": "2024-02-29T19:54:51.896228Z" + }, + "papermill": { + "duration": 0.333453, + "end_time": "2024-02-29T19:54:51.899228", + "exception": false, + "start_time": "2024-02-29T19:54:51.565775", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHCElEQVR4nO2deXhU5fXHv3f2mcySbbKRhGzsuyDIJqhURUX84UJdQVCrYluhtJZaRbSK2mqhraWtFqKtglbBWrUiooiCKLtsDWQjgewhmSWz3/v+/rgzNzNkn0zmzvJ+nmeeydxtzk1mvjnvec97DkMIIaBQKBQRkYhtAIVCoVAholAookOFiEKhiA4VIgqFIjpUiCgUiuhQIaJQKKJDhYhCoYgOFSIKhSI6MrEN6A8cx6GmpgY6nQ4Mw4htDoVCuQhCCCwWC7KysiCRdO33RLUQ1dTUICcnR2wzKBRKD1RXVyM7O7vL/VEtRDqdDgB/k3q9XmRrKBTKxZjNZuTk5Ajf1a6IaiHyDcf0ej0VIgolgukpdEKD1RQKRXSoEFEoFNGhQkShUESHChGFQhEdKkQUCkV0RBei8+fP46677kJKSgrUajXGjBmDAwcOiG0WhUIJI6JO37e0tGD69Om44oor8N///hdGoxFnzpxBUlKSmGZRKAAhQNU+oKkE0GcDBbMAqVxsq2IWUYXohRdeQE5ODjZt2iRsy8/PF9EiSjzi8nBweFjolLL2fJfyL4Cqb/mfzbWA2waMvFE8I2McUYdmH3zwASZNmoRbb70VaWlpmDBhAl599dUuj3c6nTCbzQEPCqU//K/OjFe/Kse/j9S0bzTXANXf8T8PmggwDFB/AjCdE8fIOEBUISovL8eGDRswZMgQbN++HQ899BB+8pOf4PXXX+/0+LVr18JgMAgPus6M0h9qTXZ8crwOLg+HsYMM7d5Q5df80Cx9FDD0aiB9NL/9/CHxjI1xGDHbCSkUCkyaNAl79+4Vtv3kJz/B/v378c0333Q43ul0wul0Cq9961hMJhNd4kHpE4QQvPVdFRrMTgzL0GHu6AxeiKyNwP7XeC9o8gOAJpn3kA6+DkhkwPSfAjKF2OZHDWazGQaDocfvqKgeUWZmJkaOHBmwbcSIEaiqqur0eKVSKawro+vLKP3hXIsdDWYnFDIJZg8zgmEYeFgOpcf24mStGSSliBchANBlAupEgPMArWdFtTtWEVWIpk+fjpKSkoBtp0+fxuDBg0WyiBIvHD3XCgAYkamDRsHP2XhYFvVnDsFkd6NOPaT9YIYBkgv5n5vLwmxpfCCqEC1fvhz79u3Dc889h9LSUrz11lv429/+hmXLlolpFiXGcbhZlDW0AQDGDEoUtqus55Ch8sAjUeKoPSXwpKQ8/tlUHR4j4wxRhejSSy/Ftm3bsHnzZowePRrPPPMM1q1bhzvvvFNMsygxjtnuhl4tQ6pWAaNO2b6j4RTS9Eo0awpQ2miHm+Xa9+mz+GdbM+B2hNfgOED0ekQ33HADbrjhBrHNoMQRaXoVFk/Lg9PjJzSEAM2l0CplcCUNhZslqLpgQ6FRy+9Xavk4kb0VsNQCyTTfLZSIvsSDQhEDhmGgkkvbN1hqAVcbGKkCxpwiAEB5Y1vgST6vyFwDSmihQkSJK5weFhzXScZKcyn/nFyAwjQDAKC80Rp4rDaDf7bWD7CV8QcVIkpccaCyBRu+LMPBsxcCd/iEKKUIg5LU0KvlyEpUw+UfJ0pI5Z/bmsJjbBwheoyIQgkn51pscHk4KGV+wzKHGbDU89P0KYWQShgsmZ7Xsc5ygpF/trcArAeQ0q9PqKAeESVucHk41Jn4zPycZE37jgve3CBdJqBIANBFsXelDpCrAMLxs2eUkEGFiBI31LTawRECvVoOg9qvpMeFCv45pTDgeEIIWm2u9jgRw7R7RW2NYbA4fqBCRIkbqltsAICcJHX7Ro5rX7aRFDgl/+a3Vdi0pxL1Fr+8IY030dF+UYyJ0i+oEFHihnMtdgBAdpLfsMxaxycoypT80MwPnYqPAdWZ/IRI7S3aZ28ZUFvjDSpElLjA6WFRb+YFJTvZzyPyDcuSBgMX9WbP0KsAUCEKBzTsT4kLCAEuK0hBS5sLepVffKilkn/2rSXzI9PAC1YtFaIBhwoRJS5QyaW4rOCihaweF2A+z/+c1HHJRppeCYYBTHY3bC4Pv0pflcjvdDsAtx2QqzucR+k7dGhGiV9M1QDHAipDu6fjh0ouRUoCXwRN8IpkCn7dGUC9ohBChYgS87AcQWmDBW1OT+COFl98KI+fmu+EDO/wjMaJBhY6NKPEPI0WJ/5ztBYquRQPzipoT1b0Baq7WUlfYEyAUiZBrn8CpDoJaK3mV+JTQgIVIkrMc76Vn7bPSlS1i5DTwq8ZYxggseuKoIVGbXspEB9Kb4liJ+0iEyro0IwS89QIQuQXWPbNlmnTAYWm40ndofIKkYMKUaigQkSJaQghghAN6kyIOpm2vxiHm0X1BRtMdje/Qanjn6lHFDKoEFFimlabGzYXC5mEQZqvLCwh7ULUi0qLn52qx7sHz6G0wcJvUPL1iuAw8dei9BsqRJSYxhcfSjeoIJN6P+5tTYDTypfx0Gf3eA2jlhewRou3p55vaMa6AY+zi7MofYEKESWm6XZYZsjtVU0hX4F9QYik8vZERjo8Cwl01owS00wpSEF2kiawW4d//lAvSEngz22xucFxBBIJw3tFbjsfsNamhdjq+IMKESWmMVxce4hjgVZvJ+FeCpFeLYNcysDNEpjsbiQlKPgpfEs94DSF3ug4hA7NKPGF6Rwf21Foeu3JMAzDiw+ACzYXv1HlC1jToVkoEFWInnrqKTAME/AYPny4mCZRYoiDZ1tw8GwLLA53+0b/afsulnV0RrLGK0RtXiGiSY0hRfSh2ahRo/DZZ58Jr2Uy0U2ixACEEBw62wKr04M0nRI6X+kPQYj61iBxZJYeg5LU7UFvIZfIEhqD4xzRv/UymQwZGRlim0GJMVpsblidHsgkDDIMfIEzuO18I0Wg1/EhH4NTEgI3eIvsw9XW8WBKnxE9RnTmzBlkZWWhoKAAd955J6qqqsQ2iRIDVF/g61NnJqoh9+UPtVbxCYgJqe25QMFCPaKQIqpHNGXKFBQXF2PYsGGora3FmjVrMHPmTBw/fhw6na7D8U6nE05newKZ2UzH55TOOXuhk0L5F/o2bX8xNa12tNhcKDRqofJ5RL6kRpmy+5Mp3SKqRzR37lzceuutGDt2LK655hp8/PHHaG1txTvvvNPp8WvXroXBYBAeOTk5YbaYEg14WE7wiPJT/YZUfVhf1hnbT9Th0xP1fGKjTMknNgJ0eBYCRB+a+ZOYmIihQ4eitLS00/2rVq2CyWQSHtXV1WG2kBINnGuxw+XhoFXK2hMZHSa+kBkjARJzg7puooYXno6LX+nwrL9ElBBZrVaUlZUhMzOz0/1KpRJ6vT7gQaFcjMnuhkzCID81ob3+kC+JUZcR9DAqUc1P4bfavEJEA9YhQ9QY0cqVKzFv3jwMHjwYNTU1WL16NaRSKW6//XYxzaJEOeNyEjEySw+Xh2vf2OJtohikNwQAhos9IoW3YJrLGvQ1KTyiCtG5c+dw++23o7m5GUajETNmzMC+fftgNBrFNIsSA8ilkvbZMkL8url2XY2xJ3xLRVrtvqRGrxDRoVm/EVWItmzZIubbU2IQN8u1C5APewu/FEMiBQzBT3Ak+oTI5gYhBIzgEdGhWX8RPaGRQgkl2w6dh5PlMGdEmtAgUYgP6bPaZ7qCwKCWg2EAl4eD3c1CQ4dmIYMKESVmsLtY1JjsfM6i0u+j3dr/+BAAyKQS/GBkOrRKGRRSSfvQjHpE/YYKESVmqGhqAyFAqk7Z3laaEL9AdfDxIR+jsgztLxQ0RhQqImr6nkLpDxVNvGdS6J/EaGvmPRaJDNAPCu0b+oTI4+QzrClBQ4WIEhN4WA6VzbwQFfj3IfMNywyDelUWtidMdjdO1phR2mDl85Ek3mvSOFG/oEJEiQnOt/LZ1AlKKdL1/mVhQzcsA4Bakx3bT9ThcFULX8+IxolCAhUiSkxQ3uj1hlK17dnUhAAm7zKgfgaqffjqGlkcHn6D3Nuc0WULyfXjFRqspsQE+akJcHo4DEn3G5bZW3iBkMgAXefLhvqKTsV/ZSwOD19I37fMw009ov5AhYgSE+SlJiAv9aLiZT5vSJ8ZkvgQAGgVMkgYBhwhaHN5oKMeUUigQzNK7GI6xz8bem6i2FskEgZaP68ICq8QuakQ9QcqRJSo5391ZtSbHeC4i9o/C0IU2rpVvuGZ2eEG5L4V+HTWrD9QIaJENW6Ww44T9Xjr2yq02v1yeZxWwHaBn9kKcf6Q3j9graBDs1BAY0SUqKam1Q4PR6BTyZCk8VtH5vOGElIBuSqk7zk+JxHDM3RI1SmBNm+DRTo06xdUiChRTZWvNnWypn3aHhiwYRmA9q4gAOCmeUShgA7NKFGNT4gGp2gCd/hmzEIYqO4U39DM4wA4rvtjKV1ChYgStTjcLF/IHkBOkp8QeVyAtYH/eQCEyM1yOFljxncVF0BkKj4ORQgdnvUDKkSUqKXO5AAhQJJGHlj2w3weIBzfu0xl6PoC/WD7iTrsKW2CkwUg99Y9okIUNFSIKFFLjckOAMgwqAN3DGB8CODL0KrkUgCA1enxW+ZB40TBQoPVlKjl0rxk5CZr+CJl/phr+OdQl/3wQ6uUwuFmYXV4kKpIANqaqEfUD6gQUaIWuVSC7KSLgtSEABafEGUN2HtrVTI0WV3UIwoRdGhGiS3sLYDbwS901aYN2NskKPj/4Vanh/Y3CwFUiChRyZl6C74oaRBaSwuYz/PPunS+a8cA4Vtv1ubvEdGhWdBQIaJEJWWNVhypasW5FnvgDnMt/zyAwzIA0Cr9PSK6zKO/0BgRJSpp8OYPBWQ5A34e0cAKUV5qAv5vwiAkauSAzStAtCZR0ESMR/T888+DYRg8+uijYptCiXBcHg4X2vhuq2k6v7KwrLs9kXGAPSK9So681AQkahS0SmMIiAgh2r9/P/76179i7NixYptCiQKarE4Qwg+PAhIZrfV8IqMiYcASGTtFqEnUxs/aUfqM6EJktVpx55134tVXX0VSUpLY5lCiAN+wLM2/SD7glz+UxS+7GGBO1ZrxbXkz7PAOD1kPbSsUJKIL0bJly3D99ddjzpw5PR7rdDphNpsDHpT4o8HsAAAYdd0IURjYW9aMvWXNaHGQ9lK0tEBaUIgqRFu2bMGhQ4ewdu3aXh2/du1aGAwG4ZGTMzAp/JTIxtdBI013caDaK0QhKpTfE1olnx7Q5mL9pvDt3ZxB6QrRZs2qq6vx05/+FDt27IBK1bvCVatWrcKKFSuE12azmYpRHHLzxGxYnZ7ApR1OK+AweSsyhscj0irlAByw+HKJHGaaSxQkognRwYMH0dDQgEsuuUTYxrIsdu/ejT/96U9wOp2QSgMT0pRKJZRK5cWXosQhWuVFH12LN39Ik8J3YA0DCV6PyOqg2dX9RTQhuuqqq3Ds2LGAbffeey+GDx+Oxx57rIMIUSjd4ssfCpM3BLQX0Q/MrqZDs2AQTYh0Oh1Gjx4dsC0hIQEpKSkdtlMoPg5UXsC5FjtGDzKgKM2vmWKYMqr98aUOtLlYQO2rSUQ9omAQfdaMQukL51vtqGhqg83lad/ov+J+gDOq/fEtfLW5/IdmNEYUDBG1xGPXrl1im0CJcJqtfEZ1coKifaOtmS8PK5UBCcaw2WLUKbHgkkF8vMriTSWhweqgiCgholC6w+XhYPL2LktJ8AtIC+vLMgFJ+Jx8lVyKwSleT8jpfaZCFBR0aEaJGlpsvDekUUihVvhNZoQ5kbFTfHWr6dAsKKhHRIkaOh2WAWEpDdsVp+staLW5McyggAHgPSJCwrLEJJagQhShsCwLt5uuW/KnxdwGtYSFUcPA4eCXecDjAuw2QKIBFCmAb3uYOHa2EY0WJwxDkqGUeKfw2yyATNH9iTGCXC4PSaoNQ0jflwuXl5ejoKCg32/eX8xmMwwGA0wmE/R6vdjmhARCCOrq6tDa2iq2KRGHw83CzXJQyqRQyLxRBc7DJxEyEkCpC7tNNhcLD8tBJZdC4bECIIBCF9ZYldgkJiYiIyMjsNOul95+R4PyiIqKijBr1iwsXboUt9xyS6+XaFB6xidCaWlp0Gg0nf5x4xnf/03h9+Jq4x8yZXhLf3ixOtxweDio5VIkeEwAYQFVYlx4RIQQ2Gw2NDTwNaAyM4Nf4xeUEB06dAibNm3CihUr8Mgjj2DhwoVYunQpJk+eHLQhFH445hOhlJQUsc2JDogdgBxQJgDK8P9D9EAG1uWBXC6FSqbky4AoFYA8Pv45q72JnA0NDUhLSwt6mBaU/zh+/HisX78eNTU12LhxI2prazFjxgyMHj0aL7/8MhobG4MyJt7xxYQ0Gk0PR1IEfPV/pHJR3t43AuMI4YeHAF+cLY7wfV77E9Ps10BWJpNhwYIF+Ne//oUXXngBpaWlWLlyJXJycnDPPfegtra2P5ePW+hwrCMON4smqxNWh9+HnWP5BwPxhMj7t2LjWIhC8XntlxAdOHAADz/8MDIzM/Hyyy9j5cqVKCsrw44dO1BTU4P58+f320AKBQA8LAeWI+D8p1Z83pBE1i4CYcYnRByHuBWiUBDUX+/ll1/GmDFjMG3aNNTU1OCNN97A2bNn8Zvf/Ab5+fmYOXMmiouLcejQoVDbS4khFi9ejJtuuqlXx3q8CiSV+P335XxC1B4Ynj17do8NGPLy8rBu3bo+WNo1MimDRI0cSRo5FaJ+EJQQbdiwAXfccQfOnj2L999/HzfccAMkF01XpqWl4e9//3tIjKREPr0RgP7gEyKZ1E+IWD7BUaxhGcB7REqZFDKppD1gFCNC9NRTT2H8+PFhea+gZs127NiB3NzcDuJDCEF1dTVyc3OhUCiwaNGikBhJiW8IIWB9QiR82YnogeoOCB4RK64dUUhQHlFhYSGampo6bL9w4QLy8/P7bRQluli8eDG+/PJLrF+/HgzDgGEYlJWVYenSpcjPz4darcawYcOwfv36Ts9fs2YNjEYj9Ho9HnzwQbhcroD9Pm+IYQBhZMZ50GZtwz0/+gm0hiRkZmbipZde6nDthoYGzJs3D2q1Gvn5+XjzzTc7HMMwDDZs2IC5c+dCrVajoKAA7777rrC/srISDMPgnXfewcyZM6FWq3HppZfi9OnT2L9/PyZOnAStVotrb5iPxqbmXntEGzduxKhRo6BUKpGZmYlHHnlE2FdVVYX58+dDq9VCr9fjtttuQ319vbDf56384x//QF5eHgwGA374wx/CYrEIx3AchxdffBFFRUVQKpXIzc3Fs88+K+x/7LHHMHToUGg0GhQUFOCJJ54QZr6Ki4uxZs0aHD16VPibFhcX9+q+goIEAcMwpL6+vsP2yspKotFogrlkUJhMJgKAmEymsL3nQGK328nJkyeJ3W7vsM/pZrt8uD1sr4919eLYvtLa2kqmTp1K7r//flJbW0tqa2uJw+EgTz75JNm/fz8pLy8n//znP4lGoyFvv/22cN6iRYuIVqslCxcuJMePHycffvghMRqN5Fe/+lXg78XlIXUmO2m2OvwMbyMPLV1EcnOyyWeffUa+//57csMNNxCdTkd++tOfCofNnTuXjBs3jnzzzTfkwIEDZNq0aUStVpPf//73wjEASEpKCnn11VdJSUkJ+fWvf02kUik5efIkIYSQiooKAoAMHz6cfPLJJ+TkyZPksssuIxMnTiSzZ88mH+/4guzY/Q0pLCwkDy65hxBzLSEc1+3v7M9//jNRqVRk3bp1pKSkhHz33XeCTSzLkvHjx5MZM2aQAwcOkH379pGJEyeSWbNmCeevXr2aaLVasmDBAnLs2DGye/dukpGREfC7+8UvfkGSkpJIcXExKS0tJV999RV59dVXhf3PPPMM2bNnD6moqCAffPABSU9PJy+88AIhhBCbzUZ+9rOfkVGjRgl/U5vN1um9dPe57e13tE9CtHz5crJ8+XIikUjIj370I+H18uXLyU9+8hMyZcoUMm3atL5csl/EkxC9/GlJl49th84FHPvHnae7PPad/VUBx27YVdrhmGCYNWtWgAB0xrJly8jNN98svF60aBFJTk4mbW1t7fZs2EC0Wi1h2XZBtLs8pMniICabS9hmaTxHFAoFeeefxcK25uZmolarBTtKSkoIAPLdd98Jx5w6dYoA6CBEDz74YICtU6ZMIQ899BAhpF2IXnvtNWH/5s2bCQCyc+dO0trmJHUmO1nzzG/IsCGFhJhqCGG7F/SsrCzy+OOPd7rv008/JVKplFRVtf+tTpw4EXAvq1evJhqNhpjNZuGYn//852TKlCmEEELMZjNRKpUBwtMTv/3tb8nEiROF16tXrybjxo3r8bxQCFGfYkSHDx/2eVE4duwYFIr22QqFQoFx48Zh5cqV/XfTKDHBK6+8go0bN6Kqqgp2ux0ul6tD8HPcuHEBCZxTp06F1WpFdXU1Bg8eDICv+6OSB2bslpWegcvlwpTLpgrbkpOTMWzYMOH1qVOnIJPJMHHiRGHb8OHDkZiY2MHWqVOndnh95MiRgG3+nYjT09MBAGPGjIHEO140pqWjobGZP4Bw6Cry0dDQgJqaGlx11VWd7j916hRycnICOtSMHDkSiYmJOHXqFC699FIA/OyfTte+vi4zM1NYbnHq1Ck4nc4u3wMA3n77bfzhD39AWVkZrFYrPB6PaGs2+yREX3zxBQC+yP369etjZqFpNLDsiqIu90kuyid74PLCLo+9OPdsyfSBielt2bIFK1euxEsvvYSpU6dCp9Pht7/9Lb799tv+X5xw/GJXoL2xYRiQy9uD4r4kPrlcLuQSEQCcLz7UTZzItywilPb4bOI4rlfv8c033+DOO+/EmjVrcM0118BgMGDLli2dxtnCQVDB6k2bNlERCjMKmaTLh0wq6fWx8l4cG5R9CgVYtn22aM+ePZg2bRoefvhhTJgwAUVFRSgrK+tw3tGjR2G3t3e+2LdvH7RareANED58EHgS60FhXh7kcjm+/e6AsLmlpQWnT58WXg8fPhwejwcHDx4UtpWUlHRa2WDfvn0dXo8YMaJX9y4Ikb+Z3QiRTqdDXl4edu7c2en+ESNGoLq6GtXV1cK2kydPorW1FSNHjuyVTUOGDIFare7yPfbu3YvBgwfj8ccfx6RJkzBkyBCcPXs24JiL/6YDSa//nSxYsADFxcXQ6/VYsGBBt8du3bq134ZRoou8vDx8++23qKyshFarxZAhQ/DGG29g+/btyM/Pxz/+8Q/s37+/w6yqy+XC0qVL8etf/xqVlZVYvXo1HnnkESE1hOUImttckEsl7QXRODe02gQsXXQnfv6LXyAlNRVpaWl4/PHHA1JKhg0bhmuvvRY/+tGPsGHDBshkMjz66KOdegv/+te/MGnSJMyYMQNvvvkmvvvuu17nwQWsN/PRw8zZU089hQcffBBpaWmYO3cuLBYL9uzZgx//+MeYM2cOxowZgzvvvBPr1q2Dx+PBww8/jFmzZmHSpEm9skmlUuGxxx7DL37xCygUCkyfPh2NjY04ceIEli5diiFDhqCqqgpbtmzBpZdeio8++gjbtm0LuEZeXh4qKipw5MgRZGdnQ6fTDVhfwV7/+zMYDII76t/2ubMHJf5YuXIlpFIpRo4cCaPRiGuuuQYLFizAwoULMWXKFDQ3N+Phhx/ucN5VV12FIUOG4PLLL8fChQtx44034qmnnhL2+6buCfy+5N5Ext8+vxYzZ87EvHnzMGfOHMyYMSMgHgTw3ntWVhZmzZqFBQsW4IEHHkBaWloHO9asWYMtW7Zg7NixeOONN7B58+Zeex/+QzOBHoRo0aJFWLduHf785z9j1KhRuOGGG3DmzBkA/BDr3//+N5KSknD55Zdjzpw5KCgowNtvv90re3w88cQT+NnPfoYnn3wSI0aMwMKFC4UY0o033ojly5fjkUcewfjx47F371488cQTAefffPPNuPbaa3HFFVfAaDRi8+bNfXr/vhBUYbRIIdYKozkcDlRUVCA/P5/WePJidXrQ5vRAJZfCoPbGRKwN/GJXTXJIuroyDINt27b1ernJxXCEwM1ykDAM5G4rXx9JkQCoov8z2Ru6+9z29jsaVEDAbrfDZmsvEn727FmsW7cOn376aTCXo1C6hGV5z0Lmi8hHwIr7i/Et85BLJYDEO7tHs6v7RFBCNH/+fLzxxhsAgNbWVkyePBkvvfQS5s+fjw0bNoTUQEp8UlVVBa1Wi0FpySjISkV6SiK0Wi20egO0WUWoOl8v2or7bvHapDXm8PZ28vjqq69ENjLyCLpC4+9//3sAwLvvvouMjAwcPnwY7733Hp588kk89NBDvbrOhg0bsGHDBlRWVgIARo0ahSeffBJz584NxixKDJGVlYXDhw8LnTsSE+T8OjOnBXDZkZWd08MVek8oohMONwsPR6Bi+C/Vkb07AU3nVTYHDQp/t5FIJyghstlsQiLVp59+igULFkAikeCyyy7rMAXYHdnZ2Xj++ecxZMgQEELw+uuvY/78+Th8+DBGjRoVjGmUGEEmkyG/oBD6NhcY8F1VGYYB2pq85Vgjq4qlw83C6eEgVTCQASgqyAO06WKbFTUE5dsWFRXh/fffR3V1NbZv346rr74aAJ8x2peg8bx583DddddhyJAhGDp0KJ599llotdoOOR2U+EUpk0Apk/AiRLj2GkQREh/yIRRII95YFuEuSiyidEdQQvTkk09i5cqVyMvLw5QpU4T0+E8//RQTJkwIyhCWZbFlyxa0tbV1SLf34XQ6YTabAx6U2EUmlSBRo4BB480fYt38HLlEyldljCB8sXQWPiECFaI+ENRf85ZbbsGMGTNQW1uLcePGCduvuuoq/N///V+frnXs2DFMnToVDocDWq0W27Zt6zJ/Y+3atVizZk0wJlNigUirP+SHb70ZR8CvoyGk2/VmlEBEzyNyuVyoqqqCyWTCu+++i9deew1ffvllp2LkdDrhdDqF12azGTk5OTSPKEZhOQIJ41ec3XYB8Dj5/BxFgrjGXYTDzcJkd/MZ4KQ1pHlOkU4o8oiC8oja2trw/PPPY+fOnWhoaBAW2vkoLy/v9bUUCgWKivgFnRMnTsT+/fuxfv16/PWvf+1wrFKpHLAUc0pkQQhBk9UJBkCKVgEpw0REadiuaI8R+bp5sHRo1geCEqL77rsPX375Je6++25kZmaGtP0Nx3EBXg8lPmH92nVIGIZfbU+It0xjJAoR/8xxBJDRkrF9JSgh+u9//4uPPvoI06dP79ebr1q1CnPnzkVubi4sFgveeust7Nq1C9u3b+/XdSnhZ/bs2Rg/fnzIumP4d+1gLvaGevGPL9T29IRUwiBJw5cEIS7ek4uVIvrhICghSkpKQnJycr/fvKGhQWjEaDAYMHbsWGzfvh0/+MEP+n1tSvThcrmEYnvtXTu83oUgRJHZU55hGChk3uUdMdbNIyz0sopkAP/4xz/ILbfcElDiUwziqVRsJLNo0SIC74S171FaWkqWLFlC8vLyiEqlIkOHDiXr1q3rcN78+fPJb37zG5KZmUny8vIIIYTs2bOHjB4zliiVSjLhkkvItq1bCQBy+KtPCXHzdauPHTtGrr32WpKQkEDS0tLIXXfdRRobG7u0p6KiIny/EKeVLxdruxC+9xSRsJeK9fHSSy+hrKwM6enpyPMWqPKHNlYMIf5tc8JJL4dAALB+/XqcPn0ao0ePxtNPPw2A95qzs7Pxr3/9CykpKdi7dy8eeOABZGZm4rbbbhPO3blzJ/R6PXbs2AGAn2WZN28errr6GrzyWjFa68/j5ytX8AczDCBVoLW1FVdeeSXuu+8+/P73v4fdbsdjjz2G2267DZ9//nmn9hiNxhD+cjpHWOZBvEMN6hH1mqCEKNhyCZQgYN3AVyKU75z5M0DWu2GQwWCAQqGARqNBRkaGsN0/5ys/Px/ffPMN3nnnnQAhSkhIwGuvvSYMyf7yl7+AYRj8dv2foVKpkDJpHBrOn8X9Dz3ibS3N4E9/+hMmTJiA5557TrjOxo0bkZOTg9OnT2Po0KGd2jPQCMs85AwVoj4SlBCtXr061HZQYpDeFM8fM2ZMQBOGkpISjB07Fsn6BLAcgZRhMPkSb9Ksd9r+6NGj+OKLL6DVaju8Z1lZGYYOHTpg99QdnS7zoPSKoPPkW1tb8e6776KsrAw///nPkZycjEOHDiE9PZ2uLg4lUjnvnYjxvv2gt8XzExI6T0zUqbzvT0iHHvdWqxXz5s3DCy+80OG8zMzMftndH3zZ1e3LPLj2lANKtwQlRN9//z3mzJkDg8GAyspK3H///UhOTsbWrVtRVVUl1CqihACG6fUQSUy6K57vo7Pi+RczbNgw/POf/4TT6eSTVzkP9h/wxhy9HTsuueQSvPfee8jLy4NM1vlHOJyF331IfblEAevNOICRdnkOhSeohTArVqzA4sWLcebMmYCU7uuuuw67d+8OmXGU6MG/eH5TUxOGDBmCAwcOYPv27Th9+jSeeOIJ7N+/v8fr3HHHHeA4Dvff/wBOnjyJ7f/9GL/7418AAIx3WnzZsmW4cOECbr/9duzfvx9lZWXYvn077r33XkF8Lrbn4uz/gcA3NGM5tHtBdHjWK4ISov379+NHP/pRh+2DBg1CXV1dv42iRB/BFs+/GL1ej3+8/R4OHj6MCRMm4PEnV+PJx5YDgPBPLysrC3v27AHLsrj66qsxZswYPProo0hMTBS6eFxsT1VV1cDdvJf2GBHavSAqRL0jmLwBo9FIDh06RAghRKvVkrKyMkII3yo3Ozs7mEsGBc0jij04jiN1JjupM9mJx8MSYq4l/3z1T0Qul3fZez1S8LCsYDtnbeRziVyRbXMoCEUeUVAe0Y033oinn34abjcfRGQYBlVVVXjsscdw8803h1AmKZEEIQQON4s2pydgLVgo8XAE72x+E999swdny0/j/f98jMdWP4vbbrstZB1SBwoJwy/zSElQUI+ojwQlRC+99BKsViuMRiPsdjtmzZqFoqIi6HQ6PPvss6G2kRIhtDn5UhdWpwctba6Q1Hq+GA9H0FBfh2UPLMHIMeOxfNVTuHXBTfjb3/4W8vcKNb5lHjKpRIhnUSHqHUHNmhkMBuzYsQN79uzB0aNHYbVacckll2DOnDmhto8SIXhYDm0uvt88wwA6tSykVRf83+eRR3+Gn//8F9BzrQDrAdSJgDyyvaEOMFSI+kKfhYjjOBQXF2Pr1q2orKwEwzDIz89HRkYGCCED8uGkiI/Dzc9GyaUSJGnkA/Z39rC8lyWXEMDt8fYvi/z0BR9ONws3R6DiaHZ1X+jT0IwQghtvvBH33Xcfzp8/jzFjxmDUqFE4e/YsFi9e3OcysZTogBACu5v/QmkU0gETIUIIPN5pdhnxJTHK25sWRgG+GJrbN2qlQtQr+uQRFRcXY/fu3di5cyeuuOKKgH2ff/45brrpJrzxxhu45557QmpkvBGOnJe+4OEIOELAAFB4i36xHIHd5QFHAL06dIXKtEoZ3ByBjPN2Eo6yUqvttavjJ48oFJ/XPgnR5s2b8atf/aqDCAHAlVdeiV/+8pd48803qRAFiUKhgEQiQU1NDYxGIxQKRUQMdV0eFpybhUQCuLzFM90sB5PdDQaAHKGzkwGgIByc9jZ+eYQUAHGE5NrhwO3ywO1iYZdwkHJugGEBafTY3xcIIXC5XGhsbIREIglYM9hX+iRE33//PV588cUu98+dOxd/+MMfgjYm3pFIJMjPz0dtbS1qamrENqcDhBBc8BMci8PDb1NI2wuYhQLWDbhtfMBX2Ra664YBN8vB7mIhkwAa4vXqlOaYXm+m0WiQm5srJJMGQ5+E6MKFC0hP77p7ZXp6OlpaWoI2hsJ7Rbm5ufB4PGFfK9VXvj7TiNIGK0YPMmBSXv8rdp5vsUEmlSC5bg/k5lIgYywweGwILA0f51ts2HuyHkkaOea7vuA3TrgbUERWZ9pQIZVKIZP1fwa1T0LEsmyXiwx9Rnk8nn4ZROHzUeRyeYeCc2LAcQSMf0sfPzJT9DhWZ0eNhQ1J+6OvK+rQYrVjETkGnYIA6UVAlLVV0msZ2DkpGBcDlZQD3A5AwkbdfYSbPgkRIQSLFy/usqUP7b4Re5xusGDnqQaMyNThyuGB3nCWgc/tqTc74GG5fg3P3CyHFpsLekcNEjQsoNAC+ugrJ5Og5Gf47G4WnFIDidsBuNsADHyFyGimT0K0aNGiHo+hgerYosHshMvT+axIokYOtUIKu4tFo9WJTEPwSYdNVicIAQa5yqHQSQDj8PYi9FGEWi7FLROzkaCUgSnRAPYLgNsutlkRT5+EaNOmTQNlByVCabDwXm6aruPQgmEYZBpUqGl1oM3ZvyF5vdkJCedBDlsNQA2kdd52PNJhGAY5yd54kK8brcsmnkFRQtAVGimxDyEEDRZ+6jlN3/lw/JpRGVDKJP0OVtaZHEiyV0InJ4DKABiy+3W9iEDuFSR3dM38iQEVIkqXWJweON0cpBIGKQmdC5FKHpqs53qzA0ZbGbQGGZA+Mqqnuyua2lBncmCIS4pUgHpEvSD6BuGUsNHaxi+zMKjlkEp6FoZgV+M7PSysllYk2qugVcqA9NFBXSdSOFNvwb7yZtTbvSLtpkLUE6IK0dq1a3HppZdCp9MhLS0NN910E0pKSsQ0ieJHi43vrpqo6T6N4OszTXjtq3KUNVqDeh+ZRIKF2a0oTNVAnpQNJKQGdZ1IIUHJDzSs8HqRVIh6RFQh+vLLL7Fs2TLs27cPO3bsgNvtxtVXX422NjqmjgQSlDLkpyYgK7H72TCHm4XF4UGtKbilDFIGSLWcQrpeBWRNCOoakYRGwXtCbax3yQMdmvWIqDGiTz75JOB1cXEx0tLScPDgQVx++eUiWUXxUZSmRVFax95hF5NhUOHYeRPqghQitFQA9lZ+gWvaiOCuEUH4PCIL5/UkqUfUIxEVrDaZTACA5OTOlws4nc6ApEmz2RwWuyjdk67np/YbLE5wHBFWoPeWkoO7kGR3IGnoBMj72U8tEvB5RBafR+S2AxwbVeVMwk3EBKs5jsOjjz6K6dOnY/TozoOVa9euhcFgEB45OTlhtjJ+4DgiFEPriZQEBRQyCVweDs1trj69j7WlHheqTqKsyQYu65JgTI04EhT8/3eTR95eqdFFww3dETFCtGzZMhw/fhxbtmzp8phVq1bBZDIJj+rq6jBaGF+YHW5s2FWG174q73E2TCJhkKbjA7P15r4Nz0wlXwMAPEmFUOrTgjM2wtB4l3m4WAKPzJtL5AoukB8vRMTQ7JFHHsGHH36I3bt3Izu760Q2pVLZ5To3SmhpsfFT90p57yoyZhrUONdiR53JgdGDDL17E6cFzvNHAQDS3KlB2xppKKQSYZmH5JQOcFsBJxWi7hBViAgh+PGPf4xt27Zh165dyM/PF9Mcih/C1H0vqy9mJqqQrlfB0MNUfwBn98Jqd8KizMDgQQXBmBmRBCzzUGoBCwCXRVSbIh1RhWjZsmV466238O9//xs6nU7oEmswGCK+h1WsY/J6REma3lXdKzRqUWjseYZNwHYBnnOHYHOyqE6bhKmJMVomQ6njn6lH1C2ixog2bNgAk8mE2bNnIzMzU3i8/fbbYppFQe+TGYOmYjesDhdaVNlA0mDoVNE/W+ZPVbMNe8ua0OD0/q+nMaJuEX1oRolMfDGipIS+1SF2sxycHo5fqtEVF8qBhlOwuThUJ05Gbgx6Q+VNVhyuaoVaL0MaQD2iHoiIYDUlsvCwHCwOXoh6GyMCgBM1Jnx2sgGFaQm4YWxW5wexbuD0dgBA1qgZuDN3stDLLJbQqbxJjcQ7uUJjRN1ChYjSAQ9HMC47EWaHW0jO6w16lRwcId1nWJ/ZwWdRK3VA/uVI6Kb0cDTjy65uZb1CRD2ibonNTwGlX6jkUlwxvO85Pel6FRiG7+5hdXo6Ds9qjgC1R/kSH8Ovj7qeZX1Be7EQue18+2wp/cp1RsQkNFKiH4VMghQt/8Xr4BXVnwROe9cWDp6Or5t1eO/gOZxtjs2MY52SH9KaXDIQIbuaekVdQYWI0gGzw93r5R0Xk+FddyYIEesByr8ETv6bb5iYOQ7Im4HyJiuqLti6rIcd7fiK6HsI4Jb5SsZSIeoK6idSOrD9eB3Otdhx3ZhMDMvQ9f5EQpCVQFDhakLb+Tr+01V3DHB4FydnTwIKr4LJ4UGz1QWGQXviX4whk0qgUUhhc7FwSNRQwELjRN1AhYjSgVbv1L1e3cPHw+MCms8AzWVAWyNgb0Ge3Q5znQlSCQPCJfHLQ5Q6oPBKvgQsgLJGvspCVqI6ZKVmI5H54wdBJZdAW3YccDZQj6gbqBBRAnB5OFi9HTm6zKpmPcC574Dqb/kGgn5o5FIkJSZCrUsCa8yEzDgUMA4D/Mp7lDbwX8je1DqKZjIM3vwolS+7mk7hdwUVIkoArXY+o1qtkHburZhrgf99CLQ18a/VSbzQGLIBdTIYlR7Du6kpZHN5UNPK9/mKdSESUHjvk3pEXUKFiBKAb1jWaSJj3TGg5L98kS9FAj/cShvZp0aIp2otIIT3FvQxtqzjYhrMDpQ2WmFsYzAEoB5RN1AhogTQ0uZbY+Y3LCMEqNgNnN3Lv04dAgy7DlB0HWg2O9wob2xDoTEhYB1ZSoICuckaDE3vQxA8SmmyuvBt+QUMV0t4IXLQiqJdQYWIEkCr3bfq3iseHAec/i9Q+z3/evBUIH9Wj33HfDNvNlcyphW2d+XIS01AXmpCXKwzFJIaOW+syGnhRT2Ke7YNFDSPiBJAfmoCxmYb+M4dHAuc+oAXIYYBhl8HFMzu1RdpbHYiAOD4eVOnuUL97QwbDWi9681aPEr+d8Z5aMnYLqAeESWAoek6ftjEeoAT24CmM3zR95Hz+aB0LylK08KglsNkd+PL041IUErh9HCYUZQKuTQ+/v/5PCIny8Cj0EDmbgOcZr5YGiWA+PhEUPoGx/qJkAwYfXOfRAgApBIGs4cZAfBe0bflF3CkqhVn6uNn5kghk0Ap579idqlXfBwmES2KXKgQUQTsLhYNZjvcJ/4DNJfyIjT2ViClMKjrFRi1uHpUOjTeVIBZw4wYkRn7QWp/fDODNsa7zIMGrDuFDs0oApXNbTj11TYMc5/CqEFJwKj/A5Ly+nXNUVkGjMrqZTH9GMSglqPR4oQFCUgH+KEZpQNUiCgCznNHkWk5BpVOCYy4AUgtEtukqGdGUSpmDkmFrtkClB6jQ7MuoEJE4bE2QFOxA3YAnpypQPoosS2KCYRSuzavV0iFqFNojIgiTNM7XS60qnIgLZgltkWxh8orRHRo1ilUiCjA2T0g1gZYWDnKUmYhsY8F8yld43Cz+KasGZ9XehcHux181QJKAFSI4h3bBaBqH9wsQWniNHhkmj4VzKd0j4RhsK+8GUdr7XBLvAJPh2cdoEIU75R/AXAsLOpsNKsLoFfJIYuThMNwoJBJhAYEdqk3dcHRKp5BEYqon7jdu3dj3rx5yMrKAsMweP/998U0J/5oOQs0ngYYBvJhP8DMoUaMz00U26qYQ+/1MNskXiGyt4hoTWQiqhC1tbVh3LhxeOWVV8Q0I36p/Jp/zhwPfWoWJuUl45LcJHFtikF8SY0WhgpRV4g6fT937lzMnTtXTBPil9ZqoLWKX0c2eJrY1sQ0vpK7Jnizq6kQdSCq8oicTiecTqfw2mymU6FB46stlDEGUOlR0dQGrVKGlAQFJJLYXxkfThLVfJD6AkeFqCuiKiq5du1aGAwG4ZGTkyO2SdGJpY7vP89IgNzLwHIEHxypwT/3nYUtyDZClK5J9NZ2avR4C8k5THzuFkUgqoRo1apVMJlMwqO6ulpsk6KTc/v557QRgDoJrTYXOEKgkEmQ0IcW05TekWFQYdG0PNw+YyTf6ZUQOoV/EVE1NFMqlVAqY7dNcVhwtQENp/ifsycBAFpsvqqMirgoWBZu5FIJkn1JouokwMq3XoImWVzDIoio8ogoIaD2KD8s0GcC+iwAQIuNz/QVysNSBg61d1aSxokCENUjslqtKC0tFV5XVFTgyJEjSE5ORm5uroiWxSgcB9Qc5n8eNFHYfMFbMD+JLu0YMMoarThTb8UIuxqDgfZ2TBQAIgvRgQMHcMUVVwivV6xYAQBYtGgRiouLRbIqhmk+wxfmkqsB4whhc5OVn4lM1VIhGigaLU6cqjVDr9F4hahRbJMiClGFaPbs2XHRzSFiOH+Qf84azwdNAXAcwQUr7xGlJND420Dh65rbyOn5DbYm2tHDj6gKVlP6QVsTv6SDYYCsCcJmAmDumEw0W50w0MWuA4Yv/lbrTgBhAMbt4Du/KuOrdG5X0GB1vOCLDaUUtdfGAV/kvihNiykFKTSRcQBJTlBAwjCwswyccu/vnw7PBKgQxQMeF1DnbZA46BJxbYlTZFIJkhJ4r8gk8QlRs4gWRRZUiOKBhpO8GKmTgKT8gF2n6y0obbDCQTOqBxyjlo/BXUAiv4F6RAJUiGIdQoCaQ/zPWRM6BEf3lDbhP0dr0GB2dnIyJZSk6pRgGMAiS+Q3tDWIak8kQYPVsY6lFrDU8z3KMsYE7HK4WbR6s6qNOjpjNtCMzTZgfE4i5C4jsG83YG3gk0sldFkN9YhinfNebyhtOKDQBOyqN/N1lA1qOdR0jdmAo5RJ+XbbqkQ+l4tjeTGiUCGKaVy29nVlWR2D1PXe4ViGQRVOqygMA+gy+Z8tNeLaEiHQoVksU3MY4DyALl1YV+ZPndcjStfTYVm4OH7ehOPnTZhEtCgCAHMtMEhsq8SHekSxCusBzh/gf86Z0mkGb4MgRNQjChc2F4takwPnPIn8BkutqPZEClSIYpX64/zQTKUHjMM77LY43LA4PJAwDNJ0VIjCRaZ3GFzhNICAALZmwG0X2SrxoUOzWITjgOrv+J+zL+10VkarlOHe6XlosjqhkNH/R+EiXa+ChGHQyirg0CRB7Wrl64cbh4ptmqjQT2As0nCS/08rUwKZ4zo9hGEYJGoUKEqja53CiUImEVIlmmQZ/MbWsyJaFBlQIYo1OBao/Ir/OfcyXowoEUV2khoAcJYz8htaKsUzJkKgQhRr1H0P2Fv5nKFBkzo9xGR348Pva/D9udawmkbhyUvhu3mUOBJBAL4ygiO+O9JQIYol3HagYjf/c+40QNZ5obOKpjacqbfif7WWMBpH8ZGVqIJeLUdWago8Wm8+UfMZcY0SGRqsjiUqdvMzZQmp3a6yL2+0AgAK0xLCZRnFD5lUgiXT8/hGBVUjAGst3/rbr3xvvEE9olihtaq95tCQq7tcv+RwszjXwk8XF6Rqw2Ud5SKEbimpQ/jn1iq+w0qcQoUoFnDZgJMf8CvtM8cCSYO7PPR0vQUsR5CqVdBi+SJDCEG9RwObKg0gHFB3XGyTRIMKUbTDscCpDwCnhe+TVfSDbg8/WcMHRUdm6cNhHaUbvjzdiLe+rcIJFPAbao/w/0ziECpE0QwhQMl/gQsVfDH8kTd1GaAG+CUdtSYHJAyD4RlUiMQmP5WP0R2wZcDNyAHbBaC5tIezYhMqRNEKxwKn/gPUHeN72I9awC9u7YHsJDWGZWiRoKTzFGKTm6xBqk4JByfFGZk3VlSxOy69IipE0YjDDBzdAtSf4EVoxA1ASmGPp6XpVbh1Ug7mjOhZsCgDD8MwmJzHt53+0l4AJ2R8fSLfpEMcQYUomiCEbxl9YCM/yyKVA6NvBtJHdXsax5GA/nEyKf2zRwpD07UYlKiGA0p8w43i/05ln8ddYf2I+ES+8soryMvLg0qlwpQpU/Ddd9+JbVJkwXF8gbODm4D/fcwnLmrTgElLgNSiHk4l+OREHXaeagDLxZ/LH+kwDIMrR6RBLmVwmC1EqScZYN3A928D9haxzQsbogvR22+/jRUrVmD16tU4dOgQxo0bh2uuuQYNDXFeQpPjANN5oHQn8O0G4MT7fO1pmQIougqYuJifJesGq9OD/3xfg5I6C07UmNFooQXyI5FUrRJzRqZDo5QhadJt/N/VYQIOvs4Pv+MgZsQQkXs+T5kyBZdeein+9Kc/AQA4jkNOTg5+/OMf45e//GW355rNZhgMBphMJuj1UTgLxHGAx8E/nBbA0crPnFjrAfN5vgWQD7maz5YeNBFQdJ4RTQiBye5Go8WJymYbSurMcLMEMgmDuWMy6Er7CMfhZqGSS/kY4PH3UF1VDrVcCrUhDaqskZAm5fAipdRHTcH93n5HRZ06cblcOHjwIFatWiVsk0gkmDNnDr755psOxzudTjid7f/VzeZeLhS8UAGUftb5PkGHSSfb/Lb3dluP1yRgOYLvqy9Awrn9dhHhjES1HPmpWt77SS7Ee+d0aJVkg5yTg1TX8wW1vOdlJ2lw/dhM4XXx3soAszIMKlw1Io0WP4sCVHKvuKj0OF9wC76p/AhZjUchbSgHzpRDwvCdeSUMg4xkPbKSdYBEBpub4HiN2VuFk8/YJt5nMECmQY3cJDXAMHC4WRw9ZxLeM1Ejx7D0IP9BjbmNL7wXAkQVoqamJrAsi/T0wFmc9PR0/O9//+tw/Nq1a7FmzZq+v5HHya9wjhAYQuBwOITXLCOHW6qGQ6aHQ2ZAijEL+RPHAwlGQCLBudoz4FwEgKfDtZye9saIEgkDvUoOhUyCrEQVhqbrMChR3b6cgBI1KBUKpI+9CueaJoFpLoG67Rx0zgYoWCskhAXndgIuXriIywO2zdT1xaTq9g4ubhacpbV9H6cAEhydntYjhAvuvE6IqmSSVatWYcWKFcJrs9mMnJycnk9MzAHG/fCius1+PwvbO9vmt72323q4JgMgZ5gTkClBZCqAkQiHMwwDtVwK+C2/+OHknPYrCf/oGDAM+PY0fiyZEdjJlRKdpGqVmDXUCAw1gpBCOD0cnB4OLjcLztUGjYzjv72cG3K3G7nDnB1jSd7XOpUEUPHtrqUcQW5R+6hCKWeAhCBrVnURIggGUYUoNTUVUqkU9fX1Advr6+uRkZHR4XilUgmlMohfmiIBSI6cLygDILsPf0Na3D6+YRgGKrmUH7qp5QACPw8KAINSenctOYBBqaG2sP+IOmumUCgwceJE7Ny5U9jGcRx27tyJqVOnimgZhUIJJ6IPzVasWIFFixZh0qRJmDx5MtatW4e2tjbce++9YptGoVDChOhCtHDhQjQ2NuLJJ59EXV0dxo8fj08++aRDAJtCocQuoucR9YeozyOiUGKc3n5HRc+splAoFCpEFApFdKgQUSgU0RE9WN0ffOGtXi/1oFAoYcX33ewpFB3VQmSx8H25epVdTaFQRMNiscBgMHS5P6pnzTiOQ01NDXQ63YCtp/ItI6muro6Lmbl4u1+A3vNA3jMhBBaLBVlZWZBIuo4ERbVHJJFIkJ2dHZb30uv1cfMhBeLvfgF6zwNFd56QDxqsplAookOFiEKhiA4Voh5QKpVYvXp1cKv+o5B4u1+A3nMkENXBagqFEhtQj4hCoYgOFSIKhSI6VIgoFIroUCGiUCiiQ4UIfes0++qrr2LmzJlISkpCUlIS5syZE3WdaYPtrLtlyxYwDIObbrppYA0cAPp6z62trVi2bBkyMzOhVCoxdOhQfPzxx2GyNjT09Z7XrVuHYcOGQa1WIycnB8uXLw/oNjOgkDhny5YtRKFQkI0bN5ITJ06Q+++/nyQmJpL6+vpOj7/jjjvIK6+8Qg4fPkxOnTpFFi9eTAwGAzl37lyYLQ+Ovt6vj4qKCjJo0CAyc+ZMMn/+/PAYGyL6es9Op5NMmjSJXHfddeTrr78mFRUVZNeuXeTIkSNhtjx4+nrPb775JlEqleTNN98kFRUVZPv27SQzM5MsX748LPbGvRBNnjyZLFu2THjNsizJysoia9eu7dX5Ho+H6HQ68vrrrw+UiSElmPv1eDxk2rRp5LXXXiOLFi2KOiHq6z1v2LCBFBQUEJfLFS4TQ05f73nZsmXkyiuvDNi2YsUKMn369AG100dcD818nWbnzJkjbOuu02xn2Gw2uN1uJCd334c+Egj2fp9++mmkpaVh6dKl4TAzpARzzx988AGmTp2KZcuWIT09HaNHj8Zzzz0HlmU7PT7SCOaep02bhoMHDwrDt/Lycnz88ce47rrrwmJzVC967S997TTbGY899hiysrIC/uiRSjD3+/XXX+Pvf/87jhw5EgYLQ08w91xeXo7PP/8cd955Jz7++GOUlpbi4YcfhtvtxurVq8Nhdr8I5p7vuOMONDU1YcaMGSCEwOPx4MEHH8SvfvWrcJhMg9X94fnnn8eWLVuwbds2qFSx1wTRYrHg7rvvxquvvorU1AjsyjdAcByHtLQ0/O1vf8PEiROxcOFCPP744/jLX/4itmkDxq5du/Dcc8/hz3/+Mw4dOoStW7fio48+wjPPPBOW949rj6ivnWb9+d3vfofnn38en332GcaOHTuQZoaMvt5vWVkZKisrMW/ePGEbx/H9zmUyGUpKSlBYWDiwRveTYP7GmZmZkMvlkEqlwrYRI0agrq4OLpcLCoWi0/MihWDu+YknnsDdd9+N++67DwAwZswYtLW14YEHHsDjjz/ebS2hUBDXHlGwnWZffPFFPPPMM/jkk08wadKkcJgaEvp6v8OHD8exY8dw5MgR4XHjjTfiiiuuwJEjR6KiMmYwf+Pp06ejtLRUEF0AOH36NDIzMyNehIDg7tlms3UQG58Qk3AsRw1LSDyC2bJlC1EqlaS4uJicPHmSPPDAAyQxMZHU1dURQgi5++67yS9/+Uvh+Oeff54oFAry7rvvktraWuFhsVjEuoU+0df7vZhonDXr6z1XVVURnU5HHnnkEVJSUkI+/PBDkpaWRn7zm9+IdQt9pq/3vHr1aqLT6cjmzZtJeXk5+fTTT0lhYSG57bbbwmJv3AsRIYT88Y9/JLm5uUShUJDJkyeTffv2CftmzZpFFi1aJLwePHgwAdDhsXr16vAbHiR9ud+LiUYhIqTv97x3714yZcoUolQqSUFBAXn22WeJx+MJs9X9oy/37Ha7yVNPPUUKCwuJSqUiOTk55OGHHyYtLS1hsZWWAaFQKKIT1zEiCoUSGVAholAookOFiEKhiA4VIgqFIjpUiCgUiuhQIaJQKKJDhYhCoYgOFSJKVFFcXIzExETh9VNPPYXx48cLrxcvXhyVFSTjHSpElE5ZvHgxGIbBgw8+2GHfsmXLwDAMFi9eHHB8qAUgLy8P69atC9i2cOFCnD59ustz1q9fj+LiYuH17Nmz8eijj4bULkrooUJE6ZKcnBxs2bIFdrtd2OZwOPDWW28hNzdXFJvUajXS0tK63G8wGAI8Jkp0QIWI0iWXXHIJcnJysHXrVmHb1q1bkZubiwkTJvTr2p15KjfddJPgZc2ePRtnz57F8uXLwTAMGIYB0HFodjH+ntnixYvx5ZdfYv369cI1KioqUFRUhN/97ncB5x05cgQMw6C0tLRf90UJDipElG5ZsmQJNm3aJLzeuHEj7r333gF/361btyI7OxtPP/00amtrUVtb2+drrF+/HlOnTsX9998vXCM3N7fDPQHApk2bcPnll6OoqChUt0DpA1SIKN1y11134euvv8bZs2dx9uxZ7NmzB3fdddeAv29ycjKkUil0Oh0yMjJ6LFTXGQaDAQqFAhqNRriGVCrF4sWLUVJSItRndrvdeOutt7BkyZJQ3wall8R1hUZKzxiNRlx//fUoLi4GIQTXX3991JeNzcrKwvXXX4+NGzdi8uTJ+M9//gOn04lbb71VbNPiFuoRUXpkyZIlKC4uxuuvvx4yr0EikXSo/Od2u0Ny7d5w3333CYH4TZs2YeHChdBoNGF7f0ogVIgoPXLttdfC5XLB7XbjmmuuCck1jUZjQNyHZVkcP3484BiFQtHvFj5dXeO6665DQkICNmzYgE8++YQOy0SGDs0oPSKVSnHq1Cnh564wmUwd2g6lpKR0Wtv6yiuvxIoVK/DRRx+hsLAQL7/8MlpbWwOOycvLw+7du/HDH/4QSqUyqCFhXl4evv32W1RWVkKr1SI5ORkSiUSIFa1atQpDhgzptkY5ZeChHhGlV+j1euj1+m6P2bVrFyZMmBDwWLNmTafHLlmyBIsWLcI999yDWbNmoaCgAFdccUXAMU8//TQqKytRWFgIo9EYlN0rV66EVCrFyJEjYTQaUVVVJexbunQpXC5XWGYBKd1DS8VS4pavvvoKV111Faqrqzs0I6SEFypElLjD6XSisbERixYtQkZGBt58802xTYp76NCMEnds3rwZgwcPRmtrK1588UWxzaGAekQUCiUCoB4RhUIRHSpEFApFdKgQUSgU0aFCRKFQRIcKEYVCER0qRBQKRXSoEFEoFNGhQkShUESHChGFQhGd/wfbCGeSTV1ttAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:54:51.947353Z", + "iopub.status.busy": "2024-02-29T19:54:51.946553Z", + "iopub.status.idle": "2024-02-29T19:54:52.204304Z", + "shell.execute_reply": "2024-02-29T19:54:52.203216Z" + }, + "papermill": { + "duration": 0.285339, + "end_time": "2024-02-29T19:54:52.206666", + "exception": false, + "start_time": "2024-02-29T19:54:51.921327", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0tklEQVR4nO3de1hU1d4H8C8MMIBclTsCg6KgJpCYgOnxEjdN0+gUmsklozeU81oTadgJIj1SeSM7JO/xhKink5avma8aSqgdTfACaaAIgiBeAEWFEdBhmFnvHx72aZwBBgQH9vw+zzOP7bXXXrMWe+bX3nutWUuPMcZACCE8pK/tChBCSF+hAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0DbVegP1IoFLhx4wbMzc2hp6en7eoQQh7BGMO9e/fg5OQEff2Or9MowKlx48YNuLi4aLsahJAuXL16FUOHDu1wPwU4NczNzQE8/ONZWFhouTZ9RyaT4dChQwgJCYGhoaG2q0Meky6dT4lEAhcXF+672hEKcGq035ZaWFjwPsCZmprCwsKC918IXaCL57OrR0jUyUAI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iYSKEDFAtLS24ePEit910X4oTRRWwtjkDMxOhUl4vLy+Ympo+6SpqHQU4Qgaoixcvws/PTyX9MzV5CwoKMG7cuL6vVD9DAY6QAcrLywsFBQXcdmlNA8TfFWH9y2Ph6WilklcXUYDTIXRLwy+mpqZKV2X6V25DeOw+Rj3lA1+3IVqsWf9BAU6H0C0N0TUU4HQI3dIQXUMBTofQLQ3RNTQOjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvKX1AJeeng6RSARjY2P4+/vj1KlTneZvaGjAkiVL4OjoCKFQiJEjR+LAgQPc/o8++gh6enpKLxrTRYhu0uo4uJ07d0IsFiMjIwP+/v5IS0tDaGgoSktLYWdnp5K/tbUVwcHBsLOzw65du+Ds7IwrV67AyspKKd+YMWPw008/cdsGBjTcjxBdpNVv/vr16xEbG4uYmBgAQEZGBvbv34/MzEy8//77KvkzMzNx584dnDhxgls1SCQSqeQzMDCAg4NDn9adENL/aS3Atba2oqCgAImJiVyavr4+goKCkJeXp/aYvXv3IjAwEEuWLMEPP/wAW1tbvPrqq1i+fDkEAgGX79KlS3BycoKxsTECAwORmpoKV1fXDusilUohlUq5bYlEAuDhMmwymexxm9pvtbW1cf/yuZ26QpfOp6bt01qAq6+vh1wuh729vVK6vb290owXv3f58mUcPnwYCxYswIEDB1BeXo7FixdDJpMhOTkZAODv74+srCx4enqipqYGKSkpmDx5MoqLiztcJDY1NRUpKSkq6YcOHeL1jBpXmwDAAPn5+bherO3akMelS+ezpaVFo3wD6uGUQqGAnZ0d/va3v0EgEMDPzw/Xr1/HmjVruAA3Y8YMLr+3tzf8/f3h5uaGb7/9FosWLVJbbmJiIsRiMbfdvmp2SEgIrxd+Pld9Byg6g4CAAPi4DtZ2dchj0qXz2X6X1RWtBTgbGxsIBALU1dUppdfV1XX4/MzR0RGGhoZKt6OjRo1CbW0tWltbYWRkpHKMlZUVRo4cifLy8g7rIhQKIRQKVdINDQ15vUJ4e+eLgYEBr9upK3TpfGraPq0NEzEyMoKfnx9yc3O5NIVCgdzcXAQGBqo95tlnn0V5eTkUCgWXVlZWBkdHR7XBDQCamppQUVEBR0fH3m0AIaTf0+o4OLFYjM2bN2Pr1q0oKSlBXFwcmpubuV7VyMhIpU6IuLg43LlzB0uXLkVZWRn279+P1atXY8mSJVyehIQE/Pzzz6iqqsKJEyfw4osvQiAQYP78+U+8fYQQ7dLqM7iIiAjcunULSUlJqK2tha+vL7Kzs7mOh+rqaujr/ycGu7i44ODBg3jnnXfg7e0NZ2dnLF26FMuXL+fyXLt2DfPnz8ft27dha2uLSZMmIT8/H7a2tk+8fYQQ7dJ6J0N8fDzi4+PV7jt69KhKWmBgIPLz8zssb8eOHb1VNULIAKf1n2oRQkhfoQBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLa0PEyGEaK6yvhnN0ja1+ypuNXP/djYH4iChAdxtBvVJ/fobCnCEDBCV9c2YtvZol/ne3VXUZZ4jCVN1IshRgCNkgGi/ckuL8IWHnZnq/vtS7Duah1lTAzHIRHXyCAAov9mEt3ee7fAqkG8owBEywHjYmeEpZ0uVdJlMhlpbYJybNe9nE9EUdTIQQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4i3pReayzQaEADQwl/EcBjqc0HRQK0MBQwl8U4Hiqq0GhAA0MJfxHAY7nOhoUCtDAUMJ/Wu9kSE9Ph0gkgrGxMfz9/XHq1KlO8zc0NGDJkiVwdHSEUCjEyJEjceDAgccqkxDCT1oNcDt37oRYLEZycjIKCwvh4+OD0NBQ3Lx5U23+1tZWBAcHo6qqCrt27UJpaSk2b94MZ2fnHpdJCOEvrQa49evXIzY2FjExMRg9ejQyMjJgamqKzMxMtfkzMzNx584d7NmzB88++yxEIhGmTJkCHx+fHpdJCOEvrT2Da21tRUFBgdLCzvr6+ggKCkJeXp7aY/bu3YvAwEAsWbIEP/zwA2xtbfHqq69i+fLlEAgEPSoTAKRSKaRSKbctkUgAPHxGJZPJHrepWtHW1sb921Eb2tM7a6Mm5ZAno6tzoUvnU9O6ay3A1dfXQy6Xc4s8t7O3t8fFixfVHnP58mUcPnwYCxYswIEDB1BeXo7FixdDJpMhOTm5R2UCQGpqKlJSUlTSDx06BFNT0x60TvuuNgGAAY4fP44r6jtROTk5Ob1SDulbmp4LXTifLS0tGuUbUL2oCoUCdnZ2+Nvf/gaBQAA/Pz9cv34da9asQXJyco/LTUxMhFgs5rYlEglcXFwQEhICCwuL3qj6E3f+hgRri/IxadIkjHFS3waZTIacnBwEBwd32IuqSTnkyejqXOjS+Wy/y+qK1gKcjY0NBAIB6urqlNLr6urg4OCg9hhHR0cYGhpCIBBwaaNGjUJtbS1aW1t7VCYACIVCCIWq48AMDQ0H7PCJ9l8mGBgYdNmGztrZnXJI39L0XOjC+dS07lrrZDAyMoKfnx9yc3O5NIVCgdzcXAQGBqo95tlnn0V5eTkUCgWXVlZWBkdHRxgZGfWoTEIIf2m1F1UsFmPz5s3YunUrSkpKEBcXh+bmZsTExAAAIiMjlToM4uLicOfOHSxduhRlZWXYv38/Vq9ejSVLlmhcJiFEd2j1GVxERARu3bqFpKQk1NbWwtfXF9nZ2VwnQXV1NfT1/xODXVxccPDgQbzzzjvw9vaGs7Mzli5diuXLl2tcJiFEd2i9kyE+Ph7x8fFq9x09elQlLTAwEPn5+T0ukxCiO7T+Uy1CCOkrFOAIIbyl9VtUQohmpPIH0De+jkpJKfSNVUfptrW14UbbDZTcKelwAtNKSRP0ja9DKn8AQP0sM3xCAY6QAeJG8xUMcv8CK7qYHOfL7C873T/IHbjR7As/8L/jjQIcIQOE0yA3NFf+CZ9H+GK4mklM29ra8MvxX/DspGc7vIKruNmEpTvPwmmaW19Xt1+gAEfIACEUGEPxwBnuFp4YPUT9yvaVBpUYNXhUhyP9FQ8aoXhwC0KBcV9Xt1+gAMdTXT2vAeiZDeE/CnA8penzGoCe2RD+ogDHU109rwHomQ3hPwpwPNXV8xqAntkQ/qOBvoQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDe6hcBLj09HSKRCMbGxvD398epUx3/gDIrKwt6enpKL2Nj5VH20dHRKnnCwsL6uhmEkH5G6z/V2rlzJ8RiMTIyMuDv74+0tDSEhoaitLQUdnZ2ao+xsLBAaWkpt62np6eSJywsDFu2bOG21S3sTAjht25fwV2+fLlXK7B+/XrExsYiJiYGo0ePRkZGBkxNTZGZmdnhMXp6enBwcOBe6pYEFAqFSnmsra17td6EkP6v21dwHh4emDJlChYtWoQ//vGPKreH3dHa2oqCggKlxZ319fURFBSEvLy8Do9ramqCm5sbFAoFxo0bh9WrV2PMmDFKeY4ePQo7OztYW1tj+vTpWLVqFYYMGaK2PKlUCqlUym1LJBIAD3+MLpPJetw+bWpra+P+7agN7emdtVGTcsiT0dW50KXzqWndux3gCgsLsWXLFojFYsTHxyMiIgKLFi3ChAkTul3J+vp6yOVylSswe3t7XLx4Ue0xnp6eyMzMhLe3NxobG7F27VpMnDgR58+fx9ChQwE8vD0NDw+Hu7s7KioqsGLFCsyYMQN5eXkQCAQqZaampiIlJUUl/dChQzA1Ne12u/qDq00AYIDjx4/jivrZkjg5OTm9Ug7pW5qeC104ny0tLRrl02OMsZ68QVtbG/bu3YusrCxkZ2dj5MiReP3117Fw4ULY2tpqVMaNGzfg7OyMEydOIDAwkEtftmwZfv75Z5w8ebLLMmQyGUaNGoX58+dj5cqVavNcvnwZw4cPx08//YTnnntOZb+6KzgXFxfU19fDwsJCo7b0N+dvSDB3Uz72xAVgjJP6NshkMuTk5CA4OLjD6ZI0KYc8GV2dC106nxKJBDY2NmhsbOz0O9rjTgYDAwOEh4fj+eefx5dffonExEQkJCRgxYoVeOWVV/Dpp5/C0dGx0zJsbGwgEAhQV1enlF5XVwcHBweN6mFoaIinn34a5eXlHeYZNmwYbGxsUF5erjbACYVCtZ0QhoaGHX5Q+rv2CSwNDAy6bENn7exOOaRvaXoudOF8alr3Hg8TOXPmDBYvXgxHR0esX78eCQkJqKioQE5ODm7cuIE5c+Z0WYaRkRH8/PyQm5vLpSkUCuTm5ipd0XVGLpejqKio02B67do13L59u8uASwjhl25fwa1fvx5btmxBaWkpZs6ciW3btmHmzJnQ138YK93d3ZGVlQWRSKRReWKxGFFRURg/fjwmTJiAtLQ0NDc3IyYmBgAQGRkJZ2dnpKamAgA+/vhjBAQEwMPDAw0NDVizZg2uXLmCN954A8DDDoiUlBS89NJLcHBwQEVFBZYtWwYPDw+EhoZ2t7mEkAGs2wFu06ZNeP311xEdHd3hFZGdnR2++uorjcqLiIjArVu3kJSUhNraWvj6+iI7O5vreKiuruaCJwDcvXsXsbGxqK2thbW1Nfz8/HDixAmMHj0aACAQCPDbb79h69ataGhogJOTE0JCQrBy5UoaC0eIjul2gMvJyYGrq6tS0AEAxhiuXr0KV1dXGBkZISoqSuMy4+PjER8fr3bf0aNHlbY3bNiADRs2dFiWiYkJDh48qPF7E0L4q9vP4IYPH476+nqV9Dt37sDd3b1XKkUIIb2h2wGuo1ElTU1NjzXolxBCepvGt6hisRjAw59JJSUlKQ2AlcvlOHnyJHx9fXu9goQQ0lMaB7hff/0VwMMruKKiIhgZGXH7jIyM4OPjg4SEhN6vISGE9JDGAe7IkSMAgJiYGHz++ecDdoQ/IUR3dLsX9fdTEBFCSH+mUYALDw9HVlYWLCwsEB4e3mne3bt390rFCCHkcWkU4CwtLblJJS0tLfu0QoQQ0ls0CnC/vy2lW1RCyEDRL9ZkIISQvqDRFdzTTz+tdt0DdQoLCx+rQoQQ0ls0CnBz587t42qQ3nZfJgcAFF9v7DBP830pztwCHK7cxSAT9RMRlN9s6pP6ke7r6pzS+VSlUYBLTk7u63qQXlbx7w/y+7uLushpgO3lp7ssb5BQ6wuw6TzNzimdz9/TjVbqoJAxD2dEHm5nBhND1XUoAKC0phHv7irCuj+Ohadjx73jg4QGcLcZ1Cf1JJrr6pzS+VSlUYAbPHgwysrKYGNjA2tr606fx925c6fXKkd6bvAgI8yb4NppnvYVlobbDsJTzjT8p7/r6pzS+VSlUYDbsGEDzM3Nuf/WtMOBEEK0SaMA9/vJK6Ojo/uqLoQQ0qu6PQ5OIBDg5s2bKum3b99Wu+YoIYRoS69NeCmVSpWmUCKEEG3TuBd148aNAB5OePn3v/8dZmb/WRZbLpfjX//6F7y8vHpUifT0dKxZswa1tbXw8fHBF198gQkTJqjNm5WVxa241U4oFOLBgwfcNmMMycnJ2Lx5MxoaGvDss89i06ZNGDFiRI/qRwgZmDQOcO0LvTDGkJGRoXQ7amRkBJFIhIyMjG5XYOfOnRCLxcjIyIC/vz/S0tIQGhqK0tJS2NnZqT3GwsICpaWl3PajnR6fffYZNm7ciK1bt8Ld3R0ffvghQkNDceHCBZpWnRBdwrpp6tSp7M6dO909rEMTJkxgS5Ys4bblcjlzcnJiqampavNv2bKFWVpadlieQqFgDg4ObM2aNVxaQ0MDEwqF7JtvvtGoTo2NjQwAa2xs1KwRA9SvVfXMbfk+9mtVvbarQnqBLp1PTb+j3R7o2z6zb29obW1FQUEBEhMTuTR9fX0EBQUhLy+vw+Oamprg5uYGhUKBcePGYfXq1RgzZgwAoLKyErW1tQgKCuLyW1pawt/fH3l5eZg3b55KeVKpFFKplNuWSCQAAJlMBplM9tjt7K/ax021tbXxup26QpfOp6bt63aAe/311zvdn5mZqXFZ9fX1kMvl3CLP7ezt7XHx4kW1x3h6eiIzMxPe3t5obGzE2rVrMXHiRJw/fx5Dhw5FbW0tV8ajZbbve1RqaipSUlJU0g8dOqS0uA7fXG0CAAPk5+fjerG2a0Mely6dz5aWFo3ydTvA3b17V2lbJpOhuLgYDQ0NmD59eneL67bAwEAEBgZy2xMnTsSoUaPwP//zP1i5cmWPykxMTORWDQMeXsG5uLggJCSE12tPnKu+AxSdQUBAAHxcB2u7OuQx6dL5bL/L6kq3A9z333+vkqZQKBAXF4fhw4d3qywbGxsIBALU1dUppdfV1cHBwUGjMgwNDfH000+jvLwcALjj6urq4OjoqFRmR8saCoVCCIWqsy8YGhrC0NBQo3oMRAYGBty/fG6nrtCl86lp+3plwkt9fX2IxWKup1VTRkZG8PPzQ25uLpemUCiQm5urdJXWGblcjqKiIi6Yubu7w8HBQalMiUSCkydPalwmIYQfem02kYqKCu4hZ3eIxWJERUVh/PjxmDBhAtLS0tDc3MyNdYuMjISzszNSU1MBAB9//DECAgLg4eGBhoYGrFmzBleuXMEbb7wB4OGQkbfffhurVq3CiBEjuGEiTk5ONK8dITqm2wHu98+qgIfj4mpqarB//36l36xqKiIiArdu3UJSUhJqa2vh6+uL7OxsrpOguroa+vr/udC8e/cuYmNjUVtbC2tra/j5+eHEiRMYPXo0l2fZsmVobm7Gm2++iYaGBkyaNAnZ2dk0Bo4QHaPHWAe/verAtGnTlLb19fVha2uL6dOn4/XXX+eeAwxkEokElpaWaGxs5HUnw9krtzF3Uz72xAXA122ItqtDHpMunU9Nv6NaHQdHCCF9iVbVIoTwFgU4QghvUYAjhPAWBThCCG/1WoC7du0a3nzzzd4qjhBCHluvBbjbt2/jq6++6q3iCCHksdEtKiGEtyjAEUJ4iwIcIYS3NP4lQ3h4eKf7GxoaHrcuhBDSqzQOcJaWll3uj4yMfOwKEUJIb9E4wG3ZsqUv60EIIb2OnsERQnhL4yu4rhabadedRWcIIaQvaRzgsrKy4ObmhqeffhrdnEKOEEK0QuMAFxcXh2+++QaVlZWIiYnBa6+9hsGD+b1yDyFkYNP4GVx6ejpqamqwbNky/N///R9cXFzwyiuv4ODBg3RFRwjpl7rVySAUCjF//nzk5OTgwoULGDNmDBYvXgyRSISmpqa+qiMhhPRIj3tR9fX1oaenB8YY5HL5Y1UiPT0dIpEIxsbG8Pf3x6lTpzQ6bseOHdDT01NZLSs6Ohp6enpKr7CwsMeqIyFk4OlWgJNKpfjmm28QHByMkSNHoqioCH/9619RXV0NMzOzHlVg586dEIvFSE5ORmFhIXx8fBAaGoqbN292elxVVRUSEhIwefJktfvDwsJQU1PDvb755pse1Y8QMnBpHOAWL14MR0dHfPLJJ5g1axauXr2K7777DjNnzlRa1q+71q9fj9jYWMTExGD06NHIyMiAqalpp8NN5HI5FixYgJSUFAwbNkxtHqFQCAcHB+5lbW3d4zoSQgYmjXtRMzIy4OrqimHDhuHnn3/Gzz//rDbf7t27NX7z1tZWFBQUIDExkUvT19dHUFAQ8vLyOjzu448/hp2dHRYtWoRjx46pzXP06FHY2dnB2toa06dPx6pVqzBkCL+XUiOEKNM4wEVGRkJPT69X37y+vh5yuZxb5Lmdvb09Ll68qPaY48eP46uvvsLZs2c7LDcsLAzh4eFwd3dHRUUFVqxYgRkzZiAvLw8CgUAlv1QqhVQq5bYlEgkAQCaTQSaT9aBlA0NbWxv3L5/bqSt06Xxq2r5uDfTVtnv37mHhwoXYvHkzbGxsOsw3b9487r/Hjh0Lb29vDB8+HEePHsVzzz2nkj81NRUpKSkq6YcOHYKpqWnvVL4futoEAAbIz8/H9WJt14Y8Ll06ny0tLRrl0+oy9DY2NhAIBKirq1NKr6urg4ODg0r+iooKVFVVYfbs2VyaQqEAABgYGKC0tBTDhw9XOW7YsGGwsbFBeXm52gCXmJgIsVjMbUskEri4uCAkJITXK9ufq74DFJ1BQEAAfFxp0PZAp0vns/0uqytaDXBGRkbw8/NDbm4uN9RDoVAgNzcX8fHxKvm9vLxQVFSklPbnP/8Z9+7dw+effw4XFxe173Pt2jXcvn0bjo6OavcLhUIIhUKVdENDQxgaGnazVQOHgYEB9y+f26krdOl8ato+rQY4ABCLxYiKisL48eMxYcIEpKWlobm5GTExMQAePvtzdnZGamoqjI2N8dRTTykdb2VlBQBcelNTE1JSUvDSSy/BwcEBFRUVWLZsGTw8PBAaGvpE20YI0S6tB7iIiAjcunULSUlJqK2tha+vL7Kzs7mOh+rq6m4NQxEIBPjtt9+wdetWNDQ0wMnJCSEhIVi5cqXaqzRCCH9pPcABQHx8vNpbUuDhcI/OPNr5YWJigoMHD/ZSzQghAxlNeEkI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3uoXAS49PR0ikQjGxsbw9/fHqVOnNDpux44d0NPT49ZUbccYQ1JSEhwdHWFiYoKgoCBcunSpD2o+sLS0tKCwsJB7lRSfg7S2HCXF55TSCwsLNV45nJB+jWnZjh07mJGREcvMzGTnz59nsbGxzMrKitXV1XV6XGVlJXN2dmaTJ09mc+bMUdr3ySefMEtLS7Znzx527tw59sILLzB3d3d2//59jerU2NjIALDGxsaeNqtfKigoYAA0ehUUFGi7uqSbfq2qZ27L97Ffq+q1XZU+p+l3VI8xxrQTWh/y9/fHM888g7/+9a8AHq5s7+Ligj/96U94//331R4jl8vxhz/8Aa+//jqOHTuGhoYG7NmzB8DDqzcnJye8++67SEhIAAA0NjbC3t4eWVlZmDdvXpd1kkgksLS0RGNjIywsLHqnof1AS0sLLl68yG033Zdi/5E8PD8tEGYmymvGenl5wdTU9ElXkTyGs1duY+6mfOyJC4Cv2xBtV6dPafod1eq6qK2trSgoKEBiYiKXpq+vj6CgIOTl5XV43Mcffww7OzssWrQIx44dU9pXWVmJ2tpaBAUFcWmWlpbw9/dHXl6e2gAnlUohlUq5bYlEAgCQyWSQyWQ9bl9/Y2hoiLFjx3LbMpkMd+tvYvzTPjA0NFTJz6e264K2tjbuX76fO03bp9UAV19fD7lczq1i387e3l7pSuP3jh8/jq+++gpnz55Vu7+2tpYr49Ey2/c9KjU1FSkpKSrphw4d0omrmJycHG1XgfSCq00AYID8/HxcL9Z2bfqWps+I+8XK9pq6d+8eFi5ciM2bN8PGxqbXyk1MTIRYLOa2JRIJXFxcEBISwqtb1EfJZDLk5OQgODhY7RUcGVjOVd8Bis4gICAAPq6DtV2dPtV+l9UVrQY4GxsbCAQC1NXVKaXX1dXBwcFBJX9FRQWqqqowe/ZsLk2hUAAADAwMUFpayh1XV1cHR0dHpTJ9fX3V1kMoFEIoFKqkGxoa6sQXX1fayXcGBgbcv3w/n5q2T6vDRIyMjODn54fc3FwuTaFQIDc3F4GBgSr5vby8UFRUhLNnz3KvF154AdOmTcPZs2fh4uICd3d3ODg4KJUpkUhw8uRJtWUSQvhL67eoYrEYUVFRGD9+PCZMmIC0tDQ0NzcjJiYGABAZGQlnZ2ekpqbC2NgYTz31lNLxVlZWAKCU/vbbb2PVqlUYMWIE3N3d8eGHH8LJyUllvBwhhN+0HuAiIiJw69YtJCUloba2Fr6+vsjOzuY6Caqrq6Gv370LzWXLlqG5uRlvvvkmGhoaMGnSJGRnZ8PY2LgvmkAI6ae0Pg6uP+LrOLhHyWQyHDhwADNnzuT9MxtdQOPgVPWLn2oRQkhfoABHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN7S+oSXhJCeeXSd29KaBkhry1FSbALFbSulvLq6zi0FOEIGqIsXL8LPz08l/dWtqnkLCgowbty4J1Cr/oUCHCEDlJeXFwoKCrjtpvtS7D+Sh+enBcLMRKiSVxdRgCNkgDI1NVW6KpPJZLhbfxOBE8bTFPT/1i86GdLT0yESiWBsbAx/f3+cOnWqw7y7d+/G+PHjYWVlhUGDBsHX1xfbt29XyhMdHQ09PT2lV1hYWF83gxDSz2j9Cm7nzp0Qi8XIyMiAv78/0tLSEBoaitLSUtjZ2ankHzx4MD744AN4eXnByMgI+/btQ0xMDOzs7BAaGsrlCwsLw5YtW7htdQs7E0L4TetXcOvXr0dsbCxiYmIwevRoZGRkwNTUFJmZmWrzT506FS+++CJGjRqF4cOHY+nSpfD29sbx48eV8gmFQjg4OHAva2vrJ9EcQkg/otUruNbWVhQUFCAxMZFL09fXR1BQEPLy8ro8njGGw4cPo7S0FJ9++qnSvqNHj8LOzg7W1taYPn06Vq1ahSFD1C+lJpVKIZVKuW2JRALg4TMNmUzWk6YNCO1t43MbdYkunU9N26jVAFdfXw+5XM4t8tzO3t5eaXzPoxobG+Hs7AypVAqBQIAvv/wSwcHB3P6wsDCEh4fD3d0dFRUVWLFiBWbMmIG8vDwIBAKV8lJTU5GSkqKSfujQIZ0YO5STk6PtKpDHJJfLceHCBdy9exdFRUUYPXq02s86X7S0tGiUT+vP4HrC3NwcZ8+eRVNTE3JzcyEWizFs2DBMnToVADBv3jwu79ixY+Ht7Y3hw4fj6NGjeO6551TKS0xMhFgs5rYlEglcXFwQEhLC+4Wfc3JyEBwcTL1uA9j333+P5cuXo6qqiksTiUT49NNP8eKLL2qvYn2o/S6rK1oNcDY2NhAIBKirq1NKr6urg4ODQ4fH6evrw8PDAwDg6+uLkpISpKamcgHuUcOGDYONjQ3Ky8vVBjihUKi2E8LQ0FAnvvi60k4+2r17N+bNm4dZs2Zh+/btuHbtGoYOHYrPPvsM8+bNw65duxAeHq7tavY6TT+vWu1kMDIygp+fH3Jzc7k0hUKB3NxcBAYGalyOQqFQeob2qGvXruH27dtwdHR8rPoS0p/I5XK8++67mDVrFvbs2QN/f3+YmJjA398fe/bswaxZs5CQkAC5XK7tqmqN1ntRxWIxNm/ejK1bt6KkpARxcXFobm5GTEwMACAyMlKpEyI1NRU5OTm4fPkySkpKsG7dOmzfvh2vvfYaAKCpqQnvvfce8vPzUVVVhdzcXMyZMwceHh5Kw0gIGeiOHTuGqqoqrFixAvr6yl9lfX19JCYmorKyEseOHdNSDbVP68/gIiIicOvWLSQlJaG2tha+vr7Izs7mOh6qq6uVTl5zczMWL16Ma9euwcTEBF5eXvjHP/6BiIgIAIBAIMBvv/2GrVu3oqGhAU5OTggJCcHKlStpLBzhlZqaGgDAU089pXZ/e3p7Pl2k9QAHAPHx8YiPj1e77+jRo0rbq1atwqpVqzosy8TEBAcPHuzN6hHSL7U/cikuLkZAQIDK/uLiYqV8ukjrt6iEkJ6ZPHkyRCIRVq9eDYVCobRPoVAgNTUV7u7umDx5spZqqH394gqOENJ9AoEA69atwx//+EfMmTMHwcHBuHTpEq5cuYKcnBzs378fu3bt4vV4uK5QgCNkAAsPD0dCQgI2bNiAffv2cekGBgZISEjg5RCR7qAAR8gAtnv3bqxduxbPP/88dwU3YsQI5OTkYO3atQgICNDpIKfHGGParkR/I5FIYGlpicbGRt7/kuHAgQOYOXMmDfQdgORyOTw8PDB27Fjs2bMHcrmcO58CgQBz585FcXExLl26xLvbVE2/o9TJQMgARePgukYBjpABisbBdY0CHCED1O/HwalD4+AowBEyYNE4uK5RgCNkgGofB7dv3z7MnTsX+fn5uH//PvLz8zF37lzs27cPa9eu5V0HQ3fQMBFCBrDw8HDs2rUL7777Lv7whz9w6e7u7rydKqk7KMARMsCFh4djzpw5OHLkCH788UfMmDED06ZN0+krt3YU4AjhAYFAgClTpqC5uRlTpkyh4PZv9AyOEMJbFOAIIbxFAY4Qwlv0DE6N9p/narpyz0Alk8nQ0tICiURCv0XlAV06n+3fza5+Sk8BTo179+4BAFxcXLRcE0JIZ+7duwdLS8sO99NsImooFArcuHED5ubm0NPT03Z1+kz7+q9Xr17l9awpukKXzidjDPfu3YOTk5PKRAO/R1dwaujr62Po0KHarsYTY2FhwfsvhC7RlfPZ2ZVbO+pkIITwFgU4QghvUYDTYUKhEMnJybReLE/Q+VRFnQyEEN6iKzhCCG9RgCOE8BYFOEIIb1GA6wXR0dGYO3dur5Y5depUvP32253mEYlESEtL69X3JYRPKMA9QpPAQgaWjz76CL6+vtquRof622euv9XncVCAI4QHWltbtV2FfokC3O9ER0fj559/xueffw49PT3o6emhoqICixYtgru7O0xMTODp6YnPP/9c7fEpKSmwtbWFhYUF3nrrLY0/dM3NzYiMjISZmRkcHR2xbt06lTw3b97E7NmzYWJiAnd3d3z99dcqefT09LBp0ybMmDEDJiYmGDZsGHbt2sXtr6qqgp6eHr799ltMnjwZJiYmeOaZZ1BWVobTp09j/PjxMDMzw4wZM3Dr1i0N/2pAZmYmxowZA6FQCEdHR8THx3P7qqurMWfOHJiZmcHCwgKvvPIK6urquP3tV1fbt2+HSCSCpaUl5s2bx014ADz8bfBnn30GDw8PCIVCuLq64i9/+Qu3f/ny5Rg5ciRMTU0xbNgwfPjhh5DJZACArKwspKSk4Ny5c9w5zcrK0rhtfa2nn7n2xyJ/+ctf4OTkBE9PTwDAiRMn4OvrC2NjY4wfPx579uyBnp4ezp49yx1bXFyMGTNmwMzMDPb29li4cCHq6+s7rE9VVdWT+nP0PkY4DQ0NLDAwkMXGxrKamhpWU1PDHjx4wJKSktjp06fZ5cuX2T/+8Q9mamrKdu7cyR0XFRXFzMzMWEREBCsuLmb79u1jtra2bMWKFRq9b1xcHHN1dWU//fQT++2339isWbOYubk5W7p0KZdnxowZzMfHh+Xl5bEzZ86wiRMnMhMTE7ZhwwYuDwA2ZMgQtnnzZlZaWsr+/Oc/M4FAwC5cuMAYY6yyspIBYF5eXiw7O5tduHCBBQQEMD8/PzZ16lR2/PhxVlhYyDw8PNhbb72lUd2//PJLZmxszNLS0lhpaSk7deoUVye5XM58fX3ZpEmT2JkzZ1h+fj7z8/NjU6ZM4Y5PTk5mZmZmLDw8nBUVFbF//etfzMHBQelvt2zZMmZtbc2ysrJYeXk5O3bsGNu8eTO3f+XKleyXX35hlZWVbO/evcze3p59+umnjDHGWlpa2LvvvsvGjBnDndOWlhaN2vYkPO5nbuHChay4uJgVFxezxsZGNnjwYPbaa6+x8+fPswMHDrCRI0cyAOzXX39ljDF29+5dZmtryxITE1lJSQkrLCxkwcHBbNq0aR3Wp62tTRt/ml5BAe4RU6ZMUQos6ixZsoS99NJL3HZUVBQbPHgwa25u5tI2bdrEzMzMmFwu77Sse/fuMSMjI/btt99yabdv32YmJiZcPUpLSxkAdurUKS5PSUkJA6AS4B4NTP7+/iwuLo4x9p8A9/e//53b/8033zAALDc3l0tLTU1lnp6enda7nZOTE/vggw/U7jt06BATCASsurqaSzt//rxSW5KTk5mpqSmTSCRcnvfee4/5+/szxhiTSCRMKBQqBbSurFmzhvn5+XHbycnJzMfHR+Pjn7Sefubs7e2ZVCrl0jZt2sSGDBnC7t+/z6Vt3rxZKcCtXLmShYSEKJV99epVBoCVlpZqXJ+BgmYT0UB6ejoyMzNRXV2N+/fvo7W1VeWhtY+PD0xNTbntwMBANDU14erVq3Bzc+uw7IqKCrS2tsLf359LGzx4MHfLAQAlJSUwMDCAn58fl+bl5QUrKyuV8gIDA1W2f397AgDe3t7cf9vb2wMAxo4dq5R28+bNDuvc7ubNm7hx4waee+45tftLSkrg4uKiNK/e6NGjYWVlhZKSEjzzzDMAHvYGm5ubc3kcHR259y8pKYFUKu3wPQBg586d2LhxIyoqKtDU1IS2trYBP5uGJp+5sWPHwsjIiNsuLS2Ft7c3jI2NubQJEyYoHXPu3DkcOXIEZmZmKu9ZUVGBkSNH9m5DtIyewXVhx44dSEhIwKJFi3Do0CGcPXsWMTExA/qh7u9ne22f7+7RtEdXSlfHxMSk1+vz6Pt39R55eXlYsGABZs6ciX379uHXX3/FBx98MKDPj6afuUGDBnW77KamJsyePRtnz55Vel26dElpXVW+oAD3CCMjI8jlcm77l19+wcSJE7F48WI8/fTT8PDwQEVFhcpx586dw/3797nt/Px8mJmZdTkr8PDhw2FoaIiTJ09yaXfv3kVZWRm37eXlhba2NhQUFHBppaWlaGhoUCkvPz9fZXvUqFGd1qGnzM3NIRKJkJubq3b/qFGjcPXqVVy9epVLu3DhAhoaGjB69GiN3mPEiBEwMTHp8D1OnDgBNzc3fPDBBxg/fjxGjBiBK1euKOV59Jz2Nz39zD3K09MTRUVFkEqlXNrp06eV8owbNw7nz5+HSCSCh4eH0qs9YPb3v1d3UIB7hEgkwsmTJ1FVVYX6+nqMGDECZ86cwcGDB1FWVoYPP/xQ5UMDPOymX7RoES5cuIADBw4gOTkZ8fHxnc42CgBmZmZYtGgR3nvvPRw+fBjFxcWIjo5WOs7T0xNhYWH4r//6L5w8eRIFBQV444031F7dfPfdd8jMzERZWRmSk5Nx6tQppV7N3vbRRx9h3bp12LhxIy5duoTCwkJ88cUXAICgoCCMHTsWCxYsQGFhIU6dOoXIyEhMmTIF48eP16h8Y2NjLF++HMuWLcO2bdtQUVGB/Px8fPXVVwAeBsDq6mrs2LEDFRUV2LhxI77//nulMkQiESorK3H27FnU19crBYD+oKefuUe9+uqrUCgUePPNN1FSUoKDBw9i7dq1AP5zpb5kyRLcuXMH8+fPx+nTp1FRUYGDBw8iJiaGC2qP1keTq/l+S9sPAfub0tJSFhAQwExMTBgAdvHiRRYdHc0sLS2ZlZUVi4uLY++//77SQ+uoqCg2Z84clpSUxIYMGcLMzMxYbGwse/DggUbvee/ePfbaa68xU1NTZm9vzz777DOVB701NTXs+eefZ0KhkLm6urJt27YxNzc3lU6G9PR0FhwczIRCIROJREo9b+2dDO0PnBlj7MiRIwwAu3v3Lpe2ZcsWZmlpqfHfLCMjg3l6ejJDQ0Pm6OjI/vSnP3H7rly5wl544QU2aNAgZm5uzl5++WVWW1vL7VfXAbBhwwbm5ubGbcvlcrZq1Srm5ubGDA0NmaurK1u9ejW3/7333uP+7hEREWzDhg1K9X/w4AF76aWXmJWVFQPAtmzZonHbnoTH+cw96pdffmHe3t7MyMiI+fn5sX/+859cme3KysrYiy++yKysrJiJiQnz8vJib7/9NlMoFGrrU1lZ2cd/gb5D0yXxiJ6eHr7//vte/9kYGbi+/vprxMTEoLGxsdeemQ4k1ItKCI9s27YNw4YNg7OzM86dO4fly5fjlVde0cngBlCA63PV1dWdPlC/cOECXF1dn2CNukfdcIJ2P/74IyZPnvwEa0O6Ultbi6SkJNTW1sLR0REvv/yy0q8+dA3dovaxtra2Tn/qIhKJYGDQf/8/U15e3uE+Z2dnnb0yIAMDBThCCG/RMBFCCG9RgCOE8BYFOEIIb1GAI4TwFgU4olXR0dHcxIqGhoawt7dHcHAwMjMzu/UToaysLLWzq/S1vliPg/QeCnBE68LCwlBTU4Oqqir8+OOPmDZtGpYuXYpZs2ahra1N29UjA5k2fydGSEe/qczNzWUAuIku161bx5566ilmamrKhg4dyuLi4ti9e/cYY//5Pe3vX8nJyYwxxrZt28b8/PyYmZkZs7e3Z/Pnz2d1dXXc+9y5c4e9+uqrzMbGhhkbGzMPDw+WmZnJ7a+urmYvv/wys7S0ZNbW1uyFF17gfpuZnJys8r5Hjhzpk78T6Rm6giP90vTp0+Hj44Pdu3cDAPT19bFx40acP38eW7duxeHDh7Fs2TIAwMSJE5GWlgYLCwvU1NSgpqYGCQkJAACZTIaVK1fi3Llz2LNnD6qqqhAdHc29z4cffogLFy7gxx9/RElJCTZt2gQbGxvu2NDQUJibm+PYsWP45ZdfYGZmhrCwMLS2tiIhIQGvvPIKdwVaU1ODiRMnPtk/FOmctiMs0W0dXcExxlhERAQbNWqU2n3fffcdGzJkCLet6Qwop0+fZgC4q7/Zs2ezmJgYtXm3b9/OPD09uVk2GGNMKpUyExMTdvDgwS7rT7SPruBIv8UY4+Yx++mnn/Dcc8/B2dkZ5ubmWLhwIW7fvo2WlpZOyygoKMDs2bPh6uoKc3NzTJkyBcDD3wgDQFxcHHbs2AFfX18sW7YMJ06c4I49d+4cysvLYW5uDjMzM5iZmWHw4MF48OCBRhNQEu2jAEf6rZKSEri7u6OqqgqzZs2Ct7c3/vd//xcFBQVIT08H0Pl6oM3NzQgNDYWFhQW+/vprnD59mpsMs/24GTNm4MqVK3jnnXe49SXab2+bmprg5+enMr13WVkZXn311T5uPekN/fdX3kSnHT58GEVFRXjnnXdQUFAAhUKBdevWcTMdf/vtt0r51U2zffHiRdy+fRuffPIJN3X8mTNnVN7L1tYWUVFRiIqKwuTJk/Hee+9h7dq1GDduHHbu3Ak7O7sOF7Hh0/TefERXcETrpFIpamtrcf36dRQWFmL16tWYM2cOZs2ahcjISHh4eEAmk+GLL77A5cuXsX37dmRkZCiVIRKJ0NTUhNzcXNTX16OlpQWurq4wMjLijtu7dy9WrlypdFxSUhJ++OEHlJeX4/z589i3bx+3hsWCBQtgY2ODOXPm4NixY6isrMTRo0fx3//937h27Rr3vr/99htKS0tRX1/PLThN+gltPwQkui0qKoobYmFgYMBsbW1ZUFAQy8zMVFpTdv369czR0ZGZmJiw0NBQtm3bNpWp1t966y02ZMgQpWEi//znP5lIJGJCoZAFBgayvXv3qqwTOmrUKGZiYsIGDx7M5syZwy5fvsyVWVNTwyIjI5mNjQ0TCoVs2LBhLDY2ljU2NjLGGLt58yYLDg5mZmZmNEykH6LpkgghvEW3qIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4a3/BwsY9IUMJXtgAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T19:54:52.260366Z", + "iopub.status.busy": "2024-02-29T19:54:52.259965Z", + "iopub.status.idle": "2024-02-29T19:54:52.582364Z", + "shell.execute_reply": "2024-02-29T19:54:52.581357Z" + }, + "papermill": { + "duration": 0.352247, + "end_time": "2024-02-29T19:54:52.584806", + "exception": false, + "start_time": "2024-02-29T19:54:52.232559", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAEmCAYAAAD8/yLTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABm80lEQVR4nO2deVyUVffAv8/MMMMO4gKiKG6J+w5hJloYmmWWby5ZLlm2aIuUqS2i9b7hm+ZSmf5abHXvzSwrC8ldRMN930NFcEH2ZZiZ+/tjZGTYZBAYkPv9fJ6PM/ee53nOM8Mcz7333HMUIYRAIpFIaikqeysgkUgk9kQaQYlEUquRRlAikdRqpBGUSCS1GmkEJRJJrUYaQYlEUquRRlAikdRqpBGUSCS1Go29FaipmEwmEhIScHNzQ1EUe6sjkUgKIIQgPT0dX19fVKrSfT1pBMtJQkICfn5+9lZDIpGUwvnz52ncuHGpMtIIlhM3NzfA/CG7u7vbWRuJRFKQtLQ0/Pz8LL/T0pBGsJzkD4Hd3d2lEZRIqillmaqSCyMSiaRWI42gRCKp1UgjKJFIajVyTrCKEUJgMBgwGo32VkUiqdE4ODigVqtv+zrSCFYher2eS5cukZWVZW9VJJIaj6IoNG7cGFdX19u6jjSCVYTJZOLs2bOo1Wp8fX3RarUyyFoisRGTSaBSKQghuHLlChcuXKBVq1a35RFKI1hF6PV6TCYTfn5+ODs721sdiaTGoTcYOX05gzrOWhp6OFK/fn3OnTtHXl7ebRlBuTBSxdxqC49EIimeKxl6jCZBTp4RRVEqbCRl91/kwoUL8ff3x9HRkaCgIHbt2lWi7OHDhxkyZAj+/v4oisL8+fOLyOT3FT4mTJhgkenTp0+R/ueff74yHk8ikVQAeUYTyZl6ABq46Sr02nY1gitXriQ8PJyIiAj27NlDp06dCAsL4/Lly8XKZ2Vl0bx5c2bNmoWPj0+xMrt37+bSpUuWIyoqCoDHH3/cSu7ZZ5+1kvvggw8q9uEkEkmFcTUjFyEEzloNLrqKncWzqxGcO3cuzz77LGPHjqVt27YsXrwYZ2dnlixZUqx8jx49mD17NsOHD0enK/5/g/r16+Pj42M51q1bR4sWLQgJCbGSc3Z2tpKTW98qljFjxjB48OAKvWafPn149dVXS5Xx9/cvdoQgqXnkf98Gk4nkjJteYEUvKNrNCOr1euLi4ggNDb2pjEpFaGgoMTExFXaP77//nqeffrrIB7d06VLq1atH+/btmTZtmgxbKYWyGB9JzWLGjBl07tzZ3mqUieQMPUYhcHRQ4+ZY8Wu5dlsdvnr1KkajEW9vb6t2b29vjh07ViH3+Omnn0hJSWHMmDFW7U888QRNmzbF19eXAwcOMGXKFI4fP86PP/5Y4rVyc3PJzc21vE9LS6sQHSWSO5m8vDwcHBzKfb4QgquV6AVCNVgYqUy+/PJLBgwYgK+vr1X7+PHjCQsLo0OHDowcOZJvv/2WNWvWcPr06RKvFRkZiYeHh+WoiFyCQgiy9Aa7HEKIMuk4ZswYNm/ezIIFCyyLSKdPn2bcuHE0a9YMJycnWrduzYIFC4o9f+bMmdSvXx93d3eef/559Hp9me6bmZnJqFGjcHV1pWHDhnz44YdFZC5fvszDDz+Mk5MTzZo1Y+nSpUVkFEVh0aJFDBgwACcnJ5o3b84PP/xg6T937hyKorBq1SruvfdenJyc6NGjBydOnGD37t10794dV1dXBgwYwJUrV8qkO8CSJUto164dOp2Ohg0bMnHiREtffHw8jzzyCK6urri7uzN06FCSkpIs/fle2nfffYe/vz8eHh4MHz6c9PR0i4zJZOKDDz6gZcuW6HQ6mjRpwn/+8x9L/5QpU7jrrrtwdnamefPmvPPOO+Tl5QHw9ddfM3PmTPbv32/5Tr/++utbPtOxY8fo1asXjo6OtG3blg0bNqAoCj/99JPVZ7ly5UpCQkJwdHRk6dKlXLt2jREjRtCoUSOcnZ3p0KEDy5cvt7p2Sd93Tp4Jg8mEVqPCw6n8xrQ07OYJ1qtXD7VabfXlAyQlJZW46GEL//zzDxs2bCjVu8snKCgIgFOnTtGiRYtiZaZNm0Z4eLjlfX6+stshO89I2+l/3NY1ysuRd8Nw1t7661+wYAEnTpygffv2vPvuuwDUqVOHxo0bs3r1aurWrcuOHTsYP348DRs2ZOjQoZZzo6OjcXR0ZNOmTZw7d46xY8dSt25dqx9rSUyePJnNmzezdu1aGjRowJtvvsmePXushnBjxowhISGBjRs34uDgwMsvv1zsoto777zDrFmzWLBgAd999x3Dhw/n4MGDtGnTxiITERHB/PnzadKkCU8//TRPPPEEbm5uLFiwAGdnZ4YOHcr06dNZtGjRLXVftGgR4eHhzJo1iwEDBpCamsr27dsBs/HKN4CbN2/GYDAwYcIEhg0bxqZNmyzXOH36ND/99BPr1q3j+vXrDB06lFmzZlk+u2nTpvH5558zb948evXqxaVLl6xGUG5ubnz99df4+vpy8OBBnn32Wdzc3HjjjTcYNmwYhw4dYv369WzYsAEADw+PUp/JaDQyePBgmjRpQmxsLOnp6bz22mvFyk6dOpUPP/yQLl264OjoSE5ODt26dWPKlCm4u7vz66+/8tRTT9GiRQsCAwOBkr/vJq3aAlDftXK8QACEHQkMDBQTJ060vDcajaJRo0YiMjLyluc2bdpUzJs3r8T+iIgI4ePjI/Ly8m55rW3btglA7N+/v0x6CyFEamqqAERqamqZ5LOzs8WRI0dEdna2pS0zN080nbLOLkdm7q0/l3xCQkLEK6+8UqrMhAkTxJAhQyzvR48eLby8vERmZqalbdGiRcLV1VUYjcZSr5Weni60Wq1YtWqVpe3atWvCycnJosfx48cFIHbt2mWROXr0qACs/i4A8fzzz1tdPygoSLzwwgtCCCHOnj0rAPHFF19Y+pcvXy4AER0dbWmLjIwUrVu3LlXvfHx9fcVbb71VbN+ff/4p1Gq1iI+Pt7QdPnzY6lkiIiKEs7OzSEtLs8hMnjxZBAUFCSGESEtLEzqdTnz++edl0kcIIWbPni26detmeR8RESE6depU5vN///13odFoxKVLlyxtUVFRAhBr1qwRQtz8LOfPn3/L6w0cOFC89tprQojSv++R454XRxJShdFkKnKN4n5T+djy+7TrjpHw8HBGjx5N9+7dCQwMZP78+WRmZjJ27FgARo0aRaNGjYiMjATMCx1HjhyxvL548SL79u3D1dWVli1bWq5rMpn46quvGD16NBqN9SOePn2aZcuW8eCDD1K3bl0OHDjApEmT6N27Nx07dqyiJzfj5KDmyLthVXrPgve+HRYuXMiSJUuIj48nOzsbvV5fZKK9U6dOVrtjgoODycjI4Pz58zRt2rTEa58+fRq9Xm/x0AG8vLxo3bq15f3Ro0fRaDR069bN0hYQEICnp2eR6wUHBxd5v2/fPqu2gt99/jx1hw4drNpKCt0qyOXLl0lISOD+++8vtv/o0aP4+flZjSLatm2Lp6cnR48epUePHoB5lbtgVuSGDRta7n/06FFyc3NLvAeYw88++ugjTp8+TUZGBgaD4bYiII4fP46fn5/VKC3fiytM9+7drd4bjUbef/99Vq1axcWLF9Hr9eTm5lr+Nor7vuvUqYN/i1YA1HPVoarELaZ2NYLDhg3jypUrTJ8+ncTERDp37sz69estf4Tx8fFWOywSEhLo0qWL5f2cOXOYM2cOISEhVkOJDRs2EB8fz9NPP13knlqtlg0bNlgMrp+fH0OGDOHtt9+uvActAUVRyjQkrW6sWLGC119/nQ8//JDg4GDc3NyYPXs2sbGx9lat3BScvM8fdhVuM5lMt7yOk5NThetT+P63ukdMTAwjR45k5syZhIWF4eHhwYoVK4qdV60MXFxcrN7Pnj2bBQsWMH/+fDp06ICLiwuvvvpqqfPDqdl5mIRAAbxctJWqr91/gRMnTrSaNC5IQcMG5v8dRRkm9B944IES5fz8/Ni8ebPNepaXtJw8XGqgoSuIVqu1Sv21fft2evbsyYsvvmhpK25Raf/+/WRnZ1t+tDt37sTV1fWWc6ktWrTAwcGB2NhYmjRpAsD169c5ceKEJd4zICAAg8FAXFycxXs6fvw4KSkpRa63c+dORo0aZfW+4H+mFYmbmxv+/v5ER0fTt2/fIv1t2rTh/PnznD9/3vI5HDlyhJSUFNq2bVume7Rq1QonJyeio6N55plnivTv2LGDpk2b8tZbb1na/vnnHyuZwt/prWjdujXnz58nKSnJ4qTs3r27TOdu376dRx55hCeffBIwj9ROnDhhed7C37cQgpPnE/nnzGl69+6NWlW5iUZq9q+zBjB7/XG27DtCj9bNGNmuYryEqsbf35/Y2FjOnTuHq6srrVq14ttvv+WPP/6gWbNmfPfdd+zevZtmzZpZnafX6xk3bhxvv/02586dIyIigokTJ95y/7Srqyvjxo1j8uTJ1K1blwYNGvDWW29Znde6dWv69+/Pc889x6JFi9BoNLz66qvFekmrV6+me/fu9OrVi6VLl7Jr1y6+/PLLivlwimHGjBk8//zzNGjQgAEDBpCens727dt56aWXCA0NtUQlzJ8/H4PBwIsvvkhISEiRYWRJODo6MmXKFN544w20Wi333HMPV65c4fDhw4wbN45WrVoRHx/PihUr6NGjB7/++itr1qyxuoa/vz9nz55l3759NG7cGDc3txI3IAD069ePFi1aMHr0aD744APS09Mto6dbLVi0atWKH374gR07dlCnTh3mzp1LUlKSxQgW/r6d3esw7U3z9+14m9M2ZeGODpGxOyYTdx+bxZ/iBa6cPUBypnkDeE3j9ddfR61W07ZtW+rXr09YWBiPPfYYw4YNIygoiGvXrll5hfncf//9tGrVit69ezNs2DAGDRrEjBkzynTP2bNnc++99/Lwww8TGhpKr169rOb/AL766it8fX0JCQnhscceY/z48TRo0KDItWbOnMmKFSvo2LEj3377LcuXLy+z11UeRo8ezfz58/n0009p164dDz30ECdPngTMBmPt2rXUqVOH3r17ExoaSvPmzVm5cqVN93jnnXd47bXXmD59Om3atGHYsGGWOcNBgwYxadIkJk6cSOfOndmxYwfvvPOO1flDhgyhf//+9O3bl/r16xcJWSmMWq3mp59+IiMjgx49evDMM89YPE1HR8dSz3377bfp2rUrYWFh9OnTBx8fnyK7iQp+3w8/2J8ugXfTqXOXSp0LzEcRZRlfSoqQlpaGh4cHqamppU44m34Yh+rQD+xy60dq8Jt0bNMS7zpyi15VoSgKa9asqfAtfBLzMLdXr16lhpbZSmaugdNXMlAUhQAfNxzUJftpOTk5nD17lmbNmhUxxGX9fYIcDlc6qvvfgSNr6SiOsRs9adl5eNext1YSie2sWbPGMh1y6tQpXnnlFe65554KM4AAl9PNu7LqODuUagArEjkcrmzq+EPgeAA8lEyy9Ub0htpdXyQ+Ph5XV9cSj/j4eHurWCql6b5161Z7q1culi5dWuIztWvXDoD09HQmTJhAQEAAY8aMoUePHqxdu7bCdMjWG0jPyUMB6ldwuqzSkJ5gVdD7dTi+EQcMOJLF9Sx3vN0rf8K3uuLr61skTq9wf0VRGbM9peneqFGjCr9fVTBo0CCrOL2C5IfrjBo1ymqVvaLJ9wI9nLXoNFX3+5BGsCpw9oLuYwCoq6RxKbNOpW0GrwloNBqr4PaaRk3WvSTc3NysgrOrmpw8I6nZ5r3NpXqB2Sng6A5KxQ1i5XC4qug4DKHSoFUMuJtSyNLX7iGxRFKQKze8QHdHh5J3M+WkwfWzcOU4iFsHrpcVaQSrCgdHFEfzJvUGpJCSkW1nhSSS6oHeYCIl6xZeoBCQlmB+rXOTnmCNxcEZo0qHWjGhy7lSI2MGJZKK5mpGLgKBq66U1PnZ18GQDYoaXG8/y1RBpBGsShQFlZt5y5EXqWRkZtpZIYnEvhQsoFSiF2gy3fQCXb1BfQfVGKmNKDpXctUuqBRQZyTaWx2JxK5cy8jFJATOWjWuJXmBWVfAlAcqB3CpX+E6SCNoB1QejRACXEU6edkZ9lan0ilc/KhgNmJJ7cVoMnEtI98LdCw+WsJogPQbiZfdG0Il1O2WITJ2wMHRhXS1G26mdETaRXC8C2pRuMylS5eoU0dum6ntXMsvoKRR415SAaWMJBBG0DiBk1el6CE9QTthcm2ISShojVmI3NpVtMnHx6fUjCVVgRACg8FgVx1qMybTzQJK9UuKmTXkQuaNui7uvpXmKEgjaCfcnJ1JFm6Ql43p6inIzQB9ZtUdNu6kSE9PZ+TIkbi4uNCwYUPmzZtX7lKcxRXn+fHHH+nbty/Ozs506tSpSNnVbdu2WQoh+fn58fLLL5NZYGHpu+++o3v37ri5ueHj48MTTzxhlQl606ZNKIrC77//Trdu3dDpdGzbts1m3SUVQ3KW3lxASa3Cw7mEAkrplwABWjdzWEwlIYfDdkKlUshzcIMvumKXDXRvJoDW5dZyNwgPD2f79u38/PPPeHt7M3369CKFj26Ht956izlz5tCqVSveeustRowYwalTp9BoNJw+fZr+/fvz73//myVLlnDlyhVLMt6vvvoKMJd2fO+992jdujWXL18mPDycMWPG8Ntvv1ndZ+rUqcyZM4fmzZvLIbmdMAlhCY6u71ZC6nx9ljksBirVCwRpBO2Ku7N9h4RlJT09nW+++YZly5ZZ6lrk5/KrKF5//XUGDhwImPP/tWvXjlOnThEQEEBkZCQjR460eJ2tWrXio48+IiQkhEWLFuHo6GhVSqF58+Z89NFH9OjRg4yMDFxdXS197777Lv369aswvSW2k5KVR57RhEaloo5zManzhYC0i+bXTnVA61xUpgKRRtCOOLu4cXLccZoa49EqBnDzMcdBVQUOZf/DOnPmDHl5eVaFdTw8PKwKH90uBQsdNWzYEDAXLQoICGD//v0cOHDAqq6wEAKTycTZs2dp06YNcXFxzJgxg/3793P9+nVLPY74+HirBKplzd4sqRyElReoRVVc6vzcdNBnAAq4Nax0new+J7hw4UL8/f1xdHQkKCiIXbt2lSh7+PBhhgwZgr+/P4qiWIVd5DNjxgxLQen8IyAgwEomJyeHCRMmULduXVxdXRkyZEiR+sdVgaJSUc/Li0SNLzg4IXLTQa01D1Mr+6hmq9HFFTrKN2QZGRk899xz7Nu3z3Ls37+fkydP0qJFCzIzMwkLC8Pd3Z2lS5eye/duSzr5wsV8ChcBklQtadl55BqMqFUKXi7FjIQKeoEu9UFT+aMluxrBlStXEh4eTkREBHv27KFTp06EhYWVWNowKyuL5s2bM2vWrFILtLdr145Lly5ZjsIT4JMmTeKXX35h9erVbN68mYSEBB577LEKfbay4unsQI6DB1lCiyJM5pCAakbz5s1xcHCwKqyTmprKiRMnquT+Xbt25ciRI7Rs2bLIodVqOXbsGNeuXWPWrFnce++9BAQElKk8pqRqEUJY0mXVddEVX0ApOxkMOebtcW5VMyqyqxGcO3cuzz77LGPHjqVt27YsXrwYZ2dnlixZUqx8jx49mD17NsOHDy81xEKj0eDj42M56tWrZ+lLTU3lyy+/ZO7cudx3331069aNr776ih07drBz584Kf8ZboSgKPu6OJIq6AIjMq+Y/gmqEm5sbo0ePZvLkyWzcuNFS0EelUlVJOrApU6awY8cOJk6cyL59+zh58iRr1661VCls0qQJWq2Wjz/+mDNnzvDzzz/z3nvvVbpeEtvIyDWQnWdEpSjUcy1mLtBkhLRL5tdu3qCqmtk6uxlBvV5PXFwcoaGhN5VRqQgNDS0SHmErJ0+exNfXl+bNmzNy5EirTMVxcXHk5eVZ3TcgIIAmTZqUet/c3FzS0tKsjorCzVGD0LmSJpxQEDf/EKoRc+fOJTg4mIceeojQ0FDuuece2rRpc8siOxVBx44d2bx5MydOnODee++lS5cuTJ8+3bIwU79+fb7++mtWr15N27ZtmTVrFnPmzKl0vSS2ke8Ferlo0RSXOj/zxvY4tRacK357XEnYbWHk6tWrGI1GSw3TfLy9vTl27Fi5rxsUFMTXX39N69atuXTpEjNnzuTee+/l0KFDuLm5kZiYiFarxdPTs8h9ExNL3ssbGRnJzJkzy61XaeR7gxeveOHGRZScFHMsnw0hLJWNm5ub1cJEZmYmM2fOZPz48bc899y5c1bvC2Z7Lq6WtKenZ5G2Hj168Oeff5Z4jxEjRjBixIgS79OnT59KyTItKRuZuQYycw0oikI912JGcca8m1NBbpWzPa4k7L4wUtEMGDCAxx9/nI4dOxIWFsZvv/1GSkoKq1atuq3rTps2jdTUVMtx/vz5CtLYjItOg87JhevcCOdIu2hzQHNlsnfvXpYvX87p06fZs2cPI0eOBOCRRx6xs2aSmsCVAgWUtJpizE5GkjlRqsbJHBZThdjNE6xXrx5qtbrIqmxSUlKpix624unpyV133cWpU6cA85YtvV5PSkqKlTd4q/vqdLpK3+rl7e7I2ew6eJKJSp9pzqTr5FGp97SFOXPmcPz4cbRaLd26dWPr1q0cPXqUAQMGlHhORsadnyBCUjrZegNp+QWUivMCDTmQedX82qNRlUcu2M0I5v+QoqOjLTVhTSYT0dHRlgnviiAjI4PTp0/z1FNPAdCtWzccHByIjo5myJAhABw/fpz4+HiCg4Mr7L7lwdFBjZuLM1ez3GlAKiL9Ioqje7UIZ+nSpQtxcXFF2rOzs0stPCSR5HuBHk5adMWlzk+7sT1OV7nb40rCrsHS4eHhjB49mu7duxMYGMj8+fPJzMxk7NixgLm6VaNGjYiMjATMiylHjhyxvL548SL79u3D1dXVUvzm9ddf5+GHH6Zp06YkJCQQERGBWq22zBd5eHgwbtw4wsPD8fLywt3dnZdeeong4GDuvvtuO3wK1ni7OXIyqw5eIh2NIReyroFLvVufaCecnJzuyMJDkooh91YFlPSZkJNifu1un0p9djWCw4YN48qVK0yfPp3ExEQ6d+7M+vXrLYsl8fHxqApMkCYkJNClSxfL+zlz5jBnzhxCQkLYtGkTABcuXGDEiBFcu3aN+vXr06tXL3bu3En9+jdXm+bNm4dKpWLIkCHk5uYSFhbGp59+WiXPfKvJeQeNijqujlzOqIMv1xDpl1Cc6oCq9pbolNRcrmTkIrhRQElb6G/YanucFzg42XTtilroUoRcMisXaWlpeHh4kJqairu7+y3ljUYjJ06coEGDBtStW7dUWYPRxImkNFqI8+jyt9NVwfYhiaQi0RtMHE9KRwhBi/quReuHZKfC9TOAAg3agqaY2MFSSE1NJSEhgZYtW1rtOALbfp9y73AVoVar8fT0tOxkcHZ2LjXQ2FOncCHdnUaqa4jrSSgqV1CXkHJIIqmGXE7LwZSnx0mrRi0M5OQUyN8oBCSfB6MA57pgMNm0ScBkMnHlyhWcnZ3RaG7PjEkjWIXkrz6XZUuXEIKktFwyRApa8uBqZqVl1pVIKhqTSZCYloNJQD1XLWdTCw2FczPMW+QUNbjr4IrtRcdUKhVNmjS57V1L0ghWIYqi0LBhQxo0aEBeXt4t5Y8fvMTKqBjmaz9FoEZ5YgV4NasCTSWS2+OrbWf5PvYyrbxdWTSyjbWh0mfC9y9A1lXo9To071Wue2i1Wqs1g/IijaAdUKvVqNW3Xuh4uGtTFm5tx9brDeinjoMt/4bhS295nkRiT9Jz8li0LZ60HCNvD2qBk1OhBY+dC+DyXqjjD92fsHkusKK543aM3Elo1CreCGvNLMNwjEKBY+vgn9vbVy2RVDZLY+NJyzHQor4LYe0KbUDIuAzbF5hf3x9hdwMI0ghWe/q19cazSXtWGvuaG6LeqVbb6SSSguTkGfli61kAXujTsmjS1E2zIC8TfLtCu0ftoGFRpBGs5iiKwpT+AcwzDCFL6ODCbjj6s73VkkiKZfXf57makUsjTyce6Vyo/MLVkxD3tfn1A+9Vi51QII1gjSCwmRcdA1rzudFcg4MNM8xZNySSakSe0cT/bTkDwPjezXEonC5rwwxzDeG7BoB/+RZDKgNpBGsIb/QP4HPjQK4Id0g+c/N/VImkmvDL/gQuXM+mnquWYT38rDvjd5rntBUVhM6wi34lIY1gDaG1jxthXVqxwGBO+iA2zTJnmZFIqgEmk+DTTacBeLpXMxwLJkoQAv58x/y6y1PQIKCYK9gPaQRrEJP6teJH7ue0qSFK1lXY8ZG9VZJIAPjzSBKnLmfg5qjhybubWnce/QUu7DJXOOwzzT4KloI0gjWIxnWceSK4BR8YhgMgdnxSLVPxS2oXQggWbTLn6xwd7I+7Y4HtncY881wgQPBEcK9+e+ClEaxhTOjbkh0Od/O36S4UQzZset/eKklqOdtPXWP/hVQcHVSMvcffujPua0g+Dc714J6X7aHeLZFGsIZRx0XL831a8n7eEwCIvd/D5aN21kpSm1m40ewFDu/RhLoFM0fnppvjAgH6TLVLwtSycFtGMCMjo9IqsElKZuw9/px37cDvxh7mWsX5ww2JpIqJ++c6MWeuoVEpjO/d3Lpz+0fm/cFeLaDbGLvoVxZsNoJnz55l4MCBuLi44OHhQZ06dahTpw6enp7UqVO1BVJqK85aDa+GtuIDw3AMqODEeji71d5qSWoh+XOBj3VthK9ngT3CaZcg5hPz69AZ1ToNnM0JFJ588kmEECxZsgRvb+8qKb4tKcrQ7n58sbUly1LuZ5Qmyryd7pm/qrRUoaR2cywxjQ1HL6Mo8FxIC+vOTZGQlwWNA6HNw/ZRsIzYbAT3799PXFwcrVu3rgx9JGXEQa1iclhrpi99jCHqrbgk7IUja6D9EHurJqklLNxojgt8sH1DWtR3vdlx+Rjs/c78uhptjysJm92GHj16VGjN3YULF+Lv74+joyNBQUHs2rWrRNnDhw8zZMgQ/P39URSF+fPnF5GJjIykR48euLm50aBBAwYPHszx48etZPr06YOiKFbH888/X2HPVFUMaO9Do8ZNWGx4yNywYSYYcu2rlKRW8EPcBX7ZnwDAC30KeYEbZphrCAc8BE3sX7zsVthsBL/44gv++9//8s033xAXF8eBAwesDltYuXIl4eHhREREsGfPHjp16kRYWFiJmZezsrJo3rw5s2bNKrFG8ObNm5kwYQI7d+4kKiqKvLw8HnjgATIzrTPXPvvss1y6dMlyfPDBBzbpXh3IT67whfFBLgtPSPkH/l5ib7Ukdzi7ziYz7Ufzb31i35a0b1SgNva5bXDid3PG6Gq2Pa5EhI3ExMSIZs2aCUVRLIdKpbL8awuBgYFiwoQJlvdGo1H4+vqKyMjIW57btGlTMW/evFvKXb58WQBi8+bNlraQkBDxyiuv2KRrYVJTUwUgUlNTb+s6FcFTX8aKKW+GCxHhLsQsfyGyrttbJckdyj9XM0XnmX+IplPWiRe+/1sYjaabnSaTEP/Xx/x3+Msk+ykpbPt92uwJPv3003Tp0oWYmBjOnDnD2bNnrf4tK3q9nri4OEJDQy1tKpWK0NBQYmIqLnFoamoqAF5e1vU5li5dSr169Wjfvj3Tpk0jKyur1Ovk5uZW23CgN8Jas9oYwklTI3Pdhu3z7a2S5A4kLSePp7/ZzfWsPDo08uDDxztb5ws8vAYS9oCDizkusIZg88LIP//8w88//3zbBbevXr2K0Wi01BjOx9vbm2PHjt3WtfMxmUy8+uqr3HPPPbRv397S/sQTT9C0aVN8fX05cOAAU6ZM4fjx4/z4448lXisyMpKZM2dWiF4VTftGHjzU2Y9ZB4bzpfZD2LkIejwDHo3trZrkDsFgNDFx2V5OXc7Ax92RL0Z3t64jbNBD9I3fxz2vgGsD+yhaDmw2gvfddx/79++/bSNYFUyYMIFDhw6xbds2q/bx48dbXnfo0IGGDRty//33c/r0aVq0aFH4MgBMmzaN8PBwy/u0tDT8/PyKlbUHr/Vrzf0HE4g1BRBkOAYb34fBVVNQXnLn8966I2w5cQUnBzVfjO6Ot7ujtcDfS+D6OXD1huAJdtGxvNhsBB9++GEmTZrEwYMH6dChQ5Gix4MGDSrTderVq4darSYpKcmqPSkpqcRFD1uYOHEi69atY8uWLTRuXLpHFBQUBMCpU6dKNII6nQ6dTldsX3WgSV1nRgb5837ME6zVTUfsW4Zy94vg0/7WJ0skpfBdzDm+ifkHgHnDOlkvhADkpMLm/5pf95kGOldqEjYbwfxQknfffbdIn6IoGI3GMl1Hq9XSrVs3oqOjGTx4MGAevkZHRzNx4kRb1bIghOCll15izZo1bNq0iWbNbl2ict++fQA0bFj9MlzYwsT7WhLy93nWGe/mIfVO2BABT/7P3mpJajBbT15hxi9HAJgc1pr+7Yv5jWybb56LrneXOV9gDcNmI2gymSrs5uHh4YwePZru3bsTGBjI/PnzyczMZOzYsQCMGjWKRo0aERkZCZgXU44cOWJ5ffHiRfbt24erq6tleD5hwgSWLVvG2rVrcXNzIzExEQAPDw+cnJw4ffo0y5Yt48EHH6Ru3bocOHCASZMm0bt3bzp27Fhhz2YP6rnqeObe5nzw1zDC1H/jcGoDnN4ILfraWzVJDSQxNYcXl+7BaBI81rURLxaOBwRIvQg7b0y7hM4EdQ2s4mvLsrNerxdqtVocPHiwvCvXRfj4449FkyZNhFarFYGBgWLnzp2WvpCQEDF69GjL+7NnzwqgyBESEmKRKa4fEF999ZUQQoj4+HjRu3dv4eXlJXQ6nWjZsqWYPHmyzaEu1SlEpiDpOXmi67t/iiVvPW4OVVjUSwij0d5qSWog7/96RDSdsk48/PFWkZNnKF5ozYvmv7Mv+5tDZKoJtvw+FSFsq9/YvHlz1qxZQ6dOnSrQFNc80tLS8PDwIDU1FXd3d3urY8XX28/y0S872eIYjitZ8NjnxDd6CB8PR7QaubdYcmsycw3cHRlNeo6BL0Z1J7Std1GhxEOwuBcg4JloaNy9yvUsCVt+nzb/It566y3efPNNkpOTy62gpHJ5IqgpLl7efJpn3rie+XsE/Wb/wew/Kib0SHLn80PcBdJzDDSr58J9ASWEu2yIAAS0HVytDKCt2DyA/+STTzh16hS+vr40bdoUFxcXq/49e/ZUmHKS8qHVqHj9gdZMWdGf0ZoovLMTeEodxedbtbw1sK291ZNUc4wmwVfbzQXUn77Hv2gBdTDPNZ/aACoN3D+9ijWsWGw2gvkruZLqzcMdffm/zfWZc/lfzHb4jJc0a9jr9aC91ZLUAKKPJnHuWhYeTg4M6VZMeJnJBFE3DF/3cVC3+LCymoLNRjAiIqIy9JBUMCqVwpQBAYxdksI49e8EqM4zQv8D8JC9VZNUc77cZvYCRwQ2wVlbjIk49AMkHgCtG4S8UcXaVTzlniWPi4vj+++/5/vvv2fv3r0VqZOkgujdqh4Pd27MLMMIAAbl/gLX/7GSyckzkpNXtthOyZ3PoYupxJ5NRqNSGN2zaVGBvByIfs/8uter4FKvSvWrDGz2BC9fvszw4cPZtGkTnp6eAKSkpNC3b19WrFhB/fr1K1pHSTlRFIUFw7twok8Lti/8lXvUh+Gvf8OQzwHIM5ro+l4UTg5qdr8VWvzcj6RWke8FDuzYkIYeTkUFdn8OqfHg5gt3v1jF2lUONnuCL730Eunp6Rw+fJjk5GSSk5M5dOgQaWlpvPxy9SypV9tx1mmIvOENcnAVJOwD4OL1bLL0Rq5l6snUG+ynoKRakJSWY0mUOq5XMTutspJhy2zz6/veAq1zFWpXedhsBNevX8+nn35KmzZtLG1t27Zl4cKF/P777xWqnKRicNFqOCSas8Z4j7kh6h0Qgozcm4Yvz2hTuKjkDuTbmHMYTIIe/nXo2NizqMC2ueZ9wg3aQqcRVa5fZWGzETSZTEWSJgA4ODhU6JY6ScXhojPPenxoGIpQa+HsFjgVTWp2nkVGb5DfXW0mW29kaWw8UIIXeP0fiP0/8+t+74JKXVSmhmKzEbzvvvt45ZVXSEhIsLRdvHiRSZMmcf/991eocpKKQatR4aBWuCDqk9n5aXNj1HSSM7ItMrkGuThSm/nfngukZOXh5+VEv7bFZHHa+B8w6qFZb2gZWrS/BmOzEfzkk09IS0vD39+fFi1a0KJFC5o1a0ZaWhoff/xxZegoqQDyQx0ud5oIjh5w+TBep24mkZWeYO1FCMG3MecAGNuzGerCC2SX9sOBlebX/d6t9tXjbMXm1WE/Pz/27NnDhg0bLBmg27RpY5UmX1L9cNGqSc3OI0PlBve+DlHv0PHEJ+j4gFy05EojWGvZE3+dE0kZODqo+Ff3QsHRQsCf75hfd3gcfLtUvYKVTLny3iiKQr9+/ejXr19F6yOpJPLnBTNzjRA4HnZ9hlvqeZ5Wr2eRcZA0grWY5bvMJXQf6uiLu2Oh+f7T0XB2M6i1cN/bdtCu8imXEYyOjiY6OprLly8XWQxZskSWfKyOOFuMoAEcHOG+d2DNeF7QrGWFsY8cDtdSUrPzWHfAPL8/IrCJdafJCFE3dogFjoc6/lWrXBVh85zgzJkzeeCBB4iOjubq1atcv37d6pBUT1xuFMWxxAN2eJxzDi1wV7J5SfOTXBippfy87yI5eSbu8nalaxNP684DKyHpkHkO+d7X7KJfVWCzJ7h48WK+/vprnnqq5qXRrs3kD4ez9DeMnUrFZ7qxvJ/3Nk+qo9h27QzQgNTsPM5ezeR6pp5ererhoJb5B+9UhBAsuzEUHt6jCUrBBY+8bPPuIjAbQGevYq5wZ2CzEdTr9fTs2bMydJFUIhZPsECA9Ma8tmw2diREfYC8qJl0/yucqxl6S//zIS2YOiCgynWVVA0HLqRy9FIaWo2Kx7o2su6MXQxpF8HDDwKfs4+CVYTN/80/88wzLFu2rDJ0kVQizgUXRjB7AcmZemYZRmASCmFiB40yzfVb6rlqAfh6x1kup+fYR2FJpbNitzk4+sH2Png6a292ZF6DrXPNr+972zyHfAdjsxHMyclh7ty5hISE8NJLLxEeHm512MrChQvx9/fH0dGRoKAgdu3aVaLs4cOHGTJkCP7+/iiKwvz588t1zZycHCZMmEDdunVxdXVlyJAhRUp/3mnke4JZN+YEs/OM5BpMHBVNSb1rCADL/H/j0IwH2P1WKF2aeJKTZ+LTjaftprOk8sjINbB2XwkLIlvnQG4aeHeADkPtoF3VYrMRPHDgAJ07d0alUnHo0CH27t1rOfJLV5aVlStXEh4eTkREBHv27KFTp06EhYVx+fLlYuWzsrJo3rw5s2bNKrE2cVmuOWnSJH755RdWr17N5s2bSUhI4LHHHrNJ95pG/pxg/n7h5EzzsFerUeE5cAaodbhc2onrP9EoisLrD7QGYFlsPAkp2cVeU1Jz+WV/All6I83ruxDYrMB8X/JZ2GXOMsQD74KqFswJV27Np9IJDAwUEyZMsLw3Go3C19dXREZG3vLcpk2binnz5tl8zZSUFOHg4CBWr15tkTl69KgARExMTJl1r67V5kris82nRdMp68SrK/YKIYQ4cD5FNJ2yTgT9Z4NZ4M/p5qphnwQKYcgTJpNJDF28QzSdsk5M+/GA/RSXVAoPf7xVNJ2yTny2+bR1x+qx5r+DbwfbR7EKwpbfp93MvF6vJy4uzmqniUqlIjQ0lJiYmEq7ZlxcHHl5eVYyAQEBNGnSpNT75ubmkpaWZnXUJJx11gsjyVlmT7COy425oF6TwKkOXDkG+5aiKAqv3fAGV+0+T/y1rKpXWlIpHE5I5cCFVBzUivWCyMU4OPQ/QDHXEK4l2M0IXr16FaPRiLe3dSk/b29vS8H0yrhmYmIiWq3WkhC2rPeNjIzEw8PDcvj5+ZVLR3vhmr8wcmNO8PqN4bCXy40dAk6e0PtGqvSN74M+k8BmXtzbqh4Gk+Cjv05WtcqSSmLFjbCYB9r5UNdVZ24UAv68UTek03Bo2NFO2lU9tWDAXzFMmzaN1NRUy3H+/Hl7q2QT+QkU8leH8+cE6xRcFewxDjybQkYixHwKYPEGf9xzgdNXMqpQY0llkKU38NPeiwA8UXBB5OSf8M82UOug71t20s4+2M0I1qtXD7VaXWRVNikpqcRFj4q4po+PD3q9npSUFJvuq9PpcHd3tzpqEoVXh69n5XuCBYygRnezfOL2+ZBxhc5+noS2aYBJwIIN0hus6fx64BLpuQaaeDkT3LyuudFouFk97u7nwbNmjXJuF5uN4JYtWzAYiqZiNxgMbNmypczX0Wq1dOvWjejoaEubyWQiOjqa4OBgW9Uq8zW7deuGg4ODlczx48eJj48v931rAi664j1BKyMI0O4xc6YQfQZs/i8Ak/rdBcAvBxI4npheRRpLKoPlu8yxgcMD/W7WlNm/zDwX7FQHetke5lbTsdkI9u3bl+Tk5CLtqamp9O3b16ZrhYeH8/nnn/PNN99w9OhRXnjhBTIzMxk7diwAo0aNYtq0aRZ5vV7Pvn372LdvH3q9nosXL7Jv3z5OnTpV5mt6eHgwbtw4wsPD2bhxI3FxcYwdO5bg4GDuvvtuWz+OGoOLznrvcIlGUKUy54wDiPsKrp6ina8HD3bwQQiYF3WiynSWVCwnktLZE5+CRqXwr/x6wvpM8xwwQO/J5rnhWobN2+aEENZ7DG9w7do1XFxcbLrWsGHDuHLlCtOnTycxMZHOnTuzfv16y8JGfHw8qgJxSgkJCXTpcjOf2Zw5c5gzZw4hISFs2rSpTNcEmDdvHiqViiFDhpCbm0tYWBiffvqpTbrXNPLnBLNKmxPMp1lvaBUGJ/+A6Jkw7Dsmhd7F74cSWX84kUMXU2nfyKPKdJdUDPleYGgbbxq43dgFsvNTSL8Enk2gxzN21M5+KEKIMlXYyQ8mXrt2Lf3790en01n6jEYjBw4coHXr1qxfv75yNK1mpKWl4eHhQWpqao2YH0zNyqPTu38CcOLfA3jo462cSMpg6TNB3NOymNqxSUdg8T0gTDAuCvwCeXXFXn7al8B9AQ1YMqZHFT+B5HZ5+ONtHLyYyqcju/Jgh4aQcQU+6gL6dBjyJXT4l71VrDBs+X2WeTicHxoihMDNzc0qXMTHx4fx48fz/fff37byksohP04QzIsjyZnmIkvFeoIA3m2h80jz6z/N1eleCb0LtUrhr2OXiftHpk2raeQX1vJ2v+HAbPnAbAAbdjbPBddSyjwc/uqrrwDw9/fn9ddft3noK7EvDmoVWo0KvcFERq6h+NXhwvR9Ew7+AOd3wrFfadbmIYZ0bcSqvy8wL+oE3z8TVEXaSyqCfCPo4eQA107D3zcSID/wXu3YHlcCNj95RESENIA1lPwwmaS0HIwm8yyIp3PR8qkW3H0heIL59YYIMObx0n2tcFArbDt1lZjT1ypbZcltciIpHb3BhMkkSM8xG0F3JwfYMANMBmj1gHkOuBZjsxFMSkriqaeewtfXF41Gg1qttjok1Zf8MJnzyeaECC5aNY4Ot/jO7nkFnOvCtVOw51v8vJwZ1sMcR/b51jOVqq/k9th0/DIPzNvCzF8Ok6E3cOP/PTyu7oWjP4OiqlXb40rC5tXhMWPGEB8fzzvvvEPDhg2LXSmWVE9cbqwQX7hu3gdcp7ShcD6O7hAyFX6fDJsioeNQnr6nGd/vjGfT8ctcTMmmkadTZaotKSd/HDZvGrhwPZu0G0NhnUZB99cMs0DnJ8xzv7Ucm43gtm3b2Lp1K507d64EdSSVSf7iyIXrZk+w1PnAgnQbA7GLIPkM7PiE5n2nEdy8LjFnrrFy93nCbwRTS6oXMaevAmASwjIfOEi31zzHq3GCPm/aU71qg83DYT8/P8oYVSOpZuR7gufzPcGSVoYLo9HC/Teqju34GNITGRFk3ne6cnc8BqOsVFfduJiSzbkbmX+MJrMR1GDgZbHULBD8Ing0KuUKtQebjeD8+fOZOnUq586dqwR1JJWJS3k9QYC2j0Cj7pCXCZtmEdbOGy8XLUlpufx1rPgkuBL7UXDRyiQEadkGhqk34We6aJ7jvecV+ylXzbDZCA4bNoxNmzbRokUL3Nzc8PLysjok1Zd8TzA/U3SZPUEARTGHUgDs+Rbd9VM8fmPrVf5OBEn1YceNoTCAyQRZ6Sm8qvnB3BAyxVxGUwKUY06wpLoekupP/pxgntE8nWHJJVhWmvaE1gPh+K+wYSbD+33O/205w6YTV7hwPYvGdZwrWmVJORBCsLOAJ2gUgqbHl1BfSeOKgy/1u421o3bVD5uN4OjRoytDD0kVkB8ik0+ZVocLEzoDTqyH47/SrOd+eraoy47T11i1+zzhN3IPSuzLP9eySEi9WSXQPe8aHa58C8BfjV5gmKYc3/sdTLnCxE+fPs3bb7/NiBEjLAWMfv/9dw4fPlyhykkqlvzhcD5etgyH86l/F3QdZX7959uMuBEzuPLv83KBpJqw44YXmJ8pa2jWUrSmbPaaWnKh4QN21Kx6YrMR3Lx5Mx06dCA2NpYff/yRjAxztuH9+/cTERFR4QpKKg5nrXVgdLk8QYA+08DBBS7G0V8VS90bCyTRcoGkWpA/H9ixsSctlIs8kPMHAO/nPYFHef7ju8Ox2QhOnTqVf//730RFRaHV3vxA77vvPnbu3FmhykkqlsLDYZtWhwvi5g33vAyAw8Z3GdrVnKZMLpDYHyGEZWX43lb1mKJZgRoTe5x6slsEmLfMSayw2QgePHiQRx99tEh7gwYNuHr1ajFnSKoLReYEb8crCJ4ILg3g+lmecdwEwOYTVzifLKvS2ZMTSRlcy9Tj5KCmr9MpHlDHYUTFV07muXx3R2kEC2OzEfT09OTSpUtF2vfu3UujRjL4sjrjUmg4XGryhFuhc4W+5qzfdePmE9rcESFg7b6Lt6Oi5DbJHwr38K9Dq/0fAPCbwwMcNTQEbmSQkVhhsxEcPnw4U6ZMITExEUVRMJlMbN++nddff51Ro0ZVho6SCsK5wMKIu6MGB/Vtpk/qMgrqtoKsa7ygWQeYPZFxX+/m8y0yuYI9yF8UGem2D7er+8gUOr52GG6dRktihc2/gvfff5+AgAD8/PzIyMigbdu29O7dm549e/L2229Xho6SCsK1wHC43POBBVFroJ85C0nnC0vx4Ro/708g+thl/vPb0du/vsQmjCZB7JlrOGCgzwVzuYjPjQO5pnhajKC7k81RcXc8NhtBrVbL559/zunTp1m3bh3ff/89x44d47vvvit3Kq2FCxfi7++Po6MjQUFB7Nq1q1T51atXExAQgKOjIx06dOC3336z6lcUpdhj9uzZFhl/f/8i/bNmzSqX/jWFgtmly70yXJjWD0KTYNSmXCZp/lcx15SUiyMJaaTlGBjruAld2jnynOrxuWEgWXojeoM5fEl6gkUp93ioSZMmPPjggwwdOpRWrVqVW4GVK1cSHh5OREQEe/bsoVOnToSFhVniDwuzY8cORowYwbhx49i7dy+DBw9m8ODBHDp0yCJz6dIlq2PJkiUoisKQIUOsrvXuu+9ayb300kvlfo6aQME4wXLFCBaHokA/83a6f6k301qRK8T2Ysfpq7iSxUsq839GiV0mkYkTKVlmL1ClWI8GJGbK9ImEh4fz3nvv4eLiQnh46XVJ586da5MCc+fO5dlnn7WUxFy8eDG//vorS5YsYerUqUXkFyxYQP/+/Zk8eTIA7733HlFRUXzyyScsXrwYoEgR9bVr19K3b1+aN29u1e7m5lbuQu81kUrxBAH8epDefCBuZ35limYFT+e9UXHXlpSZHaev8ZxmHW6mVKjbipSA4fBXLPobQezuTg4y/2cxlMkT3Lt3L3l5eZbXJR379u2z6eZ6vZ64uDhCQ0NvKqRSERoaSkxMTLHnxMTEWMkDhIWFlSiflJTEr7/+yrhx44r0zZo1i7p169KlSxdmz55dbFH5OwkrT7AijSCQds808oSa+9T7CFbJnUNVjd5g4p9zp3hGfWNqKHQGKo310FcOhYunTJ7gxo0bi319u1y9ehWj0WhVExjA29ubY8eOFXtOYmJisfKJiYnFyn/zzTe4ublZSobm8/LLL9O1a1e8vLzYsWMH06ZN49KlSyV6srm5ueTm5lrep6Wl3fL5qhtqlYKjg4qcPNPtxQgWd+16rVhqvJ8xmj+ZplnGI/r3KvT6ktI5cCGF500rcdLoEX53owQMRJWYbiUjjWDx3PETBEuWLGHkyJE4OjpatRcc1nfs2BGtVstzzz1HZGSkVU3lfCIjI5k5s+bXY3DRasjJ09ueQeYWaDUqPjY8yhD1VjqqzvKwaifwcIXeQ1Iyxw7EMkK9GQDlgfdAUVCrrIe+MlC6eMpkBAt7UaXx448/llm2Xr16qNVqkpKSrNqTkpJKnKvz8fEps/zWrVs5fvw4K1euvKUuQUFBGAwGzp07R+vWRbOhTJs2zcpwpqWl4efnd8vrVjdcdBquZeor3BPUalRcw4PFhoeZ7LCKyZqVYJhOzD8ZXE7PYVAnXzkfVYm0PTwXtSL4xzuUpn6BAKgKfd7SEyyeMs0JFiy07u7uTnR0NH///belPy4ujujoaDw8bEvUqNVq6datG9HR0ZY2k8lEdHQ0wcHBxZ4THBxsJQ8QFRVVrPyXX35Jt27d6NSp0y112bdvHyqVigYNGhTbr9PpcHd3tzpqIm0auqFRKbRpWLH6a28EXn9pHECiqIOf6gpi1+e8tHwvr6zYR/iq/ZhMsixDZaA/uYmuubvIE2pUodMt7UU8QWkEi6VMnmB+4XWAKVOmMHToUBYvXmyJCzQajbz44ovlMgzh4eGMHj2a7t27ExgYyPz588nMzLSsFo8aNYpGjRoRGRkJwCuvvEJISAgffvghAwcOZMWKFfz999989tlnVtdNS0tj9erVfPjhh0XuGRMTQ2xsLH379sXNzY2YmBgmTZrEk08+SZ06dWx+hprEJ090JTU7j3quRYf8t4ODWkFRIEfomGv4Fx84fA5bZqPP+ABwZc3ei/Tw9+KJG7VJJBWDMBm5umYqvsBP6gf4V8uOlj61UtgI3vGzX+XC5jjBJUuW8Prrr1sFRqvVasLDw1myZInNCgwbNow5c+Ywffp0OnfuzL59+1i/fr1l8SM+Pt5qr3LPnj1ZtmwZn332GZ06deKHH37gp59+on379lbXXbFiBUIIRowYUeSeOp2OFStWEBISQrt27fjPf/7DpEmTihjSOxEHtarCDSCYA9TzvcH/GXtz3NQYJSeFiQ6/WGS+jTlX4fet7ez+9Ut8s46SIRzZ3fQZqymHwrMPcjhcPDb/12AwGDh27FiRebNjx45hMpUvqebEiROZOHFisX2bNm0q0vb444/z+OOPl3rN8ePHM378+GL7unbtKtN+VQJajYpcgwkjamYZRvCVdjajVev5mn4kUI+rGXp7q3hHkZqWgW+ceRfUYsPD9Ghn/ZssPByWRrB4bDaCY8eOZdy4cZw+fZrAQPMEbGxsLLNmzbIMYSW1E51GRX5QxkZTZwxN7kEXv53XHFbzWt4L5BqMdtXvTmLN3gsc/GEW0x0ukyQ8cev7Co92sc7iJFeHy4bNRnDOnDn4+Pjw4YcfWoapDRs2ZPLkybz22msVrqCk5qDTFNw7rpBx73Q8l4bxqGobXyoDOGVoXuK5krJzNSOXd1fH8JfDGgAygqfwXGiHInJydbhs2DwnqFKpeOONN7h48SIpKSmkpKRw8eJF3njjjXInUJDcGWg11n9OOQ0687MxGJUimKpZjt5gQgi5Qny7/LjnAuNVa6mjZJDs3JwW/Yqf9pHD4bJxWwnlanKoiKTi0RbKT5hnNDHbMBS9UNNbfZB7VQfYdz6Fp76M5cCFFPsoWQOJOX2NkV/s5NTlDIQQbIzdw9Pq9QB4PRJpTmlWDIVsoAyRKYFyrZn/8MMPrFq1ivj4ePR668nuPXv2VIhikppHYU9QbzRxXnjznfEBxml+Z5pmOQM/bY9AxdaTV1n2bBA9W9Szk7Y1hxGfmxfxZv5ymFdDWzEk7Vt06jwMfj3R3BVW4nkq6QmWCZs9wY8++oixY8fi7e3N3r17CQwMpG7dupw5c4YBAwZUho6SGoKusBG8kcPuY8Ng0oQzbVX/MFi13dL/zDd/k5MnF0vKSm6eiTOHYnlMtRUATdi/i8bBFKBwnKCbo4wTLA6bjeCnn37KZ599xscff4xWq+WNN94gKiqKl19+mdTU1MrQUVJDKOwJ5t1I4ZSCG58aBgHwusMqdJhHD1l6I3vir1etkjWMLP3NzEbN67vQ9fg8VIrgsFcoNO5W6rkF5wRdtOrbL6dwh2LzpxIfH0/Pnj0BcHJyIj3dHBTx1FNPsXz58orVTlKjKDIcNtyMG12hGkiC8KKRco3R6j8s7TtOXasy/Woiu8/d/E+is34PLdJi0Qs1B+66dQLggqvDcihcMjYbQR8fH5KTkwFzdun8oOOzZ8/Klb9aTuGFkfxkngAqrRMf5g0FYKJmLZ43Igq3n5ZlWktj442C9gom+sR/AsD3xn7oGrS85bkFPUG5KFIyNhvB++67j59//hkwB05PmjSJfv36MWzYsGLrEUtqD0WHwzf/U3RyULPG1Iujpia4K1lM0KwF4MCFVNJz8qzO++NwIo98so2zVzMrX+lqjBCCv24YwcGq7fhknyQTZz42DKa+2623PhZcF5FGsGRsnin97LPPLNvjJkyYQN26ddmxYweDBg3iueeeq3AFJTUH62Bp6+Gwo4MKEyoiDSP4VvtfRqn/5BvjA1wwNSD2TDKhbW8myn3uuzgAJq/ezw8v9Kwa5ashZ65mEp+chQ49rzusAmCJ8ijXcS/T/m9zATEQQg6HS8MmT9BgMPDvf//bKovz8OHD+eijj3jppZfQais2R52kZlHSwohKAUcHs4HcYurIVmN7dIqB1zXmH3ZJQ+LrWbV7r3H+UHi0+g8aKddI0dRnYU4/gDInwchfIZZb5krGJiOo0Wj44IMP7vhaHJLyUThE5qYRVAr0KcwyPAHAYPUO2itnSlwcqY1JWM9ezeTFpXEcupjKX8cu40k6LzmYpw5Wu48mR2hRKWWvEZMfKyg9wZKxeU7w/vvvZ/PmzZWhi6SGU9gTzDUUNII3h8qHhT8/GnsB8KZmGceT0uj1379Ytft81SlbTXl15T5+O5jIQx9vY/e5ZCZo1uJGFkdNfvxoMH9mXi66IlviSiLfE5RGsGRsnhMcMGAAU6dO5eDBg3Tr1g0XFxer/kGDBlWYcpKahYPa+odp8QRVoHOwNpAf5j3Ow5pYeqqP0Me4n03XO/PG/w4wtMfNkgW1zw+E05czLK+9TUmM1v4JwCzDEySkmxeQyrIokk++rZQJVUvG5k/mxRdfBIqvL6woCkaj3AFQW1EKma08Q3HDYTMXqc92ryH0ubaCqZrlbNF3xIQKQ4Gwmlo4GsZVpyEj1zzd9LpmFVrFQIJXEJsTOkK22QjWcy373LscDt8am4fDJpOpxEMawNqNwDpOVG8sfjicz7aGo8jTehCgOs8Q9RYAJv9woPIVrcbkb21rr5xhsHoHAPtaT6KgX2yLJ6iWRvCWyH00kkojP05QpRRdNAHAsQ6mXuYclK9pVuNILmv2XrR0F/YsawPOOg0geFOzDIBY11DS6rSzkqlvQ3kEy+qwNIIlUubhcHZ2NtHR0Tz00EOAuQRlwWLkarWa9957r0h9X0ntJT9OUKVSiswJgjlsRtfzeTJiFuOTncDT6t/51Di4irWsXugNJvqo9tNTfYRcoeFcp/AiiyC2eIIN3B25nqXHr45zRat6x1BmT/Cbb77h//7v/yzvP/nkE3bs2MHevXvZu3cv33//PYsWLSqXEgsXLsTf3x9HR0eCgoLYtWtXqfKrV68mICAAR0dHOnTowG+//WbVP2bMmBuBojeP/v37W8kkJyczcuRI3N3d8fT0ZNy4cWRkZCApP4V3Td5qOOzooAKNDpcB5qL2L2h+wYs0S//xpHSy9bVjimX3uWQOJ6SSkZXDVI15D/7XxjDatGmHptCCky2Fsr4c3Z01L96Dj4d0TkqizEZw6dKlRQoXLVu2jI0bN7Jx40Zmz57NqlWrbFZg5cqVhIeHExERwZ49e+jUqRNhYWFcvny5WPkdO3YwYsQIxo0bx969exk8eDCDBw/m0KFDVnL9+/fn0qVLlqNwcoeRI0dy+PBhoqKiWLduHVu2bCmxMJOkbBTeOV7awgjc3GGitP8XiS6tcVOyeUmzxkrmy21nAJgbdYKPok9WvNLVgOOJ6Ty+OIaBH23j7ow/CVCdJ0W4sNDwCG0buqNRWX92tniCvp5OdPLzrGCN7yzKbARPnTpFhw436xg4OjqiKvDlBAYGcuTIEZsVmDt3Ls8++yxjx46lbdu2LF68GGdn5xLLdy5YsID+/fszefJk2rRpw3vvvUfXrl355JNPrOR0Oh0+Pj6Wo2A94aNHj7J+/Xq++OILgoKC6NWrFx9//DErVqwgISHB5meQFI/BVHBOsARPEEClYpv/KwA8qd5AU+XmjqRjielcTs/ho+iTzI06ccflHzyemE7YfPOikCO5hGt+AOATw2DScEWjVqFRld8TlNyaMhvBlJQUqznAK1eu4O/vb3lvMpms+suCXq8nLi6O0NDQmwqpVISGhhITE1PsOTExMVbyAGFhYUXkN23aRIMGDWjdujUvvPAC165ds7qGp6cn3bt3t7SFhoaiUqmIjY0t9r65ubmkpaVZHZLSsQqWLmZOUOdw0zAmeAWy0dgJB8XIZM1KS/uRhDQupeRUvrJ24sc9Fyyvn1avp6GSzAVRjxVKGItGdgWK1gqxxROU3JoyG8HGjRsXGXIW5MCBAzRu3Nimm1+9ehWj0WgptJ6Pt7e31f7kgiQmJt5Svn///nz77bdER0fz3//+l82bNzNgwABLCE9iYiINGjSwuoZGo8HLy6vE+0ZGRuLh4WE5/Pz8ipWrzQxo72P1PvNGvJtapRRJswU39xPny8wyjMAkFB5Sx9JZOQWYkwg8snB7kXPvFPL3R3uRxgsac3amecZhHHxvEAM6NASwmhNUqxQ85UpvhVJmI/jggw8yffp0cnKK/q+cnZ3NzJkzGThwYIUqV16GDx/OoEGD6NChA4MHD2bdunXs3r272ELuZWXatGmkpqZajvPn5RavwnRs7Mlfr4Uwpqc/AOsOmKcWFKXojhEAxwLzhGqVwnHRhB+MvQGY5rCMorOMdx7Xs8wB0C9p1uCmZHPI5M8mbYjVvml1gWmneq7aIrVDJLdHmY3gm2++SXJyMq1bt2b27NmsXbuWtWvX8sEHH9C6dWuuX7/Om2++adPN69Wrh1qtJikpyao9KSkJHx+fYs/x8fGxSR6gefPm1KtXj1OnTlmuUXjhxWAwkJycXOJ1dDqdpbqerLJXMs3ruzI5rDXdmtbhxpRgKavDBTzBGz/6uYZ/kSMcCFIdI1RVtGjXnZa393hiOk2VRJ5UbwDgfcMTuDpZ7wgpOCco5wMrnjIbQW9vb3bs2EGbNm2YOnUqjz76KI8++ijTpk2jbdu2bNu2rcgw9VZotVq6detGdHS0pc1kMhEdHU1wcHCx5wQHB1vJA0RFRZUoD3DhwgWuXbtGw4YNLddISUkhLi7OIvPXX39hMpkICgqy6RkkRXHRafhqbA86NPIAzHuKi1sdLmgE8x2fROrypdFcsGuqZjlq7qyFkIKkZucRn5zFZM0qHBQjm4yd2GFqXyTtlVoawUrFpr3DzZo1Y/369SQnJ1u8qpYtW+Ll5VVuBcLDwxk9ejTdu3cnMDCQ+fPnk5mZydixYwEYNWoUjRo1IjIyEoBXXnmFkJAQPvzwQwYOHMiKFSv4+++/+eyzzwDIyMhg5syZDBkyBB8fH06fPs0bb7xBy5YtCQszlyds06YN/fv359lnn2Xx4sXk5eUxceJEhg8fjq+vb7mfRXITd0cHvn06kH//epRereoWyTADJewiARYbBjFC/RctVQkMVW9iufF+S1/hrXk1mX3nU+isnOIh9U5MwjwnCkWrwhX0BOWiSMVTrm1zXl5eBAYGEhgYeFsGEGDYsGHMmTOH6dOn07lzZ/bt28f69estXmV8fDyXLl2yyPfs2ZNly5bx2Wef0alTJ3744Qd++ukn2rdvD5h3rhw4cIBBgwZx1113MW7cOLp168bWrVvR6W7+AS1dupSAgADuv/9+HnzwQXr16mUxpJKKoY6Llg+HduLRLo2LHcYW9AQL9qfjzEeGxwCYpPkfztych/71wM2/hZpO3LnkG3Of8D/jvRwTTYBijKC64JygNIIVTbXIrzNx4kQmTpxYbF9xixmPP/44jz/+eLHyTk5O/PHHH8X2FcTLy4tly5bZpKek/JiKsYKOBRZLCvcvNYYyVr2epqrLPKP+jY+MZqM4+YcDPNqlkZVhqKmI478TpDpGjnBgruHm37NboeGw9AQrl5r/lySpERhNxRnBAp5gob62jeuys7n5P8bnNL9Qj5s1rS+l1vy4QaMhj8FXzSOPJcYBXKKupa/0OUFZwqKikUZQUiXULSYdvKOm+OEwmHeYPPbkRPaZmuOi5PKK5n+WvvjkrErTs6q4tOkLWigXuS7cWGSwTkQs5wSrFmkEJVVCcIu6TOzbkifvbmJp05UyHNY5qHDQqGkzagEAI9R/0Vwxxx3OizpRYxMrfBtzjr7v/4p26ywA1td9inSsM7wUNoIFPUFb0mhJyoY0gpIqQVEUXg9rTVi7m3GYJa0Ow81C7rqWvYkydkWjmHjjxna6v/+5zoIamEwhPSeP6WsP81DmjzRQUvjH1ID0DqOLyBUeDhf8D0J6ghWPNIKSKiXPKn3+TQ9HFOMJ5tNxzDyMQqG/ejfdlOMAbDlxpZI1rXie+nIX9UjlOc06AFZ7jmVkz5Y4OVgHkheuB9LA3RGtRoWbTiMzRFcC0ghKqpQGbsXntSu8blJwh4l3i86s15rr7b55YztdE6+akyQ09sw1Jizdw77zKbys+RFXJYfcBp15/dWpuOg0VqvkUHR12N3RgahJvYkKD6mVZUgrm2oRIiOpPbRv5MF/Hm1fxIgVXRixNgz+Q/5N1vLNdFOdJEy1m/WHAytb1Qpj2Gc7AWiuJPCE2rzbSXngXXMZPsDJQc118izyhecEAZrWdSnSJqkYpCcoqXJGBjXl3lb1rdoK7wQpbATbBbTGdPcEAKZoVqDBYOmbG3WCJ7+ItaTztzdCCH6Iu8DxxHSr9smalWgUExuMXdC2DLG0O2qth8OFPUFJ5SKNoKRaUGQ47FA04YLrfeFcFe40VyUyXL2RuH+uA/BR9Em2nbrKn0eKT4NW1Ww8fpnXV++3JEsF6KYcZ4B6N0ahcKbTZCt5x0LJJdyL8QQllYc0gpLqQeGFkeJWjnVu/J8yFIBXNf9j1KJoS85CoNqEzZy7ejOO0RwkLm7MZcIqYx/G/8s65ZyT9ATtijSCkmpBr0LD45LCZwaOncoZkw/1lDTGa9bR5d0oS19xW/PsQcEwlg4z/iBMtZtuqpNkCy3zDP8qIl94YaS4ZBOSykN+2pJqQWAzL+4LuJntu7j8gwBtGnnxX8NwAJ5V/4an8WbZhGJ25tmFgjV+9fpcpmhWAPC58UEuU6eIfOEQGUnVIo2gpNrwaJdGltfFZaIGs3H8z7Rp7BV34azk8uqNwkRQfTzBgjGPw9Ubaa5K5Kpw5zPDQ8XKFzf/Kak6pBGUVBt6triZRCA9x1CiXD03R9bWfx6AYepNtFTMxYqqiyeYr4YL2bx6Y8/zAsNjZFB8bGNBT1CWx6x6pBGUVBvqFtgXW7jCWmFOObZnvbEHakVYhpvXM/UcSUjjkU+2cSk1u1J1LZUbVnC8Zh31lDTOmHxYbryvRPEW9V0tr799uubEP94pSCMoqVZ8Ny6Qx7s1ZkSPJqXK5RlNfGAYhkGo6KfeQ5BylLlRJ3jwo63sv5BKcORfVaSxmfPJWYSv3MfRS2mkZufRgOs8q/4NgA8MwzGUsi/h6V7+jApuyldje8htcXZABiRJqhX3tqpfJJC6ONJzDJwRviw33sdTmg1Mc1jKYP17gH22lY3/Lo6jl9L4ce9FAN7X/ICzkguNe7D+VI9Sz9Vp1Lz7SPuqUFNSDNITlNRI0nPN28wWGIaQKXR0Vp1hoCrWbvocS0yzvG6pXGCYepP5TT/7GWZJ2agWRnDhwoX4+/vj6OhIUFAQu3btKlV+9erVBAQE4OjoSIcOHfjtt98sfXl5eUyZMoUOHTrg4uKCr68vo0aNIiEhweoa/v7+KIpidcyaNatSnk9S8WTcWDi5igf/Z3gYgDc0K3Cg5AWVqmKKZgVqRfCHsTs0LbkKoqR6YHcjuHLlSsLDw4mIiGDPnj106tSJsLCwInWB89mxYwcjRoxg3Lhx7N27l8GDBzN48GAOHToEQFZWFnv27OGdd95hz549/Pjjjxw/fpxBgwYVuda7777LpUuXLMdLL71Uqc8qqTgKrh5/YXyQy8KTpqrLjLxRv7eqyY+KCVKO0k+9B4NQ4fnwv+2ii8Q27G4E586dy7PPPsvYsWNp27YtixcvxtnZmSVLlhQrv2DBAvr378/kyZNp06YN7733Hl27duWTTz4BwMPDg6ioKIYOHUrr1q25++67+eSTT4iLiyM+Pt7qWm5ubvj4+FgOFxeZqaOmYCgQD5OFI/MMQwB4WfMjbtgr/b5gmsNSAFYY+xIUKL3AmoBdjaBerycuLo7Q0FBLm0qlIjQ0lJiYmGLPiYmJsZIHCAsLK1EeIDU1FUVR8PT0tGqfNWsWdevWpUuXLsyePRuDoeShVG5uLmlpaVaHpPqwytiHUyZfvJQMXtD8bBcdBqpi6aw6Q6bQseCGUZZUf+xqBK9evYrRaLTUGM7H29ubxMTiM4IkJibaJJ+Tk8OUKVMYMWIE7u7ulvaXX36ZFStWsHHjRp577jnef/993njjjRJ1jYyMxMPDw3L4+fmV9TElVYARtaV4+dPq32nINS5crxqPcP2hSzhg4I0b8YqfGR7iCp5Vcm/J7XNHh8jk5eUxdOhQhBAsWrTIqi88PNzyumPHjmi1Wp577jkiIyOtirTnM23aNKtz0tLSpCGsZmwwdSXWFECQ6hjhmtX0+m9dBnZoiLuThsjHOlbKPY9eSuP57/cwRr2BpqrLXBaefG4cWKL8vGGdKkUPSfmxqydYr1491Go1SUlJVu1JSUn4+PgUe46Pj0+Z5PMN4D///ENUVJSVF1gcQUFBGAwGzp07V2y/TqfD3d3d6pBUNxTez3sCgCHqrQQo8fx68BLLd51n3/mUSrljcqYeN7J4WfMjAPMMQ8ii+BICc4d24tEujStFD0n5sasR1Gq1dOvWjejoaEubyWQiOjqa4ODiJ5WDg4Ot5AGioqKs5PMN4MmTJ9mwYQN169YtfJki7Nu3D5VKRYMGDW4pK6m+7BctWWe8G5UimKpZbmn/YP2xCr+XEIIsvZHnNT/jpWRwyuTLKmOfInJbJvfl4xFdrBJESKoPdh8Oh4eHM3r0aLp3705gYCDz588nMzOTsWPHAjBq1CgaNWpEZGQkAK+88gohISF8+OGHDBw4kBUrVvD333/z2WefAWYD+K9//Ys9e/awbt06jEajZb7Qy8sLrVZLTEwMsbGx9O3bFzc3N2JiYpg0aRJPPvkkdeoUTXUkqX7MebwT8zec4ML1onuEPzAM4wHVbvqo93OP8SDbTR1Iy8kr5irlQwhB3D/XWbjxFEePH2OT7ncAfvN5HmN80YwwTeo606RuzSkMVduwuxEcNmwYV65cYfr06SQmJtK5c2fWr19vWfyIj49HpbrpsPbs2ZNly5bx9ttv8+abb9KqVSt++ukn2rc3bzu6ePEiP/9sXh3s3Lmz1b02btxInz590Ol0rFixghkzZpCbm0uzZs2YNGmS1ZyfpHrzr26N+Ve3xvhP/bVIX7zwZqkxlLGaP5imWc7D+nacSMrgfHIWfhVQpW79oUReWLoHgA80P+Co5BFrCiDRpy/En7/t60uqFkUULvgqKRNpaWl4eHiQmpoq5wftSPy1LHrP3lik3Ys0Nukm4a5k84r+RdaaegEQM+0+Gno43dY9Jyzdw68HL9Faied37TRUimBw7ru0C7yPpbE3Y1HPzSp5gURSudjy+7R7sLREcjs0qetcrLFJxp0tDZ4EYLLDKnToARj71e7bv+mNrcBTNctRKYJ1xiD2iZbIksA1E2kEJXcsDzw9g0vCi8bKVZ5Sm2uRHCtUBtNWTCaBAvRUHaKvej95Qs1swzAAFJkooUYijaDkjmD5s3fTtqE70wYEWNo0OhdOtHkZgJc0a/Ag47bukWsw8sD8Lfx64CLTNObqcd8bQ/lHmMOzChZM+lc3GQpTU5BGUHJHENyiLr+9ci+dC6SnV6kUQoa+zDGTHx5KFi9q1t7WPWLPJHPqcgYPq2LooDpHunDiY8OjN++nKBx9tz9/vx3K7H9VTnC2pOKRRlByR+FQuFyl6uZ2ujHqP2isXCn3tfUGE1ryeMNhJQCLDA+TzM1Jd4G5hnA9Vx2KnCCsMUgjKLmj6NzYk9A23oy9x9/StsnUie3GdugUA+Ga1VxOzynXtfOMJp5S/0lj5SqJog5LjAMqSGuJPZFGUHJHoVIpfDG6OxEPtyvQqhB5wxt8TL2Nse9/SXkiw7YdPMVLmp8A+NDwODlY7zF/qGPD8qotsSPSCEpqBYdEc34y9gRgmmYpr67Yi8FoKvP5GbkG/I4sxlPJ5LipMf8z9rbqX/ZMEB0be1akypIqQhpBSa1hjmEouUJDL/Vhrh9cz30fbi7zuaPm/sBY9R8AnOs6BVOBn84PzwfTs2W9CtdXUjVIIyipNVwQDfjW+AAA0zTLuZBctpCZzFwDI7O+Q6fkscPYlgcGPckb/VsD8NGILnT396o0nSWVjzSCkjue9x/tYHn9iWEwqcKZNqp4HlVt45f9CaWcCbvOJjN0xv/xqGo7AJm9p6OoVLzYpyVn3n+QQZ18K1V3SeUjjaDkjueJoJuF3OeN6cNnwhzb95rDKl5fHkteKXODQ/9vB1M1y1ApgrXGnvTrd3NFWKWSYTB3AnbPIiORVAXnZg1ECIGiKCQPDOfCb3/QWLnKWPV6ElIeoGndokW2zl3NpLfqAPeqD6EXano/t8AOmksqG+kJSmoN+QHMDjonPsx7HIAXNWv5NnpPEVkhBPfN+YtpNxKzLqM/dRrfVXXKSqoMaQQltQ6NSsVPpns4YmqKu5KN74GFReIGL6Zk86hqG21U8aQJZx6e+KGdtJVUNtIISmodrX3cEKh432CuR/KU+k9WRW21ksnKzOA1h1UALNc9Tt36MhD6TkUaQUmto2UDV8LaebPN1IEtxg5oFSNOW9/nWkauRebA//6Lr5LMRVGX0a++b0dtJZWNNIKSWsniJ7sR/VoIH6ufwiQUBqljGPuf/wPgxJlzPJC8FIBrgW/g6OxqT1UllUy1MIILFy7E398fR0dHgoKC2LVrV6nyq1evJiAgAEdHRzp06MBvv/1m1S+EYPr06TRs2BAnJydCQ0M5efKklUxycjIjR47E3d0dT09Pxo0bR0bG7eWbk9QcFEWhRX1XGrYOZM2N1PtvOiwjJTOXM2tm4q5kc8TUlA79n7GzppLKxu5GcOXKlYSHhxMREcGePXvo1KkTYWFhXL58uVj5HTt2MGLECMaNG8fevXsZPHgwgwcP5tChQxaZDz74gI8++ojFixcTGxuLi4sLYWFh5OTczB4ycuRIDh8+TFRUFOvWrWPLli2MHz++0p9XUr2YMagd8Z0nkSscuFt1lHnvT+a+NHPewQ2NX0RRyyiyOx5hZwIDA8WECRMs741Go/D19RWRkZHFyg8dOlQMHDjQqi0oKEg899xzQgghTCaT8PHxEbNnz7b0p6SkCJ1OJ5YvXy6EEOLIkSMCELt377bI/P7770JRFHHx4sUy6Z2amioAkZqaWrYHlVRrFr01UogId8ux+e17xLaTV+ytlqSc2PL7tKsnqNfriYuLIzQ01NKmUqkIDQ0lJiam2HNiYmKs5AHCwsIs8mfPniUxMdFKxsPDg6CgIItMTEwMnp6edO/e3SITGhqKSqUiNja22Pvm5uaSlpZmdUjuHDwfeMPqvd+wD7hHJkWoFdjVCF69ehWj0WipMZyPt7e3pWB6YRITE0uVz//3VjINGjSw6tdoNHh5eZV438jISDw8PCyHn59fGZ9SUhMY3rsjqxqY65HsrvMQzdr3tLNGkqrC7nOCNYVp06aRmppqOc6fl0W27zSGvvgeiSP/ouuEr+2tiqQKseusb7169VCr1SQlJVm1JyUl4ePjU+w5Pj4+pcrn/5uUlETDhg2tZDp37myRKbzwYjAYSE5OLvG+Op0OnU5XbJ/kzsGnVTd7qyCpYuzqCWq1Wrp160Z0dLSlzWQyER0dTXBwcLHnBAcHW8kDREVFWeSbNWuGj4+PlUxaWhqxsbEWmeDgYFJSUoiLi7PI/PXXX5hMJoKCgirs+SQSSQ2gChZqSmXFihVCp9OJr7/+Whw5ckSMHz9eeHp6isTERCGEEE899ZSYOnWqRX779u1Co9GIOXPmiKNHj4qIiAjh4OAgDh48aJGZNWuW8PT0FGvXrhUHDhwQjzzyiGjWrJnIzs62yPTv31906dJFxMbGim3btolWrVqJESNGlFlvuToskVRfbPl92t0ICiHExx9/LJo0aSK0Wq0IDAwUO3futPSFhISI0aNHW8mvWrVK3HXXXUKr1Yp27dqJX3/91arfZDKJd955R3h7ewudTifuv/9+cfz4cSuZa9euiREjRghXV1fh7u4uxo4dK9LT08usszSCEkn1xZbfpyJEOcpuSUhLS8PDw4PU1FTc3d1vfYJEIqkybPl9ytVhiURSq5FGUCKR1GqkEZRIJLUauTu8nORPpcrtcxJJ9SP/d1mWJQ9pBMtJeno6gNw+J5FUY9LT0/Hw8ChVRq4OlxOTyURCQgJubm6WAj41nbS0NPz8/Dh//nytXfGu7Z/BnfL8QgjS09Px9fVFpSp91k96guVEpVLRuHFje6tRKbi7u9foH0BFUNs/gzvh+W/lAeYjF0YkEkmtRhpBiURSq5FGUGJBp9MRERFRq7Pl1PbPoDY+v1wYkUgktRrpCUokklqNNIISiaRWI42gRCKp1UgjKJFIajXSCNZykpOTGTlyJO7u7nh6ejJu3DgyMjJKPeezzz6jT58+uLu7oygKKSkpVaNsBbFw4UL8/f1xdHQkKCiIXbt2lSq/evVqAgICcHR0pEOHDvz2229VpGnlYMvzHz58mCFDhuDv74+iKMyfP7/qFK0ipBGs5YwcOZLDhw8TFRXFunXr2LJlC+PHjy/1nKysLPr378+bb75ZRVpWHCtXriQ8PJyIiAj27NlDp06dCAsLK1J4K58dO3YwYsQIxo0bx969exk8eDCDBw/m0KFDVax5xWDr82dlZdG8eXNmzZpVYhGyGk/lJbiWVHeOHDkiALF7925L2++//y4URREXL1685fkbN24UgLh+/XolalmxBAYGigkTJljeG41G4evrKyIjI4uVHzp0qBg4cKBVW1BQkHjuuecqVc/KwtbnL0jTpk3FvHnzKlE7+yA9wVpMTEwMnp6edO/e3dIWGhqKSqUiNjbWjppVDnq9nri4OEJDQy1tKpWK0NBQYmJiij0nJibGSh4gLCysRPnqTHmevzYgjWAtJjExkQYNGli1aTQavLy8SExMtJNWlcfVq1cxGo14e3tbtXt7e5f4vImJiTbJV2fK8/y1AWkE70CmTp2KoiilHseOHbO3mhJJtUCm0roDee211xgzZkypMs2bN8fHx6fIhLjBYCA5OfmOnASvV68earWapKQkq/akpKQSn9fHx8cm+epMeZ6/NiA9wTuQ+vXrExAQUOqh1WoJDg4mJSWFuLg4y7l//fUXJpOJoKAgOz5B5aDVaunWrRvR0dGWNpPJRHR0NMHBwcWeExwcbCUPEBUVVaJ8daY8z18rsPfKjMS+9O/fX3Tp0kXExsaKbdu2iVatWokRI0ZY+i9cuCBat24tYmNjLW2XLl0Se/fuFZ9//rkAxJYtW8TevXvFtWvX7PEINrFixQqh0+nE119/LY4cOSLGjx8vPD09RWJiohBCiKeeekpMnTrVIr99+3ah0WjEnDlzxNGjR0VERIRwcHAQBw8etNcj3Ba2Pn9ubq7Yu3ev2Lt3r2jYsKF4/fXXxd69e8XJkyft9QgVjjSCtZxr166JESNGCFdXV+Hu7i7Gjh0r0tPTLf1nz54VgNi4caOlLSIiQgBFjq+++qrqH6AcfPzxx6JJkyZCq9WKwMBAsXPnTktfSEiIGD16tJX8qlWrxF133SW0Wq1o166d+PXXX6tY44rFlufP//4LHyEhIVWveCUhU2lJJJJajZwTlEgktRppBCUSSa1GGkGJRFKrkUZQIpHUaqQRlEgktRppBCUSSa1GGkGJRFKrkUZQIpHUaqQRlNwxjBkzptiMOf3797e3apJqjMwiI7mj6N+/P1999ZVVm06nK1Y2Ly8PBwcHqza9Xo9Wq7X5vuU9T2J/pCcouaPQ6XT4+PhYHXXq1AFAURQWLVrEoEGDcHFx4T//+Q8zZsygc+fOfPHFFzRr1gxHR0cA4uPjeeSRR3B1dcXd3Z2hQ4dapaAq6TxJzUMaQUmtYsaMGTz66KMcPHiQp59+GoBTp07xv//9jx9//JF9+/ZhMpl45JFHSE5OZvPmzURFRXHmzBmGDRtmda3C50lqJnI4LLmjWLduHa6urlZtb775pqUy3hNPPMHYsWOt+vV6Pd9++y3169cHzPkCDx48yNmzZ/Hz8wPg22+/pV27duzevZsePXoUe56kZiKNoOSOom/fvixatMiqzcvLy/K6YFGpfJo2bWplyI4ePYqfn5/FAAK0bdsWT09Pjh49ajGChc+T1EykEZTcUbi4uNCyZctS+8vSVtZ7SWo+ck5QIilEmzZtOH/+POfPn7e0HTlyhJSUFNq2bWtHzSSVgfQEJXcUubm5RcpHajQa6tWrV+ZrhIaG0qFDB0aOHMn8+fMxGAy8+OKLhISEFDucltRspCcouaNYv349DRs2tDp69epl0zUURWHt2rXUqVOH3r17ExoaSvPmzVm5cmUlaS2xJzK9vkQiqdVIT1AikdRqpBGUSCS1GmkEJRJJrUYaQYlEUquRRlAikdRqpBGUSCS1GmkEJRJJrUYaQYlEUquRRlAikdRqpBGUSCS1GmkEJRJJrUYaQYlEUqv5f/IWEH9Sbf++AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.026767, + "end_time": "2024-02-29T19:54:52.637961", + "exception": false, + "start_time": "2024-02-29T19:54:52.611194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4069.545582, + "end_time": "2024-02-29T19:54:55.386766", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tab_ddpm_concat/4/mlu-eval.ipynb", + "output_path": "eval/treatment/tab_ddpm_concat/4/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "path": "eval/treatment/tab_ddpm_concat/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T18:47:05.841184", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/model.pt b/treatment/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f35689bb21ed40dc09f6ba93eaf2d86040a5e39 --- /dev/null +++ b/treatment/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d39a61bbc2abce434d4ef30a4412c24c5565e755254e859508c745a68183eb2 +size 74520513 diff --git a/treatment/tab_ddpm_concat/params.json b/treatment/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..2edb074bfe21242b0e7dd650bd82cc146e39d8c9 --- /dev/null +++ b/treatment/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "tab_ddpm_concat", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tvae/eval.csv b/treatment/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..6a9b23397df9ebdd55b3d808bfd2de6960b7adec --- /dev/null +++ b/treatment/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.06831869456822515,0.07396524139990789,0.0023402947514378617,4.434747219085693,0.13169732689857483,1.3246768712997437,0.3612377345561981,1.4730316252098419e-05,2.4250926971435547,0.035608064383268356,0.06660553812980652,0.04837659373879433,0.06654548645019531,0.0213878583163023,6.859839916229248 diff --git a/treatment/tvae/history.csv b/treatment/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..75220d1a1f6fefc44c81527deef914c0bb8d90e3 --- /dev/null +++ b/treatment/tvae/history.csv @@ -0,0 +1,30 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.25079594189301135,172.3493912117849,0.0756463885481935,0.0,0.0,0.0,0.0,0.0,0.25079594189301135,320,80,100.33574557304382,1.2541968196630477,0.31354920491576194,0.0283103398049775,0.15434771333821118,0.8378194279823219,0.03735067891972328,0.0,0.0,0.0,0.0,0.0,0.15434771333821118,80,20,19.603960514068604,0.9801980257034302,0.24504950642585754,0.15204731123521925 +1,0.03561886841698651,0.3229346345896033,0.003486672353001108,0.0,0.0,0.0,0.0,0.0,0.03561886841698651,320,80,99.75122737884521,1.2468903422355653,0.3117225855588913,0.2135042036534287,0.007464131528354301,14.743032303131127,0.0003255664299921697,0.0,0.0,0.0,0.0,0.0,0.007464131528354301,80,20,19.27774691581726,0.963887345790863,0.24097183644771575,0.033036651482689194 +2,0.008308970414873329,2.2638450761613056,0.0002922931804504517,0.0,0.0,0.0,0.0,0.0,0.008308970414873329,320,80,99.78807187080383,1.2473508983850479,0.31183772459626197,0.19011376096532331,0.008792089688631677,7.008411024302973,0.00045460039846188566,0.0,0.0,0.0,0.0,0.0,0.008792089688631677,80,20,19.60298991203308,0.9801494956016541,0.24503737390041352,0.05645044087141286 +3,0.008624525008644923,0.4896122309531961,0.00020031151738803265,0.0,0.0,0.0,0.0,0.0,0.008624525008644923,320,80,99.84555792808533,1.2480694741010665,0.3120173685252666,0.20240991505852435,0.007710156342363916,3.4492962674491308,0.00016955623478978054,0.0,0.0,0.0,0.0,0.0,0.007710156342363916,80,20,19.69949173927307,0.9849745869636536,0.2462436467409134,0.057611686212476344 +4,0.005540997025855176,0.2870428302302061,0.00010962202765298911,0.0,0.0,0.0,0.0,0.0,0.005540997025855176,320,80,99.68231582641602,1.2460289478302002,0.31150723695755006,0.20185390445403756,0.007617261353880167,0.8548410554893053,0.00021040458378775994,0.0,0.0,0.0,0.0,0.0,0.007617261353880167,80,20,19.60577392578125,0.9802886962890625,0.24507217407226561,0.06373736902605742 +5,0.004228771507609963,0.8442374373333109,7.225186645620774e-05,0.0,0.0,0.0,0.0,0.0,0.004228771507609963,320,80,99.56437826156616,1.244554728269577,0.31113868206739426,0.19006720920442605,0.007288631894334685,1.333325219784001,0.00021377515046261815,0.0,0.0,0.0,0.0,0.0,0.007288631894334685,80,20,19.61089253425598,0.9805446267127991,0.24513615667819977,0.05554712610319257 +6,0.004288671186168358,0.7424422502960226,7.162387234466161e-05,0.0,0.0,0.0,0.0,0.0,0.004288671186168358,320,80,99.62058591842651,1.2452573239803315,0.3113143309950829,0.19745058890111977,0.006442523133591749,0.5144731080438725,0.00011468025654934877,0.0,0.0,0.0,0.0,0.0,0.006442523133591749,80,20,19.81056571006775,0.9905282855033875,0.24763207137584686,0.06324401344172656 +7,0.003485065951008437,0.5401337196294321,3.538603915114859e-05,0.0,0.0,0.0,0.0,0.0,0.003485065951008437,320,80,99.80776119232178,1.2475970149040223,0.3118992537260056,0.18830189779400824,0.009077594889095052,0.3387085039643353,0.0002953080845562894,0.0,0.0,0.0,0.0,0.0,0.009077594889095052,80,20,19.67119312286377,0.9835596561431885,0.24588991403579713,0.07019970323890448 +8,0.0029438145053859444,0.8214074499624902,2.2453812508680688e-05,0.0,0.0,0.0,0.0,0.0,0.0029438145053859444,320,80,99.6238522529602,1.2452981531620027,0.31132453829050066,0.19482076268177478,0.007072782384057064,0.48727775096755294,0.00015469519749622407,0.0,0.0,0.0,0.0,0.0,0.007072782384057064,80,20,19.500911951065063,0.9750455975532532,0.2437613993883133,0.06650436315685511 +9,0.002503083477677137,0.7746123696918176,1.2513642564668465e-05,0.0,0.0,0.0,0.0,0.0,0.002503083477677137,320,80,99.6650288105011,1.2458128601312637,0.3114532150328159,0.18602521782158873,0.007042999230907299,0.4210409534451173,0.00016206004065456026,0.0,0.0,0.0,0.0,0.0,0.007042999230907299,80,20,19.607900619506836,0.9803950309753418,0.24509875774383544,0.06358127733692527 +10,0.0019445733821157774,0.45887769680392837,2.8811827562555402e-05,0.0,0.0,0.0,0.0,0.0,0.0019445733821157774,320,80,99.54072690010071,1.2442590862512588,0.3110647715628147,0.17861799619859084,0.007491035046405159,0.4072774214367428,0.0001684058567391844,0.0,0.0,0.0,0.0,0.0,0.007491035046405159,80,20,19.5883047580719,0.979415237903595,0.24485380947589874,0.0683793492615223 +11,0.0016902872650462087,0.14312844078152837,1.2574293510679222e-05,0.0,0.0,0.0,0.0,0.0,0.0016902872650462087,320,80,99.5855655670166,1.2448195695877076,0.3112048923969269,0.18321805561427026,0.00778989009122597,0.30623174232314343,0.0002285926662562332,0.0,0.0,0.0,0.0,0.0,0.00778989009122597,80,20,19.662875652313232,0.9831437826156616,0.2457859456539154,0.0688946488313377 +12,0.0016234107402169685,0.26942976858223344,1.856327916652254e-05,0.0,0.0,0.0,0.0,0.0,0.0016234107402169685,320,80,100.64648413658142,1.2580810517072678,0.31452026292681695,0.18938408511457966,0.007789343649346847,0.39892919776157215,0.00019050486260980827,0.0,0.0,0.0,0.0,0.0,0.007789343649346847,80,20,19.436559677124023,0.9718279838562012,0.2429569959640503,0.06721605993807316 +13,0.0014992240631727326,0.32569619336183353,1.7869830239917398e-06,0.0,0.0,0.0,0.0,0.0,0.0014992240631727326,320,80,99.36593246459961,1.242074155807495,0.31051853895187376,0.19987344325636514,0.007941817007667851,0.3285603669361308,0.00020670617296367766,0.0,0.0,0.0,0.0,0.0,0.007941817007667851,80,20,19.3479266166687,0.967396330833435,0.24184908270835875,0.06898661321029068 +14,0.0015280315952168166,0.1436633735797855,2.658071664495944e-05,0.0,0.0,0.0,0.0,0.0,0.0015280315952168166,320,80,99.75400042533875,1.2469250053167342,0.31173125132918356,0.189214165561134,0.007610848201147746,0.30814517063022323,0.00022678597003369382,0.0,0.0,0.0,0.0,0.0,0.007610848201147746,80,20,19.45919370651245,0.9729596853256226,0.24323992133140565,0.06733945356681943 +15,0.0012301572283377026,0.10807322694710982,5.3408418359659e-06,0.0,0.0,0.0,0.0,0.0,0.0012301572283377026,320,80,99.28717947006226,1.2410897433757782,0.31027243584394454,0.19792793567758055,0.006984197727433639,0.247640823103211,0.0001665852697917726,0.0,0.0,0.0,0.0,0.0,0.006984197727433639,80,20,19.52495002746582,0.976247501373291,0.24406187534332274,0.06778875123709441 +16,0.0010839548700573686,0.07997205620506662,9.080777822098943e-06,0.0,0.0,0.0,0.0,0.0,0.0010839548700573686,320,80,99.66356492042542,1.2457945615053176,0.3114486403763294,0.19800062356516718,0.008054383638955187,0.28627982361469717,0.00025097979236513577,0.0,0.0,0.0,0.0,0.0,0.008054383638955187,80,20,19.489768266677856,0.9744884133338928,0.2436221033334732,0.06728147398680448 +17,0.0007286213950465026,0.10312503655737952,6.776996160624913e-07,0.0,0.0,0.0,0.0,0.0,0.0007286213950465026,320,80,99.96071243286133,1.2495089054107666,0.31237722635269166,0.1800330831320025,0.007334689510025782,0.2440898734063012,0.00019387806316268908,0.0,0.0,0.0,0.0,0.0,0.007334689510025782,80,20,19.55093002319336,0.977546501159668,0.244386625289917,0.06980508081614971 +18,0.0007013096967625643,0.0806437349208462,3.612507871512266e-07,0.0,0.0,0.0,0.0,0.0,0.0007013096967625643,320,80,99.62839913368225,1.245354989171028,0.311338747292757,0.193802969326498,0.008005985312775011,0.25233456931382536,0.00024166704105831326,0.0,0.0,0.0,0.0,0.0,0.008005985312775011,80,20,19.874733448028564,0.9937366724014283,0.24843416810035707,0.06946740932762623 +19,0.000650154861421015,0.13836135070985306,6.324771196684101e-07,0.0,0.0,0.0,0.0,0.0,0.000650154861421015,320,80,100.67735123634338,1.2584668904542924,0.3146167226135731,0.1874849540356081,0.007276729341538157,0.19176392750296145,0.0001885933500119812,0.0,0.0,0.0,0.0,0.0,0.007276729341538157,80,20,19.5623676776886,0.97811838388443,0.2445295959711075,0.06984148817136884 +20,0.0005832819898387243,0.04374472918600412,9.791867645731814e-07,0.0,0.0,0.0,0.0,0.0,0.0005832819898387243,320,80,99.7720296382904,1.24715037047863,0.3117875926196575,0.19292465539183468,0.007389668950054329,0.19728227270163642,0.00020410724985005512,0.0,0.0,0.0,0.0,0.0,0.007389668950054329,80,20,19.41769814491272,0.970884907245636,0.242721226811409,0.06944395890459418 +21,0.0005352863459663126,0.01976681658542425,6.443602266463554e-07,0.0,0.0,0.0,0.0,0.0,0.0005352863459663126,320,80,100.6177191734314,1.2577214896678925,0.31443037241697314,0.19984434806974605,0.007991572895844002,0.22150783375837904,0.00023175869880640577,0.0,0.0,0.0,0.0,0.0,0.007991572895844002,80,20,20.159916162490845,1.0079958081245421,0.25199895203113554,0.07001060470938683 +22,0.0005078368830425007,0.02868203009257253,8.534584292643494e-07,0.0,0.0,0.0,0.0,0.0,0.0005078368830425007,320,80,99.94173312187195,1.2492716640233994,0.31231791600584985,0.19320008249487727,0.007862234280037229,0.2152756543153373,0.0002432201600976569,0.0,0.0,0.0,0.0,0.0,0.007862234280037229,80,20,19.478534698486328,0.9739267349243164,0.2434816837310791,0.06986867655068636 +23,0.000520391069062498,0.03020876141404938,4.458749571339605e-07,0.0,0.0,0.0,0.0,0.0,0.000520391069062498,320,80,100.07614707946777,1.2509518384933471,0.3127379596233368,0.1921757934615016,0.007205282323411666,0.20713103588941523,0.0001827256362942009,0.0,0.0,0.0,0.0,0.0,0.007205282323411666,80,20,19.527496099472046,0.9763748049736023,0.24409370124340057,0.06974663501605391 +24,0.0005546999067291836,0.027485956521196276,6.895572416364425e-07,0.0,0.0,0.0,0.0,0.0,0.0005546999067291836,320,80,99.80755043029785,1.247594380378723,0.31189859509468076,0.1784662834368646,0.007224415110249538,0.17986427203068162,0.00017955067020931638,0.0,0.0,0.0,0.0,0.0,0.007224415110249538,80,20,19.42549467086792,0.971274733543396,0.242818683385849,0.0726472232490778 +25,0.0005678469070971914,0.0783861569140182,2.3522705448089696e-06,0.0,0.0,0.0,0.0,0.0,0.0005678469070971914,320,80,99.91300010681152,1.248912501335144,0.312228125333786,0.18876876458525657,0.008989003312308341,0.17780489571450744,0.00034604211847950596,0.0,0.0,0.0,0.0,0.0,0.008989003312308341,80,20,19.550805807113647,0.9775402903556824,0.2443850725889206,0.0718118923716247 +26,0.0004598389233777311,0.03701534456806623,6.740662266656723e-08,0.0,0.0,0.0,0.0,0.0,0.0004598389233777311,320,80,99.83816456794739,1.2479770570993423,0.31199426427483556,0.19492796149570496,0.008001866593258456,0.1477779638089487,0.0002354497060985228,0.0,0.0,0.0,0.0,0.0,0.008001866593258456,80,20,19.476824283599854,0.9738412141799927,0.24346030354499817,0.07350850850343704 +27,0.0004317075494100209,0.016361890759745458,3.335215542313308e-07,0.0,0.0,0.0,0.0,0.0,0.0004317075494100209,320,80,99.95542669296265,1.2494428336620331,0.3123607084155083,0.18771503504831344,0.008291181290405802,0.19831402401805462,0.0002830501877054914,0.0,0.0,0.0,0.0,0.0,0.008291181290405802,80,20,19.60791325569153,0.9803956627845765,0.24509891569614412,0.06914721243083477 +28,0.0004966751410051984,0.013397608251197823,1.0833947258724608e-07,0.0,0.0,0.0,0.0,0.0,0.0004966751410051984,320,80,100.00627398490906,1.2500784248113632,0.3125196062028408,0.19725441149203107,0.008539365028264,0.21735998928961636,0.0002757914544125217,0.0,0.0,0.0,0.0,0.0,0.008539365028264,80,20,19.56255578994751,0.9781277894973754,0.24453194737434386,0.06783094126731157 diff --git a/treatment/tvae/mlu-eval.ipynb b/treatment/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..979fcbdd61e8c9579db332a18a4ae54b7664f50c --- /dev/null +++ b/treatment/tvae/mlu-eval.ipynb @@ -0,0 +1,2773 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.074576Z", + "iopub.status.busy": "2024-02-29T04:29:11.073762Z", + "iopub.status.idle": "2024-02-29T04:29:11.113694Z", + "shell.execute_reply": "2024-02-29T04:29:11.112810Z" + }, + "papermill": { + "duration": 0.055415, + "end_time": "2024-02-29T04:29:11.115700", + "exception": false, + "start_time": "2024-02-29T04:29:11.060285", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.141741Z", + "iopub.status.busy": "2024-02-29T04:29:11.140838Z", + "iopub.status.idle": "2024-02-29T04:29:11.148404Z", + "shell.execute_reply": "2024-02-29T04:29:11.147580Z" + }, + "papermill": { + "duration": 0.02269, + "end_time": "2024-02-29T04:29:11.150466", + "exception": false, + "start_time": "2024-02-29T04:29:11.127776", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.174255Z", + "iopub.status.busy": "2024-02-29T04:29:11.173792Z", + "iopub.status.idle": "2024-02-29T04:29:11.177803Z", + "shell.execute_reply": "2024-02-29T04:29:11.176979Z" + }, + "papermill": { + "duration": 0.018365, + "end_time": "2024-02-29T04:29:11.179910", + "exception": false, + "start_time": "2024-02-29T04:29:11.161545", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.203700Z", + "iopub.status.busy": "2024-02-29T04:29:11.203202Z", + "iopub.status.idle": "2024-02-29T04:29:11.207400Z", + "shell.execute_reply": "2024-02-29T04:29:11.206583Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018933, + "end_time": "2024-02-29T04:29:11.209878", + "exception": false, + "start_time": "2024-02-29T04:29:11.190945", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.234360Z", + "iopub.status.busy": "2024-02-29T04:29:11.233845Z", + "iopub.status.idle": "2024-02-29T04:29:11.239162Z", + "shell.execute_reply": "2024-02-29T04:29:11.238293Z" + }, + "papermill": { + "duration": 0.019487, + "end_time": "2024-02-29T04:29:11.241190", + "exception": false, + "start_time": "2024-02-29T04:29:11.221703", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e997d4e6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.266264Z", + "iopub.status.busy": "2024-02-29T04:29:11.266000Z", + "iopub.status.idle": "2024-02-29T04:29:11.270684Z", + "shell.execute_reply": "2024-02-29T04:29:11.269865Z" + }, + "papermill": { + "duration": 0.019149, + "end_time": "2024-02-29T04:29:11.272484", + "exception": false, + "start_time": "2024-02-29T04:29:11.253335", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tvae/2\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011209, + "end_time": "2024-02-29T04:29:11.294824", + "exception": false, + "start_time": "2024-02-29T04:29:11.283615", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.318911Z", + "iopub.status.busy": "2024-02-29T04:29:11.318297Z", + "iopub.status.idle": "2024-02-29T04:29:11.327653Z", + "shell.execute_reply": "2024-02-29T04:29:11.326895Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023753, + "end_time": "2024-02-29T04:29:11.329797", + "exception": false, + "start_time": "2024-02-29T04:29:11.306044", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tvae/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:11.353602Z", + "iopub.status.busy": "2024-02-29T04:29:11.353337Z", + "iopub.status.idle": "2024-02-29T04:29:13.595630Z", + "shell.execute_reply": "2024-02-29T04:29:13.594714Z" + }, + "papermill": { + "duration": 2.256654, + "end_time": "2024-02-29T04:29:13.597744", + "exception": false, + "start_time": "2024-02-29T04:29:11.341090", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:13.624599Z", + "iopub.status.busy": "2024-02-29T04:29:13.623911Z", + "iopub.status.idle": "2024-02-29T04:29:13.639836Z", + "shell.execute_reply": "2024-02-29T04:29:13.639010Z" + }, + "papermill": { + "duration": 0.031202, + "end_time": "2024-02-29T04:29:13.641867", + "exception": false, + "start_time": "2024-02-29T04:29:13.610665", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:13.666451Z", + "iopub.status.busy": "2024-02-29T04:29:13.666195Z", + "iopub.status.idle": "2024-02-29T04:29:13.673384Z", + "shell.execute_reply": "2024-02-29T04:29:13.672673Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021525, + "end_time": "2024-02-29T04:29:13.675216", + "exception": false, + "start_time": "2024-02-29T04:29:13.653691", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:13.699417Z", + "iopub.status.busy": "2024-02-29T04:29:13.699140Z", + "iopub.status.idle": "2024-02-29T04:29:13.800471Z", + "shell.execute_reply": "2024-02-29T04:29:13.799702Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.116049, + "end_time": "2024-02-29T04:29:13.802564", + "exception": false, + "start_time": "2024-02-29T04:29:13.686515", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:13.829096Z", + "iopub.status.busy": "2024-02-29T04:29:13.828793Z", + "iopub.status.idle": "2024-02-29T04:29:18.514659Z", + "shell.execute_reply": "2024-02-29T04:29:18.513888Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.702039, + "end_time": "2024-02-29T04:29:18.517136", + "exception": false, + "start_time": "2024-02-29T04:29:13.815097", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 04:29:16.137890: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 04:29:16.137945: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 04:29:16.139721: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:18.543026Z", + "iopub.status.busy": "2024-02-29T04:29:18.542205Z", + "iopub.status.idle": "2024-02-29T04:29:18.548277Z", + "shell.execute_reply": "2024-02-29T04:29:18.547581Z" + }, + "papermill": { + "duration": 0.021126, + "end_time": "2024-02-29T04:29:18.550299", + "exception": false, + "start_time": "2024-02-29T04:29:18.529173", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:18.576519Z", + "iopub.status.busy": "2024-02-29T04:29:18.576245Z", + "iopub.status.idle": "2024-02-29T04:29:40.762246Z", + "shell.execute_reply": "2024-02-29T04:29:40.760891Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.202272, + "end_time": "2024-02-29T04:29:40.764735", + "exception": false, + "start_time": "2024-02-29T04:29:18.562463", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:41.281815Z", + "iopub.status.busy": "2024-02-29T04:29:41.281494Z", + "iopub.status.idle": "2024-02-29T04:29:41.349146Z", + "shell.execute_reply": "2024-02-29T04:29:41.348155Z" + }, + "papermill": { + "duration": 0.083785, + "end_time": "2024-02-29T04:29:41.351100", + "exception": false, + "start_time": "2024-02-29T04:29:41.267315", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T04:29:41.380722Z", + "iopub.status.busy": "2024-02-29T04:29:41.380408Z", + "iopub.status.idle": "2024-02-29T04:29:41.922049Z", + "shell.execute_reply": "2024-02-29T04:29:41.921052Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.558849, + "end_time": "2024-02-29T04:29:41.924289", + "exception": false, + "start_time": "2024-02-29T04:29:41.365440", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:41.953848Z", + "iopub.status.busy": "2024-02-29T04:29:41.953527Z", + "iopub.status.idle": "2024-02-29T04:29:41.957492Z", + "shell.execute_reply": "2024-02-29T04:29:41.956677Z" + }, + "papermill": { + "duration": 0.021605, + "end_time": "2024-02-29T04:29:41.959458", + "exception": false, + "start_time": "2024-02-29T04:29:41.937853", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:41.989740Z", + "iopub.status.busy": "2024-02-29T04:29:41.989123Z", + "iopub.status.idle": "2024-02-29T04:29:41.996421Z", + "shell.execute_reply": "2024-02-29T04:29:41.995567Z" + }, + "papermill": { + "duration": 0.024554, + "end_time": "2024-02-29T04:29:41.998337", + "exception": false, + "start_time": "2024-02-29T04:29:41.973783", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18701313" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:42.026419Z", + "iopub.status.busy": "2024-02-29T04:29:42.026154Z", + "iopub.status.idle": "2024-02-29T04:29:42.124245Z", + "shell.execute_reply": "2024-02-29T04:29:42.123367Z" + }, + "papermill": { + "duration": 0.114636, + "end_time": "2024-02-29T04:29:42.126241", + "exception": false, + "start_time": "2024-02-29T04:29:42.011605", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 95] --\n", + "├─Adapter: 1-1 [2, 2648, 95] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 98,304\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 95] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,701,313\n", + "Trainable params: 18,701,313\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 74.05\n", + "========================================================================================================================\n", + "Input size (MB): 2.51\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.81\n", + "Estimated Total Size (MB): 1156.80\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T04:29:42.157988Z", + "iopub.status.busy": "2024-02-29T04:29:42.157605Z", + "iopub.status.idle": "2024-02-29T05:32:09.727771Z", + "shell.execute_reply": "2024-02-29T05:32:09.726851Z" + }, + "papermill": { + "duration": 3747.588598, + "end_time": "2024-02-29T05:32:09.729982", + "exception": false, + "start_time": "2024-02-29T04:29:42.141384", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.25079594189301135, 'avg_role_model_std_loss': 172.3493912117849, 'avg_role_model_mean_pred_loss': 0.0756463885481935, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.25079594189301135, 'n_size': 320, 'n_batch': 80, 'duration': 100.33574557304382, 'duration_batch': 1.2541968196630477, 'duration_size': 0.31354920491576194, 'avg_pred_std': 0.0283103398049775}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.15434771333821118, 'avg_role_model_std_loss': 0.8378194279823219, 'avg_role_model_mean_pred_loss': 0.03735067891972328, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.15434771333821118, 'n_size': 80, 'n_batch': 20, 'duration': 19.603960514068604, 'duration_batch': 0.9801980257034302, 'duration_size': 0.24504950642585754, 'avg_pred_std': 0.15204731123521925}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03561886841698651, 'avg_role_model_std_loss': 0.3229346345896033, 'avg_role_model_mean_pred_loss': 0.003486672353001108, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03561886841698651, 'n_size': 320, 'n_batch': 80, 'duration': 99.75122737884521, 'duration_batch': 1.2468903422355653, 'duration_size': 0.3117225855588913, 'avg_pred_std': 0.2135042036534287}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007464131528354301, 'avg_role_model_std_loss': 14.743032303131127, 'avg_role_model_mean_pred_loss': 0.0003255664299921697, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007464131528354301, 'n_size': 80, 'n_batch': 20, 'duration': 19.27774691581726, 'duration_batch': 0.963887345790863, 'duration_size': 0.24097183644771575, 'avg_pred_std': 0.033036651482689194}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008308970414873329, 'avg_role_model_std_loss': 2.2638450761613056, 'avg_role_model_mean_pred_loss': 0.0002922931804504517, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008308970414873329, 'n_size': 320, 'n_batch': 80, 'duration': 99.78807187080383, 'duration_batch': 1.2473508983850479, 'duration_size': 0.31183772459626197, 'avg_pred_std': 0.19011376096532331}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008792089688631677, 'avg_role_model_std_loss': 7.008411024302973, 'avg_role_model_mean_pred_loss': 0.00045460039846188566, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008792089688631677, 'n_size': 80, 'n_batch': 20, 'duration': 19.60298991203308, 'duration_batch': 0.9801494956016541, 'duration_size': 0.24503737390041352, 'avg_pred_std': 0.05645044087141286}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008624525008644923, 'avg_role_model_std_loss': 0.4896122309531961, 'avg_role_model_mean_pred_loss': 0.00020031151738803265, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008624525008644923, 'n_size': 320, 'n_batch': 80, 'duration': 99.84555792808533, 'duration_batch': 1.2480694741010665, 'duration_size': 0.3120173685252666, 'avg_pred_std': 0.20240991505852435}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007710156342363916, 'avg_role_model_std_loss': 3.4492962674491308, 'avg_role_model_mean_pred_loss': 0.00016955623478978054, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007710156342363916, 'n_size': 80, 'n_batch': 20, 'duration': 19.69949173927307, 'duration_batch': 0.9849745869636536, 'duration_size': 0.2462436467409134, 'avg_pred_std': 0.057611686212476344}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005540997025855176, 'avg_role_model_std_loss': 0.2870428302302061, 'avg_role_model_mean_pred_loss': 0.00010962202765298911, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005540997025855176, 'n_size': 320, 'n_batch': 80, 'duration': 99.68231582641602, 'duration_batch': 1.2460289478302002, 'duration_size': 0.31150723695755006, 'avg_pred_std': 0.20185390445403756}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007617261353880167, 'avg_role_model_std_loss': 0.8548410554893053, 'avg_role_model_mean_pred_loss': 0.00021040458378775994, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007617261353880167, 'n_size': 80, 'n_batch': 20, 'duration': 19.60577392578125, 'duration_batch': 0.9802886962890625, 'duration_size': 0.24507217407226561, 'avg_pred_std': 0.06373736902605742}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004228771507609963, 'avg_role_model_std_loss': 0.8442374373333109, 'avg_role_model_mean_pred_loss': 7.225186645620774e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004228771507609963, 'n_size': 320, 'n_batch': 80, 'duration': 99.56437826156616, 'duration_batch': 1.244554728269577, 'duration_size': 0.31113868206739426, 'avg_pred_std': 0.19006720920442605}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007288631894334685, 'avg_role_model_std_loss': 1.333325219784001, 'avg_role_model_mean_pred_loss': 0.00021377515046261815, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007288631894334685, 'n_size': 80, 'n_batch': 20, 'duration': 19.61089253425598, 'duration_batch': 0.9805446267127991, 'duration_size': 0.24513615667819977, 'avg_pred_std': 0.05554712610319257}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004288671186168358, 'avg_role_model_std_loss': 0.7424422502960226, 'avg_role_model_mean_pred_loss': 7.162387234466161e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004288671186168358, 'n_size': 320, 'n_batch': 80, 'duration': 99.62058591842651, 'duration_batch': 1.2452573239803315, 'duration_size': 0.3113143309950829, 'avg_pred_std': 0.19745058890111977}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006442523133591749, 'avg_role_model_std_loss': 0.5144731080438725, 'avg_role_model_mean_pred_loss': 0.00011468025654934877, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006442523133591749, 'n_size': 80, 'n_batch': 20, 'duration': 19.81056571006775, 'duration_batch': 0.9905282855033875, 'duration_size': 0.24763207137584686, 'avg_pred_std': 0.06324401344172656}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003485065951008437, 'avg_role_model_std_loss': 0.5401337196294321, 'avg_role_model_mean_pred_loss': 3.538603915114859e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003485065951008437, 'n_size': 320, 'n_batch': 80, 'duration': 99.80776119232178, 'duration_batch': 1.2475970149040223, 'duration_size': 0.3118992537260056, 'avg_pred_std': 0.18830189779400824}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009077594889095052, 'avg_role_model_std_loss': 0.3387085039643353, 'avg_role_model_mean_pred_loss': 0.0002953080845562894, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009077594889095052, 'n_size': 80, 'n_batch': 20, 'duration': 19.67119312286377, 'duration_batch': 0.9835596561431885, 'duration_size': 0.24588991403579713, 'avg_pred_std': 0.07019970323890448}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029438145053859444, 'avg_role_model_std_loss': 0.8214074499624902, 'avg_role_model_mean_pred_loss': 2.2453812508680688e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029438145053859444, 'n_size': 320, 'n_batch': 80, 'duration': 99.6238522529602, 'duration_batch': 1.2452981531620027, 'duration_size': 0.31132453829050066, 'avg_pred_std': 0.19482076268177478}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007072782384057064, 'avg_role_model_std_loss': 0.48727775096755294, 'avg_role_model_mean_pred_loss': 0.00015469519749622407, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007072782384057064, 'n_size': 80, 'n_batch': 20, 'duration': 19.500911951065063, 'duration_batch': 0.9750455975532532, 'duration_size': 0.2437613993883133, 'avg_pred_std': 0.06650436315685511}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002503083477677137, 'avg_role_model_std_loss': 0.7746123696918176, 'avg_role_model_mean_pred_loss': 1.2513642564668465e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002503083477677137, 'n_size': 320, 'n_batch': 80, 'duration': 99.6650288105011, 'duration_batch': 1.2458128601312637, 'duration_size': 0.3114532150328159, 'avg_pred_std': 0.18602521782158873}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007042999230907299, 'avg_role_model_std_loss': 0.4210409534451173, 'avg_role_model_mean_pred_loss': 0.00016206004065456026, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007042999230907299, 'n_size': 80, 'n_batch': 20, 'duration': 19.607900619506836, 'duration_batch': 0.9803950309753418, 'duration_size': 0.24509875774383544, 'avg_pred_std': 0.06358127733692527}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019445733821157774, 'avg_role_model_std_loss': 0.45887769680392837, 'avg_role_model_mean_pred_loss': 2.8811827562555402e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019445733821157774, 'n_size': 320, 'n_batch': 80, 'duration': 99.54072690010071, 'duration_batch': 1.2442590862512588, 'duration_size': 0.3110647715628147, 'avg_pred_std': 0.17861799619859084}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007491035046405159, 'avg_role_model_std_loss': 0.4072774214367428, 'avg_role_model_mean_pred_loss': 0.0001684058567391844, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007491035046405159, 'n_size': 80, 'n_batch': 20, 'duration': 19.5883047580719, 'duration_batch': 0.979415237903595, 'duration_size': 0.24485380947589874, 'avg_pred_std': 0.0683793492615223}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016902872650462087, 'avg_role_model_std_loss': 0.14312844078152837, 'avg_role_model_mean_pred_loss': 1.2574293510679222e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016902872650462087, 'n_size': 320, 'n_batch': 80, 'duration': 99.5855655670166, 'duration_batch': 1.2448195695877076, 'duration_size': 0.3112048923969269, 'avg_pred_std': 0.18321805561427026}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00778989009122597, 'avg_role_model_std_loss': 0.30623174232314343, 'avg_role_model_mean_pred_loss': 0.0002285926662562332, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00778989009122597, 'n_size': 80, 'n_batch': 20, 'duration': 19.662875652313232, 'duration_batch': 0.9831437826156616, 'duration_size': 0.2457859456539154, 'avg_pred_std': 0.0688946488313377}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016234107402169685, 'avg_role_model_std_loss': 0.26942976858223344, 'avg_role_model_mean_pred_loss': 1.856327916652254e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016234107402169685, 'n_size': 320, 'n_batch': 80, 'duration': 100.64648413658142, 'duration_batch': 1.2580810517072678, 'duration_size': 0.31452026292681695, 'avg_pred_std': 0.18938408511457966}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007789343649346847, 'avg_role_model_std_loss': 0.39892919776157215, 'avg_role_model_mean_pred_loss': 0.00019050486260980827, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007789343649346847, 'n_size': 80, 'n_batch': 20, 'duration': 19.436559677124023, 'duration_batch': 0.9718279838562012, 'duration_size': 0.2429569959640503, 'avg_pred_std': 0.06721605993807316}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014992240631727326, 'avg_role_model_std_loss': 0.32569619336183353, 'avg_role_model_mean_pred_loss': 1.7869830239917398e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014992240631727326, 'n_size': 320, 'n_batch': 80, 'duration': 99.36593246459961, 'duration_batch': 1.242074155807495, 'duration_size': 0.31051853895187376, 'avg_pred_std': 0.19987344325636514}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007941817007667851, 'avg_role_model_std_loss': 0.3285603669361308, 'avg_role_model_mean_pred_loss': 0.00020670617296367766, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007941817007667851, 'n_size': 80, 'n_batch': 20, 'duration': 19.3479266166687, 'duration_batch': 0.967396330833435, 'duration_size': 0.24184908270835875, 'avg_pred_std': 0.06898661321029068}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015280315952168166, 'avg_role_model_std_loss': 0.1436633735797855, 'avg_role_model_mean_pred_loss': 2.658071664495944e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015280315952168166, 'n_size': 320, 'n_batch': 80, 'duration': 99.75400042533875, 'duration_batch': 1.2469250053167342, 'duration_size': 0.31173125132918356, 'avg_pred_std': 0.189214165561134}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007610848201147746, 'avg_role_model_std_loss': 0.30814517063022323, 'avg_role_model_mean_pred_loss': 0.00022678597003369382, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007610848201147746, 'n_size': 80, 'n_batch': 20, 'duration': 19.45919370651245, 'duration_batch': 0.9729596853256226, 'duration_size': 0.24323992133140565, 'avg_pred_std': 0.06733945356681943}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012301572283377026, 'avg_role_model_std_loss': 0.10807322694710982, 'avg_role_model_mean_pred_loss': 5.3408418359659e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012301572283377026, 'n_size': 320, 'n_batch': 80, 'duration': 99.28717947006226, 'duration_batch': 1.2410897433757782, 'duration_size': 0.31027243584394454, 'avg_pred_std': 0.19792793567758055}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006984197727433639, 'avg_role_model_std_loss': 0.247640823103211, 'avg_role_model_mean_pred_loss': 0.0001665852697917726, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006984197727433639, 'n_size': 80, 'n_batch': 20, 'duration': 19.52495002746582, 'duration_batch': 0.976247501373291, 'duration_size': 0.24406187534332274, 'avg_pred_std': 0.06778875123709441}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010839548700573686, 'avg_role_model_std_loss': 0.07997205620506662, 'avg_role_model_mean_pred_loss': 9.080777822098943e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010839548700573686, 'n_size': 320, 'n_batch': 80, 'duration': 99.66356492042542, 'duration_batch': 1.2457945615053176, 'duration_size': 0.3114486403763294, 'avg_pred_std': 0.19800062356516718}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008054383638955187, 'avg_role_model_std_loss': 0.28627982361469717, 'avg_role_model_mean_pred_loss': 0.00025097979236513577, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008054383638955187, 'n_size': 80, 'n_batch': 20, 'duration': 19.489768266677856, 'duration_batch': 0.9744884133338928, 'duration_size': 0.2436221033334732, 'avg_pred_std': 0.06728147398680448}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007286213950465026, 'avg_role_model_std_loss': 0.10312503655737952, 'avg_role_model_mean_pred_loss': 6.776996160624913e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007286213950465026, 'n_size': 320, 'n_batch': 80, 'duration': 99.96071243286133, 'duration_batch': 1.2495089054107666, 'duration_size': 0.31237722635269166, 'avg_pred_std': 0.1800330831320025}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007334689510025782, 'avg_role_model_std_loss': 0.2440898734063012, 'avg_role_model_mean_pred_loss': 0.00019387806316268908, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007334689510025782, 'n_size': 80, 'n_batch': 20, 'duration': 19.55093002319336, 'duration_batch': 0.977546501159668, 'duration_size': 0.244386625289917, 'avg_pred_std': 0.06980508081614971}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007013096967625643, 'avg_role_model_std_loss': 0.0806437349208462, 'avg_role_model_mean_pred_loss': 3.612507871512266e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007013096967625643, 'n_size': 320, 'n_batch': 80, 'duration': 99.62839913368225, 'duration_batch': 1.245354989171028, 'duration_size': 0.311338747292757, 'avg_pred_std': 0.193802969326498}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008005985312775011, 'avg_role_model_std_loss': 0.25233456931382536, 'avg_role_model_mean_pred_loss': 0.00024166704105831326, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008005985312775011, 'n_size': 80, 'n_batch': 20, 'duration': 19.874733448028564, 'duration_batch': 0.9937366724014283, 'duration_size': 0.24843416810035707, 'avg_pred_std': 0.06946740932762623}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000650154861421015, 'avg_role_model_std_loss': 0.13836135070985306, 'avg_role_model_mean_pred_loss': 6.324771196684101e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000650154861421015, 'n_size': 320, 'n_batch': 80, 'duration': 100.67735123634338, 'duration_batch': 1.2584668904542924, 'duration_size': 0.3146167226135731, 'avg_pred_std': 0.1874849540356081}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007276729341538157, 'avg_role_model_std_loss': 0.19176392750296145, 'avg_role_model_mean_pred_loss': 0.0001885933500119812, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007276729341538157, 'n_size': 80, 'n_batch': 20, 'duration': 19.5623676776886, 'duration_batch': 0.97811838388443, 'duration_size': 0.2445295959711075, 'avg_pred_std': 0.06984148817136884}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005832819898387243, 'avg_role_model_std_loss': 0.04374472918600412, 'avg_role_model_mean_pred_loss': 9.791867645731814e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005832819898387243, 'n_size': 320, 'n_batch': 80, 'duration': 99.7720296382904, 'duration_batch': 1.24715037047863, 'duration_size': 0.3117875926196575, 'avg_pred_std': 0.19292465539183468}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007389668950054329, 'avg_role_model_std_loss': 0.19728227270163642, 'avg_role_model_mean_pred_loss': 0.00020410724985005512, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007389668950054329, 'n_size': 80, 'n_batch': 20, 'duration': 19.41769814491272, 'duration_batch': 0.970884907245636, 'duration_size': 0.242721226811409, 'avg_pred_std': 0.06944395890459418}\n", + "Epoch 21\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005352863459663126, 'avg_role_model_std_loss': 0.01976681658542425, 'avg_role_model_mean_pred_loss': 6.443602266463554e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005352863459663126, 'n_size': 320, 'n_batch': 80, 'duration': 100.6177191734314, 'duration_batch': 1.2577214896678925, 'duration_size': 0.31443037241697314, 'avg_pred_std': 0.19984434806974605}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007991572895844002, 'avg_role_model_std_loss': 0.22150783375837904, 'avg_role_model_mean_pred_loss': 0.00023175869880640577, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007991572895844002, 'n_size': 80, 'n_batch': 20, 'duration': 20.159916162490845, 'duration_batch': 1.0079958081245421, 'duration_size': 0.25199895203113554, 'avg_pred_std': 0.07001060470938683}\n", + "Epoch 22\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005078368830425007, 'avg_role_model_std_loss': 0.02868203009257253, 'avg_role_model_mean_pred_loss': 8.534584292643494e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005078368830425007, 'n_size': 320, 'n_batch': 80, 'duration': 99.94173312187195, 'duration_batch': 1.2492716640233994, 'duration_size': 0.31231791600584985, 'avg_pred_std': 0.19320008249487727}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007862234280037229, 'avg_role_model_std_loss': 0.2152756543153373, 'avg_role_model_mean_pred_loss': 0.0002432201600976569, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007862234280037229, 'n_size': 80, 'n_batch': 20, 'duration': 19.478534698486328, 'duration_batch': 0.9739267349243164, 'duration_size': 0.2434816837310791, 'avg_pred_std': 0.06986867655068636}\n", + "Epoch 23\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000520391069062498, 'avg_role_model_std_loss': 0.03020876141404938, 'avg_role_model_mean_pred_loss': 4.458749571339605e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000520391069062498, 'n_size': 320, 'n_batch': 80, 'duration': 100.07614707946777, 'duration_batch': 1.2509518384933471, 'duration_size': 0.3127379596233368, 'avg_pred_std': 0.1921757934615016}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007205282323411666, 'avg_role_model_std_loss': 0.20713103588941523, 'avg_role_model_mean_pred_loss': 0.0001827256362942009, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007205282323411666, 'n_size': 80, 'n_batch': 20, 'duration': 19.527496099472046, 'duration_batch': 0.9763748049736023, 'duration_size': 0.24409370124340057, 'avg_pred_std': 0.06974663501605391}\n", + "Epoch 24\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005546999067291836, 'avg_role_model_std_loss': 0.027485956521196276, 'avg_role_model_mean_pred_loss': 6.895572416364425e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005546999067291836, 'n_size': 320, 'n_batch': 80, 'duration': 99.80755043029785, 'duration_batch': 1.247594380378723, 'duration_size': 0.31189859509468076, 'avg_pred_std': 0.1784662834368646}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007224415110249538, 'avg_role_model_std_loss': 0.17986427203068162, 'avg_role_model_mean_pred_loss': 0.00017955067020931638, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007224415110249538, 'n_size': 80, 'n_batch': 20, 'duration': 19.42549467086792, 'duration_batch': 0.971274733543396, 'duration_size': 0.242818683385849, 'avg_pred_std': 0.0726472232490778}\n", + "Epoch 25\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005678469070971914, 'avg_role_model_std_loss': 0.0783861569140182, 'avg_role_model_mean_pred_loss': 2.3522705448089696e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005678469070971914, 'n_size': 320, 'n_batch': 80, 'duration': 99.91300010681152, 'duration_batch': 1.248912501335144, 'duration_size': 0.312228125333786, 'avg_pred_std': 0.18876876458525657}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008989003312308341, 'avg_role_model_std_loss': 0.17780489571450744, 'avg_role_model_mean_pred_loss': 0.00034604211847950596, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008989003312308341, 'n_size': 80, 'n_batch': 20, 'duration': 19.550805807113647, 'duration_batch': 0.9775402903556824, 'duration_size': 0.2443850725889206, 'avg_pred_std': 0.0718118923716247}\n", + "Epoch 26\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004598389233777311, 'avg_role_model_std_loss': 0.03701534456806623, 'avg_role_model_mean_pred_loss': 6.740662266656723e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004598389233777311, 'n_size': 320, 'n_batch': 80, 'duration': 99.83816456794739, 'duration_batch': 1.2479770570993423, 'duration_size': 0.31199426427483556, 'avg_pred_std': 0.19492796149570496}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008001866593258456, 'avg_role_model_std_loss': 0.1477779638089487, 'avg_role_model_mean_pred_loss': 0.0002354497060985228, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008001866593258456, 'n_size': 80, 'n_batch': 20, 'duration': 19.476824283599854, 'duration_batch': 0.9738412141799927, 'duration_size': 0.24346030354499817, 'avg_pred_std': 0.07350850850343704}\n", + "Epoch 27\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004317075494100209, 'avg_role_model_std_loss': 0.016361890759745458, 'avg_role_model_mean_pred_loss': 3.335215542313308e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004317075494100209, 'n_size': 320, 'n_batch': 80, 'duration': 99.95542669296265, 'duration_batch': 1.2494428336620331, 'duration_size': 0.3123607084155083, 'avg_pred_std': 0.18771503504831344}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008291181290405802, 'avg_role_model_std_loss': 0.19831402401805462, 'avg_role_model_mean_pred_loss': 0.0002830501877054914, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008291181290405802, 'n_size': 80, 'n_batch': 20, 'duration': 19.60791325569153, 'duration_batch': 0.9803956627845765, 'duration_size': 0.24509891569614412, 'avg_pred_std': 0.06914721243083477}\n", + "Epoch 28\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004966751410051984, 'avg_role_model_std_loss': 0.013397608251197823, 'avg_role_model_mean_pred_loss': 1.0833947258724608e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004966751410051984, 'n_size': 320, 'n_batch': 80, 'duration': 100.00627398490906, 'duration_batch': 1.2500784248113632, 'duration_size': 0.3125196062028408, 'avg_pred_std': 0.19725441149203107}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008539365028264, 'avg_role_model_std_loss': 0.21735998928961636, 'avg_role_model_mean_pred_loss': 0.0002757914544125217, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008539365028264, 'n_size': 80, 'n_batch': 20, 'duration': 19.56255578994751, 'duration_batch': 0.9781277894973754, 'duration_size': 0.24453194737434386, 'avg_pred_std': 0.06783094126731157}\n", + "Epoch 29\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00043703318435177606, 'avg_role_model_std_loss': 0.012256504303220872, 'avg_role_model_mean_pred_loss': 2.395966142935274e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00043703318435177606, 'n_size': 320, 'n_batch': 80, 'duration': 99.63266062736511, 'duration_batch': 1.245408257842064, 'duration_size': 0.311352064460516, 'avg_pred_std': 0.18974662573309614}\n", + "Time out: 3631.7292110919952/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▁▂▂▃▂▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁█▇██▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▁█▄▃▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▄▁▄▄▄▄▅▄▃▄▃▄▂▂▂▃▃▃▆▃▂█▃▃▂▃▃▄▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▆▃▄▄▃▂▃▄▃▃▂▃█▁▃▁▃▄▃█▃█▄▅▄▄▄▄▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▄▁▄▄▄▄▅▄▃▄▃▄▂▂▂▃▃▃▆▃▂█▃▃▂▃▃▄▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▆▃▄▄▃▂▃▄▃▃▂▃█▁▃▁▃▄▃█▃█▄▅▄▄▄▄▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▄▁▄▄▄▄▅▄▃▄▃▄▂▂▂▃▃▃▆▃▂█▃▃▂▃▃▄▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▆▃▄▄▃▂▃▄▃▃▂▃█▁▃▁▃▄▃█▃█▄▅▄▄▄▄▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00854\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.0005\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.06783\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.19725\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00854\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.0005\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.00028\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.21736\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.0134\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.97813\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.25008\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.24453\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.31252\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 19.56256\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 100.00627\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/tvae/2/wandb/offline-run-20240229_042943-h1x5mo8p\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_042943-h1x5mo8p/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.002340294746186043, 'avg_g_mag_loss': 0.022517556619054607, 'avg_g_cos_loss': 0.06498512633163529, 'pred_duration': 2.4336907863616943, 'grad_duration': 4.428678035736084, 'total_duration': 6.862368822097778, 'pred_std': 0.06654548645019531, 'std_loss': 0.0213878583163023, 'mean_pred_loss': 1.473031716159312e-05, 'pred_rmse': 0.04837659373879433, 'pred_mae': 0.035608064383268356, 'pred_mape': 0.06660553067922592, 'grad_rmse': 0.3612377345561981, 'grad_mae': 0.13169732689857483, 'grad_mape': 1.3246768712997437}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.002340294746186043, 'avg_g_mag_loss': 0.022517556619054607, 'avg_g_cos_loss': 0.06498512633163529, 'avg_pred_duration': 2.4336907863616943, 'avg_grad_duration': 4.428678035736084, 'avg_total_duration': 6.862368822097778, 'avg_pred_std': 0.06654548645019531, 'avg_std_loss': 0.0213878583163023, 'avg_mean_pred_loss': 1.473031716159312e-05}, 'min_metrics': {'avg_loss': 0.002340294746186043, 'avg_g_mag_loss': 0.022517556619054607, 'avg_g_cos_loss': 0.06498512633163529, 'pred_duration': 2.4336907863616943, 'grad_duration': 4.428678035736084, 'total_duration': 6.862368822097778, 'pred_std': 0.06654548645019531, 'std_loss': 0.0213878583163023, 'mean_pred_loss': 1.473031716159312e-05, 'pred_rmse': 0.04837659373879433, 'pred_mae': 0.035608064383268356, 'pred_mape': 0.06660553067922592, 'grad_rmse': 0.3612377345561981, 'grad_mae': 0.13169732689857483, 'grad_mape': 1.3246768712997437}, 'model_metrics': {'tvae': {'avg_loss': 0.002340294746186043, 'avg_g_mag_loss': 0.022517556619054607, 'avg_g_cos_loss': 0.06498512633163529, 'pred_duration': 2.4336907863616943, 'grad_duration': 4.428678035736084, 'total_duration': 6.862368822097778, 'pred_std': 0.06654548645019531, 'std_loss': 0.0213878583163023, 'mean_pred_loss': 1.473031716159312e-05, 'pred_rmse': 0.04837659373879433, 'pred_mae': 0.035608064383268356, 'pred_mape': 0.06660553067922592, 'grad_rmse': 0.3612377345561981, 'grad_mae': 0.13169732689857483, 'grad_mape': 1.3246768712997437}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:32:09.773143Z", + "iopub.status.busy": "2024-02-29T05:32:09.772841Z", + "iopub.status.idle": "2024-02-29T05:32:09.777116Z", + "shell.execute_reply": "2024-02-29T05:32:09.776290Z" + }, + "papermill": { + "duration": 0.0282, + "end_time": "2024-02-29T05:32:09.779164", + "exception": false, + "start_time": "2024-02-29T05:32:09.750964", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:32:09.820437Z", + "iopub.status.busy": "2024-02-29T05:32:09.819934Z", + "iopub.status.idle": "2024-02-29T05:32:10.291755Z", + "shell.execute_reply": "2024-02-29T05:32:10.290586Z" + }, + "papermill": { + "duration": 0.496243, + "end_time": "2024-02-29T05:32:10.294946", + "exception": false, + "start_time": "2024-02-29T05:32:09.798703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:32:10.343563Z", + "iopub.status.busy": "2024-02-29T05:32:10.342710Z", + "iopub.status.idle": "2024-02-29T05:32:10.631486Z", + "shell.execute_reply": "2024-02-29T05:32:10.630522Z" + }, + "papermill": { + "duration": 0.314088, + "end_time": "2024-02-29T05:32:10.634360", + "exception": false, + "start_time": "2024-02-29T05:32:10.320272", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxSklEQVR4nO3de1AUZ7438G/PneEuyM0Q8Ea8g4vCYqKprLOiSYwac6KWFS+V6IlHsich5sJuhCRuBS9syiRaeo77eju1RtY9Mac2ySGaUcxJRE1QI4qymkVRcUAwMMAw9+f9o2caRm4zCExr/z5VXTPT80zPMyPz9Xn66X6aY4wxEEKICMj8XQFCCHGjQCKEiAYFEiFENCiQCCGiQYFECBENCiRCiGhQIBFCREPh7wr0BafTierqagQHB4PjOH9XhxByF8YYmpqaEBcXB5ms63bQAxFI1dXViI+P93c1CCE9uH79Oh566KEun38gAik4OBgA/2FDQkL8XBtCyN2MRiPi4+OF32pXHohAcnfTQkJCKJAIEbGedqnQTm1CiGhQIBFCRIMCiRAiGg/EPiRybxwOB2w2m7+rQe5jSqUScrn8nrdDgSRhjDEYDAY0NDT4uyrkARAWFoaYmJh7OhawV4G0detWbNq0CQaDAcnJyfjkk0+QlpbWadkdO3Zg7969OH/+PAAgNTUVH3zwgUf5ZcuWYc+ePR6vy8zMRFFRUW+qR7zkDqOoqChotVo6qJT0CmMMJpMJtbW1AIDY2Nheb8vnQCosLER2dja2b9+O9PR0bN68GZmZmaioqEBUVFSH8sXFxVi0aBGmTJkCjUaDDRs2YMaMGbhw4QKGDBkilJs5cyZ27dolPFar1b38SF1zOBl+uHoHrTYHpo6IhEIu3V1oDodDCKOIiAh/V4fc5wICAgAAtbW1iIqK6nX3jfN1Ctv09HRMnjwZW7ZsAcCfthEfH49XXnkFb7/9do+vdzgcCA8Px5YtW7BkyRIAfAupoaEBn3/+uVd1sFgssFgswmP3QVeNjY3dHodkdzgx4g//CwA4m/tbhGlVXr3fg8hsNqOyshKJiYnCHxMh96K1tRVXr17F0KFDodFoPJ4zGo0IDQ3t8TfqUxPBarWitLQUOp2ubQMyGXQ6HUpKSrzahslkgs1mw6BBgzzWFxcXIyoqCo888ghWrVqF+vr6LreRn5+P0NBQYfH2tBGFXAaVq1Vksjq8es2DjrpppK/0xd+ST4FUV1cHh8OB6Ohoj/XR0dEwGAxebeOtt95CXFycR6jNnDkTe/fuhV6vx4YNG3Ds2DHMmjULDkfnoZGTk4PGxkZhuX79utefQaOkQCJErAZ0lG39+vXYv38/iouLPZp0CxcuFO6PHz8eEyZMwPDhw1FcXIzp06d32I5are71PiatSgGj2Y5WCiRCRMenFlJkZCTkcjlqamo81tfU1CAmJqbb1xYUFGD9+vU4dOgQJkyY0G3ZYcOGITIyEleuXPGlel7Rqvidba02CiTSOxzHeb2/837x7rvvIiUlxd/V8C2QVCoVUlNTodfrhXVOpxN6vR4ZGRldvm7jxo1Yt24dioqKMGnSpB7f58aNG6ivr7+n4cOuaJR8IJms9j7fNiEDZffu3QgLC+uz7a1Zs8bjd+0vPo97Z2dnY8eOHdizZw8uXryIVatWoaWlBcuXLwcALFmyBDk5OUL5DRs2YO3atdi5cycSExNhMBhgMBjQ3NwMAGhubsYbb7yBEydO4OrVq9Dr9ZgzZw5GjBiBzMzMPvqYbYQWEnXZiARYrVavygUFBYni8A+fA2nBggUoKChAbm4uUlJScPbsWRQVFQk7uquqqnDr1i2h/LZt22C1WvHcc88hNjZWWAoKCgAAcrkc586dwzPPPIOkpCS8+OKLSE1Nxf/93//1y7FIAdRl6xJjDCarfcAXXy+eXFRUhMceewxhYWGIiIjA008/jZ9//hkAMGXKFLz11lse5W/fvg2lUolvv/0WAHDr1i089dRTCAgIwNChQ7Fv3z4kJiZi8+bNvfreysrK8Jvf/AYBAQGIiIjAypUrhf9wAX4EOS0tDYGBgQgLC8Ojjz6Ka9euAQB++uknPPHEEwgODkZISAhSU1Px448/dvt+xcXFWL58ORobG8FxHDiOw7vvvgsASExMxLp167BkyRKEhIRg5cqVAPjBpKSkJGi1WgwbNgxr1671OF3o7i7bsmXLMHfuXBQUFCA2NhYRERFYvXp1v59i1Kud2llZWcjKyur0ueLiYo/HV69e7XZbAQEB+Prrr3tTjV4JELpsFEh3a7U5MCZ34P4t3Mrfz4RW5f2fYktLC7KzszFhwgQ0NzcjNzcX8+bNw9mzZ7F48WJs3LgR69evF4ahCwsLERcXh6lTpwLgW/F1dXUoLi6GUqlEdna2cJSxr1paWpCZmYmMjAz88MMPqK2txUsvvYSsrCzs3r0bdrsdc+fOxYoVK/Dpp5/CarXi1KlTQt0WL16MiRMnYtu2bZDL5Th79iyUSmW37zllyhRs3rwZubm5qKioAMC3cNzcDYa8vDxhXXBwMHbv3o24uDiUlZVhxYoVCA4Oxptvvtnl+xw9ehSxsbE4evQorly5ggULFiAlJQUrVqzo1XflDcmdy0Zdtvvf/PnzPR7v3LkTgwcPRnl5OZ5//nm8+uqr+O6774QA2rdvHxYtWgSO43Dp0iV88803+OGHH4T9mX/+858xcuTIXtVl3759MJvN2Lt3LwIDAwEAW7ZswezZs7FhwwYolUo0Njbi6aefxvDhwwEAo0ePFl5fVVWFN954A6NGjQIAr+qhUqkQGhoKjuM6HUz6zW9+g9dff91j3TvvvCPcT0xMxJo1a7B///5uA8l9ALNcLseoUaPw1FNPQa/XUyD1pQDX/8TUQuooQClH+ft9v9/Om/f1xeXLl5Gbm4uTJ0+irq4OTqcTAP/jHjduHGbMmIG//OUvmDp1KiorK1FSUoL/+I//AABUVFRAoVDgV7/6lbC9ESNGIDw8vFd1v3jxIpKTk4UwAoBHH30UTqcTFRUVmDZtGpYtW4bMzEz89re/hU6nw/PPPy8M2GRnZ+Oll17Cf/3Xf0Gn0+Ff/uVfhODqrc4GjgoLC/Hxxx/j559/RnNzM+x2e4+zq44dO9bjFJDY2FiUlZXdU916IrmTuWjYv2scx0GrUgz44usRvrNnz8adO3ewY8cOnDx5EidPngTQtgN38eLF+Nvf/gabzYZ9+/Zh/PjxGD9+fJ9/X97atWsXSkpKMGXKFBQWFiIpKQknTpwAwO+7uXDhAp566ikcOXIEY8aMwcGDB+/p/dqHIwCUlJRg8eLFePLJJ/HFF1/gzJkz+MMf/tDjDu+7u44cxwnh318kF0ju/41badj/vlRfX4+Kigq88847mD59OkaPHo1ffvnFo8ycOXNgNptRVFSEffv2YfHixcJzjzzyCOx2O86cOSOsu3LlSodteGv06NH46aef0NLSIqz7/vvvIZPJ8MgjjwjrJk6ciJycHBw/fhzjxo3Dvn37hOeSkpLw2muv4dChQ3j22Wc9TjLvikql6vJMhrsdP34cCQkJ+MMf/oBJkyZh5MiRwk51sZFeIKlop/b9LDw8HBEREfjP//xPXLlyBUeOHEF2drZHmcDAQMydOxdr167FxYsXsWjRIuG5UaNGQafTYeXKlTh16hTOnDmDlStXIiAgoFfnYi1evBgajQZLly7F+fPncfToUbzyyit44YUXEB0djcrKSuTk5KCkpATXrl3DoUOHcPnyZYwePRqtra3IyspCcXExrl27hu+//x4//PCDxz6mriQmJqK5uRl6vR51dXUwmUxdlh05ciSqqqqwf/9+/Pzzz/j444/vuRXWXyQXSNRlu7/JZDLs378fpaWlGDduHF577TVs2rSpQ7nFixfjp59+wtSpU/Hwww97PLd3715ER0dj2rRpmDdvnjDidPcZ6t7QarX4+uuvcefOHUyePBnPPfccpk+fLsyGodVqcenSJcyfPx9JSUlYuXIlVq9ejX/913+FXC5HfX09lixZgqSkJDz//POYNWsW3nvvvR7fd8qUKXj55ZexYMECDB48GBs3buyy7DPPPIPXXnsNWVlZSElJwfHjx7F27VqfP+tA8Hn6ETHydmoDANh/qgpvf1aG6aOi8P+WTR6gGoqPe/qRzqaKkJobN24gPj4e33zzTafnThLvdPc35e1vVIKjbNRlk7ojR46gubkZ48ePx61bt/Dmm28iMTER06ZN83fVJE+CXTY+g6nLJl02mw2///3vMXbsWMybNw+DBw8WDpL8y1/+gqCgoE6XsWPHDlgdZ82a1WU9PvjggwGrx0CTXAuJDowkmZmZXZ4n+cwzzyA9Pb3T53o6grov/fnPf0Zra2unz909ueGDRHKBJJztb6Nhf9JRcHBwj9efHwjt55uXEgl22aiFRIhYUSARQkRDcoEknO1vc/g87QUhpH9JL5BcLSTGAIu9f8/LIYT4RnqB1O7Mcuq2ESIukgskj2uz0bFIpBcexEn+xUJygQS0m8aWzvgn96m+nuQf4Gd75TgODQ0NfbpdX0gykLR0+gghoiTJQAqgof/OMQZYWwZ+oUn++2ySf4vFgjVr1mDIkCEIDAxEenq6xzz3165dw+zZsxEeHo7AwECMHTsWX331Fa5evYonnngCAD/FC8dxWLZsWa++j3shuSO1Ac+hf9KOzQR8EDfw7/v7akAV2HM5F5rkv+tJ/rOyslBeXo79+/cjLi4OBw8exMyZM1FWVoaRI0di9erVsFqt+PbbbxEYGIjy8nIEBQUhPj4e//3f/4358+ejoqICISEhCAgI6NV3ci8kGUh0cOT9jSb573yS/6qqKuzatQtVVVWIi+P/Y1mzZg2Kioqwa9cufPDBB6iqqsL8+fOFKX2HDRsmvN59jlxUVFSf75/yliQDyT3RPwXSXZRavrXij/f1AU3y37mysjI4HA4kJSV5rLdYLMJFIH/3u99h1apVOHToEHQ6HebPn9/jpe0HkjT3ISlp2L9THMd3nQZ6oUn++2SS/+bmZsjlcpSWluLs2bPCcvHiRXz00UcAgJdeegn//Oc/8cILL6CsrAyTJk3CJ5980mef9V5JMpCEOZFo2P++Q5P88zqb5H/ixIlwOByora3FiBEjPJb2Xbv4+Hi8/PLL+Oyzz/D6669jx44dwjYBeH3xgP4gyUCiWSPvXzTJP6+zSf6TkpKwePFiLFmyBJ999hkqKytx6tQp5Ofn48svvwQAvPrqq/j6669RWVmJ06dP4+jRo8L7JSQkgOM4fPHFF7h9+7bHSOGAYQ+AxsZGBoA1NjZ6Vf79v19gCW99wT74qryfayZera2trLy8nLW2tvq7Kj47fPgwGz16NFOr1WzChAmsuLiYAWAHDx4Uynz11VcMAJs2bVqH11dXV7NZs2YxtVrNEhIS2L59+1hUVBTbvn27V+9/93udO3eOPfHEE0yj0bBBgwaxFStWsKamJsYYYwaDgc2dO5fFxsYylUrFEhISWG5uLnM4HMxisbCFCxey+Ph4plKpWFxcHMvKyvL63+Tll19mERERDADLy8tjjDFmtVpZbm4uS0xMZEqlksXGxrJ58+axc+fOMcYYy8rKYsOHD2dqtZoNHjyYvfDCC6yurk7Y5vvvv89iYmIYx3Fs6dKlXtXDrbu/KW9/o5Kb5B8A/nSoAp8cuYIlGQl4f864Aaih+NAk/21okv++QZP89xJ12aSNJvkXL0nuQ9Iq6dpsUkaT/IuXpFtIdBySNNEk/+LVqxbS1q1bkZiYCI1Gg/T0dJw6darLsjt27MDUqVMRHh6O8PBw6HS6DuUZY8jNzUVsbCwCAgKg0+lw+fLl3lTNK+4DI0007E/uEhwc3GHI3L0kJCQMWD2GDBnSZT0okNopLCxEdnY28vLycPr0aSQnJyMzM7PLc4GKi4uxaNEiHD16FCUlJYiPj8eMGTNw8+ZNoczGjRvx8ccfY/v27Th58iQCAwORmZkJs9nc+0/WjbYuG80Y+QCMaRCR6JO/JZ/G9RhjaWlpbPXq1cJjh8PB4uLiWH5+vlevt9vtLDg4mO3Zs4cxxpjT6WQxMTFs06ZNQpmGhgamVqvZp59+2uk2zGYza2xsFJbr16/7NOz/3eXbLOGtL9hvPyz2qvyDyG63s/Lyco8hX0LuRV1dHSsvL2d2u73Dc94O+/u0D8lqtaK0tBQ5OTnCOplMBp1Oh5KSEq+2YTKZYLPZhGZnZWUlDAYDdDqdUCY0NBTp6ekoKSnBwoULO2wjPz8f7733ni9V90CjbIBcLkdYWJjQstVqtb06MJAQxhhMJhNqa2sRFhYGuVze84u64FMg1dXVweFwIDo62mN9dHQ0Ll265NU23nrrLcTFxQkBZDAYhG3cvU33c3fLycnxODrXaDQiPj7e689BZ/vz3KcT9HbqDULaCwsL8zhFpTcGdJRt/fr12L9/P4qLi+/pYDy1Wg21Wt3r12uVrnPZJD7sz3EcYmNjERUVBZvN5u/qkPuYUqm8p5aRm0+BFBkZCblcjpqaGo/1NTU1PSZjQUEB1q9fj2+++cZjugP362pqaoQpGdyPU1JSfKme1zQqfl9+q+vabFLvqsjl8j75YyLkXvk0yqZSqZCamgq9Xi+sczqd0Ov1yMjI6PJ1GzduxLp161BUVCRMiuU2dOhQxMTEeGzTaDTi5MmT3W7zXrjP9mcMMNNIGyGi4XOXLTs7G0uXLsWkSZOQlpaGzZs3o6WlBcuXLwfATw86ZMgQ5OfnAwA2bNiA3NxcYd5i934h91GnHMfh1VdfxR//+EeMHDkSQ4cOxdq1axEXF4e5c+f23Sdtx+PabDaHsJObEOJfPgfSggULcPv2beTm5sJgMCAlJQVFRUXCTumqqirIZG0Nr23btsFqteK5557z2E5eXp4wMfmbb76JlpYWrFy5Eg0NDXjsscdQVFTUbyd9ymUcVAoZrHYnTFY7BgWq+uV9CCG+keTZ/gCQ8v4hNJhsOPzaNIyMDu7nGhIibd7+RiV5ci3QdrS2lI9FIkRsJBtIGhWd8U+I2Eg2kOjgSELER7qBpHSf8U+BRIhYSDaQAqjLRojoSDeQ3FOQ0JxIhIiGZANJS2f8EyI6kg0k6rIRIj7SDSQljbIRIjaSDSTqshEiPpINpLaJ/imQCBEL6QaSkv/oZtqHRIhoSDaQtHQpJEJER7KBRBP9EyI+kg0k905t6rIRIh6SDaQAmn6EENGRbiBRl40Q0ZFsILl3atOR2oSIh2QDiY7UJkR8pBtI7c5lczrv+2nFCXkgSDaQtO0ufWS2UyuJEDGQbCB5XJuNum2EiIJkA0km46BW8B+fRtoIEQfJBhLQbqJ/GmkjRBQkHkiuoX9qIREiCpIOJI2SumyEiImkA6nt4Eg6458QMZB0INHpI4SIi7QDiY7WJkRUpBVIDhtw/jPgx12A00GjbISIjMLfFRhwf1vO346ZQ102QkSmVy2krVu3IjExERqNBunp6Th16lSXZS9cuID58+cjMTERHMdh8+bNHcq8++674DjOYxk1alRvqtY9uRJQBvL3zY1tLSQKJEJEwedAKiwsRHZ2NvLy8nD69GkkJycjMzMTtbW1nZY3mUwYNmwY1q9fj5iYmC63O3bsWNy6dUtYvvvuO1+r5h1NCH9rbmzbh0RdNkJEwedA+vDDD7FixQosX74cY8aMwfbt26HVarFz585Oy0+ePBmbNm3CwoULoVaru9yuQqFATEyMsERGRnZZ1mKxwGg0eixe04S6NmJsdykkGvYnRAx8CiSr1YrS0lLodLq2Dchk0Ol0KCkpuaeKXL58GXFxcRg2bBgWL16MqqqqLsvm5+cjNDRUWOLj471/I3cgteuy0T4kQsTBp0Cqq6uDw+FAdHS0x/ro6GgYDIZeVyI9PR27d+9GUVERtm3bhsrKSkydOhVNTU2dls/JyUFjY6OwXL9+3fs3U3fsstFE/4SIgyhG2WbNmiXcnzBhAtLT05GQkIC//vWvePHFFzuUV6vV3Xb/uiW0kIw0ykaIyPjUQoqMjIRcLkdNTY3H+pqamm53WPsqLCwMSUlJuHLlSp9tU0BdNkJEy6dAUqlUSE1NhV6vF9Y5nU7o9XpkZGT0WaWam5vx888/IzY2ts+2KehklI26bISIg89dtuzsbCxduhSTJk1CWloaNm/ejJaWFixfzh9wuGTJEgwZMgT5+fkA+B3h5eXlwv2bN2/i7NmzCAoKwogRIwAAa9aswezZs5GQkIDq6mrk5eVBLpdj0aJFffU523iMslELiRAx8TmQFixYgNu3byM3NxcGgwEpKSkoKioSdnRXVVVBJmtreFVXV2PixInC44KCAhQUFODxxx9HcXExAODGjRtYtGgR6uvrMXjwYDz22GM4ceIEBg8efI8frxMeXTaaD4kQMenVTu2srCxkZWV1+pw7ZNwSExPBWPdX9di/f39vqtE77UbZ2vYh0XFIhIiBtE6uBQBNGH9LR2oTIjoSDKS2Lpt7H5LZ5qRrsxEiApIOpPbXZqNWEiH+J8FAcu1Dshihkbd9fAokQvxPgoHkaiExJ2T2FmGifxppI8T/pBdICg0gV/H32w3907FIhPif9AKJ4zo9wZa6bIT4n/QCCejiBFs6FokQf5N4INE0toSIiUQDqWOXjfYhEeJ/Eg2kjifY0j4kQvxP2oFkbqAuGyEiIs1A8hhlo2F/QsRCmoEknGBrRIDKdWAkddkI8TuJBlJncyLRsD8h/ibRQKJRNkLESKKB1DbKRju1CREPaQdSuzmRaB8SIf5HgURdNkJEQ5qBJAz7G2mif0JERJqB5G4hOSwIlNsAUJeNEDGQZiCpggCO/+hBzASAzvYnRAykGUgyGaAOBgAEsRYA1GUjRAykGUiA0G3TsmYA1GUjRAwkH0gBDj6QaJSNEP+TbiCp+UDSuALJYnfCQddmI8SvpBtIrhaS2hVIAHXbCPE3yQeSwtokrKId24T4l4QDiT84UmZpd+URCiRC/ErCgdTxBFuTjY5FIsSfehVIW7duRWJiIjQaDdLT03Hq1Kkuy164cAHz589HYmIiOI7D5s2b73mbfaLd+WwaaiERIgo+B1JhYSGys7ORl5eH06dPIzk5GZmZmaitre20vMlkwrBhw7B+/XrExMT0yTb7RLtpbGkKEkLEwedA+vDDD7FixQosX74cY8aMwfbt26HVarFz585Oy0+ePBmbNm3CwoULoVar+2SbfaLdxSKFLhsFEiF+5VMgWa1WlJaWQqfTtW1AJoNOp0NJSUmvKtCbbVosFhiNRo/FZ5102Uw07E+IX/kUSHV1dXA4HIiOjvZYHx0dDYPB0KsK9Gab+fn5CA0NFZb4+Hjf37iTq9eaqYVEiF/dl6NsOTk5aGxsFJbr16/7vhH3vNqWtjmR6Ix/QvxL4UvhyMhIyOVy1NTUeKyvqanpcod1f2xTrVZ3uT/Ka+5LIVmbEajkTxmhLhsh/uVTC0mlUiE1NRV6vV5Y53Q6odfrkZGR0asK9Mc2veIeZQMQJjMDoC4bIf7mUwsJALKzs7F06VJMmjQJaWlp2Lx5M1paWrB8+XIAwJIlSzBkyBDk5+cD4Hdal5eXC/dv3ryJs2fPIigoCCNGjPBqm/1CrgCUgYCtBeGyVgA0ykaIv/kcSAsWLMDt27eRm5sLg8GAlJQUFBUVCTulq6qqIJO1Nbyqq6sxceJE4XFBQQEKCgrw+OOPo7i42Ktt9htNKGBrQYjMBEBOXTZC/IxjjN33c24YjUaEhoaisbERISEhPb/AbeuvgdsX8feU7XjlRAienTgEHy5I6bd6EiJV3v5G78tRtj7jGmkLhntebWohEeJPEg8k/likQPdE/9RlI8SvKJAABLrn1abjkAjxK2kHkmvo3z2vNs0YSYh/STuQXC0kjZMm+idEDCiQAKjt7i4bBRIh/iTxQOK7bCobP682ddkI8S+JB5Jron87H0jUZSPEvyiQACis/HxKVro2GyF+JfFACgMAyCxtE7zRFCSE+I+0A8k17M9ZjOA4fhXtRyLEf6QdSK4uG2cxQqvkE4lG2gjxHwokAGBORCptAGjHNiH+JO1AUmoAuQoAEKGwAKAuGyH+JO1AAoRWUqSSn6SNumyE+A8FkiuQIuT8NLbUZSPEfyiQXCNt4XJXC4m6bIT4DQWSq4UUJnN32eg4JEL8hQLJHUgczRpJiL9RILmnseVaAFAgEeJPFEiuFpJ7Xm0z7UMixG8okFyBFMSohUSIv1EgqV3zajspkAjxNwokVwspwBVI1GUjxH8okDrMq03D/oT4CwWSa5RNTbNGEuJ3FEiuFpKKJvonxO8okFyBpLQZATA6dYQQP6JAcgWSzGmDGjZqIRHiRxRIqiCA47+GEJhoHxIhfkSBxHHCGf8hXAt12Qjxo14F0tatW5GYmAiNRoP09HScOnWq2/IHDhzAqFGjoNFoMH78eHz11Vcezy9btgwcx3ksM2fO7E3VesfVbQuBibpshPiRz4FUWFiI7Oxs5OXl4fTp00hOTkZmZiZqa2s7LX/8+HEsWrQIL774Is6cOYO5c+di7ty5OH/+vEe5mTNn4tatW8Ly6aef9u4T9YZwgq0JVocTdodz4N6bECLwOZA+/PBDrFixAsuXL8eYMWOwfft2aLVa7Ny5s9PyH330EWbOnIk33ngDo0ePxrp16/CrX/0KW7Zs8SinVqsRExMjLOHh4b37RL3huj5biOsEWxN12wjxC58CyWq1orS0FDqdrm0DMhl0Oh1KSko6fU1JSYlHeQDIzMzsUL64uBhRUVF45JFHsGrVKtTX13dZD4vFAqPR6LHcE3eXzTUnkpm6bYT4hU+BVFdXB4fDgejoaI/10dHRMBgMnb7GYDD0WH7mzJnYu3cv9Ho9NmzYgGPHjmHWrFlwODoPhvz8fISGhgpLfHy8Lx+jI9dO7QjXNLY00kaIfyj8XQEAWLhwoXB//PjxmDBhAoYPH47i4mJMnz69Q/mcnBxkZ2cLj41G472FkquFFE6BRIhf+dRCioyMhFwuR01Njcf6mpoaxMTEdPqamJgYn8oDwLBhwxAZGYkrV650+rxarUZISIjHck/uCiQa+ifEP3wKJJVKhdTUVOj1emGd0+mEXq9HRkZGp6/JyMjwKA8Ahw8f7rI8ANy4cQP19fWIjY31pXq95xplc8+rTUP/hPiHz6Ns2dnZ2LFjB/bs2YOLFy9i1apVaGlpwfLlywEAS5YsQU5OjlD+3//931FUVIQ//elPuHTpEt599138+OOPyMrKAgA0NzfjjTfewIkTJ3D16lXo9XrMmTMHI0aMQGZmZh99zB64p7Hl3F02moKEEH/weR/SggULcPv2beTm5sJgMCAlJQVFRUXCjuuqqirIZG05N2XKFOzbtw/vvPMOfv/732PkyJH4/PPPMW7cOACAXC7HuXPnsGfPHjQ0NCAuLg4zZszAunXroFar++hj9sAdSK5pbKnLRoh/cIwx5u9K3Cuj0YjQ0FA0Njb2bn/SP48Be5/BTWUCHm3Kx/pnx2Nh2sN9X1FCJMrb3yidywYILSQtTfRPiF9RIAFtgeSkLhsh/kSBBAiBpHa2QgE7jbIR4icUSIBwpDYABKGVumyE+AkFEgDIFfxEbeDPZ2u10bA/If5AgeQmzInUgn/UNPu5MoRIEwWSm9o9J1IrSq/9AkOj2c8VIkR6KJDcXC2klEj+4dcXOp+9gBDSfyiQ3FyBlDGEP3j9y7Jb/qwNIZJEgeTmOsF2wmD+K/nh6h3UNlG3jZCBRIHk5mohhXEmJMeHgTHg6ws1PbyIENKXKJDcXIEEcyOeGs/P1fTVOeq2ETKQKJDc3AdHmo2YNY6fh+lkZT3qmi1+rBQh0kKB5NauhRQ/SIsJD4XCyYBD1G0jZMBQILm1CyQAQivpKxptI2TAUCC5adxdNj6QnnTtRyr5Zz3utFj9VStCJIUCyc11sUhY+EBKiAjE2LgQOJwMh8vpIElCBgIFkttdXTYAeHI83237sowCiZCBQIHkJgSSEXA6AQCzxvHdtuNX6tBgom4bIf2NAslNmBOJAVb+bP9hg4MwKiYYdifD4XIabSOkv1EguSk1gNx1lZNOum002kZI/6NAaq/T/Uh8t+27K3VobLX5o1aESAYFUnvuoX+LUVg1IioYSdFBsDkYvqFuGyH9igKpvU5aSEDbQZL/e566bYT0Jwqk9roIJPd+pG//UYcmM3XbCOkvFEjttTvBtr2k6CAMHxwIq8MJ/cVaP1SMEGlQ+LsCotJFC4njOMwZE4Kvvz2Pf5yqARKnAyFDwGRyWOxOWGxOmO0O2BxOxIYGQC7jBqa+jAF1l4F/HgVkCiA+DYgaA8jkA/P+RNxsrUDNBaDmPD+CHBIHhAwBQmIBVeC9bdthBxwWwG4BFBpApe2TKlMgtecOpJs/At9/BNRfAep/Buqv4HfNNfidGsAtAB8BDsbBgEG4ySJxk0WimkWgmkVCrg7EhPhBmJgQgaFRIZDJ5AAn40NCpuQPL1AE8LfKANd916JQ91xHmxm4+h1w+Wvg8iHgl6uez6uCgCG/Ah5K4wPqocmAdpD33wFj/B+ytdm1tPCP7Wb+ve3tFpsZcPZwySin3bU4AKet7bHD9bpOvwcNoNTygwwB4W1LZ9+P1QQYbwINVUDjjbZFJge0EfwSGOm6H8l/F+pg/j8d0x2g9Q5/a6rn77f+wod7Z68NjGw7xYg52j4Xc7puHfz3ZW4AWhs63lqMADh++zJ52y3nuq/UuD7rIL6e7e8rA7r/ni1NgKEMuPVT23K7gq9TZzRhrnCK4z+fw8qHi93sum2967HFFUBW/pY527Y1cz3w61Xd189LFEjtuQPpH0X8cpdfuDAYHSrEcHeg5uwYgnoM4eoBVLQVcgK45lp8pQwEgqKAoOh2t677DitwRQ9UHgNsprbXyFVAwqMAGHCjFLA2AZXf8otbWILrD9rVcuM4/r771mEBLK7wsTZ5/rGJiVLL/5ACwvkfsvEmHyRSIFcDciX/HwYAwHXrfmw3t61rL3AwEDOB/zc1VvPfmbWZD0lzA1B74d7r5ui7sxgokNp75Emg7AD/P3TECNcynF8GDYdaHoSm2y2wygGt7Q4CTDehMVVD1VwNufEGWOMNNBibUNfUil+azQBzQAYnZGAIUACDNBy0MhvUzAql0wLOYQZna237X8zWAvxSyS/dCY4DRv4WSMoEhj4OqPmLXMLpAG5fAq6fAm78wN/WXwYaepOO4FtbSq1nC07hunU/lim734ZMwS9y161M2dY6AFwtLRPf2rKZXI9b+ftmI99iMTfwPyibiV+aqjvWMzQeCIsHQh/iF+YEWur5wDLVAS11rpZQHf8eqqC21od2ULtWySC+5XP3a1rq+Lp09qMXcPz3FRDmCs67bjUhfBmn3bOF5XTdt7W2tdLcrbfWX1wtSlcLpTshQ4DYZCA2xXWbDATHuP7jcWGMb6m5w8lYzb+XQuP699W4WvGux3J1u+fU/H+AHrdq/t+2j3CMse6+4fuC0WhEaGgoGhsbERIS0vMLBoDZ5sCxf9zG33+qhv5iLVptHZvOwWoFEiO0GBGpxohwOQZxTQiw1EFjqYPWWo8A9621HgpmxZ2odJgSdFAPmYDBwRpEBqugVfXwx2C6w4eU0+75vytjEH5cMiUfaqpg120g31qTiWTMw+nkf0TucGr9he86hA7hw0cT5vmj6w5jfAD05kfkdPBdPY5r62YJXS6593XwBWN8d6z1l7aWq/A+7Vq8qiDfuuYDzNvfaK8CaevWrdi0aRMMBgOSk5PxySefIC0trcvyBw4cwNq1a3H16lWMHDkSGzZswJNPPik8zxhDXl4eduzYgYaGBjz66KPYtm0bRo4c6VV9xBhI7Zmsdugv1uJkZT2u1plQWdeC6sZW9MV/BVqVHJFBaoRplQjRKBGsUSBYo3Ddb3scpFYgUK1AoFrO36r4x1qVHGqFDFx//JgIcem3QCosLMSSJUuwfft2pKenY/PmzThw4AAqKioQFRXVofzx48cxbdo05Ofn4+mnn8a+ffuwYcMGnD59GuPGjQMAbNiwAfn5+dizZw+GDh2KtWvXoqysDOXl5dBoNH32YcXEbHOg6g4fTpV1LbhWb4LF7oCM4yDjAA6cq4HCP7Y7GOpbrKhrtgiL2dY3+3pUchlCAlwhFqBEiCvQQgIUCNYooVHIoHIvchlUCj7EVAoZlHIOHMcJ9ZZxHDjXrYzjP4NCJoNcBshlMsjbrZNxfBuNMYC5WmuMtTXkVAoOaoUcaqUMaoUcGiX//hSe959+C6T09HRMnjwZW7ZsAQA4nU7Ex8fjlVdewdtvv92h/IIFC9DS0oIvvvhCWPfrX/8aKSkp2L59OxhjiIuLw+uvv441a9YAABobGxEdHY3du3dj4cKFffZhHySMMZisDiGcGkw2NJntaDLbYDTbYTTbYGxte2yy2NFsscNkdaDFYkeL1d5ngTaQOA58GMplUMhlkMs4KGUc5HLOFXwcFDIOaoUMGqUcASo5tCo5f9+1aJRyvtfl3iD4+5zrPwI+UCEEX1vItn/eFcAydyC7/hPhPLfDcRw48L1f97q2z8J5vK8ba7efyv3r5IO7XWjfFeBCZ7zdz7n9D5sDIJdxkMv4/yjctwoZJ9QNnNAJ9KgbY4CTMaEOTie/bSdjYIzhkZgQDI3s/jACb3+jPnWkrVYrSktLkZOTI6yTyWTQ6XQoKSnp9DUlJSXIzs72WJeZmYnPP/8cAFBZWQmDwQCdTic8HxoaivT0dJSUlHQaSBaLBRZL2w4+o9HYocyDjuM4VxdMgYSI3h1TYnc40WJ1oNlih7GVDzRjq80VZnyQNZltsNidsLoWi2uxOpyw2h2wO5jwx+lkrj9Y1x+tkzE4nAwO163dwT9ndzI4Xes5eP7xo92P1u5wwuw6xkv4YTLw6+7DMH1QvfPUaLw0dVifbMunQKqrq4PD4UB0dLTH+ujoaFy6dKnT1xgMhk7LGwwG4Xn3uq7K3C0/Px/vvfeeL1UnnVDIZQgNkCE0QIkhYT0c5+JHjDHYHAwWuwNmmxMWuwMWu1MIOYeTweZse2x38gerttocaLU5YLY50Gp1CI8tNicYY+1aFXyLw93S4MPP3RJgrhYCf9/pZEIZvoXgDuO2QHZvp/3zQNtt+26pe/v8fbRrJ3XcR97W6vJsbXkGurusJydr9x+Ek911/656MM8WWftWHt9abOueyzggKqTn3Sreui+H/XNycjxaXUajEfHx8X6sEelPHMdBpeCgUsgQ3Hd/+0SEfBrXjYyMhFwuR02N5zQcNTU1iImJ6fQ1MTEx3ZZ33/qyTbVajZCQEI+FEHL/8ymQVCoVUlNTodfrhXVOpxN6vR4ZGRmdviYjI8OjPAAcPnxYKD906FDExMR4lDEajTh58mSX2ySEPKCYj/bv38/UajXbvXs3Ky8vZytXrmRhYWHMYDAwxhh74YUX2Ntvvy2U//7775lCoWAFBQXs4sWLLC8vjymVSlZWViaUWb9+PQsLC2P/8z//w86dO8fmzJnDhg4dylpbW72qU2NjIwPAGhsbff04hJAB4O1v1OdAYoyxTz75hD388MNMpVKxtLQ0duLECeG5xx9/nC1dutSj/F//+leWlJTEVCoVGzt2LPvyyy89nnc6nWzt2rUsOjqaqdVqNn36dFZRUeF1fSiQCBE3b3+jdOoIIaTfefsbFcnJSoQQQoFECBGR+/I4pLu5e51SPGKbkPuB+7fZ0x6iByKQmpqaAIAOjiRE5JqamhAaGtrl8w/ETm2n04nq6moEBwf3eCa4+6ju69ev0w5wL9D35T36rrrGGENTUxPi4uIg62aerQeihSSTyfDQQw/59Bo6wts39H15j76rznXXMnKjndqEENGgQCKEiIbkAkmtViMvLw9qtReXHCL0ffmAvqt790Ds1CaEPBgk10IihIgXBRIhRDQokAghokGBRAgRDQokQohoSC6Qtm7disTERGg0GqSnp+PUqVP+rpLfffvtt5g9ezbi4uLAcZxwiSo3xhhyc3MRGxuLgIAA6HQ6XL582T+V9bP8/HxMnjwZwcHBiIqKwty5c1FRUeFRxmw2Y/Xq1YiIiEBQUBDmz5/fYc540jlJBVJhYSGys7ORl5eH06dPIzk5GZmZmaitrfV31fyqpaUFycnJ2Lp1a6fPb9y4ER9//DG2b9+OkydPIjAwEJmZmTCbzQNcU/87duwYVq9ejRMnTuDw4cOw2WyYMWMGWlpahDKvvfYa/v73v+PAgQM4duwYqqur8eyzz/qx1veRfp23UmTS0tLY6tWrhccOh4PFxcWx/Px8P9ZKXACwgwcPCo+dTieLiYlhmzZtEtY1NDQwtVrNPv30Uz/UUFxqa2sZAHbs2DHGGP/dKJVKduDAAaHMxYsXGQBWUlLir2reNyTTQnJfdbf9FXJ7uuou6fnKwlLX2NgIABg0aBAAoLS0FDabzeP7GjVqFB5++GH6vrwgmUDq7qq7XV0hl/TuysJS4XQ68eqrr+LRRx/FuHHjAPDfl0qlQlhYmEdZ+r6880BMP0KIP6xevRrnz5/Hd9995++qPDAk00LqzVV3Se+uLCwFWVlZ+OKLL3D06FGPubhiYmJgtVrR0NDgUV7q35e3JBNIvbnqLqErC9+NMYasrCwcPHgQR44cwdChQz2eT01NhVKp9Pi+KioqUFVVJcnvy2f+3qs+kHq66q5UNTU1sTNnzrAzZ84wAOzDDz9kZ86cYdeuXWOM3fuVhR8kq1atYqGhoay4uJjdunVLWEwmk1Dm5ZdfZg8//DA7cuQI+/HHH1lGRgbLyMjwY63vH5IKJMa6v+quVB09epQB6LC4r0B8r1cWfpB09j0BYLt27RLKtLa2sn/7t39j4eHhTKvVsnnz5rFbt275r9L3EZoPiRAiGpLZh0QIET8KJEKIaFAgEUJEgwKJECIaFEiEENGgQCKEiAYFEiFENCiQCCGiQYFECBENCiRCiGhQIBFCROP/A20RukO8001nAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:32:10.685003Z", + "iopub.status.busy": "2024-02-29T05:32:10.684611Z", + "iopub.status.idle": "2024-02-29T05:34:04.939597Z", + "shell.execute_reply": "2024-02-29T05:34:04.938519Z" + }, + "papermill": { + "duration": 114.280164, + "end_time": "2024-02-29T05:34:04.942172", + "exception": false, + "start_time": "2024-02-29T05:32:10.662008", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:34:04.987806Z", + "iopub.status.busy": "2024-02-29T05:34:04.987011Z", + "iopub.status.idle": "2024-02-29T05:34:05.008253Z", + "shell.execute_reply": "2024-02-29T05:34:05.007293Z" + }, + "papermill": { + "duration": 0.046226, + "end_time": "2024-02-29T05:34:05.010204", + "exception": false, + "start_time": "2024-02-29T05:34:04.963978", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0683190.0739650.002344.4347470.1316971.3246770.3612380.0000152.4250930.0356080.0666060.0483770.0665450.0213886.85984
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.068319 0.073965 0.00234 4.434747 0.131697 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 1.324677 0.361238 0.000015 2.425093 0.035608 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.066606 0.048377 0.066545 0.021388 6.85984 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:34:05.052398Z", + "iopub.status.busy": "2024-02-29T05:34:05.052094Z", + "iopub.status.idle": "2024-02-29T05:34:05.621766Z", + "shell.execute_reply": "2024-02-29T05:34:05.620695Z" + }, + "papermill": { + "duration": 0.594222, + "end_time": "2024-02-29T05:34:05.624883", + "exception": false, + "start_time": "2024-02-29T05:34:05.030661", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:34:05.672499Z", + "iopub.status.busy": "2024-02-29T05:34:05.672189Z", + "iopub.status.idle": "2024-02-29T05:36:07.804471Z", + "shell.execute_reply": "2024-02-29T05:36:07.803670Z" + }, + "papermill": { + "duration": 122.15973, + "end_time": "2024-02-29T05:36:07.806970", + "exception": false, + "start_time": "2024-02-29T05:34:05.647240", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:36:07.853517Z", + "iopub.status.busy": "2024-02-29T05:36:07.853197Z", + "iopub.status.idle": "2024-02-29T05:36:07.870127Z", + "shell.execute_reply": "2024-02-29T05:36:07.869295Z" + }, + "papermill": { + "duration": 0.042527, + "end_time": "2024-02-29T05:36:07.871962", + "exception": false, + "start_time": "2024-02-29T05:36:07.829435", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:36:07.914664Z", + "iopub.status.busy": "2024-02-29T05:36:07.914365Z", + "iopub.status.idle": "2024-02-29T05:36:07.919622Z", + "shell.execute_reply": "2024-02-29T05:36:07.918622Z" + }, + "papermill": { + "duration": 0.028815, + "end_time": "2024-02-29T05:36:07.921727", + "exception": false, + "start_time": "2024-02-29T05:36:07.892912", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.5567177837355095}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:36:07.965945Z", + "iopub.status.busy": "2024-02-29T05:36:07.965677Z", + "iopub.status.idle": "2024-02-29T05:36:08.337152Z", + "shell.execute_reply": "2024-02-29T05:36:08.336249Z" + }, + "papermill": { + "duration": 0.396392, + "end_time": "2024-02-29T05:36:08.339358", + "exception": false, + "start_time": "2024-02-29T05:36:07.942966", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDfUlEQVR4nO3dd3hb9b0/8PfRlrW8Z+zYcZyEELITCCFkssIFUloaCoW4aUhbnFLI5T7U7dOEUUjoBRrKpWlLwYYfIxQaRtmjJClNE0IGCSQ4yyse8ZRsWdY8398fR5ItW7YlWdKRrM/refRIOufo6Hts6aPv/nKMMQZCCBGRROwEEEIIBSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUiQojoKBCRsNq7dy/uv/9+GI1GsZNC4ggFIhJWe/fuxQMPPECBiASFAhEhRHQUiEjY3H///fif//kfAEBRURE4jgPHcdBqtVi6dOmg43meR15eHr73ve95tz322GO49NJLkZaWBrVajTlz5uD111/3+34vvvgi5syZA7VajdTUVNx8882or6+PzMWRiOJoGhASLkePHsXWrVvxyiuv4Pe//z3S09MBAGfOnMGDDz6IhoYGZGdne4/fs2cPFi9ejNdee80bjPLz83H99ddj6tSpsNvt2LFjB7744gu88847uPbaa72vffjhh/Gb3/wG3//+97F48WK0trbiqaeeglarxeHDh5GcnBzVayejxAgJo//93/9lAFh1dbV3W1VVFQPAnnrqKZ9j77zzTqbVapnFYvFu6/+YMcbsdjubNm0aW7ZsmXdbTU0Nk0ql7OGHH/Y59tixY0wmkw3aTmIfFc1IxE2aNAkzZ87Eq6++6t3mcrnw+uuv47rrroNarfZu7/+4s7MTJpMJixYtwqFDh7zbd+7cCZ7n8f3vfx9tbW3eW3Z2NkpKSvDZZ59F58JI2MjETgBJDKtXr8avfvUrNDQ0IC8vD7t27UJLSwtWr17tc9w777yD3/72tzhy5AhsNpt3O8dx3senTp0CYwwlJSV+30sul0fmIkjEUCAiUbF69WqUl5fjtddew913342//e1vMBgMuPrqq73H/Otf/8L111+Pyy+/HH/84x+Rk5MDuVyOiooKvPzyy97jeJ4Hx3F4//33IZVKB72XVquNyjWR8KFARMKqf86lv6KiIsyfPx+vvvoqNmzYgJ07d2LVqlVQKpXeY/7+979DpVLhww8/9NleUVHhc67i4mIwxlBUVIRJkyZF5kJIVFEdEQkrjUYDAH47NK5evRr79u3Dc889h7a2tkHFMqlUCo7j4HK5vNtqamrw5ptv+hx34403QiqV4oEHHgAb0OjLGEN7e3t4LoZEDTXfk7A6cOAA5s+fj5UrV+Lmm2+GXC7HddddB41Gg3PnzqGgoABarRZyuRzNzc0+9Tn//Oc/sXz5cixatAi33HILWlpa8PTTTyM7OxtHjx71CTpbt25FeXk5Lr30UqxatQo6nQ7V1dV44403sH79etx7771iXD4JlYgtdmSMeuihh1heXh6TSCSDmvIXLlzIALB169b5fe2zzz7LSkpKmFKpZFOmTGEVFRVs8+bNzN9H9e9//zu77LLLmEajYRqNhk2ZMoWVlZWxqqqqSF0aiRDKERFCREd1RIQQ0VEgIoSIjgIRIUR0FIgIIaITNRAVFhZ6p4rofysrKxMzWYSQKBO1Z/WBAwd8Oq99/fXXuOKKK3DTTTeJmCpCSLTFVPP93XffjXfeeQenTp0acqhAfzzPo7GxETqdLqDjCSHRxRhDd3c3cnNzIZEMXQCLmbFmdrsdL774IjZu3DhkULHZbD4jshsaGjB16tRoJZEQEqL6+nqMGzduyP0xE4jefPNNGI1GlJaWDnnMli1b8MADDwzaXl9fD71eH8HUEUJC0dXVhfz8fOh0umGPi5mi2VVXXQWFQoF//OMfQx4zMEfkuUiTyUSBiJAY1NXVBYPBMOJ3NCZyRLW1tfjkk0+wc+fOYY9TKpU+00MQQsaGmOhHVFFRgczMTJ/J0QkhiUP0QMTzPCoqKrBmzRrIZDGRQSOERJno3/xPPvkEdXV1WLt2bUTOzxiD0+n06a9EAiOVSiGTyahrBIk40QPRlVdeOWiWvXCx2+1oamqCxWKJyPkTQVJSEnJycqBQKMROChnDRA9EkcLzPKqrqyGVSpGbmwuFQkG/7EFgjMFut6O1tRXV1dUoKSkZtkMaIaMxZgOR3W4Hz/PIz89HUlKS2MmJS2q1GnK5HLW1tbDb7VCpVGInKWIO1nbg1HkzitI1mFeYComEfrSiacz/xNGv+Ogkyt8vPyUJzV1W7D3Tjt0nW8VOTsJJjE8ZISPI1Kuw4oIsAMCReiOaTVaRU5RYKBCRhPVtc5dPwJmWZ8AFOULv30N1nWIlKyFRIEpwhYWF2LZtm9jJiDqrw4XPvm3FjgN1aDL1erfPHp8MADjdYobVQV0+ooUCEUlIh+uMsDpcSNUokKXrq4TP1KmQrlXAxTPUtPeImMLEQoFoDLDb7WInIa5YHS5v0euSCWmDWsiK0rUAgJYu26DXkshIyEBkd/JD3pwuPuBjHQEcG4olS5Zgw4YN2LBhAwwGA9LT0/Gb3/zG2/GzsLAQDz30EG6//Xbo9XqsX78eAPD5559j0aJFUKvVyM/Px1133YWenr5f9ZaWFlx33XVQq9UoKirCSy+9FFL64t03jSbYnTzStQqUZGoH7Z+Rb8DahUW4fFKGCKlLTGO2H9Fwnv7s9JD7itI1WDUrz/v8L3vOwOHy3/N7XIoaN83N9z5/7t/V6LX71ivcc8WkkNL4/PPP48c//jG++OILfPnll1i/fj0KCgpwxx13AAAee+wxbNq0CZs3bwYAnDlzBldffTV++9vf4rnnnkNra6s3mFVUVAAASktL0djYiM8++wxyuRx33XUXWlpaQkpfvOJ5hq/qTQCAmfkpfju56lTyQdtIZCVkIIoH+fn5+P3vfw+O4zB58mQcO3YMv//9772BaNmyZfjv//5v7/Hr1q3DrbfeirvvvhsAUFJSgj/84Q9YvHgxtm/fjrq6Orz//vv44osvMG/ePADAs88+iwsuuCDq1yam6vYemHodUMmlmJIz/GRdJHoSMhCVLZ045L6BHWrXX1485LEDf0zXLiwaTbJ8XHLJJT6/1gsWLMDjjz/uHbw7d+5cn+O/+uorHD161Ke4xRjzDnU5efIkZDIZ5syZ490/ZcoUJCcnhy3N8cDpYtCpZJicrYNcOnTNRF27BYfrO5GhVeLSielRTGFiSshApJAFXjUWqWNHS6PR+Dw3m834yU9+grvuumvQsQUFBTh58mS0khbTJmfrMDFTCyc/fP2dxeHE2dYeasKPkoQMRPFg//79Ps/37duHkpISSKVSv8fPnj0bx48fx8SJ/nN7U6ZMgdPpxMGDB71Fs6qqKhiNxrCmOx5IJRykEv9/R480jTATaJvZDsYYDZiOsIRsNYsHdXV12LhxI6qqqvDKK6/gqaeewi9+8Yshj7/vvvuwd+9ebNiwAUeOHMGpU6fw1ltvYcOGDQCAyZMn4+qrr8ZPfvIT7N+/HwcPHsS6deugVqujdUmiYoyhrt0CFx/YlDOpGgUkHAe7k0e3zRnh1BEKRDHq9ttvR29vL+bPn4+ysjL84he/8DbT+zN9+nTs3r0bJ0+exKJFizBr1ixs2rQJubm53mMqKiqQm5uLxYsX48Ybb8T69euRmZkZjcsRXWu3DX8/dA6Ve2sCCkZSCYcUjdB61mGmflqRRkWzGCWXy7Ft2zZs37590L6amhq/r5k3bx4++uijIc+ZnZ2Nd955x2fbbbfdNqp0xouT580AgGy9CtIAp/hI0yjRbrajvceGwnTNyC8gIaMcEUkIZ1qFQDTRTwfGoaRphVkp2yhHFHEUiMiY19FjR0ePHVIJh/FpgU+Sl5KkiGpLaCKjolkM2rVrl9hJGFPOunND41LUUMmHby3rryRTi0lZWmoxiwIKRGTM8xTLijMCL5YBoOlio0j0fGdDQwN++MMfIi0tDWq1GhdddBG+/PJLsZNFxgib04VmkzCKniqcY5eoOaLOzk4sXLgQS5cuxfvvv4+MjAycOnUKKSkpYiaLjCFyiQSr5+WjucsKgzqIwawuB3Dmnzhz4jAa+BQUXHIjCnNoNH6kiBqIHn30UeTn53tHhwNAUVH4xmsRIpFwyDaokG0IYgUSxoDjbwFtp+Dq7Ya8pw3cNzuB7PWDBxiSsBC1aPb2229j7ty5uOmmm5CZmYlZs2bhmWeeGfJ4m82Grq4unxshYXf+a6DtFCCRwlK4Ai5ODr6zXthOIkLUQHT27Fls374dJSUl+PDDD/Gzn/0Md911F55//nm/x2/ZsgUGg8F7y8/P93scIQDQa3fh4+PnUdXcHfhqwrwLqP6X8LhwESTjZqNRPwM2pws4R3WXkSJqIOJ5HrNnz8YjjzyCWbNmYf369bjjjjvwpz/9ye/x5eXlMJlM3lt9fX2UU0ziSXOXFV83mLDvbHvgTfAtxwGrCVBqgXFzoVXKcF47FTaeA7qbgZ62yCY6QYkaiHJycjB16lSfbRdccAHq6ur8Hq9UKqHX631uhAzFszpHlj6I+qHGw8J93hxAKodWJYNTqkKbLEfY3loV5lQSQORAtHDhQlRV+f5jT548ifHjx4uUIjKWnO8S1izLCbSi2twKmBoATgJkTwcA6JRCS1uzogA8Y0DH2YikNdGJGojuuece7Nu3D4888ghOnz6Nl19+GX/5y19QVlYWmTdkDHDao38LtH4CwAsvvIC0tDTYbL4rSKxatSphBqiGA2PM238o4Bazpq+E+/SJQtEMgEougVYpgyy1EE6eAd1NQtM+CStRm+/nzZuHN954A+Xl5XjwwQdRVFSEbdu24dZbb43MG7ocwL8ej8y5h7PovwGZIqBDb7rpJtx11114++23cdNNNwEQVt949913hx1ZT3wZLQ5YHS7IJBzStcqRX8AY0Pqt8NidGwIAjuNwx+UThP3/2QPYzEBXA5BSGJmEJyjRe1b/13/9F44dOwar1YoTJ054J4dPVGq1GrfccotP36oXX3wRBQUFWLJkiXgJizPnu4ViWaZeGdi0H10NgK1b+MFI8dOXjeMAg7uV1kiNJOGWWGPNpHIhdyLG+wbhjjvuwLx589DQ0IC8vDxUVlaitLSUBl8GobNHKD5l6ALIDQF9uaG0EkA6xNdCnwe0nADM58OQQtJfYgUijgu4iCSmWbNmYcaMGXjhhRdw5ZVX4ptvvsG7774rdrLiyoLiNMzINyCgmWEZ62sNy5gyaPc3jSYcqu3EFJUC8wChGZ+EVWIFojiybt06bNu2DQ0NDVixYgV13gxBkiLAj3dXI2DtEnKuqYOLZU4XQ5vZjvNKg7DB1g3YewAFDaINF9HriIh/t9xyC86dO4dnnnkGa9euFTs5Y1ubOzeUNtFvMVqrEgJat1MCJKUKG6l4FlYUiGKUwWDAd7/7XWi1WqxatUrs5MSVZpMVbxw+hy9rOgJ7QfsZ4T7d//LgSQphMrUemxPQuhcboB7WYUWBKIY1NDTg1ltvhVIZYIUrASAM7ahps6DB2DvywZYOIahwEiB1gt9DPEW8XrsLTO3OEVnaw5VcAqojikmdnZ3YtWsXdu3ahT/+8Y9iJyfutJuFjowB9R/y5IaS8wG5/46PGneOyMkz2JWpUAKUIwozCkQxaNasWejs7MSjjz6KyZMni52cuNPRI6y6kaoJoIW0/bRwn+Z/hVwAkEklUMolsDl4WGQGIRBRjiisKBDFoKHWLSOBMVqEPkQpSSMEIqcNMLoHWA8TiAAgTaOA3cnDoUwWNjh6AbsFUAS+KggZGgUiMqbYnTzM7iWik5NG6EjacRZgvNAS5mkNG8LqeQV9T1R6obnf0k6BKEzGfGV1wBNiEb/i7e9n7BWKZWqFdOSlgwIolvmVlC7cU/EsbMZsIJLLhV9Di8Uickrim+fv5/l7xrpeuwsKmQQpI+WGeL6vojrYQKR2L+7Q2xl8AolfY7ZoJpVKkZycjJaWFgBAUlISjdUKAmMMFosFLS0tSE5OhlQa+MKEYhqfpsGdS4rhcI2Qk+tuFOp5ZErAMG7E837b3IUD1R3IT03CErW7h7XVFIYUE2AMByIAyM7OBgBvMCLBS05O9v4d4wXHcVDIRvjRaTsl3KdOACQjB1nPMA+dSg6kJgsbrcZRpZP0GdOBiOM45OTkIDMzEw4HTWYVLLlcHjc5oaAFWT+kUQpflR67E1BRjijcxnQg8pBKpWP3C0V8vHqgDiq5FMsvyIJWOcTHu9fo7k3NAWnFAZ3XM8zDYnMBqmRho90izMAZBzM6xLoxW1lNEo/V4UKj0YqzrT2QS4cpmnkqqQ3jALk6oHN7A5HdBSZT9vXCplxRWFAgImNGV69Q/E5SSKGUDZMDbnfXDwXRWuYZb8YzBpuT71c8M4aSVDIABSIyZnRZhUCkH26N+yB6U/cnlXBQyISvi8Xer3hGOaKwSIg6IpIYuqxCj2q9aphA1FkjrOaqTgGS0oI6f5pGAYeLh4tnfTmiXmNoiSU+RM0R3X///eA4zuc2ZcrgqToJCYSnaKZXD/P72r+1LMh+ZTfPL8BtCwqFebDVycJGKpqFheg5ogsvvBCffPKJ97lMJnqSSJwaMUfEWL9AFFhr2ZD6TxtLRk30b71MJou7DnMkNkk5oR5HpxriY93VKDS5yxRAcoH/YwKl1An3FIjCQvRAdOrUKeTm5kKlUmHBggXYsmULCgr8f0hsNpvPCqhdXV3RSiaJA9dOzxl+kK4nNxRgb+qBvqo34qtzRkzK0uGSce5AZO8R6pxCOB/pI2od0cUXX4zKykp88MEH2L59O6qrq7Fo0SJ0d/v/ldmyZQsMBoP3RitbkIE8dY1+hTra3s3h4tFutgvzHcnVgMT9O065olETNRBdc801uOmmmzB9+nRcddVVeO+992A0GvG3v/3N7/Hl5eUwmUzeW309rbhJAmTrBszuMYdDzE09Es+0IlaHS6jopuJZ2IheNOsvOTkZkyZNwunTp/3uVyqVNJE88au2vQd7TrWhIDUJiydlDD6g46xwr8sOeT0ytbt3da/DJWxQ6oSpQCgQjVpMdWg0m804c+YMcnJyxE4KiTOdFgfaum3eJvxBPIFoFK1laneOqNfeLxABFIjCQNRAdO+992L37t2oqanB3r178Z3vfAdSqRQ/+MEPxEwWiUN9fYj8NN3zPNBRLTwOsVgG9AtEDgpE4SZq0ezcuXP4wQ9+gPb2dmRkZOCyyy7Dvn37kJHhJ2tNyDC8wzv8Nd13NQhDO+QqQJcb8nt4imZ2Jw+ni4dMqRd22Kj1drREDUQ7duwQ8+3JGGJ2d2b024fIUyxLKQIkoRcClO4+Skq5FA4Xg8ybI6JANFoxVVlNSKg8K3dolX6KZp5ANIpiGSB0DVi3qN85qGgWNjFVWU1IKHieoccm1NtolAM6FtrMQHez8HiUgWgQ5YBOjSRkFIhI3LO7eKRo5FDJpdAoBmTyO92V1LosQKkN7xsrNAAnEcaw2c3hPXeCoaIZiXsquRS3Lyj0v9NbLBvlIFe3vWfacLrFjNkFKZiWZxCCm7VLyHl5pgYhQaMcERm7GOvXbF8UllP22l1oN9vR7a4ch8Kdy6Ic0ahQICJjV0+rsHaZVA7o88JySu8wD6e7TsjTS5sC0ahQ0YzEvS+qO1B1vhvT8wyYkZ/ct6OzRrhPLgjb6HiVXPjttnk6NXpzRD1hOX+iohwRiXsdPXa0ddtgd/G+Ozprhfvk8WF7L9Wg3tXuQGSjHNFoUCAica+vD1G/DD7vAozuQJQS/kBkdbiDnrdoRjmi0aBAROJej79A1N0EuNzzBmmzwvZePlOBAFRZHSYUiEhcY4z5zxH1rx8KcpL84ajlUmiVsr7+ShSIwoIqq0lcszl52J1CMUnjE4g8xbLCsL5fqkaBOy7v10PbWzSzCN0Fwhj0EgnliEhc8xTLlHKJdwFEuBzCiHsg7IFoEIVGCD6MBxyWyL7XGEaBiMQ1F8+QrlUgTaPo22iqFyqrlTphIcVIkkiFeiiAWs5GgYpmJK5l6lW4beDwjv7FsggUld452oiOHjuuvjAbmXqVkCuyW9z1ROGrGE8klCMiY4/JvajCaNcuG+r0vQ60m+3osVOnxnChQETGFpejb9oPw7iIvIVKRk344UZFMxLXPvi6Ca1mOy6bmI6idI2wmivvEno8R6h+aHBfIurUOFqUIyJxrd09vMO7wqunWGYYF7GmdM94s77e1ZQjGi0KRCSuDepVbTon3BsiUz8E+BmBT+PNRi1mAtHWrVvBcRzuvvtusZNC4oSr3xSxWpVMWDbIG4giUz8E9AtEdiqahUtIgejs2bNhTcSBAwfw5z//GdOnTw/recnY5hnaIZVwwppj5vNCZbVMCWgityRVkkIY5uHtQElFs1ELKRBNnDgRS5cuxYsvvgir1TqqBJjNZtx666145plnkJIS4c5nZEzxFMs0Shk4jutXP5Q/qmWDRnJBjh53XD4Byy9w9xnyBCKXQ1g/jQQtpP/WoUOHMH36dGzcuBHZ2dn4yU9+gi+++CKkBJSVleHaa6/FihUrRjzWZrOhq6vL50YSlydHpPPWD/WrqI4mmUKYBRKg4lmIQgpEM2fOxJNPPonGxkY899xzaGpqwmWXXYZp06bhiSeeQGtra0Dn2bFjBw4dOoQtW7YEdPyWLVtgMBi8t/z8/FCST8YICQekaxVI0SiEAaee+qFkET4XVDwblVHlX2UyGW688Ua89tprePTRR3H69Gnce++9yM/Px+23346mpqYhX1tfX49f/OIXeOmll6BSqQJ6v/LycphMJu+tvr5+NMkncW5ipg63LSjEFVOzAEuHMMxCIgN0ORF9X6vDhb8dqMcL/6np6zbQfxQ+CdqoAtGXX36JO++8Ezk5OXjiiSdw77334syZM/j444/R2NiIG264YcjXHjx4EC0tLZg9ezZkMhlkMhl2796NP/zhD5DJZHC5Bi9Yp1QqodfrfW6EAABMdcK9Pjds81MPRS6VoMHYi3azHTYnzdQYDiH1rH7iiSdQUVGBqqoqrFy5Ei+88AJWrlwJibuCsKioCJWVlSgsLBzyHMuXL8exY8d8tv3oRz/ClClTcN9990EqjeyHiYwxxujVD0klHBQyCexOHlaHS2jOp6LZqIQUiLZv3461a9eitLQUOTn+s8GZmZl49tlnhzyHTqfDtGnTfLZpNBqkpaUN2k6IPy/trwXPM1xzUQ7So1w/pPQGIsoRhUNIgejjjz9GQUGBNwfkwRhDfX09CgoKoFAosGbNmrAkkpCBGGPoMNvh5BnkTjNgNQnLP4dp/bKRqBVSdFudfat5UCAalZACUXFxMZqampCZmemzvaOjA0VFRX7rdwKxa9eukF5HEo/VwcPJCxXF2l53o4g2U+jMGAU0Aj+8Qqqs9rYUDGA2mwNuASNkNLptDgBCL2dpV/Sb7WkEfngFlSPauHEjAIDjOGzatAlJSUnefS6XC/v378fMmTPDmkBC/PGMMdMoZb49qqNEo5RCp5JB4hnhr3B/Fxw0iX4oggpEhw8fBiDkiI4dOwaFom+eYIVCgRkzZuDee+8NbwoJ8cNsFXpVJ8scQHebsDGKPaqXTM7Eksn9qibk7hwR7wIcvX2BiQQkqED02WefARCa2Z988knqx0NE4xnekep09+JPSusrHolBKgPkKsBhFYpnFIiCElJldUVFRbjTQUhQlHIJ0rUKpPPnhQ1iDOsYSKF1ByIzgMiN/h+LAg5EN954IyorK6HX63HjjTcOe+zOnTtHnTBChjO7IAWzC1KAg7sBG6I+0LXZZMXuky3QKuW4drq7L51CA/S0UYV1CAIORAaDQZhqwf2YENE57f0myo9ujsjFGBqNViQn9euqQi1nIQs4EPUvjlHRjMSE7kZhhVWlDlBF98dRJRswbzXQLxBRX6JghdSPqLe3FxZL3yjj2tpabNu2DR999FHYEkbIUBwuHtt3ncFHew/AxTOhfijKzeWefkQ2pws87xmB7+7USEtPBy2kQHTDDTfghRdeAAAYjUbMnz8fjz/+OG644QZs3749rAkkZKAemxNWhwuc6ZwwEWO0J0JDXyBiDDQCPwxCnqFx0aJFAIDXX38d2dnZqK2txQsvvIA//OEPYU0gIQOZbU5wzIVUVys4cBFdsWMonhH4gL/e1VQ0C1ZIgchisUCn0wEAPvroI9x4442QSCS45JJLUFtbG9YEEjKQ2eZEkr0dKgkv9N3RpIuSjkHLCtHS0yELefL8N998E/X19fjwww9x5ZVXAgBaWlqokyOJuB6bE3pbk5AjMUS/fshD6x7m4eIHzNLo6BV6WJOAhdShcdOmTbjllltwzz33YPny5ViwYAEAIXc0a9assCaQkIG6rU7obOehUEpEqR/yWD1vQJFQphamImG8UGGt1ImTsDgUUiD63ve+h8suuwxNTU2YMWOGd/vy5cvxne98J2yJI8Qfs9UBg60ZiiRJ1PsPDUsiEYZ22MxC8YwCUcBCCkQAkJ2djezsbJ9t8+fPH3WCCBlJKtcNjcwBpVIP6LJHfkE0yfsFIhKwkAJRT08Ptm7dik8//RQtLS3ged5nf7hXgiWkv0vTLMC4ZCBlfMQnyh/OiaYuHDtnQmG6BvOLUoWNCi2AFmo5C1JIgWjdunXYvXs3brvtNuTk5HiHfhASFVFY3z4QFrsLDcZe6FT9vkbUlygkIQWi999/H++++y4WLlwY7vQQMizGGDhvIBK3fkgld/cjctJ4s9EKKRClpKQgNTU13GkhZESN55vR+G01tGoFLojSRPlD6Zsutv94M5q7OhQh9SN66KGHsGnTJp/xZoREg62lGk6eoUeZKaw5L6JB81YDlCMKUUg5oscffxxnzpxBVlYWCgsLIZfLffYfOnQooPNs374d27dvR01NDQDgwgsvxKZNm3DNNdeEkiySAJzt1cKD5OgP6xhI7Q5EvRSIRi2kQLRq1aqwvPm4ceOwdetWlJSUgDGG559/HjfccAMOHz6MCy+8MCzvQcYQxgCjMIRIklIkcmL66ohsDh48zyCRcDTMI0QhBaLNmzeH5c2vu+46n+cPP/wwtm/fjn379vkNRDabDTabzfu8q6srLOkgcaK3Ey5rN3hOCkWauC1mAKCUSSGXclDJpbC7eKgk0r4ckdMGuByAVD78SQiAEOuIAGH6j7/+9a8oLy9HR0cHAKFI1tDQENL5XC4XduzYgZ6eHu+QkYG2bNkCg8HgveXnx1CvWjKkodbBC1pnDWwOHmZFJvRJ4k9OL5Vw2LCsBOsWTfDWF0GmBCTu33fKFQUspEB09OhRTJo0CY8++igee+wxGI1GAMJc1eXl5UGd69ixY9BqtVAqlfjpT3+KN954A1OnTvV7bHl5OUwmk/dWX18fSvJJFNW1W/D2V41wuPiRDx4B66yF3emCSZULvTrkQQGRxXFUTxSCkALRxo0bUVpailOnTvms7Lpy5Urs2bMnqHNNnjwZR44cwf79+/Gzn/0Ma9aswfHjx/0eq1QqodfrfW4kdtmdPD463oyzrT04XGcEAJgsDhw9Zwz+ZIyB76yFXi2HNLUQWmWMBiKAAlEIQvpvHjhwAH/+858Hbc/Ly0Nzc3NQ51IoFJg4cSIAYM6cOThw4ACefPJJv+cn8eWbRhO6rU7o1XLMKkiG2ebEi/trwfMMEzK0wQWTnlZInb2YkpeKKZfNFwaYxoC9Z9pQ32HB3MJUFGe4K6ppgrSghfTfVCqVfiuKT548iYyM0a3nxPO8T4U0iU+MMRypNwIA5o5PgVwqgUYhRYZWCSfPcMSdQwqYsU64NxSIOr5sIKPFgUajFaZeR99GyhEFLaRAdP311+PBBx+EwyH88TmOQ11dHe677z5897vfDfg85eXl2LNnD2pqanDs2DGUl5dj165duPXWW0NJFokhTSYrjBYHFDIJLsgRitAcx2FWQTIAYcCod9L5QHRUg2cMLAb6D/XnHeZBfYlGJaRA9Pjjj8NsNiMjIwO9vb1YvHgxJk6cCJ1Oh4cffjjg87S0tOD222/H5MmTsXz5chw4cAAffvghrrjiilCSRWLIyfPdAIDiDK13bmcAmJChhVohhdnmRG1HgD3zXU7AWIPqth5UfCsJrY4pQlQy92oetKzQqIRUR2QwGPDxxx/j3//+N7766iuYzWbMnj0bK1asCOo8zz77bChvT2IcYwynzgtfwpIsrc8+qYTD5CwdjtQbcep8N4rSA1iv3lQHuJwwMxVMXDLk0tioHwIApd/e1dSpMVhBByKe51FZWYmdO3eipqYGHMehqKgI2dnZwshomhIk4dmcPCZmatHcZcX41MH9fSZmanGk3oizbT19PZKH0yHMb3VengtwHJKTYqeToJrGm4VFUIGIMYbrr78e7733HmbMmIGLLroIjDGcOHECpaWl2LlzJ958880IJZXEC5VciqVTMofcn5eshkouhdPFo9NiR5pWOfwJO6rhYgzN0lwAQLJa3MGu/fXVEfkrmvUIw1Lox3lEQQWiyspK7NmzB59++imWLl3qs++f//wnVq1ahRdeeAG33357WBNJxhaJhMN3Z+chRaMYuZjVawR62mB18jAq86CUS7xf/ligkgvDPHwuw1M0453CUA+5yu9rSZ+g/qOvvPIKfvWrXw0KQgCwbNky/PKXv8RLL70UtsSR+OPiGc51WvqW2BlCpl4VWF1PpzDavkeZBZdUhWS1IqaK/zkGFTYsK/Fd0UMq75uihIpnAQkqEB09ehRXX331kPuvueYafPXVV6NOFIlf57useO3Lc6j4d3XAY8yGPa79DACgQylMghZL9UMAhg6KnlyRgwJRIIIKRB0dHcjKyhpyf1ZWFjo7O0edKBK/znX2AgCy9KoRcy7fNJrw//5Tg0N1Q3xmXA5vjkiWORlF6RrkJavDmt6IoQrroARVR+RyuSCTDf0SqVQKp9M56kSR+FXv7huU76e1bCC7k0eb2Y6zrT2YM97P1MMdZ4U+RCoDJhZNwMQYKpL19/6xJnRZHbj6whwYPDk2CkRBCbrVrLS0FEql/1YOGpqR2JwuHo1GIUc0LmXknMuEdC12VbWi0WiF1eHqm0rDo+2kcJ8xKaZbnppMwhCPHruzXyCiuauDEVQgWrNmzYjHUItZ4mrussLJMyQppEjTjNzEbkiSI02rQLvZjtp2CyZn91sZleeB9tMAAEfKRDjsTiQpYnPEvUouhanXQX2JRiGo/2xFRUWk0kHGgPoOITeUn5oUcMtWUboG7WY7qtvMvoHIVAc4rIAiCbXOVPxj91nkpybhe3PEn5lxIL99ieTuoikFooDETocMEvfOdQr1Q4EUyzw8Qzyq2yy+g2Bbq4T7tIlo6bYDgO9ChjHE/yT6VDQLBgUiEjaXT8rAZSXpGJ8WwPgxt1yD0Mva6nCh0STkqMC7gJYTwuOMKTjfbQUAZOtjs2Ogp27LRkWzkMXmTwyJS1l6FbKCDBYSCYcpOTrYHHzfKP3OGsDRCyiSwFIK0XysBgCQbYjNQKQcdsVXCw3zCAAFIiK6pZMHjEs7/41wnzkVxl4XrA4XpBIuoApwMXiGefjwBCLGAw5L33PiFwUiEhYHajqgU8lQlK6BUjaKGRSd9r5m+8ypqHfXO+UYVJDF0PQf/c3KT8bsghTfjRIpIFcLOTt7DwWiEcTmf5bEFZvThb2n2/H+sWbflqMgMMbQ0mVFa+3XQo9qdTKgz0Wdu4NkQQAdJMUy9DAPmiAtUJQjIqPW0NkLnjEY1HIY1KGNBTvWYMKnJ1owv+tzpKcycFkXAhyHC3MNSFJIA5tALdYotEBPG1VYB4ByRGTU6t3jy0aTa5mUpYOON0FirEO3zQXkzAAgNO8vm5KFzBhtMQOESdHeOtKAVw/U+Q7gpZazgFEgIqPmLT6lhR6IVHIpZsmFlTpOWNPBlPGzZp1UwuFsaw8ajVbYnENMkEaGRYGIjEqPzYm2bmGMYTAdGQdxOTFVUgMJB5zgivHawXP48JtmGC32MKU0cuRSibfrQa+d5q4OhaiBaMuWLZg3bx50Oh0yMzOxatUqVFVViZkkEiRPq1aGTjm6sWAtx6FmdozLzkKnugANnb043tiFqubuMKU0svz3rqYcUaBEDUS7d+9GWVkZ9u3bh48//hgOhwNXXnklenroHxcv2s1CjmX8KIplYAyo3w8AyJu2CP81Iw9Tc/W4YmoW5hf5mR4kBqkVQiCy2P0FImo1G4morWYffPCBz/PKykpkZmbi4MGDuPzyy0VKFQnGwonpuGicAaPqN9xxVmhdksqBnJkokatQkqUb+XUxJMkdiKhoFpqYar43mUwAgNRU/7+CNpvNZ84jf8tek+jTq0Y5fas7N4TcmXE70bxquKKZo1cYPxdDS2XHmpiprOZ5HnfffTcWLlyIadOm+T1my5YtMBgM3lt+fn6UU0n6G2mC/IB01gCdtcKXdNy80Z9PJEkKYZiHz99ErgYk7t96G/1oDidmAlFZWRm+/vpr7NixY8hjysvLYTKZvLf6+vooppD0xxhD5d4avHm4Ad1WR6gnAar3CI9zZgIqQ9jSF20Li9OxYVkJFhSn9W3kOEDpLmLa4qPSXSwxUTTbsGED3nnnHezZswfjxg098ZVSqRxymloSXec6e9HV64DN6fK2GAWt/TRgagCkMmD8gvAmMMqGXK1WqQN6OwEr5YiGI2ogYozh5z//Od544w3s2rULRUVFYiaHBOFbd7N6SaYutMGoLidw+lPhcd6cvpzDWEM5ooCIGojKysrw8ssv46233oJOp0NzczMAwGAwQK2Ok2VjEpDDxeNUi/DFmpIdYgCp3y/kFBQaoODSMKZOHKZeB3ZVtQAAbpiZ17dD5e4hToFoWKIGou3btwMAlixZ4rO9oqICpaWl0U8QCci3Td2wOXgY1PLQelObW4HavcLjicvjtqVsoLOtPZBKODDG+kbke3NEVDQbjuhFMxJfGGM4cs4IAJiRnxz88s8uJ3DiLWFd+NQJQObU8CdSBJ5+RC6ewebk+5ZGUror4CkQDStmWs1IfGgw9qKt2wa5lMOFuUEOTGUMOPWhkCNSJAFTrh0zU6j2H2/ms6yQJ0dEldXDiolWMxI/MnUqrLggC73+FkQcjqepvumoEHym/Beg1EYuoSJQyaWwO3lY7C4ke0a8eOqIHL3ChG/SUXb+HKMoEJGgKGQSXDQuyP4+Lidw5lOg4ZDwvORKIK04/IkTWZJCiq5eh+94M5lK6J7gcgoV1knxMXYu2igQkcjhXUJfoeo9wlgyjgMmrgDyZoudsohI8g58dfZt5DihnsjSLtQTUSDyiwIRCYjDxeONQw2YkqPDhbkGSD0d+Hhe+JL1tAiDOx0W4Zff1g10NQrFEUCoE5q8EkgvEe8iIkyjkEEhk8A5cOiLUucORNSEPxQKRCQg3zZ1o8HYC7PNiWm5BsDSAZz7Emg9IazdNRRFkjB8Y9w84fEYtmxKJlZMzRq8gyqsR0SBiIyIMYYj9Z0AgJm5akhOfww0HhbW7AKEClhtlvCFU2iE6S+UOkCTLmwfIy1jIxlymAd1ahwRBSIyouYuK9rMdqQ6mnBR00eA3f2FSp0AjJsLpBTSFBfDoU6NI6JAREZ0vMGE7O5jmOs8DLlaI4ySn3wNkEpjA/vzDPPgGcN3ZvUbvK1KFu57jWIkKy5QICLDcjjssH/zDxR2VSEzRw9kTxOa32U0C8JAEk4Y5iHhBgzzUCcL91aT0J8qQYqqwaBARIZmt6D93y8iueskFHIZ9BddLVQ60xfJrySFDBwH8Iyh1+HqW0xAqRf+ZrxTmL96rM40MAo0xIP412sEDr+IJGszDHodpDNWg8ufT0FoGFIJ552byWzr15dIIhWCEUDFsyFQjogMZm4Fju4AbGbok9Mw9fLVQgsYGZFGKYPF7oLF5gL6Z3zUyULRzGoEQFMcD0Q5IuKrpw346mXAZhaCz6zbKAgFQaP0kyMCqMJ6BBSISB9LB/DVK0IHRV0Wagq+g07X2JgrKFo07nohn/FmQL8Ka2NU0xMvKBARgdMGfP13ISekzQB/0Wp8UGVC5d4aNBp7xU5d3NAohWEe/MC5tihHNCyqIyJCk/KJfwjFMqUWmL4arTYpeu0uKGQSZOspVxSoSyak4dLitMETxlGOaFiUIyJAw0Gg7ZSwBte07wJKHeo6hPFj41LUQw9dIINIJZz/WSs9OSKbWZgShPigQJToLB3A2c+Ex8XLAH0uAKDeHYgKUsf2QNWokasBmUJ4TLmiQSgQJTLGgKr3hF/olELvPEFOF4+GTqFeiAJRcFw8w1tHGvDivlrYnP0qrDkOUKcIjy0d4iQuhokaiPbs2YPrrrsOubm54DgOb775ppjJSTwtJwBjvTCD4ORrvJ0Vm0xWOHkGrVKGVI1C5ETGF6mEQ4OxF63dNpitA4pgSe5VYC3t0U9YjBM1EPX09GDGjBl4+umnxUxGYnI5+opkBQv6KlMBb/1Qfqo6+FU6CHQqYV7qLgpEARO11eyaa67BNddcI2YSEte5L4WJupQ6IP9in11zC1OQY1BBraCpPUKhV8nQ1m1Dt9Xhu4MC0ZDiqvneZrPBZrN5n3d10fwuIXE5gHNfCI+LLh+0soRSJsWEjLG1wkY06d05ou7hckQ0Ct9HXFVWb9myBQaDwXvLz6cxOyFpPir0nlbpgawLxU7NmKNTCb/vg3JE6lQh+DhtwvzexCuuAlF5eTlMJpP3Vl9fL3aS4g/PA/Xu3FD+xYNmVjxU14nPT7Wh3Wzz82ISiCHriKQyYVI5gIpnA8RV0UypVEKppAm5RqX1W2GYgVwN5MwYtPvrBhPazXZk6ZVI09LfOhQ6lQxyKQe51E/RKylN+Ptb2oGU8VFPW6yKq0BERokxoO4/wuNxcwfVDZltTrSb7eA4IJ/6D4Usx6BC2dKJ/lscNelA+xmgpzX6CYthogYis9mM06dPe59XV1fjyJEjSE1NRUFBgYgpG6M6zgLmFiEA5c0ZtNvTmzpDpwxuOWniY9guD1r3ckPm89FJTJwQNRB9+eWXWLp0qff5xo0bAQBr1qxBZWWlSKkaw+r2Cfe5M4Wi2QA0rCMKvIGohVrO+hE1EC1ZsgRs4HQJJDJM5wBjnVA5PW7+oN2MMW9HRgpEo3e4rhNfN3bhwlw9Zhek9O1QpwqDi10OoLeTlqB2i6tWMzIKntxQ1oV9C/7102lxoNvqhFTCITd5cG6JBMfm5NHWbUNb94DWR4kE0GYIj6l45kWBKBH0tAnTfHDcoF7UHl29DqgVUuQmqyGX0sditFKShDF6Rotj8E6qJxqEWs0SgSc3lF4y5PzTheka/OTyCbA6+CgmbOxKSRJaJDst9sE7tZnCfTcFIg/66RvrrCbg/DfC4/xLhj2U4zgaXxYmye4ckcXugsU+oGOjTpjzCV0NQoU1oUA05tV/ATAeSC4ADHl+D3G6eGo0CDOFTIJkd66orXtArkibJXShcNqEYjOhQDSm2cxA4xHh8fhLhzzsYG0n/vqvahyu64xOuhJEhk7omd5qtvrukEi8M2HCRMOUAApEY9u5L4RljvW5wgyMQ6hu64HZJrSYkfDJ0quQrlVAJvHzNTOME+67GqKbqBhFldVjlaMXaDgkPB5/6ZAd53psTjR3Cb/YNPVHeM0rTMW8wiH6CXkCkelc9BIUwyhHNFbV7RM6zWkzgLSJQx5W3dYDxoBsgwpaJf0uRY0+D+AkwgDYXioSUyAai6wmYQZGAChaPOwwgjOtZgDAhHRNNFKWkFw8g905oFuETNnXeNBxNvqJijEUiMai6n8JdUPJ+cPmhmxOF+rahWEdVCyLjP+cacf2XadxpN44eGdqsXDfToGIAtFYY2oAzn8tPC5eNmxu6HSLGU6eISVJjnQtrdYRCUq5BA4X879sd+oE4d5YIxSjExgForGEdwnrlDEGZE/rayIeQo5BjdnjUzCzIIVW64iQce5xew3GXvD8gL5a2kxhxkaXE2g/7efViYMC0VhSu1foICdXA8XLRzw8VaPA4kkZmJmfHPm0Jah0rTC3k93Jo6lrQH8ijgOypgqPPb3fExQForGiswao/bfwuOQKQEFTecQCiYRDUbrwvzjTYh58QKZ78YKOswk9oT4ForGgtxM4/rZQJMuZPuLKHA4Xj/ePNeFcp4WGdkRBsbsh4EyrefDfW5sB6HOEYnXjYRFSFxsoEMU7ew9w9G/CvTYDKLlyxJccazDh2+ZufPTNeRpzGQUFaUmQSzkYLQ40mqyDDxg3T7hvOCjUFyUgCkTxrNcIHH4RsHQIlZ7TVw+aEH8gq8OF/Wc7AAgrukpoWEfEKWVSzBmfissnZSBN46d1MmOKMFmd3QKcOxD9BMYACkTxqv0McOh5dxDSAzNuFpaPHsHeM22wOlxI0yowLdcQhYQSAFhQnIY541P8L0ogkQor7gJCPZ818VYwpkAUb+w9QNUH7uKYRSiOzbotoLmPT7d046t6EwBgyaRMyg2JZFAvawDIcne3cDmA428JdUYJJCYC0dNPP43CwkKoVCpcfPHF+OKLL8ROUuyxdQPVe4D9f+6r1MybDcxe43cO6oHq2i14/1gzAGD2+BQUpFGrmhgajL144T81ON44INfDccAF1wEyhTAQ9sTbCRWMRB/l+Oqrr2Ljxo3405/+hIsvvhjbtm3DVVddhaqqKmRmZoqdPHHZLUDHGaC1Smje9XwwtZnAxBVBrRTaYOyFk2eYkKHBoon+p4slkVfXbkG31YmPj58HzxguzNX3dSZNSgWmrgK+/jvQ8q1QRJt0FaDLFjXN0cAxkdtvL774YsybNw//93//BwDgeR75+fn4+c9/jl/+8pfDvrarqwsGgwEmkwl6/ci5gpjDmDBLn9MqTNthNbqXI24DuhqF+p/+DHlCC0v6ZGFyrUGnY7A5eVjsLrR22yCVABMzdd59h+uNuCjPQJPji4jnGT78phnfNncDAPKS1Ziaq0dushp6lQwyqUSo/zv+lvDZAITZNdMnCQFJnQLIk/z+/2NRoN9RUXNEdrsdBw8eRHl5uXebRCLBihUr8J///GfQ8TabDTZb3/IsXV0BVup1VANnPvU/P7DPNjb0tpCPFe6/bjDB5uyX1WY8JC47ODAo5VJckN33TzrR3IVeu3BsryIVRs0EdCYVodecBk21DLdk9n0I3zh8Due7bEJMc/Fw9htGoFPJMCFdC4mEA8dxvutrEVFIJByunpaNFI0CX1R3oMHYiwb3OLSLJ6Ti0uJ0IK0YbVN+iGOfv40U82lw544COOo9BwOHrBQ9clK0gEQKi4Ph68ZuMJ9hOn2PcwyqkdeqC2WIz0XfD6haIBCiBqK2tja4XC5kZWX5bM/KysK333476PgtW7bggQceCP6NnDbALO5a445es/9KSgA8JwOUWkCpB9QpqLfY0OBKhlmRBadUJRzEANicgz4vNgfvDVoeCpkEKUkKZBuUsLt4qCQ0IX4s4TgOl0xIw4W5enzT2IW6dgtauq3QKfu6XtjkOhzRXg6lahZSLWehtzVBY2+HwmUBwOCy9wJ24cPA7E64ekxDv6FMDSgjUCfIwrfii+h1RMEoLy/3LksNCDmi/Pz8kV+YnC80bwP9In+/b7S/X5Ihfl1CPTZnshWu/v83jgNkKkCugkwmB/Qq765Z+TZMd+dsPKfwnGlgS9c103Lg4HlwAGRSCZIUUip6xQmdSo5LJqThkglpYIz5ZK7TNAp8b457Fkdc0LeD8eAcvdDLXYBSAvAuyBx2FEzy01ESABiDVikFkuSDto+aInxzWIkaiNLT0yGVSnH+vO/6TufPn0d29uAKOqVSCaVSGfwbKTRAalGoyQyL7CCm+/FMuh4Iw8APGIlLHMf5/J6p5FLkD1mc8v0wKQHkpUUsaVEh6k+nQqHAnDlz8Omnn3q38TyPTz/9FAsWLBAxZYSQaBK9aLZx40asWbMGc+fOxfz587Ft2zb09PTgRz/6kdhJI4REieiBaPXq1WhtbcWmTZvQ3NyMmTNn4oMPPhhUgU0IGbtE70c0GnHfj4iQMS7Q7yg1rxBCREeBiBAiOgpEhBDRiV5ZPRqe6q2Ah3oQQqLK890cqSo6rgNRd7cwcDCg3tWEENF0d3fDYBh6Ir64bjXjeR6NjY3Q6XQRW5fLM4ykvr4+IVrmEu16AbrmSF4zYwzd3d3Izc2FZJgZA+I6RySRSDBu3LiRDwwDvV6fMB9SIPGuF6BrjpThckIeVFlNCBEdBSJCiOgoEI1AqVRi8+bNoY36j0OJdr0AXXMsiOvKakLI2EA5IkKI6CgQEUJER4GIECI6CkSEENFRIEJwK80+88wzWLRoEVJSUpCSkoIVK1bE3cq0oa6su2PHDnAch1WrVkU2gREQ7DUbjUaUlZUhJycHSqUSkyZNwnvvvRel1IZHsNe8bds2TJ48GWq1Gvn5+bjnnntgtQ4xKX+4sQS3Y8cOplAo2HPPPce++eYbdscdd7Dk5GR2/vx5v8ffcsst7Omnn2aHDx9mJ06cYKWlpcxgMLBz585FOeWhCfZ6Paqrq1leXh5btGgRu+GGG6KT2DAJ9pptNhubO3cuW7lyJfv8889ZdXU127VrFzty5EiUUx66YK/5pZdeYkqlkr300kusurqaffjhhywnJ4fdc889UUlvwgei+fPns7KyMu9zl8vFcnNz2ZYtWwJ6vdPpZDqdjj3//PORSmJYhXK9TqeTXXrppeyvf/0rW7NmTdwFomCvefv27WzChAnMbrdHK4lhF+w1l5WVsWXLlvls27hxI1u4cGFE0+mR0EUzz0qzK1as8G4bbqVZfywWCxwOB1JTUyOVzLAJ9XoffPBBZGZm4sc//nE0khlWoVzz22+/jQULFqCsrAxZWVmYNm0aHnnkEbhcLr/Hx5pQrvnSSy/FwYMHvcW3s2fP4r333sPKlSujkua4HvQ6WsGuNOvPfffdh9zcXJ9/eqwK5Xo///xzPPvsszhy5EgUUhh+oVzz2bNn8c9//hO33nor3nvvPZw+fRp33nknHA4HNm/eHI1kj0oo13zLLbegra0Nl112GRhjcDqd+OlPf4pf/epX0UgyVVaPxtatW7Fjxw688cYbUKlUI78gznR3d+O2227DM888g/T0dLGTEzU8zyMzMxN/+ctfMGfOHKxevRq//vWv8ac//UnspEXMrl278Mgjj+CPf/wjDh06hJ07d+Ldd9/FQw89FJX3T+gcUbArzfb32GOPYevWrfjkk08wffr0SCYzbIK93jNnzqCmpgbXXXeddxvPC+tmy2QyVFVVobi4OLKJHqVQ/sc5OTmQy+WQSqXebRdccAGam5tht9uhUCgimubRCuWaf/Ob3+C2227DunXrAAAXXXQRenp6sH79evz6178edi6hcEjoHFGoK83+7ne/w0MPPYQPPvgAc+fOjUZSwyLY650yZQqOHTuGI0eOeG/XX389li5diiNHjsTFzJih/I8XLlyI06dPe4MuAJw8eRI5OTkxH4SA0K7ZYrEMCjaeQMyiMRw1KlXiMWzHjh1MqVSyyspKdvz4cbZ+/XqWnJzMmpubGWOM3XbbbeyXv/yl9/itW7cyhULBXn/9ddbU1OS9dXd3i3UJQQn2egeKx1azYK+5rq6O6XQ6tmHDBlZVVcXeeecdlpmZyX7729+KdQlBC/aaN2/ezHQ6HXvllVfY2bNn2UcffcSKi4vZ97///aikN+EDEWOMPfXUU6ygoIApFAo2f/58tm/fPu++xYsXszVr1nifjx8/ngEYdNu8eXP0Ex6iYK53oHgMRIwFf8179+5lF198MVMqlWzChAns4YcfZk6nM8qpHp1grtnhcLD777+fFRcXM5VKxfLz89mdd97JOjs7o5JWmgaEECK6hK4jIoTEBgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYjElcrKSiQnJ3uf33///Zg5c6b3eWlpaVxOZZvoKBARv0pLS8FxHH76058O2ldWVgaO41BaWupzfLgDQGFhIbZt2+azbfXq1Th58uSQr3nyySdRWVnpfb5kyRLcfffdYU0XCT8KRGRI+fn52LFjB3p7e73brFYrXn75ZRQUFIiSJrVajczMzCH3GwwGnxwTiQ8UiMiQZs+ejfz8fOzcudO7befOnSgoKMCsWbNGdW5/OZVVq1Z5c1lLlixBbW0t7rnnHnAcB47jAAwumg3UP2dWWlqK3bt348knn/Seo7q6GhMnTsRjjz3m87ojR46A4zicPn16VNdFQkOBiAxr7dq1qKio8D5/7rnn8KMf/Sji77tz506MGzcODz74IJqamtDU1BT0OZ588kksWLAAd9xxh/ccBQUFg64JACoqKnD55Zdj4sSJ4boEEgQKRGRYP/zhD/H555+jtrYWtbW1+Pe//40f/vCHEX/f1NRUSKVS6HQ6ZGdnjzhjpj8GgwEKhQJJSUnec0ilUpSWlqKqqso7UbzD4cDLL7+MtWvXhvsySIASeqpYMrKMjAxce+21qKysBGMM1157bdzPX52bm4trr70Wzz33HObPn49//OMfsNlsuOmmm8ROWsKiHBEZ0dq1a1FZWYnnn38+bLkGiUQyaApSh8MRlnMHYt26dd6K+IqKCqxevRpJSUlRe3/iiwIRGdHVV18Nu90Oh8OBq666KiznzMjI8Kn3cblc+Prrr32OUSgUo15LbKhzrFy5EhqNBtu3b8cHH3xAxTKRUdGMjEgqleLEiRPex0MxmUyD1j9LS0vzO8n+smXLsHHjRrz77rsoLi7GE088AaPR6HNMYWEh9uzZg5tvvhlKpTKkImFhYSH279+PmpoaaLVapKamQiKReOuKysvLUVJSMuxiCSTyKEdEAqLX66HX64c9ZteuXZg1a5bP7YEHHvB77Nq1a7FmzRrcfvvtWLx4MSZMmIClS5f6HPPggw+ipqYGxcXFyMjICCnd9957L6RSKaZOnYqMjAzU1dV59/34xz+G3W6PSisgGR7NWU0S1r/+9S8sX74c9fX1g1ZFJdFFgYgkHJvNhtbWVqxZswbZ2dl46aWXxE5SwqOiGUk4r7zyCsaPHw+j0Yjf/e53YieHgHJEhJAYQDkiQojoKBARQkRHgYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIEd3/B5bwo9EGInRUAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:36:08.385465Z", + "iopub.status.busy": "2024-02-29T05:36:08.385142Z", + "iopub.status.idle": "2024-02-29T05:36:08.746209Z", + "shell.execute_reply": "2024-02-29T05:36:08.745171Z" + }, + "papermill": { + "duration": 0.38658, + "end_time": "2024-02-29T05:36:08.748418", + "exception": false, + "start_time": "2024-02-29T05:36:08.361838", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEqCAYAAACx7V3bAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCKElEQVR4nO2deXxTddb/Pzd72izd95QWyiq7CAKiIIwIbuiMMi4I4jKOdeZRfjyjzIKj82h1HnVgZnzQcQaqI4jL4DajIuIAgwgiUARBlu50oWvSptlz7++Pm6RNm7ZJmuQmzXm/Xnkluffm5tw2+eR8z/ec82U4juNAEAQhICKhDSAIgiAhIghCcEiICIIQHBIigiAEh4SIIAjBISEiCEJwSIgIghAcEiKCIASHhIggCMEhISIIQnAEFaKCggIwDNPnVlxcLKRZBEFEGImQb3748GE4nU7P85MnT+IHP/gBbr31VgGtIggi0jDRVPT6yCOP4J///CfOnTsHhmEGPZ5lWdTX10OtVvt1PEEQkYXjOHR2diInJwciUf8DMEE9op7YbDa88cYbWLNmTb+iYrVaYbVaPc/r6uowYcKESJlIEESQ1NbWIi8vr9/9USNE77//PvR6PVatWtXvMSUlJXjyySf7bK+trYVGowmjdQRBBENHRwd0Oh3UavWAx0XN0Gzx4sWQyWT46KOP+j2mt0fkvkiDwUBCRBBRSEdHB7Ra7aDf0ajwiKqrq/H5559jx44dAx4nl8shl8sjZBVBEJEiKvKItmzZgoyMDFx33XVCm0IQhAAILkQsy2LLli1YuXIlJJKocNAIgogwgn/zP//8c9TU1GD16tVhOT/HcXA4HF75SkToEYvFkEgklEZBBIXgQnTNNdcgXPFym82GhoYGmEymsJyf8CYhIQHZ2dmQyWRCm0LEGIILUbhgWRaVlZUQi8XIycmBTCajX+swwXEcbDYbmpubUVlZidGjRw+YvEYQvRm2QmSz2cCyLHQ6HRISEoQ2Z9ijVCohlUpRXV0Nm80GhUIhtEkBcaS6DecuGlGYlojLClIgEtGPViQZ9j9b9MscOWL5b61LTkBjhwUHylux92yz0ObEHbH7ySGIEJKhUWDR+EwAQFmtHo0Gi8AWxRckRETc8n1jh5fgTMzVYnw2n/17tKZdKLPiEhKiKGT+/Pl45JFHhDZjWGOxO/Hv75ux/XANGgxmz/bpI5IAAOebjLDYKeUjUpAQEXHJsRo9LHYnUhJlyFR3B9Yz1AqkqWRwshyqWrsEtDC+ICGKMlatWoW9e/di48aNno6VeXl52LRpk9dxx44dg0gkQnV1NQDgxRdfxKRJk5CYmAidToeHHnoIRqPR6zX79+/HvHnzoFQqodPp8POf/xxdXfH3ZbPYnZ6h1+UjU/vMkBWmqQAATR3WPq8lwkNcCpHNwfZ7czhZv4+1+3FsoGzcuBGzZ8/G/fffj4aGBjQ0NOD222/Htm3bvI7bunUr5s6dixEjRgDgZ6z++Mc/4rvvvsNrr72GL774Ar/4xS88x5eXl+Paa6/FD3/4Q3z77bd46623sH//fjz88MMB2xjrfFdvgM3BIk0lw+gMVZ/9U3RarJ5biCvHpAtgXXwSNW1AgmGgFgMWiwWVlZUoLCzsk9Pyh11n+z1nYVoilk3L9Tz/8xfnYHf6/hPlJStx6wyd5/nLe8thtnnHFR79wRi/r8fN/PnzMXXqVGzYsAEAUFZWhunTp6Oqqgr5+flgWRb5+fn49a9/jQcffNDnOd599108+OCDaGlpAQDcd999EIvFeOWVVzzH7N+/H1dddRW6urpCkvcz0N88WmBZDqUHqmAw27FofCYm5WmFNmlY428bkLj0iGKNqVOnYvz48R6vaO/evWhqavLq7f35559j4cKFyM3NhVqtxooVK9Da2uopbzl+/DhKS0uhUqk8t8WLF3sy0OOFytYuGMx2KKRijMseuFkXETmGbWb1QBQvKOp3X++E2geuHNXvsb0rRlbPLRyKWQNy5513Ytu2bXj88cexbds2XHvttUhNTQUAVFVV4frrr8dPf/pTPP3000hJScH+/ftx7733wmazISEhAUajET/5yU/w85//vM+58/Pzw2Z3tOFwclArJBibpYZU3P/vcE2rCcdq25GukmNOUVoELYxP4lKIZBL/HcFwHTvgeWSyPt0C7rjjDvz617/GkSNH8O677+Lll1/27Dty5AhYlsULL7zgyW5+++23vV4/ffp0nDp1CkVF/YtwPDA2S42iDBUc7MDxO5PdgYrmLprCjxA0NItCCgoKcOjQIVRVVaGlpQUsy6KgoABz5szBvffeC6fTiRtvvNFzfFFREex2O/70pz+hoqICf//7372ECgAee+wxHDhwAA8//DDKyspw7tw5fPDBB3EZrBaLGMgl4gGPSU3kO4G2GG1h6w5BdENCFIWsXbsWYrEYEyZMQHp6OmpqagDww7Pjx4/j5ptvhlKp9Bw/ZcoUvPjii3juuecwceJEbN26FSUlJV7nnDx5Mvbu3YuzZ89i3rx5mDZtGtavX4+cnJyIXptQcByHmlYTnKx/opKSKIOIYWBzsOi0OsJsHRGXs2ZEeIjmv3lThwVbD9VAo5Ri1ZwCiP2orn/9qyq0Gm24eVouCtISI2Dl8INmzQiiB2cv8smdWRqFXyIEdA/PWrsosTHckBARcUF5My9ERT4SGPsjVcV3mmwx2sJiE9ENCREx7GnrsqGtywaxiMGIVP+b5CUnyEI2E0oMTFxO3xPxRYXLG8pLVkIhHXi2rCejM1QYk6miFsMRQHC5r6urw1133YXU1FQolUpMmjQJ33zzjdBmEcMI97BsVLr/wzIAEIkYEqEIIahH1N7ejrlz52LBggX45JNPkJ6ejnPnziE5OVlIs4hhhNXhRKOBDzbTzFf0IqgQPffcc9DpdNiyZYtnW2Fh+MokiPhDKhJh+WU6NHZYoFVK/X+h0w6Uf4Hy08dQxyYj//JbUJBN1fjhQtCh2YcffogZM2bg1ltvRUZGBqZNm4ZXX31VSJOIYYZIxCBLq8BUXZL/L+I44NQHQN1ROM0dkLadA/PdDn47ERYEFaKKigps2rQJo0ePxs6dO/HTn/4UP//5z/Haa6/5PN5qtaKjo8PrRhAh5+JJoOUcIBLDVLAITkYKtr2W306EBUGFiGVZTJ8+Hc888wymTZuGBx54APfff3+fOik3JSUl0Gq1nptOp/N5HEEAgNnmxK5TF3GmsdP/ejHWCVT+h39cMA+ivOmo10yB1eEELtAkSrgQVIiys7MxYcIEr23jx4/31Fb1Zt26dTAYDJ5bbW1tJMyMONHWPD/a7PGXxg4LTtYZcLCi1f/Zr6ZTgMUAyFVA3gyo5BJcVE2AlWWAzkagqyW8Rscpggar586dizNnznhtO3v2rKf9aW/kcjnkcnkkTIt5bDZb3K9B716dI1MTQN1b/TH+PvdSQCyFSuGEQ6xAiyQbgBFoPgMkUn+iUCOoR/Too4/i4MGDeOaZZ3D+/Hls27YNf/nLX1BcXCykWYLiq3l+eXk57r33XhQWFkKpVGLs2LHYuHFjn9ctW7YMTz/9NHJycjB27FgAwIEDBzB16lQoFArMmDED77//PhiGQVlZmee1J0+exJIlS6BSqZCZmYkVK1Z4Wsz6sqeqqipSf44hcbGDX7MsW+unEBmbAUMdwIiArMkAALWcn2lrlOWD5TigrSIstsY7gnpEl112Gd577z2sW7cOTz31FAoLC7Fhwwbceeed4XlDjuOnZSONWNq3nWM/bNy4EWfPnsXEiRPx1FNPAQCSk5ORl5eHd955B6mpqThw4AAeeOABZGdn47bbbvO8dvfu3dBoNNi1axcAvvL5hhtuwNKlS7Ft2zZUV1f3GWLp9XpcffXVuO+++/CHP/wBZrMZjz32GG677TZ88cUXPu1JT4/+aWyO4zz5Q1n+ClHDcf4+rYgfmgFQSEVQySWQyArgsJ+ErLOB/wyJA0gFIAZF8BKP66+/Htdff31k3sxpB/7zQmTeqyfz/h8g8W+YpNVqIZPJkJCQgKysLM/2J5980vO4sLAQX331Fd5++20vIUpMTMRf//pXz5Ds5ZdfBsMwePXVV6FQKDBhwgTU1dXh/vvv97zmz3/+M6ZNm4ZnnnnGs23z5s3Q6XQ4e/YsxowZ49OeaEdvssNid0IiYpCm8mM4z3FA8/f8Y5c3BAAMw+D+K0fy+7/aB1iNQEcdkFwQHsPjFMGFiPCPl156CZs3b0ZNTQ3MZjNsNhumTp3qdcykSZO84kJnzpzB5MmTvXoDzZw50+s1x48fx7///W+oVH3LH8rLyzFmTOCrkEQDFzv5YVmGRu5f24+OOsDayf9gJPtIqmUYQKsDmk4D+loSohATX0IklvLeiRDvOwS2b9+OtWvX4oUXXsDs2bOhVqvxv//7vzh06JDXcYmJgZcwGI1G3HDDDXjuuef67MvOzg7aZqFp7+KH4OlqPyc33N5Q6mhA3M/XQpPLC5HxYggsJHoSX0LEMH4PkYSkd/P8L7/8EnPmzMFDDz3k2VZeXj7oecaOHYs33ngDVqvVM9t4+PBhr2OmT5+Of/zjHygoKIBE4vvj4KuZf7Qze1Qqpui08KszLMfxs2EAkD6uz+7v6g04Wt2OcQoZLgP4aXwipAhefU/0pXfz/NGjR+Obb77Bzp07cfbsWfzmN7/pIyi+uOOOO8CyLB544AGcPn0aO3fuxPPPPw8Anrya4uJitLW14fbbb8fhw4dRXl6OnTt34p577vGIj69m/rFAgkwCldyP39qOesDSwXuuKX2HZQ4nhxajDRc512KM1k7AFn9LdYcTEqIopHfz/MWLF+OWW27B8uXLMWvWLLS2tnp5R/2h0Wjw0UcfoaysDFOnTsWvfvUrrF+/HgA8caOcnBx8+eWXcDqduOaaazBp0iQ88sgjSEpK8ixN1F8z/2FDi8sbSi3yOYxWKXgx63SIgIQUfiMNz0IKNc+PM7Zu3Yp77rkHBoPBayWQUBAtf/NGgwVfVbRAl5yAGQUpg7/g61f5jOkJNwGZE/rsbjCYsf3rWqgVEtyXchxo+h4oWgjoZvo4GdETf5vnx1eMKA55/fXXMXLkSOTm5uL48eOeHKFQi1A00dhhQVWLCSKGwYzBDja18SLEiICUkT4PSZDxXxOzzQlOmQIGAEytoTQ57iEhGuY0NjZi/fr1aGxsRHZ2Nm699VY8/fTTQpsVVlqNfCKjX/lDra6gf5IOkPr24hJlfHtZB8vBJk+BHKCasxBDQjTM+cUvfoFf/OIXQpsRUdq6+FU3UhL9mCFtPc/fp/a/FLdELIJcKoLVzsIk0fJCRB5RSCEhIoYdehOfQ5ScMIgQOayA3hV4H0CIACA1UQabg4VdnsRvsJsBmwmQ+b8qCNE/JETEsMLmYGF0LRGdlDBIImlbBcCx/ExYwsBB7eWX5Xc/UWj46X5TKwlRiBj20/cxPCkYc0TD31pv5odlSpl48KWD/BiW+STB1QaEhmchY9gKkVTK/xqaTCaBLYkf3H9r999eCMw2J2QSEZIH84ZYtjtQHagQKV2rzJjbAzeQ8MmwHZqJxWIkJSWhqakJAJCQkEBrVIUJjuNgMpnQ1NSEpKQkiMX+L2IYakakJuKh+aNgdw7inXXW83EeiRzQ5g163u8bO3C4sg26lATMV7oyrC2GEFhMAMNYiAB42la4xYgIL0lJSVHRKoRhGMgkg/zotJzj71NGAqLBhdNd5qFWSIGUJH6jRT8kO4luhrUQMQyD7OxsZGRkwG4XoCFaHCGVSgX1hAImwPhQoqtmrcvmABTkEYWaYS1EbsRicWx9SYigeetwDRRSMRaOz+y/4NWsd2VTM0DqKL/Om+BKajRZnYAiid9oMwEOW0x0dIh2hm2wmog/LHYn6vUWVDR3QSoeYGjmDlJr8wCpf6UuHiGyOcFJ5N1Z2OQVhQQSImLY0GHmh98JMjHkkgE84FZXfCiA2TJ3vRnLcbA62B7DM30wphK9ICEihg0dFl6INAOtcR9ANnVPxCIGMgn/dTHZegzPyCMKCXERIyLigw4Ln1GtUQwgRO1V/GquymQgITWg86cmymB3snCyXLdHZNYHZyzhhaAe0W9/+1vPWlnu27hxfVt1EoQ/uIdmGuUAv689Z8sCzCv78cx8rJhdwPfBVibxG2loFhIE94guueQSfP75557n/fVNJojBGNQj4rgeQuTfbFm/yHu0jSWGjODfeolEEhVJcETsI2b4OI5a0c/HuqOen3KXyICkfN/H+Itczd+TEIUEwYXo3LlzyMnJgUKhwOzZs1FSUoL8fN8fEqvVCqvV6nne0dERKTOJGOC6ydkDF966vSE/s6l7c7xWj+MX9BiTqcbleS4hsnXxMacgzkd0I2iMaNasWSgtLcWnn36KTZs2obKyEvPmzUNnp+9fmZKSEmi1Ws9Np9NF2GIi2nHHGn0SbLW9C7uTRavRxvc7kioBket3nLyiISOoEC1ZsgS33norJk+ejMWLF+Pjjz+GXq/H22+/7fP4devWwWAweG61tbURtpiIWaydgNFVc9hPb+rBcLcVsdidfKCbhmchQ/ChWU+SkpIwZswYnD9/3ud+uVzuWSiQIHpS3dqFfedakJ+SgKvGpPc9oK2Cv1dnAbLAV8QF+B5HAGC2uxablKv5ViAkREMmqhIajUYjysvLY3qpY0IY2k12tHRaPVP4fXAL0RBmy5Quj8hs6yFEAAlRCBBUiNauXYu9e/eiqqoKBw4cwM033wyxWIzbb79dSLOIGKQ7h8jH1D3LAm2V/OMgh2VADyGykxCFGkGHZhcuXMDtt9+O1tZWpKen44orrsDBgweRnu7DtSaIAfCUd/iauu+o40s7pApAnRP0e7iHZjYHC4eThUTuWjDQSrO3Q0VQIdq+fbuQb08MI4yuZEafOUTuYVlyISAKfhAgd+UoyaVi2J0cJB6PiIRoqERVsJoggsW9codK7mNo5haiIQzLAD414L55Pc5BQ7OQEVXBaoIIBpbl0GXl4zaJ8l6JhVYj0NnIPx6iEPVB3iupkQgaEiIi5rE5WSQnSqGQipEo6+Xkt7uC1OpMQK4K7RvLEgFGxNew2YyhPXecQUMzIuZRSMW4e3aB752eYdkQi1xdHChvwfkmI6bnJ2NirpYXN0sH73m5W4MQAUMeETF84bge0/aFITml2eZEq9GGTldwHDKXl0Ue0ZAgISKGL13N/NplYimgyQ3JKT1lHg5XTMidpU1CNCRoaEbEPF9XtuHMxU5MztViii6pe0d7FX+flB+y6niFlP/ttrqTGj0eUVdIzh+vkEdExDxtXTa0dFphc7LeO9qr+fukESF7L0Wf7GqXEFnJIxoKJEREzNOdQ9TDwWedgN4lRMmhFyKL3SV6nqEZeURDgYSIiHm6fAlRZwPgdPUNUmWG7L28WoEAFKwOESREREzDcZxvj6hnfCjAJvkDoZSKoZJLuvOVSIhCAgWriZjG6mBhc/DDpEQvIXIPywpC+n4piTLcf2WPDG3P0MzEpwuEUPTiCfKIiJjGPSyTS0WeBRDhtPMV90DIhagPskRefDgWsJvC+17DGBIiIqZxshzSVDKkJsq6Nxpq+WC1XM0vpBhORGI+DgXQzNkQoKEZEdNkaBRY0bu8o+ewLAxDpX9+W4+2LhuuvSQLGRoF7xXZTK44UegC4/EEeUTE8MPgWlRhqGuX9Xd6sx2tRhu6bJTUGCpIiIjhhdPe3fZDmxeWt1BIaAo/1NDQjIhpPj3ZgGajDVcUpaEwLZFfzZV18hnPYYoP9c0loqTGoUIeERHTtLrKOzwrvLqHZdq8sE2lu+vNurOrySMaKiREREzTJ6vacIG/14YnPgT4qMCnerMhEzVC9Oyzz4JhGDzyyCNCm0LECM4eLWJVCgm/bJBHiMITHwJ6CJGNhmahIighqqioCKkRhw8fxiuvvILJkyeH9LzE8MZd2iEWMfyaY8aLfLBaIgcSw7ckVYKML/PwJFDS0GzIBCVERUVFWLBgAd544w1YLJYhGWA0GnHnnXfi1VdfRXJymJPPiGGFe1iWKJeAYZge8SHdkJYNGozx2Rrcf+VILBzvyhlyC5HTzq+fRgRMUP+to0ePYvLkyVizZg2ysrLwk5/8BF9//XVQBhQXF+O6667DokWLBj3WarWio6PD60bEL26PSO2JD/UIVEcSiYzvAgnQ8CxIghKiqVOnYuPGjaivr8fmzZvR0NCAK664AhMnTsSLL76I5uZmv86zfft2HD16FCUlJX4dX1JSAq1W67npdLpgzCeGCSIGSFPJkJwo4wtO3fGhJAE+FzQ8GxJD8l8lEgluueUWvPPOO3juuedw/vx5rF27FjqdDnfffTcaGhr6fW1tbS3+67/+C1u3boVCofDr/datWweDweC51dbWDsV8IsYpylBjxewC/GBCJmBq48ssRBJAnR3W97XYnXj7cC1e/6qqO22gZxU+ETBDEqJvvvkGDz30ELKzs/Hiiy9i7dq1KC8vx65du1BfX4+bbrqp39ceOXIETU1NmD59OiQSCSQSCfbu3Ys//vGPkEgkcDr7Llgnl8uh0Wi8bgQBADDU8PeanJD1p+4PqViEOr0ZrUYbrA7q1BgKgsqsfvHFF7FlyxacOXMGS5cuxeuvv46lS5dC5AoQFhYWorS0FAUFBf2eY+HChThx4oTXtnvuuQfjxo3DY489BrE4vB8mYpihj1x8SCxiIJOIYHOwsNid/HQ+Dc2GRFBCtGnTJqxevRqrVq1CdrZvNzgjIwN/+9vf+j2HWq3GxIkTvbYlJiYiNTW1z3aC8MXWQ9VgWQ5LJmUjLcLxIblHiMgjCgVBCdGuXbuQn5/v8YDccByH2tpa5OfnQyaTYeXKlSExkiB6w3Ec2ow2OFgOUocRsBj45Z9DtH7ZYChlYnRaHN2reZAQDYmghGjUqFFoaGhARkaG1/a2tjYUFhb6jO/4w549e4J6HRF/WOwsHCwfKFaZXZMiqgw+mTECUAV+aAkqWO2ZKeiF0Wj0ewaMIIZCp9UOgM9yFndEftqeKvBDS0Ae0Zo1awAADMNg/fr1SEhI8OxzOp04dOgQpk6dGlIDCcIX7hqzRLnEO6M6QiTKxVArJBC5K/xlru+CnZroB0NAQnTs2DEAvEd04sQJyGTdfYJlMhmmTJmCtWvXhtZCgvCB0cJnVSdJ7EBnC78xghnV88dmYP7YHqEJqcsjYp2A3dwtTIRfBCRE//73vwHw0+wbN26kPB5CMNzlHSkOVxZ/Qmr38EgIxBJAqgDsFn54RkIUEEEFq7ds2RJqOwgiIORSEdJUMqSxF/kNQpR19EamcgmREUD4qv+HI34L0S233ILS0lJoNBrccsstAx67Y8eOIRtGEAMxPT8Z0/OTgSN7ASsiXujaaLBg79kmqORSXDfZlUsnSwS6WihgHQR+C5FWq+VbLbgeE4TgOGw9GuVH1iNychzq9RYkJfRIVaGZs6DxW4h6DsdoaEZEBZ31/AqrcjWgiOyPo0LSq2810EOIKJcoUILKIzKbzTCZuquMq6ursWHDBnz22WchM4wg+sPuZLFpTzk+O3AYTpbj40MRni535xFZHU6wrLsC35XUSEtPB0xQQnTTTTfh9ddfBwDo9XrMnDkTL7zwAm666SZs2rQppAYSRG+6rA5Y7E4whgt8I8ZIN0JDtxBxHKgCPwQE3aFx3rx5AIB3330XWVlZqK6uxuuvv44//vGPITWQIHpjtDrAcE6kOJvBgAnrih394a7AB3xlV9PQLFCCEiKTyQS1Wg0A+Oyzz3DLLbdAJBLh8ssvR3V1dUgNJIjeGK0OJNhaoRCxfO5OYpogdvRZVoiWng6aoJvnv//++6itrcXOnTtxzTXXAACampooyZEIO11WBzTWBt4j0UY+PuRG5SrzcLK9ujTazXyGNeE3QSU0rl+/HnfccQceffRRLFy4ELNnzwbAe0fTpk0LqYEE0ZtOiwNq60XI5CJB4kNull/Wa0goUfKtSDiWD1jL1cIYFoMEJUQ/+tGPcMUVV6ChoQFTpkzxbF+4cCFuvvnmkBlHEL4wWuzQWhshSxBFPH9oQEQivrTDauSHZyREfhOUEAFAVlYWsrKyvLbNnDlzyAYRxGCkMJ1IlNghl2sAddbgL4gk0h5CRPhNUELU1dWFZ599Frt370ZTUxNYlvXaH+qVYAmiJ3NSTUBeEpA8IuyN8gfidEMHTlwwoCAtETMLU/iNMhWAJpo5C5CghOi+++7D3r17sWLFCmRnZ3tKPwgiIkRgfXt/MNmcqNOboVb0+BpRLlFQBCVEn3zyCf71r39h7ty5obaHIAaE4zgwHiESNj6kkLryiBxUbzZUghKi5ORkpKSkhNoWghiU+ouNqP++EiqlDOMj1Ci/P7rbxfasN6Pe1cEQVB7R7373O6xfv96r3iwYNm3ahMmTJ3sWS5w9ezY++eSTIZ2TGN5YmyrhYDl0yTP4NecFpE/faoA8oiAJyiN64YUXUF5ejszMTBQUFEAqlXrtP3r0qF/nycvLw7PPPovRo0eD4zi89tpruOmmm3Ds2DFccsklwZhGDHMcrZX8g6TIl3X0RukSIjMJ0ZAJSoiWLVsWkje/4YYbvJ4//fTT2LRpEw4ePEhCRPSF4wA9X0IkSi4U2JjuGJHVzoJlOYhEDJV5BElQQvTEE0+E2g44nU6888476Orq8mRq98ZqtcJqtXqed3R0hNwOIooxt8Np6QTLiCFLFXbGDADkEjGkYgYKqRg2JwuFSNztETmsgNMOiKUDn4QAMISERr1ej3fffRfl5eX47//+b6SkpODo0aPIzMxEbq7/QcQTJ05g9uzZsFgsUKlUeO+99zBhwgSfx5aUlODJJ58M1mRCIDiOC02KR3sVrHYWRlkGdAnCN6cXixg8fPVo740SOSCSAKyD94qUSYLYFmsEFaz+9ttvMWbMGDz33HN4/vnnodfrAfC9qtetWxfQucaOHYuysjIcOnQIP/3pT7Fy5UqcOnXK57Hr1q2DwWDw3Gpra4Mxn4ggNa0mfHi8HnYnO/jBg8C1V8PmcMKgyIFGGfRvaHhhGIoTBUFQQrRmzRqsWrUK586d81rZdenSpdi3b19A55LJZCgqKsKll16KkpISTJkyBRs3bvR5rFwu98ywuW9E9GJzsPjsVCMqmrtwrEYPADCY7Pj2gj7wk3Ec2PZqaJRSiFMKoJJHqRABJERBENR/8/Dhw3jllVf6bM/NzUVjY+OQDGJZ1isORMQu39Ub0GlxQKOUYlp+EoxWB944VA2W5TAyXRWYmHQ1Q+wwY1xuCsZdMZMvMI0CDpS3oLbNhBkFKRiV7gpUU4O0gAlKiORyuc9A8dmzZ5Ge7v96TuvWrcOSJUuQn5+Pzs5ObNu2DXv27MHOnTuDMYuIIjiOQ1mtHgAwY0QypGIRJCIG6So56vRmlNXoccXoABqa6Wv4e22+oPVlvdGb7KjXW2Aw27s3kkcUMEH9rNx444146qmnYLfzf3yGYVBTU4PHHnsMP/zhD/0+T1NTE+6++26MHTsWCxcuxOHDh7Fz50784Ac/CMYsIopoMFigN9khk4gwPpsfQjMMg2n5SQD4glFP03l/aKsEy3HgoiB/qCeeMg/KJRoSQSc0/uhHP0J6ejrMZjOuuuoqNDY2Yvbs2Xj66af9Ps/f/va3YN6eiAHOXuwEAIxKV3l6OwPAyHQVlDIxjFYHqttMKEzzY5lopwPQV6GypQt77SLMEOsxOS8pTJYHhkLiWs2DlhUaEkEJkVarxa5du/Dll1/i+PHjMBqNmD59OhYtWhRq+4gYhOM4nLvIfwlHZ6q89olFDMZmqlFWq8e5i53+CZGhBnA6YOQUMDBJkIqjIz4EAHKf2dWU1BgoAQsRy7IoLS3Fjh07UFVVBYZhUFhYiKysrNDlixAxjdXBoihDhcYOC0ak9M33KcpQoaxWj4qWru6M5IFo4/tbXZTmAAyDpIToSRJUUr1ZSAhIiDiOw4033oiPP/4YU6ZMwaRJk8BxHE6fPo1Vq1Zhx44deP/998NkKhErKKRiLBiX0e/+3CQlFFIxHE4W7SYbUlXygU/YVgknx6FRnAMASFIKW+zak+4Yka+hWRdflkI/zoMSkBCVlpZi37592L17NxYsWOC174svvsCyZcvw+uuv4+677w6pkcTwQiRi8MPpuUhOlA0+zDLrga4WWBws9PJcyKUiz5c/GlBI+TIPr8twD81YB1/qIVX4fC3RTUD/0TfffBO//OUv+4gQAFx99dV4/PHHsXXr1pAZR8QeTpbDhXZT9xI7/ZChUfgX62nnq+275JlwihVIUsqiavifrVXg4atHe6/oIZZ2tyih4ZlfBCRE3377La699tp+9y9ZsgTHjx8fslFE7HKxw4J3vrmALV9WguP8m54f8LjWcgBAm5yvX4ym+BCA/kXR7RXZSYj8ISAhamtrQ2ZmZr/7MzMz0d7ePmSjiNjlQrsZAJCpUQzquXxXb8Dfv6rC0Zp+PjNOu8cjkmSMRWFaInKTlCG1N2xQwDogAooROZ1OSCT9v0QsFsPhcAzZKCJ2qW3ju3bqfMyW9cbmYNFitKGiuQuXjvDReritgs8hUmhRVDgSRVE0JOvJJyca0GGx49pLsqF1e2wkRAER8KzZqlWrIJf7nuWgGrH4xuFkUa/nPaK85ME9l5FpKuw504x6vQUWu9PTetVDy1n+Pn1MVM88NRj4Eo8um6OHEFHv6kAISIhWrlw56DE0Yxa/NHZY4GA5JMjESE0cfIpdmyBFqkqGVqMN1a0mjM3qsTIqywKt5wEA9uQi2G0OJMiis+JeIRXDYLZTLtEQCOg/u2XLlnDZQQwDatt4b0iXkuD3zFZhWiJajTZUthi9hchQA9gtgCwB1Y4UfLS3ArqUBPzoUuE7M/bGZy6R1DU0JSHyi+hJyCBingvtfHzIn2GZG3eJR2WLybsItvkMf59ahKZOGwB4L2QYRfhuok9Ds0AgISJCxpVj0nHF6DSMSPWjfsxFjpbPsrbYnag38B4VWCfQdJp/nD4OFzstAIAsTXQmBrpjW1YamgVNdP7EEDFJpkaBzADFQiRiMC5bDaud7a7Sb68C7GZAlgAuuQCNJ6oAAFna6BQi+YArvpqozMMPSIgIwVkwtldd2sXv+PuMCdCbnbDYnRCLGL8C4ELgLvPwwi1EHAvYTd3PCZ+QEBEh4XBVG9QKCQrTEiGXDKGDosPWPW2fMQG1rrhTtlYBSRS1/+jJNF0Spucne28UiQGpkvfsbF0kRIMQnf9ZIqawOpw4cL4Vn5xo9J45CgCO49DUYUFz9Uk+o1qZBGhyUONKkMz3I0FSKPov86AGaf5CHhExZOrazWA5DlqlFFplcLVgJ+oM2H26CTM79iMthQOTeQnAMLgkR4sEmdi/BmrRhkwFdLVQwNoPyCMihkytq75sKF7LmEw11KwBIn0NOq1OIHsKAH56/+pxmciI0hkzgG+K9kFZHd46XONdwEszZ35DQkQMGc/wKTV4IVJIxZgm5VfqOG1JAyePnTXrxCIGFc1dqNdbYHX00yCNGBBBhaikpASXXXYZ1Go1MjIysGzZMpw5c0ZIk4gA6bI60NLJ1xgGksjYB6cDE0RVEDHAaWYU3jlyATu/a4TeZAuRpeFDKhZ5Ug/MNupdHQyCCtHevXtRXFyMgwcPYteuXbDb7bjmmmvQ1UX/uFjBPauVrpYPrRas6RSUnA15WZloV+ajrt2MU/UdONPYGSJLw4vv7GryiPxF0GD1p59+6vW8tLQUGRkZOHLkCK688kqBrCICodXIeywjhjAsA8cBtYcAALkT5+F6eS4qWrqQm6TEJTmxMURTyvjCV5PNlxDRrNlgRNWsmcFgAACkpPjoTUNEJXOL0jApT4sh5Q23VfCzS2IpkD0Vo6UKjM5UD/66KCJB5vKIaGgWFFEjRCzL4pFHHsHcuXMxceJEn8dYrVavnke+lr0mIo9GMcT2rS5vCDlTY7bRvGKgoZndzNfPRdFS2dFG1MyaFRcX4+TJk9i+fXu/x5SUlECr1XpuOp0ughYSvRmsQb5ftFcB7dX8lzTvsqGfTyASZHyZh9ffRKoERK7feiv9aA4Ew/nb4TyMPPzww/jggw+wb98+FBYW9nucL49Ip9PBYDBAo4mNWMJwgeM4bP6yCqmJMiwcnwF1MF4RxwHH/g4Y6oDcS4Ex14Te0AjR70KRB18GzO3AtDuBpPy++4c5HR0d0Gq1g35HBR2acRyHn/3sZ3jvvfewZ8+eAUUIAORyeb9taonIcqHdjA6zHVaH0zNjFDCt53kREkuAEbNDa2CE6Xe1WrmaFyILeUQDIagQFRcXY9u2bfjggw+gVqvR2NgIANBqtVAqY2S1hjjle9e0+ugMdXDFqE4HcH43/zj3Uv4LOxxxX5c1NtIQhELQGNGmTZtgMBgwf/58ZGdne25vvfWWkGYRg2B3sjjXxH+xxmUFKSC1h3hPQZYI5M8JoXXCYDDb8UFZHT4oq/PeoXANR0iIBkTwoRkRe3zf0AmrnYVWKQ0um9rYDFQf4B8XLYzZmbLeVDR3QSxiwHFcd0W+xyOiodlARM2sGREbcByHsgt6AMAUXVLgyz87HcDpD/h14VNGAhkTQm+kALjziJws511vJtfy9yREA0JCRAREnd6Mlk4rpGIm8KxnjgPO7eQ9IlkCMO66YdNCtWe9mdeyQm6PiILVAxI1CY1EbJChVmDR+EyYfS2IOBAcB1TuAxq+5cVn3PWAXBU+QwVAIRXD5mBhsjmR5K54cceI7Ga+4Zt4iMmfwxQSIiIgZBIRJuVpA3uR0wGU7wbqjvLPR18DpI4KvXECkyATo6N3vZlEwacnOB18wDqBypd8QUJEhA/WyecKVe7ja8kYBihaBOROF9qysOCOE5lsju6NDMPHiUytfJyIhMgnJESEX9idLN47Wodx2WpckqOF2J3Ax7L8l6yriS/utJv4X35rJ9BRzw9HAD4mNHYpkDZauIsIM4kyCWQSERy9S1/kapcQ0RR+f5AQEX7xfUMn6vRmGK0OTMzRAqY24MI3QPNpfu2u/pAlANlT+ToyWfQ2wA8FV4/LwKIJmX13UMB6UEiIiEHhOA5lte0AgKk5SojO7wLqj/FrdgF8AFaVyX/hZIl8+wu5GkhM47cPk5mxwei3zIOSGgeFhIgYlMYOC1qMNqTYGzCp4TPA5vpCpYwE8mYAyQXU4mIgKKlxUEiIiEE5VWdAVucJzHAcg1SZCCi0wNglQMrARcrxhsFsx54zTWA5DjdPy+veoUji7816IcyKCUiIiAGx222wffcRCjrOICNbA2RN5KffJdQFoTcihi/zEDG9yjyUSfy9xcDnU8XJUDUQSIiI/rGZ0PrlG0jqOAuZVALNpGv5oDN9kXySIJOAYQCW42C2O7sXE5Br+L8Z6+D7Vw/XTgNDgEo8CN+Y9cCxN5BgaYRWo4Z4ynIwupkkQgMgFjGe3kxGa49cIpGYFyOAhmf9QB4R0RdjM/DtdsBqhCYpFROuXM7PgBGDkiiXwGRzwmR1Aj0dH2USPzSz6AFQi+PekEdEeNPVAhzfBliNvPhMW0EiFACJch8eEUAB60EgISK6MbUBx9/kExTVmajKvxntzuHRKyhSJLriQl71ZkCPgLU+ovbECiREBI/DCpz8B+8JqdLBTlqOT88YUHqgCvV6s9DWxQyJcr7Mg+3d9I88ogGhGBHBTymf/ogflslVwOTlaLaKYbY5IZOIkKUhr8hfLh+ZijmjUvs2jCOPaEDIIyKAuiNAyzl+Da6JPwTkatS08fVjecnK/ksXiD6IRYzvrpVuj8hq5FuCEF6QEMU7pjag4t/841FXA5ocAECtS4jyU4Z3oWrEkCoBiYx/TF5RH0iI4hmOA858zP9CJxd4+gQ5nCzq2vm4EAlRYDhZDh+U1eGNg9WwOnoErBkGUCbzj01twhgXxQgqRPv27cMNN9yAnJwcMAyD999/X0hz4o+m04C+lu8gOHaJJ1mxwWCBg+WgkkuQkigT2MjYQixiUKc3o7nTCqOl1xAsIZW/N7VG3rAoR1Ah6urqwpQpU/DSSy8JaUZ84rR3D8nyZ3cHUwFPfEiXogx8lQ7Cs/x2BwmR3wg6a7ZkyRIsWbJESBPilwvf8I265GpAN8tr14yCZGRrFVDKqLVHMGgUErR0WtFpsXvvICHql5iavrdarbBarZ7nHR3U3yUonHbgwtf848Ir+6wsIZeIMTJ9eK2wEUk0Lo+ocyCPiKrwvYipYHVJSQm0Wq3nptNRzU5QNH7LZ08rNEDmJUJbM+xQK/jf9z4ekTKFFx+Hle/vTXiIKSFat24dDAaD51ZbWyu0SbEHywK1Lm9IN6tPZ8WjNe3Yf64FrUarjxcT/tBvjEgs4ZvKATQ860VMDc3kcjnkcmrINSSav+fLDKRKIHtKn90n6wxoNdqQqZEjVUV/62BQKySQihlIxT6GXgmp/N/f1Aokj4i4bdFKTAkRMUQ4Dqj5in+cN6NPbMhodaDVaAPDADrKHwqabK0CxQuKfM84JqYBreVAV3PkDYtiBBUio9GI8+fPe55XVlairKwMKSkpyM/PF9CyYUpbBWBs4gUo99I+u93Z1OlqeWDLSRNeDJjyoHItN2S8GBljYgRBheibb77BggULPM/XrFkDAFi5ciVKS0sFsmoYU3OQv8+Zyg/NekFlHRHAI0RNNHPWA0GFaP78+eB6t0sgwoPhAqCv4YPTeTP77OY4zpPISEI0dI7VtONkfQcuydFgen5y9w5lCl9c7LQD5nZagtpFTM2aEUPA7Q1lXtK94F8P2k12dFocEIsY5CT19ZaIwLA6WLR0WtHS2Wv2USQCVOn8YxqeeSAhige6Wvg2HwzTJ4vaTYfZDqVMjJwkJaRi+lgMleQEvkZPb7L33Ulxoj7QrFk84PaG0kb323+6IC0RP7lyJCx2NoKGDV+SE/gZyXaTre9OVQZ/30lC5IZ++oY7FgNw8Tv+se7yAQ9lGIbqy0JEkssjMtmcMNl6JTaq+Z5P6KjjA9YECdGwp/ZrgGOBpHxAm+vzEIeTpUmDECOTiJDk8opaOnt5RapMPoXCYeWHzQQJ0bDGagTqy/jHI+b0e9iR6nb89T+VOFbTHhm74oR0NZ+Z3my0eO8QiTydMGGgMiWAhGh4c+FrfpljTQ7fgbEfKlu6YLTyM2ZE6MjUKJCmkkEi8vE10+bx9x11kTUqSqFg9XDFbgbqjvKPR8zpN3Guy+pAYwf/i02tP0LLZQUpuKygnzwhtxAZLkTOoCiGPKLhSs1BPmlOlQ6kFvV7WGVLFzgOyNIqoJLT71LE0OQCjIgvgDXTkJiEaDhiMfAdGAGg8KoBywjKm40AgJFpiZGwLC5xshxsjl5pERJ59+RBW0XkjYoySIiGI5X/4WNDSboBvSGrw4maVr6sg4Zl4eGr8lZs2nMeZbX6vjtTRvH3rSREJETDDUMdcPEk/3jU1QN6Q+ebjHCwHJITpEhT0Wod4UAuFcHu5Hwv250ykr/XV/HD6DiGhGg4wTr5dco4Dsia2D1F3A/ZWiWmj0jG1PxkWq0jTOS56vbq9GawbK9cLVUG37HR6QBaz/t4dfxAQjScqD7AJ8hJlcCohYMenpIow1Vj0jFVlxR+2+KUNBXf28nmYNHQ0SufiGGAzAn8Y3f2e5xCQjRcaK8Cqr/kH4/+ASCjVh7RgEjEoDCN/1+UNxn7HpDhWrygrSKuG+qTEA0HzO3AqQ/5IVn25EFX5rA7WXxyogEX2k1U2hEBRrkmAsqbjX3/3qp0QJPND6vrjwlgXXRAQhTr2LqAb9/m71XpwOhrBn3JiToDvm/sxGffXaSaywiQn5oAqZiB3mRHvcHS94C8y/j7uiN8vCgOISGKZcx64NgbgKmND3pOXt6nIX5vLHYnDlW0AeBXdBVRWUfYkUvEuHRECq4ck47URB+zk+nj+GZ1NhNw4XDkDYwCSIhildZy4OhrLhHSAFN+zC8fPQgHyltgsTuRqpJhYo42AoYSADB7VCouHZHse1ECkZhfcRfg43yW+FvBmIQo1rB1AWc+dQ3HTPxwbNoKv3ofn2/qxPFaAwBg/pgM8oYEok+WNQBkutItnHbg1Ad8zCiOiAoheumll1BQUACFQoFZs2bh66+/Ftqk6MPaCVTuAw690h3UzJ0OTF/pswd1b2paTfjkRCMAYPqIZOSn0qyaENTpzXj9qyqcqu/l9TAMMP4GQCLjC2FPfxhXYiR4leNbb72FNWvW4OWXX8asWbOwYcMGLF68GGfOnEFGRobQ5gmLzQS0lQPNZ/jpXfcHU5UBFC0KaKXQOr0ZDpbDyPREzCvy3S6WCD81rSZ0WhzYdeoiWI7DJTma7mTShBRgwjLg5D+Apu/5IdqYxYA6S1CbIwHDCTx/O2vWLFx22WX485//DABgWRY6nQ4/+9nP8Pjjjw/42o6ODmi1WhgMBmg0g3sFUQfH8V36HBa+bYdF71qOuAXoqOfjPz3R5vIzLGlj+eZafU7HwepgYbI50dxphVgEFGWoPfuO1eoxKVdLzfEFhGU57PyuEd83dgIAcpOUmJCjQU6SEhqFBBKxiI//nfqA/2wAfHfNtDG8ICmTAWmCz/9/NOLvd1RQj8hms+HIkSNYt26dZ5tIJMKiRYvw1Vdf9TnearXCau1enqWjw8+gXlslUL7bd39gr21c/9uCPpa/P1lngNXRw9XmWIicNjDgIJeKMT6r+590urEDZht/rFmWAn3iSLQnFMJsTEVipQR3ZHR/CN87dgEXO6y8pjlZOHqUEagVEoxMU0EkYsAwjPf6WoQgiEQMrp2YheREGb6ubEOd3ow6Vx3arJEpmDMqDUgdhZZxd+HE/g+RbDwP5sK3AL71nIMDg8xkDbKTVYBIDJOdw8n6TnBeZTrdj7O1isHXqgumxGfSbX6FBfxBUCFqaWmB0+lEZmam1/bMzEx8//33fY4vKSnBk08+GfgbOayAUdi1xu1mo+8gJQCWkQByFSDXAMpk1JqsqHMmwSjLhEOs4A/iAFgdfT4vVjvrES03MokIyQkyZGnlsDlZKETUED+aYBgGl49MxSU5GnxX34GaVhOaOi1Qy7tTL6xSNcpUV0KumIYUUwU01gYk2lohc5oAcHDazICN/zBwNgecXYb+31CiBORhiAlyoVvxRfAYUSCsW7fOsyw1wHtEOp1u8Bcm6fjpbaCH8vf4Rvv6Jenn1yXYY7PHWuDs+X9jGECiAKQKSCRSQKPw7Jqms2Kyy7Nxn8J9pt4zXUsmZsPOsmAASMQiJMjENPSKEdQKKS4fmYrLR6aC4zgv5zo1UYYfXerq4ojx3Ts4FozdDI3UCchFAOuExG5D/hgfiZIAwHFQycVAgrTP9iEjC10PK0GFKC0tDWKxGBcveq/vdPHiRWRl9Q3QyeVyyOXywN9IlgikFAZrZkjICqDdj7vpuj9oe3/AiJiEYRiv3zOFVAxdv8Mp7w+THEBuathMiwiC/nTKZDJceuml2L17t2cby7LYvXs3Zs+eLaBlBEFEEsGHZmvWrMHKlSsxY8YMzJw5Exs2bEBXVxfuueceoU0jCCJCCC5Ey5cvR3NzM9avX4/GxkZMnToVn376aZ8ANkEQwxfB84iGQsznERHEMMff7yhNrxAEITgkRARBCA4JEUEQgiN4sHoouMNbfpd6EAQRUdzfzcFC0TEtRJ2dfOGgX9nVBEEIRmdnJ7Ta/hvxxfSsGcuyqK+vh1qtDtu6XO4yktra2riYmYu36wXomsN5zRzHobOzEzk5ORAN0DEgpj0ikUiEvLy8wQ8MARqNJm4+pED8XS9A1xwuBvKE3FCwmiAIwSEhIghCcEiIBkEul+OJJ54Iruo/Bom36wXomqOBmA5WEwQxPCCPiCAIwSEhIghCcEiICIIQHBIigiAEh4QIga00++qrr2LevHlITk5GcnIyFi1aFHMr0wa7su727dvBMAyWLVsWXgPDQKDXrNfrUVxcjOzsbMjlcowZMwYff/xxhKwNDYFe84YNGzB27FgolUrodDo8+uijsFj6acofarg4Z/v27ZxMJuM2b97Mfffdd9z999/PJSUlcRcvXvR5/B133MG99NJL3LFjx7jTp09zq1at4rRaLXfhwoUIWx4cgV6vm8rKSi43N5ebN28ed9NNN0XG2BAR6DVbrVZuxowZ3NKlS7n9+/dzlZWV3J49e7iysrIIWx48gV7z1q1bOblczm3dupWrrKzkdu7cyWVnZ3OPPvpoROyNeyGaOXMmV1xc7HnudDq5nJwcrqSkxK/XOxwOTq1Wc6+99lq4TAwpwVyvw+Hg5syZw/31r3/lVq5cGXNCFOg1b9q0iRs5ciRns9kiZWLICfSai4uLuauvvtpr25o1a7i5c+eG1U43cT00c680u2jRIs+2gVaa9YXJZILdbkdKSkq4zAwZwV7vU089hYyMDNx7772RMDOkBHPNH374IWbPno3i4mJkZmZi4sSJeOaZZ+B0On0eH20Ec81z5szBkSNHPMO3iooKfPzxx1i6dGlEbI7potehEuhKs7547LHHkJOT4/VPj1aCud79+/fjb3/7G8rKyiJgYegJ5porKirwxRdf4M4778THH3+M8+fP46GHHoLdbscTTzwRCbOHRDDXfMcdd6ClpQVXXHEFOI6Dw+HAgw8+iF/+8peRMJmC1UPh2Wefxfbt2/Hee+9BoVAM/oIYo7OzEytWrMCrr76KtLQ0oc2JGCzLIiMjA3/5y19w6aWXYvny5fjVr36Fl19+WWjTwsaePXvwzDPP4P/+7/9w9OhR7NixA//617/wu9/9LiLvH9ceUaArzfbk+eefx7PPPovPP/8ckydPDqeZISPQ6y0vL0dVVRVuuOEGzzaW5dfNlkgkOHPmDEaNGhVeo4dIMP/j7OxsSKVSiMViz7bx48ejsbERNpsNMpksrDYPlWCu+Te/+Q1WrFiB++67DwAwadIkdHV14YEHHsCvfvWrAXsJhYK49oiCXWn297//PX73u9/h008/xYwZMyJhakgI9HrHjRuHEydOoKyszHO78cYbsWDBApSVlcVEZ8xg/sdz587F+fPnPaILAGfPnkV2dnbUixAQ3DWbTKY+YuMWYi4S5agRCYlHMdu3b+fkcjlXWlrKnTp1invggQe4pKQkrrGxkeM4jluxYgX3+OOPe45/9tlnOZlMxr377rtcQ0OD59bZ2SnUJQREoNfbm1icNQv0mmtqaji1Ws09/PDD3JkzZ7h//vOfXEZGBvc///M/Ql1CwAR6zU888QSnVqu5N998k6uoqOA+++wzbtSoUdxtt90WEXvjXog4juP+9Kc/cfn5+ZxMJuNmzpzJHTx40LPvqquu4lauXOl5PmLECA5An9sTTzwRecODJJDr7U0sChHHBX7NBw4c4GbNmsXJ5XJu5MiR3NNPP805HI4IWz00Arlmu93O/fa3v+VGjRrFKRQKTqfTcQ899BDX3t4eEVupDQhBEIIT1zEigiCiAxIigiAEh4SIIAjBISEiCEJwSIgIghAcEiKCIASHhIggCMEhISJiitLSUiQlJXme//a3v8XUqVM9z1etWhWTHSTjHRIiwierVq0CwzB48MEH++wrLi4GwzBYtWqV1/GhFoCCggJs2LDBa9vy5ctx9uzZfl+zceNGlJaWep7Pnz8fjzzySEjtIkIPCRHRLzqdDtu3b4fZbPZss1gs2LZtG/Lz8wWxSalUIiMjo9/9Wq3Wy2MiYgMSIqJfpk+fDp1Ohx07dni27dixA/n5+Zg2bdqQzu3LU1m2bJnHy5o/fz6qq6vx6KOPgmEYMAwDoO/QrDc9PbNVq1Zh79692Lhxo+cclZWVKCoqwvPPP+/1urKyMjAMg/Pnzw/puojgICEiBmT16tXYsmWL5/nmzZtxzz33hP19d+zYgby8PDz11FNoaGhAQ0NDwOfYuHEjZs+ejfvvv99zjvz8/D7XBABbtmzBlVdeiaKiolBdAhEAJETEgNx1113Yv38/qqurUV1djS+//BJ33XVX2N83JSUFYrEYarUaWVlZgzaq84VWq4VMJkNCQoLnHGKxGKtWrcKZM2c8/Zntdju2bduG1atXh/oyCD+J6w6NxOCkp6fjuuuuQ2lpKTiOw3XXXRfzbWNzcnJw3XXXYfPmzZg5cyY++ugjWK1W3HrrrUKbFreQR0QMyurVq1FaWorXXnstZF6DSCTq0/nPbreH5Nz+cN9993kC8Vu2bMHy5cuRkJAQsfcnvCEhIgbl2muvhc1mg91ux+LFi0NyzvT0dK+4j9PpxMmTJ72OkclkQ17Cp79zLF26FImJidi0aRM+/fRTGpYJDA3NiEERi8U4ffq053F/GAyGPssOpaam+uxtffXVV2PNmjX417/+hVGjRuHFF1+EXq/3OqagoAD79u3Dj3/8Y8jl8qCGhAUFBTh06BCqqqqgUqmQkpICkUjkiRWtW7cOo0ePHrBHORF+yCMi/EKj0UCj0Qx4zJ49ezBt2jSv25NPPunz2NWrV2PlypW4++67cdVVV2HkyJFYsGCB1zFPPfUUqqqqMGrUKKSnpwdl99q1ayEWizFhwgSkp6ejpqbGs+/ee++FzWaLyCwgMTDUKpaIW/7zn/9g4cKFqK2t7bMYIRFZSIiIuMNqtaK5uRkrV65EVlYWtm7dKrRJcQ8NzYi4480338SIESOg1+vx+9//XmhzCJBHRBBEFEAeEUEQgkNCRBCE4JAQEQQhOCREBEEIDgkRQRCCQ0JEEITgkBARBCE4JEQEQQgOCRFBEILz/wFPCPvQaR/QDgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:36:08.796581Z", + "iopub.status.busy": "2024-02-29T05:36:08.796231Z", + "iopub.status.idle": "2024-02-29T05:36:08.962034Z", + "shell.execute_reply": "2024-02-29T05:36:08.961064Z" + }, + "papermill": { + "duration": 0.194298, + "end_time": "2024-02-29T05:36:08.965546", + "exception": false, + "start_time": "2024-02-29T05:36:08.771248", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvkUlEQVR4nO3deViUR54H8C/dQHMoiHKKaKOo4AEoPgJG16hcOhoNmQSPyKGDuyi7Jh1Gg5vAEh2ZREUms0Q2bvDKJDLjGOOqAUkrrgd4YDSggIIgHoAiQgvEpumu/cPlnbTdSINgw9u/z/P0Q956662u8oVf3vetequMGGMMhBDCQwJ9V4AQQnoLBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG8Z67sCfZFKpcL9+/cxcOBAGBkZ6bs6hJDnMMbw5MkTDB06FAJBx9dpFOC0uH//PlxcXPRdDUJIJ+7cuYNhw4Z1uJ8CnBYDBw4E8Owfz8rKSs+16T0KhQLHjx9HUFAQTExM9F0d8pIM6XzKZDK4uLhwf6sdoQCnRfttqZWVFe8DnIWFBaysrHj/B2EIDPF8dvYIiToZCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWzRMhJB+qqWlBSUlJdx20y9ynCssh43tJQwwF6nldXd3h4WFxauuot5RgCOknyopKYGPj49G+mda8hYUFGDy5Mm9X6k+hgIcIf2Uu7s7CgoKuO3S6gZI/laIlLcnYqzTII28hogCHCH9lIWFhdpVmeD2I4hO/wKPCV7wHjFEjzXrO6iTgRDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbeg9waWlpEIvFMDMzg6+vLy5cuPDC/A0NDVizZg2cnJwgEokwZswYHDt2jNv/H//xHzAyMlL7GOprKoQYOr2+qpWZmQmJRIL09HT4+voiNTUVwcHBKC0thb29vUb+1tZWBAYGwt7eHgcOHICzszNu376NQYMGqeUbP348fvzxR27b2JjeSCPEEOn1Lz8lJQXR0dGIiooCAKSnp+Po0aPIyMjAhx9+qJE/IyMD9fX1OHfuHLdqkFgs1shnbGwMR0fHXq17f0TT6xBDo7cA19raioKCAsTHx3NpAoEAAQEByMvL03rM4cOH4e/vjzVr1uD777+HnZ0dli5divXr10MoFHL5bt68iaFDh8LMzAz+/v5ITk7G8OHDO6yLXC6HXC7ntmUyGYBny7ApFIqXbWqfUVRUBF9fX410bdPrnD9/HpMmTer9SpEe09bWxv3k0++tNrq2T28Brq6uDkqlEg4ODmrpDg4OalcZv3br1i2cOHECy5Ytw7Fjx1BWVobVq1dDoVAgMTERAODr64vdu3dj7NixqK6uRlJSEmbMmIGioqIOF4lNTk5GUlKSRvrx48d5dRUjl8uxbds2bru2BdhbJkS4mxIOzzWzsrIS1dXVr7iG5GXcaQIAY+Tn5+Nekb5r07taWlp0ytevHk6pVCrY29vjyy+/hFAohI+PD+7du4ctW7ZwAW7u3Llcfk9PT/j6+mLEiBH461//ipUrV2otNz4+HhKJhNtuXzU7KCiI1ws/X62qR+bOS3h7yRR4DR+s7+qQl3S1qh4ovAQ/Pz/en8/2u6zO6C3A2draQigUora2Vi29tra2w+dnTk5OMDExUbsd9fDwQE1NDVpbW2FqaqpxzKBBgzBmzBiUlZV1WBeRSASRSKSRbmJiwusVwts7X4yNjXndTkNhSOdT1/bpbZiIqakpfHx8IJVKuTSVSgWpVAp/f3+tx7z22msoKyuDSqXi0m7cuAEnJyetwQ0AmpqaUF5eDicnp55tACGkz9PrODiJRIKdO3diz549KC4uRkxMDJqbm7le1fDwcLVOiJiYGNTX12Pt2rW4ceMGjh49is2bN2PNmjVcnri4OJw6dQqVlZU4d+4c3nzzTQiFQixZsuSVt48Qol96fQYXFhaGhw8fIiEhATU1NfD29kZWVhbX8VBVVQWB4B8x2MXFBdnZ2Xj//ffh6ekJZ2dnrF27FuvXr+fy3L17F0uWLMGjR49gZ2eH6dOnIz8/H3Z2dq+8fYQQ/dJ7J0NsbCxiY2O17svNzdVI8/f3R35+fofl7d+/v6eqRgjp5/T+qhYhhPQWCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDe0vswEUKI7irqmtEsb9O6r/xhM/fzRXMgWoqM4Wpr2Sv162sowBHST1TUNWPW1txO831woLDTPCfjXjeIIEcBjpB+ov3KLTXMG272AzT3/yLHkdw8zH/dH5bmmpNHAEDZgya8l3mlw6tAvqEAR0g/42Y/ABOcrTXSFQoFauyAySNseD+biK6ok4EQwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFw0R47EWj3gEa+U74jwIcT+k66h2gke+EvyjA8VRno94BGvlO+E/vz+DS0tIgFothZmYGX19fXLhw4YX5GxoasGbNGjg5OUEkEmHMmDE4duzYS5XJZ+2j3rV9Jo+wwZT/H/neUZ6OgiMh/YFeA1xmZiYkEgkSExNx+fJleHl5ITg4GA8ePNCav7W1FYGBgaisrMSBAwdQWlqKnTt3wtnZudtlEkL4S68BLiUlBdHR0YiKisK4ceOQnp4OCwsLZGRkaM2fkZGB+vp6HDp0CK+99hrEYjFmzpwJLy+vbpdJCOEvvT2Da21tRUFBgdrCzgKBAAEBAcjLy9N6zOHDh+Hv7481a9bg+++/h52dHZYuXYr169dDKBR2q0wAkMvlkMvl3LZMJgPw7OVlhULxsk3Vi7a2Nu5nR21oT39RG3Uph7wanZ0LQzqfutZdbwGurq4OSqWSW+S5nYODA0pKSrQec+vWLZw4cQLLli3DsWPHUFZWhtWrV0OhUCAxMbFbZQJAcnIykpKSNNKPHz8OCwuLbrRO/+40AYAxzpw5g9udPEbLycnpkXJI79L1XBjC+WxpadEpX7/qRVWpVLC3t8eXX34JoVAIHx8f3Lt3D1u2bEFiYmK3y42Pj4dEIuG2ZTIZXFxcEBQUBCsrq56o+it37b4MWwvzMX36dIwfqr0NCoUCOTk5CAwM7HB6HV3KIa9GZ+fCkM5n+11WZ/QW4GxtbSEUClFbW6uWXltbC0dHR63HODk5wcTEBEKhkEvz8PBATU0NWltbu1UmAIhEIohEmsMkTExM+u28Wu0Dd42NjTttw4va2ZVySO/S9VwYwvnUte5662QwNTWFj48PpFIpl6ZSqSCVSuHv76/1mNdeew1lZWVQqVRc2o0bN+Dk5ARTU9NulUkI4S+99qJKJBLs3LkTe/bsQXFxMWJiYtDc3IyoqCgAQHh4uFqHQUxMDOrr67F27VrcuHEDR48exebNm7FmzRqdyySEGA69PoMLCwvDw4cPkZCQgJqaGnh7eyMrK4vrJKiqqoJA8I8Y7OLiguzsbLz//vvw9PSEs7Mz1q5di/Xr1+tcJiHEcOi9kyE2NhaxsbFa9+Xm5mqk+fv7Iz8/v9tlEkIMh95f1SKEkN5CAY4Qwlt6v0UlhOhGrnwKgdk9VMhKITDTHKXb1taG+233UVxf3OH8fhWyJgjM7kGufApAc+lBvqEAR0g/cb/5Nixd/4wNnUyO80XWFy/cb+kK3G/2hg/43/FGAY6QfmKo5Qg0V/wr/hTmjVFaprFqa2vD2TNn8dr01zq8git/0IS1mVcwdNaI3q5un0ABjpB+QiQ0g+qpM1ytxmLcEO0r21cYV8BjsEeHI/1VTxuhevoQIqFZb1e3T6BOBkIIb1GAI4TwFt2i8lRnPW4A9boR/qMAx1O69rgB1OtG+IsCHE911uMGUK8b4T8KcDzVWY8bQL1uhP+ok4EQwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4Qwlt9IsClpaVBLBbDzMwMvr6+uHCh4+H3u3fvhpGRkdrHzEx9jFZkZKRGnpCQkN5uBiGkj9H7QN/MzExIJBKkp6fD19cXqampCA4ORmlpKezt7bUeY2VlhdLSUm7byMhII09ISAh27drFbWtb2JkQwm9dvoK7detWj1YgJSUF0dHRiIqKwrhx45Ceng4LCwtkZGR0eIyRkREcHR25j7YlAUUikVoeGxubHq03IaTv6/IVnJubG2bOnImVK1fit7/9rcbtYVe0traioKBAbXFngUCAgIAA5OXldXhcU1MTRowYAZVKhcmTJ2Pz5s0YP368Wp7c3FzY29vDxsYGs2fPxqZNmzBkyBCt5cnlcsjlcm5bJpMBePYqk0Kh6Hb79KmtrY372VEb2tNf1EZdyiGvRmfnwpDOp65173KAu3z5Mnbt2gWJRILY2FiEhYVh5cqVmDp1apcrWVdXB6VSqXEF5uDggJKSEq3HjB07FhkZGfD09ERjYyO2bt2KadOm4dq1axg2bBiAZ7enoaGhcHV1RXl5OTZs2IC5c+ciLy8PQqFQo8zk5GQkJSVppB8/fhwWFhZdbldfcKcJAIxx5swZ3Nb+rj0nJyenR8ohvUvXc2EI57OlpUWnfEaMMdadL2hra8Phw4exe/duZGVlYcyYMVixYgWWL18OOzs7ncq4f/8+nJ2dce7cOfj7+3Pp69atw6lTp3D+/PlOy1AoFPDw8MCSJUuwceNGrXlu3bqFUaNG4ccff8ScOXM09mu7gnNxcUFdXR2srKx0aktfc+2+DIt25ONQjB/GD9XeBoVCgZycHAQGBnb4sr0u5ZBXo7NzYUjnUyaTwdbWFo2NjS/8G+12J4OxsTFCQ0Pxm9/8Bl988QXi4+MRFxeHDRs24J133sGnn34KJyenF5Zha2sLoVCI2tpatfTa2lo4OjrqVA8TExNMmjQJZWVlHeYZOXIkbG1tUVZWpjXAiUQirZ0QJiYmHf6i9HXt0x8ZGxt32oYXtbMr5ZDepeu5MITzqWvduz1M5NKlS1i9ejWcnJyQkpKCuLg4lJeXIycnB/fv38fChQs7LcPU1BQ+Pj6QSqVcmkqlglQqVbuiexGlUonCwsIXBtO7d+/i0aNHnQZcQgi/dPkKLiUlBbt27UJpaSnmzZuHvXv3Yt68eRAInsVKV1dX7N69G2KxWKfyJBIJIiIiMGXKFEydOhWpqalobm5GVFQUACA8PBzOzs5ITk4GAHzyySfw8/ODm5sbGhoasGXLFty+fRu/+93vADzrgEhKSsJbb70FR0dHlJeXY926dXBzc0NwcHBXm0sI6ce6HOB27NiBFStWIDIyssMrInt7e3z11Vc6lRcWFoaHDx8iISEBNTU18Pb2RlZWFtfxUFVVxQVPAHj8+DGio6NRU1MDGxsb+Pj44Ny5cxg3bhwAQCgU4ueff8aePXvQ0NCAoUOHIigoCBs3bqSxcIQYmC4HuJycHAwfPlwt6AAAYwx37tzB8OHDYWpqioiICJ3LjI2NRWxsrNZ9ubm5atvbt2/H9u3bOyzL3Nwc2dnZOn83IYS/uvwMbtSoUairq9NIr6+vh6ura49UihBCekKXA1xHo0qamppeatAvIYT0NJ1vUSUSCYBnr0klJCSoDYBVKpU4f/48vL29e7yChBDSXToHuJ9++gnAsyu4wsJCmJqacvtMTU3h5eWFuLi4nq8hIYR0k84B7uTJkwCAqKgo/OlPf+q3I/wJIYajy72ov56CiBBC+jKdAlxoaCh2794NKysrhIaGvjDvwYMHe6RihBDysnQKcNbW1tykktbW2ldJJ4SQvkanAPfr21K6RSWE9Bd9Yk0GQgjpDTpdwU2aNEnrugfaXL58+aUqRAghPUWnALdo0aJergYhpDO/KJQAgKJ7jVr3N/8ix6WHgOPtx7A01z6xRNmDpl6rX1+kU4BLTEzs7XoQQjpR/v/B6cODhS/IZYx9ZRc7LctSpPcF9V4Jw2glITwQNP7ZLNej7AfA3ERzbZHS6kZ8cKAQ2347EWOdOh7tYCkyhqutZa/Vsy/RKcANHjwYN27cgK2tLWxsbF74PK6+vr7HKkcI+YfBlqZYPHV4h/vbV8waZWeJCc40nAvQMcBt374dAwcO5P5b1w4HQgjRJ50C3K8nr4yMjOytuhBCSI/q8jg4oVCIBw8eaKQ/evRI65qjhBCiLz024aVcLlebQokQQvRN517Uzz//HMCzCS//+7//GwMG/GNZbKVSif/93/+Fu7t7tyqRlpaGLVu2oKamBl5eXvjzn/+MqVOnas27e/dubsWtdiKRCE+fPuW2GWNITEzEzp070dDQgNdeew07duzA6NGju1W//qizMVMAjZsi/KdzgGtf6IUxhvT0dLXbUVNTU4jFYqSnp3e5ApmZmZBIJEhPT4evry9SU1MRHByM0tJS2Nvbaz3GysoKpaWl3PbznR6fffYZPv/8c+zZsweurq74+OOPERwcjOvXrxvMtOq6jZkCaNwU4TOdf2srKioAALNmzcLBgwdhY2PTIxVISUlBdHQ0d1WWnp6Oo0ePIiMjAx9++KHWY4yMjODo6Kh1H2MMqamp+Oijj7jFp/fu3QsHBwccOnQIixcv7pF693WdjZkCaNwU4b8u/2+5fWbfntDa2oqCggLEx8dzaQKBAAEBAcjLy+vwuKamJowYMQIqlQqTJ0/G5s2bMX78eADPAnFNTQ0CAgK4/NbW1vD19UVeXp7BBLjOxkwBNG6K8F+XA9yKFSteuD8jI0Pnsurq6qBUKrlFnts5ODigpKRE6zFjx45FRkYGPD090djYiK1bt2LatGm4du0ahg0bhpqaGq6M58ts3/c8uVwOuVzObctkMgCAQqGAQqHQuT39TXuAa2tr43U7DYUhnU9d29flAPf48WONLyoqKkJDQwNmz57d1eK6zN/fH/7+/tz2tGnT4OHhgf/6r//Cxo0bu1VmcnIykpKSNNKPHz+utnoY39xpAgBj5Ofn416RvmtDXpYhnc+Wlhad8nU5wH333XcaaSqVCjExMRg1alSXyrK1tYVQKERtba1aem1tbYfP2J5nYmKCSZMmoaysDAC442pra+Hk5KRWZkfLGsbHx3PLIgLPruBcXFwQFBTE68V1rlbVA4WX4OfnB6/hg/VdHfKSDOl8tt9ldaZHusYEAgEkEglef/11rFu3TufjTE1N4ePjA6lUyk3JpFKpIJVKERsbq1MZSqUShYWFmDdvHgDA1dUVjo6OkEqlXECTyWQ4f/48YmJitJYhEokgEmkOkzAxMYGJiYnO7elvjI2NuZ98bqehMKTzqWv7eqzvv7y8nHsG0BUSiQQRERGYMmUKpk6ditTUVDQ3N3O9quHh4XB2dkZycjIA4JNPPoGfnx/c3NzQ0NCALVu24Pbt2/jd734H4FkP63vvvYdNmzZh9OjR3DCRoUOH0rx2hBiYLge4X9/KAc+GZVRXV+Po0aNq76zqKiwsDA8fPkRCQgJqamrg7e2NrKwsrpOgqqoKAsE/Xrh4/PgxoqOjUVNTAxsbG/j4+ODcuXMYN24cl2fdunVobm7GqlWr0NDQgOnTpyMrK8tgxsARQp4xYh29e9WBWbNmqW0LBALY2dlh9uzZWLFiBXeZ3J/JZDJYW1ujsbGR18/grtx+hEU78nEoxg/eI4bouzrkJRnS+dT1b1Sv4+AIIaQ30apahBDeogBHCOEtCnCEEN6iAEcI4a0eC3B3797FqlWreqo4Qgh5aT0W4B49eoSvvvqqp4ojhJCXRreohBDeogBHCOEtCnCEEN7S+U2G0NDQF+5vaGh42boQQkiP0jnAWVu/eEpra2trhIeHv3SFCCGkp+gc4Hbt2tWb9SCEkB5Hz+AIIbyl8xVcZ4vNtOvKojOEENKbdA5wu3fvxogRIzBp0iR0cQo5QgjRC50DXExMDL799ltUVFQgKioK7777LgYP5vfCFoSQ/k3nZ3BpaWmorq7GunXr8D//8z9wcXHBO++8g+zsbLqiI4T0SV3qZBCJRFiyZAlycnJw/fp1jB8/HqtXr4ZYLEZTU1Nv1ZEQQrql272oAoEARkZGYIxBqVS+VCXS0tIgFothZmYGX19fXLhwQafj9u/fDyMjI43VsiIjI2FkZKT2CQkJeak6EkL6ny4FOLlcjm+//RaBgYEYM2YMCgsL8Z//+Z+oqqrCgAEDulWBzMxMSCQSJCYm4vLly/Dy8kJwcDAePHjwwuMqKysRFxeHGTNmaN0fEhKC6upq7vPtt992q36EkP5L5wC3evVqODk54Y9//CPmz5+PO3fu4G9/+xvmzZuntqxfV6WkpCA6OhpRUVEYN24c0tPTYWFh8cLhJkqlEsuWLUNSUhJGjhypNY9IJIKjoyP3sbGx6XYdCSH9k869qOnp6Rg+fDhGjhyJU6dO4dSpU1rzHTx4UOcvb21tRUFBAeLj47k0gUCAgIAA5OXldXjcJ598Ant7e6xcuRKnT5/Wmic3Nxf29vawsbHB7NmzsWnTJgwZwu+l1Agh6nQOcOHh4TAyMurRL6+rq4NSqeQWeW7n4OCAkpISrcecOXMGX331Fa5cudJhuSEhIQgNDYWrqyvKy8uxYcMGzJ07F3l5eRAKhRr55XI55HI5ty2TyQAACoUCCoWiGy3rH9ra2riffG6noTCk86lr+7o00Fffnjx5guXLl2Pnzp2wtbXtMN/ixYu5/544cSI8PT0xatQo5ObmYs6cORr5k5OTkZSUpJF+/PhxWFhY9Ezl+6A7TQBgjPz8fNwr0ndtyMsypPPZ0tKiUz69LkNva2sLoVCI2tpatfTa2lo4Ojpq5C8vL0dlZSUWLFjApalUKgCAsbExSktLMWrUKI3jRo4cCVtbW5SVlWkNcPHx8ZBIJNy2TCaDi4sLgoKCeL2y/dWqeqDwEvz8/OA1nAZt93eGdD7b77I6o9cAZ2pqCh8fH0ilUm6oh0qlglQqRWxsrEZ+d3d3FBYWqqV99NFHePLkCf70pz/BxcVF6/fcvXsXjx49gpOTk9b9IpEIIpFII93ExAQmJiZdbFX/YWxszP3kczsNhSGdT13bp9cABwASiQQRERGYMmUKpk6ditTUVDQ3NyMqKgrAs2d/zs7OSE5OhpmZGSZMmKB2/KBBgwCAS29qakJSUhLeeustODo6ory8HOvWrYObmxuCg4NfadsIIfql9wAXFhaGhw8fIiEhATU1NfD29kZWVhbX8VBVVdWlYShCoRA///wz9uzZg4aGBgwdOhRBQUHYuHGj1qs0Qgh/6T3AAUBsbKzWW1Lg2XCPF3m+88Pc3BzZ2dk9VDNCSH9GE14SQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLf6RIBLS0uDWCyGmZkZfH19ceHCBZ2O279/P4yMjLg1VdsxxpCQkAAnJyeYm5sjICAAN2/e7IWaE0L6Mr0HuMzMTEgkEiQmJuLy5cvw8vJCcHAwHjx48MLjKisrERcXhxkzZmjs++yzz/D5558jPT0d58+fh6WlJYKDg/H06dPeagYhpA/Se4BLSUlBdHQ0oqKiMG7cOKSnp8PCwgIZGRkdHqNUKrFs2TIkJSVh5MiRavsYY0hNTcVHH32EhQsXwtPTE3v37sX9+/dx6NChXm4NIaQv0eu6qK2trSgoKEB8fDyXJhAIEBAQgLy8vA6P++STT2Bvb4+VK1fi9OnTavsqKipQU1ODgIAALs3a2hq+vr7Iy8vD4sWLNcqTy+WQy+XctkwmAwAoFAooFIput6+va2tr437yuZ2GwpDOp67t02uAq6urg1Kp5Faxb+fg4ICSkhKtx5w5cwZfffUVrly5onV/TU0NV8bzZbbve15ycjKSkpI00o8fPw4LC4vOmtFv3WkCAGPk5+fjXpG+a0NeliGdz5aWFp3y9YmV7XX15MkTLF++HDt37oStrW2PlRsfHw+JRMJty2QyuLi4ICgoCFZWVj32PX3N1ap6oPAS/Pz84DV8sL6rQ16SIZ3P9ruszug1wNna2kIoFKK2tlYtvba2Fo6Ojhr5y8vLUVlZiQULFnBpKpUKAGBsbIzS0lLuuNraWjg5OamV6e3trbUeIpEIIpFII93ExAQmJiZdbld/YWxszP3kczsNhSGdT13bp9dOBlNTU/j4+EAqlXJpKpUKUqkU/v7+Gvnd3d1RWFiIK1eucJ833ngDs2bNwpUrV+Di4gJXV1c4OjqqlSmTyXD+/HmtZRJC+Evvt6gSiQQRERGYMmUKpk6ditTUVDQ3NyMqKgoAEB4eDmdnZyQnJ8PMzAwTJkxQO37QoEEAoJb+3nvvYdOmTRg9ejRcXV3x8ccfY+jQoRrj5Qgh/Kb3ABcWFoaHDx8iISEBNTU18Pb2RlZWFtdJUFVVBYGgaxea69atQ3NzM1atWoWGhgZMnz4dWVlZMDMz640m9BstLS1qnTel1Q2Q15ShuMgcqkeD1PK6u7vzuoOFGAYjxhjTdyX6GplMBmtrazQ2NvKqk+Hy5cvw8fHRKW9BQQEmT57cyzUiPenK7UdYtCMfh2L84D1iiL6r06t0/RvV+xUceXXc3d1RUFDAbTf9IsfRk3n4zSx/DDAXaeQlpL+jAGdALCws1K7KFAoFHtc9gP/UKbzvdSOGSe+vahFCSG+hAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S2aLomQfopmaO4cBThC+qmSkhKtMzQv3aOZ11BnaKYAR0g/RTM0d65PPINLS0uDWCyGmZkZfH19ceHChQ7zHjx4EFOmTMGgQYNgaWkJb29v7Nu3Ty1PZGQkjIyM1D4hISG93QxCXqn2GZrbP/5Tp2DaxFHwnzpFLX3y5MkGeXsK9IEruMzMTEgkEqSnp8PX1xepqakIDg5GaWkp7O3tNfIPHjwY//7v/w53d3eYmpriyJEjiIqKgr29PYKDg7l8ISEh2LVrF7etbWFnQgi/6f0KLiUlBdHR0YiKisK4ceOQnp4OCwsLZGRkaM3/+uuv480334SHhwdGjRqFtWvXwtPTE2fOnFHLJxKJ4OjoyH1sbGxeRXMIIX2IXq/gWltbUVBQgPj4eC5NIBAgICAAeXl5nR7PGMOJEydQWlqKTz/9VG1fbm4u7O3tYWNjg9mzZ2PTpk0YMkT7UmpyuRxyuZzblslkAJ4tyqJQKLrTtH6hvW18bqMhMaTzqWsb9Rrg6urqoFQquUWe2zk4OKh1fz+vsbERzs7OkMvlEAqF+OKLLxAYGMjtDwkJQWhoKFxdXVFeXo4NGzZg7ty5yMvLg1Ao1CgvOTkZSUlJGunHjx83iGcXOTk5+q4CeUlKpRLXr1/H48ePUVhYiHHjxmn9XeeLlpYWnfLp/RlcdwwcOBBXrlxBU1MTpFIpJBIJRo4ciddffx0AsHjxYi7vxIkT4enpiVGjRiE3Nxdz5szRKC8+Ph4SiYTblslkcHFxQVBQEK8Wfn6eQqFATk4OAgMDadnAfuy7777D+vXrUVlZyaWJxWJ8+umnePPNN/VXsV7UfpfVGb0GOFtbWwiFQtTW1qql19bWwtHRscPjBAIB3NzcAADe3t4oLi5GcnIyF+CeN3LkSNja2qKsrExrgBOJRFo7IUxMTAziD99Q2slHBw8exOLFizF//nzs27cPd+/exbBhw/DZZ59h8eLFOHDgAEJDQ/VdzR6n6++rXjsZTE1N4ePjA6lUyqWpVCpIpVL4+/vrXI5KpVJ7hva8u3fv4tGjR3Bycnqp+hLSlyiVSnzwwQeYP38+Dh06BF9fX5ibm8PX1xeHDh3C/PnzERcXB6VSqe+q6o3ee1ElEgl27tyJPXv2oLi4GDExMWhubkZUVBQAIDw8XK0TIjk5GTk5Obh16xaKi4uxbds27Nu3D++++y4AoKmpCb///e+Rn5+PyspKSKVSLFy4EG5ubmrDSAjp706fPo3Kykps2LABAoH6n7JAIEB8fDwqKipw+vRpPdVQ//T+DC4sLAwPHz5EQkICampq4O3tjaysLK7joaqqSu3kNTc3Y/Xq1bh79y7Mzc3h7u6Or7/+GmFhYQAAoVCIn3/+GXv27EFDQwOGDh2KoKAgbNy4kcbCEV6prq4GAEyYMEHr/vb09nyGSO8BDgBiY2MRGxurdV9ubq7a9qZNm7Bp06YOyzI3N0d2dnZPVo+QPqn9kUtRURH8/Pw09hcVFanlM0R6v0UlhHTPjBkzIBaLsXnzZqhUKrV9KpUKycnJcHV1xYwZM/RUQ/3rE1dwhJCuEwqF2LZtG377299i4cKFCAwMxM2bN3H79m3k5OTg6NGjOHDgAK/Hw3WGAhwh/VhoaCji4uKwfft2HDlyhEs3NjZGXFwcL4eIdAUFOEL6sYMHD2Lr1q34zW9+w13BjR49Gjk5Odi6dSv8/PwMOsgZMcaYvivR18hkMlhbW6OxsZH3bzIcO3YM8+bNo4G+/ZBSqYSbmxsmTpyIQ4cOQalUcudTKBRi0aJFKCoqws2bN3l3m6rr3yh1MhDST9E4uM5RgCOkn6JxcJ2jAEdIP/XrcXDa0Dg4CnCE9Fs0Dq5zFOAI6afax8EdOXIEixYtQn5+Pn755Rfk5+dj0aJFOHLkCLZu3cq7DoauoGEihPRjoaGhOHDgAD744AP80z/9E5fu6urK26mSuoICHCH9XGhoKBYuXIiTJ0/ihx9+wNy5czFr1iyDvnJrRwGOEB4QCoWYOXMmmpubMXPmTApu/4+ewRFCeIsCHCGEtyjAEUJ4i57BadH+eq6uK/f0VwqFAi0tLZDJZPQuKg8Y0vls/9vs7FV6CnBaPHnyBADg4uKi55oQQl7kyZMnsLa27nA/zSaihUqlwv379zFw4EAYGRnpuzq9pn391zt37vB61hRDYUjnkzGGJ0+eYOjQoRoTDfwaXcFpIRAIMGzYMH1X45WxsrLi/R+EITGU8/miK7d21MlACOEtCnCEEN6iAGfARCIREhMTab1YnqDzqYk6GQghvEVXcIQQ3qIARwjhLQpwhBDeogBHCOEtCnA88frrr+O9997TdzVIN/S1c9fX6vMyKMARwgOtra36rkLfxEi/FxERwQCofZydndkXX3yhlu/y5cvMyMiIVVZWMsYY27ZtG5swYQKzsLBgw4YNYzExMezJkydqx5w+fZpNnz6dmZmZsWHDhrF//dd/ZU1NTa+sbXyn7dyVlZWxFStWMLFYzMzMzNiYMWNYamqqxnELFy5kmzZtYk5OTkwsFjPGGDt79izz8vJiIpGI+fj4sO+++44BYD/99BN3bGFhIQsJCWGWlpbM3t6evfvuu+zhw4cd1qeiouJV/XP0OApwPNDQ0MD8/f1ZdHQ0q66uZtXV1SwuLo5Nnz5dLd8HH3yglrZ9+3Z24sQJVlFRwaRSKRs7diyLiYnh9peVlTFLS0u2fft2duPGDXb27Fk2adIkFhkZ+craxnfazt3Tp09ZQkICu3jxIrt16xb7+uuvmYWFBcvMzOSOi4iIYAMGDGDLly9nRUVFrKioiDU2NrLBgwezd999l127do0dO3aMjRkzRi3APX78mNnZ2bH4+HhWXFzMLl++zAIDA9msWbM6rE9bW5s+/ml6BAU4npg5cyZbu3Ytt/3TTz8xIyMjdvv2bcYYY0qlkjk7O7MdO3Z0WMbf/vY3NmTIEG575cqVbNWqVWp5Tp8+zQQCAfvll196tgEG7Plzp82aNWvYW2+9xW1HREQwBwcHJpfLubQdO3awIUOGqJ2bnTt3qgW4jRs3sqCgILWy79y5wwCw0tJSnevTX9AzOJ7y9vaGh4cHvvnmGwDAqVOn8ODBA7z99ttcnh9//BFz5syBs7MzBg4ciOXLl+PRo0doaWkBAFy9ehW7d+/GgAEDuE9wcDBUKhUqKir00i5DkZaWBh8fH9jZ2WHAgAH48ssvUVVVpZZn4sSJMDU15bZLS0vh6ekJMzMzLm3q1Klqx1y9ehUnT55UO6fu7u4AgPLy8l5skX5QgOOxZcuWcQHum2++QUhICIYMGQIAqKysxPz58+Hp6Ym///3vKCgoQFpaGoB/PLBuamrCP//zP+PKlSvc5+rVq7h58yZGjRqln0YZgP379yMuLg4rV67E8ePHceXKFURFRWl0JFhaWna57KamJixYsEDtnF65cgU3b95UW1eVL2g+OJ4wNTWFUqlUS1u6dCk++ugjFBQU4MCBA0hPT+f2FRQUQKVSYdu2bdyEgX/961/Vjp88eTKuX78ONze33m+AAXv+3J09exbTpk3D6tWruTRdrq7Gjh2Lr7/+GnK5nHvh/uLFi2p5Jk+ejL///e8Qi8UwNtb+56/td6m/ois4nhCLxTh//jwqKytRV1cHlUoFsViMadOmYeXKlVAqlXjjjTe4/G5ublAoFPjzn/+MW7duYd++fWoBEADWr1+Pc+fOITY2lvu//Pfff4/Y2NhX3Txee/7cjR49GpcuXUJ2djZu3LiBjz/+WCNQabN06VKoVCqsWrUKxcXFyM7OxtatWwGAm5l6zZo1qK+vx5IlS3Dx4kWUl5cjOzsbUVFRXFDT9rvUb+n7ISDpGaWlpczPz4+Zm5urde1/8cUXDAALDw/XOCYlJYU5OTkxc3NzFhwczPbu3csAsMePH3N5Lly4wAIDA9mAAQOYpaUl8/T0ZH/4wx9eUasMw/PnrqSkhEVGRjJra2s2aNAgFhMTwz788EPm5eXFHdM+TOR5Z8+eZZ6enszU1JT5+Piwb775hiuz3Y0bN9ibb77JBg0axMzNzZm7uzt77733mEql0lqf/jxMhKZLIoTH/vKXvyAqKgqNjY0wNzfXd3VeOXoGRwiP7N27FyNHjoSzszOuXr2K9evX45133jHI4AZQgCOEV2pqapCQkICamho4OTnh7bffxh/+8Ad9V0tv6BaVEMJb1ItKCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwRK8iIyNhZGQEIyMjmJiYwMHBAYGBgcjIyOjSK0K7d+/GoEGDeq+iHYiMjMSiRYte+fcS3VCAI3oXEhKC6upqVFZW4ocffsCsWbOwdu1azJ8/H21tbfquHunP9PumGDF0Hb1TKZVKGQC2c+dOxtiLp1c/efKkxjTbiYmJjDHG9u7dy3x8fNiAAQOYg4MDW7JkCautreW+p76+ni1dupTZ2toyMzMz5ubmxjIyMrj9VVVV7O2332bW1tbMxsaGvfHGG9y7mYmJiRrfe/LkyV75dyLdQ1dwpE+aPXs2vLy8cPDgQQCAQCDA559/jmvXrmHPnj04ceIE1q1bBwCYNm0aUlNTYWVlherqalRXVyMuLg4AoFAosHHjRly9ehWHDh1CZWUlIiMjue/5+OOPcf36dfzwww8oLi7Gjh07YGtryx0bHByMgQMH4vTp0zh79iwGDBiAkJAQtLa2Ii4uDu+88w53BVpdXY1p06a92n8o8mL6jrDEsHV0BccYY2FhYczDw0PrvuenV9+1axeztrbu9PsuXrzIAHBXfwsWLGBRUVFa8+7bt4+NHTuWm2WDMcbkcjkzNzdn2dnZndaf6B9dwZE+izHGzWPW2fTqHSkoKMCCBQswfPhwDBw4EDNnzgQAbvrvmJgY7N+/H97e3li3bh3OnTvHHXv16lWUlZVh4MCB3PTegwcPxtOnT3k5vTcfUYAjfVZxcTFcXV11ml5dm+bmZgQHB8PKygp/+ctfcPHiRXz33Xdqx82dOxe3b9/G+++/j/v372POnDnc7W1TUxN8fHw0pve+ceMGli5d2sutJz2BZhMhfdKJEydQWFiI999/X6fp1bVNs11SUoJHjx7hj3/8I1xcXAAAly5d0vguOzs7REREICIiAjNmzMDvf/97bN26FZMnT0ZmZibs7e1hZWWltZ58mt6bj+gKjuidXC5HTU0N7t27h8uXL2Pz5s1YuHAh5s+fj/DwcJ2mVxeLxWhqaoJUKkVdXR1aWlowfPhwmJqacscdPnwYGzduVDsuISEB33//PcrKynDt2jUcOXIEHh4eAJ4t2mNra4uFCxfi9OnTqKioQG5uLv7t3/4Nd+/e5b73559/RmlpKerq6qBQKF7NPxrRjb4fAhLD9uuV1I2NjZmdnR0LCAhgGRkZTKlUcvl0mV79X/7lX9iQIUPUhol88803TCwWM5FIxPz9/dnhw4c11gn18PBg5ubmbPDgwWzhwoXs1q1bXJnV1dUsPDyc2draMpFIxEaOHMmio6NZY2MjY4yxBw8ecFO6g4aJ9Dk0HxwhhLfoFpUQwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbz1fyoc3FOe3LbKAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:36:09.024114Z", + "iopub.status.busy": "2024-02-29T05:36:09.023330Z", + "iopub.status.idle": "2024-02-29T05:36:09.234183Z", + "shell.execute_reply": "2024-02-29T05:36:09.233273Z" + }, + "papermill": { + "duration": 0.236662, + "end_time": "2024-02-29T05:36:09.236192", + "exception": false, + "start_time": "2024-02-29T05:36:08.999530", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVUElEQVR4nO2dd3xT5ffH3xlNW0pbdlugZRYoeyOgbGTJcgHylSFLAREQEFCGoBZ/MhUEQQsqVGUJqIiykSFIKXtDoayW3UVHxv39EZImbdImbdo05Xm/XvdFc3PHubT55DznOc85MkmSJAQCgcDFkDvbAIFAIMgJQrwEAoFLIsRLIBC4JEK8BAKBSyLESyAQuCRCvAQCgUsixEsgELgkQrwEAoFLonS2AfmNTqfj9u3beHt7I5PJnG2OQCDIgCRJJCQkULZsWeRy6/7VMydet2/fJjAw0NlmCASCbLhx4wbly5e3+v4zJ17e3t6A/j/Gx8fHydYIBIKMxMfHExgYaPysWuOZEy/DUNHHx0eIl0BQgMkurCMC9gKBwCUR4iUQCFwSIV4CgcAleeZiXgLXQpIkNBoNWq3W2aYIHIRCoUCpVOY6VUmIl6DAkpaWxp07d3jy5ImzTRE4mCJFihAQEIBKpcrxNYR4ZYFOJ5Gi0VJEJf6b8hudTkdUVBQKhYKyZcuiUqlEUnEhQJIk0tLSuHfvHlFRUQQHB2eZiJoV4lNphfMx8UzecIoqpYsy7/V6zjbnmSMtLQ2dTkdgYCBFihRxtjkCB+Lp6YmbmxvXr18nLS0NDw+PHF1HBOyt8CRNy/Ebj9lw7Cb/XXvobHOeWXL6rSwo2Dji9yr+MqzQMKg4fZvolxFN23QajVbnZIsEAoEpQryyYFLnGhQr4sb5mAS+P3Td2eYIBAIThHhlQQkvFZM61QBgwfaLxManONkigcA1GDRoEL169crTewjxyoa+TQKpF1iMxFQNn20952xzBC5AmzZtGDt2rLPNKPQI8coGuVzGJz1rI5PB5uO3OXjlvrNNEgjyBbVa7WwTskSIlw3UKe9L/2ZBAEzffIY0jQje5zeSJPEkTeOUzZ6m8oMGDWLv3r0sWrQImUyGTCajfPnyLF261Oy4yMhI5HI516/rY6nz58+nTp06eHl5ERgYyMiRI0lMTDQ7Z//+/bzwwgt4enoSGBjImDFjSEpKssmuO3fu0K1bNzw9PalUqRLh4eFUrFiRhQsXGo+RyWQsXbqUHj164OXlxaeffopWq2XIkCFUqlQJT09PqlevzqJFi8yurdVqGT9+PMWKFaNkyZJMmjTJrv+znCLyvGxk4os1+PNUDJfvJrLyQBQjWldxtknPFMlqLTWn/+WUe5+d1cnmROVFixZx8eJFateuzaxZswCYN28e4eHhvPPOO8bj1qxZQ8uWLalQoQKgTx348ssvqVSpElevXmXkyJFMmjSJr7/+GoArV67QuXNnPvnkE8LCwrh37x6jR49m9OjRrFy5Mlu7BgwYwP3799mzZw9ubm6MHz+eu3fvZjpu5syZzJkzh4ULF6JUKtHpdJQvX55169ZRsmRJDh48yPDhwwkICOD11183Pt+qVasICwsjJCSEefPm8euvv9KuXTub/s9yikzKD4ksQMTHx+Pr60tcXJzd9bzWHb3BxPUnKaJSsPP91gT4euaRlYKUlBSioqKoVKkSHh4ePEnTuIR4gT7mVb9+faNXc/z4cRo2bMi1a9cICgpCp9MRFBTERx99xNtvv23xGuvXr+ftt9/m/n19mGLo0KEoFAq++eYb4zH79++ndevWJCUlZZnoef78eUJCQvjvv/9o3LgxAJcvXyY4OJgFCxYY43MymYyxY8eyYMGCLJ9v9OjRxMTEsH79egDKli3LuHHjmDhxIgAajYZKlSrRqFEjNm3aZPEaGX+/ptj6GXWq57V06VKWLl3KtWvXAKhVqxbTp0+nS5cuVs9Zt24d06ZN49q1awQHB/P555/TtWvXfLH3lYbl+fm/G0Rcf8Ts38/ydf9G+XJfAXi6KTg7q5PT7p0b6tevT0hICOHh4UyePJm9e/dy9+5dXnvtNeMxO3bsIDQ0lPPnzxMfH49GoyElJYUnT55QpEgRTpw4wcmTJ1mzZo3xHEmSjMuoQkJCrN7/woULKJVKGjZsaNxXtWpVihcvnulYg7iZsmTJEsLCwoiOjiY5OZm0tDTq168PQFxcHHfu3KFZs2bG45VKJY0bN87zoaNTY17ly5dnzpw5REREcPToUdq1a0fPnj05c+aMxeMPHjxIv379GDJkCJGRkfTq1YtevXpx+vTpfLFXLpcxu2dt5DLYeiqGfRfv5ct9BXqvoIhK6ZTNEWsq+/fvT3h4OADh4eF07tyZkiVLAnDt2jVeeukl6taty4YNG4iIiGDJkiWAfpkUQGJiIiNGjOD48ePG7cSJE1y6dIkqVRwXwvDy8jJ7/fPPPzNhwgSGDBnC33//zfHjxxk8eLDRLmfiVPHq3r07Xbt2JTg4mGrVqvHpp59StGhR/v33X4vHL1q0iM6dOzNx4kRCQkKYPXs2DRs2ZPHixflmc82yPgxsURGAGVvOkKoRpVoE5qhUqkwlfN544w1Onz5NREQE69evp3///sb3IiIi0Ol0zJs3j+eee45q1apx+/Zts/MbNmzI2bNnqVq1aqYtu8oM1atXR6PREBkZadx3+fJlHj16lO2zHDhwgBYtWjBy5EgaNGhA1apVuXLlivF9X19fAgICOHz4sHGfRqMhIiIi22vnlgIz26jVavn5559JSkqiefPmFo85dOgQHTp0MNvXqVMnDh06lB8mGhnXsRqlvd2Jup/Ein1X8/XegoJPxYoVOXz4MNeuXeP+/fvodDoqVqxIixYtGDJkCFqtlh49ehiPr1q1Kmq1mq+++oqrV6/y448/smzZMrNrfvDBBxw8eJDRo0dz/PhxLl26xObNmxk9enS29tSoUYMOHTowfPhwjhw5QmRkJMOHD8fT0zNbrzI4OJijR4/y119/cfHiRaZNm8Z///1ndsx7773HnDlz2LRpE+fPn2fkyJE8fvzY9v+wHOJ08Tp16hRFixbF3d2dt99+m19//ZWaNWtaPDYmJgY/Pz+zfX5+fsTExFi9fmpqKvHx8WZbbvHxcOPDrvoYw+Ldl7nxUNSbEqQzYcIEFAoFNWvWpHTp0kRHRwP6oeOJEyfo3bs3np7pkz316tVj/vz5fP7559SuXZs1a9YQGhpqds26deuyd+9eLl68yAsvvECDBg2YPn06ZcuWtcmmH374AT8/P1q1akXv3r0ZNmwY3t7e2VZ0GDFiBC+//DJ9+vShWbNmPHjwgJEjR5od8/777/Pmm28ycOBAmjdvjre3N71797bJrlwhOZnU1FTp0qVL0tGjR6XJkydLpUqVks6cOWPxWDc3Nyk8PNxs35IlS6QyZcpYvf6MGTMkINMWFxeXK7t1Op30+rKDUoUPfpeGfv9frq4lyExycrJ09uxZKTk52dmmFEpu3LghAdKOHTuccv+sfr9xcXE2fUad7nmpVCqqVq1Ko0aNCA0NpV69epmS4Az4+/sTGxtrti82NhZ/f3+r158yZQpxcXHG7caNGw6xWyaTMbtXbZRyGdvPxrLrfGz2JwkETmLXrl1s2bKFqKgoDh48SN++falYsSKtWrVytmk5xunilRGdTkdqaqrF95o3b87OnTvN9m3fvt1qjAzA3d3d2KPR0b0aq/l589bzlQCYueUsKWoRvBfkP//88w9Fixa1uoF+qc/UqVOpVasWvXv3pnTp0saEVVfFqXleU6ZMoUuXLgQFBZGQkEB4eDh79uzhr7/0yYgDBgygXLlyxvH/e++9R+vWrZk3bx7dunXj559/5ujRoyxfvtxpzzCmfTCbj98i+uETlu65wriO1Zxmi+DZpHHjxhw/fjzLYzp16kSnTs7Jk8srnCped+/eZcCAAdy5cwdfX1/q1q3LX3/9RceOHQGIjo42q7jYokULwsPD+eijj5g6dSrBwcFs2rSJ2rVrO+sRKOquZNpLNRkdHsnSvVd4uWE5KpT0yv5EgcBBeHp6UrVqVWebke+I5UEOQJIk3vzuCPsv36dt9dKEDWoimkXkkqyWjwhcH0csDypwMS9XRCaT8XHPWrgpZOy+cI+/z4rgvUCQ1wjxchBVShdl2AuVAZj121mS00TwXiDIS4R4OZDR7apSrpgntx4ns3j3JWebIxAUaoR4OZAiKn3wHmD5vqtcuZeYzRkCgSCnCPFyMJ1q+dGmemnUWomZW87kS0VJQeHBUnVTazWxnnWEeDkYmUzGxz1qoVLK+efSfbaesr7uUiDIjjt37mRZ3+5ZRohXHlChpBdvPy0TPfv3sySmapxskcBV8ff3x93d3ak2SJKERlPw/oaFeOURI9tUIbCEJzHxKXy1UwTvc40kQVqSczY7h/4JCQn0798fLy8vAgICWLBgQY7boZkOG69du4ZMJmPjxo20bduWIkWKUK9evUwlobJr1PHjjz/SuHFjvL298ff354033jCrZ79nzx5kMhl//vknjRo1wt3dnf3799tte14jGnDkER5uCj7uUYu3Vh3lu/1RvNKoPNX8vJ1tluuifgKf2Vb+xeFMvQ0q21dNjB8/ngMHDrBlyxb8/PyYPn06x44dM5ZOzi0ffvghc+fOJTg4mA8//JB+/fpx+fJllEqlTY061Go1s2fPpnr16ty9e5fx48czaNAgtm7danafyZMnM3fuXCpXrmyxZLSzEeKVh7Sr4UeHED92nItl+ubT/DTsOZF5X8hJSEjg+++/Jzw8nPbt2wOwcuVKm+tu2cKECRPo1q0bAB9//DG1atXi8uXL1KhRg9DQUPr372/08oKDg/nyyy9p3bo1S5cuxcPDg7feest4rcqVK/Pll1/SpEkTEhMTjQu5AWbNmmVcqlcQEeKVx8zoXpP9l+/x79WHbDlxm571yznbJNfErYjeA3LWvW3k6tWrqNVqmjZtatzn6+tL9erVHWZO3bp1jT8HBAQA+nXCNWrUsKlRR0REBDNnzuTEiRM8evQInU7fhzQ6OtqsEKilZhwFCSFeeUxgiSKMbluVuX9f5JM/ztG2Rhl8PFy3DInTkMnsGroVZkzL2Bg8eYMAGRp1jBkzJtN5QUFBJCUlGStMrFmzxljptVOnTpmaamRsxlHQEAH7fGBYq8pUKuXFvYRUFm4XwfvCTOXKlXFzczOr8x4XF8fFixfz5f7ZNeo4f/48Dx48YM6cObzwwgvUqFHDYvNZV0CIVz7grtQH7wG+P3SNc3dyX0dfUDDx9vZm4MCBTJw4kd27d3PmzBmGDBmCXC7Pl3hndo06goKCUKlUxmYfW7ZsYfbs2XluV14gxCufaFWtNF3r+KPVSUzbdBqdTmTeF1bmz59P8+bNeemll+jQoQMtW7YkJCQkX0r7ZNeoo3Tp0qxatYp169ZRs2ZN5syZw9y5c/PcrrxA1PPKR24/TqbD/L08SdMy97V6vNqofL7e35UoTPW8kpKSKFeuHPPmzWPIkCHONqdAIOp5uRhli3kypn0wAKFbzxH3RO1kiwR5QWRkJD/99BNXrlzh2LFjxgazPXv2dLJlhQshXvnMWy0rUbVMUR4kpTH37wvONkeQR8ydO5d69erRoUMHkpKS+Oeffzh37ly2jTIEtiNSJfIZlVLOrJ61eGPFYVYfvs7rjQOpU97X2WYJHEiDBg0strtPTk7OtlGGwHaEeDmBFlVK0aNeWbacuM20zaf5dWQLkXn/DPCsNsrIK8Sw0Ul82C0EL5WC4zcei7I5WfCMzSc9Mzji9yrEy0n4+Xgw9GnN+7l/X0Ct1TnZooKFIYv8yZMnTrZEkBcYfq+5aXrr1GFjaGgoGzdu5Pz583h6etKiRQs+//zzLNeBrVq1isGDB5vtc3d3JyUlJa/NdTjDWlVm9b/XibqfxC//3eB/z1VwtkkFBoVCQbFixYzZ30WKFBFD60KAJEk8efKEu3fvUqxYMRQKRY6v5VTx2rt3L6NGjaJJkyZoNBqmTp3Kiy++yNmzZ7NcV+Xj48OFC+kzda76R13UXcm77aoy87ezLNp5iZcblqOISoQhDfj7+wO47PIVgXWKFStm/P3mFKd+UrZt22b2etWqVZQpU4aIiAhatWpl9TyZTJbrBy8ovNGsAt8diOLGw2RWHrjGqLYioGtAJpMREBBAmTJlUKtFTlxhwc3NLVcel4EC9TUfFxcHQIkSJbI8LjExkQoVKqDT6WjYsCGfffYZtWrVsnhsamoqqampxtfx8QVrXaFKKWfCi9V57+fjLNtzhTeaBlHcS+VsswoUCoXCIX/sgsJFgQnY63Q6xo4dS8uWLaldu7bV46pXr05YWBibN29m9erV6HQ6WrRowc2bNy0eHxoaiq+vr3ELDAzMq0fIMd3rlqVmgA8JqRqW7L7sbHMEApegwKxtfOedd/jzzz/Zv38/5cvbvuZPrVYTEhJCv379LK6Ot+R5BQYGOmVtY1bsvXiPgWFHUCnk7JrQmvLFbS+AJxAUJlxqbePo0aP5/fff2b17t13CBfrxc4MGDbh82bLH4u7ujo+Pj9lWEGkVXIrmlUuSptWxQNT8EgiyxaniJUkSo0eP5tdff2XXrl1UqlTJ7mtotVpOnTplLIfrqshkMj7oUgOAjZE3OR9TsGJzAkFBw6niNWrUKFavXk14eDje3t7ExMQQExNDcnKy8ZgBAwYwZcoU4+tZs2bx999/c/XqVY4dO8b//vc/rl+/ztChQ53xCA6lfmAxutbxR5Lgi21i0bZAkBVOFa+lS5cSFxdHmzZtCAgIMG6//PKL8Zjo6Gju3LljfP3o0SOGDRtGSEgIXbt2JT4+noMHD5o1DnBlJrxYHYVcxs7zdzkS9dDZ5ggEBZYCE7DPL5xZjNBWpv56ivDD0TQMKsaGd8SibcGzhUsF7AXmvNc+GA83OceiH7P9bKyzzREICiRCvAogfj4eDHleP3nxf39dQCMWbQsEmRDiVUAZ0boKxYq4cfluIhuP3XK2OQJBgSNX4pWYmEh8fLzZJnAMPh5ujGqjX+e4YMdFUtRaJ1skEBQs7BavqKgounXrhpeXF76+vhQvXpzixYtTrFgxihcvnhc2PrO82bwCZX09uBOXwvcHrznbHIGgQGH3wuz//e9/SJJEWFgYfn5+YiYsD/FwUzCuYzUmrj/J13uu0LdJEL5Fcl68TSAoTNgtXidOnCAiIiLLgoECx/Fyw/Ks+OcqF2MTWbr3CpOfZuELBM86dg8bmzRpwo0bN/LCFoEFFHIZkzrpBWvlgShi4lyvYqxAkBfY7Xl9++23vP3229y6dYvatWtnqkFdt25dhxkn0NM+pAyNKxTn6PVHLNp5kdCXxf+xQGC3eN27d48rV66Y1ZGXyWRIkoRMJkOrFbNijkYmkzG5Sw1eXXaIn47c4Mq9JL4d2BgfDxH/Ejy72D1sfOutt2jQoAGHDh3i6tWrREVFmf0ryBsaVyxBhxA/AI5EPWTpnitOtkggcC52e17Xr19ny5YtonmmE5jUuTo7zumXC8WK2JfgGcduz6tdu3acOHEiL2wRZEM1P2/jz7XK+TrREoHA+djteXXv3p1x48Zx6tQp6tSpkylg36NHD4cZJ8jMKw3Ls+HYTdGkVvDMY7d4vf3224C+KGBGRMA+73F30zvLqWohXoJnG7vFS6cTHxpn4qHUtwBL0YgvCcGzjV0xL7VajVKp5PTp03lljyAbhOclEOixS7zc3NwICgoSQ0MnIjwvgUCP3bONH374IVOnTuXhQ1Ff3RkIz0sg0GN3zGvx4sVcvnyZsmXLUqFCBby8vMzeP3bsmMOME2TGQ6kXL0ue14+HrlG5dFFaVi2V32YJBPmO3eLVq1evPDBDYCvubvphY0bP62JsAtM2nwHg9MedKOpu969WIHAp7P4LnzFjRl7YIbARD8OwMYPn9SQt/fXvJ27Tt2lQvtolEOQ3OS4DHRERwerVq1m9ejWRkZE5ukZoaChNmjTB29ubMmXK0KtXLy5cyL7Z6rp166hRowYeHh7UqVOHrVu35uj+roi70rLnpTPpYPfLUVGySFD4sVu87t69S7t27WjSpAljxoxhzJgxNGrUiPbt23Pv3j27rrV3715GjRrFv//+y/bt21Gr1bz44oskJSVZPefgwYP069ePIUOGEBkZSa9evejVq9czk75h8Lwyxry0unTxiox+zMXYhHy1SyDIb+wWr3fffZeEhATOnDnDw4cPefjwIadPnyY+Pp4xY8bYda1t27YxaNAgatWqRb169Vi1ahXR0dFERERYPWfRokV07tyZiRMnEhISwuzZs2nYsCGLFy+291FcEmuel0Zr3jv46LVH+WaTQOAM7I55bdu2jR07dhASEmLcV7NmTZYsWcKLL76YK2Pi4uIAKFGihNVjDh06xPjx4832derUiU2bNlk8PjU1ldTUVONrV+9wZIvnBZAm8sAEhRy7PS+dTpdpMTboE1hzs3RIp9MxduxYWrZsSe3ata0eFxMTg5+fn9k+Pz8/YmJiLB4fGhqKr6+vcQsMDMyxjQUBa56XVjIXL3UGT0wgKGzkqCTOe++9x+3bt437bt26xbhx42jfvn2ODRk1ahSnT5/m559/zvE1LDFlyhTi4uKMm6vX37fueZmLmVqsQRUUcuwWr8WLFxMfH0/FihWpUqUKVapUoVKlSsTHx/PVV1/lyIjRo0fz+++/s3v3bsqXL5/lsf7+/sTGxprti42Nxd/f3+Lx7u7u+Pj4mG2ujK0xL7VGYt3RG/RcvJ9r961PgAgErordMa/AwECOHTvGjh07OH/+PAAhISF06NDB7ptLksS7777Lr7/+yp49e6hUqVK25zRv3pydO3cyduxY477t27fTvHlzu+/viribeF6GvgFgnioBoNbq2HT8FiduxvHp1nOsGNA4320VCPKSHKVhy2QyOnbsSMeOHXN181GjRhEeHs7mzZvx9vY2xq18fX3x9PQEYMCAAZQrV47Q0FAA3nvvPVq3bs28efPo1q0bP//8M0ePHmX58uW5ssVVMHhekqSPa6mUevHS6DKLl2HkuP1sLMeiH9EwSHQ0FxQeciReO3fuZOfOndy9ezdTkD4sLMzm6yxduhSANm3amO1fuXIlgwYNAiA6Ohq5PH1026JFC8LDw/noo4+YOnUqwcHBbNq0Kcsgf2HCEPMCvfelerrWMdNso1Zn5o3937bz/DTsOdHhXFBosFu8Pv74Y2bNmkXjxo0JCAjI1YdBkrKfEduzZ0+mfa+99hqvvfZaju/ryqgUcmQyveeVqtaBh35/ppiXVofpnn+vPuSfS/dpVa10/hkrEOQhdovXsmXLWLVqFW+++WZe2CPIBplMhrtSTopaR4o6fcYxU6qERsKgXkElihD98An/99d5nq9aCrlceF8C18fu2ca0tDRatGiRF7YIbMQ446hJH7JnHDaqdTqkp+o1vFVlirorOX0rnj9PW86HEwhcDbvFa+jQoYSHh+eFLQIbMeZ6mXhemQP2EgZnrKSXiqEv6Gdy5/19AY3oPCQoBNg9bExJSWH58uXs2LGDunXrZsq2nz9/vsOME1jGoueVQZDUmvSYl0wmY+gLlfnh0HWu3k9ifcRNUTJH4PLYLV4nT56kfv36AJkqOYiZrPzBWNPLLOZlfoxaqzNOiMhkUNRdyai2VZn9+1kW7rhErwbl8Hha2FAgcEXsFq/du3fnhR0CO7Ac8zL3vNJMZhsNXyn9mwURtj+KW4+T+fHQdYa1qpwP1goEeUOOixEKnEdWMS/Pp96U3vPSv2fwiD3cFLzXIRiAJXsuE5+izi+TBQKHI8TLBbHkeemeipdB2NRaKZPnBfByg3JULVOUx0/UfLvvan6YKxDkCUK8XBBLdewzel4arQ5MYl4GlAo5E16sBsC3+6O4l5Be60wgcCWEeLkgBs8rRZ05z8tDpX8vzdTzyjCP0qmWP/XK+/IkTcuS3Zfz3F6BIC+wW7z27duHRqPJtF+j0bBv3z6HGCXIGvcsPC9DR22zmBfm6iWTyZjUuQYAaw5f58bDJ3ltskDgcOwWr7Zt21rslh0XF0fbtm0dYpQgayx5XpljXukZ9ljIYGlZtRTPVy2FWiuxcMelvDVYIMgD7BYv0xpSpjx48CBT92xB3pBlzOvpsFGtSS+JYy37bmKn6gBsjLwpug0JXA6b87xefvllQD/kGDRoEO7u7sb3tFotJ0+eFGse84msYl6GgL15zMuyfNULLEaX2v78eTqGuX9dYLkoWChwIWwWL19fX0DveXl7exuLBQKoVCqee+45hg0b5ngLBZmw7HnphczdLM9LL19ZFZF4/8Vq/HUmhr9FwUKBi2GzeK1cuRKAihUrMmHCBDFEdCKWPS/9v2apEk/JGLA3pWoZb15tVJ61R28yc8sZNr7TAqVCTEILCj52/5XOmDFDCJeTcVcaPK/My4PMklSNGfZZX+/9F6vj46Hk5M04lv8jElcFroHd4hUbG8ubb75J2bJlUSqVKBQKs02Q9xgWVGe1PCjNZLYxu+Xyfj4eTO9eC4CF2y9xSQTvBS6A3QuzBw0aRHR0NNOmTct1GWhBzrDkeaWnSqR/gRgbz9rwK3qlYTn+OHmb3RfuMWH9STa83VwMHwUFGrvFa//+/fzzzz/GsjiC/Ccrz8tUvNKeiltWMS8DMpmM0Jfr0nHBXk7ceMx3+6MY0bqKI80WCByK3V+tgYGBNjXOEOQdlmNe5sNG0A8dIfuYlwF/Xw+mvVQTgHnbL3L5bqIjzBUI8gS7xWvhwoVMnjyZa9eu5YE5AlsweFepFjwvd5PWaOmel+281qg8raqVJk2jY9L6E5lq4wsEBQW7xatPnz7s2bOHKlWq4O3tTYkSJcw2e9i3bx/du3enbNmyyGQyNm3alOXxe/bsQSaTZdoMzWqfFdLXNmaOeSnlMtwUerkyipcdcUmZTMacl+tQ1F3JsejHrDwQ5SizBQKHYnfMa+HChQ67eVJSEvXq1eOtt94yZvDbwoULF/Dx8TG+LlOmjMNscgU8lNZjXgq5HDeFHLVWa/ew0UDZYp581C2EyRtP8cVfF+haJ4CyxTyzP1EgyEfsFq+BAwc67OZdunShS5cudp9XpkwZihUr5jA7XA1LnpfWzPOSA1rjvpzMB/dpEkjYgSguxiZy8macEC9BgSNHc+FXrlzho48+ol+/fty9exeAP//8kzNnzjjUOGvUr1+fgIAAOnbsyIEDB7I8NjU1lfj4eLPNZg4vh3sXc2mt47HseemFTG4Ur3Ryks4ik8koXkQFZO4JKRAUBOwWr71791KnTh0OHz7Mxo0bSUzUz0idOHGCGTNmONxAUwICAli2bBkbNmxgw4YNBAYG0qZNG44dO2b1nNDQUHx9fY1bYGCgbTc79zv8ORG+bQ8X/3bQEzgGyzEv/b9KuQyVImP9rpzdR/n0OpoMzT0EgoKA3eI1efJkPvnkE7Zv345KpTLub9euHf/++69DjctI9erVGTFiBI0aNaJFixaEhYXRokULFixYYPWcKVOmEBcXZ9xu3Lhh280Cm0JQc0iNh/DXYf8CY1llZ2PwvLQ6CfXTuJZBYBRyGW7KDJ5XDu+jkOuvo8nYV00gKADYLV6nTp2id+/emfaXKVOG+/fvO8Qoe2jatCmXL1svZezu7o6Pj4/ZZhNFy8CALdBoECDBjpmwYSikOb/qqGk6hMH7Mo15KeWZK6fmBMN1xLBRUBCxW7yKFSvGnTt3Mu2PjIykXLlyDjHKHo4fP05AQEDeXFypgu6LoNs8kCvh9HpY2RnibubN/WzE3cSzMsS9tIbyN5ZiXjm8j0G8NEK8BAUQu2cb+/btywcffMC6deuQyWTodDoOHDjAhAkTGDBggF3XSkxMNPOaoqKiOH78OCVKlCAoKIgpU6Zw69YtfvjhB0CfplGpUiVq1apFSkoK3377Lbt27eLvv/M4JtVkKJSuAWsHwJ0TsLwN9FkNQc/l7X2tIJPJUCnlpGl0Rs/LMLRTyvXvmR+fs/sYYl4ZG9oKBAUBuz2vzz77jBo1ahAYGEhiYiI1a9akVatWtGjRgo8++siuax09epQGDRrQoEEDAMaPH0+DBg2YPn06AHfu3CE6Otp4fFpaGu+//z516tShdevWnDhxgh07dtC+fXt7H8N+Kj4Pw3aDX21IugerXoKIVXl/Xyt4KM0bz2qNeV6WPK+cqZcx5iU8L0EBxG7PS6VSsWLFCqZNm8bp06dJTEykQYMGBAcH233zNm3aZLlOctWqVWavJ02axKRJk+y+j8MoXgGG/A2b3oGzm+G39yDmNHQOBYVbvpri7qaAFA2p6owxL7kxw95Ajj0vw7BRBOwFBRC7xctAUFAQQUFBjrTFNVB5wWvfw765sPsT+G8F3Duv3+dVMt/MMBQdTNGYx7wUcjJ5XjlFYSHmtXjXJQ5HPeS7gU0yDU8FgvzEJvEaP348s2fPxsvLi/Hjx2d57Pz58x1iWIFGJoPWE8GvJmwcDtf+gRVtoO9P4F87X0wwlII2eF4G70ghl6PKlKSas3ukzzamx7xW/xtNTHwKF2ISqFPeN2cXFggcgE3iFRkZiVqtNv5sjWeuMGGNbjB0B/zUDx5FwXcdofcyqNkzz2+dyfMyTZXIOGzMYcwrPUk13fNKzeDpCQTOwibx2r17t8WfBUCZEBi2C9YPhqt79DOSrSZBmykgz7thVUbPK33YmDlgn1MzlE9PNM3zSsuQVyYQOAsRtHAERUpA/w3w3Ej9633/B2vfhNS8qwWfsf2Z6WxjpmFjjmcb9eepTQL2hkoVOkni7zMxbD5+K0fXFghyi02elz3lajZu3JhjY1wahVI/6+hXG34fC+d/h287Qr9wKFHZ4bfLHPMyWR6URzEvnU4yCplGKzH8xwgAmlcpSRlvj5zdRCDIITZ5XqYLm318fNi5cydHjx41vh8REcHOnTuNjWmfaRr0h0Fboagf3DsHK9rph5MOxprnpZTLcFNmjHnljIyzjWqTwL1pw9ukVC0CQX5jk+dlaDgL8MEHH/D666+zbNkyY6szrVbLyJEjbV83WNgJbALD98Av/4NbEfDjy9DpU2j2ds7doAxkbDxrXB4kywvPS3/tNJMqFqY/K561iRpBgcDumFdYWBgTJkww69GoUCgYP348YWFhDjXOpfEpq/fA6vYFSQvbJsPm0aBJdcjlrXpeiszilVPfy9D6TGNBvEzL8SgUQrwE+Y/d4qXRaDh//nym/efPn0cn1sCZ4+ahT5148VOQyeH4aljVDRJyX3M/o+elMVse5JgMe8OwUfs0zmUI1oO5kMllcDc+hUdJaTm7kUCQA+zOsB88eDBDhgzhypUrNG3aFIDDhw8zZ84cBg8e7HADXR6ZDFqMhjI1YP1bcPM/WN4W+q6Gco1yfFl3E89Lp5OMpcaUT2vYm5mQw3sYho2GWJc1z+tufCo9lxygfHFP/pnU9tnL9xM4BbvFa+7cufj7+zNv3jxjaZyAgAAmTpzI+++/73ADCw1VO+gXdv/UD+5fgLAu0OMrqNcnR5cz9bxME0YVFmNeuUuVMAxJ1VrLAfuTNx8DcPNRMhqdlMnzEwjyAruHjXK5nEmTJnHr1i0eP37M48ePuXXrFpMmTTKLgwksULKKPiO/WhfQpsKvw+Hvj0Bn/2ydaczLNGFUobCU55UzMtbzSrUSsDfdf/JmnEhgFeQLuUpStasyqUCPhw/0DYcXnnqpB7+CNa9B8iO7LmPqeZku31E6Mub1VASNMS8rgmX68ytLD/L5tswxUYHA0eSoqsT69etZu3Yt0dHRpKWZB2mzaoYheIpcDu2ng18t2DQKruyEFe2h389QuppNl7DqecllxllC4+1yWQba0myjmZCpzT3H5fuuMrVrSI7uKRDYit2e15dffsngwYPx8/MjMjKSpk2bUrJkSa5evZqjHozPNLVfgSF/gW8gPLyi71R0YZtNp5rFvHTmMa+Mw8ackjHD3my2UWvZ8zKQohaJqwITov6BX9/JUYjEGnb/lX/99dcsX76cr776CpVKxaRJk9i+fTtjxowhLi7OYYY9MwTU0wfyg1roOxX91Bf+mZdtpyJTz8vQOUgme1rDPmOGfa5bn1kI2JuIkyXxenHBvpzdVFC4kCQ4sgJ+7AUnwuG/bx12abvFKzo6mhYtWgDg6elJQoJ+8fGbb77JTz/95DDDnimKloYBm6HRYECCnbNgw5AsOxVZ8rwMnpLjZhvNW5+ZDRtNhCw5LfO3afRD53dZEjgZTZq+2vDWCaDTQJ3XoKF9fS6ywm7x8vf35+HDh4C+mqqhV2NUVFSWJZ0F2aBUQfeF0G3+005FGyCsEzy23GfSw6TxrGlFCbAgXjk1KUOqhFmQXm0iXmKIKMhI4l34vjsc+x6QQcdZ8PIKcPN02C3sFq927dqxZcsWQJ+wOm7cODp27EifPn0s9nMU2EmTIfp+kUVKQsxJWNEWrh/KdFh6VYn0gL1hjaGjKqmmL8zOOkn1iQXPS/AMc/u4PhH7xr/g7gv910HL9xy2rteA3bONy5cvNy4DGjVqFCVLluTgwYP06NGDESNGONS4Z5aKLfULu396A2JP6b/Bun4BjdNXMJh6XpoMnpfDKqlmXJhtJUifrNZYvUbY/ijkMhjUslKObBC4GKfWP13Dmwwlg6HfT1DK/uY8tmCX56XRaPjkk0+IiUlfm9e3b1++/PJL3n33XVQqlV0337dvH927d6ds2bLIZDI2bdqU7Tl79uyhYcOGuLu7U7Vq1UwdhgoNxYL0M5E1e4FOra8R9sf7oNWX406PeWlNFmXrf50Oqyrx9DqGGl5qM88r3duy5nk9Skpj1u9nmfnbWe7EJefMCIFroNM+7So/RC9cwS/CsJ15Jlxgp3gplUr+7//+D43G+jetPSQlJVGvXj2WLFli0/FRUVF069aNtm3bcvz4ccaOHcvQoUP566+/HGJPgUPlBa+tgnZP+2H+9y380AuS7luMeRnyufIq5mXV87IiXomp6X8nzUN35dAKQYEnJU6/7G3/Av3rlmP1OYseeVvfz+5hY/v27dm7dy8VK1bM9c27dOliV27YsmXLqFSpEvPmzQMgJCSE/fv3s2DBAjp16pRrewokMhm0mghlauk7FV3fD8vb4tV9FZDB85JbjnnlVL2yinml2RDzspRCIShk3L8MP/eD+xdB6QE9FkPd1/Ll1naLV5cuXZg8eTKnTp2iUaNGeHl5mb3fo0cPhxmXkUOHDtGhQwezfZ06dWLs2LF5ds8CQ42uTzsV9YVHUZT85SW6yIfxp66ZUSSMs42ZKqk6KOZlZ8DeNC9MUAi5tENfKSU1DnzKQd81ULZBvt3ebvEaOVLfZMJSf0aZTIZWm3czTzExMfj5+Znt8/PzIz4+nuTkZDw9M0/DpqamkpqaXgAwPj4+z+zLc8rUeNqp6C1kV3ezVLWIRZrePEltDKQH6jN1D8q15/U0VcKsnlf67zk5zXIYIU14XoUTSYKDX+pjXJIOAp+DPj9C0TL5aobdqRI6nc7qlpfClVNCQ0PNavAHBgY626TcUaQE9F+P1Ez/JfKe8leq7XkHL5KzSJXIqedl3vrMWpLqEyt5XmKJUCFEnQy/joDt0/XC1XAADNyS78IFLtb6zN/fn9jYWLN9sbGx+Pj4WPS6AKZMmUJcXJxxu3HDctKnS6FQIusSygead0iV3PC7vZONqhmURz8LnDlVImcYrmOcbTRbHpT+s7XcZJH/VciIvw0ru8DJX0CmgK5zofuXoHR3ijk2DxuTk5PZuXMnL730EqAXBdPhmEKhYPbs2Xh45F0LrObNm7N161azfdu3b6d58+ZWz3F3d8fd3Tn/uXnNVmVbLqQGEO7zFdVTb7Ik6X244oebXwuz4xzV+sxazMsaCamOmZUWFABuHNE3lEmMBc8S8Pr3UKmVU02y2fP6/vvv+eabb4yvFy9ezMGDB4mMjCQyMpLVq1ezdOlSu26emJjI8ePHOX78OKBPhTh+/DjR0dGAXiAHDEhfC/X2229z9epVJk2axPnz5/n6669Zu3Yt48aNs+u+hQV3pYLjUlXWN1rNcV0VvKVEWP0yXseWA+nuUG6bzmZXEscaiSlCvAoFx37U915IjNXPeg/f7XThAjvEa82aNQwfPtxsX3h4OLt372b37t188cUXrF271q6bHz16lAYNGtCggX6GYvz48TRo0IDp06cDcOfOHaOQAVSqVIk//viD7du3U69ePebNm8e3335beNMkssFdqf/13ac4fdKmscu9PUg6PHd9xBfKb1ChT2jN6bgxU8zLShloaySmqnN2Y0HBQKuBPz+ALaNBmwYh3WHI31C8orMtA+wYNl6+fJk6deoYX3t4eCCXp2tf06ZNGTVqlF03b9OmTZaLuS1lz7dp04bIyEi77lNYMSSqJqZqSUXFYp/xtGvYAenvD3lNuY8q8tuMSBuXi0qq1j0vWyo9J2TwvHQ6iYjoR9Qq60MRlRKtTjJ6d4ICxpOHsG4gRD0tbdRmqj7fUF5wwuQ2W/L48WOzGNe9e/fMElV1Op3Z+4K8x7BE6MnTVAWlQgHNR0L/DTyWvGgov8xv7h+huJ2z6raZM+ztqxqSUbzCDkTx2rJDDFr5H2dux1Fz+jaW7L6cI9sEeUjsWX1BgKh9oCoKfdZAmw8KlHCBHeJVvnx5Tp8+bfX9kydPUr58eYcYJbANg+dlmNUzeDGyqu14VfMpl3Tl8Jc9osia7nDiZ7uvbypekiSZ5XbZQmKGgP0v/+lneo9EPWTmljOkanR88dcFu+0S5CHnfoNvO8Cja1CsAgzZDiEvOdsqi9gsXl27dmX69OmkpKRkei85OZmPP/6Ybt26OdQ4QdYYPK+kpyJhOgS7owigd9rHbNc2RKZN1efm/PWhPo5hI0qTb1qNTrI76TRjwF6lTL+eKP1WwNDpYM/n+hlFdZI+ID98D/jVdLZlVrE55jV16lTWrl1L9erVGT16NNWq6RtFXLhwgcWLF6PRaJg6dWqeGSrIjMHzSkrLLF5uSjmP04owXD2ei60icTswDw4thrtn4dUw8Cye7fUVJvliWp1kFrC3hYQMAXtT8RIUIFITYdM7cE5fp49mb+u7vCty1J8n37DZOj8/Pw4ePMg777zD5MmTjYF2mUxGx44d+frrrzMt3RHkLekxL/1wTmkqXk+z7CXkaNt8iFvZOrBpJFzZBSvaPe1UVD3L65tezxGeV8ZlS4ICwKNr+rpxd8+A3A1eWgAN33S2VTZhl7RWqlSJbdu28fDhQy5f1gdaq1atSokSJfLEOEHWGD2vp8NGuYnYZKosUas3lKgCP78BD6/qW6298i1U72z1+qaenFYrGTPtbSVjwN5deF4Fi6h9sHYgJD8ErzLQZzUENXO2VTaTo7+mEiVK0LRpU5o2bSqEy4mkx7wseV7pPxtTJQLq6uMYFVpCWoK+QsW+uVYDUKbXU+t0dnteGTPsTT0vEfJyIpIEh5fra8MlP9RXghi+x6WEC1xsbaPAnKxiXqaNZ80y7L1KwZuboPEQQIJds/VlTdKSMl1fJpMZr6nVSXbX58o8bBQ5XU5Hkwa/jYE/J4KkhTqvw+A/wbecsy2zGyFeLoy7W/YxL7CwtlGpgpfm6+MbciWc2Wi1U5HpEiF7UyUydhVSPfUUrTHrt7O8teo/sya6Agdi7OjzA8jk0HE2vLzcoR198hMhXi6Mh9J8+Y55zMtk2GjtAo3fgoG/QZFSEHMKlreB6wfNDjHmemntn23MSFaeV6pGS9iBKHadv8upW6J5scO5Han//Ro6+ryxDlqOcXhHn/xEiJcLY/C8DFj3vLL4A63QQr/Q1r8OPLmv/2Y+GmZ827QUtL0B+0z2muV5mV/r5qP0Bh0aUYHVsZxaD2GdIf6WvqPPsF0Q3CH78wo4QrxcmIyzdwqTpFIz8cruQsWC4K2/9DOSOg38Pg5+Hw9atVEQ07S6XA/nMs2AmnD9QXrMTSOGjY5Bp4XtM5529Ekx6ehT1dmWOQQhXi5Mlp6XMouYlyVUXvDqSmg/HZDB0e/gh56UlCUAjiksmFWe17X7T3J9fYEJKXH62eQDC/Wvnx+XLx198hMhXi6MRybPy0S85KapEjbGNWQyeOF9faNQlTdcP8CP2knUlF3jSaoDxMvE3ozOVfRDIV4O4/4lfR7fpb9B6QmvfAcdZoI86wkTV0OIlwuT0fNSWIh55SgeW72LvlNRicoEcJ/1qo/xvvJbbkw1swkydxa69iBzqoYgB1zarheuB5f0HX3e2gZ1XnW2VXmCEC8XJqPnZWnYmOO5pKedio4o6lNElkq9f8cyXrkWGTkPppvG6DImvF5/YO55Rd1P4rOt57ibkLkQgMACkgQHFsGa1/StyAKf0yeelq3vbMvyDCFeLkzWntfT8ji5mQr3LM6HntNZrtFXCxmj3MRytwUUJWdDPNNUCdO0C41Wx40Mw8beXx9g+b6rvPfT8Rzd65lCnQwbh+k7+iBBw4H6FBgndPTJT4R4uTBZxbwMM3u5zeKRK9z4TNOffTVnkyq50VERwUbVDCrIYuy+lmmmv9rE87oTl2I2wyhJ8PiJviJFRPSjXFj/DBB3S58GcWqdPuG461zovkifiFzIEeLlwuRZzMvCNU+V7srradOIlYpRTX6LzappPC8/Zde1JJMVjaZVWUW8K4dEH9Ynnt45ru/o8+YmaDrMpRNP7UGIlwtjWNtowFKSak47B6VfR39+UqqGE1JVuqd+SqSuKsVkSXzvNochiq3kZJm1acD+2gPrw9Bn42OYA479oO/ok3QX/Grr41uVXnC2VfmKEC8Xxj3DWkG5paoSDvK8DOsU71KcvmkfsU7TCoVMYprbaua6fYO30r42Z6biFSmGhrajVcPWSbDlXdCpIaSHPsG4eAVnW5bvCPFyYWzzvHKHoRR0skmSaioqJmpGMEv9JlpJxquKfYQrZ1OGrEXINLfLVLw2HruVSyufEZ48hNUvw5Gn/VPbfgivfQ/uRZ1rl5MoEOK1ZMkSKlasiIeHB82aNePIkSNWj121ahUymcxsy8su3QWZjJ6XpeVBjop5Zc6wlxGm7cIA9WQeS17U4RK/uX9IfZn1bkA6k/WMtq6TlNDPRn629Rx7L96z1/zCQ+wZfXzL0NGnbzi0nlTgOvrkJ05/8l9++YXx48czY8YMjh07Rr169ejUqRN37961eo6Pjw937twxbtevX89HiwsOWXpeyqepErn0vZQKa+Kl54CuDj3TZnNdHoif7DG/qGbzsnyfxWN12axZDPDVfwlJGWJon/xxjuX7rjIwzPqXWqHm3G/wbUd4fF3f8HXIdqghmt04Xbzmz5/PsGHDGDx4MDVr1mTZsmUUKVKEsLAwq+fIZDL8/f2N27NaOz+rmJfK4Z6X9ZjWdcmfKSUW8Le2Ee4yNfNVy/hQuRoF5oKXVbGIjjX98PawXJV81cFrdttdKNDpYM8ck44+rWHY7gLd0Sc/cap4paWlERERQYcO6eU55HI5HTp04NChQ1bPS0xMpEKFCgQGBtKzZ0/OnDlj9djU1FTi4+PNtsKCQi4zS/y0FPOS51K9jDEvddZrGxWePoxQj2ORpjcAw5RbWen2f/iQaDxGl0W/M4WNdn6+7TyrDkTZdGxBIC5ZzctfH+B7ewU4NRHWDYA9ofrXz42E/22EIqLsugGnitf9+/fRarWZPCc/Pz9iYiwnQVavXp2wsDA2b97M6tWr0el0tGjRgps3b1o8PjQ0FF9fX+MWGBjo8OdwJqbel3kZaMOwMXcYBNEQsLfWvsxLpURCzgLNa4xMG8MTyZ1WilNsVk2jqkz/u8lSvExsz6rc9NI9V5j521lSshHTgsI3e69wLPoxM7ZY/4LNxMMo+K6jfrioUEHPJdA5tMC3IstvnD5stJfmzZszYMAA6tevT+vWrdm4cSOlS5fmm2++sXj8lClTiIuLM243bmQudezKmMa9LBYjzO2wMUPMy9PNcmWCIu7p+7fqnuOVtJnclEpRSR7Lr6oZtJdHZFkPzNTxen/tiWztirrvGomtSan2pZBwdS+saKvvr1nUDwb9AQ3+lzfG5SOPktL4/eTtbOOe9uBUKS9VqhQKhYLY2Fiz/bGxsfj7+9t0DTc3Nxo0aGBsxZYRd3d33N3dc21rQcWa5+Wo5UFKeWbxiktWZzquqLv5n9I5qQI9Uj/ha9UinpOfY4XbfP65+YSved6iVaa2P0xKM/5srWNRvAUbnI0kSWh1klnzE5s/qpIER1bAtsn6xhhlG0LfNeBTNk9szWskSeLcnQR2X7jLrvN3iYx+hE6C30Z7Uae8Y2qKOdXzUqlUNGrUiJ07dxr36XQ6du7cSfPmzW26hlar5dSpUwQEBOSVmQUadxPPy/LyoNzJlzFJ9WnAPuMMpwFPVWaP7CE+/C9tCj9oOiKXSbS+uZTFbl/iSeZKEbbGvAwUxFqrA8KO0PLzXWZD2ixGyuloUvVJp4aOPnX7wOCtLidcSaka/joTw5SNJ2keuouuX/7DF39dIOK6Xrhq+HsTn+K4Lx2nD6LHjx/PwIEDady4MU2bNmXhwoUkJSUxePBgAAYMGEC5cuUIDdUHLmfNmsVzzz1H1apVefz4MV988QXXr19n6NChznwMp2HqeVnq25jb2Uaj5/X0A+lhZdhobTipQcl0zWDOSUF8olrFS4rDVJLFMDxtPLcobTzOdKbUFmwShXzmn0v3ATgS9ZBW1fTPllWcD4CEWFj7Jtw4/LSjzyxoPtpl1idG3U9i1/m77D5/lyNRD82qhXi6KWhZtSRta5ShTfUylCvm2C5FThevPn36cO/ePaZPn05MTAz169dn27ZtxiB+dHQ0cpNEvEePHjFs2DBiYmIoXrw4jRo14uDBg9Ss+WxOH3uYeV4mSaq5ref1FMMQyPAZtCReHm5yM+G0xE/a9lQLbkT3C5OpJb/OZvdpjEx7jyNSiN52mcyunLSMuWAFiQFhR/hp2HM0r1IyaytvHdOnQcTf0nf0eS0MqhbsxhipGi2Hrz5k9wW9YGVcl1qhZBHaVi9D2xplaFaphNUvO0fgdPECGD16NKNHj7b43p49e8xeL1iwgAULFuSDVa6BaYE/0xLxbnLHDBszipKlYaOHm8Imz+maVz16pH7CctV8asuvsUb1GTM1A1mj7WC351XQtCtjN6R+K/7l2pxu1j3Ek+tgy2h9Y4xS1aDvTwW2Mca9hFS2n41l1/m7HLxy3yxh2U0ho2mlEkbBqlzKK9d/c7ZSIMRLkHOKF0mv22S+PMgxqRKKDKJiaXjooVTYFLPSSXCbUryaNoMv3L6hu+JfPnULo6bsOhf5yC67Cph2ZbHcKcN+nRZ2fqyvegoQ3AleWVEgG2PEJatZtvcKYfujzNJX/HzcaVtdPxR8PrhUpsma/EKIl4szsEVF/jydOSfOOGx0UMzLgKVhgKdKkUnkLKF96oak4M676nc5p6vABOVa+it3cu3Kfc7JJwC2DTMKWswrY01+AzrT3cmPYcNQuLxd//r58dDuowLXGCNFreXHQ9dZvPuycWa5djkfOtfyp031MtQq65Nv3lVWCPFycZ6rXJJRbauw58I9GgYVM+5P75GY29lG82GiJfFyV8ptyuQ3z/GR8bW2J+elQBa5LaFi0gm+ko1nsGwcZ6WK2V6roMW8rImXwc7KstvwbXt4cFnf0afXEqj9Sn6aaJUHian8duI228/Fcik2kfuJqcYKIMFlijKpcw06hJQpEIJlihCvQsDETjWY2KmG2b4KJYtQ0ktF/cBiubq2LZ6Xh5ttnpelmbdduob0SpvFWp8v8Uu9wQbVTCao3+YP3XNZXqugeV5p1sRLgjbySL50WwwPksGnvD5/y8mNMSRJ4lj0Y1b/e50/Tt7JZH+ArwfjOlbjlYblbfrdOgMhXoUUbw83Dk5pl2WXaltQKrKPeXnaGLC3tjD7ilSOH2p9xwsnJ9NYc4wlqi+poYlmvuZVJCupiAPCjjClSw2GvVDZ/mB/HmAxmVaSaH0/nK5u3yCXSRDUHF7/wamNMZ6kadhy/DY//nudM7fT1/nWLe9L1zoBpKp1FPdy4/XGgXk6U+gIhHgVYjJWncgJts02ym0K2GeckTMl1c2X6V7T6flgBSOUf/CuchM1ZNGMU48kkSIWzwn98zyhf55n86iW1Mulh2krkiTxw6Hr1CnvS8Og4sb9GQP2HqTChqG8FLseZBCuaccbA35xWmOMK/cSWf3vddZH3CQhRZ9w7K6U071eWd58rkK+/f85EiFegiyxJebl4abAFudHm+XCbNDJFIRq+nNOV4HP3VbQUXGMX2UzGKYezzXJ+gqKUeHH2P9Bu+wNcAB/n401LrK+Nie9ppZpzCuAByxXzYPT19CiYIZ6AKu1HXkjn4VLo9Wx49xdVv97nf2X7xv3VyhZhP81q8CrjcpT3Mt1uwwJ8RJkSUbPK3fDRttK4mzSPc/VtACWq+YT/LRT0Wj1GP7R1bV4rulayPURN/l6z2W+HdCYyqUdXx758t1Ei/sNw8ZGsgssUy2ktCwOipRkaanprL5Y2uI5eYFGq+P4jcfsuXCPDcducidOvxRLJoP2Ncrwv+cq0Cq4dIEYaucWIV6CLPHKkMNjadjo7mZbnldWQfaMH6aTUhW6p37CN6oFNJRfZpXb53ym6c932i5knEE1FcUJ6/QVKaZsPMUvI2xbH5tTVh6Iwl2p4I1mQai1OvoodjNbGYZKpuWcLoiQYb9z+a+HwO08syEmLoULsQncfpzMvov32H/5vnFYCFDSS0WfJoH0axpEYAnLw29XRYiXIEu61wvg8t1Etpy4TYpaS4uqpTId4+Emty3Py0bPy8A9itM3bRqfKMN4XbmXaW6rqSm/zlT1EFJJH+6kanS0nLOL0e3SM9Stla12JB//dhaAXnVLU/bAND53+xGAP7RNmaB+m3PFK6CTHjrsfpIkcf3BEw5cuc+By/d5/ETNv1cfkPG/1dfTjReCS9Gxph+da/s7JPZZEBHiJcgSbw83pnevyYfdQlBrdXi4KVg5qAmjwo8ZBcLPx8PqMMRNITMGs7OKeVk7Pw03JmmGc1aqwEfK1byi+IcqstsMTxvPXdID5rceJzNlY3oT3GwXRDuI4sSjCH8FvxsHAJirfo3F2l4YvENHWbFi31VWHbzGrcfJFt9vWrEELauWolW1UtQtX6zApjc4EiFeAptQyGUonmaCt61RhsjpHUnT6Nh1/i7tQ/zYe8FyZx+VQo5aqxe5rArRZf1hk7FK25mLUnm+dltEffkVfnP/kBFp4zkuWV4P6MCad+aWmJhZXRbNt27zcL9xD43Si3eejGC7rrHZ8VnNsGbH4ydp/Hv1Af9de8R3+6OM929SoQTxKWrOxyQA+lpqa9/O2yFyQUSIlyBHuCsVuCsV9KxfDjBfFG4gY3wstzXsD+pq0yPtE1a4zaO6/Ca/qGYxVT2UDbpWmY51VMVOSZIY+v1RShZV8X+v1jPu7yw/wjy3pXjJUknxrkBE8yVs35K5P4I9VjxJ05Cm0fH7yTusORzNuTvm1ytf3JO/x7WiiErJoh2XjOJVxEIttWcBIV4Ch2C6PKhOOV8uxCbQr2kQa/9LL7u924p3BrbX84qW/Hg57WMWuH3Ni4oI5qmWUVNznc80b6A1WRd5ITaBFLU214mWl+4msvO8vg3f56/URdLpGKdcz3vKjQD8o60NHVcSjzdwLNP52Xle8SlqHiam8deZGJbvu8oDk5lTAy8El6Jf0yA6hPgZewiULZbeq9RZC6OdzbP51AKHYzrsm969Jk0q6rvcmIpXlufbEaJJwpMR6nGMlTbynnIjQ5R/Eiy7ybvqd4kjPT2i88J97JnY1vYLW8BUe3QpCXQ+O4kqyt0AfKfpwmeaN1ggFUWns748KCM6ncR/1x4yffMZLsQmWDxvROvKDHm+EmW8LTdULmtS2M+0f8CzhBAvgUMw9bys6dAbzYLYe+GexaCzvQFmfaeiVzmnC2K+21J9pyLZNIap3+eSVB4gU6E8e7j56An7Lt6nXqC+VE2gLBZZ2ItUuX+OVEnJh5ohrNe2BvT19K11VTIVr38u3ePN7yw3zq0fWIyXG5ajaaUSPEpS81zlElkuhDY06AUo4vZsfoyfzacWOBzTYZ+1z9xnvesA0Dx0pzF5Mv2cnM2ObdM15VqaPyvc5lFRHsuvqumMVY9ih65Rjq5noPPCf0hM1dC7QTlayE+zxO1L5PcSSVKV5H8JY4iUgo3HJqRorDbMvfYgvcuRNeECeP/FarwQbHsya4BvuueVXzOrBQ2Xa30mKJiYB9yzFqJgP+/M58tlFvfbwnkpiB5pszmkrUlRWQrfquYxWvErIDFt02kqTv7DLAvfFhJTNYBE0MUf+MFtDsVliWgCGrK+0Woz4QJISFFbLInTYNbfxqB6Rqa/VJNFfesbX1vrAWAN04Ynlro5PQsI8RI4BNMlkNk5UZYaMShkMua+VpepXWvQsmpJu+//CB/eVE9mleZFACa4rWOx25es//cCAA1nb7fpOrceJ/MkTYMKNZ8rVzBO+x1KmY4N2ueJ77uZJ+5+mc55nKy2OBnx6Em6qLSuZu5VDWxRER9PN+NrS92XbEWIl0CQCxRWYl6WBjTli2cWL7lchrtSwfBWVVgz9Dn2TGiDr8mH2xY0KJmpGcRk9VDSJAUvKQ6zQfUx5dALy/Of78ry/OsPkmg5ZxddP9vAT6pP6KPcgxY5s9X9eV/9Dhq55UXM4Yej2XfR+kwq6IeFputEFXKZ2fPZ63mZIsRLIMgFpgH37KqqWhKvjHliFUt5cWLGi1yb043TH3fipbq29+X8WduON9I+5J7kQ035dba4f0Qz2TluPrKcnQ76hNAd5+5SV3aFn6TJNJJfIk4qwpzis/hO2w2QkZSqZekey82NQZ+Qa/psX7yavpA8uIw3mgy5Zz4e6eJVRGV/+LmGv36Y3apa/i38LkiIgL3AIdgSsDdgadiYleAVdVey+I2GtA+5ybhfTthkz1GpBj1TP+Eb1XzqyK+xWvUZH2sGoNN1zZRT9vhJGvVnbaeXfD/rVCtwl6m5pCvHMPV4AovWAfTlZNrO3WP1fh91C2HoC5W5ci+R9vP2AtC1TgAnbj6mXLEiFoeFpsml1pr5ZsX3bzVlfcRN+jYJtPvcwkCB8LyWLFlCxYoV8fDwoFmzZhw5Yn1WBmDdunXUqFEDDw8P6tSpw9atW/PJUoE1zIeNWatXOYueV/azjfb0dQR9p6LX0mawWdsCN5mWT9xWcmDhm6BJD94npKhpOOsvpijXsFD1Ne4yNTu0Deid9jHXpABjI9nsCAnweWqj+TN90qsO77SpYvGckkXTh6EZq3fYgp+PB6PaVqVkUXe7zy0MOF28fvnlF8aPH8+MGTM4duwY9erVo1OnTty9e9fi8QcPHqRfv34MGTKEyMhIevXqRa9evTh9+nQ+Wy4wRWHF87I0i28p8TKrihOWrutjkppw5uNOVs9JwZ331KOYo+6LTpLxQvxvPFrWhTW7IriXkErLmb8S5vYFI5R/ALBY05Nh6vetVm+1hiF+ZUsjEgPuSgVHprbnyIftcctlue5nEaf/j82fP59hw4YxePBgatasybJlyyhSpAhhYWEWj1+0aBGdO3dm4sSJhISEMHv2bBo2bMjixYvz2XKBKfakaVnysm4/TrFwpHVMvY3slwDJWKbtwRD1BOIlT4rfP0qbva8xPfRTNqmm0UZxgmRJxei0d5mr6WNWN79mgA8nZrxIGe+svZtiRfTilZ1wZ6SMj4fVLHpB1jhVvNLS0oiIiKBDh/QW53K5nA4dOnDo0CGL5xw6dMjseIBOnTpZPT41NZX4+HizTeB4Snqlf7hNSwu3D9E3m6hUyivL84taSfI0xdQ7MQ2M25qcv1vXgN5ps7iiC6Cc7AFLVYuoLI/hllSSV9Nm8rsuvTLD643Lc/Wzrmx97wV8Pd149CTrPDGD52WaaZ9R0A1Z8dVzmM8mMMepAfv79++j1Wrx8zPPnfHz8+P8+fMWz4mJibF4fExM5sarAKGhoXz88ceOMVhgFX9fD9YMbYZWJ5kF5D97uQ6NKxSnax3z2cLNo1qy9fQdGgUV53DUQ15rVD7be7QP0Tc8bRBUjOEvVGHQyiMMeaESMpmMNtVLs+fCPTzdFCSr9SV4xnWoxs7zsZy8GUffJoF0r1eWaZtP0/veLL50W0wbxQkO62owMu09NJ6l+Ol/jWhWqYTFReJf9m3ABxtO8kJwaf44dcfsvd4NyuH9dOYwwNeTAc0r4K6UZ/IIVw9txrf/RDHSSgxMYCeSE7l165YESAcPHjTbP3HiRKlp06YWz3Fzc5PCw8PN9i1ZskQqU6aMxeNTUlKkuLg443bjxg0JkOLi4hzzEIICgU6nk3Q6naTV6rfkNI0kSZKk0eqkpFR1puOTU9Okiyf/lZ4kp9h1D0HeExcXZ9Nn1KmeV6lSpVAoFMTGxprtj42Nxd/f3+I5/v7+dh3v7u6Ou/uzORvzLGFYG2kYqnk8LZyokMss5lB5qNwIrtMsR/cQFAycGvNSqVQ0atSInTt3GvfpdDp27txJ8+aWK0M2b97c7HiA7du3Wz1eIBAUTpyepDp+/HgGDhxI48aNadq0KQsXLiQpKYnBgwcDMGDAAMqVK0doaCgA7733Hq1bt2bevHl069aNn3/+maNHj7J8+XJnPoZAIMhnnC5effr04d69e0yfPp2YmBjq16/Ptm3bjEH56Oho5Carflu0aEF4eDgfffQRU6dOJTg4mE2bNlG7dm1nPYJAIHACMkl6tooBxcfH4+vrS1xcHD4+Ps42RyAQZMDWz6jTk1QFAoEgJwjxEggELokQL4FA4JI4PWCf3xhCfGKZkEBQMDF8NrMLxz9z4pWQoK8pHhj4bNZAEghchYSEBHx9fa2+/8zNNup0Om7fvo23t7fDMqbj4+MJDAzkxo0bhW4GUzyb6+HqzyVJEgkJCZQtW9YsTSojz5znJZfLKV8++0XAOcHHx8cl/1hsQTyb6+HKz5WVx2VABOwFAoFLIsRLIBC4JEK8HIC7uzszZswolNUrxLO5HoX1uTLyzAXsBQJB4UB4XgKBwCUR4iUQCFwSIV4CgcAlEeIlEAhcEiFeOeThw4f0798fHx8fihUrxpAhQ0hMTMzy+HfffZfq1avj6elJUFAQY8aMIS4uLh+ttg17nw1g+fLltGnTBh8fH2QyGY8fP84fY7OgMHdit+fZzpw5wyuvvELFihWRyWQsXLgw/wzNQ4R45ZD+/ftz5swZtm/fzu+//86+ffsYPny41eNv377N7du3mTt3LqdPn2bVqlVs27aNIUOG5KPVtmHvswE8efKEzp07M3Xq1HyyMmsKcyd2e5/tyZMnVK5cmTlz5lhtVOOS5G0To8LJ2bNnJUD677//jPv+/PNPSSaTSbdu3bL5OmvXrpVUKpWkVmduzeUscvtsu3fvlgDp0aNHeWhl9jRt2lQaNWqU8bVWq5XKli0rhYaGWjz+9ddfl7p162a2r1mzZtKIESPy1M6cYO+zmVKhQgVpwYIFeWhd/iE8rxxw6NAhihUrRuPGjY37OnTogFwu5/DhwzZfx1DmVqksOEtMHfVsziQ/OrE7i5w8W2FFiFcOiImJoUyZMmb7lEolJUqUsNq5OyP3799n9uzZ2Q7H8htHPJuzyaoTu7VnsLcTu7PIybMVVoR4mTB58mRkMlmW2/nz53N9n/j4eLp160bNmjWZOXNm7g23gfx6NoEgvyg445UCwPvvv8+gQYOyPKZy5cr4+/tnCo5qNBoePnyYbUA0ISGBzp074+3tza+//oqbm1tuzbaJ/Hi2gkJ+dGJ3Fjl5tsKKEC8TSpcuTenSpbM9rnnz5jx+/JiIiAgaNWoEwK5du9DpdDRrZr2FfHx8PJ06dcLd3Z0tW7bg4eHhMNuzI6+frSBh2om9V69eQHon9tGjR1s8x9CJfezYscZ9BbETe06erdDi7BkDV6Vz585SgwYNpMOHD0v79++XgoODpX79+hnfv3nzplS9enXp8OHDkiRJUlxcnNSsWTOpTp060uXLl6U7d+4YN41G46zHsIi9zyZJknTnzh0pMjJSWrFihQRI+/btkyIjI6UHDx444xGkn3/+WXJ3d5dWrVolnT17Vho+fLhUrFgxKSYmRpIkSXrzzTelyZMnG48/cOCApFQqpblz50rnzp2TZsyYIbm5uUmnTp1yiv1ZYe+zpaamSpGRkVJkZKQUEBAgTZgwQYqMjJQuXbrkrEdwCEK8csiDBw+kfv36SUWLFpV8fHykwYMHSwkJCcb3o6KiJEDavXu3JEnpKQSWtqioKOc8hBXsfTZJkqQZM2ZYfLaVK1fm/wM85auvvpKCgoIklUolNW3aVPr333+N77Vu3VoaOHCg2fFr166VqlWrJqlUKqlWrVrSH3/8kc8W2449z2b4fWXcWrdunf+GOxBREkcgELgkYrZRIBC4JEK8BAKBSyLESyAQuCRCvAQCgUsixEsgELgkQrwEAoFLIsRLIBC4JEK8BAKBSyLES1AgGDRokMVKF507d3a2aYICiliYLSgwdO7cmZUrV5rts9b1Wa1WZ6rIkZaWhkqlsvu+OT1P4FyE5yUoMLi7u+Pv72+2FS9eHACZTMbSpUvp0aMHXl5efPrpp8ycOZP69evz7bffUqlSJWOVjujoaHr27EnRokXx8fHh9ddfNyshY+08gWshxEvgMsycOZPevXtz6tQp3nrrLQAuX77Mhg0b2LhxI8ePH0en09GzZ08ePnzI3r172b59O1evXqVPnz5m18p4nsD1EMNGQYHh999/p2jRomb7pk6dauxI9MYbbzB48GCz99PS0vjhhx+Mtcq2b9/OqVOniIqKIjAwEIAffviBWrVq8d9//9GkSROL5wlcDyFeggJD27ZtWbp0qdm+EiVKGH82bQpioEKFCmYCdO7cOQIDA43CBVCzZk2KFSvGuXPnjOKV8TyB6yHES1Bg8PLyomrVqlm+b8s+W+8lcG1EzEtQqAgJCeHGjRvcuHHDuO/s2bM8fvyYmjVrOtEygaMRnpegwJCampqpfZdSqaRUqVI2X6NDhw7UqVOH/v37s3DhQjQaDSNHjqR169YWh50C10V4XoICw7Zt2wgICDDbnn/+ebuuIZPJ2Lx5M8WLF6dVq1Z06NCBypUr88svv+SR1QJnIcpACwQCl0R4XgKBwCUR4iUQCFwSIV4CgcAlEeIlEAhcEiFeAoHAJRHiJRAIXBIhXgKBwCUR4iUQCFwSIV4CgcAlEeIlEAhcEiFeAoHAJRHiJRAIXJL/BxG+A1vkmg7wAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.023029, + "end_time": "2024-02-29T05:36:09.282713", + "exception": false, + "start_time": "2024-02-29T05:36:09.259684", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4022.381901, + "end_time": "2024-02-29T05:36:12.029238", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tvae/2/mlu-eval.ipynb", + "output_path": "eval/treatment/tvae/2/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "path": "eval/treatment/tvae/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "tvae" + }, + "start_time": "2024-02-29T04:29:09.647337", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tvae/model.pt b/treatment/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..61287d1bac33533756256703d4c22c5cd0a02830 --- /dev/null +++ b/treatment/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:874e62fc8cb0c0242dfff3592fcc65c816abc98249e4b16dba0b19135cb92b84 +size 74860097 diff --git a/treatment/tvae/params.json b/treatment/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..69bdb7f8e1ca990340fc4c488eb9eaff7551690c --- /dev/null +++ b/treatment/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "tvae", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file