diff --git a/contraceptive/lct_gan/eval.csv b/contraceptive/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..55158bc82e86458963a2b1b3c34e5e0a0c69c098 --- /dev/null +++ b/contraceptive/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.005347351378184699,0.08097088616235722,0.002836061054452633,12.46730089187622,0.03379097953438759,0.8471567034721375,0.1400071233510971,9.070246051123831e-06,4.109842538833618,0.04161286726593971,0.12254983186721802,0.05325467884540558,0.1082986444234848,0.0010494085727259517,16.57714343070984 diff --git a/contraceptive/lct_gan/history.csv b/contraceptive/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..6cb64867c08e470190ed3c978811988ff06aca2e --- /dev/null +++ b/contraceptive/lct_gan/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.017401078423782666,1.0339260609464513,0.0010806548900643828,0.16402181821875275,0.0,0.0,0.0,0.0,0.01764664788174236,900,225,261.55655670166016,1.1624735853407118,0.29061839633517794,0.1123641776252124,0.016450096456747915,1.6555652437911634,0.0006544369515005302,0.0,0.0,0.0,0.0,0.0,0.016450096456747915,450,113,90.59106540679932,0.8016908443079586,0.20131347868177626,0.10719071906595697 +1,0.007544516106006793,0.805866371501884,0.00011455891246516556,0.05731394776852944,0.0,0.0,0.0,0.0,0.007670205862442446,900,225,262.5022921562195,1.1666768540276422,0.29166921350691055,0.08921317261954148,0.004202535958288031,0.7761073129848487,4.496906836265167e-05,0.0,0.0,0.0,0.0,0.0,0.004202535958288031,450,113,90.03428149223328,0.796763553028613,0.20007618109385172,0.058546484567521685 +2,0.006512888989463035,0.7518540992810281,0.0001415838299930615,0.049427424324288344,0.0,0.0,0.0,0.0,0.0066497801841857536,900,225,262.3658037185669,1.1660702387491861,0.29151755968729653,0.09130782820491327,0.0055336219292237525,1.1570013584530916,6.174433674384281e-05,0.0,0.0,0.0,0.0,0.0,0.0055336219292237525,450,113,89.11512207984924,0.788629398936719,0.1980336046218872,0.04651118999973467 +3,0.006024511704712899,0.4761890591268285,0.0002143375033454278,0.06465743926004507,0.0,0.0,0.0,0.0,0.0061261716642830935,900,225,260.4550771713257,1.1575781207614475,0.2893945301903619,0.09767564491679272,0.004724722134932462,0.7707038955945308,5.621562246012167e-05,0.0,0.0,0.0,0.0,0.0,0.004724722134932462,450,113,87.9228572845459,0.7780783830490787,0.19538412729899088,0.056170098961586444 +4,0.005286015553380518,0.3363608939476823,7.321617665753689e-05,0.045423433695816334,0.0,0.0,0.0,0.0,0.005374089340039063,900,225,260.1741499900818,1.1563295555114745,0.28908238887786863,0.09759037269486322,0.007981064757849607,1.2652324522273108,9.855836417115466e-05,0.0,0.0,0.0,0.0,0.0,0.007981064757849607,450,113,88.88254165649414,0.7865711651017181,0.19751675923665366,0.04572650106969924 +5,0.00576256091059703,0.5418789782114618,7.038583156805957e-05,0.02929313911823556,0.0,0.0,0.0,0.0,0.005917770875255681,900,225,262.8189432621002,1.168084192276001,0.29202104806900026,0.09744484349257417,0.0038669909037222774,1.1829836725373764,2.4478697872928952e-05,0.0,0.0,0.0,0.0,0.0,0.0038669909037222774,450,113,90.46491193771362,0.8005744419266693,0.20103313763936362,0.047730479976656824 +6,0.0038541355246626253,0.3181302580921152,2.260195795584597e-05,0.02967498921504658,0.0,0.0,0.0,0.0,0.003917783604055229,900,225,261.9046974182129,1.1640208774142795,0.2910052193535699,0.10003261071940263,0.0038603629919493365,1.908042863952322,1.606306340802302e-05,0.0,0.0,0.0,0.0,0.0,0.0038603629919493365,450,113,90.71723699569702,0.8028074070415666,0.20159385999043783,0.05183640702017706 +7,0.003702066924338902,0.2994743646377197,2.3150445486223394e-05,0.018106737517001523,0.0,0.0,0.0,0.0,0.0037678834933709974,900,225,260.3814172744751,1.1572507434421115,0.2893126858605279,0.10073738541454076,0.0028049794745553906,1.4249755061280251,1.3086192913582816e-05,0.0,0.0,0.0,0.0,0.0,0.0028049794745553906,450,113,90.54383492469788,0.8012728754398042,0.20120852205488418,0.054640445479117665 +8,0.0033579401259905555,0.3882562007228762,1.8571260889600453e-05,0.031065790198707772,0.0,0.0,0.0,0.0,0.0034128941754655293,900,225,261.0693860054016,1.1603083822462295,0.29007709556155736,0.10073215851146314,0.004292678962616871,2.506649698973513,4.666494623541402e-05,0.0,0.0,0.0,0.0,0.0,0.004292678962616871,450,113,89.04813385009766,0.788036582744227,0.1978847418891059,0.05573896699857 +9,0.0032293959062533557,0.4635296431291696,1.8672718882848967e-05,0.023970585422034167,0.0,0.0,0.0,0.0,0.003291322695731651,900,225,260.9506549835205,1.1597806888156468,0.2899451722039117,0.10359380051907566,0.002744582254672423,2.2099234429464625,2.5944116722460305e-05,0.0,0.0,0.0,0.0,0.0,0.002744582254672423,450,113,89.43556237220764,0.7914651537363508,0.19874569416046142,0.05300469076635926 diff --git a/contraceptive/lct_gan/mlu-eval.ipynb b/contraceptive/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..8d8409dd96fc71608c8b1fe9148d2e211ccb8dc3 --- /dev/null +++ b/contraceptive/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2274 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.686730Z", + "iopub.status.busy": "2024-03-22T19:40:14.686341Z", + "iopub.status.idle": "2024-03-22T19:40:14.721946Z", + "shell.execute_reply": "2024-03-22T19:40:14.721015Z" + }, + "papermill": { + "duration": 0.052769, + "end_time": "2024-03-22T19:40:14.724162", + "exception": false, + "start_time": "2024-03-22T19:40:14.671393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.752739Z", + "iopub.status.busy": "2024-03-22T19:40:14.752339Z", + "iopub.status.idle": "2024-03-22T19:40:14.760219Z", + "shell.execute_reply": "2024-03-22T19:40:14.759294Z" + }, + "papermill": { + "duration": 0.025385, + "end_time": "2024-03-22T19:40:14.762717", + "exception": false, + "start_time": "2024-03-22T19:40:14.737332", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.788860Z", + "iopub.status.busy": "2024-03-22T19:40:14.788607Z", + "iopub.status.idle": "2024-03-22T19:40:14.792716Z", + "shell.execute_reply": "2024-03-22T19:40:14.791871Z" + }, + "papermill": { + "duration": 0.019374, + "end_time": "2024-03-22T19:40:14.794678", + "exception": false, + "start_time": "2024-03-22T19:40:14.775304", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.820786Z", + "iopub.status.busy": "2024-03-22T19:40:14.820487Z", + "iopub.status.idle": "2024-03-22T19:40:14.825085Z", + "shell.execute_reply": "2024-03-22T19:40:14.824184Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019954, + "end_time": "2024-03-22T19:40:14.827258", + "exception": false, + "start_time": "2024-03-22T19:40:14.807304", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.853649Z", + "iopub.status.busy": "2024-03-22T19:40:14.853326Z", + "iopub.status.idle": "2024-03-22T19:40:14.859023Z", + "shell.execute_reply": "2024-03-22T19:40:14.858209Z" + }, + "papermill": { + "duration": 0.021566, + "end_time": "2024-03-22T19:40:14.860965", + "exception": false, + "start_time": "2024-03-22T19:40:14.839399", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "dbf0f552", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.884790Z", + "iopub.status.busy": "2024-03-22T19:40:14.884530Z", + "iopub.status.idle": "2024-03-22T19:40:14.889911Z", + "shell.execute_reply": "2024-03-22T19:40:14.888448Z" + }, + "papermill": { + "duration": 0.019283, + "end_time": "2024-03-22T19:40:14.891787", + "exception": false, + "start_time": "2024-03-22T19:40:14.872504", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/lct_gan/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011443, + "end_time": "2024-03-22T19:40:14.914601", + "exception": false, + "start_time": "2024-03-22T19:40:14.903158", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.938314Z", + "iopub.status.busy": "2024-03-22T19:40:14.938007Z", + "iopub.status.idle": "2024-03-22T19:40:14.947168Z", + "shell.execute_reply": "2024-03-22T19:40:14.946392Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023214, + "end_time": "2024-03-22T19:40:14.949039", + "exception": false, + "start_time": "2024-03-22T19:40:14.925825", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/lct_gan/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:14.974444Z", + "iopub.status.busy": "2024-03-22T19:40:14.974141Z", + "iopub.status.idle": "2024-03-22T19:40:17.011622Z", + "shell.execute_reply": "2024-03-22T19:40:17.010604Z" + }, + "papermill": { + "duration": 2.052564, + "end_time": "2024-03-22T19:40:17.013629", + "exception": false, + "start_time": "2024-03-22T19:40:14.961065", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:17.038360Z", + "iopub.status.busy": "2024-03-22T19:40:17.037936Z", + "iopub.status.idle": "2024-03-22T19:40:17.050479Z", + "shell.execute_reply": "2024-03-22T19:40:17.049537Z" + }, + "papermill": { + "duration": 0.027111, + "end_time": "2024-03-22T19:40:17.052563", + "exception": false, + "start_time": "2024-03-22T19:40:17.025452", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:17.077482Z", + "iopub.status.busy": "2024-03-22T19:40:17.077181Z", + "iopub.status.idle": "2024-03-22T19:40:17.084578Z", + "shell.execute_reply": "2024-03-22T19:40:17.083643Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022091, + "end_time": "2024-03-22T19:40:17.086602", + "exception": false, + "start_time": "2024-03-22T19:40:17.064511", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:17.112737Z", + "iopub.status.busy": "2024-03-22T19:40:17.112453Z", + "iopub.status.idle": "2024-03-22T19:40:17.206554Z", + "shell.execute_reply": "2024-03-22T19:40:17.205788Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.110426, + "end_time": "2024-03-22T19:40:17.209013", + "exception": false, + "start_time": "2024-03-22T19:40:17.098587", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:17.236406Z", + "iopub.status.busy": "2024-03-22T19:40:17.236082Z", + "iopub.status.idle": "2024-03-22T19:40:21.821006Z", + "shell.execute_reply": "2024-03-22T19:40:21.820265Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.601926, + "end_time": "2024-03-22T19:40:21.823508", + "exception": false, + "start_time": "2024-03-22T19:40:17.221582", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 19:40:19.417436: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 19:40:19.417493: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 19:40:19.419102: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:21.853178Z", + "iopub.status.busy": "2024-03-22T19:40:21.852629Z", + "iopub.status.idle": "2024-03-22T19:40:21.858956Z", + "shell.execute_reply": "2024-03-22T19:40:21.857989Z" + }, + "papermill": { + "duration": 0.022425, + "end_time": "2024-03-22T19:40:21.861204", + "exception": false, + "start_time": "2024-03-22T19:40:21.838779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:21.886820Z", + "iopub.status.busy": "2024-03-22T19:40:21.885986Z", + "iopub.status.idle": "2024-03-22T19:40:30.509485Z", + "shell.execute_reply": "2024-03-22T19:40:30.508402Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.639029, + "end_time": "2024-03-22T19:40:30.512044", + "exception": false, + "start_time": "2024-03-22T19:40:21.873015", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:40:30.541861Z", + "iopub.status.busy": "2024-03-22T19:40:30.541424Z", + "iopub.status.idle": "2024-03-22T19:40:30.549638Z", + "shell.execute_reply": "2024-03-22T19:40:30.548702Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.025563, + "end_time": "2024-03-22T19:40:30.551664", + "exception": false, + "start_time": "2024-03-22T19:40:30.526101", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:30.578204Z", + "iopub.status.busy": "2024-03-22T19:40:30.577937Z", + "iopub.status.idle": "2024-03-22T19:40:30.582780Z", + "shell.execute_reply": "2024-03-22T19:40:30.581947Z" + }, + "papermill": { + "duration": 0.020255, + "end_time": "2024-03-22T19:40:30.584737", + "exception": false, + "start_time": "2024-03-22T19:40:30.564482", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:30.610675Z", + "iopub.status.busy": "2024-03-22T19:40:30.610118Z", + "iopub.status.idle": "2024-03-22T19:40:31.126454Z", + "shell.execute_reply": "2024-03-22T19:40:31.125416Z" + }, + "papermill": { + "duration": 0.531769, + "end_time": "2024-03-22T19:40:31.128570", + "exception": false, + "start_time": "2024-03-22T19:40:30.596801", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:31.157774Z", + "iopub.status.busy": "2024-03-22T19:40:31.157137Z", + "iopub.status.idle": "2024-03-22T19:40:31.488158Z", + "shell.execute_reply": "2024-03-22T19:40:31.487052Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.348027, + "end_time": "2024-03-22T19:40:31.490454", + "exception": false, + "start_time": "2024-03-22T19:40:31.142427", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:31.519124Z", + "iopub.status.busy": "2024-03-22T19:40:31.518699Z", + "iopub.status.idle": "2024-03-22T19:40:31.627077Z", + "shell.execute_reply": "2024-03-22T19:40:31.626131Z" + }, + "papermill": { + "duration": 0.125528, + "end_time": "2024-03-22T19:40:31.629325", + "exception": false, + "start_time": "2024-03-22T19:40:31.503797", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:40:31.658594Z", + "iopub.status.busy": "2024-03-22T19:40:31.658305Z", + "iopub.status.idle": "2024-03-22T19:40:32.106194Z", + "shell.execute_reply": "2024-03-22T19:40:32.105218Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.464659, + "end_time": "2024-03-22T19:40:32.108592", + "exception": false, + "start_time": "2024-03-22T19:40:31.643933", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:32.140112Z", + "iopub.status.busy": "2024-03-22T19:40:32.139297Z", + "iopub.status.idle": "2024-03-22T19:40:32.144553Z", + "shell.execute_reply": "2024-03-22T19:40:32.143580Z" + }, + "papermill": { + "duration": 0.023298, + "end_time": "2024-03-22T19:40:32.146774", + "exception": false, + "start_time": "2024-03-22T19:40:32.123476", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:32.174755Z", + "iopub.status.busy": "2024-03-22T19:40:32.174472Z", + "iopub.status.idle": "2024-03-22T19:40:32.181541Z", + "shell.execute_reply": "2024-03-22T19:40:32.180624Z" + }, + "papermill": { + "duration": 0.023501, + "end_time": "2024-03-22T19:40:32.183627", + "exception": false, + "start_time": "2024-03-22T19:40:32.160126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11889160" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:32.211912Z", + "iopub.status.busy": "2024-03-22T19:40:32.211585Z", + "iopub.status.idle": "2024-03-22T19:40:32.297306Z", + "shell.execute_reply": "2024-03-22T19:40:32.296393Z" + }, + "papermill": { + "duration": 0.102372, + "end_time": "2024-03-22T19:40:32.299408", + "exception": false, + "start_time": "2024-03-22T19:40:32.197036", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 40] --\n", + "├─Adapter: 1-1 [2, 1179, 40] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 41,984\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 40] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,889,160\n", + "Trainable params: 11,889,160\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.13\n", + "========================================================================================================================\n", + "Input size (MB): 0.47\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.56\n", + "Estimated Total Size (MB): 423.43\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:40:32.330193Z", + "iopub.status.busy": "2024-03-22T19:40:32.329920Z", + "iopub.status.idle": "2024-03-22T20:47:39.843051Z", + "shell.execute_reply": "2024-03-22T20:47:39.841995Z" + }, + "papermill": { + "duration": 4027.546289, + "end_time": "2024-03-22T20:47:39.860643", + "exception": false, + "start_time": "2024-03-22T19:40:32.314354", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.017401078423782666, 'avg_role_model_std_loss': 1.0339260609464513, 'avg_role_model_mean_pred_loss': 0.0010806548900643828, 'avg_role_model_g_mag_loss': 0.16402181821875275, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01764664788174236, 'n_size': 900, 'n_batch': 225, 'duration': 261.55655670166016, 'duration_batch': 1.1624735853407118, 'duration_size': 0.29061839633517794, 'avg_pred_std': 0.1123641776252124}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.016450096456747915, 'avg_role_model_std_loss': 1.6555652437911634, 'avg_role_model_mean_pred_loss': 0.0006544369515005302, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.016450096456747915, 'n_size': 450, 'n_batch': 113, 'duration': 90.59106540679932, 'duration_batch': 0.8016908443079586, 'duration_size': 0.20131347868177626, 'avg_pred_std': 0.10719071906595697}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007544516106006793, 'avg_role_model_std_loss': 0.805866371501884, 'avg_role_model_mean_pred_loss': 0.00011455891246516556, 'avg_role_model_g_mag_loss': 0.05731394776852944, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007670205862442446, 'n_size': 900, 'n_batch': 225, 'duration': 262.5022921562195, 'duration_batch': 1.1666768540276422, 'duration_size': 0.29166921350691055, 'avg_pred_std': 0.08921317261954148}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004202535958288031, 'avg_role_model_std_loss': 0.7761073129848487, 'avg_role_model_mean_pred_loss': 4.496906836265167e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004202535958288031, 'n_size': 450, 'n_batch': 113, 'duration': 90.03428149223328, 'duration_batch': 0.796763553028613, 'duration_size': 0.20007618109385172, 'avg_pred_std': 0.058546484567521685}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006512888989463035, 'avg_role_model_std_loss': 0.7518540992810281, 'avg_role_model_mean_pred_loss': 0.0001415838299930615, 'avg_role_model_g_mag_loss': 0.049427424324288344, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0066497801841857536, 'n_size': 900, 'n_batch': 225, 'duration': 262.3658037185669, 'duration_batch': 1.1660702387491861, 'duration_size': 0.29151755968729653, 'avg_pred_std': 0.09130782820491327}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0055336219292237525, 'avg_role_model_std_loss': 1.1570013584530916, 'avg_role_model_mean_pred_loss': 6.174433674384281e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0055336219292237525, 'n_size': 450, 'n_batch': 113, 'duration': 89.11512207984924, 'duration_batch': 0.788629398936719, 'duration_size': 0.1980336046218872, 'avg_pred_std': 0.04651118999973467}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006024511704712899, 'avg_role_model_std_loss': 0.4761890591268285, 'avg_role_model_mean_pred_loss': 0.0002143375033454278, 'avg_role_model_g_mag_loss': 0.06465743926004507, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0061261716642830935, 'n_size': 900, 'n_batch': 225, 'duration': 260.4550771713257, 'duration_batch': 1.1575781207614475, 'duration_size': 0.2893945301903619, 'avg_pred_std': 0.09767564491679272}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004724722134932462, 'avg_role_model_std_loss': 0.7707038955945308, 'avg_role_model_mean_pred_loss': 5.621562246012167e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004724722134932462, 'n_size': 450, 'n_batch': 113, 'duration': 87.9228572845459, 'duration_batch': 0.7780783830490787, 'duration_size': 0.19538412729899088, 'avg_pred_std': 0.056170098961586444}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005286015553380518, 'avg_role_model_std_loss': 0.3363608939476823, 'avg_role_model_mean_pred_loss': 7.321617665753689e-05, 'avg_role_model_g_mag_loss': 0.045423433695816334, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005374089340039063, 'n_size': 900, 'n_batch': 225, 'duration': 260.1741499900818, 'duration_batch': 1.1563295555114745, 'duration_size': 0.28908238887786863, 'avg_pred_std': 0.09759037269486322}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007981064757849607, 'avg_role_model_std_loss': 1.2652324522273108, 'avg_role_model_mean_pred_loss': 9.855836417115466e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007981064757849607, 'n_size': 450, 'n_batch': 113, 'duration': 88.88254165649414, 'duration_batch': 0.7865711651017181, 'duration_size': 0.19751675923665366, 'avg_pred_std': 0.04572650106969924}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00576256091059703, 'avg_role_model_std_loss': 0.5418789782114618, 'avg_role_model_mean_pred_loss': 7.038583156805957e-05, 'avg_role_model_g_mag_loss': 0.02929313911823556, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005917770875255681, 'n_size': 900, 'n_batch': 225, 'duration': 262.8189432621002, 'duration_batch': 1.168084192276001, 'duration_size': 0.29202104806900026, 'avg_pred_std': 0.09744484349257417}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0038669909037222774, 'avg_role_model_std_loss': 1.1829836725373764, 'avg_role_model_mean_pred_loss': 2.4478697872928952e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038669909037222774, 'n_size': 450, 'n_batch': 113, 'duration': 90.46491193771362, 'duration_batch': 0.8005744419266693, 'duration_size': 0.20103313763936362, 'avg_pred_std': 0.047730479976656824}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038541355246626253, 'avg_role_model_std_loss': 0.3181302580921152, 'avg_role_model_mean_pred_loss': 2.260195795584597e-05, 'avg_role_model_g_mag_loss': 0.02967498921504658, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003917783604055229, 'n_size': 900, 'n_batch': 225, 'duration': 261.9046974182129, 'duration_batch': 1.1640208774142795, 'duration_size': 0.2910052193535699, 'avg_pred_std': 0.10003261071940263}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0038603629919493365, 'avg_role_model_std_loss': 1.908042863952322, 'avg_role_model_mean_pred_loss': 1.606306340802302e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038603629919493365, 'n_size': 450, 'n_batch': 113, 'duration': 90.71723699569702, 'duration_batch': 0.8028074070415666, 'duration_size': 0.20159385999043783, 'avg_pred_std': 0.05183640702017706}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003702066924338902, 'avg_role_model_std_loss': 0.2994743646377197, 'avg_role_model_mean_pred_loss': 2.3150445486223394e-05, 'avg_role_model_g_mag_loss': 0.018106737517001523, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037678834933709974, 'n_size': 900, 'n_batch': 225, 'duration': 260.3814172744751, 'duration_batch': 1.1572507434421115, 'duration_size': 0.2893126858605279, 'avg_pred_std': 0.10073738541454076}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0028049794745553906, 'avg_role_model_std_loss': 1.4249755061280251, 'avg_role_model_mean_pred_loss': 1.3086192913582816e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028049794745553906, 'n_size': 450, 'n_batch': 113, 'duration': 90.54383492469788, 'duration_batch': 0.8012728754398042, 'duration_size': 0.20120852205488418, 'avg_pred_std': 0.054640445479117665}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0033579401259905555, 'avg_role_model_std_loss': 0.3882562007228762, 'avg_role_model_mean_pred_loss': 1.8571260889600453e-05, 'avg_role_model_g_mag_loss': 0.031065790198707772, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034128941754655293, 'n_size': 900, 'n_batch': 225, 'duration': 261.0693860054016, 'duration_batch': 1.1603083822462295, 'duration_size': 0.29007709556155736, 'avg_pred_std': 0.10073215851146314}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004292678962616871, 'avg_role_model_std_loss': 2.506649698973513, 'avg_role_model_mean_pred_loss': 4.666494623541402e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004292678962616871, 'n_size': 450, 'n_batch': 113, 'duration': 89.04813385009766, 'duration_batch': 0.788036582744227, 'duration_size': 0.1978847418891059, 'avg_pred_std': 0.05573896699857}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0032293959062533557, 'avg_role_model_std_loss': 0.4635296431291696, 'avg_role_model_mean_pred_loss': 1.8672718882848967e-05, 'avg_role_model_g_mag_loss': 0.023970585422034167, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003291322695731651, 'n_size': 900, 'n_batch': 225, 'duration': 260.9506549835205, 'duration_batch': 1.1597806888156468, 'duration_size': 0.2899451722039117, 'avg_pred_std': 0.10359380051907566}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002744582254672423, 'avg_role_model_std_loss': 2.2099234429464625, 'avg_role_model_mean_pred_loss': 2.5944116722460305e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002744582254672423, 'n_size': 450, 'n_batch': 113, 'duration': 89.43556237220764, 'duration_batch': 0.7914651537363508, 'duration_size': 0.19874569416046142, 'avg_pred_std': 0.05300469076635926}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026384538486470574, 'avg_role_model_std_loss': 0.23855205940207644, 'avg_role_model_mean_pred_loss': 1.205315517315884e-05, 'avg_role_model_g_mag_loss': 0.024326985160069953, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026762135176815922, 'n_size': 900, 'n_batch': 225, 'duration': 263.52970361709595, 'duration_batch': 1.1712431271870931, 'duration_size': 0.2928107817967733, 'avg_pred_std': 0.10776877377182245}\n", + "Time out: 3785.1640882492065/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.002836061065248768, 'avg_g_mag_loss': 0.04869947255703835, 'avg_g_cos_loss': 0.010633710139164967, 'pred_duration': 4.068113565444946, 'grad_duration': 12.533695220947266, 'total_duration': 16.601808786392212, 'pred_std': 0.1082986444234848, 'std_loss': 0.0010494085727259517, 'mean_pred_loss': 9.070246051123831e-06, 'pred_rmse': 0.05325468257069588, 'pred_mae': 0.04161286726593971, 'pred_mape': 0.12254984676837921, 'grad_rmse': 0.1400071233510971, 'grad_mae': 0.03379097580909729, 'grad_mape': 0.8471567034721375}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.002836061065248768, 'avg_g_mag_loss': 0.04869947255703835, 'avg_g_cos_loss': 0.010633710139164967, 'avg_pred_duration': 4.068113565444946, 'avg_grad_duration': 12.533695220947266, 'avg_total_duration': 16.601808786392212, 'avg_pred_std': 0.1082986444234848, 'avg_std_loss': 0.0010494085727259517, 'avg_mean_pred_loss': 9.070246051123831e-06}, 'min_metrics': {'avg_loss': 0.002836061065248768, 'avg_g_mag_loss': 0.04869947255703835, 'avg_g_cos_loss': 0.010633710139164967, 'pred_duration': 4.068113565444946, 'grad_duration': 12.533695220947266, 'total_duration': 16.601808786392212, 'pred_std': 0.1082986444234848, 'std_loss': 0.0010494085727259517, 'mean_pred_loss': 9.070246051123831e-06, 'pred_rmse': 0.05325468257069588, 'pred_mae': 0.04161286726593971, 'pred_mape': 0.12254984676837921, 'grad_rmse': 0.1400071233510971, 'grad_mae': 0.03379097580909729, 'grad_mape': 0.8471567034721375}, 'model_metrics': {'lct_gan': {'avg_loss': 0.002836061065248768, 'avg_g_mag_loss': 0.04869947255703835, 'avg_g_cos_loss': 0.010633710139164967, 'pred_duration': 4.068113565444946, 'grad_duration': 12.533695220947266, 'total_duration': 16.601808786392212, 'pred_std': 0.1082986444234848, 'std_loss': 0.0010494085727259517, 'mean_pred_loss': 9.070246051123831e-06, 'pred_rmse': 0.05325468257069588, 'pred_mae': 0.04161286726593971, 'pred_mape': 0.12254984676837921, 'grad_rmse': 0.1400071233510971, 'grad_mae': 0.03379097580909729, 'grad_mape': 0.8471567034721375}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:47:39.896618Z", + "iopub.status.busy": "2024-03-22T20:47:39.896258Z", + "iopub.status.idle": "2024-03-22T20:47:39.900756Z", + "shell.execute_reply": "2024-03-22T20:47:39.899964Z" + }, + "papermill": { + "duration": 0.024824, + "end_time": "2024-03-22T20:47:39.902718", + "exception": false, + "start_time": "2024-03-22T20:47:39.877894", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:47:39.935797Z", + "iopub.status.busy": "2024-03-22T20:47:39.934952Z", + "iopub.status.idle": "2024-03-22T20:47:40.035705Z", + "shell.execute_reply": "2024-03-22T20:47:40.034557Z" + }, + "papermill": { + "duration": 0.119826, + "end_time": "2024-03-22T20:47:40.038220", + "exception": false, + "start_time": "2024-03-22T20:47:39.918394", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:47:40.073678Z", + "iopub.status.busy": "2024-03-22T20:47:40.073343Z", + "iopub.status.idle": "2024-03-22T20:47:40.372089Z", + "shell.execute_reply": "2024-03-22T20:47:40.371184Z" + }, + "papermill": { + "duration": 0.319285, + "end_time": "2024-03-22T20:47:40.374396", + "exception": false, + "start_time": "2024-03-22T20:47:40.055111", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAESCAYAAAB3gfmJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABN5klEQVR4nO2deVxU9frHP8PAzLAOCMKAsqXgSmoohGtdSTLKKG+W15TM1ApzocXsqpj9Epe8WWaWlXgrCbXU3FIRl26KG4KCIG4oGAyIyAzrDMw8vz+OjIwMywwzDOD3/Xqd18x8z3O+5zkj8/F7zvN9vg+PiAgMBoPRibAwtwMMBoNhbJiwMRiMTgcTNgaD0elgwsZgMDodTNgYDEangwkbg8HodDBhYzAYnQ5LczvQnlCr1cjPz4e9vT14PJ653WEwGPUgIpSVlcHDwwMWFk2PyZiw1SM/Px+enp7mdoPBYDRBXl4eunfv3qQNE7Z62NvbA+C+OAcHBzN7w2Aw6iOXy+Hp6an5nTYFE7Z61N1+Ojg4MGFjMNopLXlMxIIHDAaj08GEjcFgdDqYsDEYjE4He8bGMApqtRpKpdLcbjA6OFZWVuDz+a3uhwkbo9UolUrk5ORArVab2xVGJ8DR0RESiaRVc0mZsDFaBRGhoKAAfD4fnp6ezU6cZDAag4hQWVmJoqIiAIC7u3urOtObr776iry9vUkoFFJQUBCdOnWqSfutW7dSr169SCgUUv/+/Wnv3r1a+3/77Td66qmnqEuXLgSAUlNTtfbn5OQQAJ3b1q1bNXa69v/yyy8tvi6ZTEYASCaTNWlXJK+mo9lFdKlA3uK+OytKpZIyMzOptLTU3K4wOgnFxcWUmZlJtbW1Wu0t/X0SEen93+uWLVsQHR2NmJgYnDt3DgMGDEBYWJhGZR/kxIkTmDhxIqZNm4bU1FREREQgIiICGRkZGpuKigoMHz4cK1as0NmHp6cnCgoKtLaPP/4YdnZ2GDt2rJZtXFycll1ERIS+l9gs645cReTG0/g1Jc/ofXc0VCoVAEAgEJjZE0ZnwcbGBgBQU1NjeCf6qmlQUBBFRUVpPqtUKvLw8KDY2Fid9hMmTKDw8HCttuDgYJo5c2YD27qR2YMjNl0MHDiQXn/9da02ALRjx47mL6IRWvo/wuaTN8l7/h6a8kPTI9WHgaqqKsrMzKSqqipzu8LoJDT2N2WyEZtSqURKSgpCQ0M1bRYWFggNDUVycrLOY5KTk7XsASAsLKxR+5aQkpKCtLQ0TJs2rcG+qKgouLi4ICgoCBs3bgQ1UatGoVBALpdrbS3Bz80OAHClsMywC2AwGCZFr+BBcXExVCoV3NzctNrd3Nxw6dIlncdIpVKd9lKpVE9X7/PDDz+gT58+GDp0qFb70qVL8Y9//AM2NjY4ePAg3n77bZSXl2P27Nk6+4mNjcXHH3+s9/n9XblctXxZNcqqa2AvstL/IhgMhsnocCGsqqoqxMfH6xytLVq0CMOGDcOgQYMwf/58fPDBB1i1alWjfS1YsAAymUyz5eW17JmZ2MYKrvZCAMDVonLDLoTx0MPj8bBz505zu2FUlixZgoEDB5rbDf2EzcXFBXw+H4WFhVrthYWFkEgkOo+RSCR62TfHr7/+isrKSkyZMqVZ2+DgYNy6dQsKhULnfqFQqEl41zfx3d+NG7VdKWTCxui4bNq0CY6Ojkbr77333kNSUpLR+jMUvYRNIBAgMDBQy3G1Wo2kpCSEhIToPCYkJKTBhSYmJjZq3xw//PADxo0bh65duzZrm5aWBicnJwiFQoPO1RQ9XbnnbJfZczbGQ0BLs0rs7Ozg7OxsYm+aR+9b0ejoaHz33Xf473//i6ysLLz11luoqKjA1KlTAQBTpkzBggULNPZz5szB/v37sXr1aly6dAlLlizB2bNnMWvWLI1NSUkJ0tLSkJmZCQDIzs5GWlpag+dwV69exZ9//ok33nijgV+7d+/G999/j4yMDFy9ehXr16/HsmXL8M477+h7iS2ibsR2md2KakFEqFTWmmVrKlCki/3792P48OFwdHSEs7Mznn32WVy7dg0AMHToUMyfP1/L/vbt27CyssKff/4JACgoKEB4eDisra3h6+uL+Ph4+Pj4YM2aNQZ9d+np6fjHP/4Ba2trODs7Y8aMGSgvv//3dfToUQQFBcHW1haOjo4YNmwYbt68CQA4f/48nnzySdjb28PBwQGBgYE4e/Zsk+c7evQopk6dCplMBh6PBx6PhyVLlgAAfHx88Mknn2DKlClwcHDAjBkzAADz58+Hv78/bGxs8Mgjj2DRokVa0zIevBV97bXXEBERgc8++wzu7u5wdnZGVFRU66ZytAC9Mw9efvll3L59G4sXL4ZUKsXAgQOxf/9+TYAgNzdXa/b50KFDER8fj4ULF+Kjjz6Cn58fdu7cif79+2tsdu3apRFGAHjllVcAADExMZovGgA2btyI7t27Y8yYMQ38srKywrp16zBv3jwQEXr27In//Oc/mD59ur6X2CL870VGr7IRmxZVNSr0XXzALOfOXBoGG0HL/6QrKioQHR2NRx99FOXl5Vi8eDFeeOEFpKWlYdKkSVi5ciWWL1+uSe3ZsmULPDw8MGLECADcf+LFxcU4evQorKysEB0d3eh8zpb4EhYWhpCQEJw5cwZFRUV44403MGvWLGzatAm1tbWIiIjA9OnT8csvv0CpVOL06dMa3yZNmoRBgwZh/fr14PP5SEtLg5VV00GtoUOHYs2aNVi8eDGys7MBcCOuOj777DMsXrwYMTExmjZ7e3ts2rQJHh4eSE9Px/Tp02Fvb48PPvig0fMcOXIE7u7uOHLkCK5evYqXX34ZAwcONNlvEwB4pO9/c50YuVwOsVgMmUzW7PM2WWUNBiw9CABIXzLmoY2MVldXIycnB76+vhCJRKhU1nYYYXuQ4uJidO3aFenp6XBzc4OHhwcOHz6sEbKhQ4di5MiRWL58OS5duoQ+ffrgzJkzGDx4MADujsLPzw+ff/455s6d2+z5eDweduzYgYiICHz33XeYP38+8vLyYGtrCwDYt28fnnvuOeTn58PKygrOzs44evQoRo0a1aAvBwcHrF27FpGRkXpd86ZNmzB37lyUlpZqtfv4+GDQoEHYsWNHk8d/9tlnSEhI0IwOlyxZgp07dyItLQ0AN2I7evQorl27pklunzBhAiwsLJCQkKCzzwf/purQ5/fJckUNpC4yWlSmwJWicjzm5WRul9oF1lZ8ZC4NM9u59eHKlStYvHgxTp06heLiYk0Sf25uLvr3748xY8Zg8+bNGDFiBHJycpCcnIxvv/0WAPe4xNLSEo899pimv549e8LJybC/g6ysLAwYMEAjagAwbNgwqNVqZGdnY+TIkXjttdcQFhaGp556CqGhoZgwYYImnzI6OhpvvPEGfvrpJ4SGhuKll15Cjx49DPKljjrBrs+WLVvw5Zdf4tq1aygvL0dtbW2zItOvXz+tFTvc3d2Rnp7eKt+ao8NN92hP3I+MstvROng8HmwElmbZ9F0N4rnnnkNJSQm+++47nDp1CqdOnQJw/0H5pEmT8Ouvv6Kmpgbx8fEICAhAQECA0b+zlhIXF4fk5GQMHToUW7Zsgb+/P06ePAmAGyldvHgR4eHhOHz4MPr27dvsaKs56osswE22nzRpEp555hns2bMHqamp+Pe//91sYOHBW2Iej2fylWCYsLWC+xkILIDQ0bhz5w6ys7OxcOFCjB49Gn369MHdu3e1bJ5//nlUV1dj//79iI+Px6RJkzT7evXqhdraWqSmpmrarl692qCPltKnTx+cP38eFRUVmrbjx4/DwsICvXr10rQNGjQICxYswIkTJ9C/f3/Ex8dr9vn7+2PevHk4ePAgXnzxRcTFxTV7XoFAoMn3bY4TJ07A29sb//73vzF48GD4+flpghftDSZsrYBFRjsuTk5OcHZ2xoYNG3D16lUcPnwY0dHRWja2traIiIjAokWLkJWVhYkTJ2r29e7dG6GhoZgxYwZOnz6N1NRUzJgxA9bW1gatIzZp0iSIRCJERkYiIyMDR44cwTvvvIPJkyfDzc0NOTk5WLBgAZKTk3Hz5k0cPHgQV65cQZ8+fVBVVYVZs2bh6NGjuHnzJo4fP44zZ86gT58+zZ7Xx8cH5eXlSEpKQnFxMSorKxu19fPzQ25uLhISEnDt2jV8+eWXrR4VmgombK3An+WMdljqHl6npKSgf//+mDdvns4slUmTJuH8+fMYMWIEvLy8tPb9+OOPcHNzw8iRI/HCCy9oIoT1H3i3FBsbGxw4cAAlJSUYMmQI/vnPf2L06NH46quvNPsvXbqE8ePHw9/fHzNmzEBUVBRmzpwJPp+PO3fuYMqUKfD398eECRMwduzYFqULDh06FG+++SZefvlldO3aFStXrmzUdty4cZg3bx5mzZqFgQMH4sSJE1i0aJHe19oWsKhoPfSJugCArKoGAz7mIqMXloyBw0MYGW0sgvUwcuvWLXh6euLQoUMYPXq0ud3psLCoqLkouADcOg2xa1+4OQhRKFfgSmE5Ar1ZZPRh4vDhwygvL0dAQAAKCgrwwQcfwMfHByNHjjS3aw897FbUEDJ+Bfa+C1zcoXnOdrWI3Y4+bNTU1OCjjz5Cv3798MILL6Br166aybqbN2+GnZ2dzq1fv35t5uPYsWMb9WPZsmVt5kdbw0ZshuDky72W5MDP1R7/u1KMyywy+tARFhaGsDDdc/bGjRuH4OBgnfuaywgwJt9//z2qqqp07uvSpUub+dHWMGEzhC73hO3uDfj5s2R4RkPs7e1hb29vbjfQrVs3c7tgFtitqCHUjdhKb8K/qzUANpeNwWhPMGEzBHF3wMIKUCnhZ8MJmlReDXm1aVcsYDAYLYMJmyFY8AFHbk6TQ2UeJA5cSJqN2hiM9gETNkNx8uFeS3JYcRcGo53BhM1QNAEELjIKgEVGGYx2AhM2Q3G6HxnVpFaxuWwMPeiMxVzaC0zYDKVLvblsrLALo4Ni7GIuALfkOI/Ha7B4ZVvChM1QnO7fitYVdpHKqyGrYpFRBsPcMGEzlLrgQbUMYirTREYf+tQqIkBZYZ6NFXMxWjEXhUKB9957D926dYOtrS2Cg4Nx9OhRzbE3b97Ec889BycnJ9ja2qJfv37Yt28fbty4gSeffBIAtzQUj8fDa6+9ZtD30RpY5oGhCGwAOzegvJALILjZQSqvvpcM33lTVZqlphJY5mGec3+UDwhsm7e7Byvm0ngxl1mzZiEzMxMJCQnw8PDAjh078PTTTyM9PR1+fn6IioqCUqnEn3/+CVtbW2RmZsLOzg6enp747bffMH78eGRnZ8PBwQHW1tYGfSetgQlba3Dy5YStJAf+bn1ZzmgHY/z48VqfN27ciK5duyIzMxMTJkzA3Llz8ddff2mELD4+HhMnTgSPx8OlS5dw6NAhrWIu33//Pfz8/AzyJT4+HtXV1fjxxx81S3J/9dVXeO6557BixQpYWVlBJpPh2Wef1dQyqL+QZG5uLt5//3307t0bAFrkh0AggFgsBo/H0ypgnpubi7i4OOTm5sLDg/tP6r333sP+/fsRFxeHZcuWITc3F+PHj9cslf7II49ojq/LQXV1dTX687uWwoStNXTxBfJOcjmjrkEAWGQUVjbcyMlc59YDVsxFN+np6VCpVPD399dqVygUmmLIs2fPxltvvYWDBw8iNDQU48ePx6OPPmrQ+UwBe8bWGuoFEOoiow99MjyPx90OmmNjxVyMUsylvLwcfD4fKSkpSEtL02xZWVn44osvAABvvPEGrl+/jsmTJyM9PR2DBw/G2rVrjXatrYUJW2vQTPm4ock+KJQrWGS0A8CKuXDoKuYyaNAgqFQqFBUVoWfPnlpb/VtWT09PvPnmm9i+fTveffddfPfdd5o+AbS4SIwpMEjY1q1bBx8fH4hEIgQHB+P06dNN2m/btg29e/eGSCRCQEAA9u3bp7V/+/btGDNmDJydncHj8TTFVuvzxBNPaCI3ddubb76pZZObm4vw8HDY2NjA1dUV77//Pmpraw25xJZRb8TmILKCu5hFRjsKrJgLh65iLv7+/pg0aRKmTJmC7du3IycnB6dPn0ZsbCz27t0LAJg7dy4OHDiAnJwcnDt3DkeOHNGcz9vbGzweD3v27MHt27e1IrttBulJQkICCQQC2rhxI128eJGmT59Ojo6OVFhYqNP++PHjxOfzaeXKlZSZmUkLFy4kKysrSk9P19j8+OOP9PHHH9N3331HACg1NbVBP6NGjaLp06dTQUGBZpPJZJr9tbW11L9/fwoNDaXU1FTat28fubi40IIFC1p8bTKZjABo9dskZUVEMQ5EMWIiZRVN/uEUec/fQ/Gnbrb4nB2dqqoqyszMpKqqKnO7ojeJiYnUp08fEgqF9Oijj9LRo0cJAO3YsUNjs2/fPgJAI0eObHB8fn4+jR07loRCIXl7e1N8fDy5urrSN99806LzP3iuCxcu0JNPPkkikYi6dOlC06dPp7KyMiIikkqlFBERQe7u7iQQCMjb25sWL15MKpWKFAoFvfLKK+Tp6UkCgYA8PDxo1qxZLf43efPNN8nZ2ZkAUExMDBERKZVKWrx4Mfn4+JCVlRW5u7vTCy+8QBcuXCAiolmzZlGPHj1IKBRS165dafLkyVRcXKzpc+nSpSSRSIjH41FkZGSL/Kijsb8pfX6fegtbUFAQRUVFaT6rVCry8PCg2NhYnfYTJkyg8PBwrbbg4GCaOXNmA9ucnJwmhW3OnDmN+rVv3z6ysLAgqVSqaVu/fj05ODiQQqFo5qo49BY2tZroUw9O3Iou0dLdF8l7/h5asiujZcd3AjqysBmbvLw8AkCHDh0ytysdGmMIm163okqlEikpKQgNDdW0WVhYIDQ0FMnJyTqPSU5O1rIHuCWVG7Nvis2bN8PFxQX9+/fHggULtGogJicnIyAgAG5ublrnkcvluHjxos7+FAoF5HK51qYXPJ7unFE25eOh4PDhw9i1axdycnJw4sQJvPLKK6yYSztBL2ErLi6GSqXSEg8AcHNzg1Qq1XmMVCrVy74x/vWvf+Hnn3/GkSNHsGDBAvz000949dVXmz1P3T5dxMbGQiwWazZPT0+9fAIAdPHhXuvnjLJnbA8FrJhL+6XDzGObMWOG5n1AQADc3d0xevRoXLt2zeD5OgsWLNB6YCyXy/UXt/pTPgZqR0bF1g9fndGHCVbMpf2il7C5uLiAz+ejsLBQq72wsFArDFwfiUSil31LqfujuXr1Knr06AGJRNIgOlt33sbOJRQKIRQKW+VH/VU+7O9FRgtk1bhSWIbBPp33D4fRNKyYi3nR61ZUIBAgMDAQSUlJmja1Wo2kpCSEhIToPCYkJETLHgASExMbtW8pdVNC6mZeh4SEID09XStXLzExEQ4ODujbt2+rztUkdcnwd3MAoN5E3YfrORvpmYDOYDSGMf6W9L4VjY6ORmRkJAYPHoygoCCsWbMGFRUVmDp1KgAuMbhbt26IjY0FAMyZMwejRo3C6tWrER4ejoSEBJw9exYbNmzQ9FlSUoLc3Fzk53OpOHUJuRKJBBKJBNeuXUN8fDyeeeYZODs748KFC5g3bx5GjhypSeMYM2YM+vbti8mTJ2PlypWQSqVYuHAhoqKiWj8qawrNrehNQK2Gv6sd/rx8+6F5zsbn8wFwgSVzJDszOh91QcHW3LLrLWwvv/wybt++jcWLF0MqlWLgwIHYv3+/5kF9bm4uLCzuDwSHDh2K+Ph4LFy4EB999BH8/Pywc+dO9O/fX2Oza9cujTACwCuvvAIAiImJwZIlSyAQCHDo0CGNiHp6emL8+PFYuHCh5hg+n489e/bgrbfeQkhICGxtbREZGYmlS5fq/63og9gTsLAEVAqgrEBTGf5hiYxaWlrCxsZGs6RP/X97BkMfiAiVlZUoKiqCo6Oj5j9NQ+ARu4fQIJfLIRaLIZPJ4ODg0PIDvxjI3Yq+thfnLPrhxa9PwNVeiNP/Dm320M6AUqlETk6OJomcwWgNjo6OkEgkDTI49Pl9dpioaLumiy8nbCU58OvLBTWKyhSQVdZAbNP5I6MCgQB+fn6a5HEGw1CsrKxaNVKrgwmbMag35cNeZAUPsQj5smpcKXp4IqMWFhYQiUTmdoPBAMBW9zAO9WqMAg9vZJTBaC8wYTMG9WqMAoDfveIuD/3abAyGmWDCZgzq5YsCuB8ZfUimfDAY7Q0mbMag7la06i5QVapZdPJhmfLBYLQ3mLAZA6EdYOvKva+3THhdZJTBYLQtTNiMRb2cUTuhJbo5crPwL7PbUQajzWHCZiweyBntyQIIDIbZYMJmLJzuj9gAsEUnGQwzwoTNWHTRjoyyRScZDPPBhM1YNDLlg03SZTDaHiZsxqJuxCa7BdQqNM/YbpcpUFrJcigZjLaECZuxsO0KWNkCIKA0VzsyykZtDEabwoTNWPB4OnJG7wUQ2HM2BqNNYcJmTB7IGX3YFp1kMNoLTNiMiWYu2w0AbC4bg2EumLAZky4PzmVjkVEGwxwwYTMmTrqXLyouV+BuBYuMMhhtBRM2Y1J/kq5aDdt6kdErRWzUxmC0FUzYjInYE+DxgdpqoFwK4H5klD1nYzDaDiZsxoRvBYi7c+9LHoyMMmFjMNoKJmzG5sGcUde6uWzsVpTBaCuYsBkbJ91z2VhklMFoO5iwGZsHpnz0ZJFRBqPNMUjY1q1bBx8fH4hEIgQHB+P06dNN2m/btg29e/eGSCRCQEAA9u3bp7V/+/btGDNmDJydncHj8ZCWlqa1v6SkBO+88w569eoFa2treHl5Yfbs2ZDJZFp2PB6vwZaQkGDIJRrOAyM2W62cUfacjcFoC/QWti1btiA6OhoxMTE4d+4cBgwYgLCwMBQVFem0P3HiBCZOnIhp06YhNTUVERERiIiIQEZGhsamoqICw4cPx4oVK3T2kZ+fj/z8fHz22WfIyMjApk2bsH//fkybNq2BbVxcHAoKCjRbRESEvpfYOh7IFwXuLzp5mT1nYzDaBtKToKAgioqK0nxWqVTk4eFBsbGxOu0nTJhA4eHhWm3BwcE0c+bMBrY5OTkEgFJTU5v1Y+vWrSQQCKimpkbTBoB27NjRsgshourqapLJZJotLy+PAJBMJmtxHw07lRPFOHBbVSkRES3bm0ne8/fQ4p3phvfLYDzkyGSyFv8+9RqxKZVKpKSkIDQ0VNNmYWGB0NBQJCcn6zwmOTlZyx4AwsLCGrVvKTKZDA4ODrC0tNRqj4qKgouLC4KCgrBx40YQUaN9xMbGQiwWazZPT89W+QQAENoDNi7c+wdW02UBBAajbdBL2IqLi6FSqeDm5qbV7ubmBqlUqvMYqVSql31L/fjkk08wY8YMrfalS5di69atSExMxPjx4/H2229j7dq1jfazYMECyGQyzZaXl2ewT1o8EEC4P+WDPWNjMNoCy+ZN2hdyuRzh4eHo27cvlixZorVv0aJFmveDBg1CRUUFVq1ahdmzZ+vsSygUQigUGt9JJ1/g1pkGFauKy5UoqVCii63A+OdkMBga9Bqxubi4gM/no7CwUKu9sLAQEolE5zESiUQv+6YoKyvD008/DXt7e+zYsQNWVlZN2gcHB+PWrVtQKBR6n6tVPDBisxVaorvTvZxRFhllMEyOXsImEAgQGBiIpKQkTZtarUZSUhJCQkJ0HhMSEqJlDwCJiYmN2jeGXC7HmDFjIBAIsGvXLohEomaPSUtLg5OTk2lGZU3xQI1RoN5EXRYZZTBMjt63otHR0YiMjMTgwYMRFBSENWvWoKKiAlOnTgUATJkyBd26dUNsbCwAYM6cORg1ahRWr16N8PBwJCQk4OzZs9iwYYOmz5KSEuTm5iI/Px8AkJ2dDYAb7UkkEo2oVVZW4ueff4ZcLodcLgcAdO3aFXw+H7t370ZhYSEef/xxiEQiJCYmYtmyZXjvvfda9w0ZwgMVqwDuOdvhS0VsxMZgtAWGhF3Xrl1LXl5eJBAIKCgoiE6ePKnZN2rUKIqMjNSy37p1K/n7+5NAIKB+/frR3r17tfbHxcURgAZbTEwMEREdOXJE534AlJOTQ0REf/zxBw0cOJDs7OzI1taWBgwYQN988w2pVKoWX5c+4eQmkRdw0z2WOBLVKIiIaNvZPPKev4de/vZE6/pmMB5S9Pl98oiamA/xkCGXyyEWizVTSQyGCFjmAdRUAu+cA5x74MKtUoz76jhc7AQ4u/Ap4znNYDwk6PP7ZLmipkBHxaoHI6MMBsN0MGEzFQ/kjNoI7kdGWc4og2FamLCZCp05o2zRSQajLWDCZiq66IiMurFFJxmMtoAJm6l44FYUAPxd63JG2YiNwTAlTNhMRf0R273As2bExpLhGQyTwoTNVIg9AZ4FN+WjnEspq4uM3qlQ4k55G6d5MRgPEUzYTIWloEHFKhuBJTy7sDqjDIapYcJmSnTljLqyyCiDYWqYsJkSXTmjbNFJBsPkMGEzJQ8sXwTcX3SSRUYZDNPBhM2U6JryUTdJlz1jYzBMBhM2U6JjxNbT1Q48HlDCIqMMhslgwmZK6oIHlcWAgrv1tBbw4elkA4A9Z2MwTAUTNlMiEgPWXbj3Op6zseIuDIZpYMJmanTmjLLUKgbDlDBhMzU6AwgstYrBMCVM2EyNjgACi4wyGKaFCZup0TFi69H1fmS0mEVGGQyjw4TN1OgYsWlHRtlzNgbD2DBhMzV1Uz5ktwBVjaa57jnbVXY7ymAYHSZspsZOAliKAFIBsjxNM4uMMhimgwmbqbGw0Fn/4H7OKBuxMRjGhglbW9BUzmhhGVhpVwbDuBgkbOvWrYOPjw9EIhGCg4Nx+vTpJu23bduG3r17QyQSISAgAPv27dPav337dowZMwbOzs7g8XhIS0tr0Ed1dTWioqLg7OwMOzs7jB8/HoWFhVo2ubm5CA8Ph42NDVxdXfH++++jtrbWkEs0LjoCCHWR0buVNbjD6owyGEZFb2HbsmULoqOjERMTg3PnzmHAgAEICwtDUVGRTvsTJ05g4sSJmDZtGlJTUxEREYGIiAhkZGRobCoqKjB8+HCsWLGi0fPOmzcPu3fvxrZt23Ds2DHk5+fjxRdf1OxXqVQIDw+HUqnEiRMn8N///hebNm3C4sWL9b1E46NjXTZrAR9eXVhklMEwCaQnQUFBFBUVpfmsUqnIw8ODYmNjddpPmDCBwsPDtdqCg4Np5syZDWxzcnIIAKWmpmq1l5aWkpWVFW3btk3TlpWVRQAoOTmZiIj27dtHFhYWJJVKNTbr168nBwcHUigULbo2mUxGAEgmk7XIvsVkHyCKcSBaF6LVPG3TGfKev4c2Hc8x7vkYjE6IPr9PvUZsSqUSKSkpCA0N1bRZWFggNDQUycnJOo9JTk7WsgeAsLCwRu11kZKSgpqaGq1+evfuDS8vL00/ycnJCAgIgJubm9Z55HI5Ll68qLNfhUIBuVyutZkEHRWrgPtVq9iIjcEwLnoJW3FxMVQqlZZ4AICbmxukUqnOY6RSqV72jfUhEAjg6OjYaD+Nnaduny5iY2MhFos1m6enZ4t90gtHLwA8oKYCqLitaWY5owyGaXioo6ILFiyATCbTbHl5ec0fZAiWwgYVqwDAr66AchGLjDIYxkQvYXNxcQGfz28QjSwsLIREItF5jEQi0cu+sT6USiVKS0sb7aex89Tt04VQKISDg4PWZjJ0VKyqW023tLIGxeUsMspgGAu9hE0gECAwMBBJSUmaNrVajaSkJISEhOg8JiQkRMseABITExu110VgYCCsrKy0+snOzkZubq6mn5CQEKSnp2tFZxMTE+Hg4IC+ffu2+FwmQ8eUD5HV/cgoK8fHYBgPS30PiI6ORmRkJAYPHoygoCCsWbMGFRUVmDp1KgBgypQp6NatG2JjYwEAc+bMwahRo7B69WqEh4cjISEBZ8+exYYNGzR9lpSUIDc3F/n5+QA40QK4kZZEIoFYLMa0adMQHR2NLl26wMHBAe+88w5CQkLw+OOPAwDGjBmDvn37YvLkyVi5ciWkUikWLlyIqKgoCIXC1n1LxkDHiA3gbkdv3qnE5cIyDO3p0vZ+MRidEUPCrmvXriUvLy8SCAQUFBREJ0+e1OwbNWoURUZGatlv3bqV/P39SSAQUL9+/Wjv3r1a++Pi4ghAgy0mJkZjU1VVRW+//TY5OTmRjY0NvfDCC1RQUKDVz40bN2js2LFkbW1NLi4u9O6771JNTU2Lr8tk0z2IiNJ/46Z8fP+UVvOKP7LIe/4e+mj7BeOfk8HoROjz++QRsafWdcjlcojFYshkMuM/b8tPBTY8Adi6Au9f0TTvTP0bc7ekIcinC7a+2fLbcwbjYUOf3+dDHRVtU+qyDyqKAMX96R0965LhWWSUwTAaTNjaCmtHwNqJe18vtaqnqx0s7kVGb7PVdBkMo8CErS3RscpH/cjoVTZRl8EwCkzY2hId67IBbNFJBsPYMGFrS3TUGAXqLTrJlglnMIwCE7a2RMetKKC96CSDwWg9TNjaEh3ZB0D9VT7KWWSUwTACTNjakroRmywPUN1f2bdHVy4yKqtikVEGwxgwYWtL7N0BvhBQ12pVrNLOGWXP2RiM1sKErS2xsACcvLn3D+aMssgog2E0mLC1NTrqHwD3F51k5fgYjNbDhK2taSSAUBcZvVrERmwMRmthwtbWNDLlQ7OaLouMMhithglbW6MZsd3Qan6kq+39yGgZi4wyGK2BCVtbU3/EVm9kJrLiw9vZFgB7zsZgtBYmbG1NXcUqZTlQeUdrV11q1RX2nI3BaBVM2NoaKxHg4MG9bySAwEZsDEbrYMJmDhoLIGjqjLIRG4PRGpiwmYMuPtzrgzmjrvcn6bLIKINhOEzYzEEjI7a6yKi8upZFRhmMVsCEzRw0MklXZMWHD4uMNk5aPPBJVyBzl7k9YbRzmLCZA02N0RsNdmmKu7DnbNooyoGDiwCVEji2UmuqDIPxIEzYzEHdrWi5FFBWau3SLDrJpnxoc3oDUFnMvS9MB/5OMa8/jHYNEzZzYNMFEIm59w8uE66JjLJbUQ3VMuD4F9x7h+7c69mN5vOH0e4xSNjWrVsHHx8fiEQiBAcH4/Tp003ab9u2Db1794ZIJEJAQAD27duntZ+IsHjxYri7u8Pa2hqhoaG4cuV+UeGjR4+Cx+Pp3M6cOQMAuHHjhs79J0+eNOQSTU8zy4RnF5ahrLqmrb1qn5xcD1SXAi7+wPjvubaM34Cqu2Z1i9F+0VvYtmzZgujoaMTExODcuXMYMGAAwsLCUFRUpNP+xIkTmDhxIqZNm4bU1FREREQgIiICGRkZGpuVK1fiyy+/xDfffINTp07B1tYWYWFhqK6uBgAMHToUBQUFWtsbb7wBX19fDB48WOt8hw4d0rILDAzU9xLbhkYCCL4utnAQWaKsuhajVx/D7vP5D/fUj8oSIHkd9/6JDwGvxwG3/kBtNXB+i3l9Y7RfSE+CgoIoKipK81mlUpGHhwfFxsbqtJ8wYQKFh4drtQUHB9PMmTOJiEitVpNEIqFVq1Zp9peWlpJQKKRffvlFZ59KpZK6du1KS5cu1bTl5OQQAEpNTdX3kjTIZDICQDKZzOA+WkziEqIYB6I90Q12nbp+h55YdYS85+8h7/l76NXvT9L12+Wm96k9cuhj7ntaF0KkUnFtp7/j2tYOIVKrzesfo83Q5/ep14hNqVQiJSUFoaGhmjYLCwuEhoYiOTlZ5zHJycla9gAQFhamsc/JyYFUKtWyEYvFCA4ObrTPXbt24c6dO5g6dWqDfePGjYOrqyuGDx+OXbuanhagUCggl8u1tjajichokG8X/DFnBOaF+kNgaYH/XSlG2Od/4j+Jl1Fdo2o7H81NRTFw8hvu/ZMLuBWIASBgAmBlCxRnAzdPmM8/RrtFL2ErLi6GSqWCm5ubVrubmxukUqnOY6RSaZP2da/69PnDDz8gLCwM3bt317TZ2dlh9erV2LZtG/bu3Yvhw4cjIiKiSXGLjY2FWCzWbJ6eno3aGp1GbkXrEFnxMSfUD4nzRmKUf1coVWp8mXQFYWv+xNFs3bf9nY7ja4CaCsB9AND72fvtIgcg4J/cexZEYOigw0VFb926hQMHDmDatGla7S4uLoiOjkZwcDCGDBmC5cuX49VXX8WqVasa7WvBggWQyWSaLS8vr1Fbo1MXPCjNBdSNj8K8nW2xaeoQfD3pMUgcRLh5pxKvxZ3B25tTIJVVt5GzZqCsEDh9L1Dw5L8BHk97/+B7o/WsXdzIjsGoh17C5uLiAj6fj8LCQq32wsJCSCQSncdIJJIm7eteW9pnXFwcnJ2dMW7cuGb9DQ4OxtWrVxvdLxQK4eDgoLW1GQ4eAF8AqGsA2a0mTXk8Hp4JcMehd0fhjeG+4FvwsC9ditGrj+L7/11HrUrdRk63IX/9B6itAroNBvzGNNzvMYjbVEogbXPb+8do1+glbAKBAIGBgUhKStK0qdVqJCUlISQkROcxISEhWvYAkJiYqLH39fWFRCLRspHL5Th16lSDPokIcXFxmDJlCqysrJr1Ny0tDe7u7i2+vjbFgg846q5Y1Rh2QkssfLYvds8ajse8HFGhVOH/9mbh2bV/IeVmiQmdbWNkf9+/xfyHjtFaHYNf517PxgHqTijuDMPRNzKRkJBAQqGQNm3aRJmZmTRjxgxydHQkqVRKRESTJ0+mDz/8UGN//PhxsrS0pM8++4yysrIoJiaGrKysKD09XWOzfPlycnR0pN9//50uXLhAzz//PPn6+lJVVZXWuQ8dOkQAKCsrq4FfmzZtovj4eMrKyqKsrCz69NNPycLCgjZu3Njia2vTqCgR0c//5KJ7Z1ruYx0qlZp+OXWTBnx8QBM9nf/reSopV5jA0TZm91zue/nh6aajnopyomXdOdurh9vOP4ZZ0Of3qbewERGtXbuWvLy8SCAQUFBQEJ08eVKzb9SoURQZGallv3XrVvL39yeBQED9+vWjvXv3au1Xq9W0aNEicnNzI6FQSKNHj6bs7OwG5504cSINHTpUp0+bNm2iPn36kI2NDTk4OFBQUBBt27ZNr+tqc2Hb+x73ozy42OAu7pQr6P1taRpxG/jxAdpyOpdUqg46DaLkBtHHztz3kvO/5u3rvsOEV03vG8Os6PP75BE9zLM/tZHL5RCLxZDJZG3zvC35a+DAAqDv88CEH1vV1ZkbJVi4IwPZ95LnB3s74f9e6I/ekjZ8bmgMfo8CUn8GfEcBkS1YxaMwE1gfAvD4QHQmYK/7WS+j46PP77PDRUU7Fc1M+dCHIT5dsGf2cHz0TG/YCPg4e/Muwr/8C5/uzUSForbV/bcJd64Bab9w7/+xsGXHuPUFPB8HSAWk/mQ63xgdCiZs5qR+VXgjDJyt+BaYMbIHDkWPwtP9JFCpCd/9Lweh/zmG/RkF7T8169gKTqB6PgV4BrX8uLogQsp/m5w6w3h4YMJmTpzuRUUVci4n0kh4OFrjm8mBiHttCDy7WKNAVo03fz6H1zedQe6dyuY7MAe3s4ELW7n3T36k37F9nwesnQBZHnD1kPF9Y3Q4mLCZEytrwP5exaoWTvnQhyd7uyJx3ii884+esOLzcCT7Np76/Bi+TLoCWVU7WznkaCwAAnqFA90e0+9YKxEwcBL3nmUiMMCEzfw0kTNqDERWfLw7phf2zx2JoT2coahV4z+JlxH4SSIm/3AKP528iUK5mTMYpBnAxR3ce31Ha3UEvsa9XjkIlLZhBgmjXcKEzdwYMYDQFD262mHzG8H44pWB8HO1Q62a8L8rxVi0MwPBy5IQse44vj56Fddum2GBy6Ox3GvfCEDS37A+XPwAnxEAqYFzrYswMzo+luZ24KGnkQUnTQGPx8PzA7vh+YHdcP12OQ5mFuLgRSnO5ZYiLY/bVu7PRo+uthjTT4KwfhI82k0MC4tGZv4bg/xU4NIeADzgiQWt62vw68CN/3HCNuoDgN98dgqjc8KEzdy00YjtQR7paoc3R9nhzVE9UCSvRmJWIQ5eLMSJa8W4drsC649ew/qj1yBxEOGpvm4Y088Njz/iDCu+kQf5R5ZxrwEvAa69W9dX72cB265cLYnL+4E+z7XeP0aHhAmbuWnDEVtjuDqIMCnYG5OCvSGvrsHR7Ns4cFGKo5eKIJVX46eTN/HTyZuwF1lidG9XjOknwSj/rrAVtvLPJ+8090yMx+dWx20tlgJg0GQugf7sRiZsDzFM2MxN3YitrACoqeIipWbEQWSFcQM8MG6ABxS1Kpy4egcHM6VIzCxEcbkSO9PysTMtHwJLC4zo6YKwfhKM7uMKZzuh/ic78in3OmAi4NzDOBcQGAn89Tlw7TBQch3o8ohx+mV0KFhKVT3aPKUK4CbmLvfi5rK9far1t2MmQqUmpObexcHMQhy4KMXNevPhLHjAYO8uGNPPDWH9JPDsYtN8hzeOA5ueASwsgXdS7keHjcHP47n5bMPmAk99bLx+GWZFn98nE7Z6mEXYAOCbEYD0AjAxAeg1tu3OayBEhMuF5ThwUYqDmVJk/K29pPoAT0e8MdwXY/tLYKnrmRwRsCkcuHkcCJwKPLfGuA5e2gsk/AuwceHyRy0NGE0y2h36/D7ZrWh7oIsvJ2xtHEAwFB6Ph14Se/SS2GP2aD/culuJxEwu+HD6RgnO55XinV9S0c3RGq8P98XLQzxhV/953PWjnKjxBcDI94zvoF8YN/G5LB/I2n1/GXHGQwObx9YeaAcBhNbQ3ckGU4f54pcZj+PUR6MxL9QfzrYC/F1ahU/2ZCIkNgmxf2RxS5kT3X+2FjgVEHdvunND4FsCj03h3p+NM37/jHYPE7b2gJmmfJgCFzsh5oT64fiH/0DsiwF4pKstyqpr8e2x6xi+4jC+2/gtcOsMYCkCRkSbzpHHpgA8C+DmX8Dty42aqdWEMzdK8PHuixix8jBe2ZDccVZDYTQKuxVtD3TwEZsuRFZ8TAzywsuDPXEkuwgb/ryOUzl3EHzzG8AC2Ct6Frb5FhjlT+A1tvR3axB3A/yfBrL3ASlxwNOxml21KjVO55Tgjwwp9l+U4naZQrMvr6QK87ak4ZtXA007MZlhUljwoB5mCx7cvQl88Sj3zOnfUq4eQifkxl9b4HNoBipIiBGKL1ACB/Rys8cbI3wxbqAHhJZGvu4ricDmfwIiMWrmZuFEbiX+SC/AwcxClFQoNWb2Iks81ccNAzwd8eneLChVakQ92QPvh7XPCPXDCgsedDTE3QELK67ikjwfcGzD+qZthVoNnwtfAABUQ2bgBQxAwulcZBeW4f1fL2DlgWy8NtQHk4K94GgjMMopq71GgWy7w7riFpYs/xSbq4dp9jnaWGFMXzeMDXDHsB4uEFhyT2XsRZaI3noe645cg5+rPSIGdTOKL4y2hY3Y6mG2ERsAfPkYUHINiNwN+I5s23O3BRnbgV+nAkIHYM55wKYLZFU1SDidi7jjNyC9t8KItRUfEwZ3x7Thj8DLuQXz4R6gSqnCsctF2JcuxeFLRZhS+xs+sNqCFLUfZgpiEdZPgrH93RH8SJdG08NW7L+E9UevQWBpgYQZj+MxL6dWXTrDOLB5bAZiVmGrm1T63Jfc7HlTcfcmkBYP+I8BugWa7jz1UauAr0OA4mwu0f2B9CllrRp70/Ox4c8cZBVwc+IseEBYPwmmj3ykWWEpV9Ti8KUi7M8owJFLt1FVc38V3b72VdhdMwN8qKCa+Rf47gHNu6smzPw5BYmZhXCxE+L3WcPQzdG8GSEMJmwGY1Zh2/secOY7YPg8IHSJac5x7TDw6+tA1V3uc4/R3CoYXo+b5nx1nN8C7JgBiByBuRcAkVinGRHhxLU72PDndRy7fFvTHujthOkjHsFTfd3Av/dAX1ZVg6SsQuxLl+LPK7ehrL1fV7SbozWeCZDg6f7uGOTpCItfXwMydwJD3gDCV7fI5QpFLf75TTKyCuTo4+6AX98MaX1uLKNVMGEzELMKW/I64MBH3JpkE/5r3L6JgONfAEkfc+uVOXpz1efp3sjGZwQ3UdZ3VOPFiQ1FVQusG8LlbY5eDIx4t0WHZUvL8P3/rmNn2t+oUXF/oj7ONogY1A1peaU4frVY0163b2yAO8b2lyCgm1g70nr9GPDjOEBgD7x7CRDatciHv0ur8PxXf6G4XIkxfd1YpNTMMGEzELMKW10akPtAYOYx4/WrrOBK2tWtUDvwVW7UUlYAHF8DpG4G1PeWCe8eBIx8H/B7yngCd+4nYNcsLr1pzvkWi0odRfJq/Df5Bn4+mdtgOXM/VzuM7S/B2AB39JbYNz5thAhYG8g9w3zui/ur7baAlJt3MXHDSRYpbQcwYTMQswpbXX1MkRj4MNc4fZbkAAmTgKKLXLL508u527H6AiC7BRz/Ejj3X6D23hLhkkc5gev9LGDRijnctUpOUGS5wJj/A4a+Y3BXFYpa/JpyC/+7UoxHu4sxtr8Efm72Le/gxFrg4ELAfQAw80+9zr393C1Ebz0PAFjz8kAWKTUTJq8rum7dOvj4+EAkEiE4OBinT59u0n7btm3o3bs3RCIRAgICsG/fPq39RITFixfD3d0d1tbWCA0NxZUrV7RsfHx8wOPxtLbly5dr2Vy4cAEjRoyASCSCp6cnVq5cacjlmYe61S2qZcapWHX1ELDhCU7UbF2ByD1A0PSGIzFxd+CZlcCcC8DQ2YCVLZe3unUysH4okP6r4SXtUn/iRM3ODRg8rVWXYyu0RORQH3wfORizR/vpJ2oAMOBfAF8IFJwH/j6n16EvPtYdbz3BLav0wW8XcC73rn7nZrQ5egvbli1bEB0djZiYGJw7dw4DBgxAWFgYioqKdNqfOHECEydOxLRp05CamoqIiAhEREQgIyNDY7Ny5Up8+eWX+Oabb3Dq1CnY2toiLCwM1dXaRUaWLl2KgoICzfbOO/dHAHK5HGPGjIG3tzdSUlKwatUqLFmyBBs2bND3Es2DwAawu1fFvDUZCETcemSbXwKqS4Fug7lbW++Qpo+zdwPGfALMy+BGa0IH4HYW8Ns04KshXHV2lR6VrWqqgT8/496PeJe7PnNi68yV6QMMqmT1/pheeKqvG5S1asz4MQV/l1YZ2UGGUSE9CQoKoqioKM1nlUpFHh4eFBsbq9N+woQJFB4ertUWHBxMM2fOJCIitVpNEomEVq1apdlfWlpKQqGQfvnlF02bt7c3ff7554369fXXX5OTkxMpFApN2/z586lXr14tvjaZTEYASCaTtfgYo/JDGFGMA9GFbYYdX11GtGUK10eMA9HOKKKaasP6qrxLdGwl0XKf+/39pz/R6e+JlFXNH5+8njtmdZ+W2bcFN05wPv2fhKiqVO/Dy6trKOzzY+Q9fw+NXfMnVShqTOAkozH0+X3qNWJTKpVISUlBaGiops3CwgKhoaFITk7WeUxycrKWPQCEhYVp7HNyciCVSrVsxGIxgoODG/S5fPlyODs7Y9CgQVi1ahVqa+8nKycnJ2PkyJEQCO7PWg8LC0N2djbu3tV966BQKCCXy7U2s9KanNGS68APT3HTGiysgPD/AOPWGr4WmbUjN3Kbmw489Ql3OyvLBfZGA18OBJK/BpSNFF9WVgL/uzetYuR7XN3P9oDX40DXPkBN5f3izHpgK7TE95GD4WInQGaBHPO2pEGtZo+o2yN6CVtxcTFUKhXc3Ny02t3c3CCVSnUeI5VKm7Sve22uz9mzZyMhIQFHjhzBzJkzsWzZMnzwwQfNnqf+OR4kNjYWYrFYs3l6mjmVydAao1fqnqdlcs+zXtsDDJlmnMim0A4YNpubfzZ2FeDQjYuoHlgArAngbnsVZdrHnPkeqCgCHL24KGx7gcfjKlkB3O2oAXGz7k42+HbyYAj4FjhwsRCrE7ON7CTDGHSYZYuio6PxxBNP4NFHH8Wbb76J1atXY+3atVAoFM0f3AgLFiyATCbTbHl5Zi60q1m+6EbL7Im4kdHmf3JBh+5DgBnHTDPh1soaCJ4BzE7lpkw4egOVxcChJcDn/YGjy7mJv4oybhoJAIyazxVYaU88OgGwtOb+E8g7ZVAXgd5OWD6ey2BYd+Qadqb+bUwPGUZAL2FzcXEBn89HYWGhVnthYSEkEonOYyQSSZP2da/69AkAwcHBqK2txY0bN5o8T/1zPIhQKISDg4PWZlb0uRVVlAPbIoGkpQAIeCwSeG0v4OBuUhdhKeTmgb2TAkR8Azj7cUGKo7HA5wHALxOByjtAlx7Ao6+Y1hdDsHYEAsZz71uxCCWLlLZv9BI2gUCAwMBAJCUladrUajWSkpIQEqI76hYSEqJlDwCJiYkae19fX0gkEi0buVyOU6dONdonAKSlpcHCwgKurq6a8/z555+oqbkfuUtMTESvXr3g5NRBkpjrRmzyfC6q2Bh3rgHfhwKZv3PP055dA4z7sm3X9udbAQMnAlGngH/GAa79AGUZV7AY4PJB+e00BanudvTijlZNrWGR0naMvpGJhIQEEgqFtGnTJsrMzKQZM2aQo6MjSaVSIiKaPHkyffjhhxr748ePk6WlJX322WeUlZVFMTExZGVlRenp6Rqb5cuXk6OjI/3+++904cIFev7558nX15eqqrho2okTJ+jzzz+ntLQ0unbtGv3888/UtWtXmjJliqaP0tJScnNzo8mTJ1NGRgYlJCSQjY0Nffvtty2+NrNHRdVqok+7cZG7oku6bS4fJIr15GxW+RHdPNm2PjaGSkWUtYdo4zNEWyOJVLXm9qhx1Gqi9cO57/D42lZ1xSKlbYc+v0+9hY2IaO3ateTl5UUCgYCCgoLo5Mn7P65Ro0ZRZGSklv3WrVvJ39+fBAIB9evXj/bu3au1X61W06JFi8jNzY2EQiGNHj2asrOzNftTUlIoODiYxGIxiUQi6tOnDy1btoyqq7WnMpw/f56GDx9OQqGQunXrRsuXL9fruswubERE64dxP7js/drtajXRsVVEMWJu/3ehRLJ8s7jYKTizkfsev3yM+25bQV5JBQV+cpC85++hGT+eIZWqdf0xdKPP75OlVNXDrClVdWx5laus9PQK4PE3uTZFGbDzLa4d4IqgjF3Bysq1BkUZsLoPd/tshDXwWE6p6TF5ShXDhDwYQKh7npa1m3ue9twXXB1OJmqtQ2gPPPoS996ATIQHYZHS9gUTtvZG/YpVlw8AG54Ebl/i0q2m7tNrZQpGM9QFEbJ2A+W6UwL14cFIaWpnipTqk07XDmDC1t6oG7Hl/AnEvwwoZIBnMJfv6RlkXt86G5IAbu6fupbLhTUC9SOl039MQX5Hj5SqaoDdc4DY7twqMB0EJmztjboRW20VAOJWxYjcA9g3PqeP0QrqRm0pcYBa3bRtC7Cw4GHNywPRW2KP4nIF3vjvWVQqO2idUmUFNy8xZRO3pFXiIm5Cdgd4LM+Erb3h0J1Li+ILuFzPZ//T/mbvdyb6vcCtgVeayy2dbgQ6RU5p+W1g07PA1UQuU2PQZK79r8+BPXMNX8qqjWDC1t7gW3ILIc5O46qZM0yLlTW3VhvAjdqMBJdTGqjJKf1PYuPV6NsdJdeBjWOA/HOAdRcuavz8V1zgCjxuBPfbNG4h0XYKE7b2iL2Eq2TOaBsGT+Ves/8AZMaLZgZ6d9FESr86crVjREr/Pgd8/xQnbo5ewLSDgOcQbl/ga8BLcVx0/uIO4JdXuNvVdgibx1aPdjGPjWEe4p4Bbh4HhkcDIbPur4zC4wHgtep1+f5sfHOMq1M6tIczXOyE9zYButoLtT472QjMVzDmyiFg6xSgpoILrEz6Vfez3atJ3HzLmkousPWvLYC16dMWWc0DA2HC9hCT/it3e2VCykmEONXT+Ko2Agrofm7Kt+Chi60AXe2EcLG/J351wmcvqCeCQnSxFWjKEbaatHhg1ztchPiRJ4AJPwGiJn4Deafvryrj2g+YvN3kAS4mbAbChO0hplYB/DAGKEgz+alKrb2w0/MDnEU/FJcrUFyuRHG5AqWV+s0Vs+ABXWzvi52Piw0GdHfEQE9H9Ohq17KRX93SV4c/4T4/+jIw7quWBawKLwI/vQCUF3LTlKbsvL+moAlgwmYgTNgecojuTWWo91rXDh37mnzVcVxuMrDvA6D83sKng17lVie26QIAqFGpceeeyN0uV6C47L7oabYy7nNJpbLJWRf2QksEdBdjoKejZnN1eGAlY7UK+OMDbmFQABg2Fxgdo19lspIc4MfngdKb3CTyyTsAt74tP14PmLAZCBM2hsmplgGHPgbO/sB9tu3KlUXsP16vFY9rVWqUVCpx+5743S5T4HJhGdJyS5H+twxVNQ2nY3iIRRhQJ3TuIgSefQ+Wl/cB4HG5x8EzDbsmeQHw84vc4p0iR+DV34Dugw3rq6nTMGEzDCZsjDYj9yQ3o//2Je5zz1CuToWTd6u7rlWpcbmwHOdvlSIttxTnb5XicmEZ6qbSiVGO7wWfYYjFZShhiW1ei2HR/wUM9HSEn6sdLPkGTJaoLAHiJwC3znAlHF/ZDPR4stXXUh8mbAbChI3RptQqgeNfAH+uBFRKwMoGePIjIPgtoy/SWa6oRcbfMly9koXRZ9+Ge81NyMkG05Xv4hT10djZCPjo302MQZ6OmtGdu1gEXktGk4pyLlp6/Qg3wXz89/dLHhoBJmwGwoSNYRaKrwC75wI3/+I+uw/gJsN6DDLueaQZXCSzrABw6IbiiM1IqXLH+bxSpOWV4sItGcoVDdO/utoLEdBNDFd7IcQ2VnCyEcDJxgpia+7V0eb+qwA1wPbp3OrOPAvguS+BxyYbxX0mbAbChI1hNoi4RPyDC7kaEjwL4PG3uRGcwLb1/V8/xo2mFHKuBOGrvwLi7lomajXh2u1ypN0TuvO3SnGpoAy1eqSD2Qj4cLbmYyE2IExxEADwh8fbuOj9GhzviaLjA2IotrZq0bQVJmwGwoSNYXbKi4D9HwIZv3GfxV5cvrDfU4b3mf4rsONNQF0DeA8DXonnitq0gOoaFS7my5BVUIbSSiXuVtbgbqUSsnuvpXWfq2qgrX+EDy0T8KYltzjqutpxWFX7MoCGAsbjAU42AqQsDG3ylpcJm4EwYWO0Gy4f5IpTy+6VhOz/T+DpWMDOVb9+ktcBBz7i3vd9Hnhhg0kKWKvVhLLqWk7squpETwnPixsw+OoXAIC/xM9hg/3buFulRmmVEqUVNSi7d+vbxVaAc4uaFm8mbAbChI3RrlCUc2UNT34NkJqbSjHm/7j5b809zFeruWWGkr/iPgfN5ITRgm9ytxtwNg7YMw8AAf1eBF74VjMBuEalhqyqBhWKWng7N33LzYTNQJiwMdol+anArtmA9AL32WcEV3LRpadu+1oFVyOj7nb2qaXA0Nl6zZMzOhnbge0zuNvhnqFcypbARq8uWM0DBqMz4TEImH6Ey1KwtOZqt64fChxb1XDpoGoZ8PN4TtQsLLlbz2FzzCtqAND/ReBfCdyUlquHgJ8igCrTLZ3OhI3B6AjwLYFhs4Gok0CP0YBKARz5P+DbkUDuKc5GXsCtUnLjf4DADpi0DRjwsnn9rk/PUGDyTm5hz7xT3EKWZYUmORW7Fa0HuxVldAiIuEjn/g+BymIAPO652/WjXLDB1pWbzuE+wNye6kaawSXPVxTplTzPbkUZjM4Mj8eVDpx1Bhj4KgACUn/iRM25J/BGYvsVNQCQ9Ade388tZHk3B9j4NFCUZdRTGCRs69atg4+PD0QiEYKDg3H69Okm7bdt24bevXtDJBIhICAA+/bt09pPRFi8eDHc3d1hbW2N0NBQXLlyRbP/xo0bmDZtGnx9fWFtbY0ePXogJiYGSqVSy4bH4zXYTp48acglMhjtH5suQMQ6YMouwC0AeORJ4PWDJl06yGg49wBeP8BNFi4rAOLGcqv2Ggt9y8wnJCSQQCCgjRs30sWLF2n69Onk6OhIhYWFOu2PHz9OfD6fVq5cSZmZmbRw4UKysrKi9PR0jc3y5ctJLBbTzp076fz58zRu3Djy9fWlqqoqIiL6448/6LXXXqMDBw7QtWvX6PfffydXV1d69913NX3k5OQQADp06BAVFBRoNqVS2eJrk8lkBIBkMpm+XwuDwTCEijtEG/5B9Os0IpWqSVN9fp96C1tQUBBFRUVpPqtUKvLw8KDY2Fid9hMmTKDw8HCttuDgYJo5cyYREanVapJIJLRq1SrN/tLSUhIKhfTLL7806sfKlSvJ19dX87lO2FJTU/W9JA1M2BgMM1BdRlTb/ABEn9+nXreiSqUSKSkpCA0N1bRZWFggNDQUycnJOo9JTk7WsgeAsLAwjX1OTg6kUqmWjVgsRnBwcKN9AoBMJkOXLl0atI8bNw6urq4YPnw4du3a1eT1KBQKyOVyrY3BYLQxQjuAb2XULvUStuLiYqhUKri5uWm1u7m5QSqV6jxGKpU2aV/3qk+fV69exdq1azFz5v2F8ezs7LB69Wps27YNe/fuxfDhwxEREdGkuMXGxkIsFms2T0/PRm0ZDEbHwbiLPrUBf//9N55++mm89NJLmD59uqbdxcUF0dHRms9DhgxBfn4+Vq1ahXHjxunsa8GCBVrHyOVyJm4MRidArxGbi4sL+Hw+Cgu1J9UVFhZCItFdoUYikTRpX/fakj7z8/Px5JNPYujQodiwYUOz/gYHB+Pq1auN7hcKhXBwcNDaGAxGx0cvYRMIBAgMDERSUpKmTa1WIykpCSEhITqPCQkJ0bIHgMTERI29r68vJBKJlo1cLsepU6e0+vz777/xxBNPIDAwEHFxcbBoQcGJtLQ0uLu763OJDAajM6BvACMhIYGEQiFt2rSJMjMzacaMGeTo6EhSqZSIiCZPnkwffvihxv748eNkaWlJn332GWVlZVFMTIzO6R6Ojo70+++/04ULF+j555/Xmu5x69Yt6tmzJ40ePZpu3bqlNZ2jjk2bNlF8fDxlZWVRVlYWffrpp2RhYUEbN25s8bWxqCiD0X4x6XQPIqK1a9eSl5cXCQQCCgoKopMnT2r2jRo1iiIjI7Xst27dSv7+/iQQCKhfv360d+9erf1qtZoWLVpEbm5uJBQKafTo0ZSdna3ZHxcXV1cLrcFWx6ZNm6hPnz5kY2NDDg4OFBQURNu2bdPrupiwMRjtF31+nyxXtB4ymQyOjo7Iy8tjz9sYjHZGXXCvtLQUYrG4SdsOFxU1JWVlZQDAIqMMRjumrKysWWFjI7Z6qNVq5Ofnw97evtlyY3X/e3TE0V1H9b2j+g0w340BEaGsrAweHh7NBg/ZiK0eFhYW6N69e/OG9ejI00Q6qu8d1W+A+d5amhup1cGWLWIwGJ0OJmwMBqPTwYTNQIRCIWJiYiAUCs3tit50VN87qt8A872tYcEDBoPR6WAjNgaD0elgwsZgMDodTNgYDEangwkbg8HodDBhYzAYnQ4mbAagb/nB9kBsbCyGDBkCe3t7uLq6IiIiAtnZ2eZ2yyCWL18OHo+HuXPnmtuVFvH333/j1VdfhbOzM6ytrREQEICzZ8+a261mUalUWLRokVbZy08++QQdYSIFEzY92bJlC6KjoxETE4Nz585hwIABCAsLQ1FRkblda5Jjx44hKioKJ0+eRGJiImpqajBmzBhUVFSY2zW9OHPmDL799ls8+uij5nalRdy9exfDhg2DlZUV/vjjD2RmZmL16tVwcnIyt2vNsmLFCqxfvx5fffUVsrKysGLFCqxcuRJr1641t2vNY8Llkzol+pYfbK8UFRURADp27Ji5XWkxZWVl5OfnR4mJiTRq1CiaM2eOuV1qlvnz59Pw4cPN7YZBhIeH0+uvv67V9uKLL9KkSZPM5FHLYSM2PTCk/GB7RSaTAYDOEobtlaioKISHhzco59ie2bVrFwYPHoyXXnoJrq6uGDRoEL777jtzu9Uihg4diqSkJFy+fBkAcP78efz1118YO3asmT1rHra6hx40VX7w0qVLZvJKf9RqNebOnYthw4ahf//+5nanRSQkJODcuXM4c+aMuV3Ri+vXr2P9+vWIjo7GRx99hDNnzmD27NkQCASIjIw0t3tN8uGHH0Iul6N3797g8/lQqVT49NNPMWnSJHO71ixM2B5CoqKikJGRgb/++svcrrSIvLw8zJkzB4mJiRCJROZ2Ry/UajUGDx6MZcuWAQAGDRqEjIwMfPPNN+1e2LZu3YrNmzcjPj4e/fr1Q1paGubOnQsPD4927zt7xqYHCoWC+Hw+7dixQ6t9ypQpNG7cOPM4pSdRUVHUvXt3un79urldaTE7duwgAMTn8zUbAOLxeMTn86m2ttbcLjaKl5cXTZs2Tavt66+/Jg8PDzN51HK6d+9OX331lVbbJ598Qr169TKTRy2HPWPTA0PKD7YXiAizZs3Cjh07cPjwYfj6+prbpRYzevRopKenIy0tTbMNHjwYkyZNQlpaGvh8vrldbJRhw4Y1mFZz+fJleHt7m8mjllNZWdlgpVo+nw+1Wm0mj/TA3Mra0Wiu/GB75a233iKxWExHjx7VKl9YWVlpbtcMoqNERU+fPk2Wlpb06aef0pUrV2jz5s1kY2NDP//8s7lda5bIyEjq1q0b7dmzh3Jycmj79u3k4uJCH3zwgbldaxYmbAbQVPnB9goaKV8YFxdnbtcMoqMIGxHR7t27qX///iQUCql37960YcMGc7vUIuRyOc2ZM4e8vLxIJBLRI488Qv/+979JoVCY27VmYeuxMRiMTgd7xsZgMDodTNgYDEangwkbg8HodDBhYzAYnQ4mbAwGo9PBhI3BYHQ6mLAxGIxOBxM2BoPR6WDCxmAwOh1M2BgMRqeDCRuDweh0/D+cz/5Tn2CldQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:47:40.411928Z", + "iopub.status.busy": "2024-03-22T20:47:40.411581Z", + "iopub.status.idle": "2024-03-22T20:51:46.581079Z", + "shell.execute_reply": "2024-03-22T20:51:46.579998Z" + }, + "papermill": { + "duration": 246.190558, + "end_time": "2024-03-22T20:51:46.583679", + "exception": false, + "start_time": "2024-03-22T20:47:40.393121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:51:46.621064Z", + "iopub.status.busy": "2024-03-22T20:51:46.620162Z", + "iopub.status.idle": "2024-03-22T20:51:46.642273Z", + "shell.execute_reply": "2024-03-22T20:51:46.641271Z" + }, + "papermill": { + "duration": 0.042627, + "end_time": "2024-03-22T20:51:46.644305", + "exception": false, + "start_time": "2024-03-22T20:51:46.601678", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0053470.0809710.00283612.4673010.0337910.8471570.1400070.0000094.1098430.0416130.122550.0532550.1082990.00104916.577143
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.005347 0.080971 0.002836 12.467301 0.033791 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.847157 0.140007 0.000009 4.109843 0.041613 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.12255 0.053255 0.108299 0.001049 16.577143 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:51:46.679178Z", + "iopub.status.busy": "2024-03-22T20:51:46.678852Z", + "iopub.status.idle": "2024-03-22T20:51:47.043914Z", + "shell.execute_reply": "2024-03-22T20:51:47.043067Z" + }, + "papermill": { + "duration": 0.384847, + "end_time": "2024-03-22T20:51:47.046035", + "exception": false, + "start_time": "2024-03-22T20:51:46.661188", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:51:47.083119Z", + "iopub.status.busy": "2024-03-22T20:51:47.082278Z", + "iopub.status.idle": "2024-03-22T20:55:57.990842Z", + "shell.execute_reply": "2024-03-22T20:55:57.989788Z" + }, + "papermill": { + "duration": 250.930339, + "end_time": "2024-03-22T20:55:57.993924", + "exception": false, + "start_time": "2024-03-22T20:51:47.063585", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:55:58.031362Z", + "iopub.status.busy": "2024-03-22T20:55:58.031011Z", + "iopub.status.idle": "2024-03-22T20:55:58.057419Z", + "shell.execute_reply": "2024-03-22T20:55:58.056562Z" + }, + "papermill": { + "duration": 0.047546, + "end_time": "2024-03-22T20:55:58.059612", + "exception": false, + "start_time": "2024-03-22T20:55:58.012066", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:55:58.094102Z", + "iopub.status.busy": "2024-03-22T20:55:58.093810Z", + "iopub.status.idle": "2024-03-22T20:55:58.099460Z", + "shell.execute_reply": "2024-03-22T20:55:58.098484Z" + }, + "papermill": { + "duration": 0.0258, + "end_time": "2024-03-22T20:55:58.101597", + "exception": false, + "start_time": "2024-03-22T20:55:58.075797", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.3770567048447473}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:55:58.139403Z", + "iopub.status.busy": "2024-03-22T20:55:58.138694Z", + "iopub.status.idle": "2024-03-22T20:55:58.530094Z", + "shell.execute_reply": "2024-03-22T20:55:58.529245Z" + }, + "papermill": { + "duration": 0.412474, + "end_time": "2024-03-22T20:55:58.532159", + "exception": false, + "start_time": "2024-03-22T20:55:58.119685", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEhElEQVR4nO3deXxU9b0//teZfZ/sG0lIgBD2TYGiIiBUWUSovVarVRBR22LVUh4P5fZbFW0Lel27XK71SiIPF6z+XHqvRaBUxHIFEWTfIYTs+8xk9uV8fn+cmSFDFmYmM3Myk/fz8QiTnDk58z4Z8s5n/3CMMQZCCBGRROwACCGEEhEhRHSUiAghoqNERAgRHSUiQojoKBERQkRHiYgQIjpKRIQQ0VEiIoSIjhIR6VNlZSU4jsPFixfFDoWkMEpEJObeffddvPrqq2KHQZIIJSISc5SISKQoERFCREeJiERs69atmDVrFvR6PQwGA6ZOnYp3330XADB79mx89tlnqK6uBsdx4DgOJSUlYV+b53k888wzKCgogEajwZw5c3DixAmUlJRg+fLlwfPa29uxZs0ajB8/HjqdDgaDAQsWLMDhw4dDrrdr1y5wHIe//vWv+N3vfofCwkKoVCrMnTsX586di8WPg8SATOwASHKprKzEihUrMHbsWKxduxZpaWn47rvv8Pnnn+Puu+/Gr3/9a5jNZtTW1uKVV14BAOh0urCvv3btWrzwwgtYvHgxbrnlFhw+fBi33HILnE5nyHkXLlzAJ598gjvuuAOlpaVoamrC66+/jlmzZuHEiRMoKCgIOX/Dhg2QSCRYs2YNzGYzXnjhBdxzzz3Yt29f/38opP8YIX2oqKhgAFhVVRUzmUxMr9ez6dOnM4fDEXIez/PBzxctWsSGDh0a8Ws1NjYymUzGli5dGnL8mWeeYQDYsmXLgsecTifz+Xwh51VVVTGlUsmeffbZ4LEvvviCAWCjR49mLpcrePy1115jANjRo0cjjpPEHlXNSNh27NiBzs5OPPnkk1CpVCHPcRzX7+vv3LkTXq8XP//5z0OO/+IXv+h2rlKphEQi/Pf1+Xxoa2uDTqdDeXk5Dh482O38+++/HwqFIvj1zJkzAQglKyI+SkQkbOfPnwcAjBs3Li7Xr66uBgCMGDEi5HhGRgbS09NDjvE8j1deeQVlZWVQKpXIyspCdnY2jhw5ArPZ3O3axcXFIV8HrtfR0RHLWyBRokREktLvf/97rF69GjfeeCPefvttbNu2DTt27MDYsWPB83y386VSaY/XYbRS8oBAjdUkbMOHDwcAHDt2rFuppatoq2lDhw4FAJw7dw6lpaXB421tbd1KLh9++CHmzJmDN998M+S4yWRCVlZWVK9PxEMlIhK2m2++GXq9HuvXr+/Wi9W1ZKHVanusHl3N3LlzIZPJsHHjxpDjf/rTn7qdK5VKu5VmPvjgA9TV1UX8ukR8VCIiYTMYDHjllVewcuVKTJ06FXfffTfS09Nx+PBh2O12vPXWWwCAa665Bu+//z5Wr16NqVOnQqfTYfHixVe9fm5uLh577DG89NJLuO222zB//nwcPnwYW7duRVZWVkhJ69Zbb8Wzzz6L+++/H9dddx2OHj2Kd955B8OGDYvb/ZM4ErnXjgxwXbvvA/72t7+x6667jqnVamYwGNi0adPYe++9F3zearWyu+++m6WlpTEAEXXle71e9pvf/Ibl5eUxtVrNbrrpJnby5EmWmZnJfvrTnwbPczqd7Fe/+hXLz89narWaXX/99ezrr79ms2bNYrNmzQqeF+i+/+CDD0Jep6qqigFgFRUVkf5ISBxwjFFrHRnYTCYT0tPT8dvf/ha//vWvxQ6HxAG1EZEBxeFwdDsWmEA7e/bsxAZDEobaiEhCtLS0wOfz9fq8QqFARkYG3n//fVRWVmLhwoXQ6XT417/+hffeew8333wzrr/++gRGTBKJEhFJiKlTpwYHLPZk1qxZ2LVrFyZMmACZTIYXXngBFosl2ID929/+NoHRkkSjNiKSEHv27Omx2hWQnp6Oa665JoERkYGEEhEhRHTUWE0IEV1StxHxPI/6+nro9fqYzP4mhMQWYwydnZ0oKCgIrpbQk6RORPX19SgqKhI7DELIVdTU1KCwsLDX55M6Een1egDCTRoMBpGjIYRcyWKxoKioKPi72pukTkSB6pjBYKBERMgAdrWmE2qsJoSIjhIRIUR0lIgIIaJL6jaicDDG4PV6+5znRHomlUohk8loaASJu5RORG63Gw0NDbDb7WKHkrQ0Gg3y8/NDdsAgJNZSNhHxPI+qqipIpVIUFBRAoVDQX/YIMMbgdrvR0tKCqqoqlJWV9TkgjZD+ED0R1dXV4YknnsDWrVtht9sxYsQIVFRU4Nprr+3Xdd1uN3ieR1FRETQaTYyiHVzUajXkcjmqq6vhdru77WWWjM40deJorRnleXqMG2IUOxziJ2oi6ujowPXXX485c+Zg69atyM7OxtmzZ7vtYdUf9Fe8f1Lp59dhc+PzY43w8QyX2u3I0SuRY0j+5JoKRE1Ezz//PIqKilBRURE81nUbGUJi6WidGT7+8mITh2pMuHlsnogRkQBR/9z97W9/w7XXXos77rgDOTk5mDx5Mt54441ez3e5XLBYLCEfhITrXLMVADCpOA0AUNVqow0WBwhRE9GFCxewceNGlJWVYdu2bfjZz36GRx99NLgtzZXWr18Po9EY/KAJr/1XUlISXBM6lTHGMK00A2MKDPheaSay9EqMyNHB46NENBCIWjXjeR7XXnstfv/73wMAJk+ejGPHjuG//uu/sGzZsm7nr127FqtXrw5+HZhQR8jVcByHcUOMwQbqe783VOSISFeilojy8/MxZsyYkGOjR4/GpUuXejxfqVQGJ7jSRNfL3G632CEQ0i+iJqLrr78ep0+fDjl25syZ4B7o8eL28r1+eH182Od6wjg3GrNnz8YjjzyCRx55BEajEVlZWfjNb34TbM8oKSnBc889h/vuuw8GgwEPPfQQAOBf//oXZs6cCbVajaKiIjz66KOw2WzB6zY3N2Px4sVQq9UoLS3FO++8E1V8yehiqw2NZmfI++vjGTpslMQHAlGrZr/85S9x3XXX4fe//z1+9KMf4ZtvvsFf/vIX/OUvf4nr6/75i3O9PleapcXSyUOCX/9l9/le2xEK09W449rLVcNNe6rgcIdOJfnl90dGFeNbb72FBx54AN988w2+/fZbPPTQQyguLsaDDz4IAHjxxRfx1FNP4emnnwYAnD9/HvPnz8dvf/tbbNq0CS0tLcFkFuiVXL58Oerr6/HFF19ALpfj0UcfRXNzc1TxJZsdJ5pgdXlx59QiFKSpYbZ78NbXFyGVcPj57OE02FVkoiaiqVOn4uOPP8batWvx7LPPorS0FK+++iruueceMcMaEIqKivDKK6+A4ziUl5fj6NGjeOWVV4KJ6KabbsKvfvWr4PkrV67EPffcg8cffxwAUFZWhj/84Q+YNWsWNm7ciEuXLmHr1q345ptvMHXqVADAm2++idGjRyf83hLN6fHB6vICADJ1wlQVrVIKnjH4vAx2tw9apehjewc10X/6t956K2699daEvuaqOSN6fU5yxR/Gh24c3uu5V/4RXXF97MZAfe973wv5Kz1jxgy89NJLwcm7V448P3z4MI4cORJS3WKMBae6nDlzBjKZLGTLnlGjRiEtLS1mMQ9UrVYXAMCglkMpkwIAZFIJ9Co5LA4PTA4PJSKRDcqfvkIWftNYvM7tL61WG/K11WrFww8/jEcffbTbucXFxThz5kyiQhtwTHYPACBdIw85nqYWElGHzY0haWoxQiN+gzIRJYN9+/aFfL13716UlZVBKpX2eP6UKVNw4sQJjBjRc2lv1KhR8Hq9OHDgQLBqdvr0aZhMppjGPRBZHEIiMqpDE1G6Vo5L7YDZ/zwRT+pMJEoxly5dwurVq3H69Gm89957+OMf/4jHHnus1/OfeOIJ/N///R8eeeQRHDp0CGfPnsWnn36KRx55BABQXl6O+fPn4+GHH8a+fftw4MABrFy5Emp16pcEzL0kIqNaaC8KlJiIeCgRDVD33XcfHA4Hpk2bhlWrVuGxxx4LdtP3ZMKECfjyyy9x5swZzJw5E5MnT8ZTTz2FgoKC4DkVFRUoKCjArFmzcPvtt+Ohhx5CTk5OIm5HVBankGgM3RKRPOR5Ip6k3nLaYrHAaDTCbDZ3G9zodDpRVVWF0tLSpFu+Yvbs2Zg0adKAmHqRzD/HgJp2O9psbgzL1sKgupyM2m1u7L/YjkytAteWZIgYYerq63e0K2ojIimvKEODoozua1JlaBW4hWbfDwhUNSOEiI5KRAPQrl27xA4hZZjtHjRanEjXyHtcBM3r42Fz+aCUS6CS99wjSeKPSkQkpV1qt+PvRxvw9YW2Hp//+Ls6bNpTheo22mBBTJSISEoLTO3QKnou/OtVwvFO6jkTFSUiktICiUin6jkR6ZRCL1qn/zwiDkpEJKXZAomol7lkgQRldVIiEhMlIpLSglWz3hKRUmigtrspEYmJEhFJaVcrEan9bUc2F21JLiZKRCRleX087P6F6npLRFqFUCJyeCgRiYnGEZGUxXEcFk8sgM3lhUre899cjUKGsQUGaJUy8DyD5MoFqUhCUCIiKUsq4TAiR9fnOQqZhDZZHAAGV9WMMcDrTvxHBPOKN2/ejMzMTLhcrpDjS5cuxb333hvrnwghA8LgKhH5PMBXLyX+dWf+CpApwjr1jjvuwKOPPoq//e1vuOOOOwAIu2989tln2L59ezyjTDnNFic67B5k6RTI1Cl7Pc/j42GnaR6iGlwloiSgVqtx9913B3feAIC3334bxcXFmD17tniBJaFTjZ34+9EGHK0z93ne1mON2LSnCmeaOhMUGbnS4CoRSeVC6USM143Agw8+iKlTp6Kurg5DhgxBZWUlli9fTlveRCgwNqi3HrOAQM8ZdeGLZ3AlIo4Lu4okpsmTJ2PixInYvHkzbr75Zhw/fhyfffaZ2GElnUDXvVrRd3Ur8DwNahTP4EpESWTlypV49dVXUVdXh3nz5qGoqOjq30RCBBKRppcJrwGBCbF2N5WIxEJtRAPU3XffjdraWrzxxhtYsWKF2OEkpcCuu9qrlIi0NM1DdKImomeeeQYcx4V8jBo1SsyQBgyj0Ygf/vCH0Ol0WLp0qdjhJB3GWARVM5rmITbRq2Zjx47FP/7xj+DXMpnoIQ0YdXV1uOeee6BU9t71THrm8vLg/eO31Ffpkg887/RSIhKL6L/1MpkMeXk0srWrjo4O7Nq1C7t27cJ//ud/ih1OUpJJhOkdTo8PMmnfBX+NQopxQ4xQy6VgjFHvpAhET0Rnz55FQUEBVCoVZsyYgfXr16O4uLjHc10uV8iIY4vFkqgwE2ry5Mno6OjA888/j/LycrHDSUoyqeSq0zsCVHIpvj8mN84Rkb6ImoimT5+OyspKlJeXo6GhAevWrcPMmTNx7Ngx6PX6buevX78e69atEyHSxLp48aLYIRCSUANqg0WTyYShQ4fi5ZdfxgMPPNDt+Z5KREVFRSm3weJAkqw/x5ZOF9ptbmRoFcjWX72NzeX1wenmoVJIoJTRNI9YScoNFtPS0jBy5EicO3eux+eVSmXEDbcDKM8mpWT9+Z1rtmLvhTZMKDRi7uirV7s+/a4edSYHbp2Qj7Lc7qVxEl8DahyR1WrF+fPnkZ+f3+9ryeXCtAq7nbaJ6Y/Azy/w80wWDo8wJuhqXfcBKlogTVSilojWrFmDxYsXY+jQoaivr8fTTz8NqVSKH//4x/2+tlQqRVpaGpqbmwEAGo2GekMiwBiD3W5Hc3Mz0tLSIJUmV3Ul3FHVASqZ8DfZQaOrRSFqIqqtrcWPf/xjtLW1ITs7GzfccAP27t2L7OzsmFw/MCwgkIxI5NLS0pJyeMXlRBReAlVTiUhUoiaiLVu2xPX6HMchPz8fOTk58HhoA71IyeXypCsJBQRKNlcbzBgQHNTo4eMWE+ndgGqsjhepVJq0v1AkOpGWiFTBREQlIjEMqMZqQmLBx7NgQgm3jYiqZuIaFCUiMvjcNqkAdpev1907rpSuUWD8ECPStcnVO5gqKBGRlCOVcBie3cv0DsYAUzXgsgKZIwC5MEgzQ6vAPJrmIRpKRGTwYAw49w+g9lvha5UBmHIfoKQBjGKjNiKSctptbpxqtKDZ4gx9ovWMkIQ4DpCrAacFOPX34NMurw9muwc+PjlHkyczSkQk5VS12rD1aCMOVHdcPsjzwPkvhM+LvyeUhCRSoP0CYK4FAFTsuYhNe6rQbnOLEPXgRomIpBxHTysztl8AHB1Cm1DxDECTAeSOFZ6rPyScT134oqFERFJOYO3pkK77hkPCY954QOafOJ03QXhsPQ34PJSIRESJiKQch+eKwYxel1AiAoC8iZdPNBYCKqOwLXj7BSj9Xf00ujrxKBGRlNNt0fz2KoD3CdUxXZd5jBwHZJUFz1HR2tWioUREUk636R1tZ4XHzOHdT04vER47LtI0DxFRIiIphTEGR6CNSC4Txg4FqmWZZd2/Ia0Y4CSAowNa3gqAqmZioAGNJOUsGJ8Ph9snbJxobwPcdkAqE9qEriRTAvpcwNKAAkk7xg8pxJB0deKDHuQoEZGUwnFXTO8wXRIeDUOEcUM90RcIiYjrQMGYa+IfJOmGqmYktZlrhEdjUe/nGAqEx876+MdDekSJiKQUk92Nkw0WNJr90zv8o6Z7rJYF+BMR62yE0+WG2UGL6CUaJSKSUmo7HPj8WCP2VbUBDpMwn4yTCFWz3qjTAbkKLpcbm3fsx9t7qxMWLxFQIiIpxd51idjOBuGgLgeQKXr/Jo4DdLmQSTloPO1we3l4fdRzlkiUiEhKCZneEUhE+jC2p9JmQyrhoPW2AwBcXkpEiUSJiKSUkAmvnU3CQX0YC55ps8CBg8FnAkCDGhONEhFJKcFR1XIJYG0UDoZVIsoRTg0kIioRJRQlIpJSAlUzHbMCHqcwdkgbxj552iwAgJo5IPM5qUSUYAMmEW3YsAEcx+Hxxx8XOxSSxIIlIneLcECb3ftAxq5kSkBlDDZYUyJKrAExsnr//v14/fXXMWHCBLFDIUnu+2NyYXf7oLf516XWR7BLrS4HaepGTEh3I03TRy8biTnRS0RWqxX33HMP3njjDaSnp4sdDklyw7J1GDfECIXDXyKKJBFpMpFvVGNqDjAkjeabJZLoiWjVqlVYtGgR5s2bd9VzXS4XLBZLyAchPbK1Co/htA8FaDKER0d77OMhfRK1arZlyxYcPHgQ+/fvD+v89evXY926dXGOiiQri9ODug4HjHIfClydwkFNVvgXUGeAgcFrbYXH7Q17l1jSf6KViGpqavDYY4/hnXfegUqlCut71q5dC7PZHPyoqamJc5QkmTSYnPj8WCMOnjwnHFDqghsohkWTgZZOF749VY3tR2vjEyTpkWgp/8CBA2hubsaUKVOCx3w+H3bv3o0//elPcLlckEpDezuUSiWUSmWiQyVJwubvujfyZuFAJNUyAJBrIFWoANjA200xjY30TbRENHfuXBw9ejTk2P33349Ro0bhiSee6JaECLmawKhqQyARRVItAwCOg0SbCaCN2okSTLREpNfrMW7cuJBjWq0WmZmZ3Y4TEo7AGCKtzwRIAWgzI76GTCs0WHOOjqucSWJJ9F4zQmIlOOHVYxIORFoiAiDTCd8jdbaDMdp6OlEGVLfArl27xA6BJDGH2wcp74bKZwWgCE7biIRcJ5SilB4zXF4+uLMHiS8qEZGUYXP7oPZ0QC7l/D1mkQ9KlOkyIeUAtdcMF+3mkTADqkRESH/MHZUDb1091K3SqKplAAB1BrL0SgA8JMwNQB7LEEkvKBGRlFGSpQVMTqBdElW1DAAgV2FYXqawTTWzA9DGNEbSs6iqZhcuXIh1HITEhr1NeNRE3mMWpE4THp3mfodDwhNVIhoxYgTmzJmDt99+G06nM9YxERIxq8uLkw0WmFr9y8NGWyICwJRGeHkebltbjKIjVxNVIjp48CAmTJiA1atXIy8vDw8//DC++eabWMdGSNgazU5sP1KDmnp/Ioq2jQjAwRZg/8UOXLhE+5wlSlSJaNKkSXjttddQX1+PTZs2oaGhATfccAPGjRuHl19+GS0tLbGOk5A+Obr2mCm0gEIT9bUkaiMAgHeYYhQduZp+dd/LZDLcfvvt+OCDD/D888/j3LlzWLNmDYqKinDfffehoaEhVnES0ie72wuNpwNyaT8aqv2kWv+6WE4aXZ0o/UpE3377LX7+858jPz8fL7/8MtasWYPz589jx44dqK+vx5IlS2IVJyF9snt8UHtNQiLqR7UMAKTBxmoLQKOrEyKq7vuXX34ZFRUVOH36NBYuXIjNmzdj4cKFkEiEvFZaWorKykqUlJTEMlZCemV3+atmGi6qOWZdyf0lIuZxAh5Hv6p5JDxRJaKNGzdixYoVWL58OfLze96qJScnB2+++Wa/giMkXHa3F9mBqlk/S0QqpQpuqQZe3iV04VMiiruoEtGOHTtQXFwcLAEFMMZQU1OD4uJiKBQKLFu2LCZBEnI1LpcTSm8n5FJD5OsQXUEll8Al08PrdQJOE2AIY1800i9RtRENHz4cra2t3Y63t7ejtLS030EREqk5hVIMz9ZCrdP3uwSjVcqQmZmDDJ2CBjUmSFQlot6WR7BarWEv+0pILA1R2gC9CtDn9PtaWqUM15SXANX1AHXhJ0REiWj16tUAAI7j8NRTT0GjufyXx+fzYd++fZg0aVJMAyQkLMFdO/rXPhRE0zwSKqJE9N133wEQSkRHjx6FQnF5EzqFQoGJEydizZo1sY2QkKuwurzoqKuB0emBoZ8N1QFMaYCP58HZO0ArEsVfRInoiy++ACCsLf3aa6/BYDDEJShCItFkcaKmqgqZUgfGx6hEtOVYJ0oudmDMEMDIGMBxMbku6VlUbUQVFRWxjoOQqNkcDii9VsgVsphVzaQqIxg4eL0ewG0FlPqYXJf0LOxEdPvtt6OyshIGgwG33357n+d+9NFH/Q6MkHC5zC0AGKRKLSCPzZgfpUIGt0wLr48J7USUiOIq7ERkNBrB+YunRqMxbgEREilfpzDJWqLLilkVSiWXwiXVw8ubhURkLIzJdUnPwk5EXatjVDUjA4nP1gIJAKmufwMZu1LJpTDL9PDyHdRzlgBRDWh0OByw2+3Br6urq/Hqq69i+/btMQuMkHAxq9B1L9fHMBHJ/KOrA1UzEldRJaIlS5Zg8+bNAACTyYRp06bhpZdewpIlS7Bx48aYBkjI1XB2IREp03Jjdk2haqaDl6dElAhRr9A4c+ZMAMCHH36IvLw8VFdXY/PmzfjDH/4Q0wAJ6ZPPg7HpPIZna6HPzIvZZTO0ChTk5UGvklEiSoCoEpHdbodeL/QibN++HbfffjskEgm+973vobq6OuzrbNy4ERMmTIDBYIDBYMCMGTOwdevWaEIig5W9DWlqOXIy0qBUx25cW1GGBjdNKkOBUU3rEiVA1Ivnf/LJJ6ipqcG2bdtw8803AwCam5sjGuRYWFiIDRs24MCBA/j2229x0003YcmSJTh+/Hg0YZHBKDi1Izv2gw6VBoCTALxXGEtE4iaqRPTUU09hzZo1KCkpwfTp0zFjxgwAQulo8uTJYV9n8eLFWLhwIcrKyjBy5Ej87ne/g06nw969e6MJiwxCnW0NaO50oh2xH+XPOA4euX9fM6qexVVUI6v/7d/+DTfccAMaGhowceLE4PG5c+fiBz/4QVSB+Hw+fPDBB7DZbMHEdiWXywWXyxX82mKxRPVaJHV0tNSjrsUGt16OWTG8rsvrw8Zd5zG60YV5BQxSGksUV1Hv9JqXl4e8vNDGwWnTpkV8naNHj2LGjBlwOp3Q6XT4+OOPMWbMmB7PXb9+PdatWxdVvCQ1+azCYEaprv/Lf3SlkErAgYNLpoePdwqJiMRNVInIZrNhw4YN2LlzJ5qbm8HzfMjzkewEW15ejkOHDsFsNuPDDz/EsmXL8OWXX/aYjNauXRtcigQQSkRFRUXR3AJJBV43eLuw04bCENtExHFccKVGj68JCkpEcRVVIlq5ciW+/PJL3HvvvcjPzw9O/YiGQqHAiBEjAADXXHMN9u/fj9deew2vv/56t3OVSiWUSmXUr0VSjL0Vbi8Pj0QNgz72bUTBsUQ0qDHuokpEW7duxWeffYbrr78+1vGA5/mQdiBCemVrhcvLw6FIh0Elj/nlVXIJrDI9fDxPiSjOokpE6enpyMjI6PeLr127FgsWLEBxcTE6Ozvx7rvvYteuXdi2bVu/r01SH7O1wO31wa5MFwYexphKLkWbTAevh10eS0TrEsVFVN33zz33HJ566qmQ+WbRaG5uxn333Yfy8nLMnTsX+/fvx7Zt2/D973+/X9clg4PH0gwfAxzydOiUsU9ESpkUbqkWHh40lijOonr3XnrpJZw/fx65ubkoKSmBXB5aLD548GBY16F9z0h/SB1tGJ2vR9aIkZBJ+7VpcY/yjSr4+DQomowAXLQuURxFlYiWLl0a4zAIiZDXBam7E2lqBdLitIXVxKI0TCxKA77LAUw1tC5RHEWViJ5++ulYx0FIZKzNwqNSB8jV8X0tlRFADTVYx1HU5VmTyYT//u//xtq1a9He3g5AqJLV1dXFLDhCemVthtnhQa3XgHabO24vw/MMbrm/OkaJKG6iKhEdOXIE8+bNg9FoxMWLF/Hggw8iIyMDH330ES5duhRcq4iQuLE2obnTicM2GUpbrcjQ9r8X90qX2uz46LtalPFOLJKDElEcRVUiWr16NZYvX46zZ8+G7Oy6cOFC7N69O2bBEdIraxNcXh42RSb0cRhDBABKuQSMAZ2gia/xFlUi2r9/Px5++OFux4cMGYLGxsZ+B0VIn3gfYGuBy8PDJs+My2BGAFDJhK0VrcFEROsSxUtUiUipVPY48/3MmTPIzo7dusGE9MjeBp/PCweTwiUzwKiOX4kIAKxQw8dAY4niKKpEdNttt+HZZ5+Fx+MBIEwQvHTpEp544gn88Ic/jGmAhHRjbYLL44NNngmlQgqVPPZjiABAKZOA4wDGSeGV64SDTlp6Jh6iegdfeuklWK1WZGdnw+FwYNasWRgxYgT0ej1+97vfxTpGkmBv763GP040wccP0GqItQlODw+7IhNpakW/Jl33RZiBL1TP3DLqOYunqHrNjEYjduzYgT179uDw4cOwWq2YMmUK5s2bF+v4iAicHh+O1pkhkQA3jYrdzhgxY22G0+uDTZ6F9DhVywJUMgkcbh9cMh30ACWiOIk4EfE8j8rKSnz00Ue4ePEiOI5DaWkp8vLywBiL218nkjg3jszGZ0cacLjGjDH5RuQZVVf/pkRhDOhsQLZOiVkjx0FuTI/ryw3N1CJb74NUmgZYQYkoTiKqmjHGcNttt2HlypWoq6vD+PHjMXbsWFRXV2P58uVRLxNLxPf1+TbUmRxgjGFkrh6j84X1ffZVtYkc2RXsbYDXDblCgaLCoXFPknNG5WDRhHykZ/g7YSgRxUVEJaLKykrs3r0bO3fuxJw5c0Ke++c//4mlS5di8+bNuO+++2IaJImvlk4X9l5ow/6L7XjoxmFQyaWYVpqBU40WXGixwWR3I02jEDtMgcU/cl+fD0ji00jdI5VReKREFBcRvZPvvfce/v3f/71bEgKAm266CU8++STeeeedmAVHEuNcs9AlXZKlDTbOZmgVGJqpAQCcqB9APUWWejDGcNyqw4l6S0Ia1HmewSP3rwDpNNNYojiIKBEdOXIE8+fP7/X5BQsW4PDhw/0OiiRWTbuwrlRppjbk+NgCoRRwosECNlB++Sz1cHl5fNuuxs6TTZDEuUnyQHUH/vDPs/jiokNYFI33Am5bfF90EIqoatbe3o7c3N57UXJzc9HR0dHvoEjiuLw+NJidAIDiDE3Ic8OytJhVno3h2bqB0QnhdQO2Fjg8PlgV2UjTxq/rPkAhFaZ5OLwQ1iJyWvzrEuni+rqDTUQlIp/PB5ms99wllUrh9Xr7HRRJnAaTEzxjMKrlMGpCu8JlUgmmFKfHbeRyxDobAMZghRpumQ6Z2vi3W6kVwq+I0+OjdqI4iqhExBjD8uXLe91Jgxa9Tz5NFqE0lD+Quuh7Y7oEAGiTCVsHZSQkEQm/Ina3D9DRukTxElEiWrZs2VXPoR6z5NJqFdbyyQ0kIt4HXPwX0NkIZJcD+RNxqqkTpxs7MWNYJnIMIiYsfyJq4oTmgYSUiPyN9w4qEcVVRImooqIiXnEQkSwcn4cZwzOhlPlr6Sf/B2g+KXzefgHwOnHOVIwLLTZk6ZTiJSKfV+gxA0MtsgAkqETkT0QuDw+fwgApQIkoDhI4EIMMRBzHIUOrgFYpA9rOC0mIkwC5Y4UTqnZjpE4oNV1oEXHmuaUO4L1wSTSwwAAJxyVkbJNKLgnuIOSk+WZxE/s9WEjyqt0vPA65BhgxF/A6gbbzKLEfgYQbjVarW7zBjaZqAIAyqwQrSofB4vRAGu++ewiJeni2DlIJB6b0N9oHxhINhJ7EFEGJaBA7Xm/GpTY7RubpMVznAdqrhF+uwmuEx5IbgLbzULSdwlB9OaosHM632HDNUBESUds5AACXMQxGTfcevnhaPLFA+IT3hY4loi78mBG1arZ+/XpMnToVer0eOTk5WLp0KU6fPi1mSINKTbsdpxo70WZ1Ay3+n3vaUEDtn0hqKBC2z+F9GC0RGorPi1E9c1qAziYhCWQOT/zrB0ikl/c1o+pZTImaiL788kusWrUKe/fuxY4dO+DxeHDzzTfDZqORq4kQ6DHL1CmCJQ5kjww9KW8cAKDYcwEAUG9ywOH2JSxGAJdjMxRg53kr9l5oS3gMPM/g9fHUcxYnolbNPv/885CvKysrkZOTgwMHDuDGG28UKarBgecZOvzb8GQqfIC5Vngi44oSR/Yo4OwOqF3tKFJawWuyYXd7oVZIExesPxG50objyHkhAUwqSkvYy395pgXfXerA9NJMzKA9zuJiQLURmc3Cm5uR0fPWMC6XK2TQZE/rZpPwmB0eeHkGuZSD0d0oNL5qMgF1WuiJcjWQMQxoPYsfFNogHTYlsYG6bULbFYBWZTEABwxqeXBybiLIJZwwzcPjpRJRnAyY7nue5/H444/j+uuvx7hx43o8Z/369TAajcGPoqKiBEeZOtpsQkLP0CrBBUpDab38PLPKAADS9nOJCC1U0wmA8YAhH41eoXE4R9/zyP54CZT+HG6qmsXLgElEq1atwrFjx7Bly5Zez1m7di3MZnPwo6amJoERppY2f/tQhlZxuVrW277uGcOFhuLORsDVCYfbB4vTE/8gGQMajwif544PTkcRLRHR6Oq4GRBVs0ceeQT/+7//i927d6OwsJdfBgjbGPU2z41ExunlIeE4ZKo5oK1JONhbIlLqhIXILPU4eew7bOvIw7gCI+aNifN61qZqYY97qQzIGY36KmHPvHxjnPe6v0Jwmof7iqoZjSWKGVFLRIwxPPLII/j444/xz3/+E6WlpWKGM6jMGpmNR24agYlGhzA+RqkDVGm9f4O/epbtqgZjwIVWa/zXKKr5RnjMmwALL0en0wsJxyV8De2Q+WZKA61LFAeiJqJVq1bh7bffxrvvvgu9Xo/GxkY0NjbC4XCIGdagIZVwUDhahC/0+X3/dc8cAQDIcDdAJfHB5vKh0V9ViouOi8KUE44DCqfCbPdALuWQrVdCIUvsf9uubUSMk9BYojgQNRFt3LgRZrMZs2fPRn5+fvDj/fffFzOswcXqr5bprlLN0mYDKgMkzIcxamHxu/PNcSoR+LzA2R3C5wVTAE0GijI0+NnsEVg8MT8+r9kHtVyK4gwNynJ1wtK01E4Uc6K2EQ2Y5UcHmeZOJ/55shl5RhVmW4V2F+jz+v4mjhNKRXUHMUzaiIPIwoVWK24oy4ptcIwB53YAtlZh6EDJDcGnpBIO+jjtc98XmVSCH17Tpf2MxhLF3IDpNSOJ025zo8HsRIvJBtj82wVdrUQEBAc75nnqIIHQ8xYYFBkTPA+c3wnUHxIS35jbAIVm4P3BohJRzFEiGoRMdqHrPUdqEcboyNWX2z36kj4UkMog91gxQissuH+mqTM2QVmbgYNvATX+FQBGfF8YSAngbLMVFXuqsP9ie2xeK0o+muYRNwOi+54kViARZcIkHNDnhdcNLZUDaSVA2zlM0bWjuLgUZbn9nIHO+4DqPUD110JSlCmBkbdcXg8JQHWbHSa7R1iuVSQ7TjTheL0ZN47MxhQ9JaJYo0Q0CJkdQnUq3ecvYYRTLQvIHAa0nUO+tw75hTf1LxB7O3DiU2GgJCBMuC27OaR0xhhDdZvQKH7lLiOJJJcK0zzsLh+QTWOJYo0S0SAUKBHpvRG0DwVkjgCwXVgx0W0HFFEmB0s9cHgL4HUBchUwcr4wwfaKX2qT3YNOpxdSCYchaYkdyNiVVin8qtjcXkCZTusSxRi1EQ0yTo9PqOIwHhqPv0R0tR6zrlRGQJcNMAZf23kcqjHhg29r4Pby4V/D2gwcfk9IQoYC4NoHgJzRPZYsLvpLQ/lGVcLHD3Wl8Y8lsru9tC5RHFAiGmQcbh+MajkypXbImE+YPtHXiOqe+HvPJO3ncehSB2o7HDjREOZKCG47cOz/EzZLTCsCJv4YUBl6Pf18i5CIhmWLW+rQ+rcVsrn87VSBn5mDNhSNBUpEg0y6VoEVN5TiJ+P81RxNFiCJ8L+Bf5Q111GFiYVCEjl0qSO8bvaz2wGHSVhuZNwPAVnvy846PT7UdQij7Idna3s9LxE0yi4lIgDQ+JeqcYjbk5cqKBENUhJ7q/CJLifybzYMEbr8PU6M0ZigkEnQYffgYpu97+8L7hLCAWOWCtfog5dnGF9oQEmWRpwF+7vQdtlokecZoPYnIjsloligRDRY2ZqFR2125N8rkQibLwJQtp3C2AKhVPTtxfbeS0VeN3Bmm/B54VTAcPWpGjqlDDeNysUPJve+IkOiBKZ5jMrTw8uzyyUie5u4gaUISkSDzKeH6vDX/TWwtDYIB6JJRMDlcT4tpzClUNhup7bDgZr2XiYsX/xKaNhVGYGSmdG9pogkEg4/vKYQ88flC43mmkzhCUe70IVP+oUS0SDTaHaiob0TEpdJOBBtIjIWCY3MXjcM1osYXyiMrfn6Qmv3UlFnE1D7rfD5yFv6bBcKaDA7UNNuH3jTOwJURmEjSp8XcMVodPkgRoloEHF5ha57tacDKhknjAGKdgwMxwG5/iV96w5gakkGSrO0uKEsG1zXbnieB85sFUZN54wKezugfRfa8eGBWnxbPbB6pXw8g8fHC134gfW9qXrWb5SIBhGzfyBjBkyQSSTRl4YChkwRfiHNtdC5WrB08pDugw5r9wOWBmHqxoh5YV3W7vai2t/wPULkbvuu/nmqCX/YeRaHakzCga7VM9IvlIgGEZNDSETZEn9VQhtFj1lXSr0wGhoAqr4MaStxenxCj1LVbuHAiLnhTawFcKqxEzxjyDOqkK4Vt7esK4VU6MK3uvxd+IGNKO0Dq9SWjCgRDSKBqR0ZzCQc0MZgLaGSG4RSUXsV0HoGPM+w51wrKnafgfXgX4VpEBmlQN6EsC7HGMOxOmG08pj83gc6ikHrH0tkdV4xloiqZv1GiWgQMdmFya4G3iQciGYM0ZU0GUJ3PACc+l9wllp0mNoxrOHvuFBVBV6uBcoXhD0xtLbDgTarGwqZBKPywytBJUpgUbbLJSIa1BgrlIgGEYVMgnS5B1o4hcSgidHqiiUzhR1AvG5w372Nm00fINPbCItXgm91sy+v3xOGQPvL6Hw9lLIE7iYbBr1KGNTYGdhKKdBG5DQLvWckapSIBpHZ5TlYPlGLLJ1SmCsVRjd6WKQyYMKPgLxxAMdBIZWguGQEjufchv9rVeFia3hrW3t9fHC/tAmFabGJLYYCicju9glrVyu0ws+QMZpz1k+0DMhgY/Xv2hGL9qGuZEpg9GKg7BaA9yJHocHwE004WmfG34814O5pxVedpiGTSnD3tGI0WpxCshxg1HIppBIOPp7B6vLCqJYLpUpLPWBvFVYlIFGhEtFgY/Mnoli0D/VEpgiuUTS7PBv5RhVcHh7/c6RBKEVcBcdxCd9AMVwcx6EsR4fRXRvRA0MgrM3iBJUiKBENEueaO/HG7gs4ec6/f31/xxCFQSaV4NaJBdCrZJhSnAappOcGa4+Px/6L7ZGtaSSSBePzMX9cnlAaAi7/HAMJnkSFqmaDRIfdA6vTDamjDdCq+j+GKEw6pQz3zSgJWdSMMRYcfc3zDNuPN+FMUyfONlnx42lFoSOzB7pAdczWKm4cSY4S0SBhsnug9pqhlvLCIviBMTAJ0DUJWV1efHywFsNzdDCo5DhRb0GdyQEJx2FmWVZSJKHANA+VXHq5ROQ0CSsMxKoDYJARtWq2e/duLF68GAUFBeA4Dp988omY4aQ0k90NrbtV+OXR5Yq24Pu+C21otbqx70I7dpxoQp3JAbmUw60T81Ek4uL44TpWZ8Yfdp7F9hP+HXIVWqFNjDEa2NgPopaIbDYbJk6ciBUrVuD2228XM5SUZ3Z4kOFuhcogiWyN6hibU56DIelqnG+2weX1IVuvxMSiNBhE2ME1GmrFFaOrAaFU5K4W2onCWGeJdCdqIlqwYAEWLFggZgiDgsfHo9PpRZG7BSqZNLJdO2JMIuEwKs+AUXkDa/pGuAJjiQLjnQAIiaijmhqs+yGp2ohcLhdcLlfwa4slzAXbBzmT3QMwBqOvHTKpTtQSUbIL9JY53D64vD5h9HdgTBYloqglVff9+vXrYTQagx9FRUVih5QUGGMYpnMjTcHASeSxm9oxCCll0mD1zOxfzSDYA0ljiaKWVIlo7dq1MJvNwY+amhqxQ0oKOQYVlgyXCtUhXXbku3aQEGn+UlFgfSfocoTGf7eNVmuMUlJVzZRKJZTKgTf0PylY6oRHwxBx40gBRrUcDWbn5RKRVC5MgLW1Cttnh7nuErmM/jQOAl4fD5j9pUdKRP1WlKHB6HwDMrou2qb395Z1NogTVJITtURktVpxLjDlAEBVVRUOHTqEjIwMFBcXixhZ6uB5hr98cQrT605j/BADFEbxt+ZJduOGGDFuyBVLm+jzgcajwkYBJGKiJqJvv/0Wc+bMCX69evVqAMCyZctQWVkpUlSpxeL0QOlogtfng0xj7HN7Z9IPev+QiM4GYXBjEowQH0hETUSzZ88euNvFpIh2mxt6VyNUcikkVBqKGR/PYLK7ka5RQCLh/KPVJZcbrCnhR4TaiFJcu80No6sBGoVU2IuM9BtjDG98dQGbv65Gh3/5XUjlgNa/YmNno3jBJSlKRCmuvdMaLBEho1TscFICx3HBgY3tNvflJ/QFwqOlVoSokhslohTnbKkCx3io9BmXt78h/RboMWvrmogCVV8zJaJIUSJKYT6egbVVAQA0uWXUgBpDWTp/IrJ2SURp/qpvZyPg8/TwXaQ3lIhSmMfrw2hZI4xqOTT5ZWKHk1IytMLA2nbb5bmPUKUJW3jzPmEdaxI2SkQpTOVowsg0hjGFmeAyhokdTkoJVM067J7La3FzHFXPokSJKJW1nBQes8qEXh0SMwaVDAqZJNiNHxTomaREFJGkmmtGIuDzwnbpMNRgkGSPFjualMNxHCYWpkEu5aCUd9kIMs0/I8B8Sdh0UUq/YuGgn1KK4ptP4Xh1I1wSDcZPL0aa2AGloBvKelhORZsttBO5rML8PhoyERaqmqUixmC98DV8PEOrYQyMGlqxIGE4Dgi0x7VfEDeWJEKJKBW1nYe9tRY8J4NkyKSk2BkjGTHGYHF6cLapM3TzSEpEEaNElGp4H1C1C1anB436scjJpNUY4+ntvdX43yMNaLN26cZPLxFKRrZWwNEhWmzJhBJRqqn5BrC2oMMjQ71+IgrSVGJHlLI4jkOeQfj51pkcl5+Qqy83WjefEiGy5EOJKJVY6oGLX8Hp8eGUbhp4mRp5RkpE8RTYi62mwxH6RI6/p7L5RIIjSk6UiFKFqxM4/jHA+9CkHIpWTRkK0lTCLhMkborShURU22EH37WdKKtcWBbE2gzYaOPFq6FElAo8DuDwFsBpATQZSJv8A9xYno0JhWliR5bycvRKKOUSuDw8mjqdl59QaC533TceESe4JEKJKNl5XcCRvwoNo0odMOFHMBp0uGZoBsrzaBH3eJNIOBT7q2fnm22hT+ZPEh4bDguDG0mvKBElM48DOPye0DYkVwET7qKlPkQwMldI+OdbrKFPZI4QVmr0OKit6CooESUrtw049C5gabichHTZ2HehDSfqLXB7ebEjHDRKMrX4/phc/OjaK1bAlEiAgsnC55f2Ajy9J72hRJSMXJ1CErI2AwotMOkewJCPTqcHey+0Y9vxxtC92UlcKWQSjBtiDO4AG2LINUJ3vr2N2or6QIko2dhage/e9rcJ6YUkpBO2PD5cYwbPGArT1cjS0bQOMTDGQkdZy5TA0OuEzy9+JVTTSDeUiJJJx0Xg4GbAYRLagibfE1yw3eby4nCtCQAwuZjaicRQ1WrDu99cwldnW0KfKJgCaDKEibBndwjbDZEQlIiSAWNCG8Ph94VeMuMQYMq9IQ3TX51tgdvLI8+owvBsrYjBDl4SDmi2uHC4xozmrl35UhkwapEw7aPpuPBekhADIhH9+c9/RklJCVQqFaZPn45vvvlG7JAGDlub0DN2/guA8UDuGGDi3ULbkN+xOjNONnSC44DZ5dk0yVUkQzO1GJ6jA88YPjvSAIfbd/lJYyEwYp7w+YVdwMU9VDLqQvRE9P7772P16tV4+umncfDgQUycOBG33HILmpubxQ5NXNYW4PRWYP9/Ax3Vwl/V8gXA6NtCFttyenz46mwrAGDGsEzkG9ViRUwAfH90LvQqGUx2Dz44UIOWzi6TYQuvBYbOED6v2i0MQqUtqgEAHBN5q9Xp06dj6tSp+NOf/gQA4HkeRUVF+MUvfoEnn3yyz++1WCwwGo0wm80wGJJ0Z02eB7wOoTve1iLsANF+QWiMDsgqA4bfBKZOh8vLw+zwQK2QwqASln+t7bDjXLMVs0ZSaWggaLe58dHBWnQ6vZBwHCYUGTGnPOfyCXUHgXM7Ad4/yNFQICwdYigQqttKPSCRpcSuK+H+joq6QqPb7caBAwewdu3a4DGJRIJ58+bh66+/7na+y+WCy3X5L4zFYgnvhdqrgPM7ey4KhxxjfR/v4Viny4PTDZbL3xk4hwn/DElXo8A/8dTm8uJ4vbnLJXhIeQ84MDAG5BlVGJImlGicXoavTOloMoyDtb0ArK0DPr4NHp9w/e8Ny8SM4UJDdWG6BoX+OU9EfBlaBe6aVowvTjXjXLMVLs/l8UNeH4+/nDVA45uHYtN+pNsvgGMmACeENdU0CmEirUQKLyfHobpOMEgAjgPjJAA4MHAAB2RqFRiRrQMA8AzYX90OABg/xAiNPAFzDCfcGbOttUVNRK2trfD5fMjNzQ05npubi1Onui+fsH79eqxbty7yF/K6hKpOPDg98Nh7T4i80weo/f8RPV74XPaQ5wOtCF6JEnZ1PlBQCqQVw6EcgmP7/cX2rm0NANQKKXhqXxjQdEoZFk8sQLPFCV+X96rT6YXLw8MFLToMsyHXTEW68xKMznqoPR3QMP8fWt4H8H3/32JMAWiERnGOMfCdQiKC1Q0oEvCrzWI3QDOp1qxeu3YtVq9eHfzaYrGgqCiM/dzTioCJdwmfhxR3ud6PhXlc7uVR3OkWnu5yPFBF0qtkgErYekbh4zHM7gbX9XVlakCmAiRSaJQyQCm8JXofj3tnqLq+KmQSCTRKKeRS0Zv2SJhyDKHLsOhVMiy/rgQenofXx/x/UMqDBWmtQgqoAHhdkHicKC6zCqVsxgNg/s8ZAAaVXAL4tzUCgOISISnJ9EpAmoBqnSJ2vbOiJqKsrCxIpVI0NYU22DU1NSEvL6/b+UqlEkplFAP1FNq4LWKuAjAkI7xz5QByjeGdK5NKaFBiCpJJJUjvkjx6P1EJicqAIfqcq58L4e9guP8PByJR/7QqFApcc8012LlzZ/AYz/PYuXMnZsyYIWJkhJBEEr1qtnr1aixbtgzXXnstpk2bhldffRU2mw3333+/2KERQhJE9ER05513oqWlBU899RQaGxsxadIkfP75590asAkhqUv0cUT9kRLjiAhJYeH+jlL3CyFEdJSICCGio0RECBGd6I3V/RFo3gp7qgchJKECv5tXa4pO6kTU2dkJAOGNriaEiKazsxNGY++jeZO614znedTX10Ov1w/IWeeBKSg1NTUp2auXyveXyvcGJO7+GGPo7OxEQUEBJJLeW4KSukQkkUhQWFgodhhXZTAYUvI/c0Aq318q3xuQmPvrqyQUQI3VhBDRUSIihIiOElEcKZVKPP3009GtGJAEUvn+UvnegIF3f0ndWE0ISQ1UIiKEiI4SESFEdJSICCGio0RECBEdJaIIRbor7QcffIBRo0ZBpVJh/Pjx+Pvf/x7yPGMMTz31FPLz86FWqzFv3jycPXs2nrfQq0ju7Y033sDMmTORnp6O9PR0zJs3r9v5y5cvB8dxIR/z58+P9230KpL7q6ys7Ba7ShW6EH6yvnezZ8/udm8cx2HRokXBcxL+3jESti1btjCFQsE2bdrEjh8/zh588EGWlpbGmpqaejx/z549TCqVshdeeIGdOHGC/b//9/+YXC5nR48eDZ6zYcMGZjQa2SeffMIOHz7MbrvtNlZaWsocDkeibosxFvm93X333ezPf/4z++6779jJkyfZ8uXLmdFoZLW1tcFzli1bxubPn88aGhqCH+3t7Ym6pRCR3l9FRQUzGAwhsTc2Noack6zvXVtbW8h9HTt2jEmlUlZRURE8J9HvHSWiCEybNo2tWrUq+LXP52MFBQVs/fr1PZ7/ox/9iC1atCjk2PTp09nDDz/MGGOM53mWl5fH/uM//iP4vMlkYkqlkr333ntxuIPeRXpvV/J6vUyv17O33noreGzZsmVsyZIlsQ41KpHeX0VFBTMajb1eL5Xeu1deeYXp9XpmtVqDxxL93lHVLEyBXWnnzZsXPNbXrrQA8PXXX4ecDwC33HJL8Pyqqio0NjaGnGM0GjF9+vRerxkP0dzblex2OzweDzIyQve02bVrF3JyclBeXo6f/exnaGtri2ns4Yj2/qxWK4YOHYqioiIsWbIEx48fDz6XSu/dm2++ibvuugtabeg+ZYl87ygRhamvXWkbGxt7/J7GxsY+zw88RnLNeIjm3q70xBNPoKCgIOQXYv78+di8eTN27tyJ559/Hl9++SUWLFgAn8/Xx5ViL5r7Ky8vx6ZNm/Dpp5/i7bffBs/zuO6661BbWwsgdd67b775BseOHcPKlStDjif6vUvq2fdkYNiwYQO2bNmCXbt2hTTo3nXXXcHPx48fjwkTJmD48OHYtWsX5s6dK0aoYZsxY0bI3nrXXXcdRo8ejddffx3PPfeciJHF1ptvvonx48dj2rRpIccT/d5RiShMke5KCwB5eXl9nh94jOSa8RDNvQW8+OKL2LBhA7Zv344JEyb0ee6wYcOQlZWFc+fO9TvmSPTn/gLkcjkmT54cjD0V3jubzYYtW7bggQceuOrrxPu9o0QUpmh2pZ0xY0bI+QCwY8eO4PmlpaXIy8sLOcdisWDfvn0J3ek22h13X3jhBTz33HP4/PPPce211171dWpra9HW1ob8/PyYxB2uWOwo7PP5cPTo0WDsyf7eAcLQEpfLhZ/85CdXfZ24v3cJaxZPAVu2bGFKpZJVVlayEydOsIceeoilpaUFu3Xvvfde9uSTTwbP37NnD5PJZOzFF19kJ0+eZE8//XSP3fdpaWns008/ZUeOHGFLliwRrQs4knvbsGEDUygU7MMPPwzp4u3s7GSMMdbZ2cnWrFnDvv76a1ZVVcX+8Y9/sClTprCysjLmdDoTem/R3N+6devYtm3b2Pnz59mBAwfYXXfdxVQqFTt+/HjwnGR97wJuuOEGduedd3Y7LsZ7R4koQn/84x9ZcXExUygUbNq0aWzv3r3B52bNmsWWLVsWcv5f//pXNnLkSKZQKNjYsWPZZ599FvI8z/PsN7/5DcvNzWVKpZLNnTuXnT59OhG30k0k9zZ06FAGoNvH008/zRhjzG63s5tvvpllZ2czuVzOhg4dyh588MFuY3ESKZL7e/zxx4Pn5ubmsoULF7KDBw+GXC9Z3zvGGDt16hQDwLZv397tWmK8d7QMCCFEdNRGRAgRHSUiQojoKBERQkRHiYgQIjpKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjRESSSmVlJdLS0oJfP/PMM5g0aVLw6+XLl2Pp0qUJj4v0DyUi0qPAmsU//elPuz23atUqcByH5cuXh5wf6wRQUlKCV199NeTYnXfeiTNnzvT6Pa+99hoqKyuDX8+ePRuPP/54TOMisUeJiPSqqKgIW7ZsgcPhCB5zOp149913UVxcLEpMarUaOTk5vT5vNBpDSkwkOVAiIr2aMmUKioqK8NFHHwWPffTRRyguLsbkyZP7de2eSipLly4NlrJmz56N6upq/PKXvwzuIgF0r5pdqWvJbPny5fjyyy/x2muvBa9RVVWFESNG4MUXXwz5vkOHDoHjuISvlUQElIhIn1asWIGKiorg15s2bcL9998f99f96KOPUFhYiGeffRYNDQ1oaGiI+BqvvfYaZsyYgQcffDB4jeLi4m73BAAVFRW48cYbMWLEiFjdAokAJSLSp5/85Cf417/+herqalRXV2PPnj1hLaTVXxkZGZBKpdDr9cjLy4tq1UOj0QiFQgGNRhO8hlQqxfLly3H69Ong3l8ejwfvvvsuVqxYEevbIGGiNatJn7Kzs7Fo0SJUVlaCMYZFixYhKytL7LD6paCgAIsWLcKmTZswbdo0/M///A9cLhfuuOMOsUMbtKhERK5qxYoVqKysxFtvvRWzUoNEIsGVS2F5PJ6YXDscK1euDDbEV1RU4M4774RGo0nY65NQlIjIVc2fPx9utxsejwe33HJLTK6ZnZ0d0u7j8/lw7NixkHMUCkW/t6/p7RoLFy6EVqvFxo0b8fnnn1O1TGRUNSNXJZVKcfLkyeDnvTGbzTh06FDIsczMTBQVFXU796abbsLq1avx2WefYfjw4Xj55ZdhMplCzikpKcHu3btx1113QalURlUlLCkpwb59+3Dx4kXodDpkZGRAIpEE24rWrl2LsrKyhC54T7qjEhEJi8FggMFg6POcXbt2YfLkySEf69at6/HcFStWYNmyZbjvvvswa9YsDBs2DHPmzAk559lnn8XFixcxfPhwZGdnRxX3mjVrIJVKMWbMGGRnZ+PSpUvB5x544AG43e6E9AKSvtGa1WTQ+uqrrzB37lzU1NR02ymVJBYlIjLouFwutLS0YNmyZcjLy8M777wjdkiDHlXNyKDz3nvvYejQoTCZTHjhhRfEDoeASkSEkAGASkSEENFRIiKEiI4SESFEdJSICCGio0RECBEdJSJCiOgoERFCREeJiBAiuv8frr9ZwG5NpJsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:55:58.568796Z", + "iopub.status.busy": "2024-03-22T20:55:58.568507Z", + "iopub.status.idle": "2024-03-22T20:55:58.924783Z", + "shell.execute_reply": "2024-03-22T20:55:58.923874Z" + }, + "papermill": { + "duration": 0.377324, + "end_time": "2024-03-22T20:55:58.927085", + "exception": false, + "start_time": "2024-03-22T20:55:58.549761", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABC+UlEQVR4nO2deXxU5b3/32f2ZDIzWcgKCQk7yC5Ccakg1AWqWNuqtVUQly723mv5cV/K7S2KvizYqhVbX7a1AnrVUm3VrooUBSxFRRAU2UMSAiRkX2Zfzvn9cTJDhiSQZWZOMnner9cwM+c8c873cDKf+T7P832+X0lRFAWBQCDQEJ3WBggEAoEQIoFAoDlCiAQCgeYIIRIIBJojhEggEGiOECKBQKA5QogEAoHmCCESCASaY9DagL4gyzKnT5/GZrMhSZLW5ggEgnNQFIXW1lYKCgrQ6br2ewa0EJ0+fZrCwkKtzRAIBBegsrKSYcOGdbl/QAuRzWYD1Iu02+0aWyMQCM6lpaWFwsLCyHe1Kwa0EIW7Y3a7XQiRQNCPudDQiRisFggEmiOESCAQaI4QIoFAoDkDeoyoOyiKQjAYJBQKaW3KoMZoNKLX67U2Q9BPSWoh8vv9VFVV4Xa7tTZl0CNJEsOGDSMtLU1rUwT9kKQVIlmWKSsrQ6/XU1BQgMlkEkGPGqEoCrW1tZw8eZLRo0cLz0jQAc2F6NSpUzzwwAO8/fbbuN1uRo0axfr165kxY0afjuv3+5FlmcLCQlJTU2NkraC3ZGdnU15eTiAQ0FSIjpxp5fOTzYzNszFxqEMzOwTRaCpEjY2NXHbZZcydO5e3336b7Oxsjh49SkZGRszOcb6wckHi6A/eaKPLzzv7qwnJCica3OTYzOTYLVqbJUBjIXr88ccpLCxk/fr1kW0lJSUaWiRIZj4/1UxIPlsrYm9lE1dflKehRYIwmroLf/nLX5gxYwbf/OY3ycnJYdq0aTz//PNdtvf5fLS0tEQ9BILucqzGCcDUonQAyupciCI2/QNNhej48eM899xzjB49mk2bNvH973+f//zP/+TFF1/stP3q1atxOByRR7IueJ0zZw7333+/1mYkFYqiMLMkkwkFdr5UksUQm5lROWkEQkKI+gOaCpEsy0yfPp2f/vSnTJs2jXvvvZd77rmHX//61522X7FiBc3NzZFHZWVlgi3uX2zduhVJkmhqatLalH6PJElMHOrgmovySDHpuf1Lw5k3PheTQYwh9gc0vQv5+flMmDAhatv48eM5ceJEp+3NZnNkgatY6CoQJA+aCtFll13G4cOHo7YdOXKE4cOHx/W8/qDc5SMYkrvdNtCNtn3F5/PxwAMPUFhYiNlsZtSoUbzwwguUl5czd+5cADIyMpAkiSVLllzweK2trXz729/GarWSn5/PL37xiw5dwf/7v/9jxowZ2Gw28vLyuO2226ipqYnsD3tiW7ZsYcaMGaSmpnLppZd2uJf9ifI6F9XN3qj7G5IVGl1+Da0ShNF01uxHP/oRl156KT/96U+5+eab+fjjj/ntb3/Lb3/727ie99n3j3W5r2SIlRunDY28/+320i7HEYZlpPDNGWfHqdbtKMPjj15K8qOvjOmTrXfccQc7d+7kmWeeYcqUKZSVlVFXV0dhYSF/+tOf+PrXv87hw4ex2+2kpKRc8HjLli1jx44d/OUvfyE3N5eVK1eyZ88epk6dGmkTCAR49NFHGTt2LDU1NSxbtowlS5bwj3/8I+pYP/7xj3nyySfJzs7me9/7HkuXLmXHjh19ut54sfnAGZy+ILdcUkhBegrN7gAv7ixHr5P4wZyR/SK8YDCjqRBdcsklvPnmm6xYsYJHHnmEkpISnn76ab797W9raVa/4ciRI7z22mts3ryZ+fPnAzBixIjI/szMTABycnJIT0+/4PFaW1t58cUXefXVV5k3bx4A69evp6CgIKrd0qVLI69HjBjBM888wyWXXILT6YxaovHYY49x5ZVXAvDggw+ycOFCvF4vFkv/is3xBkI4fUEAstJMAFjNemRFIRRUcPtDWM2ax/YOajT/3//qV7/KV7/61YSe8765o7rcpzvnh/HeL4/ssu25P6JLL4ttDNTevXvR6/WRL3tfOX78OIFAgJkzZ0a2ORwOxo4dG9Vu9+7dPPzww+zbt4/GxkZkWe3OnDhxImpMb/LkyZHX+fn5ANTU1FBUVBQTe2NFndMHgD3FiNmgRnUb9DpsFiMtngBNnoAQIo0ZlFMGJoOuy4dBr+t2W2M32vaF7nS1Yo3L5eKaa67BbrfzyiuvsGvXLt58801AXTbTHqPRGHkd7tqERas/0eQOAJCRaozanp6ivhfjRNozKIVooDBp0iRkWWbbtm2d7jeZ1G5Gd1OcjBgxAqPRyK5duyLbmpubOXLkSOT9oUOHqK+vZ82aNVxxxRWMGzcuaqB6INLiUYXIkRItRBlW9X1z236Bdggh6scUFxezePFili5dyltvvUVZWRlbt27ltddeA2D48OFIksTf/vY3amtrcTqd5z2ezWZj8eLF/Pd//zfvv/8+X3zxBXfddRc6nS7i0RQVFWEymfjlL3/J8ePH+ctf/sKjjz4a92uNJ81dCJEjRRXysMck0A4hRP2c5557jm984xv84Ac/YNy4cdxzzz24XC4Ahg4dyqpVq3jwwQfJzc3lhz/84QWP99RTTzF79my++tWvMn/+fC677DLGjx8fGWDOzs5mw4YNvP7660yYMIE1a9bwxBNPxPUa402LVxUaewchMkbtF2iHpAzgxTYtLS04HA6am5s7BDd6vV7KysooKSnpd7M4/QmXy8XQoUN58sknueuuu+J2Hi3vR2WDm3qXnxHZVuyWs2LU4PKzq7yBLKuJGcWZCbVpsHC+72h7xFTBIOPTTz/l0KFDzJw5k+bmZh555BEAFi1apLFl8aMwM5XCzI45qTKtJq4Rq+/7BUKIkohzp9fP5cCBAwA88cQTHD58GJPJxMUXX8wHH3zAkCFDEmWmQNABIURJREFBAXv37j3v/qKiInbv3p04ozSm2R2gusVLRqqx0yRowZCMyxfCbNRhMYoUtlohhCiJMBgMjBrVdbDmYOREg5t/HjzDiGwri6YO7bD/zU9PcbLRw4JJ+YzNO39ZZEH8ELNmgqQmvLTDaur8N9dmUbe3ipkzTRFCJEhqwkKUZulciNLM6ixaa1s7gTYIIRIkNa6wEHWxliwsUE6vECItEUIkSGoiXbOuhMisDlC7/UKItEQIkSCpuZBHlNI2duTyiZLkWiKESJC0BEMy7rZEdV0JkdWkekSegBAiLRFC1A/pb1U8+ps93UWSJK6fUsBV43KwGDv/U081GbiowM7UwnRkecCudhrwiDiiJMXv90fShAxW9DqJUTlp521jMuhEkcV+wODyiBQFgv7EP3qwrnjJkiVs27aNtWvXIkkSkiRRWlrKXXfdRUlJCSkpKYwdO5a1a9d2+NyNN97IY489RkFBQSTr4r///W+mTp2KxWJhxowZvPXWW0iSFBWBvX//fq677jrS0tLIzc3l9ttvp66urkt7ysvL+3wrBIL2DC6PKBSAD55M/Hmv+H9g6J53snbtWo4cOcLEiRMjC1IzMjIYNmwYr7/+OllZWfz73//m3nvvJT8/n5tvvjny2S1btmC329m8eTOgrny+/vrrWbBgAa+++ioVFRUdulhNTU1cddVV3H333fziF7/A4/HwwAMPcPPNN/Pee+91ak92dnYM/lPiT02Ll0Z3gCFpJrLSzF22C4Rk3GKZh6YMLiEaADgcDkwmE6mpqeTlne0yrFq1KvK6pKSEnTt38tprr0UJkdVq5Xe/+12kS/brX/8aSZJ4/vnnsVgsTJgwgVOnTnHPPfdEPvOrX/2KadOm8dOf/jSybd26dRQWFnLkyBHGjBnTqT0DgUPVreyuaGRaUTpzxuZ02e7t/dWU1jiZNz6HycPSE2egIMLgEiK9UfVOtDhvH3n22WdZt24dJ06cwOPx4Pf7o0oAgZpatv240OHDh5k8eXJU/p/2ifMB9u3bx/vvvx9VnSNMaWkpY8b0rRySloRjg7qaMQsTnjkTU/jaMbiESJK63UXqT2zcuJHly5fz5JNPMnv2bGw2Gz//+c/56KOPotpZrdYeH9vpdHL99dfz+OOPd9gXrswxUAlP3aeYzt/dCu8XQY3aMbiEaIBgMpmiEuLv2LGDSy+9lB/84AeRbaWlpRc8ztixY3n55Zfx+XyYzeoYSfvE+QDTp0/nT3/6E8XFxRgMnf85nGvPQCEsRKldLHgNE14Q6/YPvGtMFjSdNXv44YcjMzHhx7hx47Q0qV9QXFzMRx99RHl5OXV1dYwePZpPPvmETZs2ceTIEX7yk590EJTOuO2225BlmXvvvZeDBw+yadOmSP7pcLL8++67j4aGBr71rW+xa9cuSktL2bRpE3feeWdEfM61pz+WDOqMcNVd6wU8IqtY5qE5mk/fX3TRRVRVVUUe//rXv7Q2SXOWL1+OXq9nwoQJZGdnc80113DTTTdxyy23MGvWLOrr66O8o66w2+389a9/Ze/evUydOpUf//jHrFy5EiAyblRQUMCOHTsIhUJcffXVTJo0ifvvv5/09HR0Ol2n9pw4cSJ+Fx8jFEXpQddMLPPQGs27ZgaDYcDNxsSbMWPGsHPnzqht69evZ/369VHbVq9eHXm9YcOGTo916aWXsm/fvsj7V155BaPRGFWNdfTo0bzxxhs9sqe/4wvKyG3xWykXmJIP7/cGhRBpheZCdPToUQoKCrBYLMyePZvVq1f3u5LFA5mXXnqJESNGMHToUPbt2xeJEdKiimwiMejU5R3eQKhD9d5zSTXpmTjUQYpRj6IokW6rIHFoKkSzZs1iw4YNjB07lqqqKlatWsUVV1zB/v37sdk6pu30+Xz4fL7I+5aWlkSaOyCprq5m5cqVVFdXk5+fzze/+U0ee+wxrc2KOwa97oLLO8JYjHq+MiE3zhYJzke/qmvW1NTE8OHDeeqppzqtsfXwww9HBfaFEXXN+j/ifgxOulvXTPPB6vakp6czZswYjh071un+FStW0NzcHHlUVlYm2ELBQKG21cfh6lZqW30Xbgz4giGa3QF8YpxIE/qVEDmdTkpLS7sMpDObzdjt9qjHhehHDt+gJtH34ViNk398XsVnJ5u61f7Pn55m3Y4yTtS742uYoFM0FaLly5ezbds2ysvL+fe//83XvvY19Ho93/rWt/p8bKNRXVbhdos/rP6A3+8HQK9PzKJST0CNCbrQ1H0Yi0iQpimaDlafPHmSb33rW9TX15Odnc3ll1/Ohx9+GJPV3Xq9nvT0dGpqagBITU0VsyEaIcsytbW1pKamdhm9HWu6G1UdxmJQf5M9IrpaEzQVoo0bN8b1+OH4pLAYCbRDp9NRVFSUsB+Ds0LUPY8oRXhEmqJ5HFE8kSSJ/Px8cnJyCAREAT0tMZlMkUjtRBD2bC4UzBgmEtQYGBjLV5KNpBaiMHq9PmFjE4L+QU89IktEiIRHpAX9atZMIIgFIVmJCEp3x4hE10xbBoVHJBh83DC1ALcv1GX1jnPJSDUxaaiDDGvfk9gJeo4QIkHSoddJjMzuYnmHokBTBfickDUKjGqUd6bVxHyxzEMzhBAJBg+KAsf+CSc/Ud9b7DD9DjB3XNcoSCxijEiQdDS4/ByqbqGmxRu9o+6IKkKSBMYU8LbAoX9EdoeXeYREocWEI4RIkHSU1bl4+/Nqdlc0nt0oy1D6vvq66EuqJ6TTQ8NxaD4JwPod5azbUUaDy6+B1YMbIUSCpMPTWWbGhuPgaVTHhIpmQ2om5F6k7ju9V20vpvA1QwiRIOkI556Omrqv2qs+500CQ1uxxbzJ6nPdYQgFhBBpiBAiQdLhCZwTzBj0qR4RQN6Usw0dw8DiUMuCNxzH3DbVL6KrE48QIkHS0SFpfkMZyCG1O5bWbkG1JMGQ0ZE2FpG7WjOEEAmSjg7LO+qPqs9ZIzs2zihWnxvLxTIPDRFCJEgqFEXBEx4jMhrU2KFwtyxrdMcPpBeBpANPI1bZCYiumRaIgEZB0nHdpHw8/pBaONFdD3436A3qmNC5GMxgy4WWKgp0DUwaOoyhGcld4aQ/IoRIkFRI0jnLO5raikHah6pxQ51hK1CFSGqkYMLF8TdS0AHRNRMkN81tBRYchV23sReoz62n42+PoFOEEAmSiia3n4NVLVQ3ty3vaIua7rRbFqZNiJTWarw+P80ekUQv0QghEiQVJxs9vLO/mo/K6sHTpK4nk3Rq16wrUjLAaMHn8/PS5l28/GFFwuwVqAghEiQV7vYpYlur1I1pOWAwdf0hSYK0XAx6idRAA/6gTDAkZs4SiRAiQVIRtbwjLES2zuvkRWHNRq+TsAYbAPAFhRAlEiFEgqQiasFr6xl1o60bCc+sQ5CQsIeaABHUmGiEEAmSikhUtVEHzmp1Y7c8ohy1aViIhEeUUIQQCZKKcNcsTXFCwKvGDlm7UbDTOgSAFMWDIeQVHlGC6TdCtGbNGiRJ4v7779faFMEAJuIR+WvVDdbsrgMZ22Mwg8URGbAWQpRY+kVk9a5du/jNb37D5MmTtTZFMMD5yoRc3P4QNldbXmpbXvc/nJZDeko1kzP8pKeeZ5ZNEHM094icTiff/va3ef7558nIyNDaHMEAZ0R2GhOHOjB52jyinghRahb5jhQuyYGh6WK9WSLRXIjuu+8+Fi5cyPz58y/Y1ufz0dLSEvUQCDrFVac+d2d8KExqpvrsaYi9PYLzomnXbOPGjezZs4ddu3Z1q/3q1atZtWpVnK0SDFRavAFONXpwGEMU+FrVjalDun+AlEwUFILOOgL+YLerxAr6jmYeUWVlJf/1X//FK6+8gsVi6dZnVqxYQXNzc+RRWVkZZysFA4mqJi/v7K9mz8Fj6gZzWqSAYrdIzaS21ccnhyp49/OT8TFS0CmaSf7u3bupqalh+vTpkW2hUIjt27fzq1/9Cp/Ph14fPdthNpsxm82JNlUwQHC1Td075GZ1Q0+6ZQDGVPQmC+BCdjfF1DbB+dFMiObNm8fnn38ete3OO+9k3LhxPPDAAx1ESCC4EOGoantYiHrSLQOQJHTWLKBejBMlGM2EyGazMXHixKhtVquVrKysDtsFgu4QjiGyhppAD1izenwMg1UdsJY8jRdoKYglms+aCQSxIrLgNdCkbuipRwQY0tTP6L0NKIooPZ0o+tW0wNatW7U2QTCA8fhD6GU/lpATMEWWbfQEY5rqRZkDzfiCcqSyhyC+CI9IkDS4/CFSAo0Y9VLbjFnPgxINaVnoJUgJNuMT1TwSRr/yiASCvjBvXA7BU6dJqdP3qlsGQEomQ2xmQEan+AFjLE0UdIEQIkHSUDzECk1eaND1qlsGgNHCiLwstUy14gasMbVR0Dm96podP3481nYIBLHBXa8+p/Z8xixCSrr67G3uszmC7tErIRo1ahRz587l5Zdfxuv1xtomgaDHOH1BDla10FTXlh62tx4RoJgdBGUZv6s+RtYJLkSvhGjPnj1MnjyZZcuWkZeXx3e/+10+/vjjWNsmEHSb6mYv735WSeXpNiHq7RgRsKcWdpU3cvyEqHOWKHolRFOnTmXt2rWcPn2adevWUVVVxeWXX87EiRN56qmnqK2tjbWdAsF58bSfMTNZwZTa62PpUhwAyJ6mGFknuBB9mr43GAzcdNNNvP766zz++OMcO3aM5cuXU1hYyB133EFVVVWs7BQIzovbHyQ10IhR34eB6jb01ra8WF4RXZ0o+iREn3zyCT/4wQ/Iz8/nqaeeYvny5ZSWlrJ582ZOnz7NokWLYmWnQHBe3IEQKcEmVYj60C0D0EcGq1tARFcnhF5N3z/11FOsX7+ew4cPs2DBAl566SUWLFiATqfqWklJCRs2bKC4uDiWtgoEXeL2tXXNUqVerTFrj7HNI1ICXgh4+tTNE3SPXgnRc889x9KlS1myZAn5+Z2XasnJyeGFF17ok3ECQXdx+4Nkh7tmffSILGYLfn0qQdmnTuELIYo7vRKizZs3U1RUFPGAwiiKQmVlJUVFRZhMJhYvXhwTIwWCC+HzeTEHWzHq7T3PQ3QOFqMOn8FGMOgFbxPYu1EXTdAnejVGNHLkSOrq6jpsb2hooKSkpM9GCQQ9Ze4wPSOzraSk2frswVjNBrKycshMM4mgxgTRK4+oq/QITqez22lfBYJYMtTsApsFbDl9PpbVbODiscVQcRrEFH5C6JEQLVu2DABJkli5ciWpqWd/eUKhEB999BFTp06NqYECQbeIVO3o2/hQBLHMI6H0SIg+/fRTQPWIPv/8c0yms0XoTCYTU6ZMYfny5bG1UCC4AE5fkMZTlTi8Aex9HKgOo5jthGQZyd2IyEgUf3okRO+//z6g5pZeu3Ytdrs9LkYJBD3hTIuXyrIysvQeJsXII9q4v5Xi8kYmDAWHooAkxeS4gs7p1RjR+vXrY22HQNBrXB4P5qATo8kQs66Z3uJAQSIYDIDfCWZbTI4r6JxuC9FNN93Ehg0bsNvt3HTTTedt+8Ybb/TZMIGgu/iaawEFvdkKxtjE/JhNBvwGK8GQoo4TCSGKK90WIofDgdTmnjocjrgZJBD0lFCrushalzYkZl0oi1GPT28jKDerQuQYFpPjCjqn20LUvjsmumaC/kTIVYsO0Kf1LZCxPRajnmaDjaDcKGbOEkCvAho9Hg9utzvyvqKigqeffpp33303ZoYJBN1FcapT90ZbDIXI0BZdHe6aCeJKr4Ro0aJFvPTSSwA0NTUxc+ZMnnzySRYtWsRzzz0XUwMFggshuVUhMqfnxuyYatcsjaAshCgR9DpD4xVXXAHAH//4R/Ly8qioqOCll17imWee6fZxnnvuOSZPnozdbsdutzN79mzefvvt3pgkGKyEAlyUITMy24otKy9mh820mijIy8NmMQghSgC9mr53u93YbOoswrvvvstNN92ETqfjS1/6EhUVFd0+zrBhw1izZg2jR49GURRefPFFFi1axKeffspFF13UG9MEgw13PekpRjDaICV2cW2FmakUTh0NH6aczUskYoniRq+T57/11ltUVlayadMmrr76agBqamp6FOR4/fXXs2DBAkaPHs2YMWN47LHHSEtL48MPP+yNWYLBSGRpR3bshcJsB0kHclCNJRLEjV4J0cqVK1m+fDnFxcXMmjWL2bNnA6p3NG3atF4ZEgqF2LhxIy6XK3I8geBCtNZXUdPqpYHYR/krkkTA2FbXTHTP4kqvumbf+MY3uPzyy6mqqmLKlCmR7fPmzeNrX/taj471+eefM3v2bLxeL2lpabz55ptMmDCh07Y+nw+fzxd539LS0hvzBUlEY+1pTtW68NuMXBnD4/qCIZ7bWsr4ah/zCxT0IpYorvS60mteXh55edGDgzNnzuzxccaOHcvevXtpbm7mj3/8I4sXL2bbtm2ditHq1atZtWpVb00WJCEhpxrMqE/re/qP9pj0OiQkfAYbIdmrCpEgbvRKiFwuF2vWrGHLli3U1NQgy3LU/p5UgjWZTIwaNQqAiy++mF27drF27Vp+85vfdGi7YsWKSCoSUD2iwsLC3lyCIBkI+pHdaqUNkz22QiRJUiRTYyB0BpMQorjSKyG6++672bZtG7fffjv5+fmRpR+xQJblqO5Xe8xmM2azOWbnEgxw3HX4gzIBXQp2W+zHiCKxRCKoMe70Sojefvtt/v73v3PZZZf16eQrVqzguuuuo6ioiNbWVl599VW2bt3Kpk2b+nRcwSDBVYcvKOMxZWC3GGN+eItRh9NgIyTLQojiTK+EKCMjg8zMzD6fvKamJlKI0eFwMHnyZDZt2sRXvvKVPh9bkPworlr8wRBuc4YaeBhjLEY99YY0ggFFxBLFmV7dvUcffZSVK1fy4osvRqWL7Smi3JCgLwRaaggp4DFmkGaOvRCZDXr8eisBH2djiUQ6kLjQq7v35JNPUlpaSm5uLsXFxRiN0W7xnj17YmKcQHA+9J56xufbGDJqDAZ9n4oWd0q+w0JITsd0xgH4RF6iONIrIbrxxhtjbIZA0EOCPvT+VtJTTKTHqYTVlMJ0phSmw6c50FQp8hLFkV4J0UMPPRRrOwSCnuGsUZ/NaWBMie+5LA6gUgxYx5Fe+7NNTU387ne/Y8WKFTQ0NABql+zUqVMxM04g6BJnDc2eACeDdhpc/ridRpYV/Ma27pgQorjRK4/os88+Y/78+TgcDsrLy7nnnnvIzMzkjTfe4MSJE5FcRQJB3HCeoabVyz6XgZI6J5nWvs/insuJejdvfHqS0bKXhUaEEMWRXnlEy5YtY8mSJRw9ejSqsuuCBQvYvn17zIwTCLrEeQZfUMZlysIWhxgiALNRh6JAK2Lha7zplRDt2rWL7373ux22Dx06lOrq6j4bJRCcFzkErlp8ARmXMSsuwYwAFoNaWtEZEaK2WCJBzOmVEJnN5k5Xvh85coTs7NjlDRYIOsVdTygUxKPo8RnsOFLi5xEBOEkhpCDyEsWRXgnRDTfcwCOPPEIgEADUBYInTpzggQce4Otf/3pMDRQIOuA8gy8QwmXMwmzSYzHGPoYIwGzQIUmgSHqCxjR1o1eknokHvbqDTz75JE6nk+zsbDweD1deeSWjRo3CZrPx2GOPxdpGQYJ5+cMK/nngDCG5n3ZDnGfwBmTcpizSU0wxXXTdHnUFvto98xvEzFk86dWsmcPhYPPmzezYsYN9+/bhdDqZPn068+fPj7V9Ag3wBkJ8fqoZnQ6uGhe7yhgxw1mDNxjCZRxCRpy6ZWEsBh0efwifIQ0bCCGKEz0WIlmW2bBhA2+88Qbl5eVIkkRJSQl5eXkoihK3XydB4vjymGz+/lkV+yqbmZDvIM9hufCHEoWiQGsV2WlmrhwzEaMjI66nG55lJdsWQq9PBydCiOJEj7pmiqJwww03cPfdd3Pq1CkmTZrERRddREVFBUuWLOlxmlhB/2FnaT2nmjwoisKYXBvj89X8Ph+V1Wts2Tm46yHox2gyUThseNxFcu64HBZOzicjs20SRghRXOiRR7Rhwwa2b9/Oli1bmDt3btS+9957jxtvvJGXXnqJO+64I6ZGCuJLbauPD4/Xs6u8gXu/PAKLUc/MkkwOVbdwvNZFk9tPeqpJazNVWtoi9235oIvPIHWnWBzqsxCiuNCjO/n73/+e//mf/+kgQgBXXXUVDz74IK+88krMjBMkhmM16pR08RBrZHA202pieJaa4uXA6X40U9RyGkVR+MKZxoHTLQkZUJdlhYCxLQOkt1nEEsWBHgnRZ599xrXXXtvl/uuuu459+/b12ShBYqlscANQkmWN2n5RgeoFHKhqQekvX76W0/iCMp80pLDl4Bl0cR6S3F3RyDPvHeX9co+aFE0Ogt8V35MOQnrUNWtoaCA3t+tZlNzcXBobG/tslCBx+IIhqpq9ABRlRie5GzHEypVjsxmZndY/JiGCfnDV4gmEcJqySbfGb+o+jEmvLvPwBFFzEXlb2vISpcX1vIONHnlEoVAIg6Fr7dLr9QSDwT4bJUgcVU1eZEXBkWLEkRo9FW7Q65helBG3yOUe01oFioKTFPyGNLKs8R+3SjGpXxFvICTGieJIjzwiRVFYsmRJl5U0uqq+Iei/nGlRvaH8/jRF3xVNJwCoN6ilgzITIkTqV8TtD0GayEsUL3okRIsXL75gGzFjNrCoc6q5fHLDQiSHoPxf0FoN2WMhfwqHzrRyuLqV2SOyyLFrKFhtQnRGUocHEuIRtQ3ee4RHFFd6JETr16+Plx0CjVgwKY/ZI7MwG9p66Qf/CjUH1dcNxyHo5VhTEcdrXQxJM2snRKGgOmOGwkmGAAnyiNqEyBeQCZns6EEIURxIYCCGoD8iSRKZVhNWswHqS1URknSQe5HaoGw7Y9JUr+l4rYYrz1tOgRzEp0ulBTs6SUpIbJPFqItUEPKK9WZxI/Y1WAQDl5O71OehF8OoeRD0Qn0pxe7P0EnjqXP6tQtubKoAwDykmKUlI2jxBtDHe+4eVahHZqeh10ko5rZB+3AsUX+YSUwShBANYr443cyJejdj8myMTAtAQ5n65Rp2sfpcfDnUl2KqP8Rw21jKWiRKa11cPFwDIao/BoCUOQJHascZvnhy/ZQC9YUcio4lElP4MUPTrtnq1au55JJLsNls5OTkcOONN3L48GEtTRpUVDa4OVTdSr3TD7Vt/+/pwyGlbSGpvUAtnyOHGK9TB4pLteieeVug9YwqAlkjE3/+MDr92bpmonsWUzQVom3btnHffffx4YcfsnnzZgKBAFdffTUul4hcTQThGbOsNFPE4yB7THSjvIkAFAWOA3C6yYPHH0qYjcBZ2+wFbCl18uHx+oTbIMsKwZAsZs7ihKZds3feeSfq/YYNG8jJyWH37t18+ctf1siqwYEsKzS2leHJMoWg+aS6I/McjyN7HBzdTIqvgUKzEzk1G7c/SIpJnzhj24TIlz6Sz0pVAZhamJ6w0287UsunJxqZVZLFbFHjLC70qzGi5mb15mZmdl4axufzRQVNdpY3W9A9mj0BgrKCUS/h8Ferg6+pWZCSHt3QmAKZI6DuKF8b5kI/YnpiDfW71LEroM5cBHiwpxgji3MTgVEnqcs8AkHhEcWJfjN9L8sy999/P5dddhkTJ07stM3q1atxOByRR2FhYYKtTB7qXaqgZ1rNSGFvKL2L/88howHQNxxLhGnRnDkAigz2fKqD6uBwjq3zyP54Efb+PH7RNYsX/UaI7rvvPvbv38/GjRu7bLNixQqam5sjj8rKygRamFzUt40PZVpNZ7tlXdV1zxypDhS3VoOvFY8/RIs3EH8jFQWqP1Nf506KLEfRTIhEdHXc6Bddsx/+8If87W9/Y/v27Qwb1sWXAbWMUVfr3AQ9wxuU0UkSWSkS1J9RN3YlROY0NRFZy2kO7v+UTY15TCxwMH9CnPNZN1WoNe71BsgZz+kytWZeviPOte7PIbLMw39O10zEEsUMTT0iRVH44Q9/yJtvvsl7771HSUmJluYMKq4ck80PrxrFFIdHjY8xp4ElvesPtHXPsn0VKAocr3PGP0dR5cfqc95kWmQjrd4gOklKeA7tqPVmZrvISxQHNBWi++67j5dffplXX30Vm81GdXU11dXVeDweLc0aNOh1EiZPrfrGln/+X/esUQBk+quw6EK4fCGq27pKcaGxXF1yIkkw7BKa3QGMeolsmxmTIbF/tu3HiBRJJ2KJ4oCmQvTcc8/R3NzMnDlzyM/Pjzz+8Ic/aGnW4MLZ1i1Lu0A3y5oNFjs6JcSEFDX5XWlNnDyCUBCOblZfF0yH1EwKM1P5/pxRXD8lPz7nPA8pRj1FmamMzk1TU9OKcaKYo+kYUb9JPzrIqGn18t7BGvIcFuY41XEXbHnn/5AkqV7RqT2M0FezhyEcr3Ny+eghsTVOUeDYZnDVqaEDxZdHdul1ErY41bk/Hwa9jq9f3G78TMQSxZx+M2smSBwNLj9VzV5qm1zgaisXdCGPCCLBjnmBU+hQZ97CQZExQZahdAuc3qsK34QbwJTa/36whEcUc4QQDUKa3OrUe46+RY3RMaacHfc4HxnDQW/AGHAyyqom3D9ypjU2RjlrYM+LUNmWAWDUV9RASuBojZP1O8rYVd4Qm3P1kpBY5hE3+sX0vSCxhIUoiyZ1gy2ve9PQeiOkF0P9MaanNVBUVMLo3D6uQJdDULEDKnaqomgww5hrzuZDAirq3TS5A2q6Vo3YfOAMX5xu5stjspluE0IUa4QQDUKaPWp3KiPU5mF0p1sWJmsE1B8jP3iK/GFX9c0QdwMc+LMaKAnqgtvRV0d5Z4qiUFGvDoqfW2UkkRj16jIPty8E2SKWKNYIIRqEhD0iW7AH40NhskYB76oZE/1uMPVSHFpOw76NEPSB0QJjrlUX2J7zpW5yB2j1BtHrJIamJzaQsT1Ws/pVcfmDYM4QeYlijBgjGmR4AyG1i6PIpAbaPKILzZi1x+KAtGxQFEL1peytbOL1TyrxB+XuH8NZA/t+r4qQvQBm3AU54zv1LMrbvKF8hyXh8UPtSW2LJXL7gyIvURwQQjTI8PhDOFKMZOndGJSQunzifBHVndE2e6ZrKGXviUZONno4UNXNTAh+N+z/k1osMb0QpnwLLPYum5fWqkI0Iltbr8PaVlbI5Wsbpwr/n3lEQdFYIIRokJFhNbH08hK+M7Gtm5M6BHQ9/DNoi7KWGsuYMkwVkb0nGrs3zX70XfA0qelGJn4dDF2nnfUGQpxqVKPsR2Zbu2yXCFLN7TwigNS2VDUebWfykgUhRIMUnbtOfZGW0/MP24eqU/4BLxNSmzAZdDS6A5TXu8//uUiVEAkm3Kge4zwEZYVJw+wUD0nVJmF/O6ztCi3KsgIpbULkFkIUC4QQDVZcNeqzNbvnn9Xp1OKLgLn+EBcVqF7RJ+UNXXtFQT8c2aS+HnYJ2C+8VCPNbOCqcbl8bVrXGRkSRXiZx7g8G0FZOesRueu1NSxJEEI0yPjz3lO8tquSlroqdUNvhAjOxvnUHmL6MLXczslGD5UNXSxYLv9AHdi1OKD4it6dU0N0OomvXzyMayfmq4PmqVnqDk+DOoUv6BNCiAYZ1c1eqhpa0fma1A29FSJHoTrIHPRjd5YzaZgaW7PzeF1Hr6j1DJz8RH095przjguFqWr2UNng7n/LO8JYHGohylAQfDGKLh/ECCEaRPiC6tR9SqARi0FSY4B6GwMjSZDbltL31G4uKc6kZIiVy0dnI7WfhpdlOPK2GjWdM67b5YA+Ot7AH3ef5JOK/jUrFZIVAiFZncIP5/cW3bM+I4RoENHcFsiYSRMGna733lCYodPVL2TzSdJ8tdw4bWjHoMOTu6ClSl26MWp+tw7r9gepaBv4HqXxtH173jt0hme2HGVvZZO6oX33TNAnhBANIpo8qhBl69q6EtZezJi1x2xTo6EByrZFjZV4AyF1Rqlsu7ph1LzuLawFDlW3IisKeQ4LGVZtZ8vaY9KrU/hOX9sUfrgQpbt/eW0DESFEg4jw0o5MpUndYI1BLqHiy1WvqKEM6o4gywo7jtWxfvsRnHteU5dBZJZA3uRuHU5RFPafUqOVJ+R3HeioBda2WCKn95xYItE16zNCiAYRTW51satdblI39CaG6FxSM9XpeIBDf0NqOUljUwMjqv7B8bIyZKMVxl7X7YWhJxs91Dv9mAw6xuV3z4NKFOGkbGc9IhHUGCuEEA0iTAYdGcYAVryqMKTGKLti8RVqBZCgH+nTl7m66XWygtW0BHV8kjbnbP6ebhAefxmfb8NsSGA12W5gs6hBja3hUkrhMSJvszp7Jug1QogGEXPG5rBkipUhaWZ1rVQ3ptG7hd4Ak2+GvIkgSZj0OoqKR/FFzg38u85CeV33clsHQ3KkXtrkYemxsS2GhIXI7Q+puatNVvX/UFHEmrM+ItKADDacbVU7YjE+1B6DGcZfD6OvATlIjimVkQfO8PmpZv6xv4rbZhZdcJmGQa/jtplFVLd4VbHsZ6QY9eh1EiFZwekL4kgxql5ly2lw16lZCQS9QnhEgw1XmxDFYnyoMwymSI6iOWOzyXdY8AVk/vpZlepFXABJkhJeQLG7SJLE6Jw0xrcfRA+HQDhrtDEqSRBCNEg4VtPK89uPc/BYW/36vsYQdQODXsdXpxRgsxiYXpSOXtf5gHUgJLOrvKFnOY004rpJ+Vw7MU/1huDs/2NY4AW9QnTNBgmN7gBOrx+9px6slr7HEHWTNLOBO2YXRyU1UxQlEn0tywrvfnGGI2daOXrGybdmFkZHZvd3wt0xV522dgxwNPWItm/fzvXXX09BQQGSJPHWW29paU5S0+QOkBJsJkUvq0nwwzEwCaC9CDl9QV7+sIJ/l9ax/1Qzf9x9kiNnWtFJEleMHjIgRCgkK2rAJpz1iLxNaoYBQa/QVIhcLhdTpkzh2Wef1dKMQUGT24/VX4fFqFdzVGv0hf/oeD11Tj8fHW9g84EznGryYNRLfHVKPoUaJsfvLvtPNfPMlqO8e6CtQq7Jqo6JKYoIbOwDmnbNrrvuOq677jotTRg0NHsCZPrrsNh1PctRHWPmjs1haEYKpTUufMEQ2TYzUwrTsWtQwbU3pJjOia4G1SvyV6jjRN3IsyToiBgjGgQEQjKt3iCF/losBn3PqnbEGJ1OYlyenXF5/Wv5RncJxxKF450AVYgaK8SAdR8YUELk8/nw+XyR9y0t3UzYPshpcgdAUXCEGjDo0zT1iAY64dkyjz+ELxhSo7/DMVlCiHrNgJq+X716NQ6HI/IoLCzU2qQBgaIojEjzk25SkHTG2C3tGISYDfpI96y5LZtBZAZSxBL1mgElRCtWrKC5uTnyqKys1NqkAUGO3cKikXq1O5SW3fOqHYIo0tu8onB+J9Jy1MF/v0tka+wlA6prZjabMZv7X+j/gKDllPpsH6qtHUmAI8VIVbP3rEekN6oLYF11avnsbuZdEpxFUyFyOp0cC0f6AmVlZezdu5fMzEyKioo0tCy5CIZkDM1t3qMQoj5TmJmKJElktk/aZstvE6IqGDJaO+MGKJoK0SeffMLcuXMj75ctWwbA4sWL2bBhg0ZWJReyrPDb9w8x69RhJg21Y3JoX5pnoDNxqIOJQ89JbWLLh+rP1UIBgh6jqRDNmTOn/1ZpSBJavAHMnjMEQyEMqY7zlncW9AFbW0hEa5Ua3DgAIsT7E2LUMslpcPmx+aqxGPXohDcUM0KyQr3Tp1Z9hbZodZ0YsO4lQoiSnAaXH4evilSTXq1FJugziqLw/AfHeWlnBY1t6XfRG8HalrGxtVo74wYoQoiSnIZWZ8QjIrNEa3OSAkmSIoGNDa52C11tBepzy0kNrBrYCCFKcry1ZUiKjMWWebb8jaDPhGfM6tsLUbjr2yyEqKcIIUpiQrKCUl8GQGruaDGAGkOGpLUJkbOdEKW3dX1bqyEU6ORTgq4QQpTEBIIhxhuqcaQYSc0XsS2xJNOqBtY2uM6ufcSSrpbwlkNqHmtBtxFClMRYPGcYk64wYVgWUuYIrc1JKsJds0Z34GwubkkS3bNeIoQomak9qD4PGa3O6ghiht1iwGTQEZKVSOFK4OzMpBCiHjGg1poJekAoiOvEPlJQ0GWP19qapEOSJKYMS8eolzAb2xWCTG9bmtR8Qi26qBdfse4g/peSFLnmEF9UVOPTpTJpVhHpWhuUhFw+upN0KtZsdZzI54TmShEy0U1E1ywZURScx3cSkhXq7BNwpIqMBQlDkiA8HtdwXFtbBhBCiJKR+lLcdSeRJQO6oVMHRGWMgYiiKLR4Axw90xpdPFIIUY8RQpRsyCEo24rTG6DadhE5WSIbYzx5+cMK/vZZFfXOdtP4GcWqZ+SqA0+jZrYNJIQQJRuVH4OzlsaAgdO2KRSkW7S2KGmRJIk8u/r/e6rJc3aHMeXsoHXNIQ0sG3gIIUomWk5D+Qd4AyEOpc1ENqSQ5xBCFE/CtdgqGz3RO3LaZiprDiTYooGJEKJkwdcKX7wJcogz5uHUpY6mIN2iVpkQxI3CDFWITja6z6YEARgyVk0L4qwBlyi8eCGEECUDAQ/s2wjeFkjNJH3a1/jy2GwmD0vX2rKkJ8dmxmzU4QvInGn1nt1hSj07dV/9mTbGDSCEEA10gj747DV1YNScBpNvxmFP4+LhmYzNE0nc441OJ1HU1j0rrXFF78yfqj5X7VODGwVdIoRoIBPwwL7fq2NDRgtMvlWk+tCAMbmq4JfWOqN3ZI1SU/MGPGKs6AIIIRqo+F2w91VoqTorQmnZfHS8ngOnW/AHZa0tHDQUZ1n5yoRcbp5xTgZMnQ4KpqmvT3wIsrgnXSGEaCDia1VFyFkDJitM/TbY82n1BvjweAObvqiOrs0uiCsmg46JQx2RCrBRDL1Ync5314uxovMghGig4aqDT19uGxOyqSKUppY83lfZjKwoDMtIYUiaWNahBYqiREdZG8ww/FL1dfkHajdN0AEhRAOJxnLY8xJ4mtSxoGnfjiRsd/mC7DvZBMC0IjFOpAVldS5e/fgEHxytjd5RMB1SM9WFsEc3q+WGBFEIIRoIKIo6xrDvD+osmWMoTL89amD6g6O1+IMyeQ4LI7OtGho7eNFJUNPiY19lMzXtp/L1Bhi3UF32ceYL9V4KougXQvTss89SXFyMxWJh1qxZfPzxx1qb1H9w1aszY6XvgyJD7gSYcps6NtTG/lPNHKxqRZJgzthsschVI4ZnWRmZk4asKPz9syo8/tDZnY5hMGq++vr4VijfITyjdmguRH/4wx9YtmwZDz30EHv27GHKlClcc8011NTUaG2atjhr4fDbsOt30Fih/qqOvQ7G3xCVbMsbCPHB0ToAZo/IIt+RopXFAuAr43OxWQw0uQO8vruS2tZ2i2GHzYDhs9XXZdvVIFRRohoASdG45vOsWbO45JJL+NWvfgWALMsUFhbyH//xHzz44IPn/WxLSwsOh4Pm5mbs9gFaSlmWIehRp+NdtWoFiIbj6mB0mCGjYeRVKCkZ+IIyzZ4AKSY9doua/vVko5tjNU6uHCO8of5Ag8vPG3tO0uoNopMkJhc6mDs252yDU3vg2BaQ24Ic7QVq6hB7gdrdNttAZ0iKqivd/Y5qmqHR7/eze/duVqxYEdmm0+mYP38+O3fu7NDe5/Ph8539hWlpaeneiRrKoHRL565w1Dbl/Ns72dbqC3C4quXsJ8NtFPWfoRkpFLQtPHX5gnxxurndIWT0cgAJBUWBPIeFoemqR+MNKnzQlMEZ+0ScDQUo9Y2E5HoCIfX4XxqRxeyR6kD1sIxUhrWteRJoT6bVxK0zi3j/UA3Hapz4Amfjh4Ihmd8etZMamk9R0y4y3MeRlCbggJpTLdWkLqTV6QlKRvaeakVBB5KEIukACQUJJMiymhiVnQaArMCuigYAJg11kGpMwBrDybeoAZsxQFMhqqurIxQKkZubG7U9NzeXQ4c6pk9YvXo1q1at6vmJgj61qxMPvAEC7q4FUfaGIKXtDzEQJORzR+0PjyIEdWbcKflQUALpRXjMQ9m/q81tbz/WAKSY9MhifKFfk2Y2cP2UAmpavITa3atWbxBfQMaHlUb7HIypl5DhPYHDe5qUQCOpStsPrRwC+fx/W4piglR1UFxSFORWVYhw+sGUgK+2ErsAzQGVs3rFihUsW7Ys8r6lpYXCwm7Uc08vhCm3qq+j3F2p623d3G4MyhS1+tXd7baHu0g2iwEsaukZU0hmhNuP1P68hhQwWECnJ9VsALN6S2whmdtnW9qfFYNOR6pZj1Gv+dCeoJvk2KPTsNgsBpZcWkxAlgmGlLYflLERR9pq0oMFCPrQBbwUjXaqXrYiA0rbawVQsBh10FbWCKCoWBUlg80M+gR060yxm53VVIiGDBmCXq/nzJnoAbszZ86Ql5fXob3ZbMZs7kWgnskatyTmFmBoZvfaGoFcR/faGvQ6EZSYhBj0OjLaiUfXDc3oLHaG2nIu3Bb1d7C7f4f9EU1/Wk0mExdffDFbtmyJbJNlmS1btjB79mwNLRMIBIlE867ZsmXLWLx4MTNmzGDmzJk8/fTTuFwu7rzzTq1NEwgECUJzIbrllluora1l5cqVVFdXM3XqVN55550OA9gCgSB50TyOqC8kRRyRQJDEdPc7KqZfBAKB5gghEggEmiOESCAQaI7mg9V9ITy81e2lHgKBIKGEv5sXGooe0ELU2toK0L3oaoFAoBmtra04HF1H8w7oWTNZljl9+jQ2m61frjoPL0GprKxMylm9ZL6+ZL42SNz1KYpCa2srBQUF6HRdjwQNaI9Ip9MxbNgwrc24IHa7PSn/mMMk8/Ul87VBYq7vfJ5QGDFYLRAINEcIkUAg0BwhRHHEbDbz0EMP9S5jwAAgma8vma8N+t/1DejBaoFAkBwIj0ggEGiOECKBQKA5QogEAoHmCCESCASaI4Soh/S0Ku3rr7/OuHHjsFgsTJo0iX/84x9R+xVFYeXKleTn55OSksL8+fM5evRoPC+hS3pybc8//zxXXHEFGRkZZGRkMH/+/A7tlyxZgiRJUY9rr7023pfRJT25vg0bNnSw3WKJToQ/UO/dnDlzOlybJEksXLgw0ibh904RdJuNGzcqJpNJWbdunfLFF18o99xzj5Kenq6cOXOm0/Y7duxQ9Hq98rOf/Uw5cOCA8r//+7+K0WhUPv/880ibNWvWKA6HQ3nrrbeUffv2KTfccINSUlKieDyeRF2Woig9v7bbbrtNefbZZ5VPP/1UOXjwoLJkyRLF4XAoJ0+ejLRZvHixcu211ypVVVWRR0NDQ6IuKYqeXt/69esVu90eZXt1dXVUm4F67+rr66Oua//+/Yper1fWr18faZPoeyeEqAfMnDlTue+++yLvQ6GQUlBQoKxevbrT9jfffLOycOHCqG2zZs1Svvvd7yqKoiiyLCt5eXnKz3/+88j+pqYmxWw2K7///e/jcAVd09NrO5dgMKjYbDblxRdfjGxbvHixsmjRolib2it6en3r169XHA5Hl8dLpnv3i1/8QrHZbIrT6YxsS/S9E12zbhKuSjt//vzItvNVpQXYuXNnVHuAa665JtK+rKyM6urqqDYOh4NZs2Z1ecx40JtrOxe3200gECAzM7qmzdatW8nJyWHs2LF8//vfp76+Pqa2d4feXp/T6WT48OEUFhayaNEivvjii8i+ZLp3L7zwArfeeitWa3SdskTeOyFE3eR8VWmrq6s7/Ux1dfV524efe3LMeNCbazuXBx54gIKCgqgvxLXXXstLL73Eli1bePzxx9m2bRvXXXcdoVDoPEeKPb25vrFjx7Ju3Tr+/Oc/8/LLLyPLMpdeeiknT54Ekufeffzxx+zfv5+77747anui792AXn0v6B+sWbOGjRs3snXr1qgB3VtvvTXyetKkSUyePJmRI0eydetW5s2bp4Wp3Wb27NlRtfUuvfRSxo8fz29+8xseffRRDS2LLS+88AKTJk1i5syZUdsTfe+ER9RNelqVFiAvL++87cPPPTlmPOjNtYV54oknWLNmDe+++y6TJ08+b9sRI0YwZMgQjh071mebe0Jfri+M0Whk2rRpEduT4d65XC42btzIXXfddcHzxPveCSHqJr2pSjt79uyo9gCbN2+OtC8pKSEvLy+qTUtLCx999FFCK932tuLuz372Mx599FHeeecdZsyYccHznDx5kvr6evLz82Nid3eJRUXhUCjE559/HrF9oN87UENLfD4f3/nOdy54nrjfu4QNiycBGzduVMxms7JhwwblwIEDyr333qukp6dHpnVvv/125cEHH4y037Fjh2IwGJQnnnhCOXjwoPLQQw91On2fnp6u/PnPf1Y+++wzZdGiRZpNAffk2tasWaOYTCblj3/8Y9QUb2trq6IoitLa2qosX75c2blzp1JWVqb885//VKZPn66MHj1a8Xq9Cb223lzfqlWrlE2bNimlpaXK7t27lVtvvVWxWCzKF198EWkzUO9dmMsvv1y55ZZbOmzX4t4JIeohv/zlL5WioiLFZDIpM2fOVD788MPIviuvvFJZvHhxVPvXXntNGTNmjGIymZSLLrpI+fvf/x61X5Zl5Sc/+YmSm5urmM1mZd68ecrhw4cTcSkd6Mm1DR8+XAE6PB566CFFURTF7XYrV199tZKdna0YjUZl+PDhyj333NMhFieR9OT67r///kjb3NxcZcGCBcqePXuijjdQ752iKMqhQ4cUQHn33Xc7HEuLeyfSgAgEAs0RY0QCgUBzhBAJBALNEUIkEAg0RwiRQCDQHCFEAoFAc4QQCQQCzRFCJBAINEcIkWBAsWHDBtLT0yPvH374YaZOnRp5v2TJEm688caE2yXoG0KIBJ0SThX6ve99r8O+++67D0mSWLJkSVT7WAtAcXExTz/9dNS2W265hSNHjnT5mbVr17Jhw4bI+zlz5nD//ffH1C5B7BFCJOiSwsJCNm7ciMfjiWzzer28+uqrFBUVaWJTSkoKOTk5Xe53OBxRHpNgYCCESNAl06dPp7CwkDfeeCOy7Y033qCoqIhp06b16dideSo33nhjxMuaM2cOFRUV/OhHP4okb4eOXbNzae+ZLVmyhG3btrF27drIMcrKyhg1ahRPPPFE1Of27t2LJEkJT1EiUBFCJDgvS5cuZf369ZH369at484774z7ed944w2GDRvGI488QlVVFVVVVT0+xtq1a5k9ezb33HNP5BhFRUUdrglg/fr1fPnLX2bUqFGxugRBDxBCJDgv3/nOd/jXv/5FRUUFFRUV7Nixo1v5a/pKZmYmer0em81GXl5er5KNORwOTCYTqampkWPo9XqWLFnC4cOHIyV3AoEAr776KkuXLo31ZQi6iUgVKzgv2dnZLFy4kA0bNqAoCgsXLmTIkCFam9UnCgoKWLhwIevWrWPmzJn89a9/xefz8c1vflNr0wYtwiMSXJClS5eyYcMGXnzxxZh5DTqdjnMz0AQCgZgcuzvcfffdkYH49evXc8stt5Campqw8wuiEUIkuCDXXnstfr+fQCDANddcE5NjZmdnR437hEIh9u/fH9XGZDL1uWpEV8dYsGABVquV5557jnfeeUd0yzRGdM0EF0Sv13Pw4MHI665obm5m7969UduysrIoLCzs0Paqq65i2bJl/P3vf2fkyJE89dRTNDU1RbUpLi5m+/bt3HrrrZjN5l51CYuLi/noo48oLy8nLS2NzMxMdDpdZKxoxYoVjB49OqF5pgUdER6RoFvY7Xbsdvt522zdupVp06ZFPVatWtVp26VLl7J48WLuuOMOrrzySkaMGMHcuXOj2jzyyCOUl5czcuRIsrOze2X38uXL0ev1TJgwgezsbE6cOBHZd9ddd+H3+xMyCyg4PyJVrGDQ8sEHHzBv3jwqKys7FCgUJBYhRIJBh8/no7a2lsWLF5OXl8crr7yitUmDHtE1Eww6fv/73zN8+HCampr42c9+prU5AoRHJBAI+gHCIxIIBJojhEggEGiOECKBQKA5QogEAoHmCCESCASaI4RIIBBojhAigUCgOUKIBAKB5gghEggEmvP/Aaq811wy81CFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:55:58.966193Z", + "iopub.status.busy": "2024-03-22T20:55:58.965464Z", + "iopub.status.idle": "2024-03-22T20:55:59.192408Z", + "shell.execute_reply": "2024-03-22T20:55:59.191390Z" + }, + "papermill": { + "duration": 0.248328, + "end_time": "2024-03-22T20:55:59.194679", + "exception": false, + "start_time": "2024-03-22T20:55:58.946351", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnDklEQVR4nO3deVxU570/8M8My+CwuAQYlqCjeCuSBInwg2BiExuQJDXX1Cbyi2nAibX3qvyuyVwbSxcosQ1J44JtibRWYmLSSpPYpE2sikRyNSEuGK0rvkQRF0BQ2eMwzDy/P7xMMs4Agw6cOTOf9+vFC89zlvkejnw455kzz1EIIQSIiGRGKXUBRES3guFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLIkeXgVFRVBq9XCz88PycnJ2LdvX7/Lt7S0YMmSJQgPD4dKpcK3vvUtbN26dZiqJSJX4S3li5eWlkKv16O4uBjJyckoLCxEeno6qqurERoaarN8d3c30tLSEBoaivfeew+RkZE4d+4cRo0a5fBrms1mXLp0CYGBgVAoFE7cGyJyBiEE2tvbERERAaWyn/MrIaGkpCSxZMkSy7TJZBIRERGioKDA7vLr1q0TEyZMEN3d3bf8mufPnxcA+MUvfrn41/nz5/v9XVYIIc2oEt3d3VCr1XjvvffwxBNPWNqzsrLQ0tKCDz/80Gadxx57DGPGjIFarcaHH36IkJAQzJs3D8uXL4eXl5fd1zEYDDAYDJbp1tZWjB07FmfPnkVgYKDT98tVGI1G7Nq1CzNmzICPj4/U5ZATeMoxbW9vx/jx49HS0oKRI0f2uZxkl43Nzc0wmUzQaDRW7RqNBidPnrS7zpkzZ/DJJ5/gmWeewdatW3H69GksXrwYRqMReXl5dtcpKChAfn6+TXtlZSXUavXt74gLU6vV2Lt3r9RlkBN5wjHt6uoCgAG7dSTt8xoss9mM0NBQ/PGPf4SXlxcSEhJw8eJFvPbaa32GV05ODvR6vWW6ra0NUVFRmDlzJoKCgoar9GFnNBpRVlaGtLQ0t/4r7Uk85Zi2tbU5tJxk4RUcHAwvLy80NjZatTc2NiIsLMzuOuHh4fDx8bG6RJw8eTIaGhrQ3d0NX19fm3VUKhVUKpVNu4+Pj1v/B+jlKfvpSdz9mDq6b5LdKuHr64uEhASUl5db2sxmM8rLy5GSkmJ3nfvvvx+nT5+G2Wy2tJ06dQrh4eF2g4uI3Jek93np9XqsX78eb775Jk6cOIFFixahs7MTOp0OAJCZmYmcnBzL8osWLcLVq1exdOlSnDp1Ch9//DFefvllLFmyRKpdICKJSNrnlZGRgaamJuTm5qKhoQHx8fHYtm2bpRO/rq7O6j6PqKgobN++HS+88ALi4uIQGRmJpUuXYvny5VLtAhFJRPIO++zsbGRnZ9udV1FRYdOWkpKCL774YoirIiJXJ3l4EZGtrq4um1uGOr4y4PMjNRgdfAABI75+EyomJsbtb/uxh+FF5IJOnjyJhIQEu/N+c9N0VVUVpk6dOvRFuRiGF5ELiomJQVVVlVVbdX0L9O8eweqn7sGk8FFWy3oihheRC1Kr1TZnU8pzV6Da/RUm3z0F8ePukKgy1yH5kDhERLeC4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLfPSZm7j5Cct9PV0Z8NwnLJN7YXi5ib6esHzz05UBz33CMrkXhpebuPkJy309Xbl3WSK5Y3i5iZufsMynK5O7Y4c9EckSw4uIZInhRUSy5BLhVVRUBK1WCz8/PyQnJ2Pfvn19Lrtx40YoFAqrLz8/v2GslohcgeThVVpaCr1ej7y8PBw8eBBTpkxBeno6Ll++3Oc6QUFBqK+vt3ydO3duGCsmIlcgeXitXr0aCxcuhE6nQ2xsLIqLi6FWq1FSUtLnOgqFAmFhYZYvjUYzjBUTkSuQ9FaJ7u5uVFVVIScnx9KmVCqRmpqKysrKPtfr6OjAuHHjYDabMXXqVLz88su466677C5rMBhgMBgs021tbQAAo9EIo9HopD1xPT09PZbv7ryfnsRTjqmj+yZpeDU3N8NkMtmcOWk0GquPunzTpEmTUFJSgri4OLS2tmLlypWYNm0ajh07hjvvvNNm+YKCAuTn59u079ixw60/InO+AwC88cUXX+DiUamrIWfwlGPa1dXl0HKyu0k1JSUFKSkplulp06Zh8uTJ+MMf/oAVK1bYLJ+TkwO9Xm+ZbmtrQ1RUFGbOnImgoKBhqVkKh+uuAkcO4L777sOUsWOkLoecwFOOae/V0UAkDa/g4GB4eXmhsbHRqr2xsRFhYWEObcPHxwf33nsvTp8+bXe+SqWCSqWyaffx8YGPj8/gi5YJb29vy3d33k9P4inH1NF9k7TD3tfXFwkJCSgvL7e0mc1mlJeXW51d9cdkMuHIkSMIDw8fqjKJyAVJftmo1+uRlZWFxMREJCUlobCwEJ2dndDpdACAzMxMREZGoqCgAADw0ksv4b777sPEiRPR0tKC1157DefOncMPf/hDKXeDiIaZ5OGVkZGBpqYm5ObmoqGhAfHx8di2bZulE7+urg5K5dcniNeuXcPChQvR0NCA0aNHIyEhAZ9//jliY2Ol2gUikoDk4QUA2dnZyM7OtjuvoqLCanrNmjVYs2bNMFRFRK5M8ptUiYhuBcOLiGSJ4UVEssTwIiJZcokOeyICzjZ3otPQ0+f8mqZOy/feG1Zv5q/yxvhg/yGpz9UwvIhcwNnmTsxYWeHQsv/93pF+5+9a9pBHBBjDi8gF9J5xFWbEY2JogP1lvjLgo4pKzHooBf4jbD/ydvpyB54vPdTv2Zs7YXgRuZCJoQG4O3Kk3XlGoxENIcDUcaPd+rONjmJ4yVh/fSSO9I8AntVHQu6F4SVTjvaRDNQ/AnhOHwm5F4aXTA3URzJQ/wjgeX0k5F4YXjLXVx8J+0fI3fEmVSKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZImfbZQpg+k6lH4XcbatGko/2w9m9/T04FLPJZy4eqLPIXHOtnVA6XcRBtN1APbHkCJyVQwvmbrUeQ7+43+Hn+7rf7nXt73e73z/8cClzngkQOPE6oiGHsNLpiL8x6Hz7P/D2ox4RNsZEqenpwef7fkM9z9wf59nXjWXO7C09BAiZowb6nKJnI7hJVMqLz+Yr0difNAkxN5hf0ics95nMXnM5D6HxDFfb4X5ehNUXn5DXS6R07HDnohkiWdeRC5goDdggIHfhPG0N2AYXkQuwNE3YID+34TxpDdgGF5ELmCgN2CAgd+E8bQ3YBheRC5goDdggIHfhPG0N2AG3WF/5syZoaiDiGhQBh1eEydOxIwZM/D222/j+vXrTimiqKgIWq0Wfn5+SE5Oxr59Dlz4A9i8eTMUCgWeeOIJp9RBRPIx6PA6ePAg4uLioNfrERYWhv/4j/9wOGzsKS0thV6vR15eHg4ePIgpU6YgPT0dly9f7ne92tpaLFu2DNOnT7/l1yYi+Rp0eMXHx2Pt2rW4dOkSSkpKUF9fjwceeAB33303Vq9ejaampkFtb/Xq1Vi4cCF0Oh1iY2NRXFwMtVqNkpKSPtcxmUx45plnkJ+fjwkTJgx2F4jIDdxyh723tzfmzJmD7373u3j99deRk5ODZcuW4ac//Snmzp2LV199FeHh4f1uo7u7G1VVVcjJybG0KZVKpKamorKyss/1XnrpJYSGhmLBggXYvXt3v69hMBhgMBgs021tbQBudH4ajUZHdtUl9fT0WL7b24/etv72caBt0PBx5FgMdEzd5Xg6Wvsth9eBAwdQUlKCzZs3w9/fH8uWLcOCBQtw4cIF5OfnY/bs2QNeTjY3N8NkMkGjsb4nRaPR4OTJk3bX2bNnDzZs2IBDhw45VGdBQQHy8/Nt2nfs2AG1Wu3QNlzR+Q4A8MaePXtwzv476wCAsrKy294GDb3BHIu+jqm7HM+uri6Hlht0eK1evRpvvPEGqqur8dhjj+Gtt97CY489BqXyxhXo+PHjsXHjRmi12sFuekDt7e149tlnsX79egQHBzu0Tk5ODvR6vWW6ra0NUVFRmDlzJoKCgpxe43A5dqkNK498gQceeAB3Rdjuh9FoRFlZGdLS0vr8bONA26Dh48ixGOiYusvx7L06Gsigw2vdunV47rnnMH/+/D4vC0NDQ7Fhw4YBtxUcHAwvLy80NjZatTc2NiIsLMxm+ZqaGtTW1uLxxx+3tJnNZgA3LmOrq6sRHR1ttY5KpYJKpbLZlo+PT5+/1HLQe5Oit7d3v/vR3346ug0aeoM5Fn0dU3c5no7WPujwKisrw9ixYy1nWr2EEDh//jzGjh0LX19fZGVlDbgtX19fJCQkoLy83HK7g9lsRnl5ObKzs22Wj4mJwZEjR6zafv7zn6O9vR1r165FVFTUYHeHiGRq0OEVHR2N+vp6hIaGWrVfvXoV48ePh8lkGtT29Ho9srKykJiYiKSkJBQWFqKzsxM6nQ4AkJmZicjISBQUFMDPzw9333231fqjRo0CAJt2InJvgw4vIYTd9o6ODvj5Df5jCRkZGWhqakJubi4aGhoQHx+Pbdu2WTrx6+rqbM7yiIgcDq/eTm+FQoHc3Fyrd+pMJhP27t2L+Pj4WyoiOzvb7mUiAFRUVPS77saNG2/pNYlI3hwOry+//BLAjTOvI0eOwNfX1zLP19cXU6ZMwbJly5xfIRGRHQ6H165duwAAOp0Oa9eulfVtBkQkf4Pu83rjjTeGog4iokFxKLzmzJmDjRs3IigoCHPmzOl32S1btjilMCKi/jgUXiNHjoRCobD8m4hIag6F1zcvFXnZSESugDdQEZEsOXTmde+991ouGwdy8ODB2yqIiMgRDoUXh1kmIlfjUHjl5eUNdR1ERIPCPi8ikiWHzrzGjBmDU6dOITg4GKNHj+63/+vq1atOK46IqC8OhdeaNWsQGBho+bejnfdEREPFofD65sCC8+fPH6paiIgcNug+Ly8vL7vPVLxy5Qq8vLycUhQR0UAGHV59DUZoMBishskhIhpKDo8q8dvf/hbAjcEI//SnPyEg4OtnK5lMJvzP//wPYmJinF8hEZEdDofXmjVrANw48youLra6RPT19YVWq0VxcbHzKyQissPh8Dp79iwAYMaMGdiyZQtGjx49ZEUREQ1k0IMR9o6oSkQkpUGH13PPPdfv/JKSklsuhojIUYMOr2vXrllNG41GHD16FC0tLfjOd77jtMKIiPoz6PD629/+ZtNmNpuxaNEiREdHO6UoIqKBOOWD2UqlEnq93vKOJBHRUHPaqBI1NTXo6elx1uaIiPo16MvG3idn9xJCoL6+Hh9//LHVZyCJiIbSoMOr98nZvZRKJUJCQrBq1aoB34kkInIW3udFRLLEkVSJSJYYXkQkSwwvIpIlhhcRyZLTwuvChQv40Y9+5KzNERH1y2nhdeXKFWzYsOGW1i0qKoJWq4Wfnx+Sk5Oxb9++PpfdsmULEhMTMWrUKPj7+yM+Ph6bNm261bKJSKYkv2wsLS2FXq9HXl4eDh48iClTpiA9Pd3uOPnAjcew/exnP0NlZSX+9a9/QafTQafTYfv27cNcORFJSfLwWr16NRYuXAidTofY2FgUFxdDrVb3ObTOQw89hO9973uYPHkyoqOjsXTpUsTFxWHPnj3DXDkRSWnQN6k6U3d3N6qqqpCTk2NpUyqVSE1NRWVl5YDrCyHwySefoLq6Gq+++qrdZQwGAwwGg2W6ra0NwI2hfIxG423ugXR6P0fa09Njdz962/rbx4G2QcPHkWMx0DF1l+PpaO0Oh9ecOXP6nd/S0uLopiyam5thMpmg0Wis2jUaDU6ePNnneq2trYiMjITBYICXlxdef/11pKWl2V22oKAA+fn5Nu07duyAWq0edM2u4nwHAHhjz549OBfQ93JlZWW3vQ0aejVtAOCNzds/w53+9p/QZTQDVw3AmffL4GPnmqnxKwUAL9kfz66uLoeWczi8Ro4cOeD8zMxMRzd3WwIDA3Ho0CF0dHSgvLwcer0eEyZMwEMPPWSzbE5OjtWHydva2hAVFYWZM2ciKChoWOodCscutWHlkS/wwAMP4K4I2/0wGo0oKytDWloafHx8bmkbNHz+euACcOw4Np+5/Wefpj/8ILR3+DuhKmn0Xh0NxOHweuONN265mL4EBwfDy8sLjY2NVu2NjY0ICwvrcz2lUomJEycCAOLj43HixAkUFBTYDS+VSgWVSmXT7uPj0+cvtRx4e3tbvve3H/3tp6PboKH3aFwkvLy8EB0agBE+9gOsur4V//3eEax68h5MCrd/MuGv8sb4YPkGFwCH/y9K2ufl6+uLhIQElJeX44knngBwY1TW8vJyZGdnO7wds9ls1a9FJDdj/H3xf5PG9rtMb59WdIg/7o7s/0rIEzgcXo4OdzPYB3Do9XpkZWUhMTERSUlJKCwsRGdnJ3Q6HQAgMzMTkZGRKCgoAHCjDysxMRHR0dEwGAzYunUrNm3ahHXr1g3qdYlI3hwOr40bN2LcuHG49957IYT9DsVbkZGRgaamJuTm5qKhoQHx8fHYtm2bpRO/rq4OSuXXvZOdnZ1YvHgxLly4gBEjRiAmJgZvv/02MjIynFYTEbk+h8Nr0aJF+Mtf/oKzZ89Cp9PhBz/4AcaMGeOUIrKzs/u8TKyoqLCa/tWvfoVf/epXTnldIpIvh29SLSoqQn19PV588UX84x//QFRUFObOnYvt27c79UyMiMgRg7rDXqVS4emnn0ZZWRmOHz+Ou+66C4sXL4ZWq0VHR8dQ1UhEZOOWPx6kVCqhUCgghIDJZHJmTUREAxrUrRIGgwFbtmxBSUkJ9uzZg1mzZuH3v/89HnnkEatOdRp6Xxlv/ME4erHV7vzOrww40ASEnbsG/xG297kBwOnLPFsm+XI4vBYvXozNmzcjKioKzz33HP7yl78gODh4KGujftT8b/D8ZMuRfpbyxqbT+wfclr9K0tv9iG6Jw/9ri4uLMXbsWEyYMAGffvopPv30U7vLbdmyxWnFUd9m3nXjEwh93ZHtyN3YgHvckU2eyeHwyszMhEKhGMpaaBAGuiObd2OTuxvUTapERK6CvexEJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsuEV5FRUXQarXw8/NDcnIy9u3b1+ey69evx/Tp0zF69GiMHj0aqamp/S5PRO5J8vAqLS2FXq9HXl4eDh48iClTpiA9PR2XL1+2u3xFRQWefvpp7Nq1C5WVlYiKisLMmTNx8eLFYa6ciKQkeXitXr0aCxcuhE6nQ2xsLIqLi6FWq1FSUmJ3+XfeeQeLFy9GfHw8YmJi8Kc//Qlmsxnl5eXDXDkRSclbyhfv7u5GVVUVcnJyLG1KpRKpqamorKx0aBtdXV0wGo0YM2aM3fkGgwEGg8Ey3dbWBgAwGo0wGo23Ub1r6+npsXx35/30JJ5yTB3dN0nDq7m5GSaTCRqNxqpdo9Hg5MmTDm1j+fLliIiIQGpqqt35BQUFyM/Pt2nfsWMH1Gr14IuWifMdAOCNL774AhePSl0NOYOnHNOuri6HlpM0vG7XK6+8gs2bN6OiogJ+fn52l8nJyYFer7dMt7W1WfrJgoKChqvUYXe47ipw5ADuu+8+TBlr/6yU5MVTjmnv1dFAJA2v4OBgeHl5obGx0aq9sbERYWFh/a67cuVKvPLKK9i5cyfi4uL6XE6lUkGlUtm0+/j4wMfH59YKlwFvb2/Ld3feT0/iKcfU0X2TtMPe19cXCQkJVp3tvZ3vKSkpfa73m9/8BitWrMC2bduQmJg4HKUSkYuR/LJRr9cjKysLiYmJSEpKQmFhITo7O6HT6QAAmZmZiIyMREFBAQDg1VdfRW5uLv785z9Dq9WioaEBABAQEICAgADJ9oOIhpfk4ZWRkYGmpibk5uaioaEB8fHx2LZtm6UTv66uDkrl1yeI69atQ3d3N5588kmr7eTl5eGXv/zlcJZORBKSPLwAIDs7G9nZ2XbnVVRUWE3X1tYOfUFE5PIkv0mViOhWMLyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkySWGxKHb19XVZfXQkur6FhgaTuPE0REwXxlltWxMTIxbP3yEPAPDy02cPHkSCQkJNu3z3rRdtqqqClOnTh2GqoiGDsPLTcTExKCqqsoy3fGVAR/vqsR3Z6QgYITKZlkiuWN4uQm1Wm11NmU0GnGt+TJSkhLd+kkz5LnYYU9EssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJkuThVVRUBK1WCz8/PyQnJ2Pfvn19Lnvs2DF8//vfh1arhUKhQGFh4fAVSkQuRdLBCEtLS6HX61FcXIzk5GQUFhYiPT0d1dXVCA0NtVm+q6sLEyZMwFNPPYUXXnhBgoqJhsfNzyQA+n4ugcc+k0BIKCkpSSxZssQybTKZREREhCgoKBhw3XHjxok1a9YM+jVbW1sFANHa2jrodeWku7tbfPDBB6K7u1vqUugWVFVVCQAOfVVVVUldrlM5+jsq2ZlXd3c3qqqqkJOTY2lTKpVITU1FZWWl017HYDDAYDBYptva2gDcGCbZaDQ67XVcTe++ufM+urPo6Gjs3bvXqq3jKwO2796P9On/x+q5BNHR0W51nB3dF8nCq7m5GSaTCRqNxqpdo9HYnC7fjoKCAuTn59u079ixwyNOtcvKyqQugZxo2j3RaG+5ivaWr9vq6+slq2codHV1ObSc2z+AIycnB3q93jLd1taGqKgozJw5E0FBQRJWNrSMRiPKysqQlpbGB3C4CU85pr1XRwORLLyCg4Ph5eWFxsZGq/bGxkaEhYU57XVUKhVUKpVNu4+Pj1v/B+jlKfvpSdz9mDq6b5LdKuHr64uEhASUl5db2sxmM8rLy5GSkiJVWUQkE5JeNur1emRlZSExMRFJSUkoLCxEZ2cndDodACAzMxORkZEoKCgAcKOT//jx45Z/X7x4EYcOHUJAQAAmTpwo2X4Q0fCTNLwyMjLQ1NSE3NxcNDQ0ID4+Htu2bbN04tfV1UGp/Prk8NKlS7j33nst0ytXrsTKlSvx4IMPoqKiYrjLJyIJSd5hn52djezsbLvzbg4krVYLIcQwVEVErk7yjwcREd0Kyc+8hlvvmZujb8fKldFoRFdXF9ra2tz6nSlP4inHtPd3c6CrLI8Lr/b2dgBAVFSUxJUQUX/a29sxcuTIPucrhId1IpnNZly6dAmBgYFQKBRSlzNkem/GPX/+vFvfjOtJPOWYCiHQ3t6OiIgIqzfsbuZxZ15KpRJ33nmn1GUMm6CgILf+j+6JPOGY9nfG1Ysd9kQkSwwvIpIlhpebUqlUyMvLs/u5TpInHlNrHtdhT0TugWdeRCRLDC8ikiWGFxHJEsNLYg899BCef/55qcsgkh2Gl0xUVFRAoVCgpaVF6lJoEFztj5Or1XM7GF5ELq67u1vqElwSw8uFGAwGLF++HFFRUVCpVJg4cSI2bNiA2tpazJgxAwAwevRoKBQKzJ8/f8Dttbe345lnnoG/vz/Cw8OxZs0am7+8mzZtQmJiIgIDAxEWFoZ58+bh8uXLlvm9Z3zl5eVITEyEWq3GtGnTUF1d7ezddzvz58/Hp59+irVr10KhUEChUKCmpgYLFizA+PHjMWLECEyaNAlr1661We+JJ57Ar3/9a0RERGDSpEkAgM8//xzx8fHw8/NDYmIiPvjgAygUChw6dMiy7tGjR/Hoo48iICAAGo0Gzz77LJqbm/usp7a2drh+HM43xM+PpAE8+OCDYunSpUIIIebOnSuioqLEli1bRE1Njdi5c6fYvHmz6OnpEe+//74AIKqrq0V9fb1oaWkZcNs//OEPxbhx48TOnTvFkSNHxPe+9z0RGBhoeT0hhNiwYYPYunWrqKmpEZWVlSIlJUU8+uijlvm7du0SAERycrKoqKgQx44dE9OnTxfTpk1z9o/C7bS0tIiUlBSxcOFCUV9fL+rr68X169dFbm6u2L9/vzhz5ox4++23hVqtFqWlpZb1srKyREBAgHj22WfF0aNHxdGjR0Vra6sYM2aM+MEPfiCOHTsmtm7dKr71rW8JAOLLL78UQghx7do1ERISInJycsSJEyfEwYMHRVpampgxY0af9fT09Ejxo3EKhpfEesOrurpaABBlZWV2l+sNkWvXrjm03ba2NuHj4yPeffddS1tLS4tQq9VW4XWz/fv3CwCivb3d6nV37txpWebjjz8WAMRXX33lUC2e7Jt/nPqyZMkS8f3vf98ynZWVJTQajTAYDJa2devWiTvuuMPqZ75+/Xqr8FqxYoWYOXOm1bbPnz9v+aPnaD1ywctGF3Ho0CF4eXnhwQcfdMr2zpw5A6PRiKSkJEvbyJEjLZcgvaqqqvD4449j7NixCAwMtLx+XV2d1XJxcXGWf4eHhwOA1eUlOa6oqAgJCQkICQlBQEAA/vjHP9r8vO+55x74+vpapqurqxEXFwc/Pz9L2zePLQAcPnwYu3btQkBAgOUrJiYGAFBTUzOEeyQNjxsSx1WNGDFi2F+zs7MT6enpSE9PxzvvvIOQkBDU1dUhPT3dppP4myN39o6DZjabh7Ved7B582YsW7YMq1atQkpKCgIDA/Haa69h7969Vsv5+/sPetsdHR14/PHH8eqrr9rM6/2D404YXi7innvugdlsxqefforU1FSb+b1/hU0mk0PbmzBhAnx8fLB//36MHTsWANDa2opTp07h29/+NgDg5MmTuHLlCl555RXLyLIHDhxwxu7Q//L19bU6Zp999hmmTZuGxYsXW9ocOSuaNGkS3n77bRgMBssHs/fv32+1zNSpU/H+++9Dq9XC29v+r/bN9cgZLxtdhFarRVZWFp577jl88MEHOHv2LCoqKvDXv/4VADBu3DgoFAp89NFHaGpqQkdHR7/bCwwMRFZWFn784x9j165dOHbsGBYsWAClUmk5cxo7dix8fX3xu9/9DmfOnMHf//53rFixYsj31ZNotVrs3bsXtbW1aG5uxr/927/hwIED2L59O06dOoVf/OIXNiFkz7x582A2m/GjH/0IJ06cwPbt27Fy5UoAX58JL1myBFevXsXTTz+N/fv3o6amBtu3b4dOp7ME1s31yPnsmeHlQtatW4cnn3wSixcvRkxMDBYuXIjOzk4AQGRkJPLz8/GTn/wEGo2mz8fFfdPq1auRkpKCWbNmITU1Fffffz8mT55s6TcJCQnBxo0b8e677yI2NhavvPKK5ReCnGPZsmXw8vJCbGwsQkJCkJ6ejjlz5iAjIwPJycm4cuWK1VlYX4KCgvCPf/wDhw4dQnx8PH72s58hNzcXACzHMyIiAp999hlMJhNmzpyJe+65B88//zxGjRplGU755npu7muTEw6J40E6OzsRGRmJVatWYcGCBVKXQ7fpnXfegU6nQ2trqyR9plJjn5cb+/LLL3Hy5EkkJSWhtbUVL730EgBg9uzZEldGt+Ktt97ChAkTEBkZicOHD2P58uWYO3euRwYXwPCSrbq6OsTGxvY5//jx4wCAlStXorq6Gr6+vkhISMDu3bsRHBw8XGWSEzU0NCA3NxcNDQ0IDw/HU089hV//+tdSlyUZXjbKVE9PT78f7ejvHScid8DwIiJZ4ruNRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwoiE1f/58y6idPj4+0Gg0SEtLQ0lJyaA+V7dx40aMGjVq6ArtQ++opuR6GF405B555BHU19ejtrYW//znPzFjxgwsXboUs2bNQk9Pj9TlkVxJORIiub+srCwxe/Zsm/by8nIBQKxfv14IIcSqVavE3XffLdRqtbjzzjvFokWLbEZz/eZXXl6eEEKIt956SyQkJIiAgACh0WjE008/LRobGy2vc/XqVTFv3jwRHBws/Pz8xMSJE0VJSYllfl1dnXjqqafEyJEjxejRo8W///u/i7NnzwohhMjLy7N53V27dg3Jz4kGj2deJInvfOc7mDJlCrZs2QIAUCqV+O1vf4tjx47hzTffxCeffIIXX3wRADBt2jQUFhYiKCgI9fX1qK+vx7JlywAARqMRK1aswOHDh/HBBx+gtrbW6uEkv/jFL3D8+HH885//xIkTJ7Bu3TrLx6OMRiPS09MRGBiI3bt347PPPkNAQAAeeeQRdHd3Y9myZZg7d67lzLG+vh7Tpk0b3h8U9U3q9CT31teZlxBCZGRkiMmTJ9ud9+6774o77rjDMv3GG2+IkSNHDvh6N4/B//jjjwudTmd32U2bNolJkyYJs9lsaTMYDGLEiBFi+/btA9ZP0uKZF0lGCGEZSG/nzp14+OGHERkZicDAQDz77LO4cuUKurq6+t3GQGPwL1q0CJs3b0Z8fDxefPFFfP7555Z1Dx8+jNOnTyMwMNAy5vuYMWNw/fp1txzz3d0wvEgyJ06cwPjx41FbW4tZs2YhLi4O77//PqqqqlBUVASg/weu9o7BHxQUhHfeeQf79+/H3/72N6v1Hn30UZw7dw4vvPACLl26hIcffthyydnR0YGEhAQcOnTI6uvUqVOYN2/eEO893S4OO0CS+OSTT3DkyBG88MILqKqqgtlsxqpVqywjfvYOf93L3tjrjo7BHxISgqysLGRlZWH69On48Y9/jJUrV2Lq1KkoLS1FaGgogoKC7NbpTmO+uxueedGQMxgMaGhowMWLF3Hw4EG8/PLLmD17NmbNmoXMzExMnDgRRqPRMpb+pk2bUFxcbLUNrVaLjo4OlJeXo7m5GV1dXQ6NwZ+bm4sPP/wQp0+fxrFjx/DRRx9h8uTJAIBnnnkGwcHBmD17Nnbv3m15bsB//dd/4cKFC5bX/de//oXq6mo0NzfDaDQOzw+NBiZ1pxu5t6ysLMttBt7e3iIkJESkpqaKkpISYTKZLMutXr1ahIeHixEjRoj09HTx1ltv2Txk9z//8z/FHXfcYXWrxJ///Geh1WqFSqUSKSkp4u9//7vNg1gnT54sRowYIcaMGSNmz54tzpw5Y9lmfX29yMzMFMHBwUKlUokJEyaIhQsXitbWViGEEJcvXxZpaWkiICCAt0q4GI7nRUSyxMtGIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hk6f8DIS+djrxxQtUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:55:59.232725Z", + "iopub.status.busy": "2024-03-22T20:55:59.232388Z", + "iopub.status.idle": "2024-03-22T20:55:59.521439Z", + "shell.execute_reply": "2024-03-22T20:55:59.520399Z" + }, + "papermill": { + "duration": 0.310375, + "end_time": "2024-03-22T20:55:59.523531", + "exception": false, + "start_time": "2024-03-22T20:55:59.213156", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCGElEQVR4nO3deVxU5f4H8M9hmWHflFUG3EFcEBcMvaUWhmaL1S0zc2/xppWZJVppaoXdNP2lpde8SnUzszQ1MwtxX1MWRRAURVlkU2SHmWHm/P4YOMzADMwZZuf7fr3mpXPmnDPfAzNfnuc5z8KwLMuCEEIsjI2pAyCEEF1Q8iKEWCRKXoQQi0TJixBikSh5EUIsEiUvQohFouRFCLFIlLwIIRbJztQBGJtcLsedO3fg6uoKhmFMHQ4hpAWWZVFVVYWAgADY2GguX3W65HXnzh2IRCJTh0EIaUdeXh4CAwM1vt7pkperqysAxQ/Gzc3NxNEQQlqqrKyESCTivquadLrk1VRVdHNzo+RFiBlrr1mHGuwJIRaJkhchxCJR8iKEWKRO1+ZFDItlWTQ0NEAmk5k6FGKmbG1tYWdn1+GuSpS8iN5IJBIUFhaitrbW1KEQM+fk5AR/f38IBAKdz0HJi+iFXC5HTk4ObG1tERAQAIFAQJ2ASSssy0IikaC0tBQ5OTno06dPmx1R20LJy4qJG2RYsDMV/+jTFVNHBBv0vSQSCeRyOUQiEZycnAz6XsSyOTo6wt7eHrdv34ZEIoGDg4NO56EGeyv2S1I+/rhShPd/vWK099T1ryjpXPTxOaFPmhWrrGswdQiEGAwlL0KIRaLkRTq9MWPGYMGCBaYOw2ocO3YMDMOgvLzcoO9DycuK0c0+/TLWl5Joh5IXIaQViURi6hDaRcmLGAzLsqiVNJjkoetC8GKxGIsXL4ZIJIJQKETv3r3x3//+F7du3cLYsWMBAJ6enmAYBjNnzmz3fFVVVZg6dSqcnZ3h7++PdevWtaqmfv/99xg2bBhcXV3h5+eHF198ESUlJdzrTSW+xMREDBs2DE5OThg5ciSysrK0vq6PP/4YPj4+cHV1xcsvv4zY2FgMHjyYe33mzJmYNGkSPvnkEwQEBCAkJESr2ADg4MGD6Nu3LxwdHTF27FjcunVL67g6gvp5WTFT1xrrpDKELfvTJO+dsTIGTgL+H+/p06fj7Nmz+PLLLxEeHo6cnBzcvXsXIpEIu3fvxrPPPousrCy4ubnB0dGx3fMtXLgQp0+fxv79++Hr64tly5YhOTlZJXFIpVKsWrUKISEhKCkpwcKFCzFz5kwcPHhQ5Vzvv/8+1q5dC29vb8ydOxezZ8/G6dOn243hhx9+wCeffIKvv/4ao0aNws6dO7F27Vr06NFDZb/ExES4ubkhISFB69jy8vLwzDPPYN68eXj11Vdx8eJFvPPOO+3GpA+UvAhpdO3aNezatQsJCQmIjo4GAPTs2ZN73cvLCwDg4+MDDw+Pds9XVVWFb7/9Fjt27MAjjzwCANi+fTsCAgJU9ps9ezb3/549e+LLL7/E8OHDUV1dDRcXF+61Tz75BKNHjwYAxMbGYuLEiaivr2+3k+eGDRswZ84czJo1CwCwbNky/PXXX6iurlbZz9nZGVu3blUZstNebJs2bUKvXr2wdu1aAEBISAjS0tLw2Weftfvz6SiTJq+4uDjs2bMHmZmZcHR0xMiRI/HZZ59xRVZ14uPjuV9CE6FQiPr6ekOHS3hytLdFxsoYk703X6mpqbC1teUSREfdvHkTUqkUkZGR3DZ3d/dWn++kpCR89NFHuHTpEu7fvw+5XA4AyM3NRVhYGLffoEGDuP/7+/sDAEpKShAUFNRmHFlZWXj99ddVtkVGRuLIkSMq2wYOHNhqrGF7sV29ehUjRoxQOSYqKqrNePTFpMnr+PHjmDdvHoYPH46GhgYsXboUjz76KDIyMuDs7KzxODc3N5X6Po2hM08Mw+hUdTMVbaqB+lZTU4OYmBjExMTghx9+gLe3N3JzcxETE9Oq0dze3p77f9NnvimZ6EPL7xyf2EzBpJ+sQ4cOqTyPj4+Hj48PkpKS8NBDD2k8jmEY+Pn5GTo8i0c5nZ+BAwdCLpfj+PHjXLVRWVOpRNvpfnr27Al7e3tcuHCBKx1VVFTg2rVr3Oc7MzMT9+7dw+rVq7mFYS5evKiPy+GEhITgwoULmD59OrftwoUL7R6nTWz9+vXD/v37VbadO3dOD1G3z6zuNlZUVABoblvQpLq6GsHBwRCJRHjqqaeQnp6ucV+xWIzKykqVByHqdO/eHTNmzMDs2bOxd+9e5OTk4NixY9i1axcAIDg4GAzD4MCBAygtLW3VZtSSq6srZsyYgXfffRdHjx5Feno65syZAxsbG67kFBQUBIFAgA0bNuDmzZvYv38/Vq1apdfreuONN/Df//4X3377La5fv46PP/4Yly9fbrfGok1sc+fOxfXr1/Huu+8iKysLO3bsQHx8vF7j18RskpdcLseCBQswatQoDBgwQON+ISEh2LZtG/bt24f//e9/kMvlGDlyJPLz89XuHxcXB3d3d+5By56RtmzatAn//Oc/8frrryM0NBSvvPIKampqAADdunXDihUrEBsbC19fX8yfP7/d833xxReIiorC448/jujoaIwaNQr9+vXjGtm9vb0RHx+Pn3/+GWFhYVi9ejXWrFmj12uaOnUqlixZgkWLFmHIkCHIycnBzJkz223o1ya2oKAg7N69G3v37kV4eDg2b96MTz/9VK/xa8Saiblz57LBwcFsXl4er+MkEgnbq1cv9oMPPlD7en19PVtRUcE98vLyWABsRUWFPsI2a1uO32CDFx9ggxcfMPh71dXVsRkZGWxdXZ3B38uSVVdXs+7u7uzWrVtNGkd0dDT70ksvmez92/q8VFRUaPUdNYvW1Pnz5+PAgQM4ceJEm4tMqmNvb4+IiAhkZ2erfV0oFEIoFOojTEJ4S0lJQWZmJiIjI1FRUYGVK1cCAJ566imjxVBbW4vNmzcjJiYGtra2+PHHH3H48GGV/lyWyKTVRpZlMX/+fPz66684cuRIq05z2pDJZEhLS+NuHRNiLLm5uXBxcdH4yM3NBQCsWbMG4eHhiI6ORk1NDU6ePImuXbvqLY7+/ftrjOGHH34AwzA4ePAgHnroIQwdOhS//fYbdu/erfamhCUxaclr3rx52LFjB/bt2wdXV1cUFRUBUPSFabptPX36dHTr1g1xcXEAgJUrV+KBBx5A7969UV5ejs8//xy3b9/Gyy+/bLLrMFd0t9GwAgICkJqa2ubrQUFBSEpKMmgcBw8ehFQqVfuar68vHB0dcfjwYYPGYAomTV6bNm0CoJiSRNn27du5cWO5ubkqsy7ev38fr7zyCoqKiuDp6YmhQ4fizJkzKp35CDEGOzs79O7d29RhIDjYsFN8myuTJi9Wi8Gzx44dU3m+bt06rFu3zkAREUIshdl0lSCEED4oeRFCLBIlL0KIRaLkRQixSJS8rBjNtmEc3bt3x/r167nnDMNg7969JounszCLHvaEWJPCwkJ4enqaOgyrR8mLED0zh+maWJaFTCaDnZ31fsWp2mjFTF5pZFlAUmOaB88FOLRZKENbytXGW7dugWEY7NmzB2PHjoWTkxPCw8Nx9uxZlWNOnTqFBx98EI6OjhCJRHjzzTe52SwA7Rfp+OOPPzB06FAIhUKcOnWKd+yWxHrTMjE9aS3waUD7+xnC0juAQPNsvC1ps1BGR7z//vtYs2YN+vTpg/fffx9TpkxBdnY27OzscOPGDYwfPx4ff/wxtm3bhtLSUsyfPx/z58/H9u3bAWi/SEdsbCzWrFmDnj17Wn3VlZIX6fS0XSijIxYtWoSJEycCAFasWIH+/fsjOzsboaGhiIuLw9SpU7lSXp8+ffDll19i9OjR2LRpExwcHLRepGPlypUYN26c3uI2Z5S8iOHYOylKQKZ6by1pu1BGR2haPCM0NBSXLl3C5cuX8cMPP3D7sCwLuVyOnJwc9OvXT+tFOoYNG6a3mM0dJS8rZvKeEgzDq+pmzdpaPKO6uhqvvfYa3nzzzVbHBQUF8VoIo62Fa6wNJS/S6WmzUIYhDRkyBBkZGRpnqEhLSzP4Ih2WiO42kk5Pm4UyDGnx4sU4c+YM5s+fj9TUVFy/fh379u3j5sg3xiIdloiSFwEAfH0sG5P/cxb1Uu2W9bI27S2UYUiDBg3C8ePHce3aNTz44IOIiIjAsmXLuBsGxlikwxIxrDaTalmRyspKuLu7o6KiAm5ubqYOx6C2n87Bit8yAAC3Vk9sc9/usb8DAFZNGoBpD/Cf3K6+vh45OTno0aOHUb7whlZTU4Nu3bph7dq1mDNnjqnDsTptfV60/Y5SmxdRIe6kJS9zWCiD8EPJy4qZ+majpVmzZg2ysrIgEAgwdOhQnDx5ElevXsWECRM0HtPewrPEcCh5EQIgIiJC7UIZdXV1bS6yQUyHkhdR0blaQNvn6OhoFotskNbobiPRq052/4foSB+fE0peVkyXPkosdPtQNfUgr62t1el40rk0fU6URx7wRdVGohe2trbw8PDgpmlxcnKimVxJKyzLora2FiUlJfDw8ICtra3O56LkRfSmaRI+5XmmCFHHw8Ojw5M2UvKyYroUfDrSFMEwDPz9/eHj46Nx+XlC7O3tO1TiakLJi+idra2tXj6chLSFGuyJCrpXSCwFJS8rRs3lxJqZNHnFxcVh+PDhcHV1hY+PDyZNmoSsrKx2j/v5558RGhoKBwcHDBw4sNU83kR31E2LWAqTJq/jx49j3rx5OHfuHBISEiCVSvHoo4+qrJrS0pkzZzBlyhTMmTMHKSkpmDRpEiZNmoQrV64YMXJCiKmZtMH+0KFDKs/j4+Ph4+ODpKQkjTNY/t///R/Gjx+Pd999FwCwatUqJCQkYOPGjdi8ebPBY7Z2unZSJcTYzKrNq6KiAgDg5eWlcZ+zZ88iOjpaZVtMTEyrdfCaiMViVFZWqjw6DeokSqyY2SQvuVyOBQsWYNSoURgwYIDG/YqKiuDr66uyzdfXF0VFRWr3j4uLg7u7O/domgOcqEdtXsRSmE3ymjdvHq5cuYKdO3fq9bxLlixBRUUF98jLy9Pr+QkhpmEWnVTnz5+PAwcO4MSJEwgMDGxzXz8/PxQXF6tsKy4u1jjUQCgUQigU6i1WS0KVRmLNTFryYlkW8+fPx6+//oojR46gR48e7R4TFRWFxMRElW0JCQmIiooyVJiEEDPUoZJXdXU1t3BmEz6LWsybNw87duzAvn374OrqyrVbubu7w9HREQAwffp0dOvWDXFxcQCAt956C6NHj8batWsxceJE7Ny5ExcvXsSWLVs6cimEEAvDu+SVk5ODiRMnwtnZGe7u7vD09ISnpyc8PDzg6enJ61ybNm1CRUUFxowZA39/f+7x008/cfvk5uaisLCQez5y5Ejs2LEDW7ZsQXh4OH755Rfs3bu3zUb+zkq3gdnUYk8sA++S10svvQSWZbFt2zb4+vp2aM4mbb4ox44da7Xtueeew3PPPafz+xJCLB/v5HXp0iUkJSUhJCTEEPEQQohWeFcbhw8fTt0NCCEmx7vktXXrVsydOxcFBQUYMGBAqzmoBw0apLfgSMcwOnSWoCYvYil4J6/S0lLcuHEDs2bN4rYxDAOWZcEwDGSyzrniMiHEuHgnr9mzZyMiIgI//vhjhxvsCSFEV7yT1+3bt7F//35aiNMC6NRVQv9hEGIQvBvsH374YVy6dMkQsRBCiNZ4l7yeeOIJvP3220hLS8PAgQNbNdg/+eSTeguOGB812BNLwTt5zZ07FwCwcuXKVq9Rg715odZIYs14J6+WYxmJdaGZVIml4NXmJZVKYWdnR/PFWwhKQ8Sa8Upe9vb2CAoKoqqhFaM2L2IpeN9tfP/997F06VKUlZUZIh6iR9TmRawZ7zavjRs3Ijs7GwEBAQgODoazs7PK68nJyXoLjhgfFbyIpeCdvCZNmmSAMAghhB/eyWv58uWGiIMYAI3cItZM52mgk5KScPXqVQBA//79ERERobegiAlRiz2xELyTV0lJCV544QUcO3YMHh4eAIDy8nKMHTsWO3fuhLe3t75jJISQVnjfbXzjjTdQVVWF9PR0lJWVoaysDFeuXEFlZSXefPNNQ8RIdKTTfF4GiIMQQ+Bd8jp06BAOHz6Mfv36cdvCwsLw1Vdf4dFHH9VrcIQQognvkpdcLm81GBtQdGCloUOWj5q8iKXQaUqct956C3fu3OG2FRQU4O2338Yjjzyi1+AIIUQT3slr48aNqKysRPfu3dGrVy/06tULPXr0QGVlJTZs2GCIGImudJqMkIpexDLwbvMSiURITk7G4cOHkZmZCQDo168foqOj9R4cIYRoolM/L4ZhMG7cOIwbN07f8RBCiFZ0Sl6JiYlITExESUlJq0b6bdu26SUw0nHUwZ5YM97Ja8WKFVi5ciWGDRsGf39/Wj2IEGISvJPX5s2bER8fj2nTphkiHmJi1FWCWAredxslEglGjhxpiFiInlGpmFgz3snr5Zdfxo4dO/Ty5idOnMATTzyBgIAAMAyDvXv3trn/sWPHwDBMq0dRUZFe4iE0PIhYDt7Vxvr6emzZsgWHDx/GoEGDWvW2/+KLL7Q+V01NDcLDwzF79mw888wzWh+XlZUFNzc37rmPj4/WxxJCrAPv5HX58mUMHjwYAFotxMG3mjJhwgRMmDCBbwjw8fHhZrQg+kVtXsRS8E5eR48eNUQcvAwePBhisRgDBgzARx99hFGjRmncVywWQywWc88rKyuNEaJZoBYvYs14t3mZkr+/PzZv3ozdu3dj9+7dEIlEGDNmTJvz5sfFxcHd3Z17iEQiI0ZseWh4ELEUOs+kagohISEICQnhno8cORI3btzAunXr8P3336s9ZsmSJVi4cCH3vLKykhIYIVbAopKXOpGRkTh16pTG14VCIYRCoREjMh/UU4JYM4uqNqqTmpoKf39/U4dhPSyw1phZVImEjGJTh0GMjHfJ68SJExg5ciTs7FQPbWhowJkzZ/DQQw9pfa7q6mpkZ2dzz3NycpCamgovLy8EBQVhyZIlKCgowHfffQcAWL9+PXr06IH+/fujvr4eW7duxZEjR/DXX3/xvQxiRcavPwkA+PX1kYgI8jRxNMRYeCevsWPHorCwsFXfqoqKCowdOxYymUzrc128eBFjx47lnje1Tc2YMQPx8fEoLCxEbm4u97pEIsE777yDgoICODk5YdCgQTh8+LDKOUgzbauNUlnz4HoLLHhxrhVXUfLqRHgnL5Zl1fbnunfvXqvVs9szZswYsG10LIqPj1d5/t577+G9997j9R6kfc9uOmPqEAjhTevk1dQDnmEYzJw5U6URXCaT4fLlyzTm0UJdzq/g/t/WHxNCzInWycvd3R2A4sPt6uoKR0dH7jWBQIAHHngAr7zyiv4jJIQQNbROXtu3bwcAdO/eHYsWLeJdRSTGp9O6jVTwIhaCd5vX8uXLDREHIYTwwrufV3FxMaZNm4aAgADY2dnB1tZW5UEIIcbAu+Q1c+ZM5Obm4sMPP6RpoK0Q1RqJpeCdvE6dOoWTJ09y0+IQYi50aeMjlot3tVEkEtHtdCtmyb9amhGjc+GdvNavX4/Y2FjcunXLAOEQfaIaPbFmvKuNkydPRm1tLXr16gUnJ6dW00CXlZXpLThifJZceqFqY+fCO3mtX7/eAGEQQgg/vJPXjBkzDBEHIYTwotN8Xjdu3MAHH3yAKVOmoKSkBADwxx9/ID09Xa/BEUKIJryT1/HjxzFw4ECcP38ee/bsQXV1NQDg0qVL1PveCljy3UbSufBOXrGxsfj444+RkJAAgUDAbX/44Ydx7tw5vQZHCCGa8E5eaWlpePrpp1tt9/Hxwd27d/USFNEPGv1ArBnv5OXh4YHCwsJW21NSUtCtWze9BEUIIe3hnbxeeOEFLF68GEVFRWAYBnK5HKdPn8aiRYswffp0Q8RICCGt8E5en376KUJDQyESiVBdXY2wsDA89NBDGDlyJD744ANDxEh0pEulkYZ+EUvBu5+XQCDAN998gw8//BBXrlxBdXU1IiIi0KdPH0PERwhvdRIZhHY2sLGhNj9rpvOis0FBQQgKCtJnLMQMWHS5iwHKayUYvDIBgwLdsX/+P0wdETEgrZLXwoULsWrVKjg7O3PLk2nyxRdf6CUwQnRx/FopANVFRYh10ip5paSkQCqVcv/XhG7Nmxddfh0W3eRlybET3rRKXkePHlX7f0IIMRWdxjYSYpao4N+paFXyalpwVht79uzRORiiXzotfUZ1L2IhtCp5ubu7cw83NzckJibi4sWL3OtJSUlITEzkFqYlhBBD06rk1bTgLAAsXrwYzz//PDZv3swtdSaTyfD666/Dzc3NMFESo7HoBnvSqfBu89q2bRsWLVqkskajra0tFi5ciG3btvE614kTJ/DEE08gICAADMNg79697R5z7NgxDBkyBEKhEL1790Z8fDzPK+g86OYvsWa8k1dDQwMyMzNbbc/MzIRcLud1rpqaGoSHh+Orr77Sav+cnBxMnDgRY8eORWpqKhYsWICXX34Zf/75J6/37Yy0HfZDBS9iKXj3sJ81axbmzJmDGzduIDIyEgBw/vx5rF69GrNmzeJ1rgkTJmDChAla779582b06NEDa9euBQD069cPp06dwrp16xATE8PrvQkhlo138lqzZg38/Pywdu1abmocf39/vPvuu3jnnXf0HqCys2fPIjo6WmVbTEwMFixYoPEYsVgMsVjMPa+srDRUeFaB2ryIpeBdbbSxscF7772HgoIClJeXo7y8HAUFBXjvvfdU2sEMoaioCL6+virbfH19UVlZibq6OrXHxMXFqdwtFYlEBo3RnFCTF7FmHeqk6ubmZvZ3GJcsWYKKigrukZeXZ+qQTEL7EhUVvYhl0GlWiV9++QW7du1Cbm4uJBKJymvJycl6CUwdPz8/FBcXq2wrLi6Gm5sbHB0d1R4jFAohFAoNFhMhxDR4l7y+/PJLzJo1C76+vkhJSUFkZCS6dOmCmzdv8mp810VUVBQSExNVtiUkJCAqKsqg72upOltXiU52uZ0e7+T19ddfY8uWLdiwYQMEAgHee+89JCQk4M0330RFBb9pSKqrq5GamorU1FQAiq4QqampyM3NBaCo8ilPLT137lzcvHkT7733HjIzM/H1119j165dePvtt/leRqdDlUFibXgnr9zcXIwcORIA4OjoiKqqKgDAtGnT8OOPP/I618WLFxEREYGIiAgAinnDIiIisGzZMgBAYWEhl8gAoEePHvj999+RkJCA8PBwrF27Flu3bqVuEgQAJejOhnebl5+fH8rKyhAcHIygoCCcO3cO4eHhyMnJ4T3/+ZgxY9o8Rl3v+TFjxrQ5p5gpVdRJIbC1gaPAsHddtafDwGzKAMRC8C55Pfzww9i/fz8ARYfVt99+G+PGjcPkyZPVrufYWVSLGzD686N4+uvTpg5FSefKRNTm1bnwLnlt2bKFGwY0b948dOnSBWfOnMGTTz6J1157Te8BWor0ggqU10pRXisFy7JmN6usooTbfkxU8iKWglfyamhowKefforZs2cjMDAQgGIdxxdeeMEgwVmSa8VV3P/rpXIzqjoSYp14VRvt7Ozw73//Gw0NDYaKx2JlKSWvWom5/HxoMkJivXi3eT3yyCM4fvy4IWKxaNeKqrn/10pkJoxEPUpJxNrwbvOaMGECYmNjkZaWhqFDh8LZ2Vnl9SeffFJvwVkKlmVVSl71UvNLXoRYG97J6/XXXwegfn1GhmEgk3W+L25JlRgVdVLuubmUvDrd0mekU+GdvPhOONgZZBVVqTw3l+SlrDMkJXO7w0sMi5Y+0wPlO40AUCc1lwZ7/iw5x/HtJE0sm9Ylr7q6OiQmJuLxxx8HoBh3qDzJn62tLVatWgUHBwf9R2nmWiUviXmUTqkcQqyZ1snr22+/xe+//84lr40bN6J///7cVDSZmZkICAjolIOks4oVdxoZRlE9M5+uEvxR4YVYCq2rjT/88ANeffVVlW07duzA0aNHcfToUXz++efYtWuX3gM0d3I5i+uNJa++Pq4AgDozvNvYGfpvUZtX56J18srOzsbAgQO55w4ODrCxaT48MjISGRkZ+o3OAhSU16FWIoPAzgb9/BuTlxk22GvLHJPcnuR8bD1509RhEDOjdbWxvLxcpY2rtLRU5XW5XK7yemfRdKext7cLXBwUP05zudtoLSWRhbsuAQCi+/mie1dnjftpO36TWAetS16BgYG4cuWKxtcvX77MjXfsTJo6p4b4ucLRXjGe0SyrjdoWqMyv4MWpqrfctkSif1onr8ceewzLli1DfX19q9fq6uqwYsUKTJw4Ua/BWYKmO419fV3hKGgqedGXzFAaZHKcun4XNeLWP2NrKWkS7WhdbVy6dCl27dqFkJAQzJ8/H3379gUAZGVlYePGjWhoaMDSpUsNFqi5aqo29vV1wfUSxV1H6iphOF8mXseXR7IxoocXfnqN1i7ozLROXr6+vjhz5gz+9a9/ITY2lusQyDAMxo0bh6+//rrVmorWTiqT42ZpDQBFyaugXLF2JHVSNQwWLH68oFi67nxOmYmjIabGa3hQjx49cOjQIZSVlSE7OxsA0Lt3b3h5eRkkOHN3+14NJDI5nAW26ObhCIfGNi9zabAnxJrptG6jl5cXIiMj9R2LxclqnAanj68rbGwYOAnMK3npNjDbnMtehDSjsY0dwN1p9FX072pKXuY8JU5pldis42uPNbbjEd1Q8uqAa02N9X6K5OVob179vJSxLHCnvA7DPzmMf3x21NTh6ISh1EWUUPLqgGstSl5N89abaw/7k9cVHYvvVltmZ+L2ev9TautcKHnpqF4qw617jXca/VwAQKnNyzzuNrZs89KmH5Qlt3hZcuyEP0peOsouqYacBTyd7OHtIgQA8+5hD5ZKJsSqUPLSkXLP+qYSTXODvRxyufmVA2ysoAe6FVwC0RNKXjq61jiHV0hjYz0AlbUazaH01bKBW5svviX3lKC81rlQ8tKRcsmriYNdc/IyxzuOVGoh1oSSl46axjQql7xsbBiu3cvc+lKxrHZdDSy44GXRsRP+zCJ5ffXVV+jevTscHBwwYsQI/P333xr3jY+PB8MwKg9jz5tfVS/lxjE2zZ7axKx62be622iaMPSJ+nqRJiZPXj/99BMWLlyI5cuXIzk5GeHh4YiJiUFJSYnGY9zc3FBYWMg9bt++bcSIwc0e4efmAHcne5XXmsc3mkd3CWVadZUw40av9kIzh7Qml7M4c+MuKuul7e9MOsTkyeuLL77AK6+8glmzZiEsLAybN2+Gk5MTtm3bpvEYhmHg5+fHPYw9m0XLnvXKnMy0oyoLwMYcvt1W7n/nb+PFb87jhf+cM3UoVs+kyUsikSApKQnR0dHcNhsbG0RHR+Ps2bMaj6uurkZwcDBEIhGeeuoppKena9xXLBajsrJS5dFRzWMaXVq9xiUvM2vzAiyrzevczXv456YzuFrY8d9XE2OUKn9JygcAZOgxbqKeSZPX3bt3IZPJWpWcfH19UVRUpPaYkJAQbNu2Dfv27cP//vc/yOVyjBw5Evn5+Wr3j4uLg7u7O/cQiUQdjlvdncYmjmbU5tUyVVlSm9cLW87h4u37mLX9Aq/jNOWnFb+lY9TqI6ioNWx1Tiozl/Rv/UxebeQrKioK06dPx+DBgzF69Gjs2bMH3t7e+M9//qN2/yVLlqCiooJ75OXldTiGpqlwQtRUG7le9maQvFrSKneZ2XdPeRwmw+iegLefvoU7FfX433nDto/K5OYxi25noNN8XvrStWtX2Nraori4WGV7cXEx/Pz8tDqHvb09IiIiuMkRWxIKhRAKhR2Otcm9ajHuVovBMEBvH3XVRvOcx55lWYPM8W6o8zZRPrUZ30vgNJjhyAprZdKSl0AgwNChQ5GYmMhtk8vlSExMRFSUdvOTy2QypKWlwd/f31BhqmjqWS/ydOISlTJuZgmp+f0F1neO2ZdagIhVCTh/855+T2zBGqjaaDQmrzYuXLgQ33zzDb799ltcvXoV//rXv1BTU4NZs2YBAKZPn44lS5Zw+69cuRJ//fUXbt68ieTkZLz00ku4ffs2Xn75ZaPE21Z7F6B8t9G8Sl6AdtVGPovOvrUzFeW1Usz59qLuQbWDT78ubaqVhm60l1HJy2hMWm0EgMmTJ6O0tBTLli1DUVERBg8ejEOHDnGN+Lm5uSorc9+/fx+vvPIKioqK4OnpiaFDh+LMmTMICwszSrzN6zS2rjIC5tVgr4yF4ZYGM2hCaNnRts04DBeGtqQy8ytxWyuTJy8AmD9/PubPn6/2tWPHjqk8X7duHdatW2eEqNTj+nhpKHmZ07Q4LZOVNv28zCEBKFMO2cxCU4tKXsZj8mqjJWFZVmWFbHXMtZMq0LLx2zK+ZPouLBr6sqnB3ngoefFQVFmPqvoG2Nkw6NlVU7XRPOexZ1nVkpimL7G55TTlNq/28phWU/50LJx2NVC10WgoefHQNJNEj67OENip/9E5NY1t7EC10VATGVpaFUwXpk6+VPIyHkpePFxv7Cahbkxjk6YG+3odS16/Xy5E+Iq/cOJaqU7HK2vdw755i1zDt5zP3UZjUKnqou2bDqZOXAAlL2Oi5MVDy3Ua1eHuNkp16yoxb0cyqsQNmL5N87RAulIpeVnId0zf90cNfd3UYG88lLw0EDfIsP/SHW6gLdB+Hy9AqdookeFKQQWSbt83bKDaYlXnsNdU8rJkljR2k3QcJS8NEq+W4M0fU/DvQ5mQyhQLalxr504j0Dw86GZpDR7fcAovfnMONWLjdViVy1kUVdQDULf0mdHCIERFZlEl1h++ptf2XLPo52WOovv5oquLACVVYiReLUE/f1fUS+UQ2tkgyMtJ43GOAtW/B+IGOcrrpHAWGudH/dZPqfjt0h1smTZUZUEQQDV5NZW8Fv9y2Shx6YM2uZcStPm4Wy3GvtQ72JOcj/Q7iimCInt4YWSvrno5P5W8NBDY2eCfQxXT5+z4O5e709jH1wW2bfT2dFQz3tGYQ4V+u3QHAPD1sRsq2xXrNqp2lci/X4ufLqrOsmHs2uTRrBJEfnJY4w0Kle4deng/c7shYW3qpTIcTCvEnPgLGPFpIlYdyMCNO6UYYncTK0UpcLeR6O29qOTVhimRImw+fgMnr5fCz00xM0Vb7V1Ac5uXslt3a3HoShEmDw+Ct6v+ZrhoS71Uhm9O5qhsa1nyModBxE3zdU3f9jdurZ7Y6nUqSJk/lmWRnFuOX5Nu4/LlFHST5GCQTR6es83DIMcC+MsKwYAFSgHYjgMQoJf3peTVhuAuzniwT1ecvH4XPzc23Ld1pxFAq6oaALz8nWLg8vmcMnw/Z4TO8STn3kcXZwGCuzi3u29mURUyG0uLTZQTgZxVX8VioehomZBRjKHdPeHjatzFTYgFYFmgqgilN5ORdek8avIuo5skBx8wBXBgpIBAad+mHkPO3oCPfscfU/Jqx9QRQTh5/S5XnWqrjxcACDV0XgWA09l3dY7j1t0aPPP1GcX/1ZRQ2sOyiqXZmjdoXkE7/swtfPz7VXg5C5D84Tid4tUbnkUvddVe5aFQVniT1bDEVUDJVaA4HSjJgKwoHQ2FVyCUVsAbgHfTfo0fe5mtI2x8+4HxDQN8+gNN/7p4a3gD3VHyascj/Xzh7SpEaZViRs/2Sl5tdaL0dBJofK09LUtRulAteWnopMoCh68qJocsq1G0T6TlV+DXlAK89UifVqslGZpq3zS2zQZ5Ta8pXyrlLg1kUuBeNpekUJwBlKQD5bkqu9k2PmQsg1usH+469YZL8CD06h8Jh24DYevZA7AxTlM6Ja922NvaYPIwETYezYar0A7+7rpXozyUvvhHMovxS1I+Pn16IDx4JjWWZSFn0eaNA3Va91bXdH7V509sPAUAqKiTYu3z4WrOqziRpEGOi7fKMCTYk1sCzphYDVVhGRW3mrEsUFnQnJyKMxTJ6u41QKa+Mb0EXrgqC0QmK0KWXIRq9xAMGRqJJ4b1wggPRyNfQDNKXlqYFhWMg2mFGB3i3aE5sZRLXrPjFe1gjvZ2WPt8OMprJXhrZ2qrYyQN8lbjKF/7PglXCiqQ+M4YtW1smql2UlVfbdT8Rc8sqkRBeR0aZHKVdrematmnB68i/swtPDbQD19PHcojLoW8slqI2uiGouvP3lgdcpWrp2bRZaOuvLEU1ViaKrmqSFbiCvX7C1wh7RqKmzZBOHbfG0fKvJHFBqIcrnB3tMcT4f54aUggIkQeBp36W1uUvLTg6+aAI4vGdPg86kpYWcWK/i+Ld1/G8RbdBXYn5eOdny+hn7+byhJgf2UoqnXHr5Vg/ADtpr9uWdLSVEpp2lcdmZzFqNVHAAAZK2NavR5/5hYA4GCa+pWf2rMvtQDzH+6jsk0fXxKV3GXARCZRmlHCzpiLZDaIFSWnlqWpygL1+9vYAV36NLZHhUHatR/OVPnih0wWR6+Vcisg2dkwGBPqjWeHBOLhfj4Q2hm/NN0WSl6aSOuA458p7pD4hAFd+wJ2urdZAc3VxvLa5uK5k8AOLMviz/TiVvu/8/MlANC4dmHLQcB85uhiWZbXFMuA6gSL95WWENPXX+H2wte1zctYJS9JQ3Py4lul14pcDlTktk5S97IBuYa+hG6BXJKCb//Gz3IfsLYCXM6vwO7kfPx27A7u1zZ//gZ0c8MzEYF4cnAAuroYp2uPLih5aVKaCZxSmrHVxg7o0rvxQxDWnNQ8grVuoHSwV+y3O7n5L2I3D0fkltW22pdh2v8yKw8CZlkWk3ms0qypzYtlNd/gU+4XZqsyN5h+k0Nb1a+8sjre5zPWYGnlNRvtOtpoXVum1HiuVO2TVKvfX+iulKQa7/D59AMcPVR2K6yow6+n8rAnuQDZJc3n8nEV4umIbnhmSGCbw9/MCSUvTeydgWGzm/+6iSsVCa00E0jfo7qfTz/Fo/EvWxdU4B7cW52SZRVfzB+U1g68VyPB+ZtlOoWo/GU5cLkQf9/SfB6WZVWSoZxlVRKQyr4a36+5ZGGoG0oyOYunvjrFPS/nsUisojtI6+1t5a6U3PtY9PMlfDAxDGNDfVReu1ctxk8X8zA1Mliru6zKJa+2ZJdU42phJSYO9IeNrF7xmWr6nDXd6avWUPW2FQBdQ1T/gPqGAW7dNBY9ayUN+DO9CLuTCnD6RnO3H6GdDWL6++HZoYEY1asL7Gwta8ANJS9NvPsCjzeWvFgWqMhX/OXjiutXgbtZgLQGKLioeDRKcgBKWTdkyUW4xoq4uzS2Mm+cvXkPN0truH3vVYuRkleuU4jKC5y+8WMKr2NZFmDUVG3aKqOUVDUvAKvc2N9UbdSmtNiejDuVuFKgvpqsjXb7ebV4bdp//0a1uAGz4i+06j+35q8s/Ph3Hn44l4vTsQ9rfE+ZnEVqXjncHe1VtqmQy4D7t4DidJzedxDetTdQcuAOfBvugGE1JD2P4OaqXlNpqksvwLb9RCqXsziXcw97kgvwR1ohapTml4vs4YVnh3TDYwP94epg3K4v+kTJSxsMA3iIFI++jzZvl0mBezdU/2KWZEBedgveTCW8bdPxD6Rzu8uvMLiX5Y8t9v6479IHJyu8cbeqN6pr+7d6S22SQEcmvlv71zUseSxU/YtanFbdnUpGu0Pb1JH2KU1tXqrVa8W/2SVV6OoiRHUbM36kFSjuyhWU16FeKtPY/eO7s7ew4rcMDO/uCQDoigoMQD5w9kZz+1RJJtCgqPLOABSdpRoLlayjF5iWSconFBDyr77dLK3GnuQC/JpSgILy5ip2kJcTnh0SiKcjuiGoi+Y7upaEkldH2NorPmQ+oQCe4Tb3j92Nvkw+QmzyEMLkI4TJRYhNniKhSe/gUds7QF0SJgsASAFptj3+JQhAFhuILLmipHZNLsIddEFbXcylDXKcuFaKsAA33qHvTs7HUjXJS9v2K3X7MXooehmidUo5x7Ngcb24CuPWneDaIDWJEHlypcBjWaUYP6DFKu6SGqAkE/mJP2OZXQ5CCvIQIsxDV6ax5Pin6u71rD2us92QxQah1KkXzlX7IUMWiD1vPA2RFkO+NCmvleDA5ULsTs5HSm45t91VaIfHw/3x7JBADA32NIvuDfpEycsAQkR+SM1zwCVZb5XtXVCBvjb5iPYqxYzedUhLPou+TB6cGTHCbG4jDLcVf5EbVbJOyGIDcU3eXPXMZEWohGLxj19TCnApv0KrO0L6TgrK55PI5Ji3I1kvDeOGuDOoXLqSyYFTjcO06ttY1bykqh7fn1O0TdpChpTkcxgPOVByFWxJOmRF6bArvw2AxYeAyjdJzjK4zfqgxiMEiWVdcZ0NwufzX0D//8uGvHEczeKoUGSeyUFppRjldQ0Q8bwmqUyO41ml2JOSj8MZJVw3DRsGeKivonvDuDBfk3QWNhZKXgbwn2lD8f3Z2/D3cMD7v17htt+DO87K3fHcw+GwGxKIl5IPoVYsRZhjOfzFOQhh8hBqk4u+TD56MoVwY2oxnLmG4TbXVM5fxHoqElmRCL1tRMisEaEK3SBGe1052k8MLPhPGyNpkOP3y4W8jtH4/m28tXI1CFAMIFduZH77p0uYN7YX93xPcj6eGRKIiV+e5La1lRwraiRwl90FijPw5/6DWGufhVAmD72ZOxDelAI3FfsxUPriOPsgQx6IM1W+yGr8A3Od7YY6OAAlzedeZB8EedMJAIwJ8ca+1AIUV4pxv1Z9z3aWZVFRJ0VplRilVWKUNP57u6wGf6QV4V5N83Ghfq54dkggnooI6DSD6Sl5GYCvmwMWxYTg3M17al9/bKCiY2kXFyFqymRIr/NCOrxwGEO5UfgCSNGTKURfJg+hNnkIYfIQYpOHQOYu/Jj78LO9j9FonkiwaaxZllIJ7Rorwm3WF3LY6Fybe/rr02q3G67rlOYTz9+helNC0iJ5AcBXR5vnMVu46xKeGRKosgxd00yeLqhFXya/8Webi1CbPLhueB2oLwcATANUSsG1rBANXUPgGhSOlX8DmWwQsuQivBg1BEm37+PsffW/6yaXC1R7tYf6ucLLWfHH5n6tBEUV9cguqUZ2SRWulVTj4q0y3Lpbq9LxtaWuLgI8NbgbnhnSDf0DWt/dtnaUvAzIUU2RfWyIN1eUb6uaJYE9MtkgZLJB2K/0+XVFLfo0fumUE5snU41eTCF6oRCP2TYv3lHHCnCd7Ybj/96MTLkID9qIkCkXoRQear8YRRX1cFGa9VW5DUVZe6WzH//Oxa27NYidEMqrrYVPUhRL5WhvWGh6/l3Fz4nJQ4hNLqKv30OA+CZmOagpKdYDMtgAXr3wR6kXsuQiZLEiZLJByGO94V/thLBydxyWNXfo3Hg0W+3vuaU3le4Gd3URgGEYbrjYwl2X2vwsuDvaw9tVCG8XIXzcFP9G9eqCh/p6w97CujfoEyUvA3JSGne49LFQ3L5Xi3djQrhtD4f6cO0q2qqCE5LZvkiW9VXaysIH5Y03CPK4xNaXyYcjI8EgJgeDkKNSkrjPuiBrrQgf2TV255CLcI0NRKa2I3vaSTJL9qQBAB7t74chQerHwuXeU+2cy7Z/WhXiBrnSjQMWAbiHEJvmRBXC5KHXN3fwl1BpGTqlAhBX/W4srWaxQchmA/DmwIH4/M+sVu93p6IedxrXB1BWJ5Whj48L7tdKcLdatQq49LFQfHowU2XbgmjF766vryt+TyuETM7C1oZBsJcTevm4oLePC8IDPdA/wA0+bkKzG5ZjLih5GZDyoOmHQ33R20d1le3lT4TBSWCL/5y42fJQnhiUwBMlck+cxCCu6mkDOYKZYqWShyK5dWeK4MlU4wHmKh6wuapypny2a3OJo7Gf2g02ANIWH5VjWq4reSb7Ll7+9gIWxYRg6ohgAIopdjYdz0bi1RKVfavFDfhof7q606hwQzVCmTxIzmbjt4un8YsgB32ZPLgx6nvfV7GOuKZ0JzdLHoRMVoQKqF/1XF3ias9ro3vBw9Gem3gSUIyeePkfPeHpJMB3Z2/jfq0ET4YHYOqIoMZjeiKyhxe6uigmmNS0kDFRj2H1PbZDB1999RU+//xzFBUVITw8HBs2bEBkZKTG/X/++Wd8+OGHuHXrFvr06YPPPvsMjz32mFbvVVlZCXd3d1RUVMDNjX8XAz7KaiQYsioBAJDy4Th4Oreu4xzNKuGmQjYWISTozRQglMlDX5vmxObHqF+mTcra4ibrryiZyYMU7WpsIPJZb7A8lkFo6gTae+lBrfqoCSBFb6aAa+9ritOfUT+SQMra4gYbwLX7ZbGByGKDkM92hSEnlO7qIsTp2LFcCSm7pBo7/87F3DG9zHpsoLnS9jtq8pLXTz/9hIULF2Lz5s0YMWIE1q9fj5iYGGRlZcHHx6fV/mfOnMGUKVMQFxeHxx9/HDt27MCkSZOQnJyMAQMGmOAKNPNwtEdEkAfsbBiVubyUBZpgPiQxBEhneyCd7QEoNXu5o1olUfRtLKm5MXWK/mrIxxO2zeMnq1kHXGcDm6ucjY3YZWj7j0LLxMVAjkCmVJGclNrxejCFsGPUN1i3LCFmsSLcVFNC7Kgt04bi1e+TuOczooLx7VnVqv60B4JVqna9fVzwweP6nfKYtGbykteIESMwfPhwbNy4EQAgl8shEonwxhtvIDY2ttX+kydPRk1NDQ4cOMBte+CBBzB48GBs3ry53fczZskLaO7MqanRWtIgx4Dlf8JZaIs3H+mDFb9lINDTEXvnjYJMzuKzQ5m4Vy2Bh5M97pTXobRKjDEhPtz0M4anaEtSLqGFMnnoxRRAwMjUHtE0NCqLDWrscBuIa2wgrq5+FtVlRXhlzXeKJKnUPufMiNWeq4J14pKiclW2Cpp7iff1dcG1YsWg46kjgvDD+VyN+wJAZHcvrJzUH0czS/HZIdX2qVurJ+Lk9VJ8mXgdsRNC0T/AHaEfHuJe7+IswJ9vP0QlLD3S9jtq0uQlkUjg5OSEX375BZMmTeK2z5gxA+Xl5di3b1+rY4KCgrBw4UIsWLCA27Z8+XLs3bsXly5darW/WCyGWNz8xaisrIRIJDJa8tJGtbgBLMvyGmfGsiyqxA2Y9NVplbGSgKJ0cPrGPWSXVOPQggdhyzB4/9crbQ7c5ssODejOFKlWPZk8BNuUqN1fzjKohBM8mBq1r4tZO9xguyk1nisSVRG80F6VT2hnA7HSoOisj8ejQcbiTnkdevu4YHdyARb93PqzAQAbX4zA44MUq9nUS2V4IC6RGwz++T8H4blhrbuPpuaVo7iyHiN7dYFUxnJdHoh+WES18e7du5DJZPD19VXZ7uvri8zMTLXHFBUVqd2/qEj9bbK4uDisWLFCPwEbiIsOC9IyDAM3B3sceWcMGmRy3KuRwNXBDqVVYrWrC+2aG4XiynpsPJKNycNFeGfXJSx7IgyDRR5Y+9c1dHERQCyVIf9+HfakaJjETkkD7JDNBiKbDQTkUdx2J9SjT+PQqFBGUapqGhrlAUXiypV7c6Wypkb0W6wfGtr4OIb6uSKzqAq7/xWFNX9eQ3AXJ5RWiRHT3w/PDxeBZVn8lVGM8EAPCO1sIbQD+jSuN/DPoYEYLPLAzxfzuJsj4/v7oYe3Mx5TmszRwd4WqcseVfv+ygaLPNrdhxieydu8DG3JkiVYuHAh97yp5GVN7Gxt4Oum6FUd3EXzr9TXzQGrJinaBf98+yFu+7InVNtnvpg8mHcMDTI5bBgGEpkcDXIWDTI57tdKkVtWi4T7dRjoIUbN3Xzsz3XA8FARwvzdEeEigC3DwMXBDnY2DFe1Vp3Pq3Wp68dXH2i1jWEYxPT3a7W9SW8fFyx5rB+WPNaP97UR82TS5NW1a1fY2tqiuFh1FtHi4mL4+an/IPr5+fHaXygUQiik9ghDa+rp7mDT3HDt4SRAj65KpcCQPnhgVPvnsrYBxMQwTNqxRCAQYOjQoUhMTOS2yeVyJCYmIioqSu0xUVFRKvsDQEJCgsb9CSHWyeTVxoULF2LGjBkYNmwYIiMjsX79etTU1GDWrFkAgOnTp6Nbt26Ii4sDALz11lsYPXo01q5di4kTJ2Lnzp24ePEitmzZYsrLIIQYmcmT1+TJk1FaWoply5ahqKgIgwcPxqFDh7hG+dzcXNgoze07cuRI7NixAx988AGWLl2KPn36YO/evWbXx4sQYlgm7+dlbMbu50UI4Ufb7ygNpiKEWCRKXoQQi0TJixBikUzeYG9sTU18lZW6L69FCDGcpu9me83xnS55VVVVAYDV9bInxNpUVVXB3V3z9Nad7m6jXC7HnTt34OrqavY9uZuGMuXl5Vn8nVG6FvNkjtfCsiyqqqoQEBCg0k2qpU5X8rKxsUFgYKCpw+DFzc3NbD5YHUXXYp7M7VraKnE1oQZ7QohFouRFCLFIlLzMmFAoxPLly61iVgy6FvNkydfS6RrsCSHWgUpehBCLRMmLEGKRKHkRQiwSJS9CiEWi5GVmysrKMHXqVLi5ucHDwwNz5sxBdXV1m8ds2bIFY8aMgZubGxiGQXl5uXGCbeGrr75C9+7d4eDggBEjRuDvv/9uc/+ff/4ZoaGhcHBwwMCBA3Hw4EEjRdo+PteSnp6OZ599Ft27dwfDMFi/fr3xAm0Hn+v45ptv8OCDD8LT0xOenp6Ijo5u93doSpS8zMzUqVORnp6OhIQEHDhwACdOnMCrr77a5jG1tbUYP348li5daqQoW2ta+Xz58uVITk5GeHg4YmJiUFKifh3HppXP58yZg5SUFEyaNAmTJk3ClStXjBx5a3yvpba2Fj179sTq1as1LgRjCnyv49ixY5gyZQqOHj2Ks2fPQiQS4dFHH0VBQftL4ZkES8xGRkYGC4C9cOECt+2PP/5gGYZhCwoK2j3+6NGjLAD2/v37BoxSvcjISHbevHncc5lMxgYEBLBxcXFq93/++efZiRMnqmwbMWIE+9prrxk0Tm3wvRZlwcHB7Lp16wwYnfY6ch0sy7INDQ2sq6sr++233xoqxA6hkpcZOXv2LDw8PDBs2DBuW3R0NGxsbHD+/HkTRtY2iUSCpKQkREdHc9tsbGwQHR2Ns2fPqj3m7NmzKvsDQExMjMb9jUWXazFH+riO2tpaSKVSeHl5GSrMDqHkZUaKiorg4+Ojss3Ozg5eXl4aVwQ3B22tfK4pbr4rnxuLLtdijvRxHYsXL0ZAQECrPzLmgpKXEcTGxoJhmDYfmZmZpg6TEM7q1auxc+dO/Prrr3BwcDB1OGp1uilxTOGdd97BzJkz29ynZ8+e8PPza9WY2tDQgLKyMrNqCG7JGCufG4su12KOOnIda9aswerVq3H48GEMGjTIkGF2CJW8jMDb2xuhoaFtPgQCAaKiolBeXo6kpCTu2CNHjkAul2PEiBEmvIK2WdPK57pciznS9Tr+/e9/Y9WqVTh06JBK26tZMvUdA6Jq/PjxbEREBHv+/Hn21KlTbJ8+fdgpU6Zwr+fn57MhISHs+fPnuW2FhYVsSkoK+80337AA2BMnTrApKSnsvXv3jBb3zp07WaFQyMbHx7MZGRnsq6++ynp4eLBFRUUsy7LstGnT2NjYWG7/06dPs3Z2duyaNWvYq1evssuXL2ft7e3ZtLQ0o8WsCd9rEYvFbEpKCpuSksL6+/uzixYtYlNSUtjr16+b6hJYluV/HatXr2YFAgH7yy+/sIWFhdyjqqrKVJfQJkpeZubevXvslClTWBcXF9bNzY2dNWuWyocnJyeHBcAePXqU27Z8+XIWQKvH9u3bjRr7hg0b2KCgIFYgELCRkZHsuXPnuNdGjx7NzpgxQ2X/Xbt2sX379mUFAgHbv39/9vfffzdqvG3hcy1Nv5OWj9GjRxs/8Bb4XEdwcLDa61i+fLnxA9cCTYlDCLFI1OZFCLFIlLwIIRaJkhchxCJR8iKEWCRKXoQQi0TJixBikSh5EUIsEiUvQohFouRFzMLMmTPVzrYxfvx4U4dGzBTNKkHMxvjx47F9+3aVbZpWcpZKpbC3t1fZJpFIIBAIeL+vrscR06KSFzEbQqEQfn5+Kg9PT08AAMMw2LRpE5588kk4Ozvjk08+wUcffYTBgwdj69at6NGjBzfvVG5uLp566im4uLjAzc0Nzz//vMrUMJqOI5aFkhexGB999BGefvpppKWlYfbs2QCA7Oxs7N69G3v27EFqairkcjmeeuoplJWV4fjx40hISMDNmzcxefJklXO1PI5YHqo2ErNx4MABuLi4qGxbunQptyrSiy++iFmzZqm8LpFI8N1338Hb2xuAYk6wtLQ05OTkQCQSAQC+++479O/fHxcuXMDw4cPVHkcsDyUvYjbGjh2LTZs2qWxTXvxB3eR4wcHBKgno6tWrEIlEXOICgLCwMHh4eODq1atc8mp5HLE8lLyI2XB2dkbv3r3bfF2bbdq+F7Fs1OZFrEq/fv2Ql5eHvLw8bltGRgbKy8sRFhZmwsiIvlHJi5gNsVjcalkuOzs7dO3aVetzREdHY+DAgZg6dSrWr1+PhoYGvP766xg9erT5z8lOeKGSFzEbhw4dgr+/v8rjH//4B69zMAyDffv2wdPTEw899BCio6PRs2dP/PTTTwaKmpgKTQNNCLFIVPIihFgkSl6EEItEyYsQYpEoeRFCLBIlL0KIRaLkRQixSJS8CCEWiZIXIcQiUfIihFgkSl6EEItEyYsQYpEoeRFCLNL/A6Sxo/8MAerDAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018137, + "end_time": "2024-03-22T20:55:59.560151", + "exception": false, + "start_time": "2024-03-22T20:55:59.542014", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4549.120128, + "end_time": "2024-03-22T20:56:02.301034", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/lct_gan/1/mlu-eval.ipynb", + "output_path": "eval/contraceptive/lct_gan/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/lct_gan/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T19:40:13.180906", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/lct_gan/model.pt b/contraceptive/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5b8cc302c7ec41c8f80ff6cd23984af5ddebe57 --- /dev/null +++ b/contraceptive/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0bc27b52d8f3e6cb7d7a93df8356db260bb7d31f833b72a29a856dc5da2b511 +size 47605515 diff --git a/contraceptive/lct_gan/params.json b/contraceptive/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..d0b2370895e147c1a22f9fcee9c7468bcf1971f3 --- /dev/null +++ b/contraceptive/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/realtabformer/eval.csv b/contraceptive/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..b35996fa713eec718667924b65f820b4e2f220af --- /dev/null +++ b/contraceptive/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.010087525346489634,0.03493708366106431,0.002533305302744598,8.234226942062378,0.25083908438682556,6.304386138916016,0.44913777709007263,1.1285437722108327e-05,8.565300703048706,0.03761804848909378,0.12008487433195114,0.050331953912973404,0.09621766954660416,0.016873924061655998,16.799527645111084 diff --git a/contraceptive/realtabformer/history.csv b/contraceptive/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..a5e1cdd7d8d288f8483785f346afe77fa4eba298 --- /dev/null +++ b/contraceptive/realtabformer/history.csv @@ -0,0 +1,10 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.014773491756835332,0.5065707220365463,0.0010022340709901162,2.0442994761798117,0.0,0.0,0.0,0.0,0.015191531291671305,900,225,274.4165141582489,1.2196289518144396,0.3049072379536099,0.11575380941232045,0.007571417718944657,1.554786379107431,0.00017871774661915499,0.0,0.0,0.0,0.0,0.0,0.007571417718944657,450,113,91.69575762748718,0.8114668816591786,0.20376835028330484,0.061720233775296174 +1,0.007462754625658918,0.9658913450170937,0.00014313012270514552,0.8396876566774315,0.0,0.0,0.0,0.0,0.007641434397470827,900,225,273.0821771621704,1.213698565165202,0.3034246412913005,0.09523055639531877,0.006775144316876928,2.6273896950355087,9.84499310450578e-05,0.0,0.0,0.0,0.0,0.0,0.006775144316876928,450,113,91.63136982917786,0.8108970781343173,0.2036252662870619,0.046712872879249995 +2,0.004955892372494822,0.7045143008656533,7.975020945901844e-05,0.6091641951931848,0.0,0.0,0.0,0.0,0.005083349947817624,900,225,273.1290547847748,1.2139069101545545,0.30347672753863864,0.09947649084807685,0.0036339714314736838,3.456087975117349,5.305239131944066e-05,0.0,0.0,0.0,0.0,0.0,0.0036339714314736838,450,113,91.59913516044617,0.8106118155791696,0.20355363368988036,0.05323507617829384 +3,0.0038874727278339883,0.5685213102487542,3.110381803025462e-05,0.44001219677428405,0.0,0.0,0.0,0.0,0.003981491989947648,900,225,273.2383725643158,1.2143927669525147,0.3035981917381287,0.09838743486338192,0.00476446549566592,8.537542689921613,7.457373639723973e-05,0.0,0.0,0.0,0.0,0.0,0.00476446549566592,450,113,91.5707778930664,0.8103608663103222,0.20349061754014758,0.04942838404795353 +4,0.003344398294769538,0.42805399294468366,2.9547856750660956e-05,0.4788584218091435,0.0,0.0,0.0,0.0,0.0034413283928168108,900,225,273.74941539764404,1.2166640684339736,0.3041660171084934,0.10119029613832632,0.003117711971394278,3.474242146586163,4.511719159815966e-05,0.0,0.0,0.0,0.0,0.0,0.003117711971394278,450,113,91.81963801383972,0.8125631682640684,0.20404364003075492,0.06101849829712141 +5,0.003131055658159312,0.4019659260851485,2.7654838360707502e-05,0.4308759291966756,0.0,0.0,0.0,0.0,0.0032185360684201846,900,225,275.91382932662964,1.2262836858961317,0.3065709214740329,0.10080154451231162,0.0037931067653052095,2.7604550893965283,4.1587406659311395e-05,0.0,0.0,0.0,0.0,0.0,0.0037931067653052095,450,113,91.9511308670044,0.8137268218318973,0.20433584637112087,0.055654415548614236 +6,0.0029877640384883206,0.366934466270567,1.5397459504559052e-05,0.43426220549477473,0.0,0.0,0.0,0.0,0.003075523809238803,900,225,275.31064915657043,1.2236028851403131,0.3059007212850783,0.10231422000668115,0.003043726567929197,3.6605247210606398,3.023255854904125e-05,0.0,0.0,0.0,0.0,0.0,0.003043726567929197,450,113,93.64256858825684,0.8286952972412109,0.20809459686279297,0.04299795463994409 +7,0.0027128174370348763,0.6366902967303084,1.8083823394538535e-05,0.3685302308367358,0.0,0.0,0.0,0.0,0.0027878411782633825,900,225,278.0753390789032,1.2358903959062364,0.3089725989765591,0.0995295613238381,0.0033965686908535037,3.677261768688456,3.01374123970701e-05,0.0,0.0,0.0,0.0,0.0,0.0033965686908535037,450,113,94.00029134750366,0.8318609853761386,0.20888953632778592,0.048798565403043205 +8,0.00263197087019863,0.38799835741244154,1.5526932464568304e-05,0.40329953748318886,0.0,0.0,0.0,0.0,0.00271295235922379,900,225,275.622722864151,1.2249898793962266,0.30624746984905665,0.10333809573306806,0.0033174797300145856,3.981254602707383,1.7999465464147374e-05,0.0,0.0,0.0,0.0,0.0,0.0033174797300145856,450,113,91.82101058959961,0.8125753149522089,0.20404669019911023,0.048110959254796574 diff --git a/contraceptive/realtabformer/mlu-eval.ipynb b/contraceptive/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..5f968c82ec57aeb54af1a5e29b9518298b271ff9 --- /dev/null +++ b/contraceptive/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2273 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:32.810606Z", + "iopub.status.busy": "2024-03-22T18:25:32.810253Z", + "iopub.status.idle": "2024-03-22T18:25:32.844126Z", + "shell.execute_reply": "2024-03-22T18:25:32.843228Z" + }, + "papermill": { + "duration": 0.049518, + "end_time": "2024-03-22T18:25:32.846150", + "exception": false, + "start_time": "2024-03-22T18:25:32.796632", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:32.871980Z", + "iopub.status.busy": "2024-03-22T18:25:32.871271Z", + "iopub.status.idle": "2024-03-22T18:25:32.878825Z", + "shell.execute_reply": "2024-03-22T18:25:32.878026Z" + }, + "papermill": { + "duration": 0.022399, + "end_time": "2024-03-22T18:25:32.880774", + "exception": false, + "start_time": "2024-03-22T18:25:32.858375", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:32.905398Z", + "iopub.status.busy": "2024-03-22T18:25:32.904673Z", + "iopub.status.idle": "2024-03-22T18:25:32.908769Z", + "shell.execute_reply": "2024-03-22T18:25:32.907978Z" + }, + "papermill": { + "duration": 0.018577, + "end_time": "2024-03-22T18:25:32.910682", + "exception": false, + "start_time": "2024-03-22T18:25:32.892105", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:32.934272Z", + "iopub.status.busy": "2024-03-22T18:25:32.934005Z", + "iopub.status.idle": "2024-03-22T18:25:32.937751Z", + "shell.execute_reply": "2024-03-22T18:25:32.936972Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017915, + "end_time": "2024-03-22T18:25:32.939736", + "exception": false, + "start_time": "2024-03-22T18:25:32.921821", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:32.963325Z", + "iopub.status.busy": "2024-03-22T18:25:32.963068Z", + "iopub.status.idle": "2024-03-22T18:25:32.969414Z", + "shell.execute_reply": "2024-03-22T18:25:32.968762Z" + }, + "papermill": { + "duration": 0.020539, + "end_time": "2024-03-22T18:25:32.971254", + "exception": false, + "start_time": "2024-03-22T18:25:32.950715", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e6ac06ed", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:32.996699Z", + "iopub.status.busy": "2024-03-22T18:25:32.996164Z", + "iopub.status.idle": "2024-03-22T18:25:33.001946Z", + "shell.execute_reply": "2024-03-22T18:25:33.001170Z" + }, + "papermill": { + "duration": 0.020348, + "end_time": "2024-03-22T18:25:33.003845", + "exception": false, + "start_time": "2024-03-22T18:25:32.983497", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/realtabformer/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011496, + "end_time": "2024-03-22T18:25:33.027218", + "exception": false, + "start_time": "2024-03-22T18:25:33.015722", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:33.051797Z", + "iopub.status.busy": "2024-03-22T18:25:33.051536Z", + "iopub.status.idle": "2024-03-22T18:25:33.061460Z", + "shell.execute_reply": "2024-03-22T18:25:33.060625Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024661, + "end_time": "2024-03-22T18:25:33.063488", + "exception": false, + "start_time": "2024-03-22T18:25:33.038827", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/realtabformer/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:33.087949Z", + "iopub.status.busy": "2024-03-22T18:25:33.087654Z", + "iopub.status.idle": "2024-03-22T18:25:35.091545Z", + "shell.execute_reply": "2024-03-22T18:25:35.090625Z" + }, + "papermill": { + "duration": 2.018587, + "end_time": "2024-03-22T18:25:35.093760", + "exception": false, + "start_time": "2024-03-22T18:25:33.075173", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:35.121283Z", + "iopub.status.busy": "2024-03-22T18:25:35.120287Z", + "iopub.status.idle": "2024-03-22T18:25:35.133689Z", + "shell.execute_reply": "2024-03-22T18:25:35.132914Z" + }, + "papermill": { + "duration": 0.02882, + "end_time": "2024-03-22T18:25:35.135556", + "exception": false, + "start_time": "2024-03-22T18:25:35.106736", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:35.159865Z", + "iopub.status.busy": "2024-03-22T18:25:35.159587Z", + "iopub.status.idle": "2024-03-22T18:25:35.166500Z", + "shell.execute_reply": "2024-03-22T18:25:35.165637Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021545, + "end_time": "2024-03-22T18:25:35.168571", + "exception": false, + "start_time": "2024-03-22T18:25:35.147026", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:35.193192Z", + "iopub.status.busy": "2024-03-22T18:25:35.192895Z", + "iopub.status.idle": "2024-03-22T18:25:35.295635Z", + "shell.execute_reply": "2024-03-22T18:25:35.294503Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.118596, + "end_time": "2024-03-22T18:25:35.299353", + "exception": false, + "start_time": "2024-03-22T18:25:35.180757", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:35.345814Z", + "iopub.status.busy": "2024-03-22T18:25:35.345281Z", + "iopub.status.idle": "2024-03-22T18:25:40.035802Z", + "shell.execute_reply": "2024-03-22T18:25:40.034988Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.72311, + "end_time": "2024-03-22T18:25:40.038405", + "exception": false, + "start_time": "2024-03-22T18:25:35.315295", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 18:25:37.634605: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 18:25:37.634666: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 18:25:37.636412: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:40.064350Z", + "iopub.status.busy": "2024-03-22T18:25:40.063748Z", + "iopub.status.idle": "2024-03-22T18:25:40.069694Z", + "shell.execute_reply": "2024-03-22T18:25:40.068940Z" + }, + "papermill": { + "duration": 0.020969, + "end_time": "2024-03-22T18:25:40.071695", + "exception": false, + "start_time": "2024-03-22T18:25:40.050726", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:40.098056Z", + "iopub.status.busy": "2024-03-22T18:25:40.097756Z", + "iopub.status.idle": "2024-03-22T18:25:48.386241Z", + "shell.execute_reply": "2024-03-22T18:25:48.385081Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.30448, + "end_time": "2024-03-22T18:25:48.388661", + "exception": false, + "start_time": "2024-03-22T18:25:40.084181", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:25:48.416817Z", + "iopub.status.busy": "2024-03-22T18:25:48.415940Z", + "iopub.status.idle": "2024-03-22T18:25:48.423229Z", + "shell.execute_reply": "2024-03-22T18:25:48.422395Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023322, + "end_time": "2024-03-22T18:25:48.425255", + "exception": false, + "start_time": "2024-03-22T18:25:48.401933", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:48.452099Z", + "iopub.status.busy": "2024-03-22T18:25:48.451734Z", + "iopub.status.idle": "2024-03-22T18:25:48.457495Z", + "shell.execute_reply": "2024-03-22T18:25:48.456465Z" + }, + "papermill": { + "duration": 0.023243, + "end_time": "2024-03-22T18:25:48.460213", + "exception": false, + "start_time": "2024-03-22T18:25:48.436970", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:48.488503Z", + "iopub.status.busy": "2024-03-22T18:25:48.488167Z", + "iopub.status.idle": "2024-03-22T18:25:48.983967Z", + "shell.execute_reply": "2024-03-22T18:25:48.982986Z" + }, + "papermill": { + "duration": 0.511485, + "end_time": "2024-03-22T18:25:48.986631", + "exception": false, + "start_time": "2024-03-22T18:25:48.475146", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:49.016655Z", + "iopub.status.busy": "2024-03-22T18:25:49.016343Z", + "iopub.status.idle": "2024-03-22T18:25:49.337933Z", + "shell.execute_reply": "2024-03-22T18:25:49.337082Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.337712, + "end_time": "2024-03-22T18:25:49.339917", + "exception": false, + "start_time": "2024-03-22T18:25:49.002205", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:49.367429Z", + "iopub.status.busy": "2024-03-22T18:25:49.367146Z", + "iopub.status.idle": "2024-03-22T18:25:49.481593Z", + "shell.execute_reply": "2024-03-22T18:25:49.480556Z" + }, + "papermill": { + "duration": 0.130903, + "end_time": "2024-03-22T18:25:49.483846", + "exception": false, + "start_time": "2024-03-22T18:25:49.352943", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:25:49.517623Z", + "iopub.status.busy": "2024-03-22T18:25:49.516808Z", + "iopub.status.idle": "2024-03-22T18:25:49.965327Z", + "shell.execute_reply": "2024-03-22T18:25:49.964454Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.467912, + "end_time": "2024-03-22T18:25:49.968113", + "exception": false, + "start_time": "2024-03-22T18:25:49.500201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:50.001404Z", + "iopub.status.busy": "2024-03-22T18:25:50.000585Z", + "iopub.status.idle": "2024-03-22T18:25:50.005435Z", + "shell.execute_reply": "2024-03-22T18:25:50.004492Z" + }, + "papermill": { + "duration": 0.022319, + "end_time": "2024-03-22T18:25:50.007663", + "exception": false, + "start_time": "2024-03-22T18:25:49.985344", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:50.037114Z", + "iopub.status.busy": "2024-03-22T18:25:50.036796Z", + "iopub.status.idle": "2024-03-22T18:25:50.044082Z", + "shell.execute_reply": "2024-03-22T18:25:50.043211Z" + }, + "papermill": { + "duration": 0.023858, + "end_time": "2024-03-22T18:25:50.046140", + "exception": false, + "start_time": "2024-03-22T18:25:50.022282", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "12536352" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:50.073008Z", + "iopub.status.busy": "2024-03-22T18:25:50.072727Z", + "iopub.status.idle": "2024-03-22T18:25:50.157362Z", + "shell.execute_reply": "2024-03-22T18:25:50.156456Z" + }, + "papermill": { + "duration": 0.100317, + "end_time": "2024-03-22T18:25:50.159378", + "exception": false, + "start_time": "2024-03-22T18:25:50.059061", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 16128] --\n", + "├─Adapter: 1-1 [2, 1179, 16128] --\n", + "│ └─Embedding: 2-1 [2, 1179, 24, 672] (48,384)\n", + "│ └─TensorInductionPoint: 2-2 [24, 1] 24\n", + "│ └─Sequential: 2-3 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 689,152\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 16128] (recursive)\n", + "│ └─Embedding: 2-4 [2, 294, 24, 672] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [24, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 12,584,736\n", + "Trainable params: 12,536,352\n", + "Non-trainable params: 48,384\n", + "Total mult-adds (M): 46.91\n", + "========================================================================================================================\n", + "Input size (MB): 0.28\n", + "Forward/backward pass size (MB): 755.51\n", + "Params size (MB): 50.34\n", + "Estimated Total Size (MB): 806.13\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:50.189362Z", + "iopub.status.busy": "2024-03-22T18:25:50.189056Z", + "iopub.status.idle": "2024-03-22T19:30:55.813525Z", + "shell.execute_reply": "2024-03-22T19:30:55.812567Z" + }, + "papermill": { + "duration": 3905.656784, + "end_time": "2024-03-22T19:30:55.830437", + "exception": false, + "start_time": "2024-03-22T18:25:50.173653", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014773491756835332, 'avg_role_model_std_loss': 0.5065707220365463, 'avg_role_model_mean_pred_loss': 0.0010022340709901162, 'avg_role_model_g_mag_loss': 2.0442994761798117, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015191531291671305, 'n_size': 900, 'n_batch': 225, 'duration': 274.4165141582489, 'duration_batch': 1.2196289518144396, 'duration_size': 0.3049072379536099, 'avg_pred_std': 0.11575380941232045}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007571417718944657, 'avg_role_model_std_loss': 1.554786379107431, 'avg_role_model_mean_pred_loss': 0.00017871774661915499, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007571417718944657, 'n_size': 450, 'n_batch': 113, 'duration': 91.69575762748718, 'duration_batch': 0.8114668816591786, 'duration_size': 0.20376835028330484, 'avg_pred_std': 0.061720233775296174}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007462754625658918, 'avg_role_model_std_loss': 0.9658913450170937, 'avg_role_model_mean_pred_loss': 0.00014313012270514552, 'avg_role_model_g_mag_loss': 0.8396876566774315, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007641434397470827, 'n_size': 900, 'n_batch': 225, 'duration': 273.0821771621704, 'duration_batch': 1.213698565165202, 'duration_size': 0.3034246412913005, 'avg_pred_std': 0.09523055639531877}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006775144316876928, 'avg_role_model_std_loss': 2.6273896950355087, 'avg_role_model_mean_pred_loss': 9.84499310450578e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006775144316876928, 'n_size': 450, 'n_batch': 113, 'duration': 91.63136982917786, 'duration_batch': 0.8108970781343173, 'duration_size': 0.2036252662870619, 'avg_pred_std': 0.046712872879249995}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004955892372494822, 'avg_role_model_std_loss': 0.7045143008656533, 'avg_role_model_mean_pred_loss': 7.975020945901844e-05, 'avg_role_model_g_mag_loss': 0.6091641951931848, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005083349947817624, 'n_size': 900, 'n_batch': 225, 'duration': 273.1290547847748, 'duration_batch': 1.2139069101545545, 'duration_size': 0.30347672753863864, 'avg_pred_std': 0.09947649084807685}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036339714314736838, 'avg_role_model_std_loss': 3.456087975117349, 'avg_role_model_mean_pred_loss': 5.305239131944066e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036339714314736838, 'n_size': 450, 'n_batch': 113, 'duration': 91.59913516044617, 'duration_batch': 0.8106118155791696, 'duration_size': 0.20355363368988036, 'avg_pred_std': 0.05323507617829384}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038874727278339883, 'avg_role_model_std_loss': 0.5685213102487542, 'avg_role_model_mean_pred_loss': 3.110381803025462e-05, 'avg_role_model_g_mag_loss': 0.44001219677428405, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003981491989947648, 'n_size': 900, 'n_batch': 225, 'duration': 273.2383725643158, 'duration_batch': 1.2143927669525147, 'duration_size': 0.3035981917381287, 'avg_pred_std': 0.09838743486338192}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00476446549566592, 'avg_role_model_std_loss': 8.537542689921613, 'avg_role_model_mean_pred_loss': 7.457373639723973e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00476446549566592, 'n_size': 450, 'n_batch': 113, 'duration': 91.5707778930664, 'duration_batch': 0.8103608663103222, 'duration_size': 0.20349061754014758, 'avg_pred_std': 0.04942838404795353}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003344398294769538, 'avg_role_model_std_loss': 0.42805399294468366, 'avg_role_model_mean_pred_loss': 2.9547856750660956e-05, 'avg_role_model_g_mag_loss': 0.4788584218091435, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034413283928168108, 'n_size': 900, 'n_batch': 225, 'duration': 273.74941539764404, 'duration_batch': 1.2166640684339736, 'duration_size': 0.3041660171084934, 'avg_pred_std': 0.10119029613832632}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003117711971394278, 'avg_role_model_std_loss': 3.474242146586163, 'avg_role_model_mean_pred_loss': 4.511719159815966e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003117711971394278, 'n_size': 450, 'n_batch': 113, 'duration': 91.81963801383972, 'duration_batch': 0.8125631682640684, 'duration_size': 0.20404364003075492, 'avg_pred_std': 0.06101849829712141}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003131055658159312, 'avg_role_model_std_loss': 0.4019659260851485, 'avg_role_model_mean_pred_loss': 2.7654838360707502e-05, 'avg_role_model_g_mag_loss': 0.4308759291966756, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032185360684201846, 'n_size': 900, 'n_batch': 225, 'duration': 275.91382932662964, 'duration_batch': 1.2262836858961317, 'duration_size': 0.3065709214740329, 'avg_pred_std': 0.10080154451231162}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0037931067653052095, 'avg_role_model_std_loss': 2.7604550893965283, 'avg_role_model_mean_pred_loss': 4.1587406659311395e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037931067653052095, 'n_size': 450, 'n_batch': 113, 'duration': 91.9511308670044, 'duration_batch': 0.8137268218318973, 'duration_size': 0.20433584637112087, 'avg_pred_std': 0.055654415548614236}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029877640384883206, 'avg_role_model_std_loss': 0.366934466270567, 'avg_role_model_mean_pred_loss': 1.5397459504559052e-05, 'avg_role_model_g_mag_loss': 0.43426220549477473, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003075523809238803, 'n_size': 900, 'n_batch': 225, 'duration': 275.31064915657043, 'duration_batch': 1.2236028851403131, 'duration_size': 0.3059007212850783, 'avg_pred_std': 0.10231422000668115}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003043726567929197, 'avg_role_model_std_loss': 3.6605247210606398, 'avg_role_model_mean_pred_loss': 3.023255854904125e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003043726567929197, 'n_size': 450, 'n_batch': 113, 'duration': 93.64256858825684, 'duration_batch': 0.8286952972412109, 'duration_size': 0.20809459686279297, 'avg_pred_std': 0.04299795463994409}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0027128174370348763, 'avg_role_model_std_loss': 0.6366902967303084, 'avg_role_model_mean_pred_loss': 1.8083823394538535e-05, 'avg_role_model_g_mag_loss': 0.3685302308367358, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027878411782633825, 'n_size': 900, 'n_batch': 225, 'duration': 278.0753390789032, 'duration_batch': 1.2358903959062364, 'duration_size': 0.3089725989765591, 'avg_pred_std': 0.0995295613238381}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0033965686908535037, 'avg_role_model_std_loss': 3.677261768688456, 'avg_role_model_mean_pred_loss': 3.01374123970701e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033965686908535037, 'n_size': 450, 'n_batch': 113, 'duration': 94.00029134750366, 'duration_batch': 0.8318609853761386, 'duration_size': 0.20888953632778592, 'avg_pred_std': 0.048798565403043205}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00263197087019863, 'avg_role_model_std_loss': 0.38799835741244154, 'avg_role_model_mean_pred_loss': 1.5526932464568304e-05, 'avg_role_model_g_mag_loss': 0.40329953748318886, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00271295235922379, 'n_size': 900, 'n_batch': 225, 'duration': 275.622722864151, 'duration_batch': 1.2249898793962266, 'duration_size': 0.30624746984905665, 'avg_pred_std': 0.10333809573306806}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0033174797300145856, 'avg_role_model_std_loss': 3.981254602707383, 'avg_role_model_mean_pred_loss': 1.7999465464147374e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033174797300145856, 'n_size': 450, 'n_batch': 113, 'duration': 91.82101058959961, 'duration_batch': 0.8125753149522089, 'duration_size': 0.20404669019911023, 'avg_pred_std': 0.048110959254796574}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0024303717027133746, 'avg_role_model_std_loss': 0.2821750321023667, 'avg_role_model_mean_pred_loss': 1.0843823537028828e-05, 'avg_role_model_g_mag_loss': 0.3803096185210678, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002506096155072252, 'n_size': 900, 'n_batch': 225, 'duration': 273.3220579624176, 'duration_batch': 1.2147647020551893, 'duration_size': 0.3036911755137973, 'avg_pred_std': 0.10012033009280762}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002648681451359557, 'avg_role_model_std_loss': 3.9122540106362074, 'avg_role_model_mean_pred_loss': 1.250503946443018e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002648681451359557, 'n_size': 450, 'n_batch': 113, 'duration': 91.80838418006897, 'duration_batch': 0.8124635768147697, 'duration_size': 0.20401863151126437, 'avg_pred_std': 0.04869509997944537}\n", + "Time out: 3677.9721715450287/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0025333050786866805, 'avg_g_mag_loss': 0.021173561026940617, 'avg_g_cos_loss': 0.010191353678366709, 'pred_duration': 8.549674272537231, 'grad_duration': 8.245461463928223, 'total_duration': 16.795135736465454, 'pred_std': 0.09621766954660416, 'std_loss': 0.016873924061655998, 'mean_pred_loss': 1.128543954109773e-05, 'pred_rmse': 0.05033194646239281, 'pred_mae': 0.03761804476380348, 'pred_mape': 0.12008485943078995, 'grad_rmse': 0.44913792610168457, 'grad_mae': 0.2508392035961151, 'grad_mape': 6.304388523101807}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0025333050786866805, 'avg_g_mag_loss': 0.021173561026940617, 'avg_g_cos_loss': 0.010191353678366709, 'avg_pred_duration': 8.549674272537231, 'avg_grad_duration': 8.245461463928223, 'avg_total_duration': 16.795135736465454, 'avg_pred_std': 0.09621766954660416, 'avg_std_loss': 0.016873924061655998, 'avg_mean_pred_loss': 1.128543954109773e-05}, 'min_metrics': {'avg_loss': 0.0025333050786866805, 'avg_g_mag_loss': 0.021173561026940617, 'avg_g_cos_loss': 0.010191353678366709, 'pred_duration': 8.549674272537231, 'grad_duration': 8.245461463928223, 'total_duration': 16.795135736465454, 'pred_std': 0.09621766954660416, 'std_loss': 0.016873924061655998, 'mean_pred_loss': 1.128543954109773e-05, 'pred_rmse': 0.05033194646239281, 'pred_mae': 0.03761804476380348, 'pred_mape': 0.12008485943078995, 'grad_rmse': 0.44913792610168457, 'grad_mae': 0.2508392035961151, 'grad_mape': 6.304388523101807}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0025333050786866805, 'avg_g_mag_loss': 0.021173561026940617, 'avg_g_cos_loss': 0.010191353678366709, 'pred_duration': 8.549674272537231, 'grad_duration': 8.245461463928223, 'total_duration': 16.795135736465454, 'pred_std': 0.09621766954660416, 'std_loss': 0.016873924061655998, 'mean_pred_loss': 1.128543954109773e-05, 'pred_rmse': 0.05033194646239281, 'pred_mae': 0.03761804476380348, 'pred_mape': 0.12008485943078995, 'grad_rmse': 0.44913792610168457, 'grad_mae': 0.2508392035961151, 'grad_mape': 6.304388523101807}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:30:55.862924Z", + "iopub.status.busy": "2024-03-22T19:30:55.862624Z", + "iopub.status.idle": "2024-03-22T19:30:55.866856Z", + "shell.execute_reply": "2024-03-22T19:30:55.866123Z" + }, + "papermill": { + "duration": 0.022686, + "end_time": "2024-03-22T19:30:55.868769", + "exception": false, + "start_time": "2024-03-22T19:30:55.846083", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:30:55.899722Z", + "iopub.status.busy": "2024-03-22T19:30:55.899469Z", + "iopub.status.idle": "2024-03-22T19:30:55.989982Z", + "shell.execute_reply": "2024-03-22T19:30:55.989180Z" + }, + "papermill": { + "duration": 0.108762, + "end_time": "2024-03-22T19:30:55.992479", + "exception": false, + "start_time": "2024-03-22T19:30:55.883717", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:30:56.026229Z", + "iopub.status.busy": "2024-03-22T19:30:56.025811Z", + "iopub.status.idle": "2024-03-22T19:30:56.311006Z", + "shell.execute_reply": "2024-03-22T19:30:56.310073Z" + }, + "papermill": { + "duration": 0.304446, + "end_time": "2024-03-22T19:30:56.313060", + "exception": false, + "start_time": "2024-03-22T19:30:56.008614", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAQElEQVR4nO3de1xUdf748dcwMMNFLopyFUVTxLygaRBqWSsrGV0wy8111VpLa7Es6pu5lba1hVntuppl1i+tNvOym22pWYbZDRQvoJJKpnjnIioXQW4zn98fA5OjgAwMjAPv5+NxHsA5n3POe5Dz9nM+53M+H41SSiGEEA7Cyd4BCCGENSRpCSEciiQtIYRDkaQlhHAokrSEEA5FkpYQwqFI0hJCOBRnewfQWoxGI6dOncLT0xONRmPvcIQQl1BKUVJSQlBQEE5O9den2k3SOnXqFCEhIfYOQwhxBcePH6dr1671bm83ScvT0xMw/UK8vLzsHI0Q4lLFxcWEhISYr9X6tJukVXtL6OXlJUlLiKvYlZpvpCFeCOFQJGkJIRyKJC0hhENpN21aonkMBgNVVVX2DkM4MBcXF7RabbOPI0lLNEgpRW5uLoWFhfYORbQBPj4+BAQENKuvpCQt0aDahOXn54e7u7t0zBVNopSirKyM/Px8AAIDA5t8LElalyivMpB5sojKaiPDenW2dzh2ZTAYzAnL19fX3uEIB+fm5gZAfn4+fn5+Tb5VlIb4S3yzP497lqSS9OUBe4did7VtWO7u7naORLQVtX9LzWkflaR1iYiuPgAcyC2motpg32CuEnJLKGzFFn9LkrQu0bWjG508dFQZFPtzSuwdjhDiEpK0LqHRaBjY1RuAPScK7RuMEOIykrTqMLDmFjHjeKFd4xCOS6PR8Nlnn9k7DJt64YUXGDRokL3DkKRVl0EhtTWtIjtHIkTTLV++HB8fH5sd76mnniI5Odlmx2sqSVp1qK1pHTp9nvMV1fYNRogWVllZ2ahyHTp0uCq6vkjSqkPnDnqCfdxQCvZKbctMKUVZZbVdFmsnQt+4cSMjRozAx8cHX19fbr/9dg4dOgTAsGHDmDVrlkX506dP4+Liwvfffw9ATk4OcXFxuLm50aNHD1asWEFoaCgLFixo0u9u7969/O53v8PNzQ1fX1+mTZvG+fPnzdu3bNlCZGQkHh4e+Pj4MHz4cI4ePQrA7t27ueWWW/D09MTLy4shQ4awY8eOBs+3ZcsWHnjgAYqKitBoNGg0Gl544QUAQkNDeemll5g8eTJeXl5MmzYNgFmzZhEWFoa7uzs9e/bk+eeft+iacOnt4f333098fDyvv/46gYGB+Pr6kpCQ0OKve0nn0noM7OrNycIL7D5RSPQ19v/f5WpwocrAtXO+ssu5970Yi7uu8X+upaWlJCYmMnDgQM6fP8+cOXMYO3YsGRkZTJw4kfnz5zNv3jzzI/hVq1YRFBTEjTfeCMDkyZMpKChgy5YtuLi4kJiYaO7Nba3S0lJiY2OJjo5m+/bt5Ofn8+CDDzJjxgyWL19OdXU18fHxPPTQQ3zyySdUVlaSlpZmjm3ixIkMHjyYt99+G61WS0ZGBi4uLg2ec9iwYSxYsIA5c+aQlZUFmGpKtV5//XXmzJnD3Llzzes8PT1Zvnw5QUFB7N27l4ceeghPT0+efvrpes/z7bffEhgYyLfffsuvv/7KH/7wBwYNGsRDDz3UpN9VY0jSqsfArj58mZkrTxAd1Lhx4yx+fv/99+nSpQv79u1j/PjxPP744/z444/mJLVixQomTJiARqPhwIEDfPPNN2zfvp2hQ4cC8N5779G7d+8mxbJixQrKy8v58MMP8fDwAODNN9/kjjvu4NVXX8XFxYWioiJuv/12rrnmGgD69u1r3v/YsWP83//9H+Hh4QCNikOn0+Ht7Y1GoyEgIOCy7b/73e948sknLdY999xz5u9DQ0N56qmnWLlyZYNJq2PHjrz55ptotVrCw8OJi4sjOTlZkpY9RNQ0xu8+LreHtdxctOx7MdZu57bGwYMHmTNnDtu2baOgoACj0QiYEkD//v0ZPXo0H3/8MTfeeCPZ2dmkpqbyzjvvAJCVlYWzszPXXXed+Xi9evWiY8eOTYp9//79REREmBMWwPDhwzEajWRlZXHTTTdx//33Exsby+9//3tiYmIYP368+f28xMREHnzwQT766CNiYmK49957zcmtqWqT8cVWrVrFwoULOXToEOfPn6e6uvqKo/z269fP4nWcwMBA9u7d26zYrkTatOoxINgbjQZOFl7gzPkKe4dzVdBoNLjrnO2yWNuT+o477uDs2bO8++67bNu2jW3btgG/NTpPnDiR//znP1RVVbFixQoGDBjAgAEDbP47a6xly5aRmprKsGHDWLVqFWFhYWzduhUwtSX9/PPPxMXFsXnzZq699lrWrl3brPNdnEABUlNTmThxIrfddhvr1q0jPT2dZ5999oqN9Jfepmo0GvN/EC1FklY9PF1d6NnZ9A8rXR8cy5kzZ8jKyuK5555j1KhR9O3bl3PnzlmUueuuuygvL2fjxo2sWLGCiRMnmrf16dOH6upq0tPTzet+/fXXy47RWH379mX37t2Ulpaa1/300084OTnRp08f87rBgwcze/ZsUlJS6N+/PytWrDBvCwsL44knnuDrr7/m7rvvZtmyZVc8r06nw2Bo3KtoKSkpdO/enWeffZahQ4fSu3dv84OAq40krQZESCdTh9SxY0d8fX1ZunQpv/76K5s3byYxMdGijIeHB/Hx8Tz//PPs37+fCRMmmLeFh4cTExPDtGnTSEtLIz09nWnTpuHm5takd+cmTpyIq6srU6ZMITMzk2+//ZZHH32USZMm4e/vT3Z2NrNnzyY1NZWjR4/y9ddfc/DgQfr27cuFCxeYMWMGW7Zs4ejRo/z0009s377dos2rPqGhoZw/f57k5GQKCgooKyurt2zv3r05duwYK1eu5NChQyxcuLDZtbmWIkmrAREhPoC8zuNonJycWLlyJTt37qR///488cQTvPbaa5eVmzhxIrt37+bGG2+kW7duFts+/PBD/P39uemmmxg7dqz5SZqrq6vV8bi7u/PVV19x9uxZrr/+eu655x5GjRrFm2++ad5+4MABxo0bR1hYGNOmTSMhIYHp06ej1Wo5c+YMkydPJiwsjPHjxzNmzBj+9re/XfG8w4YN4+GHH+YPf/gDXbp0Yf78+fWWvfPOO3niiSeYMWMGgwYNIiUlheeff97qz9oaNMraDjAOqri4GG9vb4qKiho9hVj6sXOMfSsFXw8dO56LaXejHZSXl5OdnU2PHj2adLG2JSdOnCAkJIRvvvmGUaNG2Tsch9XQ31Rjr1F5etiAvoFeODtpOFNaycnCC3TtKONKtRebN2/m/PnzDBgwgJycHJ5++mlCQ0O56aab7B1au9ek28PFixcTGhqKq6srUVFRpKWlNVh+zZo1hIeH4+rqyoABA9iwYYPF9k8//ZTRo0fj6+uLRqMhIyOj3mMppRgzZkyrvJDq6qIlPNA02610fWhfqqqq+Otf/0q/fv0YO3YsXbp0MXc0/fjjj+nQoUOdS79+/VotxjFjxtQbxyuvvNJqcbQ2q2taq1atIjExkSVLlhAVFcWCBQuIjY0lKysLPz+/y8qnpKQwYcIEkpKSuP3221mxYgXx8fHs2rWL/v37A6YewyNGjGD8+PFX7JS2YMGCVr1Ni+jqQ+bJYvacKCRuYNPHtRaOJTY2ltjYuvuk3XnnnURFRdW57Uo91W3pvffe48KFC3Vu69SpU6vF0eqUlSIjI1VCQoL5Z4PBoIKCglRSUlKd5cePH6/i4uIs1kVFRanp06dfVjY7O1sBKj09vc5jpaenq+DgYJWTk6MAtXbt2kbHXVRUpABVVFTU6H2UUmpV2jHVfdY69Yd3Uqzary24cOGC2rdvn7pw4YK9QxFtREN/U429Rq26PaysrGTnzp3ExMSY1zk5ORETE0Nqamqd+6SmplqUB9P/YvWVr09ZWRl//OMfWbx4cZ2vJVyqoqKC4uJii6UpBtb0jM88WYzB2C6eWQhxVbMqaRUUFGAwGPD397dY7+/vT25ubp375ObmWlW+Pk888QTDhg3jrrvualT5pKQkvL29zUtISIhV56vVq0sH3Fy0nK+o5vDp81feQQjRohyin9bnn3/O5s2brRoWZPbs2RQVFZmX48ePN+nczlon+gebHr/ulp7xQtidVUmrc+fOaLVa8vLyLNbn5eXVe8sWEBBgVfm6bN68mUOHDuHj44OzszPOzqbnB+PGjePmm2+ucx+9Xo+Xl5fF0lS1PeOlk6kQ9mdV0tLpdAwZMsRiyFWj0UhycjLR0dF17hMdHX3ZEK2bNm2qt3xdnnnmGfbs2UNGRoZ5AfjnP//ZqHewmmtgTc94qWkJYX9Wd3lITExkypQpDB06lMjISBYsWEBpaSkPPPAAYBo8LTg4mKSkJABmzpzJyJEjeeONN4iLi2PlypXs2LGDpUuXmo959uxZjh07xqlTpwDMg5YFBARYLJfq1q0bPXr0sP5TWymiZnae/aeKqaw2onN2iLtqYUcajYa1a9cSHx9v71DaHKuvvj/84Q/mUQ8HDRpERkYGGzduNDe2Hzt2jJycHHP5YcOGsWLFCpYuXUpERAT/+c9/+Oyzz8x9tMDUZjV48GDi4uIAuO+++xg8eDBLlixp7ueziW6d3PFxd6HSYORAbtOeQgrR2mw9sQWYhnHWaDQUFhba9LjWaNJrPDNmzGDGjBl1btuyZctl6+69917uvffeeo93//33c//991sVg2rFVyZNcyH68P0vp9l9osg88YUQovXJfU4j1d4i7mnPw9QoBZWl9llkYgubTWxRUVHBU089RXBwMB4eHkRFRVlUNo4ePcodd9xBx44d8fDwoF+/fmzYsIEjR45wyy23AKbhfzQajdWVDVuQF6YbqbZ2tbs9P0GsKoNXguxz7r+eAp3HlcvVkIkt6p/YYsaMGezbt4+VK1cSFBTE2rVrufXWW9m7dy+9e/cmISGByspKvv/+ezw8PNi3bx8dOnQgJCSE//73v4wbN46srCy8vLxwc3Nr0u+kOSRpNVJtTevX/POUVlTjoZdf3dVMJraoe2KLY8eOsWzZMo4dO0ZQkOk/oKeeeoqNGzeybNkyXnnlFY4dO8a4cePMw0/37NnTvH/tO41+fn42by9rLLnyGsnPy5VAb1dyisrJPFlEVM92OK2Yi7upxmOvc1tBJrao2969ezEYDISFhVmsr6ioME/E+thjj/HII4/w9ddfExMTw7hx4xg4cGCTztcSpE3LCgNr27Xaa38tjcZ0i2aPRSa2sMnEFufPn0er1bJz506Lfo/79+/nX//6FwAPPvgghw8fZtKkSezdu5ehQ4eyaNEim33W5pKkZQVp13IMMrGFSV0TWwwePBiDwUB+fj69evWyWC6+jQwJCeHhhx/m008/5cknn+Tdd981HxNo9IQZLUGSlhUiJGk5BJnYwqSuiS3CwsKYOHEikydP5tNPPyU7O5u0tDSSkpJYv349AI8//jhfffUV2dnZ7Nq1i2+//dZ8vu7du6PRaFi3bh2nT5+2eALaalpo2JyrTlPH07pYYVml6j5rneo+a506c77ChtFdnRx5PK1Nmzapvn37Kr1erwYOHKi2bNly2RhsGzZsUIC66aabLtv/1KlTasyYMUqv16vu3burFStWKD8/P7VkyZJGnf/Sc+3Zs0fdcsstytXVVXXq1Ek99NBDqqSkRCmlVG5uroqPj1eBgYFKp9Op7t27qzlz5iiDwaAqKirUfffdp0JCQpROp1NBQUFqxowZjf43efjhh5Wvr68C1Ny5c5VSSlVWVqo5c+ao0NBQ5eLiogIDA9XYsWPVnj17lFJKzZgxQ11zzTVKr9erLl26qEmTJqmCggLzMV988UUVEBCgNBqNmjJlSqPiqGWL8bRkYgsr/e71LRwuKGX5A9dzc5/LR2ptS2Rii9/IxBa2YYuJLeT20ErtvjG+ndi8eTOff/452dnZpKSkcN9998nEFlcJSVpWMjfGt+ee8e2ATGxx9ZJ+WlaKqBl+efeJIpRS7W4uxPZCJra4eknSslK/IG+0ThoKzleQU1ROkE/rv8Yg7MvT0xNPT097h0FwcLC9Q7ALuT20kquLlj7+pj/Y9jKSaTt5ViNagS3+liRpNUHtLWJGG5/AtfZWp6yszM6RiLai9m+pObfRcnvYBAO7+vBJ2vE2X9PSarX4+PiYRzdwd3eXNjzRJEopysrKyM/Px8fHB61W2+RjSdJqgtqe8XtPFGE0Kpyc2u6FXPtqR1OHZRHiYj4+PlZNalMXSVpNEObfAVcXJ0oqqsk+U8o1XTrYO6QWo9FoCAwMxM/Pj6qqKnuHIxyYi4tLs2pYtSRpNYGz1ol+Qd7sPHqO3ccL23TSqqXVam3yBydEc0lDfBNJz3gh7EOSVhMNMs+FWGjXOIRobyRpNVHt6zw/nyqmymC0bzBCtCOStJoo1NcdL1dnKquNZOWW2DscIdqNJiWtxYsXExoaiqurK1FRUaSlpTVYfs2aNYSHh+Pq6sqAAQPYsGGDxfZPP/2U0aNH4+vri0ajMU97X+vs2bM8+uij9OnTBzc3N7p168Zjjz1GUZH92pNq50IEuUUUojVZnbRWrVpFYmIic+fOZdeuXURERBAbG1tvP56UlBQmTJjA1KlTSU9PJz4+nvj4eDIzM81lSktLGTFiBK+++mqdxzh16hSnTp3i9ddfJzMzk+XLl7Nx40amTp1qbfg2ZW6Mb+M944W4qlg17KBSKjIyUiUkJJh/NhgMKigoSCUlJdVZfvz48SouLs5iXVRUlJo+ffplZbOzsxWg0tPTrxjH6tWrlU6nU1VVVXVuLy8vV0VFRebl+PHjzR659FIbM3NU91nrVOw/v7PZMYVorxo7cqlVNa3Kykp27txJTEyMeZ2TkxMxMTGkpqbWuU9qaqpFeTAN+1Ff+caqHd3Q2bnurmZJSUl4e3ubl5CQkGadry61PeMP5p+nrLLa5scXQlzOqqRVUFCAwWDA39/fYr2/vz+5ubl17pObm2tV+cbG8dJLLzFt2rR6y8yePZuioiLzcvz48Safrz4B3q74eeoxGBU/nyq2+fGFEJdzuKeHxcXFxMXFce211/LCCy/UW06v1+Pl5WWxtAQZyVSI1mVV0urcuTNarZa8vDyL9Xl5efW+BBkQEGBV+YaUlJRw66234unpydq1a1t1lMj6DAqRnvFCtCarkpZOp2PIkCEkJyeb1xmNRpKTk4mOjq5zn+joaIvyAJs2baq3fH2Ki4sZPXo0Op2Ozz///KqZHaa2ptXWh6kR4mph9QvTiYmJTJkyhaFDhxIZGcmCBQsoLS3lgQceAGDy5MkEBweTlJQEwMyZMxk5ciRvvPEGcXFxrFy5kh07drB06VLzMc+ePcuxY8c4deoUAFlZWYCplhYQEGBOWGVlZfz73/+muLiY4mJTG1KXLl3s+iJvbbeHI2fKKCyrxMddZ7dYhGgXmvJoctGiRapbt25Kp9OpyMhItXXrVvO2kSNHXjaB4+rVq1VYWJjS6XSqX79+av369Rbbly1bpoDLltrJJb/99ts6twMqOzu7UTHbYrLW+tw0f7PqPmud+i4r3+bHFqK9kMlaL2GryVrr8tgn6Xy++xRPjQ5jxu962/TYQrQXMllrK6q9RdwtjfFCtDhJWjYQUTtMjXR7EKLFSdKygX5BXjhpIL+kgtyicnuHI0SbJknLBtx1zoTVzIUoIz4I0bIkadlIhPTXEqJVSNKykYE1PeN3yzA1QrQoSVo2cnFNq530IhHCLiRp2UifAE90zk4Ul1dz5IxMIy9ES5GkZSMuWif6BZk6xEm7lhAtR5KWDUWYh6mRdi0hWookLRv6rWd8oX0DEaINk6RlQ7/NhVhEtcyFKESLkKRlQz07e+Cpd6a8ysgveeftHY4QbZIkLRtyctIwoHZaMblFFKJFSNKyMZnAVYiWJUnLxiK6Ss94IVqSJC0bG1gzTE1WXgnlVQb7BiNEGyRJy8aCvF3p3EHmQhSipUjSsjGNRnPRLWKhfYMRog2SpNUCZFoxIVqOJK0WMFAmcBWixUjSagG17yAeLiil6EKVfYMRoo1pUtJavHgxoaGhuLq6EhUVRVpaWoPl16xZQ3h4OK6urgwYMIANGzZYbP/0008ZPXo0vr6+aDQaMjIyLjtGeXk5CQkJ+Pr60qFDB8aNG0deXl5Twm9xnTx0hHRyA2Cv1LaEsCmrk9aqVatITExk7ty57Nq1i4iICGJjY8nPz6+zfEpKChMmTGDq1Kmkp6cTHx9PfHw8mZmZ5jKlpaWMGDGCV199td7zPvHEE3zxxResWbOG7777jlOnTnH33XdbG36rkU6mQrQQa2eBjYyMVAkJCeafDQaDCgoKUklJSXWWHz9+vIqLi7NYFxUVpaZPn35Z2ezsbAWo9PR0i/WFhYXKxcVFrVmzxrxu//79ClCpqamNirslZ5iuyzvf/aq6z1qnpn24vVXOJ4Sja+w1alVNq7Kykp07dxITE2Ne5+TkRExMDKmpqXXuk5qaalEeIDY2tt7yddm5cydVVVUWxwkPD6dbt271HqeiooLi4mKLpTX9Nvyy3B4KYUtWJa2CggIMBgP+/v4W6/39/cnNza1zn9zcXKvK13cMnU6Hj49Po4+TlJSEt7e3eQkJCWn0+Wyhf7A3ThrIKSonv1jmQhTCVtrs08PZs2dTVFRkXo4fP96q5/fQO9PLrwMAu6W2JYTNWJW0OnfujFarveypXV5eHgEBAXXuExAQYFX5+o5RWVlJYWFho4+j1+vx8vKyWFqbdDIVwvasSlo6nY4hQ4aQnJxsXmc0GklOTiY6OrrOfaKjoy3KA2zatKne8nUZMmQILi4uFsfJysri2LFjVh2ntZlf55GalhA242ztDomJiUyZMoWhQ4cSGRnJggULKC0t5YEHHgBg8uTJBAcHk5SUBMDMmTMZOXIkb7zxBnFxcaxcuZIdO3awdOlS8zHPnj3LsWPHOHXqFGBKSGCqYQUEBODt7c3UqVNJTEykU6dOeHl58eijjxIdHc0NN9zQ7F9CS4moGfGhdi5EjUZj34CEaAua8mhy0aJFqlu3bkqn06nIyEi1detW87aRI0eqKVOmWJRfvXq1CgsLUzqdTvXr10+tX7/eYvuyZcsUcNkyd+5cc5kLFy6ov/zlL6pjx47K3d1djR07VuXk5DQ65tbu8qCUUhVVBtX7rxtU91nr1JGC8612XiEcUWOvUY1S7WM65OLiYry9vSkqKmrV9q273vyR3SeKWDhhMHdGBLXaeYVwNI29Rtvs08OrhbkxXoapEcImJGm1sN/ataQxXghbkKTVwmqfIO49KXMhCmELkrRaWM8uHfDQablQZeDX0zIXohDNJUmrhWmdNPQPrhkUUGboEaLZJGm1gkE17VoyTI0QzSdJqxXI2FpC2I4krVYwsKYx/kCOzIUoRHNJ0moFXTu60clDR7VRsT9H5kIUojkkabUCjUZjrm1Jfy0hmkeSViupHclUJnAVonkkabWSiJDaYWoK7RuIEA5OklYrGXjRXIgl5TIXohBNJUmrlXTuoCfYxw2lTK/0CCGaRpJWKzLfIkrPeCGaTJJWK5Ix44VoPklarUi6PQjRfJK0WtGAYG80GjhZeIGC8xX2DkcIhyRJqxV5urpwTRfTXIhyiyhE00jSamW1t4gZ0hgvRJNI0mplEdIYL0SzSNJqZRc3xreTiZCEsClJWq2sb6AXzk4azpZWcuLcBXuHI4TDaVLSWrx4MaGhobi6uhIVFUVaWlqD5desWUN4eDiurq4MGDCADRs2WGxXSjFnzhwCAwNxc3MjJiaGgwcPWpT55ZdfuOuuu+jcuTNeXl6MGDGCb7/9tinh25Wri5a+gaY53eQ9RCGsZ3XSWrVqFYmJicydO5ddu3YRERFBbGws+fn5dZZPSUlhwoQJTJ06lfT0dOLj44mPjyczM9NcZv78+SxcuJAlS5awbds2PDw8iI2Npby83Fzm9ttvp7q6ms2bN7Nz504iIiK4/fbbyc3NbcLHti/pryVEM1g7dXVkZKRKSEgw/2wwGFRQUJBKSkqqs/z48eNVXFycxbqoqCg1ffp0pZRSRqNRBQQEqNdee828vbCwUOn1evXJJ58opZQ6ffq0AtT3339vLlNcXKwAtWnTpkbF3dgpt1vDqrRjqvusdWr8khR7hyLEVaOx16hVNa3Kykp27txJTEyMeZ2TkxMxMTGkpqbWuU9qaqpFeYDY2Fhz+ezsbHJzcy3KeHt7ExUVZS7j6+tLnz59+PDDDyktLaW6upp33nkHPz8/hgwZUud5KyoqKC4utliuFgNr3kHMPFmEwSiN8UJYw6qkVVBQgMFgwN/f32K9v79/vbdpubm5DZav/dpQGY1GwzfffEN6ejqenp64urryj3/8g40bN9KxY8c6z5uUlIS3t7d5CQkJseajtqjefp6467SUVho4JHMhCmEVh3h6qJQiISEBPz8/fvjhB9LS0oiPj+eOO+4gJyenzn1mz55NUVGReTl+/HgrR10/rZOG/kG1Iz4U2jcYIRyMVUmrc+fOaLVa8vLyLNbn5eUREBBQ5z4BAQENlq/92lCZzZs3s27dOlauXMnw4cO57rrreOutt3Bzc+ODDz6o87x6vR4vLy+L5WoijfFCNI1VSUun0zFkyBCSk5PN64xGI8nJyURHR9e5T3R0tEV5gE2bNpnL9+jRg4CAAIsyxcXFbNu2zVymrKzMFKyTZbhOTk4YjUZrPsJVY2DNBK7SM14IK1nbwr9y5Uql1+vV8uXL1b59+9S0adOUj4+Pys3NVUopNWnSJPXMM8+Yy//000/K2dlZvf7662r//v1q7ty5ysXFRe3du9dcZt68ecrHx0f973//U3v27FF33XWX6tGjh7pw4YJSyvT00NfXV919990qIyNDZWVlqaeeekq5uLiojIyMRsV9NT09VEqpowWlqvusdarXX9er8qpqe4cjhN019hq1OmkppdSiRYtUt27dlE6nU5GRkWrr1q3mbSNHjlRTpkyxKL969WoVFhamdDqd6tevn1q/fr3FdqPRqJ5//nnl7++v9Hq9GjVqlMrKyrIos337djV69GjVqVMn5enpqW644Qa1YcOGRsdsVdLK2aOU0djoYzeF0WhUg/72leo+a536/pf8Fj2XEI6gsdeoRqn28QJccXEx3t7eFBUVNdy+dfYwLBoKgQPhlmehVwxoNC0S07Nr9/LxtmP08fdk3WMjcNE6xHMRIVpEY69RuUoulbMHnF3hVDp8fA/8v9/Doc3QArn9qdF96OjuQlZeCe//mG3z4wvRFknSulS/eJi5G4Y9Cs5ucGI7fDQWlo2B7O9teqqOHjpm39YXgAXfHORkobxALcSVSNKqS4cuMPrvpuR1w19MNa9jqfDBHbAsDo78ZLNT3XNdV64P7ciFKgN/+/xnmx1XiLZKklZDPP3h1iR4LAMip4NWB0d/hOW3wQd3wrGtzT6Fk5OGv8cPwNlJw9f78kjen3flnYRoxyRpNYZXINw235S8hk4FJxfI/g7ej4WP7oYTO5p1+D4Bnkwd0QOAuZ//zIVKgw2CFqJtkqRlDe9guP0f8NguGHI/ODnDoWR4bxR8fC+c3NXkQz82qjdB3q6cOHeBRZsPXnkHIdopSVpN4dMN7vgXPLoTBv8JNFo4+DW8ewt8MgFydlt9SA+9M3Pv7AfAuz8c5tf8EltHLUSbIEmrOTqGwl2LYcZ2iJgAGifI2gDv3ASr/gR51jWsj77Wn1HhflQZFM+uzZQx5IWogyQtW/C9BsYugYQ0GDAe0MD+L+DtYbB6CuTvb9RhNBoNL9zZD1cXJ7Zln+XTXSdbNm4hHJAkLVvq3BvGvQt/2Qr97gY0sO8zeCsa/jMVTv9yxUOEdHLnsVG9AXhlw34KyypbNmYhHIwkrZbgFw73LoNHUuDauwAFmf+Bt6Lg0+lw5lCDuz84oie9/TpwprSS+V9ltU7MQjgISVotyf9aGP8hPPwjhN8Oygh7VsKb18Nnf4Gzdb+6o3N24qX4/gB8knaM9GPnWjNqIa5qkrRaQ8AAuO9jmLYFwm4FZYCMj+HNobD+STBUXbbLDT19ufu6YJSCZ9dmUm1wzHHDhLA1SVqtKWgw/HEVPLjZNHqEsRq2vwc7l9dZ/K+39cXbzYV9OcV8mHq0dWMV4iolScseug6BP/0XYl8x/fzDP6Cq/LJinTvoefrWPgD8Y9Mv5BVfXkaI9kaSlj1d/yB4BkHJKUj/qM4iE67vxqAQH85XVPPiun2tHKAQVx9JWvbkrIcbE03f11PbMr1Q3R8nDazfk8P3v5xu5SCFuLpI0rK36yZfsbbVP9ibKcNCAXj+f5mUV8kL1aL9kqRlb42obQEk/j4Mfy89R8+U8daWhvt5CdGWSdK6GjSituXp6sKc200vVC/ZcojDMjO1aKckaV0NLq1tVVfUWey2AQHcFNaFSoOROf/7WV6oFu2SJK2rxcW1rV0f1llEo9Hw4p390Dk78eOvBXyxJ6eVgxTC/iRpXS0aWdsK7exBws29AHhp3T6Kyy/vTS9EW9akpLV48WJCQ0NxdXUlKiqKtLS0BsuvWbOG8PBwXF1dGTBgABs2bLDYrpRizpw5BAYG4ubmRkxMDAcPXj565/r164mKisLNzY2OHTsSHx/flPCvXo2obQE8fHNPenT24HRJBf/4+sojRwjRllidtFatWkViYiJz585l165dREREEBsbS35+fp3lU1JSmDBhAlOnTiU9PZ34+Hji4+PJzMw0l5k/fz4LFy5kyZIlbNu2DQ8PD2JjYykv/+1J2n//+18mTZrEAw88wO7du/npp5/44x//2ISPfBVrZG1L76zlpbtML1R/mHqEzJNFrRWhEPZn7dTVkZGRKiEhwfyzwWBQQUFBKikpqc7y48ePV3FxcRbroqKi1PTp05VSpunhAwIC1GuvvWbeXlhYqPR6vfrkk0+UUkpVVVWp4OBg9d577zU6zvLyclVUVGRejh8/3qgpt+2uqlyp18OVmuul1LalDRadsWKX6j5rnbpz0Q+q2mBspQCFaBlFRUWNukatqmlVVlayc+dOYmJizOucnJyIiYkhNTW1zn1SU1MtygPExsaay2dnZ5Obm2tRxtvbm6ioKHOZXbt2cfLkSZycnBg8eDCBgYGMGTPGorZ2qaSkJLy9vc1LSEiINR/VfhpZ2wJ4Pq4vnnpndp8oYkXasVYKUAj7sippFRQUYDAY8Pf3t1jv7+9Pbm5unfvk5uY2WL72a0NlDh8+DMALL7zAc889x7p16+jYsSM333wzZ8+erfO8s2fPpqioyLwcP37cmo9qX41s2/LzcuXJ0WEAzN94gPwSeaFatH0O8fTQaDSNJfXss88ybtw4hgwZwrJly9BoNKxZs6bOffR6PV5eXhaLw7CitjUpOpQBwd6UlFfzyvrGjUUvhCOzKml17twZrVZLXp7lLMh5eXkEBATUuU9AQECD5Wu/NlQmMDAQgGuvvda8Xa/X07NnT44da6O3RY2sbWmdNLw8tj8aDXyWcYqUXwtaMUghWp9VSUun0zFkyBCSk5PN64xGI8nJyURHR9e5T3R0tEV5gE2bNpnL9+jRg4CAAIsyxcXFbNu2zVxmyJAh6PV6srJ+Gy+9qqqKI0eO0L17d2s+guOworY1sKsPf4oy/R6e+18mFdXyQrVow6xt4V+5cqXS6/Vq+fLlat++fWratGnKx8dH5ebmKqWUmjRpknrmmWfM5X/66Sfl7OysXn/9dbV//341d+5c5eLiovbu3WsuM2/ePOXj46P+97//qT179qi77rpL9ejRQ124cMFcZubMmSo4OFh99dVX6sCBA2rq1KnKz89PnT17tlFxN/bJxFXFiieJhWWVashLm1T3WevUouRfWilAIWynsdeo1UlLKaUWLVqkunXrpnQ6nYqMjFRbt241bxs5cqSaMmWKRfnVq1ersLAwpdPpVL9+/dT69estthuNRvX8888rf39/pdfr1ahRo1RWVpZFmcrKSvXkk08qPz8/5enpqWJiYlRmZmajY3bIpKWUKVnN9TIlr6ryBouu3XVCdZ+1ToU9u0EdLShtpQCFsI3GXqMapdrHW7fFxcV4e3tTVFTkWI3yVeWwcLCpbeu21yHyoXqLKqWY+N42Ug6d4ZY+XXj//uvRaDStGKwQTdfYa9Qhnh62ay6ujW7b0mg0vHhXf1y0Gr7NOs1XP9fdDUUIRyZJyxEMntSoJ4kAvfw6MP2mawD42xf7KK2obo0IhWg1krQcwcW1rR//2WBtC2DG73oR0smNnKJyFnwjL1SLtkWSlqOorW0Vn6x3dNNari5aXrzT9EL1+z8dYX9OcWtEKESrkKTlKKxo2wK4JdyPMf0DMBgVz67di9HYLp63iHZAkpYjsaK2BTDnjmvx0GnZdayQ1Tsc6N1LIRogScuRWFnbCvR244nfm16onrfxAGdLK1s6QiFanCQtR2Nlbev+YaGEB3hSWFbF1A+2c+xMWSsEKUTLkaTlaKysbTlrnZh/z0A66J1JP1bImH99z5odx2UmH+GwJGk5IitrWwO7+vDlzBu5PrQjpZUG/u8/e3jk37s4J7eLwgFJ0nJEVta2AEI6ubNyWjRP39oHZycNG3/OJXbB93z3y+kWDlYI25Kk5aisrG2Baeytv9zci88ShnNNFw/ySyqY8n4aL3z+M+VVMpyNcAyStBxVE2pbtfoHe7Pu0RuZHG0ag2t5yhHuWPRj82f1KS+GDU/DFzOh6kLzjiVEPSRpObIm1LZquem0vHhXf5Y9cD1dPPUczD/P2Ld+4u0thzA0pSPq8e2wZASkvQM7l8PqKVAtbWbC9iRpObJm1LZq3dLHj68ev4nR1/pTZVC8uvEAE97dyolzjewaYTTA96/B+7FQeBS8uoKzGxz8Cj59CAzywrawLUlajm7wJPAMbFJtq1YnDx3vTBrC/HEDcddpScs+y5gFP/BZ+smGu0YUnYAP7oTNfwdlgH53wyM/wX3/Bq0O9n0Gnz8KNROTCGELkrQcnYsrjKitbV15BIj6aDQaxl8fwpczb2RwNx9KKqp5fFUGj36STlFZ1eU77Psc3h4OR38EFw+IfxvueR/cfKBXDNyzDDRa2L0Cvvw/kH5hwkYkabUF102uqW2dgPR/N+tQ3X09WDM9msTfh6F10rBuTw63/uv732b5qSw1NbSvngTlhRA0GB7+AQb9ES4eJbXv7TD2HUAD29+DTXMkcQmbkKTVFljUtprWtnUxZ60Tj43qzX8fGUaPzh7kFJXzx/e28d6azzC+M9LU0I4Ghj8Of/4afK+p+0AD74U7/mX6PmWhqe1LiGaSpNVW2LC2VWtQiA/rHxvBxMiu/Fn7JZMyp+J05iBV7n4w+TP4/d/AWdfwQYZMgdgk0/ffvgwpb9okNtF+SdJqK2xc26rlXnmWl8teZI7LR+g11WwyDGFE0d9572S3xo/RFf0X+N1zpu+/fha2/z+bxCbaJ0labYmta1sHv4G3h8Gv34CzKyWj5rGy5zzyDB34+/r9/On/bSOnqJGdSG98CkY8Yfp+/ZOwe2Xz4xPtUpOS1uLFiwkNDcXV1ZWoqCjS0tIaLL9mzRrCw8NxdXVlwIABbNiwwWK7Uoo5c+YQGBiIm5sbMTExHDx4sM5jVVRUMGjQIDQaDRkZGU0Jv+2yVW2rugI2zoaPx0HpafC7Fh76Fs8bH+G9+6/n5bH9cXPRknLoDLH//J51e05d+ZgaDYyaC5HTAQWfPQL7/te0+ES7ZnXSWrVqFYmJicydO5ddu3YRERFBbGws+fn5dZZPSUlhwoQJTJ06lfT0dOLj44mPjyczM9NcZv78+SxcuJAlS5awbds2PDw8iI2Npby8/LLjPf300wQFBVkbdvvR3NrW6Sx4dxRsfcv0c+Q0eGgz+F8LmLpGTIzqzvrHRhDR1Zvi8mpmrEjniVUZFJfX0TXiYhoN3DoPBv8JlBH+MxUObrI+RtG+WTsLbGRkpEpISDD/bDAYVFBQkEpKSqqz/Pjx41VcXJzFuqioKDV9+nSllGl26YCAAPXaa6+ZtxcWFiq9Xq8++eQTi/02bNigwsPD1c8//6wAlZ6e3ui4HXaG6abY+o5pVuo3rr3irNRmRqNSO5Yp9ZK/ad9Xeyh14MsGd6msNqg3vjqgejyzTnWftU4NS0pWWw8VXPlchmqlVt9vOs9Lfkod/r5xMYo2rbHXqFU1rcrKSnbu3ElMTIx5nZOTEzExMaSmpta5T2pqqkV5gNjYWHP57OxscnNzLcp4e3sTFRVlccy8vDweeughPvroI9zd3a8Ya0VFBcXFxRZLu2FtbavsrKnf1RczofoC9LwZHkmBPrc2uJuL1onE0X1Y83A03Tq5c7LwAve9u5UXPv+ZA7nF9femd9LC3UshbAxUl8OKP5jeXRSiEaxKWgUFBRgMBvz9/S3W+/v7k5tb92zGubm5DZav/dpQGaUU999/Pw8//DBDhw5tVKxJSUl4e3ubl5CQkEbt1yZY07Z15EfTi877vwAnF/j9S/CnteAZ0OjTDeneiQ0zb2T80K4oZRo14tYFPzD6n9+zMPkg2QWll++kdYF7l5sSZFWpqf0sZ49VH1O0Tw7x9HDRokWUlJQwe/bsRu8ze/ZsioqKzMvx4+1sNpor1bYMVZD8Eiy/3fTeYqdr4MFNMPwxcLL+z6KD3pn590Sw/IHrGX2tPzqtEwfzz/OPTb9wy+tbuH3RDyz9/hAnCy962ujiCvetgJAboLwIPhpralMTogFW/XV27twZrVZLXl6exfq8vDwCAur+nzkgIKDB8rVfGyqzefNmUlNT0ev1ODs706tXLwCGDh3KlClT6jyvXq/Hy8vLYmlXGqptnc2GZWPgh9cBZWoYn/696ZWcZrq5jx9LJw9lx/MxvH5vBCPDuqB10pB5sphXNhxg+LzN3PN2Ch+kHCG/pBx0HjBxNQQOgrIC+PAuU3xXi+oKOLABjqbIa0hXCauSlk6nY8iQISQnJ5vXGY1GkpOTiY6OrnOf6Ohoi/IAmzZtMpfv0aMHAQEBFmWKi4vZtm2buczChQvZvXs3GRkZZGRkmLtMrFq1ipdfftmaj9C+1FXb2rMaltwIJ7aD3tv0YvNdi0Hfwaan9nJ14Z4hXfngz5Gk/XUUL4/tT1SPTmg0sOPoOeZ+/jM3vJLMxPe2snJPEUX3rDJ1rSjJgQ/vhKKTNo3HasU5sPll+Gc/WDnBlOTfGwUH1suoFXamUfW2ltZt1apVTJkyhXfeeYfIyEgWLFjA6tWrOXDgAP7+/kyePJng4GCSkkyvbqSkpDBy5EjmzZtHXFwcK1eu5JVXXmHXrl3072+auv3VV19l3rx5fPDBB/To0YPnn3+ePXv2sG/fPlxdXS+L4ciRI/To0YP09HQGDRrUqLiLi4vx9vamqKiofdW6ti01jbLg1RVCR8Cemk6dITfAuHfBp1urhpNbVM76vTl8sfsUGccLzeudnTTc3lPL3889RYfSY+DbCx74Ejr4tWp8nNgB25bAz2vBWDMWmGcgXCg0PaQA6NLXNI5Zv7tB69y68bVhjb1GrU5aAG+++SavvfYaubm5DBo0iIULFxIVFQXAzTffTGhoKMuXLzeXX7NmDc899xxHjhyhd+/ezJ8/n9tuu828XSnF3LlzWbp0KYWFhYwYMYK33nqLsLCwOs8vScsKVeWwcJCpBgOgcYKRz8CNT9r9gjt+towv9pzii9057M8xPd0NooA1+hcJ1hRQ7BWG7sEvcfXq3LKBVFeaxv7atgRO7vxtfbdhEDUdwm+HC+dg29uQ9i5U1DyJ7hgKw2fCoIngrG/ZGNuBFk1ajqjdJi2AHctg3ePg3c1Uu+p2g70jusyv+SV8sdtUAzOcOcQa3Yv4aQrZq67h4z6LGH1dL0b06oLO2YbPjkryYOcy2PE+nK9pU9XqYMC9pk61QYMu36e8yDTUTupbpjY4gA4BMOxRGHK/zW+z2xNJWpdo10kL4FSG6ZbrKr+olFLsyykmNfUn7tk7DR9K2GYMZ0rlLPRuHRjTP4A7IoK4oacvWifNlQ9Yl5M7Yds7kPkpGGt68XsGwvVT4br7oUOXKx+jsgx2fWgacqe4pv3NrSNEPQJR00zfC6tI0rpEu09aDkidSse47A60VSWkaiKYciGRSlwA6NxBz5j+AfQP9qK7rwehvh74e+nRaOpJZIYq07uO296BExe9KxsSZboF7Hunqe+YtaorTe2EP/4Tzh42rdN1MCXAGxLA07/h/YWZJK1LSNJyUMe2wUfxUFXGma4x/KPjc6z/+TSFdQwB7eriRKivB9193QntbEpkvdzLCD/1KR32foimtl3PyQX6jzPViIKH2CZOo8HULvbDPyCv5r1arR6umwTDHoOO3W1znuYoOwvH00xJ+1S6KWa9pynJ6jtc9NWz5qvHRd9fsq0F2vAkaV1CkpYDO7wFPh4PhgoYcC9Vd77Nj4fP8V3WabILSjlyppQT5y5YTH3WT5PNA85fcYdTCnqN6SngGU1HfvS5kyPdx9M5MIQevh507+xBoJcrTk291byUUnDwa/j+9d9qdBotDBxvGpqnSx/bnOdKjEY4fcAUw/Ga5UzdI6c0iZPLb8ms3qR30fpr469Y65SkdQlJWg4uayOsmmjqhnDdZLhjocWY9FUGIycLijm/5390+XkZ/oXp5m0Zxl68Xx3Ll8Yoqrj8ianO2YnundxrbjN/q6V193UnyMetaW1nSplekfrhDTj8bc1KDfS9w9RdwgYdeS2UF5m6a5zYDse3wYmdUFHH5Ludw6BrJHQdaqpJVZSYxv2vPA8V56GypObrpT/XlKlq5NRyl3roWwi+rsEikrQuIUmrDfh5Lfznz6ZhbaIegVuTTImr9AzsWm4aEbW2UdzJBfqNhajpVAdex8nCCxw5U8aRmprZ0TNlHDlTyvGzZVQZ6r8EdFonQjq5EeTjRpcOejp76mu+6ujcQU8XTz2dO+jp6K6rP7md3Gm6bTyw7rd114wydTsJHW7970EpOHOopha1zfSyef4+4JLP4eIBXYeYklRIlClRuXey/nwXMxouSmh1JbZ6kl7sy+DdtcFDS9K6hCStNiJjhWkAQTANKFhVCnvWmG4dATy6wNA/m5ZGvPRdbTCSU1TOkTOlNQmtjKNnSskuKOX42QtUGhrX+91JA74dTAmscwcdXWqT20WJLagqm+DMJej2r0WjDKYdu0WbklevGMvZjC5WWQond1ne6l04e3m5jqE1Capm8etn97541pCkdQlJWm1I2ruw4SnLdYGD4IZHTLUrGzUSG4yKnKILHCkoI7e4nILzFRSUVHD6fAUF5ys4XVJBwflKzpZWWnXcUKd8HnXdwJ3Gzbhgam/LdQ/j554PUtbrNvp5lNC9dC/aUztMNancTNNkuBfT6k23W12vN9WiQiJb/+0BG5OkdQlJWm1M6luwZR70GmVKVl2vr7+m0sKqDEbOllbWJLHfktlv3/+2/txFTz39OMeDzhuYqP0GD42pplim9LhrLh9KqMI9EE1IJLrQG0xJKmDAlWdCcjCStC4hSUtcDS5OcKdram4l5/K55vDHDM1bjYexhCql5WcVyi5jb3Yaw9hl7E0OvgAEervSN9CLvoGeNV+9CPX1aHpH26uIJK1LSNISV72K83D2MMZOvTh+XrE/p5h9OSXszylmf04xJ87VPfORm4uWsABPrq1JZOEBXoQHeuLl2oTOsnYkSesSkrSEoysur+JATRI7kGtKaFm5xZRX1f2woGtHN3NtrDahhXR0t12fNBuTpHUJSVqiLTIYFUfOlJprY/trklpO0eUzWQF46LSE19xedungiodei4feGXedFg+dM+5601cPvTMeei3uOmc8dFqctS0/yLEkrUtI0hLtybnSSvbnFptrZvtzi/kl7zyV1U0bwFDn7ESHOpOb5c/utesuKTsg2BvPK9yuNvYadZxOHEKIRuvooWPYNZ0Zds1vY5FVG4wcLjDVyrJySzhXVkVZZTWlFQbT10oDpRXVlFX89n11zatRldVGzlZXcraOOUoa47OE4QwK8bHBJ5OkJUS74ax1IszfkzB/z0bvU1ltpLSimtLKaspqk1qlgfMV1ZYJr8JQU85wyXrTOm832z0UkKQlhKiXztkJnbOOjh5XT58wh5hCTAghaknSEkI4FElaQgiHIklLCOFQJGkJIRyKJC0hhEORpCWEcCjtpp9W7dtKxcXFdo5ECFGX2mvzSm8WtpukVVJSAkBISIidIxFCNKSkpARvb+96t7ebF6aNRiOnTp3C09Oz/gk9axQXFxMSEsLx48fbzMvVbe0ztbXPA/KZlFKUlJQQFBSEk1P9LVftpqbl5ORE164NzwZyKS8vrzbzx1OrrX2mtvZ5oH1/poZqWLWkIV4I4VAkaQkhHIokrTro9Xrmzp2LXm+bqaiuBm3tM7W1zwPymRqr3TTECyHaBqlpCSEciiQtIYRDkaQlhHAokrSEEA5FkpYQwqFI0rrE4sWLCQ0NxdXVlaioKNLS0uwdUpMlJSVx/fXX4+npiZ+fH/Hx8WRlZdk7LJuaN28eGo2Gxx9/3N6hNMvJkyf505/+hK+vL25ubgwYMIAdO3bYO6wmMxgMPP/88/To0QM3NzeuueYaXnrppSu+DN0YkrQusmrVKhITE5k7dy67du0iIiKC2NhY8vPz7R1ak3z33XckJCSwdetWNm3aRFVVFaNHj6a0tImT111ltm/fzjvvvMPAgQPtHUqznDt3juHDh+Pi4sKXX37Jvn37eOONN+jYsaO9Q2uyV199lbfffps333yT/fv38+qrrzJ//nwWLVrU/IMrYRYZGakSEhLMPxsMBhUUFKSSkpLsGJXt5OfnK0B999139g6l2UpKSlTv3r3Vpk2b1MiRI9XMmTPtHVKTzZo1S40YMcLeYdhUXFyc+vOf/2yx7u6771YTJ05s9rGlplWjsrKSnTt3EhMTY17n5ORETEwMqampdozMdoqKigDo1KmTnSNpvoSEBOLi4iz+vRzV559/ztChQ7n33nvx8/Nj8ODBvPvuu/YOq1mGDRtGcnIyv/zyCwC7d+/mxx9/ZMyYMc0+drsZ5eFKCgoKMBgM+Pv7W6z39/fnwIEDdorKdoxGI48//jjDhw+nf//+9g6nWVauXMmuXbvYvn27vUOxicOHD/P222+TmJjIX//6V7Zv385jjz2GTqdjypQp9g6vSZ555hmKi4sJDw9Hq9ViMBh4+eWXmThxYrOPLUmrnUhISCAzM5Mff/zR3qE0y/Hjx5k5cyabNm3C1dXV3uHYhNFoZOjQobzyyisADB48mMzMTJYsWeKwSWv16tV8/PHHrFixgn79+pGRkcHjjz9OUFBQ8z9Ts28w24iKigql1WrV2rVrLdZPnjxZ3XnnnfYJykYSEhJU165d1eHDh+0dSrOtXbtWAUqr1ZoXQGk0GqXValV1dbW9Q7Rat27d1NSpUy3WvfXWWyooKMhOETVf165d1Ztvvmmx7qWXXlJ9+vRp9rGlTauGTqdjyJAhJCcnm9cZjUaSk5OJjo62Y2RNp5RixowZrF27ls2bN9OjRw97h9Rso0aNYu/evWRkZJiXoUOHMnHiRDIyMtBqtfYO0WrDhw+/rCvKL7/8Qvfu3e0UUfOVlZVdNvqoVqvFaDQ2/+DNTnttyMqVK5Ver1fLly9X+/btU9OmTVM+Pj4qNzfX3qE1ySOPPKK8vb3Vli1bVE5OjnkpKyuzd2g25ehPD9PS0pSzs7N6+eWX1cGDB9XHH3+s3N3d1b///W97h9ZkU6ZMUcHBwWrdunUqOztbffrpp6pz587q6aefbvaxJWldYtGiRapbt25Kp9OpyMhItXXrVnuH1GRAncuyZcvsHZpNOXrSUkqpL774QvXv31/p9XoVHh6uli5dau+QmqW4uFjNnDlTdevWTbm6uqqePXuqZ599VlVUVDT72DKelhDCoUiblhDCoUjSEkI4FElaQgiHIklLCOFQJGkJIRyKJC0hhEORpCWEcCiStIQQDkWSlhDCoUjSEkI4FElaQgiH8v8BsIpzz5ZvPRcAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:30:56.347417Z", + "iopub.status.busy": "2024-03-22T19:30:56.347111Z", + "iopub.status.idle": "2024-03-22T19:34:45.495165Z", + "shell.execute_reply": "2024-03-22T19:34:45.494328Z" + }, + "papermill": { + "duration": 229.16824, + "end_time": "2024-03-22T19:34:45.497840", + "exception": false, + "start_time": "2024-03-22T19:30:56.329600", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:34:45.534054Z", + "iopub.status.busy": "2024-03-22T19:34:45.533725Z", + "iopub.status.idle": "2024-03-22T19:34:45.554128Z", + "shell.execute_reply": "2024-03-22T19:34:45.553243Z" + }, + "papermill": { + "duration": 0.040901, + "end_time": "2024-03-22T19:34:45.556133", + "exception": false, + "start_time": "2024-03-22T19:34:45.515232", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0100880.0349370.0025338.2342270.2508396.3043860.4491380.0000118.5653010.0376180.1200850.0503320.0962180.01687416.799528
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.010088 0.034937 0.002533 8.234227 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.250839 6.304386 0.449138 0.000011 8.565301 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.037618 0.120085 0.050332 0.096218 0.016874 \n", + "\n", + " total_duration \n", + "realtabformer 16.799528 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:34:45.589403Z", + "iopub.status.busy": "2024-03-22T19:34:45.588694Z", + "iopub.status.idle": "2024-03-22T19:34:46.069865Z", + "shell.execute_reply": "2024-03-22T19:34:46.068938Z" + }, + "papermill": { + "duration": 0.500035, + "end_time": "2024-03-22T19:34:46.071881", + "exception": false, + "start_time": "2024-03-22T19:34:45.571846", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:34:46.106808Z", + "iopub.status.busy": "2024-03-22T19:34:46.106503Z", + "iopub.status.idle": "2024-03-22T19:39:08.589480Z", + "shell.execute_reply": "2024-03-22T19:39:08.588620Z" + }, + "papermill": { + "duration": 262.502998, + "end_time": "2024-03-22T19:39:08.591992", + "exception": false, + "start_time": "2024-03-22T19:34:46.088994", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:08.628489Z", + "iopub.status.busy": "2024-03-22T19:39:08.628166Z", + "iopub.status.idle": "2024-03-22T19:39:08.654544Z", + "shell.execute_reply": "2024-03-22T19:39:08.653600Z" + }, + "papermill": { + "duration": 0.047, + "end_time": "2024-03-22T19:39:08.656469", + "exception": false, + "start_time": "2024-03-22T19:39:08.609469", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:08.689934Z", + "iopub.status.busy": "2024-03-22T19:39:08.689180Z", + "iopub.status.idle": "2024-03-22T19:39:08.694739Z", + "shell.execute_reply": "2024-03-22T19:39:08.693895Z" + }, + "papermill": { + "duration": 0.024483, + "end_time": "2024-03-22T19:39:08.696729", + "exception": false, + "start_time": "2024-03-22T19:39:08.672246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.3862167485555013}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:08.731368Z", + "iopub.status.busy": "2024-03-22T19:39:08.730583Z", + "iopub.status.idle": "2024-03-22T19:39:09.084021Z", + "shell.execute_reply": "2024-03-22T19:39:09.083023Z" + }, + "papermill": { + "duration": 0.373135, + "end_time": "2024-03-22T19:39:09.086205", + "exception": false, + "start_time": "2024-03-22T19:39:08.713070", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABED0lEQVR4nO3deXhTVf4/8PfNvjRJ942utBVEZF8EREAYRZkRxhHFDSrDMg6MwzA8P8VnRNFR1FEEmRlGHW1BUVy+uIyOuIOjIsi+WtYulG50S9s06z2/P26SNjQtSZrmNunn9Tx50t6c3JybNJ+e/XCMMQZCCBGRROwMEEIIBSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUi0qnHHnsMHMcF9ZyFhYXgOA579+71Kf3f/vY39O/fH1KpFMOGDQtqXkjvQYGI+OWpp57CBx98EJLX+vzzz/H//t//w4QJE1BQUICnnnoqJK9LQk8mdgZIeHnqqadw2223YdasWT3+Wl9//TUkEgleffVVKBSKHn89Ih4qEUWIlpYWsbMQdNXV1VCr1UELQowxtLa2BuVc3WEymcTOQq9DgSgMudpujh8/jrvuugsxMTG49tprAQBvvPEGRo4cCbVajdjYWMyZMwdlZWUez//f//6H2bNnIyMjA0qlEunp6fjTn/502S8px3FoaWnBpk2bwHEcOI5Dfn4+AKCkpAS///3vMWDAAKjVasTFxWH27NkoLi72ei6TyYTFixcjLi4Oer0ec+fORX19vcdrFRQUoKWlxf1ahYWFAAC73Y4nnngCOTk5UCqVyMrKwsMPPwyLxeLxGllZWfjlL3+Jzz77DKNGjYJarcZLL72EHTt2gOM4vPPOO1i9ejX69esHnU6H2267DY2NjbBYLFi2bBkSExMRFRWF++67r8O5fX2vJ0+ejMGDB2Pfvn247rrroNFo8PDDD3f5PvdFVDULY7Nnz0ZeXh6eeuopMMbw5JNP4pFHHsHtt9+OBQsWoKamBhs2bMB1112HAwcOIDo6GgDw7rvvwmQy4f7770dcXBz27NmDDRs24Pz583j33Xc7fb3XX38dCxYswJgxY7Bo0SIAQE5ODgDgp59+wg8//IA5c+YgLS0NxcXF2LhxIyZPnozjx49Do9F4nGvp0qWIjo7GY489hqKiImzcuBElJSXuIPH666/j5Zdfxp49e/Dvf/8bADB+/HgAwIIFC7Bp0ybcdttt+POf/4zdu3djzZo1OHHiBN5//32P1ykqKsKdd96JxYsXY+HChRgwYID7sTVr1kCtVuOhhx7C6dOnsWHDBsjlckgkEtTX1+Oxxx7Djz/+iMLCQmRnZ2PVqlXu5/r6XgNAbW0tbrrpJsyZMwf33HMPkpKS/Pyk+wBGws6jjz7KALA777zTfay4uJhJpVL25JNPeqQ9cuQIk8lkHsdNJlOHc65Zs4ZxHMdKSko6vE57Wq2WzZs3r8PzvZ1z165dDADbvHmz+1hBQQEDwEaOHMmsVqv7+LPPPssAsA8//NB9bN68eUyr1Xqc8+DBgwwAW7BggcfxFStWMADs66+/dh/LzMxkANj27ds90n7zzTcMABs8eLBHHu68807GcRy76aabPNKPGzeOZWZmun/3572eNGkSA8D+9a9/dXh/SBuqmoWx3/3ud+6ft23bBp7ncfvtt+PixYvuW3JyMvLy8vDNN9+406rVavfPLS0tuHjxIsaPHw/GGA4cOBBQXtqf02azoba2Frm5uYiOjsb+/fs7pF+0aBHkcrn79/vvvx8ymQz//e9/u3wd1+PLly/3OP7nP/8ZAPDJJ594HM/OzsaNN97o9Vxz5871yMPYsWPBGMP8+fM90o0dOxZlZWWw2+0A/HuvAUCpVOK+++7r8rr6OqqahbHs7Gz3z6dOnQJjDHl5eV7Ttv/ClZaWYtWqVfjoo4882mUAoLGxMaC8tLa2Ys2aNSgoKEB5eTlYu4U/vZ3z0nxGRUUhJSWl0zYll5KSEkgkEuTm5nocT05ORnR0NEpKSjyOt3+PLpWRkeHxu8FgAACkp6d3OM7zPBobGxEXF+fXew0A/fr1o16/y6BAFMbal0J4ngfHcfj0008hlUo7pI2KigIAOBwO/OIXv0BdXR0efPBBDBw4EFqtFuXl5cjPzwfP8wHl5Q9/+AMKCgqwbNkyjBs3DgaDARzHYc6cOQGfsyu+DrRs/x5dytv71NVxV3D19b32JQ9EQIEoQuTk5IAxhuzsbFxxxRWdpjty5AhOnjyJTZs2Ye7cue7jX3zxhU+v01kAeO+99zBv3jw8//zz7mNmsxkNDQ1e0586dQpTpkxx/97c3IyKigrcfPPNXb5+ZmYmeJ7HqVOncOWVV7qPV1VVoaGhAZmZmT5dR3f4+l4T31EbUYS49dZbIZVKsXr1ao9qESD8J6+trQXQ9t++fRrGGNavX+/T62i1Wq/BRSqVdnjdDRs2wOFweD3Pyy+/DJvN5v5948aNsNvtuOmmm7p8fVegWrduncfxtWvXAgBmzJhxuUvoNl/fa+I7KhFFiJycHPz1r3/FypUrUVxcjFmzZkGn0+HcuXN4//33sWjRIqxYsQIDBw5ETk4OVqxYgfLycuj1evzf//1fh7aizowcORJffvkl1q5di9TUVGRnZ2Ps2LH45S9/iddffx0GgwGDBg3Crl278OWXXyIuLs7reaxWK6ZOnYrbb78dRUVF+Oc//4lrr70Wt9xyS5evP3ToUMybNw8vv/wyGhoaMGnSJOzZswebNm3CrFmzPEpZPcXX95r4jgJRBHnooYdwxRVX4IUXXsDq1asBCA2vN9xwg/sLLpfL8Z///AcPPPAA1qxZA5VKhV//+tdYunQphg4detnXWLt2LRYtWoS//OUvaG1txbx58zB27FisX78eUqkUW7ZsgdlsxoQJE/Dll1922mP197//HVu2bMGqVatgs9lw55134sUXX/Sp7eff//43+vfvj8LCQrz//vtITk7GypUr8eijj/rxbnWPL+818R3HLi1bEkJIiFEbESFEdBSICCGio0BECBEdBSJCiOgoEBFCREeBiBAiurAeR8TzPC5cuACdThf0Rd4JId3HGENTUxNSU1MhkXRR7gn1uiPt2e129pe//IVlZWUxlUrF+vfvzx5//HHG87xPzy8rK2MA6EY3uvXyW1lZWZffZVFLRM888ww2btyITZs24aqrrsLevXtx3333wWAw4IEHHrjs83U6HQCgrKwMer2+p7NLCPGT0WhEenq6+7vaGVED0Q8//ICZM2e6JypmZWXhrbfewp49e3x6vqs6ptfrKRAR0otdrulE1Mbq8ePH46uvvsLJkycBAIcOHcJ3333X6Qxsi8UCo9HocSOEhD9RS0QPPfQQjEYjBg4cCKlUCofDgSeffBJ333231/Rr1qxxTzAkhEQOUUtE77zzDrZs2YI333wT+/fvx6ZNm/Dcc89h06ZNXtOvXLkSjY2N7tulW7cQQsKTqLPv09PT8dBDD2HJkiXuY3/961/xxhtv4Oeff77s841GIwwGAxobGzttI2KMwW63d7pAF+mcVCqFTCajoREkYL58RwGRq2Ymk6nD2AKpVBq0NY6tVisqKipoZ81u0Gg0SElJocXfSY8SNRD96le/wpNPPomMjAxcddVVOHDgANauXdthO5dA8DyPc+fOQSqVIjU1FQqFgv6z+4ExBqvVipqaGpw7dw55eXldD0gjpBtEDUQbNmzAI488gt///veorq5GamoqFi9e7LGjZqCsVit4nkd6enqHXUaJb9RqNeRyOUpKSmC1WqFSqcTOEolQogYinU6HdevWdVgIPZjov3j3RNr7Z3Pw4ADIpJF1XeGOPg3Sp5ysakLB98U4dqERDp7BYqdOjN6AAhHpUyoazGi22PH5sSps3HEaB0obxM4SAQWiPi8rK6tHq8a9TUVjKwCgX4waNgdDTZNF5BwRgAIR6UN4nqGuRdjUcVCKMKalrsUqZpaIEwWiCGC10pfJF01mO3jGIJNwyIgTelIbTDY4eNHG9BKnPhmIrHa+05vdwfuc1uZD2kBMnjwZS5cuxdKlS2EwGBAfH49HHnnEvb1xVlYWnnjiCcydOxd6vR6LFi0CAHz33XeYOHEi1Go10tPT8cADD6ClpcV93urqavzqV7+CWq1GdnY2tmzZElD+wlVDqxCwDRo5dEoZpBIOPGNotthFzhkJ6xUaA/WPb053+lh2vBazhvdz//7yt2dgc3j/j5kWo8bsUenu31/7/hxarZ69MH/6xRUB5XHTpk347W9/iz179mDv3r1YtGgRMjIysHDhQgDAc889h1WrVrl3Nz1z5gymT5+Ov/71r3jttddQU1PjDmYFBQUAgPz8fFy4cAHffPMN5HI5HnjgAVRXVweUv3DUYBKqZdEaYXCrTiVDg8kGY6sNBrVc5Nz1bX0yEIWD9PR0vPDCC+A4DgMGDMCRI0fwwgsvuAPR9ddfjz//+c/u9AsWLMDdd9+NZcuWAQDy8vLw4osvYtKkSdi4cSNKS0vx6aefYs+ePRg9ejQA4NVXX8WVV14Z8msTi1wqQZJehfgoYbqKTiVHg8mGJjOViMTWJwPRkim5nT4muWQWyKLrcjpNe+mMkfkTsruTLQ/XXHONx5SUcePG4fnnn3dP3h01apRH+kOHDuHw4cMe1S3GmHuqy8mTJyGTyTBy5Ej34wMHDkR0dHTQ8tzbDUrVY1Bq28TLtBg1lDIJtEqpiLkiQB8NRAqZ701jPZW2u7Rarcfvzc3NWLx4sdcldjMyMtyLz5E21/SPEzsLxKlPBqJwsHv3bo/ff/zxR+Tl5UEq9f7fe8SIETh+/Dhyc72X9gYOHAi73Y59+/a5q2ZFRUVoaGgIar4JCUSf7DULB6WlpVi+fDmKiorw1ltvYcOGDfjjH//YafoHH3wQP/zwA5YuXYqDBw/i1KlT+PDDD7F06VIAwIABAzB9+nQsXrwYu3fvxr59+7BgwQKo1epQXZKoGGP4547TKLikQ4ExFnDvJgkeCkS91Ny5c9Ha2ooxY8ZgyZIl+OMf/+jupvdmyJAh2LlzJ06ePImJEydi+PDhWLVqFVJTU91pCgoKkJqaikmTJuHWW2/FokWLkJiYGIrLEV2rzQGLjUdjqw1KZxW6ymjGhq9P440fS0TOHaGqWS8ll8uxbt06bNy4scNjxcXFXp8zevRofP75552eMzk5GR9//LHHsXvvvbdb+QwXrrFCGoUUEmePhFImgYNnMFmp10xsVCIifUKLRaiOaZVt/3s1CuFnm4OqZ2KjQET6hBZniSiqXSBSyCSQS4XSEZWKxEVVs15ox44dYmch4riqZlqF55+8WiGDrdUGk9WBaFrIUzRUIiJ9gqvEo1F4Dn/QOn+nEpG4KBCRPkGnkiPZoIJB4zmnTO0MRK1WaiMSE1XNSJ8wOisWo7NiOxxPjVZDwnGIUtFXQUz07pM+zVtwIqFHVTNCiOioRET6hNe+OwcG4LYRaR3aiRhjcPCMthgSEQUiEvEYYzCabWAMkEo91245VdWE/x6pRL8YNW4bmSZSDgn9CyARz2Ln4VxlF6pLlmqRSyXgGYPZRvubiYkCEYl4FpvQNS+Xch2qXyq50H1PgUhcfSsQMQbYraG/Md93idi8eTPi4uJgsXjutzVr1qw+M0E12MzO3VxdQac9lVz4Clhorpmo+lYbkcMG/O/50L/uxD8DMoVPSWfPno0HHngAH330EWbPng1A2H3jk08+6XJmPemcq7Sj9BqIhGNWOw8HzyC9dK1gEhJ9q0QUBtRqNe666y73zhsA8MYbbyAjIwOTJ08WL2NhzOysmim9LOWrkErca49T9Uw8fatEJJULpRMxXtcPCxcuxOjRo1FeXo5+/fqhsLAQ+fn5HovpE9/JpRySDW27d7QnkXBQyqQw2xww2xwey4SQ0Olb7zrH+VxFEtPw4cMxdOhQbN68GTfccAOOHTuGTz75ROxsha3+CVHonxDV6eNZcRrYqVomqr4ViMLIggULsG7dOpSXl2PatGlIT0+//JNIQG66OkXsLPR51EbUS9111104f/48XnnlFcyfP1/s7BDSoygQ9VIGgwG/+c1vEBUVhVmzZomdnbC2/WgFXv3uHE5WNXWaRtiM0vdhFiS4qGrWi5WXl+Puu++GUqkUOythrclsh7HV1ulwrq9/rsKR80aMz42j2fgioUDUC9XX12PHjh3YsWMH/vnPf4qdnbBndg5WdA1evBTHceBpfzNRUSDqhYYPH476+no888wzGDBggNjZCXsWW+cjqwFA6Zz2QYFIPBSIeqHO9i0jgXENVFTJvAcihYymeYiNGqtJRLM7eNgcQuOQspOqmdIZoCx2GlktlogPRMyPCaeko3B//1ztQxznfYoH0FYioqqZeCK2aiaXC9MqTCYT1Gq1yLkJXyaTCUDb+xlueMaQbFCBMXQ6RcYdiBwUiMQSsYFIKpUiOjoa1dXVAACNRkNztfzAGIPJZEJ1dTWio6MhlXpvX+nt9Co57hyT0WWaKKUM6bEaxGrDM9hGgogNRACQnJwMAO5gRPwXHR3tfh8jVYJOScvEikz0QFReXo4HH3wQn376KUwmE3Jzc1FQUIBRo0Z1+9wcxyElJQWJiYmw2WxByG3fIpfLw7YkRMKLqIGovr4eEyZMwJQpU/Dpp58iISEBp06dQkxMTFBfRyqV0heqjzp8vgE/FddjQJIO1+bFd5nW1TBPVfjQEzUQPfPMM0hPT/dYBCw7O1vEHJFI02wRpndYHZ13zTPG8NK3Z2Gx8VgwMZvWJBKBqN33H330EUaNGoXZs2cjMTERw4cPxyuvvNJpeovFAqPR6HEjpCuuhfM7G8wItE3xoGke4hE1EJ09exYbN25EXl4ePvvsM9x///144IEHsGnTJq/p16xZA4PB4L7RGj3kcrpar7o9hZRGV4tJ1EDE8zxGjBiBp556CsOHD8eiRYuwcOFC/Otf//KafuXKlWhsbHTfysrKQpxjEm7advDo+k9dSYMaRSVqIEpJScGgQYM8jl155ZUoLS31ml6pVEKv13vcCOmKa+H8zia8urQNaqRpHmIQNRBNmDABRUVFHsdOnjyJzMxMkXJEIs3lZt67uOabuQIXCS1RA9Gf/vQn/Pjjj3jqqadw+vRpvPnmm3j55ZexZMkSMbNFIoheLUe0Rg61zyUiCkRiELWfcvTo0Xj//fexcuVKPP7448jOzsa6detw9913i5ktEkFuHeHbiOn4KCVMsQ5oFdR1LwaOhfH0aqPRCIPBgMbGRmovIqQX8vU7GvHLgBBCej8qh5KIVd1kxn8OVSBWK8evh9Ok1t6MAhGJWK1WB4ytNiikl5879nOlEd/8XIOMWA1mDKENF0ONqmYkYrlGSV9uVLWL2eZAq43GEYmBAhGJWGYfxxABgNw5xcNG3feioEBEIpZ7VHUna1W3p6AthURFgYhELH9KRK4BjVQiEgcFIhKx3G1E/pSIKBCJggIRiVgquQTRGjmiVJfvHJa3m30fxmN8wxZ135OINTEvARPzEnxKq5BKkKRXQS7lwDPAhx5/EkQUiAiB0EZ019iutx0iPYeqZoQQ0VGJiESszbuKIZVwmDmsH6JoQfxejT4dEpHsDh61zVYAgEziW4PPe/vO42KzBb8amop+0bRNeShR1YxEJFfXPcf51n0vPMeBVquDBjWKgAIRiUju3TtkUp83TKTR1eKhQEQiktnuWjTf9z9xGl0tHgpEJCL5M73DhfY2Ew8FIhKRXDu8+to+BNAMfDFRICIRSSrhEKORQ6+S+/wcqpqJh7rvSUQakKzDgGSdX8/Rq+VI1CuhoZ08Qo7ecUKchqVHY1h6tNjZ6JOoakYIER2ViEhE+uxYJWqaLLg2Nx5Z8Vqxs0MugwIRiUj1LVbUNFlg531fW6iszoQvjlchhrYfCjkKRCQitY0j8q/1obHVBqmPc9NI8FAbEYlIbSOr/RjQSN33ogkoEJ09ezbY+SAkaBhj7gGNNLI6PAQUiHJzczFlyhS88cYbMJvNwc4TId1idfDgnetOdxhZbaoDjn8EHHwLqDzi8ZC8XYmI1q0OrYAC0f79+zFkyBAsX74cycnJWLx4Mfbs2RPsvBESENd+ZjIJ5562AQBoqQX2bwKqjgH1xcCJj4GSXe6H5c6FqhkDbA4KRKEUUCAaNmwY1q9fjwsXLuC1115DRUUFrr32WgwePBhr165FTU1NsPNJiM8cPEOMRo5oTbvpHYwBP38M2MyALhlIGyUcP/ct0Cz8vSraBS1qJwqtbjVWy2Qy3HrrrXj33XfxzDPP4PTp01ixYgXS09Mxd+5cVFRUBCufhPgsVqtA/oRs3Dsuq+1g9XHAeAGQyoGrbwNypwHxeQDjgZLvAAAcxyFBp0SiXumu2pHQ6FYg2rt3L37/+98jJSUFa9euxYoVK3DmzBl88cUXuHDhAmbOnBmsfBISOMaAMmfTQcY1gFInLN2YfZ1wrKYIaG0AANxzTSbuHpsJnR+TZUn3BTSOaO3atSgoKEBRURFuvvlmbN68GTfffDMkEiGuZWdno7CwEFlZWcHMKyGBMZYDTZWARAakjmg7HpUIxGQC9SVCu1HWBPHy2McFFIg2btyI+fPnIz8/HykpKV7TJCYm4tVXX+1W5ggJxOHzDTh8vhEDk3UYlRULVBwWHki8ElBoPBMnDW4LRJnjhZISCbmAAtEXX3yBjIwMdwnIhTGGsrIyZGRkQKFQYN68eUHJJCH+aDDZUNNkQXqsBuAdwMUi4YHkwR0TJwwATm4HTLVAaz2+LLahrN6E665IQE5CVGgz3ocF1EaUk5ODixcvdjheV1eH7OzsbmeKkO5wTe9Qy6VAQ6nQUyZXAwYvO7nKlIDBOa+s9gxarHY0mGxotTpCmGMSUCDqbLBXc3MzVCpVtzJESHd5LJxf4ywNJQwAJJ38ucfmCPd1Z2l0tUj8qpotX74cgNDNuWrVKmg0bfVth8OB3bt3Y9iwYUHNICH+ck94lUmA2tPCwfgrOn9CbH/gzNdAQykUcUIAonFEoeVXIDpw4AAAoUR05MgRKBQK92MKhQJDhw7FihUrgptDQvxkcQYijb0RsDQJvWXRXqplLtp4oRHbakKU7SIANe1tFmJ+BaJvvvkGAHDfffdh/fr10Ov1PZIpQrqj1RWIWkqFA4Y0YSBjZzhOSFNzElpzBYD+VCIKsYDaiAoKCigIkV5LrZBBrZBC1VImHIjJuvyTDOkAAE1rJQDa7TXUfC4R3XrrrSgsLIRer8ett97aZdpt27Z1O2OEBOreazIBnge+/0A44FMgEnrOoiyVMETJoFL4vnwI6T6fS0QGg8G9h7jBYOjyFoinn34aHMdh2bJlAT2fEA9NFYDdCshVQFTS5dNHJQESGRJVDPNHRmPKgMSezyNx87lEVFBQ4PXnYPjpp5/w0ksvYciQIUE9L+nDjOXCvSG982779iRSYcqH8YIQxDSxPZs/4iGgNqLW1laYTCb37yUlJVi3bh0+//xzv8/V3NyMu+++G6+88gpiYmICyQ4hbuUNrXjjxxIcOn5COKBP9f3JumThvqky+BkjXQooEM2cORObN28GADQ0NGDMmDF4/vnnMXPmTGzcuNGvcy1ZsgQzZszAtGnTLpvWYrHAaDR63Ahpr8ViR02TBXyjs0Sk8z4X0itdMsw2B3YdOoa3fyrtmQwSrwJeoXHixIkAgPfeew/JyckoKSnB5s2b8eKLL/p8nq1bt2L//v1Ys2aNT+nXrFnj0RaVnp4eSPZJBDPbHJA5WqHhm4VueX8CUZRQInIYK1BjpCWQQymgQGQymaDTCfuKf/7557j11lshkUhwzTXXoKSkxKdzlJWV4Y9//CO2bNni87SQlStXorGx0X0rKysLJPskgpltPKKs1ZBJOEATJzRW+0obD4lMBilvg8TSCN6PPdFI9wS8eP4HH3yAsrIyfPbZZ7jhhhsAANXV1T6PL9q3bx+qq6sxYsQIyGQyyGQy7Ny5Ey+++CJkMhkcjo6TDpVKJfR6vceNkPZabQ5EWWogk/pZGgIAiRRSndDDFmWtgZUGNYZMQIFo1apVWLFiBbKysjB27FiMGzcOgFA6Gj58uE/nmDp1Ko4cOYKDBw+6b6NGjcLdd9+NgwcPQiqlcRzEf2abw1kikgB6PwMRAKkuCRIO0NjqKRCFUEDrEd1222249tprUVFRgaFDh7qPT506Fb/+9a99OodOp8PgwZ7rw2i1WsTFxXU4ToivzFY7Yq01kEXJAJ0fPWYu2gRIJRzUtnrYaHR1yAS85XRycjKSk5M9jo0ZM6bbGSKkO7QOI9ScFTK5WhgX5PcJ4iHhOGhsdVQiCqGAAlFLSwuefvppfPXVV6iurgbPe35gge4Eu2PHjoCeR4jLtDQeaIoFDP2EQYr+ikqEUiaBwdECZrcBUAc9j6SjgALRggULsHPnTtx7771ISUlxT/0gRHRNzi2sAqmWAYBCi6uyUgCrCZC3AKAOkVAIKBB9+umn+OSTTzBhAu16QHoZ4wXhPoCGajdtAmAtAVpqunce4rOAes1iYmIQG0tzcUjvYrZYcPBEEY5XGMFHBVgiAoRABAiBiIREQIHoiSeewKpVqzzmmxEiNnN9JVotVjTYZJBoogM+z/FGBY6UN6K0zLfBuaT7AqqaPf/88zhz5gySkpKQlZUFudxz9bv9+/cHJXOE+MPacB4AYNMmdWt/skZJNGCxQ9tEJaJQCSgQzZo1K8jZIKT7HA3CRFeHtpvtOlHxAADOYnRuRUQ70/S0gALRo48+Gux8ENJtrFFoqOZ1yZdJ2TW5Qo0WqRYO3i5svGjoF4zskS4E1EYECMt//Pvf/8bKlStRV1cHQKiSlZeXBy1zhPjM1gq+pRYAwOm7FzgUMgla5dFwMCYEItLjAioRHT58GNOmTYPBYEBxcTEWLlyI2NhYbNu2DaWlpe61iggJmaYK2HgGs0wPpbp7W0UrZBK0yqLhcFRTIAqRgEpEy5cvR35+Pk6dOuWxhMfNN9+Mb7/9NmiZI8RnxgpIAFg1ydAouzdhWiF1loh4KhGFSkAlItca05fq168fKitpmU0igqYKZMZpkZk7HEjv3pLDSrkUvDoWUjMHmOqClEHSlYBKREql0usyrSdPnkRCQkK3M0WIXxhrWyzfnzWqO9EvWo17pgzHoBQ90FoP8B3XxiLBFVAguuWWW/D444/DZrMBADiOQ2lpKR588EH85je/CWoGCbksc6MwN4yT+LZ1kC+UOmF3WMYDrQ3BOSfpVECB6Pnnn0dzczMSEhLQ2tqKSZMmITc3FzqdDk8++WSw80hI15oqwMCwu0aG9w5WwmwLQgmG49q2FKJ2oh4XUBuRwWDAF198ge+//x6HDh1Cc3MzRowY4dNOHIQEnfECHDxDBR+DsjqTsF51Nzh4hvcPlCOpnGF8FIOUAlGP8zsQ8TyPwsJCbNu2DcXFxeA4DtnZ2UhOTgZjjJYEIaFnvACbg6FZkQCFTAKZNODhcQAACQecrzeBt2ngYBSIQsGvT4wxhltuuQULFixAeXk5rr76alx11VUoKSlBfn6+z8vEEhI0PA80V8Lm4NGkTIImCHvWcxwHhUwCM3Xhh4xfJaLCwkJ8++23+OqrrzBlyhSPx77++mvMmjULmzdvxty5c4OaSUI61VIDOOywcnKYpQbEyIOz6YJC6hzU6ApEjHVrIi3pml8lorfeegsPP/xwhyAEANdffz0eeughbNmyJWiZI+SymoT5ZSZFIsBx0CgDXobdgzDNwwAHA2C3ANaWoJyXeOdXIDp8+DCmT5/e6eM33XQTDh061O1MEeIz54qMRoUwfi2qm6OqXRRSCRgng1UmbCRK1bOe5VcgqqurQ1JS5+M0kpKSUF9f3+1MEeIzZyCyaVOgUUihVQSnRCR3Nnhblc5R2hSIepRfn5rD4YBM1vlTpFIp7HZ7tzNFiE9srUDLRQDA8KuuwnCFBowFZ5tolVwKlVwKuzIGMF2gqR49zK9AxBhDfn4+lEql18ctFktQMkWIT1wL5atjAIUGAII2fGTGEOfiaheMQNExKhH1ML8C0bx58y6bhnrMSMi4d+zo/vyyTmnihHsKRD3Kr0BUUFDQU/kgxH9G14qMqdiyqxgahQy/HJoCpSw4DdYA2gKRxQg4bML8MxJ0wWnZIyTU2s24N6mTcLG5FRLOBkU3R1W7nK5uwsGyRqRFq3CNXCWsXW2qA3RBmlRLPATnUyMk1Ex1wvgeqQwtMmFyqlYpDVobUYvFgbI6E2qarVQ9CwEKRCQ8udYf0qWg2Sb0lGmDNJgRAJRy4athsfMUiEKAAhEJT43CHmbQp6LFIgwZCcY8MxdXO5PF7qBAFAIUiEh4aigV7g0ZaHYGoqhglohkzhKRjUpEoUCBiIQfs1FYwpXjAEMams09GIjaV81a64RGchJ0FIhI+GksE+6jkgC5ClIJB41CCp0qeF3rSucsfqudB1MZAIkUcNiFZWlJ0FH3PQk/rmpZdDoAYOqVSZh6ZVLQpncAQomI44RZ+Daeg0IdI0wnMdUC6uigvQ4RUCAi4ccdiDI9DgdzdVCZhMMfp+a1nVMT6wxEdUBcTtBehwgoEJHwYmkSggHHAYb0HnuZDkGNGqx7FLURkfBSXyzcRyUCchWqjGYUfn8O249W9OzrUiDqUVQiIuGl9oxwHytUjxpbbag32aAO4hgil+9OXUSV0Yyx/WORRoGoR1GJiIQPngfqzwk/O9tpmszCJp/6IPaYuVQ3mVFaZ0KT2d5WIrK2CPPOSFBRICLhw1guBAG5CtAJS38YW4UxRMHsundpG13NAzIloIwSHqBSUdBRICId8Q5hm+XeNniv7qxwH5MNSIQ/3YZWKwDAoA5+IFK4R1c7d46l6lmPoTYi4qn2DHDiP8IyrNp44KpfA9p49xgdUTfQvHhSuG/Xfd5gEqpm0ZqeKBG1G10NCIGovoQCUQ+gQETaNNcAR7cBvHPd8ZaLYIfewu7YmdhXIZQ8RmbGYGx2bOgDUnONMI5HIgXi8gAIW0M3tgqBKEarCPpLeg1EAAWiHkBVM9LmzNdCEIrtD4xfCmjicL6yGtUHt8Nq52G189h1pha7zojwRaw+LtzH9hfaiACYbQ4k6VXQqWTQ9kCvmWuah8XuqpoJ6x7RQvrBJ2ogWrNmDUaPHg2dTofExETMmjULRUVFYmap72osF9pgOAmQ9wtAqUN9xi9QXt+KhJaTmJbOMGVgIgCguskCng9h+xFjQM3Pws+JV7oPa5Uy3DkmAwsm9u+REpprmof7Ut2TX+uFdjQSNKJWzXbu3IklS5Zg9OjRsNvtePjhh3HDDTfg+PHj0Gq1Ymat76k4KNwnDXL/599dq4Jdk4NclGIg/zOQPgBxWgXSYtShrZo1lAilEKkciMsN2csOSNJhYLKu7VqVeiEPDpvQmK+NC1leIp2ogWj79u0evxcWFiIxMRH79u3DddddJ1Ku+iC7Fag+IfycMtR9+Nq8BByyT0R61ftA9c9A/8lIjzWEPn8XDgj3SYOFbnQnxliPBkSJ5JJzc5wQpJuqhHYiCkRB06vaiBobhSUWYmNjvT5usVhgNBo9biQI6ouF//Iqg8f8rSilDBOGDYY2KQdgPFDRtp242ebA0fLGgGe8t1odKK01ob7F2nVCSxNQ4+wt6zfC46H39p3Hph+Kcb7eFFAeAkIN1j2i1/Sa8TyPZcuWYcKECRg8eLDXNGvWrMHq1atDnLM+oPa0cB+fJ/zXv1TKMKHbuuIwkHkt7Awo/KEYrVYHojVypMVofH4pxhh2nanF3pJ6OJyNL/0TtLjxqmSo5F4anEt/FIJgdLowv6zdeWqaLbDY+OBuH9SO2ebAlyeqYHPwmDWsn1D6cgeiiz3ymn1VrykRLVmyBEePHsXWrVs7TbNy5Uo0Nja6b2VlZSHMYYRiDKhzzt9yjs+pbjLjv0cqcLq6STgef4XQU2VpAurPQSaVIC9RGGW8r6Tej5di+OxYFXafq4ODZ9Cr5ZBwHM7WtODYBS8LjpkbgQsHhZ8zJ3g81GSxw2LjIZVwiO2BrnsAkEo4nKpqRvFFE6wOZxe+NkG4b6npkdfsq3pFiWjp0qX4+OOP8e233yItLa3TdEqlstPtrkmAmqsAS7PQCGvIAACcqW5BUWUTHDxDbqIOkMqE9pnze4XqWVwORmTE4Eh5I87WtKC22YK4qMt/LgfKGnCiwggJx+EXg5IwKFWPmiYLKhvNuDrtkrYnxoBTXwjDCaIzgJgsj4cvNgnbm8doFZBe2pYTJHKpBDIJBzvPYHaVvNyBqFaY+ybpNf/Lw5qo7yJjDEuXLsX777+Pr7/+GtnZ2WJmp29yT5vIEgIOgLMXmwEIVSa35CHCfe1pwGpCjFaB/glCqWivj6UiB8/AccDkAQkYlKoHACTolB2DEABUHgEunhIGMObd0KHKWOMMRAlRPVMacnHN6je7pnmoY4T3ibcL3fgkKEQNREuWLMEbb7yBN998EzqdDpWVlaisrERra6uY2epbGp37gzlXOzSabag2WsBxQHZ8u0CkSxLaaHiHu4dtTJbQqfBzRRManVMtujI6Kxb3XJOJId4CD4AWix0/nLkIR+054KSzRzVzPBCV0CFtpVGYAZ+gU/l0mYFytVu1Wp2BiOMATbwzw1Q9CxZRA9HGjRvR2NiIyZMnIyUlxX17++23xcxW38EYYHTuD2boBwAovtgCAEg1qKFRXFJzd3XtVwq9Z8kGFTLjNOAZw96Szkcbtx/8GB+l9NrlzvMMb+8+i/MHvsTFH14XAl58Xoe2ISHbDBcahEDUL1rt27UGSO0KRLZ2AxhdjeYUiIJG1DaiYC52TgJgqhWW1ZDKhB0xAJTWCV3hmXFeesISBwnTQJqqgOZqICoRY7JjUVpncs9Uv1RRZRN+Kq7DTYOTO29H4nlIKg/jhqbPUdZYjfNSCeIyx0A2aJbXXjybgyE3MQpVRjMSdD3bZuiqmnkEInc7UXWPvnZf0isaq4lIXNvy6FIBiRSMMZyvF6rFabFeApFCI4xsrikSuvLzpiEtRoN547K8TjqtbbbgyxNVsNp5FFU1Yby3QFR7RghuLReRqmY4rzbgmGYkeN1YjJZ6//NUyCT4xaCkgC/bHyq5MM3D7mj3T1PrqppRF36wUCDqy1ztQwahp9JkdSBKKYODZ0jWd9L2kjxECERVR4GcKYBE6hGEGltt0KtkqGux4v0D5bDaeaTFqHFN9iWjkK0tQjuQa7CiXA1J5gToruiP2hO12FvSgKvTor2PLQqh6/ISMGVAomd1UuusmrXWCwNBpcFfgqSvoUDUl7n2j3cGIq1ShnuuyYTVznfeJR7bH1BohUBSewZIuML9UKvVgXf3loFnDK1WHjxjiItSYMaQFM/pEi0XgcPvCOOEOAmQNkpoC5KrcCXPsK+sCbXNVuwrqceE3HiPl3fwDBWNrUg1qDtOwegBMqmXKqdCC8jVwppNLRcBfUqP5yPS0SCIvsrS3LZts76fx0OdtfcAEMbNJDtHvlce9niopskCi51Hi8UBnjH0T9DitpFpno3elmbg8NtCEFLHAKPuA3Knupf2kEg4jM8RSk8HSuvR4tzX3qWktgXv7j2P138sEa+NkeNoYGOQUYmorzI6q2WaOECuAs8zOBiD3FsJ4FLJQ4DS3UKJyNLsXss5I06DhRP7o8poRpRS1rHdyGEHjv6fsHe9JhYYfq/Q7nSJnIQo9ItRI1Gn7FAyO3ZBmF+YFa8NyQoAtc0W7DpbC6VM6tkuFZUobPRIDdZBQYGor3JXy4RJrtVNFryztwxZ8VrcMjS16+dq44XqiLFCGGmd1dbFrpBJkO6toZsx4OSngPGCMIP+6tlegxAgLEc7e2Rah0BT22zBmRphsOVVzgGRPc3OM5yqaoZOdclXxdVg3UwlomCgqllf1eg5fuh8vQkOnvle3ek3Srgv3ys02F5O2W6g8qjQJnTVr9tWO+xE+yBktjlwqqoJ249VgjEgNzEK8T5MKQmG9gMaPd6bqGThvrmy920yEIYoEPVFDpswxwxwN1SXOZfS8Fqa8SZxkLBsiNXksTyIVxdPA2d3CD/nTgNifZ/KY7Y58MGBcnx8uALVRgtUcikmDeg40rqnuAY02nkGm0cXfoIw/cRmFtq7SLdQIOqLmiqEkcsKLaCKhoNvG6mc7uuSHhIJkDFW+Lnk+843HWyuAY5/IJQaUod3WFPocjgOSNKrYFDLkRmnwe2j0npkM8XOyKWcu53KY1CjVNauelYVsvxEKmoj6ovajx/iOFQ2tsJq56FWSBHvzyTSlGFA+X6hC/vsN8CAmzwfNxuBI+8IJbDoDGEtbD8bmJUyKaYMTMQUv54VPBzHQS2Xotlih9nm8Nw/LSpZGGXeVAEkDBAph5GBSkR90SXjh847p3Wkx2j864mSSIXgAgjrBrnWDgKENZ0Pv+3sIYsT2oUk4g5ODJRGKeT70qEE0Dl70ZqoRNRdVCLqazwmurrah5zTOmICmEAakyX0mhV/DxR9KnRpKzTCFBC7RejaH3J7pz1k4SBKKcNFztq2v5mLzjmQ0dVgLebmk2GOAlFf42Wia15iFORSDhm+NlRfKmsi4LACZT8BVcfajuuSgcG3Co3aYeymwSmQS7mOpUVtgtALaDUBFmPYX6eYKBD1Na6Jrvp+7qrS0PRoDE2PDvycHCf0hiVcCdScEAYuRmcACQMjYgXDTkeaS+XCTh7NNUL1jAJRwCgQ9TWXtA8FlaGfe1xSn6FLcQaiCo95d8Q/4f/vivjnkkB0pqbZvX888a6myYKPD1/AVye8NErrnAMbmypCm6kIQyWivsRsFHqznBNdLXYHPj5UAZ4x/HZidkjH54QTm4PHqapm6NVe3h+9s2RpLKfF9LuB3rW+xDXRNSoRkClxocEMnjEY1HIKQl3QKoX/1yaLveMUGG2C0FZkt9JM/G6gQNSXXDLRtazOz2kdfZRW0TbNo0MXvkTStoyKa1gE8RsFor6kvli4dw1kdI4fSo/t2QXow51MKoFSLnxVOgxqBNoa/l0j1onfKBD1FZYmYSoGxwHRmTDbHKhuEuaH+bNldF8V5ayeNXsNRK4SEQWiQFEg6itcpaGoJEChwfn6VjAGxEUp3F8y0jlXG1qT2Usg0vcTAnxrgxDwid8oEPUVrkDk3Lq5pFbYv8zn2fZ9nF4tg1TCwWJ3dHxQpmxbOrahLLQZixD0r7AvYAyoOyf87FwLaEJuPDJiNTBoqLfMFxO97ebRXnSmsNdbQwmQNCi0mYsAVCLqC4wXhF03pHL3uBeVXIq8JB0Se3jL5kghl0q6XpnAtdibq+RJ/EKBqC+4WCTcx+UKk11J8BnShbl7rQ2AqfPtt4l3FIgiHWNtmxg6F+/aUVSNH8/Weu8BIl5Z7Tw+OVyBrXtK4eC9rFEtUwB656YDVCryGwWiSNdcLexfJpEBsTmw2nkcPt+IXWdqYb10cB7plFzK4WxNMyoazWgydzI3z9kRgPpzIctXpKBAFOlcmyDG5QAyBUrrWuDgGfRqOWKoodpnHMch2rlPW72pk0AU21+4rzsnLIVCfEaBKJI57MIe9QCQMhQAcLpa2BcsNzEqJBsURpJYjRCI6los3hPoUgClTlijm6pnfqFAFMmqjwmrMar0QEw2HDzDmRph/FBuYpTImQs/MVqhBFnX0kmJiOOAeOeaRBdPhihXkYECUaTieaD0R+HnfiMBiQSldSZY7Ty0SilSDdRt769YV9Wsxdp5ooR2gYinNjhfUSCKVFVHhW5kuQpIFfYSO1UlTD+gallgXIGoztRFIDJkCO+5rVUY3Eh8QoEoEtnMwj5jAJAxTuhahrD2skImQV6iTsTMha8YjQJSCQetUuZ9qgcgLAuScKXwc+WR0GUuzNHotkjDGHByu7CzhCYOSBvtfmjygESMz4mHXEqloUDIpRLcPzkHcull/n+nDAEuHABqioR/CnKqBl8OlYgizfm9QPUJYZubgTd32NRQIbvMVAXSpcsGIUDoPdPGA7xd6DAgl0WBKJJcOAic/lL4OWeKe8Euo9mGCw2t4uUrAnVYMrY9jhO24waEfwzUaH1ZFIgiAWNAyQ/CTquAUB1rVyXbV1KPt38qw86TtKZyd7VY7Hj7p1K8/O1Z71M9XFKGClUyUx1Qeyp0GQxTFIjCncMGnPgIOLtT+D1tNJA71b39sdnmwPELRgBAdpxWrFxGDI1CigaTDSaro+tSpkzh7q1E8XdUKroMCkThzFQHHHgdqDoutAldcSOQN81jD/a9xfWw2nnE65S0NnUQcByHTGdAL3YuLteptNHComnN1W1TbYhXFIjCVdVxYF+BsNWxXA0MvQPoN8IjSZPZhgOl9QCA8Tlx1EgdJFnxwqqWZ6qbu24rUmiArInCz+d2ApbmEOQuPFEgCjd2q9AWdPxD4WdDGjBqftvMbyfGGL7+uRp2nqFfjBr946laFizZ8VooZBLUm2y40GjuOnG/EUIPmtUE/Pyx0J5HOqBAFE7qi4G9rwq9YxwHZI4Hht0tzCW7xJHyRpytaYFUwuH6gV0scUr8ppRJkeecq7e/pL7rxBIpMGiWsCBd3Tng1BcUjLzoFYHoH//4B7KysqBSqTB27Fjs2bNH7Cz1Lq0NwIn/AAffEn5W6YEhdwD9J3W6xXGyXoUopQzjc+IQH6UMaXb7gpGZMeA4YTWD2uZOZuO7RCUAA24W/nmU7xNKtLRMiAfRR1a//fbbWL58Of71r39h7NixWLduHW688UYUFRUhMTFR7OyJhzGgqRK4sF9oD+Kdf7ipw4UxQrKOwcXBM0g4oUE1Ua/CPddkQiXvFf9rIk5clBLDM2KQpFcizpdAn3QV4LACJz8DKg4Je6D1nyKsE0WlVXCsy9a2njd27FiMHj0af//73wEAPM8jPT0df/jDH/DQQw91+Vyj0QiDwYDGxkbo9R2rJ2HDYQNsJmFPrOZqoLlKKMabG9vSxGQC/Se3LUcKgOcZTDYH6pqtKKs34fgFIyZeEY+ByWH8XoSxMzXNMKjliNUoIJF0ElxqzwilW5uz619lENYSN6QBmljhd5kqYoKTr99RUUtEVqsV+/btw8qVK93HJBIJpk2bhl27dnVIb7FYYLG0FYONRqNvL1R3Djjzlfe6uccx5uW4t2OXT2t3OHCorBEMAAfW7qk8GIA4rUJoQObt4O02HCxr6JAtxklRr82GInMUJg0d7v7jfGnnGTgYg83OwF9yTT8V12NAko7ahELM5uCx/WglrHYeHCe0IylkEqGECiA7IQqTrkgA4nLAj16Er7/4ELHGE5A6GoDTnrP09So5spNjAKkC4CTYW9oAxjgwjkNmXBSS9L0kUF19u9f2yUCIGoguXrwIh8OBpKQkj+NJSUn4+eefO6Rfs2YNVq9e7f8L2S1Ac4hHFfM8rK2d7/rpkNkBW9s8MLMDsEo0aJVHw6SIg1GZDKOyH3iJDP3lWo8/PLONdwcgjgN0Kjn6RauQFa9FbgIt8SEGk8WBZL0KlUYzrHYeZpsDZlvbDP2k9hsVyNU4ohwJSdxQRJtLoTdXIMpaA6W9CXK+FUqZROgRtQvLjTha6uEaxM1UZkDaSybRsuAN0hS9jcgfK1euxPLly92/G41GpKenX/6J0enA0DnCz+4vabsvq8cXN8DjlxyTMIb0PDM4iedxjhOSquQyQCP8x+PkGvQfwQCOc5+Ga3dqpcxz4uq94zIBCAu6axWyzqsBJGQMGjl+MzINjDGYrEIQsjp48EwYSqGWt32GHAf8ZkSa87cszxPxdqikNkDFCW1KjEdaf5OziMwjSi0DlNLe0fOmCN6QEFEDUXx8PKRSKaqqqjyOV1VVITk5uUN6pVIJpTKAHiCFtm0DvBCRAEjzsdTKAUiO9v3crgW6SO/DccJ6RVpl518tjuOQEef7Vt/9+kCTn6hdKgqFAiNHjsRXX33lPsbzPL766iuMGzdOxJwRQkJJ9KrZ8uXLMW/ePIwaNQpjxozBunXr0NLSgvvuu0/srBFCQkT0QHTHHXegpqYGq1atQmVlJYYNG4bt27d3aMAmhEQu0ccRdUfEjCMiJEL5+h2lYbeEENFRICKEiI4CESFEdKI3VneHq3nL56kehJCQcn03L9cUHdaBqKlJmELh0+hqQohompqaYDAYOn08rHvNeJ7HhQsXoNP1zkmerikoZWVlEdmrF8nXF8nXBoTu+hhjaGpqQmpqKiSdrJ0FhHmJSCKRIC0t7fIJRabX6yPyj9klkq8vkq8NCM31dVUScqHGakKI6CgQEUJER4GoBymVSjz66KOBrRgQBiL5+iL52oDed31h3VhNCIkMVCIihIiOAhEhRHQUiAghoqNARAgRHQUiP/m7K+27776LgQMHQqVS4eqrr8Z///tfj8cZY1i1ahVSUlKgVqsxbdo0nDp1qicvoVP+XNsrr7yCiRMnIiYmBjExMZg2bVqH9Pn5+eA4zuM2ffr0nr6MTvlzfYWFhR3yrlJ57p4Rrp/d5MmTO1wbx3GYMWOGO03IPztGfLZ161amUCjYa6+9xo4dO8YWLlzIoqOjWVVVldf033//PZNKpezZZ59lx48fZ3/5y1+YXC5nR44ccad5+umnmcFgYB988AE7dOgQu+WWW1h2djZrbW0N1WUxxvy/trvuuov94x//YAcOHGAnTpxg+fn5zGAwsPPnz7vTzJs3j02fPp1VVFS4b3V1daG6JA/+Xl9BQQHT6/Ueea+srPRIE66fXW1trcd1HT16lEmlUlZQUOBOE+rPjgKRH8aMGcOWLFni/t3hcLDU1FS2Zs0ar+lvv/12NmPGDI9jY8eOZYsXL2aMMcbzPEtOTmZ/+9vf3I83NDQwpVLJ3nrrrR64gs75e22XstvtTKfTsU2bNrmPzZs3j82cOTPYWQ2Iv9dXUFDADAZDp+eLpM/uhRdeYDqdjjU3N7uPhfqzo6qZj1y70k6bNs19rKtdaQFg165dHukB4MYbb3SnP3fuHCorKz3SGAwGjB07ttNz9oRAru1SJpMJNpsNsbGxHsd37NiBxMREDBgwAPfffz9qa2uDmndfBHp9zc3NyMzMRHp6OmbOnIljx465H4ukz+7VV1/FnDlzoNV67lMWys+OApGPutqVtrKy0utzKisru0zvuvfnnD0hkGu71IMPPojU1FSPL8T06dOxefNmfPXVV3jmmWewc+dO3HTTTXA4HF2cKfgCub4BAwbgtddew4cffog33ngDPM9j/PjxOH/+PIDI+ez27NmDo0ePYsGCBR7HQ/3ZhfXse9I7PP3009i6dSt27Njh0aA7Z84c989XX301hgwZgpycHOzYsQNTp04VI6s+GzdunMfeeuPHj8eVV16Jl156CU888YSIOQuuV199FVdffTXGjBnjcTzUnx2ViHzk7660AJCcnNxlete9P+fsCYFcm8tzzz2Hp59+Gp9//jmGDBnSZdr+/fsjPj4ep0+f7nae/dGd63ORy+UYPny4O++R8Nm1tLRg69at+O1vf3vZ1+npz44CkY8C2ZV23LhxHukB4IsvvnCnz87ORnJyskcao9GI3bt3h3Sn20B33H322WfxxBNPYPv27Rg1atRlX+f8+fOora1FSkpKUPLtq2DsKOxwOHDkyBF33sP9swOEoSUWiwX33HPPZV+nxz+7kDWLR4CtW7cypVLJCgsL2fHjx9miRYtYdHS0u1v33nvvZQ899JA7/ffff89kMhl77rnn2IkTJ9ijjz7qtfs+Ojqaffjhh+zw4cNs5syZonUB+3NtTz/9NFMoFOy9997z6OJtampijDHW1NTEVqxYwXbt2sXOnTvHvvzySzZixAiWl5fHzGZzSK8tkOtbvXo1++yzz9iZM2fYvn372Jw5c5hKpWLHjh1zpwnXz87l2muvZXfccUeH42J8dhSI/LRhwwaWkZHBFAoFGzNmDPvxxx/dj02aNInNmzfPI/0777zDrrjiCqZQKNhVV13FPvnkE4/HeZ5njzzyCEtKSmJKpZJNnTqVFRUVheJSOvDn2jIzMxmADrdHH32UMcaYyWRiN9xwA0tISGByuZxlZmayhQsXdhiLE0r+XN+yZcvcaZOSktjNN9/M9u/f73G+cP3sGGPs559/ZgDY559/3uFcYnx2tAwIIUR01EZECBEdBSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARMJKYWEhoqOj3b8/9thjGDZsmPv3/Px8zJo1K+T5It1DgYh45Vqz+He/+12Hx5YsWQKO45Cfn++RPtgBICsrC+vWrfM4dscdd+DkyZOdPmf9+vUoLCx0/z558mQsW7YsqPkiwUeBiHQqPT0dW7duRWtrq/uY2WzGm2++iYyMDFHypFarkZiY2OnjBoPBo8REwgMFItKpESNGID09Hdu2bXMf27ZtGzIyMjB8+PBundtbSWXWrFnuUtbkyZNRUlKCP/3pT+5dJICOVbNLtS+Z5efnY+fOnVi/fr37HOfOnUNubi6ee+45j+cdPHgQHMeFfK0kIqBARLo0f/58FBQUuH9/7bXXcN999/X4627btg1paWl4/PHHUVFRgYqKCr/PsX79eowbNw4LFy50nyMjI6PDNQFAQUEBrrvuOuTm5gbrEogfKBCRLt1zzz347rvvUFJSgpKSEnz//fc+LaTVXbGxsZBKpdDpdEhOTg5o1UODwQCFQgGNRuM+h1QqRX5+PoqKitx7f9lsNrz55puYP39+sC+D+IjWrCZdSkhIwIwZM1BYWAjGGGbMmIH4+Hixs9UtqampmDFjBl577TWMGTMG//nPf2CxWDB79myxs9ZnUYmIXNb8+fNRWFiITZs2Ba3UIJFIcOlSWDabLSjn9sWCBQvcDfEFBQW44447oNFoQvb6xBMFInJZ06dPh9Vqhc1mw4033hiUcyYkJHi0+zgcDhw9etQjjUKh6Pb2NZ2d4+abb4ZWq8XGjRuxfft2qpaJjKpm5LKkUilOnDjh/rkzjY2NOHjwoMexuLg4pKend0h7/fXXY/ny5fjkk0+Qk5ODtWvXoqGhwSNNVlYWvv32W8yZMwdKpTKgKmFWVhZ2796N4uJiREVFITY2FhKJxN1WtHLlSuTl5YV0wXvSEZWIiE/0ej30en2XaXbs2IHhw4d73FavXu017fz58zFv3jzMnTsXkyZNQv/+/TFlyhSPNI8//jiKi4uRk5ODhISEgPK9YsUKSKVSDBo0CAkJCSgtLXU/9tvf/hZWqzUkvYCka7RmNemz/ve//2Hq1KkoKyvrsFMqCS0KRKTPsVgsqKmpwbx585CcnIwtW7aInaU+j6pmpM956623kJmZiYaGBjz77LNiZ4eASkSEkF6ASkSEENFRICKEiI4CESFEdBSICCGio0BECBEdBSJCiOgoEBFCREeBiBAiuv8Ptpmir3Bwr/IAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:09.120943Z", + "iopub.status.busy": "2024-03-22T19:39:09.120162Z", + "iopub.status.idle": "2024-03-22T19:39:09.457604Z", + "shell.execute_reply": "2024-03-22T19:39:09.456678Z" + }, + "papermill": { + "duration": 0.35686, + "end_time": "2024-03-22T19:39:09.459669", + "exception": false, + "start_time": "2024-03-22T19:39:09.102809", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEnCAYAAAANc04FAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAs0lEQVR4nO3deXxTZb4/8M/JnrRJuqX7ClRkp4ggIArCuMBVUEfwugwVRUdxZpTLHeD+RhxwFFwHrjroOELhKqKjqDM6gogCAyIKyL4XukBbujdN06zn+f1xmrRp0zZJ0xyafN+vV15pT56ePKdJvnn2h2OMMRBCiIgkYmeAEEIoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUiQojoZGJnoCd4nkdZWRm0Wi04jhM7O4SQdhhjaGxsRGpqKiSSLso9TEQOh4P94Q9/YNnZ2UylUrF+/fqx5cuXM57nffr70tJSBoBudKPbFX4rLS3t8rMsaonoxRdfxJo1a7B+/XoMGTIE+/fvx0MPPQS9Xo/f/va33f69VqsFAJSWlkKn0/V2dgkhfjIajcjIyHB/VjsjaiD6/vvvMWPGDEyfPh0AkJ2djQ8++AA//vijT3/vqo7pdDoKRIRcwbprOhG1sXr8+PHYvn07zpw5AwA4fPgwdu/ejdtuu81reqvVCqPR6HEjhPR9opaIFi9eDKPRiKuvvhpSqRROpxPPP/887r//fq/pV6xYgWXLloU4l4SQ3iZqieijjz7C+++/j40bN+LgwYNYv349XnnlFaxfv95r+iVLlqChocF9Ky0tDXGOCSG9gWNMvIXRMjIysHjxYsyfP9997E9/+hPee+89nDp1qtu/NxqN0Ov1aGhooDaiADHG4HA44HQ6xc4K6YOkUilkMlmnbUC+fkZFrZqZzeYOYwukUil4nhcpR5HFZrOhvLwcZrNZ7KyQPkyj0SAlJQUKhSLgc4gaiG6//XY8//zzyMzMxJAhQ/Dzzz/jtddew9y5c8XMVkTgeR4XLlyAVCpFamoqFAoFDQolfmGMwWazoaqqChcuXEBubm7Xgxa7IGogev311/HMM8/giSeeQGVlJVJTU/HYY49h6dKlYmYrIthsNvA8j4yMDGg0GrGzQ/ootVoNuVyO4uJi2Gw2qFSqgM4jaiDSarVYtWoVVq1aJWY2Ilqg32B9Fc8YOHQ/roX4Lhjvoch6F5KIZ7U7UW2yodnmAGMMPG1ic0WgQEQiit0pBB+jxYGqRivMNuotvBJQICIRxe4UemTlUgkYAEfL70VFReA4DocOHerR+X09z549ezBs2DDI5XLMnDmzR88ZDigQkYjBGIODF6piarnw1nf93t6OHTvAcRzq6+t7JS8LFizAyJEjceHCBRQUFPTKc/QlFIhIn2ez2XxK52wTdBQyifuYGGN6CwsLcdNNNyE9PR0xMTEBncPX6w4G18DX3kKBiHRgc/Cd3lxVGV/S2n1M669JkybhySefxFNPPYWEhATccsstOHbsGG677TZER0cjKSkJDz74IKqrq91/s2XLFtxww0RclZmMQdlpmHHHHSg6fx4A4GwXiIqKijB58mQAQGxsLDiOQ35+vvs8119/PWJiYhAfH4//+I//QGFhYYc8njp1CuPHj4dKpcLQoUOxc+dO97k5jkNNTQ3mzp0LjuPcJaKdO3dizJgxUCqVSElJweLFiz0+/N6u21Vy27p1K/Ly8qBWq3HTTTehsrISX331FQYNGgSdTof77rvPY+Aqz/NYsWIFcnJyoFarMWLECHz88cfux13n/eqrr3DNNddAqVRi9+7dfr9WvurTKzSS3vHmd+c6fSwnIQoz89Lcv/91VyHsTu8livRYNe4ZneH+fe2eC2j20jj89C+u8juP69evx+OPP449e/agvr4eN910Ex555BH8+c9/RnNzMxYtWoRZs2bh22+/BQA0NTXhN797Cjm5g2G3mPHyiucw94HZ+Gb3PvDtqmcZGRn45JNPcPfdd+P06dPQ6XRQq9Xu8yxYsADDhw+HyWTC0qVLceedd+LQoUMe3dj//d//jVWrVmHw4MF47bXXcPvtt+PChQvIyMhAeXk5Bg4ciOXLl2P27NnQ6/W4dOkSpk2bhvz8fGzYsAGnTp3CvHnzoFKp8Mc//tHrdQNAeXk5AOCPf/wj3njjDWg0GsyaNQuzZs2CUqnExo0bYTKZcOedd+L111/HokWLAAgTyN977z289dZbyM3Nxa5du/DAAw/AYDDgxhtvdD/f4sWL8corr6Bfv36IjY31+3XyFQUi0ifl5ubipZdeAiDMT8zLy8MLL7zgfnzt2rXIyMjAmTNncNVVV+Huu+9Gs80Js80BpUyCtWvXwmAw4PSpkxg7aqTHuaVSKeLi4gAAiYmJHlWnu+++2yOt6zwnTpzA0KFD3ceffPJJd9o1a9Zgy5YtePfdd/H73/8eycnJ4DgOer0eycnJAIC//OUvyMjIwBtvvAGO43D11VejrKwMixYtwtKlS91Bru11A62B6E9/+hMmTJgAAHj44YexZMkSFBYWol+/fgCAX/7yl/juu++waNEiWK1WvPDCC/jmm28wbtw4AEC/fv2we/duvP322x6BaPny5fjFL37hz0sTEApEpIP5kwd0+pik3TjAR2/o32na9mMG507I6Um2PFxzzTXunw8fPozvvvsO0dHRHdIVFhbiqquuwtmzZ7F06VLs27cP1dXV7vmMlWUXIRk90ufn7ew8JSUlHoHI9QEHAJlMhtGjR+PkyZOdnvfkyZMYN26cx0DLCRMmwGQy4eLFi8jMzOxw3W0NHz7c/XNSUhI0Go07CLmOuRYcPHfuHMxmc4cAY7PZkJeX53Fs9OjRneY5mCgQkQ5cDblipu1OVFSU+2eTyYTbb78dL774Yod0KSkpAIR5jVlZWXjnnXeQmpoKnucxdOhQKCQMSpnU5+ft7Dyhajhue91tyeVy988cx3n87jrmCpomkwkA8OWXXyItLc0jnVKp9On5go0CEenzRo0ahU8++QTZ2dmQyTq+pWtqanD69Gm88847mDhxIgB02/DqmknednkUf87zww8/4IYbbgAAOBwOHDhwAE8++WSnzzdo0CB88sknYIy5S0V79uyBVqtFenp6l3n11+DBg6FUKlFSUuJRDRMT9ZqRPm/+/Pmora3Ff/7nf+Knn35CYWEhtm7dioceeghOpxOxsbGIj4/H/765Bj8ePoFvvtmOBQsWAECn0zyysrLAcRy++OILVFVVwWQyuc/z17/+FefOncO3337rPk97b775Jj799FOcOnUK8+fPR11dXZerSjzxxBMoLS3Fb37zG5w6dQqff/45nn32WSxYsCDo8wG1Wi0WLlyIp59+GuvXr0dhYSEOHjyI119/vdNFCXsbBSLS56WmpmLPnj1wOp24+eabMWzYMDz11FOIiYmBRCKBRCLB+xs/wOFDP+OGsaPwX/+1AC+//DIAoKHZjlpTx2pVWloali1bhsWLFyMpKQlPPvkkJBIJNm3ahAMHDmDo0KF4+umn3edpb+XKlVi5ciVGjBiB3bt34x//+AcSEhI6vYa0tDT861//wo8//ogRI0bg17/+NR5++GH84Q9/CM4/qZ3nnnsOzzzzDFasWIFBgwbh1ltvxZdffomcnOC14/lD1BUae4pWaAycxWLBhQsXkJOTE/DSDX2J3cmjtskGCQcYtML1Opw8apps4AAk6sL/f9Bbunov+foZpRIRiQiusUKSNr1SkpYuQAbQLHyRUSAiEcEVaKRtxh9IOA6u39oPaiShRYGIRATX4G9Ju8FNrh4qikPiokBEIkJr1czzuLTlE0BVM3FRICIRQSrhIJNyHlUzoLWERIFIXDSgkUSEKKUMUcqOb3d5S5FISmtYi4oCEYlo3oITCT2qmhFCREdfByQiVDdawQDEauSQST2/f11jemmLIfFQICJhjzHmXoWxfbCx2J1oaLZDLpUgLirwLZNJz1DVjPQ5kyZNwlNPPeVz+rYdYu277zl3msB7zfzND+mIAhEJe66u+bY7vLrWD5LQgMYrAgUi0ooxwGET5+ZjiSQ/Px87d+7E6tWrwXEcOI5DYWEhHn74YfdC8AMHDsTq1atbLwvAbx+fhzn3zcLzzz+P1NRUDBw4EACwd+/3mHL9WGQY9Bg9ejQ+++yzDvuSdbUwv7f8FBUVBesViRjURkRaOe3Av18V57kn/hcg676NZvXq1Thz5gyGDh2K5cuXAxB22khPT8ff//53xMfH4/vvv8ejjz6KlJQUzJo1y10i+vfO72CIi8G2bdsACDPD75w5Azf94hb85W8FMFWX4+mnn/Z4vu4W5veWH4PBEMz/TESgQET6FL1eD4VCAY1G4154HgCWLVvm/jknJwd79+7FRx99hFmzZrkLW1GaKPztb39zr7741ltvgeM4vPK/f4FKpUJC9Ej8d1kZ5s2b5z7XG2+80e3C/N7yQ/xDgYi0ksqFkolYz90Db775JtauXYuSkhI0NzfDZrNh5MiRAIS2IQkHDB461B2EAOD06dMYPnw41GoVGBMarMeMGeNxXl8W5ic9R4GItOI4n6pHV5pNmzZh4cKFePXVVzFu3DhotVq8/PLL2LdvHwBAKZdCKZNCr+0YTABAKZWgsxYqXxbmJz1HgYj0OQqFwmNR+z179mD8+PF44okn3Me87b7a3sCBA/Hee+9BJWXu3St++uknjzTdLczvLT/Ef9RrRvqc7Oxs7Nu3D0VFRaiurkZubi7279+PrVu34syZM3jmmWc6BBRv7rvvPvA8j0cffRQnT57E1q1b8corrwBo7ebvbmF+b/lxbdtDfEeBiPQ5CxcuhFQqxeDBg2EwGHDLLbfgrrvuwuzZszF27FjU1NR4lI4amm2w2J1wthsspNPp8M9//hOHDh3CyJEj8f/+3//D0qVLAcC99nJ3C/N7y09JSUmI/hPhgxbPj1CRtHh+bZMNdicPvVoOlbzjZorGZjua7U5EK2X47OMP8dBDD6GhocG93z3pWjAWz6c2IhL2Wie1dnxsw4YNSErLRKwhGYWnjrnHCFEQCi0KRCTs8Z2sVw0AFRUVeOaZpbh8uQJJySm455578Pzzz4c4h4QCEQl7rM1cs/Z+//vfY/7vFsBkdUAll0Kv7tl4JhIYaqwmYY0x5h4j5K1EBLRW2fpwc2mfR4EowoX7h69tR1ln657RDPyeCcZ7iAJRhJLLhSqI2WwWOSe9jUEm5SCTcJ2uwBiMNYkimes95HpPBYLaiCKUVCpFTEwMKisrAQAajSZsl0qNaumxt1gsXh+3O3kwhwNMwsFioWDkK8YYzGYzKisrERMTA6m049AIX1EgimCu2eKuYESAKrEz0AfFxMT0eOUB0QPRpUuXsGjRInz11Vcwm80YMGAA1q1bh9GjR4udtbDHcRxSUlKQmJgIu90udnZIHySXy3tUEnIRNRDV1dVhwoQJmDx5Mr766isYDAacPXsWsbGxYmYr4kil0qC8ma5ERy7W46eiOgxM0uL63IQu09JuHuIRNRC9+OKLyMjIwLp169zHcnJyRMwRCTcmqwPGZjtsXcyOZ4zh7V3nYbXzeGRiDm26KAJRe83+8Y9/YPTo0bjnnnuQmJiIvLw8vPPOO52mt1qtMBqNHjdCumK1CzPhVbLOS3wcx4FnDDxjsDlo5rwYRA1E58+fx5o1a5Cbm4utW7fi8ccfx29/+1usX7/ea/oVK1ZAr9e7bxkZGSHOMelrLHahJKT0Mtm1LUXLpotWCkSiEDUQ8TyPUaNG4YUXXkBeXh4effRRzJs3D2+99ZbX9EuWLEFDQ4P7VlpaGuIck77G4hACkUre9VtdKRMepxKROEQNRCkpKRg8eLDHsUGDBnW6notSqYROp/O4EdIVi6tq1l2JyBWIaKVFUYgaiCZMmIDTp097HDtz5gyysrJEyhEJN1a7q0TUdSBStrQhuQIXCS1RA9HTTz+NH374AS+88ALOnTuHjRs34q9//Svmz58vZrZIGNGp5YjRyKH2uUREgUgMovZTXnvttfj000+xZMkSLF++HDk5OVi1ahXuv/9+MbNFwshdo9J9SpcQrYQ5zokoBXXdi4GWiiWE9BpfP6M0+54QIjoqh5KwVdlowT8PlyMuSo4783yrohFxUCAiYavZ5oSx2Q6FtPu5Y6cqjPjuVBUy4zSYPpx2cA01qpqRsOUaJd3dqGoXi92JZjuNIxIDBSIStiw+jiECAHnLFA87dd+LggIRCVvuUdWy7t/mrrlmNMVDHBSISNjyp0TkGtBIJSJxUCAiYcvdRuRPiYgCkSgoEJGwpZJLEKORI1rVfeewvM3s+z48xrfPou57ErYm5howMdfgU1qFVIIknQpyKQeeAT70+JMgokBECIQ2ovvGZoqdjYhFVTNCiOioRETC1oa9RZBKOMwYmYZoWhD/ikavDglLDiePGpMNACCT+Nbg8/GBi6g2WXH7iFSkxah7M3ukHaqakbDk6rrnON+674W/caLZ5qRBjSKgQETCknv3DpnU5w0TaXS1eCgQkbBkcbgWzff9LU6jq8VDgYiEJX+md7jQ3mbioUBEwpJrh1df24cAmoEvJgpEJCxJJRxiNXLoVHKf/4aqZuKh7nsSlgYmazEwWevX3+jUciTqlNDQTh4hR/9xQlqMzIjByIwYsbMRkahqRggRHZWISFjaerwCVY1WXD8gAdkJUWJnh3SDAhEJS3VNNlQ1WuHgfV9bqLTWjG0nLiOWth8KOQpEJCy1jiPyr/WhodkOqY9z00jwUBsRCUutI6v9GNBI3feioUBEwg5jzD2gkUZW9w0BBaLz588HOx+EBI3NyYNvWXe6w8hqcy1w4h/AoQ+AiqMeD8nblIho3erQCigQDRgwAJMnT8Z7770Hi8US7DwR0iOu/cxkEs49bQMA0FQDHFwPXD4O1BUBJ78Aive6H5a3LFTNGGB3UiAKpYAC0cGDBzF8+HAsWLAAycnJeOyxx/Djjz8GO2+EBMTJM8Rq5IjRtJnewRhw6gvAbgG0yUD6aOH4hV2AqQpAa9UMoHaiUAsoEI0cORKrV69GWVkZ1q5di/Lyclx//fUYOnQoXnvtNVRVVQU7n4T4LC5KgfwJOXhwXHbrwcoTgLEMkMqBYb8EBkwFEnIBxgPFuwEAHMfBoFUiUad0V+1IaHAsCJVhq9WKv/zlL1iyZAlsNhsUCgVmzZqFF198ESkpKcHIp1dGoxF6vR4NDQ3Q6XS99jykj2MMOFAANFYAOROB7OuF46ZK4Kd3hWUcx/4aUMeImcuw5OtntEe9Zvv378cTTzyBlJQUvPbaa1i4cCEKCwuxbds2lJWVYcaMGT05PSHBYbwkBCGJDEgd1Xo8OhGIzRIC1eXj4uWPBDag8bXXXsO6detw+vRpTJs2DRs2bMC0adMgkQhxLScnBwUFBcjOzg5mXgnxyZGL9ThysQFXJ2sxOjsOKD8iPJA4CFBoPBMnDQXqioVAlDVeKB2RkAsoEK1ZswZz585Ffn5+p1WvxMREvPvuuz3KHCGBqDfbUdVoRUacBuCdQPVp4YHkoR0TGwYCZ7YA5hqguQ7fFNlRWmfGDVcZ0N8QHdqMR7CAAtG2bduQmZnpLgG5MMZQWlqKzMxMKBQKzJkzJyiZJMQfrukdarkUqC8ResrkakDvZSdXmRLQpwuloppCNNlSUW+2o9nmDHGuI1tAbUT9+/dHdXV1h+O1tbXIycnpcaYI6QmPhfOrWkpDhoGApJO3e1x/4b72PI2uFklAgaizjjaTyQSVStWjDBHSU+4JrzIJUHNOOJhwVed/ENdPuK8vgYITAhCNIwotv6pmCxYsACCMt1i6dCk0mtaGP6fTiX379mHkyJFBzSAh/rK2BCKNowGwNgq9ZTFeqmUuUQlCI7bNjGh7NQA17W0WYn4Fop9//hmAUCI6evQoFAqF+zGFQoERI0Zg4cKFwc0hIX5qdgWiphLhgD5dGMjYGY4T0lSdQZSlHEA/KhGFmF+B6LvvvgMAPPTQQ1i9ejUNIiRXJLVCBp4BqqZS4UBsdvd/pM8Aqs5A01wBoB+ViEIsoF6zdevWBTsfhATNg9dlATwP7PlMOOBTIBJWZIy2VkAfLYNK4fvyIaTnfA5Ed911FwoKCqDT6XDXXXd1mXbz5s1+Z2TlypVYsmQJfve732HVqlV+/z0hHhrLAYcNkKuA6KTu00cnARIZElUOzL0mBtDE9XoWSSufA5FerwfXMupUr9cHNRM//fQT3n77bQwfPjyo5yURzHhJuNdndN5t35ZEKkz5MJYJQYwCUUj5HIjaVseCWTUzmUy4//778c477+BPf/pT0M5LItOl+mZ8d6oSwxpOYoQCgC7V9z/WJrcEogogaUiv5ZF0FNA4oubmZpjNZvfvxcXFWLVqFb7++mu/zzV//nxMnz4dU6dO7Tat1WqF0Wj0uBHSVpPVgapGK/iGlhKR1o/VH7TJsNid2Hv4OD78qaR3Mki8CigQzZgxAxs2bAAA1NfXY8yYMXj11VcxY8YMrFmzxufzbNq0CQcPHsSKFSt8Sr9ixQro9Xr3LSMjI5DskzBmsTshczZDw5uEbnl/AlF0MgDAaSxHlZFWHg2lgFdonDhxIgDg448/RnJyMoqLi7Fhwwb87//+r0/nKC0txe9+9zu8//77Po/GXrJkCRoaGty30tLSQLJPwpjFziPaVgmZhAM08UJjta+iEiCRySDl7ZBYG8D7sSca6ZmAuu/NZjO0Wi0A4Ouvv8Zdd90FiUSC6667DsXFxT6d48CBA6isrMSoUa3rwzidTuzatQtvvPEGrFYrpFLPLlSlUgmlUhlIlkmEaLY7EW2tgkzqZ2kIACRSSLVJAKoRbauCzclDJaFu/FAIePH8zz77DKWlpdi6dStuvvlmAEBlZaXPgxynTJmCo0eP4tChQ+7b6NGjcf/99+PQoUMdghAhvrDYnS0lIgmg8391UKk2CRIO0NjrYKPR1SETUIlo6dKluO+++/D0009jypQpGDduHAChdJSXl+fTObRaLYYO9VwfJioqCvHx8R2OE+Iri82BOFsVZNEyQOtHj5lLlAFSCQe1vQ52Gl0dMgEFol/+8pe4/vrrUV5ejhEjRriPT5kyBXfeeWfQMkeIv6KcRqg5G2RytTAuyO8TJEDCcdDYa6lEFEIBBSIASE5ORnJyssexMWPG9CgzO3bs6NHfEzI1nQca4wB9mjBI0V/RiVDKJNA7m8AcdgDqoOeRdBRQIGpqasLKlSuxfft2VFZWguc9vzloJ1gimsZy4T6QahkAKKIwJDsFsJkBeRMAmtgdCgEFokceeQQ7d+7Egw8+iJSUFPfUD0JEZywT7gNoqHaLMgC2YqCpqmfnIT4LKBB99dVX+PLLLzFhwoRg54eQgFmsVpw6eRoKCcPVY1ID3ysryiCsYd1EG4WGSkCBKDY2FnFxNCmQXFksdRVottpglqkh0cQEfJ4TDQo4LzVAj2JkDghe/kjnAvrSeO6557B06VKP+WaEiM1WfxEAYI9K6tH+ZA2SGJisDjgaqUQUKgGViF599VUUFhYiKSkJ2dnZkMs9l+E8ePBgUDJHiD+c9cJEV2dUD9t1ohMAAJzV2LIVEW0I0dsCCkQzZ84McjYI6TnWIDRU89rkblJ2Ta5Qo0kaBSfvEDZe1KcFI3ukCwEFomeffTbY+SCkZ+zN4JtqAACcrmeBQyGToFoeAyerokAUIgF3LNTX1+Nvf/sblixZgtraWgBClezSpUtByxwhPmssh51nsMh0UKp7tlW0QiZBsywGTp4JgYj0uoBKREeOHMHUqVOh1+tRVFSEefPmIS4uDps3b0ZJSYl7rSJCQsZYDgkAmyYZemXPJkwrpBI0y2PgtFMgCpWASkQLFixAfn4+zp4967GW0LRp07Br166gZY4QnzWWIys+Crdcl4dRmbE9OpVSLgWvjoNUwgHm2iBlkHQloED0008/4bHHHutwPC0tDRUVFT3OFCF+Yax1sXx/1qjuRFqMGg9MzsPgFB3QXAfwzh6fk3QtoECkVCq9rhd95swZGAyGHmeKEL9YGoS5YZzEt62DfKHUCrvDMh5org/OOUmnAgpEd9xxB5YvXw673Q4A4DgOJSUlWLRoEe6+++6gZpCQbjWWg4FhX5UMHx+qgMUehBIMx7VuKUTtRL0uoED06quvwmQywWAwoLm5GTfeeCMGDBgArVaL559/Pth5JKRrxjI4eYZyPhaltWZhveoecPIMHx+4iH9fYtRzFiIB9Zrp9Xps27YNe/bsweHDh2EymTBq1CiftgQiJOiMZbA7GUwKAxQyCWTSgEelAAAkHHCxzgzeroGTMUgpEPU6vwMRz/MoKCjA5s2bUVRUBI7jkJOTg+TkZDDGaEkQElo8D5gqYHfyaFQmQROEPes5joNCJoFFTmOJQsWvrw7GGO644w488sgjuHTpEoYNG4YhQ4aguLgY+fn5tEwsCb2mKsDpgI2TwyLTQy0PzqYLCmm7QY2MthbqTX6ViAoKCrBr1y5s374dkydP9njs22+/xcyZM7Fhwwb86le/CmomCelUozC/zKxIBJwcNMqAVz/2oJBJUCvXw+kE4LACtiZA2bMR26RzfpWIPvjgA/zP//xPhyAEADfddBMWL16M999/P2iZI6RbLSsyGhXCsJHoHo6qdlFIJWCcDDaZsH8fVc96l1+B6MiRI7j11ls7ffy2227D4cOHe5wpQnzWEojsUSnQKKSIUgSnRCRvafC2KVtGaVMg6lV+vWq1tbVISup8wFhSUhLq6up6nClCfGJvBpqqAQB5Q4YgT6EBC1JbjkouhUouhUMZC5jLaKpHL/MrEDmdTshknf+JVCqFw+HocaYI8YlroXx1LKDQAEDQem2nD29ZXK3MCJw+TiWiXuZXIGKMIT8/v9P9561Wa1AyRYhP3Dt29Hx+Wac08cI9BaJe5VcgmjNnTrdpqMeMhIzRtSJjKt7fWwSNQob/GJECpSw4DdYAWgOR1Qg47cL8MxJ0fgWidevW9VY+CPFPmxn3ZnUSqk3NkHB2KHo4qtrlXGUjDpU2ID1GhevkKmHtanMtoA3SpFriITivGiGhZq4VxvdIZWiSCZNTo5TSoLURNVmdKK01o8pko+pZCFAgIn2Ta/0hbQpMdqGnLCpIgxkBQCkXPhpWB0+BKAQoEJG+qUHYwwy6VDRZhZ7aYMwzc3G1M1kdTgpEIUCBiPRN9SXCvT4TppZAFB3MEpGspURkpxJRKFAgIn2PxSgs4cpxgD4dJksvBqK2VbPmWpr82ksoEJG+p6FUuI9OAuQqSCUcNAoptKrgda0rW2bx2xw8mEoPSKSA0yEsS0uCLnhfIYSEiqtaFpMBAJgyKAlTBiUFbXoHIJSIOE6YhW/nOSjUscJ0EnMNoI4J2vMQAQUi0ve4A1GWx+FgLsonk3D43ZTc1nNq4loCUS0Q3z9oz0MEFIhI32JtFIIBxwH6jF57mg5BjRqsexW1EZG+pa5IuI9OBOQqXDZaULDnArYcK+/d56VA1KuoRET6lppC4T5OqB41NNtRZ7ZDHcQxRC67z1bjstGCsf3ikE6BqFdRiYj0HTwP1F0Qfm5pp2m0CHvr6YLYY+ZS2WhBSa0ZjRZHa4nI1iTMOyNBRYGI9B3GS0IQkKsArbD0h7FZGEMUzK57l9bR1TwgU7auWU2loqCjQEQ64p3CNstX2uC92vPCfWwOIBHeuvXNNgCAXh38QKRwj65u2TmWqme9htqIiKeaQuDkP4VlWKMSgCF3AlEJ7jE6ou5bV31GuG/TfV5vFqpmMZreKBG1GV0NCIGorpgCUS+gQERamaqAY5sBvmW536ZqsMMfYF/cDBwoF0oe12TFYmxOXOgDkqlKGMcjkQLxuQCEraEbmoVAFBulCPpTeg1EAAWiXkBVM9Kq8FshCMX1A8Y/CWjicbGiEpWHtsDm4GFz8NhbWIO9hSJ8ECtPCPdx/YQ2IgAWuxNJOhW0KhmieqHXzDXNw+pwVc2EdY9oIf3go0BEBA2XhDYYTgLk/gJQalGX+QtcqmuGoekMpmYwTL46EQBQ2WgFz4ew/YgxoOqU8HPiIPfhKKUM/zkmE49M7NcrJTTXNA/3pbonv9YJ7WgkaEStmq1YsQKbN2/GqVOnoFarMX78eLz44osYOHCgmNmKTOWHhPukwe5v/n01Kjg0/TEAJbiaPwVkDER8lALpserQVs3qi4VSiFQOxA8I2dMOTNLi6mRt67UqdUIenHahMT8qPmR5CXeiloh27tyJ+fPn44cffsC2bdtgt9tx8803o6mpScxsRR6HDag8KfycMsJ9+PpcA2IHTURGnAaoPAVYGpARpwl9+1DZz8J90lChG71FMCe5eiORcJ7XynFtqmfUThRMopaItmzZ4vF7QUEBEhMTceDAAdxwww0d0lutVo8ti4xGY6/nMSLUFQnf8iq9x/ytaKUME0YOBQ4dEXqLyg8DOcLrYrE7ca7ShCGpuoACU7PNiapGK7QqWdcNzdZGoKqltyxtlMdDHx+4CLPNiSmDEpEeq/E7DwHRxAONlykQBdkV1WvW0CCs9RIXF+f18RUrVmDZsmWhzFJkqDkn3CfkCt/67aWMbAlER4Cs6+FgQMH3RWi2ORGjkfsVBBhj2FtYg/3FdXC2NL70M0ThliHJUMm9NDiX/AAwXljyIzrR4zxVJiusdj642we1YbE78c3Jy7A7ecwcmSYEXHfPWXWvPGekumIaq3mex1NPPYUJEyZg6NChXtMsWbIEDQ0N7ltpaWmIcxmGGANqW+ZvtYzPqWy04F9Hy3GuslE4nnCV0FNlbQTqLkAmlSA3URhlfKDY9y3GGWPYevwy9l2ohZNn0KnlkHAczlc14XiZlwXHLA1A2SHh56wJHg81Wh2w2nlIJRzieqHrHgCkEg5nL5tQVG2GzdnShR9lEO6bqnrlOSPVFVMimj9/Po4dO4bdu3d3mkapVHa6yywJkOkyYDUJjbD6TABAYWUTTlc0wskzDEjUAlKZ0D5zcb9QPYvvj1GZsTh6qQHnq5pQY7IiPrr71+Xn0nqcLDdCwnH4xeAkDE7VoarRiooGC4al6z0TMwac3SYMJ4jJBGKzPR6ubhSq6LFRCkglvdNmJZdKIJNwcPAMFlfJyx2IaoS5b5Ir5ru8T7si/otPPvkkvvjiC3z33XdIT08XOzuRxT1tIlsIOADOV5sACFUmt+Thwn3NOcBmRmyUAv0MQqlov4+lIifPwHHApIEGDE7VAQAMWmXHIAQAFUeB6rPCAMbcmztUGataApEhundKQy6uWf0W1zQPdazwf+IdQjc+CQpRAxFjDE8++SQ+/fRTfPvtt8jJyREzO5GpoWV/sJbVDo0WOyqNVnAckJPQJhBpk4Q2Gt7p7mEbky205Z0qb0RDy1SLrlybHYcHrsvCcG+BB0CT1YHvC6vhrLkAnGnpyMgaD0QbOqStMAoz4A1alU+XGShXu1WzrSUQcRygSWjJMFXPgkXUQDR//ny899572LhxI7RaLSoqKlBRUYHm5mYxsxU5GAOMLfuD6dMAAEXVwtCJVL0aGkW7mrura7/iMAAgWa9CVrwGPGPYX9z5aOO2gx8TopVee9l4nuHDfedx8edvUP39/wkBLyG3Q9uQkG2GsnohEKXFqH271gCpXYHI3mYAo6vRnAJR0IgaiNasWYOGhgZMmjQJKSkp7tuHH34oZrYih7lGWFZDKhN2xABQUmsGAGTFe+kJSxwsVJUaLwOmSgDAmJw49yLz3pyuaMTGH0tQY7J6fRwAwPOQVBzGzY2fIaNhPy7WmOCIzwUGz/Tai2d3MgxIjIZBq4RB27tthq6qmUcgcrcTVfbqc0cSURure3tAGumGa1sebSogkYIxhot1Qmk0Pc5LIFJohJHNVaeFrvzcqUiP1WDOuGyvY4FqTFZ8c/IybA4epy83Yry3Bu2aQmGOW1M1UtUMF9V6HNdcA147FtdKvb89FTIJfjE4KeDL9odKLkzzcDjbvFejXFUz6sIPlium14yIwNU+pBc6CMw2J6KVMjh5hmRdJ20vycOFQHT5GNB/MiCRegShhmY7dCoZapts+PTnS7A5eKTHqnFdTrvpELYmoR3INVhRroYkawK0V/VDzcka7C+ux7D0GO9ji0LohlwDJg9M9KxORrVUzZrrhIGg0uAvQRJpKBBFMtf+8S2BKEopwwPXZcHm4DvvEo/rByiihEBSUwgYrnI/1Gxz4u/7S8EzhmYbD54xxEcrMH14CiRtz9dUDRz5SBgnxEmA9NFCW5BchUE8w4HSRtSYbDhQXIcJAxI8nt7JM5Q3NCNVr/Y8Zy+RSb1UORVRgFwtrNnUVA3oUno9H+Huiui+JyKwmlq3bdaleTzUWXsPAGHcTHLLgNOKIx4PVTVaYXXwaLI6wTOGfoYo/PKadM9Gb6sJOPKhEITUscDoh4ABU9xLe0gkHMb3F0pPP5fUoallX3uX4pom/H3/RfzfD8XiVe05jgY2BhmViCKVsaVapokH5CrwPIOTMci9lQDaSx4OlOwTSkRWk3st58x4DeZN7IfLRguilV7mkDkdwLFPhL3rNXFA3oNCu1M7/Q3RSItVI1Gr7FAyO14mzC/MTogKyeTbGpMVe8/XQCmTerZLRScKGz1Sg3VQUCCKVO5qmTDJtbLRio/2lyI7IQp3jEjt+m+jEoTqiLFcGGmd3drFrpBJhNn67TEGnPkKMJYJM+iH3eM1CAHCcrT3XJPeIdDUmKworBIGWw5pGRDZ2xw8w9nLJmhV7T4qrgZrE5WIgoGqZpGqwXP80MU6M5w88726kzZauL+0X2iw7U7pPqDimNAmNOTO1uU0OtE2CFnsTpy93IgtxyvAGDAgMRoJPkwpCYa2Axo9/jfRycK9qeLK22SgD6JAFImcdmGOGeBuqC6tE8YPeS3NeJM4WFg2xGYWSkVdqT4HnN8h/DxgKhDn+wh6i92Jz36+hC+OlKPSaIVKLsWNAzuOtO4trgGNDp7B7tGFbxDGVNktQnsX6REKRJGosVwYuayIAlQxcPKtI5UzfF3SQyIBMscKPxfv6XzTQVMVcOIzodSQmtdhTaHucByQpFNBr5YjK16DWaPTe2Uzxc7IpZy7ncpjUKNU1qZ6djlk+QlX1EYUidqOH+I4VDQ0w+bgoVZIkeDPJNKUkcClg0IX9vnvgIG3eT5uMQJHPxJKYDGZwlrYfjYwK2VSTL46EZP9+qvg4TgOarkUJqsDFrvTc/+06GRhlHljOWCg5Y17gkpEkajd+KGLLdM6MmL9XAZWIhWCCyCsG+RaOwgQ1nQ+8mFLD1m80C4kEXdwYqA0SiHf7YcSQNvSi9ZIJaKeohJRpPGY6OpqH2qZ1hEbwATS2Gyh16xoD3D6K6FLW6ERpoA4rELX/vBZnfaQ9QXRShmqOVvr/mYu2paBjK4GazE3n+zjKBBFGi8TXXMToyGXcsj0taG6veyJgNMGlP4EXD7eelybDAy9S2jU7sNuG5oCuZTrWFqMMgi9gDYzYDX2+esUEwWiSOOa6KpLc1eVRmTEYERGTODn5DihN8wwCKg6KQxcjMkEDFeHxQqGnY40l8qFLYVMVUL1jAJRwCgQRZp27UNBpU9zj0uKGNqUlkBU7jHvjvin739dEf+0C0SFVSb3/vHEu6pGK744UobtJ700SmtbBjY2loc2U2GGSkSRxGIUerNaJrpaHU58cbgcPGN4eGJOSMfn9CV2J4+zl03Qqb38f3QtJUvjJVpMvwfovxZJXBNdoxMBmRJl9RbwjEGvllMQ6kKUUvi+NlsdHafARBmEtiKHjWbi9wAFokjSbqJraa2f0zoiVJSidZpHhy58iaR1GRXXsAjiNwpEkaSuSLh3DWRsGT+UEde7C9D3dTKpBEq58FHpMKgRaG34d41YJ36jQBQprI3CVAyOA2KyYLE7UdkozA8L2b7xfVh0S/XM5DUQuUpEFIgCRYEoUrhKQ9FJgEKDi3XNYAyIj1a4P2Skc642tEaLl0CkSxMCfHO9EPCJ3ygQRQpXIGrZurm4Rti/zOfZ9hFOp5ZBKuFgdTg7PihTti4dW18a2oyFCfoqjASMAbUXhJ9b1gKaMCABmXEa6DXUW+aLid5282grJkvY662+GEgaHNrMhQEqEUUCY5mw64ZU7h73opJLkZukRWIvb9kcLuRSSdcrE7gWe3OVPIlfKBBFgurTwn38AGGyKwk+fYYwd6+5HjB3vv028Y4CUbhjrHUTw5bFu3acrsQP52u89wARr2wOHl8eKcemH0vg5L2sUS1TALqWTQeoVOQ3CkThzlQp7F8mkQFx/WFz8DhysQF7C2tgaz84j3RKLuVwvsqE8gYLGi2dzM1r6QhA3YWQ5StcUCAKd65NEOP7AzIFSmqb4OQZdGo5Yqmh2mccxyGmZZ+2OnMngSiun3Bfe0FYCoX4jAJROHM6hD3qASBlBADgXKWwL9iAxOiQbFAYTuI0QiCqbbJ6T6BNAZRaYY1uqp75hQJROKs8LqzGqNIBsTlw8gyFVcL4oQGJ0SJnru+JjRJKkLVNnZSIOA5IaFmTqPpMiHIVHigQhSueB0p+EH5OuwaQSFBSa4bNwSNKKUWqnrrt/RXnqpo12TpPZGgTiHhqg/MVBaJwdfmY0I0sVwGpwl5iZy8L0w+oWhYYVyCqNXcRiPSZwv/c3iwMbiQ+oUAUjuwWYZ8xAMgcJ3QtQ1h7WSGTIDdRK2Lm+q5YjQJSCYcopcz7VA9AWBbEMEj4ueJo6DLXx9HotnDDGHBmi7CzhCYeSL/W/dCkgYkY3z8BcimVhgIhl0rw+KT+kEu7+f5OGQ6U/QxUnRa+FORUDe4OlYjCzcX9QOVJYZubq6d12NRQIetmqgLpUrdBCBB6z6ISAN4hdBiQblEgCidlh4Bz3wg/95/sXrDLaLGjrL5ZvHyFoQ5LxrbFccJ23IDwxUCN1t2iQBQOGAOKvxd2WgWE6libKtmB4jp8+FMpdp6hNZV7qsnqwIc/leCvu857n+rhkjJCqJKZa4Gas6HLYB9Fgaivc9qBk/8Azu8Ufk+/Fhgwxb39scXuxIkyIwAgJz5KrFyGDY1CinqzHWabs+tSpkzh7q1E0W4qFXWDAlFfZq4Ffv4/4PIJoU3oqluA3Kkee7DvL6qDzcEjQauktamDgOM4ZLUE9KKWxeU6lX6tsGiaqbJ1qg3xigJRX3X5BHBgnbDVsVwNjJgNpI3ySNJosePnkjoAwPj+8dRIHSTZCcKqloWVpq7bihQaIHui8POFnYDVFILc9U0UiPoah01oCzrxufCzPh0YPbd15ncLxhi+PVUJB8+QFqtGvwSqlgVLTkIUFDIJ6sx2lDVYuk6cNkroQbOZgVNfCO15pAMKRH1JXRGw/12hd4zjgKzxwMj7hblk7Ry91IDzVU2QSjjcdHUXS5wSvyllUuS2zNU7WFzXdWKJFBg8U1iQrvYCcHYbBSMvKBD1Bc31wMl/Aoc+EH5W6YDhs4F+N3a6xXGyToVopQzj+8cjIVoZ0uxGgmuyYsFxwmoGNaZOZuO7RBuAgdOEL49LB4QSLS0T4uGKCERvvvkmsrOzoVKpMHbsWPz4449iZ0l8jAHGcuDUl8CP7wAVLct5pOYB1z7SukZyG06eudssEnUqPHBdFq7Jig1lriNGfLQSeZmxuG1YMuJ9CfRJQ4TOBI4Dyg8L7XvV56h01EL0KR4ffvghFixYgLfeegtjx47FqlWrcMstt+D06dNITEwUO3uh4bQDdrOwJ5apEjBdForxlobWNLFZQL9JrcuRAuB5BrPdiVqTDaV1ZpwoM2LiVQm4OlmoqqkVUpDec+NVBo/fC6tM0KvliNMoIJF4qQqn5gFKnVC6baoGjv4dUOmFtcT16YAmTvhdpvLo+YwEHOuy2b/3jR07Ftdeey3eeOMNAADP88jIyMBvfvMbLF682COt1WqF1dpaDDYajcjIyEBDQwN0uo7tJG61F4DC7d6/fTyOMS/HvR3rPq3D6cTh0gYwABxYmz/lwQDERymEBmTeAd5hx6HS+g7ZYpwUdVE5UGSNxo2j89xvzrd3FsLJGOwOBr7dNSVolXhgbCa1CYWY3cnjr7vOw+bgwXFCO5JCJoGEAzgAOYZod+DirWZ8u+1zxBlPQursWK3TqeTISY4FpAqAk2B/ST0Y48A4Dlnx0UjSXSGBatgsr+2TbRmNRuj1+m4/o6KWiGw2Gw4cOIAlS5a4j0kkEkydOhV79+7tkH7FihVYtmyZ/0/ksAKmEI8q5nnYmjvf9dMpcwD21hKLxQnYJBo0y2NgVsTDqEyGUZkGXiJDP3mUxxvPYufdAYjjAK1KjrQYFbITojDAQEt8iMFsdSJZp0KF0QKbg4fF7oTF3jpDP6ntRgVyNY4qr4EkfgRiLCXQWcoRbauC0tEIOd8MpUwi9Ig6hOVGnE11cA3iZioLIL1CJtGy4A3SFLVEVFZWhrS0NHz//fcYN26c+/jvf/977Ny5E/v27fNIH3CJyNYkVHmANh/oNh9Wjw9ugMfbHeMZQ1m9BZzE8zjHCUlVchliNcI3HpNrcLmJARznPg3X5tRKmRR6dev60rUtC3PJpRyiFDLv1QAiCsYYzDYhCNmcPHgmHFPLpe62JMYYSms7GZXNO6CS2pGo4gCnDWA8LtWZW4rIPPRqGaKV0iujbUmfLuyV14U+USLyl1KphFIZQA+QIspr425vkgBI77rU6sYBSI7x/dyuBbrIlYfjhPWKopSdf7Q4jkNmvO9bfaf5+D7qy0TtNUtISIBUKsXly5c9jl++fBnJycki5YoQEmqiBiKFQoFrrrkG27dvdx/jeR7bt2/3qKoRQsKb6FWzBQsWYM6cORg9ejTGjBmDVatWoampCQ899JDYWSOEhIjogWj27NmoqqrC0qVLUVFRgZEjR2LLli1ISkoSO2uEkBARfRxRT/jaIk8IEYevn9ErYooHISSyUSAihIhO9DainnDVKo1Go8g5IYR44/psdtcC1KcDUWOjMIUiIyND5JwQQrrS2NgIvV7f6eN9urGa53mUlZVBq9VekfOrXFNQSktLw7IxPZyvL5yvDQjd9THG0NjYiNTUVEg6WTsL6OMlIolEgvT0dLGz0S2dTheWb2aXcL6+cL42IDTX11VJyIUaqwkhoqNARAgRHQWiXqRUKvHss88GtmJAHxDO1xfO1wZcedfXpxurCSHhgUpEhBDRUSAihIiOAhEhRHQUiAghoqNA5Cd/N4P8+9//jquvvhoqlQrDhg3Dv/71L4/HGWNYunQpUlJSoFarMXXqVJw9e7Y3L6FT/lzbO++8g4kTJyI2NhaxsbGYOnVqh/T5+fngOM7jduutt/b2ZXTKn+srKCjokHeVynP3jL762k2aNKnDtXEch+nTp7vThPy1Y8RnmzZtYgqFgq1du5YdP36czZs3j8XExLDLly97Tb9nzx4mlUrZSy+9xE6cOMH+8Ic/MLlczo4ePepOs3LlSqbX69lnn33GDh8+zO644w6Wk5PDmpubQ3VZjDH/r+2+++5jb775Jvv555/ZyZMnWX5+PtPr9ezixYvuNHPmzGG33norKy8vd99qa2tDdUke/L2+devWMZ1O55H3iooKjzR99bWrqanxuK5jx44xqVTK1q1b504T6teOApEfxowZw+bPn+/+3el0stTUVLZixQqv6WfNmsWmT5/ucWzs2LHsscceY4wxxvM8S05OZi+//LL78fr6eqZUKtkHH3zQC1fQOX+vrT2Hw8G0Wi1bv369+9icOXPYjBkzgp3VgPh7fevWrWN6vb7T84XTa/fnP/+ZabVaZjKZ3MdC/dpR1cxHrs0gp06d6j7W1WaQALB3716P9ABwyy23uNNfuHABFRUVHmn0ej3Gjh3b6Tl7QyDX1p7ZbIbdbkdcXJzH8R07diAxMREDBw7E448/jpqamqDm3ReBXp/JZEJWVhYyMjIwY8YMHD9+3P1YOL127777Lu69915ERUV5HA/la0eByEfV1dVwOp0d1tJOSkpCRUWF17+pqKjoMr3r3p9z9oZArq29RYsWITU11eMDceutt2LDhg3Yvn07XnzxRezcuRO33XYbnE5nF2cKvkCub+DAgVi7di0+//xzvPfee+B5HuPHj8fFixcBhM9r9+OPP+LYsWN45JFHPI6H+rXr07PvyZVh5cqV2LRpE3bs2OHRoHvvvfe6fx42bBiGDx+O/v37Y8eOHZgyZYoYWfXZuHHjPLa0Gj9+PAYNGoS3334bzz33nIg5C653330Xw4YNw5gxYzyOh/q1oxKRjwLZDDI5ObnL9K57sTeY7MlGl6+88gpWrlyJr7/+GsOHD+8ybb9+/ZCQkIBz5871OM/+CMZGnnK5HHl5ee68h8Nr19TUhE2bNuHhhx/u9nl6+7WjQOSjQDaDHDdunEd6ANi2bZs7fU5ODpKTkz3SGI1G7Nu3L6QbTAa60eVLL72E5557Dlu2bMHo0aO7fZ6LFy+ipqYGKSkpQcm3r4KxkafT6cTRo0fdee/rrx0gDC2xWq144IEHun2eXn/tQtYsHgY2bdrElEolKygoYCdOnGCPPvooi4mJcXfrPvjgg2zx4sXu9Hv27GEymYy98sor7OTJk+zZZ5/12n0fExPDPv/8c3bkyBE2Y8YM0bqA/bm2lStXMoVCwT7++GOPLt7GxkbGGGONjY1s4cKFbO/evezChQvsm2++YaNGjWK5ubnMYrGE9NoCub5ly5axrVu3ssLCQnbgwAF27733MpVKxY4fP+5O01dfO5frr7+ezZ49u8NxMV47CkR+ev3111lmZiZTKBRszJgx7IcffnA/duONN7I5c+Z4pP/oo4/YVVddxRQKBRsyZAj78ssvPR7neZ4988wzLCkpiSmVSjZlyhR2+vTpUFxKB/5cW1ZWFgPQ4fbss88yxhgzm83s5ptvZgaDgcnlcpaVlcXmzZvXYSxOKPlzfU899ZQ7bVJSEps2bRo7ePCgx/n66mvHGGOnTp1iANjXX3/d4VxivHa0DAghRHTURkQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiI4CESFEdBSICCGio0BE+pSCggLExMS4f//jH/+IkSNHun/Pz8/HzJkzQ54v0jMUiIhXrjWLf/3rX3d4bP78+eA4Dvn5+R7pgx0AsrOzsWrVKo9js2fPxpkzZzr9m9WrV6OgoMD9+6RJk/DUU08FNV8k+CgQkU5lZGRg06ZNaG5udh+zWCzYuHEjMjMzRcmTWq1GYmJip4/r9XqPEhPpGygQkU6NGjUKGRkZ2Lx5s/vY5s2bkZmZiby8vB6d21tJZebMme5S1qRJk1BcXIynn37avYsE0LFq1l7bkll+fj527tyJ1atXu89x4cIFDBgwAK+88orH3x06dAgcx4V8rSQioEBEujR37lysW7fO/fvatWvx0EMP9frzbt68Genp6Vi+fDnKy8tRXl7u9zlWr16NcePGYd68ee5zZGZmdrgmAFi3bh1uuOEGDBgwIFiXQPxAgYh06YEHHsDu3btRXFyM4uJi7Nmzx6eFtHoqLi4OUqkUWq0WycnJAa16qNfroVAooNFo3OeQSqXIz8/H6dOn3Xt/2e12bNy4EXPnzg32ZRAf0ZrVpEsGgwHTp09HQUEBGGOYPn06EhISxM5Wj6SmpmL69OlYu3YtxowZg3/+85+wWq245557xM5axKISEenW3LlzUVBQgPXr1wet1CCRSNB+KSy73R6Uc/vikUcecTfEr1u3DrNnz4ZGownZ8xNPFIhIt2699VbYbDbY7XbccsstQTmnwWDwaPdxOp04duyYRxqFQtHj7Ws6O8e0adMQFRWFNWvWYMuWLVQtExlVzUi3pFIpTp486f65Mw0NDTh06JDHsfj4eGRkZHRIe9NNN2HBggX48ssv0b9/f7z22muor6/3SJOdnY1du3bh3nvvhVKpDKhKmJ2djX379qGoqAjR0dGIi4uDRCJxtxUtWbIEubm5IV3wnnREJSLiE51OB51O12WaHTt2IC8vz+O2bNkyr2nnzp2LOXPm4Fe/+hVuvPFG9OvXD5MnT/ZIs3z5chQVFaF///4wGAwB5XvhwoWQSqUYPHgwDAYDSkpK3I89/PDDsNlsIekFJF2jNatJxPr3v/+NKVOmoLS0tMNOqSS0KBCRiGO1WlFVVYU5c+YgOTkZ77//vthZinhUNSMR54MPPkBWVhbq6+vx0ksviZ0dAioREUKuAFQiIoSIjgIRIUR0FIgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENH9f3oqPtzGWURhAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:09.496676Z", + "iopub.status.busy": "2024-03-22T19:39:09.496344Z", + "iopub.status.idle": "2024-03-22T19:39:09.652658Z", + "shell.execute_reply": "2024-03-22T19:39:09.651769Z" + }, + "papermill": { + "duration": 0.178936, + "end_time": "2024-03-22T19:39:09.656531", + "exception": false, + "start_time": "2024-03-22T19:39:09.477595", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApJ0lEQVR4nO3de1wU97038A/LZRG5KCIXEV3RRjQKKBTEJMeYCKRJPOaxTXjMBcTU06r0GDckhrSBEnNE6400h8iJDV6SNNp4SdJoUIPSRuMVogcVsCCKF0BQAYG4LLu/5w8fJm52gQWBZZbP+/XaF8xvfjPzHWb9ODM7O2MjhBAgIpIZhaULICLqCoYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEcmSxcMrIyMDKpUKjo6OCA8Px/Hjx9vtX1tbi0WLFsHHxwdKpRIPPPAA9uzZ00vVElFfYWfJhW/btg1qtRqZmZkIDw9Heno6oqOjUVxcDE9PT6P+zc3NiIyMhKenJ7Zv3w5fX19cunQJgwYNMnuZer0e165dg4uLC2xsbLpxbYioOwghcPv2bQwbNgwKRTv7V8KCwsLCxKJFi6RhnU4nhg0bJtLS0kz2X79+vfD39xfNzc1dXubly5cFAL744quPvy5fvtzuv2UbISxzV4nm5mY4OTlh+/bteOaZZ6T2uLg41NbW4osvvjCa5sknn4S7uzucnJzwxRdfYOjQoXj++eexdOlS2NramlyORqOBRqORhuvq6jBixAiUlZXBxcWl29err9BqtTh48CCmT58Oe3t7S5dD3aC/bNPbt29j1KhRqK2thZubW5v9LHbYWFNTA51OBy8vL4N2Ly8vFBUVmZzmwoULOHDgAF544QXs2bMHJSUlWLhwIbRaLVJSUkxOk5aWhtTUVKP2I0eOwMnJ6f5XpA9zcnLCsWPHLF0GdaP+sE2bmpoAoMPTOhY959VZer0enp6e+OCDD2Bra4uQkBBcvXoVq1atajO8kpKSoFarpeH6+nr4+fkhKioKrq6uvVV6r9Nqtdi/fz8iIyOt+n/p/qS/bNP6+nqz+lksvDw8PGBra4uqqiqD9qqqKnh7e5ucxsfHB/b29gaHiOPGjUNlZSWam5vh4OBgNI1SqYRSqTRqt7e3t+o3QKv+sp79ibVvU3PXzWKXSjg4OCAkJAQ5OTlSm16vR05ODiIiIkxO89BDD6GkpAR6vV5qO3/+PHx8fEwGFxFZL4te56VWq7FhwwZs3rwZhYWFWLBgARobGxEfHw8AiI2NRVJSktR/wYIFuHnzJhYvXozz589j9+7dWL58ORYtWmSpVSAiC7HoOa+YmBhUV1cjOTkZlZWVCA4ORnZ2tnQSv7y83OA6Dz8/P+zduxdLlixBYGAgfH19sXjxYixdutRSq0BEFmLxE/YJCQlISEgwOS43N9eoLSIiAkePHu3hqoior7N4eBGRsaamJqNLhhp+0OC7glIM9jgJ5wE/fggVEBBg9Zf9mMLwIuqDioqKEBISYnLcn34ynJeXh8mTJ/d8UX0Mw4uoDwoICEBeXp5BW3FFLdSfFWDtsxMx1meQQd/+iOFF1Ac5OTkZ7U0pLt2A8tsfMG5CEIJHDrFQZX2HxW+JQ0TUFQwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLUJ8IrIyMDKpUKjo6OCA8Px/Hjx9vsu2nTJtjY2Bi8HB0de7FaIuoLLB5e27Ztg1qtRkpKCvLz8xEUFITo6Ghcv369zWlcXV1RUVEhvS5dutSLFRNRX2Dx8Fq7di3mz5+P+Ph4jB8/HpmZmXByckJWVlab09jY2MDb21t6eXl59WLFRNQX2Fly4c3NzcjLy0NSUpLUplAoMGPGDBw5cqTN6RoaGjBy5Ejo9XpMnjwZy5cvx4MPPmiyr0ajgUajkYbr6+sBAFqtFlqttpvWxPKamppQXFwsDTf8oMF3BaVwGXQUzgOUBn3Hjh0LJyen3i6R7lNLS4v005reuz9l7rpZNLxqamqg0+mM9py8vLxQVFRkcpqxY8ciKysLgYGBqKurw+rVqzF16lScPXsWw4cPN+qflpaG1NRUo/Z9+/ZZ1T/g0tJSvPrqq0btfzLRd82aNRg9enTPF0Xd6nIDANjh6NGjuHrG0tX0nKamJrP6WTS8uiIiIgIRERHS8NSpUzFu3Dj8z//8D5YtW2bUPykpCWq1Whqur6+Hn58foqKi4Orq2is194ampiY8/PDD0vD5ijq8tuscVv2f8XjAx82gL/e85Ol0+U2g4CSmTJmCoBHuli6nx7QeHXXEouHl4eEBW1tbVFVVGbRXVVXB29vbrHnY29tj0qRJKCkpMTleqVRCqVQatdvb28Pe3r7zRfdRbm5uCAsLk4YdLt2A8kgzJgRPRvDIIRasjLqLnZ2d9NOa3rs/Ze66WfSEvYODA0JCQpCTkyO16fV65OTkGOxdtUen06GgoAA+Pj49VSYR9UEWP2xUq9WIi4tDaGgowsLCkJ6ejsbGRsTHxwMAYmNj4evri7S0NADA22+/jSlTpmDMmDGora3FqlWrcOnSJfz617+25GoQUS+zeHjFxMSguroaycnJqKysRHBwMLKzs6WT+OXl5VAoftxBvHXrFubPn4/KykoMHjwYISEh+O677zB+/HhLrQIRWYDFwwsAEhISkJCQYHJcbm6uwfC6deuwbt26XqiKiPoyi1+kSkTUFQwvIpIlhhcRyRLDi4hkieFFRLLUJz5tJCKgrKYRjZqWNseXVjdKP1uvtv+pgUo7jPIY2CP19TUML6I+oKymEdNX55rV99XtBe2OP5j4aL8IMIYXUR/QuseVHhOMMZ7Opvv8oMFXuUfw9KMRGDjA+Pu6Jdcb8Mq2U+3uvVkThpeMtXeYYc4hBtC/DjPkYIynMyb4upkcp9VqUTkUmDxysFV/MdtcDC+ZMvcwo6NDDKD/HGaQdWF4yVRHhxkdHWIA/e8wg6wLw0vm2jrM4CEGWTte50VEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJF6nKlEZ3BwrHqyirL4bC0fgK+5aWFlxruYbCm4VtfrexrL4BCser0OjuADD9fTqivorhJVPXGi9h4Kj38Obx9vu9n/1+u+MHjgKuNQYjBF7dWB1Rz2N4ydSwgSPRWPY7vBsTjNEmvtvY0tKCw4cO46GHH2pzz6v0egMWbzuFYdNH9nS5RN2O4SVTSltH6O/4YpTrWIwfYvq7jWV2ZRjnPq7N7zbq79RBf6caSlvHni6XqNvxhD0RyRLDi4hkieFFRLLE8CIiWeIJe6I+oKPr9oCOr93rb9ftMbyI+gBzr9sD2r92rz9dt8fwIuoDOrpuD+j42r3+dt0ew4uoD+jouj2g42v3+tt1e50+YX/hwoWeqIOIqFM6HV5jxozB9OnT8fHHH+POnTvdUkRGRgZUKhUcHR0RHh6O48fNOPAHsHXrVtjY2OCZZ57pljqISD46HV75+fkIDAyEWq2Gt7c3fvOb35gdNqZs27YNarUaKSkpyM/PR1BQEKKjo3H9+vV2p7t48SISExPxyCOPdHnZRCRfnQ6v4OBgvPvuu7h27RqysrJQUVGBhx9+GBMmTMDatWtRXV3dqfmtXbsW8+fPR3x8PMaPH4/MzEw4OTkhKyurzWl0Oh1eeOEFpKamwt/fv7OrQERWoMsn7O3s7DB79mw89dRTeP/995GUlITExES8+eabeO6557By5Ur4+Pi0O4/m5mbk5eUhKSlJalMoFJgxYwaOHDnS5nRvv/02PD098fLLL+Pbb79tdxkajQYajUYarq+vB3D35KdWqzVnVfuklpYW6aep9Whta28dO5oH9R5ztkVH29Ratqe5tXc5vE6ePImsrCxs3boVAwcORGJiIl5++WVcuXIFqampmDVrVoeHkzU1NdDpdPDyMrwmxcvLC0VFRSanOXToED788EOcOnXKrDrT0tKQmppq1L5v3z44OTmZNY++6HIDANjh0KFDuGT6k3UAwP79++97HtTzOrMt2tqm1rI9m5qazOrX6fBau3YtNm7ciOLiYjz55JPYsmULnnzySSgUd49AR40ahU2bNkGlUnV21h26ffs2XnrpJWzYsAEeHh5mTZOUlAS1Wi0N19fXw8/PD1FRUXB1de32GnvL2Wv1WF1wFA8//DAeHGa8HlqtFvv370dkZGSbt8TpaB7Ue8zZFh1tU2vZnq1HRx3pdHitX78e8+bNw9y5c9s8LPT09MSHH37Y4bw8PDxga2uLqqoqg/aqqip4e3sb9S8tLcXFixcxc+ZMqU2v1wO4exhbXFyM0aNHG0yjVCqhVCqN5mVvb9/mP2o5aL1I0c7Ort31aG89zZ0H9bzObIu2tqm1bE9za+90eO3fvx8jRoyQ9rRaCSFw+fJljBgxAg4ODoiLi+twXg4ODggJCUFOTo50uYNer0dOTg4SEhKM+gcEBKCgoMCg7Q9/+ANu376Nd999F35+fp1dHSKSqU6H1+jRo1FRUQFPT0+D9ps3b2LUqFHQ6XSdmp9arUZcXBxCQ0MRFhaG9PR0NDY2Ij4+HgAQGxsLX19fpKWlwdHRERMmTDCYftCgQQBg1E5E1q3T4SWEMNne0NAAR8fOfy0hJiYG1dXVSE5ORmVlJYKDg5GdnS2dxC8vLzfayyMiMju8Wk9629jYIDk52eCTOp1Oh2PHjiE4OLhLRSQkJJg8TASA3NzcdqfdtGlTl5ZJRPJmdnh9//33AO7ueRUUFMDBwUEa5+DggKCgICQmJnZ/hUREJpgdXgcPHgQAxMfH491335X1ZQZEJH+dPue1cePGnqiDiKhTzAqv2bNnY9OmTXB1dcXs2bPb7btz585uKYyIqD1mhZebmxtsbGyk34mILM2s8Lr3UJGHjUTUF/ACKiKSJbP2vCZNmiQdNnYkPz//vgoiIjKHWeHF2ywTUV9jVnilpKT0dB1ERJ3Cc15EJEtm7Xm5u7vj/Pnz8PDwwODBg9s9/3Xz5s1uK46IqC1mhde6devg4uIi/W7uyXsiop5iVnjde2PBuXPn9lQtRERm6/Q5L1tbW5PPVLxx4wZsbW27pSgioo50OrzauhmhRqMxuE0OEVFPMvuuEn/+858B3L0Z4V/+8hc4O//4bCWdTod//vOfCAgI6P4KiYhMMDu81q1bB+DunldmZqbBIaKDgwNUKhUyMzO7v0IiIhPMDq+ysjIAwPTp07Fz504MHjy4x4oiIupIp29G2HpHVSIiS+p0eM2bN6/d8VlZWV0uhojIXJ0Or1u3bhkMa7VanDlzBrW1tXjssce6rTAiovZ0Orx27dpl1KbX67FgwQKMHj26W4oiIupIt3wxW6FQQK1WS59IEhH1tG67q0RpaSlaWlq6a3ZERO3q9GFj65OzWwkhUFFRgd27dxt8B5KIqCd1Orxan5zdSqFQYOjQoVizZk2Hn0QSEXUXXudFRLLEO6kSkSwxvIhIlhheRCRLDC8ikqVOn7Bvy5UrV/D222/jgw8+6K5ZUjt+0OoAAGeu1pkc3/iDBierAe9LtzBwgNJkn5LrDT1WH1FP67bwunHjBj788MMuhVdGRgZWrVqFyspKBAUF4b333kNYWJjJvjt37sTy5ctRUlICrVaLn/3sZ3j11Vfx0ksv3e8qyErp/w+eN3YWtNPLDh+VnOhwXgOV3fY2IOo1Fn/Xbtu2DWq1GpmZmQgPD0d6ejqio6NRXFwMT09Po/7u7u74/e9/j4CAADg4OOCrr75CfHw8PD09ER0dbYE1sIyoB70BAKM9nTHA3vjZAcUVdXh1ewHW/Goixvq4tTmfgUo7jPIY2GN1EvUUi4fX2rVrMX/+fMTHxwMAMjMzsXv3bmRlZeGNN94w6v/oo48aDC9evBibN2/GoUOH+lV4uQ90wP8NG9Hm+Navao0eOhATfNsOLyK5smh4NTc3Iy8vD0lJSVKbQqHAjBkzcOTIkQ6nF0LgwIEDKC4uxsqVK0320Wg00Gg00nB9fT2Au7fy0Wq197kGfVdreLW0tFj1eloLc7ZXa3tb461lm5tbu9nhNXv27HbH19bWmjsrSU1NDXQ6Hby8vAzavby8UFRU1OZ0dXV18PX1hUajga2tLd5//31ERkaa7JuWlobU1FSj9n379sHJyanTNcvF5QYAsMPRo0dx9Yylq6GOlNYDgB227j2M4QNNP6FLqwduaoALO/bD3sR1AlU/2ACwxaFDh3DJ2Xi8XDQ1NZnVz+zwcnNr/9DDzc0NsbGx5s7uvri4uODUqVNoaGhATk4O1Go1/P39jQ4pASApKcngy+T19fXw8/NDVFQUXF1de6VeSzhdfhMoOIkpU6YgaIS7pcuhDvzt5BXg7DlsvXD/zz6NfnwaVEPkex6z9eioI2aH18aNG7tcTFs8PDxga2uLqqoqg/aqqip4e3u3OZ1CocCYMWMAAMHBwSgsLERaWprJ8FIqlVAqjS8VsLe3h729/f2tQB9mZ2cn/bTm9bQWvwj0ha2tbZsfwADmfQhjDR/AmPt+teg5LwcHB4SEhCAnJwfPPPMMgLt3Zc3JyUFCQoLZ89Hr9QbntYjkpqMPYAB+CPNTZoeXube76ewDONRqNeLi4hAaGoqwsDCkp6ejsbFR+vQxNjYWvr6+SEtLA3D3HFZoaChGjx4NjUaDPXv24KOPPsL69es7tVwikjezw2vTpk0YOXIkJk2aBCFMn1DsipiYGFRXVyM5ORmVlZUIDg5Gdna2dBK/vLwcCsWPZycbGxuxcOFCXLlyBQMGDEBAQAA+/vhjxMTEdFtNRNT3mR1eCxYswKeffoqysjLEx8fjxRdfhLt795wITkhIaPMwMTc312D4nXfewTvvvNMtyyUi+TL7i9kZGRmoqKjA66+/jr///e/w8/PDc889h71793brnhgRkTk6dVcJpVKJOXPmYP/+/Th37hwefPBBLFy4ECqVCg0N/JIvEfWeLt8SR6FQwMbGBkII6HS67qyJiKhDnQovjUaDTz/9FJGRkXjggQdQUFCA//7v/0Z5eTmcnWV8SS8RyY7ZJ+wXLlyIrVu3ws/PD/PmzcOnn34KDw+PnqyNiKhNZodXZmYmRowYAX9/f/zjH//AP/7xD5P9du7c2W3FERG1xezwio2NhY2NTU/WQkRktk5dpEpE1FfwARxEJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEt9IrwyMjKgUqng6OiI8PBwHD9+vM2+GzZswCOPPILBgwdj8ODBmDFjRrv9icg6WTy8tm3bBrVajZSUFOTn5yMoKAjR0dG4fv26yf65ubmYM2cODh48iCNHjsDPzw9RUVG4evVqL1dORJZk8fBau3Yt5s+fj/j4eIwfPx6ZmZlwcnJCVlaWyf6ffPIJFi5ciODgYAQEBOAvf/kL9Ho9cnJyerlyIrIkO0suvLm5GXl5eUhKSpLaFAoFZsyYgSNHjpg1j6amJmi1Wri7u5scr9FooNFopOH6+noAgFarhVarvY/q+7aWlhbppzWvZ3/SX7apuetm0fCqqamBTqeDl5eXQbuXlxeKiorMmsfSpUsxbNgwzJgxw+T4tLQ0pKamGrXv27cPTk5OnS9aJi43AIAdjh49iqtnLF0NdYf+sk2bmprM6mfR8LpfK1aswNatW5GbmwtHR0eTfZKSkqBWq6Xh+vp66TyZq6trb5Xa606X3wQKTmLKlCkIGmF6r5Tkpb9s09ajo45YNLw8PDxga2uLqqoqg/aqqip4e3u3O+3q1auxYsUKfPPNNwgMDGyzn1KphFKpNGq3t7eHvb191wqXATs7O+mnNa9nf9Jftqm562bRE/YODg4ICQkxONneevI9IiKizen+9Kc/YdmyZcjOzkZoaGhvlEpEfYzFDxvVajXi4uIQGhqKsLAwpKeno7GxEfHx8QCA2NhY+Pr6Ii0tDQCwcuVKJCcn469//StUKhUqKysBAM7OznB2drbYehBR77J4eMXExKC6uhrJycmorKxEcHAwsrOzpZP45eXlUCh+3EFcv349mpub8atf/cpgPikpKfjjH//Ym6UTkQVZPLwAICEhAQkJCSbH5ebmGgxfvHix5wsioj7P4hepEhF1BcOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJUp+4wp7uX1NTk8E90IoraqGpLEHhmQHQ3xhk0DcgIMCq72VG/QPDy0oUFRUhJCTEqP35zcZ98/LyMHny5F6oiqjnMLysREBAAPLy8qThhh802H3wCJ6aHgHnAUqjvkRyx/CyEk5OTgZ7U1qtFrdqriMiLNSqb1xH/RdP2BORLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYuHV0ZGBlQqFRwdHREeHo7jx4+32ffs2bP45S9/CZVKBRsbG6Snp/deoUTUp1j06UHbtm2DWq1GZmYmwsPDkZ6ejujoaBQXF8PT09Oof1NTE/z9/fHss89iyZIlFqiYqHf89CHCQNsPEu63DxEWFhQWFiYWLVokDet0OjFs2DCRlpbW4bQjR44U69at6/Qy6+rqBABRV1fX6WnlpLm5WXz++eeiubnZ0qVQF+Tl5QkAZr3y8vIsXW63MvffqMX2vJqbm5GXl4ekpCSpTaFQYMaMGThy5Ei3LUej0UCj0UjD9fX1AO4+11Cr1Xbbcvqa1nWz5nW0ZqNHj8axY8cM2hp+0GDvtycQ/cjPDR4kPHr0aKvazuaui8XCq6amBjqdDl5eXgbtXl5eRrvL9yMtLQ2pqalG7fv27esXu9r79++3dAnUjaZOHI3btTdxu/bHtoqKCovV0xOamprM6mf1T8xOSkqCWq2Whuvr6+Hn54eoqCi4urpasLKepdVqsX//fkRGRvKJ2Vaiv2zT1qOjjlgsvDw8PGBra4uqqiqD9qqqKnh7e3fbcpRKJZRKpVG7vb29Vb8BWvWX9exPrH2bmrtuFrtUwsHBASEhIcjJyZHa9Ho9cnJyEBERYamyiEgmLHrYqFarERcXh9DQUISFhSE9PR2NjY2Ij48HAMTGxsLX1xdpaWkA7p7kP3funPT71atXcerUKTg7O2PMmDEWWw8i6n0WDa+YmBhUV1cjOTkZlZWVCA4ORnZ2tnQSv7y8HArFjzuH165dw6RJk6Th1atXY/Xq1Zg2bRpyc3N7u3wisiCLn7BPSEhAQkKCyXE/DSSVSgUhRC9URUR9ncW/HkRE1BUW3/Pqba17buZ+HCtXWq0WTU1NqK+vt+pPpvqT/rJNW/9tdnSU1e/C6/bt2wAAPz8/C1dCRO25ffs23Nzc2hxvI/rZSSS9Xo9r167BxcUFNjY2li6nx7RejHv58mWrvhi3P+kv21QIgdu3b2PYsGEGH9j9VL/b81IoFBg+fLily+g1rq6uVv1G74/6wzZtb4+rFU/YE5EsMbyISJYYXlZKqVQiJSXF5Pc6SZ64TQ31uxP2RGQduOdFRLLE8CIiWWJ4EZEsMbws5OLFi7CxscGpU6d6ZT6HDx/GxIkTYW9vj2eeeea+lknUFzC8+ojc3FzY2Nigtra2R+avVqsRHByMsrIybNq0qUeWQcYeffRRvPLKK5YuQ9LX6rkfDK8uaG5utnQJnVZaWorHHnsMw4cPx6BBg7o0j95cbyEEWlpaem15fZkc32+9omefwGYdpk2bJhYtWiQWL14shgwZIh599FFRUFAgnnjiCTFw4EDh6ekpXnzxRVFdXS1N8/XXX4uHHnpIuLm5CXd3d/HUU0+JkpISaXxZWZkAIL7//nvp93tfcXFxnZrPp59+KiIiIoRSqRQPPvigyM3NNRh/72vjxo1CCCFyc3PFz3/+c+Hg4CC8vb3F0qVLhVarbXe9Dx48KACI7OxsERwcLBwdHcX06dNFVVWV2LNnjwgICBAuLi5izpw5orGxUZqXTqcTy5cvFyqVSjg6OorAwEDx2WefSeNb57tnzx4xefJkYW9vLw4ePNidm7HXxcXFGf3tS0pKxLx586S/wwMPPCDS09ONpps1a5Z45513hI+Pj1CpVEIIIQ4fPiyCgoKEUqkUISEhYteuXdJ7qFV770tT9ZSVlfXWn6PbMbzMMG3aNOHs7Cxee+01UVRUJI4ePSqGDh0qkpKSRGFhocjPzxeRkZFi+vTp0jTbt28XO3bsEP/617/E999/L2bOnCkmTpwodDqdEMIwvFpaWsSOHTsEAFFcXCwqKipEbW1tp+YzfPhwsX37dnHu3Dnx61//Wri4uIiamhrR0tIiKioqhKurq0hPTxcVFRWiqalJXLlyRTg5OYmFCxeKwsJCsWvXLuHh4SFSUlLaXO+ioiIpZKZMmSIOHTok8vPzxZgxY8S0adNEVFSUyM/PF//85z/FkCFDxIoVK6R5vfPOOyIgIEBkZ2eL0tJSsXHjRqFUKqWQbZ1vYGCg2LdvnygpKRE3btzo6U3bo2pra0VERISYP3++qKioEBUVFeLOnTsiOTlZnDhxQly4cEF8/PHHwsnJSWzbtk2aLi4uTjg7O4uXXnpJnDlzRpw5c0bU1dUJd3d38eKLL4qzZ8+KPXv2iAceeMAgvG7dutXu+9JUPS0tLZb403QLhpcZpk2bJiZNmiQNL1u2TERFRRn0uXz5shQ+plRXVwsAoqCgQAhhGF5C/PiP99atW+3W0tZ87g0KrVYrhg8fLlauXCm1ubm5SXtcQgjx5ptvirFjxwq9Xi+1ZWRkCGdnZykYf7re99b5zTffSG1paWkCgCgtLZXafvOb34jo6GghhBB37twRTk5O4rvvvjOY18svvyzmzJljMN/PP/+83fWXm2nTponFixe322fRokXil7/8pTQcFxcnvLy8hEajkdrWr18vhgwZIn744QepbcOGDQbvIXPel+bUIxc852WmkJAQ6ffTp0/j4MGDcHZ2ll4BAQEA7p5bAoB//etfmDNnDvz9/eHq6gqVSgXg7n35O8Pc+dz7xCU7OzuEhoaisLCwzfkWFhYiIiLC4LZADz30EBoaGnDlyhWT632vwMBA6XcvLy84OTnB39/foO369esAgJKSEjQ1NSEyMtLgb7Zlyxbp79UqNDS0zZqtRUZGBkJCQjB06FA4Ozvjgw8+MNqeEydOhIODgzRcXFyMwMBAODo6Sm1hYWEG05jzvrQm/e6WOF01cOBA6feGhgbMnDkTK1euNOrn4+MDAJg5cyZGjhyJDRs2YNiwYdDr9ZgwYUKnT75213y66t71vte9d/K0sbExurOnjY0N9Ho9gLt/LwDYvXs3fH19Dfr99Ht6bS3PWmzduhWJiYlYs2YNIiIi4OLiglWrVuHYsWMG/brydzDnfWlNGF5dMHnyZOzYsQMqlQp2dsZ/whs3bqC4uBgbNmzAI488AgA4dOhQu/Ns/V9Wp9N1aT5Hjx7Fv/3bvwEAWlpakJeX1+aDTQBg3Lhx2LFjB4QQ0t7X4cOH4eLi0u33Oxs/fjyUSiXKy8sxbdq0bp13X+fg4GCwTQ8fPoypU6di4cKFUps5e0Vjx47Fxx9/DI1GIwX+iRMnDPp09L40VY+c8bCxCxYtWoSbN29izpw5OHHiBEpLS7F3717Ex8dDp9Nh8ODBGDJkCD744AOUlJTgwIEDUKvV7c5z5MiRsLGxwVdffYXq6mo0NDR0aj4ZGRnYtWsXioqKsGjRIty6dQvz5s1rc3kLFy7E5cuX8bvf/Q5FRUX44osvkJKSArVa3e7dK7vCxcUFiYmJWLJkCTZv3ozS0lLk5+fjvffew+bNm7t1WX2NSqXCsWPHcPHiRdTU1OBnP/sZTp48ib179+L8+fN46623jELIlOeffx56vR7/8R//gcLCQuzduxerV68GAOk/n47el6bqad07liOGVxcMGzYMhw8fhk6nQ1RUFCZOnIhXXnkFgwYNgkKhgEKhwNatW5GXl4cJEyZgyZIlWLVqVbvz9PX1RWpqKt544w14eXkhISGhU/NZsWIFVqxYgaCgIBw6dAhffvklPDw82l3enj17cPz4cQQFBeG3v/0tXn75ZfzhD3+4r79NW5YtW4a33noLaWlpGDduHJ544gns3r0bo0aN6pHl9RWJiYmwtbXF+PHjMXToUERHR2P27NmIiYlBeHg4bty4YbAX1hZXV1f8/e9/x6lTpxAcHIzf//73SE5OBgDpPFhH70tT9XT2HGxfwlviEMnUJ598gvj4eNTV1WHAgAGWLqfX8ZwXkUxs2bIF/v7+8PX1xenTp7F06VI899xz/TK4AIYXkWxUVlYiOTkZlZWV8PHxwbPPPov/+q//snRZFsPDRiKSJZ6wJyJZYngRkSwxvIhIlhheRCRLDC8ikiWGF/WouXPnwsbGRvrytpeXFyIjI5GVldWpr6Zs2rSpy3eAvR9z587lPf/7KIYX9bgnnngCFRUVuHjxIr7++mtMnz4dixcvxtNPP81bPVPXWfRuYmT1Wm9p/FM5OTkCgNiwYYMQQog1a9aICRMmCCcnJzF8+HCxYMECcfv2bSHEjzcqvPfVesfXLVu2iJCQEOHs7Cy8vLzEnDlzRFVVlbScmzdviueff154eHgIR0dHMWbMGJGVlSWNLy8vF88++6xwc3MTgwcPFv/+7/8u3Ro5JSXFaLlyvzW1NeGeF1nEY489hqCgIOzcuRMAoFAo8Oc//xlnz57F5s2bceDAAbz++usAgKlTpyI9PR2urq6oqKhARUUFEhMTAQBarRbLli3D6dOn8fnnn+PixYuYO3eutJy33noL586dw9dff43CwkKsX79e+sK6VqtFdHQ0XFxc8O233+Lw4cNwdnbGE088gebmZiQmJuK5556T9hwrKiowderU3v1DUdssnZ5k3dra8xJCiJiYGDFu3DiT4z777DMxZMgQaXjjxo3Czc2tw+WdOHFCAJD22mbOnCni4+NN9v3oo4+MboWt0WjEgAEDxN69ezusnyyLe15kMeKeGyF+8803ePzxx+Hr6wsXFxe89NJLuHHjBpqamtqdR15eHmbOnIkRI0bAxcVFutlh661eFixYgK1btyI4OBivv/46vvvuO2na06dPo6SkBC4uLtJtk93d3XHnzh2rvG2ytWF4kcUUFhZi1KhRuHjxIp5++mkEBgZix44dyMvLQ0ZGBoD2n1nY2NiI6OhouLq64pNPPsGJEyewa9cug+l+8Ytf4NKlS1iyZAmuXbuGxx9/XDrkbGhoQEhICE6dOmXwOn/+PJ5//vkeXnu6X7yrBFnEgQMHUFBQgCVLliAvLw96vR5r1qyRbpr3t7/9zaC/qdsXFxUV4caNG1ixYgX8/PwAACdPnjRa1tChQxEXF4e4uDg88sgjeO2117B69WpMnjwZ27Ztg6enJ1xdXU3WaU23TbY23POiHqfRaFBZWYmrV68iPz8fy5cvx6xZs/D0008jNjYWY8aMgVarxXvvvYcLFy7go48+QmZmpsE8VCoVGhoakJOTg5qaGjQ1NWHEiBFwcHCQpvvyyy+xbNkyg+mSk5PxxRdfoKSkBGfPnsVXX32FcePGAQBeeOEFeHh4YNasWfj2229RVlaG3Nxc/Od//qf0BCWVSoX//d//RXFxMWpqaqDVanvnj0Yds/RJN7Ju9z6l2c7OTgwdOlTMmDFDZGVlSc+HFEKItWvXCh8fHzFgwAARHR0ttmzZYvQcy9/+9rdiyJAhBpdK/PWvfxUqlUoolUoREREhvvzyS6NnGY4bN04MGDBAuLu7i1mzZokLFy5I86yoqBCxsbHCw8NDKJVK4e/vL+bPny/q6uqEEEJcv35dREZGCmdnZ14q0cfwfl5EJEs8bCQiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkS/8PmtoaPNWQp2cAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:09.708989Z", + "iopub.status.busy": "2024-03-22T19:39:09.708145Z", + "iopub.status.idle": "2024-03-22T19:39:09.999683Z", + "shell.execute_reply": "2024-03-22T19:39:09.998652Z" + }, + "papermill": { + "duration": 0.313142, + "end_time": "2024-03-22T19:39:10.001804", + "exception": false, + "start_time": "2024-03-22T19:39:09.688662", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVHUlEQVR4nO2dd5gTVffHP5O27MIW6sLC0ntHuopIk2IFFJEiILZXsIAoRaUrqKi8goKoFKWJwCv+EFFAiiC9d+kdlrad3bT5/RES0jabZJNNsrmf58mzKTN37mYm3zn33HPPkWRZlhEIBIIAogh0BwQCgUAIkUAgCDhCiAQCQcARQiQQCAKOECKBQBBwhBAJBIKAI4RIIBAEHCFEAoEg4KgC3YG8YDQauXz5MtHR0UiSFOjuCAQCO2RZJi0tjYSEBBSKnO2ekBaiy5cvk5iYGOhuCASCXLhw4QLlypXL8fOQFqLo6GjA9E/GxMQEuDcCgcCe1NRUEhMTLb/VnAhpITIPx2JiYoQQCQRBTG6uE+GsFggEAUcIkUAgCDhCiAQCQcAJaR9RqCDLMnq9HoPBEOiuCAQ+RalUolKp8hw+I4TIz2i1Wq5cuUJmZmaguyIQ+IWoqCjKlCmDRqPxug0hRH7EaDRy5swZlEolCQkJaDQaEXgpKDDIsoxWq+X69eucOXOGatWquQxadIUQIj+i1WoxGo0kJiYSFRUV6O4UOG5laEm9o6N8sSgUCiHwgSAyMhK1Ws25c+fQarUUKlTIq3bCzlm9+cQNFm4/n6/H9PYuIXDNxduZpGbpuJGRHeiuhDW+uL7DziIavuwAl5Lv0KpaCRKLCSulIGAwivoPoU7Y3aoztHoA0rL0Ae6JQCAwE3ZCZC6epDcaA9uRMOTs2bNIksS+fft80s6xwwddbrdlyxbq1auHWq3mqaeeytMxw5UNGzYgSRLJycl+PU4YCpFJiXQGIUSBxt8X+dChQ2nYsCFnzpxh7ty5fjmGwDeEnxDd/aszCL+CJ2i12kB3IWdyOJWnTp2ibdu2lCtXjri4OK+azs//2xz46muC+tzdJeyEyHzR6gMkRLIsk6nVB+ThSVHfhx9+mMGDB/PWW29RokQJOnbsyKFDh+jcuTNFihQhPj6evn37cuPGDcs+q1ev5sEHHyQuLo7ixYvz2GOPcerUKaftnz17ljZt2gBQtGhRJEmif//+HrVz5uS/PP/UI1QqXZS6deuyceNGS9uSJHHz5k1eeOEFJEmyWEQbN26kWbNmREREUKZMGUaMGGHz43f2f5sttz/++INGjRoRGRlJ27ZtSUpK4vfff6dWrVrExMTQq1cvm8BVo9HIpEmTqFSpEpGRkTRo0IClS5daPje3+/vvv9O4cWMiIiLYvHlzrudm4sSJlCpViujoaF588UVGjBhBw4YNLZ/379+fp556ig8//JCEhARq1KgBwI8//kiTJk2Ijo6mdOnS9OrVi6SkJJu2V61aRfXq1YmMjKRNmzacPXs21/74grCbNbNYRAHyEd3RGag9+o+AHPvI+I5Eadw/5fPmzeM///kPW7ZsITk5mbZt2/Liiy/yxRdfcOfOHYYPH06PHj3466+/AMjIyGDo0KHUr1+f9PR0Ro8eTdeuXdm3b5/DFG9iYiLLli2je/fuHD9+nJiYGCIjIz1q54sPR/Pu2Ek0blCP+d99zeOPP86ZM2dITEzkypUr1KhRg/Hjx/Pss88SGxvLpUuX6NKlC/379+eHH37g2LFjvPTSSxQqVIixY8c6/b8Brly5AsDYsWOZPn06UVFR9OjRgx49ehAREcHChQtJT0+na9euTJs2jeHDhwMwadIk5s+fz8yZM6lWrRqbNm2iT58+lCxZktatW1uON2LECKZMmULlypUpWrSoy3OyYMECPvzwQ77++mseeOABFi9ezGeffUalSpVstlu3bh0xMTGsWbPG8p5Op2PChAnUqFGDpKQkhg4dSv/+/Vm1ahVgyuvVrVs3Bg0axMsvv8yuXbt4++23c79QfEDYCZHxrlUQKIsolKhWrRqffPIJYLoLN2rUiI8++sjy+ezZs0lMTOTff/+levXqdO/e3Wb/2bNnU7JkSY4cOULdunVtPlMqlRQrVgyAUqVK2Qyd3G2nZ/+XaN/lCUoUiWDGjBmsXr2a77//nnfffZfSpUsjSRKxsbGULl0agK+//prExESmT5+OJEnUrFmTy5cvM3z4cEaPHm0ROev/G+4J0cSJE3nggQcAGDhwICNHjuTUqVNUrlwZgKeffpr169czfPhwsrOz+eijj1i7di0tW7YEoHLlymzevJlvvvnGRojGjx9Phw4d3Don06ZNY+DAgQwYMACA0aNH8+eff5Kenm6zXeHChfnuu+9sll288MILlueVK1fmyy+/pGnTpqSnp1OkSBFmzJhBlSpV+OyzzwCoUaMGBw8e5OOPP3arb3khaIRo8uTJjBw5kjfffJOpU6f67TiWWbMAOasj1UqOjO8YsGN7QuPGjS3P9+/fz/r16ylSpIjDdqdOnaJ69eqcOHGC0aNHs337dm7cuIHxrtV5/vx5ByFyhbvtNLivqeW5SqWiSZMmHD16NMd2jx49SsuWLW2W2TzwwAOkp6dz8eJFypcv7/B/W1O/fn3L8/j4eKKioiwiZH5vx44dAJw8eZLMzEwHgdFqtTRq1MjmvSZNmuTYZ3uOHz/Oa6+9ZvNes2bNLFapmXr16jms/dq9ezdjx45l//793L592+Z7rV27NkePHqV58+Y2+5hF1N8EhRDt3LmTb775xuZE+wv57uBMF6AgOEmSPBoeBZLChQtbnqenp/P44487vTuWKVMGgMcff5wKFSrw7bffkpCQgNFopG7duh47S33VjrdY/9/WqNVqy3NJkmxem98z/7jNFspvv/1G2bJlbbaLiIhw63h5wb7NjIwMOnbsSMeOHVmwYAElS5bk/PnzdOzYMSic2QF3Vqenp9O7d2++/fbbXMfHviDQFlGoct9993H48GEqVqxI1apVbR6FCxfm5s2bHD9+nPfff5927dpRq1Ytbt++7bJN8x3bOj2KJ+0c2LvL8lyv17N7925q1aqV4/Fq1arF1q1bbZz2W7ZsITo62mVid2+oXbs2ERERnD9/3uH7ykvBhxo1arBz506b9+xfO+PYsWPcvHmTyZMn06pVK2rWrOngqK5Vq5bFojOzbds2r/vqCQEXokGDBvHoo4/Svn37XLfNzs4mNTXV5uEp5ktQ+Ig8Y9CgQdy6dYvnnnuOnTt3curUKf744w8GDBiAwWCgaNGiFC9enFmzZnHy5En++usvhg4d6rLNChUqIEkSK1eu5Pr166Snp3vUzk/zvmPd7ys5+e9xBg0axO3bt238IPa89tprXLhwgddff51jx46xYsUKxowZw9ChQ32+HjA6Opphw4YxZMgQ5s2bx6lTp9izZw/Tpk1j3rx5Xrf7+uuv8/333zNv3jxOnDjBxIkTOXDgQK5ZHcqXL49Go2HatGmcPn2aX3/9lQkTJths8+qrr3LixAneeecdjh8/zsKFC/Mt/iqgQrR48WL27NnDpEmT3Np+0qRJxMbGWh5e3Vnu6k+gZs1ClYSEBLZs2YLBYOCRRx6hXr16vPXWW8TFxaFQKFAoFCxevJjdu3dTt25dhgwZwqeffuqyzbJlyzJu3DhGjBhBfHw8gwcP9qidN0eMYfbXU+nQqjmbN2/m119/pUSJEi6Pt2rVKnbs2EGDBg149dVXGThwIO+//36evpucmDBhAh988AGTJk2iVq1adOrUid9++81hhssTevfuzciRIxk2bBj33XcfZ86coX///rmuei9ZsiRz587l559/pnbt2kyePJkpU6bYbFO+fHmWLVvGL7/8QoMGDZg5c6bN5IQ/kWRPgkt8yIULF2jSpAlr1qyx+IYefvhhGjZsmKOzOjs7m+zseyutzaVKUlJS3K7iUe29VegMMmMfr03/B7y/INwhKyuLM2fOUKlSJa/TIwhy5sDFZABKFIkgIS4ysJ0JIB06dKB06dL8+OOPATm+q+s8NTWV2NjYXH+jAfOa7t69m6SkJO677z7LewaDgU2bNjF9+nSys7NRKm1neSIiIhwcfZ5yb62ZGJoJQo/MzExmzpxJx44dUSqVLFq0iLVr19rEC4UiAROidu3acfCg7aLFAQMGULNmTYYPH+4gQr5CLPEQBDN16tTh3LlzTj/75ptv6NatG6tWreLDDz8kKyuLGjVqsGzZMrd8rMFMwIQoOjraIbakcOHCFC9e3KOYE0+RLQGNwkckCD5WrVqFTqdz+ll8fDyRkZGsXbs2n3vlf0IjoMWH3FviISyigkJBOpMVKlQIdBcCQlAJ0YYNG/x+DBFHJBAEHwGPI8pPrCcIhbNaIAgewkyI7j0XidEKEOKeEvKElxBZPReR1QJB8BBeQmRlEgmLSCAIHsJLiKyeizgi/1KxYkWbCHlJkvjll1/8dDRxLkOdoJo18zfWPiJRxSN/uXLlSr5kVxCEJuElRFZ3TuEjyl/MWRIDiSzLGAwGVKqwuuxDgvAamgXDrJksgzYjMA8P1zenpaXRu3dvChcuTJkyZfjiiy94+OGHeeuttzz+t62HZubk9suXL6dNmzZERUXRoEEDtm7darPP5s2badWqFZGRkSQmJvLGG2+QkZFh+fz/li3muS5tqFE+3mkyeG+T0wvyn7C9NQQsjkiXCR8lBObYoy6Dxv1sgEOHDmXLli38+uuvxMfHM3r0aPbs2WNTMSIvvPfee0yZMoVq1arx3nvv8dxzz3Hy5ElUKhWnTp2iU6dOTJw4kdmzZ3P9+nUGDx7M4MGDmTNnDgB6nZ5B74yiQd3aKLPTHJLBm/EkOb0gMISVEAWFRRQipKWlMW/ePBYuXEi7du0AmDNnDgkJvhPRYcOG8eijjwIwbtw46tSpw8mTJ6lZsyaTJk2id+/eFuurWrVqfPnll7Ru3ZoZM2ZQqFAhuvbsA0CxwhrKFY1ySAZvxpPk9ILAEF5CFAw+InWUyTIJ1LHd5PTp0+h0Opo1a2Z5LzY21lIjyxdY5yg3571OSkqiZs2a7N+/nwMHDrBgwQLLNrIsYzQaOXPmDLVq1eLIgX3M+GIyJ48eJjUl2SEZvBlPktMLAkN4CVEwzJpJkkfDo4KMfTJ6wCb5/CuvvMIbb7zhsF/58uXJyMjgP326c3/rtnw5azZ1KifmmAzeH8npBb4lvITI6rmII3JN5cqVUavV7Ny501JmJyUlhX///ZeHHnrI78e/7777OHLkCFWrVnX6+cGDB0m+fYs3R46hdrXKlCsaxa5du5xuKwh+wmrWzCgiq90mOjqafv368c4777B+/XoOHz7MwIEDUSgUuSZq9wXDhw/nn3/+YfDgwezbt48TJ06wYsUKBg8eDJisIrVGw6I5szh39ozTZPCC0CGshMhmaCYsolz5/PPPadmyJY899hjt27fngQceoFatWvmSf7t+/fps3LiRf//9l1atWtGoUSNGjx5tcZaXLFmSCZ99xZ+/raBdy8ZOk8ELQoeAJc/3Be4m5jaTkqmjwfg/AahcsjB/vf2wX/tX0JLnZ2RkULZsWT777DMGDhwY6O5YkucXi9JQrpj7jniBbwnp5PmBIChmzUKIvXv3cuzYMZo1a0ZKSgrjx48H4MknnwxwzwQFjfASIpuhmfARucOUKVM4fvw4Go2Gxo0b8/fff3P06FE6d+6c4z7mcssCgbuElxBZPRc5q3OnUaNG7N692+H9O3fusG/fvvzvkKDAEl5CZJ0qVlhEXhMZGZnjtHogELeU0Ce8Zs2snuenjyiE5wMEglzxxfUdXkJkvdYsHyKrzZHDmZmZfj+WQBAozNe3daS8p4TX0AzrgEb/WylKpZK4uDhLaoqoqKh8CQYMF2S9aSmHXiuTlRVW99SgQJZlMjMzSUpKIi4uLk/VmcNKiKzHZgajjCzLfhcGc0Iw6zw5At+QdPsOABkaJVm3NQHuTfgSFxeX58R3YSVE9jaQziCjUflXiCRJokyZMpQqVSrHUsIC73hx+QYAOtSOZ0TnSoHtTJiiVqvzZAmZCSshMto51fRGI5p8cpMplUqfnDDBPS6lGQBI1UkFInI9nAmrgbW9c1+swC8giNMY8oSXENm9FrFEAkFwEF5C5DA0E7dSgSAYCDMhsn0tchIFlsOXU9hy8kae2xG3k9AnrJzV9ogV+IHl0S9NpX3+frcNiXlI4yEi10OfsLaIRLXX4ODcTRF5Hu6ElxDZGfFavbiTBgN5vSGIsxj6hJcQCYsoKLGP7xKEH+ElRHavRRxR4DBazVjmdc5A6FjoE15CZD99L2bNAobeRoiEkoQ7eZo1S09PtxTEM+NOEvtA4RDQKH4AAcN6OJZXIRJnMfTx2CI6c+YMjz76KIULFyY2NpaiRYtStGhR4uLiKFq0qD/66DNEHFHwYC0+hjyOrcT0fejjsUXUp08fZFlm9uzZxMfH5ymNxowZM5gxYwZnz54FoE6dOowePdplYva84Dg0ExdwoDDYWETihhDueCxE+/fvZ/fu3dSoUSPPBy9XrhyTJ0+mWrVqyLLMvHnzePLJJ9m7dy916tTJc/v2OA7NxA8gUPjSWS0IfTwemjVt2pQLFy745OCPP/44Xbp0oVq1alSvXp0PP/yQIkWKsG3bNp+0b49YfR88WPvnjMJHFPZ4bBF99913vPrqq1y6dIm6des65KmtX7++Vx0xGAz8/PPPZGRk0LJlS6/ayA37gEbhIwoc1uKjFech7PFYiK5fv86pU6cYMGCA5T1JkixpVw0Gg0ftHTx4kJYtW5KVlUWRIkX43//+R+3atZ1um52dTXZ2tuV1amqqR8dyCGgUFlHAsPYR5fmGIE5jyOOxEL3wwgs0atSIRYsW5dlZDVCjRg327dtHSkoKS5cupV+/fmzcuNGpGE2aNIlx48Z5fSyHoZnwEQUM61kzrT6vSzyEEoU6HgvRuXPn+PXXX31WYE+j0Vjaaty4MTt37uS///0v33zzjcO2I0eOZOjQoZbXqampJCYmun0s+wtWWESBw/oeIIbIAo+FqG3btuzfv99vlT6NRqPN8MuaiIgIIiIivG5bxBEFD9ZDszxbROJ+EvJ4LESPP/44Q4YM4eDBg9SrV8/BWf3EE0+43dbIkSPp3Lkz5cuXJy0tjYULF7Jhwwb++OMPT7vlFSKyOnBYxw5phWUa9ngsRK+++ioA48ePd/jMU2d1UlISzz//PFeuXCE2Npb69evzxx9/0KFDB0+75RaOzmphEQUK668+rxaRIPTxWIjs15blhe+//95nbbmD4/S9uBMHCmtndV6HyGJoFgCMBlD4rjyWRwGNOp0OlUrFoUOHfNaB/ETkIwoejL70EYlZs/xDlmH9JFjyvEmMfIRHFpFaraZ8+fIexwoFCw4FFoVFFDCERRSCGA3w21DYPdf0+sQaqNHJJ017vMTjvffeY9SoUdy6dcsnHchP7K9XEdEbOPQisjq00N0xWUG75wISPPq5z0QIvPARTZ8+nZMnT5KQkECFChUoXLiwzed79uzxWed8jYisDh58OTQT+Jk7t2HRc3B+KygjoPt3UNv92XF38FiInnrqKZ92IH+xL7AofgCBwqdDs7x2RpAzKZdgfne4fhQiYuG5hVDxQZ8fxmMhGjNmjM87kV+I1ffBg1j0GgJcPw4/doPUixBdBvosg3jfp+eBPKSK3b17N0ePHgVMCc0aNWrks075C4d8ROIHEDBsFr3msayTcFb7gfPbYWEPyEqG4tWg73KIK++3w3ksRElJSfTs2ZMNGzYQFxcHQHJyMm3atGHx4sWULFnS1330GY6LXsUVHCisndXZeb4hiPPoU47/Dj8PAP0dKNcUei2BqGJ+PaTHs2avv/46aWlpHD58mFu3bnHr1i0OHTpEamoqb7zxhj/66DNEFY/gwXpophPO6uBhzw+wuLdJhKp1hOdX+F2EwAuLaPXq1axdu5ZatWpZ3qtduzZfffUVjzzyiE8752sch2biThooDMJHFFzIMvw9Bf6aaHrdsA88PhWUape7+QqvlnjYL3QFU7CjL5d/+AMxNAsejD5MjCZ8RHnEaIDfh8POb02vW70NbT+APOYa8wSPh2Zt27blzTff5PLly5b3Ll26xJAhQ2jXrp1PO+drHPMRBbdwFmTEotcgQZcFSwfcFSEJOn8C7UbnqwiBF0I0ffp0UlNTqVixIlWqVKFKlSpUqlSJ1NRUpk2b5o8++g6RjyhoyC1VrMEok56td6stYRB5SVaKKUboyApQauDp2dD8lYB0xeOhWWJiInv27GHt2rUcO3YMgFq1atG+fXufd87X2F+wIo4ocFjnI8p2YhF1m/EP+y8ks2NUO0rFFHLZliiw6AWpV2DB03DtEGiioecCqNw6YN3xKo5IkiQ6dOjgt7xB/sJh0WuQ+7QKKkajzNEraZbXziyi/ReSAfjzyDX6tKjgsj0hQx5y44QpUDHlPBSJh95LoYx31Xd8hVdCtG7dOtatW0dSUpKDg3r27Nk+6Zg/EGvNgoPJq48xa9Npy2vhI8pHLu6CBc/AnVtQrIopULFoxUD3ynMhGjduHOPHj6dJkyaUKVMmz1U88hPHoZn4AQQCaxECMMomn5BSETrXUkhyYo1pBb0uExLug94/Q+ESge4V4IUQzZw5k7lz59K3b19/9MevOAQ0iun7oEGrNxKp8S7jn3ARucG+hbBiMMgGqNIOevwAEUUC3SsLHs+aabVa7r//fn/0xe+IgMbgRQQ1+glZhs1fwC//MYlQ/Z7Q66egEiHwQohefPFFFi5c6I+++J+7uqO6OwQQQ7PgIS9+InE7yQGjEVaPhLVjTa/vfwOempFv0dKe4PHQLCsri1mzZrF27Vrq16/vEGX9+eef+6xzvsYc0KhWKtAbDWJoFkTk5aYgpu+doM+G/70Kh5ebXnf8CFoOCmyfXOCxEB04cICGDRsCOCTRD3bHtfl6VSsl7ujEYstgIieLSEiMF2Slwk994MxGUKih60yo93Sge+USj4Vo/fr1/uhHvmAWIo3KNCLViTiioEEMk31E2jVToOLVA6ApAs/+CFXaBrpXueJ1YrRQxHx3VStNQiSc1cGDs+hqgYfcPAU/doXkcxBVAvoshYTgT1gI4SZE8j0fEZim72VZDvohZTggLKI8cmmPKVAx84YpQLHPciheJdC9cpvwEqK7f81DMzCJkVophCjQ5Ljuzw1HdNj7qk+ug5/6gi4DyjQwLdkoUirQvfIIj6fvQ5l7zmorIRLDs6DAeubL6OFsZlhXej2wxJRbWpcBlR+G/r+FnAiBF0K0adMm9HrH9Ax6vZ5Nmzb5pFP+wnyxa6wsIOGwDj7sFycLcuCf6bD8JTDqoe7T0OtniIgOdK+8wmMhatOmjdMqrykpKbRp08YnnfIX9s5qEBZRsGB9FgxCiFxjNMIf78Gf75let3gNun0LKk1g+5UHPPYR5eTcvXnzpkPV12DDfH0rFBIKybTY0hdZGg1GmcvJd0gsFpXntgSm35knhJVu6bWwYhAcXGJ63WG8KWI6xCdc3Baibt26Aaagxf79+xMREWH5zGAwcODAgaBfg2b2JUiASqlAqzf6JG/164v2sOrgVf7bsyFPNiyb5/bCHU+HZmEjRNnpsKQvnPoLFCp48ito0DPQvfIJbgtRbGwsYLKIoqOjiYyMtHym0Who0aIFL730ku976EPMF6wkgVohocU30dWrDl4F4JuNp4UQeYm1mIihmRPSr8PCZ+DyXlBHQY8foVrwZ0V1F7eFaM6cOQBUrFiRYcOGBf0wzBnmy1tCQqVUAAafZmkUPx/fIFudEne+0wI/a3brDMzvBrdOQ1Rxk1O6XONA98qneOwjGjNmjD/6kS+YZ80k6Z7DWuStDg6sxURYRFZc2Q/zn4aMJFPJ5z7/gxJVA90rn+PxrNm1a9fo27cvCQkJqFQqlEqlzSMUMAmRybknZs2CD2sfUVhr0ukNMOdRkwjF14OBawqkCIEXFlH//v05f/48H3zwQeilijX7iJBQ3RUiEUcUfFgHNBrcmEwokGJ1cKkpjYdRBxVbmapsFIoNdK/8hsdCtHnzZv7++29LKpBQwjJrJoFaIRa+BhVWp8Fo89wNIfJDdwLKthmweoTpee2noNssUEW43CXU8aquWagmorLutsoyNBMWUbBh7SMKqyhrWYZ140ypXQGavQydJoMiNFweecFjH9HUqVMZMWIEZ8+e9UN3/Mu96XsJlcKckyiMLvQg4FaGNtdtbIdmbjRaEE6hQQe/vHZPhNp+YCr/HAYiBF4I0bPPPsuGDRuoUqUK0dHRFCtWzObhCZMmTaJp06ZER0dTqlQpnnrqKY4fP+5pl9zGfHeVsHZWC4sov7iels19E9Y4/cxaS4zhZhFpM2BxL9i/ECSlKVDxoWEhHy3tCR4PzaZOneqzg2/cuJFBgwbRtGlT9Ho9o0aN4pFHHuHIkSN+iVMyX9IKibtxRCIPTn6y66zjGkVnWDuoPV2JH3Jk3DStnr+0C1SR8MxcqNEp0L3KdzwWon79+vns4KtXr7Z5PXfuXEqVKsXu3bt56KGHfHYcCzZDM3MlD99d6KHqO8sv3C2gaK097sQUhWxA4+1zML873DwBkUWh1xJIbBboXgUEr/IRnTp1ivfff5/nnnuOpKQkAH7//XcOHz6cp86kpKQA5DjEy87OJjU11ebhCdZrzczJ0XwZWS1wjcpFAjo5h5kydyyikNT/q4fg+0dMIhRTDl74I2xFCLwQoo0bN1KvXj22b9/O8uXLSU9PB2D//v15iro2Go289dZbPPDAA9StW9fpNpMmTSI2NtbySExM9OgY1mvN/GERCVyjVLh3uaXe0Vmeu2cRhRhn/oY5nSH9KpSqDS+ugZI1At2rgOKxEI0YMYKJEyeyZs0aNJp7+U/atm3Ltm3bvO7IoEGDOHToEIsXL85xm5EjR5KSkmJ5XLhwwaNj3LtgJYuPKD/iiM7cyODCrUy/HyfYUbkxNEvP1vP0zK2W1wXORXT4F9O6sexUKH8/DFgFMQmB7lXA8dhHdPDgQaeVXkuVKsWNGze86sTgwYNZuXIlmzZtoly5cjluFxERYZN+xFNsVt+bZ838PDRLz9bTZsoGAE5/1AWFm36SgojCxSyQedh89IrtcLtAOat3fAur3gFkqPkYdP8O1JG57hYOeGwRxcXFceXKFYf39+7dS9mynqXAkGWZwYMH87///Y+//vqLSpUqedodz45nnY9IkT+LXq+lZlmeh8VUtAtc+YjM2G/h3hKPIP9eZRnWTYBVwwAZGg+AHj/4VIRSMnW0mbKBNxfvDf7vwwkeC1HPnj0ZPnw4V69eRZIkjEYjW7ZsYdiwYTz//PMetTVo0CDmz5/PwoULiY6O5urVq1y9epU7d+542i23sPER5VMckfUPK/QuD9/iatbM+txYE/IGkUEPv74Of08xvX54FDz2hc8DFf8+eZ0zNzJYse8y648n+bTt/MBjIfroo4+oWbMmiYmJpKenU7t2bR566CHuv/9+3n//fY/amjFjBikpKTz88MOUKVPG8vjpp5887ZZbWOcjsqw18/OVbr0oOARvVD7Fnf/ffhF1SK8102aaMiru/REkBTw2FR4e7pdAxX3nky3PJ606FnKBuh77iDQaDd9++y0ffPABhw4dIj09nUaNGlGtWjWPD57vJqRVPiLL6vt8tYiC9ieTL7hzvr0bmnnZIX+SeQsW9YQL20FVCLp/D7Ue89vh9l1Itjw/kZTOz7sv8lyz8n47nq/xusBi+fLlKV8+dP5RsLKIbBKj+VmIwsw3ve9CMtdSs+hYp7TDZ6704l7Uu+0XFpJJ0lIuwo/d4MZxU+qO536CCi39djidwcjBS6YYvL4tKvDjtnN8vuZfnmiQQOGI0Kih6lYvhw4dyoQJEyhcuDBDhw51ue3nn3/uk475A+t8RIFIjBaKvylPeeqrLQD8OeQhqsfb1thyZwbMXrjdsaKC6mtNOmoSobTLEJ0AfZZBfG2/HvLYlTSy9UZiI9W8/1gtNp24zrmbmXz792neal/dr8f2FW4J0d69e9HpdJbnORHsSdKMNkOz/Jk1kxwGG+HB2RsZjkLkjo/I7vtyZ2gWNJzbCouehawUKFHDJEJxngXdesPeC7cBaJAYR4RKybsdazJo4R5mbTpNr2blKRVTyO99yCtuCdH69eudPg81rNOAqBX54yNydvyCiN5g5EqKdaiC4zaufGTW+cStCRmf69GVsGwg6LOgXDPo9RNEeZaNwlvMjuqGiXEAdKlXmkbl49h7Ppkv1v7LpG7186UfecGrtWahyr1Zs/xba2b9wyrIzur+c3bS6pN7Nylns13uzZrZ7+PGToFW+F1zTLNj+iyo3hmeX5FvIgT3HNWNyscBphvte11qAfDTzgv8ey0t3/riLW5ZRObiiu6wfPlyrzvjb5xV8cj2QV0zAWw+aRtV760QeeOsDpgMyTJs/Bg2TDK9btTXNEWvzD8HcXKmltM3MgBoWC7O8n6TisXoWCeePw5fY/Lvx5jdv2m+9ckb3LKIrBeaxsTEsG7dOnbt2mX5fPfu3axbt85ShDHYMSVGy/9yQoG+cecnzoZmrmKCrGc0rXF3+t5olBn2834m/X40f/xKRgOsHHJPhB56B56Ylq8iBPesoYrFoyhaWGPz2fBONVEpJP46lsQ/p7xbfpVfuPWtmYsrAgwfPpwePXowc+ZMS/kgg8HAa6+9RkxMjH966SOsfUTmoZkvKr26wnZoFj44myFzJkQqhWQTVGpvEbm7LObQ5RSW7r4IwO0MLZO71fffuj7dHVj2IhxbCUjQ5VNoFpgqx/eGZUUdPqtcsgi9mpfnh63n+GjVUX4d9GDQrnX02Ec0e/Zshg0bZlPDTKlUMnToUGbPnu3Tzvkam3xEdy0irZ03NC1LZ79bnrCNrA4fKXI6NHOynWXZhyW0wq4dN+8Tx6/e84Ms2XWR0b8e8s/3fSfZND1/bCUoNdBjXsBECGCvnaPanjfbVaNIhIpDl1L5df/l/OuYh3gsRHq9nmPHjjm8f+zYMYxBnmTMcl1K95zV1rNmk34/Sr2xf/LXsWt5a9+KcF1r5nTWzMkXZL/+zGFo5maGRrNDtmbpaCQJ5m87z/iVR3wrRqmXTXmEzv8DETHQ939Q+0nfte8hsiyz/2IykLMQFS8SwX8ergLAp38cJ0tnyKfeeYbHA9oBAwYwcOBATp06RbNmpoxy27dvZ/LkyQwYMMDnHfQlNmvNzBaR1dDsm42nAZiw8ihta8b7/vhhpETOLCJn9ymlZC9EdkMzN/09x6+ZEvT1u78iSkni3WUHmLPlLBqVghGdauY9xu36cZMllHoRipQ2xQiVdp7AL784ezOT5EwdGpWCWmVydosMfLAS87ed41LyHeb+c5ZXW1fJx166h8dCNGXKFEqXLs1nn31mSQdSpkwZ3nnnHd5++22fd9CXOMtHZD808zVSmJpEzgTE6dDs7nmwHjZb45ZFJMPxq6Y8RtXjo2lcoShag5H3fznENxtPE6FSMrRDHiKML+wwJbi/cxuKV4U+y6FoBe/b8xF7z5sCGesmxFgsfGcUUit5+5EaDPt5P1+tP8mzTRIdHNuBxuOhmUKh4N133+XSpUskJyeTnJzMpUuXePfdd238RsGIs5zVzgIafenOs44ULshxRPa4O2uWW9ZGdwyi5Ewd11KzAageXwSAPi0qMPox09KKL9ed4Kv1J3NvyBnHV8O8J0wiVLYxvPBnUIgQuHZU29O1UVlqlYkhLUvPl3+d8HPPPCdPAY0xMTFBP1NmjbVFpHEyNLNs58Nj2syahY8O5RBH5Pie/SyZ/RbuDM0uJZvyV5WNiyS6kNry/gsPVmJE55qAyT/y3d+nc23Lhj0/muqN6e9A1Q7Q7/+gcHHP2vAjZiHKyT9kjVIhMaqL6buYv+0c525m+LFnnuOVEC1dupQePXrQokUL7rvvPptHMGMJaERCrcqvtWZWx/frkXzPgYvJvLZgt1cXrTPRcSbEZosoJ5H2JCaoRuloh/debV3FMiyb+NtRfth6NveGZBk2fQq/DgbZAA16wXOLQOP7WnvekqUzcOSyaTjqjhABtKpWkoeql0RnkPlktf8KmXqDx0L05ZdfMmDAAOLj49m7dy/NmjWjePHinD59ms6dO/ujjz7DfLErFK4tIv8dP7Sk6InpW1h18Cov/7Db6efJmVo++/M4J5PSHT5zPjRzfM8+rsX+K/Ikva79Ilszr7etyqA2Jgft6BWHWbTjfM6NGA2mvNJ/TTS9fnAIPPU1KNU57xMADl9OQW+UKVEkgnJF3U85O7JzTSQJfjt4hT13fUzBgMdC9PXXXzNr1iymTZuGRqPh3XffZc2aNbzxxhuWumTByr1L2mrWzN+rKgtAQOPZHCyibzadZtpfJ/nPfEehcmbJOBOV3IoueiJENZ1YRGCaiRv2SA1eamXKiT7qfwctwY826LJg6QDY+S0gQaePof3YoEwqZR0/5MmMYK0yMTzT2FSg4qPfjgbNzdFjITp//jz3338/AJGRkaSlmeI3+vbty6JFi3zbOx9j4yNS5bz63l8nJ0jOuc/4dZ8pQO6EU4vIs4BGXwzNcrKIwCRGo7rUol/LCsgyvLt0v22AX1YKLHgajqwAhRqe/h5avOr2sfObvXYLXT1haIcaFFIr2HXuNn8c9i5mztd4LESlS5fm1i1TDfPy5ctbapmdOXMmaNQ1J2wjq00zfL5c4pHbrFhBmzVzdb6dfeRse8dZM9ttPHHhlS8e5fJzSZIY83gdnmuWiFGGIT/tY/WhK5B6BeZ0gbN/gybaFCNUt7v7Bw4A5tQfjdz0D1lTOrYQL7WqDMDHq4/layqcnPBYiNq2bcuvv/4KmIIbhwwZQocOHXj22Wfp2rWrzzvoS2ziiFQ5xxH5NMGbnMPzAoArY8VpQKMbs2b2eHJzM/v9XKFQSHz4VD263VcWg1Hms0WruDOzHVw7BIVLwYDfoHJrt48ZCJLSsriUfAdJgnrlvFto/krrKpQoouHMjQwWbnfhM8snPA5onDVrlmUpx6BBgyhevDj//PMPTzzxBK+88orPO+hLnEVW6wwysizny5qwAqZDLv03zgIRnc6aWQIanW/jydDMnUqyYBKjT59uQEL6EQacG0NkZhp3ilQg8oUVUMy/tfV8gdkaqlaqiE24gicUiVDxZvvqfPDLIf677gRd7ytLjJdt+QKPLCK9Xs/EiRO5evWq5b2ePXvy5Zdf8vrrr9uUoA5KrPIRWUei5lcqkCAfuXqMK41w9r86295+iYc97gqRQnKcgXOF8tRa3r7yNsWlNA4YK9E2eRRbb4dGTJwlkDEx90BGV/RsmkjlkoW5laFl5oZTPuiZ93gkRCqVik8++QS9Xu+v/vgVqzWvNma8v2bODEaZHWdvWR0/NJUo5+RxOf8/7qYBMVuiZivUfgt3xVvlxrDMwr5FsKgnki4TY+W2zKz4JVf00Qyct5NdVucrWLHMmHnhqLZGrTStwwP4fvMZLif7p7CpO3jsI2rXrh0bN270R1/8jk3OaqsL1185ib7ffJrBC/c6HL+g4NpH5ORNZ9kJcjFi3C0n5NawTJZh81T45VUw6qFeDxS9fuLzvg/QqloJMrUG+s/ZaVMjLNgwGGUO5LLi3hM61I6nWcViZOuNfPbnv3luz1s89hF17tyZESNGcPDgQRo3bkzhwrbRpk888YTPOudrrC0SpUJCqZAwGGW/zRos3nHB7vgFC1c+Ined1bkew82hWa5CZDTCn+/Btq9Nr1sOhg4TQKGgEDCrbxNemLuTradv8vz321n4UgvqlvXOEexPTialk6E1EKVRugxXcBdJkhj1aC2e+moLy/deZOCDlaidkP9DVI+F6LXXXgOc1y+TJAmDITjznYBjfXW10iRE+ZW3OtjDGzzFlUg4XeLhoq2cnNWnb2Sw+9xtGldw7Q9xOTTTZ8Mv/4FDy0yvH5kI979us0mkRsl3/ZrQb/YOdp27TZ/vt7P45RbULB1cfiPzivv65WJzDQZ1l4aJcTzeIIH/23+Zj1Yd5ceBzfK9NJjHQzOj0ZjjI5hFCGxnzcD31V6NMi4TTxUkHdIbjKRm5ewrdDakcuojcuNYPb7Zmqv/IkeLKCsVFjxjEiGFCrp96yBCZgpHqJgzoCkNEuNIztTR+9vtnEwKrgoYnqy494R3O9ZAo1Sw+eQNNv573adtu0NYlRMy/xDM12xEDgtfvdWLk0np1PxgNbcytHlqJxR466d9Lj93d62ZPc4c+gajzNErqS73cypE6Ukw91E4sxHUhaHXEqjfw2U70YXU/DCgGXUSYriZoaXXt9s5cyN4Vqp7suLeExKLRfF8S1N6k0mrjuV7YUu3hejOnTusXLnS8nrkyJEMHTrU8njnnXfIyspy0UIQ4DA0827h6/W0bMasOGSTJ9ma5Xsu0mnqJocLuCBZRCsPXHH5uVN/kIeR2Nacuu64jMQah6HZzVPwfQe4egCiSkD/lVC1neuD3CU2Ss38gc2pWTqapLRsen27jQu3Mt3a15+kZ+s5fjclrjcR1bkxuG1VYgqpOH4tjWXO1uL5EbeFaN68eXzzzTeW19OnT+eff/5h79697N27l/nz5zNjxgy/dNJXWIZmku3QzN3pe4NR5oetZ3n0y7+Zt/UcHaducrrd/G3nOOZEpEJ1+t4bnE/fO25n74vISZBOX3dtlZgDIwG4vBe+fwRun4W4CjDwTyjrWYqaooU1zH+xOVVKFuZKShbPfbstoNPbYErLIsumvEv+KCMdF6Xh9bbVAPhszXEytfkXpuO2EC1YsICXX37Z5r2FCxeyfv161q9fz6effsqSJUt83kFfci8fkQlXWRqd8fOuC4xecZiktGyX2+UUIFmQLKLc0Btlun29hUEL9lje82bWzEyuQmQemp36C+Y+Bpk3oHQ9GLgGinuXo7lEkQgWvtSCisWjuHj7Dr2+3UZSauCsfn8Ny6x5/v4KlCsaybXUbL7/+4zfjmOP20J08uRJ6tWrZ3ldqFAhFIp7uzdr1owjR474tnc+xrqKB3g+NDt0OW9pTsJIhzh8OZU955P57eC9IZxrHXL97Zy+kcvQTKGAAz/Dgh6gTYdKD0H/VRCdtyII8TGFWPhSC8oVjeTszUx6fbedG+mub0T+IrfSQb4gQqXknY41AJi58RTXc7np+gq3hSg5OZns7Hudun79OhUrVrS8NhqNNp8HI/azZhql81QgOf1g8uq/K2jT966w/l/NwzR3Zs1y+opupGtJycy55lx37S+w/EUw6qBON+i9FAr5Zuo9IS6SRS+1oExsIU4mpdPnu+3cvjshkV/IsuxQ495fPF4/gfrlYsnQGvjvuvwJcnRbiMqVK8ehQ4dy/PzAgQOUK1fOJ53yF/ZxRJ4OzdwVkks5+BLCR4ZsBcU8lZ9XHT7lxCqSMDJStYCBGd+Z3mj+H+j+Pagi8nYwOxKLRbHwpRaUio7g2NU0+ny/nZQ7vi3G6YrLKVlcT8tGpZD8HmipUJhyNwEs2nHBaQZOnx/T3Q27dOnC6NGjnc6M3blzh3HjxvHoo4/6tHO+xr5kjXloZh/QmFMsV17rR4aRQWTzHeoN5nVkrmfN9AYj52/l7Auy9xOp0POZeiavqH4zvdF+LHSaZMoF7AcqlSjMwpeaU6KIhsOXU3l+9g6fVwbOCXMgY80y0RRS+79aTovKxWlfKx6DUWby744FVX2N22ds1KhR3Lp1ixo1avDpp5+yYsUKVqxYwSeffEKNGjW4ffs2o0aN8mdfveafUzd48qstzNlyFnCcvneII8pxaJZXJQkjJbJCf1fBnc+a3Xs+8bejvDp/j+NGd7Gewpez0/hePYVuys3oZQUz4oaZ8kv7OSK4aqlo5r/YnLgoNfsvJDNgzk4ysv0/u3QvEZpvAxldMaJzTZQKibVHr7H99E2/HsttIYqPj+eff/6hVq1ajBgxgq5du9K1a1dGjhxJ7dq12bx5M/Hxvq+O6gtS7+jYb7WQ0eIj8nRolsd+OJvSDwcMRpnULB2Xbuc8/S1jWs7hitNmIUq/DvMep7XyAJlyBC/qhrE1pqMPe+yamqVjmD+wOTGFVOw6d5sX5+3ijta/qwryY8bMnqqlitCzaSIAH6066va6P2/wyIatVKkSq1ev5vr162zbto1t27Zx/fp1Vq9eTeXKlf3VxzxjnwXQ4iPycNYsrxaR9Ur8YOZ/ey/S+7tteWrD+qvSGWQajV/Dj9vOOWxnXYBSn8sN4fT1DLh1BmY/gnR5L7fkIvTSvscGY0O3k6L5irplY5n3QjOKRKjYevomL/+4y2915XUGIwcvmWZs/e2otuet9tUprFGy/2IKKw+6DmLNC14NposVK0azZs1o1qwZxYoV8/rgmzZt4vHHHychIQFJkvjll1+8bssV9osDc4sjOn8rk5HLDzo6p8NkZDXkp/1sOZk3U1xvdfc0GOVclwwcvJTCP6ecH7NktMnxHHXzEPL3j8Ct08ixiTytHcs+uSrgfnZGX9KofFHmDGhKlEbJ3yduMGjBHr+Upzp2JY1svZHYSDWVSuRvbbWS0RG80toUh/XJ6mNk6/0jtgFda5aRkUGDBg346quv/HoclTGLOeqPaak4bHrDEllt+uts9f2iHec5esV2KOXMIrqVz9O4oYK18Ojd8PLPcJEhsEKxKB6JPMZ81QSkjCQoVQf5hT85LSdYtlF7khjNhzStWIzv+jUhQqVg3bEk3li01+dpZfZeMDmqG3hYOshXvNiqEqWiI7h4+w4/bnW0an1BQIWoc+fOTJw40e9J9ysenUUb5X7mqj+mi2KbZdFrbqvv7X9Azm7qX64LvjrinmItGr76EVlbRHpXqXjd+F210v7NDD4iWrrD8UL1YcAqiC5js42vUmJ4w/1VSvDt803QKBWsPnyVoUv2+3TRaF4qdviCKI2Ktx8xVcud9tdJl/Fc3hJSq++zs7NJTU21ebjDpXqvscrQjAhJz3T1NBpcNi1Fyc1ZrbKbBnZmEaW5SIXhDqsPXWXg3J0Bs6zSsnQ0/2gdry8y+a/yPjNowmAl4nfy4Dvpp/yD129PQinrWWVoxpMpQ8lSRTuMkm3WmgWAh6qXZEaf+1ArJf5v/2XeXXrAZ85di6M6n/1D1jzdOJEa8dGk3NExfb3vb74hJUSTJk0iNjbW8khMTHRrP0lViMG6N/hB3wGFJNPm9Kewbjyau3fRnMb19laws8sqrzfiV+fvZt2xJD5Z7f9YDWesOniFG+nZ/N/dYoO+inWytoi8Wzwp845qMePU81AgY2wykMG6N8iSNWTrHM9XIHxE9rSrFc+05xqhVEgs23OR935x4mf0kORMrWU2sWG5OB/00juUComRXUz5ref9c87n2QhCSohGjhxJSkqK5XHhwoXcd8L0JRpRMFrfn091d/PR/P0Zj5/9EBV6S0khe+zNa2fb+GrIfiM9MBaRv4IsDTZClLNF5OzrU6HnU9U3DFKZ6uetKPYCUpcpyNK9bAn2FqRHyfP9SKe6Zfji2YYoJFNU8thfD+dJjMzWUKUShSlaOLBVclpXL8mDVUugNRj55I/jPm07OM6em0RERBATE2PzcId716jEV4anWFftfZAU1L2+klnqz5G1GU79P/ZDNmc+19wKBIYr1n6hjGz3h2aRZDFL/TnPqDZhkCXe1b3E2pLPIykUNmlb7IsCqoPAIjLzRIMEPn26AZIE87ae46NV3teYD0T8UE5IkskqkiT4v/2XbWLz8kpICZG32IvFsYSu0HMhekUEbZX7eOHUmxjSbzjsp7dTpyKFHFN8+06HAhMb4K+jWltEd3TuDc2KkspCzUe0Ve4jS1bzsm4oSwxtLCJjjvvS6Y1cSbENjlT6aVmHt3RvXI6PupqyVXz79xmvK2Tkx4p7T6iTEMszjcvRu3l5EuIifdaux8nzfUl6ejonT560vD5z5gz79u2jWLFilC9f3mfHcTqjUqMzqxrN5KFdg6lw5wjGeZ0py2AuUdKyib1FlBDrmIxq0Q73hodmUjJ1xEYFrqImwJKdFzh1PZ0RnWv67Rh6d4dmd09NWa7zg2YyVRRXSJYL84L2HfbIppkasyNabZUtwX4oqw6ws9oZzzUrj85gZPSKw0xffxKNSsEb7aq5vb8sy+y/WzoovwMZXfFx9/o+DyMI6G1k165dNGrUiEaNGgEwdOhQGjVqxOjRo316nJwiq1NK3Ed37VhuqUqhuHmC5RFjqCndM/ntfUQlovO+orvVJ3/luY288u6yA3yz6TQ7ztxy8BEl+2hq1nrWLDOXoVlN6TzLI8ZQRXGFy3IxntaOsYgQ3PP/WA/NbmXYppwJ5PS9K55vWZH3HzWtZP98zb/M3Oh+RdWzNzNJztShUSmCqpqIP2KZAipEDz/8MLIsOzzmzp3r0+M4RlbfSxV7Si7LpDL/xVCiJvFSMks042kuHQVyiX/xkpwqX6w9mkSnqZtYsvNCvuUtcpbGYtz/HfZJ27etBM2VRVQ7+yBLNOOJl5I5bixHt+xxnJRt08lo7IRIZ5CD1lntjBdbVbYkG5v8+zFmb3Yv86F5xX3dhBibEukFkYL9393FQYjsAhqvSSVI772SHcYaxEiZ/KCZTCfFDifOav8KxLGraby77AD/l0tiel/x084LNpYLwAk3cs8cuuRZpsrMHHxEnRQ7GHFzFDFSJjuMNXhGO5qrFHfYzjw1H2EV93XTfmgWpBaRmUFtqlqGZeNXHmG+k3V39virdFAwEhZC5DA0u/vXEtCoN2LUxNJXO5I/DE2IkHR8rf4vCScW2uyXXxVWdp7Jn/rr644lsXinrY/LnZ9z16+3eHQcZ0OzPso1fK3+Lxp0/GloTF/tSFIp4nR/+6HZHa2BNLvUG8og9BHZM6R9NV5pbVoc/v4vh1iyy7V/MZhmzPxNWAhRbhaR1mDEKMtko+E/urdYoG+HQpKpu28c/DWRczfS0d3dJj/Ir+OAKbe0u5y9kcFvB67kWBwgJ2yHZjJDVD8zUT0HhSSzLqoL/9G9RTY5x8iYU/qqVaa/zooXqINs1swZkiQxolNNBjxQEYDhyw6wYt8lp9tm6QwcuXtugslR7S8COmuWXyjtLCKzhaRR3ZuFMaczNaLgPf0LJMlxDFEvg02f8s9fu1iZ+A5tatuub/IX+VzbzgZXfsiHp2zwqk3z9L0SAxNUs+mlWg/AVH03dsS9jOGWawtQbWcRXXNSSSNYndX2SJLE6Mdqo9UbWbD9PEOX7EetVNClnu21dfhyCnqjTIkiEZT14TR5sBL8txEfkNPNUqM0pdzU6o12wYoS/zV0Z1fd0RhR8JxqPf0vfoCkz58ie4FMsi+5NTjzjIxsAxFomameSi/VegyyxHu6F5iqf9qtQCz7oZl9DJHps9AQIjCJ0YQn69KjSTkMRpk3Fu3lz8NXbbaxjh8KxIr7/CYshMhxaGYbl6LNYdh1PPFpZpUeS7aspoNyN4/ufY1Y/J9IPD+HZvbkdM3nlrTMFYqsZBZoPqKDcjfZsprXdG+xwNDe5fGsMZ8n8+zZ1RTHoVkwz5o5Q6GQmNStPk81TEBvlBm0cA/rjydZPt+bTxU7goXQOnteYj80syTPt5qFcZa2QW+QORb3EL21I0mRoyidup+lmnGUwb/5ewM5NMuJdp9v9Gq/Mtxk7M23aaL4lxQ5ij7akaylmUdt3Buamc7c1VRHiyhUhmbWKBUSU55pwKP1yqAzyLzy4262nDRF+Ac69Ud+ExZCpMjBWX1vyYDs1AoZ8+thjl9LZ5dck2e0Y0jXlKKa4hLLI8ZQTfK+Nnif77a7XJEeSIsoJ87d9HxYWk26yLKIMVQwnOeKXIxntGPYKdf0+P+7F1mds0UUSkMza1RKBVN7NqRD7Xi0eiMD5+3ktwNXuJR8B0mCeuX8WzooWAgLIcrJIjJP35uGZs73PXrFNHPxr5zI0oazOWEsSxnpFks1Y2kieZe6Y/PJG7T6eD1tcnD+BlKHfOWPaCwdZ6lmLAnSLc5Qlu7ZY/lXNqVtsf7/XPmkKhSPAqBNjVLAPQv2ZoajEAXbWjNPUCsVTO/ViIdrlCRLZ2TQQlMlk+qlookuFNjlQPlF6J49D3C0iGzvsDq986GZPakRpXlaO4bdxmrESpnM10ziEcVOr/p0M0PLmRyqVoR6RdgOil0s0HxErJTJHmNV+srjuUwJj9v5c8hD7H6/vWVxZcTd8+Xs6wn2gMbciFApmdmnMQ9Wvfc9hUP8kJmwEKKc4oisLSJ3fvxGWSaFIvTWjmKN4T4KSTpmqKfynHKdT/trrYn5KUqHLqXkec6sp/IvZqq/oJCkY62hEb2073FFF+VVWxEqJcWL3Fvf5yovdag5q51RSK1k1vONaVbJVJCidY2SuexRcAj9s+cGOTqrrWbNDG4JkelvFhG8qhvCYv3DKCWZServeVO5DF8l1TD7UBbtOE+LSes4nk/10B6btpkjV9wPcLRF5g3lciarv0Mpyfykf5hXdEPJIsKltenJSNAc0OiMYMjQ6AuiNCoWvticP4c8ROe6pQPdnXwjLITIwX0g2U4Hy7J7tc2s154ZUDJC/xJf6p8CYIh6GR+qZqMg78nnzZo4cvlBrqVmM3zZgTy36U8UGJmoms1Q9VIApumfYrj+JQw4L43co4lpUWv7WvEeld9xbREVDCECk3VXPT46LOKHzIRlZLW9sxqclxSyx7Gap8Tn+h4kyUUZr5pLb9U6SkgpvKEb7HLJQm7YzyoFs88oAi1T1V/RWbkToywxVv88PxhcV10d/2RdHqldmvurFqfnLPcLOWpcCFEoTt8L7hEWFlFua80At6p05lRWeL6hA6/p3iRbVtFRuYsfNJOJyUPgo8P0dpDeGWPI4AfNZDord5Itqxisez1XEQKTL6R97XiiNCrSPagb78oiClRdM4FvCIuzJ0mSzW/ZPGVs7VfIclIZwp7Mu2IV7SRl7GpjM57XjiRVjqS54hhLNBMo7WXg49XUbP4+cd3yOhhv9vHc4ifNeJorjpEqR9JfN5xVxhYet5Nb0jRrXA7NgvFLErhNWAgR2A7PzNesJEmW4Zl7FpHp7t2zqfMyRtvlWvTQjuGaHEdNxQWWRYyliuR8dbUr9l9Ipu/3O6z6G1w/sirSJZZFjKWW4gJJchzPakez1VjHq7acBXYWK6zhuWaOqYJdO6vD5lIukITN2bOOJbL+XWusctzkhrlQoCthOCaXp1v2OE4Zy1BWuskyzVjuk7xLnG7m9PV0ftx61idVWPPqb2oknWCpZhzlpBucNpamm3YsR+UKXrdnn71RIcHO99ozqVs9h21d+YgKkrM6HAkbIbK2iKyjec0WkXWirZI55KY2i1VusxmXKMnT2jHsNVYlTspggeYj2il2e93325k6PlhxmAkrj3Ap2XGdlSfkRYfaKPayUPMhRaV09hkr87R2LBflUnnqj32lFKVCytHxbD25UKKI7WSAGJqFNuEjRAobJ5EFcyzR+ZumKOfW1UtSsbjzADzz3duda/42MfTSjuIvQ0MiJS3fqL+gh3K9d52/yw9bz/HA5Lwl3/dWh55RbuBb9WdESlo2GBrQS/s+t/B9QndXs1/WPqISRWxvFgUhoDGcCZuzl4MOWS5uc1nfckUjeaia84hWd4ZmNttTiJd1Q/lZ/xAqycgn6m8ZpPyFQNUwA/d8YbbIvKb8hU/Vs1BJRpYZWvGi7m0ycSyt5AvsQy2ssRYie6tVWEShTdgIkdLGR+Q4NNt51pQlsFqpIrzcujKfPdOAyiUL27Rx+e6wyJNrXo+Kd/Sv8JX+CQDeUS9hnGquTwIfvaHOmD/c3laBkbGqebyrXgLATP3jvK17Fb0fw89cW0T3PisVbSuEwkcU2oRFQCPYXuDWF7TZAWqevm9RpTgRKiXdG5fj1/2XOX393sJUS65mj2exJD7V9yRJLsoY1Q/0U62hhJTCUN1reQp8dBeDUab7jH+I0jiPdHaGBh2fq7/mMeV2AMbr+jLb0NlfXbTgaore2lldKsbeIgqbe2qBJGzOnvVwqkrJe9UirC/8olFqqpeKtry2nkl7oOq9MjfejgLmGTryuu51smUVjyp3ME/zMdF4nufH05mvfRdus+9CMv+cci+uKZpM5qo/5jHldrSykte1g/NFhMAxU4I1NkMzex+RGJqFNGEjRNaVH6qWuidE1jMxLSoXt/khZFjFuHzwWG2LAOUlruc3Ywv664aTJkfSQnGUJZrxlOK2R214OvPlyZqlktzmJ80E7lceIV0uxADdu/yf8X7PDpgHXAmKWuXCRySGZiFN2AiRNYXU94Yo1sO0FpVti/tlWE3p1ywdQ//7KwFQqYSt78hTthrr8Kz2A5LkOGopTOWWK0uX89SmM+5oDciyzK/73Gu7knSF5Zqx1Fac47ocw7PaD9hidIzn8Sfu+ojshUgs8Qhtwu7s5TQ1D9Cyiq0QJRaz3faDx2qxYdjDPFbftvTLf3s29LgfR+SKdNOO5bSxNOWkGyzVjKWhdNKtfd0xiK6lZlFr9Gr6fL+duf+czXX7+tIplmrGkqi4zlljPN214zgsV3KrP77ElRBpXEzfi0WvoU3YCVGN0tE2r49euZfrp1op20qjH3WtR6c6pVn6akvANMSpWKIwkiTx354NaV6pGDvea8eTDct61ZeLcime1o5ln7EyxaR0Fmo+5GHF3lz3azJxDbvPua4FZraCtpzM3S/UWrGfxZqJFJfSOGisyNPasZyX4937J3yMSyG6OzSLjVRbyk+bET6i0CbshOjR+gk2r+vfTU7+SO14B19KYrEoZvZtTJOKxRzaebJhWX56paXDNLKn3CKGXtr32WioT5SUzXfqz3ha6bpixu1MHf3nuE5R665bqKvib75TTyFKyuZvQ116aj/gBoFL2O5KUKqWKkK1UkV4okFCjiWiBKFJ2Ezf//bGgxy+nMrjdsOqj7vXZ/bmM/Rq7rjIMr/IpBADdcP4mFl0V25mivobSpHM14YnyKkafVqWntWHrtLJ6yx+Mi8rVzJKvQiAFYb7GaZ7FV2ALwlXSfCjNCrWDG0NgDEYay4JvCZsLKI6CbH0aJLocOdMiIvk/cdqU7lkkRz2dI8ysSbLqEmFol7tr0fFMN2rzNQ/BsC76p8Yo/oByUXg49Al+7w6loSR91XzLSL0nb4zb+leC7gIgftDLIVCYlbfxn7ujSC/CBsh8jdLXmnJfx6uwte97/O6DRkFk/W9mKDrA8AA1R9MU09Hg87p9q6WQ5y/5Tw+SY2eqeqveVH1OwAf6noxUd8XOUguBU/qvAdbehSB9wT+FlhASCwWxfBONQFoX6sUa48m5bJHznxv6MJ1OY4p6hk8ptxGMVJ5RTeUNGxn8dKy9SSlZTn4qbJ0Bn7Yes6h3cLcYab6C1opD6GTlbyre5n/GVt53U9fULywhpsZWhpXKErRKA3jn3Q/r9FD1UvSIDGO+mXDowhhQUaSgzkhci6kpqYSGxtLSkoKMTG+XwnuLTfTs/l6wym+33wmT+08qDjITPUXFJGyOGKsQD/tu1zHdujXrFIxlrzS0vL6/M1MHvrUcZV/CVKYo/mYeoqzZMgRvKZ7i43GBnnqnzdoVAr+nXgvSvtqShb/nLrBY/UTbIJLBQUDd3+j4sz7geJFIvjgsdoOMztz+jf1qJ3Nxnr01L7PdTmG2opzLNeMpZJ0xWabXWdtp/Hn/OMofuWlayzVjKWe4iw35Wie077vtQj1bl6evi3uJUJb8GJzt/Z7p2MNGpSL5aeXbdPJlo4tRLf7ygkRCnPE0MyPKCSwTrrRpqbnScQOyZXprh3HD+rJVFSYBGWA9l0OyFUAx+BG+0yTdaQzzNV8TEkplfPGkjyvG8FZuQx5IdJq8ewDVW0ruJ76qAvdvt5CWpaePi0q0CAxjqTULDrXK8OgNlXzdFxBwUXchvxI5RK5z8T1v78irz1cxeU25+V4ntaO5aCxIsWlNBZrJtJasR8wrTvbfc60Vi05U8vinRcs+z2gOMhPmgmUlFI5YqxAd+24PIuQJMGrratQqURhhrSvDkD3+0x1yh6oWhylQuKXQQ+w7u3WvPBgJRpXKErnenk7pqDgI3xEfuTczQwmrDzC2qNJPN4ggWnPNaLiiN9stjk+sRMRKiV/Hr7Kyz+6Tifrytk8pH11YiNVjP2/IwA8ofiHKeoZaCQD/xhqO3V250bzSsV4unE5lu6+yPYzpiHg6rdaUbO07Xd9R2vgr2NJtKpegphCao+OISjYhJSP6KuvvqJixYoUKlSI5s2bs2PHjtx3CgEqFC/Md/2acmZSF6Y91wiAT5+ub/l83BN1iFCZhjkdascz7blGrB3amt45BFdmEMkLundZYbgftWTgC80MXlb+HyDzxdp/MadLGqhcxZea6WgkAysNd1f75yJCtcrE8F6XWpbX/VpWYNFLLXimSSKz+jaxvF8jPtph30iNkkfrlxEiJPCagFtEP/30E88//zwzZ86kefPmTJ06lZ9//pnjx49TqpRrn0qwW0Q5YTTKGGU5xzzLmVo9tUfnnElRwsh7qgWWWKDv9J35UN+bJuWL0u7yDF5VrQRgjr4j492MEdo3ugPnbmby5FdbADj9UReblCiHLqVQSK2gailHIRIIcsLd32jAhah58+Y0bdqU6dOnA2A0GklMTOT1119nxIgRLvcNVSFyh6spWbSYtM7lNi8pV/KeeiFgWqKhR0F35WYAPtE963KJiJnf32xF2aKRFmtm07/XqVSisEPmAYHAG0JCiLRaLVFRUSxdupSnnnrK8n6/fv1ITk5mxYoVNttnZ2eTnX0vwVlqaiqJiYkFUoisuZWhZfvpm7y+aK9D+Z2uir/5RD0LtWSaLdPLCkboX2KpobXNdlOeaYBSYcq5FKFSsvPsLdrXihfpMwR+xV0hCuj0/Y0bNzAYDMTH26aciI+P59ixYw7bT5o0iXHjxuVX94KGYoU1dK5XhpN2s08frTrKrE1wUxfDDPVUFMi8pnuT9UaTP2pW38ZULVWEOzoDdRJso4871vF2saxA4HtCKo5o5MiRDB061PLabBGFK6O61GJUl1rAo5w+24v0bC1zalQPdLcEAo8JqBCVKFECpVLJtWvXbN6/du0apUs73rEjIiKIiHBehTXcqVyxYqC7IBB4TUCn7zUaDY0bN2bduntOWaPRyLp162jZsqWLPQUCQUEi4EOzoUOH0q9fP5o0aUKzZs2YOnUqGRkZDBgwINBdEwgE+UTAhejZZ5/l+vXrjB49mqtXr9KwYUNWr17t4MAWCAQFl4DHEeWFghxHJBAUBEJqiYdAIAhvhBAJBIKAI4RIIBAEnIA7q/OC2b2Vmpoa4J4IBAJnmH+bubmiQ1qI0tJMVVrDObpaIAgF0tLSiI3NuchBSM+aGY1GLl++THR0dNhX+jQvd7lw4YKYQcwj4rv0HbIsk5aWRkJCAgoXxTND2iJSKBSUK1cu0N0IKmJiYsSPx0eI79I3uLKEzAhntUAgCDhCiAQCQcARQlRAiIiIYMyYMSI7gQ8Q32X+E9LOaoFAUDAQFpFAIAg4QogEAkHAEUIkEAgCjhAigUAQcIQQhRCeVsT9+eefqVmzJoUKFaJevXqsWrUqn3oa/HjyXX777be0atWKokWLUrRoUdq3b19gqhEHDbIgJFi8eLGs0Wjk2bNny4cPH5ZfeuklOS4uTr527ZrT7bds2SIrlUr5k08+kY8cOSK///77slqtlg8ePJjPPQ8+PP0ue/XqJX/11Vfy3r175aNHj8r9+/eXY2Nj5YsXL+ZzzwsuQohChGbNmsmDBg2yvDYYDHJCQoI8adIkp9v36NFDfvTRR23ea968ufzKK6/4tZ+hgKffpT16vV6Ojo6W582b568uhh1iaBYCaLVadu/eTfv27S3vKRQK2rdvz9atW53us3XrVpvtATp27Jjj9uGCN9+lPZmZmeh0OooVK+avboYdQohCAFcVca9evep0n6tXr3q0fbjgzXdpz/Dhw0lISHAQeoH3hPTqe4Egv5k8eTKLFy9mw4YNFCpUKNDdKTAIIQoBPK2IC1C6dGmPtg8XvPkuzUyZMoXJkyezdu1a6tev789uhh1iaBYCeFMRt2XLljbbA6xZsybsK+h6W134k08+YcKECaxevZomTZrkR1fDi0B7ywXusXjxYjkiIkKeO3eufOTIEfnll1+W4+Li5KtXr8qyLMt9+/aVR4wYYdl+y5YtskqlkqdMmSIfPXpUHjNmjJi+v4un3+XkyZNljUYjL126VL5y5YrlkZaWFqh/ocAhhCiEmDZtmly+fHlZo9HIzZo1k7dt22b5rHXr1nK/fv1stl+yZIlcvXp1WaPRyHXq1JF/++23fO5x8OLJd1mhQgUZcHiMGTMm/zteQBFpQAQCQcARPiKBQBBwhBAJBIKAI4RIIBAEHCFEAoEg4AghEggEAUcIkUAgCDhCiAQCQcARQiQQCAKOECKBz+nfvz+SJDk8OnXqFOiuCYIUsfpe4Bc6derEnDlzbN7LqXKqTqdDrVbbvKfVatFoNB4f19v9BIFFWEQCvxAREUHp0qVtHkWLFgVAkiRmzJjBE088QeHChfnwww8ZO3YsDRs25LvvvqNSpUqWXD/nz5/nySefpEiRIsTExNCjRw+bFB457ScILYQQCQLC2LFj6dq1KwcPHuSFF14A4OTJkyxbtozly5ezb98+jEYjTz75JLdu3WLjxo2sWbOG06dP8+yzz9q0Zb+fIPQQQzOBX1i5ciVFihSxeW/UqFGMGjUKgF69ejFgwACbz7VaLT/88AMlS5YETPmTDh48yJkzZ0hMTATghx9+oE6dOuzcuZOmTZs63U8QegghEviFNm3aMGPGDJv3rJPNO0suVqFCBRsxOXr0KImJiRYRAqhduzZxcXEcPXrUIkT2+wlCDyFEAr9QuHBhqlat6vJzd95z91iC0Eb4iARBS61atbhw4QIXLlywvHfkyBGSk5OpXbt2AHsm8DXCIhL4hezsbIfyPCqVihIlSrjdRvv27alXrx69e/dm6tSp6PV6XnvtNVq3bi3yRhcwhEUk8AurV6+mTJkyNo8HH3zQozYkSWLFihUULVqUhx56iPbt21O5cmV++uknP/VaEChEqliBQBBwhEUkEAgCjhAigUAQcIQQCQSCgCOESCAQBBwhRAKBIOAIIRIIBAFHCJFAIAg4QogEAkHAEUIkEAgCjhAigUAQcIQQCQSCgCOESCAQBJz/B7paCiLG+ZsjAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019495, + "end_time": "2024-03-22T19:39:10.040658", + "exception": false, + "start_time": "2024-03-22T19:39:10.021163", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4421.448075, + "end_time": "2024-03-22T19:39:12.783196", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/realtabformer/4/mlu-eval.ipynb", + "output_path": "eval/contraceptive/realtabformer/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/realtabformer/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "realtabformer" + }, + "start_time": "2024-03-22T18:25:31.335121", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/realtabformer/model.pt b/contraceptive/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..79dcbc324d80825bd3a658024a609e13f1610878 --- /dev/null +++ b/contraceptive/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:173cd69ed666cb094528dcde463eb2c0dc1b60c91bcf8c38b1b592d99dbc9a98 +size 50388737 diff --git a/contraceptive/realtabformer/params.json b/contraceptive/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..f090d02aac6c1faf2ef0543dc0d3521a8d0e762c --- /dev/null +++ b/contraceptive/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/eval.csv b/contraceptive/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..09a68ffc807f2a244f9598b3cf8b0cb8cb6814c7 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.038817155207590465,0.05224676465664414,0.0031409682811326567,12.584968328475952,0.04438546299934387,1.0239506959915161,0.12085322290658951,1.278079525945941e-05,3.8802053928375244,0.04239872843027115,0.13749264180660248,0.05604434013366699,0.08971857279539108,0.03517554700374603,16.465173721313477 diff --git a/contraceptive/tab_ddpm_concat/history.csv b/contraceptive/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..25cc674a24157be5e596d94aab3cb2bba6dc7166 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.029891989165917038,1.363000964352638,0.0028323000936053095,0.03434117138773824,0.0,0.0,0.0,0.0,0.03190343896651433,900,225,264.29759979248047,1.174655999077691,0.29366399976942276,0.12323957710630364,0.018350417958055105,0.8690771571510773,0.0006744714792743404,0.0,0.0,0.0,0.0,0.0,0.018350417958055105,450,113,91.61005926132202,0.810708489038248,0.2035779094696045,0.05547503994332742 +1,0.026780042523621685,3.719150285677334,0.0013821134704785879,0.06253340480448161,0.0,0.0,0.0,0.0,0.02846383413299918,900,225,265.14594316482544,1.1784264140658909,0.2946066035164727,0.08547261928673834,0.020631530765030118,4.440361282132123,0.0007865790576680511,0.0,0.0,0.0,0.0,0.0,0.020631530765030118,450,113,92.29275822639465,0.8167500727999527,0.205095018280877,0.06341225340801696 +2,0.009797142112058484,1.345002487987807,0.00018001115952997852,0.014916595776513632,0.0,0.0,0.0,0.0,0.01049884227078615,900,225,267.053644657135,1.1869050873650444,0.2967262718412611,0.07698791329231527,0.0058155485348672506,2.1949546487962954,7.145489165522021e-05,0.0,0.0,0.0,0.0,0.0,0.0058155485348672506,450,113,92.60731506347656,0.8195337616236864,0.20579403347439237,0.03643730922346621 +3,0.00591874976532482,0.5914836981606755,8.3291523560353e-05,0.015630586181456844,0.0,0.0,0.0,0.0,0.013930044301410413,900,225,265.62826585769653,1.1805700704786513,0.2951425176196628,0.0992426086589694,0.005116828617950281,1.3365860288515323,4.988687947685675e-05,0.0,0.0,0.0,0.0,0.0,0.005116828617950281,450,113,92.46403670310974,0.8182658115319447,0.20547563711802164,0.0723611570861751 +4,0.006043252464086335,0.6006597002678468,7.193107055212267e-05,0.0193890755618405,0.0,0.0,0.0,0.0,0.006235779779187093,900,225,265.7200925350189,1.1809781890445286,0.29524454726113214,0.09385471865741743,0.0037355842368884218,2.1964674235988046,1.4399180043685393e-05,0.0,0.0,0.0,0.0,0.0,0.0037355842368884218,450,113,92.21309638023376,0.8160451007100333,0.20491799195607505,0.03963511501257596 +5,0.005593134965747595,0.5571283631080762,5.060047336453723e-05,0.005880406767505014,0.0,0.0,0.0,0.0,0.006675898930989205,900,225,265.53266191482544,1.1801451640658909,0.2950362910164727,0.09367100126213497,0.005213722620262868,1.4734984655637393,3.6257587302249085e-05,0.0,0.0,0.0,0.0,0.0,0.005213722620262868,450,113,92.31340265274048,0.8169327668384113,0.20514089478386774,0.04706298913064916 +6,0.004435698193054931,0.5322100268975697,2.625556939914274e-05,0.017856327523348026,0.0,0.0,0.0,0.0,0.004762566664511622,900,225,265.40029549598694,1.1795568688710532,0.2948892172177633,0.09591990354160468,0.0032878017918361972,2.197284267162057,1.6023078919465373e-05,0.0,0.0,0.0,0.0,0.0,0.0032878017918361972,450,113,92.23695397377014,0.8162562298563729,0.20497100883060032,0.04823664502257201 +7,0.004083349550039404,0.2640936150784081,3.919473733837115e-05,0.007234169057984319,0.0,0.0,0.0,0.0,0.004312839023510201,900,225,265.5356845855713,1.1801585981580947,0.29503964953952366,0.10409059958325492,0.0049268581152945344,2.1081838779046094,2.783397738109588e-05,0.0,0.0,0.0,0.0,0.0,0.0049268581152945344,450,113,92.30034828186035,0.8168172414323925,0.20511188507080078,0.03578016849902285 +8,0.004650288004955251,0.3868287038255513,4.264861073708936e-05,0.014303723979844815,0.0,0.0,0.0,0.0,0.004827073722052672,900,225,265.6125736236572,1.1805003272162544,0.2951250818040636,0.09734993250005775,0.00729073759013166,1.2893098778926795,7.92939556772391e-05,0.0,0.0,0.0,0.0,0.0,0.00729073759013166,450,113,92.1851875782013,0.8157981201610734,0.20485597239600287,0.04885830329442644 +9,0.005149225248670619,0.3466197312315247,4.727260695603993e-05,0.00807437449758355,0.0,0.0,0.0,0.0,0.005820203127675793,900,225,262.5310835838318,1.1668048159281412,0.2917012039820353,0.10039696198784642,0.0032142305604389142,1.9432415218951797,1.8801717722896813e-05,0.0,0.0,0.0,0.0,0.0,0.0032142305604389142,450,113,89.91775798797607,0.7957323715750095,0.19981723997328016,0.05169614580196155 diff --git a/contraceptive/tab_ddpm_concat/mlu-eval.ipynb b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..07ff1a4b50ce398327f22a8110928c104ab4dec0 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2277 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.388490Z", + "iopub.status.busy": "2024-03-22T19:39:37.388013Z", + "iopub.status.idle": "2024-03-22T19:39:37.426414Z", + "shell.execute_reply": "2024-03-22T19:39:37.425538Z" + }, + "papermill": { + "duration": 0.058091, + "end_time": "2024-03-22T19:39:37.428992", + "exception": false, + "start_time": "2024-03-22T19:39:37.370901", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.459640Z", + "iopub.status.busy": "2024-03-22T19:39:37.458799Z", + "iopub.status.idle": "2024-03-22T19:39:37.467412Z", + "shell.execute_reply": "2024-03-22T19:39:37.466266Z" + }, + "papermill": { + "duration": 0.023845, + "end_time": "2024-03-22T19:39:37.469586", + "exception": false, + "start_time": "2024-03-22T19:39:37.445741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.499019Z", + "iopub.status.busy": "2024-03-22T19:39:37.498666Z", + "iopub.status.idle": "2024-03-22T19:39:37.503716Z", + "shell.execute_reply": "2024-03-22T19:39:37.502831Z" + }, + "papermill": { + "duration": 0.02368, + "end_time": "2024-03-22T19:39:37.506421", + "exception": false, + "start_time": "2024-03-22T19:39:37.482741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.537472Z", + "iopub.status.busy": "2024-03-22T19:39:37.537182Z", + "iopub.status.idle": "2024-03-22T19:39:37.541295Z", + "shell.execute_reply": "2024-03-22T19:39:37.540543Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.021183, + "end_time": "2024-03-22T19:39:37.543345", + "exception": false, + "start_time": "2024-03-22T19:39:37.522162", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.568881Z", + "iopub.status.busy": "2024-03-22T19:39:37.568276Z", + "iopub.status.idle": "2024-03-22T19:39:37.574747Z", + "shell.execute_reply": "2024-03-22T19:39:37.573993Z" + }, + "papermill": { + "duration": 0.0214, + "end_time": "2024-03-22T19:39:37.576663", + "exception": false, + "start_time": "2024-03-22T19:39:37.555263", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8640b965", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.600634Z", + "iopub.status.busy": "2024-03-22T19:39:37.600394Z", + "iopub.status.idle": "2024-03-22T19:39:37.605081Z", + "shell.execute_reply": "2024-03-22T19:39:37.604247Z" + }, + "papermill": { + "duration": 0.018832, + "end_time": "2024-03-22T19:39:37.606932", + "exception": false, + "start_time": "2024-03-22T19:39:37.588100", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tab_ddpm_concat/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011011, + "end_time": "2024-03-22T19:39:37.629322", + "exception": false, + "start_time": "2024-03-22T19:39:37.618311", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.652460Z", + "iopub.status.busy": "2024-03-22T19:39:37.652206Z", + "iopub.status.idle": "2024-03-22T19:39:37.661394Z", + "shell.execute_reply": "2024-03-22T19:39:37.660450Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023059, + "end_time": "2024-03-22T19:39:37.663391", + "exception": false, + "start_time": "2024-03-22T19:39:37.640332", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tab_ddpm_concat/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:37.689001Z", + "iopub.status.busy": "2024-03-22T19:39:37.688684Z", + "iopub.status.idle": "2024-03-22T19:39:39.679733Z", + "shell.execute_reply": "2024-03-22T19:39:39.678771Z" + }, + "papermill": { + "duration": 2.006351, + "end_time": "2024-03-22T19:39:39.681842", + "exception": false, + "start_time": "2024-03-22T19:39:37.675491", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:39.707158Z", + "iopub.status.busy": "2024-03-22T19:39:39.706313Z", + "iopub.status.idle": "2024-03-22T19:39:39.718462Z", + "shell.execute_reply": "2024-03-22T19:39:39.717708Z" + }, + "papermill": { + "duration": 0.02696, + "end_time": "2024-03-22T19:39:39.720557", + "exception": false, + "start_time": "2024-03-22T19:39:39.693597", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:39.745761Z", + "iopub.status.busy": "2024-03-22T19:39:39.745488Z", + "iopub.status.idle": "2024-03-22T19:39:39.752474Z", + "shell.execute_reply": "2024-03-22T19:39:39.751725Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021929, + "end_time": "2024-03-22T19:39:39.754349", + "exception": false, + "start_time": "2024-03-22T19:39:39.732420", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:39.779870Z", + "iopub.status.busy": "2024-03-22T19:39:39.779540Z", + "iopub.status.idle": "2024-03-22T19:39:39.873472Z", + "shell.execute_reply": "2024-03-22T19:39:39.872695Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.109251, + "end_time": "2024-03-22T19:39:39.875637", + "exception": false, + "start_time": "2024-03-22T19:39:39.766386", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:39.900182Z", + "iopub.status.busy": "2024-03-22T19:39:39.899875Z", + "iopub.status.idle": "2024-03-22T19:39:44.646562Z", + "shell.execute_reply": "2024-03-22T19:39:44.645600Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.761517, + "end_time": "2024-03-22T19:39:44.648995", + "exception": false, + "start_time": "2024-03-22T19:39:39.887478", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 19:39:42.177065: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 19:39:42.177122: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 19:39:42.178990: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:44.676091Z", + "iopub.status.busy": "2024-03-22T19:39:44.675439Z", + "iopub.status.idle": "2024-03-22T19:39:44.681643Z", + "shell.execute_reply": "2024-03-22T19:39:44.680895Z" + }, + "papermill": { + "duration": 0.021832, + "end_time": "2024-03-22T19:39:44.683497", + "exception": false, + "start_time": "2024-03-22T19:39:44.661665", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:44.707470Z", + "iopub.status.busy": "2024-03-22T19:39:44.707201Z", + "iopub.status.idle": "2024-03-22T19:39:52.804846Z", + "shell.execute_reply": "2024-03-22T19:39:52.803750Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.112545, + "end_time": "2024-03-22T19:39:52.807357", + "exception": false, + "start_time": "2024-03-22T19:39:44.694812", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:39:52.834999Z", + "iopub.status.busy": "2024-03-22T19:39:52.834669Z", + "iopub.status.idle": "2024-03-22T19:39:52.841186Z", + "shell.execute_reply": "2024-03-22T19:39:52.840371Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022468, + "end_time": "2024-03-22T19:39:52.843226", + "exception": false, + "start_time": "2024-03-22T19:39:52.820758", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:52.867600Z", + "iopub.status.busy": "2024-03-22T19:39:52.867329Z", + "iopub.status.idle": "2024-03-22T19:39:52.871801Z", + "shell.execute_reply": "2024-03-22T19:39:52.870997Z" + }, + "papermill": { + "duration": 0.018874, + "end_time": "2024-03-22T19:39:52.873785", + "exception": false, + "start_time": "2024-03-22T19:39:52.854911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:52.897859Z", + "iopub.status.busy": "2024-03-22T19:39:52.897601Z", + "iopub.status.idle": "2024-03-22T19:39:53.346134Z", + "shell.execute_reply": "2024-03-22T19:39:53.345204Z" + }, + "papermill": { + "duration": 0.462942, + "end_time": "2024-03-22T19:39:53.348239", + "exception": false, + "start_time": "2024-03-22T19:39:52.885297", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:53.376897Z", + "iopub.status.busy": "2024-03-22T19:39:53.376578Z", + "iopub.status.idle": "2024-03-22T19:39:53.695609Z", + "shell.execute_reply": "2024-03-22T19:39:53.694765Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.336267, + "end_time": "2024-03-22T19:39:53.697798", + "exception": false, + "start_time": "2024-03-22T19:39:53.361531", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:53.725799Z", + "iopub.status.busy": "2024-03-22T19:39:53.724900Z", + "iopub.status.idle": "2024-03-22T19:39:53.827801Z", + "shell.execute_reply": "2024-03-22T19:39:53.826796Z" + }, + "papermill": { + "duration": 0.118907, + "end_time": "2024-03-22T19:39:53.829813", + "exception": false, + "start_time": "2024-03-22T19:39:53.710906", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:39:53.858319Z", + "iopub.status.busy": "2024-03-22T19:39:53.857429Z", + "iopub.status.idle": "2024-03-22T19:39:54.311997Z", + "shell.execute_reply": "2024-03-22T19:39:54.311027Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.471401, + "end_time": "2024-03-22T19:39:54.314346", + "exception": false, + "start_time": "2024-03-22T19:39:53.842945", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:54.343206Z", + "iopub.status.busy": "2024-03-22T19:39:54.342865Z", + "iopub.status.idle": "2024-03-22T19:39:54.347087Z", + "shell.execute_reply": "2024-03-22T19:39:54.346206Z" + }, + "papermill": { + "duration": 0.02052, + "end_time": "2024-03-22T19:39:54.348907", + "exception": false, + "start_time": "2024-03-22T19:39:54.328387", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:54.374945Z", + "iopub.status.busy": "2024-03-22T19:39:54.374696Z", + "iopub.status.idle": "2024-03-22T19:39:54.381345Z", + "shell.execute_reply": "2024-03-22T19:39:54.380504Z" + }, + "papermill": { + "duration": 0.021868, + "end_time": "2024-03-22T19:39:54.383210", + "exception": false, + "start_time": "2024-03-22T19:39:54.361342", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11858440" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:54.409845Z", + "iopub.status.busy": "2024-03-22T19:39:54.409280Z", + "iopub.status.idle": "2024-03-22T19:39:54.487414Z", + "shell.execute_reply": "2024-03-22T19:39:54.486496Z" + }, + "papermill": { + "duration": 0.093559, + "end_time": "2024-03-22T19:39:54.489392", + "exception": false, + "start_time": "2024-03-22T19:39:54.395833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 10] --\n", + "├─Adapter: 1-1 [2, 1179, 10] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 11,264\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 10] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,858,440\n", + "Trainable params: 11,858,440\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.00\n", + "========================================================================================================================\n", + "Input size (MB): 0.12\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.43\n", + "Estimated Total Size (MB): 422.95\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:54.519279Z", + "iopub.status.busy": "2024-03-22T19:39:54.518964Z", + "iopub.status.idle": "2024-03-22T20:48:05.007193Z", + "shell.execute_reply": "2024-03-22T20:48:05.006166Z" + }, + "papermill": { + "duration": 4090.520458, + "end_time": "2024-03-22T20:48:05.024107", + "exception": false, + "start_time": "2024-03-22T19:39:54.503649", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.029891989165917038, 'avg_role_model_std_loss': 1.363000964352638, 'avg_role_model_mean_pred_loss': 0.0028323000936053095, 'avg_role_model_g_mag_loss': 0.03434117138773824, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03190343896651433, 'n_size': 900, 'n_batch': 225, 'duration': 264.29759979248047, 'duration_batch': 1.174655999077691, 'duration_size': 0.29366399976942276, 'avg_pred_std': 0.12323957710630364}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.018350417958055105, 'avg_role_model_std_loss': 0.8690771571510773, 'avg_role_model_mean_pred_loss': 0.0006744714792743404, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.018350417958055105, 'n_size': 450, 'n_batch': 113, 'duration': 91.61005926132202, 'duration_batch': 0.810708489038248, 'duration_size': 0.2035779094696045, 'avg_pred_std': 0.05547503994332742}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.026780042523621685, 'avg_role_model_std_loss': 3.719150285677334, 'avg_role_model_mean_pred_loss': 0.0013821134704785879, 'avg_role_model_g_mag_loss': 0.06253340480448161, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02846383413299918, 'n_size': 900, 'n_batch': 225, 'duration': 265.14594316482544, 'duration_batch': 1.1784264140658909, 'duration_size': 0.2946066035164727, 'avg_pred_std': 0.08547261928673834}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.020631530765030118, 'avg_role_model_std_loss': 4.440361282132123, 'avg_role_model_mean_pred_loss': 0.0007865790576680511, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.020631530765030118, 'n_size': 450, 'n_batch': 113, 'duration': 92.29275822639465, 'duration_batch': 0.8167500727999527, 'duration_size': 0.205095018280877, 'avg_pred_std': 0.06341225340801696}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009797142112058484, 'avg_role_model_std_loss': 1.345002487987807, 'avg_role_model_mean_pred_loss': 0.00018001115952997852, 'avg_role_model_g_mag_loss': 0.014916595776513632, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01049884227078615, 'n_size': 900, 'n_batch': 225, 'duration': 267.053644657135, 'duration_batch': 1.1869050873650444, 'duration_size': 0.2967262718412611, 'avg_pred_std': 0.07698791329231527}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0058155485348672506, 'avg_role_model_std_loss': 2.1949546487962954, 'avg_role_model_mean_pred_loss': 7.145489165522021e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0058155485348672506, 'n_size': 450, 'n_batch': 113, 'duration': 92.60731506347656, 'duration_batch': 0.8195337616236864, 'duration_size': 0.20579403347439237, 'avg_pred_std': 0.03643730922346621}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00591874976532482, 'avg_role_model_std_loss': 0.5914836981606755, 'avg_role_model_mean_pred_loss': 8.3291523560353e-05, 'avg_role_model_g_mag_loss': 0.015630586181456844, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013930044301410413, 'n_size': 900, 'n_batch': 225, 'duration': 265.62826585769653, 'duration_batch': 1.1805700704786513, 'duration_size': 0.2951425176196628, 'avg_pred_std': 0.0992426086589694}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005116828617950281, 'avg_role_model_std_loss': 1.3365860288515323, 'avg_role_model_mean_pred_loss': 4.988687947685675e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005116828617950281, 'n_size': 450, 'n_batch': 113, 'duration': 92.46403670310974, 'duration_batch': 0.8182658115319447, 'duration_size': 0.20547563711802164, 'avg_pred_std': 0.0723611570861751}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006043252464086335, 'avg_role_model_std_loss': 0.6006597002678468, 'avg_role_model_mean_pred_loss': 7.193107055212267e-05, 'avg_role_model_g_mag_loss': 0.0193890755618405, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006235779779187093, 'n_size': 900, 'n_batch': 225, 'duration': 265.7200925350189, 'duration_batch': 1.1809781890445286, 'duration_size': 0.29524454726113214, 'avg_pred_std': 0.09385471865741743}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0037355842368884218, 'avg_role_model_std_loss': 2.1964674235988046, 'avg_role_model_mean_pred_loss': 1.4399180043685393e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037355842368884218, 'n_size': 450, 'n_batch': 113, 'duration': 92.21309638023376, 'duration_batch': 0.8160451007100333, 'duration_size': 0.20491799195607505, 'avg_pred_std': 0.03963511501257596}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005593134965747595, 'avg_role_model_std_loss': 0.5571283631080762, 'avg_role_model_mean_pred_loss': 5.060047336453723e-05, 'avg_role_model_g_mag_loss': 0.005880406767505014, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006675898930989205, 'n_size': 900, 'n_batch': 225, 'duration': 265.53266191482544, 'duration_batch': 1.1801451640658909, 'duration_size': 0.2950362910164727, 'avg_pred_std': 0.09367100126213497}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005213722620262868, 'avg_role_model_std_loss': 1.4734984655637393, 'avg_role_model_mean_pred_loss': 3.6257587302249085e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005213722620262868, 'n_size': 450, 'n_batch': 113, 'duration': 92.31340265274048, 'duration_batch': 0.8169327668384113, 'duration_size': 0.20514089478386774, 'avg_pred_std': 0.04706298913064916}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004435698193054931, 'avg_role_model_std_loss': 0.5322100268975697, 'avg_role_model_mean_pred_loss': 2.625556939914274e-05, 'avg_role_model_g_mag_loss': 0.017856327523348026, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004762566664511622, 'n_size': 900, 'n_batch': 225, 'duration': 265.40029549598694, 'duration_batch': 1.1795568688710532, 'duration_size': 0.2948892172177633, 'avg_pred_std': 0.09591990354160468}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032878017918361972, 'avg_role_model_std_loss': 2.197284267162057, 'avg_role_model_mean_pred_loss': 1.6023078919465373e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032878017918361972, 'n_size': 450, 'n_batch': 113, 'duration': 92.23695397377014, 'duration_batch': 0.8162562298563729, 'duration_size': 0.20497100883060032, 'avg_pred_std': 0.04823664502257201}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004083349550039404, 'avg_role_model_std_loss': 0.2640936150784081, 'avg_role_model_mean_pred_loss': 3.919473733837115e-05, 'avg_role_model_g_mag_loss': 0.007234169057984319, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004312839023510201, 'n_size': 900, 'n_batch': 225, 'duration': 265.5356845855713, 'duration_batch': 1.1801585981580947, 'duration_size': 0.29503964953952366, 'avg_pred_std': 0.10409059958325492}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0049268581152945344, 'avg_role_model_std_loss': 2.1081838779046094, 'avg_role_model_mean_pred_loss': 2.783397738109588e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0049268581152945344, 'n_size': 450, 'n_batch': 113, 'duration': 92.30034828186035, 'duration_batch': 0.8168172414323925, 'duration_size': 0.20511188507080078, 'avg_pred_std': 0.03578016849902285}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004650288004955251, 'avg_role_model_std_loss': 0.3868287038255513, 'avg_role_model_mean_pred_loss': 4.264861073708936e-05, 'avg_role_model_g_mag_loss': 0.014303723979844815, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004827073722052672, 'n_size': 900, 'n_batch': 225, 'duration': 265.6125736236572, 'duration_batch': 1.1805003272162544, 'duration_size': 0.2951250818040636, 'avg_pred_std': 0.09734993250005775}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00729073759013166, 'avg_role_model_std_loss': 1.2893098778926795, 'avg_role_model_mean_pred_loss': 7.92939556772391e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00729073759013166, 'n_size': 450, 'n_batch': 113, 'duration': 92.1851875782013, 'duration_batch': 0.8157981201610734, 'duration_size': 0.20485597239600287, 'avg_pred_std': 0.04885830329442644}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005149225248670619, 'avg_role_model_std_loss': 0.3466197312315247, 'avg_role_model_mean_pred_loss': 4.727260695603993e-05, 'avg_role_model_g_mag_loss': 0.00807437449758355, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005820203127675793, 'n_size': 900, 'n_batch': 225, 'duration': 262.5310835838318, 'duration_batch': 1.1668048159281412, 'duration_size': 0.2917012039820353, 'avg_pred_std': 0.10039696198784642}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032142305604389142, 'avg_role_model_std_loss': 1.9432415218951797, 'avg_role_model_mean_pred_loss': 1.8801717722896813e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032142305604389142, 'n_size': 450, 'n_batch': 113, 'duration': 89.91775798797607, 'duration_batch': 0.7957323715750095, 'duration_size': 0.19981723997328016, 'avg_pred_std': 0.05169614580196155}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0037408947434030577, 'avg_role_model_std_loss': 0.3032218644981286, 'avg_role_model_mean_pred_loss': 2.4328275131388343e-05, 'avg_role_model_g_mag_loss': 0.004584924664943375, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005333074951292171, 'n_size': 900, 'n_batch': 225, 'duration': 261.823570728302, 'duration_batch': 1.1636603143480089, 'duration_size': 0.2909150785870022, 'avg_pred_std': 0.09640699982229206}\n", + "Time out: 3845.4765508174896/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.00314096828578927, 'avg_g_mag_loss': 0.07551193804811766, 'avg_g_cos_loss': 0.04462564292983318, 'pred_duration': 3.8562533855438232, 'grad_duration': 12.572372913360596, 'total_duration': 16.42862629890442, 'pred_std': 0.08971857279539108, 'std_loss': 0.03517554700374603, 'mean_pred_loss': 1.2780794349964708e-05, 'pred_rmse': 0.05604434013366699, 'pred_mae': 0.04239872843027115, 'pred_mape': 0.13749265670776367, 'grad_rmse': 0.12085322290658951, 'grad_mae': 0.04438546299934387, 'grad_mape': 1.0239506959915161}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.00314096828578927, 'avg_g_mag_loss': 0.07551193804811766, 'avg_g_cos_loss': 0.04462564292983318, 'avg_pred_duration': 3.8562533855438232, 'avg_grad_duration': 12.572372913360596, 'avg_total_duration': 16.42862629890442, 'avg_pred_std': 0.08971857279539108, 'avg_std_loss': 0.03517554700374603, 'avg_mean_pred_loss': 1.2780794349964708e-05}, 'min_metrics': {'avg_loss': 0.00314096828578927, 'avg_g_mag_loss': 0.07551193804811766, 'avg_g_cos_loss': 0.04462564292983318, 'pred_duration': 3.8562533855438232, 'grad_duration': 12.572372913360596, 'total_duration': 16.42862629890442, 'pred_std': 0.08971857279539108, 'std_loss': 0.03517554700374603, 'mean_pred_loss': 1.2780794349964708e-05, 'pred_rmse': 0.05604434013366699, 'pred_mae': 0.04239872843027115, 'pred_mape': 0.13749265670776367, 'grad_rmse': 0.12085322290658951, 'grad_mae': 0.04438546299934387, 'grad_mape': 1.0239506959915161}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.00314096828578927, 'avg_g_mag_loss': 0.07551193804811766, 'avg_g_cos_loss': 0.04462564292983318, 'pred_duration': 3.8562533855438232, 'grad_duration': 12.572372913360596, 'total_duration': 16.42862629890442, 'pred_std': 0.08971857279539108, 'std_loss': 0.03517554700374603, 'mean_pred_loss': 1.2780794349964708e-05, 'pred_rmse': 0.05604434013366699, 'pred_mae': 0.04239872843027115, 'pred_mape': 0.13749265670776367, 'grad_rmse': 0.12085322290658951, 'grad_mae': 0.04438546299934387, 'grad_mape': 1.0239506959915161}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:05.057554Z", + "iopub.status.busy": "2024-03-22T20:48:05.057230Z", + "iopub.status.idle": "2024-03-22T20:48:05.061736Z", + "shell.execute_reply": "2024-03-22T20:48:05.060806Z" + }, + "papermill": { + "duration": 0.023563, + "end_time": "2024-03-22T20:48:05.063636", + "exception": false, + "start_time": "2024-03-22T20:48:05.040073", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:05.095084Z", + "iopub.status.busy": "2024-03-22T20:48:05.094772Z", + "iopub.status.idle": "2024-03-22T20:48:05.193382Z", + "shell.execute_reply": "2024-03-22T20:48:05.192337Z" + }, + "papermill": { + "duration": 0.117167, + "end_time": "2024-03-22T20:48:05.195816", + "exception": false, + "start_time": "2024-03-22T20:48:05.078649", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:05.230002Z", + "iopub.status.busy": "2024-03-22T20:48:05.229690Z", + "iopub.status.idle": "2024-03-22T20:48:05.510618Z", + "shell.execute_reply": "2024-03-22T20:48:05.509591Z" + }, + "papermill": { + "duration": 0.30074, + "end_time": "2024-03-22T20:48:05.512670", + "exception": false, + "start_time": "2024-03-22T20:48:05.211930", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABICElEQVR4nO3deVxVZf7A8c/lwgXZBWVTFjeE3FMhlzRHyopyKBsbh8QcU5u0VFrMSmmmX6EtM5aaZTXaormUTpPaQqg1BoqCC264gxugIrts9z6/Pw5cvbLdi8C9V5/363VewjnPOeeLcr+e5znPohJCCCRJkqyEjbkDkCRJMoVMWpIkWRWZtCRJsioyaUmSZFVk0pIkyarIpCVJklWRSUuSJKtia+4AWotOp+P8+fO4uLigUqnMHY4kSTcQQlBUVISfnx82NvU/T902Sev8+fP4+/ubOwxJkhpx5swZOnbsWO/x2yZpubi4AMpfiKurq5mjkSTpRoWFhfj7++s/q/W5bZJWTZXQ1dVVJi1JsmCNNd/IhnhJkqyKTFqSJFkVmbQkSbIqt02blnRztFotlZWV5g5DsmJ2dnao1eqbvo5MWlKDhBBkZ2eTn59v7lCkW4C7uzs+Pj431VdSJi2pQTUJy8vLC0dHR9kxV2oSIQSlpaXk5uYC4Ovr2+RryaRVhw17zjIg0AN/D0dzh2JWWq1Wn7A8PT3NHY5k5dq0aQNAbm4uXl5eTa4qyob4G6zbfYZZa/YR/elOcgvLzB2OWdW0YTk63t7JW2o+Nb9LN9M+KpPWDYYFtyfAw5GsvFJi/p1CfmmFuUMyO1kllJpLc/wuyaR1A29XB76aFI6Xiz1HsouYuGIXJeVV5g5LkqRqMmnVIcDTkS8nhePWxo49Wfk8/VUq5VVac4clSRIyadWru48LKyYOxFGj5n/HLjHj671UaXXmDkuyEiqViv/85z/mDqNZvf766/Tt29fcYcik1ZB+AW35JGYAGrUNPx7M5pUN6chlIiVrsWLFCtzd3Zvtei+88AKJiYnNdr2mkkmrEUO6tuODcf2wUcHa3Wd5c9NhmbikW0pFhXEvm5ydnS2i64tMWka4v6cPC8b0BuDT7adYvOW4mSMyDyEEpRVVZtlM/Y/ixx9/ZOjQobi7u+Pp6clDDz3EiRMnABg8eDCzZ882KH/x4kXs7Oz47bffALhw4QKRkZG0adOGTp06sWrVKoKCgli4cGGT/u7S09P5wx/+QJs2bfD09GTKlCkUFxfrj2/bto2wsDCcnJxwd3dnyJAhZGZmArBv3z5GjBiBi4sLrq6u9O/fn927dzd4v23btjFx4kQKCgpQqVSoVCpef/11AIKCgnjjjTeIiYnB1dWVKVOmADB79myCg4NxdHSkc+fOzJ0716Brwo3VwyeffJKoqCjeffddfH198fT0ZNq0aS0+3Et2LjXSnwb4U1hWxRsbD/FewlHcHO2IGRRk7rBa1dVKLXfM+8ks9z70j1E4aoz/dS0pKSE2NpbevXtTXFzMvHnzeOSRR9i7dy/R0dG8/fbbzJ8/X/8Kfs2aNfj5+XH33XcDEBMTw6VLl9i2bRt2dnbExsbqe3ObqqSkhFGjRjFo0CB27dpFbm4uTz31FNOnT2fFihVUVVURFRXF5MmT+frrr6moqCAlJUUfW3R0NP369WPp0qWo1Wr27t2LnZ1dg/ccPHgwCxcuZN68eWRkZADKk1KNd999l3nz5hEXF6ff5+LiwooVK/Dz8yM9PZ3Jkyfj4uLCSy+9VO99tm7diq+vL1u3buX48eM8/vjj9O3bl8mTJzfp78oYMmmZYNLQThRcreSDxGPM++4grg52RPXrYO6wpDqMGTPG4Pt///vftG/fnkOHDjF27FhmzpzJ9u3b9Ulq1apVjBs3DpVKxZEjR/jll1/YtWsXAwYMAODTTz+lW7duTYpl1apVlJWV8cUXX+Dk5ATA4sWLefjhh1mwYAF2dnYUFBTw0EMP0aVLFwBCQ0P152dlZfHiiy8SEhICYFQcGo0GNzc3VCoVPj4+tY7/4Q9/4PnnnzfY99prr+m/DgoK4oUXXmD16tUNJq22bduyePFi1Go1ISEhREZGkpiYKJOWJZkV0Y3Cq5WsSDrN8+v24WxvS8Qd3uYOq1W0sVNz6B+jzHZvUxw7dox58+axc+dOLl26hE6nvPnNysqiZ8+e3HfffaxcuZK7776bU6dOkZyczMcffwxARkYGtra23Hnnnfrrde3albZt2zYp9sOHD9OnTx99wgIYMmQIOp2OjIwMhg0bxpNPPsmoUaO49957iYiIYOzYsfrxebGxsTz11FN8+eWXRERE8Kc//Umf3JqqJhlfb82aNXzwwQecOHGC4uJiqqqqGp3lt0ePHgbDcXx9fUlPT7+p2BrTpDatJUuWEBQUhIODA+Hh4aSkpDRYft26dYSEhODg4ECvXr3YvHmzwfHXX3+dkJAQnJycaNu2LREREezcudOgTF5eHtHR0bi6uuLu7s6kSZMM2gRai0qlYt5Dd/DonR3Q6gTPrEoj+cTlVo/DHFQqFY4aW7Nspvakfvjhh8nLy+OTTz5h586d+t+nmkbn6OhovvnmGyorK1m1ahW9evWiV69ezf53Zqzly5eTnJzM4MGDWbNmDcHBwezYsQNQPh8HDx4kMjKSLVu2cMcdd7Bhw4abut/1CRQgOTmZ6OhoHnzwQTZu3MiePXt49dVXG22kv7GaqlKp9P9BtBSTk9aaNWuIjY0lLi6OtLQ0+vTpw6hRo+qt7yclJTFu3DgmTZrEnj17iIqKIioqigMHDujLBAcHs3jxYtLT09m+fTtBQUHcd999XLx4UV8mOjqagwcPkpCQwMaNG/ntt9/0DYitzcZGxdtjenPvHd5UVOl46vNd7D+bb5ZYpNouX75MRkYGr732GiNHjiQ0NJQrV64YlPnjH/9IWVkZP/74I6tWrSI6Olp/rHv37lRVVbFnzx79vuPHj9e6hrFCQ0PZt28fJSUl+n2///47NjY2dO/eXb+vX79+zJkzh6SkJHr27MmqVav0x4KDg5k1axY///wzjz76KMuXL2/0vhqNBq3WuE7RSUlJBAYG8uqrrzJgwAC6deumfxFgcYSJwsLCxLRp0/Tfa7Va4efnJ+Lj4+ssP3bsWBEZGWmwLzw8XEydOrXeexQUFAhA/PLLL0IIIQ4dOiQAsWvXLn2ZH374QahUKnHu3Dmj4q65ZkFBgVHljXG1okr8+eNkETh7o+j795/EsZzCZru2Jbh69ao4dOiQuHr1qrlDMYlWqxWenp7iiSeeEMeOHROJiYli4MCBAhAbNmzQl4uOjhZ9+vQRKpVKZGZmGlwjIiJC3HnnnWLnzp0iLS1NjBgxQrRp00YsXLjQqBiuv1dJSYnw9fUVY8aMEenp6WLLli2ic+fOYsKECUIIIU6ePClefvllkZSUJE6fPi1++ukn4enpKT788ENRWloqpk2bJrZu3SpOnz4ttm/fLrp06SJeeumlRmP4/fff9Z+jixcvipKSEiGEEIGBgeJf//qXQdnvvvtO2Nraiq+//locP35cvP/++8LDw0O4ubnpy8TFxYk+ffrov58wYYL44x//aHCdGTNmiOHDh9cbU0O/U8Z+Rk160qqoqCA1NZWIiAj9PhsbGyIiIkhOTq7znOTkZIPyAKNGjaq3fEVFBcuWLcPNzY0+ffror+Hu7m5QD4+IiMDGxqZWNbJGeXk5hYWFBltzc7BT88mEAfTp6MaV0kqe+DSFM3mlzX4fyTQ2NjasXr2a1NRUevbsyaxZs3jnnXdqlYuOjmbfvn3cfffdBAQEGBz74osv8Pb2ZtiwYTzyyCP6N2kODg4mx+Po6MhPP/1EXl4eAwcO5LHHHmPkyJEsXrxYf/zIkSOMGTOG4OBgpkyZwrRp05g6dSpqtZrLly8TExNDcHAwY8eO5YEHHuDvf/97o/cdPHgwTz/9NI8//jjt27fn7bffrrfs6NGjmTVrFtOnT6dv374kJSUxd+5ck3/WVtFgSrvBuXPnBCCSkpIM9r/44osiLCysznPs7OzEqlWrDPYtWbJEeHl5Gez7/vvvhZOTk1CpVMLPz0+kpKToj7355psiODi41rXbt28vPvzwwzrvGxcXJ4BaW3M+adXIKy4XEe9tE4GzN4rhb28ROYXW9WRSH2t90moJZ86cMXj6l5qm1Z+0WtKIESPYu3cvSUlJ3H///YwdO7bJ/WIA5syZQ0FBgX47c+ZMM0ZrqK2Thi8nhdOxbRtOXy4l5rMUCkrlfOrWbMuWLfz3v//l1KlTJCUl8ec//5mgoCCGDRtm7tBueyYlrXbt2qFWq8nJyTHYn5OTU2dfEAAfHx+jyjs5OdG1a1fuuusuPvvsM2xtbfnss8/017gxgVVVVZGXl1fvfe3t7fULs7bGAq0+bg6sfCqc9tVT2vz1812UVsgpbaxVZWUlr7zyCj169OCRRx6hffv2+o6mK1euxNnZuc6tR48erRbjAw88UG8cb731VqvF0dpM6qel0Wjo378/iYmJREVFAaDT6UhMTGT69Ol1njNo0CASExOZOXOmfl9CQgKDBg1q8F46nY7y8nL9NfLz80lNTaV///6A8j+hTqcjPDzclB+hRQV6OvHlpDDGfpRMauYVnv4qjU9jBqCxtZgHWslIo0aNYtSouvukjR49ut7fu8Z6qjenTz/9lKtXr9Z5zMPDo9XiaHWm1klXr14t7O3txYoVK8ShQ4fElClThLu7u8jOzhZCCDF+/Hjx8ssv68v//vvvwtbWVrz77rvi8OHDIi4uTtjZ2Yn09HQhhBDFxcVizpw5Ijk5WZw+fVrs3r1bTJw4Udjb24sDBw7or3P//feLfv36iZ07d4rt27eLbt26iXHjxhkdd0u8PazP7tN5IuS1H0Tg7I3ima9SRZVW1+L3bAmyTUtqbs3RpmVy0hJCiEWLFomAgACh0WhEWFiY2LFjh/7Y8OHD9a9ya6xdu1YEBwcLjUYjevToITZt2mTwQzzyyCPCz89PaDQa4evrK0aPHm3QEC+EEJcvXxbjxo0Tzs7OwtXVVUycOFEUFRUZHXNrJi0hhPjtaK7o9spmETh7o3hp3T6h01lf4pJJS2puzZG0VELcHvOsFBYW4ubmRkFBQYu3b9X48cAFnlmZhk7AlGGdmfNAiFXNt15WVsapU6fo1KlTk171S9KNGvqdMvYzKhtbWtD9PX2ZXz2lzbLfTvLhthNmjkiSrJ9MWi1s7AB/XotURuy/81MGX+6w0KERkmQlZNJqBU/d3Zln/9AVgHnfHeC7vefMHJEkWS+ZtFpJ7L3BTBgUiBDw/Np9HMspMndIUgu6FRe2sBQyabUSlUpF3MM9COvkQZVOsP34JXOHJN3imnthC1CmcVapVOTn5zfrdU0hk1YrsrFRcVcnpdPfkQvySUuSmkImrVbW3Ud5lXvEGquHQkBFiXk2ubBFsy1sUV5ezgsvvECHDh1wcnIiPDycbdu26c/NzMzk4Ycfpm3btjg5OdGjRw82b97M6dOnGTFiBKBMs6xSqXjyySeb9PdxM+R0y60sxNcFgKPZRWh1ArWN9fTborIU3vIzz71fOQ8ap8bLVZMLW9S/sMX06dM5dOgQq1evxs/Pjw0bNnD//feTnp5Ot27dmDZtGhUVFfz22284OTlx6NAhnJ2d8ff359tvv2XMmDFkZGTg6upKmzZtmvR3cjNk0mplQZ5O2NvacLVSS1ZeKZ3aGf9BlIwnF7aoe2GLrKwsli9fTlZWFn5+yn9AL7zwAj/++CPLly/nrbfeIisrizFjxuinn+7cubP+/JoxjV5eXs3eXmYsmbRamdpGRXcfF/afLSAju9C6kpado/LEY657m0AubFG39PR0tFotwcHBBvvLy8v1C7E+99xz/O1vf+Pnn38mIiKCMWPG0Lt37ybdryXINi0z6O6tVBEPW1tjvEqlVNHMscmFLZplYYvi4mLUajWpqans3btXvx0+fJj3338fgKeeeoqTJ08yfvx40tPTGTBgAIsWLWq2n/VmyaRlBiG+1Y3x2c0/BbQkF7aoUdfCFv369UOr1ZKbm0vXrl0Ntuurkf7+/jz99NOsX7+e559/nk8++UR/TcDoBTNagkxaZhDqozxpHcm2sictK9G2bVs8PT1ZtmwZx48fZ8uWLcTGxhqUcXJyIioqirlz53L48GHGjRunPxYSEkJERARTpkwhJSWFPXv2MGXKFNq0adOkAe/R0dE4ODgwYcIEDhw4wNatW3n22WcZP3483t7enDp1ijlz5pCcnExmZiY///wzx44dIzQ0lKtXrzJ9+nS2bdtGZmYmv//+O7t27TJo86pPUFAQxcXFJCYmcunSJUpLSwkODiY6OpqYmBjWr1/PqVOnSElJIT4+nk2bNgEwc+ZMfvrpJ06dOkVaWhpbt27V3y8wMBCVSsXGjRu5ePGiWZbxa9LUNNaotaemacilojIROHujCHp5oyguqzR3OPWy5qlpEhISRGhoqLC3txe9e/cW27Ztq7Uaz+bNmwUghg0bVuv88+fPiwceeEDY29uLwMBAsWrVKuHl5SU++ugjo+5/4732798vRowYIRwcHISHh4eYPHmyfmql7OxsERUVJXx9fYVGoxGBgYFi3rx5QqvVivLycvHnP/9Z+Pv7C41GI/z8/MT06dON/jd5+umnhaenpwBEXFycEEKIiooKMW/ePBEUFCTs7OyEr6+veOSRR8T+/fuFEEJMnz5ddOnSRdjb24v27duL8ePHi0uXLumv+Y9//EP4+PgIlUpVaxqqxsipaUxgjqlpGjLwzV+4WFTOhmcG0y+gaQ28LU1OTXPN2bNn8ff355dffmHkyJHmDsdqNcfUNPLtoZmE+LhwsaicI9lFFpu0bmdbtmyhuLiYXr16ceHCBV566SW5sIWFkG1aZhJa3RifIdu1LJJc2MJyySctM7nW7UG+QbREcmELyyWTlpnUDOc5kl2EEMKqpmG+3bm4uODi4mLuMOjQoYO5QzALWT00k65ezqhtVBRcrSS7sMzc4TToNnlXI7WC5vhdkknLTOxt1XRprwzrsNT+WjVVndLSUjNHIt0qan6XbqYaLauHZtTdx5WjOcUcuVDEiO5e5g6nFrVajbu7u352A0dHR1mNlZpECEFpaSm5ubm4u7ujVqubfC2ZtMwoxMeF7/dZ9nCemqEdTZ2WRZKu5+7ubjBcqClk0jKj0OrGeEvu9qBSqfD19cXLy4vKykpzhyNZMTs7u5t6wqohk5YZ1cxiejy3mIoqHRpby21iVKvVzfILJ0k3y3I/JbcBPzcHXBxsqdIJTlw0w8BTSbJCTUpaS5YsISgoCAcHB8LDw0lJSWmw/Lp16wgJCcHBwYFevXqxefNm/bHKykpmz55Nr169cHJyws/Pj5iYGM6fN5xsLigoSD/Xdc02f/78poRvMVQqFaE+cpoaSTKFyUlrzZo1xMbGEhcXR1paGn369GHUqFH1NtQmJSUxbtw4Jk2axJ49e4iKiiIqKooDBw4AyivQtLQ05s6dS1paGuvXrycjI4PRo0fXutY//vEPLly4oN+effZZU8O3ONd3MpUkyQgmzSshhAgLCxPTpk3Tf6/VaoWfn5+Ij4+vs/zYsWNFZGSkwb7w8HAxderUeu+RkpIiAJGZmanfFxgYKP71r38ZHWdZWZkoKCjQb2fOnLGYqWmu99WO0yJw9kYR89lOc4ciSWZl7NQ0Jj1pVVRUkJqaSkREhH6fjY0NERERJCcn13lOcnKyQXlQxnXVVx7QL3t048T58+fPx9PTk379+vHOO+9QVVVV7zXi4+Nxc3PTb/7+/kb8hK0vRFYPJckkJiWtS5cuodVq8fb2Ntjv7e1NdnZ2nedkZ2ebVL6srIzZs2czbtw4gzl1nnvuOVavXs3WrVuZOnUqb731Fi+99FK9sc6ZM4eCggL9dubMGWN/zFbVvXoW05zCcq6UVJg5GkmyfBbV5aGyspKxY8cihGDp0qUGx66fLrd3795oNBqmTp1KfHw89vb2ta5lb29f535L42xvi79HG87kXeVIdhGDuniaOyRJsmgmPWm1a9cOtVpNTk6Owf6cnJx6e7n6+PgYVb4mYWVmZpKQkNDo7KLh4eFUVVVx+vRpU34EiySriJJkPJOSlkajoX///iQmJur36XQ6EhMTGTRoUJ3nDBo0yKA8QEJCgkH5moR17NgxfvnlF/36aw3Zu3cvNjY2eHlZ3pg9U+kXurC2JcUkyQxMrh7GxsYyYcIEBgwYQFhYGAsXLqSkpISJEycCynLiHTp0ID4+HoAZM2YwfPhw3nvvPSIjI1m9ejW7d+9m2bJlgJKwHnvsMdLS0ti4cSNarVbf3uXh4YFGoyE5OZmdO3cyYsQIXFxcSE5OZtasWTzxxBNNXkDTkuiXFMuRSUuSGtWUV5OLFi0SAQEBQqPRiLCwMLFjxw79seHDh9daoWPt2rUiODhYaDQa0aNHD7Fp0yb9sVOnTgmgzm3r1q1CCCFSU1NFeHi4cHNzEw4ODiI0NFS89dZboqyszOiYLWk1nhsdzy0SgbM3ipDXfhBVWp25w5Eks5Cr8dzA0lbjuZ5WJ7hj3o+UV+nY+sI9dGrn1PhJknSLMfYzKsceWgC1jUrf9SFDNsZLUoNk0rIQ1xa6kO1aktQQmbQshL4xXj5pSVKDZNKyEPpuD3LgtCQ1SCYtC1HTppWVV0pJef1jKiXpdieTloXwdLanvYs9QsBR2V9Lkuolk5YFCZFVRElqlExaFiS0ujHekhe6kCRzk0nLglzr9iDfIEpSfWTSsiDXT718mwxUkCSTyaRlQbp6OaO2UVFwtZLswjJzhyNJFkkmLQtib6umS3tl3KFsjJekusmkZWFqFnCVc2tJUt1k0rIw17o9yMZ4SaqLTFoWJtS3ZrYH+aQlSXWRScvC1FQPj+cWU1GlM3M0kmR5ZNKyMH5uDrg42FKlE5y4WGzucCTJ4sikZWFUKhWhcnUeSaqXTFoW6PpOppIkGZJJywJ1l0uKSVK9ZNKyQHLxVkmqn0xaFqjmSSunsJwrJRVmjkaSLItMWhbI2d4Wf482gGzXkqQbyaTVmnQ62LkMvpkEZQ1X/WQVUZLqJpNWaym8AF89Cj+8CAe+gUP/abB4qGyMl6Q6NSlpLVmyhKCgIBwcHAgPDyclJaXB8uvWrSMkJAQHBwd69erF5s2b9ccqKyuZPXs2vXr1wsnJCT8/P2JiYjh//rzBNfLy8oiOjsbV1RV3d3cmTZpEcbGVdL48vBGWDoaTW6/tO7u7wVP0S4rJ+eIlyYDJSWvNmjXExsYSFxdHWloaffr0YdSoUeTm5tZZPikpiXHjxjFp0iT27NlDVFQUUVFRHDhwAIDS0lLS0tKYO3cuaWlprF+/noyMDEaPHm1wnejoaA4ePEhCQgIbN27kt99+Y8qUKU34kVtRRQl8PwPWRMPVPPDpDSPjlGONJK2axvij2UVodXJCQEnSEyYKCwsT06ZN03+v1WqFn5+fiI+Pr7P82LFjRWRkpMG+8PBwMXXq1HrvkZKSIgCRmZkphBDi0KFDAhC7du3Sl/nhhx+ESqUS586dMyrugoICAYiCggKjyt+0c2lCfHCnEHGuQsS5CfHzXCEqy4UoOK/se91diLLCek+v0upE8KubReDsjeLkxeLWiVmSzMjYz6hJT1oVFRWkpqYSERGh32djY0NERATJycl1npOcnGxQHmDUqFH1lgcoKChApVLh7u6uv4a7uzsDBgzQl4mIiMDGxoadO3fWeY3y8nIKCwsNtlah08L//gmfRsDl4+DiBzHfwb3/AFsNuPqCa0cQOji/p97LqG1U+qetDNkYL0l6JiWtS5cuodVq8fb2Ntjv7e1NdnZ2nedkZ2ebVL6srIzZs2czbtw4XF1d9dfw8vIyKGdra4uHh0e914mPj8fNzU2/+fv7G/Uz3pSCs/D5aEj8O+iqIHQ0/O136DzcsFzH/sqfjVUR9QtdyHYtSaphUW8PKysrGTt2LEIIli5delPXmjNnDgUFBfrtzJkzzRRlPQ6sVxrbM7eDnRP8cQmM/QIcPWqX7ThQ+fNcaoOX1DfGyyctSdKzNaVwu3btUKvV5OTkGOzPycnBx8enznN8fHyMKl+TsDIzM9myZYv+KavmGjc29FdVVZGXl1fvfe3t7bG3tzf6Z2uy8iLY/BLsW6V836E/PPoJeHap/5wO1dXcs7tACFCp6iwWKhdvlaRaTHrS0mg09O/fn8TERP0+nU5HYmIigwYNqvOcQYMGGZQHSEhIMChfk7COHTvGL7/8gqenZ61r5Ofnk5p67clky5Yt6HQ6wsPDTfkRmteZXfDRUCVhqWxg2Ivw158aTlgAvn3AxhaKc5QqZT1q2rSy8kopKa9qzsglyXqZ2sK/evVqYW9vL1asWCEOHTokpkyZItzd3UV2drYQQojx48eLl19+WV/+999/F7a2tuLdd98Vhw8fFnFxccLOzk6kp6cLIYSoqKgQo0ePFh07dhR79+4VFy5c0G/l5eX669x///2iX79+YufOnWL79u2iW7duYty4cUbH3axvD6sqhdg6X4jX2ypvAv/ZQ4jTv5t2jY/uVs5N/7bBYgP+L0EEzt4o0jLzbiJgSbJ8xn5GTU5aQgixaNEiERAQIDQajQgLCxM7duzQHxs+fLiYMGGCQfm1a9eK4OBgodFoRI8ePcSmTZv0x06dOiWAOretW7fqy12+fFmMGzdOODs7C1dXVzFx4kRRVFRkdMzNlrTyTgnx6b3VXRlchVj3VyFKr5h+nY2xyvk/vtJgsSc+3SECZ28Uq3ZmNilcSbIWxn5GVULcHksZFxYW4ubmRkFBgUF7mUn2rYFNz0NFEWhc4KF/Qu+xTbvW3q/hP0+DfzhM+rneYm9tPsyy304yYVAgf/9jz6bdS5KsgLGfUZMa4m9bV/Nh8wuQvk753j8cHl0GbYOafs2aN4gX9kFVhdKHqw413R5kY7wkKWTSakxmEqyfCgVZoFLDPS/D0FhQ3+RfnWcXcHCHsnzIOQAd7qyz2PVTLwshUNXzplGSbhcW1U/LomgrIfENWBGpJKy2QcqbweEv3XzCAqWbQ4fqTqYN9Nfq6uWM2kZFwdVKsgvLbv6+kmTlZNKqy+UT8Nl98L93leE2ff4CT28H/4HNe5+aKmIDPePtbdV0ae8EyCqiJIFMWrUd2QQf3Q3n08DBDR77NzyyFOxdmv9eHa/rZNqAmgVc5dxakiSTVm0eXUBoIXAoPP079BzTcveqqR7mnYDSvHqLheh7xsvhPJIkk9aNvEKUtqsJ/wX3Fh5k7eihJElosF0r1FfOYipJNWTSqotfX7BRt869jGjXqpkv/sTFYiqqdK0RlSRZLJm0zM2Idi1fNwdcHGyp0glOXLSSKaYlqYXIpGVuNUnrXKoy40MdVCoVoXJ1HkkCZNIyP++eYOugdDK9fKLeYtd3MpWk25lMWuamtlOmqoEGq4jd5ZJikgTIpGUZ9DOZNt4YL6uH0u1OJi1LUNNfy4gnrZzCcq6UVLRGVJJkkWTSsgQ1T1o5B6GitM4izva2BHg4ArJdS7q9yaRlCdw6grO3soLPhX31Fusue8ZLkkxaFkGlMqpdK1Q2xkuSTFoWw4h2Lf2SYjkyaUm3L5m0LIW+Z3z9YxBrqodHs4vQ6m6LWbIlqRaZtCyFXz9lGbLCs1B4oc4iQZ5O2NvacLVSS1Ze3Q32knSrk0nLUti7QPtQ5et62rXUNqrrOpnKxnjp9iSTliXRVxEb6mQqh/NItzeZtCyJEUmru+wZL93mZNKyJDXdHs7vAW1VnUVC5ZOWdJuTScuStAtWFoGtLIGLh+ssUtOmlZVXSkl53YlNkm5lMmlZEhv1tfUP66kiejrb097FHiHgqOyvJd2GmpS0lixZQlBQEA4ODoSHh5OSktJg+XXr1hESEoKDgwO9evVi8+bNBsfXr1/Pfffdh6enJyqVir1799a6xj333INKpTLYnn766aaEb9n0kwLKxnhJqovJSWvNmjXExsYSFxdHWloaffr0YdSoUeTm5tZZPikpiXHjxjFp0iT27NlDVFQUUVFRHDhwQF+mpKSEoUOHsmDBggbvPXnyZC5cuKDf3n77bVPDt3wdGm+MD63pGS+7PUi3I2GisLAwMW3aNP33Wq1W+Pn5ifj4+DrLjx07VkRGRhrsCw8PF1OnTq1V9tSpUwIQe/bsqXVs+PDhYsaMGaaGq1dQUCAAUVBQ0ORrtIqiHCHiXIWIcxPian6dRb5NPSMCZ28UYz9Kat3YJKkFGfsZNelJq6KigtTUVCIiIvT7bGxsiIiIIDk5uc5zkpOTDcoDjBo1qt7yDVm5ciXt2rWjZ8+ezJkzh9LS+nuFl5eXU1hYaLBZBWcvcA8ABJxLq7NI9+uqh6KeeeUl6VZlUtK6dOkSWq0Wb29vg/3e3t5kZ2fXeU52drZJ5evzl7/8ha+++oqtW7cyZ84cvvzyS5544ol6y8fHx+Pm5qbf/P1beA3D5tTIjA9dvZxR26gouFpJdmFZKwYmSeZna+4AjDVlyhT917169cLX15eRI0dy4sQJunTpUqv8nDlziI2N1X9fWFhoPYmrwwA48G297Vr2tmq6tHfiaE4xR7KL8HVr08oBSpL5mPSk1a5dO9RqNTk5OQb7c3Jy8PHxqfMcHx8fk8obKzw8HIDjx4/Xedze3h5XV1eDzWpcv4BrPdU/fc94ObeWdJsxKWlpNBr69+9PYmKifp9OpyMxMZFBgwbVec6gQYMMygMkJCTUW95YNd0ifH19b+o6FsmnF9jYQekluHK6ziIhchZT6TZlcvUwNjaWCRMmMGDAAMLCwli4cCElJSVMnDgRgJiYGDp06EB8fDwAM2bMYPjw4bz33ntERkayevVqdu/ezbJly/TXzMvLIysri/PnzwOQkZEBKE9pPj4+nDhxglWrVvHggw/i6enJ/v37mTVrFsOGDaN37943/ZdgcewcwLe3soDruVTw6FSrSKivnMVUuk015dXkokWLREBAgNBoNCIsLEzs2LFDf2z48OFiwoQJBuXXrl0rgoODhUajET169BCbNm0yOL58+XIB1Nri4uKEEEJkZWWJYcOGCQ8PD2Fvby+6du0qXnzxRZO6L1hNl4cam15Uuj5snl3n4XNXSkXg7I2iy5xNorxS28rBSVLzM/YzqhLi9nhnXlhYiJubGwUFBdbRvrV/LayfrDTKT06sdVgIQe+//0xRWRU/zLhb3+G0JW1Ov0DSiUu8+uAdtNGoW/x+0u3F2M+oHHtoqWqG82Tvh6ryWodVKhWhrThNzX/3nWfaqjS+2pHFyp2ZLX4/SaqPTFqWqm0ncPQEbQVkp9dZJMS3dcYgbjmSQ+yavfoXmV/tyEQn56iXzEQmLUulUjU6DrF7KywptvPkZf72VRpVOkFkb19c7G05fbmU7ccvtdg9JakhMmlZMv1MpnUvKxbSwtXD9LMFTPp8N+VVOiJCvVj4eF/G9O8IwBfJsooomYdMWpaskWlqap60cgrLuVJS0ay3Pp5bzITlKRSXVxHeyYPFf7kTO7UNT9wVAChVxrNX5IpAUuuTScuS+VVPCHjlNJTUro4529sS4OEING+71pm8Up74dCd5JRX07ujGpxMG4GCnvC3s6uXC4C6e6ASs3JnVbPeUJGPJpGXJ2rhDu+7K1421azVTFTG3qIzxn+0ku7CMbl7OrJgYhouDnUGZmEGBAKzZdYbyKm2z3FeSjCWTlqVrpIoY2oyN8QWllcR8lsLpy6V0bNuGLyeF4+GkqVUuItQbH1cH8koq2Jxe98KyktRSZNKydB36K3/W1xhfM4vpTc4XX1pRxcQVKRzJLqK9iz0rnwrHx82hzrK2ahv+Eq60bckGeam1yaRl6fRza6WBTlfrcE318Gh2Edom9p0qr9Iy9ctU0rLycWtjx5eTwgj0dGrwnD+H+WOnVrEnK58D5wqadF9JagqZtCyd1x1g5wjlhXDpaK3DQZ5O2NvacLVSS1ae6W/zqrQ6Zq7ey/+OXcJRo2b5xIH6rhQNhuXiwP09lRk2vpRPW1IrkknL0qltwa+f8nUd7VpqG9V1nUxNa4zX6QRz1qfzw4FsNGobPokZwJ0BbY0+v6ZB/rt95ygorTTp3pLUVDJpWYPG2rWasKSYEII3Nx9mXepZbFTwwbh+DOnazqSwBgS2JcTHhbJKHetSz5h0riQ1lUxa1kA/k2lqnYe7N6Fn/KItx/ls+ykA3n6sD/f3NH0mWZVKxfjqp60v5XhEqZXIpGUNaro95B6EipJah0NNfNJa/vsp/pmgtI/FPXwHj1UPzWmKqL4dcLG3JfNyKf+T4xGlViCTljVw9QMXPxA6OL+n1uGaNq2svFJKyqsavNS3qWf5+/eHAJgVEczEIbVnRTWFk72tfjzil8mnb+pakmQMmbSsRcf6Z3zwdLanvYs9QsDRBvpr/XQwm5e+3Q/AX4d04rmRXZsltJoqYuKRXM404Q2mJJlCJi1r0eiMDw1XEX8/folnV+1BqxM81r8jr0WGolKpmiW0Lu2dGdLVEyHHI0qtQCYta9HIsmI10y3X1e1hT9YVJn+xmwqtjvt7+DD/0V7Y2DRPwqox/q4gANbuPkNZpRyPKLUcmbSshW9fUKmhOBsKz9U6XN+T1pHsQp5cvovSCi13d2vH++P6Yqtu/n/2iFAv/NzkeESp5cmkZS00juDdQ/m6jnat7tclrZq1SjIvlzD+sxQKrlbSL8Cdj57oj71tyyxIIccjSq1FJi1r0sCMD129nFHbqCi4Wkl2YRnZBWU88dlOLhaVE+Ljwoonw3CyN3mZS5M8PjAAO7WKvWfyST8rxyNKLUMmLWtyfbvWDext1XRprwxy3nHyMuM/28mZvKsEeTryxaQw3Bztap3T3Nq72PNAzXjEHadb/H7S7UkmLWtSs9DF+b2grT3Wr6Zn/Oxv0jmWW4yPqwNfTgrHy6XuKWZagn484t7z5Jc27xTQkgQyaVkXz65g7wZVVyHnYK3DNY3xFVodHk4avnoqDP/q6ZhbS//AtoT6ulJepWPd7rOtem/p9tCkpLVkyRKCgoJwcHAgPDyclJSUBsuvW7eOkJAQHBwc6NWrF5s3bzY4vn79eu677z48PT1RqVTs3bu31jXKysqYNm0anp6eODs7M2bMGHJycpoSvvWysYGO1YOn62jX6tPRHVDmjv98YhhdvVxaMTiFSqVi/F3K09ZXO+V4RKn5mZy01qxZQ2xsLHFxcaSlpdGnTx9GjRpFbm5uneWTkpIYN24ckyZNYs+ePURFRREVFcWBAwf0ZUpKShg6dCgLFiyo976zZs3i+++/Z926dfz666+cP3+eRx991NTwrV8DayEO6erJu3/qw/pnBtOro1srB3ZNVD8/XByU8Yi/HbtotjikW5QwUVhYmJg2bZr+e61WK/z8/ER8fHyd5ceOHSsiIyMN9oWHh4upU6fWKnvq1CkBiD179hjsz8/PF3Z2dmLdunX6fYcPHxaASE5ONirugoICAYiCggKjylusjJ+EiHMV4oP+5o6kQa//94AInL1R/HV5irlDkayEsZ9Rk560KioqSE1NJSIiQr/PxsaGiIgIkpOT6zwnOTnZoDzAqFGj6i1fl9TUVCorKw2uExISQkBAQL3XKS8vp7Cw0GC7JdTMrXX5GFy9Yt5YGvBEdRVxS4Ycjyg1L5OS1qVLl9BqtXh7exvs9/b2Jjs7u85zsrOzTSpf3zU0Gg3u7u5GXyc+Ph43Nzf95u/vb/T9LJqTJ3h0Vr4+V/f8WpagS3tnhnZtJ8cjSs3uln17OGfOHAoKCvTbmTO30Mya+nYty01acG32hzW7suR4RKnZmJS02rVrh1qtrvXWLicnBx+fume+9PHxMal8fdeoqKggPz/f6OvY29vj6upqsN0y9J1M657xwVKMDFHGI14prWTTfjkeUWoeJiUtjUZD//79SUxM1O/T6XQkJiYyaNCgOs8ZNGiQQXmAhISEesvXpX///tjZ2RlcJyMjg6ysLJOuc8u4vttDHTM+WApbtQ3R1W1bX+yQ4xGl5mHyYLTY2FgmTJjAgAEDCAsLY+HChZSUlDBx4kQAYmJi6NChA/Hx8QDMmDGD4cOH89577xEZGcnq1avZvXs3y5Yt018zLy+PrKwszp8/DygJCZQnLB8fH9zc3Jg0aRKxsbF4eHjg6urKs88+y6BBg7jrrrtu+i/B6nj3ArW90hCfdxI8u5g7ono9PtCfhb8cZd+ZfPafzad3dV8ySWqypryaXLRokQgICBAajUaEhYWJHTt26I8NHz5cTJgwwaD82rVrRXBwsNBoNKJHjx5i06ZNBseXL18ugFpbXFycvszVq1fFM888I9q2bSscHR3FI488Ii5cuGB0zLdMl4can0QoXR/2rjZ3JI167us0ETh7o3hh7V5zhyJZMGM/oyohLLh+0YwKCwtxc3OjoKDg1mjf+nEO7PgQBk6GyHfNHU2DUjPzGLM0GXtbG3bMGUlbJ425Q5IskLGf0Vv27eEtr4FpaizNnQFtuaNmPKJcH1G6STJpWauabg/Z6VB51byxNEKlUulnf/hqR5YcjyjdFJm0rJV7ADh5ga4KLuw3dzSNGt1XGY+YlVfKr3I8onQTZNKyViqVVVURHTW2/Km/MirhSzkds3QTZNKyZo0sK2ZpanrIb5XjEaWbIJOWNbOS4Tw1OrVz4u5uynjEr3bKpy2paWTSsmZ+/QAVFGRBkXVMiFgzQeDaXXJ9RKlpZNKyZg6u4BWqfG0F7VoAI0O96eDehiullWyU4xGlJpBJy9rVzK9lJe1aahuVfn3EL5NPmzcYySrJpGXtGlhWzFI9PtAfjdqGfWcL2Hcm39zhSFZGJi1rV/MG8fwe0FlHG1E7Z3se7KVMKfSlnP1BMpFMWtaufQhonKGiGC4eMXc0Rhs/KAiA7/ed50qJXB/RrKrKzR2BSWTSsnY26uq3iFhVFfHOAHd6+CnjEdfuluMRzUKnhW3z4a0O8N00i56b7Xoyad0KrKyTKdwwHlGuj9j6Cs/D56NhWzzoKmHPV7BjqbmjMopMWreCmsZ4C17ooi6j+3TA1cGWM3lX+fWoHI/Yao7+BB8NhcztStNC7z8r+39+DU5vN29sRpBJ61ZQ0zM+9zCUWc9SaW00av40QBmP+IXs/tDyqirgp1dh1VgovQw+vWHqb/DIR9BrLAgtrHtSeQqzYDJp3QpcvMEtABDKW0QrUrM+4rajF8m63HzjEXU6QXZBGTtOXmbtrjO889MRFv5ylMvF1tXo3GzyTsK/74Pkxcr34U/DU78oU3WrVPDw++DdE0ouwtoYi26cN3mOeMlCdeyvDOc5txs6Dzd3NEbr1M6JYcHt+e3oRVbuzGTOg6FGn1tRpePslVIy80rJulxK5uVSsvJKqv8spbxKV+ucf28/xQujuhMdHojaRtWcP4rlOvAt/HcGVBSBgztELYWQBw3LaBzh8S9h2T1K2+iPc+Chf5oj2kbJpHWr6DgQDm6wqjeINWLuCuS3oxdZs/sMs+4NxsFOrT9WXF5F5uUSJSnlGSam8/lXaaj9Xm2jooN7GwI9HQn0dCQtM59DFwqZ991BVqec4Y2oHvQP9GiFn9BMKkrhx5ch7XPle/+74LHPwK1j3eU9OsOjnyrVx92fKaMt+kW3XrxGkknrVqGf8aF6WTGV9TxFjAjxooN7G87lX+XFb/Zja6NSElVeKZeKG+7D1cZOTaCnIwEeSmIK8HQisPprP/c22KmvtYBodYKVOzN596cMDl0oZMzSZMbc2ZGXHwihvYt9S/+YrSv3MKybCBcPAyoY9gIMfxnUjXzkg++De+bAtrdg4yzw7gF+fVsjYqPJhS1uFZVXId5feX09Yz+0DTR3RCb5cNtx3v4xo85jHk4afVIK9KhOTNVPT+2d7VGZmKAvF5fz9o8ZrKnuH+Zib8use4OJGRSIrdrKm3mFUJ6sfngZqq6Cszc8ugw632P8NXQ6WD0Ojv6otJVO/RUcW/6J1NjPqExat5Jl9ygN8Y/9G3qOMXc0Jikpr+L/Nh1GCEGApyOBHk7VT06OuDrYtcg992RdYd53B0k/VwBAiI8Lfx/dg/DOni1yvxZXVgDfz4SD65Xvu4xU3gw6e5l+rav5yu/TlVPQeQQ88a3SkbkFyaR1g9siaW16AXZ9Av0nQuQ/wcbKnxpagVYnWL0ri3d+yiC/tBKAP/b145UHQ/F2dTBzdCY4lwrf/BWunAYbW/jDXBj83M39DuQchE8joLIU7n4eRs4z6XQhBAfPF5J04hKT7+7c6BOxTFo3uC2S1r41sGGK8rW9qzK8p+PA6m0AOLUzb3wW7EpJBe/8nMHXKVkIAU4aNTMjgnlySJBBu5jF0emU9S9/eV1pGnAPgDH/Bv+BzXP99G/g20nK14+vhNCHGixeqdWRciqPhEM5JBzK4Vy+slLUDzPuJtS34c+dTFo3uC2SVnkRfPsUnPofVJbUPt426FoS6zAAfHqBrVw49Xr7z+Yz77uD7K2eMqerlzP/GN2DwV0tMOGXXIb/PA3Hfla+Dx0NoxdBG/fmvU/NwsAaF5iyFdp1MzhcXF7FrxkXSTiUzZYjuRSWVemPOdjZcHe39syM6EYPP7cGbyOT1g1ui6RVQ6dV3h6d3aW8TTy3u+4ZINT24NtHeQrrOEBJZm7+VvXmsSXodIJvUs8y/8cj5FXPQBHZ25fXIkPxdWtj5uiqnd6u/AdVdEH5d7w/Hgb8tWX+7bSV8MUfIfN3aNcdJieSW25HwuEcfj6YQ/KJy1Ror/WJ83DSEBHqxb13+DC0azvaaIxrC2vRpLVkyRLeeecdsrOz6dOnD4sWLSIsLKze8uvWrWPu3LmcPn2abt26sWDBAh588FrnNiEEcXFxfPLJJ+Tn5zNkyBCWLl1Kt27XMnpQUBCZmYZzL8XHx/Pyyy8bFfNtlbTqcjUfzqcpSezsbiWhXc2rXc7ZW3kKq0lifv3A3rnVw7UEBaWV/DMhgy93ZKITSveKZ0d2ZdLQTtjbtmyjdL10Wvj1bfjtbRA6aBcMjy0Hn54teltRlI32o2HYluTwu2Yo0YV/A64lyCBPR+7r4cO9d3hzZ0DbJnXcbbGktWbNGmJiYvjoo48IDw9n4cKFrFu3joyMDLy8ar+lSEpKYtiwYcTHx/PQQw+xatUqFixYQFpaGj17Kn/RCxYsID4+ns8//5xOnToxd+5c0tPTOXToEA4OSmNoUFAQkyZNYvLkyfpru7i44OTk1Kx/IbcNIZShHTVPYmd3KatV66oMy6lswOuOa0ksYJAy9MOanNkFSR9AfqbSB6n7AyadfvB8AXHfHWR35hUAOrdzIm50D4YHt2+JaOtXeB6+nawMdAbo+wQ8+DZojPsMmEqrE6RmXiHhUDYJh3LwyNvLas0baFRa3qz8C7v8nuDeO7y57w5vuno5m9z15EYtlrTCw8MZOHAgixcrY5h0Oh3+/v48++yzdT71PP7445SUlLBx40b9vrvuuou+ffvy0UcfIYTAz8+P559/nhdeeAGAgoICvL29WbFiBX/+szICPSgoiJkzZzJz5kxTwtWTScsIlVfhwr5rT2Jnd0Ph2drlgu5Wxq51f6DFX4M3mU6ntPX8/j5kJRkeC74f7p8PHp2MvpwQgvVp54j/4QiXqscvjurhzdyH7qBjW8fmjLxuR3+CDU8rT8caZ3joX9B7bLPf5mqFlv8du0jCoRy2HMnl8nUTNGrUNrzqncSEvA8QKhtU4//TrEPGWiRpVVRU4OjoyDfffENUVJR+/4QJE8jPz+e7776rdU5AQACxsbEGySYuLo7//Oc/7Nu3j5MnT9KlSxf27NlD37599WWGDx9O3759ef/99wElaZWVlVFZWUlAQAB/+ctfmDVrFra2dffwLS8vp7z82qDPwsJC/P39ZdIyVeGFa09iZ3bBmZ3KbACgvKkaOBnuHA9t2po3zhpVFZC+TnmyqmnHs7GDPo9DGw9lzihdpdIONHQWDJ0Jdsa3UxWWVbIw4RifJ59GqxM42Nkw7Z6uTB7W2WD4UbMpPA+/LoDUFcr3vn2U6mAzPu1eLi4n8UguCYdy+N+xi5RVXmufcnWw5Q8hXtzXw4dhwe1x1qjhP8/AvlXg6KnMElHfsCATGZu0TBrGc+nSJbRaLd7e3gb7vb29OXKk7ql+s7Oz6yyfnZ2tP16zr74yAM899xx33nknHh4eJCUlMWfOHC5cuMA//1n3oM74+Hj+/ve/m/LjSXVx9QXXhyH0YeX7grOw6zPlQ5SfBQlzlYnkej+uPH15hZgnzrICJaYdS5XGaVC6fQyYqMTl6qfsuzMGNr8IJ7fCr/Nh39fwwAKjq4yuDnbMe/gOxg7sSNx3B9l5Ko/3Eo7yTdpZovp2oFM7JwI8HQnydKKto13Tq0xFObD9X7D736Ct/s83/Gm49x9ga/qQIyEEF4vKOZ5bzPGLxcqf1VtukeGMDh3c2+irfQM7edTu8vHQPyHnAGTvhzXjYeIPYNd6fdqsZuxhbGys/uvevXuj0WiYOnUq8fHx2NvX/kecM2eOwTk1T1rSTXLrCBFxMPwl5Ylm58fKL3DqcmXrfI/y4ep2X+tUHQsvwM6lsHs5lFfPJebiC3f9Dfo/CQ43vGZv1w3Gb4BD38FPryjtXF//2eQqY4iPK6un3MV/953nrc2HybxcyvuJxwzKuDjYElQ95Ej/ZzunhocflVxSqrQpnyjDcEBpR/zDaxA0tNG4tDrB2SulBkmpJkkVlVXVe14PP1fuvcObe+/w5g5f14aTrV0bePwrWDZcebnzw0sw+oNGY2suJiWtdu3aoVaryckxXM04JycHHx+fOs/x8fFpsHzNnzk5Ofj6+hqUub66eKPw8HCqqqo4ffo03bt3r3Xc3t6+zmQmNRO7NspTS7/xyqvwnR/BkU1wcpuytQ2CsKnKLAE3Jo7mcDFDqQLuW6NU90B5HT/kOej1p4afRlQq6BEF3e6F396BpMXKOLsTW02qMqpUKv7YtwMjQ71Zu+sMR3OKOH1ZmYHiQkEZRWVVpJ8r0A8Tup6jRk2AR3Uya+dIsEsV4dkr8cv4ApuaPnYdBsAfXlWG0dyQRMqrtJy+dC05Hcst4nhuMaculdQ5JQ+AjQoCPBzp6uVMFy9nurZ31n9t8lCptoEw5jP4aowy1rFDf+g/wbRrNFGTGuLDwsJYtGgRoDTEBwQEMH369Hob4ktLS/n+++/1+wYPHkzv3r0NGuJfeOEFnn/+eUB5KvLy8jJoiL/RypUriYmJ4dKlS7Rt23h7imyIbwVXMmHXp8ovcVn1B9XOCfr+BcKmQPvgm79H1g7lSSRj87V9AYNgyAzoNqppw1YuHYPNLyjJFsA90KQqY13KKrVk5ZVy+pKSxGqS2enLJQZT6rhQyl/VPzDJdjOuKuXJ6oDoxNeOT5DjPYzAds4EeTriYKfmxMUSjucWc+JiMZmXS+qdlkdja0Pndk509XI22II8nZq/3e23d2HLG6DWwMQflXndmqhFuzxMmDCBjz/+mLCwMBYuXMjatWs5cuQI3t7exMTE0KFDB+Lj4wGly8Pw4cOZP38+kZGRrF69mrfeeqtWl4f58+cbdHnYv3+/vstDcnIyO3fuZMSIEbi4uJCcnMysWbN44IEH+Pzzz5v1L0RqBhUlsH+tUnW8ePja/i4jlWpbl5GmJRedDo7+oCSrMzurd6ogJFIZXxcQfvMxC3Gtylh4TtnXhLeMxqio0nEu5yJix0d0OPwp9lVKtfaETSDvlI/hR21/ru8DVR8XB1slIbU3TE4d2zq23gSHOh2sHQ9HNoJrR2VGiCYOF2vRzqWLFy/Wdy7t27cvH3zwAeHhyi/OPffcQ1BQECtWrNCXX7duHa+99pq+c+nbb79dZ+fSZcuWkZ+fz9ChQ/nwww8JDlb+Z05LS+OZZ57hyJEjlJeX06lTJ8aPH09sbKzRVUCZtMxACDj1q5K8Mn4Aqn/VPLpA+FToMw4cGvi3qCqH/Wvg9w/gcnV7kVqjnDf42VrDSZpFeTH8712lyngTbxnrVVGqPI3+vlCZpx2Uau09L8MdUVQJOJ9fpjyZ5ZWSeamE05dLuVpZRed21xJTNy9n2ruYPi1PiygrhE9GwOXj0GkYPLGh8Xm76iCH8dxAJi0zyztVXXX8Esqrq44aF6XNK2yK4Sv8q/lKo/6OpVBc3R5q7wYDJynJzqXu9tNmdfEo/PDiDVXGt6H7/U27XmWZ8jP9759Qkqvs8+isdHbtOcZy+7sZK/cIfPIHZczr4OfgvjdMvoRMWjeQSctClBcr3Qx2fnzt6QmV8rbxzhg4swN2r1DmMwdw7QB3PaM08tq7tG6szVFlrKqAPV/Ab+9BUfUqN+4BMHy2snRXE55ILNbBDcpqPgB/WgE9HjHpdJm0biCTloXR6ZS+Ujs/hmM/1T7ePlRpXO85xvwzUZQXK28Zkxcrw5zU9nB3rBJffVVGbSXsXaWcV1C9grZrB2Xa475PmP9naik/vwZJi5QXMJO3mNRvTyatG8ikZcEun4CUZcpTjWdXpXrR7V7Lm23CmCqjtkrpv/brfGVCPgBnH2USvf4TmtQx1Kpoq+DLKDj9P+XfcvIWo7u8yKR1A5m0pGYhBBz6D/z4yrXqXvD9MOotZarrbfOvVXsd2ymN+AMnNU8jvrUovqh0PC08ByEPwdgvjXpbLJPWDWTSkppVebEyPUzyktozY7Rpq1QdB06+baf14WwqLL8ftBXKNM13P9/oKcZ+Ri14HllJsmD2zso4wL8lQafqmQ7s3WDEq8pqSENn3b4JC5ROpg++o3y95f/gxJZmu/Qt9OpCksygfXeI+Q4u7IW2nZp/qmNr1v9JZcGNC/uU9q1mIpOWJN0slUqZ4VWq7YF3ANGsbXoyaUmS1HJaYMoa2aYlSZJVkUlLkiSrIpOWJElWRSYtSZKsikxakiRZFZm0JEmyKrdNl4ea0UqFhYVmjkSSpLrUfDYbG1l42yStoiJlfia5Io8kWbaioiLc3OqfGeK2GTCt0+k4f/48Li4ujU5RW7Pc2JkzZ6xucLWM3Txk7DdPCEFRURF+fn7YNDArxG3zpGVjY0PHjqathOvq6mp1v4A1ZOzmIWO/OQ09YdWQDfGSJFkVmbQkSbIqMmnVwd7enri4OKtcoVrGbh4y9tZz2zTES5J0a5BPWpIkWRWZtCRJsioyaUmSZFVk0pIkyarIpCVJklWRSesGS5YsISgoCAcHB8LDw0lJSTF3SI2Kj49n4MCBuLi44OXlRVRUFBkZGeYOq0nmz5+PSqVi5syZ5g7FKOfOneOJJ57A09OTNm3a0KtXL3bv3m3usBql1WqZO3cunTp1ok2bNnTp0oU33nij0cHKlkAmreusWbOG2NhY4uLiSEtLo0+fPowaNYrc3Fxzh9agX3/9lWnTprFjxw4SEhKorKzkvvvuo6SkxNyhmWTXrl18/PHH9O7d29yhGOXKlSsMGTIEOzs7fvjhBw4dOsR7771H27ZtzR1aoxYsWMDSpUtZvHgxhw8fZsGCBbz99tssWrTI3KE1Tkh6YWFhYtq0afrvtVqt8PPzE/Hx8WaMynS5ubkCEL/++qu5QzFaUVGR6Natm0hISBDDhw8XM2bMMHdIjZo9e7YYOnSoucNoksjISPHXv/7VYN+jjz4qoqOjzRSR8eSTVrWKigpSU1OJiIjQ77OxsSEiIoLk5GQzRma6goICADw8PMwcifGmTZtGZGSkwd+/pfvvf//LgAED+NOf/oSXlxf9+vXjk08+MXdYRhk8eDCJiYkcPXoUgH379rF9+3YeeOABM0fWuNtmlofGXLp0Ca1Wi7e3t8F+b29vjhw5YqaoTKfT6Zg5cyZDhgyhZ8+e5g7HKKtXryYtLY1du3aZOxSTnDx5kqVLlxIbG8srr7zCrl27eO6559BoNEyYMMHc4TXo5ZdfprCwkJCQENRqNVqtljfffJPo6Ghzh9YombRuMdOmTePAgQNs377d3KEY5cyZM8yYMYOEhAQcHJp/Yc+WpNPpGDBgAG+99RYA/fr148CBA3z00UcWn7TWrl3LypUrWbVqFT169GDv3r3MnDkTPz8/i49dtmlVKy8vF2q1WmzYsMFgf0xMjBg9erR5gjLRtGnTRMeOHcXJkyfNHYrRNmzYIAChVqv1GyBUKpVQq9WiqqrK3CHWKyAgQEyaNMlg34cffij8/PzMFJHxOnbsKBYvXmyw74033hDdu3c3U0TGk21a1TQaDf379ycxMVG/T6fTkZiYyKBBg8wYWeOEEEyfPp0NGzawZcsWOnXqZO6QjDZy5EjS09PZu3evfhswYADR0dHs3bsXtVpt7hDrNWTIkFpdS44ePUpgYKCZIjJeaWlprdlB1Wo1Op3OTBGZwNxZ05KsXr1a2NvbixUrVohDhw6JKVOmCHd3d5GdnW3u0Br0t7/9Tbi5uYlt27aJCxcu6LfS0lJzh9Yk1vL2MCUlRdja2oo333xTHDt2TKxcuVI4OjqKr776ytyhNWrChAmiQ4cOYuPGjeLUqVNi/fr1ol27duKll14yd2iNkknrBosWLRIBAQFCo9GIsLAwsWPHDnOH1Cigzm358uXmDq1JrCVpCSHE999/L3r27Cns7e1FSEiIWLZsmblDMkphYaGYMWOGCAgIEA4ODqJz587i1VdfFeXl5eYOrVFyPi1JkqyKbNOSJMmqyKQlSZJVkUlLkiSrIpOWJElWRSYtSZKsikxakiRZFZm0JEmyKjJpSZJkVWTSkiTJqsikJUmSVZFJS5Ikq/L/48QQiPznvKYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:05.548670Z", + "iopub.status.busy": "2024-03-22T20:48:05.548275Z", + "iopub.status.idle": "2024-03-22T20:52:10.525895Z", + "shell.execute_reply": "2024-03-22T20:52:10.525087Z" + }, + "papermill": { + "duration": 244.997818, + "end_time": "2024-03-22T20:52:10.528358", + "exception": false, + "start_time": "2024-03-22T20:48:05.530540", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:10.564325Z", + "iopub.status.busy": "2024-03-22T20:52:10.563978Z", + "iopub.status.idle": "2024-03-22T20:52:10.584415Z", + "shell.execute_reply": "2024-03-22T20:52:10.583463Z" + }, + "papermill": { + "duration": 0.040608, + "end_time": "2024-03-22T20:52:10.586520", + "exception": false, + "start_time": "2024-03-22T20:52:10.545912", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0388170.0522470.00314112.5849680.0443851.0239510.1208530.0000133.8802050.0423990.1374930.0560440.0897190.03517616.465174
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.038817 0.052247 0.003141 12.584968 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.044385 1.023951 0.120853 0.000013 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 3.880205 0.042399 0.137493 0.056044 0.089719 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.035176 16.465174 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:10.620207Z", + "iopub.status.busy": "2024-03-22T20:52:10.619875Z", + "iopub.status.idle": "2024-03-22T20:52:10.977505Z", + "shell.execute_reply": "2024-03-22T20:52:10.976573Z" + }, + "papermill": { + "duration": 0.376739, + "end_time": "2024-03-22T20:52:10.979500", + "exception": false, + "start_time": "2024-03-22T20:52:10.602761", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:11.014665Z", + "iopub.status.busy": "2024-03-22T20:52:11.014330Z", + "iopub.status.idle": "2024-03-22T20:56:26.495271Z", + "shell.execute_reply": "2024-03-22T20:56:26.494363Z" + }, + "papermill": { + "duration": 255.501341, + "end_time": "2024-03-22T20:56:26.497846", + "exception": false, + "start_time": "2024-03-22T20:52:10.996505", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:56:26.533812Z", + "iopub.status.busy": "2024-03-22T20:56:26.533500Z", + "iopub.status.idle": "2024-03-22T20:56:26.560312Z", + "shell.execute_reply": "2024-03-22T20:56:26.559584Z" + }, + "papermill": { + "duration": 0.046481, + "end_time": "2024-03-22T20:56:26.562238", + "exception": false, + "start_time": "2024-03-22T20:56:26.515757", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:56:26.595506Z", + "iopub.status.busy": "2024-03-22T20:56:26.595222Z", + "iopub.status.idle": "2024-03-22T20:56:26.600596Z", + "shell.execute_reply": "2024-03-22T20:56:26.599719Z" + }, + "papermill": { + "duration": 0.024394, + "end_time": "2024-03-22T20:56:26.602662", + "exception": false, + "start_time": "2024-03-22T20:56:26.578268", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.3847382401568549}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:56:26.639258Z", + "iopub.status.busy": "2024-03-22T20:56:26.638403Z", + "iopub.status.idle": "2024-03-22T20:56:27.027627Z", + "shell.execute_reply": "2024-03-22T20:56:27.026727Z" + }, + "papermill": { + "duration": 0.410339, + "end_time": "2024-03-22T20:56:27.029724", + "exception": false, + "start_time": "2024-03-22T20:56:26.619385", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEvklEQVR4nO3deXxTZb4/8M/JnqZJum/QjbLvBQoDiAXlAooK4nXDpRUBF7ioXO5LmfsbEJ0ZYEZZ3JjRK62MCzOOuC/AoMC4IiA7spbSfd+TZn1+f5wmbUjSJmmS06bf9+vVV9Nznpw8p2m/efaHY4wxEEKIgERCZ4AQQigQEUIER4GIECI4CkSEEMFRICKECI4CESFEcBSICCGCo0BECBEcBSJCiOAoEPUyaWlpuOWWWwL6GhzH4dlnn+0y3bPPPguO4wKaF9I3UCAKsu+//x7PPvss6uvrhc4K6QVee+015OfnC52NgKNAFGTff/891q1bR4GIeIQCESGEBAkFoiB69tln8T//8z8AgPT0dHAcB47jcOXKFeTl5eGGG25AXFwc5HI5hg8fjm3btrm91p49ezB27FgoFAoMHz4cu3bt8jo/BoMBTz31FGJjY6FWq3HbbbehuLjYZdpvv/0WWVlZUCgUyMjIwF//+leX6TiOw/Lly/HOO+9gyJAhUCgUGD9+PA4ePOj0u+A4DufPn8f9998PrVaL2NhY/O53vwNjDEVFRZg3bx40Gg0SEhLw4osven1/APDll18iOzsbarUaGo0GWVlZePfddx3SvP/++xg/fjyUSiViYmJw//33o6SkxCFNbm4uwsPDUVJSgvnz5yM8PByxsbFYtWoVLBaLQ1qr1YqtW7di1KhRUCgUiI2NxZw5c3D48GF7Gk/e77S0NJw+fRoHDhyw/61Mnz7dp99Dj8dI0Bw/fpzde++9DADbvHkz+9vf/sb+9re/sebmZpaVlcVyc3PZ5s2b2csvv8xmzZrFALBXXnnF4Rqpqals8ODBLCIigj3zzDNs06ZNbNSoUUwkErE9e/Z4lZ/777+fAWALFy5kr7zyCluwYAEbPXo0A8DWrl1rT3fixAmmVCpZSkoKW79+PXv++edZfHy8PW1HANjIkSNZTEwMe+6559jGjRtZamoqUyqV7OTJk/Z0a9euZQDY2LFj2b333stee+01NnfuXAaAbdq0iQ0ZMoQ99thj7LXXXmNTp05lANiBAwe8ur+8vDzGcRwbOXIk+8Mf/sBeffVVtnjxYvbAAw84pAHAsrKy2ObNm9kzzzzDlEolS0tLY3V1dfZ0OTk5TKFQsBEjRrBFixaxbdu2sTvuuIMBYK+99prD6+bm5jIA7KabbmJbtmxhL7zwAps3bx57+eWX7Wk8eb8//PBD1r9/fzZ06FD734q373FvQYEoyP785z8zAKygoMDhuE6nc0o7e/ZsNmDAAIdjqampDAD74IMP7McaGhpYYmIiy8zM9Dgfx44dYwDY448/7nB84cKFToFo/vz5TKFQsMLCQvuxM2fOMLFY7DIQAWCHDx+2HyssLGQKhYLdfvvt9mO2QLR06VL7MbPZzPr37884jmMbNmywH6+rq2NKpZLl5OR4fH/19fVMrVazSZMmMb1e73DOarUyxhgzGo0sLi6OjRw50iHNZ599xgCwNWvW2I/l5OQwAOy5555zuFZmZiYbP368/eevv/6aAWArVqxwypPtdRnz/P0eMWIEy87O9uCOezeqmvUQSqXS/rihoQHV1dXIzs7G5cuX0dDQ4JA2KSkJt99+u/1njUaDBx98EL/88gvKy8s9er0vvvgCALBixQqH408++aTDzxaLBbt378b8+fORkpJiPz5s2DDMnj3b5bUnT56M8ePH239OSUnBvHnzsHv3bqdqzOLFi+2PxWIxJkyYAMYYHn74YfvxiIgIDBkyBJcvX/bo3gBg7969aGpqwjPPPAOFQuFwzjbk4PDhw6isrMTjjz/ukGbu3LkYOnQoPv/8c6frPvroow4/T5s2zSFfH3zwATiOw9q1a52e23Gogzfvd19AgaiH+O677zBz5kyoVCpEREQgNjYWv/3tbwHA6Q9z4MCBTuN3Bg8eDAC4cuWKR69XWFgIkUiEjIwMh+NDhgxx+Lmqqgp6vR6DBg1yusa1aW1cpR08eDB0Oh2qqqocjncMbgCg1WqhUCgQExPjdLyurs79DV3j0qVLAICRI0e6TVNYWAjA9X0MHTrUft7G1t7TUWRkpEO+Ll26hKSkJERFRXWaP2/e775AInQGCP/He+ONN2Lo0KHYtGkTkpOTIZPJ8MUXX2Dz5s2wWq1CZzFgxGKxR8cAgAm8qrG7fHmrL7/f7lAgCjJXI5E//fRTGAwGfPLJJw4lhG+++cblNS5evAjGmMO1zp8/D4DvafFEamoqrFYrLl265FAiOHfunEO62NhYKJVKXLhwweka16a1cZX2/PnzCAsLcypRBIqtpHfq1CkMHDjQZZrU1FQA/H3ccMMNDufOnTtnP+/t6+7evRu1tbVuS0XevN99ZeQ6Vc2CTKVSAYDDgEbbJ23HT/yGhgbk5eW5vEZpaSk+/PBD+8+NjY3YsWMHxo4di4SEBI/ycdNNNwEAXnrpJYfjW7ZscfhZLBZj9uzZ+Oijj3D16lX78bNnz2L37t0ur/3DDz/g6NGj9p+Liorw8ccfY9asWX4rVXRl1qxZUKvVWL9+PVpbWx3O2X7PEyZMQFxcHP7yl7/AYDDYz3/55Zc4e/Ys5s6d6/Xr3nHHHWCMYd26dU7nbK/rzfutUqn6xOBXKhEFma0R93//939xzz33QCqV4vrrr4dMJsOtt96KRx55BM3NzXjjjTcQFxeHsrIyp2sMHjwYDz/8MH7++WfEx8dj+/btqKiocBu4XBk7dizuvfdevPbaa2hoaMCUKVOwb98+XLx40SntunXr8NVXX2HatGl4/PHHYTab8fLLL2PEiBE4ceKEU/qRI0di9uzZWLFiBeRyOV577TX7dYJFo9Fg8+bNWLx4MbKysrBw4UJERkbi+PHj0Ol0eOuttyCVSrFx40Y89NBDyM7Oxr333ouKigps3boVaWlpeOqpp7x+3RkzZuCBBx7ASy+9hAsXLmDOnDmwWq3497//jRkzZmD58uWYNWuWx+/3+PHjsW3bNvz+97/HwIEDERcX51R6CwnCddj1Xc8//zzr168fE4lE9q78Tz75hI0ePZopFAqWlpbGNm7cyLZv3+7U1Z+amsrmzp3Ldu/ezUaPHs3kcjkbOnQoe//9973Oh16vZytWrGDR0dFMpVKxW2+9lRUVFTl13zPG2IEDB9j48eOZTCZjAwYMYH/5y1/sXfAdAWDLli1jb7/9Nhs0aBCTy+UsMzOTffPNNw7pbM+tqqpyOJ6Tk8NUKpVTXrOzs9mIESO8vsdPPvmETZkyhSmVSqbRaNjEiRPZe++955Dm73//O8vMzGRyuZxFRUWx++67jxUXF3uUL1e/A7PZzP785z+zoUOHMplMxmJjY9lNN93Ejhw54pAvT97v8vJyNnfuXKZWqxmAkO3K5xijfc2I/3Ach2XLluGVV14ROiukF6E2IkKI4KiNKAR1NahRqVRCq9UGKTf+V1VV5TQwsiOZTNblOB7Ss1AgCkGJiYmdns/JyenVS0tkZWU5DTbsKDs7G/v37w9ehki3USAKQXv37u30fFJSUsBeOxhNju+88w70er3b85GRkQHPA/EvaqwmhAiOGqsJIYLr1VUzq9WK0tJSqNXqPjMUnpDehDGGpqYmJCUlQSRyX+7p1YGotLQUycnJQmeDENKFoqIi9O/f3+35Xh2I1Go1AP4mNRqNwLkhhFyrsbERycnJ9v9Vd3p1ILJVxzQaDQUiQnqwrppOqLGaECI4CkSEEMFRICKECK5XtxF5gjEGs9nc6dwk4ppYLIZEIqGhESTgQjoQGY1GlJWVQafTCZ2VXissLAyJiYmQyWRCZ4WEsJANRFarFQUFBRCLxUhKSoJMJqNPdi8wxmA0GlFVVYWCggIMGjSo0wFphHRHyAYio9EIq9WK5ORkhIWFCZ2dXkmpVEIqlaKwsBBGo9Fpf7De5OjVOlypbsHkjGgkapVdP4EEVch/xNGnePeEwu+vtF6PA+eqYLYymMw0x7sn6v1/ZYR04UxpIwCgf4QSKdFUOu6JKBCRkFdUx3dWJGh7b9Uy1FEg6uPS0tKc9jILJc0GM+p1JnAckBShRG2LEUW11Iva01AgIiGtppnfODFKJUNjqwlvfX8Fn54oFXz7auKIAlEIMBqNQmehx6pp4X83USoZosJkEHEcDCYrmgxmgXNGOuqTgchotrr9MlusHqc1eZDWF9OnT8fy5cuxfPlyaLVaxMTE4He/+539UzwtLQ3PP/88HnzwQWg0GixduhQA8O2332LatGlQKpVITk7GihUr0NLSYr9uZWUlbr31ViiVSqSnp+Odd97xKX+9jVohQZRKBolYhCiVFABQ20zBuycJ2XFEnXn1G+dtlW3SY1SYn9nP/vPrBy/BZHFdjO8fqcSdE9oXZtv+XQH0RsepJE/9x2Cf8vjWW2/h4YcfxqFDh3D48GEsXboUKSkpWLJkCQDghRdewJo1a7B27VoAwKVLlzBnzhz8/ve/x/bt21FVVWUPZratqHNzc1FaWopvvvkGUqkUK1asQGVlpU/56y3GpURiXEqkPYhrlFJUNxvRoDcJnDPSUZ8MRL1BcnIyNm/eDI7jMGTIEJw8eRKbN2+2B6IbbrgB//3f/21Pv3jxYtx333148sknAQCDBg3CSy+9hOzsbGzbtg1Xr17Fl19+iUOHDiErKwsA8Oabb2LYsGFBvzch2EbVR4TJALSgngJRj9InA9GyGQPdnhNdMwtk6fUZbtNeO2Nk0dT07mTLwW9+8xuHKSmTJ0/Giy++aJ+8O2HCBIf0x48fx4kTJxyqW4wx+1SX8+fPQyKRYPz48fbzQ4cORUREhN/y3BtolXzVjEpEPUufDEQyiedNY4FK210qlcrh5+bmZjzyyCNYsWKFU9qUlBScP38+WFnrMep1RvzzSDGiVDIsGMevl0yBqGfqk4GoN/jpp58cfv7xxx8xaNAgiMVil+nHjRuHM2fOYOBA16W9oUOHwmw248iRI/aq2blz51BfX+/XfPckTa1mNLWaIRW3f0DEquWYkhGN6HBaTaAn6ZO9Zr3B1atXsXLlSpw7dw7vvfceXn75ZTzxxBNu0z/99NP4/vvvsXz5chw7dgwXLlzAxx9/jOXLlwMAhgwZgjlz5uCRRx7BTz/9hCNHjmDx4sVQKkN3AmhjK1/qUSvaP2/D5RJMGhCNgXGdL+ZOgosCUQ/14IMPQq/XY+LEiVi2bBmeeOIJeze9K6NHj8aBAwdw/vx5TJs2DZmZmVizZo3D9tJ5eXlISkpCdnY2FixYgKVLlyIuLi4YtyOIplZ+rJBaIRU4J6QrVDXroaRSKbZs2YJt27Y5nbty5YrL52RlZWHPnj1ur5mQkIDPPvvM4dgDDzzQrXz2ZO2ByPHPvLaF776PCZdRkOohqEREQlZTW9UsXO4YiL75tRIf/VKColq9ENkiLlAgIiFL1za4VHVNILL93GKkaR49BVXNeqD9+/cLnYWQoFZIYDRboZI79jTaSkjNNN+sx6BARELWvLH9XB63BaYWCkQ9BlXNSJ9ja7xubqVA1FNQICJ9jlLGByKdkfa66ymoakZCUnGdDl+dKkdShBI3j0p0OBcm5atmehMFop6CAhEJSc0GfnqHq3YglVyC6wfHQCEVgzFG+931ABSISEhy13UP8JOTx6dGBTtLpBPURkRCks7AByKlzPUkYdKzUImIhCRd22BFlcz1n3h1swFNrWaa5tFDUImIhCRb1SzMTYnowLkqmubRg/StQMQYYDYG/8uLrWt27NiB6OhoGAwGh+Pz588P6Qmq/mYLRO6qZrYART1nPUPfqppZTMC/Xwz+6077b0Di2UJcd955J1asWIFPPvkEd955JwB+943PP/+805n1xFG4QgKDWeo04dVGYQtENJaoR+hbJaJeQKlUYuHChfadNwDg7bffRkpKCqZPny5cxnqZ28Yk4aGp6YjXuN5m2jaWSEcTX3uEvlUiEkv50okQr+uFJUuWICsrCyUlJejXrx/y8/ORm5tL4138KKytEZuqZj1D3wpEHOdxFUlImZmZGDNmDHbs2IFZs2bh9OnT+Pzzz4XOVkhRyvjKAFXNeoa+FYh6kcWLF2PLli0oKSnBzJkzkZyc3PWTCACgQWfCP48WQ62Q4K4Jrn9viraqWSuViHoEaiPqoRYuXIji4mK88cYbWLRokdDZ6VX0Jgsa9SY0drJlUGSYDNcPjsXkjJgg5oy4Q4Goh9JqtbjjjjsQHh6O+fPnC52dXsVWyrGVelxRySUYnxqJIQm0m0dPQIGoByspKcF9990HuVwudFZ6lVZz14GI9CyCB6KSkhLcf//9iI6OhlKpxKhRo3D48GGhsyWouro6fPjhh9i/fz+WLVsmdHZ6nVaTFQCgkHb+513R2IqC6hYYzNROJDRBG6vr6uowdepUzJgxA19++SViY2Nx4cIFREZGCpktwWVmZqKurg4bN27EkCFDhM5Or2Noq5rJJZ2XiD49XoqmVjMWTkpBvIZKT0ISNBBt3LgRycnJDoP30tPTBcxRz+Bu3zLimVazZyUiuVSMplYz9Zz1AIJWzT755BNMmDABd955J+Li4pCZmYk33njDbXqDwYDGxkaHL0KuJZeIEBHmfnqHjULC//nbqnJEOIIGosuXL2Pbtm0YNGgQdu/ejcceewwrVqzAW2+95TL9+vXrodVq7V+ejK1hXkw4Jc564+/vNwOi8dDUdGSmdF7Fp7FEPYeggchqtWLcuHH44x//iMzMTCxduhRLlizBX/7yF5fpV69ejYaGBvtXUVGR22tLpfy0Cp1OF5C89xW235/t9xlKKBD1HIK2ESUmJmL48OEOx4YNG4YPPvjAZXq5XO5xV7ZYLEZERAQqKysBAGFhYTRXywuMMeh0OlRWViIiIgJiceg15trakGxtSkQ4ggaiqVOn4ty5cw7Hzp8/j9TUVL9cPyEhAQDswYh4LyIiwv577C12HroKC2O4eWQiIlXu5xZSiajnEDQQPfXUU5gyZQr++Mc/4q677sKhQ4fw+uuv4/XXX/fL9TmOQ2JiIuLi4mAyuR/uT1yTSqW9siRU3WyAycIg6qIEnBIVhusHxyI2nAaMCk3QQJSVlYUPP/wQq1evxnPPPYf09HRs2bIF9913n19fRywW98p/KOI9s8UKk4VvYJd30X0fr1G4Xa+IBJfgs+9vueUW3HLLLUJng4QIQ1t7D8fx3fikd6B3ioSU1g6jqrvqnDBbrChr0ONqDfWsCo0CEQkpth4wT0pDOpMFOw8V4aNjJb1yvFQooUBEQorBgyVAbBRtc9EsVgajhbrwhUSBiIQUjuMQGSaFRtl186dUzEEs4qtvBhpLJCjBG6sJ8af0GBXSYzybOM1xHOQSEXRGCwwmK0AdaIKhEhHp02xtSbQmkbAoEJE+Td7WlmSkqpmgqGpGQsqB81UoqtUhKy3Ko/Wo20tEFIiERIGIhJR6nRFVTQaYPOwFG9lPi9RoFeLUNM1DSBSISEhp38HDs1aHwfG0i0dPQG1EJKQY7AMaaW5hb0IlIhJS7FM8PCwR6Y0W1OuNkIpFiKFZ+IKhEhEJGYyxDlsJeVYiOl/RhJ2HivDj5ZpAZo10gQIRCRlmK4PF2rYEiIcz720lJwMtoC8oqpqRkGGyWBERJoXJYoVM7GEgamtLou57YVEgIiEjTCbBQ1O92xdPRiOrewSqmpE+jQY09gwUiEifZg9EJiutSSQgCkQkZFyoaMLbPxbi2wvVHj/H1kZkZcy+1jUJPmojIiGjsdWMqiYDojvZQuhaUjGHSQOi2paWDWDmSKcoEJGQ4c3qjDYcx2FKRkygskQ8RFUzEjJazd6NqiY9B71jJGTYBiV6O8+sQW9CWYMeOqM5ENkiHqBAREKGrUTk6cx7m3+dqcDOQ0UopG2FBEOBiIQMb+eZ2dinedBYIsFQICIhQy4RQSkTex+IbNM8TDS6WijUa0ZCxoJx/X16Ho2uFh6ViEifR4FIeBSISJ9n28mDJr4Kh6pmJCQ0tZrw8bFShMslmJ/Zr+snMAbbUOqO882IMCgQkZCgN1pQ1WSA3thFqab6InBxL2BoBhLHABk3IFYtx6QBUYgM83xqCPEvCkQkJNgXze9sDFH9VeDUBwBrK/mUHAGYBTFDbqL1qgVGbUQkJNi3EXI3qtpiBs59xQeh2CHAsFv5qlnpMaC+KHgZJS5RICIhwTaY0W2JqOIUoKsBZCpgyM1Awki+agbAeuU71LYYUdagpzWJBEKBiIQEW4+Xy3lmjAHFP/OPU34DSBXtjwFYay9j58GT2HmoCGYrBSIh+BSILl++7O98ENIt7dM7XPxJNxQDLdWAWAokjG4/rowEotIh5oC4lvNt16EufCH4FIgGDhyIGTNm4O2330Zra6u/80SI10QcoJSJoXQ1vaPyLP89dkh7acgmfgQ4cIg1XAUAGGlQoyB8CkRHjx7F6NGjsXLlSiQkJOCRRx7BoUOH/J03Qjw2ZWAMHs3OwKQB0Y4nrFagqi0QxQ13fmJUBsCJoDHXQG5uotHVAvEpEI0dOxZbt25FaWkptm/fjrKyMlx33XUYOXIkNm3ahKqqKn/nkxDfNBYDRh1fEopMcz4vCwO0/SAWc4jQX6VAJJBuNVZLJBIsWLAA77//PjZu3IiLFy9i1apVSE5OxoMPPoiysjJ/5ZMQ39S2tWdGZQAiN137kemQiETQGMpomodAuhWIDh8+jMcffxyJiYnYtGkTVq1ahUuXLmHv3r0oLS3FvHnz/JVPQjr10S8leP9wEepajI4n7IGok40XI1IgFnHQtJbB0NXIbBIQPo2s3rRpE/Ly8nDu3DncfPPN2LFjB26++WaIRHxcS09PR35+PtLS0vyZV0LcKm3Q83uTdTxoaAaaKvjHkZ0EInUiItVKhMmMkEmbAUQGMKfEFZ8C0bZt27Bo0SLk5uYiMTHRZZq4uDi8+eab3cocIZ6wWlmH9ao7FPLrCvjv6nhAHu7+AmIJ4vplAHWFgLUKQHLgMktc8ikQ7d27FykpKfYSkA1jDEVFRUhJSYFMJkNOTo5fMklIZzo2MDuszmibuuGqkfpamn58IGqidk0h+NRGlJGRgepq5900a2trkZ7eSRGYkACwDUKUSUQQizrskthQzH/Xdl3CMYbFQ2+yQFdTHIgski74FIjczcdpbm6GQqFweY6QQGnfvaNDacjYws8tA/jSThcut2pwrKgehVcLAbOxy/TEv7yqmq1cuRIAvzvmmjVrEBYWZj9nsVjw008/YezYsT5lZMOGDVi9ejWeeOIJbNmyxadrkL6p1VX7UEMJ/10Vw48V6oJUpYFRHAaLxQg0VwAR1E4UTF4Fol9++QUAXyI6efIkZLL2haRkMhnGjBmDVatWeZ2Jn3/+GX/9618xevTorhMTcg0rY1DKxAiTdSgRNbS1D3lQLQP4INYii4XWXAw0lVMgCjKvAtE333wDAHjooYewdetWaDSabmegubkZ9913H9544w38/ve/7/b1SN+TERuOjOxwxyYDe/uQZzt7yCQiNMtiYTFeBZpKA5BL0hmf2ojy8vL8EoQAYNmyZZg7dy5mzpzZZVqDwYDGxkaHL0JsuLY1qGEx8aUawONAJJeI0SKL5pcBaa4MUA6JOx6XiBYsWID8/HxoNBosWLCg07S7du3y6Jo7d+7E0aNH8fPPP3uUfv369Vi3bp1HaUkf1lTOr8QoDwcUWo+eIpeIoJNGw8oAS0sNxFaL+ykhxO88DkRardb+iaPVevbmdqaoqAhPPPEE9u7d63FP2+rVq+0N5gDQ2NiI5GSqy/d131+sRmlDK8YmR2BgXHh7aUidaN+poytyiQgmiQoWkQwWiwViXQ0QHhfAXJOOPA5EeXl5Lh/76siRI6isrMS4cePsxywWCw4ePIhXXnkFBoMBYrHjJ5JcLodcToucE0dVzQYU1eowNEHNH7ANSlQneHwNjuMwPi0KCdZkiLhaoKWKAlEQ+TSyWq/n1/a1dd8XFhbiww8/xPDhwzFr1iyPrnHjjTfi5MmTDsceeughDB06FE8//bRTECLEHfvC+bbVGTuWiLwwbVAsYE0HSuv5QESCxqdANG/ePCxYsACPPvoo6uvrMXHiRMhkMlRXV2PTpk147LHHuryGWq3GyJEjHY6pVCpER0c7HSekM+3jiMSA2QDoa/kTXpSI7FSx/PdmCkTB5PMKjdOmTQMA/POf/0RCQgIKCwuxY8cOvPTSS37NICFdaS8Ridsaqhmg0PA7dnh5nXpRBEwWK5WIgsynEpFOp4NazdfH9+zZgwULFkAkEuE3v/kNCgsLfc7M/v37fX4u6ZsYY44L59fZqmXel4b2nKnAlbIW3G0wIl7cwJeuJNQmGQw+L57/0UcfoaioCLt377a3C1VWVvptfBEhnjBarLC2DWTkS0S2hmrv2ocAvufMIlbAIG6bEkKloqDxKRCtWbMGq1atQlpaGiZNmoTJkycD4EtHmZmZfs0gIZ0xWRjCZGLIJCJIRFyHhmrvS0S2uWqtsraF0XS1/som6YJPVbP//M//xHXXXYeysjKMGTPGfvzGG2/E7bff7rfMEdKVcLkEj2RngDEGztwK6Ov4Ez6ViPieWr1EC6CmffY+CTifAhEAJCQkICHB8VNn4sSJ3c4QIb7guA6lIWUEIFV6fQ3bdtUt4gjACgpEQeRTIGppacGGDRuwb98+VFZWwmp13IKFdoIlguhGtQxor5o1izVtgYiqZsHiUyBavHgxDhw4gAceeACJiYntkw0JCbILFU04VlSPtBgVspp9b6gG2qtmzaK2Dhd9HUBzzoLCp0D05Zdf4vPPP8fUqVP9nR9CvFKvN6G4Tg+NUtrtElFkmBRjUyIQqZQCV6T8LH59PaCK7vK5pHt8CkSRkZGIioryd14I8ZptMGMYZwRaG/iD4b4FouhwOWYMaZtfVhnFb0Wkr6VAFAQ+dd8///zzWLNmDXQ6nb/zQ4hX9G0bImrMbe05ykh+e+nuCmsLPtRgHRQ+lYhefPFFXLp0CfHx8UhLS4NUKnU4f/ToUb9kjpCu6NtKROHGtsGH6nifr8UYg85ogcFsRYQiiv+UpkAUFD4Fovnz5/s5G4T4xlYiCjO2bW/lY7UM4KeovX6Q7/F9dHgklAAFoiDxKRCtXbvW3/kgxCe2EpHS0BaIulEiEok4yCQiGM1WGKQR7YGIMY8XWCO+8amNCADq6+vxf//3f1i9ejVqa/n6+dGjR1FSUuK3zBHSFQ6AlBkhN3WvodrGPs1DouWDj6kVMFFbaKD5VCI6ceIEZs6cCa1WiytXrmDJkiWIiorCrl27cPXqVezYscPf+STEpdyp6WB1V4BjIn59ag/2MOuMXCpGU6sZBiYC5Bq+J05X6/WSIsQ7PpWIVq5cidzcXFy4cMFhvembb74ZBw8e9FvmCPEE11wJDly3qmU2thKRwWwFwtqGqFA7UcD5FIh+/vlnPPLII07H+/Xrh/Ly8m5nihCv2Jb+6Ga1DOgQiExWQNkWiPQ01SPQfApEcrnc5Z5i58+fR2xsbLczRYgnyhr0eP9wES5eusgf8HFEdUe2aR4Gs6XDWCIKRIHmUyC67bbb8Nxzz8FkMgHgZz5fvXoVTz/9NO644w6/ZpAQdxr1ZpTWNMLQZOu6737VLCUqDGOTIxCnVgBhbesS2ZYWIQHjUyB68cUX0dzcjNjYWOj1emRnZ2PgwIFQq9X4wx/+4O88EuKSzmiGylQDqYgD5Gp+Q8VuGp6kwYyhcUiJDutQNasDrllhgviXT71mWq0We/fuxXfffYfjx4+jubkZ48aN82jbaEL8RW+yQGWshlTM+aVa5kShBUQSwGoGWuvbG6+J33kdiKxWK/Lz87Fr1y5cuXIFHMchPT0dCQkJ/Cp5NPCLBEmryQKVsQoSmcgv1TIAsFgZWk0WWBmDWiHlq2fNVXypiAJRwHhVNWOM4bbbbsPixYtRUlKCUaNGYcSIESgsLERubi4tE0uCSmfkS0QSkf9KRAXVzXj94GV8edK22iN14QeDVyWi/Px8HDx4EPv27cOMGTMczn399deYP38+duzYgQcffNCvmSTEldbWVkSZ6iEVq/xWInLoNQOo5yxIvCoRvffee/jtb3/rFIQA4IYbbsAzzzyDd955x2+ZI6QzSkM1RGAQKzR8Y7UfOAxoBNqrYzSWKKC8CkQnTpzAnDlz3J6/6aabcPz48W5nihBPzE1lmJQeBW18it8mpcquDURUNQsKrwJRbW0t4uPdF4Hj4+NRV0djLkiQNJaA4ziItP38dklb1cxotsJiZe0lIkMzYDb67XWII68CkcVigUTivllJLBbDbDZ3O1OEeMQ2tUOT5LdL2qpmAB+MIFW2T6Sl6lnAeNVYzRhDbm4u5HLX+4EbDAa/ZIqQrlTX1KDichHC5FKk+7hrhysOaxKZLVDKxHz1zKjjG6wDMV6JeBeIcnJyukxDPWYkGFqqi9DYaoZOFo10icyv1x6epAEYIBG3lY7CooCGYmonCiCvAlFeXl6g8kGIV8x1xQAAqx9LQzb2nTxsbF34VDULGJ9XaCRESNZGfiVQTu2/9iG37D1nFIgChQIR6X2sVqCRb6gWR/g/EFmsDC0Gs33PNIcF0hjz++sRCkSkN2qugMVkgEUkg0zjnxHVHX3zayVeP3gZx4rq+QPKSH6cksUEGJv9/nqEAhHpjRqKYLIwNMoTEKaQdp3eSwopP5bIXiISiQFFBP+YqmcBQYGI9D71VwEAzYpEhMt9WsmmUwpp204epg5rENH61QHl/3eRkEBiDGgowqh+WowcNxlQux7T1h22EpF94ivQ1mB9iXrOAoQCEeldWqr5vcbEEnDqxIBsfGgrEdl2kQXQoUREU5gCgapmpHdpq5ZB059vuwkApzYigGbhBxgFItK71F5Grc6IfRVK/HQ5MO019kBk7tBGZF+/uh6wWpyfRLqFqmak97CYgfpC6I0WXEQcUnWmgLyMSibBsES1PSAB4Nc7Ekv5Lnx9PaCKDshr91UUiEiXLlY24Vx5M6LDZRifGgmpWKCCdGMxYDFBBwV00mhoFIH581XKxJgz8pqpIxzHV8+aKvjqGQUiv6JARJxZLfzAPbkGp8sased0BX+8AiisacEd4/q3TwgNptrL/Dd5P4DjoFH6fwxRp5RtgUhXA2BQcF87xFEgIo7qrgBnPgaMOpjCYvFT4zgA4RgYF47iOj1K61vxa3kTRvbTBj9vNZcAAJUSflqHOkAlIqB9Nw+ZRNReAqT1qwOGAhFpp6sFTn1gX4mwqvQqBjSWoChjIeaOSkRJvR4WK0NajEqYvLVUg3EcSkUJAAO/3U+AvHfoKqqaDLg9s1/7/VLPWcBQrxlpV3CQD0La/sBvHkWFUQa5uQnXic9AJOKQHBUmTBACgKpfAQAGdQp0TA6OQ8DaiICOPWfXDmoEja4OAEED0fr165GVlQW1Wo24uDjMnz8f586dEzJLfVdLjf2fHYNmAcpIjLzhHgyJD0d//VmgtdExucGMisbW4OWv8iwAQB8xCFqlFFqlNKDtVJ0OajTq+EGVxG8EDUQHDhzAsmXL8OOPP2Lv3r0wmUyYNWsWWlpahMxW31R+gp8+EZ0BqPkZ7dLYgYjqNxBSjgHFh+xJi+t0yP/+Cr44WcYvMB9oLTVAcyXAiRCZPAKLrktHzuS0gL6k0j6oscNYIokckIfzj6l65leCthF99dVXDj/n5+cjLi4OR44cwfXXXy9QrvogxoCK0/zjhNGO51ImA/VFQOkxIO16QCJDnFoBqZhDvc6EY0X1GJ8aGdj8lf3Cf48aYF/IXiQK7NbmLqtmAF89MzTzbVZ+XLS/r+tRbUQNDQ0AgKgo13uMGwwGNDY2OnwRP2gsAQxN/Cd+9EBYrQzvHy7CwfNVMGhS+fV4LCagiq8eySQiTMmIAQAcKqh1nArhbxYzUH6Kf5yUGbjXuYatama49t5oFn5A9JhAZLVa8eSTT2Lq1KkYOXKkyzTr16+HVqu1fyUnJwc5lyGqrVscUQMAsQRVzQYU1+lxsqQBUrEYSGwrJZWdsD9leKIGMWo5Wk0W/BigqRYAgIpTgEkPKDRA1AD843AR/vFzEaqbA7tjjG1/M4eqGUDrVwdIjwlEy5Ytw6lTp7Bz5063aVavXo2Ghgb7V1FRURBzGMJqOwQiAGUNfENsvwglXwVKGMWPLG4o5ttrwFeNrh/El4qOFzWgrsXzzQfNFiuuVLfgfEUTmg2d7INntQJXf+Af98+CiQGl9XqU1Osdp18EQHS4DMMS1UiOCnM8QetXB0SPGEe0fPlyfPbZZzh48CD69+/vNp1cLne7pxrxkaGZHy0M2ANRTVtpI9a21o9cDURlADUX+RLKgGwAQGq0CukxKhRUt+DA+SrMG5sEzoNlORiAPWfK0WKwQCziMHVgNMalRDo/t+wYP69LqgQSx6K62QDGAJVcHJAF0TpK1CqRqFU6n+g4loixgCxD0hcJWiJijGH58uX48MMP8fXXXyM9PV3I7PRNtmU1wuPsPUI1zXzpJjq8w35hCW3V5YpTDgvITxsUA4mIQ5ym8w8I1uE5UjHfxhSjlsNiZTh4vhpHCq9Z58fYAhQc4B+nXQdIZKhquiZACkERAXAivu3KQG2U/iJoIFq2bBnefvttvPvuu1Cr1SgvL0d5eTn0er2Q2epbGvj9wRCRAoAPGNUt/D98tKrDP3z0QL4xu7WxPXgBiA6XI3dqGqZkxLgtDemMZuw6WoILFU32YyP7aXH/pBRc11a9+/ZiNUrq2953qxU4+yk/Vic8DkgaBwCobGwLROGK7t+3ByxWhmaDGdaOQxREIr7xHqDqmR8JGoi2bduGhoYGTJ8+HYmJifavv//970Jmq29paGtn0/JV4maDGQaTFSKOQ2RYhykUYikQN4x/XHHK4RIdp1q0GMz46lQ5imp1qG0x4lhRPd7+sRBXa3XYf64KZkt74y/HcchKi8LwJA0YA/aeLofJbAbOfQHUFgBiCTDsNv6fH0BZ2wDK+C5KX/7AGMO2/RfxxsHLaGq9ph0rjNqJ/E3QNiJGe0QJy2wAWqr4x22BqMVggUouhlwidh65HD+CH09UeZYffS12nuu179dKXKpsxtkyx2pLdLgMc0cluhwNnT04FldrdLDWFKB836dIlrXw1Z+htwLhsQD4Ec7VbVWzpAgXbTd+xnEcFFIxTBYzdCYztOhwrzTnzO96RGM1EUhjCd/eo4zgG6QBJGgVWHp9Boxmq3N6bTKfVl8PVJ/nA9M1pmREQy4RoahWB4PZiiiVDEMT1BjVT+t2SobCWI9bRd+BWS8gQargG6eHzAViB9vTGMwWDIhVQW+0QBXghmqbMJkETa1m6IwuBjUCVCLyIwpEfZmtfUjr3FMpk7gIGhzHB58r3/GDDF0EophwOWaPSPDs9S0mfqJt8WEkMCsQoQL6jQfSpvLBqIOIMBnmje3n2XX9JEzGDxHQXxuIaFCj31Eg6ss6CURuxY/kA1FdAT8au60k5bWmCuD0Lr50BfCN4Rk3AKpomC1W1DUZhO0dA79SIwDnEpGKry6itYGv3kpoSEl39ZgBjSTIrFa+agbwVS7wbXb53xXgw1+KnUsBNmFRgLZf2/y0M769dm0B8Mvf+CCk0ACj7wJG3wmoolHXYsTffizER7+U2PcVK2vQ28c2BVOYPRBd01gtVbYHYFsbG+kWCkR9VUsVPxZGIrNPW2jUm1GnM6GoVg+5q6qZTXzbmCLbjH1v6GqB0x/y1bLINGDCw/yM/za2VRebDWZ8fbYSrSYLvjxZjh0/FDo1gAea26oZwA8rAPhVAUi3USDqq5pK+e/qJPvoYNv4oSiVrPPZ7XHD+D3FWqq9+0c0G9tWgDTwparRdwFSxzFBErEINw6Nh4jj8Gt5E7btv4QGvQlqhQQDYoO7KFtsuALDEtXoF+mil07Fj3+iEpF/UCDqqxrbApGmfbcK24jqmI4jql2RKvk2HQCoOOnZ6zHGjw9qqQZkKmDE7W43SEyJDsPc0YmQt82AVyskuG1Mkn0iarCkRIdhzshEjO4f4XxSRSUif6LG6r7KHojae6Js7TBRKg8aXxNGAVXn+HWM0qfzgw87U/wzP/6IE/FBqItG7oFx4UiJGoDGVhMiw2QQB3j9Ia/ZqmYtVTTnzA+oRNQXmQ3tXc/q9hJRdYuLOWbuRA3gg4lRB5Qf7zxtXSFw6Rv+8cCZQIRny7fIJCLEhMsFDUK2aR5Og2/DovmgajbQnDM/oEDUFzWV8Z/iCo19oqvVyuxLecR4UiISifnVGwGg8Ae+4duV1kbgzEcAs/ITZ/uN88MNBIfFyvDSvgt44+Bl53WJROL2TRabqZ2ouygQ9UWNZfz3DkudtpotSNAooFZIoFF6WGNPHMOXigxN7esGdWRrnDbq+KrM4Dm9qgojFnH2sUQu102yjSdqoXai7qJA1Bd17DFrEyaT4K6sZCyeNsCjNYUA8O1CGTfwjwu/BxpK2s9ZzPxGjU3lfOP2yAUu56b1dLbpJC0uAxE1WPsLBaK+yEWPmc/ihgGxQ/iq18l/8FM/aguAY+/wC6mJJMCo/2xfOqOXUcvbxzU5obFEfkO9Zn1NayO/KiMncmiotlqZbztjcBww9BbA2MyXiM5+2n5OIgNG3uHdFJIeRtVZIFK3zanT19JUj26iQNTXNLW1D6liHKpK7/xUCLOVYe7oRMSpvVx4TCIDxizk24kqzwBWMxCRyq+sqIzwX94FYFuStvnaNYkAfjyUQsMH96ZyIDI1yLkLHRSI+hrb/LIODdVmixW1LSZYGbNvLOg1sQRIn8Z/hRBbIGq5dr6ZjTqBApEfUBtRX+Oix6xOxwchuVQU8EXpe5sYNb+bR8q1u3nY2Kq3tpIm8Qn91fUlVovLHrOatjlmMSq55z1mfYTb3TxsbO1ETeXByVCIohJRX9JcwXerSxXtkzbhZtcO4hlbiUhfx28ESXxCgagvqbctlJ/sMLDQtmtqdDj1+rhisTI0tpocFv63kyrbG+SpVOQzCkR9yTU7dtjYS0QqKhG58tb3V/DmvwtQ0eRmcTZ79YzaiXxFgaivYKzD0rDJHQ4z9I9UIl6joKqZG7bF2hr1JtcJNG2BvePIcuIVaqzuK3Q1fBuGWNL+CQ5+25xZni5230dplVIU1+nR4C4Q2UqYDUW0JIiPqETUV9QX8t81/dwuSEZc0yj5gZ9uS0Th8fzgULOBX/iNeI0CUV9RW8B/j0xzONxiMMNipY0uO6NtC0RuS0QiUfsCc7Z2OOIVCkR9gdUC1F3hH0cNcDj1xckyvPrNRVysbA5+vnoJe4nI1TQPG60tEBUHIUehhwJRX9BQzO+aIQvjqxFtGGOobjbCYmXQKKi50B1biaip1eS+9GhvJ6JA5Av66+sLai/z3yPTHRpSmw1mtJosEHEcoqjr3i2VTIxhiWqoFVJYrMz10rWa/nzbW2sDv2WSbTdY4hEKRKGOMX6fesBh/zAAqG4bPxSpkrrdl57wPYtzRnaxdpNExrcT1V/ld8GlQOQV+usLdc2V/Ce0SNK+BVAb24jqGBpR7R9R6fx3W8cA8RgFolBXdZb/Hj3AaeGuisZWABB8j/negDF+mkd1Z1tfR7YFovpCvoOAeIwCUSizWvmlWwEgdpjT6fIGPhAlaLxcCK0POl/RjDf/XYCvz3ayLGx4PD/3zGxsX46XeIQCUSirucjvsCFVAjGDHU4xxjA2OQID48IRp6ESUVciVXzPWU2L0XmPMxuRqH2cVs3F4GQsRFAgCmUlR/jviaOddmLlOA4T0qJwqwBbOfdGkWEycBzQarJAZ+yk2hU7hP9edY7vKCAeoUAUquqL+EGMnAhI6j2bGvZUUrHIPp6o03aiqAy+Y0Bfx29HTTxCgSgUMQYUHOAfJ452uYB9QXWL+ykLxKX4tra0isZOApFE1t57VvVrEHIVGigQhaLyE3yJSCwBUqc4nTZbrPjseCm2f1tg32aadM0WiMrbehvdih3Kf684TdUzD1EgCjUtNcDFf/GP06YBCq1TkrKGVpitDOFyCSLCet/uq0JJ0LYFoga9+wZrgG8nksgAfX37qgekUxSIQolJD5zexXcfa/sD/bNcJiuq1QEA+kcqabF8L8Sp5RiXGonpQ+I6TyiWAnEj+MdlxwOfsRBAgShUmPTA8ff49XDk4cCI+W7XHSqoaQEAJLvbIoe4JBWLkD04FoPj1V0H8MQx/Peqc/wQCtIpCkShQF8H/PI20FTBz7AffQ8gV7tM2qA3obLRAI4DBsSqgpzRPkSTyJdKrRag6JDQuenxKBD1dvVFwJG32ktCY+4FwmPdJr9Uxa871C9CiTAZzXn2ltXKUFynw4+XazpvJwLaOwpKfwGMusBnrhejQNSblR3nq2MmPb8O9bgcILzz9ovLVXy1bGBceDByGHIYgE+Ol+KHSzUorutiH7OoAYA6nl8LquBgUPLXW1Eg6o0sJuDXz4Ffv+CL/rFDgMz7AYWmy6fOG5uE/xgej6EJXaclzsQiDoPj+Grv2bLGzhNzHDBwJv+47BjNP+sEBaLeRlcLHH0LKDvB/6EPyAZG3M731HhAKhZhZD8tlDKa1uGrYUl8ED9f0QSdsZPlYwEgIgWIH8GPJzrzCWDqYgxSH0WBqLdgDCg+DBx+E2iu4hulx9zDt0N40AXfoDPB5GqnUuK1JK0C8RoFTBaGo4X1XT9h0H/w47n0dcCZj/ltv4mDHhGIXn31VaSlpUGhUGDSpEk4dIh6GRw0lAC//A24sJf/I45MBSYsctqRwx2j2YqPj5fg3Z+uoqazeVLEIxzHYdIAfgXGo1frUNnURSlHquSHU4gl/LK9J9+nxutrCB6I/v73v2PlypVYu3Ytjh49ijFjxmD27NmorOxk3Ze+wGoFai4BJ94Hju7gg5FYAgyexfeMuemev1aryYJPj5eiptkIg9lCPWV+MiBGhYy4cFisDJ8eL+u6tKlJAkbdxb+HdVf4km3ZCVpArQ3HuuyDDKxJkyYhKysLr7zyCgDAarUiOTkZ//Vf/4Vnnnmm0+c2NjZCq9WioaEBGk0vbnw1GwGTjl94vaWKb9Ssvcz3hgF81SthVNuUja7v02Sxok5nxJVqHY4V1aHFYIFMIsLtmf2QFKEM8M30HXqjBf88WoystEh7479tMwKZxM1nfFMFXz3T1fA/y8L4taK0/YGwaECuAWSqkNkt1tP/UUE/Ho1GI44cOYLVq1fbj4lEIsycORM//PCDU3qDwQCDob1q0djYRa+FTW0BcGmf6wmI9mPMxbEOx10d6+bzz1c0oUVvgIiZO5zmH3AchzFp8UDCaKDfOOwuMKLo52oA1fbLsQ6vufT69oXxPz5Wap/GAQARYVLMHZWIOFqJ0a+UMjHum5gCUYddPQ5fqcPPV2ohk4ggE4sgEnEQc4BIxIEDcOeEZCgmPASUHMHVY9+grqEUuHxtbxoHq0iCYf2joJApAE6Eq3U6lDcaAHCQSziMSe6wOL9QQWvUXR59MHpC0EBUXV0Ni8WC+Ph4h+Px8fH49VfnJRTWr1+PdevWef9CZgPfwNvDmFp1aDXwDZdWTgyTOAw6aRR00ii0qPpjzNTr7NM09MYSNHW2wV8HGoUEcqkIiVoFBsWpMTRBTbt0BIjomq2Falr4D0qj2Qqj2bm6ZmWM7+FM+Q0utqTiyoUz0LaWQG2shNzcBJlFB/7DyMy3I1n567GWFlia+bYoi0QEtPSAjgfmvzz0qgaD1atXY+XKlfafGxsbkZyc3PUTI5L5Hiagw6dHhz+gTo91OO7qWDeeH9vcCq2FA5OGASIpuLY/ao7jPz07zhXLHhyLyRnR7VezXbLtAWPMPv/pxmHxmDUiAST45o3tB4PZAp3BApPVCouVwWJl9hJvx9Uwx6ZEY1D8ZMcLWC3gzK2A1QSxSgTAAoBBqzNCYjADjEEsAqBRAGDCLjMi898UIUEDUUxMDMRiMSoqKhyOV1RUICHB+R9JLpdDLvdhfWWZqn2xqh4kzov3MdKLDRBdbgBIgkYuEXu0/G6USuZmY0vnjogILRDR/az1WIKW12UyGcaPH499+/bZj1mtVuzbtw+TJ0/u5JmEkFAieNVs5cqVyMnJwYQJEzBx4kRs2bIFLS0teOihh4TOGiEkSAQPRHfffTeqqqqwZs0alJeXY+zYsfjqq6+cGrAJIaFL8HFE3REy44gICVGe/o9Sny4hRHAUiAghgqNARAgRnOCN1d1ha97yeKoHISSobP+bXTVF9+pA1NTE747g0ehqQohgmpqaoNU677Fn06t7zaxWK0pLS6FWe7C9iwBsU1CKiopCslcvlO8vlO8NCN79McbQ1NSEpKQkiETuW4J6dYlIJBKhf//+QmejSxqNJiT/mG1C+f5C+d6A4NxfZyUhG2qsJoQIjgIRIURwFIgCSC6XY+3atb6tGNALhPL9hfK9AT3v/np1YzUhJDRQiYgQIjgKRIQQwVEgIoQIjgIRIURwFIi85O2utO+//z6GDh0KhUKBUaNG4YsvvnA4zxjDmjVrkJiYCKVSiZkzZ+LChQuBvAW3vLm3N954A9OmTUNkZCQiIyMxc+ZMp/S5ubn8RgAdvubMmRPo23DLm/vLz893yrtC4bgdU29976ZPn+50bxzHYe7cufY0QX/vGPHYzp07mUwmY9u3b2enT59mS5YsYREREayiosJl+u+++46JxWL2pz/9iZ05c4b9v//3/5hUKmUnT560p9mwYQPTarXso48+YsePH2e33XYbS09PZ3q9Pli3xRjz/t4WLlzIXn31VfbLL7+ws2fPstzcXKbVallxcbE9TU5ODpszZw4rKyuzf9XW1gbrlhx4e395eXlMo9E45L28vNwhTW9972pqahzu69SpU0wsFrO8vDx7mmC/dxSIvDBx4kS2bNky+88Wi4UlJSWx9evXu0x/1113sblz5zocmzRpEnvkkUcYY4xZrVaWkJDA/vznP9vP19fXM7lczt57770A3IF73t7btcxmM1Or1eytt96yH8vJyWHz5s3zd1Z94u395eXlMa1W6/Z6ofTebd68manVatbc3Gw/Fuz3jqpmHrLtSjtz5kz7sc52pQWAH374wSE9AMyePduevqCgAOXl5Q5ptFotJk2a5PaageDLvV1Lp9PBZDIhKirK4fj+/fsRFxeHIUOG4LHHHkNNTY1f8+4JX++vubkZqampSE5Oxrx583D69Gn7uVB67958803cc889UKkc97cK5ntHgchDne1KW15e7vI55eXlnaa3fffmmoHgy71d6+mnn0ZSUpLDP8ScOXOwY8cO7Nu3Dxs3bsSBAwdw0003wWKx+DX/XfHl/oYMGYLt27fj448/xttvvw2r1YopU6aguLgYQOi8d4cOHcKpU6ewePFih+PBfu969ex70jNs2LABO3fuxP79+x0adO+55x7741GjRmH06NHIyMjA/v37ceONNwqRVY9NnjzZYW+9KVOmYNiwYfjrX/+K559/XsCc+debb76JUaNGYeLEiQ7Hg/3eUYnIQ97uSgsACQkJnaa3fffmmoHgy73ZvPDCC9iwYQP27NmD0aNHd5p2wIABiImJwcWLF7udZ2905/5spFIpMjMz7XkPhfeupaUFO3fuxMMPP9zl6wT6vaNA5CFfdqWdPHmyQ3oA2Lt3rz19eno6EhISHNI0Njbip59+CupOt77uuPunP/0Jzz//PL766itMmDChy9cpLi5GTU0NEhMT/ZJvT/ljR2GLxYKTJ0/a897b3zuAH1piMBhw//33d/k6AX/vgtYsHgJ27tzJ5HI5y8/PZ2fOnGFLly5lERER9m7dBx54gD3zzDP29N999x2TSCTshRdeYGfPnmVr16512X0fERHBPv74Y3bixAk2b948wbqAvbm3DRs2MJlMxv75z386dPE2NTUxxhhrampiq1atYj/88AMrKChg//rXv9i4cePYoEGDWGtra1DvzZf7W7duHdu9eze7dOkSO3LkCLvnnnuYQqFgp0+ftqfpre+dzXXXXcfuvvtup+NCvHcUiLz08ssvs5SUFCaTydjEiRPZjz/+aD+XnZ3NcnJyHNL/4x//YIMHD2YymYyNGDGCff755w7nrVYr+93vfsfi4+OZXC5nN954Izt37lwwbsWJN/eWmprKADh9rV27ljHGmE6nY7NmzWKxsbFMKpWy1NRUtmTJEqexOMHkzf09+eST9rTx8fHs5ptvZkePHnW4Xm997xhj7Ndff2UA2J49e5yuJcR7R8uAEEIER21EhBDBUSAihAiOAhEhRHAUiAghgqNARAgRHAUiQojgKBARQgRHgYgQIjgKRKRXyc/PR0REhP3nZ599FmPHjrX/nJubi/nz5wc9X6R7KBARl2xrFj/66KNO55YtWwaO45Cbm+uQ3t8BIC0tDVu2bHE4dvfdd+P8+fNun7N161bk5+fbf54+fTqefPJJv+aL+B8FIuJWcnIydu7cCb1ebz/W2tqKd999FykpKYLkSalUIi4uzu15rVbrUGIivQMFIuLWuHHjkJycjF27dtmP7dq1CykpKcjMzOzWtV2VVObPn28vZU2fPh2FhYV46qmn7LtIAM5Vs2t1LJnl5ubiwIED2Lp1q/0aBQUFGDhwIF544QWH5x07dgwcxwV9rSTCo0BEOrVo0SLk5eXZf96+fTseeuihgL/url270L9/fzz33HMoKytDWVmZ19fYunUrJk+ejCVLltivkZKS4nRPAJCXl4frr78eAwcO9NctEC9QICKduv/++/Htt9+isLAQhYWF+O677zxaSKu7oqKiIBaLoVarkZCQ4NOqh1qtFjKZDGFhYfZriMVi5Obm4ty5c/a9v0wmE959910sWrTI37dBPERrVpNOxcbGYu7cucjPzwdjDHPnzkVMTIzQ2eqWpKQkzJ07F9u3b8fEiRPx6aefwmAw4M477xQ6a30WlYhIlxYtWoT8/Hy89dZbfis1iEQiXLsUlslk8su1PbF48WJ7Q3xeXh7uvvtuhIWFBe31iSMKRKRLc+bMgdFohMlkwuzZs/1yzdjYWId2H4vFglOnTjmkkclk3d6+xt01br75ZqhUKmzbtg1fffUVVcsERlUz0iWxWIyzZ8/aH7vT0NCAY8eOORyLjo5GcnKyU9obbrgBK1euxOeff46MjAxs2rQJ9fX1DmnS0tJw8OBB3HPPPZDL5T5VCdPS0vDTTz/hypUrCA8PR1RUFEQikb2taPXq1Rg0aFBQF7wnzqhERDyi0Wig0Wg6TbN//35kZmY6fK1bt85l2kWLFiEnJwcPPvggsrOzMWDAAMyYMcMhzXPPPYcrV64gIyMDsbGxPuV71apVEIvFGD58OGJjY3H16lX7uYcffhhGozEovYCkc7RmNemz/v3vf+PGG29EUVGR006pJLgoEJE+x2AwoKqqCjk5OUhISMA777wjdJb6PKqakT7nvffeQ2pqKurr6/GnP/1J6OwQUImIENIDUImIECI4CkSEEMFRICKECI4CESFEcBSICCGCo0BECBEcBSJCiOAoEBFCBPf/AWuRcrhLTGQ4AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:56:27.064880Z", + "iopub.status.busy": "2024-03-22T20:56:27.064578Z", + "iopub.status.idle": "2024-03-22T20:56:27.411710Z", + "shell.execute_reply": "2024-03-22T20:56:27.410735Z" + }, + "papermill": { + "duration": 0.366976, + "end_time": "2024-03-22T20:56:27.413792", + "exception": false, + "start_time": "2024-03-22T20:56:27.046816", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBe0lEQVR4nO2deXgUZbb/v9V7d3rJQlZISAIB2RdZBkER4acsKgxegaszsrnj3EEuM8o4guh1wB3mjpcZGSEiCi6DOuqMwKCAgyAogmyyhGxA9j3dnV6q3t8fla6kk3TS3ekl6ZzP8/TT3VVvVZ1Kd7593vO+7zkcY4yBIAgijMjCbQBBEAQJEUEQYYeEiCCIsENCRBBE2CEhIggi7JAQEQQRdkiICIIIOyREBEGEHUW4DegMgiDg2rVrMBgM4Dgu3OYQBNECxhjq6uqQkpICmcyz39OthejatWtITU0NtxkEQXRAYWEh+vTp43F/txYig8EAQLxJo9EYZmsIgmhJbW0tUlNTpf9VT3RrIXJ1x4xGIwkRQXRhOgqdULCaIIiwQ0JEEETYISEiCCLsdOsYUaTCGIPT6QTP8+E2hSDaRS6XQ6FQdHr6DAlRF8Nut6OoqAgWiyXcphCEV+h0OiQnJ0OlUvl9DhKiLoQgCMjNzYVcLkdKSgpUKhVN1CS6LIwx2O12lJWVITc3F1lZWe1OWmwPEqIuhN1uhyAISE1NhU6nC7c5EYXZ5oTdKSBKrYBKQaHRQKHVaqFUKpGfnw+73Q6NRuPXeegT6YL4+6tCtI3dKaDe5gQDwEAp2gNNIL6v9I0nIp4Ghxj0V8k5qBXyMFtDtAUJERHx2HkBAKCU09e9q0KfDBE2Fi1ahDlz5gT0nDfffDOWL18uvecFBl4Qu2NKuQxOXkB6ejo2bNgQ0OsSnYOEiAgILQWgq+AURG9ILuPAM4YKsx0CY6Byfh3zzDPPYOTIkSG5FgkREdE4eVFwFDIOCpk4FYI1PoiuAwlRN8HuFDw+nI0xEG/aOrxo6yuLFi3CgQMHsHHjRnAcB47jkJOTg6VLlyIjIwNarRYDBw7Exo0b2zx+7dq1iI+Ph9FoxMMPPwy73e7Vdc1mM+677z7o9XokJyfjlVdeadWmtLQU982/C8lxJmRmZuKjD3YCAAShSYo4jsOmTZswY8YMaLVaZGZm4sMPP5T25+XlgeM4vP/++7jxxhuh1WoxduxYXLhwAceOHcOYMWOg1+sxY8YMlJWVef1327JlC4YMGQK1Wo3k5GQ89thj0r6CggLMnj0ber0eRqMR8+bNQ0lJibTf5a28/fbbSE9Ph8lkwoIFC1BXVye1EQQBL774Ivr37w+1Wo20tDQ8//zz0v4nnngCAwYMgE6nQ2ZmJp5++mk4HA4AQHZ2NtauXYuTJ09Kn2l2drbX9+YrNI+om/D6V5c87svoFYU5o3pL7984mAMH3/Zvfp8YLe4e05RMbsuhXFjt7ktJHv9/A3yybePGjbhw4QKGDh2KZ599FgAQExODPn364IMPPkBcXBy++eYbPPjgg0hOTsa8efOkY/ft2weNRoP9+/cjLy8PixcvRlxcnNs/jCd+85vf4MCBA/jkk0+QkJCA3/3udzh+/Lhbd2LZQ/ejpOgavvzyS6hUKjz62K9QUVYGoUXX7Omnn8b69euxceNGvP3221iwYAFOnTqFQYMGSW3WrFmDDRs2IC0tDUuWLME999wDg8GAjRs3QqfTYd68eVi9ejU2bdrUoe2bNm3CihUrsH79esyYMQM1NTU4dOgQAFFAXCJ04MABOJ1OLFu2DPPnz8f+/fulc+Tk5ODjjz/GZ599hqqqKsybNw/r16+X/narVq3C5s2b8dprr2HSpEkoKirCTz/9JB1vMBiQnZ2NlJQUnDp1Cg888AAMBgN++9vfYv78+Th9+jS++OIL/Otf/wIAmEymDu/Lb1g3pqamhgFgNTU14TYlIFitVnb27FlmtVpb7Xt1z3mPj4+OX3Fr+7/7Lnhs+/6xAre2m/ZfatXGHyZPnsx+/etft9tm2bJl7K677pLeL1y4kMXGxjKz2dxkz6ZNTK/XM57n2z1XXV0dU6lU7P3335e2VVRUMK1WK9lx/vx5BoAdPXpUavPdiR8ZALbuxZelbQDYww8/7Hb+8ePHs0ceeYQxxlhubi4DwP76179K+3fs2MEAsH379knb1q1bxwYOHNiu3S5SUlLYU0891ea+PXv2MLlczgoKmj6rM2fOuN3LmjVrmE6nY7W1tVKb3/zmN2z8+PGMMcZqa2uZWq1mmzdv9soexhh76aWX2PXXXy+9X7NmDRsxYkSHx7X3vfX2f5Q8om7Csin9Pe6TtVgF8uBN/Ty2bbliZMnEjM6Y1S6vv/46tmzZgoKCAlitVtjt9lbBzxEjRrjNIp8wYQLq6+tRWFiIvn37ejx3Tk4O7HY7xo8fL22LjY3FwIEDpffnzp2DQqHA9ddfL20bNGgQTKboVsHqCRMmtHp/4sQJt23Dhw+XXicmJgIAhg0b5rattLTUo80uSktLce3aNUydOrXN/efOnUNqaqpbGuTBgwcjOjoa586dw9ixYwEA6enpbpkPk5OTpeufO3cONpvN4zUA4L333sMf//hH5OTkoL6+Hk6nM2wJBilG1E1QKWQeH4oW82Paa9tyLk1bbQLBzp07sXLlSixduhR79uzBiRMnsHjxYq/jP4GAbxwxqzI3XVPeqMSCH9FqpVIpvXatAWy5TRA6jrFptVrfL96BPS2v39E1Dh8+jHvvvRczZ87EZ599hh9++AFPPfVUSD+f5pAQEQFBpVK5pS05dOgQbrjhBjz66KMYNWoU+vfvj5ycnFbHnTx5ElarVXp/5MgR6PX6Dosi9OvXD0qlEt9++620raqqChcuXJDeZw0YCKfTiePHv5e25Vy6iJqa6laCe+TIkVbvm8eHAonBYEB6ejr27dvX5v5BgwahsLAQhYWF0razZ8+iuroagwcP9uoaWVlZ0Gq1Hq/xzTffoG/fvnjqqacwZswYZGVlIT8/361Ny880mFDXjAgI6enp+Pbbb5GXlwe9Xo+srCxs27YNu3fvRkZGBt5++20cO3YMGRnuXUG73Y6lS5fi97//PfLy8rBmzRo89thjHa5f0uv1WLp0KX7zm98gLi4OCQkJeOqpp9yO6581AFOm3YqVv34Mb/zlz1AoFFi+fLm4ULOFZ/jBBx9gzJgxmDRpEt555x0cPXoUb775ZuD+QC145pln8PDDDyMhIQEzZsxAXV0dDh06hF/96leYNm0ahg0bhnvvvRcbNmyA0+nEo48+ismTJ2PMmDFenV+j0eCJJ57Ab3/7W6hUKkycOBFlZWU4c+YMli5diqysLBQUFGDnzp0YO3YsPv/8c3z00Udu50hPT0dubi5OnDiBPn36wGAwQK1WB+PPQR4RERhWrlwJuVyOwYMHIz4+Hrfddhvmzp2L+fPnY/z48aioqMCjjz7a6ripU6ciKysLN910E+bPn48777wTzzzzjFfXfOmll3DjjTfijjvuwLRp0zBp0iS3eBDPgI3/9xckJ6dg8uTJmDt3Lh588EEkJCS0OtfatWuxc+dODB8+HNu2bcOOHTu89j78YeHChdiwYQP+7//+D0OGDMHtt9+OixcvAhC7WJ988gliYmJw0003Ydq0acjMzMR7773n0zWefvpp/Pd//zdWr16NQYMGYf78+VIM6c4778Tjjz+Oxx57DCNHjsQ333yDp59+2u34u+66C9OnT8eUKVMQHx+PHTt2BObm24BjLaN23Yja2lqYTCbU1NRERBWPhoYG5ObmIiMjw+90CkQTtVYHrA4eUWoF9Oom59/JC+AFBoWcg1wmA8dx+OijjwK+3KSn0N731tv/UfKIiIjFtcZM3mKksK7BiWqrA3Znt/0NjjhIiIguSUFBAfR6vcdHQUFBh+dwTVqUtZizIJNxbvuDQXu2f/3110G7bneFgtVElyQlJaXVPJ6W+ztCLuPAWJPwuHC95RuFKBjRifZs7927t8d9PRUSIqJLolAo0L+/50mc3hCtazuZuzSXyJ/JRF7SWdt7GtQ1I3ocoeiaEb5BQkT0OGSSRxRmQwgJ6poREYndyaPG6oRKzsHUoovmihGRR9R1ICEiIhKeiULDs9Z14WQyDnq1AjJODFRT7bjwQ0JERCSChzlEgNg1i1LTV78rQTEiIiJxdbu4ljlSiC4JCRERELpa8vwZt07D00+ulIbqW+LgBdgcvJQqhAgvJERElyEYuXA8OUT1tMyjS0FC1NVhDHDaQ//wYUTJ3+T5rrpmzz//PFJSUqTsit988w1GjhwJjUaDMWPG4OOPPwbHcW6zlU+fPo0ZM2ZAr9cjMTERv/zlL1FeXi6d99DXB7F50+vQqZXgOA55eXlu16aRs64FRey6OrwD+Lp1dYqgc+N/A4q2Zya3pLPJ841GI/bu3QtAXK19xx13YObMmXj33XeRn5/fqstXXV2NW265Bffffz9ee+01WK1WPPHEE5g3bx6+/PJLbNy4EWd/Oo/rBg3G8//zHJRyGeLj493OwdGkxi4FCRHRaUwmE1QqFXQ6HZKSkqTta9eulV5nZGTg8OHDeP/9992EKCoqCn/961+hUomi9+c//xkcx2Hz5s3QaDQYPHgwrl69igceeEA65k9/+hNGjRqFP/zhD9K2LVu2IDU1FRcuXBBL5GjUiDUZkNq77TVpsk6kjCUCDwlRV0euFL2TcFy3k3iTPH/YsGGSCAHA+fPnMXz4cLe8NuPGjXM75uTJk/jqq6+g1+tbXTMnJwcDBnRcDsnVNevG6bgiChKirg7Hed1F6kq4kue/8sormDBhAgwGA1566SW3HNOA6BH5Sn19Pe644w688MILrfYlJyd7dY4mj4iEqCtAQkQEhPaS57toK3l+SwYOHIjt27fDZrNJ+ZGPHTvm1mb06NH429/+hvT0dCgUrb/CTl4AZAqYGzyPwjUFqzs0iQgBNGpGBITmyfPLy8uRlZWF7777Drt378aFCxfw9NNPtxKUtrjnnnsgCAIefPBBnDt3Drt378bLL78MoKmEz7Jly1BZWYn//M//xLFjx5CTk4Pdu3dj8eLF4HkejAGpaX3x/bFjkj0ty/zIZTLoW6SQJcIHCREREPxNnt8So9GITz/9FCdOnMDIkSPx1FNPYfXq1QAgxY1SUlJw6NAh8DyPW2+9FcOGDcPy5csRHR0NmUwGgTE88l/L3expmdFRLhOXeWiU8sD/MQjf8boebZC4cuUKu/fee1lsbCzTaDRs6NCh7NixY14d25NKTvdktm/fzpRKJbNYLF61t9idrLjGyirqbUG2jGAsAkpOV1VVYeLEiZgyZQr++c9/Ij4+HhcvXkRMTEw4zSLCzLZt25CZmYnevXvj5MmT0hwhbyukMilXdfvtHLwAQWBQKmSt8loToSWsQvTCCy8gNTUVW7dulba1LMBH9DyKi4uxevVqFBcXIzk5GXfffTeef/55r493BaA7EpdqiwMCY4iNUkHW1jJ9ImSENUb097//HWPGjMHdd9+NhIQEjBo1Cps3bw6nSUQX4Le//S3y8vKkelmvvfYadDqd18e7PKKOnBxa5tF1CKsQXb58GZs2bUJWVhZ2796NRx55BP/1X/+Ft956q832NpsNtbW1bg+CaAnHcZDLOI8r75u3A3xaVkcEibB2zQRBwJgxY6Sp+qNGjcLp06fx5z//GQsXLmzVft26dW7LBiIVRv8ZncLbYXnyiAJDIL6vYfWIkpOTW9UXHzRokMfieatWrUJNTY30KCwsDIWZIUOpFJdVWCyWMFvSM6D1ZoHB9X11fX/9Iawe0cSJE3H+/Hm3bRcuXEDfvn3bbK9Wq6XZtpGIXC5HdHQ0SktLAQA6nY7yKQcRh90Bh0OAjTmgYJ1fW9fTYIzBYrGgtLQU0dHRkMv9n5MVViF6/PHHccMNN+APf/gD5s2bh6NHj+KNN97AG2+8EU6zwopr9bpLjAjfMducAACtUt6qymtzbE4xS6NSIYOWJjb6TXR0tFvWBX/gWJgDEp999hlWrVqFixcvIiMjAytWrHBL+dAetbW1MJlMqKmpgdFoDLKloYXneTgcjnCb0S3ZfiQPTp7hrtF9YNB69nTK620oqWlAjE6FlBjv5igR7iiVynY9IW//R8O+0Ob222/H7bffHm4zuhxyubxTrm5PxckLqHOIoU+DXtfuEo4+Gg369DKFyjSiHWitGRFR2Jzi4laOA9QK+np3F+iTIiKKBoeYikStkHcY6HfyAopqrCiooFHKcENCREQUDY0ekTfekMXBY+fRQnx84irN3QozJERERGFr9Ii8Se+hUYhteIHBzlN9s3BCQkREFBzHIUanhFHb8TiMUi4uBQGaYktEeAj7qBlBBJKMXlHI6OVdBgeO46BWyGCx87A5BEDT8TFEcCCPiOjRuGJJNiffQUsimJAQET0adWMsyU5ds7BCXTMiojhwoQyFlRaMTY/FwCRDh+2bPCISonBCQkREFNUWO8rqbHB4OQo2tLcJfeOikGCI3MXU3QESIiKiaJCG772LOgxI7NhrIoIPxYiIiMImTWikdXrdCfKIiIhCWuLhpUdktfOottqhlMvQS0/ds3BBHhERMTDG0OAQPSJvCydeKKnDzqOFOHK5IpimER1AQkREDE6BgW/M++rtynuX52Rz0KhZOKGuGRExOHgB0TolHLwAldxLIWqMJdHwfXghISIiBp1KgcUTfSvQqaKZ1V0C6poRPRqa0Ng1ICEiejSSEDkEykkURkiIiIjhYkkdth/Jx78vlnt9jCtGJDAGB09CFC4oRkREDLUNTpTV2RAXpfL6GKWcw/jM2MbUskE0jmgXEiIiYvAlO6MLjuNwQ79ewTKJ8BLqmhERQ4PTt1nVRNeBPjEiYnBNSvR1nVmN1YGiGissdmcwzCK8gISIiBhcHpG3K+9d/OtsCXYeLUQ+lRUKGyRERMTg6zozF9IyD5pLFDZIiIiIQa2QQauS+y5ErmUeDppdHS5o1IyIGOaO7uPXcTS7OvyQR0T0eEiIwg8JEdHjcVXyoIWv4YO6ZkREUNfgwCcnrkGvVmDOqN4dH8AYXFOpm683I8IDCREREVjtPMrqbLDaO/Bqyi8Bl/YCtnogeQTQ7xbEG9QYnxmLGJ33S0OIwEJCREQEUtL89uYQVRcAp/8GsEbP5+r3AOPRa+AMylcdZihGREQEUhkhT7OqeSdw/gtRhOIHAoPuELtm104A1YWhM5RoExIiIiJwTWb06BGVnAYsFYAqChg4E0gaKnbNAAh5h1BptqOoxko5icIECREREbhGvNpcZ8YYcOWY+DrtZ4BS0/QagFB5GTsPnsLOo4VwCiRE4cAvIbp8+XKg7SCITtG0vKONr3TNFcBcDsiVQNLwpu3aGCA2A3IOSDBfaDwPDeGHA7+EqH///pgyZQq2b9+OhoaGQNtEED4j4wCtSg5tW8s7Ss+Jz/EDm7whF4lDwIFDvK0AAGCnSY1hwS8hOn78OIYPH44VK1YgKSkJDz30EI4ePRpo2wjCa27o3wsPT+6H8Zlx7jsEAShrFKKEwa0PjO0HcDIYnRVQO+todnWY8EuIRo4ciY0bN+LatWvYsmULioqKMGnSJAwdOhSvvvoqysrKAm0nQfhH7RXAbhE9oZj01vtVOsDUG3I5h2hrAQlRmOhUsFqhUGDu3Ln44IMP8MILL+DSpUtYuXIlUlNTcd9996GoqChQdhKEf1Q2xjNj+wEyD0P7MRlQyGQw2opomUeY6JQQfffdd3j00UeRnJyMV199FStXrkROTg727t2La9euYfbs2YGykyDa5eMfruKD7wpRZba775CEqJ3Ci9FpkMs4GBuKYOtoZjYRFPyaWf3qq69i69atOH/+PGbOnIlt27Zh5syZkMlEXcvIyEB2djbS09MDaStBeORajVWsTdZ8o60eqCsRX8e0I0SGZMQYtNCp7FAp6wHEBNFSoi38EqJNmzZhyZIlWLRoEZKTk9tsk5CQgDfffLNTxhGENwgCa5avupmTX5UrPhsSAbXe8wnkCiT07gdU5QNCGYDU4BlLtIlfQrR3716kpaVJHpALxhgKCwuRlpYGlUqFhQsXBsRIgmiP5gFmt+yMrqUbbQWpW2LsLQpRHcU1w4FfMaJ+/fqhvLx1Nc3KykpkZLTjAhNEEHBNQlQpZJDLmlVJrLkiPps69nDsukRYHTwsFVeCYSLRAX4Jkaf1OPX19dBoNG3u64j169eD4zgsX77cr+OJnktT9Y5m3pDdLK4tA0RvpwMuNxhxorAa+QX5gNPeYXsisPjUNVuxYgUAsTrm6tWrodPppH08z+Pbb7/FyJEjfTbi2LFj+Mtf/oLhw4d33JggWtDQVnyo5qr4HNVLnCvUAcooI+xyHXjeDtSXANEUJwolPgnRDz/8AED0iE6dOgWVqimRlEqlwogRI7By5UqfDKivr8e9996LzZs343/+5398OpYgAEBgDFqVHDpVM4+opjE+5EW3DBBFzKyKh8l5BagrJiEKMT4J0VdffQUAWLx4MTZu3Aij0dhpA5YtW4ZZs2Zh2rRpJESEX/SL16PfZL17yECKD3lX2UOlkKFeFQ/eXgDUXQuClUR7+DVqtnXr1oBcfOfOnTh+/DiOHTvmVXubzQabzSa9r62tDYgdRGTANeagBu8QvRrAayFSK+Qwq+LENCD1pUGykPCE10I0d+5cZGdnw2g0Yu7cue223bVrV4fnKywsxK9//Wvs3bvX6wD3unXrsHbtWq/aEj2YumIxE6NaD2hMXh2iVshgUcZBYABvroBc4D0vCSECjtdCZDKZpF8ck8m7D7c9vv/+e5SWlmL06NHSNp7ncfDgQfzpT3+CzWaDXO7+RVi1apUUMAdEjyg1lfryPZ1vLpXjWk0DRqZGo3+CvskbMiRLlTo6Qq2QwaGIAi9Tged5yC0VgD4hiFYTzfFaiJp3xwLRNZs6dSpOnTrltm3x4sW47rrr8MQTT7QSIQBQq9VQqynJOeFOWb0NhZUWXJdkEDe4JiUakrw+B8dxuD49FklCKmRcJWAuIyEKIX7FiKxWMbeva/g+Pz8fH330EQYPHoxbb73Vq3MYDAYMHTrUbVtUVBTi4uJabSeI9pAS57uyMzb3iHzgxqx4QMgArlWLQkSEDL8mNM6ePRvbtm0DAFRXV2PcuHF45ZVXMHv2bGzatCmgBhJERzTNI5IDThtgrRR3+OARSUTFi8/1JEShxO8MjTfeeCMA4MMPP0RSUhLy8/Oxbds2/PGPf/TbmP3792PDhg1+H0/0TJo8InljoJoBGqNYscPH81TLouHgBfKIQoxfQmSxWGAwiP3xPXv2YO7cuZDJZPjZz36G/Pz8gBpIEO3BGHNPnC91y3z3hvacLcG2H82oNNuBhhrRuyJCgt/J8z/++GMUFhZi9+7dUlyotLQ0IJMcCcJb7LwAoXEio+gRuQLVvsWHAHHkjJdrYJM3Lgkhryhk+CVEq1evxsqVK5Geno7x48djwoQJAETvaNSoUQE1kCDaw8Ez6FRyqBQyKGRcpzwi11q1BlVjYjRLZaDMJDrAr1Gz//iP/8CkSZNQVFSEESNGSNunTp2Kn//85wEzjiA6Qq9W4KHJ/cAYA+dsAKxV4g6/PCJxyohVYQJQ0bR6nwg6fgkRACQlJSEpyf1XZ9y4cZ02iCD8geOaeUPaaECp9fkcrnLVZnk0IICEKIT4JURmsxnr16/Hvn37UFpaCkFwL8FClWCJsNCJbhnQ1DWrlxsbhYi6ZqHCLyG6//77ceDAAfzyl79EcnJy02JDgggxF0vqcKKwGum9ojC23v9ANdDUNauXNQ64WKsAWnMWEvwSon/+85/4/PPPMXHixEDbQxA+UW114EqVFUatstMeUYxOiZFp0YjRKoE8pbiK31oNRMV1eCzROfwSopiYGMTGxgbaFoLwGddkRh3XOPcHAPT+CVGcXo0pAxvXl5XGiqWIrJUkRCHAr+H75557DqtXr4bFYgm0PQThE9bGgohGZ2M8RxsjlpfuLLpG8aGAdUjwyyN65ZVXkJOTg8TERKSnp0OpVLrtP378eECMI4iOsDZ6RHp74+RDQ6Lf52KMwWLnYXMKiNbEir/SJEQhwS8hmjNnToDNIAj/cHlEOntjeSs/u2WAuETtjYPiiO/Dg2OgBUiIQoRfQrRmzZpA20EQfuHyiLS2RiHqhEckk3FQKWSwOwXYlNFNQsSY1wnWCP/wK0YEiOk//vrXv2LVqlWorBT758ePH8fVq1cDZhxBdAQHQMnsUDs6F6h2IS3zUJhE8XE0AA6KhQYbvzyiH3/8EdOmTYPJZEJeXh4eeOABxMbGYteuXSgoKJByFRFEsFk0MQOsKg84IRPzU3tRw6w91Eo56hqcsDEZoDaKI3GWSp9TihC+4ZdHtGLFCixatAgXL150S3w/c+ZMHDx4MGDGEYQ3cPWl4MB1qlvmwuUR2ZwCoGucokJxoqDjlxAdO3YMDz30UKvtvXv3RnFxcaeNIgifcKX+6GS3DGgmRA4B0DYKkZWWegQbv4RIrVa3WVPswoULiI+P77RRBOENRTVWfPBdIS7lXBI3+DmjujmuZR42J99sLhEJUbDxS4juvPNOPPvss3A4HADElc8FBQV44okncNdddwXUQILwRK3ViWsVtbDVuYbuO981S4vVYWRqNBIMGkDXmJfIlVqECBp+CdErr7yC+vp6xMfHw2q1YvLkyejfvz8MBgOef/75QNtIEG1isTsR5aiAUsYBaoNYULGTDE4xYsp1CUiL0zXrmlUBLTJMEIHFr1Ezk8mEvXv34tChQzh58iTq6+sxevRoTJs2LdD2EYRHrA4eUfZyKOVcQLplrdCYAJkCEJxAQ3VT8JoIOD4LkSAIyM7Oxq5du5CXlweO45CRkYGkpCQxSx5N/CJCRIODR5S9DAqVLCDdMgDgBYYGBw+BMRg0SrF7Vl8mekUkREHDp64ZYwx33nkn7r//fly9ehXDhg3DkCFDkJ+fj0WLFlGaWCKkWOyiR6SQBc4jyi2vxxsHL+Ofp1zZHmkIPxT45BFlZ2fj4MGD2LdvH6ZMmeK278svv8ScOXOwbds23HfffQE1kiDaoqGhAbGOaijlUQHziNxGzQAaOQsRPnlEO3bswO9+97tWIgQAt9xyC5588km88847ATOOINpDayuHDAxyjVEMVgcAtwmNQFN3jOYSBRWfhOjHH3/E9OnTPe6fMWMGTp482WmjCMIbZvVlGJ8RC1NiWsAWpapaChF1zUKCT0JUWVmJxETPLnBiYiKqqmjOBREiaq+C4zjITL0DdkpX18zuFMALrMkjstUDTnvArkO445MQ8TwPhcJzWEkul8PpdHbaKILwCtfSDmNKwE7p6poBohhBqW1aSEvds6DhU7CaMYZFixZBrVa3ud9mo1rhRGgor6hAyeVC6NRKZPhZtaMt3HISOXloVXKxe2a3iAHrYMxXInwTooULF3bYhkbMiFBgLi9EbYMTFlUcMhSqgJ57cIoRYIBC3ugd6WKBmisUJwoiPgnR1q1bg2UHQfiEs+oKAEAIoDfkQqrk4cI1hE9ds6Dhd4ZGgggnQq2YCZQzBC4+5BFp5IyEKFiQEBHdD0EAasVAtTw68ELECwxmm1OqmeaWII2xgF+PICEiuiP1JeAdNvAyFVTGwMyobs5XP5XijYOXcaKwWtygjRHnKfEOwF4f8OsRJEREd6SmEA6eoVadBJ1G2XF7H9EoxblEkkckkwOaaPE1dc+CAgkR0f2oLgAA1GuSoVf7lcmmXTTKxkoejmY5iCh/dVAJ/KdIEMGEMaCmEMN6mzB09ATA0Pacts7g8oikha9AY8A6h0bOggQJEdG9MJeLtcbkCnCG5KAUPnR5RK4qsgCaeUS0hCkYUNeM6F40dstg7CPGboJAqxgRQKvwgwwJEdG9qLyMSosd+0q0+PZycOI1khA5m8WIpPzV1YDAtz6I6BTUNSO6D7wTqM6H1c7jEhLQ1+IIymWiVAoMSjZIggRAzHckV4pD+NZqICouKNfuqZAQER1yqbQO54vrEadX4fq+MVDKw+RI114BeAcs0MCijINRE5yvr1Ylx/ShLZaOcJzYPasrEbtnJEQBhYSIaI3AixP31EacKarFnjMl4vYSIL/CjLtG92laEBpKKi+LT+reAMfBqA38HKJ20TYKkaUCQFZorx3hkBAR7lTlAWc/AewWOHTx+LZ2NAA9+ifocaXKimvVDfipuA5De5tCb1tFDgCgVCEu6zAEySMCmqp5qBSyJg+Q8lcHDRIioglLJXD6b1ImwrJrBcisvYrCfvdg1rBkXK22ghcY0ntFhcc2czkYx+GaLAlgEMv9BIkdRwtQVmfDz0f1brpfGjkLGjRqRjSRe1AUIVMf4GcPo8SugtpZh0nys5DJOKTG6sIjQgBQ9hMAwGZIg4WpwXEIWowIaD5y1nJSI2h2dRAIqxCtW7cOY8eOhcFgQEJCAubMmYPz58+H06Sei7lC+mdH1q2ANgZDb1mAgYl69LGeAxpq3ZvbnCipbQidfaXnAADW6CyYtEqYtMqgxqnandRot4iTKomAEVYhOnDgAJYtW4YjR45g7969cDgcuPXWW2E2m8NpVs+k+Edx+URcP8AgrmhXxvdHbO/+UHIMuHJUanqlyoLsb/Lwj1NFYoL5YGOuAOpLAU6GmNQhWDIpAwsnpAf1klppUmOzuUQKNaDWi6+pexZQwhoj+uKLL9zeZ2dnIyEhAd9//z1uuummMFnVA2EMKDkjvk4a7r4vbQJQXQhcOwGk3wQoVEgwaKCUc6i2OHCisBrX940Jrn1FP4jPsZlSInuZLLilzdvsmgFi98xWL8asApi0v6fTpWJENTU1AIDY2LZrjNtsNtTW1ro9iABQexWw1Ym/+HH9IQgMH3xXiIMXymAz9hXz8fAOoEzsHqkUMtzQrxcA4GhupftSiEDDO4Hi0+LrlFHBu04LXF0zW8t7o1X4QaHLCJEgCFi+fDkmTpyIoUOHttlm3bp1MJlM0iM1NTXEVkYojcPiiM0E5AqU1dtwpcqKU1droJTLgeRGL6noR+mQwclG9DKo0eDgcSRISy0AACWnAYcV0BiB2Ey8/10h3j9WiPL64FaMcdU3c+uaAZS/Okh0GSFatmwZTp8+jZ07d3pss2rVKtTU1EiPwsLCEFoYwVQ2EyIARTViILZ3tFbsAiUNE2cW11wR4zUQu0Y3ZYle0cnCGlSZvS8+6OQF5JWbcaGkDvW2durgCQJQcFh83WcsHAy4Vm3F1Wqr+/KLIBCnV2FQsgGpsTr3HZS/Oih0iXlEjz32GD777DMcPHgQffr08dhOrVZ7rKlG+ImtXpwtDEhCVNHobcS7cv2oDUBsP6DikuihZE4GAPSNi0JGryjklptx4EIZZo9MAedFWg4GYM/ZYphtPOQyDhP7x2F0WkzrY4tOiOu6lFogeSTK621gDIhSy4OSEK05ySYtkk3a1juazyViLChpSHoiYfWIGGN47LHH8NFHH+HLL79ERkZGOM3pmbjSaugTpBGhinrRu4nTN6sXltTYXS457ZZA/sasXlDIOCQY2/+BYM2OUcrFGFMvgxq8wHDwQjm+z2+R58duBnIPiK/TJwEKFcrqWghkONBEA5xMjF3ZKEYZKMIqRMuWLcP27dvx7rvvwmAwoLi4GMXFxbBareE0q2dRI9YHQ3QaAFEwys3iP3xcVLN/+Lj+YjC7obZJvADE6dVYNDEdN/Tr5dEbstid2HX8Ki6W1EnbhvY24Rfj0zCpsXv370vluFrd+LkLAnDuU3Gujj4BSBkNACitbRQivabz9+0FvMBQb3NCaD5FQSYTg/cAdc8CSFiFaNOmTaipqcHNN9+M5ORk6fHee++F06yeRU1jnM0kdonrbU7YHAJkHIcYXbMlFHIlkDBIfF1y2u0UzZdamG1OfHG6GIWVFlSa7ThRWI3tR/JRUGnB/vNlcPJNwV+O4zA2PRaDU4xgDNh7phgOpxM4/w+gMheQK4BBd4r//ACKGidQJnbgfQUCxhg27b+EzQcvo66hRRxLR3GiQBPWGBGjGlHhxWkDzGXi60YhMtt4RKnlUCvkrWcuJw4R5xOVnhNnX8tbr/Xa91Mpckrrca7IvdsSp1dh1rDkNmdDTx4Qj4IKC4SKXBTv+xSpKrPY/bnuDkAfD0Cc4Vze2DVLiW4jdhNgOI6DRimHg3fC4nDChGb3SmvOAk6XCFYTYaL2qhjv0UaLAWkASSYNHrypH+xOoXV7U6rY1loNlF8QhakFN/SLg1ohQ2GlBTangNgoFa5LMmBYb5PHJRkaezXukB0CEy4iSakRg9MDZwHxA6Q2NiePzPgoWO08ooIcqHahUylQ1+CExd7GpEaAPKIAQkLUk3HFh0ytRypVijZEg+NE8ck7JE4ybEOIeunVuG1IknfX5x3iQtsr3yGJCUB0FND7eiB9oihGzYjWqTB7ZG/vzhsgdCpxioC1pRDRpMaAQ0LUk2lHiDySOFQUoqpccTZ2oyflM3UlwJldoncFiMHwfrcAUXFw8gKq6mzhHR2DmKkRQGuPKErsLqKhRuzeKmhKSWfpMhMaiRAjCGLXDBC7XBBjdtmHcvHRD1daewEudLGAqXfj+rSz/l27Mhf44W1RhDRGYPg8YPjdQFQcqsx2vH0kHx//cFWqK1ZUY5XmNoUSnSRELYLVSm2TALtibESnICHqqZjLxLkwCpW0bKHW6kSVxYHCSivUbXXNXCQ2zilyrdj3BUslcOYjsVsWkw6MWSqu+G/ElXWx3ubEl+dK0eDg8c9Txdh2OL9VADzYeOyaAeK0AkDMCkB0GhKinkrdNfHZkCLNDnbNH4qNUrW/uj1hkFhTzFzu2z+i096YAdImelXD5wFK9zlBCrkMU69LhIzj8FNxHTbtz0GN1QGDRoHM+NAmZYvXazAo2YDeMW2M0kWJ85/IIwoMJEQ9ldpGITI2Vatwzaju1XxGdVsotWJMBwBKTnl3PcbE+UHmckAVBQz5uccCiWlxOswangx14wp4g0aBO0ekSAtRQ0VanA7ThyZjeJ/o1jujyCMKJBSs7qlIQtQ0EuWKw8RGeRF8TRoGlJ0X8xhl3CxOPmyPK8fE+UecTBShDoLc/RP0SIvNRG2DAzE6FeRBzj/kM66umbmM1pwFAPKIeiJOW9PQs6HJIyo3t7HGzBOxmaKY2C1A8cn221blAzlfia/7TwOivUvfolLI0EuvDqsIuZZ5tJp8q4sTRdVpozVnAYCEqCdSVyT+imuM0kJXQWBSKo9e3nhEMrmYvREA8g+Lge+2aKgFzn4MMEFcONt7dABuIDTwAsMf913E5oOXW+clksmbiizWU5yos5AQ9URqi8TnZqlOG5w8kowaGDQKGLVe9tiTR4heka2uKW9Qc1zBabtF7MoMmN6tujByGSfNJWozb5JrPpGZ4kSdhYSoJ9J8xKwRnUqBeWNTcf+NmV7lFAIgxoX63SK+zv8GqLnatI93ioUa64rF4PbQuW2uTevquJaTmNsUIgpYBwoSop5IGyNmfpMwCIgfKHa9Tr0vLv2ozAVOvCMmUpMpgGH/0ZQ6o5thUDfNa2oFzSUKGDRq1tNoqBWzMnIyt0C1IDD/KmNwHHDd7YC9XvSIzn3atE+hAobe5dsSki5GVHtCZGhcU2etpKUenYSEqKdR1xgfiurl1lV659t8OAWGWcOTkWDwMfGYQgWMuEeME5WeBQQnEN1XzKyojQ6c7WHAlZK2vmVOIkCcD6UxiuJeVwzE9A2xdZEDCVFPw7W+rFmg2skLqDQ7IDAmFRb0GbkCyLhRfEQQLiEyt1xv5sKQREIUAChG1NNoY8SsyiKKkFopC3pS+u5GL4NYzSOtZTUPF67urcvTJPyCvnU9CYFvc8SsonGNWa8otfcjZj0Ej9U8XLjiRHXFoTEoQiGPqCdRXyIOqys1TYs24aFqB+EdLo/IWiUWgiT8goSoJ1HtSpSf6jax0FU1NU5Poz5twQsMtQ0Ot8T/EkptU0CevCK/ISHqSbSo2OFC8oiiyCNqi7e+ycObX+eipM5Dcjape0ZxIn8hIeopMNYsNWxqs80MfWK0SDRqqGvmAVeytlqro+0GxkZhbz6znPAJClb3FCwVYgxDrmj6BYdYNudWb5Pd91BMWiWuVFlR40mIXB5mTSGlBPET8oh6CtX54rOxt8eEZETbGLXixE+PHpE+UZwc6rSJid8InyEh6ilU5orPMelum802J3iBCl22h6lRiDx6RDJZU4I5VxyO8AkSop6AwANVeeLr2Ey3Xf84VYTXv7qES6X1obermyB5RG0t83BhcgnRlRBYFHmQEPUEaq6IVTNUOrEb0QhjDOX1dvACg1FD4UJPuDyiugaHZ+9RihOREPkDfft6ApWXxeeYDLdAar3NiQYHDxnHIZaG7j0SpZJjULIBBo0SvMDaTl1r7CPG3hpqxJJJrmqwhFeQEEU6jIl16gG3+mEAUN44fygmSumxLj0hjixOH9pB7iaFSowTVReIVXBJiHyCvn2RTn2p+AstUzSVAGrENaO6F82oDgyxGeKza2CA8BoSokin7Jz4HJfZKnFXSW0DAIS9xnx3gDFxmUd5e6WvYxqFqDpfHCAgvIaEKJIRBDF1KwDED2q1u7hGFKIko4+J0HogF0rq8ebXufjyXDtpYfWJ4tozp70pHS/hFSREkUzFJbHChlIL9BrgtosxhpGp0eifoEeCkTyijoiJEkfOKsz21jXOXMhkTfO0Ki6FxrAIgYQokrn6vficPLxVJVaO4zAmPRZ3hKGUc3ckRqcCxwENDh4WezvdrviB4nPZeXGggPAKEqJIpbpQnMTIyYCU7lPUsKuilMuk+UTtxoli+4kDA9YqsRw14RUkRJEIY0DuAfF18vA2E9jnlps9L1kg2iSxMZZWUtuOEClUTaNnZT+FwKrIgIQoEin+UfSI5Aqg7w2tdjt5AZ+dvIYt/86VykwTHeMSouLG0UaPxF8nPpecoe6Zl5AQRRrmCuDSv8TX6TcCGlOrJkU1DXAKDHq1AtG67ld9NVwkmRqFqMbqOWANiHEihQqwVjdlPSDahYQoknBYgTO7xOFjUx+gz9g2mxVWWgAAfWK0lCzfBxIMaozuG4ObBya031CuBBKGiK+LTgbfsAiAhChScFiBkzvEfDhqPTBkjse8Q7kVZgBAqqcSOUSbKOUyTB4QjwGJho4FPHmE+Fx2XpxCQbQLCVEkYK0CftgO1JWIK+yHLwDUhjab1lgdKK21geOAzPioEBvagzAmi16pwAOFR8NtTZeHhKi7U10IfP9Wkyc04j8BfbzH5jllYt6h3tFa6FS05tlXBIHhSpUFRy5XtB8nApoGCq79ANgtwTeuG0NC1J0pOil2xxxWMQ/16IWAvv34xeUysVvWP0EfCgsjDgbg7yev4XBOBa5UdVDHLDYTMCSKuaByD4bEvu4KCVF3hHcAP30O/PQP0fWPHwiM+gWgMXZ46OyRKfh/gxNxXVLHbYnWyGUcBiSI3d5zRbXtN+Y4oP808XXRCVp/1g4kRN0NSyVw/C2g6Efxi545GRjyc3GkxguUchmG9jZBq6JlHf4yKEUU8QsldbDY20kfCwDRaUDiEHE+0dm/A44O5iD1UEiIuguMAVe+A757E6gvE4PSIxaIcQgvhuBrLA442qpUSvhMikmDRKMGDp7heH51xwdk/T9xPpe1Cjj7iVj2m3CjSwjR66+/jvT0dGg0GowfPx5Hj9Iogxs1V4Ef3gYu7hW/xDF9gTFLWlXk8ITdKeCTk1fx7rcFqGhvnRThFRzHYXymmIHxeEEVSus68HKUWnE6hVwhpu099QEFr1sQdiF67733sGLFCqxZswbHjx/HiBEjcNttt6G0tJ28Lz0BQQAqcoAfPwCObxPFSK4ABtwqjox5GJ5vSYODx6cnr6Gi3g6bk6eRsgCR2SsK/RL04AWGT08WdextGlOAYfPEz7AqT/Rsi36kBGqNcKzDMcjgMn78eIwdOxZ/+tOfAACCICA1NRW/+tWv8OSTT7Z7bG1tLUwmE2pqamA0duPgq9MOOCxi4nVzmRjUrLwsjoYBYtcraVjjko2O79PBC6iy2JFXbsGJwiqYbTxUChl+Pqo3UqK1Qb6ZnoPVzuPD41cwNj1GCv67ihGoFB5+4+tKxO6ZpUJ8r9KJuaJMfQBdHKA2AqqoiKkW6+3/aFh/Hu12O77//nusWrVK2iaTyTBt2jQcPny4VXubzQabralrUVvbwaiFi8pcIGdf2wsQpW2sjW3Ntre1rZPHXyipg9lqg4w5m+0WX3AchxHpiUDScKD3aOzOtaPwWDmAcul0rNk1H7ypKTH+JyeuScs4ACBap8SsYclIoEyMAUWrkuPecWmQNavq8V1eFY7lVUKlkEEll0Em4yDnAJmMAwfg7jGp0IxZDFz9HgUnvkJVzTXgcsvRNA6CTIFBfWKhUWkAToaCKguKa20AOKgVHEakNkvOHy7RGjbPqx9GbwirEJWXl4PneSQmJrptT0xMxE8/tU6hsG7dOqxdu9b3CzltYoC3i+FosKDBJgYuBU4Oh1wHizIWFmUszFF9MGLiJGmZhtV+FXXtFfhrhlGjgFopQ7JJg6wEA65LMlCVjiAha1FaqMIs/lDanQLsztbdNYExcYQz7We4ZO6LvItnYWq4CoO9FGpnHVS8BeKPkVOMIwni+ZjZDL5ejEXxChlg7gIDDyxwNnSrgMGqVauwYsUK6X1tbS1SU1M7PjA6VRxhApr9ejT7ArW7rdn2trZ14vj4+gaYeA5MqQNkSnCNX2qOE389m68VmzwgHhP6xTWdzXXKxheMMWn909RBibh1SBKI0DN7ZG/YnDwsNh4OQQAvMPACkzze5tkwR6bFIStxgvsJBB6cswEQHJBHyQDwABhMFjsUNifAGOQyAEYNABbeNCOqwC0RCqsQ9erVC3K5HCUlJW7bS0pKkJTU+h9JrVZDrfYjv7IqqilZVRciwYfPMcaHAohtFgAkQoZaIfcq/W5slMpDYcvWAxHRJiC686Z1WcLqr6tUKlx//fXYt2+ftE0QBOzbtw8TJkxo50iCICKJsHfNVqxYgYULF2LMmDEYN24cNmzYALPZjMWLF4fbNIIgQkTYhWj+/PkoKyvD6tWrUVxcjJEjR+KLL75oFcAmCCJyCfs8os4QMfOICCJC8fZ/lMZ0CYIIOyREBEGEHRIigiDCTtiD1Z3BFd7yeqkHQRAhxfW/2VEoulsLUV2dWB3Bq9nVBEGEjbq6OphMrWvsuejWo2aCIODatWswGLwo7xIGXEtQCgsLI3JUL5LvL5LvDQjd/THGUFdXh5SUFMhkniNB3dojkslk6NOnT7jN6BCj0RiRX2YXkXx/kXxvQGjurz1PyAUFqwmCCDskRARBhB0SoiCiVquxZs0a/zIGdAMi+f4i+d6Arnd/3TpYTRBEZEAeEUEQYYeEiCCIsENCRBBE2CEhIggi7JAQ+YivVWk/+OADXHfdddBoNBg2bBj+8Y9/uO1njGH16tVITk6GVqvFtGnTcPHixWDegkd8ubfNmzfjxhtvRExMDGJiYjBt2rRW7RctWiQWAmj2mD59erBvwyO+3F92dnYr2zUa93JM3fWzu/nmm1vdG8dxmDVrltQm5J8dI7xm586dTKVSsS1btrAzZ86wBx54gEVHR7OSkpI22x86dIjJ5XL24osvsrNnz7Lf//73TKlUslOnTklt1q9fz0wmE/v444/ZyZMn2Z133skyMjKY1WoN1W0xxny/t3vuuYe9/vrr7IcffmDnzp1jixYtYiaTiV25ckVqs3DhQjZ9+nRWVFQkPSorK0N1S274en9bt25lRqPRzfbi4mK3Nt31s6uoqHC7r9OnTzO5XM62bt0qtQn1Z0dC5APjxo1jy5Ytk97zPM9SUlLYunXr2mw/b948NmvWLLdt48ePZw899BBjjDFBEFhSUhJ76aWXpP3V1dVMrVazHTt2BOEOPOPrvbXE6XQyg8HA3nrrLWnbwoUL2ezZswNtql/4en9bt25lJpPJ4/ki6bN77bXXmMFgYPX19dK2UH921DXzEldV2mnTpknb2qtKCwCHDx92aw8At912m9Q+NzcXxcXFbm1MJhPGjx/v8ZzBwJ97a4nFYoHD4UBsbKzb9v379yMhIQEDBw7EI488goqKioDa7g3+3l99fT369u2L1NRUzJ49G2fOnJH2RdJn9+abb2LBggWIinKvbxXKz46EyEvaq0pbXFzc5jHFxcXttnc9+3LOYODPvbXkiSeeQEpKits/xPTp07Ft2zbs27cPL7zwAg4cOIAZM2aA5/mA2t8R/tzfwIEDsWXLFnzyySfYvn07BEHADTfcgCtXrgCInM/u6NGjOH36NO6//3637aH+7Lr16nuia7B+/Xrs3LkT+/fvdwvoLliwQHo9bNgwDB8+HP369cP+/fsxderUcJjqNRMmTHCrrXfDDTdg0KBB+Mtf/oLnnnsujJYFljfffBPDhg3DuHHj3LaH+rMjj8hLfK1KCwBJSUnttnc9+3LOYODPvbl4+eWXsX79euzZswfDhw9vt21mZiZ69eqFS5cuddpmX+jM/blQKpUYNWqUZHskfHZmsxk7d+7E0qVLO7xOsD87EiIv8acq7YQJE9zaA8DevXul9hkZGUhKSnJrU1tbi2+//TaklW79rbj74osv4rnnnsMXX3yBMWPGdHidK1euoKKiAsnJyQGx21sCUVGY53mcOnVKsr27f3aAOLXEZrPhF7/4RYfXCfpnF7KweASwc+dOplarWXZ2Njt79ix78MEHWXR0tDSs+8tf/pI9+eSTUvtDhw4xhULBXn75ZXbu3Dm2Zs2aNofvo6Oj2SeffMJ+/PFHNnv27LANAftyb+vXr2cqlYp9+OGHbkO8dXV1jDHG6urq2MqVK9nhw4dZbm4u+9e//sVGjx7NsrKyWENDQ0jvzZ/7W7t2Ldu9ezfLyclh33//PVuwYAHTaDTszJkzUpvu+tm5mDRpEps/f36r7eH47EiIfOR///d/WVpaGlOpVGzcuHHsyJEj0r7JkyezhQsXurV///332YABA5hKpWJDhgxhn3/+udt+QRDY008/zRITE5larWZTp05l58+fD8WttMKXe+vbty8D0OqxZs0axhhjFouF3XrrrSw+Pp4plUrWt29f9sADD7SaixNKfLm/5cuXS20TExPZzJkz2fHjx93O110/O8YY++mnnxgAtmfPnlbnCsdnR2lACIIIOxQjIggi7JAQEQQRdkiICIIIOyREBEGEHRIigiDCDgkRQRBhh4SIIIiwQ0JEdCuys7MRHR0tvX/mmWcwcuRI6f2iRYswZ86ckNtFdA4SIqJNXKlCH3744Vb7li1bBo7jsGjRIrf2gRaA9PR0bNiwwW3b/PnzceHCBY/HbNy4EdnZ2dL7m2++GcuXLw+oXUTgISEiPJKamoqdO3fCarVK2xoaGvDuu+8iLS0tLDZptVokJCR43G8ymdw8JqJ7QEJEeGT06NFITU3Frl27pG27du1CWloaRo0a1alzt+WpzJkzR/Kybr75ZuTn5+Pxxx+XkrcDrbtmLWnumS1atAgHDhzAxo0bpXPk5uaif//+ePnll92OO3HiBDiOC3mKEkKEhIholyVLlmDr1q3S+y1btmDx4sVBv+6uXbvQp08fPPvssygqKkJRUZHP59i4cSMmTJiABx54QDpHWlpaq3sCgK1bt+Kmm25C//79A3ULhA+QEBHt8otf/AL//ve/kZ+fj/z8fBw6dMir/DWdJTY2FnK5HAaDAUlJSX4lGzOZTFCpVNDpdNI55HI5Fi1ahPPnz0sldxwOB959910sWbIk0LdBeAmliiXaJT4+HrNmzUJ2djYYY5g1axZ69eoVbrM6RUpKCmbNmoUtW7Zg3Lhx+PTTT2Gz2XD33XeH27QeC3lERIcsWbIE2dnZeOuttwLmNchkMrTMQONwOAJybm+4//77pUD81q1bMX/+fOh0upBdn3CHhIjokOnTp8Nut8PhcOC2224LyDnj4+Pd4j48z+P06dNubVQqVaerRng6x8yZMxEVFYVNmzbhiy++oG5ZmKGuGdEhcrkc586dk157oqamBidOnHDbFhcXh9TU1FZtb7nlFqxYsQKff/45+vXrh1dffRXV1dVubdLT03Hw4EEsWLAAarXary5heno6vv32W+Tl5UGv1yM2NhYymUyKFa1atQpZWVkhzTNNtIY8IsIrjEYjjEZju23279+PUaNGuT3Wrl3bZtslS5Zg4cKFuO+++zB58mRkZmZiypQpbm2effZZ5OXloV+/foiPj/fL7pUrV0Iul2Pw4MGIj49HQUGBtG/p0qWw2+0hGQUk2odSxRI9lq+//hpTp05FYWFhqwKFRGghISJ6HDabDWVlZVi4cCGSkpLwzjvvhNukHg91zYgex44dO9C3b19UV1fjxRdfDLc5BMgjIgiiC0AeEUEQYYeEiCCIsENCRBBE2CEhIggi7JAQEQQRdkiICIIIOyREBEGEHRIigiDCDgkRQRBh5/8DlspEo7RWUAAAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:56:27.451530Z", + "iopub.status.busy": "2024-03-22T20:56:27.451208Z", + "iopub.status.idle": "2024-03-22T20:56:27.636663Z", + "shell.execute_reply": "2024-03-22T20:56:27.635658Z" + }, + "papermill": { + "duration": 0.207051, + "end_time": "2024-03-22T20:56:27.639085", + "exception": false, + "start_time": "2024-03-22T20:56:27.432034", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArJ0lEQVR4nO3de1xUdf4/8BfXgXFARBSQwFFJ8ZKgkIibX3UTycrVtVW2LJBcv5vKfq1Z02XbILKiixe0JfmuhaZdpFq7fMu8hNKqoSikoQKuiEIKKCpy02GY+fz+8MdpxxlgUGA48Ho+Hjz0fM7nnHkfzvjynM+cOcdGCCFARCQzttYugIjoTjC8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWrB5eKSkpUKvVcHJyQmhoKLKzs1vsX1VVhSVLlsDb2xsKhQJDhw7Fjh07OqlaIuoq7K354unp6dBoNEhNTUVoaCiSk5MRERGBwsJC9O/f36R/Q0MDwsPD0b9/f3z22Wfw8fHB+fPn4ebmZvFrGgwGXLx4ES4uLrCxsWnHrSGi9iCEQE1NDQYMGABb2xaOr4QVjRs3TixZskSa1uv1YsCAASIpKcls/w0bNojBgweLhoaGO37N0tJSAYA//OFPF/8pLS1t8d+yjRDWuatEQ0MDlEolPvvsM8yaNUtqj46ORlVVFb788kuTZR5++GG4u7tDqVTiyy+/RL9+/fDEE09gxYoVsLOzM/s6Wq0WWq1Wmr5+/Tr8/PxQXFwMFxeXdt+urkKn02Hfvn2YMmUKHBwcrF0OtYOesk9ramowaNAgVFVVoXfv3s32s9ppY2VlJfR6PTw9PY3aPT09UVBQYHaZs2fPYu/evZg3bx527NiBM2fOYPHixdDpdEhISDC7TFJSEhITE03as7KyoFQq735DujClUonDhw9buwxqRz1hn9bX1wNAq8M6Vh3zaiuDwYD+/fvjH//4B+zs7BAcHIwLFy7grbfeaja84uLioNFopOnq6mr4+vpi2rRpcHV17azSO51Op8OePXsQHh7erf+X7kl6yj6trq62qJ/VwsvDwwN2dnaoqKgwaq+oqICXl5fZZby9veHg4GB0ijh8+HCUl5ejoaEBjo6OJssoFAooFAqTdgcHh279BmjSU7azJ+nu+9TSbbPapRKOjo4IDg5GRkaG1GYwGJCRkYGwsDCzy/zqV7/CmTNnYDAYpLbTp0/D29vbbHARUfdl1eu8NBoNNm7ciPfffx/5+flYtGgR6urqEBMTAwCIiopCXFyc1H/RokW4evUqli5ditOnT+Obb77Ba6+9hiVLllhrE4jISqw65hUZGYnLly8jPj4e5eXlCAoKws6dO6VB/JKSEqPrPHx9fbFr1y4899xzGD16NHx8fLB06VKsWLHCWptARFZi9QH72NhYxMbGmp2XmZlp0hYWFoZDhw51cFVE1NVZPbyIyFR9fb3JJUO1N7T4Ia8IfTyOQuX8y4dQAQEB3f6yH3MYXkRdUEFBAYKDg83Oe/O26ZycHIwdO7bji+piGF5EXVBAQABycnKM2grLqqD5NA9r5tyHYd5uRn17IoYXURekVCpNjqZsz1+BYv8NDB8ViKCBfa1UWddh9VviEBHdCYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEcmSvbULoPZRX1+PgoICabr2hhY/5BWhj8dRqJwVRn0DAgKgVCo7u0SidsXw6iYKCgoQHBxs0v6mmb45OTkYO3ZsxxdF1IEYXt1EQEAAcnJypOnCsipoPs3Dmjn3YZi3m0lfIrnrEmNeKSkpUKvVcHJyQmhoKLKzs5vtu3nzZtjY2Bj9ODk5dWK1XZNSqcTYsWOln+GjAqHw8sfwUYFG7WPHjuUpI3ULVg+v9PR0aDQaJCQkIDc3F4GBgYiIiMClS5eaXcbV1RVlZWXSz/nz5zuxYiLqCqweXmvWrMHChQsRExODESNGIDU1FUqlEmlpac0uY2NjAy8vL+nH09OzEysmoq7AqmNeDQ0NyMnJQVxcnNRma2uLqVOnIisrq9nlamtrMXDgQBgMBowdOxavvfYaRo4cabavVquFVquVpqurqwEAOp0OOp2unbak62lsbJT+7M7b2ZP0lH1q6bZZNbwqKyuh1+tNjpw8PT2NPvb/T8OGDUNaWhpGjx6N69evY9WqVZgwYQJOnjyJe+65x6R/UlISEhMTTdp3797drcd+SmsBwB6HDh3ChRPWrobaQ0/Zp/X19Rb1k92njWFhYQgLC5OmJ0yYgOHDh+N///d/sXLlSpP+cXFx0Gg00nR1dTV8fX0xbdo0uLq6dkrN1nC85CqQdxTjx49HoJ+7tcuhdtBT9mnT2VFrrBpeHh4esLOzQ0VFhVF7RUUFvLy8LFqHg4MDxowZgzNnzpidr1AooFAoTNodHBzg4ODQ9qJlwt7eXvqzO29nT9JT9qml22bVAXtHR0cEBwcjIyNDajMYDMjIyDA6umqJXq9HXl4evL29O6pMIuqCrH7aqNFoEB0djZCQEIwbNw7Jycmoq6tDTEwMACAqKgo+Pj5ISkoCALz88ssYP348/P39UVVVhbfeegvnz5/HH/7wB2tuBhF1MquHV2RkJC5fvoz4+HiUl5cjKCgIO3fulAbxS0pKYGv7ywHitWvXsHDhQpSXl6NPnz4IDg7GDz/8gBEjRlhrE4jICqweXgAQGxuL2NhYs/MyMzONpteuXYu1a9d2QlVE1JVZ/SJVIqI7wfAiIllieBGRLDG8iEiWusSAPREBxZV1qNM2Nju/6HKd9GfTBau366WwxyCPXh1SX1fD8CLqAoor6zBlVaZFff/8WV6L8/ctm9wjAozhRdQFNB1xJUcGwb+/ynyfG1p8nZmFRyeHoZez6VfezlyqxbPpx1o8eutOGF5EXYh/fxVG+fQ2O0+n06G8HzB2YJ9u/d1GS3HAnohkieFFRLLE8CIiWWJ4EZEsMbyISJb4aaOMtXRRoyUXNAI966JG6l4YXjJl6UWNrV3QCPScixqpe2F4yVRrFzW2dkEj0PMuaqTuheElc81d1MgLGqm744A9EckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJV6kKlNa/U3YOl1AcXUhbJ1Mr7BvbGzExcaLyL+a3+x3G4ura2HrdAFa/U0A5u/eSdRVMbxk6mLdefQa9Db+mt1yv3d2vtPi/F6DgIt1QQiGZztWR9TxGF4yNaDXQNQV/wnrIoMwxMx3GxsbG3HwwEH86oFfNXvkVXSpFkvTj2HAlIEdXS5Ru2N4yZTCzgmGmz4Y5DoMI/qa/25jsX0xhrsPb/a7jYab12G4eRkKO6eOLpda0dowAND6UEBPGwZgeBF1AZYOAwAtDwX0pGEAhhdRF9DaMADQ+lBATxsGYHgRdQGtDQMArQ8F9LRhgDZf53X27NmOqIOIqE3aHF7+/v6YMmUKPvjgA9y8ebNdikhJSYFarYaTkxNCQ0ORnW3BiT+Abdu2wcbGBrNmzWqXOohIPtocXrm5uRg9ejQ0Gg28vLzwxz/+0eKwMSc9PR0ajQYJCQnIzc1FYGAgIiIicOnSpRaXO3fuHJYtW4aJEyfe8WsTkXy1ObyCgoKwbt06XLx4EWlpaSgrK8MDDzyAUaNGYc2aNbh8+XKb1rdmzRosXLgQMTExGDFiBFJTU6FUKpGWltbsMnq9HvPmzUNiYiIGDx7c1k0gom7gjgfs7e3tMXv2bDzyyCN45513EBcXh2XLluGvf/0r5s6dizfeeAPe3t4trqOhoQE5OTmIi4uT2mxtbTF16lRkZWU1u9zLL7+M/v37Y8GCBdi/f3+Lr6HVaqHVaqXp6upqALcGP3U6nSWb2iU1NjZKf5rbjqa2lraxtXVQ57FkX7S2T7vL/rS09jsOr6NHjyItLQ3btm1Dr169sGzZMixYsAA///wzEhMTMXPmzFZPJysrK6HX6+HpaXxNiqenJwoKCswuc+DAAbz33ns4duyYRXUmJSUhMTHRpH337t1QKpUWraMrKq0FAHscOHAA581/sg4A2LNnz12vgzpeW/ZFc/u0u+zP+vp6i/q1ObzWrFmDTZs2obCwEA8//DC2bNmChx9+GLa2t85ABw0ahM2bN0OtVrd11a2qqanBU089hY0bN8LDw8OiZeLi4qDRaKTp6upq+Pr6Ytq0aXB1dW33GjvLyYvVWJV3CA888ABGDjDdDp1Ohz179iA8PLzZK+xbWwd1Hkv2RWv7tLvsz6azo9a0Obw2bNiAp59+GvPnz2/2tLB///547733Wl2Xh4cH7OzsUFFRYdReUVEBLy8vk/5FRUU4d+4cZsyYIbUZDAYAt05jCwsLMWTIEKNlFAoFFArT5xY6ODjI+pFgTRcp2tvbt7gdLW2npeugjteWfdHcPu0u+9PS2tscXnv27IGfn590pNVECIHS0lL4+fnB0dER0dHRra7L0dERwcHByMjIkC53MBgMyMjIQGxsrEn/gIAA5OUZPwH6b3/7G2pqarBu3Tr4+vq2dXOISKbaHF5DhgxBWVkZ+vfvb9R+9epVDBo0CHq9vk3r02g0iI6ORkhICMaNG4fk5GTU1dUhJiYGABAVFQUfHx8kJSXByckJo0aNMlrezc0NAEzaiah7a3N4CSHMttfW1sLJqe1fS4iMjMTly5cRHx+P8vJyBAUFYefOndIgfklJiclRHhGRxeHVNOhtY2OD+Ph4o0/q9Ho9Dh8+jKCgoDsqIjY21uxpIgBkZma2uOzmzZvv6DWJSN4sDq8ff/wRwK0jr7y8PDg6OkrzHB0dERgYiGXLlrV/hUREZlgcXvv27QMAxMTEYN26dbK+zICI5K/NY16bNm3qiDqIiNrEovCaPXs2Nm/eDFdXV8yePbvFvtu3b2+XwoiIWmJRePXu3Rs2NjbS34mIrM2i8PrPU0WeNhJRV8ALqIhIliw68hozZox02tia3NzcuyqIiMgSFoUXb7NMRF2NReGVkJDQ0XUQEbUJx7yISJYsOvJyd3fH6dOn4eHhgT59+rQ4/nX16tV2K46IqDkWhdfatWvh4uIi/d3SwXsioo5iUXj9540F58+f31G1EBFZrM1jXnZ2dmafqXjlyhXY2dm1S1FERK1pc3g1dzNCrVZrdJscIqKOZPFdJdavXw/g1s0I3333XahUvzxbSa/X41//+hcCAgLav0IiIjMsDq+1a9cCuHXklZqaanSK6OjoCLVajdTU1PavkIjIDIvDq7i4GAAwZcoUbN++HX369OmwooiIWtPmmxE23VGViMia2hxeTz/9dIvz09LS7rgYIiJLtTm8rl27ZjSt0+lw4sQJVFVV4de//nW7FUZE1JI2h9fnn39u0mYwGLBo0SIMGTKkXYoiImpNu3wx29bWFhqNRvpEkoioo7XbXSWKiorQ2NjYXqsjImpRm08bm56c3UQIgbKyMnzzzTdG34EkIupIbQ6vpidnN7G1tUW/fv2wevXqVj+JJCJqL7zOi4hkiXdSJSJZavORF3UNN3R6AMCJC9fNzq+7ocXRy4DX+Wvo5aww2+fMpdoOq4+oozG8ZKro/wfPX7bntdDLHlvPHGl1Xb0UfBuQ/PBdK1PTRnoBAIb0V8HZwfQmkIVl1/Hnz/Kw+nf3YZh372bX00thj0EevTqsTqKO0m7h9fPPP+Pll1/GP/7xj/ZaJbXAvZcjfj/Or9n5TdfcDenXC6N8mg8vIrlqtwH7K1eu4L333rujZVNSUqBWq+Hk5ITQ0FBkZ2c323f79u0ICQmBm5sbevXqhaCgIGzduvVOyyYimbL6p43p6enQaDRISEhAbm4uAgMDERERYfY++cCtx7C98MILyMrKwk8//YSYmBjExMRg165dnVw5EVmT1cNrzZo1WLhwIWJiYjBixAikpqZCqVQ2e2udyZMn47e//S2GDx+OIUOGYOnSpRg9ejQOHDjQyZUTkTVZdcC+oaEBOTk5iIuLk9psbW0xdepUZGVltbq8EAJ79+5FYWEh3njjDbN9tFottFqtNF1dXQ3g1q18dDrdXW5B19U05tXY2Nitt7O7sGR/NbU3N7+77HNLa7c4vGbPnt3i/KqqKktXJamsrIRer4enp6dRu6enJwoKCppd7vr16/Dx8YFWq4WdnR3eeecdhIeHm+2blJSExMREk/bdu3dDqVS2uWa5KK0FAHscOnQIF05YuxpqTVE1ANhj266DuKeX+Sd06QzAVS1w9p974GDmnKnihg0AOxw4cADnVabz5aK+vt6ifhaHV+/eLX9i1bt3b0RFRVm6urvi4uKCY8eOoba2FhkZGdBoNBg8eDAmT55s0jcuLs7oy+TV1dXw9fXFtGnT4Orq2in1WsPxkqtA3lGMHz8egX7u1i6HWvHJ0Z+Bk6ew7ezdP/s04sFJUPeV7+UvTWdHrbE4vDZt2nTHxTTHw8MDdnZ2qKioMGqvqKiAl5dXs8vZ2trC398fABAUFIT8/HwkJSWZDS+FQgGFwvQKcwcHBzg4ONzdBnRh9vb20p/deTu7i+mjfWBnZ9fsdXuAZdfudYfr9ix9v1p1zMvR0RHBwcHIyMjArFmzANy6K2tGRgZiY2MtXo/BYDAa1yKSm9au2wN47d7tLA4vS29309YHcGg0GkRHRyMkJATjxo1DcnIy6urqEBMTAwCIioqCj48PkpKSANwawwoJCcGQIUOg1WqxY8cObN26FRs2bGjT6xKRvFkcXps3b8bAgQMxZswYCGF+QPFOREZG4vLly4iPj0d5eTmCgoKwc+dOaRC/pKQEtra/jE7W1dVh8eLF+Pnnn+Hs7IyAgAB88MEHiIyMbLeaiKjrszi8Fi1ahI8//hjFxcWIiYnBk08+CXf39hkIjo2NbfY0MTMz02j6lVdewSuvvNIur0tE8mXxRaopKSkoKyvD8uXL8X//93/w9fXF3LlzsWvXrnY9EiMiskSbrrBXKBR4/PHHsWfPHpw6dQojR47E4sWLoVarUVvLe0MRUee5468H2drawsbGBkII6PX69qyJiKhVbQovrVaLjz/+GOHh4Rg6dCjy8vLw97//HSUlJVCpZHxJLxHJjsUD9osXL8a2bdvg6+uLp59+Gh9//DE8PDw6sjYiomZZHF6pqanw8/PD4MGD8f333+P7778322/79u3tVhwRUXMsDq+oqCjY2Nh0ZC1ERBZr00WqRERdhdVvRkhEdCcYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJa6RHilpKRArVbDyckJoaGhyM7Obrbvxo0bMXHiRPTp0wd9+vTB1KlTW+xPRN2T1cMrPT0dGo0GCQkJyM3NRWBgICIiInDp0iWz/TMzM/H4449j3759yMrKgq+vL6ZNm4YLFy50cuVEZE1WD681a9Zg4cKFiImJwYgRI5CamgqlUom0tDSz/T/88EMsXrwYQUFBCAgIwLvvvguDwYCMjIxOrpyIrMnemi/e0NCAnJwcxMXFSW22traYOnUqsrKyLFpHfX09dDod3N3dzc7XarXQarXSdHV1NQBAp9NBp9PdRfVdW2Njo/Rnd97OnqSn7FNLt82q4VVZWQm9Xg9PT0+jdk9PTxQUFFi0jhUrVmDAgAGYOnWq2flJSUlITEw0ad+9ezeUSmXbi5aJ0loAsMehQ4dw4YS1q6H20FP2aX19vUX9rBped+v111/Htm3bkJmZCScnJ7N94uLioNFopOnq6mppnMzV1bWzSu10x0uuAnlHMX78eAT6mT8qJXnpKfu06eyoNVYNLw8PD9jZ2aGiosKovaKiAl5eXi0uu2rVKrz++uv47rvvMHr06Gb7KRQKKBQKk3YHBwc4ODjcWeEyYG9vL/3ZnbezJ+kp+9TSbbNqeDk6OiI4OBgZGRmYNWsWAEiD77Gxsc0u9+abb+LVV1/Frl27EBIS0knVdm319fVGp9qFZVXQlp9B/glnGK64GfUNCAjo1qfM1DNY/bRRo9EgOjoaISEhGDduHJKTk1FXV4eYmBgAQFRUFHx8fJCUlAQAeOONNxAfH4+PPvoIarUa5eXlAACVSgWVSmW17bC2goICBAcHm7Q/8b5p35ycHIwdO7YTqiLqOFYPr8jISFy+fBnx8fEoLy9HUFAQdu7cKQ3il5SUwNb2lys6NmzYgIaGBvzud78zWk9CQgJeeumlziy9SwkICEBOTo40XXtDi2/2ZeGRKWFQOStM+hLJndXDCwBiY2ObPU3MzMw0mj537lzHFyRDSqXS6GhKp9PhWuUlhI0L6dbjI9RzWf0iVSKiO8HwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEsmT18EpJSYFarYaTkxNCQ0ORnZ3dbN+TJ0/iscceg1qtho2NDZKTkzuvUCLqUuyt+eLp6enQaDRITU1FaGgokpOTERERgcLCQvTv39+kf319PQYPHow5c+bgueees0LFRJ2jvr4eBQUFRm2FZVXQlp9B/glnGK64Se0BAQFQKpWdXGEXIKxo3LhxYsmSJdK0Xq8XAwYMEElJSa0uO3DgQLF27do2v+b169cFAHH9+vU2LysnDQ0N4osvvhANDQ3WLoXuQE5OjgBg0U9OTo61y21Xlv4btdqRV0NDA3JychAXFye12draYurUqcjKymq319FqtdBqtdJ0dXU1AECn00Gn07Xb63Q1TdvWnbexOxsyZAgOHz5s1FZ7Q4td+48gYuL9UDkrjPp2p/1s6bZYLbwqKyuh1+vh6elp1O7p6WlyuHw3kpKSkJiYaNK+e/fuHnGovWfPHmuXQO1own1DUFN1FTVVv7SVlZVZrZ6OUF9fb1E/q455dYa4uDhoNBppurq6Gr6+vpg2bRpcXV2tWFnH0ul02LNnD8LDw+Hg4GDtcqgd9JR92nR21BqrhZeHhwfs7OxQUVFh1F5RUQEvL692ex2FQgGFQmHS7uDg0K3fAE16ynb2JN19n1q6bVa7VMLR0RHBwcHIyMiQ2gwGAzIyMhAWFmatsohIJqx62qjRaBAdHY2QkBCMGzcOycnJqKurQ0xMDAAgKioKPj4+SEpKAnBrkP/UqVPS3y9cuIBjx45BpVLB39/fattBRJ3PquEVGRmJy5cvIz4+HuXl5QgKCsLOnTulQfySkhLY2v5ycHjx4kWMGTNGml61ahVWrVqFSZMmITMzs7PLJyIrsvqAfWxsLGJjY83Ouz2Q1Go1hBCdUBURdXVW/3oQEdGdsPqRV2drOnKz9ONYudLpdKivr0d1dXW3/mSqJ+kp+7Tp32ZrZ1k9LrxqamoAAL6+vlauhIhaUlNTg969ezc730b0sEEkg8GAixcvwsXFBTY2NtYup8M0XYxbWlrarS/G7Ul6yj4VQqCmpgYDBgww+sDudj3uyMvW1hb33HOPtcvoNK6urt36jd4T9YR92tIRVxMO2BORLDG8iEiWGF7dlEKhQEJCgtnvdZI8cZ8a63ED9kTUPfDIi4hkieFFRLLE8CIiWWJ4tWL+/PmYNWtWu65z8uTJePbZZ1vso1ar+Wg3ohb0qPCyJDRIXl566SUEBQVZu4xmdbX3XFer5270qPAikqOGhgZrl9Al9Zjwmj9/Pr7//nusW7cONjY2sLGxQVFRERYsWIBBgwbB2dkZw4YNw7p168wun5iYiH79+sHV1RXPPPOMxW+ouro6REVFQaVSwdvbG6tXrzbpc+nSJcyYMQPOzs4YNGgQPvzwQ5M+NjY22LBhA6ZPnw5nZ2cMHjwYn332mTT/3LlzsLGxwSeffIKJEyfC2dkZ999/P06fPo0jR44gJCQEKpUK06dPx+XLly38rQFpaWkYOXIkFAoFvL29je69VlJSgpkzZ0KlUsHV1RVz5841eiZB01HR1q1boVar0bt3b/z+97+XvhwP3Pqu6Ztvvgl/f38oFAr4+fnh1VdfleavWLECQ4cOhVKpxODBg/Hiiy9Kj8bavHkzEhMTcfz4cWmfbt682eJt62h3+p5rGqp49dVXMWDAAAwbNgwA8MMPPyAoKAhOTk4ICQnBF198ARsbGxw7dkxa9sSJE5g+fTpUKhU8PT3x1FNPobKystl6zp0711m/jvbXwc+P7DKqqqpEWFiYWLhwoSgrKxNlZWXi5s2bIj4+Xhw5ckScPXtWfPDBB0KpVIr09HRpuejoaKFSqURkZKQ4ceKE+Prrr0W/fv3EX//6V4ted9GiRcLPz09899134qeffhKPPvqocHFxEUuXLpX6TJ8+XQQGBoqsrCxx9OhRMWHCBOHs7Gz0UF0Aom/fvmLjxo2isLBQ/O1vfxN2dnbi1KlTQgghiouLBQAREBAgdu7cKU6dOiXGjx8vgoODxeTJk8WBAwdEbm6u8Pf3F88884xFtb/zzjvCyclJJCcni8LCQpGdnS3VpNfrRVBQkHjggQfE0aNHxaFDh0RwcLCYNGmStHxCQoJQqVRi9uzZIi8vT/zrX/8SXl5eRr+75cuXiz59+ojNmzeLM2fOiP3794uNGzdK81euXCkOHjwoiouLxVdffSU8PT3FG2+8IYQQor6+Xvz5z38WI0eOlPZpfX29RdvWGe72PffUU0+JEydOiBMnTojr168Ld3d38eSTT4qTJ0+KHTt2iKFDhwoA4scffxRCCHHt2jXRr18/ERcXJ/Lz80Vubq4IDw8XU6ZMabaexsZGa/xq2kWPCS8hhJg0aZJRaJizZMkS8dhjj0nT0dHRwt3dXdTV1UltGzZsECqVSuj1+hbXVVNTIxwdHcUnn3witV25ckU4OztLdRQWFgoAIjs7W+qTn58vAJiE1+2hExoaKhYtWiSE+CW83n33XWn+xx9/LACIjIwMqS0pKUkMGzasxbqbDBgwQLzwwgtm5+3evVvY2dmJkpISqe3kyZNG25KQkCCUSqWorq6W+jz//PMiNDRUCCFEdXW1UCgURmHVmrfeeksEBwdL0wkJCSIwMNDi5Tvbnb7nPD09hVarldo2bNgg+vbtK27cuCG1bdy40Si8Vq5cKaZNm2a07tLSUgFAFBYWWlyPXPS4u0rcLiUlBWlpaSgpKcGNGzfQ0NBgMgAcGBho9IDasLAw1NbWorS0FAMHDmx23UVFRWhoaEBoaKjU5u7uLp0GAEB+fj7s7e0RHBwstQUEBMDNzc1kfbc/VSksLMzolAEARo8eLf296VkA9913n1HbpUuXmq25yaVLl3Dx4kU8+OCDZufn5+fD19fX6L5oI0aMgJubG/Lz83H//fcDuPWpqYuLi9TH29tbev38/HxotdpmXwMA0tPTsX79ehQVFaG2thaNjY2yv6OCJe+5++67D46OjtJ0YWEhRo8eDScnJ6lt3LhxRsscP34c+/btg0qlMnnNoqIiDB06tH03xMp6zJiXOdu2bcOyZcuwYMEC7N69G8eOHUNMTIysB0j/8w6bTfcru73NYDC0uh5nZ+d2r+f212/tNbKysjBv3jw8/PDD+Prrr/Hjjz/ihRdekPX+sfQ916tXrzavu7a2FjNmzMCxY8eMfv7973/jv/7rv9prE7qMHhVejo6O0Ov10vTBgwcxYcIELF68GGPGjIG/vz+KiopMljt+/Dhu3LghTR86dAgqlarVu7EOGTIEDg4OOHz4sNR27do1nD59WpoOCAhAY2MjcnJypLbCwkJUVVWZrO/QoUMm08OHD2+xhjvl4uICtVpt9FzN/zR8+HCUlpaitLRUajt16hSqqqowYsQIi17j3nvvhbOzc7Ov8cMPP2DgwIF44YUXEBISgnvvvRfnz5836nP7Pu1q7vQ9d7thw4YhLy8PWq1Wajty5IhRn7Fjx+LkyZNQq9Xw9/c3+mkKw67++2qLHhVearUahw8fxrlz51BZWYl7770XR48exa5du3D69Gm8+OKLJm8I4NZH1QsWLMCpU6ewY8cOJCQkIDY2tsW7PAKASqXCggUL8Pzzz2Pv3r04ceIE5s+fb7TcsGHD8NBDD+GPf/wjDh8+jJycHPzhD38we1Ty6aefIi0tDadPn0ZCQgKys7ObffJSe3jppZewevVqrF+/Hv/+97+Rm5uLt99+GwAwdepU3HfffZg3bx5yc3ORnZ2NqKgoTJo0CSEhIRat38nJCStWrMDy5cuxZcsWFBUV4dChQ3jvvfcA3Aq3kpISbNu2DUVFRVi/fj0+//xzo3Wo1WoUFxfj2LFjqKysNPrH3RXc6Xvudk888QQMBgP++7//G/n5+di1axdWrVoF4Jcj7CVLluDq1at4/PHHceTIERQVFWHXrl2IiYmRAuv2eiw5Cu+yrD3o1pkKCwvF+PHjhbOzswAgCgoKxPz580Xv3r2Fm5ubWLRokfjLX/5iNAAcHR0tZs6cKeLj40Xfvn2FSqUSCxcuFDdv3rToNWtqasSTTz4plEql8PT0FG+++abJoGlZWZl45JFHhEKhEH5+fmLLli1i4MCBJgP2KSkpIjw8XCgUCqFWq40+oWoasG8avBVCiH379gkA4tq1a1Lbpk2bRO/evS3+naWmpophw4YJBwcH4e3tLf70pz9J886fPy9+85vfiF69egkXFxcxZ84cUV5eLs03N5i+du1aMXDgQGlar9eLV155RQwcOFA4ODgIPz8/8dprr0nzn3/+een3HhkZKdauXWtU/82bN8Vjjz0m3NzcBACxadMmi7etM9zNe+52Bw8eFKNHjxaOjo4iODhYfPTRR9I6m5w+fVr89re/FW5ubsLZ2VkEBASIZ599VhgMBrP1FBcXd/BvoOPwljgyYWNjg88//7zdv6pE8vXhhx8iJiYG169fb7cxSjnp8Z82EsnFli1bMHjwYPj4+OD48eNYsWIF5s6d2yODC2B43ZWSkpIWB6dPnToFPz+/Tqyobcx9pN7k22+/xcSJEzuxGmpNeXk54uPjUV5eDm9vb8yZM8fo2wg9DU8b70JjY2OLX69Qq9Wwt++6/z+cOXOm2Xk+Pj499n90kgeGFxHJUo+6VIKIug+GFxHJEsOLiGSJ4UVEssTwIiJZYnhRh5o/f750104HBwd4enoiPDwcaWlpbfpe3ebNm83eJqijdcQDWKh9MLyowz300EMoKyvDuXPn8O2332LKlClYunQpHn30UTQ2Nlq7PJIra36xkrq/5r5knJGRIQBId1FdvXq1GDVqlFAqleKee+4RixYtEjU1NUKIX75g/p8/CQkJQgghtmzZIoKDg4VKpRKenp7i8ccfFxUVFdLrXL16VTzxxBPCw8NDODk5CX9/f5GWlibNLykpEXPmzBG9e/cWffr0Eb/5zW+kLysnJCSYvO6+ffs65PdEbccjL7KKX//61wgMDMT27dsBALa2tli/fj1OnjyJ999/H3v37sXy5csBABMmTEBycjJcXV1RVlaGsrIyLFu2DACg0+mwcuVKHD9+HF988QXOnTuH+fPnS6/z4osv4tSpU/j222+Rn5+PDRs2wMPDQ1o2IiICLi4u2L9/Pw4ePAiVSoWHHnoIDQ0NWLZsGebOnSsdOZaVlWHChAmd+4ui5lk7Pal7a+7ISwghIiMjxfDhw83O+/TTT0Xfvn2laUtv5XPkyBEBQDpqmzFjhoiJiTHbd+vWrWLYsGHS7WKEEEKr1QpnZ2exa9euVusn6+KRF1mNEEK6kd53332HBx98ED4+PnBxccFTTz2FK1euoL6+vsV15OTkYMaMGfDz84OLiwsmTZoE4NaX5gFg0aJF2LZtG4KCgrB8+XL88MMP0rLHjx/HmTNn4OLiApVKBZVKBXd3d9y8edOiu5uSdTG8yGry8/MxaNAgnDt3Do8++ihGjx6Nf/7zn8jJyUFKSgqAlh+4WldXh4iICLi6uuLDDz/EkSNHpDutNi03ffp0nD9/Hs8995z0QJGmU87a2loEBweb3PP99OnTeOKJJzp46+ludd1bHlC3tnfvXuTl5eG5555DTk4ODAYDVq9eLd0i+5NPPjHqb+7e6wUFBbhy5Qpef/116XkCR48eNXmtfv36ITo6GtHR0Zg4cSKef/55rFq1CmPHjkV6ejr69+/f7BOJutM937sbHnlRh9NqtSgvL8eFCxeQm5uL1157DTNnzsSjjz6KqKgo+Pv7Q6fT4e2338bZs2exdetWpKamGq1DrVajtrYWGRkZqKysRH19Pfz8/ODo6Cgt99VXX2HlypVGy8XHx+PLL7/EmTNncPLkSXz99dfSQ0vmzZsHDw8PzJw5E/v370dxcTEyMzPxP//zP/j555+l1/3pp59QWFiIyspK6Wnd1AVYe9CNurfo6GjpMgN7e3vRr18/MXXqVJGWlmb00N41a9YIb29v4ezsLCIiIsSWLVtM7r//zDPPiL59+xpdKvHRRx8JtVotFAqFCAsLE1999ZXJg1iHDx8unJ2dhbu7u5g5c6Y4e/astM6ysjIRFRUlPDw8hEKhEIMHDxYLFy4U169fF0IIcenSJREeHi5UKhUvlehieD8vIpIlnjYSkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRL/w92qaSHaFtbbQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:56:27.677145Z", + "iopub.status.busy": "2024-03-22T20:56:27.676822Z", + "iopub.status.idle": "2024-03-22T20:56:27.889740Z", + "shell.execute_reply": "2024-03-22T20:56:27.888860Z" + }, + "papermill": { + "duration": 0.234701, + "end_time": "2024-03-22T20:56:27.891811", + "exception": false, + "start_time": "2024-03-22T20:56:27.657110", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATIAAAEmCAYAAADhgRxIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAsUlEQVR4nO2deVwT1/r/PyGQsG/KLggqVdytioLWpcVStVVv7+9qbW9du1ilrVLrvmFvi60btlq91muprdbtuvSrrb1KFatYqygWFbAiCJVVlBACJCSZ3x/ImEgCGchCmOf9es0L5sw5M89Mcj55zjNnETAMw4AgCMKKsbG0AQRBEC2FhIwgCKuHhIwgCKuHhIwgCKuHhIwgCKuHhIwgCKuHhIwgCKuHhIwgCKvH1tIGmBu1Wo2CggK4uLhAIBBY2hyCIJ6AYRhIpVL4+/vDxsYwX4t3QlZQUIDAwEBLm0EQRBPk5+ejQ4cOBuXlnZC5uLgAqHtIrq6uFraGIIgnqaioQGBgIFtXDYF3QlbfnHR1dSUhI4hWDJfQDwX7CYKwekjICIKwekjICIKwengXI2sJDMNAqVRCpVJZ2hSCsFqEQiFsbW2N2v3JokIWHx+PQ4cOITMzEw4ODoiMjMSnn36Krl276i2TmJiI6dOna6WJxWLU1NSY1FaFQoHCwkJUVVWZ9DoEwQccHR3h5+cHkUhklPNZVMiSk5MxZ84cDBw4EEqlEkuWLMHzzz+PmzdvwsnJSW85V1dXZGVlsfum7tiqVquRk5MDoVAIf39/iEQi6kxLEM2AYRgoFAqUlpYiJycHoaGhBnd6bQyLCtmJEye09hMTE+Ht7Y3U1FQMGzZMbzmBQABfX19Tm8eiUCigVqsRGBgIR0dHs13XEArKqyEA4OfuYGlTCMIgHBwcYGdnh7t370KhUMDe3r7F52xVwX6JRAIA8PT0bDRfZWUlOnbsiMDAQIwfPx43btzQm1cul6OiokJray7G+OUwJkqVGvcr5SitlEOpVlvaHIIwGGPXpVZTM9VqNebOnYshQ4agZ8+eevN17doVO3fuxNGjR/Hdd99BrVYjMjISf/31l8788fHxcHNzY7e2NDyJ0btDEPxC0FpWUXrnnXfw008/4dy5cwaPrwKA2tpahIWFYfLkyfjoo48aHJfL5ZDL5ex+/fAHiURicM/+mpoa5OTkICQkxChusLGoVamRUVjnYXb3c4WtsNX8LhFEozRWpyoqKuDm5sapjraKb35MTAyOHTuG06dPcxIxALCzs0O/fv1w+/ZtncfFYjE7HImGJRnGtGnTMGHCBKOec8SIEZg7d26jeYKDg5GQkGDU6xKWwZDP25hYVMgYhkFMTAwOHz6MX375BSEhIZzPoVKpkJ6eDj8/PxNYaP2Y+wtFmJ5Vq1ahb9++ljajVWHRt5Zz5szBnj17cPToUbi4uKCoqAgA4ObmBgeHurdwU6ZMQUBAAOLj4wEAq1evxuDBg9GlSxeUl5dj7dq1uHv3Lt544w2L3QdBtEVqa2thZ2dnaTMMwqIe2datWyGRSDBixAj4+fmx2759+9g8eXl5KCwsZPcfPnyIN998E2FhYRgzZgwqKiqQkpKC7t27m9V2hmFQpVCafdMMaTbVk23atGlITk7Gpk2bIBAIIBAIkJ2djZkzZyIkJAQODg7o2rUrNm3apLN8XFwcvLy84OrqilmzZkGhUBj0bGQyGaZMmQJnZ2f4+flh/fr1DfKUlJTgpZdegoODA0JCQrB79+4GeQQCAbZu3YrRo0fDwcEBnTp1wsGDB9njubm5EAgE2L9/P5555hk4ODhg4MCBuHXrFi5duoQBAwbA2dkZo0ePRmlpqUG2A8DOnTvRo0cPiMVi+Pn5ISYmhj2Wl5eH8ePHw9nZGa6urpg4cSKKi4vZ4/Xe0rfffovg4GC4ubnhlVdegVQqZfOo1Wp89tln6NKlC8RiMYKCgvDxxx+zxxcuXIinnnoKjo6O6NSpE5YvX47a2loAdV2U4uLicO3aNfYzTUxMbPKeMjMzMXToUNjb26N79+44deoUBAIBjhw5ovUs9+3bh+HDh8Pe3h67d+9GWVkZJk+ejICAADg6OqJXr174/vvvtc5tyOdtaizqkRnynuHMmTNa+xs3bsTGjRtNZJHhVNeq0H3Fz2a/7s3V0XAUGfaxbdq0Cbdu3ULPnj2xevVqAICHhwc6dOiAAwcOoF27dkhJScFbb70FPz8/TJw4kS2blJQEe3t7nDlzBrm5uZg+fTratWunVeH08eGHHyI5ORlHjx6Ft7c3lixZgitXrmg1h6ZNm4aCggKcPn0adnZ2eO+991BSUtLgXMuXL8eaNWuwadMmfPvtt3jllVeQnp6OsLAwNs/KlSuRkJCAoKAgzJgxA6+++ipcXFywadMmODo6YuLEiVixYgW2bt3apO1bt25FbGws1qxZg9GjR0MikeD8+fMA6gSoXsSSk5OhVCoxZ84cTJo0Set7mp2djSNHjuDYsWN4+PAhJk6ciDVr1rDPbvHixfjqq6+wceNGDB06FIWFhcjMzGTLu7i4IDExEf7+/khPT8ebb74JFxcXLFiwAJMmTcL169dx4sQJnDp1CkBdC6YxVCoVJkyYgKCgIFy8eBFSqRQffPCBzryLFi3C+vXr0a9fP9jb26Ompgb9+/fHwoUL4erqiuPHj+P1119H586dER4eDsCwz9vkMDxDIpEwABiJRGJwmerqaubmzZtMdXU1myaT1zIdFx4z+yaT17I21CpVzLX8h8y1/IeMQqnSafvw4cOZ999/v9H7mzNnDvP3v/+d3Z86dSrj6enJyGQyNm3r1q2Ms7Mzo1Lpvk49UqmUEYlEzP79+9m0srIyxsHBgbUjKyuLAcD8/vvvbJ6MjAwGALNx40Y2DQAza9YsrfMPGjSIeeeddxiGYZicnBwGALNjxw72+Pfff88AYJKSkti0+Ph4pmvXro3aXY+/vz+zdOlSncf+97//MUKhkMnLy2PTbty4oXUvK1euZBwdHZmKigo2z4cffsgMGjSIYRiGqaioYMRiMfPVV18ZZA/DMMzatWuZ/v37s/srV65k+vTpY3D5n376ibG1tWUKCwvZtJMnTzIAmMOHDzMM8/hZJiQkNHm+sWPHMh988AHDMIZ93rrQVafqaU4dpUHjzcTBToibq6Mtct2WsmXLFuzcuRN5eXmorq6GQqFo8OvZp08frVEMERERqKysRH5+Pjp27Kj33NnZ2VAoFBg0aBCb5unpqTV+NiMjA7a2tujfvz+b1q1bN7i7uzc4X0RERIP9tLQ0rbTevXuz//v4+AAAevXqpZWmy9t7kpKSEhQUFOC5557TeTwjIwOBgYFafRG7d+8Od3d3ZGRkYODAgQDq3r5qzm7q5+fHXj8jIwNyuVzvNQBg3759+Pzzz5GdnY3KykoolcoWvW3PyspCYGCg1miYem/qSQYMGKC1r1Kp8Mknn2D//v24d+8eFAoF5HI5+90w5PM2ByRkzUQgEBjcxGtN7N27F/Pnz8f69esREREBFxcXrF27FhcvXrS0ac1GMyBdPwb2yTS1ASMf6l8wGdOeJ6/f1DUuXLiA1157DXFxcYiOjoabmxv27t1rtrjTk2Oc165di02bNiEhIQG9evWCk5MT5s6da3C81Fy0in5khOkQiURa0w6dP38ekZGRmD17Nvr164cuXbogOzu7Qblr166hurqa3f/tt9/g7Ozc5MiIzp07w87OTksYHz58iFu3brH73bp1g1KpRGpqKpuWlZWF8vLyBuf77bffGuxrxseMiYuLC4KDg5GUlKTzeFhYGPLz85Gfn8+m3bx5E+Xl5Qa/bAoNDYWDg4Pea6SkpKBjx45YunQpBgwYgNDQUNy9e1crz5OfaVN07doV+fn5Wi8lLl26ZFDZ8+fPY/z48fjnP/+JPn36oFOnTlqfpSGftzmwPpeC4ERwcDAuXryI3NxcODs7IzQ0FLt27cLPP/+MkJAQfPvtt7h06VKDPnwKhQIzZ87EsmXLkJubi5UrVyImJqbJMXLOzs6YOXMmPvzwQ7Rr1w7e3t5YunSpVrmuXbvihRdewNtvv42tW7fC1tYWc+fO1emtHDhwAAMGDMDQoUOxe/du/P777/jPf/5jnIejg1WrVmHWrFnw9vbG6NGjIZVKcf78ebz77ruIiopCr1698NprryEhIQFKpRKzZ8/G8OHDGzTJ9GFvb4+FCxdiwYIFEIlEGDJkCEpLS3Hjxg3MnDkToaGhyMvLw969ezFw4EAcP34chw8f1jpHcHAwcnJykJaWhg4dOsDFxQVisVjvNUeNGoXOnTtj6tSp+OyzzyCVSrFs2TIATc8cExoaioMHDyIlJQUeHh7YsGEDiouLWeE25PM2B+SRtXHmz58PoVCI7t27w8vLC9HR0Xj55ZcxadIkDBo0CGVlZZg9e3aDcs899xxCQ0MxbNgwTJo0CePGjcOqVasMuubatWvxzDPP4KWXXkJUVBSGDh2qFQ8DgK+//hr+/v4YPnw4Xn75Zbz11lvw9vZucK64uDjs3bsXvXv3xq5du/D999+btKvN1KlTkZCQgC+//BI9evTAiy++iD///BNAXaU/evQoPDw8MGzYMERFRaFTp05a3YUMYfny5fjggw+wYsUKhIWFYdKkSWwMbdy4cZg3bx5iYmLQt29fpKSkYPny5Vrl//73v+OFF17AyJEj4eXl1aA7xJMIhUIcOXIElZWVGDhwIN544w0sXboUAJoccrds2TI8/fTTiI6OxogRI+Dr69tg1Ichn7epaTVjLc1Fc8ZxtdaxlkqVGjcfjbUM83OFXRsbaykQCHD48GGjD5ci6pqMQ4cOxe3bt9G5c2ezX9/YYy2paUkQPODw4cNsaOH27dt4//33MWTIEIuImCloWz/hhMnJy8uDs7Oz3i0vL8/SJjZKY7b/+uuvljavWezevVvvPfXo0QMAIJVKMWfOHHTr1g3Tpk3DwIEDcfToUQtbbjzIIyM44e/v36Af15PHjYUpoh6N2R4QEGD065mDcePGafXj0qS+K8iUKVMwZcoUc5plVkjICE7Y2tqiS5culjaj2Viz7fpwcXHR6oDLR6hpSRCE1UNCRhCE1UNCRhCE1UNCZs3Q0poEAYCEjCCINgAJWVvBBOMznlwMRHNGUYJoTVD3CyvG3C3LwsJCeHh4mPmqBNE0JGSEwWhOzGcpGIaBSqWCrS19dYnHUNOyuTAMoJCZf9PT211fy1IqleK1116Dk5MT/Pz8sHHjxmYvEadrsYpDhw5h5MiRcHR0RJ8+fXDhwgWtMufOnWMXBgkMDMR7770HmUzGHv/2228xYMAAuLi4wNfXF6+++qrWbK5nzpyBQCDATz/9hP79+0MsFuPcuXOcbSfaNvSz1lxqq4BPjDccx2CWFAAip6bzPSI2Nhbnz5/HDz/8AB8fH6xYscKoC0MsXboU69atQ2hoKJYuXYrJkyfj9u3bsLW1RXZ2Nl544QX861//ws6dO1FaWoqYmBjExMTg66+/BlC35NhHH32Erl27oqSkBLGxsZg2bRp+/PFHressWrQI69atQ6dOnah5SzSAhKwNI5VK8c0332DPnj3sHPH184AZi/nz52Ps2LEA6uYO69GjB27fvo1u3bohPj4er732Guv9hYaG4vPPP8fw4cOxdetW2NvbY8aMGey5OnXqhM8//xwDBw5EZWUlnJ2d2WOrV6/GqFGjjGY30bYgIWsudo513pElrmsgd+7cQW1trdZCE25ubkZdGEJz4Y/61d5LSkrQrVs3XLt2DX/88YfWmpUMw0CtViMnJwdhYWFITU3FqlWrcO3aNTx8+JCd2z4vL09rAkVDZ2Al+AkJWXMRCDg18UxkhIWvr3vhj3oxqqysxNtvv4333nuvQbmgoCDIZDJER0cjOjoau3fvhpeXF/Ly8hAdHd1gcYsnF8UgCE1IyNownTp1gp2dHS5duoSgoCAAgEQiwa1btzBs2DCTX//pp5/GzZs39c44kZ6ejrKyMqxZs4Zd1OTy5csmt4toe9BbyzaMi4sLpk6dig8//BCnT59mF7iwsbFpctEJY7Bw4UKkpKQgJiYGaWlp+PPPP3H06FHExMQAqPPKRCIRvvjiC9y5cwc//PADPvroI5PbRbQ9SMjaOBs2bEBERARefPFFREVFYciQIQgLCzPL2gO9e/dGcnIybt26hWeeeQb9+vXDihUr2JcNXl5eSExMxIEDB9C9e3esWbMG69atM7ldRNuDFh8xgNa6+IhKrcaNgrrFR7r5ukBk2/Qq5DKZDAEBAVi/fj1mzpxpahMJQie0+AjBiatXryIzMxPh4eGQSCRYvXo1AGD8+PEWtowgjAcJGQ9Yt24dsrKyIBKJ0L9/f/z666/IyMjA6NGj9ZaprKw0o4UE0TIsKmTx8fE4dOgQMjMz4eDggMjISHz66adN9nM6cOAAli9fjtzcXISGhuLTTz/FmDFjzGS1ddGvXz+kpqY2SK+urm50IQ6CsCYsKmTJycmYM2cOBg4cCKVSiSVLluD555/HzZs39fYbSklJweTJkxEfH48XX3wRe/bswYQJE3DlyhX07NnTzHdgvTg4OLTJhTgIftKqgv2lpaXw9vZGcnKy3n5OkyZNgkwmw7Fjx9i0wYMHo2/fvti2bVuT12hJsD84OBgODg6G3YwZaE6wnyBaA9XV1cjNzTVasL9Vdb+QSCQAAE9PT715Lly4gKioKK206OjoBrMuGJP63utVVVUmuwZB8In6uqQ5MqQltJpgv1qtxty5czFkyJBGm4hFRUXw8fHRSvPx8UFRUZHO/HK5HHK5nN2vqKjgbJtQKIS7uzs7vYyjo6NZOpQ2hUqtBqOsG8pTU1MDNXlkRCuHYRhUVVWhpKQE7u7uEAqN851tNUI2Z84cXL9+3ehzTcXHxyMuLq7F56mfVFBzrixLo2YYlJTXAABsZGLY2rQqB5sg9OLu7m7UiTpbhZDFxMTg2LFjOHv2LDp06NBoXl9fXxQXF2ulFRcX630oixcvRmxsLLtfUVHBjuvjgkAggJ+fH7y9vVFbW8u5vCmQyWvx1uHzAIDvZg6Cn3vrid8RhD7s7OyM5onVY1EhYxgG7777Lg4fPowzZ84gJCSkyTIRERFISkrSmuH05MmTiIiI0JlfLBZDLBYby2QIhUKjfwjNpRZC3JOqAABCO3GrGnVAEObEokI2Z84c7NmzB0ePHoWLiwsb53Jzc2PfDk6ZMgUBAQGIj48HALz//vsYPnw41q9fj7Fjx2Lv3r24fPkytm/fbrH7IAjCslg0qLJ161ZIJBKMGDECfn5+7LZv3z42T15eHgoLC9n9yMhI7NmzB9u3b0efPn1w8OBBHDlyhPqQEQSPsXjTsinOnDnTIO0f//gH/vGPf5jAIuuFMcXClgRhJdBrLoIgrB4SMoIgrB4SsjZC6xloRhDmh4TMiiHtIog6SMjaCCRqBJ8hISMIwuohISMIwuppUT+yyspKdjHWegydP4gwLq1oWjmCMDucPbKcnByMHTsWTk5OcHNzg4eHBzw8PODu7g4PDw9T2EgQBNEonD2yf/7zn2AYBjt37oSPj0+rmJeLAH0OBK/hLGTXrl1DampqkwuEEOaFmpYEn+HctBw4cCDy8/NNYQtBEESz4OyR7dixA7NmzcK9e/fQs2fPBnNu9+7d22jGEYZD/hjBZzgLWWlpKbKzszF9+nQ2TSAQgGEYCAQCqFQqoxpIEATRFJyFbMaMGejXrx++//57CvYTBNEq4Cxkd+/exQ8//ECLu7YyKNZP8BnOwf5nn30W165dM4UtBEEQzYKzR/bSSy9h3rx5SE9PR69evRoE+8eNG2c04/hOrUqNKoUKbg7GWcSUINoqnIVs1qxZAIDVq1c3OEbBfuMSvfEs7tyX4fclz8HbtakVkqhtSfAXzk1LtVqtdyMRMy537ssAAGdulVrYEoJo3XASstraWtja2uL69eumsofgAAX4CaIOTkJmZ2eHoKAg8rzMjCEdXEjUCD7DuWm5dOlSLFmyBA8ePDCFPQRBEJzhHOzfvHkzbt++DX9/f3Ts2BFOTk5ax69cuWI044g6DOl0TA4ZwWc4C9mECRNMYAbRGDR2giAah7OQrVy50hR2EARBNJtmT3WdmpqKjIwMAECPHj3Qr18/oxlFaGPIcFYK9hN8hrOQlZSU4JVXXsGZM2fg7u4OACgvL8fIkSOxd+9eeHl5GdtG3kPj8gmicTi/tXz33XchlUpx48YNPHjwAA8ePMD169dRUVGB9957zxQ2EgZAYkfwGc4e2YkTJ3Dq1CmEhYWxad27d8eWLVvw/PPPG9U4og6BAeF+aloSfKZZQ5SeHCgO1HWWfXJpOIIgCHPQrGl83n//fRQUFLBp9+7dw7x58/Dcc89xOtfZs2fx0ksvwd/fHwKBAEeOHGk0/5kzZyAQCBpsRUVFXG/DqjAo2E89yQgew1nINm/ejIqKCgQHB6Nz587o3LkzQkJCUFFRgS+++ILTuWQyGfr06YMtW7ZwKpeVlYXCwkJ28/b25lSeIIi2BecYWWBgIK5cuYJTp04hMzMTABAWFoaoqCjOFx89ejRGjx7NuZy3tzf7xpQgCKJZ/cgEAgFGjRqFUaNGGdseg+jbty/kcjl69uyJVatWYciQIRaxw1wYNESJWpYoq5Qjq0iKiM7taC0JntEsIUtKSkJSUhJKSkoaBPh37txpFMN04efnh23btmHAgAGQy+XYsWMHRowYgYsXL+Lpp5/WWUYul0Mul7P7FRUVJrOPsCzPrk+GpLoWm1/thxd7+1vaHMKMcBayuLg4rF69GgMGDICfn59Zf/m6du2qtcJ5ZGQksrOzsXHjRnz77bc6y8THxyMuLs5cJpoE8i0MQ1JdCwBIyighIeMZnIVs27ZtSExMxOuvv24KezgTHh6Oc+fO6T2+ePFixMbGsvsVFRUIDAw0h2lGg4YocYNalfyDs5ApFApERkaawpZmkZaWBj8/P73HxWIxxGKxGS0yIyReOrEhJeMdnIXsjTfewJ49e7B8+fIWX7yyshK3b99m93NycpCWlgZPT08EBQVh8eLFuHfvHnbt2gUASEhIQEhICHr06IGamhrs2LEDv/zyC/73v/+12JbWjEE9+0nVWEjG+AdnIaupqcH27dtx6tQp9O7du0Ev/w0bNhh8rsuXL2PkyJHsfn0TcOrUqUhMTERhYSHy8vLY4wqFAh988AHu3bsHR0dH9O7dG6dOndI6R1uEHAxu0PPiH5yF7I8//kDfvn0BoMEiJFwD/yNGjADTSHAnMTFRa3/BggVYsGABp2sQ/MMQD5ZoW3AWstOnT5vCDqIRaPERbthwHq9CWDv0kRNtEPLI+AYJmRVAMR9u0PPiHyRkVgHVTC7Q0+IfJGREm4P6kfEPzkJ29uxZKJXKBulKpRJnz541ilGENlQvuWFDz4t3cBaykSNH6lxlXCKRtPn+XJaC6iU3aOYL/sFZyBiG0flFKSsra7DqOEEQhDkwuB/Zyy+/DKDu127atGla4xdVKhX++OOPVjUGsy1B85Fxgxwy/mGwkLm5uQGo88hcXFzg4ODAHhOJRBg8eDDefPNN41tIEByhnv38w2Ah+/rrrwEAwcHBmD9/PjUjzQhVS25QsJ9/cB6itHLlSlPYQTQCraLEDWpa8g/Owf7i4mK8/vrr8Pf3h62tLYRCodZGEJaG+pHxD84e2bRp05CXl4fly5ebfaprQhtNL4yC/RrQV5J3cBayc+fO4ddff2Wn8iGI1gYF+/kH56ZlYGBgo3OIEcaHnF5uULCff3AWsoSEBCxatAi5ubkmMIfQhWFTXRP1kPDzD85Ny0mTJqGqqgqdO3eGo6Njg6mudQ1fIghzQk1L/sFZyBISEkxgBtEoVC85QR4Z/+AsZFOnTjWFHUQjGDbVNTUu6yEd4x/Nmo8sOzsby5Ytw+TJk1FSUgIA+Omnn3Djxg2jGsdnSJiaD3UJ4h+chSw5ORm9evXCxYsXcejQIVRWVgIArl27Rr3+TQRVTG5Qh1j+wVnIFi1ahH/96184efIkRCIRm/7ss8/it99+M6pxfIarQ8Z3/03TgyUd4x+chSw9PR1/+9vfGqR7e3vj/v37RjGK0IbqZdOoNZScnhf/4Cxk7u7uKCwsbJB+9epVBAQEGMUoQtvDIg+jadTkkfEazkL2yiuvYOHChSgqKoJAIIBarcb58+cxf/58TJkyxRQ28hIK9nNDW8hIyfgGZyH75JNP0K1bNwQGBqKyshLdu3fHsGHDEBkZiWXLlpnCRt5DHTybRlP3Scf4B+d+ZCKRCF999RWWL1+O69evo7KyEv369UNoaKgp7OMtXJuWfHfgVBpBMhJ+/sFZyOoJCgpCUFCQMW0hiGaj2bSkQeP8wyAhi42NxUcffQQnJyfExsY2mnfDhg1GMYzvMPQWjhNqalryGoOE7OrVq6itrWX/1wcFWY0H96mr+d221OpHRtLPOwwSstOnT+v8v6WcPXsWa9euRWpqKgoLC3H48GFMmDCh0TJnzpxBbGwsbty4gcDAQCxbtgzTpk0zmk2tEj31ku9xMU3II+M3zRpraSxkMhn69OmDLVu2GJQ/JycHY8eOxciRI5GWloa5c+fijTfewM8//2xiS82PdtOSX+tayuRK/OdcDvIfVBlcRt2WHgDBGYM8svrFeQ3h0KFDBucdPXo0Ro8ebXD+bdu2ISQkBOvXrwcAhIWF4dy5c9i4cSOio6MNPg/Ruon/KQPf/ZaHhJO3kB5n2OdKQsZvDPLI3Nzc2M3V1RVJSUm4fPkyezw1NRVJSUnsIr6m4sKFC4iKitJKi46OxoULF/SWkcvlqKio0NqsDb41lVJulwEApHKlwWU0dYw0jX8Y5JHVL84LAAsXLsTEiROxbds2dvk3lUqF2bNnw9XV1TRWPqKoqAg+Pj5aaT4+PqioqEB1dbXW6uf1xMfHIy4uzqR2mQIaNM4NzX5ktMYn/+AcI9u5cyfmz5+vtYalUChEbGwsdu7caVTjjMHixYshkUjYLT8/39ImcYZnDlmzbpialvyGc4dYpVKJzMxMdO3aVSs9MzMTarXaaIbpwtfXF8XFxVppxcXFcHV11emNAYBYLIZYLDapXaZA06ugbi1NQ01LfsNZyKZPn46ZM2ciOzsb4eHhAICLFy9izZo1mD59utEN1CQiIgI//vijVtrJkycRERFh0utaAs5NS55XXk2PjOePgpdwFrJ169bB19cX69evZ6fz8fPzw4cffogPPviA07kqKytx+/Ztdj8nJwdpaWnw9PREUFAQFi9ejHv37mHXrl0AgFmzZmHz5s1YsGABZsyYgV9++QX79+/H8ePHud6GVUEOWdOoySPjNZyFzMbGBgsWLMCCBQvYN4DNDfJfvnwZI0eOZPfrhz9NnToViYmJKCwsRF5eHns8JCQEx48fx7x587Bp0yZ06NABO3bsaJNdL7QGjRuSvw3V3ubotrZH1naeBWEYzR40DjRfwOoZMWJEoxUwMTFRZ5nGhkkR1k3c/91AdqmMc7m2JOQEd5olZAcPHsT+/fuRl5cHhUKhdezKlStGMYzv8HUO+q/P5zarHDUt+Q3n7heff/45pk+fDh8fH1y9ehXh4eFo164d7ty5w6mXPtE4VBe5odmPjOAfnIXsyy+/xPbt2/HFF19AJBJhwYIFOHnyJN577z1IJBJT2EjwrycZZ7RiZOSS8Q7OQpaXl4fIyEgAgIODA6RSKQDg9ddfx/fff29c63gMTd3MDepHxm84C5mvry8ePHgAoG6W2Pq1LHNycuiX0ILw/clTz35+w1nInn32Wfzwww8A6jrHzps3D6NGjcKkSZN0rndJNBOaIZYTWsF+y5lBWAjOby23b9/ODkWaM2cO2rVrh5SUFIwbNw5vv/220Q3kK4b0hdLMwXeHRDtGZkFDCIvASciUSiU++eQTzJgxAx06dABQt87lK6+8YhLjCMJQGOoQy2s4NS1tbW3x2WefQak0fJ4oonloB/upcdkU1I+M33COkT333HNITk42hS1EC+C7F0L9yPgN5xjZ6NGjsWjRIqSnp6N///5wcnLSOj5u3DijGcdnuI615Ds0+wW/4Sxks2fPBqB7/UqBQACVStVyqwheDlFqSfcdht588BrOQmbqyROJZtIG6m5L9Ic8Mn5j0eXgCP1oNy354ZK1RIAo2M9vDPbIqqurkZSUhBdffBFA3Vz4crmcPS4UCvHRRx/B3t7e+FbyEFp8hBvUs5/fGCxk33zzDY4fP84K2ebNm9GjRw92rvzMzEz4+/tj3rx5prGUaPO0LEZG/cj4jMFNy927d+Ott97SStuzZw9Onz6N06dPY+3atdi/f7/RDeQrfKyMLbljlUbolpwz/mGwkN2+fRu9evVi9+3t7WFj87h4eHg4bt68aVzrCINpC5WXgv1EczG4aVleXq4VEystLdU6rlartY4TLYRqIycYGmvJawz2yDp06IDr16/rPf7HH3+w4y+JlsPHutiS5jR17Oc3BgvZmDFjsGLFCtTU1DQ4Vl1djbi4OIwdO9aoxhH8wnhNS1I1vmFw03LJkiXYv38/unbtipiYGDz11FMAgKysLGzevBlKpRJLliwxmaF8g5pH3NDyyOjZ8Q6DhczHxwcpKSl45513sGjRIjYmIRAIMGrUKHz55Zfw8fExmaFE4/DdC2Eo2M9rOA1RCgkJwYkTJ/DgwQN2hfAuXbrA09PTJMbxGT4Kk9GaluTO8o5mrWvp6emJ8PBwY9tCaGBIXWxrFbYl4q2iIcC8hsZathHamKZxhqa65jckZK0UPtbFlggQxcj4DQkZ0Wqg2S+I5kJC1krhGv9qC3W3JTE/6kfGb0jIzIRSpcaJ60UoqzRsGBd5Fdygnv38hoTMTOw8n4NZ36XipS/OmfW61+9JMGfPFeTel5n1us2hJVpEYy35TasQsi1btiA4OBj29vYYNGgQfv/9d715ExMTIRAItDZrmMzxfzeKAQAFkoZDvAwhu7QSh678BbUe10Nfs+zFL87h+B+FeGPX5WZd15y0qB8ZuWS8pln9yIzJvn37EBsbi23btmHQoEFISEhAdHQ0srKy4O3trbOMq6srsrKy2H1rWPfRVsjNxicr9XPr65bgsxEIMKFfAOfr3ymt5FzG7LRAi1RawX4SNb5hcY9sw4YNePPNNzF9+nR0794d27Ztg6OjI3bu3Km3jEAggK+vL7tZw9AoO6FxHvXVvIc60/ledan7Bb+xqJApFAqkpqYiKiqKTbOxsUFUVBQuXLigt1xlZSU6duyIwMBAjB8/Hjdu3NCbVy6Xo6KiQmuzBLY2HD0yHlbHlk3jw7/nRTzGokJ2//59qFSqBh6Vj48PioqKdJbp2rUrdu7ciaNHj+K7776DWq1GZGQk/vrrL5354+Pj4ebmxm6BgYFGvw9DsDWSR9aWq2vLxloa5zyEdWLxpiVXIiIiMGXKFPTt2xfDhw/HoUOH4OXlhX//+9868y9evBgSiYTd8vPzzWxxHXYtjJE1XYBj/lZIyzrEUj8yPmPRYH/79u0hFApRXFyslV5cXAxfX1+DzmFnZ4d+/fqxs3E8iVgshlgsbrGtLUVow+03w9hV0RpeiLQEhjwyXmNRj0wkEqF///5ISkpi09RqNZKSkhAREWHQOVQqFdLT0+Hn52cqM40C1xgZH2lRz341Bfv5jMW7X8TGxmLq1KkYMGAAwsPDkZCQAJlMhunTpwMApkyZgoCAAMTHxwMAVq9ejcGDB6NLly4oLy/H2rVrcffuXbzxxhuWvI0m4Rzs56FbYayxlgT/sLiQTZo0CaWlpVixYgWKiorQt29fnDhxgn0BkJeXp7Xs3MOHD/Hmm2+iqKgIHh4e6N+/P1JSUtC9e3dL3YJBGC3YTxVWJyrq2c9rLC5kABATE4OYmBidx86cOaO1v3HjRmzcuNEMVhkXzsF+rf/19OY3II81YaxpfKhxyT+s7q2ltSLk3LQ07vWtIUJnrH5k5JHxDxIyM2Gsnv1tGupHRjQTql1mgvtbS47zkTFArUqNaoWK43XaBtSPjN+QkJkJzWC/qgWv2BqrpMM/O42wFSesVsxaNo2P0cwgrBASMjOh6ZHVGrDkT3MqZv0UQTcLLTOetKUYaxofEjX+QUJmJjSn8TFIyDieX7Py6urEbw0d+1sW7Nc8D8E3SMjMhJ1GXzilyrRVzQo0y+jQW0t+Q0JmJjQ9IlM0LTWzW+u4Sl33XFBebVBZCvbzGxIyC1BrQLBfbydYPUU1K7J1ypjuJmHc/+mfa04Tmo+M35CQmQnNeqY0wCNryfmt1CHTOb5UJjfsDazWbwNpGu8gITMTmh5WS5qW+kRK2yNrmElXmjVgqCjTVNf8hoTMTGgKU20Lgv0GNS2tU7Na2P1C8zwkZXyDhMwCmCLY31TYra0HwNXkkfEaEjIzoVm5DPHIDJnxQiu9DXhkLYHmI+M3JGRmQrtpafxgf1MxMmugZYuPUD8yPkNCZgEM6RCrN9ivJ7+mNurs2W8F4qbLCzW0Txw1LfkNCZmZ0HprqW6+R6avkvI+2E8rjfMaEjIzodW0VBqnaalvUlRr8L6MDXlk/IaEzAIoDenZz7E2qtqCR6Yj7clbUet5duSF8RsSMgtg+mC/daJLjDRFuaxSjgEfn8KKo9cb5NNqrZOm8Y5WsfgIH9CspC3pfqGPJp08K1C3pm7hmwt38UCmwK4Ld7F6fE+tYzRovJWgUgJVZUBlMVBZ8uhvMSArrfv77HLAM8TolyUhMyEZhRVwdbBDgLuD0bpf6GtBPTlEh2EYzrNglEhroFQx8Hd3aLZ9xsbQO6A5+00IwwDVDx+LUmWpxv8lgKzksWjJ7qPRn6Snp5CQWROFkmqM3vQrACB3zVitj9aQQeOce/Zr1OQqhQpRG5IxqFM7TucI/7huxff0Vc/Dxd6OmwFGoMl7biQD9SPjCMMAculjT4n1oEq0vSlZad3/6lrDzy2wAZy8ACdvwNkbcPZ59Ncb8DC+iAEkZCYjq0iq91hNLbcZYo//Udhkfs3W6k/phcgulSG7VNZkOV0USmqaLWS3SyoR/2MG3n0uFH0D3TmWNs5ycLymtvqxIMmeEKUnhUpp2FxvLA4ej0XJSVOgfABnr0d/fQDHdoCN0DT3pwcSMhMhfGKOfs169vGPGahVqzF7RBeDzvXvs3eazGOq2R/yyqpw94EMz4R6GZR/5jeXcLesCkmZJchdM5bTtXRpkeEdYjXO09ZiZKrauiablqekw3uqLAXkEm7nFrloiNAjUWrgSfnUeVi2ItPcnxEgITMRQo0KWKVQNahcn53IalTIbjXi0emiyTn7m3muYWtPAwAOzY7E00EeTZa9W1bV6PGsIimSMosxY0gI7O24/Wo3Jk+MtTUt1Wqg+oEOT0kjMF6fVlXG7dxCsXZzTlOUnJ5o6omcTHN/ZoaEzERoeghcl2erlCux4L9/6Dmqu5Zq9iMTmqAjWfpfEoOErCmiE84CAKrkKsyP7qp1rCX6o6tD7PwD13CjoAJH5wyByNa4PY3KqxT44VoBJvQLgGt9M5xhAHmFtihVPiFKmnEnhsP3QiCs84pYUfJp6EnVe072btbbmbCZkJCZCIXq8Ze0SqE02Es4nVmC6YmXOF9PsyLbtPBLrKtpZsN5geHGuZL3sOF1dTUtDTyf9nxkdX8Ppv4FAPjtThmGPWVY07ie5Ful6OjpiOD2ToCi6glPqRiXfk+HsDAf2b9Wo6+HAoJ6oVLJOV0Hju0eC9CToqQZd3LwBGyo26c+SMj0UVEIHP/g8ZdJ89ew/ksndtZbXK4R0K9rWmrj4ag7mL762M1mmaspAkbWHADG9/IMGd0A1DkW0ppauNjbNfpj8GSwX/MtrliXN6ZUaDfhNALjsrIC2P95G2qUQy2uhE1tZYPio4C62lP1aNNE7Ko7xsQK1KNmnZMXINT+HkhrapGWX47Uuw9x71Y1XurjjWE+7Rt9Rq0RpUoNmVwFqbwWlXIlZHIlpDVK9Av0gJue735LICHTR8U9IOt443nsHHUIXN2X1LPEFv0FD1AKd8irKhq4G0HtGsYmSqVy5Nxv/E1jQXmNznTNimvIywGuCI3sDOjqgqLLEzyVUYLecf/DzmkDGz2fmmFgAzU8IUUHRSUUtx7iZZuz8BJI0OnKaeBquXYTr7qhR1iPE4BB9fdb3+vA1p4Vob9qXXDmngD34Y4Sxh1ycTvEvfosnNv5133+dob1w2MYBnfvy5B69yFS8x7iyt2HyCqWan1VDqT+heFPeWHJmDB09XUx6LzNRa1mIFMoIZOrUCmvhbTmyf+VqJQrIX0kTJU1dfvsVqNE5aP8+t7M7387AuEhnka3nYRMH24dgLEbnvjVLn0c96iVAbVVQPnduu0JBgH4r/jRzm6gt409/i5yxX244T7jBkmpB3D6rIbH5403Em/BAY6ohr1es6oUSp3pmg6OXMegdC4OlS7PR6FiIFeqILbVDtArlGpcyn2A/h09GgTvS6VytHcW6XzzqMsj0+1xMXBhZPhX4hF4C8rxko0EXgIJmP/9DoHs8Ru87SV/wVVcDqGAAe4B2AtsqH/J1nBEUx02tjr7Op0ttMHem3KUMu64DzeM7N8TK/4+iH2Ic7em4LLyIeZFPYWL1+4hu1QG2zQnfPr/Ouq5UB01tSqk35PUCdfdOuEqkyka5AvydGSf58HUfCTfKsWvf5Zi0sBAzBv1FLxdHn8/GIZBTa0aUnltnejUKB//L6/VEhet/+WP/q+p95hUqJTr/m61BJGtDVzEtnC2t4WTyFbrbb4xaRVCtmXLFqxduxZFRUXo06cPvvjiC4SHh+vNf+DAASxfvhy5ubkIDQ3Fp59+ijFjxhjXKBdfYOBMdlepUkNao0RFTd2vU6W0HApJMVQVxVBLS+BnK0GYcw0EVXWB3JLCfFQ/LISXQAJHgRx26hp0tKlBR5Q8vkbySa1LHgUAe0DGiHGfcUMp3HGfccN9xpWtVC7V/qi4xSCr0h4dg4PZsioDg3AMw2Dq15cgEtrg88l9cTStAFFhPvByETdabvmR6/jsRCb+WPm8ljB98mMGElNyMaaXL758rb9WmYEfn8LMoSFY/mJ3nL99Hyt/eLy0m1BZBTy4A1SW4Oaff0J6/x462cvwiW06vAQSeAnK4SWQoD0kEAt0VLAU7V0PABAAakYAqdANDh7++K1EiFK4IbxnGAIDg594g+cD2LvrjDvt/vYyflYX45nQ9si9fR87L5ehV+d7+Fu/DrhTWonLdx9CaCPA5PBARHZph39su4B9l/Mxvp8/Ijs/bgYWV9SwopV69yFuFEgaDE8TCW3Qq4Mb+nf0wNNBHni6o7uWUL09rBM+PZGJn64X4fvf83E0rQCBHo51nlFNLWQKFVRGnh7X1kbACo+LvS2cxbZweiRGLvX/i+uO1f/v/Cif5uYktjX6Sxa9NpvlKo2wb98+xMbGYtu2bRg0aBASEhIQHR2NrKwseHt7N8ifkpKCyZMnIz4+Hi+++CL27NmDCRMm4MqVK+jZs6eOKzSPGwUSvL83DRXVdcJVXavvDZMNAF8AvugV4IalY8MwuFM7fH/qT2w8dQsA4IgahDpVw7aqBO0FkkcVVYIOdpUIsJOis4OMFT0HgQJOAjmcBCXaolePBMAeoL6hdV1sj/uMGyp/80CYncsj4asXQVfcZ9xQAnfcr3VD8KLjmNDXH2dvlQIAFv43Hf93rQA7vXNwMnY4ewl9miitUUJSXYsfrhUg5XYZNk3ui8SUXADAj+lF+PlaHvxxH16CcvY+HS+U41h6NYRVJYgXSOAlqhMpp3I58HndebtrXONVPd9ICeOIUsa9bnvk1bb3DcT5IiHKBO4oVLqihHHDA7hi6FO+WDymG6Yk1I2seNu1ExZHhun5/OooKK/GlbyHyC6R4ecbxQCAOSO74OkgD2xK+hNLDl1HD383HL56DwAw/CkveLvaw9vVHv8cHITvfsvDkkPpmD4khBWuezoWF27vLMaAjh51wtXRAz0DXBt4uZoEt3fC1n/2x+XcB/jX8Qyk5Zcjq7hh1xyBAHAWPRYUJ/ETIvSE8Lg8EiotAXr0v9jWxuoWeRYwFp7/ZNCgQRg4cCA2b94MAFCr1QgMDMS7776LRYsWNcg/adIkyGQyHDt2jE0bPHgw+vbti23btjV5vYqKCri5uUEikcDV1VVvvpsFFRjz+a8N0h1FQrjY28LF3o796yQS4tc/77Ou+ajuPjh5s7hJWxrCwAk1aP/IE/ESSB4LH+rEob1AAi/U/XUQNGyWNEYlY49Sxo1t3pYyjzw+uCGgQ0ecymNQCjd89c4YhAX5AAA6L/oBnpCyHpKXoJy9fphLNQSVJaxwuQu4jSSoYsQoeeRplj6yp96++v9LGXeUwRVytKwzprujHWJGdkGvADfIFHXNqmJJDa7mP8SVu+UoqmgYe7weFw0HOyGmff07fv3zPjp7OUFao0SJVI6trz2N0b38AAAVNbV4fsPZBuewEQDdfF3R/5Fw9e/ogQ4eDs0WCYZhkHr3IRRKNesh1YuQo0hodeKjD0PrqCYWFTKFQgFHR0ccPHgQEyZMYNOnTp2K8vJyHD16tEGZoKAgxMbGYu7cuWzaypUrceTIEVy7dq1BfrlcDrn88SvxiooKBAYGNvmQqhRKXMuXwMXeFq6PRMvZ3hZ2eqLepVI5NiXdwve/5+t19cf08sUHz3fFc+uT9V7XcOpEz0tQjvaQoL2g4rEnpCGCdYJYDnsBh7FyAGRwQDVjB09IYSMw/CuiYIQoxWMRqvMO3Vhv6j7j+ui4O6oaiQU2l54Brrh+r4JzOaGNAGF+LujYzgkXssvwTGh7bHqlH4C66YPGfn6OFSpPJxF+W/ycVrPp/O37WHbkOhvf6t/RA30C3eEstnijx+pojpBZ9Cnfv38fKpUKPj4+Wuk+Pj7IzMzUWaaoqEhn/qKiIp354+PjERcXx9k2R5EtIjobPujay0WMf03ohWmRwVjzUyZOZdQ1CyM7t0NKdhn6Brpj8+SnYWMjQO6asTh89S/M29dQeA1HABkcIGMckAu/JnqTMnBGtZY31/6Rd1UndI9FsD0ksBfUwgnVcBLUNY1UjABlrBfnxgpVvVdXCjeUPPKgKuAEY84Z5OMqRty4HrhbVoXBndrhmwu56BXghsGd2uHPkkpE9/DBV2fvICmzBEvGhCHQwxGD45PY8tOHBOP/9e+AN765DDXDwElc16RyEgvh7iBCrw5ueDrIA30C3eAo0l0d2jmLsWPqACQ8ChVMHBDYIPYzpEt7nJ4/wmj3TXDDoh5ZQUEBAgICkJKSgoiICDZ9wYIFSE5OxsWLFxuUEYlE+OabbzB58mQ27csvv0RcXByKixs255rrkbUUpUoN2yb6LNSq1LC1EUAgELDT7jAMA4VKrRU3UasZ2NgIoFSpYSMQwMZGgIqaWqjVDKoUKvi7O0CuVKGmVg1HkRDSGiUeyOTo4OGIP4sr4epgCzcHOziIhCiWyFEmk+NGQQVy78vQJ9AdHo4iVNTUIrdMhk7tHOEjVkBYfR811VVILxdBLvIAIxDCTiiAp5MYvq72qFWrkVkoRRdvZ4R6O6O9ixi/ZJZAJLTBs928IRAAKjUDsa0NymQKOIlskZZfDrlShX6BHqhUKCGpqkUXb2fklsnQxcvZaJ1ulSo1+yyb+gyI1ofVeWTt27eHUChsIEDFxcXw9fXVWcbX15dTfrFYDLG48TdypsCQCqTZTK2PbwgEggbB3/oKrnnO+mEx7o51+2JbIVvO00kET6e6mFKvDm5a5wpq54igdo7o1+Rwo2AAgP53x8DIrtovY8b18dfar++N0d657vlrerhujnXztAHAUz7G7R/1+Dm1jZgR0TQW/bkSiUTo378/kpIeNwXUajWSkpK0PDRNIiIitPIDwMmTJ/XmJwii7WPxSGRsbCymTp2KAQMGIDw8HAkJCZDJZJg+fToAYMqUKQgICEB8fDwA4P3338fw4cOxfv16jB07Fnv37sXly5exfft2S94GQRAWxOJCNmnSJJSWlmLFihUoKipC3759ceLECTagn5eXBxuNTouRkZHYs2cPli1bhiVLliA0NBRHjhwxah8ygiCsC4v3IzM3zQkkEgRhPppTR+mVDkEQVg8JGUEQVg8JGUEQVo/Fg/3mpj4kWFHBfRgLQRCmp75ucgnf807IpNK6mQMCAwMtbAlBEI0hlUrh5ubWdEbw8K2lWq1GQUEBXFxc2sxsAaagfihXfn4+vd1tAnpW3GjqeTEMA6lUCn9/f62uV43BO4/MxsYGHTp0sLQZVoOrqytVTgOhZ8WNxp6XoZ5YPRTsJwjC6iEhIwjC6iEhI3QiFouxcuVKi8wcYm3Qs+KGKZ4X74L9BEG0PcgjIwjC6iEhIwjC6iEhIwjC6iEhIwjC6iEh4zFbtmxBcHAw7O3tMWjQIPz++++N5j9w4AC6desGe3t79OrVCz/++KOZLLU8XJ7VV199hWeeeQYeHh7w8PBAVFRUk8+2rcH1u1XP3r17IRAItJaHNAiG4CV79+5lRCIRs3PnTubGjRvMm2++ybi7uzPFxcU6858/f54RCoXMZ599xty8eZNZtmwZY2dnx6Snp5vZcvPD9Vm9+uqrzJYtW5irV68yGRkZzLRp0xg3Nzfmr7/+MrPlloHr86onJyeHCQgIYJ555hlm/PjxnK5JQsZTwsPDmTlz5rD7KpWK8ff3Z+Lj43XmnzhxIjN27FittEGDBjFvv/22Se1sDXB9Vk+iVCoZFxcX5ptvvjGVia2K5jwvpVLJREZGMjt27GCmTp3KWcioaclDFAoFUlNTERUVxabZ2NggKioKFy5c0FnmwoULWvkBIDo6Wm/+tkJzntWTVFVVoba2Fp6enqYys9XQ3Oe1evVqeHt7Y+bMmc26Lu8GjRPmWeG9rdCcZ/UkCxcuhL+/f4MfgrZIc57XuXPn8J///AdpaWnNvi4JGUGYkDVr1mDv3r04c+YM7O3tLW1Oq0MqleL111/HV199hfbt2zf7PCRkPMQcK7y3FZrzrOpZt24d1qxZg1OnTqF3796mNLPVwPV5ZWdnIzc3Fy+99BKbplarAQC2trbIyspC586dm7wuxch4CK3wbjjNeVYA8Nlnn+Gjjz7CiRMnMGDAAHOY2irg+ry6deuG9PR0pKWlsdu4ceMwcuRIpKWlGT6Tc3PfTBDWzd69exmxWMwkJiYyN2/eZN566y3G3d2dKSoqYhiGYV5//XVm0aJFbP7z588ztra2zLp165iMjAxm5cqVvOp+weVZrVmzhhGJRMzBgweZwsJCdpNKpZa6BbPC9Xk9SXPeWpKQ8ZgvvviCCQoKYkQiERMeHs789ttv7LHhw4czU6dO1cq/f/9+5qmnnmJEIhHTo0cP5vjx42a22HJweVYdO3ZkADTYVq5caX7DLQTX75YmzREymsaHIAirh2JkBEFYPSRkBEFYPSRkBEFYPSRkBEFYPSRkBEFYPSRkBEFYPSRkBEFYPSRkBEFYPSRkRKtg2rRpEAgEDbYXXnjB0qYRVgDNfkG0Gl544QV8/fXXWmn6VqOura2FnZ2dVppCoYBIJOJ83eaWI1oP5JERrQaxWAxfX1+tzcPDAwAgEAiwdetWjBs3Dk5OTvj444+xatUq9O3bFzt27EBISAg731deXh7Gjx8PZ2dnuLq6YuLEiVrTyugrR1gvJGSE1bBq1Sr87W9/Q3p6OmbMmAEAuH37Nv773//i0KFDSEtLg1qtxvjx4/HgwQMkJyfj5MmTuHPnDiZNmqR1rifLEdYNNS2JVsOxY8fg7OyslbZkyRIsWbIEAPDqq69i+vTpWscVCgV27doFLy8vAHVzpKWnpyMnJ4edy2rXrl3o0aMHLl26hIEDB+osR1g3JGREq2HkyJHYunWrVprmgh26Jijs2LGjlhhlZGQgMDBQa0K+7t27w93dHRkZGayQPVmOsG5IyIhWg5OTE7p06dLocUPSDL0W0XagGBnRpggLC0N+fj7y8/PZtJs3b6K8vBzdu3e3oGWEKSGPjGg1yOXyBsvL2draclpdJyoqCr169cJrr72GhIQEKJVKzJ49G8OHD+fV3Pl8gzwyotVw4sQJ+Pn5aW1Dhw7ldA6BQICjR4/Cw8MDw4YNQ1RUFDp16oR9+/aZyGqiNUBTXRMEYfWQR0YQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNVDQkYQhNXz/wFhNblZ8YS4XAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018188, + "end_time": "2024-03-22T20:56:27.928530", + "exception": false, + "start_time": "2024-03-22T20:56:27.910342", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4614.695564, + "end_time": "2024-03-22T20:56:30.671491", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tab_ddpm_concat/4/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tab_ddpm_concat/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tab_ddpm_concat/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-22T19:39:35.975927", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/model.pt b/contraceptive/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce73156fb714ae8c179c752381e50f680d1913a4 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b3ac873b73d5d19d404bc1940f245ef136580c00b5a6df3a257860ada70fa96 +size 47482955 diff --git a/contraceptive/tab_ddpm_concat/params.json b/contraceptive/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..1f4f3974b8e9096f8eb8fca2963f2f1e6f4af44f --- /dev/null +++ b/contraceptive/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "tab_ddpm_concat", "mse_mag": true, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tvae/eval.csv b/contraceptive/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..78c9674d0fa7c4388d92acb6cfd784eb724e5eaa --- /dev/null +++ b/contraceptive/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.029584258647425615,0.02876510211709655,0.0026995846250792966,12.484404563903809,0.019254347309470177,0.5103664398193359,0.03382880240678787,4.6931290853535756e-05,4.136843204498291,0.03707587346434593,0.11939960718154907,0.05195752531290054,0.09320646524429321,0.024370625615119934,16.6212477684021 diff --git a/contraceptive/tvae/history.csv b/contraceptive/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..8bbba089a99aa1885490897935ea76f302c57456 --- /dev/null +++ b/contraceptive/tvae/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.02038899842772581,0.7097008107104881,0.0013612247566806238,0.2463153438932366,0.0,0.0,0.0,0.0,0.020629282327491737,900,225,260.0577940940857,1.1558124181959364,0.2889531045489841,0.1322056857123971,0.013955928831257755,0.32679080615364564,0.0004168977530578862,0.0,0.0,0.0,0.0,0.0,0.013955928831257755,450,113,87.91127800941467,0.7779759115877405,0.19535839557647705,0.10100915132964079 +1,0.0060118012685173505,0.5180407320536465,8.332759030961423e-05,0.06306991064869281,0.0,0.0,0.0,0.0,0.00608574254250723,900,225,260.53746366500854,1.1579442829555935,0.28948607073889837,0.09532983317143387,0.004431044542773937,4.37402327783609,5.788037800186684e-05,0.0,0.0,0.0,0.0,0.0,0.004431044542773937,450,113,90.43914699554443,0.8003464335888888,0.20097588221232096,0.04492716361002057 +2,0.003981401668424951,0.5578023322469762,3.4067052916835076e-05,0.02774549509638746,0.0,0.0,0.0,0.0,0.004037349244463258,900,225,259.8459405899048,1.1548708470662434,0.28871771176656086,0.09886109303269121,0.005920900385180074,3.167294240776193,6.510238445815651e-05,0.0,0.0,0.0,0.0,0.0,0.005920900385180074,450,113,88.16130113601685,0.7801885056284676,0.19591400252448188,0.04382448092585149 +3,0.003003391056942443,0.5015569428249138,1.917405674469519e-05,0.019288771962617628,0.0,0.0,0.0,0.0,0.003044349568921866,900,225,258.8182637691498,1.1503033945295547,0.28757584863238866,0.09951512091689639,0.0036749178177625354,5.0160115570675385,3.591052332840726e-05,0.0,0.0,0.0,0.0,0.0,0.0036749178177625354,450,113,88.94795989990234,0.7871500876097552,0.1976621331108941,0.050722146361439895 +4,0.0026951473932907296,0.46114019461917133,1.2756809318339692e-05,0.018491813861118214,0.0,0.0,0.0,0.0,0.0027525624157472826,900,225,260.7371289730072,1.1588316843244764,0.2897079210811191,0.09791627071694367,0.0032899318864413846,2.2600422874186443,1.737884139242202e-05,0.0,0.0,0.0,0.0,0.0,0.0032899318864413846,450,113,89.44571375846863,0.7915549890129967,0.19876825279659696,0.05520178424967123 +5,0.002675653763451717,0.38926185907018096,1.1465693117683688e-05,0.018026919938856734,0.0,0.0,0.0,0.0,0.002713999592718513,900,225,261.18925762176514,1.1608411449856229,0.2902102862464057,0.10430583260332545,0.00328355419371898,3.905635658147522,2.2026288097660226e-05,0.0,0.0,0.0,0.0,0.0,0.00328355419371898,450,113,89.86624097824097,0.7952764688339908,0.19970275772942436,0.047640036822469756 +6,0.002428213983172706,0.4563769066303573,1.070831390430716e-05,0.01809621533375725,0.0,0.0,0.0,0.0,0.0024591475264686678,900,225,262.2440469264984,1.1655290974511041,0.29138227436277603,0.10204424848676556,0.003275549128625749,3.9855575082005594,1.929245272713863e-05,0.0,0.0,0.0,0.0,0.0,0.003275549128625749,450,113,90.92798852920532,0.8046724648602241,0.2020621967315674,0.04646150635676953 +7,0.0023822973380447365,0.44478256372083136,9.005847006119572e-06,0.01762347323496619,0.0,0.0,0.0,0.0,0.002411980113861824,900,225,263.1176962852478,1.1694119834899903,0.2923529958724976,0.10207737949159411,0.0031462487001489435,3.1703384120910294,2.179660826174649e-05,0.0,0.0,0.0,0.0,0.0,0.0031462487001489435,450,113,88.3796284198761,0.7821206054856292,0.19639917426639134,0.047558308290564906 +8,0.0021461909939373275,0.3496849273867641,8.150395166715502e-06,0.016793633546072266,0.0,0.0,0.0,0.0,0.0021742351456487084,900,225,264.955197095871,1.1775786537594266,0.29439466343985665,0.10324391664730179,0.00289536593284639,3.853903681079736,2.701310116102227e-05,0.0,0.0,0.0,0.0,0.0,0.00289536593284639,450,113,92.01904273033142,0.8143278117728444,0.2044867616229587,0.05265144564250517 +9,0.002258309722690481,0.5195468653853743,7.549715697529972e-06,0.019823850064721128,0.0,0.0,0.0,0.0,0.0022869745167554355,900,225,265.574116230011,1.1803294054667155,0.2950823513666789,0.10302899842564431,0.0032758567545291347,1.3863763298404954,4.0907979645609235e-05,0.0,0.0,0.0,0.0,0.0,0.0032758567545291347,450,113,89.21779704093933,0.7895380269109675,0.1982617712020874,0.0622333479762918 diff --git a/contraceptive/tvae/mlu-eval.ipynb b/contraceptive/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..75f4532397ff9f07b74d47a6bc61ea0797186c08 --- /dev/null +++ b/contraceptive/tvae/mlu-eval.ipynb @@ -0,0 +1,2280 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:02.940942Z", + "iopub.status.busy": "2024-03-22T18:24:02.940605Z", + "iopub.status.idle": "2024-03-22T18:24:02.974127Z", + "shell.execute_reply": "2024-03-22T18:24:02.973246Z" + }, + "papermill": { + "duration": 0.048631, + "end_time": "2024-03-22T18:24:02.976269", + "exception": false, + "start_time": "2024-03-22T18:24:02.927638", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.003179Z", + "iopub.status.busy": "2024-03-22T18:24:03.002821Z", + "iopub.status.idle": "2024-03-22T18:24:03.009533Z", + "shell.execute_reply": "2024-03-22T18:24:03.008703Z" + }, + "papermill": { + "duration": 0.022249, + "end_time": "2024-03-22T18:24:03.011518", + "exception": false, + "start_time": "2024-03-22T18:24:02.989269", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.034982Z", + "iopub.status.busy": "2024-03-22T18:24:03.034720Z", + "iopub.status.idle": "2024-03-22T18:24:03.038779Z", + "shell.execute_reply": "2024-03-22T18:24:03.037972Z" + }, + "papermill": { + "duration": 0.018138, + "end_time": "2024-03-22T18:24:03.040841", + "exception": false, + "start_time": "2024-03-22T18:24:03.022703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.064726Z", + "iopub.status.busy": "2024-03-22T18:24:03.064446Z", + "iopub.status.idle": "2024-03-22T18:24:03.068482Z", + "shell.execute_reply": "2024-03-22T18:24:03.067689Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018191, + "end_time": "2024-03-22T18:24:03.070313", + "exception": false, + "start_time": "2024-03-22T18:24:03.052122", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.094148Z", + "iopub.status.busy": "2024-03-22T18:24:03.093890Z", + "iopub.status.idle": "2024-03-22T18:24:03.099456Z", + "shell.execute_reply": "2024-03-22T18:24:03.098647Z" + }, + "papermill": { + "duration": 0.019665, + "end_time": "2024-03-22T18:24:03.101336", + "exception": false, + "start_time": "2024-03-22T18:24:03.081671", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1a5dc951", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.127355Z", + "iopub.status.busy": "2024-03-22T18:24:03.127077Z", + "iopub.status.idle": "2024-03-22T18:24:03.131879Z", + "shell.execute_reply": "2024-03-22T18:24:03.131076Z" + }, + "papermill": { + "duration": 0.020354, + "end_time": "2024-03-22T18:24:03.133762", + "exception": false, + "start_time": "2024-03-22T18:24:03.113408", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011105, + "end_time": "2024-03-22T18:24:03.156067", + "exception": false, + "start_time": "2024-03-22T18:24:03.144962", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.179681Z", + "iopub.status.busy": "2024-03-22T18:24:03.179174Z", + "iopub.status.idle": "2024-03-22T18:24:03.188041Z", + "shell.execute_reply": "2024-03-22T18:24:03.187211Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022732, + "end_time": "2024-03-22T18:24:03.189956", + "exception": false, + "start_time": "2024-03-22T18:24:03.167224", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:03.213790Z", + "iopub.status.busy": "2024-03-22T18:24:03.213530Z", + "iopub.status.idle": "2024-03-22T18:24:05.212106Z", + "shell.execute_reply": "2024-03-22T18:24:05.211109Z" + }, + "papermill": { + "duration": 2.012769, + "end_time": "2024-03-22T18:24:05.214125", + "exception": false, + "start_time": "2024-03-22T18:24:03.201356", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:05.242305Z", + "iopub.status.busy": "2024-03-22T18:24:05.241844Z", + "iopub.status.idle": "2024-03-22T18:24:05.254590Z", + "shell.execute_reply": "2024-03-22T18:24:05.253855Z" + }, + "papermill": { + "duration": 0.029051, + "end_time": "2024-03-22T18:24:05.256688", + "exception": false, + "start_time": "2024-03-22T18:24:05.227637", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:05.281449Z", + "iopub.status.busy": "2024-03-22T18:24:05.280934Z", + "iopub.status.idle": "2024-03-22T18:24:05.288363Z", + "shell.execute_reply": "2024-03-22T18:24:05.287666Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021749, + "end_time": "2024-03-22T18:24:05.290199", + "exception": false, + "start_time": "2024-03-22T18:24:05.268450", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:05.314864Z", + "iopub.status.busy": "2024-03-22T18:24:05.314602Z", + "iopub.status.idle": "2024-03-22T18:24:05.411455Z", + "shell.execute_reply": "2024-03-22T18:24:05.410650Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.111724, + "end_time": "2024-03-22T18:24:05.413655", + "exception": false, + "start_time": "2024-03-22T18:24:05.301931", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:05.441068Z", + "iopub.status.busy": "2024-03-22T18:24:05.440695Z", + "iopub.status.idle": "2024-03-22T18:24:10.140482Z", + "shell.execute_reply": "2024-03-22T18:24:10.139521Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.716252, + "end_time": "2024-03-22T18:24:10.142843", + "exception": false, + "start_time": "2024-03-22T18:24:05.426591", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 18:24:07.693870: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 18:24:07.693938: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 18:24:07.695597: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:10.167560Z", + "iopub.status.busy": "2024-03-22T18:24:10.167007Z", + "iopub.status.idle": "2024-03-22T18:24:10.172801Z", + "shell.execute_reply": "2024-03-22T18:24:10.171934Z" + }, + "papermill": { + "duration": 0.020077, + "end_time": "2024-03-22T18:24:10.174683", + "exception": false, + "start_time": "2024-03-22T18:24:10.154606", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:10.201405Z", + "iopub.status.busy": "2024-03-22T18:24:10.201109Z", + "iopub.status.idle": "2024-03-22T18:24:19.205995Z", + "shell.execute_reply": "2024-03-22T18:24:19.204935Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.021688, + "end_time": "2024-03-22T18:24:19.208711", + "exception": false, + "start_time": "2024-03-22T18:24:10.187023", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:24:19.237135Z", + "iopub.status.busy": "2024-03-22T18:24:19.236780Z", + "iopub.status.idle": "2024-03-22T18:24:19.243528Z", + "shell.execute_reply": "2024-03-22T18:24:19.242662Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023498, + "end_time": "2024-03-22T18:24:19.245747", + "exception": false, + "start_time": "2024-03-22T18:24:19.222249", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:19.271717Z", + "iopub.status.busy": "2024-03-22T18:24:19.271449Z", + "iopub.status.idle": "2024-03-22T18:24:19.276110Z", + "shell.execute_reply": "2024-03-22T18:24:19.275293Z" + }, + "papermill": { + "duration": 0.019869, + "end_time": "2024-03-22T18:24:19.277993", + "exception": false, + "start_time": "2024-03-22T18:24:19.258124", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:19.302520Z", + "iopub.status.busy": "2024-03-22T18:24:19.302224Z", + "iopub.status.idle": "2024-03-22T18:24:19.869747Z", + "shell.execute_reply": "2024-03-22T18:24:19.868753Z" + }, + "papermill": { + "duration": 0.582657, + "end_time": "2024-03-22T18:24:19.872198", + "exception": false, + "start_time": "2024-03-22T18:24:19.289541", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:19.901865Z", + "iopub.status.busy": "2024-03-22T18:24:19.901564Z", + "iopub.status.idle": "2024-03-22T18:24:20.224677Z", + "shell.execute_reply": "2024-03-22T18:24:20.223796Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.340084, + "end_time": "2024-03-22T18:24:20.226791", + "exception": false, + "start_time": "2024-03-22T18:24:19.886707", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:20.254943Z", + "iopub.status.busy": "2024-03-22T18:24:20.254652Z", + "iopub.status.idle": "2024-03-22T18:24:20.363030Z", + "shell.execute_reply": "2024-03-22T18:24:20.362116Z" + }, + "papermill": { + "duration": 0.125241, + "end_time": "2024-03-22T18:24:20.365312", + "exception": false, + "start_time": "2024-03-22T18:24:20.240071", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:24:20.395222Z", + "iopub.status.busy": "2024-03-22T18:24:20.394907Z", + "iopub.status.idle": "2024-03-22T18:24:20.850984Z", + "shell.execute_reply": "2024-03-22T18:24:20.850027Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.472973, + "end_time": "2024-03-22T18:24:20.853026", + "exception": false, + "start_time": "2024-03-22T18:24:20.380053", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:20.883359Z", + "iopub.status.busy": "2024-03-22T18:24:20.883030Z", + "iopub.status.idle": "2024-03-22T18:24:20.887066Z", + "shell.execute_reply": "2024-03-22T18:24:20.886243Z" + }, + "papermill": { + "duration": 0.022094, + "end_time": "2024-03-22T18:24:20.889070", + "exception": false, + "start_time": "2024-03-22T18:24:20.866976", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:20.915312Z", + "iopub.status.busy": "2024-03-22T18:24:20.915025Z", + "iopub.status.idle": "2024-03-22T18:24:20.921808Z", + "shell.execute_reply": "2024-03-22T18:24:20.921016Z" + }, + "papermill": { + "duration": 0.022203, + "end_time": "2024-03-22T18:24:20.923743", + "exception": false, + "start_time": "2024-03-22T18:24:20.901540", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11895304" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:20.950779Z", + "iopub.status.busy": "2024-03-22T18:24:20.950516Z", + "iopub.status.idle": "2024-03-22T18:24:21.031109Z", + "shell.execute_reply": "2024-03-22T18:24:21.030301Z" + }, + "papermill": { + "duration": 0.096946, + "end_time": "2024-03-22T18:24:21.033405", + "exception": false, + "start_time": "2024-03-22T18:24:20.936459", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 48,128\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,895,304\n", + "Trainable params: 11,895,304\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.58\n", + "Estimated Total Size (MB): 423.53\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:24:21.065187Z", + "iopub.status.busy": "2024-03-22T18:24:21.064897Z", + "iopub.status.idle": "2024-03-22T19:31:29.310852Z", + "shell.execute_reply": "2024-03-22T19:31:29.309758Z" + }, + "papermill": { + "duration": 4028.279722, + "end_time": "2024-03-22T19:31:29.328132", + "exception": false, + "start_time": "2024-03-22T18:24:21.048410", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02038899842772581, 'avg_role_model_std_loss': 0.7097008107104881, 'avg_role_model_mean_pred_loss': 0.0013612247566806238, 'avg_role_model_g_mag_loss': 0.2463153438932366, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.020629282327491737, 'n_size': 900, 'n_batch': 225, 'duration': 260.0577940940857, 'duration_batch': 1.1558124181959364, 'duration_size': 0.2889531045489841, 'avg_pred_std': 0.1322056857123971}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013955928831257755, 'avg_role_model_std_loss': 0.32679080615364564, 'avg_role_model_mean_pred_loss': 0.0004168977530578862, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013955928831257755, 'n_size': 450, 'n_batch': 113, 'duration': 87.91127800941467, 'duration_batch': 0.7779759115877405, 'duration_size': 0.19535839557647705, 'avg_pred_std': 0.10100915132964079}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0060118012685173505, 'avg_role_model_std_loss': 0.5180407320536465, 'avg_role_model_mean_pred_loss': 8.332759030961423e-05, 'avg_role_model_g_mag_loss': 0.06306991064869281, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00608574254250723, 'n_size': 900, 'n_batch': 225, 'duration': 260.53746366500854, 'duration_batch': 1.1579442829555935, 'duration_size': 0.28948607073889837, 'avg_pred_std': 0.09532983317143387}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004431044542773937, 'avg_role_model_std_loss': 4.37402327783609, 'avg_role_model_mean_pred_loss': 5.788037800186684e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004431044542773937, 'n_size': 450, 'n_batch': 113, 'duration': 90.43914699554443, 'duration_batch': 0.8003464335888888, 'duration_size': 0.20097588221232096, 'avg_pred_std': 0.04492716361002057}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003981401668424951, 'avg_role_model_std_loss': 0.5578023322469762, 'avg_role_model_mean_pred_loss': 3.4067052916835076e-05, 'avg_role_model_g_mag_loss': 0.02774549509638746, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004037349244463258, 'n_size': 900, 'n_batch': 225, 'duration': 259.8459405899048, 'duration_batch': 1.1548708470662434, 'duration_size': 0.28871771176656086, 'avg_pred_std': 0.09886109303269121}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005920900385180074, 'avg_role_model_std_loss': 3.167294240776193, 'avg_role_model_mean_pred_loss': 6.510238445815651e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005920900385180074, 'n_size': 450, 'n_batch': 113, 'duration': 88.16130113601685, 'duration_batch': 0.7801885056284676, 'duration_size': 0.19591400252448188, 'avg_pred_std': 0.04382448092585149}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003003391056942443, 'avg_role_model_std_loss': 0.5015569428249138, 'avg_role_model_mean_pred_loss': 1.917405674469519e-05, 'avg_role_model_g_mag_loss': 0.019288771962617628, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003044349568921866, 'n_size': 900, 'n_batch': 225, 'duration': 258.8182637691498, 'duration_batch': 1.1503033945295547, 'duration_size': 0.28757584863238866, 'avg_pred_std': 0.09951512091689639}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036749178177625354, 'avg_role_model_std_loss': 5.0160115570675385, 'avg_role_model_mean_pred_loss': 3.591052332840726e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036749178177625354, 'n_size': 450, 'n_batch': 113, 'duration': 88.94795989990234, 'duration_batch': 0.7871500876097552, 'duration_size': 0.1976621331108941, 'avg_pred_std': 0.050722146361439895}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026951473932907296, 'avg_role_model_std_loss': 0.46114019461917133, 'avg_role_model_mean_pred_loss': 1.2756809318339692e-05, 'avg_role_model_g_mag_loss': 0.018491813861118214, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027525624157472826, 'n_size': 900, 'n_batch': 225, 'duration': 260.7371289730072, 'duration_batch': 1.1588316843244764, 'duration_size': 0.2897079210811191, 'avg_pred_std': 0.09791627071694367}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032899318864413846, 'avg_role_model_std_loss': 2.2600422874186443, 'avg_role_model_mean_pred_loss': 1.737884139242202e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032899318864413846, 'n_size': 450, 'n_batch': 113, 'duration': 89.44571375846863, 'duration_batch': 0.7915549890129967, 'duration_size': 0.19876825279659696, 'avg_pred_std': 0.05520178424967123}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002675653763451717, 'avg_role_model_std_loss': 0.38926185907018096, 'avg_role_model_mean_pred_loss': 1.1465693117683688e-05, 'avg_role_model_g_mag_loss': 0.018026919938856734, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002713999592718513, 'n_size': 900, 'n_batch': 225, 'duration': 261.18925762176514, 'duration_batch': 1.1608411449856229, 'duration_size': 0.2902102862464057, 'avg_pred_std': 0.10430583260332545}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00328355419371898, 'avg_role_model_std_loss': 3.905635658147522, 'avg_role_model_mean_pred_loss': 2.2026288097660226e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00328355419371898, 'n_size': 450, 'n_batch': 113, 'duration': 89.86624097824097, 'duration_batch': 0.7952764688339908, 'duration_size': 0.19970275772942436, 'avg_pred_std': 0.047640036822469756}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002428213983172706, 'avg_role_model_std_loss': 0.4563769066303573, 'avg_role_model_mean_pred_loss': 1.070831390430716e-05, 'avg_role_model_g_mag_loss': 0.01809621533375725, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024591475264686678, 'n_size': 900, 'n_batch': 225, 'duration': 262.2440469264984, 'duration_batch': 1.1655290974511041, 'duration_size': 0.29138227436277603, 'avg_pred_std': 0.10204424848676556}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003275549128625749, 'avg_role_model_std_loss': 3.9855575082005594, 'avg_role_model_mean_pred_loss': 1.929245272713863e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003275549128625749, 'n_size': 450, 'n_batch': 113, 'duration': 90.92798852920532, 'duration_batch': 0.8046724648602241, 'duration_size': 0.2020621967315674, 'avg_pred_std': 0.04646150635676953}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023822973380447365, 'avg_role_model_std_loss': 0.44478256372083136, 'avg_role_model_mean_pred_loss': 9.005847006119572e-06, 'avg_role_model_g_mag_loss': 0.01762347323496619, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002411980113861824, 'n_size': 900, 'n_batch': 225, 'duration': 263.1176962852478, 'duration_batch': 1.1694119834899903, 'duration_size': 0.2923529958724976, 'avg_pred_std': 0.10207737949159411}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0031462487001489435, 'avg_role_model_std_loss': 3.1703384120910294, 'avg_role_model_mean_pred_loss': 2.179660826174649e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031462487001489435, 'n_size': 450, 'n_batch': 113, 'duration': 88.3796284198761, 'duration_batch': 0.7821206054856292, 'duration_size': 0.19639917426639134, 'avg_pred_std': 0.047558308290564906}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021461909939373275, 'avg_role_model_std_loss': 0.3496849273867641, 'avg_role_model_mean_pred_loss': 8.150395166715502e-06, 'avg_role_model_g_mag_loss': 0.016793633546072266, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021742351456487084, 'n_size': 900, 'n_batch': 225, 'duration': 264.955197095871, 'duration_batch': 1.1775786537594266, 'duration_size': 0.29439466343985665, 'avg_pred_std': 0.10324391664730179}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00289536593284639, 'avg_role_model_std_loss': 3.853903681079736, 'avg_role_model_mean_pred_loss': 2.701310116102227e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00289536593284639, 'n_size': 450, 'n_batch': 113, 'duration': 92.01904273033142, 'duration_batch': 0.8143278117728444, 'duration_size': 0.2044867616229587, 'avg_pred_std': 0.05265144564250517}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002258309722690481, 'avg_role_model_std_loss': 0.5195468653853743, 'avg_role_model_mean_pred_loss': 7.549715697529972e-06, 'avg_role_model_g_mag_loss': 0.019823850064721128, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022869745167554355, 'n_size': 900, 'n_batch': 225, 'duration': 265.574116230011, 'duration_batch': 1.1803294054667155, 'duration_size': 0.2950823513666789, 'avg_pred_std': 0.10302899842564431}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032758567545291347, 'avg_role_model_std_loss': 1.3863763298404954, 'avg_role_model_mean_pred_loss': 4.0907979645609235e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032758567545291347, 'n_size': 450, 'n_batch': 113, 'duration': 89.21779704093933, 'duration_batch': 0.7895380269109675, 'duration_size': 0.1982617712020874, 'avg_pred_std': 0.0622333479762918}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021910814406065683, 'avg_role_model_std_loss': 0.2070099846257626, 'avg_role_model_mean_pred_loss': 8.39444046341834e-06, 'avg_role_model_g_mag_loss': 0.017909487343212176, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002219982292638936, 'n_size': 900, 'n_batch': 225, 'duration': 260.76630783081055, 'duration_batch': 1.1589613681369357, 'duration_size': 0.28974034203423393, 'avg_pred_std': 0.10706099790003565}\n", + "Time out: 3783.828666448593/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0026995842230168082, 'avg_g_mag_loss': 0.019575848251313577, 'avg_g_cos_loss': 0.035776930846345964, 'pred_duration': 4.142037391662598, 'grad_duration': 12.572051525115967, 'total_duration': 16.714088916778564, 'pred_std': 0.0932064801454544, 'std_loss': 0.02437058463692665, 'mean_pred_loss': 4.6931305405450985e-05, 'pred_rmse': 0.051957521587610245, 'pred_mae': 0.037075866013765335, 'pred_mape': 0.11939958482980728, 'grad_rmse': 0.033828798681497574, 'grad_mae': 0.01925434172153473, 'grad_mape': 0.510366678237915}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0026995842230168082, 'avg_g_mag_loss': 0.019575848251313577, 'avg_g_cos_loss': 0.035776930846345964, 'avg_pred_duration': 4.142037391662598, 'avg_grad_duration': 12.572051525115967, 'avg_total_duration': 16.714088916778564, 'avg_pred_std': 0.0932064801454544, 'avg_std_loss': 0.02437058463692665, 'avg_mean_pred_loss': 4.6931305405450985e-05}, 'min_metrics': {'avg_loss': 0.0026995842230168082, 'avg_g_mag_loss': 0.019575848251313577, 'avg_g_cos_loss': 0.035776930846345964, 'pred_duration': 4.142037391662598, 'grad_duration': 12.572051525115967, 'total_duration': 16.714088916778564, 'pred_std': 0.0932064801454544, 'std_loss': 0.02437058463692665, 'mean_pred_loss': 4.6931305405450985e-05, 'pred_rmse': 0.051957521587610245, 'pred_mae': 0.037075866013765335, 'pred_mape': 0.11939958482980728, 'grad_rmse': 0.033828798681497574, 'grad_mae': 0.01925434172153473, 'grad_mape': 0.510366678237915}, 'model_metrics': {'tvae': {'avg_loss': 0.0026995842230168082, 'avg_g_mag_loss': 0.019575848251313577, 'avg_g_cos_loss': 0.035776930846345964, 'pred_duration': 4.142037391662598, 'grad_duration': 12.572051525115967, 'total_duration': 16.714088916778564, 'pred_std': 0.0932064801454544, 'std_loss': 0.02437058463692665, 'mean_pred_loss': 4.6931305405450985e-05, 'pred_rmse': 0.051957521587610245, 'pred_mae': 0.037075866013765335, 'pred_mape': 0.11939958482980728, 'grad_rmse': 0.033828798681497574, 'grad_mae': 0.01925434172153473, 'grad_mape': 0.510366678237915}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:29.361660Z", + "iopub.status.busy": "2024-03-22T19:31:29.361330Z", + "iopub.status.idle": "2024-03-22T19:31:29.365755Z", + "shell.execute_reply": "2024-03-22T19:31:29.364986Z" + }, + "papermill": { + "duration": 0.023195, + "end_time": "2024-03-22T19:31:29.367546", + "exception": false, + "start_time": "2024-03-22T19:31:29.344351", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:29.398613Z", + "iopub.status.busy": "2024-03-22T19:31:29.398328Z", + "iopub.status.idle": "2024-03-22T19:31:29.489852Z", + "shell.execute_reply": "2024-03-22T19:31:29.488974Z" + }, + "papermill": { + "duration": 0.110023, + "end_time": "2024-03-22T19:31:29.492433", + "exception": false, + "start_time": "2024-03-22T19:31:29.382410", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:29.529036Z", + "iopub.status.busy": "2024-03-22T19:31:29.528298Z", + "iopub.status.idle": "2024-03-22T19:31:29.800678Z", + "shell.execute_reply": "2024-03-22T19:31:29.799685Z" + }, + "papermill": { + "duration": 0.292892, + "end_time": "2024-03-22T19:31:29.802904", + "exception": false, + "start_time": "2024-03-22T19:31:29.510012", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3EUlEQVR4nO3dfVxUZf4//teZYW64nYFBboZ7TcNVNFcXwkxrZaNiK8qyXFJyzZsWy2Ir19qkm1+ibf7qk7mV9Vnd3STNVveza9ZmqLUqiookhpI3CCp3IjLcz+31/eMwRwZmYIaZYUDez8fjPBjOuebMNSPz8pzrXOe6OMYYAyGEDBEiT1eAEEIcQaFFCBlSKLQIIUMKhRYhZEih0CKEDCkUWoSQIYVCixAypHh5ugIDxWQyoaqqCv7+/uA4ztPVIYR0wxhDc3Mz1Go1RCLbx1PDJrSqqqoQFRXl6WoQQvpw8eJFREZG2tw+bELL398fAP+BBAQEeLg2hJDumpqaEBUVJXxXbRk2oWU+JQwICKDQImQQ66v5hhriCSFDCoUWIWRIodAihAwpw6ZNizjHaDRCr9d7uhpkCJNIJBCLxU7vh0KL9IoxhpqaGjQ2Nnq6KuQGoFQqERYW5lRfSQot0itzYIWEhMDHx4c65pJ+YYyhra0NdXV1AIDw8PB+74tCq5vmDj1O1zTDYGRIHqXydHU8ymg0CoGlUg3vz4I4z9vbGwBQV1eHkJCQfp8qUkN8N4fON+CRDwvw5q5ST1fF48xtWD4+Ph6uCblRmP+WnGkfpdDqJi6Y/1Av1LeBhs/n0SkhcRVX/C1RaHUTGegDjgNatAZcbdV5ujqEkG4otLqRS8RQK/hz74qrrR6uDSGku36F1vr16xEbGwu5XI6kpCQUFhb2Wn7btm2Ij4+HXC5HQkICdu3aJWzT6/VYvnw5EhIS4OvrC7VajXnz5qGqqspiHw0NDcjIyEBAQACUSiUWLFiAlpaW/lS/T7Gdp4jl9W1u2T+58XEch3/+85+eroZLvfrqq7jllls8XQ3HQ2vr1q3Izs5GTk4OioqKMHHiRKSmpgqXMrs7ePAg5syZgwULFuD48eNIT09Heno6Tp48CQBoa2tDUVERXnnlFRQVFWH79u0oKyvD/fffb7GfjIwM/Pjjj9i9ezd27tyJ77//HosWLerHW+5bjMoXAB1pkaFt06ZNUCqVLtvf888/j/z8fJftr9+YgxITE1lWVpbwu9FoZGq1muXm5lotP3v2bJaWlmaxLikpiS1evNjmaxQWFjIArKKigjHGWGlpKQPAjhw5IpT56quvGMdx7PLly3bVW6PRMABMo9H0Wfaj786ymOU72dK8Irv2faNqb29npaWlrL293dNVGXIAsB07dni0Dhs3bmQKhaLPclqt1v2V6dTb35S931GHjrR0Oh2OHTuGlJQUYZ1IJEJKSgoKCgqsPqegoMCiPACkpqbaLA8AGo0GHMcJ/0sUFBRAqVRiypQpQpmUlBSIRCIcPnzY6j60Wi2amposFnvFdh5pXainI62uGGNo0xk8sjAHr+R+/fXXmDZtGpRKJVQqFX7961/j3LlzAICpU6di+fLlFuWvXLkCiUSC77//HgBQXV2NtLQ0eHt7Iy4uDnl5eYiNjcW7777br8+upKQEv/zlL+Ht7Q2VSoVFixZZNG/s27cPiYmJ8PX1hVKpxG233YaKigoAwA8//IA777wT/v7+CAgIwOTJk3H06NFeX2/fvn2YP3++8F3iOA6vvvoqACA2NhZvvPEG5s2bh4CAAOGMZfny5RgzZgx8fHwwcuRIvPLKKxZdE7qfHj7xxBNIT0/H22+/jfDwcKhUKmRlZbn9di+HOpfW19fDaDQiNDTUYn1oaChOnz5t9Tk1NTVWy9fU1Fgt39HRgeXLl2POnDnCuFc1NTUICQmxrLiXF4KCgmzuJzc3F6+99ppd76u72ODO0LraCsYYXfLv1K434mcr/+OR1y59PRU+Uvv/XFtbW5GdnY0JEyagpaUFK1euxIMPPoji4mJkZGTgrbfewurVq4V/261bt0KtVuP2228HAMybNw/19fXYt28fJBIJsrOzbTaB2FOX1NRUJCcn48iRI6irq8OTTz6JpUuXYtOmTTAYDEhPT8fChQvx2WefQafTobCwUKhbRkYGJk2ahA8++ABisRjFxcWQSCS9vubUqVPx7rvvYuXKlSgrKwMA+Pn5CdvffvttrFy5Ejk5OcI6f39/bNq0CWq1GiUlJVi4cCH8/f3x4osv2nydvXv3Ijw8HHv37sXZs2fx6KOP4pZbbsHChQv79VnZY1D1iNfr9Zg9ezYYY/jggw+c2teKFSuQnZ0t/G4eFdEe0UF8Q3xzhwHX2vQI8pU6VRcy8GbNmmXx+1/+8heMGDECpaWlmD17Np599lns379fCKm8vDzMmTMHHMfh9OnT+Pbbb3HkyBHh6P6TTz7B6NGj+1WXvLw8dHR04G9/+xt8ffn/EN9//33cd999WLNmDSQSCTQaDX79619j1KhRAICxY8cKz6+srMQLL7yA+Ph4ALCrHlKpFAqFAhzHISwsrMf2X/7yl/j9739vse6Pf/yj8Dg2NhbPP/88tmzZ0mtoBQYG4v3334dYLEZ8fDzS0tKQn58/eEIrODgYYrEYtbW1Futra2utfjAAEBYWZld5c2BVVFRgz549FqOLhoWF9fhfzmAwoKGhwebrymQyyGQyu99bV3y3BzmqNB0or2+l0OrkLRGj9PVUj722I86cOYOVK1fi8OHDqK+vh8lkAsAHwPjx43HXXXdh8+bNuP3221FeXo6CggJ89NFHAICysjJ4eXnh5z//ubC/m266CYGBgf2q+6lTpzBx4kQhsADgtttug8lkQllZGaZPn44nnngCqamp+NWvfoWUlBTMnj1buD8vOzsbTz75JP7+978jJSUFjzzyiBBu/dW1qcVs69ateO+993Du3Dm0tLTAYDD0OcrvuHHjLG7HCQ8PR0lJiVN164tDbVpSqRSTJ0+2uIJgMpmQn5+P5ORkq89JTk7uccVh9+7dFuXNgXXmzBl8++23Pe5zS05ORmNjI44dOyas27NnD0wmE5KSkhx5C3ajK4g9cRwHH6mXRxZHT9Hvu+8+NDQ04OOPP8bhw4eFtk+dju8wnJGRgS+++AJ6vR55eXlISEhAQkKCyz8ze23cuBEFBQWYOnUqtm7dijFjxuDQoUMA+LakH3/8EWlpadizZw9+9rOfYceOHU69XtcABfh244yMDNx7773YuXMnjh8/jpdffln4vGzpfprKcZzwH4S7ONzlITs7Gx9//DH++te/4tSpU3jqqafQ2tqK+fPnA+DbAlasWCGUX7ZsGb7++musXbsWp0+fxquvvoqjR49i6dKlAPjAevjhh3H06FFs3rwZRqMRNTU1qKmpET6wsWPH4u6778bChQtRWFiIAwcOYOnSpXjsscegVqtd8Tn0YO6rdeEq9dUaaq5evYqysjL88Y9/xMyZMzF27Fhcu3bNoswDDzyAjo4OfP3118jLy0NGRoaw7eabb4bBYMDx48eFdWfPnu2xD3uNHTsWP/zwA1pbr/8HeODAAYhEItx8883CukmTJmHFihU4ePAgxo8fj7y8PGHbmDFj8Nxzz+Gbb77BQw89hI0bN/b5ulKpFEaj0a46Hjx4EDExMXj55ZcxZcoUjB49WrgQMNg4HFqPPvqo0Ih3yy23oLi4GF9//bXQ2F5ZWYnq6mqh/NSpU5GXl4cNGzZg4sSJ+OKLL/DPf/4T48ePBwBcvnwZ//rXv3Dp0iXccsstCA8PF5aDBw8K+9m8eTPi4+Mxc+ZM3HvvvZg2bRo2bNjg7Pu3ia4gDl2BgYFQqVTYsGEDzp49iz179li0bwL8kUZ6ejpeeeUVnDp1CnPmzBG2xcfHIyUlBYsWLUJhYSGOHz+ORYsWwdvbu18XZTIyMiCXy5GZmYmTJ09i7969ePrppzF37lyEhoaivLwcK1asQEFBASoqKvDNN9/gzJkzGDt2LNrb27F06VLs27cPFRUVOHDgAI4cOWLR5mVLbGwsWlpakJ+fj/r6erS12f4PePTo0aisrMSWLVtw7tw5vPfee04fzbmNO/piDEaO9NNijLGvSqpZzPKd7P51/3VzzQavodxPa/fu3Wzs2LFMJpOxCRMmsH379vXoO7Vr1y4GgE2fPr3H86uqqtg999zDZDIZi4mJYXl5eSwkJIR9+OGHdr1+99c6ceIEu/POO5lcLmdBQUFs4cKFrLm5mTHGWE1NDUtPT2fh4eFMKpWymJgYtnLlSmY0GplWq2WPPfYYi4qKYlKplKnVarZ06VK7/02WLFnCVCoVA8BycnIYY4zFxMSwd955p0fZF154galUKubn58ceffRR9s4771j088rJyWETJ04Ufs/MzGQPPPCAxT6WLVvGZsyYYbM+ruinxTE2PIYyaGpqgkKhgEajsWsKsdM1Tbj73f9C4S3BDzl3DUANB5+Ojg6Ul5cjLi4Ocrnc09XxqEuXLiEqKgrffvstZs6c6enqDFm9/U3Z+x0dVF0eBpOYIP70UNOux7VWHQLpCuKwsmfPHrS0tCAhIQHV1dV48cUXERsbi+nTp3u6asMejfJgg7dUjLAA/n+CC3QFcdjR6/V46aWXMG7cODz44IMYMWKE0NF08+bN8PPzs7qMGzduwOp4zz332KzHqlWrBqweA42OtHoRo/JBTVMHKq62YVJ0//rokKEpNTUVqanW+6Tdf//9Nrva9NVT3ZU++eQTtLe3W90WFBQ0YPUYaBRavYgL9sXh8gaU0xVE0oW/vz/8/f09XQ1ERER4ugoeQaeHvaAOpoQMPhRavYhVUQdTQgYbCq1edB3tgRAyOFBo9SKm80irsU2Pxjaa5IKQwYBCqxc+Ui+E+PMjRVTQKSIhgwKFVh+EexDpFJE44Eac2GKwoNDqQ2yXyVsJGUpcPbEFwA/jzHEcGhsbXbpfR1Bo9YG6PRAyuFBo9cF8elhOoQUwBuhaPbPQxBYum9hCq9Xi+eefR0REBHx9fZGUlIR9+/YJz62oqMB9992HwMBA+Pr6Yty4cdi1axcuXLiAO++8EwA//A/HcXjiiSf69Xk4g3rE98F8ekgN8QD0bcAq9wy62KeXqgCpb9/lOtHEFrYntli6dClKS0uxZcsWqNVq7NixA3fffTdKSkowevRoZGVlQafT4fvvv4evry9KS0vh5+eHqKgo/OMf/8CsWbNQVlaGgIAAeHt79+szcQaFVh/Mp4cNrTpo2vVQeA/cvWWk/2hiC+sTW1RWVmLjxo2orKwURv19/vnn8fXXX2Pjxo1YtWoVKisrMWvWLGH46ZEjRwrPN9/TGBIS4vL2MntRaPXBT+aFYD8Z6lu0qLjaigmRSk9XyXMkPvwRj6de2wE0sYV1JSUlMBqNGDNmjMV6rVYrzM3wzDPP4KmnnsI333yDlJQUzJo1CxMmTOjX67kDtWnZIY7Gi+dxHH+K5omFJrZwycQWLS0tEIvFOHbsGIqLi4Xl1KlT+J//+R8AwJNPPonz589j7ty5KCkpwZQpU7Bu3TqXvVdnUWjZQbiCSKM9DAk0sQXP2sQWkyZNgtFoRF1dHW666SaLpetpZFRUFJYsWYLt27fj97//PT7++GNhnwDsnjDDHSi07EA3Tg8tNLEFz9rEFmPGjEFGRgbmzZuH7du3o7y8HIWFhcjNzcWXX34JAHj22Wfxn//8B+Xl5SgqKsLevXuF14uJiQHHcdi5cyeuXLlicQV0wPQ6gvwNxNGJLbr69w+XWczyneyhPx9wQ80GL5rY4sac2EKn07GVK1ey2NhYJpFIWHh4OHvwwQfZiRMnGGOMLV26lI0aNYrJZDI2YsQINnfuXFZfXy/s8/XXX2dhYWGM4ziWmZlpVz3MaGILBzg6sUVXJy9r8Ot1+xHsJ8XRP/7KTTUcfGhii+toYgvXoIktBoh5tIf6Fh2aO/Twl1O3hxsdTWwxeFGblh385RIE+/ENkNTJdHigiS0GLzrSslOMyhf1LTpcuNqK8REKT1eHuBlNbDF4UWjZKUblg2MV1+hIi9DEFh5Gp4d2ijPfOD0M+2oNk2s1ZAC44m+JQstOMcHDb4ga86lOWxsdXRLXMP8tOXMaTaeHdhqOHUzFYjGUSqUwuoGPj0+/OlcSwhhDW1sb6urqoFQqIRaL+70vCi07mW/ludKsRYvWAD/Z8PjozLd29HdYFkK6UiqVFrcL9cfw+Oa5gMJbgiBfKRpadai42opx6uFxBZHjOISHhyMkJAR6vd7T1SFDmEQiceoIy4xCywExKp/O0GobNqFlJhaLXfIHR4izqCHeAcP5CiIhgwWFlgNokgtCPI9CywGxNBggIR5HoeUAYeJWOj0kxGMotBxgDq26Zi3adAYP14aQ4YlCywEKHwmUPnxPXroHkRDPoNByEJ0iEuJZFFoOGo638xAymFBoOYi6PRDiWRRaDooLpg6mhHgShZaDzOPFU0M8IZ5BoeUgc0N8TVMH2nWem7CSkOGKQstBgb5SKLw7uz000CkiIQONQqsfhCuI9XSKSMhA61dorV+/HrGxsZDL5UhKSkJhYWGv5bdt24b4+HjI5XIkJCRg165dFtu3b9+Ou+66CyqVChzHobi4uMc+7rjjDnAcZ7EsWbKkP9V3Gl1BJMRzHA6trVu3Ijs7Gzk5OSgqKsLEiRORmppqc2TLgwcPYs6cOViwYAGOHz+O9PR0pKen4+TJk0KZ1tZWTJs2DWvWrOn1tRcuXIjq6mpheeuttxytvkvEdl5BvEChRcjAYw5KTExkWVlZwu9Go5Gp1WqWm5trtfzs2bNZWlqaxbqkpCS2ePHiHmXLy8sZAHb8+PEe22bMmMGWLVvmaHUFGo2GAWAajabf+zD7x7GLLGb5TvbYRwVO74sQwrP3O+rQkZZOp8OxY8eQkpIirBOJREhJSUFBQYHV5xQUFFiUB/iJMG2V783mzZsRHByM8ePHY8WKFb3OEqPVatHU1GSxuAqdHhLiOQ4Nt1xfXw+j0YjQ0FCL9aGhoTh9+rTV59TU1FgtX1NT41BFf/Ob3yAmJgZqtRonTpzA8uXLUVZWhu3bt1stn5ubi9dee82h17CXuYNplaYDHXoj5BIahpiQgTJkxohftGiR8DghIQHh4eGYOXMmzp07h1GjRvUov2LFCmRnZwu/NzU1ISoqyiV1CfSRwF/uheYOAyob2jAm1POzDRMyXDh0ehgcHAyxWIza2lqL9bW1tTanBQoLC3OovL2SkpIAAGfPnrW6XSaTISAgwGJxFY7jaLQHQjzEodCSSqWYPHky8vPzhXUmkwn5+flITk62+pzk5GSL8gCwe/dum+XtZe4WER4e7tR++ouuIBLiGQ6fHmZnZyMzMxNTpkxBYmIi3n33XbS2tmL+/PkAgHnz5iEiIgK5ubkAgGXLlmHGjBlYu3Yt0tLSsGXLFhw9ehQbNmwQ9tnQ0IDKykpUVVUBAMrKygDwR2lhYWE4d+4c8vLycO+990KlUuHEiRN47rnnMH36dEyYMMHpD6E/aIgaQjykP5cm161bx6Kjo5lUKmWJiYns0KFDwrYZM2awzMxMi/Kff/45GzNmDJNKpWzcuHHsyy+/tNi+ceNGBqDHkpOTwxhjrLKykk2fPp0FBQUxmUzGbrrpJvbCCy841H3BlV0eGGNs21G+28NvPqZuD4S4gr3fUY4xxjyYmQOmqakJCoUCGo3GJe1bRy804OEPCxCh9MaBP/zSBTUkZHiz9ztK9x72U6zQ7aEdHXoa7YGQgUKh1U8qXyn8ZF5gDLh0jdq1CBkoFFr9xHGcMCBgOY32QMiAodDqrkMD/PQNUPZVn0XNp4h0Ow8hA2fI9IgfMOf3AZ/PA8ISgJvv6bXo9W4PFFqEDBQ60uouYgr/s7YU0PV+2hcj9Iqn00NCBgqFVneKCMA/HGBGoLq416Jx1CuekAFHoWVNZOfR1qUjvRYzN8RXNbZDa6BuD4QMBAota8yniJeO9lpshJ8MvlIxTAy42NA+ABUjhFBoWRP5C/5nH6HFd3ugK4iEDCQKLWvUtwCcGGiuAjSXey0aG0w3ThMykCi0rJH6AqE/4x9f7v1oK4bG1SJkQFFo2WJnu1aciq4gEjKQKLRssbNdy3wFsYJODwkZEBRatpi7PVQdB4wGm8XMt/JcutYGncE0EDUjZFij0LJFNRqQKQBDO1D3o81iIf4yeEv4bg802gMh7kehZYtIBET8nH/cyyli19Ee6BSREPej0OqNne1a5pl5yukKIiFuR6HVG3O7Vh/dHmiIGkIGDoVWb8zdHup/Atqv2SxGM/MQMnAotHrjqwIC4/jHl4tsFouhvlqEDBgKrb7Y0a4VJ3R7aIfeSN0eCHEnCq2+2NGuFeIvg1wigtHEcPkajfZAiDtRaPUlssvtPDamiBSJOMQEdV5BpFNEQtyKQqsvoQmAWAa0NwAN520WM4/2UEHdHghxKwqtvnhJgfCJ/ONe2rVihcZ4uoJIiDtRaNnDjnYtuoJIyMCg0LJHxGT+Zy9jxgunh3SkRYhbUWjZw9ztoeYkoO+wWsR8enixoQ0G6vZAiNtQaNlDGQ34jgBMeqDmhNUiYQFyyLxEMJgYLjdStwdC3IVCyx4c16WTqfVTRJHo+mgP1BhPiPtQaNlLaNfquzGebpwmxH0otOxlx+085hunaYgaQtyHQste6kkAOEBTCTTXWi1yfYgaOj0kxF0otOwlDwBCxvKPbfTXiqW+WoS4HYWWI/po1zI3xFO3B0Lch0LLEX1cQVQrvCH1EkFvZKjWWO/PRQhxDoWWI7pOK2Yy9tgsEnGIDjJ3e6BTRELcgULLESPiAakfoGsBrpy2WoSGXibEvSi0HCESd15FhM12LaExnro9EOIWFFqO6qNdK4Zm5iHErSi0HCUMU3PM6mY6PSTEvSi0HGWeVqzuFNDR1GOz+fSw8mobjCbrwzMTQvqPQstR/qGAIhoA468idqNWekMi5qAzmlCtodEeCHE1Cq3+ECa76NmuJRZxiAqiAQEJcRcKrf7os12rc2YeuoJIiMv1K7TWr1+P2NhYyOVyJCUlobCwsNfy27ZtQ3x8PORyORISErBr1y6L7du3b8ddd90FlUoFjuNQXFzcYx8dHR3IysqCSqWCn58fZs2ahdpa6zcuu13XK4hWphWLpSFqCHEbh0Nr69atyM7ORk5ODoqKijBx4kSkpqairq7OavmDBw9izpw5WLBgAY4fP4709HSkp6fj5MmTQpnW1lZMmzYNa9assfm6zz33HP79739j27Zt+O6771BVVYWHHnrI0eq7RtgEQCQBWq8AjZU9NpvHi6criIS4AXNQYmIiy8rKEn43Go1MrVaz3Nxcq+Vnz57N0tLSLNYlJSWxxYsX9yhbXl7OALDjx49brG9sbGQSiYRt27ZNWHfq1CkGgBUUFNhVb41GwwAwjUZjV/k+fTSDsZwAxk5s67FpX1kdi1m+k6Ws3eea1yJkGLD3O+rQkZZOp8OxY8eQkpIirBOJREhJSUFBQYHV5xQUFFiUB4DU1FSb5a05duwY9Hq9xX7i4+MRHR1tcz9arRZNTU0Wi0uZTxGttGvFmU8PG9pgom4PhLiUQ6FVX18Po9GI0NBQi/WhoaGoqamx+pyamhqHytvah1QqhVKptHs/ubm5UCgUwhIVFWX369klwvYVRLVSDi8RB53BhJomGu2BEFe6Ya8erlixAhqNRlguXrzo2hcwX0GsPgEYtBabvMQiodsD3YNIiGs5FFrBwcEQi8U9rtrV1tYiLCzM6nPCwsIcKm9rHzqdDo2NjXbvRyaTISAgwGJxqaCRgHcQYNTy8yF2Q7fzEOIeDoWWVCrF5MmTkZ+fL6wzmUzIz89HcnKy1eckJydblAeA3bt32yxvzeTJkyGRSCz2U1ZWhsrKSof241Ic16W/Vs8RH2hmHkLcw8vRJ2RnZyMzMxNTpkxBYmIi3n33XbS2tmL+/PkAgHnz5iEiIgK5ubkAgGXLlmHGjBlYu3Yt0tLSsGXLFhw9ehQbNmwQ9tnQ0IDKykpUVVUB4AMJ4I+wwsLCoFAosGDBAmRnZyMoKAgBAQF4+umnkZycjFtvvdXpD6HfIqYAZ77hh6lJWmyxiWbmIcRN+nNpct26dSw6OppJpVKWmJjIDh06JGybMWMGy8zMtCj/+eefszFjxjCpVMrGjRvHvvzyS4vtGzduZAB6LDk5OUKZ9vZ29rvf/Y4FBgYyHx8f9uCDD7Lq6mq76+zyLg+MMXbmW77bw7sTe2zae7qWxSzfye76/79z3esRcgOz9zvKMWalS/cNqKmpCQqFAhqNxnXtW+2NwJoY/vEL5wFflbDpQn0r7nh7H+QSEUpfuxsiEeea1yTkBmXvd/SGvXo4ILyVQPAY/nG3dq2IQG+IRRw69CbUNlO3B0JchULLWUJ/LcvQkohFiAr0BgBcqKcriIS4CoWWs3oZpoauIBLiehRazhK6PRQBJssJWoUriBRahLgMhZazQsYBXt6AVgNcPWOxKdY8yQWdHhLiMhRazhJ72ZxWTJhOjI60CHEZCi1XsNGuFaO6PuzyMOlZQojbUWi5go3beSIDfSAWcWjXG1HXrLXyREKIoyi0XME8tlbtj4Du+qmg1EuECKW52wOdIhLiChRarhCgBvzVADMBVcUWm2KE0R4otAhxBQotV7HRrhUXbG6MpyuIhLgChZar2GjXog6mhLgWhZar2Lid5/oQNXSkRYgrUGi5ivoWgBMDzdWA5rKwWuhgerWVuj0Q4gIUWq4i9QVCf8Y/7tKuFRnoDREHtOmMuNJC3R4IcRaFlisJ04pdP0WUeYmhVtJoD4S4CoWWK9lo17p+BZEa4wlxFoWWK5mPtKqKAaNeWH39dh4KLUKcRaHlSqqbAJkCMLTzveM7CTdO0+khIU6j0HIlkQiInMw/7tKuRaM9EOI6FFquJrRrHRNWxQbTaA+EuAqFlquZ27Usuj34gOOAFq0B9S06D1WMkBsDhZarRXSeHl49A7RfAwDIJWKoFXy3B2qMJ8Q5FFqu5qsCgkbyjy/3PEWkG6cJcQ6FljtYadeKEa4g0pEWIc6g0HIHK+1acXQFkRCXoNByh67dHjqvFnYdL54Q0n8UWu4QmgCIZXxDfMN5ANdHe7hQT6M9EOIMCi138JIC4RP5x533IUYH8d0emrUGNLRStwdC+otCy126tWvJJWKEB8gB0BVEQpxBoeUuVm7noSuIhDiPQstdzEdaNSWAvh2A5SimhJD+odByF0UU4BsCmAxA9QkA18eLp9NDQvqPQstdOK7HtGIx1FeLEKdRaLlTt2nFzLfylFO3B0L6jULLnboNvxwTxB9pNXcY0Nimt/UsQkgvKLTcKeLnADhAcxForoG3VIywzm4PP9U2e7ZuhAxRFFruJPMHQsbyjzuPthIiFQCA57/4AZeuUYM8IY6i0HK3bu1arz8wDrEqH1xsaMdjGw7hYoMLg+vCAeDjmcCeN4V7Hgm50VBouVu3dq1whTe2LEpGXLAvLl1zUXAZDcCe/w/YlMaH4/dvAftWO1lxQgYnCi13EyZwLQJMRgBAmEKOzxbeipHBvrjcyAdXZX/7bl27AGy8B/j+TwAYEDONX//dauDwBqerT8hgQ6HlbiNuBqR+gL4VqDslrA5TyPHZolsxcoQ5uAoc7ylf8gXw4e3ApUJAFgA8/Bdg/pfAHS/x2796kS9DyA2EQsvdROLOq4iwuA8RAEID5Niy8FaMGuGLKk0HHttwyL77ErXNwI6ngH8sALRNQFQSsGQ/MH4Wv33Gi0DiIgAM2LEYOPOta98TIR5EoTUQurVrdRUSwB9x3RTih+rO4CrvLbguFwEfTQd+yAM4ETBjOfDELiAw5noZjgPuXgOMf5i/jejzucDFQhe/KUI8g0JrIAjD1PQMLQAI8efbuEaH+KGmqQOPbSjA+SstloVMJmD/u8D//oofWDAgEnjiS+DOlwCxV8+dikRA+gfATSmAvg3Y/IjF6SkhQxWF1kAwd3u4chroaLJaZIS/DJ8tuhVjQv1Q26TFYxsO4Zw5uJprgE8fBL7N4Y+cxt4PPLUfiJna++t6SYHZfwMiE4GORuDvDwLXKlz3vgjxgH6F1vr16xEbGwu5XI6kpCQUFvZ+6rFt2zbEx8dDLpcjISEBu3btstjOGMPKlSsRHh4Ob29vpKSk4MyZMxZlYmNjwXGcxbJ69RC5rO8XAiijATCgqshmsWA/GT5beCviw/xR18wHV1XhDuCDqcD5fYDEB7jvPT6IvAPte22pL/CbrcCIsUBzNR9cLVdc8rYI8QSHQ2vr1q3Izs5GTk4OioqKMHHiRKSmpqKurs5q+YMHD2LOnDlYsGABjh8/jvT0dKSnp+PkyZNCmbfeegvvvfcePvzwQxw+fBi+vr5ITU1FR0eHxb5ef/11VFdXC8vTTz/taPU9p5d2ra5UfjJsfjIJCaEy/K59A9S7ngDargJhCcCi74DJmXyblSN8goC52wFFNNBwDtg8y+YRHyGDHnNQYmIiy8rKEn43Go1MrVaz3Nxcq+Vnz57N0tLSLNYlJSWxxYsXM8YYM5lMLCwsjP3pT38Stjc2NjKZTMY+++wzYV1MTAx75513HK2uQKPRMABMo9H0ex9OObiesZwAxjY/2nfZ2lNMv+5WvnxOANv86hx25vIV5+tw5Qxja0by+92Yxpiu3fl9EuIi9n5HHTrS0ul0OHbsGFJSUoR1IpEIKSkpKCgosPqcgoICi/IAkJqaKpQvLy9HTU2NRRmFQoGkpKQe+1y9ejVUKhUmTZqEP/3pTzAYDDbrqtVq0dTUZLF4VNfbeWzdYsMYcOR/gQ0z4FVfCpPPCKz0exUvtWfgsb8cd/4m6+CbgMe/AKT+wIX/8l0mjLY/Q0IGI4dCq76+HkajEaGhoRbrQ0NDUVNTY/U5NTU1vZY3/+xrn8888wy2bNmCvXv3YvHixVi1ahVefPFFm3XNzc2FQqEQlqioKPvfqDuETQBEEqD1CtBopTG8rQHY+jjwZTZg6ABGzYTodweR/bssjFMHoL5FhzkbDqGsxsngUk8C5uQBYilweiew81m6T5EMKUPm6mF2djbuuOMOTJgwAUuWLMHatWuxbt06aLVaq+VXrFgBjUYjLBcvXhzgGncjkfPtUkDPdq3y74EPbuNDRCQB7noTyPgC8AuB0kfKt3FFKHC1VYc5Hx/C6RonjxrjpvO95zkRcPzvQP5rzu2PkAHkUGgFBwdDLBajtrbWYn1tbS3CwsKsPicsLKzX8uafjuwTAJKSkmAwGHDhwgWr22UyGQICAiwWj+veX8uoB/JfB/56P9BcBahGAwvzgalL+X5WnZQ+Uny6IAkTIhVoaOWPuEqrnAyusfcBv36Xf7z/HeDg+87tj5AB4lBoSaVSTJ48Gfn5+cI6k8mE/Px8JCcnW31OcnKyRXkA2L17t1A+Li4OYWFhFmWamppw+PBhm/sEgOLiYohEIoSEhDjyFjyra7tWQznwl7uB/64FwIBJc4HF312f5LUbhY8Ef1+QhImRClxr0yPjk0P4sUrjXH0mZwIpr/KPv3kZKM5zbn+EDARHW/i3bNnCZDIZ27RpEystLWWLFi1iSqWS1dTUMMYYmzt3LvvDH/4glD9w4ADz8vJib7/9Njt16hTLyclhEomElZSUCGVWr17NlEol+7//+z924sQJ9sADD7C4uDjW3s5f3Tp48CB75513WHFxMTt37hz79NNP2YgRI9i8efPsrrfHrx4yxlj9Wf7K3Wsqxt6M4B+vimKs5B9276KxTcfuf38/i1m+k0187T+s5FKjc3UymRj7+iW+Lq8GMnZ6l3P7I6Sf7P2OOhxajDG2bt06Fh0dzaRSKUtMTGSHDh0Sts2YMYNlZmZalP/888/ZmDFjmFQqZePGjWNffvmlxXaTycReeeUVFhoaymQyGZs5cyYrKysTth87dowlJSUxhULB5HI5Gzt2LFu1ahXr6Oiwu86DIrRMJsZWxwpdGdgndzF2rcLh3WjadeyBzuCa8KqLgmv7Er5Ob4QwVr7fuf0R0g/2fkc5xobHpaOmpiYoFApoNBrPtm/962ng+GZ+JIbbn7d+36Admjv0yPxLIYoqGxEg98LmJ28VhnLuF6OBv3r501f8MDfzd12/cEDIALD3O0qhNdAY44eWkTtfh+YOPZ7YeATHKq4hQO7Ft3lFKfu/Q3078PeHgMqD/ESzC/4DBI10up6E2MPe7+iQ6fJww+A4lwQWAPjLJfjrbxMxJSYQTR0GPP6/h1F8sbH/O5R4A3M+A0ITgNY64G/p/M3ahAwiFFpDnJ/MC5t+m4jE2CA0dxgw95PDOHC2vv879FYCj/8DCIzjO8F+Ogtob3RVdQlxGoXWDcBP5oWN83+BxLggNGsNyPjkMOZsOIQDZ+v7N5O1fygwdwfgFwrUngQ+ewzQ0XRnZHCg0LpB+Mq8sGn+LzAnMRoSMYeC81eR8clhPPjng/i2tNbx8AqKAx7fDsgUQGUB8MV8vjMsIR5GDfE3oKrGdmz4/jw+K6yE1mACAMSH+SPrzptwb0I4xCIHhrapKAD+ns7fDznhMX40VBH9X0dcj64edjOcQsvsSrMWn+w/j08LKtCq46cvGxnsiyV3jMKDkyIgEdsZPj/9B/hsDsCMwC8WAlOf5gc1dHRcL0J6QaHVzXAMLbPGNh02HbyAjQcuQNPOn+JFKL2xZMZIPDIlCnKJuO+d/LCFn9nHTKYAwsbzfbnMy4h4wEvmpndBbnQUWt0M59Aya9EasPlQBT7+bznqW/jRMUb4y7Dw9jhkJMXAV9ZHR9fiz4CC9fxY9yYr7VsiLyD45s4Q6wy00ATAV+WGd0NuNBRa3VBoXdehN2LrkYv46LtzqNLwQ1orfSSYPzUOT0yNhcJH0vsODDqg/iegpqRzOcH/7Gi0Xj4gAgjtdlQWGEdtY8QChVY3FFo96Qwm/PP4Zfx531lcuMp3afCTeWFucgwWTItDsJ8Dp3qMAU2XuwRZ53Kt3Hp5qR8QOq7zaGw8P0hiyFhA6uOCd0aGIgqtbii0bDOaGL4sqcb6PWdR1jmks1wiwmO/iMbiGSMRrvDu/847moC6UssjsrpT/NXIHjjAS975kON/t/jZWYaDlXXWynXbh9X92ihr90/z80V8EMsV9i+ygBvvaNNk4uflrC0Bak7y/961J4HF/+2zmYBCqxsKrb6ZTAzfnqrF+r1n8cMlfqwuiZjDw5MjsWTGKMSofF3zQkYDcPXs9SCrPQlUnwDanOjJPyRxfHB5m4NMaTvgfFSAIhJQRLnsNjCnaZuB2lLLgKor5ScH7m7e/wEj7+h1dxRa3VBo2Y8xhv1n6/H+nrM4XN4AABBxwP0T1Xjy9pG4Oczf/u4S9r8o0Frf+QfPOsetZ5bj15vXdX0sbLf2HNbLc1jn6u7PYwAzWVln6yeuP0fXAnRo7FusHmnaSa7gp4NTRgPKKD7IhJ/RfMC5sjsKY4DmIh9MtV2OnhrOWy/vJQdCfsZfjAntvCgTPpGfg7MXFFrdUGj1z5ELDXh/z1l899P1CV7FIg6Rgd6IUfkiVuWDWJUvYoN9EKPyRVSgD6ReN9gpjzvoOwBtU5cga+w95Frq+OBov9b3viU+/FGZMrpLoHUJOP8wQGSjm4u+A7hyqktAneSPpDpsjJLrF3b9arH5YkvQqH4NuUSh1Q2FlnNKLmnw531nsbesDh16k81yIg5QK70RF+yLmM5AM4dbVJCPfX3CiG3aFj68Gi8CmkqgsbLzcee6FjtG5RBJgAB155FaNB9ijZV8QNX/xHci7vGcbt1ZzAHlG+yyt0ah1Q2FlmswxlDXrEV5fSsqrrbiwtU2/md9Gy5cbUWbzsoffCeOA9QKb8SofK4fpQX7Ilbli+ggH3hLKdCcZtACmkvXQ6yx0jLkmqoAUx9zXXoHXr+iaw6oETe7veMwhVY3FFruxxjDlRYtKq624UJ9KyqutqH8aqsQai3a3r8soQEy+Msl8BJxkIhF8BJzkIhEEIs4/rFYBK/Ox16i69v53zl4dXuORNy5rnN/3lIxIpXeiAz0QZhCPjxPY01GoLm6y9FZZ5AFhPMhFTqePwrzwC1aFFrdUGh5FmMMDa06XBAC7fpRWnl9K5o6Bnama44DwgLkiFB6IzKQD7KIwOuP1Uo5ZF505DeQKLS6odAa3BrbdKi42oZWnQFGE4PByKA3mmAw8T+FdSYTDEYGg4nB0GW7tXVGE4PeyGDofE5Thx6XG9tx+Vq7MPpFb0L8ZZ1B5oPIQG/LgFN60+msi9n7He3frAqEuJjSRwqlj3RAXosxhvoWHS5da8PlxnZcusYH2aVrbfzjxna06Yyoa9airlmL45WNVvcT7CftDDL+KM1P5gUvMQepWASJsHCQeomEU1uJl6jLdv60Vep1/RTWXNa8TSIWOTaU0DBAoUWGHY7jMMJfhhH+MkyKDuyxnTGGa216PtSutQtBZg61S9fa0aI1oL5Fh/oWndAR111EHD/Io1rhjXClHGqlN9QKOcIV3vxjpRxhiuFzOkuhRUg3HMchyFeKIF8pJkQqe2xnjKGp3YCLXY7Uqhrb0a43Qm8wQW80QW9k0BlNwqmr+bHeaILewJ++6jq3mR+bn2c0WbbYmBjQ3GFAWUezcJuVNcF+MkQoLcNMrfRGuIJvuwv2k0HkwqM2ncGEFq0BLR0GNGv1aOkw8L9rDWg2P+78+bs7RyHEX+6S16XQIsRBHMdB4SOBwkeB8RFOzDVpg8nEt93pjUwIwaYOPaoaO1Ctacflxg5UN7ajStOO6sYOXG7k2+jqW7Sob9HaPPKTiDmEdR6hRXSGmVrpjbAAOYyMWQmd60HU3GUbH1IG6OxoFzR7eHIkhRYhNyqRiINMJIbMC0Bn16iQADluCvG3Wt58OlvV2C4s1Ro+zKo1fMDVNHVAb2S42NCOiw3tLq2vj1QMP5kX/ORe8O/86Sfzgp9MAv/Ox4G+rmuvpNAiZIjrejpr68jPYDShrlnLh5qmgw+2Rv6orbapAxIxBz+5hA+dLsFjDh3L3yXC775SMbxcfR9qHyi0CBkGvMSiznYuJ4YZGiSGYZdgQshQRqFFCBlSKLQIIUMKhRYhZEih0CKEDCkUWoSQIYVCixAypAybflrmEXiampo8XBNCiDXm72Zfo2UNm9BqbuZvNI2KivJwTQghvWluboZCYfuezmEzCKDJZEJVVRX8/f3B9TGUbFNTE6KionDx4sUhN2Ag1d0zqO7OY4yhubkZarUaol4msR02R1oikQiRkZEOPScgIGDI/QGaUd09g+runN6OsMyoIZ4QMqRQaBFChhQKLStkMhlycnIgk7l3njd3oLp7BtV94AybhnhCyI2BjrQIIUMKhRYhZEih0CKEDCkUWoSQIYVCixAypFBodbN+/XrExsZCLpcjKSkJhYWFnq5Sn3Jzc/GLX/wC/v7+CAkJQXp6OsrKyjxdrX5ZvXo1OI7Ds88+6+mq2OXy5ct4/PHHoVKp4O3tjYSEBBw9etTT1eqT0WjEK6+8gri4OHh7e2PUqFF44403+rxZeTCg0Opi69atyM7ORk5ODoqKijBx4kSkpqairq7O01Xr1XfffYesrCwcOnQIu3fvhl6vx1133YXW1lZPV80hR44cwUcffYQJEyZ4uip2uXbtGm677TZIJBJ89dVXKC0txdq1axEYGOjpqvVpzZo1+OCDD/D+++/j1KlTWLNmDd566y2sW7fO01XrGyOCxMRElpWVJfxuNBqZWq1mubm5HqyV4+rq6hgA9t1333m6KnZrbm5mo0ePZrt372YzZsxgy5Yt83SV+rR8+XI2bdo0T1ejX9LS0thvf/tbi3UPPfQQy8jI8FCN7EdHWp10Oh2OHTuGlJQUYZ1IJEJKSgoKCgo8WDPHaTT8tOhBQUEeron9srKykJaWZvH5D3b/+te/MGXKFDzyyCMICQnBpEmT8PHHH3u6WnaZOnUq8vPz8dNPPwEAfvjhB+zfvx/33HOPh2vWt2EzykNf6uvrYTQaERoaarE+NDQUp0+f9lCtHGcymfDss8/itttuw/jx4z1dHbts2bIFRUVFOHLkiKer4pDz58/jgw8+QHZ2Nl566SUcOXIEzzzzDKRSKTIzMz1dvV794Q9/QFNTE+Lj4yEWi2E0GvHmm28iIyPD01XrE4XWDSYrKwsnT57E/v37PV0Vu1y8eBHLli3D7t27IZfLPV0dh5hMJkyZMgWrVq0CAEyaNAknT57Ehx9+OOhD6/PPP8fmzZuRl5eHcePGobi4GM8++yzUavWgrzu1aXXSarVMLBazHTt2WKyfN28eu//++z1TKQdlZWWxyMhIdv78eU9XxW47duxgAJhYLBYWAIzjOCYWi5nBYPB0FW2Kjo5mCxYssFj35z//manVag/VyH6RkZHs/ffft1j3xhtvsJtvvtlDNbIftWl1kkqlmDx5MvLz84V1JpMJ+fn5SE5O9mDN+sYYw9KlS7Fjxw7s2bMHcXFxnq6S3WbOnImSkhIUFxcLy5QpU5CRkYHi4mKIxWJPV9Gm2267rUfXkp9++gkxMTEeqpH92traeowOKhaLYTKZPFQjB3g6NQeTLVu2MJlMxjZt2sRKS0vZokWLmFKpZDU1NZ6uWq+eeuopplAo2L59+1h1dbWwtLW1ebpq/TJUrh4WFhYyLy8v9uabb7IzZ86wzZs3Mx8fH/bpp596ump9yszMZBEREWznzp2svLycbd++nQUHB7MXX3zR01XrE4VWN+vWrWPR0dFMKpWyxMREdujQIU9XqU8ArC4bN270dNX6ZaiEFmOM/fvf/2bjx49nMpmMxcfHsw0bNni6SnZpampiy5YtY9HR0Uwul7ORI0eyl19+mWm1Wk9XrU80nhYhZEihNi1CyJBCoUUIGVIotAghQwqFFiFkSKHQIoQMKRRahJAhhUKLEDKkUGgRQoYUCi1CyJBCoUUIGVIotAghQ8r/A8TaRLLnILPgAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:29.838480Z", + "iopub.status.busy": "2024-03-22T19:31:29.837985Z", + "iopub.status.idle": "2024-03-22T19:35:32.709073Z", + "shell.execute_reply": "2024-03-22T19:35:32.708051Z" + }, + "papermill": { + "duration": 242.891902, + "end_time": "2024-03-22T19:35:32.711706", + "exception": false, + "start_time": "2024-03-22T19:31:29.819804", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:35:32.748889Z", + "iopub.status.busy": "2024-03-22T19:35:32.748580Z", + "iopub.status.idle": "2024-03-22T19:35:32.770477Z", + "shell.execute_reply": "2024-03-22T19:35:32.769593Z" + }, + "papermill": { + "duration": 0.04302, + "end_time": "2024-03-22T19:35:32.772721", + "exception": false, + "start_time": "2024-03-22T19:35:32.729701", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0295840.0287650.002712.4844050.0192540.5103660.0338290.0000474.1368430.0370760.11940.0519580.0932060.02437116.621248
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.029584 0.028765 0.0027 12.484405 0.019254 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.510366 0.033829 0.000047 4.136843 0.037076 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.1194 0.051958 0.093206 0.024371 16.621248 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:35:32.808159Z", + "iopub.status.busy": "2024-03-22T19:35:32.807431Z", + "iopub.status.idle": "2024-03-22T19:35:33.154322Z", + "shell.execute_reply": "2024-03-22T19:35:33.153206Z" + }, + "papermill": { + "duration": 0.365867, + "end_time": "2024-03-22T19:35:33.156318", + "exception": false, + "start_time": "2024-03-22T19:35:32.790451", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:35:33.192853Z", + "iopub.status.busy": "2024-03-22T19:35:33.192556Z", + "iopub.status.idle": "2024-03-22T19:39:46.938700Z", + "shell.execute_reply": "2024-03-22T19:39:46.937596Z" + }, + "papermill": { + "duration": 253.767885, + "end_time": "2024-03-22T19:39:46.941488", + "exception": false, + "start_time": "2024-03-22T19:35:33.173603", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:46.979157Z", + "iopub.status.busy": "2024-03-22T19:39:46.978304Z", + "iopub.status.idle": "2024-03-22T19:39:47.004942Z", + "shell.execute_reply": "2024-03-22T19:39:47.004246Z" + }, + "papermill": { + "duration": 0.047363, + "end_time": "2024-03-22T19:39:47.006828", + "exception": false, + "start_time": "2024-03-22T19:39:46.959465", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:47.039712Z", + "iopub.status.busy": "2024-03-22T19:39:47.039439Z", + "iopub.status.idle": "2024-03-22T19:39:47.044859Z", + "shell.execute_reply": "2024-03-22T19:39:47.043967Z" + }, + "papermill": { + "duration": 0.02409, + "end_time": "2024-03-22T19:39:47.046909", + "exception": false, + "start_time": "2024-03-22T19:39:47.022819", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.383920996813547}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:47.081866Z", + "iopub.status.busy": "2024-03-22T19:39:47.081333Z", + "iopub.status.idle": "2024-03-22T19:39:47.474641Z", + "shell.execute_reply": "2024-03-22T19:39:47.473717Z" + }, + "papermill": { + "duration": 0.413249, + "end_time": "2024-03-22T19:39:47.476838", + "exception": false, + "start_time": "2024-03-22T19:39:47.063589", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAQElEQVR4nO3deXxTZb4/8M/JvjRL9720lLKDgFAFRFAYRVBhnFF4uVER9M7AdZTLvJS5P2HQccAZFxzH4c54pcBVwdFBnavjwkURRWXfl7J13/c0TZvtPL8/TpMS2rRJmuS06ff9evWV9JwnyXOa5ptnfzjGGAMhhIhIInYGCCGEAhEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIhIUH3//ff47W9/i6amJrGzQgYQCkQkqL7//nusX7+eAhHxCwUiQojoKBCRoPntb3+LX//61wCArKwscBwHjuMQFRWFW265pUt6nueRmpqKn//85+5jL730EqZNm4bY2Fio1Wpcf/31+OCDD7p9vbfffhvXX3891Go1YmJisHjxYpSWlobm4khIcbQMCAmWkydPYuPGjdixYwdeffVVxMXFAQAuX76M5557DuXl5UhKSnKn37dvH2bOnIn333/fHYzS09Nx9913Y/To0bDZbNi5cycOHjyITz75BPPnz3c/9oUXXsCzzz6L++67DzNnzkRtbS1ef/11REVF4dixYzAajWG9dtJHjJAg+uMf/8gAsMLCQvexgoICBoC9/vrrHml/+ctfsqioKGaxWNzHrr7PGGM2m42NHTuW3Xrrre5jRUVFTCqVshdeeMEj7alTp5hMJutynPR/VDUjITd8+HBMmDAB7733nvuY0+nEBx98gLvuugtqtdp9/Or7jY2NaG5uxowZM3D06FH38V27doHnedx3332oq6tz/yQlJSEnJwdff/11eC6MBI1M7AyQwWHRokX4zW9+g/LycqSmpmLv3r2oqanBokWLPNJ98skn+N3vfofjx4/DarW6j3Mc575/8eJFMMaQk5PT7WvJ5fLQXAQJGQpEJCwWLVqENWvW4P3338eTTz6Jv//97zAYDJg7d647zbfffou7774bN998M/7yl78gOTkZcrkc+fn5ePfdd93peJ4Hx3H47LPPIJVKu7xWVFRUWK6JBA8FIhJUV5dcrpaVlYXc3Fy89957WLlyJXbt2oWFCxdCqVS60/zjH/+ASqXCF1984XE8Pz/f47mys7PBGENWVhaGDx8emgshYUVtRCSotFotAHQ7oHHRokX48ccfsWXLFtTV1XWplkmlUnAcB6fT6T5WVFSEjz76yCPdPffcA6lUivXr14Nd0+nLGEN9fX1wLoaEDXXfk6A6dOgQcnNzMW/ePCxevBhyuRx33XUXtFotysrKkJGRgaioKMjlclRVVXm053z11VeYPXs2ZsyYgfvvvx81NTV44403kJSUhJMnT3oEnY0bN2LNmjWYNm0aFi5cCJ1Oh8LCQnz44Yd47LHHsHr1ajEunwRKxB47EqGef/55lpqayiQSSZeu/OnTpzMAbNmyZd0+9q233mI5OTlMqVSykSNHsvz8fLZu3TrW3b/qP/7xD3bTTTcxrVbLtFotGzlyJFuxYgUrKCgI1aWREKESESFEdNRGRAgRHQUiQojoKBARQkRHgYgQIjoKRIQQ0VEgIoSIbkBP8eB5HhUVFdDpdF6nFhBCxMMYQ0tLC1JSUiCReC/3DOhAVFFRgfT0dLGzQQjpRWlpKdLS0ryeH9CBSKfTARAuUq/Xi5wbQsi1TCYT0tPT3Z9VbwZ0IHJVx/R6PQUiQvqx3ppOqLGaECI6CkSEENFRICKEiG5AtxH5gjEGh8PhsdgW8Y1UKoVMJqOhESTkIjoQ2Ww2VFZWwmKxiJ2VAUuj0SA5ORkKhULsrJAIFrGBiOd5FBYWQiqVIiUlBQqFgr7Z/cAYg81mQ21tLQoLC5GTk9PjgDRC+iJiA5HNZgPP80hPT4dGoxE7OwOSWq2GXC5HcXExbDYbVCqV2Fnqk8ZWG74uqEGSXoWp2bH0xdSPiPoV53Q68eyzzyIrKwtqtRrZ2dl4/vnnuyyI3hf0Ld43kfT3+/ZSHYrrLThQ2ICK5naxs0OuImqJ6MUXX8TmzZuxbds2jBkzBocPH8YjjzwCg8GAJ554QsyskQjjcPIoqmt1/36pxoxUo7qHR5BwEjUQff/991iwYAHmz58PAMjMzMSOHTtw8OBBMbNFIlBNixVOXihp3zk+GSkUhPoVUcvd06ZNw549e3DhwgUAwIkTJ/Ddd9/hjjvu6Da91WqFyWTy+CF9k5mZiU2bNomdjZCr7KiKZSdEISdRB60yYptHByRR341nnnkGJpMJI0eOhFQqhdPpxAsvvIAHHnig2/QbNmzA+vXrw5xLEgkaW20AgLgoGobQH4laIvr73/+Od955B++++y6OHj2Kbdu24aWXXsK2bdu6Tb9mzRo0Nze7f0pLS8Oc4/7JZrOJnYV+74ahMfjpxFSMTNKjpN6CQ0UNqDNbxc4W6SBqIPr1r3+NZ555BosXL8a4cePw0EMP4amnnsKGDRu6Ta9UKt0z7fsy497m4L3+OJy8z2ntPqQNxKxZs7By5UqsXLkSBoMBcXFxePbZZ929iZmZmXj++efx8MMPQ6/X47HHHgMAfPfdd5gxYwbUajXS09PxxBNPoLW1s4G2pqYGd911F9RqNbKysvDOO+8ElL+BSKeSIzNOixitAsdKG/HdxTqUN7aJnS3SQdSqmcVi6dI9LJVKwfOBfYB99cbXl7yey4rTYuHEVPfvf9t3GXZn98MJ0qLVuHdy58JsW/YXos3mOZXkqZ8MDyiP27Ztw6OPPoqDBw/i8OHDeOyxx5CRkYHly5cDAF566SWsXbsW69atAwBcvnwZc+fOxe9+9zts2bIFtbW17mCWn58PAMjLy0NFRQW+/vpryOVyPPHEE6ipqQkofwOZXiVsc222OkTOCXERNRDdddddeOGFF5CRkYExY8bg2LFjeOWVV7B06VIxs9UvpKen49VXXwXHcRgxYgROnTqFV1991R2Ibr31VvzHf/yHO/2yZcvwwAMP4MknnwQA5OTk4E9/+hNmzpyJzZs3o6SkBJ999hkOHjyIKVOmAADeeustjBo1KuzXFm4WmwOnypqhV8sxKlmPKJXwb9/SToGovxA1EL3++ut49tln8ctf/hI1NTVISUnB448/jrVr14b0dVfcMszrOck1g20fuznba9prB+YunZ7Vl2x5uPHGGz1G/k6dOhUvv/yye/Lu5MmTPdKfOHECJ0+e9KhuMcbcU10uXLgAmUyG66+/3n1+5MiRMBqNQctzf9XQasP3l+thcAWijh4zKhH1H6IGIp1Oh02bNoW9+1gh871pLFRp+0qr1Xr8bjab8fjjj3c7EDQjI8M9RGIwarUKwdtVEnIHona7aHkinmgwRT914MABj99//PFH5OTkQCqVdpt+0qRJOHv2LIYN6760N3LkSDgcDhw5csRdNSsoKEBTU1NQ890fuUo+rgDkum210dIw/UXkTCSKMCUlJVi1ahUKCgqwY8cOvP766/jVr37lNf3TTz+N77//HitXrsTx48dx8eJFfPzxx1i5ciUAYMSIEZg7dy4ef/xxHDhwAEeOHMGyZcugVkf+CONrA5FGKQTz7no+iTioRNRPPfzww2hra0Nubi6kUil+9atfubvpuzN+/Hh88803+M///E/MmDEDjDFkZ2dj0aJF7jT5+flYtmwZZs6cicTERPzud7/Ds88+G47LEVVrRyByjaZWSCVYODEVGoUUUpqB3y9QIOqn5HI5Nm3ahM2bN3c5V1RU1O1jpkyZgi+//NLrcyYlJeGTTz7xOPbQQw/1KZ8DwbUlIo7jkBWn7ekhJMyoakYinmtsl0bRffsaER+ViEjEu3N8MlqtTsTrlO5jxfWtqG2xIj1Gg0T9wF7wLRJQIOqH9u7dK3YWIkpslBKxUZ7HzlWacK6yBTcPj6NA1A9Q1YwMSiq5UE2zUBd+v0CBiES0VqsDBwsbcLbCc+0qjUKoDFAg6h8oEJGI1mixYf+lOhwsrPc47mq4brdTIOoPKBCRiOYKNOpresyUHdNxKBD1DxSISERrswkjp11tQi6u360BrhlFgosCEYloba4SkZxKRP0Zdd+TiGaxCaOqr62aGTUK3DMptUuAIuKgQEQiWruXEpFCJsGQWJrm0V9Q1YxEtDYvjdWkfxlcJSLGAKcIi2FJ5V2Xc/Ri+/bteOqpp1BRUQGlsnNKwsKFC6HT6fA///M/ocplRJo1PAHmIQ5Ea7tuI3Su0oRWqwOjkvW0z5nIBtdf32kHvn05/K874z8AmW/7ad1777144okn8M9//hP33nsvAGH3jU8//bTHmfWke9FaRbdBCAB+uFyP5jY7UoxqCkQio6pZP6NWq3H//fe7d94AgLfffhsZGRmYNWuWeBmLQEq58O9PXfjiG1xfA1K5UDoR43X9sHz5ckyZMgXl5eVITU3F1q1bkZeX57GYPukdzzMcLm6EUibB2FQDpNfsjKCS0ejq/mJwBSKO87mKJKaJEyfiuuuuw/bt23HbbbfhzJkz+PTTT8XO1oBjc/LYf6kOADA21dDlPJWI+o/BFYgGkGXLlmHTpk0oLy/HnDlzkJ6e3vuDiAdXSUchk3QpDQFUIupPqI2on7r//vtRVlaGN998kzacDJCrpKP0ss0TlYj6DwpE/ZTBYMDPfvYzREVFYeHChWJnZ0Cy2nsJRFQi6jeoataPlZeX44EHHvAYT0R8Z3UIAUbpZRrHiEQdEvVK6FX+dSaQ4KNA1A81NjZi79692Lt3L/7yl7+InZ0Bq72XEpFBI4dBQ0GoP6BA1A9NnDgRjY2NePHFFzFixAixszNguUtEMpre0d9RIOqHvO1bRvwzMlmPRL2qy1pELhabA5drWsFx3Xfvk/ChQEQiVpRS5t5UsTvmdgf+71w1dCoZBSKRRXyvGWNM7CwMaJH891PIqPu+v4jYQCSXC42QFotF5JwMbK6/n+vvOZAUVLXgRGkTmi3dr7jgCkQ2Bx/RAXcgiNiqmVQqhdFoRE1NDQBAo9HQXC0/MMZgsVhQU1MDo9EIqXTgNfgeK2lEZXM77roupdveMYW083vY5uSpUVtEERuIACApKQkA3MGI+M9oNLr/jgONa6CiSt59wV8q4SCVcHDyDDYHBSIxRXQg4jgOycnJSEhIgN0uwoJoA5xcLh+QJSGXzike3V8Dx3FQyCRoszlho3YiUUV0IHKRSqUD+gNF/McY6wxEXkpEgFA9a4MTNicFIjENikBEBh8Hz+DkhQZobyOrAWD2qAQAQLSm/y8PE8koEJGI5GofknCcR6P0tWgnj/4hYrvvyeB2dbWMekv7PyoRkYikV8nx8+vT3NUzbyqb21BvtiFBp0SCXhWm3JFrUSAiEUkhkyA9RtNrujPlJpwqb8bU7FgKRCKiqhkZ1K4eXU3EI3ogKi8vx4MPPojY2Fio1WqMGzcOhw8fFjtbZICrNrXjRGkTypvaekxHgah/ELVq1tjYiOnTp+OWW27BZ599hvj4eFy8eBHR0dFiZotEgMK6VvxwuR7jUg1INaq9pnMHIhpHJCpRA9GLL76I9PR0j80Es7KyvKa3Wq2wWq3u300mU0jzRwYuV6+Zt7WIXFxd+1QiEpeoVbN//vOfmDx5Mu69914kJCRg4sSJePPNN72m37BhAwwGg/uHttgh3rjGEfU0qhroHOxIgUhcogaiK1euYPPmzcjJycEXX3yBX/ziF3jiiSewbdu2btOvWbMGzc3N7p/S0tIw55gMFL1tJeTiXpOIqmaiErVqxvM8Jk+ejN///vcAhLWaT58+jf/6r//CkiVLuqRXKpW0owXxidXu23rV8Tol7hyfDLWC5iKKSdQSUXJyMkaPHu1xbNSoUSgpKREpRyRSdLYR9fwvrlHIkJOoQ1p072OOSOiIGoimT5+OgoICj2MXLlzAkCFDRMoRiRS9LQFC+hdRq2ZPPfUUpk2bht///ve47777cPDgQfztb3/D3/72NzGzRSLA7WMS0W53wtjLvmU8z3Cp1gybg8eoZD2kEpqXJgZRA9GUKVPw4YcfYs2aNXjuueeQlZWFTZs24YEHHhAzWyQC+FPV+vRkJQBgaLwWGgXNehKD6H/1O++8E3feeafY2SCDlETCQS7lYHcy2B0MoGWJRCH6FA9Cgq3d7sTJsiZcrG7xKX1nF74zlNkiPRC9RERIsJna7NhzrgZapRQ5ibpe08ulEgBO2J20pZBYqEREIo6/PWY08VV8FIhIxLE6XIMZffv3lnfMN7PT6GrRUCAiEafd3vvuHVej+WbiozYiEnH8rZpNyojGqGQ9EnW0QqNYKBCRiONv1cyXJWVJaFHVjEQcmt4x8FCJiESc8akGpEerYVD7NjqxyWJDTYsVUUoZUnpYzZGEDgUiEnFio5SIjfJ9uZjLta3Yd6EWI5N0FIhEQlUzMui5l4ul7nvRUImIRJwL1S1gDMiI0fi04BkNaBQflYhIxPn2Yh3+daoSzW12n9LTTh7io0BEIo7/I6uFNYjsVCISDQUiElEYY+4qlq8jq6lEJD4KRCSiWB08WMckelcjdG8U7rlmNPteLNRYTSKKazCjXMpB5mMg0ihk+MnoRHfJiIQfBSISUTrbh3wfVa2QSTA21RCqLBEf0FcAiShWP2fek/4hoBLRlStXMHTo0GDnhZA+i41S4K7rkiHh/NuNo7TBgna7E+kxGqjkNEct3AL62hg2bBhuueUWvP3222hvbw92nggJmEYhw7AEHYbGR/n1uC/OVOGTk5Vosvg29ogEV0CB6OjRoxg/fjxWrVqFpKQkPP744zh48GCw80ZI2NDiaOIKKBBNmDABr732GioqKrBlyxZUVlbipptuwtixY/HKK6+gtrY22PkkxCfVpnYUVLWgzmz163Fymm8mqj616MlkMtxzzz14//338eKLL+LSpUtYvXo10tPT8fDDD6OysjJY+STEJ2crTfjXqUoUVPm2lZALzTcTV58C0eHDh/HLX/4SycnJeOWVV7B69WpcvnwZu3fvRkVFBRYsWBCsfBLiE3evmZ9jgqhEJK6Aes1eeeUV5Ofno6CgAPPmzcP27dsxb948SCTCm5mVlYWtW7ciMzMzmHklpFeBjCMCOktEtJOHOAIKRJs3b8bSpUuRl5eH5OTkbtMkJCTgrbfe6lPmCPGXtbd5Zq11QNF3gK0VSBwDJF8HcBxVzUQWUCDavXs3MjIy3CUgF8YYSktLkZGRAYVCgSVLlgQlk4T4qnO96m4CUUs1cPxtwGETfm8qAdqbgaEzMTxRhzitEgl631d2JMETUBtRdnY26urquhxvaGhAVlZWnzNFSKCsdqFq1mVQIu8Ezv1TCEKGNCDjRuF48fdAczlSjWqMSzMgUU9bCokhoEDEWPezlM1mM1QqeiOJeNpdgejaNqKK40K1TK4Gxv4MyL4FSBornCvcF95Mki78qpqtWrUKAMBxHNauXQuNpnM/KKfTiQMHDmDChAlBzSAhvmKMYe7YJLTbeWiUVwUingdKDwj3s2YAio7/28wZQM05oLEIloYy1LJoyKQSpNIC+mHnVyA6duwYAOENP3XqFBSKzu1aFAoFrrvuOqxevTq4OSTERxzHYViCruuJugtCW5BcDSSN7zyuNgJxOUDNeTRdPopdTSORbFBhcW5G2PJMBH4Foq+//hoA8Mgjj+C1116DXq8PSaYICaqqU8JtygRAKvc8lzQeqDkPdeN5AMOp+14kAbUR5efnUxAi/Y7Z6kBBVQvKm9o6D9rbgIYrwv3EsV0fFJ0FyNWQ81borNXuXjcSXj6XiO655x5s3boVer0e99xzT49pd+3a1eeMEeKvquZ2/OtUpWf1qrYAYDwQlQBo47o+SCIBYoZC0noC0W3FqNGlhTfTBIAfgchgMIDrWOPFYKDV7Ej/095d133NOeE2YbT3B8blQFp+EtHtJShzTAVjzP2/TsLD50CUn5/f7X1C+gvX9A6Va1S1wwY0lwr340d4f2B0FqRSCdT2BsjsLXDyDDIpBaJwCqiNqK2tDRaLxf17cXExNm3ahC+//DJoGSPEX+3uZWI7SkRNxcJARrURUEd7f6BcBalemKqkt1bSxFcRBBSIFixYgO3btwMAmpqakJubi5dffhkLFizA5s2bg5pBQnzVZTCjq5E6ZijQS1WLM2YgM1aDGfEWyCS03nW4BbxC44wZMwAAH3zwAZKSklBcXIzt27fjT3/6U1AzSIivXCUid9Xs6kDUG2MGkg1qZMoaaFshEQT0F7dYLNDphIFjX375Je655x5IJBLceOONKC4uDigjGzduBMdxePLJJwN6PCEejdWWBqCtCZBIAaMPAxQNaUKpydIAWM2hzSjpIuDF8z/66COUlpbiiy++wG233QYAqKmpCWh80aFDh/DXv/4V48eP7z0xIV7kZsXgJ6MTkWJQCzPrAUCfAsh8mFEvV8Ms1aO5zQ5LfVloM0q6CCgQrV27FqtXr0ZmZiZuuOEGTJ06FYBQOpo4caJfz2U2m/HAAw/gzTffRHR0Dw2KhPQiPUaDsakGGDRywFQuHNSn+vz4E01qnK00oa6yKDQZJF4FFIh+/vOfo6SkBIcPH8bnn3/uPj579my8+uqrfj3XihUrMH/+fMyZM6fXtFarFSaTyeOHkG6ZKoRbg+8DFB1RSQAArqUqFDkiPQh4y+mkpCQkJSV5HMvNzfXrOXbu3ImjR4/i0KFDPqXfsGED1q9f79drkMGB5xku1LRAJZMiQ8dB0tqxXpY+xefncGqFLnzOXAkw1mtPGwmegAJRa2srNm7ciD179qCmpgY87znu4sqVK70+R2lpKX71q19h9+7dPq9htGbNGvdSJABgMpmQnp7uX+ZJRGp3OPHZqSpwHPDEhI7ue3U0oND6/iTaeDBOAmazAO1NPY89IkEVUCBatmwZvvnmGzz00ENITk4OaDj8kSNHUFNTg0mTJrmPOZ1O7Nu3D3/+859htVohlXoubqVUKqFU0lKepCuLrXPRfElLR/uQwff2IQCQyRWwyGPA82agpYoCURgFFIg+++wzfPrpp5g+fXrALzx79mycOnXK49gjjzyCkSNH4umnn+4ShAjpSVtHINIopECzq6Ha92oZIOzk0aiIg5NvAcw1QMKoYGeTeBFQIIqOjkZMTEyfXlin02HsWM9lGbRaLWJjY7scJ6Q3bR1jiNQyDmjpaKjW+zeTXiGToFUeC6f9ihCISNgE1Gv2/PPPY+3atR7zzQgRk6tEZGDNwmRXqRzQxvv1HKlGNcYOz0ZclBJopUAUTgGViF5++WVcvnwZiYmJyMzMhFzuuerd0aNHA8rM3r17A3ocIa42omhHrXBAnyKsNeSHRL0KiSOGA7UKoN0kLKomp/WrwyGgQLRw4cIgZ4OQvnFN79DbqoUDfgxk9CBXASqDsMZ1a61v00NInwUUiNatWxfsfBDSJ6NT9IjXKZFyqQHgEVAgsjt51JttUHJGRKNZaCeiQBQWAU8zbmpqwn//939jzZo1aGhoACBUycrLy4OWOUJ8lahXYWy8HHq+RTjgZ48ZADRabNhxsAT7qzqGo1CDddgEVCI6efIk5syZA4PBgKKiIixfvhwxMTHYtWsXSkpK3GsVERJWrmkdmpjOvcv8oOwYMmKSRgMoogbrMAqoRLRq1Srk5eXh4sWLHqOi582bh337aNdMEn4XqltQVXYZTsYCbh+Sy4SSULM0RtjNuLVW2JyRhFxAgejQoUN4/PHHuxxPTU1FVRVNGCThxRjDZ6eqcOLMGTh55veIaheFVPg4tMv0cHBSwOkQpnqQkAsoECmVym5nvl+4cAHx8f6N3SCkr9rtPHjeCa21FjIJF3CJSCaVQCrhAE4Ch6pjeoelPog5Jd4EFIjuvvtuPPfcc7Db7QCErX5LSkrw9NNP42c/+1lQM0hIb9rsTmjsDVByDkjkSkDTzf5lPpJ3lIocyo6ZA65Z/CSkAgpEL7/8MsxmM+Lj49HW1oaZM2di2LBh0Ol0eOGFF4KdR0J6ZLE5oLNWQy7lAJ3/Axmv5lqv2uYKRBYKROEQUK+ZwWDA7t27sX//fpw4cQJmsxmTJk3yaXEzQoKt1epElK1GKM0E2D7kMinDCCfPoJLZgRpQiShM/A5EPM9j69at2LVrF4qKisBxHLKyspCUlEQ7ZBJRmK1CiUihkgQ+orrDxIyOtqFWJtxa6mmRtDDwqwzLGMPdd9+NZcuWoby8HOPGjcOYMWNQXFyMvLw8/PSnPw1VPgnxqt1igsphEkpEAQxk7JbaKOwA4rQL0z1ISPlVItq6dSv27duHPXv24JZbbvE499VXX2HhwoXYvn07Hn744aBmkpCejNS0wBmvhcKQ0OdJqi3tdlhsTmgUUujU0ULVzFIvBCYSMn6ViHbs2IHf/OY3XYIQANx666145pln8M477wQtc4T4ItZRiwSdCsbEzD4/1w+X6/HugRKcq2wBtB29b9ROFHJ+BaKTJ09i7ty5Xs/fcccdOHHiRJ8zRYhfXFM7+tg+BADyjl4zu5PvHAZAPWch51fVrKGhAYmJiV7PJyYmorGxsc+ZIsRnPI+aikIoOCeiolIC35amg7JjHJHNwQN6KhGFi1/vm9PphEzm/SFSqRQOh6PPmSLEV3ZTFS5XNcEpUWCyKqbPgchVIrJdWyKinrOQ8ut9Y4whLy/P604aVqs1KJkixFfW+lIAgEWVCKW87xsuKK4uEanjAE4iLD1rbQFU/m+nTnzjVyBasmRJr2mox4yEk7WhGADgiEoJyhg2xdVtRFKZsKWQpV74oUAUMn4Fovz8/FDlg5CAOBvLAAAsCA3VQOdcM5ujY/kPbWxnIIrJCsprkK76WqUmRDztJjgsTWDgIDEGJxDFahXIzYqBQd2xIYQmDsAFYW0iEjIUiMjA1VwGq4OHRRELncaPraV7EK1VYPqwq2bvu8YS0XIgIRX4NGVCxGYqh9XuRIsyETpViL5TXT1nrbVCzxkJCSoRkYGruRRp0RpIU8cgLsb/Naq7w/MMzW122J084nVKcJoYodve3g7YLYAiOCUv4olKRGRgctgAcy2ilDLk5IxEtFYRlKe18zy2fl+Edw6UwMEzYcdYlVE4SQMbQ4YCERmYTOUA44Uu9SB2q7vGEQEdXfjAVe1EFIhChQIRGZhM5bA5eZQ4Y1DWaAna03Ic17lKo6sLXxMr3LZSg3WoUCAiA1NjMcxWB76rUWNvQXC71hVdxhJRiSjUKBARr3ie4WhJI/7vbDVqW/rR9B2nHTCVo93uRLMqBdGa4LQPuSiunm8GXNVzRoEoVCgQEa9+vFKPbwpqcbnWLGzT0180lwK8E2Zo0C4zIFojD+rTdxld7aqa2VoBW/CqgaQTBSLSLbPVgUNFwpIut45MCFqvVFA0CvPL6mTJAMfBGKISkd3ZMW5IpgBUBuE+DWwMCQpEpFtnypvBM4YUowo5iTqxs+OpsQgAUCVJAAAYg1wiGpGoQ25WDKK1Vz0vtROFFA1oJN0qqG4BAIxLNQIQSkiXa8zQKKTiBiZ7G2CuhpMxVHBJABD0NqJxaYauBzWxQP1l6jkLESoRkS5M7XbUm23gOGBovDCS+HKNGV+dr8GxkiZxM9dYBDAGi8wAu0wLlVwKtaLv6xD1ikpEIUUlItJFSb3QIJukV0HVsdjY0HgtvjoPVDS3wWJzQKMQ6V+n/hIAQJ00Aj+PS0Ob3Rn0l7A5eFhsDnAcd80sfFDPWYhQiYh0kaBXYnJmNMakdFZRdCo5EvRKMAYU1YnUc8Tz7kAkSxiO9BgNhoegmnimohn5+4vw3cWrgo6r58zaIsw7I0FFgYh0kaBTYUZOfJe2koyOiaXBHMnsF1OZEATkKkCfFrKXcZUC268ubclVgLIj6FHPWdBRICJdWVuEtphrvvnTooVAVN7UJkKmANRdFG5jsvH9lQacqWjunA8WRO5A5Lim2kftRCFDbUTEg+nSj3Be2A2dUgKZUgOMXgDEDAUApBhV4DigyWJHS7sdOlVwu817xBhQWwAAsOiH4sC5BnAckJMQ/KqZsmMcUbv9miCniQMaCqmdKASoREQ6NVxB88lPcb6yGVcabEKJ6PQ/3F3WSpkUCTohGNWEe8pHc5mwB71MgQqpsCxsrFbhHnwYTK4SkbVLiaijnYiqZkFHJSIi4Hng0h60Wh2ojhoF1ZQFQNOXwijmgk+BiQ8BHIc7xiZBrZC6P6xhU31GuI0fiYoWYe+8ZEPf9rn3RiUXgpvVzoPnGSSu6S3UcxYyopaINmzYgClTpkCn0yEhIQELFy5EQUGBmFkavOoKgNY6mBwylBhvQIJRC4ycLywM1lzubp+J1irCH4ScdqD2nHA/YTQqm4U2qhRjaAKRUtZ5fVbHVdUzV89Ze7OwMBsJGlED0TfffIMVK1bgxx9/xO7du2G323HbbbehtbVVzGwNThXH4eB5FKtGwSlRIFbbMb8qbbJwvuhb8dZsrj4tVBPVRtj16ag2CdXCFKMqJC8nlXCYmGFEblaM5+auCk3nUrHUYB1UolbNPv/8c4/ft27dioSEBBw5cgQ333yzSLkahNpNQFMx2u08aqNGQKu8quqVlguUHQbMNUBTMRCdiaMljbhY3YLcrFhkxYV4DWfGhNcHgNTrUdZkhZNn0KlknYMNQ2DWiITuT2hihVn4rXWAPiVkrz/Y9KvG6ubmZgBATExMt+etVitMJpPHDwmCmrMAY2hWJsEq0yFGe9WW4goNkDROuF9+FABQb7ahoqk9POOJGq4IH3qpHEgajxqTMKQgK04blJ1d/UZd+CHRbwIRz/N48sknMX36dIwdO7bbNBs2bIDBYHD/pKenhzmXEaruAgCgRi1008dorylppEzsSHcRsLa4q0QVoR5PxBhwZW9nHuQq3DA0FstmZGFyZvdfVsHSbneiyWLzHNQIdAYiarAOqn4TiFasWIHTp09j586dXtOsWbMGzc3N7p/S0tIw5jBC2dsAUwUAIHP4dfjJ6ESMSLpmMfqoBMCYLixWX3EcaUZhYGO1yRqSAYVuFceEKqFMAWRMdR/WqeQhrZYBwO6z1cjfX4SCqhbPE1GJwq25OqSvP9j0i+77lStX4pNPPsG+ffuQluZ96L5SqYRSqfR6ngSgoVAoeWjjEB0Th2hv6VImAk2lQOUJ6IdMQ5RSBrPVgWpTu3vEdVBZGoDLXwn3M28GFBpYHU6PHq1Q6naaBwBo44Vbq1lYrVERgmsfhEQtETHGsHLlSnz44Yf46quvkJWVJWZ2BqeGy8Jtx+hpr+JGAHI1YG0B11Do7jovbwxB9cxmAU59IHTbG9KAtMkwtdvx5r4r+Ph4edeBhiHgHkvkuKbEJ1MC6o5wTaWioBE1EK1YsQJvv/023n33Xeh0OlRVVaGqqgptbSLNZRpsGBNKRABaoobgZFmTe4xOF1IZkNTRdld5vLOdyFv6QJkqgGNvC6OXlTpgzEKA4/D9pXrYnQx2JwtLqcj1Gl1KRAAQ1VEqag3u7iGDmahVs82bNwMAZs2a5XE8Pz8feXl54c/QYNPWKHRFS6Qo52Ow51wNUowqLJqS0X365IlA6SGg/hLSkmZCo5AGvC5RjakdZytNqG2xguM4JEqaMbL9GOLbO9r9VHpg/CJAqcO5ShPOVZrAccD0YbEBXqx/XCWi9mtLRIDQTlR7gUpEQSRqIGJiDZAjguaOD70uGQ1twgfOo+v+WtpYodG6qRRx5vN47Oab/O5Cd/IM+y7W4njHSo8qexPSm49AZrkMk06J+AQdkDgW7ek3odQsRVlpDU6UCWmnZMaEbFrHtdSuNiJbNyUibccYI3NNWPIyGPSLxmoikqaOQGRMR4NZmLIQ09tuHR2N1lzVSWDIdMCPQMTzDP86VYlLNWZIeAdulJxFlvMsOC0Ps1yFqPTxwNjZgDYWtQ0WfHKyzP3Y69INmJYdntIQAPfysxabo+vJqI5AZKkHeCcgCfOUlwhEgWgwc5WIDOloqBYCUWxvgShuhLBIWLsJaCwEixmKNrvTpyoaxwFxUUpUVFXhp/IfkcA1A1FyIDYb8UNndX7AO6QYVTCo5RiVrMeQ2BCP4L6GXi3H+DQD9N0NE1AZhEZrh1UIRlFeRmETn1EgGqzaTUBbE8BxcOpS0dgqBKWYqF4CkVQmjLQuPYSWwkPYeUYoES2bkdVrNY3jOExNV2NCxQ9QO0xC1/eI+UDcsC5p02M0WBTjpa0qDPQqOWaPSuz+JMcJwaepVKieUSDqs34zoJGEmas0FJWIJhsHnjEoZBLolD58NyVPAABoWorAtzfDbHX0uCW1wykspwGnHTj9gRCEVAbg+rxug9CA4G4nogbrYKBANFg1d7S/GNPRaBGqZdEahW+Nz9o4wJgOKRjGSYRdVy/VmL0m//ZSHd47VIKWYx8KS4rIlMD4+zp3T+2n2u1ONLbaOreevloUNVgHEwWiwaqpRLg1ZCA9RoN7J6dhRk6c749PmQQAGGY7BylvxaXa7gNReVMbTpQ2QVH8DZzV54SG3bE/65yz1Y+9f6QMW78v6n5slS5ZuG2pFG95lAhCgWgwslk6J20a0qCUSZEWrUF6jB/TFeJHAto4xCh5pJlPo95sQ7XJc7F9u5PH7jNVSDCdwQQUCHvUj5gHRA8J4sWEjkbu6jnrrgs/DpDIhAbrtsYw5yzyUCAajFzVMm1c4HOlJBIgcwZkEgnG8uehdJhwrMTzA/ndxTqg7iJGtPyIIbEaIOvmztHZA4BG0UMgkkgBXUdjdsekYRI4CkSD0VXd9owxfHuxFqfLm+HwdyZ9/AggeghSdDJk1+/Fpapm97ibi9UtuHLxLHLq9yA7TgNZ6kRgyLQgX0houcYStXUXiICrqmdVYcpR5KJANBi5A1EaTG0OHC5qxNfnayDxd6ExjgOGz0WURo1R6mY8ZDwFjYRHeaMFh37ch5G1nyNVL4cxYwww/Ha/Bj/2B66xUd0OagSuCkRUIuorGkc02DhsQEtHl7MxHfUd3e7RWkXnbhX+0MQAoxci4/Q/gJZLwA+vI0miwBhTGdQaKTJyxgOjFw7I0ceuqllbdxNfgc6lYluqaYR1H1GJaLAxlQsLnKn0gMqA+lYfR1T3JG4YcN0iQG0EHDZIbWaMSotFzo3zIBn3c2EQ5ADkqpq1Wr0EInW0MBSBd9CKjX00MP9DSOCuah8ChPWnAR/mmPUmOhPIfVwY4MfboYhKElZWHMBiNAqMSzUg1ttoc44TqmeNRUL1TOdlJDbpFQWiwaaps30IABpcJaLepnb4QiIB9Ml9f55+IlqrwJzRvQQXXZIQiEyVnWt7E79R1Www4Z2dXc3GDDDG0NAqtBH1uPwH8a6jZOkeEkECQoFoMGmpFNoz5GpAEwtTuwN2J4NUwsEY4sXoByqrw4l6s7X7lRoBwJAqVNEs9cI61iQgVDUbTBqLhFtjOsBx0KtkWH7zUJja7IH1mA0CHx0rR0VTO+66LhnDEnRdE8jVwsBQc61QKkoYGf5MRgAqEQ0mrkAULWxSwHEcopSykO0hHwm0HasRtLR7GUsEAIaO5UpcHQHEbxSIBguHVZj5DgAxtFuKr6I6ApHZ2kMgMrraiSgQBYoC0WDRVCKMH1JHu7fD+fp8Db6/VNfzh2yQ06k6AlGPJaKOvfjMNYC93Xs64hUFosGiY9sgV2nI4eRxsqwZBwob4ORpGQtvopRCI36PVTOlTgjujFHvWYAoEA0GjAH1l4T7He1DDa028IxBJZdCr6I+C29cW1s3t9l7ThidKdw2FoY2QxGKAtFg0FIFtDcLUy06SkQ1HXPMEnRKv7cEGkyMGiEQma0O2HtancDV7tZwJQy5ijz0VTgY1J4XbmOHAVLhg+VaYzpeRwMZe6KSS3FdugFRSjn4nlZiNA4BOAlgaRAWSnNtS018QiWiSMcYUHdBuB/fOcalpkVoVE3QUyDqza0jE5GbFdPzVtdylTC4EaBSUQAoEEU6U4XwLS2VATHZAISNDus6JrvGR1EgCpqYocJtA7UT+YsCUaSrPC7cxo9yz4Y3tdvBcYBSLun7rPtBwOHkUW+29rhlEgCh6gsIgcjRS1rigdqIIpm9Dag5K9xPmeA+bNQo8IuZ2TC1O6ih2gcF1S348kw10qLVuHdyuveE2nhhoThLg9BLmTgmfJkc4KhEFMnKDgFOh7AHlz7V4xTHce6uadIzV/W11mwF66nBmuM62+FqzoUhZ5GDAlGksrcDZYeF+0OmDbj1ovuTGK0CEo6D1c6jpbdR6AmjhFuqnvmFAlGkKtwnfBC0cR69ZU0WG7Z8V4gvzlT1/O1O3GRSCWK0nsMevNLGA5pYYbkVKhX5jAJRJGoqBSqOCvdzfuJRGiqut6C5zQ5Tm53ah/zgGm9V11sg4jggaZxw39VRQHpFgSjSWM3A2Y+E8UNJ4zqnHnQoqm8FAAyJ1YY/bwOYKxBVmXyY1Jo8XtjRw1TZuWMK6REFokhiswAndwrBSBsH5Nzmcdru5FHWKOzjnhkb4A6vg1SqUfh7lTe1ge9tkrBCC8QNF+6XHQpxziIDBaJI0dYIHH9HWClQoQXG/qzLLhrF9a2wOXjoVDKa2uGnBJ0SuVkxmDfWx80B0qYIt9VnhPeG9IgCUSRoLAaObBP21lJGARPuF8azXKOgSlhTeUSSjtqH/CSRcJg+LA6ZcVrfltU1pAoTYRkPFP8Q+gwOcBSIBjKeF3rHTuwQBi/qk4Hr84Rq2TUsNgeu1HYEosRu1l4mwTdkunBbdVJoLyJeUSAaqNqagONvA0X7OxumJzwgLNLVDQ4cJg2JRlaclqplfVDWaMFX56thau9lfSJAWEI2cbTw/lz8QvjiIN2iKR4DDWNCu8PFL4VxQjIFMHxur9MJ1Aoppg/rWlIi/vnhcj3KGtuglPn498y+VZjuYaoECvcKv5MuqEQ0kLQ3A6feB879rxCEDKnA5KW9BiFaCjZ4JmYI6wydKGuCxebDWt9KHTBivnC/5ABQcTx0mRvA+kUgeuONN5CZmQmVSoUbbrgBBw8eFDtL/YvDKrQFHXwTqL8sjFHJuhmY8GCvC3AV1bXif34oQo0v419Ir7Ljhaqt1c7j6/O1vo1OTxgJZNwo3C/4TGi8plHtHkQPRO+99x5WrVqFdevW4ejRo7juuutw++23o6amRuysia+tEbj8NfDjZqEtyGkXdoyY/CiQOV3Ya74Hl2rM+ORkBRotdpyuaA5TpiMbx3GYPSoBEo7DheoW7D5bDZvDh7afobM6u/Sv7AWOvysstE8BCQDAMZEnHN1www2YMmUK/vznPwMAeJ5Heno6/v3f/x3PPPNMj481mUwwGAxobm6GXq8PR3ZDw+kA7BbAZha64M01wiLsrXWdaTQxQNZMIH6E1wmsTp7B3O5ARXMbzleZUFRnAQBkxWlx5/hkyKSif+9EjLMVJnx5tgqMAXq1HEunZ7qHRNSY2sFxHKQSDhyuersYg6L2FDTFXwtz0QC0KWPAG4cC+hRINDGQaY2QyZXgevmSGSh8/YyK2lhts9lw5MgRrFmzxn1MIpFgzpw5+OGHrmMvrFYrrNbOuT4mk8m3F2ooBC7v6f7bx+MY837M43jXYyUNFtS0tHU57To/Nk0PVUcgKG20oKpZqCpxzAkJ79kDMypZD7VcCnAcCp3xOGwbgiZJJth5Djh/xSN7P52YigS9CgBw4Eo9DhQ2uM9xHDApIxrTh8VBSltKB9XoFD20Sin2nKtBtFbuMS7r/SNlXktJqdGJuC93OVC8H6g+izMXLsHuvHhNKg5MKodGrcaEIXEAhOc+VtYMm4MHE8IbOA4d94XOiOvSDO5nOFnWjDa7s9s8KGUSTEg3un8/U26C2Ut7l0zC4fohndV/m4NBIeu41nH3AargFABEDUR1dXVwOp1ITEz0OJ6YmIjz5893Sb9hwwasX7/e/xdyWIURxyHkbGuF3dJDO4xVBsiFNY95mwUOq2daBg52qRrtciOyk0ZCnTgEiBmKqhILyq7UA7bu/7GdV0WlKJUMUgmHuCglMmI0GJdqgEFDaw6FypBYLfKmZXZZGkSjkEIu5eDgWZfvPrmUA9RGYOR8IHs2Slq/hsJcDrW1DgpHC2S8FQAD57RBZueBts6PKGttBH9VgPN4aoUU6Fj+FwCYuQm8rftAxMskgLnzy483N4P3srwJk3KAufMc5+QBV8maBW84woDqvl+zZg1WrVrl/t1kMiE9vYcV81yM6cB1i4X77m+uq0oIHlWd7o73ntbYZofc6vQcscxx7hRSvcrdphNtdUBpdQjfaJwETK4GJ1UL6TlAo1W43+zxaXJkJ3ROUHU9o+tlrl7cbEyKAeNSDTRqOowkkq4LzD0y3cctveUq3DPvDvevPM9gt7fDaWuHw24Dx9sAuQQAAxhD2jArnLwQXBjPcHUokkk44KrxYUlDrXA4u291kUq7prV5SctxAAyqzutl6Py3VwRv4rSogSguLg5SqRTV1Z4zlKurq5GUlNQlvVKphFIZwGA8hTbk+71H6wFfN5DRRwG+Fmi1Shm0St/eJqp+DWwSCQelUg0o1d2eTzD6/lyJXWf4eBXvR9oe9jHpE1FbxBQKBa6//nrs2bPHfYzneezZswdTp04VMWeEkHASvWq2atUqLFmyBJMnT0Zubi42bdqE1tZWPPLII2JnjRASJqIHokWLFqG2thZr165FVVUVJkyYgM8//7xLAzYhJHKJPo6oLyJmHBEhEcrXz2hkjJoihAxoFIgIIaKjQEQIEZ3ojdV94Wre8nmqByEkrFyfzd6aogd0IGppaQEA30ZXE0JE09LSAoPB4PX8gO4143keFRUV0On652LwrikopaWlEdmrF8nXF8nXBoTv+hhjaGlpQUpKCiQ9rCgwoEtEEokEaWlpYmejV3q9PiL/mV0i+foi+dqA8FxfTyUhF2qsJoSIjgIRIUR0FIhCSKlUYt26dYGtGDAARPL1RfK1Af3v+gZ0YzUhJDJQiYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIj85O+utO+//z5GjhwJlUqFcePG4V//+pfHecYY1q5di+TkZKjVasyZMwcXL167vUx4+HNtb775JmbMmIHo6GhER0djzpw5XdLn5eWB4ziPn7lz54b6Mrzy5/q2bt3aJe8qlcojzUB972bNmtXl2jiOw/z5891pwv7eMeKznTt3MoVCwbZs2cLOnDnDli9fzoxGI6uuru42/f79+5lUKmV/+MMf2NmzZ9n/+3//j8nlcnbq1Cl3mo0bNzKDwcA++ugjduLECXb33XezrKws1tbWFq7LYoz5f233338/e+ONN9ixY8fYuXPnWF5eHjMYDKysrMydZsmSJWzu3LmssrLS/dPQ0BCuS/Lg7/Xl5+czvV7vkfeqqiqPNAP1vauvr/e4rtOnTzOpVMry8/PdacL93lEg8kNubi5bsWKF+3en08lSUlLYhg0buk1/3333sfnz53scu+GGG9jjjz/OGGOM53mWlJTE/vjHP7rPNzU1MaVSyXbs2BGCK/DO32u7lsPhYDqdjm3bts19bMmSJWzBggXBzmpA/L2+/Px8ZjAYvD5fJL13r776KtPpdMxsNruPhfu9o6qZj1y70s6ZM8d9rKddaQHghx9+8EgPALfffrs7fWFhIaqqqjzSGAwG3HDDDV6fMxQCubZrWSwW2O12xMR47k2zd+9eJCQkYMSIEfjFL36B+vr6oObdF4Fen9lsxpAhQ5Ceno4FCxbgzJkz7nOR9N699dZbWLx4MbRaz33KwvneUSDyUU+70lZVVXX7mKqqqh7Tu279ec5QCOTarvX0008jJSXF4wMxd+5cbN++HXv27MGLL76Ib775BnfccQeczu53IA2VQK5vxIgR2LJlCz7++GO8/fbb4Hke06ZNQ1lZGYDIee8OHjyI06dPY9myZR7Hw/3eDejZ96R/2LhxI3bu3Im9e/d6NOguXrzYfX/cuHEYP348srOzsXfvXsyePVuMrPps6tSpHnvrTZs2DaNGjcJf//pXPP/88yLmLLjeeustjBs3Drm5uR7Hw/3eUYnIR/7uSgsASUlJPaZ33frznKEQyLW5vPTSS9i4cSO+/PJLjB8/vse0Q4cORVxcHC5dutTnPPujL9fnIpfLMXHiRHfeI+G9a21txc6dO/Hoo4/2+jqhfu8oEPkokF1pp06d6pEeAHbv3u1On5WVhaSkJI80JpMJBw4cCOtOt4HuuPuHP/wBzz//PD7//HNMnjy519cpKytDfX09kpOTg5JvXwVjR2Gn04lTp0658z7Q3ztAGFpitVrx4IMP9vo6IX/vwtYsHgF27tzJlEol27p1Kzt79ix77LHHmNFodHfrPvTQQ+yZZ55xp9+/fz+TyWTspZdeYufOnWPr1q3rtvveaDSyjz/+mJ08eZItWLBAtC5gf65t48aNTKFQsA8++MCji7elpYUxxlhLSwtbvXo1++GHH1hhYSH7v//7PzZp0iSWk5PD2tvbw3ptgVzf+vXr2RdffMEuX77Mjhw5whYvXsxUKhU7c+aMO81Afe9cbrrpJrZo0aIux8V47ygQ+en1119nGRkZTKFQsNzcXPbjjz+6z82cOZMtWbLEI/3f//53Nnz4cKZQKNiYMWPYp59+6nGe53n27LPPssTERKZUKtns2bNZQUFBOC6lC3+ubciQIQxAl59169YxxhizWCzstttuY/Hx8Uwul7MhQ4aw5cuXdxmLE07+XN+TTz7pTpuYmMjmzZvHjh496vF8A/W9Y4yx8+fPMwDsyy+/7PJcYrx3tAwIIUR01EZECBEdBSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAaUrVu3wmg0un//7W9/iwkTJrh/z8vLw8KFC8OeL9I3FIhIt1xrFv/bv/1bl3MrVqwAx3HIy8vzSB/sAJCZmYlNmzZ5HFu0aBEuXLjg9TGvvfYatm7d6v591qxZePLJJ4OaLxJ8FIiIV+np6di5cyfa2trcx9rb2/Huu+8iIyNDlDyp1WokJCR4PW8wGDxKTGRgoEBEvJo0aRLS09Oxa9cu97Fdu3YhIyMDEydO7NNzd1dSWbhwobuUNWvWLBQXF+Opp55y7yIBdK2aXevqklleXh6++eYbvPbaa+7nKCwsxLBhw/DSSy95PO748ePgOC7sayURAQUi0qOlS5ciPz/f/fuWLVvwyCOPhPx1d+3ahbS0NDz33HOorKxEZWWl38/x2muvYerUqVi+fLn7OTIyMrpcEwDk5+fj5ptvxrBhw4J1CcQPFIhIjx588EF89913KC4uRnFxMfbv3+/TQlp9FRMTA6lUCp1Oh6SkpIBWPTQYDFAoFNBoNO7nkEqlyMvLQ0FBgXvvL7vdjnfffRdLly4N9mUQH9Ga1aRH8fHxmD9/PrZu3QrGGObPn4+4uDixs9UnKSkpmD9/PrZs2YLc3Fz87//+L6xWK+69916xszZoUYmI9Grp0qXYunUrtm3bFrRSg0QiwbVLYdnt9qA8ty+WLVvmbojPz8/HokWLoNFowvb6xBMFItKruXPnwmazwW634/bbbw/Kc8bHx3u0+zidTpw+fdojjUKh6PP2Nd6eY968edBqtdi8eTM+//xzqpaJjKpmpFdSqRTnzp1z3/emubkZx48f9zgWGxuL9PT0LmlvvfVWrFq1Cp9++imys7PxyiuvoKmpySNNZmYm9u3bh8WLF0OpVAZUJczMzMSBAwdQVFSEqKgoxMTEQCKRuNuK1qxZg5ycnLAueE+6ohIR8Yler4der+8xzd69ezFx4kSPn/Xr13ebdunSpViyZAkefvhhzJw5E0OHDsUtt9zikea5555DUVERsrOzER8fH1C+V69eDalUitGjRyM+Ph4lJSXuc48++ihsNltYegFJz2jNajJoffvtt5g9ezZKS0u77JRKwosCERl0rFYramtrsWTJEiQlJeGdd94RO0uDHlXNyKCzY8cODBkyBE1NTfjDH/4gdnYIqERECOkHqERECBEdBSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghovv/jdNWaZd83W8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:47.511883Z", + "iopub.status.busy": "2024-03-22T19:39:47.511612Z", + "iopub.status.idle": "2024-03-22T19:39:47.873180Z", + "shell.execute_reply": "2024-03-22T19:39:47.872271Z" + }, + "papermill": { + "duration": 0.381365, + "end_time": "2024-03-22T19:39:47.875111", + "exception": false, + "start_time": "2024-03-22T19:39:47.493746", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+5UlEQVR4nO3deZxT5b0/8M/JvkyW2feNHQQcKEIBFxSqCCpoq3hFC6joVbSX8uNV5d6KV61FW6FStbTXlu0KUrVUb2srUhRURGSRfR9mZfYtmUz2nOf3x5lkJmSWJJPkzGS+79drXsmc8yR5zmTyzbM/HGOMgRBCRCQROwOEEEKBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUiQojoZGJnoC94nkdVVRV0Oh04jhM7O4SQqzDG0NraiqysLEgk3Zd7BnQgqqqqQm5urtjZIIT0oqKiAjk5Od2eH9CBSKfTARAuUq/Xi5wbQsjVzGYzcnNzfZ/V7gzoQOStjun1egpEhPRjvTWdUGM1IUR0FIgIIaKjQEQIEd2AbiMKBmMMbrcbHo9H7KzENalUCplMRsMoSFjiOhA5nU5UV1fDarWKnZVBQaPRIDMzEwqFQuyskAEmbgMRz/MoKSmBVCpFVlYWFAoFfVtHCWMMTqcT9fX1KCkpwfDhw3scvEbI1eI2EDmdTvA8j9zcXGg0GrGzE/fUajXkcjnKysrgdDqhUqnEzlKA5jYnPj9fhwy9ClOHJtMXUz8i6teWx+PBc889h8LCQqjVagwdOhQvvfQSIrmMNn0zx05//1t/eakBZY1WHCxpQpXJLnZ2SCeiloheffVVbNiwAVu2bME111yDw4cPY8mSJTAYDPjJT34iZtZInHF7eJQ2tPl+v1RnQbZRLWKOSGeiBqKvv/4a8+bNw9y5cwEABQUFePfdd/Htt9+KmS0Sh+paHfDwQkn7jvGZyKIg1K+IWpaeNm0a9uzZgwsXLgAAjh8/jq+++gq33357l+kdDgfMZrPfDyHBqG6vig1NS8DwdB20yrhtHh2QRH03nn32WZjNZowaNQpSqRQejwcvv/wyFi5c2GX6NWvW4IUXXohxLmNvxowZKCoqwuuvvy52VuJGc5sTAJCSQEML+iNRS0Tvvfcetm3bhu3bt+Po0aPYsmULXnvtNWzZsqXL9KtWrYLJZPL9VFRUxDjHZKCaMiQJd0/IxqgMPcobrThU2oQGi0PsbBEvJqKcnBz25ptv+h176aWX2MiRI4N6vMlkYgCYyWQKOGez2diZM2eYzWYLOOdwebr9cbk9Qad1BpE2VIsWLWIA/H6ys7PZ7373O790R48eZRzHsdLSUsYYY2vXrmVjx45lGo2G5eTksCeeeIK1trb6PebLL79k119/PVOpVCwnJ4c9/fTTzGKxhJzH7vT0N+9PPvyukq379Dw7Vt4sdlbiXk+f0c5ErZpZrdaALl+pVAqe56P6um99fqnbc4UpWsyfkO37/X++KIbL0/VwgpxENe6d1LEw28b9JbA5/aeS/PQHI0LK2/r163HhwgWMHTsWL774IgBg7dq12L59O5544glfum3btmH69OnIz88HIHSd//a3v0VhYSEuX76MJ598Ej/72c/wu9/9DgBQXFyM2bNn4xe/+AU2btyI+vp6PPXUU3jqqaewadOmkPI40OlVcgCAxeEWOSfES9Sq2Z133omXX34ZH3/8MUpLS/HXv/4V69atw9133y1mtkRlMBigUCig0WiQkZGBjIwMLFy4EPv370d5eTkAYdT4jh07/NrSli9fjptvvhkFBQW45ZZb8Itf/ALvvfee7/yaNWuwcOFCLF++HMOHD8e0adPw29/+Flu3boXdHt9jaqxONw5ebsTZaqFzI0ElfP+22ikQ9ReilojeeOMNPPfcc3jyySdRV1eHrKwsPP7441i9enVUX3fZzcO6PSe5arDtYzcO7Tbt1QNzH55e2JdsdauoqAijR4/G9u3b8eyzz2Lfvn2oq6vDvffe60vzr3/9C2vWrMG5c+dgNpvhdrtht9thtVqh0Whw/PhxnDhxAtu2bfM9hjHmmwozevToqOS9P2hqc+Lr4kYY1HKMztQjob3HjEpE/YeogUin0+H111+Pee+QQhZ8QTBaaUO1cOFCXyDavn07Zs+ejeTkZABAaWkp7rjjDjzxxBN4+eWXkZSUhK+++gqPPPIInE4nNBoNLBYLHn/88S4Hiubl5UUt3/1Bm0OoLntLQr5AZHeJlifijwZT9EMKhSJg2ZIHHngAP//5z3HkyBF88MEH+P3vf+87d+TIEfA8j7Vr1/ra3DpXywBg4sSJOHPmDIYN6740GK+8JR9vAPLetjlpaZj+on9PDhqkCgoKcPDgQZSWlqKhoQE8z6OgoADTpk3DI488Ao/Hg7vuusuXftiwYXC5XHjjjTdw+fJl/O///q9foAKAZ555Bl9//TWeeuopHDt2DBcvXsRHH32Ep556KtaXF3NXByKNUgoAcLp5uDzR7RghwaFA1A+tXLkSUqkUY8aMQWpqqq+ReuHChTh+/DjuvvtuqNUdUxSuvfZarFu3Dq+++irGjh2Lbdu2Yc2aNX7POX78eOzbtw8XLlzADTfcgAkTJmD16tXIysqK6bWJoa09EHlHUyukEsyfkI0HpuRBSjPw+wWOsQhOdY8xs9kMg8EAk8kUsIuH3W5HSUkJCgsL++WSFPGov/7N3ztcgSvNNswZl4mRGT1va0Miq6fPaGdUIiJxzzu2S6OQipwT0h1qrCZx747xmWhzeJCqU/qOlTW2ob7VgdwkDdL1/af0NlhRiYjEveQEJfKSNVDJO0pEZ6vN+PJiAyqbaT3z/oACERmUvEHJSl34/QIFIhLX2hxufFvShDNV/mtXaRRCqwQFov6BAhGJa81WJ/ZfasC3JY1+x70N13YXBaL+gAIRiWveQKO+qsdM2T4dhwJR/0CBiMQ1m1MYOd25obrz7w43jazuDygQkbhm85aI5FQi6s8oEPVDM2bMwPLly8XOhk9/y08orE5hesfVVTOjRoF7JmZjflF2Vw8jMUYDGuOU0+mkPejRqY3oqhKRQiZBfrJWjCyRLlCJqJ9ZvHgx9u3bh/Xr14PjOHAch+LiYjzyyCO+HXFHjhyJ9evXBzxu/vz5ePnll5GVlYWRI0cCEPaOKyoqgkqlwqRJk/Dhhx+C4zgcO3bM99hTp07h9ttvR0JCAtLT0/HQQw+hoaGh2/yUlpbG6s/RZ7ZuGqtJ/zK4SkSMAR4RFsOSygOXc+xGV2tWJyYmIicnB++//z6Sk5Px9ddf47HHHkNmZibuu+8+32P37NkDvV6P3bt3AxAmHN55552YM2cOtm/fjrKysoAqVktLC2655RY8+uij+M1vfgObzYZnnnkG9913Hz777LMu85OamhqBP0pszBiRBku+G4nawNLh2Woz2hxujM7U0z5nIhtcf32PC/hybexf94b/B8iCqyZdvWa1V+f93AoLC3HgwAG89957foFIq9Xij3/8o69K9vvf/x4cx+Htt9+GSqXCmDFjcOXKFSxdutT3mDfffBMTJkzAL3/5S9+xjRs3Ijc3FxcuXMCIESO6zM9AkahVdBmEAOBAcSNMNheyjGoKRCKjv/4A8dZbb2Hjxo0oLy+HzWaD0+lEUVGRX5px48b5tQudP38e48eP91uSY/LkyX6POX78OD7//HMkJCQEvGZxcTFGjAhtF5KBRCmXADbqwu8PBlcgksqF0okYr9sHO3bswMqVK7F27VpMnToVOp0Ov/71r3Hw4EG/dFpt6I2vFosFd955J1599dWAc5mZmWHnuT/geYbDZc1QyiQYm22A9KqdEVQyGl3dXwyuQMRxQVeRxHT1mtX79+/HtGnT8OSTT/qOFRcX9/o8I0eOxDvvvAOHwwGlUlgC49ChQ35pJk6ciL/85S8oKCiATNb1v0NXa2gPBE4Pj/2XhEb3sdmGgPNKudBXQyUi8VGvWT909ZrVw4cPx+HDh7Fr1y5cuHABzz33XEBA6coDDzwAnufx2GOP4ezZs9i1axdee+01AADX3ni+bNkyNDU14d/+7d9w6NAhFBcXY9euXViyZIkv+HS1hvZA4C3pKGSSgNIQQCWi/oQCUT909ZrVt912G+655x4sWLAAU6ZMQWNjo1/pqDt6vR5/+9vfcOzYMRQVFeG//uu/fHvGeduNsrKysH//fng8Htx6660YN24cli9fDqPR6NsRpLs1tPs7b0lH2c02T1Qi6j9ozepBZtu2bViyZAlMJpPfAvyR0N/+5uWNVvzlaCVSEhR4aGpBwPlvS5qw/1IDxmTpcds1A69HcCAIds3qwdVGNAht3boVQ4YMQXZ2No4fP+4bIxTpINQfOdxClUsp73ow48h0HdL1SuhVfetMIH1HgSjO1dTUYPXq1aipqUFmZibuvfdevPzyy2JnKybsrp6rZgaNHAYNBaH+gAJRnPvZz36Gn/3sZ2JnQxS+EpGMpnf0dxSISNwalalHul4VsBaRl9XpRnFdGziu6+59EjsUiEjcSlDKfNtMd8Vid+NfZ2uhU8koEIks7rvvB3Cn4IAz0P7WChl13/cXcRuI5HKhEdJqpX2rYsX7t/b+7cV2vqYVxytaYLJ2veKCNxA53fyAC6LxJm6rZlKpFEajEXV1dQAAjUbjG01MIosxBqvVirq6OhiNRkil/aNx+LvyZlSb7Ljz2qwue8cU0o7vYaeHp0ZtEcVtIALgW7bCG4xIdBmNxn61VIh36oZK3nXBXyrhIJVw8PAMTjcFIjHFdSDiOA6ZmZlIS0uDyyXCgmiDiFwu7zclIa+OKR5d54vjOChkEticHjipnUhUcR2IvKRSab/7kJDoYox1BKJuSkSAUD2zwQOnhwKRmAZFICKDj5tn8PBCA3R3I6sBYOboNABAoqb/Lw8TzygQkbjkbR+ScJxfo/TVaCeP/iFuu+/J4Na5Wka9pf0flYhIXNKr5PjR93J81bPuVJtsaLQ4kaZTIk0v/tIlgxUFIhKXFDIJcpM0vaY7fcWMk1dMmDo0mQKRiESvml25cgUPPvggkpOToVarMW7cOBw+fFjsbJFBovPoaiIeUUtEzc3NmD59Om6++Wb885//RGpqKi5evIjExEQxs0XiQK3ZjhqTHSk6JbKN3S8CR4GofxA1EL366qvIzc3Fpk2bfMcKCwtFzBGJFyUNbThQ3Ihx2YbgAhGNIxKVqFWz//u//8OkSZNw7733Ii0tDRMmTMDbb7/dbXqHwwGz2ez3Q0hXvL1m3a1F5OXt2qcSkbhEDUSXL1/Ghg0bMHz4cOzatQtPPPEEfvKTn2DLli1dpl+zZg0MBoPvJzc3N8Y5JgOFdxxRT6OqgY7BjhSIxCVqIOJ5HhMnTsQvf/lLTJgwAY899hiWLl2K3//+912mX7VqFUwmk++noqIixjkmA0VvWwl5+dYkoqqZqERtI8rMzMSYMWP8jo0ePRp/+ctfukyvVCp9O5YS0hOHK7j1qlN1StwxPhNqBc1FFJOogWj69Ok4f/6837ELFy4gPz9fpByReNHRRtRziUijkGF4ui4WWSI9ELVq9tOf/hTffPMNfvnLX+LSpUvYvn07/ud//gfLli0TM1skDvS2BAjpX0Tf6fXvf/87Vq1ahYsXL6KwsBArVqzA0qVLg3pssLtIksGnstkKu8uDnERNjz1nPM9wqd4Cp5vH6Ew9pBKalxZJA2an1zvuuAN33HGH2NkgcSYnsffpHV4fn6gGAAxJ1UKjEP0jMSiJPsWDEDFJJBzkUqEU5HLTAvpioUBE4o7d5cGJyhZcrG0NKn1HF74nmtkiPaByKIk7ZpsLe87WQauUBtUjJpdKAHjg8lCJSCxUIiJxJ9QeM5r4Kj4KRCTuONzewYzB/XvL2+ebuWh0tWgoEJG4Y3f1vntHZzTfTHzURkTiTqhVs4l5iRidqUe6jlZoFAsFIhJ3Qq2aBbOkLIkuqpqRuEPTOwYeKhGRuDM+24DcRDUM6uA2TWyxOlHX6kCCUoasHlZzJNFDgYjEneQEJZITgl8upri+DV9cqMeoDB0FIpFQ1YwMer7lYqn7XjRUIiJx50JtKxgD8pI0QS14RgMaxUclIhJ3vrzYgH+crIbJ5goqPe3kIT4KRCTuhD6y2jv7ngKRWCgQkbjCGPNVsYIdWU0lIvFRICJxxeHm4V1z1NsI3RuFb64Zzb4XCzVWk7jiHcwol3KQBRmINAoZfjAm3VcyIrFHgYjElY72oeBHVStkEozNNkQrSyQI9BVA4oojxJn3pH8Iq0R0+fJlDBkyJNJ5IaTPkhMUuPPaTEi40HbjqGgSdv3ITep51w8SHWF9bQwbNgw333wz3nnnHdjt9kjniZCwaRQyDEvTYUhqQkiP23W6Bn8/UY0Wa3Bjj0hkhRWIjh49ivHjx2PFihXIyMjA448/jm+//TbSeSMkZmhxNHGFFYiKioqwfv16VFVVYePGjaiursb111+PsWPHYt26daivr490PgkJSq3ZjvM1rWiwOEJ6nJzmm4mqTy16MpkM99xzD95//328+uqruHTpElauXInc3Fz8+Mc/RnV1daTySUhQzlSb8Y+T1ThfE9xWQl4030xcfQpEhw8fxpNPPonMzEysW7cOK1euRHFxMXbv3o2qqirMmzcvUvkkJCi+XrMQxwRRiUhcYfWarVu3Dps2bcL58+cxZ84cbN26FXPmzIFEIryZhYWF2Lx5MwoKCiKZV0J6Fc44IqCjREQ7eYgjrEC0YcMGPPzww1i8eDEyMzO7TJOWloY//elPfcocIaFy9DbPrK0BKP0KcLYB6dcAmdcCHEdVM5GFFYh2796NvLw8XwnIizGGiooK5OXlQaFQYNGiRRHJJCHB6livuotA1FoLHHsHcDuF31vKAbsJGHITRqTrkKJVIk0f/MqOJHLCaiMaOnQoGhoaAo43NTWhsLCwz5kiJFwOl1A1CxiUyHuAs/8nBCFDDpD3feF42deA6QqyjWqMyzEgXU9bCokhrEDEWNezlC0WC1QqeiOJeOzeQHR1G1HVMaFaJlcDY38IDL0ZyBgrnCv5IraZJAFCqpqtWLECAMBxHFavXg2NpmM/KI/Hg4MHD6KoqCiiGSQkWIwxzB6bAbuLh0bZKRDxPFBxULhfeAOgaP+/LbgBqDsLNJfC2lSJepYImVSCbFpAP+ZCCkTfffcdAOENP3nyJBSKju1aFAoFrr32WqxcuTKyOSQkSBzHYViaLvBEwwWhLUiuBjLGdxxXG4GU4UDdObQUH8XOllHINKhw/+S8mOWZCEIKRJ9//jkAYMmSJVi/fj30en1UMkVIRNWcFG6zigCp3P9cxnig7hzUzecAjKDue5GE1Ua0adMmCkKk37E43Dhf04orLbaOgy4b0HRZuJ8+NvBBiYWAXA0574DOUevrdSOxFXSJ6J577sHmzZuh1+txzz339Jh2586dfc4YIaGqMdnxj5PV/tWr+vMA44GENECbEvggiQRIGgJJ23Ek2spQp8uJbaYJgBACkcFgANe+xovBQKvZkf7H3lXXfd1Z4TZtTPcPTBkO6ZUTSLSXo9I9FYwx3/86iY2gA9GmTZu6vE9If+Gd3qHyjqp2OwFThXA/dWT3D0wshFQqgdrVBJmrFR6eQSalQBRLYbUR2Ww2WK1W3+9lZWV4/fXX8emnn0YsY4SEyu5bJra9RNRSJgxkVBsBdWL3D5SrINULU5X0jmqa+CqCsALRvHnzsHXrVgBAS0sLJk+ejLVr12LevHnYsGFDRDNISLACBjN6G6mThgC9VLU4Yx4KkjW4IdUKmYTWu461sFdovOGGGwAAH3zwATIyMlBWVoatW7fit7/9bUQzSEiwvCUiX9WscyDqjTEPmQY1CmRNtK2QCML6i1utVuh0wsCxTz/9FPfccw8kEgm+//3vo6ysLKyMvPLKK+A4DsuXLw/r8YT4NVZbmwBbCyCRAsYgBigacoRSk7UJcFiim1ESIOzF8z/88ENUVFRg165duPXWWwEAdXV1YY0vOnToEP7whz9g/PjxvScmpBuTC5PwgzHpyDKohZn1AKDPAmRBzKiXq2GR6mGyuWBtrIxuRkmAsALR6tWrsXLlShQUFGDKlCmYOnUqAKF0NGHChJCey2KxYOHChXj77beRmNhDgyIhvchN0mBstgEGjRwwXxEO6rODfvzxFjXOVJvRUF0anQySboUViH70ox+hvLwchw8fxieffOI7PnPmTPzmN78J6bmWLVuGuXPnYtasWb2mdTgcMJvNfj+EdMlcJdwagh+g6E7IAABwrTXRyBHpQdhbTmdkZCAjI8Pv2OTJk0N6jh07duDo0aM4dOhQUOnXrFmDF154IaTXIIMDzzNcqGuFSiZFno6DpK19vSx9VtDP4dEKXficpRpgrNeeNhI5YQWitrY2vPLKK9izZw/q6urA8/7jLi5fvtzrc1RUVOA//uM/sHv37qDXMFq1apVvKRIAMJvNyM3NDS3zJC7Z3R7882QNOA74SVF79706EVBog38SbSoYJwFzWgF7S89jj0hEhRWIHn30Uezbtw8PPfQQMjMzwxoOf+TIEdTV1WHixIm+Yx6PB1988QXefPNNOBwOSKX+i1splUoolbSUJwlkdXYsmi9pbW8fMgTfPgQAMrkCVnkSeN4CtNZQIIqhsALRP//5T3z88ceYPn162C88c+ZMnDx50u/YkiVLMGrUKDzzzDMBQYiQntjaA5FGIQVM3obq4KtlgLCTR7MiBR6+FbDUAWmjI51N0o2wAlFiYiKSkpL69MI6nQ5jx/ovy6DVapGcnBxwnJDe2NrHEKllHNDa3lCtD20mvUImQZs8GR7XZSEQkZgJq9fspZdewurVq/3mmxEiJm+JyMBMwmRXqRzQpob0HNlGNcaOGIqUBCXQRoEolsIqEa1duxbFxcVIT09HQUEB5HL/Ve+OHj0aVmb27t0b1uMI8bYRJbrrhQP6LGGtoRCk61VIHzkCqFcAdrOwqJqc1q+OhbAC0fz58yOcDUL6xju9Q++sFQ6EMJDRj1wFqAzCGtdt9cFNDyF9FlYgev755yOdD0L6ZEyWHqk6JbIuNQE8wgpELg+PRosTSs6IRJiEdiIKRDER9jTjlpYW/PGPf8SqVavQ1NQEQKiSXblyJWKZIyRY6XoVxqbKoedbhQMh9pgBQLPViXe/Lcf+mvbhKNRgHTNhlYhOnDiBWbNmwWAwoLS0FEuXLkVSUhJ27tyJ8vJy31pFhMSUd1qHJqlj77IQKNuHjJiliQBKqcE6hsIqEa1YsQKLFy/GxYsX/UZFz5kzB198Qbtmkti7UNuKmspieBgLu31ILhNKQiZpkrCbcVu9sDkjibqwAtGhQ4fw+OOPBxzPzs5GTQ1NGCSxxRjDP0/W4Pjp0/DwLOQR1V4KqfBxsMv0cHNSwOMWpnqQqAsrECmVyi5nvl+4cAGpqaGN3SCkr+wuHjzvgdZRD5mEC7tEJJNKIJVwACeBW9U+vcPaGMGcku6EFYjuuusuvPjii3C5XACErX7Ly8vxzDPP4Ic//GFEM0hIb2wuDzSuJig5NyRyJaDpYv+yIMnbS0VuZfvMAe8sfhJVYQWitWvXwmKxIDU1FTabDTfddBOGDRsGnU6Hl19+OdJ5JKRHVqcbOkct5FIO0IU+kLEz73rVTm8gslIgioWwes0MBgN2796N/fv34/jx47BYLJg4cWJQi5sREmltDg8SnHVCaSbM9iGviXlGeHgGlcwF1IFKRDESciDieR6bN2/Gzp07UVpaCo7jUFhYiIyMDNohk4jC4hBKRAqVJPwR1e0m5LW3DbUx4dbaSIukxUBIZVjGGO666y48+uijuHLlCsaNG4drrrkGZWVlWLx4Me6+++5o5ZOQbtmtZqjcZqFEFMZAxi6pjcIOIB6XMN2DRFVIJaLNmzfjiy++wJ49e3DzzTf7nfvss88wf/58bN26FT/+8Y8jmklCejJK0wpPqhYKQ1qfJ6m22l2wOj3QKKTQqROFqpm1UQhMJGpCKhG9++67+M///M+AIAQAt9xyC5599lls27YtYpkjJBjJ7nqk6VQwphf0+bkOFDdi+8FynK1uBbTtvW/UThR1IQWiEydOYPbs2d2ev/3223H8+PE+Z4qQkHindvSxfQgA5O29Zi4P3zEMgHrOoi6kqllTUxPS09O7PZ+eno7m5uY+Z4qQoPE86qpKoOA8SEjICn9bmnbK9nFETjcP6KlEFCshvW8ejwcyWfcPkUqlcLvdfc4UIcFymWtQXNMCj0SBSaqkPgcib4nIeXWJiHrOoiqk940xhsWLF3e7k4bD4YhIpggJlqOxAgBgVaVDKe/7hguKziUidQrASYSlZx2tgCr07dRJcEIKRIsWLeo1DfWYkVhyNJUBANwJWREZw6bo3EYklQlbClkbhR8KRFETUiDatGlTtPJBSFg8zZUAABaBhmqgY66Z092+/Ic2uSMQJRVG5DVIoL5WqQkRj90Mt7UFDBwkxsgEomStApMLk2BQt28IoUkBcEFYm4hEDQUiMnCZKuFw87AqkqHThLC1dA8StQpMH9Zp9r53LBEtBxJV4U9TJkRs5itwuDxoVaZDp4rSd6q356ytXug5I1FBJSIycJkqkJOogTT7GqQkhb5GdVd4nsFkc8Hl4ZGqU4LTJAnd9i474LICisiUvIg/KhGRgcntBCz1SFDKMHz4KCRqFRF5WhfPY/PXpdh2sBxungk7xqqMwkka2Bg1FIjIwGS+AjBe6FKPYLe6dxwR0N6FD3RqJ6JAFC0UiMjAZL4Cp4dHuScJlc3WiD0tx3EdqzR6u/A1ycJtGzVYRwsFIjIwNZfB4nDjqzo19p6PbNe6ImAsEZWIoo0CEekWzzMcLW/Gv87Uor61H03f8bgA8xXYXR6YVFlI1ESmfchL0Xm+GdCp54wCUbRQICLd+uZyI/adr0dxvUXYpqe/MFUAvAcWaGCXGZCokUf06QNGV3urZs42wBm5aiDpQIGIdMnicONQqbCkyy2j0iLWKxURzcL8sgZZJsBxMEapROTytI8bkikAlUG4TwMbo4ICEenS6Ssm8Iwhy6jC8HSd2Nnx11wKAKiRpAEAjBEuEY1M12FyYRIStZ2el9qJoooGNJIuna9tBQCMyzYCEEpIxXUWaBRScQOTywZYauFhDFVcBgBEvI1oXI4h8KAmGWgspp6zKKESEQlgtrvQaHGC44AhqcJI4uI6Cz47V4fvylvEzVxzKcAYrDIDXDItVHIp1Iq+r0PUKyoRRRWViEiA8kahQTZDr4KqfbGxIalafHYOqDLZYHW6oVGI9K/TeAkAoM4YiR+l5MDm8kT8JZxuHlanGxzHXTULH9RzFiVUIiIB0vRKTCpIxDVZHVUUnUqONL0SjAGlDSL1HPG8LxDJ0kYgN0mDEVGoJp6uMmHT/lJ8dbFT0PH2nDlahXlnJKIoEJEAaToVbhieGtBWktc+sTSSI5lDYq4UgoBcBehzovYy3lKgvXNpS64ClO1Bj3rOIo4CEQnkaBXaYq765s9JFALRlRabCJkC0HBRuE0aiq8vN+F0laljPlgE+QKR+6pqH7UTRQ21ERE/5kvfwHNhN3RKCWRKDTBmHpA0BACQZVSB44AWqwutdhd0qsh2m/eIMaD+PADAqh+Cg2ebwHHA8LTIV82U7eOI7K6rgpwmBWgqoXaiKKASEenQdBmmEx/jXLUJl5ucQono1F98XdZKmRRpOiEY1cV6yoepUtiDXqZAlVRYFjZZq/ANPowkb4nIEVAiam8noqpZxIkaiNasWYPrrrsOOp0OaWlpmD9/Ps6fPy9mlgYvngcu7UGbw43ahNFou+5pIDEf8LiB8x/7Vie8fWwG/v2moRiamhDb/NWeFm5TR6GqVdg7L9PQt33uu6OSCx8Lh4sHz3dalZF6zqJG1EC0b98+LFu2DN988w12794Nl8uFW2+9FW1tbWJma3BqOA+0NcDslqHcOAVpRi0waq6wMJjpiq99JlGr8JUYYsbjAurPCvfTxqDaJLRRZRmjE4iUso7rc7g7Vc+8PWd2k7AwG4kYUduIPvnkE7/fN2/ejLS0NBw5cgQ33nijSLkapKqOwc3zKFONhkeiQLJWAcjVQM4koOwAUPolkDJcnN1Oa08J1US1ES59LmrNJQCENqtokEo4TMgzQi6V+F+uQiMsFetsExqs9VlRef3BqF+1EZlMJgBAUlKSyDkZZOxmoKUMdheP+oSR0CqlHaWenMlCqchSB7QIk02Pljfjz4fKUdIQg5IrY0DlYeF+9vdQ2eKAh2fQqWQdgw2jYMbINEwflhJY+vMtkkbVs0jqN4GI53ksX74c06dPx9ixY7tM43A4YDab/X5IBNSdARiDSZkBh0yHJG2nLcUVGiBjnHD/ylEAQKPFiaoWe2zGEzVdFj70UjmQMR51ZmFIQWGKNiI7u4aMuvCjot8EomXLluHUqVPYsWNHt2nWrFkDg8Hg+8nNzY1hDuNYwwUAQJ1a6KZP0l5V0sia0J7uIuBo9VWJqqI9nogx4PLejjzIVZgyJBmP3lCISQXRLTXbXR60WJ3+gxqBjkBEJaKI6heB6KmnnsLf//53fP7558jJ6X7E7KpVq2AymXw/FRUVMcxlnHLZAHMVAKBgxLX4wZh0jMy4ajH6hDTAmCssVl91DDlGYWBjrdkRlQGFPlXfCVVCmQLIm+o7rFPJo1otA4DdZ2qxaX8pzte0+p9ISBduLbVRff3BRtTGasYYnn76afz1r3/F3r17UVjY897iSqUSSqWyxzQkRE0lQslDm4LEpBQkdpcuawLQUgFUH4c+fxoSlDJYHG7Umu2+EdcRZW0Cij8T7hfcCCg0cLg9fj1a0dTlNA8A0KYKtw6LsFqjIgrXPgiJWiJatmwZ3nnnHWzfvh06nQ41NTWoqamBzSbSFILBqKlYuG0fPd2tlJFCL5qjFVxTia/r/EpzFN4rpxU4+YHQbW/IAXImwWx34e0vLuOjY1cCBxpGgW8skfuqEp9MCajbwzWViiJG1EC0YcMGmEwmzJgxA5mZmb6fP//5z2Jma/BgTCgRAWhNyMeJyhbfGJ0AUhmQ0d6JUH2so52ou/ThMlcB370jjF5W6oBr5gMch68vNcLlYXB5WExKRd7XCCgRAUBCe6moLbK7hwxmolfNiIhszcKYGIkUV/gk7DlbhyyjCguuy+s6feYEoOIQ0HgJORk3QaOQhr0uUZ3ZjjPVZtS3OsBxHNIlJoyyf4dUe3u7n0oPjF8AKHU4W23G2WozOA6YPiw5zIsNjbdEZL+6RAQI7UT1F6hEFEE06XUwM7V/6HWZaLIJHzi/rvuraZOFRuuWCqRYzuGxG68PuQvdwzN8cbEex9pXelS5WpBrOgKZtRhmnRKpaTogfSzsudejwiJFZUUdjlcKaa8rSIratI6rqb1tRM4uSkRaYa1sWOpikpfBgALRYNbSHoiMuWiyCFMWknrbraO90ZqrOQHkTw9ppDXPM/zjZDUu1Vkg4d34vuQMCj1nwGl5WOQqJOSOB8bOBLTJqG+y4u8nKn2PvTbXgGlDY1MaAuBbftbqdAeeTGgPRNZGgPcAkhhPeYlDFIgGM2+JyJCLplohECX3FohSRgqLhNnNQHMJWNIQ2FyeoKpoHAekJChRVVODu+XfII0zAQlyIHkoUofM6PiAt8syqmBQyzE6U4/8ZG04Vxg2vVqO8TkG6LsaJqAyCI3WbocQjK7KNwkdBaLBym4GbC0Ax8Gjy0ZzmxCUkhJ6CURSmTDSuuIQWksOYcdpoUT06A2FvVbTOI7D1Fw1iqoOQO02C13fI+cCKcMC0uYmabAgqZu2qhjQq+SYOTq965McJwSflgqhekaBqM/6xYBGIgJvaSghHS1ODjxjUMgk0CmD+G7KLAIAaFpLwdtNsDjcPW5J7fa0L6fhcQGnPhCCkMoAfG9xl0FoQPC1E1GDdSRQIBqsTO3tL8ZcNFuFalmiRhFc47M2BTDmQgqGcRJhIuylOku3yb+81IA/HypH63d/FZYUkSmB8fd17J7aT9ldHjS3OTu2nu4sgRqsI4kC0WDVUi7cGvKQm6TBvZNycMPwlOAfnzURADDMeRZS3oFL9V0HoistNhyvaIGibB88tWeFht2xP+yYs9WPvX+kEpu/Lu16bJUuU7htrfYtGkfCR4FoMHJaOyZtGnKglEmRk6hBblII0xVSRwHaFCQpeeRYTqHR4kSt2X+xfZeHx+7TNUgzn0YRzgt71I+cI6z8OABo5N6es6668FMAiUxosLY1xzhn8YcC0WDkrZZpU8KfKyWRAAU3QCaRYCx/Dkq3Gd+V+38gv7rYADRcxMjWb5CfrAEKb+wYnT0AaBQ9BCKJFNC1N2a3Txom4aNANBh16rZnjOHLi/U4dcUEd6gz6VNHAon5yNLJMLRxLy7VmHzjbi7WtuLyxTMY3rgHQ1M0kGVPAPKnRfhCoss7lsjWVSACOlXPamKUo/hFgWgw8gWiHJhtbhwubcbn5+ogCXWhMY4DRsxGgkaN0WoTHjKehEbC40qzFYe++QKj6j9Btl4OY941wIjbxFlmtg+8Y6O6HNQIdApEVCLqKxpHNNi4nUBre5ezMReN7d3uiVoFJJIwAoUmCRgzH3mn/gK0XgIOvIEMiQLXmCuh1kiRN3w8MGb+gBx97K2a2bqa+Ap0rFndWksjrPuISkSDjfmKsMCZSg+oDGhsC3JEdU9ShgHXLgDURsDthNRpweicZAz//hxIxv1IGAQ5AHmrZm2ObgKROlEYisC7acXGPhqY/yEkfJ3ahwBh/WkgiDlmvUksACY/Lgzw411QJGQIKysOYEkaBcZlG5Dc3WhzjhOqZ82lQvVM181IbNIrCkSDTUtH+xAANHlLRL1N7QiGRALoM/v+PP1EolaBWWN6CS66DCEQmas71vYmIaOq2WDCezq6mo15YIyhqU1oI+px+Q/SvfaSpW9IBAkLBaLBpLVaaM+QqwFNMsx2N1weBqmEgzHKi9EPVA63B40WR9crNQKAIVuoolkbhXWsSVioajaYNJcKt8ZcgOOgV8mw9MYhMNtc4fWYDQIffncFVS123HltJoal6QITyNXCwFBLvVAqShsV+0zGASoRDSbeQJQo7JbCcRwSlLKo7SEfD7TtqxG02rsZSwQAhvblSrwdASRkFIgGC7dDmPkOAEk9b9tEOiS0ByKLo4dAZPS2E1EgChcFosGipVwYP6RO9G2H8/m5Onx9qaHnD9kgp1O1B6IeS0Ttm4Ja6gCXvft0pFsUiAaL9m2DvKUht4fHiUoTDpY0wcPTMhbdSVAKjfg9Vs2UOiG4M0a9Z2GiQDQYMAY0XhLut7cPNbU5wTMGlVwKvYr6LLrj3draZHP1nDCxQLhtLoluhuIUBaLBoLUGsJuEqRbtJaK69jlmaTplyFsCDSZGjRCILA43XD2tTuBtd2u6HINcxR/6KhwM6s8Jt8nDAKnwwfKuMZ2qo4GMPVHJpbg214AEpRx8TysxGvMBTgJYm4SF0rzbUpOgUIko3jEGNFwQ7qd2jHGpaxUaVdP0FIh6c8uodEwuTOp5q2u5ShjcCFCpKAwUiOKduUr4lpbKgKShAISNDhvaJ7umJlAgipikIcJtE7UThYoCUbyrPibcpo72zYY3213gOEApl/R91v0g4PbwaLQ4etwyCYBQ9QWEQOTuJS3xQ21E8cxlA+rOCPezinyHjRoFnrhpKMx2NzVUB+F8bSs+PV2LnEQ17p2U231CbaqwUJy1SeilTL8mdpkc4KhEFM8qDwEet7AHlz7b7xTHcb6uadIzb/W13uIA66nBmuM62uHqzsYgZ/GDAlG8ctmBysPC/fxpA2696P4kSauAhOPgcPFo7W0Uetpo4ZaqZyGhQBSvSr4QPgjaFL/esharExu/KsGu0zU9f7sTH5lUgiSt/7CHbmlTAU2ysNwKlYqCRoEoHrVUAFVHhfvDf+BXGiprtMJkc8Fsc1H7UAi8460aegtEHAdkjBPuezsKSK8oEMUbhwU486EwfihjXMfUg3aljW0AgPxkbezzNoB5A1GNOYhJrZnjhR09zNUdO6aQHlEgiidOK3BihxCMtCnA8Fv9Trs8PCqbhX3cC5LD3OF1kMo2Cn+vKy028L1NElZogZQRwv3KQ1HOWXygQBQvbM3AsW3CSoEKLTD2hwG7aJQ1tsHp5qFTyWhqR4jSdEpMLkzCnLFBbg6Qc51wW3taeG9IjygQxYPmMuDIFmFvLWUCUPSAMJ7lKudrhDWVR2boqH0oRBIJh+nDUlCQog1uWV1DtjARlvFA2YHoZ3CAo0A0kPG80Dt2/F1h8KI+E/jeYqFadhWr043L9e2BKL2LtZdJ5OVPF25rTgjtRaRbFIgGKlsLcOwdoHR/R8N00UJhka4ucOAwMT8RhSlaqpb1QWWzFZ+dq4XZ3sv6RICwhGz6GOH9ubhL+OIgXaIpHgMNY0K7w8VPhXFCMgUwYnav0wnUCimmDwssKZHQHChuRGWzDUpZkH/PobcI0z3M1UDJXuF3EoBKRAOJ3QScfB84+zchCBmygUkP9xqEaCnYyJmQJ6wzdLyyBVZnEGt9K3XAyLnC/fKDQNWx6GVuAOsXgeitt95CQUEBVCoVpkyZgm+//VbsLPUvbofQFvTt20BjsTBGpfBGoOjBXhfgKm1ow/8eKEVdMONfSK+GpgpVW4eLx+fn6oMbnZ42Csj7vnD//D+Fxmsa1e5H9ED05z//GStWrMDzzz+Po0eP4tprr8Vtt92Guro6sbMmPlszUPw58M0GoS3I4xJ2jJj0CFAwXdhrvgeX6iz4+4kqNFtdOFVlilGm4xvHcZg5Og0SjsOF2lbsPlMLpzuItp8hMzq69C/vBY5tFxbap4AEAOCYyBOOpkyZguuuuw5vvvkmAIDneeTm5uLpp5/Gs88+2+NjzWYzDAYDTCYT9Hp9LLIbHR434LICTovQBW+pExZhb2voSKNJAgpvAlJHdjuB1cMzWOxuVJlsOFdjRmmDFQBQmKLFHeMzIZOK/r0TN85UmfHpmRowBujVcjw8vcA3JKLObAfHcZBKOHDo9HYxBkX9SWjKPhfmogGwKZPAG4cA+ixINEmQaY2QyZXgevmSGSiC/YyK2ljtdDpx5MgRrFq1yndMIpFg1qxZOHAgcOyFw+GAw9Ex18dsNgf3Qk0lQPGerr99/I6x7o/5HQ88Vt5kRV2rLeC09/zYHD1U7YGgotmKGpNQVeKYBxLevwdmdKYearkU4DiUeFJx2JmPFkkB2DkOOHfZL3t3T8hGml4FADh4uREHS5p85zgOmJiXiOnDUiClLaUjakyWHlqlFHvO1iFRK/cbl/X+kcpuS0nZiem4b/JSoGw/UHsGpy9cgstz8apUHJhUDo1ajaL8FADCc39XaYLTzYMJ4Q0ch/b7QmfEtTkG3zOcqDTB5vJ0mQelTIKiXKPv99NXzLB0094lk3D4Xn5H9d/pZlDI2q913H2AKjIFAFEDUUNDAzweD9LT0/2Op6en49y5cwHp16xZgxdeeCH0F3I7hBHHUeSxtcFl7aEdxiED5MKax7zTCrfDPy0DB5dUDbvciKEZo6BOzweShqCm3IrKy42As+t/bE+nqJSgkkEq4ZCSoERekgbjsg0waGjNoWjJT9Zi8bSCgKVBNAop5FIObp4FfPfJpRygNgKj5gJDZ6K87XMoLFegdjRA4W6FjHcAYOA8TshcPGDr+IiytmbwnQKc31MrpED78r8AwCwt4J1dByJeJgEsHV9+vMUEvpvlTZiUAywd5zgPD3hL1ixywxEGVPf9qlWrsGLFCt/vZrMZubk9rJjnZcwFrr1fuO/75upUQvCr6nR1vPe0RpsLcofHf8Qyx/lSSPUqX5tOosMNpcMtfKNxEjC5GpxULaTnAI1W4Xuzx+fIMTStY4Kq9xm9L9N5cbNrsgwYl22gUdMxJJEELjC3ZHqQW3rLVbhnzu2+X3meweWyw+O0w+1yguOdgFwCgAGMIWeYAx5eCC6MZ+gcimQSDug0PixjiANuT9etLlJpYFpnN2k5DoBB1XG9DB3/9orITZwWNRClpKRAKpWittZ/hnJtbS0yMjIC0iuVSiiVYQzGU2ijvt97oh4IdgMZfQIQbIFWq5RBqwzubaLq18AmkXBQKtWAUt3l+TRj8M+VHjjDp1upIaTtYR+TPhG1RUyhUOB73/se9uzZ4zvG8zz27NmDqVOnipgzQkgsiV41W7FiBRYtWoRJkyZh8uTJeP3119HW1oYlS5aInTVCSIyIHogWLFiA+vp6rF69GjU1NSgqKsInn3wS0IBNCIlfoo8j6ou4GUdESJwK9jMaH6OmCCEDGgUiQojoKBARQkQnemN1X3ibt4Ke6kEIiSnvZ7O3pugBHYhaW1sBILjR1YQQ0bS2tsJgMHR7fkD3mvE8j6qqKuh0/XMxeO8UlIqKirjs1Yvn64vnawNid32MMbS2tiIrKwuSHlYUGNAlIolEgpycHLGz0Su9Xh+X/8xe8Xx98XxtQGyur6eSkBc1VhNCREeBiBAiOgpEUaRUKvH888+Ht2LAABDP1xfP1wb0v+sb0I3VhJD4QCUiQojoKBARQkRHgYgQIjoKRIQQ0VEgClGou9K+//77GDVqFFQqFcaNG4d//OMffucZY1i9ejUyMzOhVqsxa9YsXLx49fYysRHKtb399tu44YYbkJiYiMTERMyaNSsg/eLFi8FxnN/P7Nmzo30Z3Qrl+jZv3hyQd5VK5ZdmoL53M2bMCLg2juMwd+5cX5qYv3eMBG3Hjh1MoVCwjRs3stOnT7OlS5cyo9HIamtru0y/f/9+JpVK2a9+9St25swZ9vOf/5zJ5XJ28uRJX5pXXnmFGQwG9uGHH7Ljx4+zu+66ixUWFjKbzRary2KMhX5tDzzwAHvrrbfYd999x86ePcsWL17MDAYDq6ys9KVZtGgRmz17Nquurvb9NDU1xeqS/IR6fZs2bWJ6vd4v7zU1NX5pBup719jY6Hddp06dYlKplG3atMmXJtbvHQWiEEyePJktW7bM97vH42FZWVlszZo1Xaa/77772Ny5c/2OTZkyhT3++OOMMcZ4nmcZGRns17/+te98S0sLUyqV7N13343CFXQv1Gu7mtvtZjqdjm3ZssV3bNGiRWzevHmRzmpYQr2+TZs2MYPB0O3zxdN795vf/IbpdDpmsVh8x2L93lHVLEjeXWlnzZrlO9bTrrQAcODAAb/0AHDbbbf50peUlKCmpsYvjcFgwJQpU7p9zmgI59quZrVa4XK5kJTkvzfN3r17kZaWhpEjR+KJJ55AY2NjRPMejHCvz2KxID8/H7m5uZg3bx5Onz7tOxdP792f/vQn3H///dBq/fcpi+V7R4EoSD3tSltTU9PlY2pqanpM770N5TmjIZxru9ozzzyDrKwsvw/E7NmzsXXrVuzZswevvvoq9u3bh9tvvx0eT9c7kEZLONc3cuRIbNy4ER999BHeeecd8DyPadOmobKyEkD8vHfffvstTp06hUcffdTveKzfuwE9+570D6+88gp27NiBvXv3+jXo3n///b7748aNw/jx4zF06FDs3bsXM2fOFCOrQZs6darf3nrTpk3D6NGj8Yc//AEvvfSSiDmLrD/96U8YN24cJk+e7Hc81u8dlYiCFOqutACQkZHRY3rvbSjPGQ3hXJvXa6+9hldeeQWffvopxo8f32PaIUOGICUlBZcuXepznkPRl+vzksvlmDBhgi/v8fDetbW1YceOHXjkkUd6fZ1ov3cUiIIUzq60U6dO9UsPALt37/alLywsREZGhl8as9mMgwcPxnSn23B33P3Vr36Fl156CZ988gkmTZrU6+tUVlaisbERmZmZEcl3sCKxo7DH48HJkyd9eR/o7x0gDC1xOBx48MEHe32dqL93MWsWjwM7duxgSqWSbd68mZ05c4Y99thjzGg0+rp1H3roIfbss8/60u/fv5/JZDL22muvsbNnz7Lnn3++y+57o9HIPvroI3bixAk2b9480bqAQ7m2V155hSkUCvbBBx/4dfG2trYyxhhrbW1lK1euZAcOHGAlJSXsX//6F5s4cSIbPnw4s9vtMb22cK7vhRdeYLt27WLFxcXsyJEj7P7772cqlYqdPn3al2agvnde119/PVuwYEHAcTHeOwpEIXrjjTdYXl4eUygUbPLkyeybb77xnbvpppvYokWL/NK/9957bMSIEUyhULBrrrmGffzxx37neZ5nzz33HEtPT2dKpZLNnDmTnT9/PhaXEiCUa8vPz2cAAn6ef/55xhhjVquV3XrrrSw1NZXJ5XKWn5/Pli5dGjAWJ5ZCub7ly5f70qanp7M5c+awo0eP+j3fQH3vGGPs3LlzDAD79NNPA55LjPeOlgEhhIiO2ogIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiI4CERlQNm/eDKPR6Pv9v//7v1FUVOT7ffHixZg/f37M80X6hgIR6ZJ3qdB///d/Dzi3bNkycByHxYsX+6WPdAAoKCjA66+/7ndswYIFuHDhQrePWb9+PTZv3uz7fcaMGVi+fHlE80UijwIR6VZubi527NgBm83mO2a327F9+3bk5eWJkie1Wo20tLRuzxsMBr8SExkYKBCRbk2cOBG5ubnYuXOn79jOnTuRl5eHCRMm9Om5uyqpzJ8/31fKmjFjBsrKyvDTn/7Ut3g7EFg1u1rnktnixYuxb98+rF+/3vccJSUlGDZsGF577TW/xx07dgwcx8V8iRIioEBEevTwww9j06ZNvt83btyIJUuWRP11d+7ciZycHLz44ouorq5GdXV1yM+xfv16TJ06FUuXLvU9R15eXsA1AcCmTZtw4403YtiwYZG6BBICCkSkRw8++CC++uorlJWVoaysDPv37w9q/Zq+SkpKglQqhU6nQ0ZGRliLjRkMBigUCmg0Gt9zSKVSLF68GOfPn/dtueNyubB9+3Y8/PDDkb4MEiRaKpb0KDU1FXPnzsXmzZvBGMPcuXORkpIidrb6JCsrC3PnzsXGjRsxefJk/O1vf4PD4cC9994rdtYGLSoRkV49/PDD2Lx5M7Zs2RKxUoNEIsHVK9C4XK6IPHcwHn30UV9D/KZNm7BgwQJoNJqYvT7xR4GI9Gr27NlwOp1wuVy47bbbIvKcqampfu0+Ho8Hp06d8kujUCj6vGtEd88xZ84caLVabNiwAZ988glVy0RGVTPSK6lUirNnz/rud8dkMuHYsWN+x5KTk5GbmxuQ9pZbbsGKFSvw8ccfY+jQoVi3bh1aWlr80hQUFOCLL77A/fffD6VSGVaVsKCgAAcPHkRpaSkSEhKQlJQEiUTiaytatWoVhg8fHtN1pkkgKhGRoOj1euj1+h7T7N27FxMmTPD7eeGFF7pM+/DDD2PRokX48Y9/jJtuuglDhgzBzTff7JfmxRdfRGlpKYYOHYrU1NSw8r1y5UpIpVKMGTMGqampKC8v95175JFH4HQ6Y9ILSHpGS8WSQevLL7/EzJkzUVFREbBBIYktCkRk0HE4HKivr8eiRYuQkZGBbdu2iZ2lQY+qZmTQeffdd5Gfn4+Wlhb86le/Ejs7BFQiIoT0A1QiIoSIjgIRIUR0FIgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiO7/A9uS0aQenSxWAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:47.912480Z", + "iopub.status.busy": "2024-03-22T19:39:47.911777Z", + "iopub.status.idle": "2024-03-22T19:39:48.135371Z", + "shell.execute_reply": "2024-03-22T19:39:48.134453Z" + }, + "papermill": { + "duration": 0.244727, + "end_time": "2024-03-22T19:39:48.137641", + "exception": false, + "start_time": "2024-03-22T19:39:47.892914", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAmJUlEQVR4nO3de1hU17038C/DZbiriFxEdLxUwSSA4gtizUlsEWxiapPW8GoiOPGYHpUekwknhl7gWNOQNIokOSitDTEmsdKm5vIkqZeipF4wKkaPN7AoiFFAEAWBOAwz6/3Dl52MM8BgBjZ75vt5Hh6ctdee+W33+HXvtdfscRFCCBARKYxK7gKIiO4Gw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJDe5CxhoJpMJV65cgZ+fH1xcXOQuh4juIITAzZs3MXLkSKhU3R9fOV14XblyBeHh4XKXQUS9uHTpEkaNGtXtcqcLLz8/PwC3/2L8/f1lrqb/GAwG7Nq1C0lJSXB3d5e7HLIDZ9mnLS0tCA8Pl/6tdsfpwqvrVNHf39/hw8vb2xv+/v4O/UZ3Js62T3sb1uGAPREpEsOLiBSJ4UVEisTwIiJFkj288vPzodFo4Onpifj4eBw+fLjH/jdu3MCKFSsQGhoKtVqNiRMn4rPPPhugaolosJD1amNRURF0Oh0KCgoQHx+PvLw8JCcno6KiAkFBQRb9Ozo6MHv2bAQFBeH9999HWFgYLl68iKFDhw588UQkK1nDKzc3F0uXLoVWqwUAFBQU4NNPP0VhYSFeeOEFi/6FhYVoamrCwYMHpUvFGo1mIEsetNrb21FeXi49bv1aj4Mnz2NY4FH4eqnN+kZERMDb23ugSySyK9nCq6OjA2VlZcjMzJTaVCoVEhMTUVpaanWdjz/+GAkJCVixYgU++ugjjBgxAgsXLsSqVavg6upqdR29Xg+9Xi89bmlpAXB7zozBYLDjFsnr1KlTiI+Pt2j/vZW+X3zxBaZMmdL/RZFddb1fHel9a42t2ydbeDU2NsJoNCI4ONisPTg42OwI4tsuXLiAPXv24IknnsBnn32GyspKLF++HAaDAdnZ2VbXycnJwerVqy3ad+3a5VBHH3q9HuvWrZMe17cDWypdkTrBiOA7NrO6uhq1tbUDXCHZy+7du+UuoV+1t7fb1E9RM+xNJhOCgoLwxz/+Ea6uroiNjcXly5fx6quvdhtemZmZ0Ol00uOujx4kJSU59Az7EzVNKNp0FPMXTEP06AC5yyE7MBgM2L17N2bPnu3QM+y7zo56I1t4BQYGwtXVFfX19Wbt9fX1CAkJsbpOaGgo3N3dzU4RIyMjUVdXh46ODnh4eFiso1aroVarLdrd3d0d+g3g5uYm/Xbk7XRGjv7etXXbZJsq4eHhgdjYWBQXF0ttJpMJxcXFSEhIsLrO97//fVRWVsJkMklt586dQ2hoqNXgIiLHJes8L51Oh02bNuHtt9/G2bNnsWzZMrS1tUlXH1NTU80G9JctW4ampiasXLkS586dw6effoqXXnoJK1askGsTiEgmso55paSkoKGhAVlZWairq0NMTAx27NghDeLX1NSY3YwsPDwcO3fuxLPPPouoqCiEhYVh5cqVWLVqlVybQEQykX3APj09Henp6VaXlZSUWLQlJCTg0KFD/VwVEQ12socXEVm6c9Ix0P3EY2eddMzwIhqEysvLERsba3XZnROPy8rKMHXq1P4vapBheBENQhERESgrKzNrq6i9Ad1fTyJ3/n2YFDrUrK8zYngRDULe3t4WR1Oqi9eg3vc1Iu+NRsyY4TJVNnjIfkscIqK7wfAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIg2K8MrPz4dGo4Gnpyfi4+Nx+PDhbvtu3rwZLi4uZj+enp4DWC0RDQayh1dRURF0Oh2ys7Nx7NgxREdHIzk5GVevXu12HX9/f9TW1ko/Fy9eHMCKiWgwkD28cnNzsXTpUmi1WkyePBkFBQXw9vZGYWFht+u4uLggJCRE+gkODh7AioloMHCT88U7OjpQVlaGzMxMqU2lUiExMRGlpaXdrtfa2ooxY8bAZDJh6tSpeOmll3DPPfdY7avX66HX66XHLS0tAACDwQCDwWCnLRl8Ojs7pd+OvJ3OxFn2qa3bJmt4NTY2wmg0Whw5BQcHo7y83Oo6kyZNQmFhIaKiotDc3Iy1a9dixowZOH36NEaNGmXRPycnB6tXr7Zo37VrF7y9ve2zIYPQpVYAcMOhQ4dw+ZTc1ZA9OMs+bW9vt6mfrOF1NxISEpCQkCA9njFjBiIjI/GHP/wBa9asseifmZkJnU4nPW5paUF4eDiSkpLg7+8/IDXL4URNE3DyKKZPn47o0QFyl0N24Cz7tOvsqDeyhldgYCBcXV1RX19v1l5fX4+QkBCbnsPd3R1TpkxBZWWl1eVqtRpqtdrqeu7u7n0vWiHc3Nyk3468nc7EWfaprdsm64C9h4cHYmNjUVxcLLWZTCYUFxebHV31xGg04uTJkwgNDe2vMoloEJL9tFGn0yEtLQ3Tpk1DXFwc8vLy0NbWBq1WCwBITU1FWFgYcnJyAAC//e1vMX36dEyYMAE3btzAq6++iosXL+Lf//3f5dwMIhpgsodXSkoKGhoakJWVhbq6OsTExGDHjh3SIH5NTQ1Uqm8OEK9fv46lS5eirq4Ow4YNQ2xsLA4ePIjJkyfLtQlEJAPZwwsA0tPTkZ6ebnVZSUmJ2eP169dj/fr1A1AVEQ1msk9SJSK6GwwvIlIkhhcRKRLDi4gUaVAM2BMRUNXYhjZ9Z7fLzze0Sb+7JqzeyUfthrGBPv1S32DD8CIaBKoa2zBrbYlNfZ97/2SPy/dmPOgUAcbwIhoEuo648lJiMCHI13qfr/X4pKQUcx9MgI+X5UfeKq+24pmi4z0evTkShhfRIDIhyBf3hg2xusxgMKBuBDB1zDCH/myjrThgT0SKxPAiIkVieBGRInHMS8F6urRuy2V1wLkurZNjYXgplK2X1nu7rA44z6V1ciwML4Xq7dJ6b5fVAee7tE6OheGlcN1dWudldXJ0HLAnIkVieBGRIjG8iEiRGF5EpEgMLyJSJF5tVCi98RZUnpdR1VIBlaflVInOzk5c6byCs01nu52kWtXSCpXnZeiNtwBY/zAw0WDF8FKoK20X4TP2DfzycM/9NuzY0ONyn7HAlbYYxCLYjtUR9T+Gl0KN9BmDtqpf4LWUGIy3Mkm1s7MTB/YfwPdnfr/bI6/zV1uxsug4Rs4a09/lEtkdw0uh1K6eMN0Kw1j/SZg83Pok1Sq3KkQGRHY7SdV0qxmmWw1Qu3r2d7lEdtfnAfsLFy70Rx1ERH3S5/CaMGECZs2ahXfffRe3bt3qj5qIiHrV5/A6duwYoqKioNPpEBISgp///Oc4fLiXUWMiIjvr85hXTEwMXnvtNaxbtw4ff/wxNm/ejJkzZ2LixIl46qmnsGjRIowYMaI/aiVyWL1NfQF6n/7ibFNf7nrA3s3NDY899hgefvhhbNiwAZmZmcjIyMAvf/lLPP7443jllVcQGhpqz1qJHJatU1+Anqe/ONPUl7sOr6NHj6KwsBDbtm2Dj48PMjIysGTJEnz11VdYvXo15s2bx9NJIhv1NvUF6H36i7NNfelzeOXm5uKtt95CRUUFHnroIWzZsgUPPfQQVKrbw2djx47F5s2bodFo7F0rkcPqbeoL0Pv0F2eb+tLnAfuNGzdi4cKFuHjxIj788EPMnTtXCq4uQUFBePPNN21+zvz8fGg0Gnh6eiI+Pt7mI7Zt27bBxcUFP/nJT/qyCUTkAPocXrt378aqVassxrOEEKipqQEAeHh4IC0tzabnKyoqgk6nQ3Z2No4dO4bo6GgkJyfj6tWrPa5XXV2NjIwM3H///X3dBCJyAH0Or/Hjx6OxsdGivampCWPHju1zAbm5uVi6dCm0Wi0mT56MgoICeHt7o7CwsNt1jEYjnnjiCaxevRrjxo3r82sSkfL1ObyEEFbbW1tb4enZt3Ptjo4OlJWVITEx8ZuCVCokJiaitLS02/V++9vfIigoCEuWLOnT6xGR47B5wF6n0wEAXFxckJWVBW9vb2mZ0WjEF198gZiYmD69eGNjI4xGI4KDzS/rBgcHo7y83Oo6+/fvx5tvvonjx4/b9Bp6vR56vV563NLSAuD24KfBYOhTvYNJZ2en9NvadnS19bSNvT0HDRxb9kVv+9RR9qettdscXl9++SWA20deJ0+ehIeHh7TMw8MD0dHRyMjI6GOZfXPz5k0sWrQImzZtQmBgoE3r5OTkYPXq1Rbtu3btMgtgpbnUCgBu2L9/Py5av7IO4PYY5Xd9Dup/fdkX3e1TR9mf7e3tNvWzObz27t0LANBqtXjttdfg7+9/d5V9S2BgIFxdXVFfX2/WXl9fj5CQEIv+58+fR3V1NR555BGpzWQyAbg9abaiogLjx483WyczM1M6agRuH3mFh4cjKSnJLtsgl9NXWrD25CHMnDkT94y03A6DwYDdu3dj9uzZ3d5VorfnoIFjy77obZ86yv7sOjvqTZ/neb311lt9LqY7Hh4eiI2NRXFxsTTdwWQyobi4GOnp6Rb9IyIicPKk+TdA//rXv8bNmzfx2muvITw83GIdtVoNtdryS1fd3d0V/X2GXZMU3dzcetyOnrbT1ueg/teXfdHdPnWU/Wlr7TaF12OPPYbNmzfD398fjz32WI99t2/fbtMLd9HpdEhLS8O0adMQFxeHvLw8tLW1QavVAgBSU1MRFhaGnJwceHp64t577zVbf+jQoQBg0U5Ejs2m8BoyZAhcXFykP9tTSkoKGhoakJWVhbq6OsTExGDHjh3SIH5NTY3FJFgiIpvC69univY8beySnp5u9TQRAEpKSnpcd/PmzXavh4gGPx7SEJEi2XTkNWXKFOm0sTfHjh37TgUREdnCpvDiB5+JaLCxKbyys7P7uw4ioj7hmBcRKZJNR14BAQE4d+4cAgMDMWzYsB7Hv5qamuxWHBFRd2wKr/Xr18PPz0/6s62D90RE/cWm8Pr2jQUXL17cX7UQEdmsz2Nerq6uVu9yeu3aNbi6utqlKCKi3tjtZoR6vd7sNjlERP3J5rtKvP766wBu34zwT3/6E3x9v7lhkNFoxD//+U9ERETYv0IiIitsDq/169cDuH3kVVBQYHaK6OHhAY1Gg4KCAvtXSERkhc3hVVVVBQCYNWsWtm/fjmHDhvVbUUREvenzzQi77qhKRCSnPofXU0891ePynr6yjIjIXvocXtevXzd7bDAYcOrUKdy4cQM/+MEP7FYYEVFP+hxeH3zwgUWbyWTCsmXLLL78goiov9jlg9kqlQo6nU66IklE1N/sdleJ8+fPS196SUTU3/p82vjt70AEbs/7qq2txaeffmr2GUgiov7U5/Dq+ubsLiqVCiNGjMC6det6vRJJRGQvnOdFRIrEO6kSkSIxvIhIkRheRKRIDC8iUiS7hddXX32Fp59+2l5PR0TUI7uF17Vr1/Dmm2/a6+mIiHrE00YiUiSGFxEpUp8nqdLg8LXBCAA4dbnZ6vK2r/U42gCEXLwOHy+11T6VV1v7rT6i/mZzeD322GM9Lr9x48Z3rYX64Pz/D54Xtp/soZcb3qk80utz+aj5fxgpj83v2iFDhvS6PDU19a6KyM/Px6uvvoq6ujpER0fjjTfeQFxcnNW+27dvx0svvYTKykoYDAZ873vfw3PPPYdFixbd1WsrVdI9IQCA8UG+8HK3/L7MitpmPPf+Saz72X2YFNr9vvNRu2FsoE+/1UnUX2wOr7feeqtfCigqKoJOp0NBQQHi4+ORl5eH5ORkVFRUICgoyKJ/QEAAfvWrXyEiIgIeHh745JNPoNVqERQUhOTk5H6pcTAK8PHA/40b3e3yrtsTjR/hg3vDev6Ph0iJZB+wz83NxdKlS6HVajF58mQUFBTA29u723vhP/jgg3j00UcRGRmJ8ePHY+XKlYiKisL+/fsHuHIikpPNR1623u6mL1/A0dHRgbKyMmRmZkptKpUKiYmJKC0t7XV9IQT27NmDiooKvPLKK1b76PV66PV66XFLSwuA2/feNxgMNteqNF1HXp2dnQ69nY7i5te336Mnapq6valn263bF2ECLzTAx9PyIkxlQxsA5e9zW2u3Obw2b96MMWPGYMqUKRBC3HVh39bY2Aij0Yjg4GCz9uDgYJSXl3e7XnNzM8LCwqDX6+Hq6ooNGzZg9uzZVvvm5ORg9erVFu27du2Ct7f3d9uAQexSKwC44dChQ7h8Su5qqDel9S4AXPGrj8700tMN71R+2WOPI6X7cdHLbqUNuPb2dpv62Rxey5Ytw5///GdUVVVBq9XiySefREBAwF0X+F34+fnh+PHjaG1tRXFxMXQ6HcaNG4cHH3zQom9mZqbZ3V9bWloQHh6OpKQk+Pv7D2DVA+tETRNw8iimT5+O6NHy7Cey3fS2Dtx39irGjfCxegEGAM7VNeP5D87i949GYmKI9XFMH7UrNMOVfQGm6+yoNzaHV35+PnJzc7F9+3YUFhYiMzMTDz/8MJYsWYKkpCS4uLj0ucjAwEC4urqivr7erL2+vh4hISHdrqdSqTBhwgQAQExMDM6ePYucnByr4aVWq6FWWx5iu7u7w93dvc81K4Wbm5v025G301EED3XHEwljbeo7MWQIYsYM7+eK5GPr+7VPA/ZqtRoLFizA7t27cebMGdxzzz1Yvnw5NBoNWlv7PuHRw8MDsbGxKC4ultpMJhOKi4uRkJBg8/OYTCazcS0icnx3PTtRpVLBxcUFQggYjca7LkCn0yEtLQ3Tpk1DXFwc8vLy0NbWBq1WCwBITU1FWFgYcnJyANwew5o2bRrGjx8PvV6Pzz77DO+88w42btx41zUQkfL0Kbz0er102rh//37MnTsX//M//4M5c+ZApbq7WRcpKSloaGhAVlYW6urqEBMTgx07dkiD+DU1NWbP3dbWhuXLl+Orr76Cl5cXIiIi8O677yIlJeWuXp+IlMnm8Fq+fDm2bduG8PBwPPXUU/jzn/+MwMBAuxSRnp6O9PR0q8tKSkrMHr/44ot48cUX7fK6RKRcNodXQUEBRo8ejXHjxuHzzz/H559/brXf9u3b7VYcEVF3bA6v1NTUu7qiSETUH/o0SZWIaLCQ/bONRER3g+FFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiDYrwys/Ph0ajgaenJ+Lj43H48OFu+27atAn3338/hg0bhmHDhiExMbHH/kTkmGQPr6KiIuh0OmRnZ+PYsWOIjo5GcnIyrl69arV/SUkJFixYgL1796K0tBTh4eFISkrC5cuXB7hyIpKT7OGVm5uLpUuXQqvVYvLkySgoKIC3tzcKCwut9n/vvfewfPlyxMTEICIiAn/6059gMplQXFw8wJUTkZzc5Hzxjo4OlJWVITMzU2pTqVRITExEaWmpTc/R3t4Og8GAgIAAq8v1ej30er30uKWlBQBgMBhgMBi+Q/WDW2dnp/TbkbfTmTjLPrV122QNr8bGRhiNRgQHB5u1BwcHo7y83KbnWLVqFUaOHInExESry3NycrB69WqL9l27dsHb27vvRSvEpVYAcMOhQ4dw+ZTc1ZA9OMs+bW9vt6mfrOH1Xb388svYtm0bSkpK4OnpabVPZmYmdDqd9LilpUUaJ/P39x+oUgfciZom4ORRTJ8+HdGjrR+VkrI4yz7tOjvqjazhFRgYCFdXV9TX15u119fXIyQkpMd1165di5dffhn/+Mc/EBUV1W0/tVoNtVpt0e7u7g53d/e7K1wB3NzcpN+OvJ3OxFn2qa3bJuuAvYeHB2JjY80G27sG3xMSErpd7/e//z3WrFmDHTt2YNq0aQNRKhENMrKfNup0OqSlpWHatGmIi4tDXl4e2traoNVqAQCpqakICwtDTk4OAOCVV15BVlYWtm7dCo1Gg7q6OgCAr68vfH19ZdsOIhpYsodXSkoKGhoakJWVhbq6OsTExGDHjh3SIH5NTQ1Uqm8OEDdu3IiOjg787Gc/M3ue7Oxs/Pd///dAlk5EMpI9vAAgPT0d6enpVpeVlJSYPa6uru7/ghSovb3d7AptRe0N6OsqcfaUF0zXhpr1jYiIcOgrreQcBkV40XdXXl6O2NhYi/aFb1v2LSsrw9SpUwegKqL+w/ByEBERESgrK5Met36tx6d7S/HwrAT4eqkt+hIpHcPLQXh7e5sdTRkMBlxvvIqEuGkOfVmdnJfsn20kIrobDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBRJ9vDKz8+HRqOBp6cn4uPjcfjw4W77nj59Gj/96U+h0Wjg4uKCvLy8gSuUiAYVNzlfvKioCDqdDgUFBYiPj0deXh6Sk5NRUVGBoKAgi/7t7e0YN24c5s+fj2effVaGiokGRnt7O8rLy83aKmpvQF9XibOnvGC6NlRqj4iIgLe39wBXOAgIGcXFxYkVK1ZIj41Goxg5cqTIycnpdd0xY8aI9evX9/k1m5ubBQDR3Nzc53WVpKOjQ3z44Yeio6ND7lLoLpSVlQkANv2UlZXJXa5d2fpvVLYjr46ODpSVlSEzM1NqU6lUSExMRGlpqd1eR6/XQ6/XS49bWloAAAaDAQaDwW6vM9h0bZsjb6MjGz9+PL744guzttav9di57wiS7/8/8PVSm/V1pP1s67bIFl6NjY0wGo0IDg42aw8ODrY4XP4ucnJysHr1aov2Xbt2OcWh9u7du+Uugexoxn3jcfNGE27e+KattrZWtnr6Q3t7u039ZB3zGgiZmZnQ6XTS45aWFoSHhyMpKQn+/v4yVta/DAYDdu/ejdmzZ8Pd3V3ucsgOnGWfdp0d9Ua28AoMDISrqyvq6+vN2uvr6xESEmK311Gr1VCr1Rbt7u7uDv0G6OIs2+lMHH2f2rptsk2V8PDwQGxsLIqLi6U2k8mE4uJiJCQkyFUWESmErKeNOp0OaWlpmDZtGuLi4pCXl4e2tjZotVoAQGpqKsLCwpCTkwPg9iD/mTNnpD9fvnwZx48fh6+vLyZMmCDbdhDRwJM1vFJSUtDQ0ICsrCzU1dUhJiYGO3bskAbxa2pqoFJ9c3B45coVTJkyRXq8du1arF27Fg888ABKSkoGunwikpHsA/bp6elIT0+3uuzOQNJoNBBCDEBVRDTYyf7xICKiuyH7kddA6zpys/VyrFIZDAa0t7ejpaXFoa9MORNn2add/zZ7O8tyuvC6efMmACA8PFzmSoioJzdv3sSQIUO6Xe4inGwQyWQy4cqVK/Dz84OLi4vc5fSbrsm4ly5dcujJuM7EWfapEAI3b97EyJEjzS7Y3cnpjrxUKhVGjRoldxkDxt/f36Hf6M7IGfZpT0dcXThgT0SKxPAiIkVieDkotVqN7Oxsq5/rJGXiPjXndAP2ROQYeORFRIrE8CIiRWJ4EZEiMbyISJEYXgrw4IMP4plnnpG7DLoLg23fDbZ6vguGF9Eg19HRIXcJg1M/fwUbfUdpaWkW39MXFhYmNmzYYNbv2LFjwsXFRVRXVwshhFi3bp249957hbe3txg1apRYtmyZuHnzptk6+/btEzNnzhSenp5i1KhR4he/+IVobW0dsG1zdNb2XWVlpXjqqaeERqMRnp6eYuLEiSIvL89ivXnz5okXX3xRhIaGCo1GI4QQ4sCBAyI6Olqo1WoRGxsrPvjgAwFAfPnll9K6J0+eFHPmzBE+Pj4iKChIPPnkk6KhoaHbeqqqqgbqr8PuGF6D3I0bN0RCQoJYunSpqK2tFbW1tSIjI0PMnDnTrN9zzz1n1rZ+/XqxZ88eUVVVJYqLi8WkSZPEsmXLpOWVlZXCx8dHrF+/Xpw7d04cOHBATJkyRSxevHjAts3RWdt3t27dEllZWeLIkSPiwoUL4t133xXe3t6iqKhIWi8tLU34+vqKRYsWiVOnTolTp06J5uZmERAQIJ588klx+vRp8dlnn4mJEyeahdf169fFiBEjRGZmpjh79qw4duyYmD17tpg1a1a39XR2dsrxV2MXDC8FeOCBB8TKlSulx19++aVwcXERFy9eFELc/qbxsLAwsXHjxm6f469//asYPny49HjJkiXi6aefNuuzb98+oVKpxNdff23fDXBid+47a1asWCF++tOfSo/T0tJEcHCw0Ov1UtvGjRvF8OHDzfbNpk2bzMJrzZo1Iikpyey5L126JACIiooKm+tRCo55KVBMTAwiIyOxdetWAMDnn3+Oq1evYv78+VKff/zjH/jhD3+IsLAw+Pn5YdGiRbh27Zr0hZ4nTpzA5s2b4evrK/0kJyfDZDKhqqpKlu1yFvn5+YiNjcWIESPg6+uLP/7xj6ipqTHrc99998HDw0N6XFFRgaioKHh6ekptcXFxZuucOHECe/fuNdunERERAIDz58/34xbJg+GlUE888YQUXlu3bsWcOXMwfPhwAEB1dTXmzp2LqKgo/O1vf0NZWRny8/MBfDP429raip///Oc4fvy49HPixAn861//wvjx4+XZKCewbds2ZGRkYMmSJdi1axeOHz8OrVZrMSjv4+PT5+dubW3FI488YrZPjx8/jn/961/4t3/7N3ttwqDhdPfzUiIPDw8YjUaztoULF+LXv/41ysrK8P7776OgoEBaVlZWBpPJhHXr1kk3c/vLX/5itv7UqVNx5swZfmVcP7tz3x04cAAzZszA8uXLpTZbjoomTZqEd999F3q9Xvpg9pEjR8z6TJ06FX/729+g0Wjg5mb9n7a195JS8chLATQaDb744gtUV1ejsbERJpMJGo0GM2bMwJIlS2A0GvHjH/9Y6j9hwgQYDAa88cYbuHDhAt555x2zcAOAVatW4eDBg0hPT5f+d/7oo4+6/SYnujt37rvvfe97OHr0KHbu3Ilz587hN7/5jUUIWbNw4UKYTCY8/fTTOHv2LHbu3Im1a9cCgHRH4BUrVqCpqQkLFizAkSNHcP78eezcuRNarVYKLGvvJcWSe9CNeldRUSGmT58uvLy8zC5vb9iwQQAQqampFuvk5uaK0NBQ4eXlJZKTk8WWLVsEAHH9+nWpz+HDh8Xs2bOFr6+v8PHxEVFRUeJ3v/vdAG2Vc7hz35WXl4vFixeLIUOGiKFDh4ply5aJF154QURHR0vrdE2VuNOBAwdEVFSU8PDwELGxsWLr1q3Sc3Y5d+6cePTRR8XQoUOFl5eXiIiIEM8884wwmUxW61HyVAneEodIod577z1otVo0NzfDy8tL7nIGHMe8iBRiy5YtGDduHMLCwnDixAmsWrUKjz/+uFMGF8DwIlKMuro6ZGVloa6uDqGhoZg/fz5+97vfyV2WbHjaSESKxKuNRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwon61ePFiuLi4wMXFBe7u7ggODsbs2bNRWFjYp4+mbN68GUOHDu2/QruxePFi/OQnPxnw16XeMbyo382ZMwe1tbWorq7G3//+d8yaNQsrV67E3Llz0dnZKXd5pFTyfjqJHF13n9MrLi4WAMSmTZuEED3ftnrv3r0Wty/Ozs4WQgixZcsWERsbK3x9fUVwcLBYsGCBqK+vl16nqalJLFy4UAQGBgpPT08xYcIEUVhYKC2vqakR8+fPF0OGDBHDhg0TP/7xj6XP+2VnZ1u87t69e/vl74n6jkdeJIsf/OAHiI6Oxvbt2wEAKpUKr7/+Ok6fPo23334be/bswfPPPw8AmDFjBvLy8uDv74/a2lrU1tYiIyMDAGAwGLBmzRqcOHECH374Iaqrq7F48WLpdX7zm9/gzJkz+Pvf/46zZ89i48aNCAwMlNZNTk6Gn58f9u3bhwMHDsDX1xdz5sxBR0cHMjIy8Pjjj0tHjrW1tZgxY8bA/kVR9+ROT3Js3R15CSFESkqKiIyMtLrszttWv/XWW2LIkCG9vt6RI0cEAOmo7ZFHHhFardZq33feeUdMmjRJuuOCEELo9Xrh5eUldu7c2Wv9JC8eeZFshBDSvah6u211d8rKyvDII49g9OjR8PPzwwMPPAAA0m2Vly1bhm3btiEmJgbPP/88Dh48KK174sQJVFZWws/PT7ptckBAAG7duuWQt012NAwvks3Zs2cxduxYm25bbU1bWxuSk5Ph7++P9957D0eOHMEHH3xgtt6PfvQjXLx4Ec8++yyuXLmCH/7wh9IpZ2trK2JjYy1um3zu3DksXLiwn7eeviveVYJksWfPHpw8eRLPPvusTbettnb74vLycly7dg0vv/wywsPDAQBHjx61eK0RI0YgLS0NaWlpuP/++/Ff//VfWLt2LaZOnYqioiIEBQXB39/fap2OdNtkR8MjL+p3er0edXV1uHz5Mo4dO4aXXnoJ8+bNw9y5c5GammrTbas1Gg1aW1tRXFyMxsZGtLe3Y/To0fDw8JDW+/jjj7FmzRqz9bKysvDRRx+hsrISp0+fxieffILIyEgAt7/EJDAwEPPmzcO+fftQVVWFkpIS/Od//ie++uor6XX/93//FxUVFWhsbITBYBiYvzTqndyDbuTYvv0tzW5ubmLEiBEiMTFRFBYWCqPRKPWz5bbV//Ef/yGGDx9uNlVi69atQqPRCLVaLRISEsTHH39s8V2GkZGRwsvLSwQEBIh58+aJCxcuSM9ZW1srUlNTRWBgoFCr1WLcuHFi6dKlorm5WQghxNWrV6VbZYNTJQYV3s+LiBSJp41EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFOn/AYwQzl3adqwTAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:39:48.175632Z", + "iopub.status.busy": "2024-03-22T19:39:48.175312Z", + "iopub.status.idle": "2024-03-22T19:39:48.479425Z", + "shell.execute_reply": "2024-03-22T19:39:48.478436Z" + }, + "papermill": { + "duration": 0.325609, + "end_time": "2024-03-22T19:39:48.481640", + "exception": false, + "start_time": "2024-03-22T19:39:48.156031", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABgSElEQVR4nO2deVyT9R/A3zsYlwgqCqIoeORRKiZKWmoHhmWlnWrllVlpdpGZmkempZkZZqZpmWaWdtevwzISyyItPPO+8QIB5RZ2Pb8/5sYGG2ywMTa+b197+ezZ93mezx62zz7f7+eSSZIkIRAIBF6I3N0CCAQCgasQCk4gEHgtQsEJBAKvRSg4gUDgtQgFJxAIvBah4AQCgdciFJxAIPBahIITCARei9LdAtRF9Ho9586dIygoCJlM5m5xBAJBOSRJoqCggIiICORy23aaUHBWOHfuHJGRke4WQyAQVMHp06dp2bKlzdeFgrNCUFAQYLh5DRs2dLM0AoGgPPn5+URGRpq+q7YQCs4Kxmlpw4YNhYITCOowVS0hCSeDQCDwWoSCEwgEXotQcAKBwGsRCk4gEHgtQsEJBAKvpU4ouKVLlxIVFYWfnx9xcXFs377d5tivvvqK2NhYQkJCCAwMJCYmhrVr11qMGT16NDKZzOIxcOBAV78NgUBQE0rywMkFxt0eJrJhwwYSExNZvnw5cXFxJCUlkZCQwKFDh2jWrFmF8Y0bN+all16iY8eOqFQqvv/+e8aMGUOzZs1ISEgwjRs4cCAffvih6bmvr2+tvB+BQFANtGr4+F4ICofB74Kfc8KzZO7uyRAXF0fPnj155513AEOaVGRkJE899RRTpkyx6xzXXnstgwYNYs6cOYDBgsvNzeWbb76plkz5+fkEBweTl5cn4uAEgtrgh0nwz0rwC4bHtkDj6EqH2/sddesUVa1Wk5aWRnx8vGmfXC4nPj6e1NTUKo+XJInk5GQOHTpEv379LF5LSUmhWbNmdOjQgfHjx5OTk2PzPKWlpeTn51s8BAJBLbF7vUG5AdzzfpXKzRHcOkXNzs5Gp9MRFhZmsT8sLIyDBw/aPC4vL48WLVpQWlqKQqHg3XffZcCAAabXBw4cyD333EN0dDTHjh1j2rRp3HbbbaSmpqJQKCqcb968ecyePdt5b0wgENhHxl7437OG7f4vwlW3OvX0bl+Dqw5BQUHs2rWLwsJCkpOTSUxMpE2bNtx4440ADBs2zDS2S5cudO3albZt25KSksItt9xS4XxTp04lMTHR9NyY5yYQCFzI5VzYMAK0l6FdvEHBORm3KrjQ0FAUCgWZmZkW+zMzMwkPD7d5nFwup127dgDExMRw4MAB5s2bZ1Jw5WnTpg2hoaEcPXrUqoLz9fUVTgiBoDbR6+HrJ+DSCQhpBfesBHnF2VVNcesanEqlokePHiQnJ5v26fV6kpOT6d27t93n0ev1lJaW2nz9zJkz5OTk0Lx58xrJKxAInMTWN+HwT6DwhQfWQkBjl1zG7VPUxMRERo0aRWxsLL169SIpKYmioiLGjBkDwMiRI2nRogXz5s0DDOtlsbGxtG3bltLSUn788UfWrl3LsmXLACgsLGT27Nnce++9hIeHc+zYMSZPnky7du0swkgEAoGbOJoMv71q2L5jEUTEuOxSbldwQ4cOJSsri5kzZ5KRkUFMTAwbN240OR7S09MtKnYWFRUxYcIEzpw5g7+/Px07duTjjz9m6NChACgUCvbs2cOaNWvIzc0lIiKCW2+9lTlz5ohpqEDgbi6dgi/HAhL0GA3dH3bp5dweB1cXEXFwAoEL0JTAqgQ4vwsiroVHNoKyekaHR8TBCQSCesRPLxiUm39jeOCjais3RxAKTiAQuJ4dHxkeyOC+DyCkdsKwhIITCASu5dxOQyoWwM3Toe3NtXZpoeAEAoHrKL4IG0aCrhQ63A43JFZ9jBMRCk4gELgGvQ6+fBTy0qFxGxiyDCrpYeoKhIITCASuIWU+HEsGpT8M/Rj8Q2pdBKHgBAKB8zm0EX5fYNi+620Iu9otYggFJ6iT6PUSO9MvUarVuVsUgaNcPA5fPWbY7vUYdH3AbaIIBVdLfL/nHEPfS+VCfom7RfEIlm05xt3v/sVzG3a5WxSBI6iLDU6F0jxo2QtufdWt4ggFV0tM/GQn205cZO4PB9wtikfwwdYTAPy4N8PNkgjsRpLg++cgcy8ENoUH1oBS5VaRhIKrZfIua9wtgkfgqxQfTY/j3w9gz3qQKeC+D6FhhLslEgpOUDfx83F+bTCBCzn9D/x0pYfKgNkQ3de98lxBKLhaRiZztwR1k5nf/seMb/4zPRcWnAdRmAWfjQS9BjoPht4T3S2RCfEpEridvGINH6WeYu3fp8gpNBQuFQrOQ9Bp4YsxUHAOQq+CwUvr1K+4+BQJ3I7OrGKXTm/Y9lWKKapH8NscOPkHqBoYgnl9g9wtkQVCwQncjtzsB9+o6nx9xEezzrP/O/gzybA9+B1o2sGt4lhDfIoEbkdGmYbTS8KC8wiyj8A3EwzbvSfC1Xe7Vx4bCAUncD9WlmyEBVeHKS2EDQ+DugBa3wDxdbensPgUCdyO+Zq0cTnOz8yCE1X16xCSBN89BVkHIag53P8hKNze2sUmdULBLV26lKioKPz8/IiLi2P79u02x3711VfExsYSEhJCYGAgMTExrF271mKMJEnMnDmT5s2b4+/vT3x8PEeOHHH127CLuuNfqjuY3xPTFNXMgivV6mtZIoFN/l4G+74CuRLuXwMNmrlbokpxu4LbsGEDiYmJzJo1ix07dtCtWzcSEhK4cOGC1fGNGzfmpZdeIjU1lT179jBmzBjGjBnDzz//bBqzYMEC3n77bZYvX862bdsIDAwkISGBkhKRB1oXkZmZcEZjTaUo+2heVouE+zrBqb/gl+mG7YTXoFWce+WxA7cruEWLFjFu3DjGjBlD586dWb58OQEBAaxatcrq+BtvvJG7776bTp060bZtW5555hm6du3K1q1bAYP1lpSUxPTp0xk8eDBdu3blo48+4ty5c3zzzTe1+M4ENUFupvQua4SCczsFGfD5aJB00OV+Q5UQD8CtCk6tVpOWlkZ8fLxpn1wuJz4+ntTU1CqPlySJ5ORkDh06RL9+/QA4ceIEGRkZFucMDg4mLi7O5jlLS0vJz8+3eLgKWR0KgqyLGC04ibJ1N6Hg3IxOY1BuhZnQrDPcubhOBfNWhlsVXHZ2NjqdztTk2UhYWBgZGbarSOTl5dGgQQNUKhWDBg1iyZIlDBgwAMB0nCPnnDdvHsHBwaZHZGTtdPwRVMS4BmfuVxBTVDezaSakp4JvQ0MwryrQ3RLZjdunqNUhKCiIXbt28c8///Dqq6+SmJhISkpKtc83depU8vLyTI/Tp087T1gbfPbvaVZdKQkkKMOo1/RmGq5EWHDuY+8X8Pe7hu27l0OTtu6Vx0Hc6t8NDQ1FoVCQmZlpsT8zM5Pw8HCbx8nlctq1awdATEwMBw4cYN68edx4442m4zIzM2nevLnFOWNiYqyez9fXF19f1zehNSJJEpO/2APAbV3CaR7sX2vXrouYh4EYt80VnJiiuokLBwwhIWDohtVxkHvlqQZuteBUKhU9evQgOTnZtE+v15OcnEzv3r3tPo9er6e01JCkHR0dTXh4uMU58/Pz2bZtm0PnrC2KSrXuFqFOYVRrYorqZkryDMG8mmJoc6Ohn6kH4vYIvcTEREaNGkVsbCy9evUiKSmJoqIixowZA8DIkSNp0aIF8+bNAwzrZbGxsbRt25bS0lJ+/PFH1q5dy7JlywDDIv6zzz7L3Llzad++PdHR0cyYMYOIiAiGDBnirrdpExHDaonxfujNFZyw4GoXSTKkYeUchYYt4d4PQO6ZqXNuV3BDhw4lKyuLmTNnkpGRQUxMDBs3bjQ5CdLT05Gb9VIsKipiwoQJnDlzBn9/fzp27MjHH3/M0KFDTWMmT55MUVERjz32GLm5udxwww1s3LgRPz+/Wn9/5ZEhlFplSCYng1iDcxt/LoaD34NCBQ98BIGh7pao2rhdwQFMnDiRiROtF8kr7zyYO3cuc+fOrfR8MpmMV155hVdeecVZIgpciGRlW0xR3cTxLZB8Jbf0tgXQsod75akhHulFFXgvZVNUcyeDSNWqFfLOwBePgKSHmIehx2h3S1RjhIJzA2KGahtjgK9Yg6tltKXw2SgozobwrjBooccE81aGUHC1jBd8ZlyK/oqxJtbgapmNU+Hsv+AXAkPXgo93hC4JBedmhDVnud5mtODM74tYg3Mxuz41tPxDBve+D42i3C2R0xAKTlCnsL4GJxScyzi/B75/1rB94xRoP8Ct4jgboeDcgCjgWDViDa4WuHwJPhsB2hJofyv0m+xuiZyOUHCCOoWpmoj5GpyYojofvR6+ehwunYSQ1nD3eyD3PnXgfe+oziNj+8mLpmfCmMNiwc20Bme2T6MXN8np/LEQjvwMSj+DUyGgsbslcglCwdUyx7MKeXDlNneLUWextgan04s4OKdy5FfY/Jphe9AiaN7NvfK4EKHgapnj2UXuFqFOo7dSTUSjExac07h0Er4cC0jQYwx0f8jdErkUoeDcjCQCRSywlqqlE1NU56Apgc9GQkkuRFwLt73ubolcjlBwArdjruStVRPR6sQU1Sn8OAnO74aAJoYkemXt1UB0F0LBCeoYFauJaIUFV3PS1sDOtSCTw32rIKR+lOUXCk5Qp9BbcTJoxRpczTibZrDeAG6eYShgWU8QCs7NiDARS8q6apWhFV7U6lOUY0ii16mh4x1ww3PulqhWEQpO4HYsclFNXtSyfWKKWk30OoPHNO80NG4LQ96td9UehIIT1CnKvKhiilpjNr8GxzeDT4AhmNcv2N0S1To1quhbWFiIvtz0oWHDhjUSSFC/KUvVKtsnpqjV4OCPhmwFgLuWQNjV7pXHTThswZ04cYJBgwYRGBhIcHAwjRo1olGjRoSEhNCoUaNqCbF06VKioqLw8/MjLi6O7du32xy7cuVK+vbta7pufHx8hfGjR49GJpNZPAYOHFgt2QS1i7W2gcKCc5CcY/D1E4btuCegy33ulceNOGzBPfzww0iSxKpVqwgLC0NWwzn9hg0bSExMZPny5cTFxZGUlERCQgKHDh2iWbNmFcanpKQwfPhw+vTpg5+fH6+//jq33nor+/bto0WLFqZxAwcO5MMPPzQ9r82+pwLHsNaTQS/CRKqHuhg2jIDSPIi8DgbMcbdEbsVhBbd7927S0tLo0KGDUwRYtGgR48aNM7UJXL58OT/88AOrVq1iypQpFcavW7fO4vn777/Pl19+SXJyMiNHjjTt9/X1rbR5dF1BeFEtEYG+NUCSDLXdLuyDwGZw/2pQqtwtlVtxeIras2dPTp8+7ZSLq9Vq0tLSiI+PLxNILic+Pp7U1FS7zlFcXIxGo6FxY8tqCCkpKTRr1owOHTowfvx4cnJynCKzwLWYshqEF9Vx/nkf9mwAmcKg3Bo2d7dEbsdhC+7999/niSee4OzZs1xzzTX4+PhYvN61a1e7z5WdnY1OpzP1QDUSFhbGwYMH7TrHiy++SEREhIWSHDhwIPfccw/R0dEcO3aMadOmcdttt5GamopCUbGBbWlpKaWlpabn+fn5dr8HgXOxGugrFFzVpG+DjVdmPANegajr3StPHcFhBZeVlcWxY8dMU0ow9CGVJAmZTIZOV3vFCefPn8/69etJSUmxaOo8bNgw03aXLl3o2rUrbdu2JSUlhVtuuaXCeebNm8fs2bNrRWZBRaxVOLYslySZPl8CKxRegM9HgV4LnYdA7yfdLVGdweEp6iOPPEL37t1JTU3l+PHjnDhxwuJ/RwgNDUWhUJCZmWmxPzMzs8r1s4ULFzJ//nx++eWXKq3GNm3aEBoaytGjR62+PnXqVPLy8kwPZ03BHUGr0/NF2hlOXyyu9WvXJUyd7cvtF1acDXRaQy/TgvMQ2gEGv1Pvgnkrw2EL7tSpU3z33Xe0a9euxhdXqVT06NGD5ORkhgwZAoBeryc5Odlmp3uABQsW8Oqrr/Lzzz8TGxtb5XXOnDlDTk4OzZtbX5Pw9fV1u5d19V8nmfvDAQBOzh/kVlnciTUnAxisOJ+KqwuC5Nlw8g9QNYChH4NvkLslqlM4bMHdfPPN7N6922kCJCYmsnLlStasWcOBAwcYP348RUVFpinwyJEjmTp1qmn866+/zowZM1i1ahVRUVFkZGSQkZFBYWEhYAg+fuGFF/j77785efIkycnJDB48mHbt2pGQkOA0uZ2FcVH97+PCCQLmJcstNZxGeFIrsv9b+Ottw/aQd6HpVe6Vpw7isAV355138txzz7F37166dOlSwclw1113OXS+oUOHkpWVxcyZM8nIyCAmJoaNGzeaHA/p6enIzZphLFu2DLVazX33WQYvzpo1i5dffhmFQsGePXtYs2YNubm5REREcOuttzJnzhy3W2mVU3+nFRZxcFacDCCCfSuQdRi+mWDY7vM0dB7sXnnqKA4ruCeeMERIv/LKKxVeq66TYeLEiTanpCkpKRbPT548Wem5/P39+fnnnx2Wwd2IZRMD1lK1QKzBWVBaABseBnUhRPWFW2a5W6I6i8MKrnzuqcA51Df9ptNLKOQV37XeSjUREPmoJiQJvp0I2YcgqLmheKWiRinlXo1Da3AajQalUsl///3nKnnqHUZLpT5ZcG9tOkznmRs5nFlQ4TVr1URATFFNpC6F/d+A3MdQdrxBxXRGQRkOKTgfHx9atWpVq7Fu9QV5PdJwi5OPUKrVM+9Hg9fYej24cgpOTFHh5J+waaZhe+A8iOzlXnk8AIe9qC+99BLTpk3j4sWLVQ8W2E090m+VYmsNrt73Rs0/D5+PBkkHXYdCz0fdLZFH4PDk/Z133uHo0aNERETQunVrAgMDLV7fsWOH04QT1D+sVROBet4bVas2ZCoUXYCwa+COJPGLaCcOKzhjQK7AuYg0JANGxVbRgqvHCm7TDDi9DXyDDetuqgB3S+QxOKzgZs0SLmlX4M3q7XzeZd785TCj+0RxTYuKZbOt9UUtr87qbaDvns9h23LD9t3LoUlb98rjYVTbv5yWlsaBA4ZF4quvvpru3bs7Taj6iDdbcM98uovtJy/yRdoZizQ0a+9ZL5wMZWTuh/89bdjuOwk63u5eeTwQhxXchQsXGDZsGCkpKYSEhACQm5vLTTfdxPr162natKmzZayzSJLE2dzLALRsVLNpg/eqNzh8oWI4SFXU+0yGkjxDMK+mGNrcBDdNc7dEHonDXtSnnnqKgoIC9u3bx8WLF7l48SL//fcf+fn5PP30066Qsc7yxs+HuOH1zaz83bEqKtbwYgPOIUwWXLkZab0K9NXr4evxcPEYBEfCvR+AXFQaqA4OW3AbN27k119/pVOnTqZ9nTt3ZunSpdx6661OFa6u0yHcULnhv3M1L5DpzfrN1nuTrCy42SrhXq+mqH8mwaEfQKEyOBUCm7hbIo/FYQtOr9dXSLAHQxBwfUvjujrCsGC+/1x+tb18ZZkM3qviHLkz5Sv6+igM96XeTFGPbYbfrjSKuX0htLjWvfJ4ONUql/TMM89w7tw5076zZ8/y3HPPWa2W681EhwYSoFJwWaPjeFahxWv/nLzIgyv/5lCGfetP3qvebGNNqZfPZPBRGD6i9SLQN/e0oRO9pIfuD0OPUe6WyONxWMG988475OfnExUVRdu2bWnbti3R0dHk5+ezZMkSV8hYZ1HIZXRubmh0/d+5PIvX7l+eyl/Hchj9oe0erxZ4sYZz5K2V5aIa/jcqOK8P9NWWwmcjoTgHmnczWG+CGuPwGlxkZCQ7duzg119/NTWG6dSpk0XTl/rENS2C+ffUJf47m8/dViJlzueV2HWe+pSLWh7LenCW1UTKLDgvV3A/vQjndoB/I3hgLfj4u1sir6BacXAymYwBAwYwYMAAZ8vjcRgDV/87m1fFyMqpv+rNkrJcVMOGr9JowXnxFHXnOkj7EJDBPe9Do9bulshrqJaCS05OJjk5mQsXLlRwLKxatcopgnkK17QwTFH3n8tHr5eQW6lxZg/12ICzwJaTwWstuPO74YdEw/ZN06B9/ZwJuQqH1+Bmz57NrbfeSnJyMtnZ2Vy6dMniUd9o17QBvko5BaVaTlWjI5YxTUlWD204a+/Y1JPhynPTGpw3Krjii7BhBGhLoH2CIVtB4FQctuCWL1/O6tWrGTFihCvk8TiUCjkdmzdk9+lc/jubR3RoYNUHWcGbLbiqQmDMY99MFpze0ouq9bYpql4PXz0GuaegURTc8x7IHbY3BFXg8B1Vq9X06dPHqUIsXbqUqKgo/Pz8iIuLY/t2257HlStX0rdvXxo1akSjRo2Ij4+vMF6SJGbOnEnz5s3x9/cnPj6eI0eOOFVmc7q0qOhJdVRhebOCs9bYGWzEx5WrJqJSeqmT4fcFcHQTKP0MTgX/Ru6WyCtxWME9+uijfPLJJ04TYMOGDSQmJjJr1ix27NhBt27dSEhI4MKFC1bHp6SkMHz4cDZv3kxqaiqRkZHceuutnD171jRmwYIFvP322yxfvpxt27YRGBhIQkICJSX2eTQd5ZorAb/7zpZlNBgXx6ui7LvvxRrOAYx6zHhbVN4YJnL4F0iZb9i+IwmaV964XFB9HJ6ilpSUsGLFCn799Ve6du1aIath0aJFDp1v0aJFjBs3ztQHdfny5fzwww+sWrWKKVOmVBi/bt06i+fvv/8+X375JcnJyYwcORJJkkhKSmL69OkMHmxopfbRRx8RFhbGN998w7BhwxySzx6MntS9Z/OQJAmZTIZKIadEY/+0qpq+CY/AkSyNCoG+SqOTwUumqJdOwlfjAAlix0LMcHdL5HSM34G6gMMKbs+ePcTExABUaD7j6JtSq9WkpaVZNHaWy+XEx8eTmppq1zmKi4vRaDQ0btwYgBMnTpCRkWERlxccHExcXBypqalWFVxpaSmlpaWm5/n5juWWtg9rgI9CRt5lDWcuXSaycQC+Pgoo0dp9jjryeahVjG/Zoh7clf+NCs6rLDjNZYNToSQXWsQa+ip4Gek5xdy3/C+ua9OE+fd2IUDl3o5fDl998+bNTrt4dnY2Op3O1OTZSFhYmCmIuCpefPFFIiIiTAotIyPDdI7y5zS+Vp558+Yxe/ZsR8U34atUcFVYEPvO5bPvXJ5Bwdk5Ra0POKK7y8JEDP97TaCvJMEPz0PGHggINSTRK+tyI/Lq8eWOM1woKOW73ec4llXIypGxRIS4L2jZo7+F8+fPZ/369Xz99df4+flV+zxTp04lLy/P9Dh9+rTD5+hiCvg1WH92r8Fd+d+bMxkcUU3lK4z4GAN9PX2KmrYadq0DmdzQyzS4hbslcgm/HsgEDGmM+87lM3jpn+xMd1/4mFsVXGhoKAqFgszMTIv9mZmZhIeHV3rswoULmT9/Pr/88gtdu5Yt0hqPc+Scvr6+NGzY0OLhKFebrcOBwapzBO9Vb44hmSw4yymqzpOnqGfS4KfJhu1bZkKb/u6Vx0Wcz7vMvnP5yGTw1fg+dAgLIquglKEr/ubbXWerPoELcKuCU6lU9OjRg+TkZNM+vV5PcnIyvXv3tnncggULmDNnDhs3biQ2NtbitejoaMLDwy3OmZ+fz7Zt2yo9Z025JuJKqMgVR4OvT9mt1dsxvaori7KuoKp3ZtEXFUsng1HBeWw9uKJsQxK9Tg0d74Drn3W3RC4j+YAh8uHaVo3oFhnClxP6cEvHZqi1ep5Zv4s3fzlk13fBmbh9ipqYmMjKlStZs2YNBw4cYPz48RQVFZm8qiNHjrRwQrz++uvMmDGDVatWERUVRUZGBhkZGRQWGsoVyWQynn32WebOnct3333H3r17GTlyJBERES7tCNapeUMUchk5RWoy80tNX0yAUq2HT69chDWdXmEN7ooX1SMr+up1hvJH+WegSTsY8q5Xe5OM09P4Tob17wa+SlaMjOXxfm0AWPLbUZ78ZAfFavudbzXFYQX3+++/o9VWFFCr1fL77787LMDQoUNZuHAhM2fOJCYmhl27drFx40aTkyA9PZ3z58+bxi9btgy1Ws19991H8+bNTY+FC8vKy0yePJmnnnqKxx57jJ49e1JYWMjGjRtrtE5XFX4+Cto1bQAYrDiV2RrcZY2uyuO9eQ3OFtbifyXJMjC4LJPBAy24za/C8RTwCYChH4NfxY5i3kJRqZa/juUAEN+pmWm/Qi5j6u2deOO+rvgoZPz0Xwb3L0/lfN7lWpHLYS/qTTfdxPnz52nWrJnF/ry8PG666SZ0uqq/zOWZOHEiEydOtPpaSkqKxfOTJ09WeT6ZTMYrr7zCK6+84rAsNeGaFsEcyizgv3N5FlNOexRcPdRvVtFLkoXi89gwkYM/wB9vGrbvWgLNOlU+3sP540g2aq2e1k0CaNesQYXX74+NJCo0kMfXprHvXD53vfMnK0fGEhMZ4lK5HLbgbAXx5eTkVOhyX98wVhb572y+hRVyWW1bwdlKY6pPlL8D5h21ylK1PGiKmnMMvn7CsH3dBOhyn3vlqQWSr0xPb+kYZnM9uWdUY7598voy58N7qXy3+5zVsc7CbgvunnvuAQzW0ejRo/H1LYvh0el07Nmzx+k5qp5G82DDFDi3WI2fT5kXtcSuKarLxHI7jlinkiRZKDyPqyaiLjK0+yvNh1a9YUDtziLcgU4v8dtBg4MhvnOzSsdGNg7gywl9eObTnSQfvMDTn+7kaGYBz8ZfVe1SY5Vht4ILDjasH0iSRFBQEP7+ZcF7KpWK6667jnHjxjldQE/C+MullyQLK8QeBefNXlRbRqotJ4PeyhqcR4SJSBL87xm4sB8ahMH9q0FRsUGTt7HrdC45RWqC/JT0jGpc5Xij82HBxoO89/tx3v7tKEezCnnz/hj8Vc5tj2i3gvvwww8BiIqKYtKkSfV+OmoNuUnBWX5J7VqDc5lUnoXByVD23DhF9Qgv6vYVsPdzkCkMyi2o8lhOb8HoPb2pQzPTD1JVGJ0P7Zo1YNrXe/lxbwanL6aycmQs4cHOcwY6vAY3a9YsodxsYLSwJcrCHKCKNTjjhhdruKqMU/N1yPLWr8rYNrCuT1HT/4afr3Sfv3UutK4/yzWm9bdOlU9PrXF/bCSfjLuOxoEq9p7N4653tnKsXIe6muCwgsvMzGTEiBFERESgVCpRKBQWj/qM0YKTJMnSyWCXBeedGu5ikdqh8RKWFpxHhIkUZMLno0GvhavvgevGu1uiWiM9p5jDmYUo5TJuvMpxBQeWzofmIf60cGLuqsNhIqNHjyY9PZ0ZM2bQvHlzr147chTjrTBYIWX7K1uDK2v87ELB3MTHf59i+jf/VT3QDKmcBWdScHV1iqrTwBdjoOA8NO1oCAnxxj+mDYzT055RjQkOqP56Y2TjAL4Y35sSjd7CQVdTHFZwW7du5Y8//jCVTBKUYVqD05dbg6tkimrEG78Sjio3MCh88x8HY7J9nbXgfn0ZTv0JqiBDMK9vxRgwbyb5YPWnp+UJ8vMhyMmx+A5PUSMjI0Xslg3kFl7Usv0llaZqXWk6440azgoFJZoK+yx7MkgWgXHGNbg6GSay72tIfcewPeRdCG3vXnlqmfwSDduOXwRgQOewKka7B4cVXFJSElOmTLEro6C+YXIylEs3mv9T1bXtvHUNrjxdXv6l0tcNDhoPCPTNOgTfPGnYvv4Z6HyXe+VxA1sOZaHVS7Rr1oDWTeqm49HhKerQoUMpLi6mbdu2BAQEVChZfvHiRacJ52nYioOrDOMwbw70dQTJRhxcnZqilhYYgnk1RRDVF26e6W6J3EL55Pq6iMMKLikpyQVieAdycyeDowZHfZmjVoFUbnqvlNexckmSBN8+CdmHISgC7vsQFO4ty+0ONDo9m43ZC05Yf3MVDv9lRo0a5Qo5vAJjqkl5K6QyjKPqp3qr+K4lymrCyWVlne3rTF/U1Hdg/7cg9zGUHW/Q1N0SuYV/T14iv0RL40AV3VvV3ZaH1aoHd+zYMaZPn87w4cNN7f1++ukn9u3b51ThPA1zC85RP4ww4Azo9ZLZtF2GQl6HAn1P/AGbZhm2b5sPkT3dK48bSTbLXlDU4fUVhxXcli1b6NKlC9u2beOrr74yFZrcvXs3s2bNcrqAnoTMRqpWZZji4OqpDVcecyeDXCarO2tw+ecM8W6SDroOM7T8q6dIkmRafxtQRXK9u3FYwU2ZMoW5c+eyadMmVCqVaf/NN9/M33//7VThPA15NZwMRswtuPochmMRYiOjblhwWjV8NgqKsiDsGrjjrXptch/LKuJkTjEqhZy+7ev2FN1hBbd3717uvvvuCvubNWtGdna2U4TyVCzDROw7xqjMzL8udWE2VptY9GQwC7GxWINzZ5jILy/Bme3gGwxD14IqwH2y1AGM1tt1bZsQ6Fu3HSwOK7iQkBCLEuJGdu7cSYsW3tkKzV6EBVdzJKn8GpybyyXt+cxQJQTgnhXQuI175KhDGNffBtRh76kRhxXcsGHDePHFF8nIyEAmk6HX6/nzzz+ZNGkSI0eOdIWMHoOtXNTKMM3GzDRcfbPgzDFfg5MBSrkxk8ENFlzGf/Dd04btfpOhw8Dal6GOcbFITdopQ5/Tm+tw/JsRhxXca6+9RseOHYmMjKSwsJDOnTvTr18/+vTpw/Tp0x0WYOnSpURFReHn50dcXBzbt2+3OXbfvn3ce++9REVFIZPJrMbkvfzyy8hkMotHx44dHZarOhgdBY44GawhOdQq2bswn97LZTKUpjCRWr4nl3PhsxGgvQxtb4Ebp9Tu9esomw9eQC9B5+YNnVr1w1U4rOBUKhUrV67k2LFjfP/993z88cccPHiQtWvXOlwuacOGDSQmJjJr1ix27NhBt27dSEhIMIWelKe4uJg2bdowf/78ShtDX3311Zw/f9702Lp1q0NyVZcrsymLaVZVmH+Zy++rL5grdPPpvUxmGehba1N3vd7QU+HicQhuBfe+D/L6XQrMiDG5vi4H95pT7RXCVq1a0apVqxpdfNGiRYwbN87UA3X58uX88MMPrFq1iilTKv5i9uzZk549DbFH1l43olQqK1WArsJWRV97sFyDc6ZUtc/BjHze2nS4WseaFwuVy2WmKSoYav8bLTqXsnURHP4JFL7wwBoIqLoMd32gVKtjy6EsAOLraHJ9eexScImJicyZM4fAwEASExMrHbto0SK7LqxWq0lLS7No6iyXy4mPjyc1NdWuc9jiyJEjRERE4OfnR+/evZk3b16lyri0tJTS0lLT8/z8/Gpd1yJVy+5MhopeVE+fog5b8Te5xRWrhpTHWqSFebFQGVgoNK1eQulqQ+poMvw217A9aCG0uNbFF/Qcth2/SJFaR7MgX66J8Iwer3YpuJ07d6LRaEzbtnCk+GV2djY6nc7U4NlIWFgYBw9WXX3DFnFxcaxevZoOHTpw/vx5Zs+eTd++ffnvv/8ICgqyesy8efOYPXt2ta9pxPj+y9c0s+/Ysm1PdzLYo9xsYV4tSS6TmaaoUAuxcLnp8OWjgATXjjQ8BCZ+NZUmD3NJByxXYJeC27x5s9Xtushtt91m2u7atStxcXG0bt2azz77jLFjrUefT5061cIyzc/PJzIy0uFrm4eJ2L1eZCWToT6FiZinZkH5NTiZhQXn0lARTQl8NhIuX4TmMXDbG667lgciSRLJB+p+cn15qpWL6gxCQ0NRKBRkZmZa7M/MzHTq+llISAhXXXUVR48etTnG19eXhg0bWjyqg3mgb00suPqi3jbtz+TauZs4m3vZtE+SMFViMTgZym6MS0NFNr4I53aCfyNDMK+Pk0vLejgHzhdwNvcyfj5yrm8X6m5x7MYuC87Y9NkevvrqK7vGqVQqevToQXJyMkOGDAFAr9eTnJzMxIkT7b5eVRQWFnLs2DFGjBjhtHPaojqBvtZGSXWkcIY9nMop4qsdZxlzfRQhAaqqDyhHbrGGxb8eMT3XS5bVRGRXEu51egmdq6aoO9ZC2mpABvd+ACE1c555I8bg3hvaNXVqzwRXY5eCMzZ9BoOp+vXXXxMcHExsbCwAaWlp5ObmOqQIweC8GDVqFLGxsfTq1YukpCSKiopMXtWRI0fSokUL5s2bBxgcE/v37zdtnz17ll27dtGgQQPatWsHwKRJk7jzzjtp3bo1586dY9asWSgUCoYPH+6QbNXBItDXzi+j1TARD7Lh7liylYISLQcz8nlvRGy1zmHedUxCqnBPlFcUnMYVJZPO7YIfnjds3/QStLvF+dfwAsqKW3rO9BTsVHDGps8AL774Ig888ADLly83xb3pdDomTJjg8NRu6NChZGVlMXPmTDIyMoiJiWHjxo0mx0N6ejpys0Xmc+fO0b17d9PzhQsXsnDhQvr3709KSgoAZ86cYfjw4eTk5NC0aVNuuOEG/v77b5o2dX1SsHmYSE3KJXmSk6GgRAvAPycvVfsce8/mlT2RLKuJgEHBleKCYN/ii4ZgXl0pXDUQ+j7v3PN7CRfyS9h9xvA3utkbFZw5q1atYuvWrRZBvQqFgsTERPr06cMbbzi2ODtx4kSbU1Kj0jISFRVV5QL8+vXrHbq+M7Hoi2rnMdZGeqKT4WKRmvwSDQ39qt86DqynuSkVckDnXC+qXmfwmOamQ6NouPu9skhtgQXJVyr3dosMoZmz2165GIf/olqt1moYx8GDB9HXtcYgtUxZHFw1An3Ntj3JgjNn6pd7gZpVEpIwqyZy5dOpNJVMcuLna8vrcCwZlP4Gp4J/iPPO7WV4UnJ9eRy24MaMGcPYsWM5duwYvXr1AmDbtm3Mnz/ftHZWX6lJ0xnz0Z60BmfOb1d+6eUyGbpqWqF6Mw+0aYrq7HzUwz8bFBzAnYshvItzzuuFXFbr2HrUUAbtFg9Iri+Pwwpu4cKFhIeH8+abb5rKJjVv3pwXXniB55+v32sYFmEiNTE2PFO/mVDIZOiq+SbKZzKAkxvPXDwBX40zbPccB92G1vycXsyfR7Mp0ehpEeJPx3DrgfJ1GYenqHK5nMmTJ3P27Flyc3PJzc3l7NmzTJ482eFke2/D3BOqMwWrVn6Mta/s4x+nUWLmWfQkCku1qGvg7ZQqseBq3BtVXQwbRkBJHrTsCQmv1ex89QDz5HpHMpXqCjVaVa1JUKw3YqHgrnxLlXamtJjP6Ham57I29ZRTZastPv67ZnIbwkQsfxyMZcs1NZmiShL8kAiZeyEgFO5fA0rH4/bqE3q9xK/G7AUPSa4vT7WqiXzxxRd89tlnpKeno1arLV7bsWOHUwTzRGRWfi4UclmlX0xbHtNLxWqr++syMhnM/6n6ecRgmNqXt+B8jFV9azJF/XcV7P7U8Ee6/0MIrt/Vp+1h79k8sgpKaeCrJC66ibvFqRYOW3Bvv/02Y8aMISwsjJ07d9KrVy+aNGnC8ePHLfJA6yNyKya8spqhB57qSa0p1iw44xS12oG+Z/6Fn140bMe/DNH9aihl/cAY3NvvqlBUSs8MoXFY6nfffZcVK1awZMkSVCoVkydPZtOmTTz99NPk5eVVfQIvxtpstKqekbb0mKd6UmuKvlw1ESib5lfLgivKNiTR6zXQ6U7o87STJPV+TNNTD/SeGnFYwaWnp9OnTx8A/P39KSgoAGDEiBF8+umnzpXOw7BmwfnYWaCx/FfXA2N9nYJkFkMoMzkZDB9Th9fgdFpDL9P8s9CkPQx+t163+3OEM5eKOXA+H7nM0NzZU3FYwYWHh3Px4kXAUNXX2Av1xIkTHhmB70ysfXeq7Ppt45ZVlstaotHx097z5JdUv+5a3UUyW4Mz/K+obqDv5rlw4nfwCYShH4OfcIjZizGmMbZ1YxoFeq4zxmEFd/PNN/Pdd98BhqDf5557jgEDBjB06FCr/VLrE9a607tiDe7VHw4wft0OHl39b7XOXZfRW1hwhn0+impMUQ/8D7a+Zdge/A40q53GQ97Cpv3G4paea71BNbyoK1asMKVkPfnkkzRp0oS//vqLu+66i8cff9zpAnoS1oy1qnoIGNfaylu/ekli2/Ec/HwUdIsMsXjti7QzAGw/ebH6wtZRJLOSvsYpv7E3qt1T1Oyj8PV4w/Z1T8I1jlW5qe8UlGj4+3gO4LnhIUYcUnBarZbXXnuNRx55hJYtWwKGPqnDhg1ziXCehrU1uCqnqDa4VKxm6ArD9P/h61oxd0hZOpE3LyPprazB+ZicDHZMUUsLYcPDoC6AVn1gQM1L0dc3/jiSjUYnER0aSNumDdwtTo1waP6kVCpZsGABWq3WVfJ4NNYUj08VU1Rby5YX8sua4Hz8d3pNxKo1nKF3zbtqGc9nd6CvJMF3T0HWAWgQDvevBkXNqpvURzy19ps1HF4guuWWW9iyZYsrZPF4DI2mLfdV14LzRJzhYjLPRZWb1uDsDPTdthz2fQVypUG5BXn29Mod6PQSm684GDwxub48Dq/B3XbbbUyZMoW9e/fSo0cPAgMDLV6/6667nCacJ1K+kkaVa3A2vrOVxcF5s8q0lotaZsFVMkU9lQq/TDds3/oqtO7tSjG9lh3pl7hUrCHY34fY1o3cLU6NcVjBTZgwAbDe/1Qmk6HTeWaSuLOQy8D8DtiTi1qs1jL3hwN2X8MTk57tRW9hwZVPtreh9Asy4PNRoNfCNfdBXP12dtUE4/T0pg5NTfGHnozD70Cv19t81HflBhWVjz2ZDM5IrN9/Lp+53+8n18U5rPvP5fOfeYlxJ2PRkcyYqmWKg7Oi4HQa+HwMFGZC006G+m5e/APgan7dX9b71BuoVrK9wDYqhRy11jCVksuse1bLU6R27IfB2hlvf/sPALIKS1k8rLuVETVHrdWbrrNvdgKBvpYfH2fEeUtX/kHZGpzRktDqDM18Tl0sJqpJgOHHZNMsSP8LVEGGYF5fz/b6uZMT2UUcyypCKZfRv4Pre5jUBnZbcJcvX+b77783PTc2SzY+XnjhBUpKShwWYOnSpURFReHn50dcXBzbt2+3OXbfvn3ce++9REVFIZPJSEpKqvE5nY15UrJcJqtSwVUr+6OSU+47l+/4+eykVFumiK1lUTjDcLJW0dfHLJNh7g8HuGlhCh9sPQH/fQl/LzUMvnsZhLaruQD1GGNp8rg2jWvcW6OuYLeCW7NmDe+9957p+TvvvMNff/3Fzp072blzJx9//DHLli1z6OIbNmwgMTGRWbNmsWPHDrp160ZCQgIXLlywOr64uJg2bdowf/58m82hHT1njSjIgI8GGxoGX0GlKKfg7LnDTkxxq610OWuXcUpJcQkr9eDKAn1X/XkCgM9+2gTfPmUYcMNzhkR6QY0oCw/xjukpOKDg1q1bx2OPPWax75NPPmHz5s1s3ryZN954g88++8yhiy9atIhx48YxZswYOnfuzPLlywkICGDVqlVWx/fs2ZM33niDYcOG4evr65Rz1ojkOXA8BTaMNLSgw9KCk9kxRXW2OnKlfjNfX7R2mZpU8jVi3s/CZMGVq+jbgGKWKd8CTRFE94ebptf4uvWdvGKNqfVjvVRwR48epUuXsmh6Pz8/i56lvXr1MjVltge1Wk1aWhrx8fFlwsjlxMfHk5qaavd5nHHO0tJS8vPzLR52kfAqNIqCvHRDCzq9zuEpKmD33G7fuTyeXLfD1IvUGq6038yldJWlaOiqdeV6FcJEJEDiDZ/3aCs/Dw1bwH2rQCGWkmtKyuEL6PQSHcKCiGwc4G5xnIbdCi43N5fS0rLo+qysLKKiokzP9Xq9xetVkZ2djU6nMzV5NhIWFkZGRobd53HGOefNm0dwcLDpERkZad8F/UMMC9tKP0MLui2vl5ui2uFFtRUHZ2X/Xe/8yQ97z1sZK1nddja14Zw074ta3smw+q+TPKb4ntsU/6CWFPDARxAY6nqh6gHeklxfHrsVXMuWLfnvv/9svr5nzx5TfqqnMXXqVPLy8kyP06dP239weBe4I8mwveV1+uj+Mb1ksODsOIcVpVR+T1ZBqc04MPMy4bVVsMpVetSiHtyVfcYwkd7yfbyoNDT2nq0dBS1jXSNEPUOj07PlcBbgPeEhRuxWcLfffjszZ8606im9fPkys2fPZtCgQXZfODQ0FIVCQWZmpsX+zMxMmw4EV53T19fX1ECnWo10YoZDz0cBeLbgTVrJDNeXyeB4VlEVB9tMZbDAmD5jjfd+P152mCvX4Gohh8JaNRGlQkY4OSzxWYJCJvGFrh/rdLe4XJb6wj8nLlJQoiW0gYqYcpVrPB27Fdy0adO4ePEiHTp04I033uDbb7/l22+/ZcGCBXTo0IFLly4xbdo0uy+sUqno0aMHycnJpn16vZ7k5GR6965emo0rzmk3CfOgZU8aSIUs90nCj1LkchkZ+Y6HzkDFVC17S5inXyzm651nqnXNqjCforpKkerMnAzGNTgVGpapFhMqy2efvjUvaR7BuxPWXIckSeQWqzmYkc+Ww1l89s9plm05Bhgq93pb7rTdq7NhYWH89ddfjB8/nilTppi58mUMGDCAd999t8LaV1UkJiYyatQoYmNj6dWrF0lJSRQVFTFmzBgARo4cSYsWLZg3bx5gcCIYHRlqtZqzZ8+ya9cuGjRoQLt27ew6p8tQquD+NeQv7kNnTvGqzype5ekrieK2A3ntXYNzpNbjcxt2c3d31y4XuKpnhFqrr7AGd/3RRXSSHyVPCuAJzbOU4rkVZl1JiUZHRl4JmfklZOSXcCG/lIx8w3PDo5TM/BJKtda93QM8vPabNRxyP0VHR7Nx40YuXrzI0aNHAWjXrh2NGzeu1sWHDh1KVlYWM2fOJCMjg5iYGDZu3GhSlOnp6Rae2nPnztG9e1mU/sKFC1m4cCH9+/cnJSXFrnO6lOAWfBA+g6fPTuJexR8cpiNfKhKqdary6qMuNIKuDSdDqVZvWdF393o6nTGEHz2jeZLTkvd9CatCp5fILiw1KS+jsjJXXhl5JeRX4l0vT6MAH8Ia+hHW0I/whn50CA/yqvAQI9Xyrzdu3JhevXo5RYCJEycyceJEq68ZlZaRqKgou7yElZ3T1Zxs2IMFp4Yx1edTJuk/5KiiLcm0sjnelmVW/n3O/p/9ITgAJ7OLiAoNrHqgA5ivwblqilqi0ZmUe6T6OPzvWQAWa+8hRW+ZgiZJktcUHjibe5ntJ3I4e+lyBeWVVVBqtwXv5yMn/IriCmvoR3iwcdvXtL9pkC9+PgrXvqE6ggggcjIqhZz3dHfQXX6UgYp/eE37BrczlxyCrY7X29AUNe2LeuPCFHbPupVgf+el3JjrEr0kVdoYp7qUavVIkkRDCnk842XQXuZ80xtYfLpi2fFSrd5jv6hFpVr+Pp7DH0ey+f1IVpXOKLkMmgUZFJW58moW5Et4sMEKa9bQj4Z+Sq9R+s5AKDgnYwj0lTFJ8zidlGdpLZ1jic8SRmimoqPil9GWgrNXdZjnh5bn7KXLTlVw5oz4YDt+Ps4vp1Oq0aPX6Vjks4ymmnMQ0op/e7yO/nTFiiuFpVqPUXA6vcS+c3kGhXY4ix3plywqFMtl0LVlCO2aNTBYWsF+hF1RXmEN/Qht4Ot1DoDaQCg4J2PMZCgkgJd8XmSl+kX6KPYzSfqM17XDK4y3peDOXCy263q7T9suXeRsR4D51+ts7mWnnttIiVZHl+Mf0EOxE43MB58H1iJlNQYqKriiUi2hDayn7NUFzuZeZuuRLH4/ks2fR7PJLbYsUNCykT/9rmpKv/ah9G4b6rIfo/qMUHBOxjxV66SiFYsDn2ZK0QLGK//HLn07ftb3JLZ1IwJ8lfx+OAudHk5ZUWY5Ra6t61YdamPq01e2m2uPvwvA52GJPBgRgzKnYvYGGCy4uoT5tPOPI1kcKzftDPJV0rttE/pe1ZS+7UJpbSz5JHAZQsE5Gd9y1URSA27k/by9PKr8iYU+yzmsbolM1ghjJXO9JPHtrnNuktYxXP1VbCnLYrHPUmRIfKK9mX8a3c6D2K6KXFTqXs+yPdPObpEh9G1vsNK6RYaY+ksIageh4JxM+WoiPnIZ87XD6SI/QZz8IMt93mKufglyWRBQeQd7e6iN8kjf7jrLbwcv8Pq9XV12DV/UvOuTRCNZIecCO/NyzijuuPKarb4WRW6w4M7lXuaPI1n8cWXaeamyaWebUIIDxLTTnQgF52TM69hf0yKYnMJStCiZqH6KH3xfooP8DE/kJ7E6cAZQc29pbfDM+l2AYRHcVbysXENX+QkuSg34NGoO6pxS0/RNaaOoXm1MUYtKtWw7kcPvh61POxtcmXb2ax9K3/ZNxbSzjiEUnJMxT4ifdnsnJn+xG4AsGjFB/TSfql7l+pItnMm/mk3cwIWC6qVy1QZ/Hc22iKW75KJ1waGKzQxXbkYvyXha8xTNaQqcKasmYnOK6nwFp9dL/Hdl2vnHkSzSTolppycjFJyTGRwTwe+HsxjdJ4oWIf4W9eD+lTrymvZBZvms5b6c9/hcFkrSrzW73oxvbVd4qQl/Hc3mwfe3WexzhWHSRXacV5SrAViovZ+t+i7ccSVrw3g9W92djBacXq9Hra6+8r2QX0LaqUv8e+oiO05dssgIaBYgJ6yhHz2jGtOjdQjdIxsRZObt1GnU6CpWbxfUEB8fHxSKmocACQXnZFo2CmDD42WJ/eWnKx/qBhLfMJ3rL2/hXdXbDCp9lSyq33/ycGZhtY+tjL9PXKywz9n6rRH5LFMl4SvTsEnXg2U6Q09dY1pa+b6o5Skq1aFWqzlx4gR6vf3VhPWShFqrp0Sjp1SrQ6OTaADcGCHjxojGyGXgq5Tj66PATyk3U7BFZGcUkV3tdyxwhJCQEMLDw2s05RcKzsVU/G7KWBn8HNG6U0SoT/KOagkPqaeh9YA/xbe7neftlaNnsc9SWsqyOaEP43nNE0hXitsUq40WnGXJ8vIUqTWcP38ehUJBZGSkRd6yOZIkUaLRUazWUaTWotHokUsSAUAABsXt66MgwFdJoEqBn4/CvkrMApcgSRLFxcWmPirNmzev9rnq/rfKw7H2NVEr/Fndci5PHRtHnPwgU5SfMlc7wunXromD1Zrcp3LsCz62h2eUX9JPsZfLkoonNM+RT9la32WTBWd4bsvJoFZrKS4uJiIigoAAyzLbaq2ewlIthSUaCku1Zj1VFaBQ4KuQ08BPSZCvkkBfpVc0OfYm/P39Abhw4QLNmjWr9nRVKDgXY80SkMkg2y+SSZoneE/1Fo8qf2KXvh3f611cs66OcLN8B88ovwZgiuZRDkmWxQguq8uvwVm3prRaLeCDSqVCp5coKtVSWKqloERbIYVNIZMR6KskyE9JA18lKqVceDvrOMYfLY1GIxRcXUVuZf1IhgyFTMbP+p4s097JeOX/eN1nBYfUkRyR6kbZd1d991vLMkjyMWQqrNbeyrf6GyqMKb8GZ8uLei6vhIISJacvFVMqqS1iAmWAv8qgzIL8lPirxLTT03DGD5Cwy12Mrfxo45dtofYB/tRdTaCslOU+b9EA500D6xp+lLLcJ4mGsmLS9O15Vfuw1XGXKyg46x/TQxn55F3WclmtQ5IkVAo5jQNVtG4cQKfmDQ2J68F+BPoqhXKrpwgF52JsTVGNlp0OBU9rJnJOakxb+Xne8HmP2msdU5tIvObzAZ3k6WRJDZmgfgaNjQmEPVNUlVJOgEqJv4+cZkF+dAgLokN4EC0bBRAcoBJranWc0aNHM2TIEJdfR3wKXIwtw8HcssshmAnqZ1FLCm5T/MNjiu9rR7hKcHaDmYcVv3KPYitaSc5TmqfJxHYV6BKN3kIGa1PUvS/fylcT+tCkgS+NAlX4+ig8Zk3txhtv5Nlnn3W3GPUCoeBcjLUvnUwmqxDbtUtqZ2iFB7yoXE9v+b5aka826C47wkzlRwDM1w7nb33nSserdQYFV74vqjm+SoXIIHAzGk3dj3CuE5+QpUuXEhUVhZ+fH3FxcWzfvr3S8Z9//jkdO3bEz8+PLl268OOPP1q8Pnr0aGQymcVj4MCBrnwLNrE6RbWxf53uFr7Q9UMhk1jis4Rwcmp07f/tOcdjH/1LQYnjH0RnGUNNyONd1WJUMh0/6Hrxvu52u481TuO9qdDj6NGj2bJlC4sXLzZ9Nlu2bMmyZcssxu3cuRO5XM6pU4Y6eIsWLaJLly4EBgYSGRnJhAkTKCy0DPLeunUrffv2xd/fn8jISJ5++mmKiqpqW2ng/PnzDBo0CH9/f6Kjo/nkk0+IiooiKSnJNEYmk7Fs2TLuuusuAgMDefXVV9HpdIwdO5bo6Gj8/f3p0KEDixcvtji3TqcjMTGRkJAQmjRpwuTJk2ulSATUAQW3YcMGEhMTmTVrFjt27KBbt24kJCSYgvzK89dffzF8+HDGjh3Lzp07GTJkCEOGDKnQlHrgwIGcP3/e9Pj0009r4+1UwNp3UyazruBAxkuaR9inb02oLJ9lqsWoqP6v5HtbjvPL/ky6vPwLy6+0hqtNFOhY4rOE5rKLHNVHMFnzOJXlQ/SKbkxDv7J1OeNIW4G+5ZEkiWK11i0Pe7+wixcvpnfv3owbN8702Rw+fDiffPKJxbh169Zx/fXX07p1awDkcjlvv/02+/btY82aNfz2229MnjzZNP7YsWMMHDiQe++9lz179rBhwwa2bt1qd2+SkSNHcu7cOVJSUvjyyy9ZsWKF1e/gyy+/zN13383evXt55JFH0Ov1tGzZks8//5z9+/czc+ZMpk2bxmeffWY65s0332T16tWsWrWKrVu3cvHiRb7++mu75KopMqm2VKkN4uLi6NmzJ++88w5gyCuMjIzkqaeeYsqUKRXGDx06lKKiIr7/vmyd6rrrriMmJobly5cDhl/J3Nxcvvnmm2rJlJ+fT3BwMHl5eY43gS5H4oZdfLXzrMW+mzo0pV2zBqz840SF8cN7tWLrP//wveolgmXFfKQdwEytc1oenpxvf2Put5OPsGjT4Rpdb4ryU55Q/o9CyY/B6jkck1pUOj6+UxjdWgbz5pXrPtG/LVNu60ipVkeH6Rstxp6cP4iSkhJOnDhBdHQ0fn5+FKu1dJ75c41kri77X0kgQGVf1NWNN95ITEyMyTratWsX1157LSdPnqRVq1bo9XpatWrF9OnTeeKJJ6ye44svvuCJJ54gO9uQOPboo4+iUCh47733TGO2bt1K//79KSoqws/Pz6Y8Bw8epFOnTvzzzz/ExsYCcPToUdq3b89bb71lWi+UyWQ8++yzvPXWW5W+v4kTJ5KRkcEXX3wBQEREBM899xwvvPACYIhfjI6OpkePHpV+R8v/fc2x9zvqVgtOrVaTlpZGfHy8aZ9cLic+Pp7U1FSrx6SmplqMB0hISKgwPiUlhWbNmtGhQwfGjx9PTk7NpnvVxqoFJ7MZthDVJIDTUhjPap4EYKRyE3fL/3ClhFap6aRwoHw7Tyj/B8BkzWNVKjcjfdo1MW0brV8fG2Ei3kJMTAydOnUyWXFbtmzhwoUL3H///aYxv/76K7fccgstWrQgKCiIESNGkJOTQ3GxIaxo9+7drF69mgYNGpgeCQkJ6PV6Tpyo+ENqzqFDh1AqlVx77bWmfe3ataNRo4o50kYFaM7SpUvp0aMHTZs2pUGDBqxYsYL09HQA8vLyOH/+PHFxcabxSqXS6nlcgVsDfbOzs9HpdBV6loaFhXHw4EGrx2RkZFgdn5GRYXo+cOBA7rnnHqKjozl27BjTpk3jtttuIzU11WpEdGlpKaWlpabn+fn5NXlbFthSZNYCgKFs7WuzvjuLtffwjPIrXvP5gIPqVhyQWtdIlqMXCmjXLKhG57CHtrKzV8JdYIV2ED/qr7PrONmVzlHmz8H2vSqPv4+C/a9Urw9tTfGvYfObhx56iE8++YQpU6bwySefMHDgQJo0MSj7kydPcscddzB+/HheffVVGjduzNatWxk7dixqtZqAgAAKCwt5/PHHefrppyucu1Ur220rHSUw0LIV5fr165k0aRJvvvkmvXv3JigoiDfeeINt27bZOEPt4pWZDMOGDTNtd+nSha5du9K2bVtSUlK45ZZbKoyfN28es2fPdoksVtfgbOw3vFb2wmLtPXSTHeNGxW6W+7zFneq55NOg2rI8uW4nPz/Xz66x1XUyBFDCcp8kgmSX+Vvfide1w6o+yIxmDcuayORfdqzem0wms3ua6E5UKhU6nWUq2YMPPsj06dNJS0vjiy++MC23AKSlpaHX63nzzTdNBQXM17gArr32Wvbv30+7du0clqdDhw5otVp27txJjx49AMMU9dKlS1Ue++eff9KnTx8mTJhg2nfsWNl6b3BwMM2bN2fbtm3062f47Gm1WtLS0iwsRlfhVts/NDQUhUJBZmamxf7MzEzCw8OtHhMeHu7QeIA2bdoQGhrK0aNHrb4+depU8vLyTI/Tp087+E5sY82C8/NRoLChQSx6jyI3dHPXN6W1/AKLfJYhw/6yQOW5WOzqRjYSr/usoL38LBlSIyaqn7baKrEyfJVl47MKSisZ6blERUWxbds2Tp48SXZ2Nnq9nqioKPr06cPYsWPR6XTcddddpvHt2rVDo9GwZMkSjh8/ztq1ay0UIMCLL77IX3/9xcSJE9m1axdHjhzh22+/tcvJ0LFjR+Lj43nsscfYvn07O3fu5LHHHsPf37/K2ML27dvz77//8vPPP3P48GFmzJjBP//8YzHmmWeeYf78+XzzzTccPHiQCRMmkJuba/8NqwFuVXAqlYoePXqQnJxs2qfX60lOTqZ3b+uJ571797YYD7Bp0yab4wHOnDlDTk6OzbIrvr6+NGzY0OLhLKx9QO7o2tzuoNQ8GvCE5llKJR/iFTt5UvFttWVxdbTFI4qN3Kn4G42kYIL6GbJtNLu2xZ4zuRbPswq9U8FNmjQJhUJB586dadq0qWm96qGHHmL37t3cfffdpmoaAN26dWPRokW8/vrrXHPNNaxbt4558+ZZnLNr165s2bKFw4cP07dvX7p3787MmTOJiIiwS6aPPvqIsLAw+vXrx9133824ceMICgqq1DkB8Pjjj3PPPfcwdOhQ4uLiyMnJsbDmAJ5//nlGjBjBqFGjTNPYu+++2y65aorb7fnExERGjRpFbGwsvXr1IikpiaKiIsaMMXgOR44cSYsWLUx/0GeeeYb+/fvz5ptvMmjQINavX8+///7LihUrACgsLGT27Nnce++9hIeHc+zYMSZPnky7du1ISKj99RlzpaJSylFr9dzUsRlHLlgvVGlN8e2TopmuHcMbPitIVH7BHqkNv+u7VUMW+zWco1kBPWUHmaZcB8Bc7cPskK5y6HiAzHxLhVaXy7nXhKuuusqqE238+PGMHz/e6jHPPfcczz33nMW+ESMsS2z17NmTX375pVoyNW/e3CKe9MyZM1y4cMFiymst4MLX15cPP/yQDz/80GK/uQJWKpUkJSVZxNTVFm5XcEOHDiUrK4uZM2eSkZFBTEwMGzduNDkS0tPTLQoZ9unTh08++YTp06czbdo02rdvzzfffMM111wDgEKhYM+ePaxZs4bc3FwiIiK49dZbmTNnDr6+td8k2DxIddtUw/qfn4+CHBvWiS218rnuRrrLjvCgcjOLfZZyp/pVzkhNHZLFVQnnTbnEUtXbKGV6vtH1YY3u1hqdL8hXSUGplo7hzrOkBZXz22+/UVhYSJcuXTh//jyTJ08mKirKtG7mqbhdwYEhbsbWWkFKSkqFfffff7+FC90cf39/fv7ZPbFQ1jBXKY0CVabtflc1ZU1qxW7tlemg2dpRXC0/RTf5cd71SeJ+9SxKUdk+wIFzVxclWt5VLaaZLJeD+kimah6lpkEm/3vqBj7dns7YvtHOEbKe88cff3DbbbfZfL2wsBCNRsO0adM4fvw4QUFB9OnTh3Xr1uHj49ltD+uEgvNmbE31bukUxs4ZA+g+Z5Np39qxvThmY+oKUIqKCepn+J/vS3SVn+Bl5RqmasfZLYsrLLhpyk/oKT9MvuTPE5pnuUzlazbmTLr1Khb+UjGYOCo0kKm3d3KmmPWa2NhYdu3aVemYhIQEtyzhuBqh4FxMZUrF3KKTyaBv+6Ycz6o8d/AsTXla8xQf+cxnuHIzu6R2bNDdZJcs6ReLOX2xmMjGAVUPtoO75H/xiNKQYfC8ZjwnJcdq54tk+drB39+/WuEj3oD4hLkYez2XxvVbe4ysrfouLNQapuivKFdzjey43fKMX5dm99jKaC87w3yflQC8ox3MJr3jkeneWPVOULcQCs7F2BuFb8Te0ct0d7FJ1wNfmYblqiRCKLDruCN2thk0b2BdniCKWe7zFgGyUv7QXcMirfX10KrQVzMNultkSLWOE9Q/hIJzMY4ue10ssq96iISc5zVPcEIfRktZNot9liKvQRCwORqdvpJEe4mFPstpKz/PWakJz2gmoq/mx6i6ZR7WPRpX9SCBAKHgXI6jC/sfbLV/uplPIOM1z3FZUtFfsYdnlF86Kp4JSZLQX7HaVvxuW4YnFP8jQfEvpZKSCepnuEj1QznKx1UN6lr1Gt4tHZvRwFcsHQvsQ3xSXIyj2QPGhiv2clBqxVTNoySp3uUZ5dfs1rflN73jOX6PrP6HUxeL+XrC9bzx8yGrY/rI/+MF5QYAXtaOYrdU84Xrx/q14ffDWQzv1YrbrrGdbrdzxgAOnM+nd9smNscIBOURFpyLqcqCC7pijRiHPRAb6fA1vtHfwGqtIbg2yeddWssyqjiiIpsPZXE8q4h/Tly0+npzcljiswSFTOIzbX8+1d3s8DXKI0kw7fZObHy2H6P6RNGsoe0Qk0aBKvq0C/WYvguCuoFQcC6mqi/kp49dR1x0Y74c3wcwFLysDq9qHyZN356GsmKW+yThR/XyOK2VXlOh4V3VYprICvhPH8UM7RjscYf8UkXlkkr8GIJKsFZKvLrFXb0doeBcTFVT1GtaBLPh8d5c28pQXNDPp3p/Eg2GNbEsqSGd5Om86vMB1QnEsKaQZyjX0l1+lFwp0JD4b2f2hJ+y8koikggUcQrnz5+vNFOhPiMUnItx1MlQPvi1Q5j9BSozacxTmqfRSnLuVWzlYcWvFcaUaiv3tK7aaln99V7574xQ/opekvGs5knOSM3slqeqty4sOOcQHh7uljxrcyRJQqt1rH5fbSAUnItx1MlQvkWeo1bO3/rOzNcOB2Cm8iO6y45UGFNZl60/jmSbtjvLTl6xBA3FN1P0MQ7JUlUM4N3d7StjbjeSBOoi9zwcjHkpKCjgoYceIjAwkObNm/PWW29Vu1+q+RT15MmTyGQyvvrqK2666SYCAgLo1q1bheolVXXgWrt2LbGxsQQFBREeHs6DDz5o0YQmJSUFmUzGTz/9RI8ePfD19WXr1q0Oy+5qhBfVxTi6KF5VB6l3H7qWCet2VDrmfd3tdJcfYZBiO++qFnNH6avkmNVm6/LyL7RuEkDKpBttyteQQpb7vIWfTMNvuhje1jlev6sq5R4dGlj5AEfRFMNr9tU/czrTzoHK/veTmJjIn3/+yXfffUdYWBgzZ85kx44dxMTEOEWcl156iYULF9K+fXteeuklhg8fztGjR1EqlaYOXHPnzmXVqlVkZWWZCl4Yyx5pNBrmzJlDhw4duHDhAomJiYwePbpCi84pU6awcOFC2rRpY7WHg7sRCs7FODpFVZWz4K6JCOawWfZBfKew8odYQcZkzeN0kJ2hnfwcS3yWMEIz1aK67qmcYs7nlRAR4k+xWlvuaD1JPu/SSp5Fur4pz2kmIFXD2JfVuHWNd1JQUMCaNWv45JNPTCX0P/zwQ7uLU9rDpEmTGDTI0EVt9uzZXH311Rw9epSOHTsyb948HnroIZO12L59e95++2369+/PsmXL8PPz45FHHjGdq02bNrz99tv07NmTwsJCGjQoK5v/yiuvMGDAAKfJ7WyEgnMxjk5Ry6/BdWrekN13Xc3byUdIuDrc7h6hRfjzuOY5vlXNoI9iPy9In5mmrmWyGc41/yfLBj9PKb7hZsUuSiQfxmueI8+OPhBzhlzDjG8se9PWer9mnwCDJeUOfOwvYHD8+HE0Gg29evUy7QsODqZDhw5OE6dr166mbWMl6wsXLtCxY0d2797Nnj17WLdunWmMIdDb0IGrU6dOpKWl8fLLL7N7924uXbqEXm9Yu01PT6dz586m42qrO1Z1EQrOxXRp4VjZbqWZAmvVOIBRfaJQKeXMuKNzJUdZ55jUgsmax3hX9TZPKP/HTn1bftaXfankckOZ8I/M6tL1l+/m2SsZEdO1j7BPiqryOiEBPjwQ27KCgmvo78MLCR2sBg67pFu9TObQNNGbMa/jZlyGMCqpqjpwFRUVmconrVu3zlRWPSEhAbXasq9H+S5bdQ3hZHAxfdqF8u5D11YZE2bEvAdo4oCrUClr9if6UX8dK7SGqcpCn/doIyuzcBQyGeM/LlvPaym7wGKfd5DLJNZpb+ELXX+7rqG34g79/YWb8PNRcIdZ+tWTN7U1badOrXmgsKfSpk0bfHx8LJqz5OXlcfhwzRpt24t5B67yD5VKxcGDB8nJyWH+/Pn07duXjh07Wu1y7wkIBVcL3N6lOVfZGe5h7nmsrKKHI7yuHcbf+k4EyS4bqoBg6HUgYUisB/BFzTKfJEJkRezSt2G2dqTd55eouN7Wqolhymbe5/S+HpFsf+kWjr92u8X++kZQUBCjRo3ihRdeYPPmzezbt4+xY8cil8trJVOjqg5crVq1QqVSmbp4fffdd8yZM8flcrkCoeDqID2jGtE82I+bOlYec/biwI52nU+Hgonqp8mQGnGV/Cyv+6wAJCZ9vvtKySKJOcoP6SI/SY4UxAT1s6ixXaq631VNWTwsxvR8WE/b6WX+KgUfju7J/Hu6EB0aSLMgP4dLSHkjixYtonfv3txxxx3Ex8dz/fXX06lTpyq7WDmDqjpwNW3alNWrV/P555/TuXNn5s+fz8KFC10ulyuQSdZa5dQyS5cu5Y033iAjI4Nu3bqxZMkSiwXY8nz++efMmDGDkydP0r59e15//XVuv/120+uSJDFr1ixWrlxJbm4u119/PcuWLaN9+/Z2yZOfn09wcDB5eXlObSFoL3q9hFqnx89Gt/Tr5//G2dzL/PZ8f25+c4vd571WdpgNqjn4yHS8ohnBKp0h+n2Y4jfm+7yPTpIxQjOVv/TXVDh26m0deSA2kqzCUpM1mlVQyt6zudzQztD85qrpP5nGn5w/yG65qktJSQknTpwgOjq6VhSDKykqKqJFixa8+eabjB071t3i1Akq+/va+x11uwW3YcMGEhMTmTVrFjt27KBbt24kJCTYnPP/9ddfDB8+nLFjx7Jz506GDBnCkCFD+O+/sgXuBQsW8Pbbb7N8+XK2bdtGYGAgCQkJlJR4Rhs6uVxmU7kBJD/fn23TbqFN0waEBBgsrRHXtQYqT/XaIV3FXO3DAExTrqOn7CBdZceYrVwNwELtUKvKDeDRvm1oFKiymGo3DfLl5o5hqJRylzS08WZ27tzJp59+yrFjx9ixYwcPPfQQAIMHD3azZN6F2y24uLg4evbsyTvvvAMYPD2RkZE89dRTTJkypcL4oUOHUlRUxPfff2/ad9111xETE8Py5cuRJImIiAief/55Jk2aBBgWcMPCwli9ejXDhg2rUiZ3W3COYLSibryqGYcyC2ge7EdIgApJkoieagjKnD6oE3N/OHDlCIkkn6UMUfyFRlIgASqZjp91sTyueQ5rSfTfTbyeri1DKpXD/Hpv3NeV+6tRFcVRPNmC27lzJ48++iiHDh0yNUBftGgRubm5VXbAqi84w4Jza5iIWq0mLS2NqVOnmvbJ5XLi4+OtNsYFSE1NJTEx0WJfQkKCKVXlxIkTZGRkEB8fb3o9ODiYuLg4UlNTrSq40tJSSkvLqm/k5+fX5G3VKkYrCgwxc0ZkMhmbnutH2qlLPBAbyX09WhLzyiZAxlTNo3SUnaaj/DQAx/XhTNI8gblye37AVYQ19OPeHi3tCumQyWSkTY/nZE4xPVrXvYj2ukb37t1JS6vYH+Py5ctVdsAS2I9bFVx2djY6nc7U5NlIWFgYBw8etHpMRkaG1fEZGRmm1437bI0pz7x585g9e3a13kNdpn1YEO2vTClDAlScmHc70VN/5DJ+PKF5lm9UM1GgZ7zmWQoweD3DGvqSOuWWajkCmjTwpUkD9yZ9ezr1uQOWKxCBvsDUqVMtrML8/HwiI10/xaptZDKZxeJ/TvadnM8rZl1YBKEerJjqgJ9M4AKc8Xd1q4ILDQ1FoVCQmZlpsT8zM5PwcOvlq8PDwysdb/w/MzPTlKJifG4rkdnX19ft5WbcQZPQMJqEuluK6qNQGBwxarUaf39/N0sjcDbFxcWAZVaGo7hVwRkXV5OTkxkyZAhgcDIkJyebgg7L07t3b5KTky3KymzatInevXsDEB0dTXh4OMnJySaFlp+fz7Zt2xg/frwr346gllEqlQQEBJCVlYWPjw9ya+WIBR6HJEkUFxdz4cIFQkJCTD9k1cHtU9TExERGjRpFbGwsvXr1IikpiaKiIsaMGQPAyJEjadGiBfPmzQPgmWeeoX///rz55psMGjSI9evX8++//7JixQrAMA179tlnmTt3Lu3btyc6OpoZM2YQERFhUqIC70Amk9G8eXNOnDjBqVOnqj5A4FGEhITYnMnZi9sV3NChQ8nKymLmzJlkZGQQExPDxo0bTU6C9PR0i1/mPn368MknnzB9+nSmTZtG+/bt+eabb7jmmrL4rcmTJ1NUVMRjjz1Gbm4uN9xwAxs3bvS4UAJB1ahUKtq3b18hCVzg2fj4+NTIcjPi9ji4uognxcEJBPURj8lkEAgEAlchFJxAIPBahIITCARei9udDHUR47KkJ6VsCQT1CeN3syoXglBwVigoKADwymwGgcCbKCgoIDjYdlsA4UW1gl6v59y5cwQFBdVKhVV3YUxJO336tPAWX0HcE+vUtfsiSRIFBQVERERUGuAtLDgryOVyWrZs6W4xao2GDRvWiQ9tXULcE+vUpftSmeVmRDgZBAKB1yIUnEAg8FqEgqvH+Pr6MmvWrHpZScUW4p5Yx1Pvi3AyCAQCr0VYcAKBwGsRCk4gEHgtQsEJBAKvRSg4gUDgtQgF5+UsXbqUqKgo/Pz8iIuLY/v27ZWO//zzz+nYsSN+fn506dKFH3/8sZYkrT0cuScrV66kb9++NGrUiEaNGhEfH1/lPfRUHP2sGFm/fj0ymaxuVsyWBF7L+vXrJZVKJa1atUrat2+fNG7cOCkkJETKzMy0Ov7PP/+UFAqFtGDBAmn//v3S9OnTJR8fH2nv3r21LLnrcPSePPjgg9LSpUulnTt3SgcOHJBGjx4tBQcHS2fOnKllyV2Lo/fFyIkTJ6QWLVpIffv2lQYPHlw7wjqAUHBeTK9evaQnn3zS9Fyn00kRERHSvHnzrI5/4IEHpEGDBlnsi4uLkx5//HGXylmbOHpPyqPVaqWgoCBpzZo1rhLRLVTnvmi1WqlPnz7S+++/L40aNapOKjgxRfVS1Go1aWlpxMfHm/bJ5XLi4+NJTU21ekxqaqrFeICEhASb4z2N6tyT8hQXF6PRaGjcuLGrxKx1qntfXnnlFZo1a8bYsWNrQ8xqIZLtvZTs7Gx0Op2peY+RsLAwDh48aPWYjIwMq+MzMjJcJmdtUp17Up4XX3yRiIiICj8Enkx17svWrVv54IMP2LVrVy1IWH2EghMI7GT+/PmsX7+elJSUet2hraCggBEjRrBy5UpCQ+t253Ch4LyU0NBQFAoFmZmZFvszMzNt9poMDw93aLynUZ17YmThwoXMnz+fX3/9la5du7pSzFrH0fty7NgxTp48yZ133mnap9frAUMz7kOHDtG2bVvXCm0nYg3OS1GpVPTo0YPk5GTTPr1eT3JyMr1797Z6TO/evS3GA2zatMnmeE+jOvcEYMGCBcyZM4eNGzcSGxtbG6LWKo7el44dO7J371527dpletx1113cdNNN7Nq1q25Vwna3l0PgOtavXy/5+vpKq1evlvbv3y899thjUkhIiJSRkSFJkiSNGDFCmjJlimn8n3/+KSmVSmnhwoXSgQMHpFmzZnllmIgj92T+/PmSSqWSvvjiC+n8+fOmR0FBgbvegktw9L6Up656UYWC83KWLFkitWrVSlKpVFKvXr2kv//+2/Ra//79pVGjRlmM/+yzz6SrrrpKUqlU0tVXXy398MMPtSyx63HknrRu3VoCKjxmzZpV+4K7GEc/K+bUVQUnyiUJBAKvRazBCQQCr0UoOIFA4LUIBScQCLwWoeAEAoHXIhScQCDwWoSCEwgEXotQcAKBwGsRCk4gEHgtQsEJPIbRo0cjk8kqPAYOHOhu0QR1FFFNROBRDBw4kA8//NBin61u6xqNBh8fH4t9arUalUrl8HWre5zAvQgLTuBR+Pr6Eh4ebvFo1KgRADKZjGXLlnHXXXcRGBjIq6++yssvv0xMTAzvv/8+0dHRpjpu6enpDB48mAYNGtCwYUMeeOABi3JBto4TeBZCwQm8ipdffpm7776bvXv38sgjjwBw9OhRvvzyS7766it27dqFXq9n8ODBXLx4kS1btrBp0yaOHz/O0KFDLc5V/jiB5yGmqAKP4vvvv6dBgwYW+6ZNm8a0adMAePDBBxkzZozF62q1mo8++oimTZsChhp3e/fu5cSJE6baZR999BFXX301//zzDz179rR6nMDzEApO4FHcdNNNLFu2zGKfeQMYawUpW7dubaGkDhw4QGRkpEVhxs6dOxMSEsKBAwdMCq78cQLPQyg4gUcRGBhIu3btKn3dnn32Xkvg2Yg1OEG9o1OnTpw+fZrTp0+b9u3fv5/c3Fw6d+7sRskEzkZYcAKPorS0tEIbQ6VS6VB3p/j4eLp06cJDDz1EUlISWq2WCRMm0L9/f6/suVCfERacwKPYuHEjzZs3t3jccMMNDp1DJpPx7bff0qhRI/r160d8fDxt2rRhw4YNLpJa4C5EyXKBQOC1CAtOIBB4LULBCQQCr0UoOIFA4LUIBScQCLwWoeAEAoHXIhScQCDwWoSCEwgEXotQcAKBwGsRCk4gEHgtQsEJBAKvRSg4gUDgtQgFJxAIvJb/A4Y7LJUPyoA4AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018928, + "end_time": "2024-03-22T19:39:48.519433", + "exception": false, + "start_time": "2024-03-22T19:39:48.500505", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4549.760739, + "end_time": "2024-03-22T19:39:51.260347", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/1/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tvae/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tvae/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tvae" + }, + "start_time": "2024-03-22T18:24:01.499608", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tvae/model.pt b/contraceptive/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f99755c1daae71d86981360e6fd5731ef03ea75 --- /dev/null +++ b/contraceptive/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0666886122f68ede68dcce3fa85c996284dd7da4c4e0e081f59def727cc47185 +size 47629899 diff --git a/contraceptive/tvae/params.json b/contraceptive/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..4ecac3b2e74e186db5efa67bc2f0ff7332c427bd --- /dev/null +++ b/contraceptive/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/lct_gan/eval.csv b/insurance/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..be4172f704d537ead381cf20309e42686994e529 --- /dev/null +++ b/insurance/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0011455664967208923,0.01398279099403832,0.0009290176306413265,6.546527147293091,0.018045689910650253,0.6029329299926758,0.056619517505168915,9.035532457346562e-06,2.36279296875,0.018268784508109093,0.8310969471931458,0.03047979064285755,0.14884799718856812,2.9791326596750878e-06,8.90932011604309 diff --git a/insurance/lct_gan/history.csv b/insurance/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..a43754690f280126cc84448938cd8f269f261c84 --- /dev/null +++ b/insurance/lct_gan/history.csv @@ -0,0 +1,18 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03242904957539092,0.8712406825969596,0.006855423091376893,0.38299333698219723,0.0,0.0,0.0,0.0,0.03276979403259853,900,113,157.391695022583,1.3928468586069294,0.17487966113620335,0.13700437690831918,0.006329906290500528,0.46090240039369457,5.679718335689661e-05,0.0,0.0,0.0,0.0,0.0,0.006329906290500528,450,57,52.734307527542114,0.9251632899568792,0.1171873500612047,0.06423525986049258 +1,0.007500169986807224,1.1092925603606028,0.00019880676637914756,0.05190713240040673,0.0,0.0,0.0,0.0,0.007618296743732773,900,113,157.97713208198547,1.3980277175396945,0.17553014675776163,0.0854555291609954,0.005137449055619072,0.6586513487276945,2.703822330727585e-05,0.0,0.0,0.0,0.0,0.0,0.005137449055619072,450,57,53.497430086135864,0.9385514050199274,0.1188831779691908,0.05167298022200141 +2,0.004720190377233343,0.5550237497275922,4.577079138076836e-05,0.02987108025078972,0.0,0.0,0.0,0.0,0.004798027821105077,900,113,156.7364845275879,1.3870485356423707,0.17415164947509765,0.09026794963046512,0.004182245211753374,0.24288320739538394,0.00014376330885359007,0.0,0.0,0.0,0.0,0.0,0.004182245211753374,450,57,52.81124806404114,0.9265131239305463,0.11735832903120252,0.07229022658838515 +3,0.004049827164530547,0.43805476618803796,5.451335522204717e-05,0.03185774852211277,0.0,0.0,0.0,0.0,0.004109540785429999,900,113,155.9995777606964,1.3805272368203223,0.17333286417855157,0.08781432131288854,0.005127925792023436,0.8044920190759293,3.6524204451706925e-05,0.0,0.0,0.0,0.0,0.0,0.005127925792023436,450,57,52.598074197769165,0.92277323153981,0.11688460932837592,0.04705578919393909 +4,0.004900188291147869,0.6228896175905209,0.00012289669273024944,0.031160058855182596,0.0,0.0,0.0,0.0,0.005004006718000811,900,113,156.05951523780823,1.3810576569717543,0.17339946137534248,0.08326448575980895,0.00360431135011216,0.1391123805354558,0.00017833255974409213,0.0,0.0,0.0,0.0,0.0,0.00360431135011216,450,57,52.58910870552063,0.9226159422021163,0.11686468601226807,0.07722438271402528 +5,0.00338898796432962,0.3835711898057332,5.93510390443841e-05,0.04150173789097203,0.0,0.0,0.0,0.0,0.0034270015977866325,900,113,156.12185072898865,1.3816092984866253,0.17346872303220962,0.09039239540893947,0.004042578568138803,0.16148795401670143,2.5237032979771928e-05,0.0,0.0,0.0,0.0,0.0,0.004042578568138803,450,57,53.37223672866821,0.9363550303275125,0.11860497050815158,0.07075024978257716 +6,0.002903090084760657,0.2928461281526556,1.3530937127217902e-05,0.03275197486082713,0.0,0.0,0.0,0.0,0.0029446042016045087,900,113,157.1868932247162,1.3910344533160723,0.174652103583018,0.09083371352305454,0.004153869318348977,0.08165856703591276,0.0004118713491015787,0.0,0.0,0.0,0.0,0.0,0.004153869318348977,450,57,52.67383909225464,0.9241024402149937,0.11705297576056586,0.08548504119869649 +7,0.0026706402006998866,0.20781586027989052,0.00013795001301429672,0.037672711697717506,0.0,0.0,0.0,0.0,0.0026999003414271607,900,113,155.8716015815735,1.3793947042617123,0.17319066842397055,0.09302399396500756,0.0036898051684774043,0.04890317060488071,0.00016461873778845238,0.0,0.0,0.0,0.0,0.0,0.0036898051684774043,450,57,52.411773443222046,0.9195047972495096,0.11647060765160455,0.08178644566878415 +8,0.002238100019361203,0.3384687315130162,1.1111032782711483e-05,0.03593144379142258,0.0,0.0,0.0,0.0,0.002262775602414169,900,113,155.77839064598083,1.3785698287254942,0.1730871007177565,0.09169465267157133,0.00233326900155387,0.18033007517983266,6.032218973090211e-05,0.0,0.0,0.0,0.0,0.0,0.00233326900155387,450,57,52.599425315856934,0.9227969353659111,0.11688761181301541,0.07049635971545062 +9,0.0012590437038711064,0.15078724619232048,4.081550376790824e-06,0.0263645450067189,0.0,0.0,0.0,0.0,0.0012720353449630138,900,113,156.98792576789856,1.3892736793619342,0.1744310286309984,0.09350108472317194,0.001966165854424212,0.3141539510364837,9.730311759192344e-05,0.0,0.0,0.0,0.0,0.0,0.001966165854424212,450,57,53.111929416656494,0.9317882353799385,0.11802650981479221,0.06570776830950197 +10,0.001532604441874557,0.1929226224414785,3.391608327091929e-06,0.03449563190340996,0.0,0.0,0.0,0.0,0.00154851471255016,900,113,156.23109221458435,1.3825760372972067,0.17359010246064926,0.0936134795996204,0.002808738038454774,0.16682867217481775,0.0001628730561616128,0.0,0.0,0.0,0.0,0.0,0.002808738038454774,450,57,52.47261381149292,0.9205721721314547,0.11660580846998427,0.07371748221646014 +11,0.0019407396270738294,0.3653798322266867,2.2870681320568346e-05,0.03755757513559527,0.0,0.0,0.0,0.0,0.001960661863623601,900,113,157.5604350566864,1.3943401332450125,0.1750671500629849,0.09302689506779466,0.0022611901594912828,0.223774224152935,9.727328464069852e-05,0.0,0.0,0.0,0.0,0.0,0.0022611901594912828,450,57,52.695061922073364,0.9244747705626906,0.11710013760460748,0.06702947569602545 +12,0.0015795660438016057,0.20200802482515098,3.2623586478791824e-06,0.03782492588998543,0.0,0.0,0.0,0.0,0.0015964161236964476,900,113,157.62508010864258,1.3949122133508194,0.17513897789849175,0.09343803705301433,0.0019024741732250226,0.16194683409677263,0.00010373163840122158,0.0,0.0,0.0,0.0,0.0,0.0019024741732250226,450,57,53.39478373527527,0.9367505918469345,0.11865507496727837,0.07240477975523263 +13,0.0012615600261617348,0.17734704436294219,6.4640958073491015e-06,0.032005395059370334,0.0,0.0,0.0,0.0,0.0012749444810389024,900,113,157.78692436218262,1.3963444633821471,0.17531880484686957,0.09475085942025206,0.003454031080505552,0.19643008009194018,0.00048754797153507685,0.0,0.0,0.0,0.0,0.0,0.003454031080505552,450,57,53.61415338516235,0.9405991821958307,0.11914256307813856,0.08090297263850899 +14,0.001146070581356374,0.12761228089974574,5.205212322840684e-06,0.029757227330572074,0.0,0.0,0.0,0.0,0.001157807001274907,900,113,158.21624660491943,1.4001437752647738,0.1757958295610216,0.09795315121918653,0.0023975990749345835,0.05156347854368074,0.00027527655832503,0.0,0.0,0.0,0.0,0.0,0.0023975990749345835,450,57,53.09577012062073,0.9315047389582584,0.11799060026804606,0.07778614515177253 +15,0.0008811901032135615,0.11619655411389741,1.1726831726480937e-06,0.026715771118178962,0.0,0.0,0.0,0.0,0.0008903484087042872,900,113,157.73501706123352,1.3958851067365798,0.17526113006803726,0.09735740258036989,0.0022739232058585105,0.07977884817566928,0.0003206384845873516,0.0,0.0,0.0,0.0,0.0,0.0022739232058585105,450,57,52.94484996795654,0.9288570169816938,0.11765522215101454,0.08302696749339239 +16,0.0007510672794726109,0.07160761223355694,8.719960328579189e-07,0.025627725821816258,0.0,0.0,0.0,0.0,0.0007592219165558668,900,113,155.887836933136,1.3795383799392564,0.17320870770348443,0.10122318941671236,0.001969620921461481,0.14806897564466326,0.0002989729056185993,0.0,0.0,0.0,0.0,0.0,0.001969620921461481,450,57,52.3817937374115,0.9189788374984473,0.11640398608313667,0.07865735263514675 diff --git a/insurance/lct_gan/mlu-eval.ipynb b/insurance/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3722323aa04cb752ed501e8abbb4cfa6a339b2ad --- /dev/null +++ b/insurance/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2475 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.775742Z", + "iopub.status.busy": "2024-03-22T13:31:10.775388Z", + "iopub.status.idle": "2024-03-22T13:31:10.809260Z", + "shell.execute_reply": "2024-03-22T13:31:10.808445Z" + }, + "papermill": { + "duration": 0.049185, + "end_time": "2024-03-22T13:31:10.811626", + "exception": false, + "start_time": "2024-03-22T13:31:10.762441", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.837166Z", + "iopub.status.busy": "2024-03-22T13:31:10.836720Z", + "iopub.status.idle": "2024-03-22T13:31:10.843587Z", + "shell.execute_reply": "2024-03-22T13:31:10.842788Z" + }, + "papermill": { + "duration": 0.022047, + "end_time": "2024-03-22T13:31:10.845565", + "exception": false, + "start_time": "2024-03-22T13:31:10.823518", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.868871Z", + "iopub.status.busy": "2024-03-22T13:31:10.868586Z", + "iopub.status.idle": "2024-03-22T13:31:10.872665Z", + "shell.execute_reply": "2024-03-22T13:31:10.871831Z" + }, + "papermill": { + "duration": 0.017967, + "end_time": "2024-03-22T13:31:10.874536", + "exception": false, + "start_time": "2024-03-22T13:31:10.856569", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.898144Z", + "iopub.status.busy": "2024-03-22T13:31:10.897843Z", + "iopub.status.idle": "2024-03-22T13:31:10.902004Z", + "shell.execute_reply": "2024-03-22T13:31:10.901136Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018546, + "end_time": "2024-03-22T13:31:10.904116", + "exception": false, + "start_time": "2024-03-22T13:31:10.885570", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.928121Z", + "iopub.status.busy": "2024-03-22T13:31:10.927759Z", + "iopub.status.idle": "2024-03-22T13:31:10.933667Z", + "shell.execute_reply": "2024-03-22T13:31:10.932778Z" + }, + "papermill": { + "duration": 0.020413, + "end_time": "2024-03-22T13:31:10.935553", + "exception": false, + "start_time": "2024-03-22T13:31:10.915140", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fa29630e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.960407Z", + "iopub.status.busy": "2024-03-22T13:31:10.960101Z", + "iopub.status.idle": "2024-03-22T13:31:10.965227Z", + "shell.execute_reply": "2024-03-22T13:31:10.964361Z" + }, + "papermill": { + "duration": 0.019757, + "end_time": "2024-03-22T13:31:10.967135", + "exception": false, + "start_time": "2024-03-22T13:31:10.947378", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/lct_gan/0\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011013, + "end_time": "2024-03-22T13:31:10.989313", + "exception": false, + "start_time": "2024-03-22T13:31:10.978300", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:11.013132Z", + "iopub.status.busy": "2024-03-22T13:31:11.012736Z", + "iopub.status.idle": "2024-03-22T13:31:11.022355Z", + "shell.execute_reply": "2024-03-22T13:31:11.021514Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024011, + "end_time": "2024-03-22T13:31:11.024335", + "exception": false, + "start_time": "2024-03-22T13:31:11.000324", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/lct_gan/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:11.048821Z", + "iopub.status.busy": "2024-03-22T13:31:11.048470Z", + "iopub.status.idle": "2024-03-22T13:31:13.092182Z", + "shell.execute_reply": "2024-03-22T13:31:13.091205Z" + }, + "papermill": { + "duration": 2.058715, + "end_time": "2024-03-22T13:31:13.094289", + "exception": false, + "start_time": "2024-03-22T13:31:11.035574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.121361Z", + "iopub.status.busy": "2024-03-22T13:31:13.120262Z", + "iopub.status.idle": "2024-03-22T13:31:13.134109Z", + "shell.execute_reply": "2024-03-22T13:31:13.133344Z" + }, + "papermill": { + "duration": 0.029569, + "end_time": "2024-03-22T13:31:13.136200", + "exception": false, + "start_time": "2024-03-22T13:31:13.106631", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.161072Z", + "iopub.status.busy": "2024-03-22T13:31:13.160367Z", + "iopub.status.idle": "2024-03-22T13:31:13.167785Z", + "shell.execute_reply": "2024-03-22T13:31:13.167005Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021972, + "end_time": "2024-03-22T13:31:13.169788", + "exception": false, + "start_time": "2024-03-22T13:31:13.147816", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.194948Z", + "iopub.status.busy": "2024-03-22T13:31:13.194311Z", + "iopub.status.idle": "2024-03-22T13:31:13.291036Z", + "shell.execute_reply": "2024-03-22T13:31:13.289868Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.112234, + "end_time": "2024-03-22T13:31:13.293545", + "exception": false, + "start_time": "2024-03-22T13:31:13.181311", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.321198Z", + "iopub.status.busy": "2024-03-22T13:31:13.320221Z", + "iopub.status.idle": "2024-03-22T13:31:18.059075Z", + "shell.execute_reply": "2024-03-22T13:31:18.058235Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.755204, + "end_time": "2024-03-22T13:31:18.061484", + "exception": false, + "start_time": "2024-03-22T13:31:13.306280", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 13:31:15.593181: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 13:31:15.593242: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 13:31:15.594874: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:18.086449Z", + "iopub.status.busy": "2024-03-22T13:31:18.085796Z", + "iopub.status.idle": "2024-03-22T13:31:18.092975Z", + "shell.execute_reply": "2024-03-22T13:31:18.092258Z" + }, + "papermill": { + "duration": 0.02172, + "end_time": "2024-03-22T13:31:18.094921", + "exception": false, + "start_time": "2024-03-22T13:31:18.073201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:18.122003Z", + "iopub.status.busy": "2024-03-22T13:31:18.121152Z", + "iopub.status.idle": "2024-03-22T13:31:26.530810Z", + "shell.execute_reply": "2024-03-22T13:31:26.529807Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.426043, + "end_time": "2024-03-22T13:31:26.533331", + "exception": false, + "start_time": "2024-03-22T13:31:18.107288", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T13:31:26.562557Z", + "iopub.status.busy": "2024-03-22T13:31:26.562182Z", + "iopub.status.idle": "2024-03-22T13:31:26.568987Z", + "shell.execute_reply": "2024-03-22T13:31:26.568139Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023527, + "end_time": "2024-03-22T13:31:26.571019", + "exception": false, + "start_time": "2024-03-22T13:31:26.547492", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:26.596763Z", + "iopub.status.busy": "2024-03-22T13:31:26.596424Z", + "iopub.status.idle": "2024-03-22T13:31:26.601516Z", + "shell.execute_reply": "2024-03-22T13:31:26.600685Z" + }, + "papermill": { + "duration": 0.020363, + "end_time": "2024-03-22T13:31:26.603428", + "exception": false, + "start_time": "2024-03-22T13:31:26.583065", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:26.628735Z", + "iopub.status.busy": "2024-03-22T13:31:26.628397Z", + "iopub.status.idle": "2024-03-22T13:36:01.416808Z", + "shell.execute_reply": "2024-03-22T13:36:01.415837Z" + }, + "papermill": { + "duration": 274.815243, + "end_time": "2024-03-22T13:36:01.430583", + "exception": false, + "start_time": "2024-03-22T13:31:26.615340", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:36:01.459705Z", + "iopub.status.busy": "2024-03-22T13:36:01.458691Z", + "iopub.status.idle": "2024-03-22T13:36:01.798957Z", + "shell.execute_reply": "2024-03-22T13:36:01.798123Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.357702, + "end_time": "2024-03-22T13:36:01.801276", + "exception": false, + "start_time": "2024-03-22T13:36:01.443574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:36:01.832024Z", + "iopub.status.busy": "2024-03-22T13:36:01.831204Z", + "iopub.status.idle": "2024-03-22T13:41:58.932397Z", + "shell.execute_reply": "2024-03-22T13:41:58.931173Z" + }, + "papermill": { + "duration": 357.136543, + "end_time": "2024-03-22T13:41:58.951952", + "exception": false, + "start_time": "2024-03-22T13:36:01.815409", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T13:41:58.986118Z", + "iopub.status.busy": "2024-03-22T13:41:58.985668Z", + "iopub.status.idle": "2024-03-22T13:41:59.466837Z", + "shell.execute_reply": "2024-03-22T13:41:59.465865Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.5011, + "end_time": "2024-03-22T13:41:59.469128", + "exception": false, + "start_time": "2024-03-22T13:41:58.968028", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.500820Z", + "iopub.status.busy": "2024-03-22T13:41:59.500461Z", + "iopub.status.idle": "2024-03-22T13:41:59.504956Z", + "shell.execute_reply": "2024-03-22T13:41:59.504022Z" + }, + "papermill": { + "duration": 0.023179, + "end_time": "2024-03-22T13:41:59.506852", + "exception": false, + "start_time": "2024-03-22T13:41:59.483673", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.535267Z", + "iopub.status.busy": "2024-03-22T13:41:59.534881Z", + "iopub.status.idle": "2024-03-22T13:41:59.542637Z", + "shell.execute_reply": "2024-03-22T13:41:59.541738Z" + }, + "papermill": { + "duration": 0.024826, + "end_time": "2024-03-22T13:41:59.544773", + "exception": false, + "start_time": "2024-03-22T13:41:59.519947", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9631369" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.579995Z", + "iopub.status.busy": "2024-03-22T13:41:59.578894Z", + "iopub.status.idle": "2024-03-22T13:41:59.686595Z", + "shell.execute_reply": "2024-03-22T13:41:59.685598Z" + }, + "papermill": { + "duration": 0.129451, + "end_time": "2024-03-22T13:41:59.689262", + "exception": false, + "start_time": "2024-03-22T13:41:59.559811", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 29] --\n", + "├─Adapter: 1-1 [2, 1071, 29] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 30,720\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 29] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,631,369\n", + "Trainable params: 9,631,369\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.31\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.53\n", + "Estimated Total Size (MB): 346.32\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.723605Z", + "iopub.status.busy": "2024-03-22T13:41:59.723221Z", + "iopub.status.idle": "2024-03-22T14:46:40.208564Z", + "shell.execute_reply": "2024-03-22T14:46:40.207591Z" + }, + "papermill": { + "duration": 3880.522327, + "end_time": "2024-03-22T14:46:40.227825", + "exception": false, + "start_time": "2024-03-22T13:41:59.705498", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03242904957539092, 'avg_role_model_std_loss': 0.8712406825969596, 'avg_role_model_mean_pred_loss': 0.006855423091376893, 'avg_role_model_g_mag_loss': 0.38299333698219723, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03276979403259853, 'n_size': 900, 'n_batch': 113, 'duration': 157.391695022583, 'duration_batch': 1.3928468586069294, 'duration_size': 0.17487966113620335, 'avg_pred_std': 0.13700437690831918}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006329906290500528, 'avg_role_model_std_loss': 0.46090240039369457, 'avg_role_model_mean_pred_loss': 5.679718335689661e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006329906290500528, 'n_size': 450, 'n_batch': 57, 'duration': 52.734307527542114, 'duration_batch': 0.9251632899568792, 'duration_size': 0.1171873500612047, 'avg_pred_std': 0.06423525986049258}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007500169986807224, 'avg_role_model_std_loss': 1.1092925603606028, 'avg_role_model_mean_pred_loss': 0.00019880676637914756, 'avg_role_model_g_mag_loss': 0.05190713240040673, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007618296743732773, 'n_size': 900, 'n_batch': 113, 'duration': 157.97713208198547, 'duration_batch': 1.3980277175396945, 'duration_size': 0.17553014675776163, 'avg_pred_std': 0.0854555291609954}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005137449055619072, 'avg_role_model_std_loss': 0.6586513487276945, 'avg_role_model_mean_pred_loss': 2.703822330727585e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005137449055619072, 'n_size': 450, 'n_batch': 57, 'duration': 53.497430086135864, 'duration_batch': 0.9385514050199274, 'duration_size': 0.1188831779691908, 'avg_pred_std': 0.05167298022200141}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004720190377233343, 'avg_role_model_std_loss': 0.5550237497275922, 'avg_role_model_mean_pred_loss': 4.577079138076836e-05, 'avg_role_model_g_mag_loss': 0.02987108025078972, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004798027821105077, 'n_size': 900, 'n_batch': 113, 'duration': 156.7364845275879, 'duration_batch': 1.3870485356423707, 'duration_size': 0.17415164947509765, 'avg_pred_std': 0.09026794963046512}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004182245211753374, 'avg_role_model_std_loss': 0.24288320739538394, 'avg_role_model_mean_pred_loss': 0.00014376330885359007, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004182245211753374, 'n_size': 450, 'n_batch': 57, 'duration': 52.81124806404114, 'duration_batch': 0.9265131239305463, 'duration_size': 0.11735832903120252, 'avg_pred_std': 0.07229022658838515}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004049827164530547, 'avg_role_model_std_loss': 0.43805476618803796, 'avg_role_model_mean_pred_loss': 5.451335522204717e-05, 'avg_role_model_g_mag_loss': 0.03185774852211277, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004109540785429999, 'n_size': 900, 'n_batch': 113, 'duration': 155.9995777606964, 'duration_batch': 1.3805272368203223, 'duration_size': 0.17333286417855157, 'avg_pred_std': 0.08781432131288854}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005127925792023436, 'avg_role_model_std_loss': 0.8044920190759293, 'avg_role_model_mean_pred_loss': 3.6524204451706925e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005127925792023436, 'n_size': 450, 'n_batch': 57, 'duration': 52.598074197769165, 'duration_batch': 0.92277323153981, 'duration_size': 0.11688460932837592, 'avg_pred_std': 0.04705578919393909}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004900188291147869, 'avg_role_model_std_loss': 0.6228896175905209, 'avg_role_model_mean_pred_loss': 0.00012289669273024944, 'avg_role_model_g_mag_loss': 0.031160058855182596, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005004006718000811, 'n_size': 900, 'n_batch': 113, 'duration': 156.05951523780823, 'duration_batch': 1.3810576569717543, 'duration_size': 0.17339946137534248, 'avg_pred_std': 0.08326448575980895}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00360431135011216, 'avg_role_model_std_loss': 0.1391123805354558, 'avg_role_model_mean_pred_loss': 0.00017833255974409213, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00360431135011216, 'n_size': 450, 'n_batch': 57, 'duration': 52.58910870552063, 'duration_batch': 0.9226159422021163, 'duration_size': 0.11686468601226807, 'avg_pred_std': 0.07722438271402528}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00338898796432962, 'avg_role_model_std_loss': 0.3835711898057332, 'avg_role_model_mean_pred_loss': 5.93510390443841e-05, 'avg_role_model_g_mag_loss': 0.04150173789097203, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034270015977866325, 'n_size': 900, 'n_batch': 113, 'duration': 156.12185072898865, 'duration_batch': 1.3816092984866253, 'duration_size': 0.17346872303220962, 'avg_pred_std': 0.09039239540893947}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004042578568138803, 'avg_role_model_std_loss': 0.16148795401670143, 'avg_role_model_mean_pred_loss': 2.5237032979771928e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004042578568138803, 'n_size': 450, 'n_batch': 57, 'duration': 53.37223672866821, 'duration_batch': 0.9363550303275125, 'duration_size': 0.11860497050815158, 'avg_pred_std': 0.07075024978257716}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002903090084760657, 'avg_role_model_std_loss': 0.2928461281526556, 'avg_role_model_mean_pred_loss': 1.3530937127217902e-05, 'avg_role_model_g_mag_loss': 0.03275197486082713, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029446042016045087, 'n_size': 900, 'n_batch': 113, 'duration': 157.1868932247162, 'duration_batch': 1.3910344533160723, 'duration_size': 0.174652103583018, 'avg_pred_std': 0.09083371352305454}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004153869318348977, 'avg_role_model_std_loss': 0.08165856703591276, 'avg_role_model_mean_pred_loss': 0.0004118713491015787, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004153869318348977, 'n_size': 450, 'n_batch': 57, 'duration': 52.67383909225464, 'duration_batch': 0.9241024402149937, 'duration_size': 0.11705297576056586, 'avg_pred_std': 0.08548504119869649}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026706402006998866, 'avg_role_model_std_loss': 0.20781586027989052, 'avg_role_model_mean_pred_loss': 0.00013795001301429672, 'avg_role_model_g_mag_loss': 0.037672711697717506, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026999003414271607, 'n_size': 900, 'n_batch': 113, 'duration': 155.8716015815735, 'duration_batch': 1.3793947042617123, 'duration_size': 0.17319066842397055, 'avg_pred_std': 0.09302399396500756}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036898051684774043, 'avg_role_model_std_loss': 0.04890317060488071, 'avg_role_model_mean_pred_loss': 0.00016461873778845238, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036898051684774043, 'n_size': 450, 'n_batch': 57, 'duration': 52.411773443222046, 'duration_batch': 0.9195047972495096, 'duration_size': 0.11647060765160455, 'avg_pred_std': 0.08178644566878415}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002238100019361203, 'avg_role_model_std_loss': 0.3384687315130162, 'avg_role_model_mean_pred_loss': 1.1111032782711483e-05, 'avg_role_model_g_mag_loss': 0.03593144379142258, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002262775602414169, 'n_size': 900, 'n_batch': 113, 'duration': 155.77839064598083, 'duration_batch': 1.3785698287254942, 'duration_size': 0.1730871007177565, 'avg_pred_std': 0.09169465267157133}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00233326900155387, 'avg_role_model_std_loss': 0.18033007517983266, 'avg_role_model_mean_pred_loss': 6.032218973090211e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00233326900155387, 'n_size': 450, 'n_batch': 57, 'duration': 52.599425315856934, 'duration_batch': 0.9227969353659111, 'duration_size': 0.11688761181301541, 'avg_pred_std': 0.07049635971545062}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012590437038711064, 'avg_role_model_std_loss': 0.15078724619232048, 'avg_role_model_mean_pred_loss': 4.081550376790824e-06, 'avg_role_model_g_mag_loss': 0.0263645450067189, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012720353449630138, 'n_size': 900, 'n_batch': 113, 'duration': 156.98792576789856, 'duration_batch': 1.3892736793619342, 'duration_size': 0.1744310286309984, 'avg_pred_std': 0.09350108472317194}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001966165854424212, 'avg_role_model_std_loss': 0.3141539510364837, 'avg_role_model_mean_pred_loss': 9.730311759192344e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001966165854424212, 'n_size': 450, 'n_batch': 57, 'duration': 53.111929416656494, 'duration_batch': 0.9317882353799385, 'duration_size': 0.11802650981479221, 'avg_pred_std': 0.06570776830950197}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001532604441874557, 'avg_role_model_std_loss': 0.1929226224414785, 'avg_role_model_mean_pred_loss': 3.391608327091929e-06, 'avg_role_model_g_mag_loss': 0.03449563190340996, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00154851471255016, 'n_size': 900, 'n_batch': 113, 'duration': 156.23109221458435, 'duration_batch': 1.3825760372972067, 'duration_size': 0.17359010246064926, 'avg_pred_std': 0.0936134795996204}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002808738038454774, 'avg_role_model_std_loss': 0.16682867217481775, 'avg_role_model_mean_pred_loss': 0.0001628730561616128, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002808738038454774, 'n_size': 450, 'n_batch': 57, 'duration': 52.47261381149292, 'duration_batch': 0.9205721721314547, 'duration_size': 0.11660580846998427, 'avg_pred_std': 0.07371748221646014}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019407396270738294, 'avg_role_model_std_loss': 0.3653798322266867, 'avg_role_model_mean_pred_loss': 2.2870681320568346e-05, 'avg_role_model_g_mag_loss': 0.03755757513559527, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001960661863623601, 'n_size': 900, 'n_batch': 113, 'duration': 157.5604350566864, 'duration_batch': 1.3943401332450125, 'duration_size': 0.1750671500629849, 'avg_pred_std': 0.09302689506779466}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022611901594912828, 'avg_role_model_std_loss': 0.223774224152935, 'avg_role_model_mean_pred_loss': 9.727328464069852e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022611901594912828, 'n_size': 450, 'n_batch': 57, 'duration': 52.695061922073364, 'duration_batch': 0.9244747705626906, 'duration_size': 0.11710013760460748, 'avg_pred_std': 0.06702947569602545}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015795660438016057, 'avg_role_model_std_loss': 0.20200802482515098, 'avg_role_model_mean_pred_loss': 3.2623586478791824e-06, 'avg_role_model_g_mag_loss': 0.03782492588998543, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015964161236964476, 'n_size': 900, 'n_batch': 113, 'duration': 157.62508010864258, 'duration_batch': 1.3949122133508194, 'duration_size': 0.17513897789849175, 'avg_pred_std': 0.09343803705301433}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019024741732250226, 'avg_role_model_std_loss': 0.16194683409677263, 'avg_role_model_mean_pred_loss': 0.00010373163840122158, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019024741732250226, 'n_size': 450, 'n_batch': 57, 'duration': 53.39478373527527, 'duration_batch': 0.9367505918469345, 'duration_size': 0.11865507496727837, 'avg_pred_std': 0.07240477975523263}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012615600261617348, 'avg_role_model_std_loss': 0.17734704436294219, 'avg_role_model_mean_pred_loss': 6.4640958073491015e-06, 'avg_role_model_g_mag_loss': 0.032005395059370334, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012749444810389024, 'n_size': 900, 'n_batch': 113, 'duration': 157.78692436218262, 'duration_batch': 1.3963444633821471, 'duration_size': 0.17531880484686957, 'avg_pred_std': 0.09475085942025206}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003454031080505552, 'avg_role_model_std_loss': 0.19643008009194018, 'avg_role_model_mean_pred_loss': 0.00048754797153507685, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003454031080505552, 'n_size': 450, 'n_batch': 57, 'duration': 53.61415338516235, 'duration_batch': 0.9405991821958307, 'duration_size': 0.11914256307813856, 'avg_pred_std': 0.08090297263850899}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001146070581356374, 'avg_role_model_std_loss': 0.12761228089974574, 'avg_role_model_mean_pred_loss': 5.205212322840684e-06, 'avg_role_model_g_mag_loss': 0.029757227330572074, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001157807001274907, 'n_size': 900, 'n_batch': 113, 'duration': 158.21624660491943, 'duration_batch': 1.4001437752647738, 'duration_size': 0.1757958295610216, 'avg_pred_std': 0.09795315121918653}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023975990749345835, 'avg_role_model_std_loss': 0.05156347854368074, 'avg_role_model_mean_pred_loss': 0.00027527655832503, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023975990749345835, 'n_size': 450, 'n_batch': 57, 'duration': 53.09577012062073, 'duration_batch': 0.9315047389582584, 'duration_size': 0.11799060026804606, 'avg_pred_std': 0.07778614515177253}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008811901032135615, 'avg_role_model_std_loss': 0.11619655411389741, 'avg_role_model_mean_pred_loss': 1.1726831726480937e-06, 'avg_role_model_g_mag_loss': 0.026715771118178962, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008903484087042872, 'n_size': 900, 'n_batch': 113, 'duration': 157.73501706123352, 'duration_batch': 1.3958851067365798, 'duration_size': 0.17526113006803726, 'avg_pred_std': 0.09735740258036989}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022739232058585105, 'avg_role_model_std_loss': 0.07977884817566928, 'avg_role_model_mean_pred_loss': 0.0003206384845873516, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022739232058585105, 'n_size': 450, 'n_batch': 57, 'duration': 52.94484996795654, 'duration_batch': 0.9288570169816938, 'duration_size': 0.11765522215101454, 'avg_pred_std': 0.08302696749339239}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007510672794726109, 'avg_role_model_std_loss': 0.07160761223355694, 'avg_role_model_mean_pred_loss': 8.719960328579189e-07, 'avg_role_model_g_mag_loss': 0.025627725821816258, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007592219165558668, 'n_size': 900, 'n_batch': 113, 'duration': 155.887836933136, 'duration_batch': 1.3795383799392564, 'duration_size': 0.17320870770348443, 'avg_pred_std': 0.10122318941671236}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001969620921461481, 'avg_role_model_std_loss': 0.14806897564466326, 'avg_role_model_mean_pred_loss': 0.0002989729056185993, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001969620921461481, 'n_size': 450, 'n_batch': 57, 'duration': 52.3817937374115, 'duration_batch': 0.9189788374984473, 'duration_size': 0.11640398608313667, 'avg_pred_std': 0.07865735263514675}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006008681999972194, 'avg_role_model_std_loss': 0.07322366024851942, 'avg_role_model_mean_pred_loss': 2.489029027904118e-07, 'avg_role_model_g_mag_loss': 0.019117836964627107, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006073361239072659, 'n_size': 900, 'n_batch': 113, 'duration': 156.76917052268982, 'duration_batch': 1.3873377922361931, 'duration_size': 0.17418796724743313, 'avg_pred_std': 0.0980093978114624}\n", + "Time out: 3743.244676589966/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'pred_duration': 2.398756980895996, 'grad_duration': 6.545684576034546, 'total_duration': 8.944441556930542, 'pred_std': 0.14884799718856812, 'std_loss': 2.9791326596750878e-06, 'mean_pred_loss': 9.03553154785186e-06, 'pred_rmse': 0.030479786917567253, 'pred_mae': 0.018268786370754242, 'pred_mape': 0.8310969471931458, 'grad_rmse': 0.05661950632929802, 'grad_mae': 0.018045680597424507, 'grad_mape': 0.6029329299926758}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'avg_pred_duration': 2.398756980895996, 'avg_grad_duration': 6.545684576034546, 'avg_total_duration': 8.944441556930542, 'avg_pred_std': 0.14884799718856812, 'avg_std_loss': 2.9791326596750878e-06, 'avg_mean_pred_loss': 9.03553154785186e-06}, 'min_metrics': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'pred_duration': 2.398756980895996, 'grad_duration': 6.545684576034546, 'total_duration': 8.944441556930542, 'pred_std': 0.14884799718856812, 'std_loss': 2.9791326596750878e-06, 'mean_pred_loss': 9.03553154785186e-06, 'pred_rmse': 0.030479786917567253, 'pred_mae': 0.018268786370754242, 'pred_mape': 0.8310969471931458, 'grad_rmse': 0.05661950632929802, 'grad_mae': 0.018045680597424507, 'grad_mape': 0.6029329299926758}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'pred_duration': 2.398756980895996, 'grad_duration': 6.545684576034546, 'total_duration': 8.944441556930542, 'pred_std': 0.14884799718856812, 'std_loss': 2.9791326596750878e-06, 'mean_pred_loss': 9.03553154785186e-06, 'pred_rmse': 0.030479786917567253, 'pred_mae': 0.018268786370754242, 'pred_mape': 0.8310969471931458, 'grad_rmse': 0.05661950632929802, 'grad_mae': 0.018045680597424507, 'grad_mape': 0.6029329299926758}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.265195Z", + "iopub.status.busy": "2024-03-22T14:46:40.264825Z", + "iopub.status.idle": "2024-03-22T14:46:40.269352Z", + "shell.execute_reply": "2024-03-22T14:46:40.268411Z" + }, + "papermill": { + "duration": 0.025753, + "end_time": "2024-03-22T14:46:40.271239", + "exception": false, + "start_time": "2024-03-22T14:46:40.245486", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.307605Z", + "iopub.status.busy": "2024-03-22T14:46:40.307245Z", + "iopub.status.idle": "2024-03-22T14:46:40.388209Z", + "shell.execute_reply": "2024-03-22T14:46:40.387226Z" + }, + "papermill": { + "duration": 0.101979, + "end_time": "2024-03-22T14:46:40.390711", + "exception": false, + "start_time": "2024-03-22T14:46:40.288732", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.428666Z", + "iopub.status.busy": "2024-03-22T14:46:40.428323Z", + "iopub.status.idle": "2024-03-22T14:46:40.724550Z", + "shell.execute_reply": "2024-03-22T14:46:40.723557Z" + }, + "papermill": { + "duration": 0.317504, + "end_time": "2024-03-22T14:46:40.726627", + "exception": false, + "start_time": "2024-03-22T14:46:40.409123", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+qElEQVR4nO3de1xUZf4H8M/MwMxwHW7CAHLxhhoimsaElWSyoZGGuaHEKrmmtmleWDeXNqXa34Zpba6XzbVard0MdFfbTc0i1MwYQbl4l1VDQGFAQBjul5nn98dxjg4MODNcBuT7fr3OCznnOec8MzGfnjnnOc8jYIwxEEJIPyG0dAUIIcQUFFqEkH6FQosQ0q9QaBFC+hUKLUJIv0KhRQjpVyi0CCH9ipWlK9BbtFotiouL4eDgAIFAYOnqEELaYIyhpqYGXl5eEAo7bk8NmNAqLi6Gj4+PpatBCLmPoqIiDB48uMPtAya0HBwcAHBviKOjo4VrQwhpS61Ww8fHh/+sdmTAhJbuK6GjoyOFFiF92P0u39CFeEJIv0KhRQjpVyi0CCH9yoC5pkW6RqPRoKWlxdLVIP2YtbU1RCJRl49DoUU6xRiDSqVCVVWVpatCHgBOTk6Qy+Vd6itJoUU6pQssd3d32NraUsdcYhbGGOrr61FWVgYA8PT0NPtYFFptVDe04FKJGgIAiqGulq6ORWk0Gj6wXF0H9ntBus7GxgYAUFZWBnd3d7O/KtKF+DZyCm9j7o6TeOvri5auisXprmHZ2tpauCbkQaH7W+rK9VEKrTZc7SQAgMq6JgvXpO+gr4Sku3TH3xKFVhvOdtYAgNt1LaA5Pwjpeyi02tC1tJo1WtQ2tVq4NoSQtii02rARiyC15t6W23XUL4mYRyAQ4KuvvrJ0NbrVW2+9hXHjxlm6GhRahuhaWxV0XYv0Y7t27YKTk1O3HW/16tVIS0vrtuOZi0LLAP66Vn2zhWtCSM9rbjbu79ze3r5PdH2h0DLAhb+DSF8P78UYQ31zq0UWU2+KHD58GI8//jicnJzg6uqKZ599FteuXQMATJo0CWvWrNErf+vWLVhbW+P48eMAgJKSEkRGRsLGxgZDhgzB7t274e/vj02bNpn13p07dw5PPfUUbGxs4OrqisWLF6O2tpbffuzYMYSEhMDOzg5OTk547LHHUFBQAAA4c+YMpkyZAgcHBzg6OmLChAk4ffp0p+c7duwYFixYgOrqaggEAggEArz11lsAAH9/f/zxj3/E/Pnz4ejoiMWLFwMA1qxZg4CAANja2mLo0KFYu3atXteEtl8PX3rpJURFReH999+Hp6cnXF1dsXTp0h5/3Is6lxrgYsu1tKjbg76GFg0eWvetRc598Z0I2IqN/3Otq6tDfHw8xo4di9raWqxbtw6zZs1Cbm4uYmNjsWHDBqxfv56/BZ+SkgIvLy888cQTAID58+ejvLwcx44dg7W1NeLj4/ne3Kaqq6tDREQEQkNDcerUKZSVleHll1/GsmXLsGvXLrS2tiIqKgqLFi3Cl19+iebmZmRmZvJ1i42Nxfjx4/HRRx9BJBIhNzcX1tbWnZ5z0qRJ2LRpE9atW4e8vDwAXEtJ5/3338e6deuQmJjIr3NwcMCuXbvg5eWFc+fOYdGiRXBwcMDrr7/e4XmOHj0KT09PHD16FFevXsWcOXMwbtw4LFq0yKz3yhgUWgZQS6v/mz17tt7vf//73zFo0CBcvHgR0dHRWLlyJU6cOMGH1O7duxETEwOBQIDLly/j+++/x6lTpzBx4kQAwCeffIIRI0aYVZfdu3ejsbERn3/+Oezs7AAAW7duxYwZM/Dee+/B2toa1dXVePbZZzFs2DAAwOjRo/n9CwsL8bvf/Q6jRo0CAKPqIRaLIZPJIBAIIJfL221/6qmn8Nvf/lZv3Ztvvsn/29/fH6tXr0ZycnKnoeXs7IytW7dCJBJh1KhRiIyMRFpaGoVWb3Oxo5aWITbWIlx8J8Ji5zbFlStXsG7dOmRkZKC8vBxarRYAFwBjxozB008/jS+++AJPPPEE8vPzoVQq8be//Q0AkJeXBysrKzz88MP88YYPHw5nZ2ez6n7p0iUEBwfzgQUAjz32GLRaLfLy8jB58mS89NJLiIiIwC9+8QuEh4cjOjqafz4vPj4eL7/8Mv7xj38gPDwcL7zwAh9u5tKF8b1SUlKwefNmXLt2DbW1tWhtbb3vKL+BgYF6j+N4enri3LlzXarb/dA1LQOopWWYQCCArdjKIoupPalnzJiByspKfPzxx8jIyEBGRgaAuxedY2Nj8a9//QstLS3YvXs3goKCEBQU1O3vmbF27twJpVKJSZMmISUlBQEBATh58iQA7lrShQsXEBkZiSNHjuChhx7C/v37u3S+ewMUAJRKJWJjY/HMM8/gwIEDyMnJwR/+8If7XqRv+zVVIBDw/4PoKRRaBlBLq3+rqKhAXl4e3nzzTUydOhWjR4/G7du39co899xzaGxsxOHDh7F7927Exsby20aOHInW1lbk5OTw665evdruGMYaPXo0zpw5g7q6On7dTz/9BKFQiJEjR/Lrxo8fj4SEBKSnp2PMmDHYvXs3vy0gIACrVq3Cd999h+effx47d+6873nFYjE0Go1RdUxPT4efnx/+8Ic/YOLEiRgxYgR/I6CvodAyQNfSul1PLa3+yNnZGa6urtixYweuXr2KI0eOID4+Xq+MnZ0doqKisHbtWly6dAkxMTH8tlGjRiE8PByLFy9GZmYmcnJysHjxYtjY2Jj17FxsbCykUini4uJw/vx5HD16FK+99hrmzZsHDw8P5OfnIyEhAUqlEgUFBfjuu+9w5coVjB49Gg0NDVi2bBmOHTuGgoIC/PTTTzh16pTeNa+O+Pv7o7a2FmlpaSgvL0d9fX2HZUeMGIHCwkIkJyfj2rVr2Lx5c5dbcz2FQssAXUuropZaWv2RUChEcnIysrKyMGbMGKxatQobN25sVy42NhZnzpzBE088AV9fX71tn3/+OTw8PDB58mTMmjWLv5MmlUpNro+trS2+/fZbVFZW4pFHHsEvf/lLTJ06FVu3buW3X758GbNnz0ZAQAAWL16MpUuXYsmSJRCJRKioqMD8+fMREBCA6OhoTJ8+HW+//fZ9zztp0iS88sormDNnDgYNGoQNGzZ0WHbmzJlYtWoVli1bhnHjxiE9PR1r1641+bX2CmaGrVu3Mj8/PyaRSFhISAjLyMjotPyePXvYyJEjmUQiYWPGjGEHDx7U256YmMhGjhzJbG1tmZOTE5s6dSo7efKkXpmKigr24osvMgcHByaTydivf/1rVlNTY3Sdq6urGQBWXV1937IVtU3Mb80B5rfmAGtu1Rh9jgdNQ0MDu3jxImtoaLB0VSyuqKiIAWDff/+9pavSr3X2N2XsZ9TkllZKSgri4+ORmJiI7OxsBAcHIyIiosM+LOnp6YiJicHChQuRk5ODqKgoREVF4fz583yZgIAAbN26FefOncOJEyfg7++Pp59+Grdu3eLLxMbG4sKFC0hNTcWBAwdw/PhxvlNcd5PZWEP3LYB6xQ9MR44cwX//+1/k5+cjPT0dc+fOhb+/PyZPnmzpqhFTkzIkJIQtXbqU/12j0TAvLy+WlJRksHx0dDSLjIzUW6dQKNiSJUs6PIcucXX/V7t48SIDwE6dOsWX+eabb5hAIGA3b940qt6mtLQYY2z8O98xvzUH2OUStVHlH0QDuaV1+PBhFhgYyGxsbJi7uzuLiopi169fZ4wx9s9//pPZ2dkZXB566KFeq+O0adM6rMef/vSnXquHKbqjpWVSP63m5mZkZWUhISGBXycUChEeHg6lUmlwH6VS2e4iaERERIdPwDc3N2PHjh2QyWQIDg7mj+Hk5KTXtyQ8PBxCoRAZGRmYNWtWu+M0NTWhqenuNSm1Wm306wQAZ1trVNY133louvNpusmDJyIiAhERhvukzZw5EwqFwuC2+/VU706ffPIJGhoaDG5zcXHptXr0NpNCq7y8HBqNBh4eHnrrPTw8cPnyZYP7qFQqg+VVKpXeugMHDmDu3Lmor6+Hp6cnUlNT4ebmxh/D3d1dv+JWVnBxcWl3HJ2kpCSjLlZ2xNVOgmu36mh4GtKOg4MDHBws/z8yb29vS1fBIvrM3cMpU6YgNzcX6enpmDZtGqKjo81+1gsAEhISUF1dzS9FRUUm7e9MfbUI6ZNMCi03NzeIRCKUlpbqrS8tLTX4fBMAyOVyo8rb2dlh+PDhePTRR/Hpp5/CysoKn376KX+MtgHW2tqKysrKDs8rkUjg6Oiot5iCesUT0jeZFFpisRgTJkzQGwhMq9UiLS0NoaGhBvcJDQ1tN3BYampqh+XvPa7umlRoaCiqqqqQlZXFbz9y5Ai0Wm2H1xa6inrFE9I3mfzAdHx8POLi4jBx4kSEhIRg06ZNqKurw4IFCwBwQ3p4e3sjKSkJALBixQqEhYXhgw8+QGRkJJKTk3H69Gns2LEDADdsx5/+9CfMnDkTnp6eKC8vx7Zt23Dz5k288MILALjHIKZNm4ZFixZh+/btaGlpwbJlyzB37lx4eXl113uhx9lWDACopF7xhPQpJofWnDlzcOvWLaxbtw4qlQrjxo3D4cOH+YvthYWFEArvNuAmTZqE3bt3480338Qbb7yBESNG4KuvvsKYMWMAACKRCJcvX8Znn32G8vJyuLq64pFHHsGPP/6IwMBA/jhffPEFli1bhqlTp0IoFGL27NnYvHlzV19/h1zt74QWtbQI6VMEjA2MebLUajVkMhmqq6uNur51LK8ML+08hdGejvhmxRO9UMO+p7GxEfn5+RgyZIhZj68MZAKBAPv370dUVJSlq9KndPY3ZexntM/cPexraNJW0t9198QWADeMs0AgQFVVVbce1xQUWh2gSVsJ6ZsotDpAk7YawBjQXGeZhSa26LaJLZqamrB69Wp4e3vDzs4OCoUCx44d4/ctKCjAjBkz4OzsDDs7OwQGBuLQoUO4fv06pkyZAoAb/kcgEOCll14y6/3oChpuuQO6SVsbW7S4XdcCB2nvPZ7RZ7XUA+/2zN3a+3qjGBDb3b/cHTSxRccTWyxbtgwXL15EcnIyvLy8sH//fkybNg3nzp3DiBEjsHTpUjQ3N+P48eOws7PDxYsXYW9vDx8fH/z73//G7NmzkZeXB0dHR9jY2Jj1nnQFhVYnXO0kuFnVgIq6Jvi62lq6OsQENLGF4YktCgsLsXPnThQWFvLdhVavXo3Dhw9j586dePfdd1FYWIjZs2fzw08PHTqU31/3TKO7u3u3Xy8zFoVWJ5ztrHGzqoGGp9GxtuVaPJY6twloYgvDzp07B41Gg4CAAL31TU1N/ESsy5cvx29+8xt89913CA8Px+zZszF27FizztcT6JpWJ3SP8lTUUmgBAAQC7iuaJRaa2KJbJraora2FSCRCVlYWcnNz+eXSpUv4y1/+AgB4+eWX8fPPP2PevHk4d+4cJk6ciC1btnTba+0qCq1O6CZtpZZW/0ITW3AMTWwxfvx4aDQalJWVYfjw4XrLvV8jfXx88Morr2Dfvn347W9/i48//pg/JgCjJ8zoCRRaneBbWnUUWv0JTWzBMTSxRUBAAGJjYzF//nzs27cP+fn5yMzMRFJSEg4ePAgAWLlyJb799lvk5+cjOzsbR48e5c/n5+cHgUCAAwcO4NatW3p3QHtNjwxP2AeZOnIpY4xtSfsf81tzgP1ub24P1qzv6s8jl6amprLRo0cziUTCxo4dy44dO8YAsP379/NlDh06xACwyZMnt9u/uLiYTZ8+nUkkEubn58d2797N3N3d2fbt2406f9tznT17lk2ZMoVJpVLm4uLCFi1axM9xoFKpWFRUFPP09GRisZj5+fmxdevWMY1Gw5qamtjcuXOZj48PE4vFzMvLiy1btszo/yavvPIKc3V1ZQBYYmIiY4yx5uZmtm7dOubv78+sra2Zp6cnmzVrFjt79ixjjLFly5axYcOGMYlEwgYNGsTmzZvHysvL+WO+8847TC6XM4FAwOLi4oyqh053jFxKj/F0YndGId7Yfw7ho93xSdwjPVzDvoce47nrxo0b8PHxwffff4+pU6daujr9Vnc8xkN3Dztxd3ga+no40Bw5cgS1tbUICgpCSUkJXn/9dZrYoo+ga1qduDsQIIXWQNPS0oI33ngDgYGBmDVrFgYNGsR3NP3iiy9gb29vcLl3ZJKeNn369A7r8e677/ZaPXobtbQ6QS2tgYsmtui7KLQ6oWtpqRtb0aLRwlpEDVNCE1tYGn0KO0GTtnIGyL0a0gu642+JQqsTIqHg7rDLA/Arou6rTn19vYVrQh4Uur+lrnyNpq+H96GbtHUghpZIJIKTkxM/uoGtra1ZnSsJYYyhvr4eZWVlcHJygkgkMvtYFFr3oZu0dSCGFgD+0Y6uzEFJiI6Tk1OH0/4Zi0LrPu6OYDowQ0sgEMDT0xPu7u5oaaGZiYj5rK2tu9TC0qHQug96/pAjEom65Q+OkK6iC/H34TLAW1qE9DVmhda2bdvg7+8PqVQKhUKBzMzMTsvv3bsXo0aNglQqRVBQEA4dOsRva2lpwZo1axAUFAQ7Ozt4eXlh/vz5KC7WH2zO39+fH+tat6xfv96c6puEWlqE9C0mh1ZKSgri4+ORmJiI7OxsBAcHIyIiosMLtenp6YiJicHChQuRk5ODqKgoREVF4fz58wC4W6DZ2dlYu3YtsrOzsW/fPuTl5WHmzJntjvXOO++gpKSEX1577TVTq28yvqU1gPtpEdKnmDSuBGMsJCSELV26lP9do9EwLy8vlpSUZLB8dHQ0i4yM1FunUCjYkiVLOjxHZmYmA8AKCgr4dX5+fuzDDz80tbo8c4amYYyxY3llzG/NARbx4Q9mn5sQcn/GfkZNamk1NzcjKysL4eHh/DqhUIjw8HAolUqD+yiVSr3yAPdcV0flAfDTHrUdOH/9+vVwdXXF+PHjsXHjRrS2djy1V1NTE9Rqtd5iDpc7nUuppUVI32DS3cPy8nJoNBp4eHjorffw8MDly5cN7qNSqQyWV6lUBss3NjZizZo1iImJ0RtTZ/ny5Xj44Yfh4uKC9PR0JCQkoKSkBH/+858NHicpKQlvv/22KS/PIBf7uz3iGWPUuZIQC+tTXR5aWloQHR0Nxhg++ugjvW33Dpc7duxYiMViLFmyBElJSZBIJO2OlZCQoLePWq2Gj4+PyXXStbRaNAy1Ta00/yEhFmbS10M3NzeIRCKUlpbqrS8tLe2wl6tcLjeqvC6wCgoKkJqaet/RRRUKBVpbW3H9+nWD2yUSCRwdHfUWc9iIRbCx5vonDdRe8YT0JSaFllgsxoQJE5CWlsav02q1SEtLQ2hoqMF9QkND9coDQGpqql55XWBduXIF33//PT//Wmdyc3MhFArh7u5uykswi4vdwH1ompC+xuSvh/Hx8YiLi8PEiRMREhKCTZs2oa6uDgsWLADATSfu7e2NpKQkAMCKFSsQFhaGDz74AJGRkUhOTsbp06exY8cOAFxg/fKXv0R2djYOHDgAjUbDX+9ycXGBWCyGUqlERkYGpkyZAgcHByiVSqxatQq/+tWvzJ5A0xQudmLcrGqg0CKkLzDn1uSWLVuYr68vE4vFLCQkhJ08eZLfFhYW1m6Gjj179rCAgAAmFotZYGAgO3jwIL8tPz+fATC4HD16lDHGWFZWFlMoFEwmkzGpVMpGjx7N3n33XdbY2Gh0nc3t8sAYY/M+zWB+aw6wPacKTd6XEGIcmo2nDXNm49FZlZKL/Tk3kTB9FJaEmTcdOSGkc8Z+RunZQyPwAwFSXy1CLI5Cywiuur5atRRahFgahZYRnKlXPCF9BoWWEXRdHmikB0Isj0LLCLrQojG1CLE8Ci0jUEuLkL6DQssIutCquTNpKyHEcii0jCCzsYZQN2krtbYIsSgKLSOIhAI4UV8tQvoECi0j8Q9NU18tQiyKQstILtTSIqRPoNAyEg1PQ0jfQKFlJGcKLUL6BAotI7lSaBHSJ1BoGYlaWoT0DRRaRqKWFiF9A4WWkailRUjfQKFlJGppEdI3UGgZSdfSul3PTdpKCLEMCi0j3Ttpa01Tq4VrQ8jARaFlpHsnbaWHpgmxHAotE9C4WoRYnlmhtW3bNvj7+0MqlUKhUCAzM7PT8nv37sWoUaMglUoRFBSEQ4cO8dtaWlqwZs0aBAUFwc7ODl5eXpg/fz6Ki4v1jlFZWYnY2Fg4OjrCyckJCxcuRG1trTnVNxuNYEqI5ZkcWikpKYiPj0diYiKys7MRHByMiIgIlJWVGSyfnp6OmJgYLFy4EDk5OYiKikJUVBTOnz8PAKivr0d2djbWrl2L7Oxs7Nu3D3l5eZg5c6becWJjY3HhwgWkpqbiwIEDOH78OBYvXmzGSzYftbQI6QNMnQU2JCSELV26lP9do9EwLy8vlpSUZLB8dHQ0i4yM1FunUCjYkiVLOjxHZmYmA8AKCgoYY4xdvHiRAWCnTp3iy3zzzTdMIBCwmzdvGlXvrswwrbMyOYf5rTnAth+7avYxCCGGGfsZNaml1dzcjKysLISHh/PrhEIhwsPDoVQqDe6jVCr1ygNAREREh+UBoLq6GgKBAE5OTvwxnJycMHHiRL5MeHg4hEIhMjIyDB6jqakJarVab+kqGumBEMszKbTKy8uh0Wjg4eGht97DwwMqlcrgPiqVyqTyjY2NWLNmDWJiYvipsVUqFdzd3fXKWVlZwcXFpcPjJCUlQSaT8YuPj49Rr7EzFFqEWF6funvY0tKC6OhoMMbw0UcfdelYCQkJqK6u5peioqIu149CixDLszKlsJubG0QiEUpLS/XWl5aWQi6XG9xHLpcbVV4XWAUFBThy5AjfytIdo+2F/tbWVlRWVnZ4XolEAolEYvRrM4YzjV5KiMWZ1NISi8WYMGEC0tLS+HVarRZpaWkIDQ01uE9oaKheeQBITU3VK68LrCtXruD777+Hq6tru2NUVVUhKyuLX3fkyBFotVooFApTXkKXuNpTS4sQizP1Cn9ycjKTSCRs165d7OLFi2zx4sXMycmJqVQqxhhj8+bNY7///e/58j/99BOzsrJi77//Prt06RJLTExk1tbW7Ny5c4wxxpqbm9nMmTPZ4MGDWW5uLispKeGXpqYm/jjTpk1j48ePZxkZGezEiRNsxIgRLCYmxuh6d8fdwyulNcxvzQE2JvGw2ccghBhm7GfU5NBijLEtW7YwX19fJhaLWUhICDt58iS/LSwsjMXFxemV37NnDwsICGBisZgFBgaygwcP8tvy8/MZAIPL0aNH+XIVFRUsJiaG2dvbM0dHR7ZgwQJWU1NjdJ27I7Qqa5uY35oDzG/NAdbUojH7OISQ9oz9jAoYGxhDFqjVashkMlRXV+tdLzOFVssw/A+HoGVA5htT4e4o7eZaEjJwGfsZ7VN3D/s6oVDAX4ynXvGEWAaFlomc6flDQiyKQstEfF8t6vZAiEVQaJmIn2maWlqEWASFlolcqK8WIRZFoWUiamkRYlkUWiai5w8JsSwKLRNRaBFiWRRaJqLQIsSyKLRMRKFFiGVRaJnIhSZtJcSiKLRMpAstmrSVEMug0DKR1FoEWzFN2kqIpVBomYEemibEcii0zKAbwZRaWoT0PgotM1BLixDLodAygysNT0OIxVBomcGZ+moRYjEUWmagDqaEWA6FlhkotAixHAotM9DopYRYDoWWGailRYjlmBVa27Ztg7+/P6RSKRQKBTIzMzstv3fvXowaNQpSqRRBQUE4dOiQ3vZ9+/bh6aefhqurKwQCAXJzc9sd48knn4RAINBbXnnlFXOq32UUWoRYjsmhlZKSgvj4eCQmJiI7OxvBwcGIiIhAWVmZwfLp6emIiYnBwoULkZOTg6ioKERFReH8+fN8mbq6Ojz++ON47733Oj33okWLUFJSwi8bNmwwtfrdQjd6aU1jK5pbtRapAyEDlqmzwIaEhLClS5fyv2s0Gubl5cWSkpIMlo+OjmaRkZF66xQKBVuyZEm7srrZpnNyctptCwsLYytWrDC6no2Njay6uppfioqKujzDtI5Go2VDfs/NNF1a3dDl4xFCjJ9h2qSWVnNzM7KyshAeHs6vEwqFCA8Ph1KpNLiPUqnUKw8AERERHZbvzBdffAE3NzeMGTMGCQkJqK+v77BsUlISZDIZv/j4+Jh8vo7QpK2EWI6VKYXLy8uh0Wjg4eGht97DwwOXL182uI9KpTJYXqVSmVTRF198EX5+fvDy8sLZs2exZs0a5OXlYd++fQbLJyQkID4+nv9drVZ3a3C52IlRUddMveIJ6WUmhZYlLV68mP93UFAQPD09MXXqVFy7dg3Dhg1rV14ikUAikfRYfXS94qmlRUjvMunroZubG0QiEUpLS/XWl5aWQi6XG9xHLpebVN5YCoUCAHD16tUuHcdcrveMYEoI6T0mhZZYLMaECROQlpbGr9NqtUhLS0NoaKjBfUJDQ/XKA0BqamqH5Y2l6xbh6enZpeOYi29p1VJoEdKbTP56GB8fj7i4OEycOBEhISHYtGkT6urqsGDBAgDA/Pnz4e3tjaSkJADAihUrEBYWhg8++ACRkZFITk7G6dOnsWPHDv6YlZWVKCwsRHFxMQAgLy8PANdKk8vluHbtGnbv3o1nnnkGrq6uOHv2LFatWoXJkydj7NixXX4TzEEtLUIsxJxbk1u2bGG+vr5MLBazkJAQdvLkSX5bWFgYi4uL0yu/Z88eFhAQwMRiMQsMDGQHDx7U275z504GoN2SmJjIGGOssLCQTZ48mbm4uDCJRMKGDx/Ofve735nUfcHY26nG+vTHn5nfmgPs1S+yuuV4hAx0xn5GBYwNjCll1Go1ZDIZqqur4ejo2OXj/Sf3JlYk52LSMFfsXvRoN9SQkIHN2M8oPXtoJl0/LXqUh5DeRaFlJnr+kBDLoNAyE03aSohlUGiZiSZtJcQyKLTMdO+krZXUV4uQXkOh1QX8xXjqq0VIr6HQ6gLdpK3U0iKk91BodQG1tAjpfRRaXeBK3R4I6XUUWl3gTDNNE9LrKLS6wIXG1CKk11FodYELtbQI6XUUWl1ALS1Ceh+FVhe40JhahPQ6Cq0u4B+apn5ahPQaCq0u4CdtbaJJWwnpLRRaXSCzsYZQwP2bviIS0jsotLrg3klbqYMpIb2DQquLaDBAQnoXhVYXOVNoEdKrKLS6iJ4/JKR3mRVa27Ztg7+/P6RSKRQKBTIzMzstv3fvXowaNQpSqRRBQUE4dOiQ3vZ9+/bh6aefhqurKwQCAT8R670aGxuxdOlSuLq6wt7eHrNnz243c7UlUEuLkN5lcmilpKQgPj4eiYmJyM7ORnBwMCIiIlBWVmawfHp6OmJiYrBw4ULk5OQgKioKUVFROH/+PF+mrq4Ojz/+ON57770Oz7tq1Sp8/fXX2Lt3L3744QcUFxfj+eefN7X63Y5aWoT0MlMnVAwJCWFLly7lf9doNMzLy4slJSUZLB8dHc0iIyP11ikUCrZkyZJ2ZfPz8xkAlpOTo7e+qqqKWVtbs7179/LrLl26xAAwpVJpVL27e7JWHZq0lZDuYexn1KSWVnNzM7KyshAeHs6vEwqFCA8Ph1KpNLiPUqnUKw8AERERHZY3JCsrCy0tLXrHGTVqFHx9fTs8TlNTE9Rqtd7SE2j0UkJ6l0mhVV5eDo1GAw8PD731Hh4eUKlUBvdRqVQmle/oGGKxGE5OTkYfJykpCTKZjF98fHyMPp8pdP20qHMpIb3jgb17mJCQgOrqan4pKirqkfPQSA+E9C4rUwq7ublBJBK1u2tXWloKuVxucB+5XG5S+Y6O0dzcjKqqKr3WVmfHkUgkkEgkRp/DXPeOqcUYg0Ag6PFzEjKQmdTSEovFmDBhAtLS0vh1Wq0WaWlpCA0NNbhPaGioXnkASE1N7bC8IRMmTIC1tbXecfLy8lBYWGjScXqCLrRatQzqRpq0lZCeZlJLCwDi4+MRFxeHiRMnIiQkBJs2bUJdXR0WLFgAAJg/fz68vb2RlJQEAFixYgXCwsLwwQcfIDIyEsnJyTh9+jR27NjBH7OyshKFhYUoLi4GwAUSwLWw5HI5ZDIZFi5ciPj4eLi4uMDR0RGvvfYaQkND8eijj3b5TegK3aSt9c0a3K5rhszG2qL1IeSBZ86tyS1btjBfX18mFotZSEgIO3nyJL8tLCyMxcXF6ZXfs2cPCwgIYGKxmAUGBrKDBw/qbd+5cycD0G5JTEzkyzQ0NLBXX32VOTs7M1tbWzZr1ixWUlJidJ17qssDY4w9tj6N+a05wE5fr+z2YxMyUBj7GRUwxpgFM7PXqNVqyGQyVFdXw9HRsVuPPXPrCZy9UY1P5k9E+EMe99+BENKOsZ/RB/buYW+ikR4I6T0UWt3AhWaaJqTXUGh1A2ppEdJ7KLS6AY30QEjvodDqBjTSAyG9h0KrG1BLi5DeQ6HVDailRUjvodDqBs73PH9ICOlZFFrdwM2OezC7pqkVBRV1Fq4NIQ82Cq1uILO1RuhQVwDAypRctGhotmlCegqFVjfZ+MJYOEitkFNYhS1pVyxdHUIeWBRa3WSwsy3+NCsIALD16FVk5ldauEaEPJgotLrRzGAvPP+wN7QMWJWSi+qGFktXiZAHDoVWN3vnuTHwdbHFzaoGrP3qPAbIIBqE9BoKrW5mL7HCX+aOg0gowH/PFGN/zk1LV4mQBwqFliFNtV3afbyvM1ZOHQEAWPefCyisqO+OWhFCQKHVXn0lsOVh4HAC0Fht9mFenTIcj/g7o7apFStSctBK3SAI6RYUWm1d/A9QWwqc/CuwZQKQ809Aa3rgiIQCfDhnHN8NYvORqz1QWUIGHgqttiYuAH61D3AdAdTdAv6zFPj0F8DNbJMPpdcN4sgVnLpO3SAI6SoKLUOGTwV+kw784o+A2B64eRr4+Cngv8uBugqTDnVvN4iVydQNgpCuotDqiJUYeGw5sOw0EBQNgAHZn3HXuzI/BjTGz3F4bzeIdf8533N1JmQAoNC6H0dPYPbHwIJvAI8xQGMVcGg1sONJoEBp1CHsJVbYdKcbxH9yi7E/50aPVpmQB5lZobVt2zb4+/tDKpVCoVAgMzOz0/J79+7FqFGjIJVKERQUhEOHDultZ4xh3bp18PT0hI2NDcLDw3Hliv7ze/7+/hAIBHrL+vXrzam+efwmAYt/AJ55H5DKgNJzwM5pwL8XAeqS++7+sK8zVtzpBrH2K+oGQYi5TA6tlJQUxMfHIzExEdnZ2QgODkZERATKysoMlk9PT0dMTAwWLlyInJwcREVFISoqCufP3/2atGHDBmzevBnbt29HRkYG7OzsEBERgcbGRr1jvfPOOygpKeGX1157zdTqd43ICghZBLyWDTwcB0AAnNsDbJ0I/PQXoLXz8bSW3tMNYiV1gyDEPKbOAhsSEsKWLl3K/67RaJiXlxdLSkoyWD46OppFRkbqrVMoFGzJkiWMMca0Wi2Ty+Vs48aN/PaqqiomkUjYl19+ya/z8/NjH374oanV5fXIDNM3TjO24ynGEh25ZVMwY6d3MdbS2OEuRZV1bEziYea35gD783d5pp2v/Cpj/3mNsb+MYyz7nybtWtvYwrRarWnnI6QXGfsZNaml1dzcjKysLISHh/PrhEIhwsPDoVQavr6jVCr1ygNAREQEXz4/Px8qlUqvjEwmg0KhaHfM9evXw9XVFePHj8fGjRvR2trxxfCmpiao1Wq9pdt5TwAWpgLPbQPsBgG384GvlwN/GQcotxnsWX9vN4gtR67gtDHdIIpzgD1xXL+x7M+Ayp+B/7wKfPUq0Nzx10zGGE5cKceCnZkITPwWCz87jSqam5H0cyaFVnl5OTQaDTw89Kd+9/DwgEqlMriPSqXqtLzu5/2OuXz5ciQnJ+Po0aNYsmQJ3n33Xbz++usd1jUpKQkymYxffHx8jH+hphAKgfG/ApbnAk//CXDwBGqKgW/fADaNAY69x/Wyv8e93SCWf5mDPaeL2neFYAzIPw58HsVd9L/4FQAGBEwDJi0HBEIg9wvgk6nArf/p7drUqsGe00WY/pcf8atPM3A07xYA4MjlMszYegIXis3v6U+IpVlZugLGio+P5/89duxYiMViLFmyBElJSZBIJO3KJyQk6O2jVqt7LrgAQGIPTFrGXfM68yVwYhPX8jr2LpC+GZjwEhC6jLsbCeDtmYHIKriNgop6vP6vs3hz/3lMDnDDs0FyRFhlwSZjM3Azizu2QAQE/RJ4bAXgEcitGx4O/PtloOwiF2oz/oKKoTPxRUYhPlcWoLy2CQBgKxYheqIPJge4IfG/F1BU2YDn/5qOpOeD8PzDg3vu/SCkh5gUWm5ubhCJRCgtLdVbX1paCrlcbnAfuVzeaXndz9LSUnh6euqVGTduXId1USgUaG1txfXr1zFy5Mh22yUSicEw63FWEi6gxv2Kax2d+BAoPQ8otwKZO4BxLwKPrYCDy1D8+zeTkHKqCF+fKcY11W04/28Pgq59DRshdzdSI5SAjZ8Hq8eXA85++ucZGga8cgL490Lg+o/AvpeRqg3HtuZfoQlieMqkeGmSP+Y+4guZrTUA7g7mypRcHMu7hfg9Z5BTWIW1zz4EsRX1fCH9h0l/rWKxGBMmTEBaWhq/TqvVIi0tDaGhoQb3CQ0N1SsPAKmpqXz5IUOGQC6X65VRq9XIyMjo8JgAkJubC6FQCHd3d1NeQu8RWXGto1dOAC/uBXweBTTNQNYu7trUvxbCrfYKlk6S47DiPC66rsFG6x0YJiyBmtliS2sUQuo3YeypX2DFt5VIvViKplYNf3jGGH5UCfGS5g/Y3BoFLRNgrvB7fGP3Dj551hnHX5+CJWHD+MACACdbMf4e9wiW3+l68Y+TBZizQwlVdWPb2hPSZwkYM22UupSUFMTFxeFvf/sbQkJCsGnTJuzZsweXL1+Gh4cH5s+fD29vbyQlJQHgujyEhYVh/fr1iIyMRHJyMt59911kZ2djzJgxAID33nsP69evx2effYYhQ4Zg7dq1OHv2LC5evAipVAqlUomMjAxMmTIFDg4OUCqVWLVqFaZPn47PPvvMqHqr1WrIZDJUV1fD0dHRxLepmxSkAz9+AFz9/u46sQPQXMP9294D7NGluOQ9G/+9XIuvzxTjZlUDX9RBaoWIQDke8nREyqki5JVy+wkEwEr/QrxauQHWTZWAxBF4bivw0HMdViXtUilWpeRC3dgKN3sxtsQ8jNBhrp3XX6sBmmruLGrD/25UAxIHwCcE8BwHWEvNfbfIAGPsZ9Tk0AKArVu3YuPGjVCpVBg3bhw2b94MhUIBAHjyySfh7++PXbt28eX37t2LN998E9evX8eIESOwYcMGPPPMM/x2xhgSExOxY8cOVFVV4fHHH8df//pXBAQEAACys7Px6quv4vLly2hqasKQIUMwb948xMfHG/0VsE+Elk7JGeDHP3MjSoABLkO561Vj5+p9yBljyC2qwtdnSnDwXDFK1U16h9Fdr1rwmD/8XO2A6pvAv34NFJ3kCihe4Z6ftBIbrEZBRR2W/CMLl1U1EAkF+P20UXg5WALBzSzgxinuIfHa0ruB1GLi9GgiMeA1HvBRAL6Pcj/t3Ew7BhkwejS0+qM+FVo6lT8DNSruwywUdVpUq2U4db0SB86W4LJKjfDRHpgb4guZjbV+QU0LcOSPXGdXgOuW8cud7a+JAUBzPZqKcnD4269hVZKF8cKr8BIY0QXDSsq1pvjF8c7iwN2QqFEBRRncKBltuQ7nvir7KrifbiO4piIZ8Ci02uiTodWT8r4B9r/CPSsplQFR24FBI7kWlG4pvQBo9fu6aZgA+SJ/DBr9GGTDQwHnIW3CyaHDlpsexrhQLsoACk9yP29dbl/OxoUL7aFh3A0Ma5tuefmk/6HQamPAhRYA3C4A/rXgbtcJQ+w9gMGPAIMnIs96JJZ8r8H1GiHsJVZ4/4WxmDbGs+N9TVVfyYWlLsRuZgGt99wEcAsAoj4CBk/svnOSfoNCq40BGVoA9zxk6jog4yNAJAE8g/mQwuBHANlgva9nZTWNWLY7h5+3MTLIE48Nd0PIEGcMG2QPQXd+lWttBlRnuRsUyq3c9TOBEHhsJfDk77nuI6RjjAEZfwNObgMCpgOTVwP2ffRuuhEotNoYsKGl01gNWNkY9dWuRaPFe99cxicn8vXWu9iJ8Yi/Mx7xd0HIEBc85OkIK1E39fGqrwS+WcM9gA4A7g9xrS6vcd1z/AdNUw3w39eAC/vvrrO2BR79DffEhI2TxapmLgqtNgZ8aJkhu/A2juXdQmZ+BXIKq9DUqj8qhZ1YhIf9nBHi74JHhrhgnI8TpNad31C4r0tfA1+vBOrLuScBJq8Gnlht3HW0gaLsEpAyD6i4AgituJD6+RhQfGdIcKmMa60qlgBiO0vW1CQUWm1QaHVNc6sW525WIzO/Eqeuc0tNo/5FfLFIiLGDZQgaLMNgZ1t4O9lgsDO3yGysjf9qWVcOHPztnectAciDuFaXPKh7X1R/dHYv91B+Sz3g4AVEf8b1iWMMuHyQu3Osu+Fh7wFM/h03jFI/CH0KrTYotLqXRsuQp6rBqeuVyLxeiVP5lSiraeqwvJ1YBG9nG3g72dz5acv/7uNsAzd7CYTCNqF2/t9ceDXcBoTWQNga4PGVgMja4Dm6HWPAjdPAmd1cq2/ok8CQJ7iWTG9rbeIewj/1Cff70CeB2Z+27/em1QDn9gJH3wWqCrh1Tr7AkwnA2Dn37VpjSRRabVBo9SzGGAoq6pF5vRJXSmtws6oBN2834GZVA8pr7z8cjo21CEGDZRjv44Txvk4Y7+sMD0cpUFMKHFgF5B3kCnqOA2ZtB9xH99yLaWkAzv0LOPUx1xH4XgIRdwNj2BRg2FOA18PcI1s9qaoI2Bt39y7w5N9xIdRZALU2c8MYHd/I3eAAALeRwFNvAqNn9Mm+cRRabVBoWU5ji0YvxHQ/b9yux83bDVCpG6E18FfoKZNivK8Txg2WIbz1Bww59TYEjdVcT/spb3DXcrqz5VCZD5z+lJvrsuE2t04kAcbM5jrNXjsCVLSZv1Ii41pfuhBzGdp99QG4R77+vQhoqASkTsDzO4CACOP3b67nHtQ/8SHXZw/gnlKYug4YOqVPhReFVhsUWn1Xi0aL6+V1yCmqQk5hFXKLqpCnUrcLMk/hbWy224lHWk4DABo9Hob04bnchCMegebdMdNquWA49TFwJRXAnZM6+QITFwLj5wF29zyTWVUIXDvKBVj+D3fDTcfJjwuvYVMA/ycAWxfT66Sr1/ENwLH1XJ08xwHRnxt+ssEYDVVctxLlX+8+jiWScF91pY53fsq4DsS6f7ddJI7c0Eoynx75mkmh1QaFVv9S19SKszeqkVtUhZzC28gpqsKtmiYADC+IfsBaq3/AUdCgv5PMlwsv+Zg7QTaGa/kIDXTLqK/kWlSnPwVuX7+7fng48MgiYMQv7v/B1GqAklwuwK4d4575bPOEARy9ue4bHoF36vQQNxFwZxfG6yqAfYuAa3dGPpmwAJi2vnsePq+9BZz4M3DqU0DT8TXITokk3PvqNpx7LMt1BPfTbYT5IQ0KrXYotPo3xhiKqxuRU3gbuYVVKLx+BUGlX2E08jFKWITBgnLDO1rbcqGhCzInP+DSf7hrVrre+FIZ16Ka+GvAdZj5lWyqAa7/BPx8FJqraRBVXDFcTmjNPVLFh9mdxcGTe0h9bxxQXcT1q3v2Q2BcjPl16khzPdetpFHN9eHTLU33/l7Vfrv6JjfEUkdsXO4GmOvwu/92GXbfO5gUWm1QaD14blY1YPuxa0g5VQSpRo1RgiI87XoLke4VkDdeg6DsEtDa0PEB5EFcqyroBUBsa3Y9Gpo1uFhSjTNF1Th3sxpnblTh51t1cEA9QuxUeNyhFEHWN+Hfmg/n2qsQtdQYPpCNMzevgLaF+5DP+cfdkWr7Cq2GC9Tyq1w/sYqrQPmdn+qbHe83/7/c86WdoNBqg0LrwaWqbsT2H67hy8xCvgNs8GAZlk8ZiqfcayAovcCNHlt6gftweY3nwsonxOQL0U2tGuSpanDmRjXO3ajC2RvVuFJWC42BOwkCAddrQh+Dj7ACU5zK8IitCqMEhfBqugbbmusQsDuDPI6ewU2WYomuFV3RXAdUXLsTZtfuhNkVLuCWZQKOXp3uTqHVBoXWg69M3Ygdx3/GPzMK0NjChVeglyOWTx2BX4z2aN8PrBMaLUNxVQMKK+txvaIOF4vVOHezGpdLatBsYL7KQQ4SBA+WIcjbCWN9ZAjylsFObIX/ldbgskqNSyU1uFSixmVVTftJTABI0IzxNmUY5iqBwPthDHd3uLPYw8NR0r3PfPY2XcTc5zVQaLVBoTVwlNc24ZMf8/G58jrqm7nWyyi5A5ZPHYFpgXI+vBpbNCiqrEdBRT0KKutRUFGHgop6FFbW48bterRoDH80nGytMXawE8Z6yzB2sAxjBzsZHSyMMajUjbhUwgXZZRUXZj/fqjXY7QMAHCRWGOpuj+GD7DHc/e7i42zTfc9+9gEUWm1QaA08lXXN+PuJfOxKv47aJu6u3nB3e7jYiVFYUQ+VuvOx8cUiIQa72MDPxRYjPBwwdrAMwYOdMNjZpttbPo0tGlwtq8VlVQ2ultXi2q1aXCurRUFlvcGvnrr6DXGzw9BBdnC2E8NBagVHqTXsJVZwkFrBQWp95ye33kFqBXuJVZ8NOgqtNii0Bq7q+hb8/ad8/P2n/HbPSzpIrODrags/V1v4udrBz8X2zu92kDtKITLhK2VPaGrVoKCiHlfLavkw0/3UfQU2lY21CI42VpA7SuHjYgvfexYfF1t4yqQWCTYKrTYotIi6sQWHz6kgsRbC14ULJmdbEx7k7kO0WoabVQ24eqsW18vroG5oRU1jC2oaW1HTxP1UN95dV9vYioYWzf0PDMBKKIC3sw0fYveGmrujBI5S666P5mEAhVYbFFpkoGvRaFHb2IqaxlZUN7SguLoBRZXcNTzdcqOyweCNhrYkVkLIbKzhaGMN2Z3FUWp199/3bHOUWmOMtyMcpJ0/6G7sZ7TfzDBNCOkaa5EQznZiONtxnTyDBrfvUqHVMpTWNKLwzg0J/VBrQEVdExgDmlq1KKtp6nRkj3vtf3USxvs6d8vroNAihPCEQgE8ZTbwlNlAMbT9PJhaLUNNUyvUDS2obmi5+7OR+8mta+X/rdvmYteN43kxM2zdupX5+fkxiUTCQkJCWEZGRqfl9+zZw0aOHMkkEgkbM2YMO3jwoN52rVbL1q5dy+RyOZNKpWzq1Knsf//7n16ZiooK9uKLLzIHBwcmk8nYr3/9a1ZTU2N0naurqxkAVl1dbfwLJYT0GmM/oybfIkhJSUF8fDwSExORnZ2N4OBgREREoKyszGD59PR0xMTEYOHChcjJyUFUVBSioqJw/vx5vsyGDRuwefNmbN++HRkZGbCzs0NERAQaG+/eko6NjcWFCxeQmpqKAwcO4Pjx41i8eLHJIU0I6edMTcOQkBC2dOlS/neNRsO8vLxYUlKSwfLR0dEsMjJSb51CoWBLlixhjHGtLLlczjZu3Mhvr6qqYhKJhH355ZeMMcYuXrzIALBTp07xZb755hsmEAjYzZs3jao3tbQI6dt6pKXV3NyMrKwshIeH8+uEQiHCw8OhVCoN7qNUKvXKA0BERARfPj8/HyqVSq+MTCaDQqHgyyiVSjg5OWHixLvz4YWHh0MoFCIjI8PgeZuamqBWq/UWQkj/Z1JolZeXQ6PRwMPDQ2+9h4cHVCqVwX1UKlWn5XU/71fG3V1/PjcrKyu4uLh0eN6kpCTIZDJ+8fHxMfJVEkL6sr7Zn78bJCQkoLq6ml+KioosXSVCSDcwKbTc3NwgEolQWlqqt760tBRyudzgPnK5vNPyup/3K9P2Qn9raysqKys7PK9EIoGjo6PeQgjp/0wKLbFYjAkTJiAtLY1fp9VqkZaWhtDQUIP7hIaG6pUHgNTUVL78kCFDIJfL9cqo1WpkZGTwZUJDQ1FVVYWsrCy+zJEjR6DVaqFQKEx5CYSQfs7kzqXx8fGIi4vDxIkTERISgk2bNqGurg4LFiwAAMyfPx/e3t5ISkoCAKxYsQJhYWH44IMPEBkZieTkZJw+fRo7duwAAAgEAqxcuRL/93//hxEjRmDIkCFYu3YtvLy8EBUVBQAYPXo0pk2bhkWLFmH79u1oaWnBsmXLMHfuXHh5dT6wmA6787QSXZAnpG/SfTbZ/Z4sNOfW5JYtW5ivry8Ti8UsJCSEnTx5kt8WFhbG4uLi9Mrv2bOHBQQEMLFYzAIDAzvsXOrh4cEkEgmbOnUqy8vL0ytTUVHBYmJimL29PXN0dGQLFiwwqXNpUVERAzfVCi200NKHl6Kiok4/ywPmgWmtVovi4mI4ODjc96l+tVoNHx8fFBUVDehrYfQ+cOh94PT0+8AYQ01NDby8vCA0NIPSHQPm2UOhUIjBgwebtA9dwOfQ+8Ch94HTk++DTCa7b5kHtssDIeTBRKFFCOlXKLQMkEgkSExMhEQisXRVLIreBw69D5y+8j4MmAvxhJAHA7W0CCH9CoUWIaRfodAihPQrFFqEkH6FQosQ0q9QaLWxbds2+Pv7QyqVQqFQIDMz09JV6nVvvfUWBAKB3jJq1ChLV6vHHT9+HDNmzICXlxcEAgG++uorve2MMaxbtw6enp6wsbFBeHg4rly5YpnK9qD7vQ8vvfRSu7+PadOm9Vr9KLTuYeqkHQ+ywMBAlJSU8MuJEycsXaUeV1dXh+DgYGzbts3gdmMmYHkQ3O99AIBp06bp/X18+eWXvVdBo4dJGABMnbTjQZWYmMiCg4MtXQ2LAsD279/P/27MBCwPorbvA2OMxcXFseeee84i9WHMjCnEHlTmTNrxILty5Qq8vLwwdOhQxMbGorCw0NJVsihjJmAZSI4dOwZ3d3eMHDkSv/nNb1BRUdFr56bQusOcSTseVAqFArt27cLhw4fx0UcfIT8/H0888QRqamosXTWLMWYCloFi2rRp+Pzzz5GWlob33nsPP/zwA6ZPnw6NRtMr5x8wQ9MQ402fPp3/99ixY6FQKODn54c9e/Zg4cKFFqwZ6Qvmzp3L/zsoKAhjx47FsGHDcOzYMUydOrXHz08trTvMmbRjoHByckJAQACuXr1q6apYjDETsAxUQ4cOhZubW6/9fVBo3WHOpB0DRW1tLa5duwZPT09LV8VijJmAZaC6ceMGKioqeu3vg74e3uN+k3YMFKtXr8aMGTPg5+eH4uJiJCYmQiQSISYmxtJV61G1tbV6rYX8/Hzk5ubCxcUFvr6+952A5UHR2fvg4uKCt99+G7Nnz4ZcLse1a9fw+uuvY/jw4YiIiOidClrsvmUf1dmkHQPFnDlzmKenJxOLxczb25vNmTOHXb161dLV6nFHjx41ONGCbqIWYyZgeRB09j7U19ezp59+mg0aNIhZW1szPz8/tmjRIqZSqXqtfjSeFiGkX6FrWoSQfoVCixDSr1BoEUL6FQotQki/QqFFCOlXKLQIIf0KhRYhpF+h0CKE9CsUWoSQfoVCixDSr1BoEUL6lf8HXCPIJJjQZeQAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.769360Z", + "iopub.status.busy": "2024-03-22T14:46:40.768952Z", + "iopub.status.idle": "2024-03-22T14:48:58.416842Z", + "shell.execute_reply": "2024-03-22T14:48:58.416011Z" + }, + "papermill": { + "duration": 137.673684, + "end_time": "2024-03-22T14:48:58.419405", + "exception": false, + "start_time": "2024-03-22T14:46:40.745721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:48:58.458680Z", + "iopub.status.busy": "2024-03-22T14:48:58.458035Z", + "iopub.status.idle": "2024-03-22T14:48:58.478208Z", + "shell.execute_reply": "2024-03-22T14:48:58.477275Z" + }, + "papermill": { + "duration": 0.042235, + "end_time": "2024-03-22T14:48:58.480335", + "exception": false, + "start_time": "2024-03-22T14:48:58.438100", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0011460.0139830.0009296.5465270.0180460.6029330.056620.0000092.3627930.0182690.8310970.030480.1488480.0000038.90932
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.001146 0.013983 0.000929 6.546527 0.018046 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.602933 0.05662 0.000009 2.362793 0.018269 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.831097 0.03048 0.148848 0.000003 8.90932 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:48:58.517267Z", + "iopub.status.busy": "2024-03-22T14:48:58.516680Z", + "iopub.status.idle": "2024-03-22T14:48:58.914736Z", + "shell.execute_reply": "2024-03-22T14:48:58.913883Z" + }, + "papermill": { + "duration": 0.418878, + "end_time": "2024-03-22T14:48:58.917032", + "exception": false, + "start_time": "2024-03-22T14:48:58.498154", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:48:58.956992Z", + "iopub.status.busy": "2024-03-22T14:48:58.955979Z", + "iopub.status.idle": "2024-03-22T14:51:25.628364Z", + "shell.execute_reply": "2024-03-22T14:51:25.627328Z" + }, + "papermill": { + "duration": 146.695763, + "end_time": "2024-03-22T14:51:25.631495", + "exception": false, + "start_time": "2024-03-22T14:48:58.935732", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:25.673508Z", + "iopub.status.busy": "2024-03-22T14:51:25.673079Z", + "iopub.status.idle": "2024-03-22T14:51:25.702166Z", + "shell.execute_reply": "2024-03-22T14:51:25.701069Z" + }, + "papermill": { + "duration": 0.053561, + "end_time": "2024-03-22T14:51:25.705052", + "exception": false, + "start_time": "2024-03-22T14:51:25.651491", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:25.743874Z", + "iopub.status.busy": "2024-03-22T14:51:25.743504Z", + "iopub.status.idle": "2024-03-22T14:51:25.749616Z", + "shell.execute_reply": "2024-03-22T14:51:25.748693Z" + }, + "papermill": { + "duration": 0.027677, + "end_time": "2024-03-22T14:51:25.751678", + "exception": false, + "start_time": "2024-03-22T14:51:25.724001", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.03444043153561541}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:25.792667Z", + "iopub.status.busy": "2024-03-22T14:51:25.792298Z", + "iopub.status.idle": "2024-03-22T14:51:26.187177Z", + "shell.execute_reply": "2024-03-22T14:51:26.186182Z" + }, + "papermill": { + "duration": 0.417606, + "end_time": "2024-03-22T14:51:26.189398", + "exception": false, + "start_time": "2024-03-22T14:51:25.771792", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA910lEQVR4nO3deXhU9b0/8PdZZs0sSchOEhIIi7JGBETUgFBRrEptvVapggjaFqo2ch/N7S0qrcVaF2jrQ729JZGnImp/uNxaVIqCSgUF2QQLBEMIISEhyySzz5zz/f1xJgNDZpLJZJIzCZ+XzzwyM98585ksn3z3L8cYYyCEEBXxagdACCGUiAghqqNERAhRHSUiQojqKBERQlRHiYgQojpKRIQQ1VEiIoSojhIRIUR1lIhIlyoqKsBxHE6ePKl2KGQQo0RE4m7jxo1Ys2aN2mGQAYQSEYk7SkSkpygREUJUR4mI9NiWLVtQUlICs9kMi8WCKVOmYOPGjQCAmTNn4r333kN1dTU4jgPHcSgoKIj62rIs48knn0ROTg6MRiNmzZqFI0eOoKCgAIsWLQqWa25uxooVKzB+/HiYTCZYLBbcdNNNOHDgQMj1tm/fDo7j8MYbb+Dpp59Gbm4u9Ho9Zs+ejcrKynh8OUgciGoHQAaWiooKLF68GGPHjkVZWRmSk5Oxb98+vP/++7j77rvxi1/8AjabDadPn8aLL74IADCZTFFfv6ysDM8++yxuueUWzJ07FwcOHMDcuXPhdrtDyn377bd4++23cccdd6CwsBBnz57Fyy+/jJKSEhw5cgQ5OTkh5Z955hnwPI8VK1bAZrPh2WefxYIFC7B79+7ef1FI7zFCulBeXs4AsKqqKtba2srMZjObNm0ac7lcIeVkWQ7+++abb2bDhg3r8XvV19czURTZ/PnzQx5/8sknGQC2cOHC4GNut5tJkhRSrqqqiul0OrZq1argYx9//DEDwC677DLm8XiCj69du5YBYIcOHepxnCT+qGlGorZ161a0t7fj8ccfh16vD3mO47heX3/btm3w+/346U9/GvL4z372s05ldTodeF758ZUkCU1NTTCZTBg9ejS++uqrTuXvu+8+aLXa4P1rr70WgFKzIuqjRESiduLECQDAuHHj+uT61dXVAICioqKQx1NTU5GSkhLymCzLePHFFzFy5EjodDqkpaUhPT0dBw8ehM1m63Tt/Pz8kPsd12tpaYnnRyAxokREBqTf/OY3KC0txXXXXYe//vWv+OCDD7B161aMHTsWsix3Ki8IQtjrMNopOSFQZzWJ2ogRIwAAX3/9daday4VibaYNGzYMAFBZWYnCwsLg401NTZ1qLn/7298wa9Ys/OUvfwl5vLW1FWlpaTG9P1EP1YhI1G644QaYzWasXr260yjWhTWLpKSksM2j7syePRuiKGLdunUhj//xj3/sVFYQhE61mTfffBO1tbU9fl+iPqoRkahZLBa8+OKLWLJkCaZMmYK7774bKSkpOHDgAJxOJ1555RUAwOTJk/H666+jtLQUU6ZMgclkwi233NLt9TMzM/Hwww/j+eefx6233oobb7wRBw4cwJYtW5CWlhZS0/rud7+LVatW4b777sPVV1+NQ4cO4dVXX8Xw4cP77POTPqTyqB1JcBcO33d499132dVXX80MBgOzWCxs6tSp7LXXXgs+b7fb2d13382Sk5MZgB4N5fv9fvbLX/6SZWVlMYPBwK6//nr2zTffsCFDhrAf//jHwXJut5s9+uijLDs7mxkMBjZjxgz2+eefs5KSElZSUhIs1zF8/+abb4a8T1VVFQPAysvLe/olIX2AY4x660hia21tRUpKCn7961/jF7/4hdrhkD5AfUQkobhcrk6PdSygnTlzZv8GQ/oN9RGRftHY2AhJkiI+r9VqkZqaitdffx0VFRWYN28eTCYTPvvsM7z22mu44YYbMGPGjH6MmPQnSkSkX0yZMiU4YTGckpISbN++HRMmTIAoinj22WfR1tYW7MD+9a9/3Y/Rkv5GfUSkX+zcuTNss6tDSkoKJk+e3I8RkURCiYgQojrqrCaEqG5A9xHJsowzZ87AbDbHZfU3ISS+GGNob29HTk5OcLeEcAZ0Ijpz5gzy8vLUDoMQ0o2amhrk5uZGfH5AJyKz2QxA+ZAWi0XlaAghF2tra0NeXl7wdzWSAZ2IOppjFouFEhEhCay7rhPqrCaEqI4SESFEdZSICCGqG9B9RNFgjMHv93e5zomEJwgCRFGkqRGkzw3qROT1elFXVwen06l2KAOW0WhEdnZ2yAkYhMSb6omotrYWjz32GLZs2QKn04mioiKUl5fjyiuv7NV1ZVlGVVUVBEFATk4OtFot/WXvAcYYvF4vGhsbUVVVhZEjR3Y5IY2Q3lA1EbW0tGDGjBmYNWsWtmzZgvT0dBw/frzT0TGx8Hq9kGUZeXl5MBqNcYj20mMwGKDRaFBdXQ2v19vpLDMShr0ROLENyJoAZF6udjQDhqqJ6Le//S3y8vJQXl4efOzC0xsu5vF44PF4gvfb2tq6fQ/6K9479PXroaodQHOVcksbBQiqNzoGBFV/yt59911ceeWVuOOOO5CRkYHi4mL8+c9/jlh+9erVsFqtwRst7yCJxOuT8M2/D+NEo105YcTZpHZIA4aqiejbb7/FunXrMHLkSHzwwQf4yU9+goceeih4GsTFysrKYLPZgreampp+jpiQyI6eqkVrWzsa2j2wuXyAo0HtkAYMVRORLMu44oor8Jvf/AbFxcV44IEHsHTpUvzpT38KW16n0wWXc9CyjvgoKCgI7glNeqehsREAoBU46LUC4GxWOaKBQ9VElJ2djcsvD+3Qu+yyy3Dq1CmVIiIkdu2tSlOsKMMMvSgAnnaVIxo4VE1EM2bMwNGjR0MeO3bsWPDoYRIdr9erdgiXPElm8DmU0231msCvldeuYkQDi6qJ6Oc//zl27dqF3/zmN6isrMTGjRvxP//zP1i2bFmfvq/XL0e8+SU56rK+KMrGYubMmVi+fDmWL18Oq9WKtLQ0/PKXvwwesVxQUIBf/epXuPfee2GxWPDAAw8AAD777DNce+21MBgMyMvLw0MPPQSHwxG8bkNDA2655RYYDAYUFhbi1VdfjSk+0lm72weNvx08B3iMWWh2eMCoRhQ1VccWp0yZgrfeegtlZWVYtWoVCgsLsWbNGixYsKBP3/eljysjPleYloT5xUOD9//nkxPwSeG39c5NMeCOK8+P3K3fWQWXN3Qpyc+/MyqmGF955RXcf//9+OKLL7Bnzx488MADyM/Px9KlSwEAzz33HFauXIknnngCAHDixAnceOON+PWvf43169ejsbExmMw6pkcsWrQIZ86cwccffwyNRoOHHnoIDQ3UoRoP7W4/tJIdWpHHR7UCMtvtmGxqA81Hj47qkxy++93v4rvf/a7aYSScvLw8vPjii+A4DqNHj8ahQ4fw4osvBhPR9ddfj0cffTRYfsmSJViwYAEeeeQRAMDIkSPx+9//HiUlJVi3bh1OnTqFLVu24IsvvsCUKVMAAH/5y19w2WWX9ftnG4zSTDpMydFBdCThWzkNaAd8Hhe0jAE0o79bqiciNSybVRTxOf6in5kHrhsRsezFP1+LZ0SejNlTV111VciSlOnTp+P5558PLt69eAnMgQMHcPDgwZDmFmMsuNTl2LFjEEUx5MieMWPGIDk5OW4xX8oMWgEGvQwwLThZWRngl2TA7wE0NCO9O5dkItKK0XeN9VXZ3kpKSgq5b7fb8eCDD+Khhx7qVDY/Px/Hjh3rr9AuXT5lcTVvsELmRKUP0e+iRBSFSzIRDQS7d+8Oub9r1y6MHDkSgiCELX/FFVfgyJEjKCoKX9sbM2YM/H4/9u7dG2yaHT16FK2trXGN+1L17VkbzC3tMBtEaJOT4Od18MsM8LkBg9rRJT5aSJSgTp06hdLSUhw9ehSvvfYa/vCHP+Dhhx+OWP6xxx7Dv/71Lyxfvhz79+/H8ePH8c4772D58uUAgNGjR+PGG2/Egw8+iN27d2Pv3r1YsmQJDAb6LYmHgyfrUdloR7tHhk5nhJ/Xnq8RkW5RIkpQ9957L1wuF6ZOnYply5bh4YcfDg7ThzNhwgTs2LEDx44dw7XXXovi4mKsXLkSOTk5wTLl5eXIyclBSUkJbr/9djzwwAPIyMjoj48z6PlcypwhUWeEQScqNaKOPiLSLWqaJSiNRoM1a9Zg3bp1nZ47efJk2NdMmTIFH374YcRrZmVl4e9//3vIY/fcc0+v4iQKyav0Dwk6IwqGJME9NB1D3E7ARzWiaFAiIiQOWCARiTojcpINQHYaUHcG8LtVjmxgoKYZIb0kywyyT0k4ojawCZ+oU/5PNaKoUI0oAW3fvl3tEEgPePwyBFlZ7ydodfD6ZbS5AK3bB4vkUzm6gYFqRIT0ktcvQ5B9EHgOgsYAm8uHfx5rxbGz7YBEndXRoBoRIb1k0Aq4ptAMbX0SIGqhFXlIvAaSzACqEUWFEhEhvaQVeWQncUCSDhB00Ik8JE4DJQ+5EX4KKrkQNc0IiYeOJpiohVZQakQA4PdR0ywaVCMipJea7B64z9lg8fhgFnTgeQ5CYH2Z5KXh+2hQIiKkl6qbnWiqaUSB1g1zYNhe0Cg7EUk+2j0zGtQ0I6SXvH4ZAlNGzTrmD4lUI+oRqhER0ksd84gEngMEJRGNG5aBlBYj9IIE0OZo3aJEREgveXwSkmQvRF4LiEqTbGxeGlAV2NlA8p6faU3CurSaZowBfm//31j4Pa/D2bBhA4YMGRJytDYAzJ8/nxaoJiiPzw+e+UNqROBFgAv8eknUT9SdS6tGJPmAT5/v//e99tHgX8ru3HHHHXjooYfw7rvv4o477gCgnL7x3nvvdbmynqjHH+gHEi/oI2rz+AEfBy0k6P1egCpEXbq0akQDgMFgwN133x08eQMA/vrXvyI/Px8zZ85ULzASUUci4kUR4JXpi3tONmNfrQON7R5AptnV3bm0akSCRqmdqPG+PbB06VJMmTIFtbW1GDp0KCoqKrBo0aKQzfRJ4ri6wAxtiwlJSabgYxqBh58TIcseWuYRhUsrEXFc1E0kNRUXF2PixInYsGEDbrjhBhw+fBjvvfee2mGRCIaaOMCkA3Tnt93VCDy8nAiJualGFAVVm2ZPPvkkOI4LuY0ZM0bNkBLGkiVLUFFRgfLycsyZMwd5eXndv4iowx/ojBbO/5HTCBwkXoQsM0DyqxTYwKF6H9HYsWNRV1cXvH322Wdqh5QQ7r77bpw+fRp//vOfsXjxYrXDIRFIMsOJ+iY0O72QQxIRD5kTITHQqFkUVE9EoigiKysreEtLS1M7pIRgtVrx/e9/HyaTCfPnz1c7HBKByydh97E6HKtvB3fBXCGNoKzAl2RGTbMoqJ6Ijh8/jpycHAwfPhwLFizAqVOnIpb1eDxoa2sLuQ1mtbW1WLBgAXQ6GvtNVB6fFFzecXEikjlqmkVL1UQ0bdo0VFRU4P3338e6detQVVWFa6+9Fu3t7WHLr169GlarNXgbrP0mLS0teOutt7B9+3YsW7ZM7XBIF0KWd4jnT3QdkqTFyOwUZFh0VCOKgqqjZjfddFPw3xMmTMC0adMwbNgwvPHGG7j//vs7lS8rK0NpaWnwfltb26BMRsXFxWhpacFvf/tbjB49Wu1wSBe8gUQk8lxIZ3VKkhYpQ1MBpqfh+ygk1PB9cnIyRo0ahcrKyrDP63S6S6KZEuncMpJ4PH4ZAvOGrLwP4gO/XlQj6pbqfUQXstvtOHHiBLKzs9UOhZCoePxScOP8C2tEsszQ6gXsHj/1EUVB1US0YsUK7NixAydPnsS//vUvfO9734MgCLjrrrvi9h6sBwtOSWf09eua1y9DZF6IPB9SI3L7Jbx/5BwO1dog0/B9t1Rtmp0+fRp33XUXmpqakJ6ejmuuuQa7du1Cenp6r6+t0SjLKpxOJwwGQzelSSROp3KCacfXk4QqTEvC0CwDkry68yvvcX7UDAAkvzexmh4JSNVEtGnTpj67tiAISE5ORkNDAwDAaDTSWq0eYIzB6XSioaEBycnJEAQ6iyKcISYdYOIBpyakRiTyHFigj0jyeUFpvGsJ1Vkdb1lZWQAQTEak55KTk4NfRxJBx/n2FyQijuPAB9Y1yn5qmnVnUCcijuOQnZ2NjIwM+Hw0ctFTGo2GakLdqGl2wtjWDrOGg1YIXVDNBXZdkPz0s9edQZ2IOgiCQL9QpE/sqmxA7plWjMo0YchFw/cc1YiiRn1ohPSCz+sCAGXUTAhNRLyo1IhkqhF165KoERHSV/xeZW9xQaMB+NC/66NzUpHSYoCel9UIbUChRERIL0iBGhGv6TxFZEJ+GnDGCAiUiLpDTTNCYiTLDHLgbHtRG2bpER8YtJf9PTrJ5VJEiYiQGHklZcErAIjazjUip8TB6fXD55cAWerv8AYUSkSExMjTcdQ0B/BhDlDcUdmKA6dtaLR7aJfGblAfESEx0mt4zCgww8QnhT2UQRRFMI5XNkejFfhdokRESIx0ooDhKRqgRd9p6B4ARIFTdmlkoBX43aCmGSG9EVzeoe/0lIbnIXEiZEY1ou5QIiIkRjanD3XNNji8/vBNs2CNiNEujd2gRERIjL49Z8eeE/WobXGFb5rxgUQkg2pE3aBEREiMlP2qfcp+1WFrRPz5GhEN33eJOqsJiVHICR5hakQZZh306RakeJ3UNOsGJSJCYhScRxRu43wAOckG5OSkAk0t1DTrBjXNCImRxy9BlD0QBT5sIgJw/iQPGr7vEiUiQmLU0UcUqWkmyQwuiYfbLynrzUhElIgIiZHHJ0GQPYGmWefO6rNtbnz473P4pq6NmmbdoD4iQmI0dZgFSfVGGLVC2AmNIs8pExplUGd1NygRERKjESkiYNEDXOjhih1o+D561DQjJFYde1ELWiUZXUSZWS3QEo8oUCIiJAaSzHCyoQltbh9YmNoQoKw161j0Shvody1hEtEzzzwDjuPwyCOPqB0KId1y+yR8dOg0jpxpC9s/BJxfawbQBvrdSYhE9OWXX+Lll1/GhAkT1A6FkKh4/DJEpoyYcZoIiYg/n4gk6qzukuqJyG63Y8GCBfjzn/+MlJQUtcMhJCoev9TlHCJAOeCzKCsZWVY9eJpH1CXVE9GyZctw8803Y86cOd2W9Xg8aGtrC7kRogZlMqMn4vKODleNzELhkCRoOBo164qqw/ebNm3CV199hS+//DKq8qtXr8ZTTz3Vx1ER0j1PyMr7yIkouMSDakRdUq1GVFNTg4cffhivvvoq9PrwbeyLlZWVwWazBW81NTV9HCUh4Xl85/uIukpEHsbDK8mQqLO6S6rViPbu3YuGhgZcccUVwcckScInn3yCP/7xj/B4PJ3Oq9fpdNDpuvjrQ0g/cQf6iESh81HTF/r7oUbkVLdgdL4eqf0Y30CjWiKaPXs2Dh06FPLYfffdhzFjxuCxxx7rlIQISSTDUo1Iy9IhxaHtskbEa5RDFmVafd8l1RKR2WzGuHHjQh5LSkrCkCFDOj1OSKLJsOiBZA3Auk5EgtCRiGhCY1dUHzUjZMDq4gSPDrwYOHZaomOnu5JQi163b9+udgiERKXO5oKh3Q6TLEOMsMQDAPjA9iCSHDjJI8x2IYRqRITEZMfRRhw62YA2l7/LGpEQqBHJDDSE3wVKRITEwOOTIDAvRCH8pmgdNKKgHDtNK/C7lFBNM0IGCq/XA47JgXlEkWtE2VYDBKsJSVpG+1Z3gRIRIT3EGIPfq3RUCwIfdlO0DpdlW4DsFMBjp6ZZF2Jqmn377bfxjoOQAcMvMyAwHC9q9GE3RQsRXOZBTbNIYkpERUVFmDVrFv7617/C7XbHOyZCEprHL0OU3eAACFpDl2VlmcEHAX5Zpn2ruxBTIvrqq68wYcIElJaWIisrCw8++CC++OKLeMdGSELy+DrOM+PAabpORF+fsWH78RacaHTQvtVdiCkRTZo0CWvXrsWZM2ewfv161NXV4ZprrsG4cePwwgsvoLGxMd5xEpIwDFoBV+cbMTTZAGiMXZYVeV7Zt1qmUbOu9Gr4XhRF3H777XjzzTfx29/+FpWVlVixYgXy8vJw7733oq6uLl5xEpIwjFoRo4dokG01ABF2Z+ygCWwXKzNGTbMu9CoR7dmzBz/96U+RnZ2NF154AStWrMCJEyewdetWnDlzBrfddlu84iQksfhcyv/Frptm548UAtWIuhDT8P0LL7yA8vJyHD16FPPmzcOGDRswb9488LyS1woLC1FRUYGCgoJ4xkpIQrC5fPC3tsLkl6Drpo9I5DnIvAhZorPNuhJTIlq3bh0WL16MRYsWITs7O2yZjIwM/OUvf+lVcIQkon/XtaHxaC0uE1wY0V0iEjjIEM6vNSNhxZSItm7divz8/GANqANjDDU1NcjPz4dWq8XChQvjEiQhicTlk6CR3dBo+S5nVQOBzmpepCUe3Yipj2jEiBE4d+5cp8ebm5tRWFjY66AISWRunwxR8ij7VXdTIzJoBWSnmpFq1NLM6i7EVCNiEfZVsdvtUe8/TchA5fZJ0AfmEXWXiEw6EVeNyAQEE60160KPElFpaSkA5bymlStXwmg8P4dCkiTs3r0bkyZNimuAhCQal09CuuyGKBi6bZoBAAK7NFLTLLIeJaJ9+/YBUGpEhw4dglZ7frGfVqvFxIkTsWLFivhGSEiCcbs94JkfGp7rdkIjALBAHxEv+dDNqrRLVo8S0ccffwxA2eR+7dq1sFgsfRIUIYnM53ECgHLAQ1dnmkH5o/3anjoUNjWjONkL6rgIL6Y+ovLy8njHQciAwBjD1cOMsLYboNUbu115z3EcOEH5NZPpbLOIok5Et99+OyoqKmCxWHD77bd3WXbz5s29DoyQRMRxHMala4AUI6BLiuo1fKCPSPLTSR6RRJ2IrFYruED2t1qtfRYQIQnP61D+r40uEXV0Vst+GjWLJOpEdGFzjJpm5FLl8kpob2qGxSdBH2UiCm6gT2ebRRTThEaXywWn0xm8X11djTVr1uDDDz+MW2CEJKLaVic+PXwSlY32qGtEHWebMeojiiimRHTbbbdhw4YNAIDW1lZMnToVzz//PG677TasW7curgESkkhcXhkayaXMqo62RhTY01qmeUQRxbxD47XXXgsA+Nvf/oasrCxUV1djw4YN+P3vfx/1ddatW4cJEybAYrHAYrFg+vTp2LJlSywhEdIvXD4JGskFjcBHnYgyU0xITdJCw8l9HN3AFdPwvdPphNlsBgB8+OGHuP3228HzPK666ipUV1dHfZ3c3Fw888wzGDlyJBhjeOWVV3Dbbbdh3759GDt2bCyhEdKnlAWvHTUiU1SvuaooE2hQfl/AWPeb7V+CYt48/+2330ZNTQ0++OAD3HDDDQCAhoaGHk1yvOWWWzBv3jyMHDkSo0aNwtNPPw2TyYRdu3bFEhYhfc7llaCRnBB7UCMKLvEAaOFrBDElopUrV2LFihUoKCjAtGnTMH36dABK7ai4uDimQCRJwqZNm+BwOILXu5jH40FbW1vIjZD+5PH5A02z6JZ3AAB4ESzwHyWi8GJqmv3gBz/ANddcg7q6OkycODH4+OzZs/G9732vR9c6dOgQpk+fDrfbDZPJhLfeeguXX3552LKrV6/GU089FUvIhMSF22kHBwZREKKuEe2obIa2qgW5yXrkSb5uV+xfimLeszorKwvFxcUhm6NNnToVY8aM6dF1Ro8ejf3792P37t34yU9+goULF+LIkSNhy5aVlcFmswVvNTU1sYZPSEwmZ/HITTHAkGQGeCGq13AAJE4MnORBNaJwYqoRORwOPPPMM9i2bRsaGhogy6GjAT05CVar1aKoqAgAMHnyZHz55ZdYu3YtXn755U5ldToddLquFxkS0peKLFCWd5hTon6NKHDwcyJkJlMiiiCmRLRkyRLs2LED99xzD7Kzs4NLP+JBlmV4PJ64XY+QuPIE+iV15qhfohF4eDkBEqPTXiOJKRFt2bIF7733HmbMmNGrNy8rK8NNN92E/Px8tLe3Y+PGjdi+fTs++OCDXl2XkL7g9kloO9cIs0+CQRf9ekuRV842Y8xLm6NFEFMiSklJQWpqaq/fvKGhIXgQo9VqxYQJE/DBBx/gO9/5Tq+vTUi8NbZ7cOCbk8j1tWPSZdFPU9EEzjaTZDpSKJKYEtGvfvUrrFy5Eq+88krIdrE9RccNkYHE6ZWglRzK0H0PmmYCz50/ZJGaZmHFlIief/55nDhxApmZmSgoKIBGowl5/quvvopLcIQkEofXD63fDo2eB3TR14jMehEpZiNMfhs1zSKIKRHNnz8/zmEQkvjcbjd0kgMaUQcYkqN+XW6KEbnDM4BzNqoRRRBTInriiSfiHQchCc/b3gQdGASNPvpZ1R2CJ3lQH1E4MU9obG1txf/+7/+irKwMzc3NAJQmWW1tbdyCIySRSHblUFHBlNbzhat84G8+Nc3CiqlGdPDgQcyZMwdWqxUnT57E0qVLkZqais2bN+PUqVPBvYoIGUxkp/IHVzAN6dHrmh1e/OtwI7LbWzB5GCWicGKqEZWWlmLRokU4fvx4yMmu8+bNwyeffBK34AhJJJdZfchLMcCUktmj1wkcB48sQJJpZnUkMdWIvvzyy7BLMIYOHYr6+vpeB0VIIsrVOpXlHT1MRKLAQYYAmQGMDlkMK6YakU6nC7sFx7Fjx5Cent7roAhJOIwBLqVpBkPPJvOKAgeZF8FAZ5tFElMiuvXWW7Fq1Sr4fMoXleM4nDp1Co899hi+//3vxzVAQhKBy26Drd0Bt18GDNEveAUAkVdmVgOARCd5hBVTInr++edht9uRnp4Ol8uFkpISFBUVwWw24+mnn453jISo7uzZMzhS14bDLTwg9KxHQ+C54KiZRDWisGLqI7Jardi6dSt27tyJAwcOwG6344orrsCcOXPiHR8hCcHTpgzdc8bY1lhywUMWKRGF0+NEJMsyKioqsHnzZpw8eRIcx6GwsBBZWVlgjMV1SxBCEoWvvRFAYA5RDNKsSbC2a8DRqFlYPWqaMcZw6623YsmSJaitrcX48eMxduxYVFdXY9GiRT3eJpaQgUJyNAEAND2cQ9RhztihuDzbAqPI4hnWoNGjGlFFRQU++eQTbNu2DbNmzQp57qOPPsL8+fOxYcMG3HvvvXENkhC1sUAi0lpiHBXumFktUY0onB7ViF577TX813/9V6ckBADXX389Hn/8cbz66qtxC46QhCD5wVytAAC9JSO2awTXmlEfUTg9SkQHDx7EjTfeGPH5m266CQcOHOh1UIQkEuZqhs8vQeI1MJmj3/7jQh/+uwlfnmzGuTZHnKMbHHrUNGtubkZmZuRZpZmZmWhpael1UIQkEuZoQkFaEtq1GUjSa7p/QRhexsEvMxo1i6BHiUiSJIhi5JcIggC/n9rAZHDh3S3IMOuRkZkHCLFtWMEJyukzsuSjY6fD6FEiYoxh0aJFEY/0odM3yKDkVDqqYYxtxAwARDEwj0iWlT2JejgpcrDr0Vdj4cKF3ZahETMy2DhbG+Bz+SCIVphivAYvagEolSHIfkpEF+nRV6O8vLyv4iAkMTGGcw11qD3XBm8Wh5Lc2C4jigIALnDaqw+AvruXXFJi3qGRkEuC1wG/xwWAg84ce9NMIwiQOBESY7RvdRiUiAjpiqsZXkmGR0yCyWiI+TIWg4gkox5akad9q8OghiohXXE2w+OX4RaTYdLF/usyITcZGJYGuNtoUmMYqtaIVq9ejSlTpsBsNiMjIwPz58/H0aNH1QyJkFCuFnj9MlyiFWZ9L/9u84E5SNQ060TVRLRjxw4sW7YMu3btwtatW+Hz+XDDDTfA4aDZpyQxeO3n4JcZ3BoLzDFOZgzqGCmjFfidqNo0e//990PuV1RUICMjA3v37sV1112nUlSEnOdtV+YQMX2K0r8To28b7Th9ogWZrB2jKRF1klB9RDabDQCQmhp+8ymPxxMyaTLcvtmExA1j0PnaMCI9CeaRhb26lMwAp5+Hj8nUNAsjYUbNZFnGI488ghkzZmDcuHFhy6xevRpWqzV4y8vL6+coySXFa4cGEjIsRowbkd+rS2kFZd9qiTHqrA4jYRLRsmXL8PXXX2PTpk0Ry5SVlcFmswVvNTU1/RghueQEDlSE3gLwQq8upRE5yJwYmNBIw/cXS4im2fLly/H3v/8dn3zyCXJzI09d1el0Ede5ERJ3rhbYXD74tElI8kowaGNPRhqBh8wJNKExAlVrRIwxLF++HG+99RY++ugjFBb2rh1OSFy5WnC6xYlPT0s41ezs1aU0gabZ+SUe5EKq1oiWLVuGjRs34p133oHZbA6eEmu1WmEwxD6LlZC4cAUmMxosvZ5DpBX4wBIPQPb7E6dPJEGo+vVYt24dbDYbZs6ciezs7ODt9ddfVzMsQgAAslOZzOiOw2RGjcDBZNDBpBPpkMUwVK0RMUYnGpAExRi89iYwAD6ttVfLOwBAFHjMnZAHfPstAOqsvhjVEAkJx2uH1+MBAwdNUkp8zuvrWOJBExo7oURESDjOZnj9MryiCSZDnEZqO6YA0KhZJwkxfE9IwnG1wOOX4RJTe7/GLGD7iVZYT7WgQO9ESlyuOHhQIiIkHFcLUowaXJY1DHx2bEcIXczu5aDzy/D7qLP6YpSICAnH1QyjVkRBbi4wxBiXSwqBDfQlOlKoE+ojIiQcV+B8PmP4BdixEDTKBvqyn2pEF6NERMjFGANcLWhs9+C0SwdJjs80E0pEkVEiIuRinnb4fV4cb3Tgb0fa45iIlJM7mM8dl+sNJpSICLmYqwUenwyPaIZBp+3VhmgXEjSB0179gdNeSRAlIkIu5mqG2y/BLVpgNcRn6B4AjAYDDBoeIg+aS3QRGjUj5GKBGpFbY41rIpo8PBOoSVVqQ5IHCJz+SqhGREhnzo4aUXwTETgOEALJhzqsQ1AiIuRiwVnVVljimYgAQAwsF5E8XZe7xFDTjJALMQa4WuHxSXFvmtU0O3HmlB2paMdI2gokBCUiQi7ktgGyH4UZFojjRiDNFL+tiWXG0ObloIdETbOLUCIi5EKBGdXWlAxYc+O7NFUr8pB4LSQ/o6bZRaiPiJALuQIndxjivz5eJwqQeI0yQZJqRCEoERFyIWcLHB4/Tjp1aGiP7wxonchD4pRExKhGFIISESEXcjSiyeHFZ3Ucvq61xfXSStNMo2w/66FlHheiRETIhRyNcPkkuDTJSDHGd8KhyHNggtL57af1ZiEoERHSwesEvA64vBJcmhSkJsU3EXEcB0NgmYfspabZhWjUjJAOznOQGYONGSHxWqTEOREBwE0T84Fj/wY0ctyvPZBRjYiQDo5GePwyHJoUaEUe5l4eIRRWoGlGo2ahKBER0sHeGGyWJRs18TlC6GK0xCMsVRPRJ598gltuuQU5OTngOA5vv/22muGQS137Gbh8EhzaNKTGuaO6w95aBw6ebkVdU1ufXH+gUjURORwOTJw4ES+99JKaYRCi7A9kb0SmRYdZU4pRnN83B/7Y/QIcXglej6tPrj9QqdpZfdNNN+Gmm26KurzH44HHc75K29ZGf1VInNjPAkyGqDcjLztT2bKjD2h0BgCATMP3IQZUH9Hq1athtVqDt7y8PLVDIoOFrVb5vyWnz5IQAIg65Wgi2e+lXRovMKASUVlZGWw2W/BWU1OjdkhksGj+Fg6vH/vtFnzbaO+zt9Fo9WDgAuvNqFbUYUDNI9LpdNDp4rctAyEAlKF0Ww1aHF58LiVjWH07hqeb+uStdBoRdl4HSZYAnxvQmfvkfQaaAVUjIqRPtJwEZAlNfj3cohVDkw199lY6DQ8/r4NfZoCfOqw7UCIipP4gZMZwks8HOA65KX2XiAwaAZzWAJHnlBoRAaBy08xut6OysjJ4v6qqCvv370dqairy8/NVjIwMRowxOL0SDBoBPB/okLY3Ak2VaHX6UGccBbNejPsaswvlJBuQMzYfaPJRjegCqiaiPXv2YNasWcH7paWlAICFCxeioqJCpajIYFTZYMf2ow1od/uh0/C4clgqJucmQTj6D4AxnORz4dKkoDjD1Dczqi8kKie+Uo3oPFUT0cyZM8HoxEvSxw6fseHDw2eD9z1eP47t3wntnsOYmCbDKWuwm58EABibY+37gDSBph/ViIIG1KgZIT1V0+zEP480AAAm5llx7RA7mva/j5rWGuSmJYHTpsJVcDP0J3nkJGmRbu77UdnPTzlgPNOKwtR2WPr83QYGSkRkUDPrRaSatEhL0mKWsQrc1x8jSwSGjMiBOOwqYOhkpIla3J0pKXN7+kGrlwe8EjwuZ7+830BAiYgMaslGLX54ZS64bz8Gd+JL5cGcYmiGzwQ0+mA5nSj0W0y8LgkA4He399t7JjpKRGTQcXj8qLO5UZRhAmQJmuP/AOq/Vp4ccT2QP03V+DQGZRKj3+1QNY5EQomIDGj1Nje+rrWh2emFT5IhyQw2pw9+meGGUVaMbf0YaK4COB4YMw/IGq92yMFEJHuoRtSBEhEZsPadasGOY40IN/A6XNuK/Op/ApINEETg8u8BaUX9H2QYWoMFMgDJ61IWvgrxO9Z6oKJERAas4Wkm7K1uwdBkA4anm6AVeYiyF9b6f8HcdBAcAOhMwLgfAJZstcMNMhiNaOcEeP0y4HUAhmS1Q1IdJSIyYFmNGtwzfdj5juamE0DlFqCjyZM1XukT0hrVCzIMk16Ddk0SBN4NeO2UiECJiAwwjDG0ufywGpXmjE4UAMkPVP4TOLNPKWRIAUbNBVILVYw0ssK0JAy/fBjQdkapERFKRGRgOXbWjve/rsfkYSm4ZmSaskzi6/8HtJ5SCuROAYaXJHS/C8dxSpMRUGpEhBIRGTj8koydlcrZYxqBUzp6D72h7K4oaoHL5wNDRqgdZnS0gUTkoUQEUCIiA8jBWhtsLh9MOhHFeVbgyNuBJKQDJi0AzJlqhxi1L+t80J5uRa65CUOGqx2N+mg/IjIguH0SvqhqBgBcVZgKbdVHwLnjAC8A438woJIQALTISXB6JXjaz6kdSkKgREQGhD0nW+DyShhi0mKs9A1Qu1fZ5P6yW4Dkgbd3ldacCgDw2VtUjiQxUCIiCc/m8mHfKeUXdlZKI/iq7coTI2YDGZepF1gvmJLTAQA+p42OnwYlIjIANNk94HkOY3RNyK3/SHkwb4pyG6CSLRb4eR2cPglwt6odjuooEZGENzzdhPsm6jHT/yk4JgMZY5Ta0ACWmqSFW7TA7ZXAnM1qh6M6SkQk8TmbYfzm/8HASUDKMGDMLX16CGJ/sBo08GqtkBjgtDWoHY7qaPieJCRZZvjwSD1GW3woPP0u4HUCpgxg7O3KItYBTuA5CJZsmH0nIdvq1Q5HdQP/O0oGHZ8kY+uRs6itPg5z0z8xdKgOWksGMOHOkM3MBrobpk4Af/AI4KMhfEpEJGE4PH4cb7Djq5PnkNTwFcbZ9mJUhgFaaxYw8YeANkntEOOKt2QrTUy3TZlhreub02UHAkpERHWtDjd2HDiO1nN1MHqbMNxxFBY4MSLLBOuw8cDoecrs6cFGowdMmfC2ngF37gQ0QyeqHZFqKBGRfscc5+BpqITedRawN8DkaEZWVSMyGGDWiRhi1SJ9yFCIRbOUrTwGeMd0V/a0J8NffRj5+sPIoUSkrpdeegm/+93vUF9fj4kTJ+IPf/gDpk6dqnZYiYkxoLVa2f7U1QxwPCTDEJzmstAqpGNifsoFRVnUhwW6vBLOtrnh8Poh8BxMOhGZFj00QpwGVr1OeM4cxNlje9BytgYcgPG5VnDgIAIoykqBMTkdBmsGkDpcmaiYwCvo40VOGwP27U601BxGziXcPFM9Eb3++usoLS3Fn/70J0ybNg1r1qzB3LlzcfToUWRkZKgdXuLw2IGzXwNn9gOuFjAwOD0SGuweNNk98EkMXo0V8ozrwGePB4yp+OjfDbB7/BiVacaIwA6GkGVl6wl3q9I34W7D2XYPtlW2wS1a4NSkwscbAI4Dz3HIsuowITcZl2X38AQuxgC/B1JLNc5VHURr9SG02p2QGcDAwWEciuFDi2FKHQokDcEQffKgrvlEMrqoCF8eyAScZ3H24D+ROWW+2iGpgmMqH7U6bdo0TJkyBX/84x8BALIsIy8vDz/72c/w+OOPd/natrY2WK1W2Gw2WCyD7Kg6yackCttpoKlS2X2QyXB4/TjnAqowFHVyMjgmw+RtRKavBql6DsOGGCHyPGRDKraekuCWNeCZBD1zYYjoRhJzQJYkWI0a5FiVE0clmWFPdTN0ogCdhoeP16MZVjRzyXBqhmBiUS7GD8sCeBFOtxtHTjcjSfBCDw80kgui5IHgdwE+J8yiD3rmAbxO1LfacarZGTwvzKkZAk/mBAwddQVG52ZAr+m/I3wS2ed7vwL2b4TAc8gaV4KssddBaxwcP8/R/o6qWiPyer3Yu3cvysrKgo/xPI85c+bg888/71Te4/HA4/EE77e1tUX3Rs1VwIltCLfL+ukWJ860KAfddTzLgQX/PTbHApNW+TKdaXWiJlD2wmsxABxjuCzHAoteAzCG+jY3qpucAJPDhjQ6y4xkg3LdhnYPqhrbg5flwCDI5z/n8HQTUo1awJKNU9wIfNiYCpnXQOQ5DE83YWyOBflWEXzTcaDhCNBcBd7VjGutfjQ5vDhn98Dtk+EB4EGgRiKYkJMyDNBbIQAoTnVA620BXC0AY2BwwuO3o81VBUvdAeCckjQc7R74Gu1ojfClHplhgt6kdCxrBR4O3gyXtQDWwkkYP3wk0i36vj9bfoC5ctIk7Gysgq72c9Qe3I4zB7djQmEmjIYkgBdwstmFs20eABwQ5ks3IdcKQyCp1zQ7ccbmjvhe43KsSNIpZWtb3DjdGvmQx8uzLTDrlZ/ROpsbJp0YvA8AGP8fgD4+CVPVRHTu3DlIkoTMzNAtHDIzM/Hvf/+7U/nVq1fjqaee6vkb+T2AvTHsU8zhhOTo4gxypwRIgS+TywXZGf4bxwAwFwAE+jU8bshdnFvFPBwgaJU7Xjdk7/kYGKCc8sBp4NQOQVb2OKSOmgyY0pFu92Ckphl5KUaMzDSF1iqyxik3rxNor4PR2Qyj5EUuL6BN0uGcTwcnnwROb0GKSQ8kG4Iv1Xb8Q/IBjnPgHA3QOxqhtzcCPoeyE6Lsh2DUYUi6CU6mhYfTwcvp4OUN8HE6SKIBzsJsIDMN0Bhg4g0Y6wXSTTpKPl3QCDyu+c73cPhALtqOfQrRUQ8t8wJuPwCAORyQ7ZGTC2f3AoGfA2Z3Qm7v4ufZ4QF8YnRlzW7ArwmUdSmv81/Qbxfhj2wsVO8j6omysjKUlpYG77e1tSEvL6/7FybnKfNQgIv6IThY3X7wgW946FPKHY1JBwQ2Z7d4/CjwSBcU4UKK641aQAiU9UkY4VbKcnznX0KDXhP44eFg8UsY6fGj488dx3FgGiMgKrUHk14MxpBm0mHe+G5OpNAalZ0KA7sVcgCsgVu3BI1y4kWEUy9SA7doaAFkDMJR974gCjwmXjENuGIanE4HBNmh/FFgMlKdHug8vrDHJgGAYNEBgZ+xZLcPglsKXxCAaNYCgQEIi9uP/MDPfjhakxYQlbImjx9agQ/eVwrEb16XqokoLS0NgiDg7NmzIY+fPXsWWVlZncrrdDrodDH8ZGuTIm6kbgncomEK3KJhDNyioQ/cCAEAozEJwPlfcmtylH9E0LOf556W7UuqLnrVarWYPHkytm3bFnxMlmVs27YN06dPVzEyQkh/Ur1pVlpaioULF+LKK6/E1KlTsWbNGjgcDtx3331qh0YI6SeqJ6I777wTjY2NWLlyJerr6zFp0iS8//77nTqwCSGDl+rziHpjUM8jImQQiPZ3lDZGI4SojhIRIUR1lIgIIapTvbO6Nzq6t6Je6kEI6Vcdv5vddUUP6ETU3q6sz4pqdjUhRDXt7e2wWiNPyxzQo2ayLOPMmTMwm80DZi1Tx7KUmpqaQTXSNxg/12D8TED/fi7GGNrb25GTkwOej9wTNKBrRDzPIzc3V+0wYmKxWAbVD3eHwfi5BuNnAvrvc3VVE+pAndWEENVRIiKEqI4SUT/T6XR44oknYttFIIENxs81GD8TkJifa0B3VhNCBgeqERFCVEeJiBCiOkpEhBDVUSIihKiOElEfe/rpp3H11VfDaDQiOTk5qtcwxrBy5UpkZ2fDYDBgzpw5OH78eN8G2kPNzc1YsGABLBYLkpOTcf/998Nut3f5mpkzZ4LjuJDbj3/8436KOLyXXnoJBQUF0Ov1mDZtGr744osuy7/55psYM2YM9Ho9xo8fj3/84x/9FGnP9ORzVVRUdPq+6PX9vIs6I31q5cqV7IUXXmClpaXMarVG9ZpnnnmGWa1W9vbbb7MDBw6wW2+9lRUWFjKXy9W3wfbAjTfeyCZOnMh27drFPv30U1ZUVMTuuuuuLl9TUlLCli5dyurq6oI3m83WTxF3tmnTJqbVatn69evZ4cOH2dKlS1lycjI7e/Zs2PI7d+5kgiCwZ599lh05coT993//N9NoNOzQoUP9HHnXevq5ysvLmcViCfm+1NfX92vMlIj6SXl5eVSJSJZllpWVxX73u98FH2ttbWU6nY699tprfRhh9I4cOcIAsC+//DL42JYtWxjHcay2tjbi60pKStjDDz/cDxFGZ+rUqWzZsmXB+5IksZycHLZ69eqw5f/jP/6D3XzzzSGPTZs2jT344IN9GmdP9fRzRfuz2ZeoaZZgqqqqUF9fjzlz5gQfs1qtmDZtWtjTb9Xw+eefIzk5GVdeeWXwsTlz5oDneezevbvL17766qtIS0vDuHHjUFZWBmeEAyv7Wscpwxd+nbs6ZRhQPveF5QFg7ty5CfN9AWL7XABgt9sxbNgw5OXl4bbbbsPhw4f7I9ygAb3odTCqr68HgLCn33Y8p7b6+npkZGSEPCaKIlJTU7uM8e6778awYcOQk5ODgwcP4rHHHsPRo0exefPmvg65k56eMgwonzuRvy9AbJ9r9OjRWL9+PSZMmACbzYbnnnsOV199NQ4fPtxvi8qpRhSDxx9/vFPn3sW3SN/0RNbXn+uBBx7A3LlzMX78eCxYsAAbNmzAW2+9hRMnTsTxU5Cemj59Ou69915MmjQJJSUl2Lx5M9LT0/Hyyy/3WwxUI4rBo48+ikWLFnVZZvjw4TFdu+OE27NnzyI7+/yxz2fPnsWkSZNiuma0ov1cWVlZaGhoCHnc7/ejubk57Am9kUybNg0AUFlZiREjRvQ43t7o6SnDgPK96Ul5NcTyuS6m0WhQXFyMysrKvggxLEpEMUhPT0d6enqfXLuwsBBZWVnYtm1bMPG0tbVh9+7d+MlPftIn79kh2s81ffp0tLa2Yu/evZg8eTIA4KOPPoIsy8HkEo39+/cDQEjC7S8XnjI8f/58AOdPGV6+fHnY10yfPh3btm3DI488Enxs69atCXUqcSyf62KSJOHQoUOYN29eH0Z6EVW7yi8B1dXVbN++feypp55iJpOJ7du3j+3bt4+1t7cHy4wePZpt3rw5eP+ZZ55hycnJ7J133mEHDx5kt912W0IO3xcXF7Pdu3ezzz77jI0cOTJk+P706dNs9OjRbPfu3YwxxiorK9mqVavYnj17WFVVFXvnnXfY8OHD2XXXXafWR2CbNm1iOp2OVVRUsCNHjrAHHniAJScnB4eu77nnHvb4448Hy+/cuZOJosiee+459s0337AnnngiYYfve/K5nnrqKfbBBx+wEydOsL1797If/vCHTK/Xs8OHD/dbzJSI+tjChQsZgE63jz/+OFgGACsvLw/el2WZ/fKXv2SZmZlMp9Ox2bNns6NHj/Z/8F1oampid911FzOZTMxisbD77rsvJLlWVVWFfM5Tp06x6667jqWmpjKdTseKiorYf/7nf6o6j4gxxv7whz+w/Px8ptVq2dSpU9muXbuCz5WUlLCFCxeGlH/jjTfYqFGjmFarZWPHjmXvvfdeP0ccnZ58rkceeSRYNjMzk82bN4999dVX/RovbQNCCFEdjZoRQlRHiYgQojpKRIQQ1VEiIoSojhIRIUR1lIgIIaqjREQIUR0lIkKI6igRkQGloqIiZMvdJ598MmQx8KJFi4JrrMjAQYmIhLVo0aKIe0ovW7YMHMeFrNTviwRQUFCANWvWhDx255134tixYxFfs3btWlRUVATvz5w5M2SRKklMlIhIRHl5edi0aRNcLlfwMbfbjY0bNyI/P1+VmAwGQ6dN2S5ktVqjPqSAJA5KRCSiK664Anl5eSE7KG7evBn5+fkoLi7u1bXD1VTmz58frGXNnDkT1dXV+PnPfx7clA3o3DS72IU1s0WLFmHHjh1Yu3Zt8BpVVVUoKirCc889F/K6/fv3g+O4ft2Dh5xHiYh0afHixSgvLw/eX79+Pe67774+f9/NmzcjNzcXq1atQl1dHerq6np8jbVr12L69OlYunRp8Br5+fmdPhMAlJeX47rrrkNRUVG8PgLpAUpEpEs/+tGP8Nlnn6G6uhrV1dXYuXMnfvSjH/X5+6ampkIQBJjNZmRlZcW0C6LVaoVWq4XRaAxeQxAELFq0CEePHg2e9eXz+bBx40YsXrw43h+DRIl2aCRdSk9Px80334yKigowxnDzzTcjLS1N7bB6JScnBzfffDPWr1+PqVOn4v/+7//g8Xhwxx13qB3aJYtqRKRbixcvRkVFBV555ZW41Rp4nsfFW2H5fL64XDsaS5YsCXbEl5eX484774TRaOy39yehKBGRbt14443wer3w+XyYO3duXK6Znp4e0u8jSRK+/vrrkDJarRaSJPXqfSJdY968eUhKSsK6devw/vvvU7NMZdQ0I90SBAHffPNN8N+R2Gy24Ib4HYYMGYK8vLxOZa+//nqUlpbivffew4gRI/DCCy+gtbU1pExBQQE++eQT/PCHP4ROp4upSVhQUIDdu3fj5MmTMJlMSE1NBc/zwb6isrIyjBw5MqE2wL8UUY2IRMViscBisXRZZvv27SguLg65PfXUU2HLLl68GAsXLsS9996LkpISDB8+HLNmzQops2rVKpw8eRIjRoyI+dSUFStWQBAEXH755UhPT8epU6eCz91///3wer39MgpIukZ7VpNL1qefforZs2ejpqam08mopH9RIiKXHI/Hg8bGRixcuBBZWVl49dVX1Q7pkkdNM3LJee211zBs2DC0trbi2WefVTscAqoREUISANWICCGqo0RECFEdJSJCiOooERFCVEeJiBCiOkpEhBDVUSIihKiOEhEhRHX/H07YC6mtqHbFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:26.231009Z", + "iopub.status.busy": "2024-03-22T14:51:26.230151Z", + "iopub.status.idle": "2024-03-22T14:51:26.577757Z", + "shell.execute_reply": "2024-03-22T14:51:26.576763Z" + }, + "papermill": { + "duration": 0.370764, + "end_time": "2024-03-22T14:51:26.579890", + "exception": false, + "start_time": "2024-03-22T14:51:26.209126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8ZElEQVR4nO3deXhU9b348fc5Z7ZsM0kgK0nYwiI7ilDckMqVirVau1hrK4hLW2lvLZdWuf2VFn0stlUrbX1ob1uJ3qpU70W7KnJRUFEpoiyCgmCAsIQEsmf2c87vj5kMBCbbZDmT8Hn5nEdm5syZz8nyyXf/KqZpmgghhIVUqwMQQghJREIIy0kiEkJYThKREMJykoiEEJaTRCSEsJwkIiGE5SQRCSEsZ7M6gO4wDINjx46RkZGBoihWhyOEOItpmjQ2NlJYWIiqtl3u6deJ6NixYxQXF1sdhhCiAxUVFRQVFbX5er9ORBkZGUDkJt1ut8XRCCHO1tDQQHFxcex3tS39OhG1VMfcbrckIiGSWEdNJ9JYLYSwnCQiIYTlJBEJISzXr9uIOsM0TcLhMLquWx3Kec1ut6NpmtVhiCRleSI6evQo9957Ly+99BJer5fS0lJWr17NtGnTun3tYDDI8ePH8Xq9PRCp6A5FUSgqKiI9Pd3qUEQSsjQR1dbWcumllzJ79mxeeuklcnJy+Pjjj8nKyur2tQ3DoLy8HE3TKCwsxOFwyKBHi5imSXV1NUeOHGHUqFFSMhLnsDQR/exnP6O4uJjVq1fHnhs+fHiPXDsYDGIYBsXFxaSmpvbINUXicnJyOHjwIKFQaGAnoqZqOLAB8idB3jiro+k3LG2s/utf/8q0adP40pe+RG5uLlOnTuX3v/99m+cHAgEaGhpaHR1pb1i56DvnTWm0fBPUlMOev4AetjqafsPS39JPPvmEVatWMWrUKNatW8e3vvUt/v3f/50nn3wy7vkrVqzA4/HEDpneIZJJMKTz4Ue7OVDdhGma4D1ldUj9hqWJyDAMLrzwQn76058ydepU7rrrLu68805++9vfxj1/6dKl1NfXx46Kioo+jliItu09fJS6hkaqGgPU+0LQXGV1SP2GpYmooKCAceNa16MvuOACDh8+HPd8p9MZm84xkKd1XHnlldxzzz1WhyG6qKq6GgCHpuByaOCtsTii/sPSRHTppZeyd+/eVs/t27ePoUOHWhRR/7Jx40YURaGurs7qUATQWBepipXmZuCyaRBotDii/sPSRPS9732Pd955h5/+9Kfs37+fZ555hv/6r/9i0aJFVoYlRJfphkmouR4Alz36axVssjCi/sXSRHTxxRfzwgsv8OyzzzJhwgQeeOABHnvsMW655ZZe/dxg2GjzCOtGp88NdeLc7goEAtx7770UFxfjdDopLS3lj3/8IwcPHmT27NkAZGVloSgKCxYs6PB6jY2N3HLLLaSlpVFQUMAvf/nLc6qC//3f/820adPIyMggPz+fr371q1RVnW7vaCmJbdiwgWnTppGamsoll1xyTun2fNLoD2EPN6IqEEjNp6Y5gCklok6zfGT1Zz/7WT772c/26Wc+/tr+Nl8bPjiNG6YOiT3+r9cPENLj78pdlJXCl6ad7rl7YnM5vmDrqSTf+7fR3Yr11ltv5e233+ZXv/oVkydPpry8nJMnT1JcXMz//u//8oUvfIG9e/fidrtJSUnp8HqLFy9m8+bN/PWvfyUvL49ly5bx3nvvMWXKlNg5oVCIBx54gDFjxlBVVcXixYtZsGAB//znP1td64c//CGPPPIIOTk5fPOb32ThwoVs3ry5W/fbXzX6wzj0Jhw2lVePauQ1NnFRegMOqwPrJyxPRKJt+/bt47nnnmP9+vXMmTMHgBEjRsRez87OBiA3N5fMzMwOr9fY2MiTTz7JM888w1VXXQXA6tWrKSwsbHXewoULY/8eMWIEv/rVr7j44otpampqNUXjwQcfZNasWQDcd999XHvttfj9flwuV2I33I8NTndycaETW3ManxiDoRFCAR8O04TzZQxVN5yXiWjR7NI2X1PP+pm564qRbZ579s/Xwkt7ZlR4i+3bt6NpWuyXvbs++eQTQqEQ06dPjz3n8XgYM2ZMq/O2bdvGT37yE3bs2EFtbS2GEaliHj58uFUv56RJk2L/LigoAKCqqoqSkpIeibc/SXFopLgMMB0oRmSKUlg3IBwA+/mXmLvqvExEDlvnm8Z669zO6ExVq6c1Nzczd+5c5s6dy9NPP01OTg6HDx9m7ty5BIPBVufa7fbYv1tGTrckrfNSKDK5Wk3xYCi2SBti2CeJqBNk/kMSmzhxIoZhsGnTprivOxyRFojOLnEyYsQI7HY7W7dujT1XX1/Pvn37Yo8/+ugjTp06xUMPPcTll1/O2LFjWzVUi/g+OVFPdW0j/rCOIyWNsOokbJgQ8lsdWr8giSiJDRs2jPnz57Nw4UJefPFFysvL2bhxI8899xwAQ4cORVEU/v73v1NdXU1TU/vdxRkZGcyfP5/vf//7vPbaa+zevZvbb78dVVVjJZqSkhIcDge//vWv+eSTT/jrX//KAw880Ov32t/tPFjJ/uomGgMGTmcqYdVxukQkOiSJKMmtWrWKL37xi9x9992MHTuWO++8k+bmZgCGDBnC8uXLue+++8jLy+Pb3/52h9d79NFHmTlzJp/97GeZM2cOl156KRdccEGsgTknJ4eysjKef/55xo0bx0MPPcTDDz/cq/c4EIR8kT8CNmcqKU5bpETU0kYkOqSYphm/b7ofaGhowOPxUF9ff850D7/fT3l5OcOHDz8ve3E6q7m5mSFDhvDII49w++2399rnDPTvx5pX3mBoxV8YP2oETZNuw//+cwz2H8I96bMw5EKrw7NMe7+jZzovG6vPZ++//z4fffQR06dPp76+nvvvvx+A66+/3uLI+jczGGmotjlTKcxMgYLBcPwYhKWNqDMkEQ0gZ3evn23Pnj0APPzww+zduxeHw8FFF13EG2+8weDBg/sqzAHHMEyMaKO0zRFdhM/mjPw/JG1EnSGJaAApLCxk+/bt7b5eUlLCtm3b+i6o80AgbKAZkaENmsNJMGzQ4AOHP4RbD1kcXf8giWgAsdlslJa2PVhT9I5g2EAzQmiqgmZPocYXYsO+OkobG5k2VBqrO0MSkRDdlOLQuGx4Bo7KNLA5cNhUdNWObpggJaJOkUQkRDc5bCoFaQqkOUFz4rSp6IqdSB7yM4C3CugxMo5IiJ6gR6tgNgcOLVIiAgiHpGrWGVIiEqKbTjUF8J+sxx0IkaE5UVUFLTq/TA9K931nSCISopsO1Xg5VVHNMIefjGi3vWaPzgMMBdt7q4iSqpkQ3RQMG2hmpNesZfyQTUpEXSKJKAkl2y4eyRZPsmkZR6SpCmiRRDRhaC5DB6Xi0nTov7Oo+owkogHq7LWDRO8JhHRsRhCbqoItUiUbXzyYQk9KZDcPXb4XHTm/EpFpQjjY90cX/iIuWLCATZs2sXLlShRFQVEUDhw4wO23387w4cNJSUlhzJgxrFy58pz33XDDDTz44IMUFhbGVl186623mDJlCi6Xi2nTpvHiiy+iKEqrEdgffPAB11xzDenp6eTl5fH1r3+dkydPthnPwYMHu/2tGEgCoTCqGW5VIkK1gRL99ZJE1KHzq7FaD8Ebj/T9517+H7G/lB1ZuXIl+/btY8KECbEJqVlZWRQVFfH8888zaNAg3nrrLe666y4KCgr48pe/HHvvhg0bcLvdrF+/HojMfL7uuuuYN28ezzzzDIcOHTqnilVXV8enP/1p7rjjDn75y1/i8/m49957+fKXv8yrr74aN56cnJwe+KIMHOFoO5DtjDaihkAYQgoOdFzhIDitjDD5nV+JqB/weDw4HA5SU1PJz8+PPb98+fLYv4cPH87bb7/Nc8891yoRpaWl8Yc//CG2cuNvf/tbFEXh97//PS6Xi3HjxnH06FHuvPPO2Ht+85vfMHXqVH7605/GnnviiScoLi5m3759jB49Om484rSWRKTabKBGhi++e7AG7WgzI9wGxYaMru7I+ZWINHukdGLF53bT448/zhNPPMHhw4fx+XwEg8FWWwBBZGnZliQEsHfvXiZNmtRq/Z8zF84H2LFjB6+99lqr3TlaHDhwgNGju7cd0vngkmEZOGrTSUs7/TW0ayphxYZhBGSaRyecX4lIUTpdRUoma9asYcmSJTzyyCPMnDmTjIwMfvGLX7Bly5ZW56WlpXX52k1NTVx33XX87Gc/O+e1lp05RPuGpCuQ7gTn6c0O7JpKULGhm36QElGHLG2s/slPfhJrAG05xo4da2VIScHhcLRaEH/z5s1ccskl3H333UydOpXS0lIOHDjQ4XXGjBnDrl27CAROTzM4c+F8gAsvvJDdu3czbNgwSktLWx0tie3seMRZwtHGaO30Hzm7pqCrNgzDBD1sUWD9h+W9ZuPHj+f48eOx480337Q6JMsNGzaMLVu2cPDgQU6ePMmoUaN49913WbduHfv27eNHP/rROQklnq9+9asYhsFdd93Fhx9+yLp162LrT7cslr9o0SJqamq4+eab2bp1KwcOHGDdunXcdtttseRzdjzn9ZZBZ9ENkwOVp6jxBjFaJSIVQ7Ghm0ivWSdYnohsNhv5+fmxQ1YKhCVLlqBpGuPGjSMnJ4e5c+dy4403ctNNNzFjxgxOnTrF3Xff3eF13G43f/vb39i+fTtTpkzhhz/8IcuWLQOItRsVFhayefNmdF3n6quvZuLEidxzzz1kZmaiqmrceA4fPtx7N9/P+EI6W/YdZ19lI4rtdNeYXYvMwNcNU6pmnWB5G9HHH39MYWEhLpeLmTNnsmLFijZ3Cg0EAq2qGQ0NDX0VZp8aPXo0b7/9dqvnVq9ezerVq1s9t2LFiti/y8rK4l7rkksuYceOHbHHTz/9NHa7vdXXeNSoUaxdu7ZL8YiIQEiPTe84OxEZig1Dl6pZZ1haIpoxYwZlZWW8/PLLrFq1ivLyci6//HIaGxvjnr9ixQo8Hk/sKC4u7uOI+5+nnnqKN998k/Lycl588cXYGCErdpEdiFpN77Cd7p0clOZgVEEWuW6nlIg6wdIS0TXXXBP796RJk5gxYwZDhw7lueeei7u1zdKlS1m8eHHscUNDgySjDlRWVrJs2TIqKyspKCjgS1/6Eg8++KDVYQ0YwWgisqlKq8bqrDQHWUOywXRJ930nWF41O1NmZiajR49m//79cV93Op04nTJEtSt+8IMf8IMf/MDqMAasQNhAM4OtZt7HqNFfLykRdcjyxuozNTU1ceDAARm/IvqNQFiPLZx/ZonIMEzqgtAUCEsbUSdYmoiWLFnCpk2bOHjwIG+99Raf//zn0TSNm2++ucc+ox9vZDugDNTvQzBsYDNbZt6fLhH5wzov7znJrqP1GNJ93yFLq2ZHjhzh5ptv5tSpU+Tk5HDZZZfxzjvv9MikSrs9Mq3C6/VKw2wSaFmWRNMG1lLywwenMSQ/hbSg8/TMe073mgHo4WByVT2SkKWJaM2aNb12bU3TyMzMpKqqCoDU1NTYID7RtwzDoLq6mtTUVGy2pGqW7LZB6U5IV8Frb1UisqkKZrSNSA8F6f5sw4FtYP1UnKVltnhLMhLWUVWVkpKSgfnHoGV/+zMSkaIoqNF5jUZYqmYdGdCJSFEUCgoKyM3NJRSSngsrORyO2EjtgaSixktqQyMZdgWH1npCtRJddUEPy89eRwZ0ImqhadqAa5sQyeGd/VUUHatjdF46g87qvlekRNRpA+9PlBB9KBT0AUR6zbTWiUi1RUpEhpSIOnRelIiE6C3hYGTuo2a3w1lVzzGF2WTVpuBSZbWCjkgiEqIb9GiJSLWfO0RkUslgOJYKmiSijkjVTIgEGYaJEd3b3uaIM/VIjXbaG2HZ26wDkoiESFBQj0x4BbA5zi0ReXUFbzBMKKyDIStctkcSkRAJCrRsNa2AevaEV2DT/jp2HKmnuikgqzR2QNqIhEiQy65y6bAM0tW0uJsy2Gw2TEWNrFstM/DbJYlIiAQ5bRojsuxQ6zqn6x7ApimRVRpNZAZ+B6RqJkR3xKZ3uM55ya6q6IoNw5QSUUckEQmRoHpviOM19TQHw/GrZrESkSmrNHZAEpEQCfrkZBPvHqjkaK0vftVMjSYiAykRdUASkRAJiqxXHYqsVx23RKSeLhFJ9327pLFaiAS12sEjTokoN8OJK8dNVtArVbMOSCISIkGxcUTxFs4HCjNTKCzMhlO1UjXrgFTNhEhQIKxjMwLYNDVuIgJO7+Qh3fftkkQkRIJa2ojaqprpholPV/GH9ch8M9EmSURCJCgQ0tGMQLRqdm5j9YkGP698dJIPjzdI1awD0kYkRIKmD3WTVplKqkOLO6DRpiqRAY0G0ljdAUlEQiRoZJYN3C5QWm+u2EK67ztPqmZCJKplLWrNEUlGZ4mMrNZkikcnSCISIgG6YXKw6hQN/hBmnNIQROaatUx6lQX025c0ieihhx5CURTuueceq0MRokP+kM6ru46w51hD3PYhOD3XDGQB/Y4kRSLaunUrv/vd75g0aZLVoQjRKYGwgc2M9Jgp9jYSkXo6EenSWN0uyxNRU1MTt9xyC7///e/JysqyOhwhOiUQ1tsdQwSRDT5L8zPJ97hQZRxRuyxPRIsWLeLaa69lzpw5HZ4bCARoaGhodQhhhchgxkCb0ztafGpUPsMHpWFXpNesPZZ2369Zs4b33nuPrVu3dur8FStWsHz58l6OSoiOBVrNvG87EcWmeEiJqF2WlYgqKir47ne/y9NPP43LFb+OfbalS5dSX18fOyoqKno5SiHiC4ROtxG1l4gCpkpQN9ClsbpdlpWItm3bRlVVFRdeeGHsOV3Xef311/nNb35DIBA4Z796p9OJ09nOXx8h+og/2kZk087davpMf99VTeGhWsaUuMjuw/j6G8sS0VVXXcWuXbtaPXfbbbcxduxY7r333nOSkBDJZGh2KoPznWQ1O9otEan2yCaLhsy+b5dliSgjI4MJEya0ei4tLY1Bgwad87wQySbX7YJMO5jtJyJNa0lEMqCxPZb3mgnRb7Wzg0cL1RbddlqXbafbk1STXjdu3Gh1CEJ0yvF6HymNTaQbBrY2pngAqNHlQXQjupNHnOVChJSIhEjIpr3V7DpYRYMv3G6JSIuWiAwT6cJvhyQiIRIQCOloZhCbFn9RtBZ2mxbZdlpm4LcrqapmQvQXwWAAxTSi44jaLhEVeFLQPOmkOUxZt7odkoiE6CLTNAkHIw3VmqbGXRStxQUFbijIgkCTVM3akVDV7JNPPunpOIToN8KGCdHueJvdFXdRtFZi0zykataWhBJRaWkps2fP5k9/+hN+v7+nYxIiqQXCBjbDjwJojpR2zzUMkxAaYcOQdavbkVAieu+995g0aRKLFy8mPz+fb3zjG/zrX//q6diESEqBUMt+ZgqKvf1E9MGxejZ+XMuB6mZZt7odCSWiKVOmsHLlSo4dO8YTTzzB8ePHueyyy5gwYQKPPvoo1dXVPR2nEEkjxaFxSUkqQzJTwJ7a7rk2VY2sW21Ir1l7utV9b7PZuPHGG3n++ef52c9+xv79+1myZAnFxcXceuutHD9+vKfiFCJppDpsjBlkp8CTAm2sztjCHl0u1jBNqZq1o1uJ6N133+Xuu++moKCARx99lCVLlnDgwAHWr1/PsWPHuP7663sqTiGSS8gX+b+t/arZ6S2FkBJROxLqvn/00UdZvXo1e/fuZd68eTz11FPMmzcPVY3kteHDh1NWVsawYcN6MlYhkkK9L0S4ro70sI6zgzYim6pgqDYMXfY2a09CiWjVqlUsXLiQBQsWUFBQEPec3Nxc/vjHP3YrOCGS0UfHG6jee5QLNB8jO0pEmoKBdnqumYgroUS0fv16SkpKYiWgFqZpUlFRQUlJCQ6Hg/nz5/dIkEIkE19Ix274sTvUdkdVQ7SxWrXJFI8OJNRGNHLkSE6ePHnO8zU1NQwfPrzbQQmRzPwhA5seiKxX3UGJKMWhUZCdQXaqQ0ZWtyOhEpHZxroqTU1NnV5/Woj+yh/ScUXHEXWUiNKdNj41Mg+0dJlr1o4uJaLFixcDkf2ali1bRmrq6TEUuq6zZcsWpkyZ0qMBCpFsfCGdHMOPTUvpsGoGQHSVRqmata1Liej9998HIiWiXbt24XCcnuzncDiYPHkyS5Ys6dkIhUgyfn8A1QxjV5UOBzQCmNE2IlUP0cGstPNWlxLRa6+9BkQWuV+5ciVut7tXghIimYUCXoDIBg/t7WlG5I/2s+8eZ/ipGqZmBpGGi/gSaiNavXp1T8chRL9gmiaXDE3F05iCw5Xa4cx7RVFQtMivmSF7m7Wp04noxhtvpKysDLfbzY033tjuuWvXru12YEIkI0VRmJBjh6xUcKZ16j1qtI1ID8tOHm3pdCLyeDwo0ezv8Xh6LSAhkl6wOfJ/R+cSUUtjtRGWXrO2dDoRnVkdk6qZOF/5gjqNp2pwh3RcnUxEsQX0ZW+zNiU0oNHn8+H1emOPDx06xGOPPcYrr7zSY4EJkYyO1nl5Y/dB9lc3dbpE1LK3mSltRG1KKBFdf/31PPXUUwDU1dUxffp0HnnkEa6//npWrVrVowEKkUx8QQO77ouMqu5siSi6prUh44jalPAKjZdffjkA//M//0N+fj6HDh3iqaee4le/+lWnr7Nq1SomTZqE2+3G7XYzc+ZMXnrppURCEqJP+EI6dt2HXVM7nYjystLJTnNgV4xejq7/Sqj73uv1kpGRAcArr7zCjTfeiKqqfOpTn+LQoUOdvk5RUREPPfQQo0aNwjRNnnzySa6//nref/99xo8fn0hoQvSqyITXlhJReqfe86nSPKiK/L5gmh0vtn8eSnjx/BdffJGKigrWrVvH1VdfDUBVVVWXBjled911zJs3j1GjRjF69GgefPBB0tPTeeeddxIJS4he5wvq2HUvti6UiGJTPEAmvrYhoUS0bNkylixZwrBhw5gxYwYzZ84EIqWjqVOnJhSIruusWbOG5ubm2PXOFggEaGhoaHUI0ZcCoXC0ata56R0AqDbM6H+SiOJLqGr2xS9+kcsuu4zjx48zefLk2PNXXXUVn//857t0rV27djFz5kz8fj/p6em88MILjBs3Lu65K1asYPny5YmELESP8HubUDCxaVqnS0Sb9tfgKK+lKNNFsR7qcMb++SjhNavz8/OZOnVqq8XRpk+fztixY7t0nTFjxrB9+3a2bNnCt771LebPn8+ePXvinrt06VLq6+tjR0VFRaLhC5GQi/JVirJSSEnLAFXr1HsUQFds0Z08pEQUT0IloubmZh566CE2bNhAVVUVhtG6N6ArO8E6HA5KS0sBuOiii9i6dSsrV67kd7/73TnnOp1OnM72JxkK0ZtK3USmd2Rkdfo9Nk0hrNgwTEMSURsSSkR33HEHmzZt4utf/zoFBQWxqR89wTAMAoFAj11PiB4ViLZLOjM6/Ra7phJUNHRTdnttS0KJ6KWXXuIf//gHl156abc+fOnSpVxzzTWUlJTQ2NjIM888w8aNG1m3bl23ritEb/CHdBpOVpMR0klxdn6+pU2N7G1mmkFZHK0NCSWirKwssrOzu/3hVVVVsY0YPR4PkyZNYt26dfzbv/1bt68tRE+rbgyw48ODFIUamXJB54ep2KN7m+mGbCnUloQS0QMPPMCyZct48sknWy0X21Wy3ZDoT7xBHYfeHOm670LVTFOV05ssStUsroQS0SOPPMKBAwfIy8tj2LBh2O32Vq+/9957PRKcEMmkORjGEW7C7lLB2fkSUYbLRlZGKunheqmatSGhRHTDDTf0cBhCJD+/349Tb8Zuc0JKZqffV5SVStGIXDhZLyWiNiSUiH784x/3dBxCJL1g4ymcmGh2V+dHVbeI7eQhbUTxJDygsa6ujj/84Q8sXbqUmpoaIFIlO3r0aI8FJ0Qy0Zsim4pq6YO7PnFVjf7Nl6pZXAmViHbu3MmcOXPweDwcPHiQO++8k+zsbNauXcvhw4djaxUJMZAY3sgfXC19UJfeV9Mc5K3d1RQ01nLRUElE8SRUIlq8eDELFizg448/brWz67x583j99dd7LDghkskFnhDFWSmkZ+V16X2aohAwNHRDRla3JaES0datW+NOwRgyZAiVlZXdDkqIZFTk8Eamd3QxEdk0BQMNwwRTNlmMK6ESkdPpjLsEx759+8jJyel2UEIkHdMEX6RqRkrXBvPaNAVDtWEie5u1JaFE9LnPfY7777+fUCjyRVUUhcOHD3PvvffyhS98oUcDFCIZ+JrqqW9sxh82IKXzE14BbGpkZDWALjt5xJVQInrkkUdoamoiJycHn8/HrFmzKC0tJSMjgwcffLCnYxTCcidOHGPP8QZ216qgda1FQ1OVWK+ZLiWiuBJqI/J4PKxfv57NmzezY8cOmpqauPDCC5kzZ05PxydEUgg0RLruldTE5lgqsU0WJRHF0+VEZBgGZWVlrF27loMHD6IoCsOHDyc/Px/TNHt0SRAhkkWosRqIjiFKwGBPGp5GO4r0msXVpaqZaZp87nOf44477uDo0aNMnDiR8ePHc+jQIRYsWNDlZWKF6C/05lMA2Ls4hqjFnPFDGFfgJtVm9mRYA0aXSkRlZWW8/vrrbNiwgdmzZ7d67dVXX+WGG27gqaee4tZbb+3RIIWwmhlNRA53gr3CLSOrdSkRxdOlEtGzzz7Lf/7nf56ThAA+/elPc9999/H000/3WHBCJAU9jOmrA8Dlzk3sGrG5ZtJGFE+XEtHOnTv5zGc+0+br11xzDTt27Oh2UEIkE9NXQyiso6t20jM6v/zHmV756BRbD9ZwsqG5h6MbGLpUNaupqSEvr+1RpXl5edTW1nY7KCGSidl8imGD02h05JLmsnf8hjiCpkLYMKXXrA1dSkS6rmOztf0WTdMIh6UOLAYW1V9LboaL3Lxi0BJbsELRIrvPGHpItp2Oo0uJyDRNFixY0OaWPrL7hhiQvJGGalIT6zEDsNmi44gMI7ImURcHRQ50XfpqzJ8/v8NzpMdMDDTeuipCvhCazUN6gtdQbQ4gUhjCCEsiOkuXvhqrV6/urTiESE6mycmq4xw92UAwX2FWUWKXsdk0QInu9hoCXB295byS8AqNQpwXgs2EAz5AwZmReNXMrmnoig3dNGXd6jgkEQnRHl8NQd0gYEsjPTUl4cu4U2ykpbpw2FRZtzoOSxPRihUruPjii8nIyCA3N5cbbriBvXv3WhmSEK15awiEDfy2TNKdibfrTCrKZOrQwRR6UmRQYxyWJqJNmzaxaNEi3nnnHdavX08oFOLqq6+muVkGfYkk4aslGDbw2TxkuLrZwKxGxyBJ1ewcljbdv/zyy60el5WVkZuby7Zt27jiiissikqI04JNJwkbJn67m4wEBzPGtPSUyQz8cyRVH2J9fT0A2dmJrfkiRE8LNkbGEJmurEj7ToI+qW7iyIFa8sxGxkgiOkfSJCLDMLjnnnu49NJLmTBhQtxzAoFAq0GT8dbNFqLHmCbOUAMjc9LIGDW8W5cyTPCGVUKmIVWzOJKm12zRokV88MEHrFmzps1zVqxYgcfjiR3FxcV9GKE47wSbsKOT605lwsiSbl3KoUXWrdZNUxqr40iKRPTtb3+bv//977z22msUFbU9Ymzp0qXU19fHjoqKij6MUpx3ohsq4nKDqnXrUnabgqHYogMapfv+bJZWzUzT5Dvf+Q4vvPACGzduZPjw9ou/TqezzXluQvQ4Xy31vhAhRxppQZ0UR+LJyK6pGIomAxrbYGmJaNGiRfzpT3/imWeeISMjg8rKSiorK/H5fFaGJUSEr5YjtV7eOKJzuMbbrUvZo1Wz01M8xJksTUSrVq2ivr6eK6+8koKCgtjx5z//2cqwhIjwRQcz2t3dHkPk0NToFA8wZKmcc1heNRMiWRneyGBGfw8MZrRrCukpTtKDNnQ9mByNs0kkabrvhUgqpkmw6RQmEHJ4ujW9A8CmqcydVAyffAJIY/XZJDELEU+wiWAggImCPS2rZ/bra5niIQMazyGJSIh4vDUEwwZBWzrpKT3UU9syBEB6zc4hVTMh4vHVEggb+GzZ3Z9jFrXxQB2ew7UMc3nJ6pErDhySiISIx1dLVqqdC/KHohYktoXQ2ZqCCs6wQTgU7JHrDSSSiISIx1dDqsPGsKIiGJTaI5fUogvo67Kl0DmkjUiIeHzR/flSe24lCM0eWUDfCEuJ6GySiIQ4m2mCr5bqxgBHfE50o2fGu0kiapskIiHOFmgkHArycXUz/7OnsQcTUWTnDjPk75HrDSSSiIQ4m6+WQMggYMsgxeno1oJoZ9Ls0d1ew9HdXkWMJCIhzuarwR/W8dvceFJ6puseIDUlhRS7ik1FxhKdRXrNhDhbtETkt3t6NBFdNCIPKrIjpSE9ANHdX4WUiIQ4l7elRNSziQhFAS2afKTBuhVJREKcLTaq2oO7JxMRgC06XUQPtH/eeUaqZkKcyTTBV0cgpPd41ayixsuxw01k08goXUpEZ5JEJMSZ/PVghBme68Y2YSSD03tuaWLDNGkIKrjQpWp2FklEQpwpOqLak5WLp6hnp6Y6bCq66kAPm1I1O4u0EQlxJl90546Unp8f77Rp6Ko9MkBSSkStSCIS4kzeWpoDYQ56nVQ19uwIaKdNRVciiciUElErkoiEOFNzNaeag7x5XOGDo/U9eulI1cweWX42INM8ziSJSIgzNVfjC+n47JlkpfbsgEObqmBqkcbvsMw3a0USkRAtgl4INuML6vjsWWSn9WwiUhSFlOg0DyMoVbMzSa+ZEC28JzFMk3ozFV11kNXDiQjgmsklsO8jsBs9fu3+TEpEQrRoriYQNmi2Z+GwqWR0cwuhuKJVM+k1a83SRPT6669z3XXXUVhYiKIovPjii1aGI853TdWxallmqr1nthA6m0zxiMvSRNTc3MzkyZN5/PHHrQxDiIjGY/hCOs2OwWT3cEN1i21Hm9l5pI7jpxp65fr9laVtRNdccw3XXHONlSEIEaGHoKmaPLeT2RdMxZ6W2Ssf0xTWcAR1PAFfr1y/v+pXjdWBQIBA4HSRtqFB/qqIHtJ0AkwDmyuD4oK8yJIdvcDuTAHAkO77VvpVY/WKFSvweDyxo7i42OqQxEBRfzTyf3dhryUhAJszsjWREQ7KKo1n6FeJaOnSpdTX18eOiooKq0MSA0XNJzQHw2xvcvNJdVOvfYzd4cJEic43k1JRi35VNXM6nTidPbcsgxBApCu9voLa5iBv65kMrWxkRE56r3yU026jSXWiGzqE/ODM6JXP6W/6VYlIiF5RexAMnVNhF36bhyGZKb32UU67Slh1EjZMCEuDdQtLS0RNTU3s378/9ri8vJzt27eTnZ1NSUmJhZGJ80rlTgzT5KBaAopCUVbvJaIUu4biSMFmeiMlIgFYnIjeffddZs+eHXu8ePFiAObPn09ZWZlFUYmByjRNvEGdFLuGqkYbpJuq4dR+6rwhjqeOJsNl6/E5ZmcqzEyhcHwJnApJiegMliaiK6+8ElM2mhN9YH9VExv3VtHoD+O0q0wbms1FRWloe/8JpslBtQifPYupuem9M6L6TLbIjq9SIjqtXzVWC5GI3cfqeWX3idjjQDDMvu2bcby7m8mDDbyGnS3qFADGF3p6PyB7tOonJaIYSURiQKuo8fJ/e6oAmFzs4fJBTZza/jIVdRUUDU5DcWTjG3YtroMqhWkOcjJ6v1f27cPNpB6rY3h2I+5e/7T+QRKRGNAyXDay0x0MTnMwO7Uc5YPXyLfBoJGF2IZ+CoZcxGCbg6/m6ZGxPX2gLqhCUCfg8/bJ5/UHkojEgJaZ6uAr04pQPnkN5cDWyJOFU7GPuBLsrth5TpvWZzGpzjQAwv7GPvvMZCeJSAw4zYEwx+v9lOamg6Fj//ifUPlB5MWRn4aSGZbGZ0+JDGIM+5stjSOZSCIS/VplvZ8PjtZT4w0S0g10w6TeGyJsmFw92sP4utegphwUFcbOg/yJVoccS0RGQEpELSQRiX7r/cO1bNpXTbwRICMcdZQc+j/Q60GzwbjPw+DSvg8yDkeKGwPQg77IxFet57a17q8kEYl+a8TgdLYdqmVIZgojctJx2FRsRhBP5VtknNqJAuBMhwlfBHeB1eHGpKSm0qhoBMMGBJshJdPqkCwniUj0W55UO1+fOfR0Q/OpA7D/JWip8uRPjLQJOVKtCzKOdJedRnsamuqHYJMkIiQRiX7GNE0afGE8qZHqjNOmgR6G/f8Hx96PnJSSBaPnQvZwCyNt2/DBaYwYNxQajkVKREISkehf9p1o4uUPKrloaBaXjRocmSbxwf9C3eHICUUXw4hZSd3uoihKpMoIkRKRkEQk+o+wbrB5f2TvMbumRBp6dz0XWV3R5oBxN8CgkVaH2TmOaCIKSCICSUSiH9l5tJ56X4h0p42pxR7Y82I0CTlhyi2QkWd1iJ229XgIx5E6ijJOMWiE1dFYTxZGE/2CP6Tzr/IaAD41PBtH+atw8mNQNZj4xX6VhABqjTS8QZ1A40mrQ0kKkohEv/DuwVp8QZ1B6Q7G6x/C0W2RRe4vuA4y+98ieo6MbABCTbUWR5IcJBGJpFfvC/H+4cgv7OysatTyjZEXRl4FuRdYF1g3pGfmABDy1sv200giEv3AqaYAqqow1nmKospXI08WXxw5+qlMt5uw6sQb0sFfZ3U4lpNEJJLeiJx0bpvs4srwGyimAbljI6Whfiw7zYHf5sYf1DG9NVaHYzlJRCL5eWtI/fB/SVF0yBoKY6/r1U0Q+4InxU7Q4UE3wVtfZXU4lpPue5GUDMPklT2VjHGHGH7krxD0QnoujL8xMom1n9NUBc1dQEboIEZ9pdXhWK7/f0fFgBPSDdbvOcHRQx+Tcer/GDLEicOdC5NuarWYWX939fRJqDv3QEi68CURiaTRHAjzcVUT7x08SVrVe0yo38bo3BQcnnyY/BVwpFkdYo9S3QWRKqa/PjLC2tk7u8v2B5KIhOXqmv1s2vExdSePkxo8xYjmvbjxMjI/Hc/QiTBmXmT09EBjd0F6HsG6YygnD2AfMtnqiCwjiUj0ObP5JIGq/bh8J6CpivTmGvLLq8k1IcNpY5DHQc6gIdhKZ0eW8ujnDdPtebcxk/Ch3ZS4dlMoichajz/+OL/4xS+orKxk8uTJ/PrXv2b69OlWh5WcTBPqDkWWP/XVgKKipwziiJJPnZbD5JKsM041O71ZoC+oc6LBT3MwjKYqpDtt5Lld2LUe6lgNegkc28mJfe9Se6ICBZhY5EFBwQaU5meRmplDiicXskdEBiom8Qz6nmIMHov5yWZqK3ZTeB5XzyxPRH/+859ZvHgxv/3tb5kxYwaPPfYYc+fOZe/eveTm5lodXvIINMGJD+DYdvDVYmLiDehUNQU41RQgpJsE7R6MS69ALZgIqdm8+lEVTYEwo/MyGBldwRDDiCw94a+LtE34GzjRGGDD/gb8NjdeezYhNQUUBVVRyPc4mVSUyQUFXdyByzQhHECvPcTJ8p3UHdpFXZMXwwQThebUIYwYMpX07CGQNohBrswBXfJpy5jSUrbuyAPvCU7s/D/yLr7B6pAsoZgW7/k8Y8YMLr74Yn7zm98AYBgGxcXFfOc73+G+++5r970NDQ14PB7q6+txuwfYVnV6KJIo6o/Aqf2R1QdNg+ZgmJM+KGcIx41MFNMgPVhNXqiCbJfC0EGp2FQVIyWb9Yd1/IYd1dRxmT4G2fykmc0Yuo4n1U6hJ7LjqG6YvHuoBqdNw2lXCakuavBQo2TitQ9icmkRE4fmg2rD6/ez50gNaVoQFwHsug+bHkAL+yDkJcMWwmUGIOilsq6JwzXe2H5hXvsgAnmTGDL6QsYU5eKy990WPsns7W3vwfZn0FSF/AmzyB9/BY7UgfHz3NnfUUtLRMFgkG3btrF06dLYc6qqMmfOHN5+++1zzg8EAgQCgdjjhoaGzn1QTTkc2EC8VdaP1Ho5VhvZ6K7lVQUz9u/xhW7SHZEv07E6LxXRc8+8lgkopskFhW7cLjuYJpUNfg6d8oJpxA1pTH4GmSmR61Y1BiivboxdVsFEM07f54icdLJTHeAu4LAykleqszFUOzZVYUROOuML3ZR4bKinPoaqPVBTjuqr4XJPmFPNQU42BfCHDAJAgGiJREunMGsouDxowNTsZhzBWvDVgmli4iUQbqLBV477+A44GUkazY0BQtVN1LXxpR6Vm44rPdKw7NBUmtUMfJ5heIZPYeKIUeS4Xb2/t3w/M23KFDZXl+M8+jZHd27k2M6NTBqeR2pKGqgaB2t8nGgIAArE+dJNKvKQEk3qFTVejtX72/ysCYUe0pyRc4/W+jlS1/Ymj+MK3GS4Ij+jx+v9pDttsccATPwyuHomYVqaiE6ePImu6+TltV7CIS8vj48++uic81esWMHy5cu7/kHhADRVx33JbPaiN7ezB7lXBz36ZfL5MLzxv3EmYPoAou0aAT9GO/tWmQEFNEfkQdCPETwdgwmRXR4UO17HIPILJpA9+iJIzyGnKcAoew3FWamMyktvXarInxA5gl5oPE6qt4ZUPUiRqtGgOzkZcuJV01BcbrLSXZCZEnuro+UfegiaT6I0V+FqrsbVVA2h5shKiEYYLdXJoJx0vKaDgOIkqDgJqimEFCe6LQXv8ALIGwz2FNLVFMYHISfdKcmnHXZN5bJ/+zy7dxTRsO8NbM2VOMwg+MMAmM3NGE1tJxelKQjRnwOzyYvR2M7Pc3MAQrbOnZvhh7A9eq4v8r7wGe12bfyRTYTlbURdsXTpUhYvXhx73NDQQHFxccdvzCyOjEOBs9ohFDz+MGr0G976pcgDe7oToouzuwNhhgX0M05RWp3uSnWAFj03pDPSHzlXUc/9JUxx2aM/PArusM6oQJiWP3eKomDaU8EWKT2ku2yxGAanO5k3sYMdKRypkZUKo6sVKoAnenRIs0d2vGhj14vs6NEZDiB3APa69wabpjL5whlw4Qy83mY0oznyR8E0yPYGcAZCcbdNAtDcToj+jGX6Q2h+Pf6JgC3DAdEOCLc/TEn0Zz8eR7oDbJFz0wNhHJoaexw5oefGdVmaiAYPHoymaZw4caLV8ydOnCA/P/+c851OJ05nAj/ZjrQ2F1J3R4/OSI8enZEaPTrDFT2EAEhNTQNO/5J7Mjv5R4Su/Tx39dzeZOmkV4fDwUUXXcSGDRtizxmGwYYNG5g5c6aFkQkh+pLlVbPFixczf/58pk2bxvTp03nsscdobm7mtttuszo0IUQfsTwR3XTTTVRXV7Ns2TIqKyuZMmUKL7/88jkN2EKIgcvycUTdMaDHEQkxAHT2d1QWRhNCWE4SkRDCcpKIhBCWs7yxujtamrc6PdVDCNGnWn43O2qK7teJqLExMj+rU6OrhRCWaWxsxONpe1hmv+41MwyDY8eOkZGR0W/mMrVMS6moqBhQPX0D8b4G4j1B396XaZo0NjZSWFiIqrbdEtSvS0SqqlJUVGR1GAlxu90D6oe7xUC8r4F4T9B399VeSaiFNFYLISwniUgIYTlJRH3M6XTy4x//OLFVBJLYQLyvgXhPkJz31a8bq4UQA4OUiIQQlpNEJISwnCQiIYTlJBEJISwniaiXPfjgg1xyySWkpqaSmZnZqfeYpsmyZcsoKCggJSWFOXPm8PHHH/duoF1UU1PDLbfcgtvtJjMzk9tvv52mpqZ233PllVeiKEqr45vf/GYfRRzf448/zrBhw3C5XMyYMYN//etf7Z7//PPPM3bsWFwuFxMnTuSf//xnH0XaNV25r7KysnO+Ly5XH6+ibopetWzZMvPRRx81Fy9ebHo8nk6956GHHjI9Ho/54osvmjt27DA/97nPmcOHDzd9Pl/vBtsFn/nMZ8zJkyeb77zzjvnGG2+YpaWl5s0339zue2bNmmXeeeed5vHjx2NHfX19H0V8rjVr1pgOh8N84oknzN27d5t33nmnmZmZaZ44cSLu+Zs3bzY1TTN//vOfm3v27DH/3//7f6bdbjd37drVx5G3r6v3tXr1atPtdrf6vlRWVvZpzJKI+sjq1as7lYgMwzDz8/PNX/ziF7Hn6urqTKfTaT777LO9GGHn7dmzxwTMrVu3xp576aWXTEVRzKNHj7b5vlmzZpnf/e53+yDCzpk+fbq5aNGi2GNd183CwkJzxYoVcc//8pe/bF577bWtnpsxY4b5jW98o1fj7Kqu3ldnfzZ7k1TNkkx5eTmVlZXMmTMn9pzH42HGjBlxd7+1wttvv01mZibTpk2LPTdnzhxUVWXLli3tvvfpp59m8ODBTJgwgaVLl+JtY8PK3tayy/CZX+f2dhmGyH2feT7A3Llzk+b7AondF0BTUxNDhw6luLiY66+/nt27d/dFuDH9etLrQFRZWQkQd/fbltesVllZSW5ubqvnbDYb2dnZ7cb41a9+laFDh1JYWMjOnTu599572bt3L2vXru3tkM/R1V2GIXLfyfx9gcTua8yYMTzxxBNMmjSJ+vp6Hn74YS655BJ2797dZ5PKpUSUgPvuu++cxr2zj7a+6cmst+/rrrvuYu7cuUycOJFbbrmFp556ihdeeIEDBw704F2Irpo5cya33norU6ZMYdasWaxdu5acnBx+97vf9VkMUiJKwH/8x3+wYMGCds8ZMWJEQtdu2eH2xIkTFBSc3vb5xIkTTJkyJaFrdlZn7ys/P5+qqqpWz4fDYWpqauLu0NuWGTNmALB//35GjhzZ5Xi7o6u7DEPke9OV862QyH2dzW63M3XqVPbv398bIcYliSgBOTk55OTk9Mq1hw8fTn5+Phs2bIglnoaGBrZs2cK3vvWtXvnMFp29r5kzZ1JXV8e2bdu46KKLAHj11VcxDCOWXDpj+/btAK0Sbl85c5fhG264ATi9y/C3v/3tuO+ZOXMmGzZs4J577ok9t379+qTalTiR+zqbruvs2rWLefPm9WKkZ7G0qfw8cOjQIfP99983ly9fbqanp5vvv/+++f7775uNjY2xc8aMGWOuXbs29vihhx4yMzMzzb/85S/mzp07zeuvvz4pu++nTp1qbtmyxXzzzTfNUaNGteq+P3LkiDlmzBhzy5Ytpmma5v79+83777/ffPfdd83y8nLzL3/5izlixAjziiuusOoWzDVr1phOp9MsKysz9+zZY951111mZmZmrOv661//unnffffFzt+8ebNps9nMhx9+2Pzwww/NH//4x0nbfd+V+1q+fLm5bt0688CBA+a2bdvMr3zlK6bL5TJ3797dZzFLIupl8+fPN4Fzjtdeey12DmCuXr069tgwDPNHP/qRmZeXZzqdTvOqq64y9+7d2/fBt+PUqVPmzTffbKanp5tut9u87bbbWiXX8vLyVvd5+PBh84orrjCzs7NNp9NplpaWmt///vctHUdkmqb561//2iwpKTEdDoc5ffp085133om9NmvWLHP+/Pmtzn/uuefM0aNHmw6Hwxw/frz5j3/8o48j7pyu3Nc999wTOzcvL8+cN2+e+d577/VpvLIMiBDCctJrJoSwnCQiIYTlJBEJISwniUgIYTlJREIIy0kiEkJYThKREMJykohEv1JWVtZqpcuf/OQnrebgLViwIDa1QfQfkohEXAsWLGhzKddFixahKEqrCbK9kQCGDRvGY4891uq5m266iX379rX5npUrV1JWVhZ7fOWVV7aaGyaSkyQi0abi4mLWrFmDz+eLPef3+3nmmWcoKSmxJKaUlJRz1kI6k8fj6fTa4CJ5SCISbbrwwgspLi5utXDZ2rVrKSkpYerUqd26drySyg033BArZV155ZUcOnSI733ve7G1kODcqtnZziyZLViwgE2bNrFy5crYNcrLyyktLeXhhx9u9b7t27ejKEqfLn0hTpNEJNq1cOFCVq9eHXv8xBNPcNttt/X6565du5aioiLuv/9+jh8/zvHjx7t8jZUrVzJz5kzuvPPO2DVKSkrOuSeA1atXc8UVV1BaWtpTtyC6QBKRaNfXvvY13nzzTQ4dOsShQ4fYvHkzX/va13r9c7Ozs9E0jYyMDPLz8xNafMzj8eBwOEhNTY1dQ9M0FixYwN69e2Nb7IRCIZ555hkWLlzY07chOkkWRhPtysnJ4dprr6WsrAzTNLn22msZPHiw1WF1S2FhIddeey1PPPEE06dP529/+xuBQIAvfelLVod23pISkejQwoULKSsr48knn+yxUoOqqpy9Ak0oFOqRa3fGHXfcEWuIX716NTfddBOpqal99vmiNUlEokOf+cxnCAaDhEIh5s6d2yPXzMnJadXuo+s6H3zwQatzHA4Huq5363Pausa8efNIS0tj1apVvPzyy1Its5hUzUSHNE3jww8/jP27LfX19bF1qFsMGjSI4uLic8799Kc/zeLFi/nHP/7ByJEjefTRR6mrq2t1zrBhw3j99df5yle+gtPpTKhKOGzYMLZs2cLBgwdJT08nOzsbVVVjbUVLly5l1KhRSbXu9PlISkSiU9xuN263u91zNm7cyNSpU1sdy5cvj3vuwoULmT9/PrfeeiuzZs1ixIgRzJ49u9U5999/PwcPHmTkyJEJb1awZMkSNE1j3Lhx5OTkcPjw4dhrt99+O8FgsE96AUX7ZKlYcd564403uOqqq6ioqDhnQ0LRtyQRifNOIBCgurqa+fPnk5+fz9NPP211SOc9qZqJ886zzz7L0KFDqaur4+c//7nV4QikRCSESAJSIhJCWE4SkRDCcpKIhBCWk0QkhLCcJCIhhOUkEQkhLCeJSAhhOUlEQgjLSSISQlju/wM8DZSjgrZJdgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:26.621078Z", + "iopub.status.busy": "2024-03-22T14:51:26.620698Z", + "iopub.status.idle": "2024-03-22T14:51:26.847753Z", + "shell.execute_reply": "2024-03-22T14:51:26.846754Z" + }, + "papermill": { + "duration": 0.249855, + "end_time": "2024-03-22T14:51:26.849809", + "exception": false, + "start_time": "2024-03-22T14:51:26.599954", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA43ElEQVR4nO3deVgT194H8C9hCSC7bGLRYFHRCxUXRGkVFcQNr1Sp1mJFa/VWL91wKfa2Wm/flqrgVr362FatVdSqlFpqVVxQXqWKW4uKqAhiWURECYskITnvH97MS2QLCiST+X2ex0cycyb5DcP8cubMmXOMGGMMhBBi4ES6DoAQQtoDJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIPAu2W3YsAESiQTm5ubw9/fHuXPnGi37zTffYMiQIbC3t4e9vT2Cg4ObLE8IMVy8SnZ79uxBdHQ0li5diosXL6JPnz4YNWoUSkpKGiyfmpqKqVOn4sSJE0hPT4e7uztCQkJQUFDQzpETQnTNiE8DAfj7+8PPzw/r168HAKhUKri7u+Pdd99FTExMs9srlUrY29tj/fr1mD59ulafqVKpUFhYCGtraxgZGT1X/ISQ1sUYQ0VFBdzc3CASNV13M2mnmJ6bXC7HhQsXsHjxYm6ZSCRCcHAw0tPTtXqP6upqKBQKODg4NFpGJpNBJpNxrwsKCtC7d+9nD5wQ0ubu3r2LF154ockyvEl2paWlUCqVcHFx0Vju4uKC69eva/UeH330Edzc3BAcHNxomdjYWCxbtqze8m+//RaWlpYtC5oQ0qaqq6vx9ttvw9rautmyvEl2z+urr77C7t27kZqaCnNz80bLLV68GNHR0dxrqVQKd3d3hIWFwcbGpj1C1QmFQoGUlBSMHDkSpqamug6HPCehHE+pVIq3335bqyYm3iQ7R0dHGBsb4969exrL7927B1dX1ya3jYuLw1dffYWjR4/ipZdearKsWCyGWCyut9zU1NSg/2jUhLKfQmHox7Ml+8abu7FmZmbo378/jh07xi1TqVQ4duwYBg8e3Oh2K1aswOeff45Dhw5hwIAB7REqIUQP8aZmBwDR0dGIjIzEgAEDMHDgQKxZswZVVVWYOXMmAGD69Ono3LkzYmNjAQDLly/HkiVLkJCQAIlEguLiYgCAlZUVrKysdLYfhJD2x6tkN2XKFNy/fx9LlixBcXExfH19cejQIe6mRX5+vsbt540bN0IulyM8PFzjfZYuXYrPPvusPUMnhOgYr5IdAERFRSEqKqrBdampqRqv8/Ly2j4gQggv8KbNjhBCngfvanaEkPqqq6s1+ptWPpbhTGYO7B3Pw8pCs3eBl5eXIPuMUrITKG1PDqGeGHxz/fp19O/fv97yFQ2UvXDhAvr169f2QekZSnYCpe3JIdQTg2+8vLxw4cIF7nV20SNE783Eqtd80LOTXb2yQkTJTqC0PTmEemLwjaWlpcaXkujOA4jTHqOXdx/4du2ow8j0ByU7gaKTgwgNJTtCeCq3tApVstoG1+Xcr+L+NzFp/DTvIDaBh2OHNolP31CyI4SHckurMDwutdly8/dlNlvmxIJhgkh4lOwEoqlaAEA1Ab5RH8s1U3zh6Vz/0ceqxzIkp6YjdNhgdLCoP7AFANwqqcQHey43+XdhSCjZCYC2tQCAagJ84+lsBe/OtvWWKxQKFDsB/braG/SoJy1ByU4AmqsFAFQTIIaPkp2ANFYLAKgmQAwfJTtCeEimrIHIvAC50myIzOvX1mtra1FYW4issqxG22BzpZUQmRdApqwB0PCXoCGhZEcIDxVW3UEHj6/xcTPTIP/n0H+aXN/BAyis8kV/uDRZzhBQshOA5moBANUE+MatQ1dU5b6LtVN88WID7bC1tbU4/b+n8fIrLzd6PHNKKvH+nstwG961rcPVC5TsBEDbWgBANQG+EBubQ1XTGR42PdG7Y8N3Y3NNctHLoVejbbCqmnKoau5DbNz4BFSGhJKdADRXCwCoJkAMHyU7AWiuFgBQTYAYPhqpmBAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7AghgkDJjhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7AghgkDJjhAiCJTsCCGCwLshnjZs2ICVK1eiuLgYffr0wddff42BAwc2Wn7v3r349NNPkZeXh+7du2P58uUYO3ZsO0ase48VSgDAlYLyRstUPZbh/H3A9c7DJmcXI/qhuWNKx7M+XiW7PXv2IDo6Gps2bYK/vz/WrFmDUaNGITs7G87OzvXKnzlzBlOnTkVsbCxCQ0ORkJCAsLAwXLx4Ed7e3jrYA93I+e8fdUxic3PCmuCHWxnNvl8HMa/+bAySdseUjmddRowxpusgtOXv7w8/Pz+sX78eAKBSqeDu7o53330XMTEx9cpPmTIFVVVVSE5O5pYNGjQIvr6+2LRpU4OfIZPJIJPJuNdSqRTu7u4oLS2FjY1NK+9R+yirkuNoVgm6OXWAhalxg2VuFJdj0U9ZWPFqL/RwbXx+iQ5iY0g60gTZutbcMRXK8ZRKpXB0dER5eXmz5ydvUrpcLseFCxewePFibplIJEJwcDDS09Mb3CY9PR3R0dEay0aNGoWkpKRGPyc2NhbLli2rt/zIkSOwtLR8tuD1gBWAkpLG1z+pKJig5FYmxMVNv9e1VoyLPLumjqlQjmd1dbXWZXmT7EpLS6FUKuHiojnRi4uLC65fv97gNsXFxQ2WLy5u/OgvXrxYI0Gqa3YhISG8rdlp44/8MiDzPAYNGoQ+XRx0HQ55TkI5nlKpVOuyvEl27UUsFkMsrt+ga2pq2ujcDIZAPcmOiYmJQe+nUAjleLZk33jT9cTR0RHGxsa4d++exvJ79+7B1dW1wW1cXV1bVJ4QYrh4k+zMzMzQv39/HDt2jFumUqlw7NgxDB48uMFtBg8erFEeAFJSUhotTwgxXLy6jI2OjkZkZCQGDBiAgQMHYs2aNaiqqsLMmTMBANOnT0fnzp0RGxsLAHj//fcRGBiI+Ph4jBs3Drt378b58+exefNmXe4GIUQHeJXspkyZgvv372PJkiUoLi6Gr68vDh06xN2EyM/Ph0j0/5XVgIAAJCQk4JNPPsHHH3+M7t27IykpSVB97AghT/Aq2QFAVFQUoqKiGlyXmppab9lrr72G1157rY2jIoToO9602RFCyPOgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkR6BUKnE+/X9Rde0kzqf/L5RKpa5DIqTVUbITuMTERHh6emL21DCU/rISs6eGwdPTE4mJiboOjZBWRclOwBITExEeHg5vb2/E/Hs5HMa8j5h/L4e3tzfCw8Mp4RGDYsQYY7oOQp9JpVLY2tqivLwcNjY2ug6n1VRUVMDLywt2dnZ4+PAhioqKuHWdOnWCvb09ysvLkZWVBWtrax1GSlqqsrIS4ydOxulL1/By3974JfFHWFlZ6TqsNtGS85NqdgK1a9cuFBYW4tq1axqJDgCKiopw7do1FBQUYNeuXTqKkDyLgQMHwtraGqkpv0FRegepKb/B2toaAwcO1HVoOkfJTqBMTExatRzRvYEDByIjI6PBdRkZGYJPeJTsBOrRo0etWo7oVklJSaOJTi0jIwOnTp1CdXV1O0WlXyjZCVTdS9fg4GB4e3vDwcEB3t7eCA4ObrAc0V+vvfaaVuUCAwNx/fr1No5GP9ENimYY6g2KHj164ObNm82W6969O27cuNEOEZHn0bVrV+Tn5zdbrlOnTrh16xYsLS3bIaq2RzcoSLOkUmmrliO6pe2lqUKhMJhE11KU7ATK2dm5VcsR3XJzc2vVcoaIN8murKwMERERsLGxgZ2dHWbNmoXKysomy7/77rvo2bMnLCws0KVLF7z33nsoLy9vx6j1V01NTauWI7oll8tbtZwh4k2yi4iIwNWrV5GSkoLk5GScOnUKc+bMabR8YWEhCgsLERcXhytXrmDbtm04dOgQZs2a1Y5R66/c3NxWLUd0i5olmtfiGxS3b99Gt27d2iqeBmVlZaF3797IyMjAgAEDAACHDh3C2LFj8ddff2ldNd+7dy+mTZuGqqqqRvuPyWQyyGQy7rVUKoW7uztKS0sN6gaFmZmZ1mWFXBvgC2dnZ626CdnZ2aGkpKTtA2onUqkUjo6OWt2gaHGPUU9PTwQGBmLWrFkIDw+Hubn5MweqrfT0dNjZ2XGJDnjSXUIkEuHs2bN49dVXtXof9S+kqY6ysbGxWLZsWb3lR44cEWzD7sGDB3UdAmlGU006T5czpOPZkj6DLa7ZXb58GVu3bsWuXbsgl8sxZcoUzJo1q017Z3/55Zf4/vvvkZ2drbHc2dkZy5Ytw9y5c5t9j9LSUvTv3x/Tpk3DF1980Wg5qtnVRzU7/ff08bSxsYFCoYCpqWm9S1dDOp5tWrPz9fXF2rVrER8fjwMHDmDbtm145ZVX0KNHD7z11lt488034eTkpNV7xcTEYPny5U2WycrKammI9UilUowbNw69e/fGZ5991mRZsVgMsVhcb7mpqSlMTU2fOxY+Eup+85k6wT1+/LjeOkM6ni3Zl+fuVCyTyfCf//wHixcvhlwuh5mZGSZPnozly5ejU6dOTW57//59PHjwoMky3bp1w44dOzB//nw8fPiQW15bWwtzc3Ps3bu3ycvYiooKjBo1CpaWlkhOTm7xZbehdioWiUTQ5tAbGRlBpVK1Q0TkeRgZGWld1pCeI2jR+cmeUUZGBps7dy6zt7dnL7zwAvvXv/7Fbt++zU6dOsWCgoKYn5/fs751PdeuXWMA2Pnz57llhw8fZkZGRqygoKDR7crLy9mgQYNYYGAgq6qqeqbPLi8vZwBYeXn5M22vr6ysrBiAZv9ZWVnpOlSihdDQUK2OZ2hoqK5DbVUtOT9b3PVk1apV8PHxQUBAAAoLC7F9+3bcuXMH//M//wMPDw8MGTIE27Ztw8WLF1v61o3q1asXRo8ejdmzZ+PcuXM4ffo0oqKi8Prrr3N3YgsKCuDl5YVz584BeJLxQ0JCUFVVhe+++w5SqRTFxcUoLi6mYceh/be7tuUI0XctbrPbuHEj3nrrLcyYMaPRy1RnZ2d89913zx1cXTt37kRUVBSCgoIgEokwadIkrFu3jluvUCiQnZ3N3Z25ePEizp49C+DJHeS6cnNzIZFIWjU+vqFOqIaFOok3r8XJLiUlBV26dIFIpFkpZIzh7t276NKlC8zMzBAZGdlqQQKAg4MDEhISGl0vkUg0aiHDhg2jWkkTFApFq5YjumVhYdGq5QxRiy9jX3zxRZSWltZbXlZWBg8Pj1YJihDSMpTsmtfiZNdYbamysrJdOhiT1vF0zfx5yxHdaqiLyfOUM0RaX8ZGR0cDeHKLe8mSJRpPEyiVSpw9exa+vr6tHiBpG6amphqdp5sqR/Tf/fv3W7WcIdI62V26dAnAk5pdZmamRo9tMzMz9OnTBwsWLGj9CEmbsLCw0CrZCfmyh0/qJjEjIyONK7C6rynZaeHEiRMAgJkzZ2Lt2rUG1cFWiCQSCS5fvqxVOaL/ysrKuJ+fbmqq+7puOaFpcYPM1q1bKdEZgH79+rVqOaJbDT3i+DzlDJFWNbuJEydi27ZtsLGxwcSJE5ssS7PI84O2zxy3xrPJpO25uLiguLhYq3JCpVWys7W15Z69s7W1bdOASPvQdsRmGtmZH7R9NrYlz9AaGq2S3datWxv8mfBX3busxsbGGo/Q1X1Nd2P5gboSNY+mexeouqPaGhsbY8iQIVCpVBCJRDhz5gyX7GiSbH5wd3fX6nl0d3f3dohGP2mV7Pr27at19bc1BwAgbadu1yG5XI7U1NRmyxH9NW7cOPz8889alRMqrZJdWFhYG4dB2puXl5dWk2R7eXm1QzTkeWk71PrBgwcxe/bsNo5GP2mV7JYuXdrWcZB29uqrr+KXX37RqhzRf+o5KExNTRscvEG9XNu5KgwRtdkJFN2NNSxWVlYAnoxSM3bsWJiZmSEnJwcvvvgi5HI5V/NTlxMirZKdg4MDbty4AUdHR9jb2zfZfifkHtp8op4nRD2k9dPUy7WdT4ToVlhYGJKSkmBiYoLMzEzcvXsXAJCZmYkuXbrAxMQEtbW1gm6S0irZrV69GtbW1tzPQu6rYyg6d+4M4EnNzdnZGS4uLigrK4ODgwPu3bvHzS2qLkf0W9euXQE8mZtFnejU8vPz65UToueecMfQGeqEO3K5HB06dICZmRlkMlm9fnZisRhyuRxVVVV0R5YHlEolOnXq1OSD/s7OzigsLISxsXE7Rta2WnJ+triHobGxcYMzij948MCgfomG7syZM6itrUV1dTUcHBwQHh6OESNGIDw8HA4ODqiurkZtbS3OnDmj61CJltSj2Dg4OKBTp07o0KEDOnXqBAcHBwDCHpIdeIYbFI1VBGUyGdUAeKSgoAAA4OHhgTt37mDfvn3cOpFIBA8PD+Tm5nLliH5LTU2FVCqFlZWVRrt5VVUVgCc3JqRSKVJTUxEUFKSrMHVK62SnntzGyMgI3377rcZdHaVSiVOnTlGfLB5RX+7k5ubWW6dSqbjlQh7/jE/UncIrKythZmaGiRMnwsLCAo8fP0ZiYiLX5YSSnRZWr14N4EnNbtOmTRqXrGZmZpBIJNi0aVPrR0jaRMeOHVu1HNEt9SxwpqamqKiogJGREQ4ePIixY8fi+++/h5WVFRQKhaBni9M62am/6YcPH47ExETY29u3WVCk7RUWFrZqOaJb6qG4nJ2dYWJionHDycTEBE5OTigsLBT0kF0tvkFx4sQJSnQG4MiRI9zPT7e11h3gsW45or/UNx8KCgowYcIE/P7773j8+DF+//13TJgwgfvSEvJNihbfoHjrrbeaXL9ly5ZnDoa0n7/++ov7+enHi+pe6tQtR/RXjx49kJKSAgA4evQokpOTuXV15xHp0aNHu8emL1pcs3v48KHGv5KSEhw/fhyJiYk0HBCP1L3B9HQn8bqvhfx4EZ+sXLkSwJOuYU+3y8lkMq6NXV1OiFpcs/vpp5/qLVOpVJg7dy5efPHFVgmKtL3hw4dzw3GNHDkS3bt3x40bN9CjRw/cvHkThw8f5soR/WdhYQE/Pz9kZGQAeDJ3iPpurPo4+/n5CXq2uFYZCEAkEiE6OhrDhg3DokWLWuMtSRurrq7mfj58+DCX3J5uo6tbjugvpVKJ+/fvc08TPD2upK2tLUpLS6FUKgXb+b/VxmjOyclBbW1ta70daWM0Z4FhSUtLQ15eHqRSKUaPHg1vb2907NgR3t7eGD16NKRSKXJzc5GWlqbrUHWmxTW76OhojdeMMRQVFeHXX39FZGRkqwVG2la3bt1atRzRLfWTLhKJBEeOHIFKpQLw5DHOa9euQSKRCP6JmBYnu0uXLmm8FolEcHJyQnx8fLN3aon+8PHx4X42NzfX6JJQ93XdckR/0RMxzWtxsjtx4kRbxEHaWd0/ehsbG7zzzjuorq6GpaUlEhISuGQn5JODT+r2fTUyMtJ4hr3uayH3kaWRigVKncRGjRqFo0ePYs2aNdw6ExMThISE4MiRI5TseCI9PZ37ualkl56eLtjmJt5MIllWVoaIiAjY2NjAzs4Os2bN0no8fcYYxowZAyMjIyQlJbVtoDyhHoG4pKSk3gCdbm5uXJKjkYr54c8//+R+buqJmLrlhIY3NbuIiAgUFRUhJSUFCoUCM2fOxJw5c5CQkNDstmvWrKG7ik9RJ7hLly7BxcUFH3zwAXcZu2vXLq5tlkYq5oeKigru56CgIISEhODmzZvo3r07jhw5gl9//bVeOaHhRbLLysrCoUOHkJGRgQEDBgAAvv76a4wdOxZxcXFwc3NrdNvLly8jPj4e58+fR6dOnZr9LJlMxg2CCDwZCRV48khVQ7M28ZWfnx9MTExgZmaG0tJSjctYY2NjWFpaQi6Xw8/Pz6D221Cpa+BmZma4cuUKl9yAJ3dozczMIJfL4eTkZFDHsyX70mrJ7q+//sK///1vbN68ubXekpOeng47Ozsu0QFAcHAwRCIRzp492+h0f9XV1XjjjTewYcMGuLq6avVZsbGxWLZsWb3lR44cgaWl5bPtgB7KzMxEbW0tamtrYWNjAx8fH4jFYshkMmRmZnJJfvXq1XRHlgfUNx7kcjkePnyIv//973B1dUVxcTFSU1O5R8js7e21nmOWD1rS6b3Vkt2DBw/w3XfftUmyKy4uhrOzs8YyExMTODg4oLi4uNHtPvzwQwQEBGDChAlaf9bixYs1+hJKpVK4u7sjJCTEoOagUD/HLJFIcPfuXZw+fZpbZ2JiAolEgry8PLi7u2Ps2LE6ipJoy9zcHImJiQCe/M0eOHCgwXJz5szBiBEj2jO0NqX+UtaGTi9jY2JisHz58ibLPOv4WwcOHMDx48fr9Qtsjlgs1mjQVTM1NYWpqekzxaKPHj58CAC4c+cOxo0bh5EjR3JtPCkpKdxl0MOHDw1qvw1VcHAwnJycmp1wJzg42KAeF2vJ36ZOk938+fMxY8aMJst069YNrq6u9Sb5qa2tRVlZWaOXp8ePH0dOTg7s7Ow0lk+aNAlDhgzhhrEWKvUIxE5OTvjpp5/AGONGtp03bx46d+6MkpISGqmYJ4yNjbFp0yZMmjSp0TIbN240qETXUjpNdk5OTlp1bRg8eDAePXqECxcuoH///gCeJDOVSgV/f/8Gt4mJicHbb7+tsczHxwerV6/G+PHjnz94nnvw4AGAJ11PJk6ciIULF3KDPa5cuZL7clGXI/zx9BMx6tFPhE7rZDdx4sQm17flWHa9evXC6NGjMXv2bGzatAkKhQJRUVF4/fXXuTuxBQUFCAoKwvbt2zFw4EC4uro2WOvr0qULPDw82ixWvlB/yfTt2xeZmZkYOnQot87DwwN9+/bFpUuXqJ8dTyiVSsyfPx/jx4/H/v37cfLkSfz2228YM2YMAgMDMWnSJCxYsAATJkwQbO1O62Rna2vb7Prp06c/d0CN2blzJ6KiohAUFASRSIRJkyZxM54BT25BZ2dn05BEWqrbzy40NBQffvihRpudeqRb6mfHD+pRT3bt2gVTU1MEBgaiqqoKgYGBMDU1xeLFixEQEIC0tDQMGzZM1+HqhNbJbuvWrW0ZR7McHBya7EAskUgandNWrbn1QjJkyBBIJBI4OjoiMzNTYxhviUSCAQMG4MGDBxgyZIgOoyTaKioqAgB4e3s3uF69XF1OiHjRqZi0PmNjY8THxyM8PBzjxo1DdHR0vbux+/btE+wlD9+oO8xfuXIFgwYNqrf+ypUrGuWEyIhpWd3RdvgmQ5twRyqVcqO/GlI/O7XExETMnz8feXl53DIPDw/ExcU1205L9IdSqYSnpyd8fHyQlJQEpVLJ3V03NjZGWFgYrly5gps3bxrUF1hLzk+tk51IJELXrl3Rt2/fJi8HG5qjgs8MPdkBT06UEydOcA3aw4cPN6gTQigSExMRHh6O0NBQLFy4EAUFBejcuTNWrlyJ5ORk7Nu3z+C+wFp0fjItzZs3j9nb2zNfX1+2du1a9uDBA2035bXy8nIGgJWXl+s6lDYll8tZUlISk8vlug6FPIf9+/cziUTCAHD/PDw82P79+3UdWptoyfmp9RBPGzZsQFFRERYtWoRffvkF7u7umDx5Mg4fPkwN/4ToiYkTJ+LWrVtISUlBdHQ0UlJScPPmTYOr0T2LFo1nJxaLMXXqVKSkpODatWv429/+hnnz5kEikWg9thwhpG0ZGxsjMDAQQ4cORWBgIDVJ/NczD94pEom4EVCVSmVrxkQIIa2uRclOJpNh165dGDlyJHr06IHMzEysX78e+fn5NHM8IXpCqVTi5MmTOHXqFE6ePEmVkf/Sup/dvHnzsHv3bri7u+Ott97Crl274Ojo2JaxEUJa6OmuRKtWrYJEIkF8fLzg2+20TnabNm1Cly5d0K1bN5w8eRInT55ssJx6TC1CSPuq2/Xkhx9+wF9//YUXXngBK1asQHh4uEF2PWkJrZPd9OnTaR4HQvSUeiCA0NBQrlPxgwcP4O/vj6SkJISFhdFAANoW3LZtWxuGQQh5HnUHAhCJRBrtdCKRiAYCAI+mUiRthxq0+a/uQAANHU8aCICSneAlJibC09MTI0eOxKpVqzBy5Eh4enpS2yvPqB/wX79+fYPHc/369RrlhIiSnYCpG7R9fHyQlpaGXbt2IS0tDT4+PggPD6eExyNDhgyBk5MTFi9eDG9vb43j6e3tjY8//hjOzs6CHrKLkp1APd2g7e/vDwsLC65BOzQ0FAsWLKBLWh6pewNR/QgnPcr5/yjZCZS6Qfvjjz+GSKT5Z6Bu0M7NzUVaWpqOIiQtkZaWhpKSEsTGxuLKlSsYOnQopk6diqFDh+Lq1av48ssvUVJSIujjSclOoGhkW8OiPk5RUVENDgQQFRWlUU6IKNkJVN2RbRtCI9vyS93j2dBAAHQ8KdkJlnoOii+//BIqlUpjnUqlQmxsLDw8PATdoM0ndDybR8lOoNRzUCQnJyMsLAy///47N29sWFgYkpOTERcXJ9je9nxDx1MLbTyQKO8Z+kjFQhvZ1tAJ7Xi25PzUeg4KoaI5KAjfCOl4tuT8pKkUCdegrZ5U2VBPDKGg49kwarMjhAgCJTtCiCBQsiOECAIlO0KIIFCyIzSeHREESnYCR+PZEaGgZCdgNJ4dERLeJLuysjJERETAxsYGdnZ2mDVrFiorK5vdLj09HSNGjECHDh1gY2ODoUOH4vHjx+0QsX6j8eyI0PAm2UVERODq1atISUlBcnIyTp06hTlz5jS5TXp6OkaPHo2QkBCcO3cOGRkZiIqKqjd+mxDVHc+OMabRZscYo/HsiOFp40fXWsW1a9cYAJaRkcEt++2335iRkRErKChodDt/f3/2ySefPNdnG+qzsQkJCQwA27FjR71nKSUSCduxYwcDwBISEnQdKnkGcrmcJSUlMblcrutQ2lRLzk9ePC6Wnp4OOzs7DBgwgFsWHBwMkUiEs2fP4tVXX623TUlJCc6ePYuIiAgEBAQgJycHXl5e+OKLL/DKK680+lkymQwymYx7LZVKAQAKhQIKhaIV90q3nJycAADTpk3D2LFj8f777yMnJwcvvvgiUlJSMG3aNK6cIe23ECiVSqSmpuLUqVMQi8UYNmyYwT4y1pK/TV4ku+LiYjg7O2ssMzExgYODA4qLixvc5vbt2wCAzz77DHFxcfD19cX27dsRFBSEK1euoHv37g1uFxsbi2XLltVbfuTIEVhaWj7nnugPuVwOkUgEc3NzZGRk4ODBg9w6JycnWFpaQiaT4eHDhxrriH5LT0/H1q1bUVJSAgBYtWoVnJ2dMXPmTAwePFjH0bW+6upqrcvqNNnFxMRg+fLlTZbJysp6pvdWD2D4j3/8AzNnzgQA9O3bF8eOHcOWLVsQGxvb4HaLFy9GdHQ091oqlcLd3R0hISEGNerJyZMnoVKpUF1dDSsrK7z33nuoqamBubk5du/ezf0R2dvbIzAwUMfREm389NNPWLFiBcaOHYsFCxaguLgYrq6uiIuLw4oVK7B79+4Gr4L4TH3lpQ2dJrv58+djxowZTZbp1q0bXF1duW8qtdraWpSVlcHV1bXB7dTDT/fu3Vtjea9evZCfn9/o54nFYojF4nrLTU1NYWpq2mSsfHLv3j0AgIeHB/Lz87Fu3TpunbGxMTw8PJCbm4t79+4Z1H4bKqVSiY8++oi7u65UKnHw4EG8/PLLGDp0KMLCwhATE4NJkyYZ1CVtS/42dZrsnJycuLajpgwePBiPHj3ChQsX0L9/fwDA8ePHoVKp4O/v3+A2EokEbm5uyM7O1lh+48YNjBkz5vmD57n79+8DAHJzcyEWizW6mJiYmCA3N1ejHNFv6rvru3btgkgk0jie6tniAgICkJaWhmHDhukuUB3iRR+MXr16YfTo0Zg9ezbOnTuH06dPIyoqCq+//jrc3NwAAAUFBfDy8sK5c+cAPJlDc+HChVi3bh327duHW7du4dNPP8X169cxa9YsXe6OXujYsWOrliO6RbPFNY8XNygAYOfOnYiKikJQUBBEIhEmTZqkcemlUCiQnZ2t0WD5wQcfoKamBh9++CHKysrQp08fpKSk4MUXX9TFLuiVus0CcrlcY13d1083HxD9VHd2sUGDBtVbT7OL8SjZOTg4ICEhodH1EomkwdnPY2JiEBMT05ah8VJpaWmrliO6VXd2saSkJI11NLvYE7xJdqR11b1J4+TkhIiICFRVVaFDhw7YuXMnV6Nr6mYO0R/q2cXCw8MxYcIEjBw5Ejdv3sSdO3eQkpKCX3/9Ffv27TOomxMtRclOoNS1YHNzc1hYWGD16tXcOolEAnNzc9TU1DRYWyb6aeLEiViwYAFWrVqF5ORkbrmxsTEWLFiAiRMn6jA63aNkJ1Dq54Nramrg7e2N6Oho3Lx5E927d8eRI0eQl5enUY7ov8TERKxcuRIWFhYag12YmZlh5cqVGDRokKATHiU7geratSv387Fjx/Drr79yry0sLBosR/SXUqnEO++802SZuXPnYsKECYK9lKWvbYEaMWJEq5YjupWamsr1iQwKCtIYnzAoKAjAkzvrqampOoxStyjZCdSwYcPqPW/8NGdnZ8F2QOWb48ePAwAGDRqExMRE1NTUICMjAzU1NUhMTOS6o6jLCRElO4EyNjbGxo0bYWRkBCMjI4116mUbN24U7CUP39y9exfAk87DPXr00Bhmv0ePHtxjk+pyQkTJTsAmTpyIffv2wcXFRWO5i4sL9u3bJ+jGbL5xd3cHAHz77bfw9vbWuIz19vbGli1bNMoJESU7gZs4cSJu3bqFlJQUREdHIyUlBTdv3qRExzN1R6ZhjHFdhur+/HQ5oaG7sQTGxsYIDAxEVVUVAgMD6dKVh+oes+PHjzd6d13Ix5ZqdoQYgKaeYa7bJivkZ50p2RFiANQP+MfGxta7y+7s7Iwvv/xSo5wQUbIjxACoBwI4c+YMbt68qdEGe+PGDaSnpwt+IABKdoQYAPVAAMnJyZg0aRLEYjH8/PwgFosxadIkJCcnIy4uTtBtdnSDghADoe5KFB0djaFDh3LLJRIJdSUC1ewIMThPdxInT1CyI8RAJCYmIjw8HD4+Phqdin18fBAeHo7ExERdh6hTlOwIMQBKpRLz58/nZhfz9/eHhYUF/P39kZSUhNDQUCxYsEBjIh6hoWRHiAFQzy728ccf1xuDUD27WG5uLtLS0nQUoe5RsiPEANDsYs2jZEeIAag7u1hDaHYxSnaEGIS6s4upVCqNdTS72BOU7AgxAHU7FYeFheH333/H48eP8fvvvyMsLIw6FYM6FRNiMNSdiufPn6/RqdjDw4M6FQMwYjRXXpOkUilsbW1RXl4OGxsbXYfTJuRyOb7++mscP34cI0aMwLvvvgszMzNdh0WekVKpxIkTJ/Dbb79hzJgxGD58uMHW6FpyflKya4ahJ7tFixZh1apVGv2vjI2NER0djRUrVugwMvI8FAoFDh48iLFjx8LU1FTX4bSZlpyfdBkrYIsWLcLKlSvrLVcqldxySnjEUNANCoGSy+WIi4sDUP9ZSvXruLg4yOXydo+NkLZAyU6g1q5dy81N8PRljvo1Ywxr165t99gIaQuU7ATq559/5n5+uvG67uu65QjhM0p2AlVeXt6q5QjRd5TsBMrJyYn7efjw4RpDAg0fPrzBcoTwGW+SXVlZGSIiImBjYwM7OzvMmjULlZWVTW5TXFyMN998E66urujQoQP69euH/fv3t1PE+q3upCznz59HZmYmHj9+jMzMTJw/f77BcoTwGW+6nkRERKCoqAgpKSlQKBSYOXMm5syZg4SEhEa3mT59Oh49eoQDBw7A0dERCQkJmDx5Ms6fP4++ffu2Y/T6x8Tk/w99SUkJ5s2b12w5QviMFzW7rKwsHDp0CN9++y38/f3xyiuv4Ouvv8bu3btRWFjY6HZnzpzBu+++i4EDB6Jbt2745JNPYGdnhwsXLrRj9Pqpa9eurVqO6A+5XI5169Zh8+bNWLduHXUf+i9efG2np6fDzs4OAwYM4JYFBwdDJBLh7NmzePXVVxvcLiAgAHv27MG4ceNgZ2eHH3/8ETU1NRg2bFijnyWTySCTybjXUqkUwJMe6QqFonV2SA8MHTqUm0vUzMxM44QQi8Xc72Do0KEGtd+GLiYmBmvWrOFGPjl48CAWLVqEDz74AF999ZWOo2t9Lfnb5EWyKy4urtd2ZGJiAgcHBxQXFze63Y8//ogpU6agY8eOMDExgaWlJX766Sd4eno2uk1sbCyWLVtWb/mRI0dgaWn57DuhZ5RKJWxsbLhkXpe6/52trS2qqqpw8ODB9g6PPINt27YhKSmp3nKVSoVVq1bh9u3bmDFjRrvH1Zaqq6u1LqvTZBcTE4Ply5c3WSYrK+uZ3//TTz/Fo0ePcPToUTg6OiIpKQmTJ0/mJiFpyOLFixEdHc29lkqlcHd3R0hIiME9G/vNN99gypQpjfaz27x5M8aPH6+L0EgLyeXyRq9w1H7++Wfs2LHDoAZ5aOjLujE6TXbz589v9pumW7ducHV1RUlJicby2tpalJWVwdXVtcHtcnJysH79ely5cgV/+9vfAAB9+vRBWloaNmzYgE2bNjW4nVgshlgsrrfc1NTU4B6onjx5MkxMTBAdHY07d+5wy11cXBAfHy/4IYH4ZM2aNag7pse0adPQv39/XLhwATt27ADwpMb+n//8BwsXLtRVmK2uReck44Fr164xAOz8+fPcssOHDzMjIyNWUFDQ4DZ//vknA8CuXbumsTwkJITNnj1b688uLy9nAFh5efmzBc8DtbW1LCUlhUVHR7OUlBRWW1ur65BICw0aNIgBYADY48ePmVwuZ0lJSUwul7PHjx9z6wYNGqTrUFtVS85PXtyN7dWrF0aPHo3Zs2fj3LlzOH36NKKiovD666/Dzc0NAFBQUAAvLy+cO3cOAODl5QVPT0/84x//wLlz55CTk4P4+HikpKQgLCxMh3ujf4yNjREYGIihQ4ciMDDQYMc+M2TZ2dkAgH79+sHc3Fxjnbm5OXx9fTXKCREvkh0A7Ny5E15eXggKCsLYsWPxyiuvYPPmzdx6hUKB7OxsrsHS1NQUBw8ehJOTE8aPH4+XXnoJ27dvx/fff4+xY8fqajcIaRPqdrjr16+jtrZWY11tbS1u3LihUU6IeHE3FgAcHBya7EAskUg02iwAoHv37vTEBBEEPz8/JCcno7q6Gp07d8Znn30Gc3NzfPvtt/jss8+4SoCfn5+OI9Ud3iQ7Qkjjdu3aBWtrawBNPxGza9eu9gxLr/DmMpYQ0jgrK6tma21+fn6wsrJqp4j0DyU7QgzEuXPnGk14fn5+3M07oaJkR4gBOXfuHCoqKjB+/Hh07doV48ePR0VFheATHUBtdoQYHCsrK+zfv18Qs4u1BNXsCCGCQMmOECIIlOwIIYJAbXbNUHdUbsnoCnykUChQXV0NqVRKbTwGQCjHU31ePv1AQUMo2TWjoqICAODu7q7jSAghjamoqICtrW2TZYyYNilRwFQqFQoLC2FtbQ0jIyNdh9Nm1OP23b171+DG7RMioRxPxhgqKirg5uYGkajpVjmq2TVDJBLhhRde0HUY7cbGxsagTw6hEcLxbK5Gp0Y3KAghgkDJjhAiCJTsCIAnw9EvXbq0wSHpCf/Q8ayPblAQQgSBanaEEEGgZEcIEQRKdoQQQaBkxyPDhg3DBx98oOswCOElSnYGKDU1FUZGRnj06JGuQyEtoG9fZvoWz/OiZEeIAZHL5boOQW9RsuMpmUyGjz76CO7u7hCLxfD09MR3332HvLw8DB8+HABgb28PIyMjzJgxo9n3q6ioQEREBDp06IBOnTph9erV9b7Zf/jhBwwYMADW1tZwdXXFG2+8gZKSEm69ukZ57NgxDBgwAJaWlggICBD0xMzamjFjBk6ePIm1a9fCyMgIRkZGyMnJwaxZs+Dh4QELCwv07NkTa9eurbddWFgYvvjiC7i5uaFnz54AgDNnzsDX1xfm5uYYMGAAkpKSYGRkhMuXL3PbXrlyBWPGjIGVlRVcXFzw5ptvorS0tNF48vLy2uvX0TYY4Y3AwED2/vvvM8YYmzx5MnN3d2eJiYksJyeHHT16lO3evZvV1tay/fv3MwAsOzubFRUVsUePHjX73m+//Tbr2rUrO3r0KMvMzGSvvvoqs7a25j6PMca+++47dvDgQZaTk8PS09PZ4MGD2ZgxY7j1J06cYACYv78/S01NZVevXmVDhgxhAQEBrf2rMDiPHj1igwcPZrNnz2ZFRUWsqKiI1dTUsCVLlrCMjAx2+/ZttmPHDmZpacn27NnDbRcZGcmsrKzYm2++ya5cucKuXLnCysvLmYODA5s2bRq7evUqO3jwIOvRowcDwC5dusQYY+zhw4fMycmJLV68mGVlZbGLFy+ykSNHsuHDhzcaT21trS5+Na2Gkh2PqJNddnY2A8BSUlIaLKdOOg8fPtTqfaVSKTM1NWV79+7llj169IhZWlpqJLunZWRkMACsoqJC43OPHj3Klfn1118ZAPb48WOtYhGyul9mjfnnP//JJk2axL2OjIxkLi4uTCaTccs2btzIOnbsqPE7/+abbzSS3eeff85CQkI03vvu3bvcl6S28fAJXcby0OXLl2FsbIzAwMBWeb/bt29DoVBg4MCB3DJbW1vukkjtwoULGD9+PLp06QJra2vu8/Pz8zXKvfTSS9zPnTp1AgCNy12ivQ0bNqB///5wcnKClZUVNm/eXO/37ePjAzMzM+51dnY2XnrpJZibm3PL6h5bAPjjjz9w4sQJWFlZcf+8vLwAADk5OW24R7pDQzzxkIWFRbt/ZlVVFUaNGoVRo0Zh586dcHJyQn5+PkaNGlWvUbzuyLjqMQBVKlW7xmsIdu/ejQULFiA+Ph6DBw+GtbU1Vq5cibNnz2qU69ChQ4vfu7KyEuPHj8fy5cvrrVN/QRkaSnY85OPjA5VKhZMnTyI4OLjeevW3vFKp1Or9unXrBlNTU2RkZKBLly4AgPLycty4cQNDhw4FAFy/fh0PHjzAV199xY3afP78+dbYHfJfZmZmGsfs9OnTCAgIwLx587hl2tS6evbsiR07dkAmk3EDAWRkZGiU6devH/bv3w+JRAITk4bTwNPx8B1dxvKQRCJBZGQk3nrrLSQlJSE3Nxepqan48ccfAQBdu3aFkZERkpOTcf/+fVRWVjb5ftbW1oiMjMTChQtx4sQJXL16FbNmzYJIJOJqZl26dIGZmRm+/vpr3L59GwcOHMDnn3/e5vsqJBKJBGfPnkVeXh5KS0vRvXt3nD9/HocPH8aNGzfw6aef1ktaDXnjjTegUqkwZ84cZGVl4fDhw4iLiwPw/zXtf/7znygrK8PUqVORkZGBnJwcHD58GDNnzuQS3NPx8L12TsmOpzZu3Ijw8HDMmzcPXl5emD17NqqqqgAAnTt3xrJlyxATEwMXFxdERUU1+36rVq3C4MGDERoaiuDgYLz88svo1asX1+7j5OSEbdu2Ye/evejduze++uor7gQirWPBggUwNjZG79694eTkhFGjRmHixImYMmUK/P398eDBA41aXmNsbGzwyy+/4PLly/D19cW//vUvLFmyBAC44+nm5obTp09DqVQiJCQEPj4++OCDD2BnZ8cNb/50PE+3FfINDfFEGlRVVYXOnTsjPj4es2bN0nU45Dnt3LkTM2fORHl5uU7afPUBtdkRAMClS5dw/fp1DBw4EOXl5fj3v/8NAJgwYYKOIyPPYvv27ejWrRs6d+6MP/74Ax999BEmT54s2EQHULIThPz8fPTu3bvR9deuXQMAxMXFITs7G2ZmZujfvz/S0tLg6OjYXmGSVlRcXIwlS5aguLgYnTp1wmuvvYYvvvhC12HpFF3GCkBtbW2Tj/o0dUeOEENByY4QIgh0N5YQIgiU7AghgkDJjhAiCJTsCCGCQMmOECIIlOyI3pgxYwY3Kq6pqSlcXFwwcuRIbNmypUXPZW7btg12dnZtF2gj1KMGE/1EyY7oldGjR6OoqAh5eXn47bffMHz4cLz//vsIDQ1FbW2trsMjfKbLkUMJqSsyMpJNmDCh3vJjx44xAOybb75hjDEWHx/PvL29maWlJXvhhRfY3Llz642WXPff0qVLGWOMbd++nfXv359ZWVkxFxcXNnXqVHbv3j3uc8rKytgbb7zBHB0dmbm5OfP09GRbtmzh1ufn57PXXnuN2draMnt7e/b3v/+d5ebmMsYYW7p0ab3PPXHiRJv8nsizoZod0XsjRoxAnz59kJiYCAAQiURYt24drl69iu+//x7Hjx/HokWLAAABAQFYs2YNbGxsUFRUhKKiIixYsAAAoFAo8Pnnn+OPP/5AUlIS8vLyNCYj+vTTT3Ht2jX89ttvyMrKwsaNG7nH5RQKBUaNGgVra2ukpaXh9OnTsLKywujRoyGXy7FgwQJMnjyZq5kWFRUhICCgfX9RpGm6zraEqDVWs2OMsSlTprBevXo1uG7v3r2sY8eO3OutW7cyW1vbZj/v6Tk0xo8fz2bOnNlg2R9++IH17NmTqVQqbplMJmMWFhbs8OHDzcZPdI9qdoQXGGPcwJNHjx5FUFAQOnfuDGtra7z55pt48OABqqurm3yP5ubQmDt3Lnbv3g1fX18sWrQIZ86c4bb9448/cOvWLVhbW3NzNjg4OKCmpsZg52wwNJTsCC9kZWXBw8MDeXl5CA0NxUsvvYT9+/fjwoUL2LBhA4CmJ4hWz6FhY2ODnTt3IiMjAz/99JPGdmPGjMGdO3fw4YcforCwEEFBQdwlcGVlJfr374/Lly9r/Ltx4wbeeOONNt570hpoqAui944fP47MzEx8+OGHuHDhAlQqFeLj47kRddXD0as1NHeCtnNoODk5ITIyEpGRkRgyZAgWLlyIuLg49OvXD3v27IGzszNsbGwajNPQ5mwwNFSzI3pFJpOhuLgYBQUFuHjxIr788ktMmDABoaGhmD59Ojw9PaFQKLi5MH744Qds2rRJ4z0kEgkqKytx7NgxlJaWorq6Wqs5NJYsWYKff/4Zt27dwtWrV5GcnIxevXoBACIiIuDo6IgJEyYgLS2Nm/fjvffew19//cV97p9//ons7GyUlpZCoVC0zy+NaEfXjYaEqEVGRnLdNkxMTJiTkxMLDg5mW7ZsYUqlkiu3atUq1qlTJ2ZhYcFGjRrFtm/fXm9S8HfeeYd17NhRo+tJQkICk0gkTCwWs8GDB7MDBw7Umzi6V69ezMLCgjk4OLAJEyaw27dvc+9ZVFTEpk+fzhwdHZlYLGbdunVjs2fPZuXl5YwxxkpKStjIkSOZlZUVdT3RQzSeHSFEEOgylhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7AghgkDJjhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgj/B6O+PK7MkOS0AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:26.891264Z", + "iopub.status.busy": "2024-03-22T14:51:26.890884Z", + "iopub.status.idle": "2024-03-22T14:51:27.172219Z", + "shell.execute_reply": "2024-03-22T14:51:27.171220Z" + }, + "papermill": { + "duration": 0.304536, + "end_time": "2024-03-22T14:51:27.174494", + "exception": false, + "start_time": "2024-03-22T14:51:26.869958", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWUElEQVR4nO2dd3iTZffHP0madNFBGR3QsilDlixBEZEiKOJ+ReSVraIsrYOhgoiKvgLyAxRkCSoIuHkF8cUKKLL3LqvQMroo3SNt8vz+CEmTNEmTNm2Scn+uK5fNM8+DyTfnPve5z5FJkiQhEAgEHobc1QYIBAJBRRDiJRAIPBIhXgKBwCMR4iUQCDwSIV4CgcAjEeIlEAg8EiFeAoHAIxHiJRAIPBIhXgKBwCPxcrUB1Y1Wq+XatWsEBAQgk8lcbY5AIDBDkiRycnKIiIhALrfhX0kuZMeOHdLDDz8shYeHS4D0008/lXvOtm3bpE6dOkkqlUpq1qyZ9OWXXzp0z6SkJAkQL/ESLzd/JSUl2fwuu9TzysvLo0OHDowaNYonnnii3OMTEhIYOHAgY8eOZc2aNcTFxTFmzBjCw8Pp37+/XfcMCAgAICkpicDAwErZLxAInE92djaRkZGG76o1ZJLkHguzZTIZP/30E4899pjVYyZPnsymTZs4ceKEYdszzzxDZmYmW7Zsses+2dnZBAUFkZWVJcRLIHBD7P2OelTAfvfu3cTExJhs69+/P7t377Z6TlFREdnZ2SYvgUDg+XiUeCUnJxMaGmqyLTQ0lOzsbAoKCiyeM3v2bIKCggyvyMjI6jBVIBBUMR4lXhVh6tSpZGVlGV5JSUmuNkkgEDgBj0qVCAsLIyUlxWRbSkoKgYGB+Pr6WjzH29sbb2/v6jBPAEiSRElJCRqNxtWmCNwUhUKBl5dXpVOVPEq8evTowebNm022bd26lR49erjIIoExarWa69evk5+f72pTBG6On58f4eHhqFSqCl/DpeKVm5vL+fPnDe8TEhI4cuQIISEhREVFMXXqVK5evcpXX30FwNixY1m0aBFvvvkmo0aN4s8//2TDhg1s2rTJVY8guIVWqyUhIQGFQkFERAQqlUokAQvKIEkSarWatLQ0EhISaNGihe1EVBu4VLwOHDhAnz59DO9jY2MBGD58OKtWreL69eskJiYa9jdp0oRNmzbx6quv8n//9380bNiQ5cuX253jVRFm/XoKf28vYvu1rLJ71ATUajVarZbIyEj8/PxcbY7AjfH19UWpVHL58mXUajU+Pj4Vuo5Lxeu+++7DVprZqlWrLJ5z+PDhKrSqlLScIlbsTADgoXZhtAoTeWHlUdFfUcHthTM+J+KTZoNijdbw97p9YpZSIHAnhHjZQKMt9Qp/OnyVwmIxgyYQuAtCvGxgPKLNKijm95PJrjNGUGXcd999vPLKK642o8awfft2ZDIZmZmZVXofIV420JjF48TQ8famur6UAvsQ4mUD/bBRqZAhk8Huize4lJ7nYqsEgqpHrVa72oRyEeJlA+0tzyvQR0nvlvUAWH9AeF/2IkkS+eoSl7wqWiylqKiIyZMnExkZibe3N82bN2fFihVcunTJkNZTu3ZtZDIZI0aMKPd6OTk5DB06FH9/f8LDw/n000/LDFO//vprunTpQkBAAGFhYTz77LOkpqYa9us9vri4OLp06YKfnx89e/YkPj7e7ud6//33qV+/PgEBAYwZM4YpU6bQsWNHw/4RI0bw2GOP8cEHHxAREUF0dLRdtgFs3ryZli1b4uvrS58+fbh06ZLddlUGj8qwr270npdcLuOZrpFsj0/juwNXiO3XEqVC6H55FBRraDP9d5fc+9R7/fFTOf7xHjZsGLt372bBggV06NCBhIQE0tPTiYyM5IcffuDJJ58kPj7e5pI0Y2JjY/nnn3/YuHEjoaGhTJ8+nUOHDpkIR3FxMbNmzSI6OprU1FRiY2MZMWJEmdUkb731FnPnzqVevXqMHTuWUaNG8c8//5Rrw5o1a/jggw/4/PPPufvuu1m3bh1z586lSZMmJsfFxcURGBjI1q1b7bYtKSmJJ554gnHjxvHCCy9w4MABXnvttXJtcgZCvGxgEC8Z9G0dSt1a3qTnFvHnmVT6tw1zsXUCZ3P27Fk2bNjA1q1bDaWXmjZtatgfEhICQP369QkODi73ejk5OaxevZq1a9fSt29fAL788ksiIiJMjhs1apTh76ZNm7JgwQK6du1Kbm4utWrVMuz74IMP6N27NwBTpkxh4MCBFBYWlpvkuXDhQkaPHs3IkSMBmD59Ov/73//Izc01Oc7f35/ly5ebLNkpz7bFixfTrFkz5s6dC0B0dDTHjx/n448/Lvffp7II8bKBftiokMlQKuQ81bkhS3ZcYN2+RCFeduCrVHDqvapb/VDevR3lyJEjKBQKg0BUlosXL1JcXEy3bt0M24KCggxDMj0HDx7k3Xff5ejRo9y8eROtVpdfmJiYSJs2bQzHtW/f3vB3eHg4AKmpqURFRdm0Iz4+npdfftlkW7du3fjzzz9NtrVr167MWsPybDt9+jTdu3c3Oae61hoL8bKB8bARYHDXSJbsuMCOs2lcyywgIrj8YcPtjEwmq9DQzVXYMwx0Nnl5efTv35/+/fuzZs0a6tWrR2JiIv379y8TNFcqlYa/9etG9WLiDPz9/StsmysQgRsb6HNUFbfEq0ldf+5qGoJWgu8OXHGhZYKqoF27dmi1Wnbs2GFxv94rsbfcT9OmTVEqlezfv9+wLSsri7Nnzxrenzlzhhs3bvDRRx/Rq1cvWrVqVSYgXlmio6NNbADKvLeEPba1bt2affv2mWzbs2dP5Y22AyFeNjAeNuoZ0k3nom84kGSSgS/wfBo3bszw4cMZNWoUP//8MwkJCWzfvp0NGzYA0KhRI2QyGb/++itpaWllYkbmBAQEMHz4cN544w22bdvGyZMnGT16NHK53OA5RUVFoVKpWLhwIRcvXmTjxo3MmjXLqc81YcIEVqxYwerVqzl37hzvv/8+x44dK7fqhz22jR07lnPnzvHGG28QHx/P2rVrLa5JrgqEeNnAfNgI0L9tGMF+Sq5mFrDx6FVXmSaoIhYvXsxTTz3Fyy+/TKtWrXj++efJy9Pl9jVo0ICZM2cyZcoUQkNDGT9+fLnXmzdvHj169ODhhx8mJiaGu+++m9atWxuC7PXq1WPVqlV89913tGnTho8++og5c+Y49ZmGDh3K1KlTef3117nzzjtJSEhgxIgR5Qb67bEtKiqKH374gZ9//pkOHTqwZMkSPvzwQ6fabw236R5UXTjSPWjX+XSeXb6X6NAAfn/1XsP2z7ef5z9b4okM8SUu9j5UXuI3oLCwkISEBJo0aVLhEie3A3l5eTRo0IC5c+cyevRol9nRr18/wsLC+Prrr11yf1ufF3u/o54TTXUB+uVBxp4XwMieTVj1zyWSMgpYtz+RYT0au8A6gSdw+PBhzpw5Q7du3cjKyuK9994D4NFHH602G/Lz81myZAn9+/dHoVDw7bff8scff5jkc3kiwmWwgXGelzG+KgUT+rYAYEHcefLVJdVtmsANSExMpFatWlZf+kKac+bMoUOHDsTExJCXl8fff/9N3bp1nWZH27ZtrdqwZs0aZDIZmzdv5t5776Vz587897//5YcffijTRtDTEJ6XDQwBe3P1Ap7pGsnyvy9y+UY+X/5ziXF9mle3eQIXExERwZEjR2zuj4qK4uDBg1Vqx+bNmykuLra4LzQ0FF9fX/74448qtcEVCPGygb4WodzCrIxSISe2X0smrTvCkh0XGNo9imC/ijcTEHgeXl5eNG/u+h+tRo0audoElyCGjTbQDxsteV4Ag9pH0CosgJzCEhbvuFCdpgkEtz1CvGwgWcjzMkYul/HmAN1Sj1X/XCI5q7DabBMIbneEeNmgdLbR+jF9ouvTpVFtikq0LPjzXDVZJhAIhHjZoLxhI+jWmE1+sBUA6/cnkSCKFQoE1YIQLxvoZxstBeyN6do4hD7R9dBoJeZtPWvzWIFA4ByEeNnA1myjOa/318W+/nv0GievZVWlWQI3o3HjxsyfP9/wXiaT8fPPP7vMntsFIV420NoxbNTTNiKIRzroiszN+d3+8ryCmsf169d58MEHXW1GjUeIlw00dg4b9cT2a4mXXMa2+DT2JWRUpWkCNyYsLAxvb2+X2iBJEiUlNXvlhxAvG5QG7O07vnFdf57uGgnAf7acqXATiBqDJIE6zzUvB//t7WmUYS/Gw8ZLly4hk8n48ccf6dOnD35+fnTo0IHdu3ebnLNz50569eqFr68vkZGRTJw40VDNAuxv0vHbb7/RuXNnvL292blzp8O2exIiw94Gko3lQdaY1LcFPxy8woHLN/nzTCp9W4dWlXnuT3E+fBhR/nFVwbRroPIv/7hb2NMoozK89dZbzJkzhxYtWvDWW28xZMgQzp8/j5eXFxcuXGDAgAG8//77rFy5krS0NMaPH8/48eP58ssvAfubdEyZMoU5c+bQtGlTateu7RTb3RUhXjYoXZhtv3iFBvow4u7GfLHjIp/8Hk+f6PplqlII3At7G2VUhtdff52BAwcCMHPmTNq2bcv58+dp1aoVs2fPZujQoQYvr0WLFixYsIDevXuzePFifHx87G7S8d5779GvXz+n2e3OCPGygcasDLS9vNS7GWv3JnImOYf/HrvGox0bVIF1HoDST+cBueredmJvo4zKYK15RqtWrTh69CjHjh1jzZo1hmMkSUKr1ZKQkEDr1q3tbtLRpUsXp9ns7gjxsoFhttEBzwsg2E/F2N7N+OT3eOb+7ywP3hF+exYslMkcGrrVZGw1z8jNzeXFF19k4sSJZc6LiopyqBGGeRONmsxt+I2yH/1sY3m1vi0x8u7G1K3lTWJGvuiy7ebY0yijKrnzzjs5deoUzZs3L/NSqVTV0qTDE3G5eH322Wc0btwYHx8funfvXqYTiTnz588nOjraMCvz6quvUlhYNQuiHZ1tNMZP5cXEvrpyKQvizomChW6MPY0yqpLJkyeza9cuxo8fz5EjRzh37hy//PKLoUZ+dTTp8ERcKl7r168nNjaWGTNmcOjQITp06ED//v2t/qqsXbuWKVOmMGPGDE6fPs2KFStYv34906ZNqxL7HElStcQzXaOIDPElLaeIVbsuOdEygbMpr1FGVdK+fXt27NjB2bNn6dWrF506dWL69OmGCYPqaNLhkUgupFu3btK4ceMM7zUajRQRESHNnj3b4vHjxo2T7r//fpNtsbGx0t133233PbOysiRAysrKKvfYT7fGS40m/ypN+/GY3dc358dDSVKjyb9K7WZskTLz1BW+jrtTUFAgnTp1SiooKHC1KU4hNzdXCgoKkpYvX+5qU2oktj4v9n5HXeZ5qdVqDh48aFJHWy6XExMTUyaBT0/Pnj05ePCgYWh58eJFNm/ezEMPPWT1PkVFRWRnZ5u87KWynhfAIx0aEB0aQHZhCUv+EgUL3ZXDhw/z7bffcuHCBQ4dOsTQoUOB6m2UIXAMl4lXeno6Go2G0FDTJM7Q0FCSk5MtnvPss8/y3nvvcc8996BUKmnWrBn33XefzWHj7NmzCQoKMrwiIyPttlHfU9aRPC9zFHKZYdH2l/8kkJotCha6K5YaZZw+fdpmkw2B6/CoVInt27fz4Ycf8vnnn9O9e3fOnz/PpEmTmDVrFu+8847Fc6ZOnUpsbKzhfXZ2tt0CpqlAhr0lYlrX586oYA4lZrLgz3O8/1i7Sl1P4Hw6depksVFGQUGBzSYbAtfhMvGqW7cuCoWClJQUk+0pKSmEhYVZPOedd97hueeeY8yYMQC0a9eOvLw8XnjhBd566y3kFkqeent7V3iRrDOGjaBLtXhzQCueWbqHdfuSeL5XUxrVuX3ycTwZX19ft2iyISiLy4aNKpWKzp07ExcXZ9im1WqJi4ujR48eFs/Jz88vI1AKhQKgShZB61MlnDFbflfTOvRuWY+SGl6wsCr+PwhqHs74nLg0VSI2NpZly5axevVqTp8+zUsvvUReXh4jR44EYNiwYUydOtVw/KBBg1i8eDHr1q0jISGBrVu38s477zBo0CCDiDkTTTkNOBzljVuxr41Hr3H6uv0TB56APoM8Pz/fxZYIPAH958R45YGjuDTmNXjwYNLS0pg+fTrJycl07NiRLVu2GIL4iYmJJp7W22+/jUwm4+233+bq1avUq1ePQYMG8cEHH1SJfc4aNuq5o0EQA9uHs+nYdeb8Hs+KEV2dcl13QKFQEBwcbMjR8/Pzq5YET4FnIUkS+fn5pKamEhwcXCmnQybdZn5+dnY2QUFBZGVlERgYaPPYt38+zjd7EpnUtwWv9mvplPtfTMul36d/odFKfD+2B10ahzjluu6AJEkkJyeTmZnpalMEbk5wcDBhYWEWf+Ds/Y561GxjdaOvYe8szwugab1aPN2lId/uS+LjLWfY8GKPGuOhyGQywsPDqV+/vtX28wKBUql0SphHiJcNJCelSpgzsW8Lfjx0lf2XbrI9Po0+reo79fquRqFQVEkMUiAwxuULs92ZihQjtIfwIF+G92wMwH9+jzfE1gQCgf0I8bJBaZKq86/9Uu9mBHh7cfp6Nv895qKCfQKBByPEywbaKvK8AGr7q3jh3qYAzNt6lmJ9gE0gENiFEC8baJywttEWo+5pQt1aKi7fyGeDKFgoEDiEEC8bODvPyxx/by/G99EtPfm/P85RoNZUyX0EgpqIEC8bGAL2Vdj9Z0j3KBoE+5KaU8Tq3Zeq7D4CQU1DiJcNnL08yBLeXgpDAuzi7RfIKhD5UQKBPQjxsoFUhbONxjzeqQEt6tciq6CYpaJgoUBgF0K8bFBVeV7mGBcsXLnzEqk5omChQFAeQrxsUNGmsxXhgTahdIwMpqBYw6I/z1f5/QQCT6dS4pWbm1vh+vCeQFXmeZmjK1io876+3ZdI4g1RWkYgsIXD4pWQkMDAgQPx9/cnKCiI2rVrU7t2bYKDg6ldu3ZV2OgyqmO20ZiezerSq0VdijUSn/5RcwsWCgTOwOGF2f/+97+RJImVK1cSGhpaYyoiWKI6ZhvNeaN/NH+fS+fnI1d5sXdTWoXZLtsjENyuOCxeR48e5eDBg0RHR1eFPW6FthIdsytK+4bBPNQujM3Hk5nzezzLh9ecgoUCgTNx+GvZtWtXkpJuj6Uses+rOmJexrz2QDQKuYw/Tqdy8HJGtd5bIPAUHPa8li9fztixY7l69Sp33HFHmRrU7du3d5pxrqaqlwdZo1m9Wjx1Z0PWH0ji4y3xrH/hrho9PBcIKoLD4pWWlsaFCxcMTTJAN1MmSRIymQyNpuaszzM0na1m8QKYFNOCn45cZV9CBjvOpnFfdM0qWCgQVBaHxWvUqFF06tSJb7/9tuYH7LXVH7DXExHsy7C7GrF8ZwKf/B7PvS3quUREBQJ3xWHxunz5Mhs3brwtGnFqXRTz0vNyn+as25/EyWvZbDp+nUEdIlxih0DgjjgcsL///vs5evRoVdjidpTmebnm/iH+Kp7vJQoWCgSWcNjzGjRoEK+++irHjx+nXbt2ZQL2jzzyiNOMczWuyPMyZ3SvJny1+xIJ6Xl8d+AKz3aPcpktAoE74bB4jR07FoD33nuvzL4aF7B30WyjMbW8vXi5T3Nm/XqK/4s7yxN3NsBHKTrzCAQOD4i0Wq3VV00SLjDK83JxoHzorYKFKdlFfCUKFgoEgIPiVVxcjJeXFydOnKgqe9wKrb7prItnVH2UCibFtADg8+0XyC4UBQsFAofES6lUEhUVVeM8LGtoq6jpbEV4olMDmtevRWZ+Mcv+uuhqcwQCl+PwsPGtt95i2rRpZGTU/GUr1VWM0B68FHJef0BXLnrFzgTScopcbJFA4FocDtgvWrSI8+fPExERQaNGjfD39zfZf+jQIacZ52oMeV5uUrKxf9swOjQM4uiVLD7bdp53H2nrapMEApfhsHg99thjVWCGe+LKDHtL6AoWtmLo8r2s2XuZ0fc0ITLEz9VmCQQuwWHxmjFjRlXY4ZZUdzFCe7i7eV3ubl6Hf87f4Js9l5n6UGtXmyQQuASHxUvPwYMHOX36NABt27alU6dOTjPKXdAvzHYXz0vPoPYR/HP+BqeuV77sdm5RCS98dYCY1qGMuqeJE6wTCKoHh6M5qamp3H///XTt2pWJEycyceJEOnfuTN++fUlLS3PYgM8++4zGjRvj4+ND9+7d2bdvn83jMzMzGTduHOHh4Xh7e9OyZUs2b97s8H3tQeMGSaqWiA4LAOBMck6lr7UjPo1dF27wzd7Llb6WQFCdOCxeEyZMICcnh5MnT5KRkUFGRgYnTpwgOzubiRMnOnSt9evXExsby4wZMzh06BAdOnSgf//+pKamWjxerVbTr18/Ll26xPfff098fDzLli2jQYMGjj6GXbhLkqo5LUJ14pWWU0RGnrpS1zp1PQuAAvXtkf4iqDk4PGzcsmULf/zxB61bl8Za2rRpw2effcYDDzzg0LXmzZvH888/b6gNtmTJEjZt2sTKlSuZMmVKmeNXrlxJRkYGu3btMqypbNy4saOPYDdaNwvY66nl7UXD2r5cuVnA2ZQc7mpap8LXOn1d570VFAvxEngWFVoeZL4YG3QJrFqt/VUP1Go1Bw8eJCYmptQYuZyYmBh2795t8ZyNGzfSo0cPxo0bR2hoKHfccQcffvhhlSXNuluqhDHRt7yvsymVGzqeuqaLm+ULz0vgYVSoJM6kSZO4du2aYdvVq1d59dVX6du3r93XSU9PR6PREBoaarI9NDSU5ORki+dcvHiR77//Ho1Gw+bNm3nnnXeYO3cu77//vtX7FBUVVai3pCRJpZVU3czzAmh5K+4VX4m4V0aemuRsXXdudYnWEOMTCDwBh8Vr0aJFZGdn07hxY5o1a0azZs1o0qQJ2dnZLFy4sCpsNKDVaqlfvz5Lly6lc+fODB48mLfeeoslS5ZYPWf27NkEBQUZXpGRkfbdy+h77G7DRnCO53XabLZSDB0FnoTDMa/IyEgOHTrEH3/8wZkzZwBo3bq1yfDPHurWrYtCoSAlJcVke0pKCmFhYRbPCQ8PR6lUolCUloRp3bo1ycnJqNVqVCpVmXOmTp1KbGys4X12drZdAmbshbhbwB5KZxzjk3MM/QMcxVy88tUl1PKucPaMQFCtVOiTKpPJ6NevH/369avwjVUqFZ07dyYuLs6Qta/VaomLi2P8+PEWz7n77rtZu3YtWq0W+a1A1NmzZwkPD7coXADe3t54e3s7bJ8+3gXulyoB0LSePwq5jOzCEpKzCwkP8nX4GuZ5YmLGUeBJVEi84uLiiIuLIzU1tUyQfuXKlXZfJzY2luHDh9OlSxe6devG/PnzycvLM8w+Dhs2jAYNGjB79mwAXnrpJRYtWsSkSZOYMGEC586d48MPP3Q4RcMejD0vdxw2enspaFLXn/OpucQn51RMvK6JYaPAc3FYvGbOnMl7771Hly5dCA8Pr1T3oMGDB5OWlsb06dNJTk6mY8eObNmyxRDET0xMNHhYoBuy/v7777z66qu0b9+eBg0aMGnSJCZPnlxhG6yhkYyHjU6/vFOIDg3gfGouZ1NyHG6Npi7RciEtFwBvLzlFJVox4yjwKBwWryVLlrBq1Sqee+45pxgwfvx4q8PE7du3l9nWo0cP9uzZ45R720Lr5p4XQMvQADYdv058cq7D555LzaFYIxHo40VYkA9nU3LFsFHgUTgsXmq1mp49e1aFLW6FyWyjG8a8AKLDagEVm3HUJ6e2Dg+ksEQ39BfiJfAkHB4QjRkzhrVr11aFLW6FRisRLUvEG7XbNtZteStd4lxqjsM5Wvp4V5uIQPxuNfTIFzEvgQfhsOdVWFjI0qVL+eOPP2jfvn2ZbPt58+Y5zThXIkuPZ71qFuekhpDfG/xCXG1SGRrV8cfbS05hsZbEjHya1PUv/6Rb6NMkWocHkngjH4ACdUmV2CkQVAUOi9exY8fo2LEjQJlGHO7qoVSI/BvIkegqj4eV/WHo91C7kautMkEhl9EitBYnrmYTn5xjt3hJksTp5FueV3ggO87qqoGIYaPAk3BYvLZt21YVdrgdBeHdeVY9g9WqjwlPPwsr+sGzGyCio6tNM6FlaAAnrmZzNiWHAXdYTu4153pWIZn5xXjdEj8xbBR4Im6aBOB6tJLEWSmSZ6UPIPQOyE2BLx+Cc3+42jQT9MuE4h0I2uuHjM3r18LbS4GfSidewvMSeBJCvKygD4Cny+vAyM3QpDcU58Hap+HQ1y62rhT9Au2zDizQ1gfrW4cHAuAjxEvggQjxsoJJz0afIF3Mq/1gkDSwcTxsmw2S66sw6D2vhPQ8ikrsEx/jeBeAn1IXPRDDRoEnIcTLChrzbtleKnj8C+j1mu79jo/gl/GgcW336vAgHwJ8vCjRSiSk59l1jnGOFyCGjQKPxGHx+uuvvygpKTulXlJSwl9//eUUo9wBvedlMoMqk0Hf6fDwpyCTw5FvYO1gKKp8LfmKIpPJSuNedg4dM/N1paNDA3UL1sWwUeCJOCxeffr0sdgtOysriz59+jjFKHegtPmGhZ1dRsEz34LSDy7E6QL5OZYLKFYHFS1MqNdlMdso8EQcFi9rtaNu3LhRpnu2J2OIeVnLXYseAMN/Bb+6kHwMlveDtPhqtNDIlEoWJiwdNookVYHnYHee1xNPPAHohikjRowwqZGl0Wg4duxYjVrzGB0WwN9vluNJNuwMY7bCN09CxkVY8QAMWQeNelSPkbdoWYF0CWMMw0bheQk8CLvFKygoCNB5XgEBAfj6ltaPUqlU3HXXXTz//PPOt9BFeHspiAzxK//AkKYweit8+wxc2Q9fPQpPLIW2j1W5jXr0VVWTMgrIKyrB38FqqIZho4h5CTwIuz/lX375JaBrNfb666/XqCFipfGvC8M2wg9jIH4TfDcCsj+AHuOq5fYh/irqBXiTllPEudRcOkYG2zzePMHDT6X7GIiAvcCTcDjmNWPGDCFcllD5weCvoesYQILfp8GWqeBAO7jKYIh7ORS018XzfFW6j4EYNgo8CYfFKyUlheeee46IiAi8vLxQKBQmr9sauQIemgMxM3Xv93wO34+A4sIqv7U+7nWmAq3QfG95XmLYKPAkHF6YPWLECBITE3nnnXcqXQa6RiKTwT2vQFBD+GksnPoFclPhmbVVWlanMoUJ9TEvfe9Gdy2+KBAY47B47dy5k7///ttQFkdghXZPQa36sO7fkLi7ysvqVGbG0VdV6jEXFGtE+zOBR+DwsDEyMhLJDdb0eQRN7oVRWyCwAejL6lw7UiW3anFLvNJyisjIU9s81vx/n7eX3JCwmi9yvQQegsPiNX/+fKZMmcKlS5eqwJwaSGgbGPNHlZfVqeXtRWSILn3F0aGjTCYzDB3FjKPAU3BYvAYPHsz27dtp1qwZAQEBhISEmLwEFgiMqJayOo5m2huHK31FoqrAw3A4uDF//vwqMOM2QF9WZ+N4OLZe99+sK3DfFFMVqQQtQwP443Sqw2scoVS8xIyjwFNwWLyGDx9eFXbcHujL6gQ1hL/n6srqZF2BQfNBoSz39PKIruACbSit6SWGjQJPoUL1vC5cuMDbb7/NkCFDSE1NBeC3337j5MmTTjWuRlKFZXWMZxxtTapY2ifK4gg8DYfFa8eOHbRr1469e/fy448/kpur69Z89OhRZsyY4XQDayxVUFanaT1/FHIZOYUlJGc7lhgryuIIPA2HxWvKlCm8//77bN26FZVKZdh+//33s2fPHqcaV+Nxclkdby8FTW+1P3N06CjK4gg8DYfF6/jx4zz++ONlttevX5/09HSnGHVboS+rE9IUshJ1ZXUu767w5QwNOeyYcTSeJhDDRoGn4bB4BQcHc/369TLbDx8+TIMGDZxi1G2HvqxOw65QmKkrq3Py5wpdqrQkdK5D51kbNpZoqmdhuUDgKA6L1zPPPMPkyZNJTk5GJpOh1Wr5559/eP311xk2bFhV2Hh7oC+rEz0QNEW6sjq7P3P4Mi3tyPWyFMq31IRj9ubTdHxvK4k38h22QyCoahwWrw8//JBWrVoRGRlJbm4ubdq04d5776Vnz568/fbbVWHj7YOhrM7zVLSsTrTRsFFfh98eLA0bv/jrIrlFJcyPO2v3dQSC6sLhPC+VSsWyZct45513OHHiBLm5uXTq1IkWLVpUhX23H3IFPPQJBEfC1um6sjrZV+HxpaD0Kff0qBA/fJRyCou1JGbk06SufbXXbPVuLCoWQ0eB+1Hh8gFRUVFERUU50xaBHpkM7p6kW9DtYFkdhVxGi/oBHL+aRXxyjv3iZSNgXyjSJwRuiF3iFRsby6xZs/D39yc2NtbmsfPmzXPYiM8++4xPPvmE5ORkOnTowMKFC+nWrVu5561bt44hQ4bw6KOP8vPPPzt8X7engmV1WobqxOtsSg4D7gizepxxLTYfw/KgsqkSRSXC8xK4H3aJ1+HDhykuLjb8bY2KFCZcv349sbGxLFmyhO7duzN//nz69+9PfHw89evXt3repUuXeP311+nVq5fD9/Qo9GV11jxVWlbn2Q0Q0dHqKfrChFZre1kIhRmqSlgYIgrPS+CO2CVe27Zts/i3M5g3bx7PP/88I0eOBGDJkiVs2rSJlStXMmXKFIvnaDQahg4dysyZM/n777/JzMx0qk1uh76szpp/QcoJXTb+019BixiLh7esQD17W0mqhSVCvATuR4XWNjoLtVrNwYMHiYkp/RLK5XJiYmLYvdt6ouZ7771H/fr1GT16dLn3KCoqIjs72+TlkThQVkc/43gxPY8iO4XHx0ZVCRGwF7gjdnle+oaz9vDjjz/afWx6ejoajYbQ0FCT7aGhoZw5c8biOTt37mTFihUcOXLErnvMnj2bmTNn2m2TW2MoqzMBjq2zWlYnLNCHAB8vcgpLuJiWR+vwwHIvXTpstCBeIuYlcEPs8ryCgoIMr8DAQOLi4jhw4IBh/8GDB4mLizM0pq0qcnJyeO6551i2bBl169a165ypU6eSlZVleCUlJVWpjVWOlwoeXwK9Xte93/ER/DIeNMWGQ2QyGa0cWCYEtns3ipiXwB2xy/PSN5wFmDx5Mk8//TRLliwxtDrTaDS8/PLLBAaW/wtvTN26dVEoFKSkpJhsT0lJISys7CzZhQsXuHTpEoMGDTJs095K4PTy8iI+Pp5mzZqZnOPt7Y23t7dDdrk9Mhn0fQeCGsCm13RldXKuw9OrwVsnWi1DA9h/6abFBdr6eL3x9Iq+d6OlYaMQL4E74nDMa+XKlbz++usmPRoVCgWxsbGsXLnSoWupVCo6d+5MXFycYZtWqyUuLo4ePXqUOb5Vq1YcP36cI0eOGF6PPPIIffr04ciRI0RGRjr6OJ6NjbI60Q56XvrejWLYKPAUHBavkpISi/GoM2fOGLwgR4iNjWXZsmWsXr2a06dP89JLL5GXl2eYfRw2bBhTp04FwMfHhzvuuMPkFRwcTEBAAHfccYdJiZ7bBitldRxthWbeu9EYZ4nX/ksZfL3nsug+JXAKDmfYjxw5ktGjR3PhwgVDIunevXv56KOPDILjCIMHDyYtLY3p06eTnJxMx44d2bJliyGIn5iYiFzu0klR90dfVuebJyHjIqx4gDaP62YikzIKyCsqwb+cXozGvRvz1SUE+FS+LLU5/1qim0FuUsefe1rYF7MUCKzhsHjNmTOHsLAw5s6dayiNEx4ezhtvvMFrr71WISPGjx/P+PHjLe7bvn27zXNXrVpVoXvWOPRldb59Bq7sJ3DDUwz2n8D6vDs5m5JDp6jaNk/X926UJN3QsSrES8/ljDzuQYiXJ3H5Rh5vfH+MNuGBTHmwFT5KRfknVTEOuzRyuZw333yTq1evkpmZSWZmJlevXuXNN980iYMJXIBZWZ3ZmrmMUvxWJu6lvTVsM14QUZ29GxVGN37ju6NM/v5Yld5PUDmuZhbw7LK97EvIYNWuSzz9xW6uZha42qzKJakGBgY6PMMoqGKMyurIkZiu/JpG+983lNUpUGsMM4q1/U1jhLbanxU7sSihXK4Tr8x8Nd8dvML6A0ncyC1y2vUFziMlu5Bnl+3hamYBjer4UdtPybErWQxauJNdF1xbOblCVSW+//57NmzYQGJiImq1aWv5Q4cOOcUwQSW4VVbnaE4AHc7M467U9fB9ITy+lNQcnTD5KhUEmMXBbDWezVdrCPJ1TuxR73kp5KUeWL5aQx2nXF3gLNJzixi6fC+Xb+QTGeLLuhfuokQjMfabg5y8ls1zK/Yx9cFWjL6nSYXWNVcWhz+NCxYsYOTIkYSGhnL48GG6detGnTp1uHjxIg8++GBV2CioCDIZ0t2TmKgeTzFeurI6Xz9Gepoupy400LvMB868d6PxbkvVJiqKfv7FeM5RNLt1LzLz1Ty3Yh/nU3MJD/Jh7Zi7CA/yJTLEjx9e6skTdzZAo5V4f9NpJq474tTPh704LF6ff/45S5cuZeHChahUKt588022bt3KxIkTycrKqgobBRWkRf1abNT2ZJh6MlrvQEjcTctfn6ShLI36gWULGxqvb9RqJYwzGiorLlqj9Av5LVU0vr4lb0/gGrILixm+ch+nr2dTL8CbNWO6ExniZ9jvo1Qw918dmPlIW7zkMv579BpPfL6LyzfyqtVOh8UrMTGRnj17AuDr60tOji4Y/Nxzz/Htt9861zpBpfD39iIyxJfd2rYcf2A9BDYgIPciP6pm0FmVWOZ44/WNGrNcrPyiyolLsVEOoGG4aCKOouWaO5CvLmHUl/s5eiWL2n5K1ozpTtN6tcocJ5PJGN6zMWufv4u6tbw5k5zDoIU72RafWm22OixeYWFhZGRkALpqqvpejQkJCSL50A2JDtVNqBwpioAxf5Di25z6skxeTZoE5/4wOda4LI55ompeJcWlWFN6vXMpuTz/1QFOJ5dW+BAt11xPYbGGMasPcODyTQJ8vPh6dHdDsrM1ujUJYdPEe7gzKpjswhJGrdrPgrhzJp52VeGweN1///1s3LgR0CWsvvrqq/Tr14/Bgwdb7OcocC0mhQkDI5jXcD47NW1RaQvKlNUxGTaae16VFC/jFmrfH7zC1lMprN9fukheDBtdS1GJhrHfHGTXhRv4qxSsHtWNOxrYV2ghNNCHdS/0YGj3KCQJ5m09y4vfHCS7sLj8kyuBw7ONS5cuNSwDGjduHHXq1GHXrl088sgjvPjii043UFA5zAsTXs7zYmTxZP5s/D2RSRtNyuqYDBu15uJVyWGjkeeVmlNo8l9nXF9QcYo1WiZ+e5jt8Wn4KOWsHNGVO8tJajZH5SXng8fb0aFhMG//coKtp1J4bNE/LB3Wmeb1bXtvFcUhz6ukpIT333+f5ORkw7ZnnnmGBQsWMGHChNtzbaGbo1+gHZ+SgyRJpGYXUYwXV+/7tExZnVpKncAUqDVlPKFKx7yMPC+9kKXllOZ22Vu5QpIkh1q6CWyj0Uq8tuEov59MQeUlZ9mwLnRvWvGklae7RvLdiz2ICPLhYnoejy76hy0nyjapdgYOiZeXlxf/+c9/KCkRwVVPoWndWnjJZeQUlpCcXUhKts7bCQ3y1ZXVefhTkMnhyDc8lzAZfwrIV2vKxKAqG/Mq0ZQVHGPxstfzmvDtYbp/GEdOFQ9Jbge0WokpPxxj49FreMllLB56J71a1Kv0dTtEBrNxwj3c1TSEPLWGsd8c4j9bzjj9R8fhmFffvn3ZsWOHU40QVB0qL7mh/dmhy5nk3RKJ+gG3apwZldVpmrWH9apZeOWnlBGTyg7r1BYy9G/mlwqQvdf/9dh10nOL+O1EcvkHC6wiSRIzNp7ku4NXkMtgwZBO9G0dWv6JdlK3ljffjO7OmHuaALB5x042LHiTzHx1OWfaj8MxrwcffJApU6Zw/PhxOnfujL+/aV/ARx55xGnGCZxDdFgA51Jz+ftcGgAB3l6mVSZuldUpWP0kdxRfIuLcWK5Hf2NyjUoH7Mspl2Sp8cfByzfZdiaV8fc3L7MQ2Ete/RndNQVJkvhw82m+3nMZmQzmPt2Bh9qFO/0+Xgo5bz/chnv9LtFux7vUzsxhz28NuevJSc65vqMnvPzyy4Dl/owymQyNRgRe3Y3o0AB+5Tp/ndWJV/1AC5VlG3ZmS49v6Lh9FE2KkwnY9CRdZJM4ILUCIK+yMa8S20OGPAue15OLdwFQy8eLsb1NK+QqhHhVmI1Hr7Hs7wQAPny8HY93alh1NzuziXt3jQJZIdf9W9P9gWecdmmHh41ardbqSwiXe9LyVtD+WtateJeF7HoAbVBjnlTP5LyqNUp1FmtUs3lQvheovOdVXK7nZf2zcym9bOa2EK+Ks+fiDQCG92jEkG5V2PV+3zJY/28oKYQW/QmfFIcswHlDU1Hl7zYg2izR0Jp4+akUZBDIjNofcj28L96yYj5TLmCU4rdKx7wsBeyNySuyLo4qr7IfU4ULFgLXFBIz8gFo1zC4am6g1cLWGbD5dZC0cOdweGYtqPzLP9cB7B42FhQUEBcXx8MPPwzouvIUFZXOFikUCmbNmoWPj+UvhsB1RIb44aOUU3ir/6LFYSOlSaqZxUr+6j6XwqQ3GO61lenKr/lfch5oV1AswaHLN+kYFYy3l/3128orqWNLHFUKC+IlPK8KoxevKKP1ik6jpAh+GQfHv9O9v/9tXUpOFfzY2O15rV69mi+++MLwftGiRezatYvDhw9z+PBhvvnmGxYvXux0AwWVRyGX0cIoUTA0wIrnZZSkmlcMM0pGMLtkCAAPZP8I349g/e5zDF66h8XbLzhkQ/niVb7nZbzk5HYXr6ISDVtOJJNV4FjKSIlGy7VMXfggMsTXuUYVZOpKkR//DuRe8NhiuPeNKhEucEC81qxZwwsvvGCybe3atWzbto1t27bxySefsGHDBqcbKHAO+mRVsDVsLC2Jo0tSlfGj71MmZXV6/DOaIHJJynCskmax2bDRfLbQpud1S7wsLu6+Tfn4t3jGfnOQUav2O3Te9axCNFoJlUJu9UesQmRdgS8fhEt/g6oWPLsBOj7rvOtbwG7xOn/+PO3atTO89/HxMWmM0a1bN06dOuVc6wROwzjuFWpl2Gjcu1EfQK/jr2KjtidTfGeAdxDNCo7zg+pd/PKvOnT/EjPPq5lZpQJz8TL21PTipRGel4HvD+rWhR68fNOh85JuDRkbhvgaKtpWmpSTuq5VqaegVhiM/A2a93XOtW1gt3hlZmaaxLjS0tJo3Lix4b1WqzXZL3AvWtrheRn3btSLSd1aOqHbpW2LduRvXJdCaC6/xmtJ4+DaEbvvb56k2ibCtHy4+bDReHmSPuZl7L3d7uJV0Vx1fbwrsraT4l0Xd8DKAZBzDepG67pYhbd3zrXLwW7xatiwISdOnLC6/9ixYzRsWIX5IoJK0eqWeHnJZdQLsOx5GfduzC3SxVLq1tKtV80rKuGqqgmPFb3HaW0UwZoMXZNbs7I61jCfbWwdbjoDau55FRq91wuVieclMx92ljDh28NsOlY16+hqCk4N1h/7ThfjKsqGRnfD6N8huApTL8ywW7weeughpk+fTmFhYZl9BQUFzJw5k4EDBzrVOIHzCA30YdajbfnwiXZW21YZ927MyNMt49B7XvlqDedTc0khhH+pp3NE2RGK88qU1bGGeYZ98/plh43GAXljMdOLlvHQ07yE9dK/LvLfo9cYt/Y26aFQQdcr6aYuVlmpYL0kwc5P4ccxoC2Gto/Dv38EX8cqUVQWu1Mlpk2bxoYNG4iOjmb8+PG0bNkSgPj4eBYtWkRJSQnTpk2rMkMFlee5Ho1t7jfu3XjjlnjVuSVeJVqJU9d1xQNz8WOa7ztsbv0dHFtnUlbH2syS2szzsvTLX1iiKZ00MBo26muLlRiJm3nhy2tu0IrLE6i056XVwG9vwv7luvc9xkO/WaWNCaoRu8UrNDSUXbt28dJLLzFlyhTDh0cmk9GvXz8+//xzQ5drgWei792Yp9YYPK86tUrLHB1JyjT8nV0sR/PoYvak+3D3tVW6sjpZV2DQfFCUbVhrHrAPDyr7y59XZFm89KcaDz3NHQ/z2UyBZa7oY14VES91PvwwBuI3ATLo/yH0eNm5BjqAQ2sbmzRpwpYtW8jIyOD8+fMANG/enJCQkCoxTlD9+Kp04nUjVydegT5eqBRy1Botx65kGo4rLNZwOCmToRcf4AU/b6ZKy5Ad+QZyrsPTq8HbNKZlnuflpyo7dNUF7XWeXmp2aXhCc2vIaTz0NK84bqlqhcCUvKISg0ftsHjl3YBvB8OV/aDwhieWQtvHnG+kA1TI1wsJCaFbt25069ZNCFcNQx/3yr21XMdX5YWft25bSrZp/S29d7Y0vzcpD30JSj+4EKcL5OeYlqwx94xkMhn+ZgJmHOcyTgEweF426kGZe3aCsiTd1HldwX5KAn3KesdWybgIK/rphMsnGIb94nLhArG2UWCGvnej4b1Kgb+qrINeUKwhp7A0veEvOsPwX8GvLiQf0+X9pMUb9lsqRKePp+kxTpc4YCJetzwvk2Gj6fVut2FjRZ428UYF4l1XD+r+X2ZcgKAoGP0/aNSjAnd3PkK8BCb4mHlDvkqFySykPsFVF9Qv9cT2X8qAhp11eT4hzSArEVY8AJd3G443xzieBqWeV2GxhhNXS3uAagwBeyPvyux65S0/qmlUpFOXwzle8Vtg1cOQnw5h7XX/b+tFO3zfqkKIl8AEP7M0Cl+VwmR4165BsOFv42Hk/ksZHLiUQZfPL/Jb96+gYVcozISvHoWTP5fxlADq+Jt6XvqaYceuZJl4UpaGjWUD9reXeJmz81w6zyzdzcW0XKvHXDGkSdghXge+hHVDoDgfmvWFkZshIMxZ5joFIV4CE8wD6X4qhWEGEHTJrvrlOqlGNegv3cjnqSW7Sc8tYuLGRBi2EVo9DJoi+G4Ena6uLXOvumaeV0Gxbth44HKGyXaLw8YyntftNWw0598r9rLnYgaT1h2xeozB87KV4yVJ8Of78OsrunI2Hf8Nz64vMwHjDgjxEphgPmz0U3rh7126rUVoLXyV+gB+2YRlgNp+KlD5wdNfkXnHcECid8KnvOP1NTJKPSTzYaPe8zp4SRfvKl3TqNtvMttYJuZ1e3teevSTKJZIKi/Hq0QNP78Ef32ie997Cjy6yGLqizvgFuL12Wef0bhxY3x8fOjevTv79u2zeuyyZcvo1asXtWvXpnbt2sTExNg8XuAYloaNvkaeV/P6tQzemb77j3mxwIa1db/saq2Mjgce4MNiXVmd0V6/sUi5AG90X7BgX/OYl87zOpSoE6/uTXQz2fZ4XuqS20u8rPmZXgrLScKSJNlOUC3M1q2WOPotyBTwyELoM7XKytk4A5eL1/r164mNjWXGjBkcOnSIDh060L9/f1JTUy0ev337doYMGcK2bdvYvXs3kZGRPPDAA1y96liVA4FljIeNXnIZKi+5IeYlk+mqQeg9L30u1hOdGphcQz8L+eeZFEDGUs0gJjMRtaRgoGIfX6tmQ35Gme9FvlqDukRr6CqkrzyhD9hrbMS8bKVR3E5YW7CellNEUYkWuQwigs2GjdnXdektF7eB0l83TLxzWDVYWzlcLl7z5s3j+eefZ+TIkbRp04YlS5bg5+fHypUrLR6/Zs0aXn75ZTp27EirVq1Yvnw5Wq2WuLi4ara8ZmI8bNSLlD7mFRXih4/R7KO+acZ90fX4enQ32oTrKkXoxWvDgSuGa60vvIvhxVPIlvzoJo+Hlf3xz79mcu98tcakH2Ogr264oh8RGg8NzWfbbO2riVh7RGtdlfQ5XuFBviiNK9OmnoHlMZByHPzrw8hN0KKfs82tElwqXmq1moMHDxITE2PYJpfLiYmJYffu3XZdIz8/n+LiYqvJskVFRWRnZ5u8BNYxzvPSi5TeG2txazG1r9nQMsBHSa8W9fh86J0ABgHan2AaeN+tbctT6hkU+IZD+lkePTictrIEw/58dYlB+PxUCrwNMS/trf/amG00Gjaae2Gbj183Sb2oySisrDG0OGS89A+sfACyr0Cd5rpUiIhO1WGmU3CpeKWnp6PRaMqsiQwNDSU52b6mopMnTyYiIsJEAI2ZPXs2QUFBhldkZGSl7a7JGA8b9X93blwblUJOzK2mpL4qc/HyMvlvnlpDiUZrsct2j7vuwfelPyH0DvzU6WxQvUdv+VEA8otKE18DfLyQy/SlcHTnFhuJknmJHeN9xiJ3KPEmL685xMMLd9r7T+ARWEo9ARueV4ZZNYkTP8LXj0FhFkR2h9FboXbjKrC06nD5sLEyfPTRR6xbt46ffvrJauOPqVOnkpWVZXglJSVVs5Wehcmw8dZwsU90fU7M7M8zt9pkWfK8QDfM08dcrmYWoNeQBkYxFplMBoERMPI3rtTujr+siBXKT/iXYvutrP1iwzW9DHW8tCb/BfjPljMmNhgPG43/Ppuc4+C/gOtIyS6stIdoLeZl8Lxq+8KuRfD9SNCodeksw34BP89b5udS8apbty4KhYKUlBST7SkpKYSF2U6ImzNnDh999BH/+9//aN/eeuVGb29vAgMDTV4C6xjPNhp7YcYziua5YHqPS6mQ06yerr3V4cRMw/67m9cx/G0I0vsE8lv7BfyguQcvmZZPlEsZkL6K7IJiwzX1ZYr1TpZxLte5VNNkTONho7HnpfGg+Ff3D+N4eOFOLthINC0Pa+KVlJGPHC0PXl0A/3tLt7Hbi/D0V6B0ciOOasKl4qVSqejcubNJsF0ffO/Rw/r6qf/85z/MmjWLLVu20KVLl+ow9bbBz0LA3hxrw0aAVmG6H4fDt9IdFHIZ3ZvUwRIauZLXil9iYcljADya+RXR+6biRYmJ56XVShy7ksmb3x+zardxnOuej7dxLiXHcK6ncfxKxb0va+KVcuMmi5QLaHbxVuHIB96HBz8Guf3t69wNlw8bY2NjWbZsGatXr+b06dO89NJL5OXlMXLkSACGDRvG1KlTDcd//PHHvPPOO6xcuZLGjRuTnJxMcnIyubkV/7USlGI6bLT8wTauxKrykpv0b2x9a8bx8K3aX75KBV0al1bYlFH65dI5RTLmljzNtOLRaJHTJOknVijnUFepNnheJVptuV1yjB2s3KISXvtOF0eztCDc3alMapXSQp5XUXYanxTO4CHFPiSFCp5cAT0nuHUOlz04VM+rKhg8eDBpaWlMnz6d5ORkOnbsyJYtWwxB/MTERJMuRYsXL0atVvPUU0+ZXGfGjBm8++671Wl6jcTasNHkGKPtgT6mH6FWt2rTn7qmm9X1UcqJCvGjbi0V6blqk++LcdB5raYvAfUieT37I3pzjKZXXmF/lK5PqEZLuf0JzYeHx65kcfDyTTxQu8qUuLaE8ePO+19p9Y4ys403LyFb/QRd5RfIlvwI+Pd6aHKvs0x1KS4XL4Dx48czfvx4i/u2b99u8v7SpUtVb9BtjPE6RmviZTycrOVt+hHS53rph3E+SgUymYxOUbXZeioF46+leThqj1dXNnVezj37Xiay6Bwhe/5NM9mraKX6t2YerSuRJQ/rycW7mPpgK6vnuCv2NEYyftoFf543/G0y23jtCKz5F6q8VK5KdZgZMJOlNUS4wA2GjQL3Qt+7EbDRqKNUsALMitrVD/Cmtl/pNr3QPdIhAqVCRseoYKv3VmskUgPa8IR6JqmqhvgXXOMH1bs0zT9uNQUAbMe1ijxw2ZC8EsM5Q8zr3B+6rPm8VG4GtOSJoplo63mekNtCiJfABF8HPa8As2GjTCYzxL1019MdO6hDBKfeG8DD7SOs3rtYo0WSIFEKZWGjz8io3YFgWR5vpr7JAPleq+fZmlE0XvNYmVm86qQykSgvuUzXzWnt07ruTk3vY2XLxaQQUrG69W6MEC+BCaYxL8tRBWNRMxcvKJ1xBFPvzWRZCpaX+Oi35CmD2X3Pl/yu6YKKYj7hU0YpfrNoj62gvHFt+75zd1BYrLF6rCsx9h7tiXlZRuLhm1/pujlJGmj/DDz7HadvLXRwSq9GN0KIl8AEXztSJXxMPK+y5VKMG8pauwZYqMlVojW0OZPLZKD046XiV9jiNwg5EtOVX5cpqwOlrdEsYV5tIrucwL+rME71sKsZuNkje1HCx17LGHhjlW5Dr9fg8SWUyLzYm3ADgE5R1dtXsaoR4iUwQd+7EeybbbTkeZkMG22Jl9l7tUYyCJoMXfxGi5xltV6yWlYHbHte5jEvd60+oTERL8c8Lz8KWa6cy2Cv7WiRw8B50Hc6yGQcvZJFTmEJQb5K2jUIcrbZLkWIl8AEfe9GsJ7n5auy7Xk1r1/LEDi2dg1LpOcW8cnvuml/idLgc4kESzWDmKgeX6asDpj2eDTn232JJu/dNe/LuNCiPdqlTzOpRybrVe9xn+IoBZKK1VEfQNfRhuN2nksHdKscrCWweipukSohcC/0vRutZtgrred5gW5Y2ayeP2dTcvFRVuz30UsuQx8iK7olThu1PUktDmapch7d5PFIK/vztt+7rDlr/3Xd0fNKSM/jv0dLywPZ63k1lV1jtfJjIuVp3JACGKV+gzbBvUyO+ftcGgC9WtRznsFugvC8BGUoLYVj+bfNt5xhI5QG7a2lW4D1mlQAD7YLNyRc3jAqbbxH24Yn1e9yVaqDLP0sky6/bFJWpzzc0fPqM2c787Y6oMBAJ+L5QfUukfI0LmlDeUI9k6NSc5OUkpzCYsNKh3ua13WmyW6BEC9BGe5pXo8Qf5UhW94c45hXLW/L9c0HdYgg2E/J3c2sf2mslXVZ9Gwneresh+KWB5KeW2Sy/5zUkCeKZlIQ0pr6skyTsjrl4Y7iZY61fxcDpzbytdcH1JblckTbjCfUM7ks6QoZGJeB3n3hBhqtRJO6/jUuTQKEeAksMPuJduyb1pe6Zk1h9djK89LTr00oh9/pR0ybUIv7wbrnpb+vPkZj6bgUQjg1YD1/a+4wKatTHiVaLZIkMXvzaX454p6lw7W28mr3fgEbhuEtK2ar5k6eUb9NBqUTJMae187zunhXTfS6QIiXwApeCusfDXuGjVB+vpI1/6JegKl4WWPc9+cZVfymSVmdSYofbFxZJwzb4lP54q+LNtuEVRX2NAqxmHSr1cL/3obf3gQkvi6JYWzxqxRi+gNjvLbx71vB+l4thHgJBACoFHJDLpKl2cbKUup52T4uObuQYrxMyuq8qvyBj72W4UXZKq6g87zGfn3Imebazfu/nqLl278ZFq1bQ5+8W1is4YsdFzh/LR1+GA27FuoO6DuDH8JeRUPZeKLe80rKyCchPQ+FXMZdzSyXJPJ0hHgJHEYmk9GzWV2iQvwMbc4qhJVxo34G01o9dgsWGcrqaCQZg722s0I5B38Kyhyp0UomWffVyfKduomFT/+wHZyf+uNx4pNzWLz9Ap/9doD0JQ/DyR9BroTHl0KvWEO5oJjW9U3O1W/XDxk7RgYTWAU/MO6ASJUQVIivR3dDo5VsDi/Lw9rgTj/cVDiYrLlW05dkqTaLlAvprTjGetksRqrfII3SzHJ3SJUo77lu5hcz5qv93BmYx/eqmbSUXwXvQBj8NTS9DyitLlvbz7T3pd7z2lnDh4wgPC9BBZHJZJUSLrDseBnH0CqSVPmnVhfETpcCuUN+iZ+8Z9BMVhqYd4fKqvY8V62bZ5iROomW8qtcl0Jg5G8G4YLSoWWIv6l4KeQyNFrJ4HkJ8RIIqgBLKQFBvqVDHHvFyzwR9pikSx+4qA2joSydH1Tv0kWma9hx4PLNSljsHLbHW26orOdu+XE2qN4jRHuDeK0uLYSwO0yO0VoRL7lMxomrWWQVFBPg7UWHhsFOtd2dEOIlcBn+3mWjFsbxGXsdO0vDsEQplCfV73JI25xgWR5rVLN5UL63TDLogUsZZc6tavTNei3xuPxvVin/Q4CsgJOq9vxLPYPrlA2469MpapuJl0xWmlXfo1mdSnvH7kzNfTKB2zOiZ2Publ6HWY+2NWwz9bxKP571AyznnOmOs+yh3SSQZ9Vv8bumC96yYj5TLihTVuepJbtJziqs6CM4EYmXFT/zqWoxSpmGjZoeTPF9l2z8LR5t8LzMYl5glCLRsuYtCTJGiJfAZfipvFgz5i6e69HYsM1EvIw8qr5ms2rG2BpeFuLNS8WvsLqkH3KZ5bI617PKzkpWJwo0vO+1kjeVGwBYUjKIScXjSMop9dDM+zmeudWP0kshM3QWB8gvKuHQrc5NvWpocqoeIV4CtyLYqIS0caZE31bWM/XLW8isRc6MkhFWy+oYdz8y5sTVLDYfv05JVaZWqPNYovyUf3vFoZVkTC8ezkclQ5CQk5lfWnvMuOO3cTOSyzfyTf7N9iRkUKyRiAzxpVGdmrckyBghXgK3YGD7cABG39PEsC2vqNTzuLt5Xfq20nlfb/SPNjnXvsC+zFBWp0jyMpTVCSLXpKGunjPJ2Ty8cCcvrznEZ9suVOCJ7CA3DVYPop/iEIWSkpeKX+ErTf9yTzubUtoF3FepINi3dOi4L0EXw7uneb1KVGT1DIR4CdyChc904uiMB2gRWroYXF+22Ecpx1elYNmwLhx/9wEesLFesjw2ansyvHgK2ZIf3eS6ygzvrN5UZlgWd7p0RrBK1kDeuAAr+sHVg9yUaulic9qudp2qHzLqCfIrm4R6bw1OkdAjxEvgFsjlMpN4F+jWUB579wGOTH/AcEyAj7JMhQRbZaAtYVxWp7n8Gv+X+yaTF31tSOwEXePaqqKj7DyaZTFwMwGCG+lmRaWW5Z6nH76eNRYvGQSb/bvJZdDTRjWPmoIQL4FbE+ijLFMTzEepMInnVCRrXl9W57Q2ylBWZ9mXS1m3L5Evdlxg8fbKDRUlSSI+OadMw48Y+UG+Vb2PojADwjvCmD+4KFnvqGSMvqR1vJnnZZ5l375hsEVvrKYhxEvgkfz+Smnz1BJNxbLmUwjhX+rpJmV1Dv6ykNm/nTE57mJ6XplOR+Xx24lk+s//i2eW7iGroJhL6Xn8W7GVL5Tz8JWp+VPTkS1dV0At67Oo5ny7LxFJkjiTbLqwO9hMqGpyVr0xQrwEHomxN1aZxhK5+NlVVkcfCLcXfe38I0mZdJy5hc3zx/K+8ksUMolvS/rwfPFrjN0Q79A13990muTsQrILTYe05l5WTSz5bAmxMFvg8QzqEMHjnRpwZ6NgYub95fD5+rI616S6TPD6mVeVPxAhu8FbJaMoufUVMS5F7QhKSviP8gseV/wDwNzip1ioeZyKtpY9fsV0YqFBsG8Zz7OTja7kNQnheQk8Ho1Wy9NdI2leP4CPnmhHeJBPBa6iK6sz1UpZHUfjahl5agLI50vlxzyu+IdiScHrxS+yUPMExsI19cfjDl131qZTADSq48fbA1vTs1mdMsNG8+a+NZXb4ykFNZpiI8/jmW5R7J7at8LX+lbTl+eLXyNf8taV1VHNoh432XvxBtmFthvWFpWUBudvXEtgg2om9yhOkiv5MLr4db7X9C57P7PWbOWRlFFAVIgfG8fdw5heTZHJZLeNp2WOEC+Bx2OpUUhYoM77stX01hqWyurs2bebQUZZ7ua8+f1Rot/ewsW0XKSUk/zoPYPW8iRSpWAGq6fzl7aDw3ZYwttLzpJ/dzaJc4UH+dKkrm4N5F1NQ5xyH09AJjk6jeLhZGdnExQURFZWFoGBgeWfIHBbzqbkcD41l4fahZfZdyEtl2V/XeSl+5rx0+GrzP/jnMPXj5KlsEr5MU3lyWRK/oxRv8b6D2ItZvQ3nrIJgB7ykyz3no+/lMd5bQQjiidzRXIsgH5nVDBP3NmQ+X+co0ldP0q0Egue6cTGo9fo3Kg2dzUtW2UiIT2P9fuTeL5XE+pYaZziKdj7HRXiJajxSJJEk6mbK3RubbJZoZrDnfLzFElKXil+mddeeYPm9U29vcZTNvGIfBefKJfgLSthnzaa59WvkUUtu+7z8n3N+PxWbtneaX0JDaxI3K5mYO931C2GjZ999hmNGzfGx8eH7t27s2/fPpvHf/fdd7Rq1QofHx/atWvH5s0V+2AKbg9kMhnP99KtmezWJIT3jErwlIelsjpr/28qGUazj4XqEl5Q/JcFqkV4y0rYpOnGc+qpdgsXwGsPRPP92B6cmTXgthYuR3C5eK1fv57Y2FhmzJjBoUOH6NChA/379yc11XK1yV27djFkyBBGjx7N4cOHeeyxx3jsscc4ceJENVsu8CTeHNCKr0Z1Y9XIrgzr0ZipD7bilZgWdp1rqazOT7OHsSguHk1JCRs/+jfTlN8CsLzkQcYXT6SIsnW2zNHff0DbMBRyGV0ah9jsMC4wxeXDxu7du9O1a1cWLVoEgFarJTIykgkTJjBlypQyxw8ePJi8vDx+/fVXw7a77rqLjh07smTJknLvJ4aNAnP2JWTw9Be77ThS4gXFrwah2qTphgKJAYr9aCUZH5QMZYXmIbvu+euEe2gbEciJq9m0DKtltSzP7YhHDBvVajUHDx4kJibGsE0ulxMTE8Pu3ZY/TLt37zY5HqB///5Wjy8qKiI7O9vkJRAY061JCIuH3mnHkbqyOhOMyuoMUOynSFIyvniCTeFSKeREhvjy95t9SJj9EHc0CEImk9GuYZAQrgri0gz79PR0NBoNoaGmJU5CQ0M5c+aMxXOSk5MtHp+cnGzx+NmzZzNz5kznGCyosTzYLpxLHw002SZJEsnZhTz22T+kZBcZtv9X25O04mC+UM5DQsbz6tfYL7UyObdXi7p8+Hg7dp5Pp0SjNakWK3AONX550NSpU4mNjTW8z87OJjIy0oUWCTwFmUxGeJAve6eZevqSJJFd8ACf/683YYF+LLurJcEWaskDDOkWVR2m3pa4VLzq1q2LQqEgJSXFZHtKSgphYWEWzwkLC3PoeG9vb7y9PTvvReBeyGQygvyUTH2sm6tNua1xacxLpVLRuXNn4uLiDNu0Wi1xcXH06NHD4jk9evQwOR5g69atVo8XCAQ1E5cPG2NjYxk+fDhdunShW7duzJ8/n7y8PEaOHAnAsGHDaNCgAbNnzwZg0qRJ9O7dm7lz5zJw4EDWrVvHgQMHWLp0qSsfQyAQVDMuF6/BgweTlpbG9OnTSU5OpmPHjmzZssUQlE9MTERu1EamZ8+erF27lrfffptp06bRokULfv75Z+644w5rtxAIBDUQl+d5VTciz0sgcG88Is9LIBAIKooQL4FA4JEI8RIIBB6JywP21Y0+xCeWCQkE7on+u1leOP62E6+cHF3PO5FlLxC4Nzk5OQQFWe8MddvNNmq1Wq5du0ZAQAAyWcU6uFQE/bKkpKSkGj3Lebs8J4hnrSokSSInJ4eIiAiTNClzbjvPSy6X07BhQ5fdPzAwsMZ/0OH2eU4Qz1oV2PK49IiAvUAg8EiEeAkEAo9EiFc14e3tzYwZM2p8hYvb5TlBPKurue0C9gKBoGYgPC+BQOCRCPESCAQeiRAvgUDgkQjxEggEHokQryoiIyODoUOHEhgYSHBwMKNHjyY3N9fm8RMmTCA6OhpfX1+ioqKYOHEiWVlZ1Wh1xXD0WQGWLl3KfffdR2BgIDKZjMzMzOox1kFul27ujjznyZMnefLJJ2ncuDEymYz58+dXn6FGCPGqIoYOHcrJkyfZunUrv/76K3/99RcvvPCC1eOvXbvGtWvXmDNnDidOnGDVqlVs2bKF0aNHV6PVFcPRZwXIz89nwIABTJs2rZqsdJzbpZu7o8+Zn59P06ZN+eijj6w2vqkWJIHTOXXqlARI+/fvN2z77bffJJlMJl29etXu62zYsEFSqVRScXFxVZjpFCr7rNu2bZMA6ebNm1VoZcXo1q2bNG7cOMN7jUYjRURESLNnz7Z4/NNPPy0NHDjQZFv37t2lF198sUrtrCyOPqcxjRo1kj799NMqtM46wvOqAnbv3k1wcDBdunQxbIuJiUEul7N37167r6Mvg+vl5b5LUJ31rO5GdXRzdwcq8pzughCvKiA5OZn69eubbPPy8iIkJMRqZ29z0tPTmTVrVrnDL1fjjGd1R2x1c7f2XI52c3cHKvKc7oIQLweYMmUKMpnM5uvMmTOVvk92djYDBw6kTZs2vPvuu5U3vAJU17MKBBXFfccjbshrr73GiBEjbB7TtGlTwsLCygQ7S0pKyMjIKDfAmZOTw4ABAwgICOCnn35CqVRW1uwKUR3P6s5URzd3d6Aiz+kuCPFygHr16lGvXr1yj+vRoweZmZkcPHiQzp07A/Dnn3+i1Wrp3r271fOys7Pp378/3t7ebNy4ER8fH6fZ7ihV/azujnE398ceewwo7eY+fvx4i+fou7m/8sorhm3u3s29Is/pNrhkmuA2YMCAAVKnTp2kvXv3Sjt37pRatGghDRkyxLD/ypUrUnR0tLR3715JkiQpKytL6t69u9SuXTvp/Pnz0vXr1w2vkpISVz2GXTj6rJIkSdevX5cOHz4sLVu2TAKkv/76Szp8+LB048YNVzyCRdatWyd5e3tLq1atkk6dOiW98MILUnBwsJScnCxJkiQ999xz0pQpUwzH//PPP5KXl5c0Z84c6fTp09KMGTMkpVIpHT9+3FWPYBeOPmdRUZF0+PBh6fDhw1J4eLj0+uuvS4cPH5bOnTtXrXYL8aoibty4IQ0ZMkSqVauWFBgYKI0cOVLKyckx7E9ISJAAadu2bZIklaYMWHolJCS45iHsxNFnlSRJmjFjhsVn/fLLL6v/AWywcOFCKSoqSlKpVFK3bt2kPXv2GPb17t1bGj58uMnxGzZskFq2bCmpVCqpbdu20qZNm6rZ4orhyHPq/3+av3r37l2tNouSOAKBwCMRs40CgcAjEeIlEAg8EiFeAoHAIxHiJRAIPBIhXgKBwCMR4iUQCDwSIV4CgcAjEeIlEAg8EiFeArdgxIgRFitXDBgwwNWmCdwUsTBb4DYMGDCAL7/80mSbtQ7NxcXFZSpuqNVqVCqVw/et6HkC1yI8L4Hb4O3tTVhYmMmrdu3aAMhkMhYvXswjjzyCv78/H3zwAe+++y4dO3Zk+fLlNGnSxFCFIzExkUcffZRatWoRGBjI008/bVLyxdp5As9CiJfAY3j33Xd5/PHHOX78OKNGjQLg/Pnz/PDDD/z4448cOXIErVbLo48+SkZGBjt27GDr1q1cvHiRwYMHm1zL/DyB5yGGjQK34ddff6VWrVom26ZNm2boMPTss88ycuRIk/1qtZqvvvrKUHts69atHD9+nISEBCIjIwH46quvaNu2Lfv376dr164WzxN4HkK8BG5Dnz59WLx4scm2kJAQw9/GTT70NGrUyESATp8+TWRkpEG4ANq0aUNwcDCnT582iJf5eQLPQ4iXwG3w9/enefPmNvfbs83eewk8GxHzEtQoWrduTVJSEklJSYZtp06dIjMzkzZt2rjQMoGzEZ6XwG0oKioq027Ly8uLunXr2n2NmJgY2rVrx9ChQ5k/fz4lJSW8/PLL9O7d2+KwU+C5CM9L4DZs2bKF8PBwk9c999zj0DVkMhm//PILtWvX5t577yUmJoamTZuyfv36KrJa4CpEGWiBQOCRCM9LIBB4JEK8BAKBRyLESyAQeCRCvAQCgUcixEsgEHgkQrwEAoFHIsRLIBB4JEK8BAKBRyLESyAQeCRCvAQCgUcixEsgEHgkQrwEAoFH8v+BgjL1SdrB8gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020824, + "end_time": "2024-03-22T14:51:27.215878", + "exception": false, + "start_time": "2024-03-22T14:51:27.195054", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4820.639414, + "end_time": "2024-03-22T14:51:29.960241", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/lct_gan/0/mlu-eval.ipynb", + "output_path": "eval/insurance/lct_gan/0/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/lct_gan/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T13:31:09.320827", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/lct_gan/model.pt b/insurance/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8b2e28ba8662bb15d7086ecd1fb2fbe3019e1d4 --- /dev/null +++ b/insurance/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a36059ea0c87193f58c8c86e84042e2cf8924cc753130163b6610f3adaaf99a9 +size 38583573 diff --git a/insurance/lct_gan/params.json b/insurance/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..f29ec0a8931fd9a1d483b7e2fae0a45d67199683 --- /dev/null +++ b/insurance/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/realtabformer/eval.csv b/insurance/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..6a8993b262d28ed9535ef59afc23f32b98d00c16 --- /dev/null +++ b/insurance/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.010992311685847762,0.00499268272087041,0.0005800863841570736,4.444354057312012,0.20095407962799072,9.563033103942871,0.47066447138786316,4.327647218360653e-07,5.621514081954956,0.015106264501810074,1.2521374225616455,0.02408498153090477,0.1404879093170166,0.001446777256205678,10.065868139266968 diff --git a/insurance/realtabformer/history.csv b/insurance/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff6134efcaa092074906fc804921964f60d47c50 --- /dev/null +++ b/insurance/realtabformer/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.029822943683765413,0.7290258940829034,0.00741166268384139,4.97963041305542,0.0,0.0,0.0,0.0,0.030589072092229292,900,113,167.66863656044006,1.4837932438976997,0.18629848506715563,0.11875602419990881,0.011320239888348927,0.8666478302036187,0.0016214420898055298,0.0,0.0,0.0,0.0,0.0,0.011320239888348927,450,57,53.24812388420105,0.9341776120035272,0.11832916418711345,0.05855883448793177 +1,0.008769770529793783,0.7716960565807296,0.00043396698297856246,1.4175501622094049,0.0,0.0,0.0,0.0,0.008993633385075049,900,113,166.8349289894104,1.4764153007912424,0.18537214332156712,0.09298099682921857,0.006308909958720001,0.1587160864532443,0.0010557503438670954,0.0,0.0,0.0,0.0,0.0,0.006308909958720001,450,57,54.493409872055054,0.9560247345974571,0.12109646638234456,0.07172276902322967 +2,0.0052478324769375225,0.4535665847330644,0.00015836110432902994,0.8015708598825667,0.0,0.0,0.0,0.0,0.005376618540095579,900,113,167.3072214126587,1.4805948797580415,0.18589691268073189,0.09632732554347115,0.005088021330302581,0.6213396075144173,0.00011936115010066849,0.0,0.0,0.0,0.0,0.0,0.005088021330302581,450,57,52.64925193786621,0.9236710866292318,0.11699833763970269,0.052176826527309525 +3,0.0030236450636746464,0.41981709804675077,4.2389841936938424e-05,0.5216422769096163,0.0,0.0,0.0,0.0,0.0031053333073052473,900,113,167.31541466712952,1.4806673864347746,0.18590601629681058,0.09325551584494852,0.0019911888805735443,0.2196139024221849,1.918494357696417e-05,0.0,0.0,0.0,0.0,0.0,0.0019911888805735443,450,57,53.6285560131073,0.9408518598790754,0.11917456891801623,0.07260068974523037 +4,0.0016811677904075219,0.24856906805416656,1.2625722286744727e-05,0.3810585221648216,0.0,0.0,0.0,0.0,0.001737628386148976,900,113,167.82943487167358,1.4852162378024212,0.1864771498574151,0.09434814767631809,0.0027509150341696416,0.07755351969834473,2.1412663001758838e-05,0.0,0.0,0.0,0.0,0.0,0.0027509150341696416,450,57,53.66714072227478,0.941528784601312,0.11926031271616618,0.08398696716482702 +5,0.001128127839474473,0.21778014314879,6.985574985683568e-06,0.28518845240275065,0.0,0.0,0.0,0.0,0.001170292465992841,900,113,160.84526014328003,1.4234093817989384,0.1787169557147556,0.09777768919310342,0.0021597746671694847,0.3930031767821128,3.3223664668092175e-05,0.0,0.0,0.0,0.0,0.0,0.0021597746671694847,450,57,49.775447607040405,0.8732534667901826,0.11061210579342312,0.07324986261511712 +6,0.0010482495646445184,0.11189875107239877,8.224311742242863e-06,0.2797618282172415,0.0,0.0,0.0,0.0,0.0010887739290612647,900,113,162.833313703537,1.4410027761374955,0.1809259041150411,0.09814084033621887,0.0019042760821240436,0.35190855250582836,1.6172142232353588e-05,0.0,0.0,0.0,0.0,0.0,0.0019042760821240436,450,57,52.383286476135254,0.9190050258971098,0.11640730328030056,0.06503926100732203 +7,0.001563700584617133,0.21518477240972333,4.5778025960834125e-06,0.30026551425457,0.0,0.0,0.0,0.0,0.0016092181016897989,900,113,165.6556372642517,1.465979090834086,0.1840618191825019,0.09587243578470914,0.003619586681533191,0.5286780524212229,0.00025007503121890473,0.0,0.0,0.0,0.0,0.0,0.003619586681533191,450,57,52.2312707901001,0.9163380840368438,0.11606949064466689,0.07745434307460591 +8,0.0011870296497040222,0.15783826198738787,2.913096641859988e-06,0.26780749612384375,0.0,0.0,0.0,0.0,0.0012269385414159235,900,113,164.9515302181244,1.4597480550276494,0.18327947802013822,0.0955763464315539,0.000885065957877992,0.12204710721440885,1.0928920943189783e-06,0.0,0.0,0.0,0.0,0.0,0.000885065957877992,450,57,50.95037126541138,0.8938661625510768,0.11322304725646973,0.07468080061912667 +9,0.0007502347028801321,0.07230353024762727,2.765409718966213e-06,0.2302845541636149,0.0,0.0,0.0,0.0,0.0007828545368586977,900,113,159.93213367462158,1.4153286165895715,0.17770237074957954,0.09794058360620937,0.002954557936366958,0.5175472071741417,0.0001666847069735606,0.0,0.0,0.0,0.0,0.0,0.002954557936366958,450,57,48.75002574920654,0.8552636096352025,0.10833339055379232,0.06763517065790661 +10,0.0008960025814141975,0.1276295194669632,9.211538751067532e-06,0.24726980176236896,0.0,0.0,0.0,0.0,0.0009317982033179659,900,113,158.89508271217346,1.406151174444013,0.17655009190241497,0.09495367640546992,0.002206301508348487,0.6127840376083674,3.638367311774459e-05,0.0,0.0,0.0,0.0,0.0,0.002206301508348487,450,57,48.69274377822876,0.8542586627759432,0.1082060972849528,0.07679086615609233 +11,0.0012414162013576263,0.15656802731034372,7.65507348889812e-06,0.2784161967039108,0.0,0.0,0.0,0.0,0.0012826652981392625,900,113,159.0150740146637,1.4072130443775548,0.17668341557184855,0.09698942330031268,0.0015209214665810578,1.1925622708846475,1.541826602484448e-05,0.0,0.0,0.0,0.0,0.0,0.0015209214665810578,450,57,48.32934379577637,0.8478832244873047,0.10739854176839193,0.060592792895540856 +12,0.000802709650840067,0.1468484333481462,6.78528227740518e-07,0.21540525201294158,0.0,0.0,0.0,0.0,0.000833754398206818,900,113,158.784077167511,1.4051688244912477,0.17642675240834554,0.09867627354981624,0.001989033992609216,0.6257210954253898,2.2637783627021217e-05,0.0,0.0,0.0,0.0,0.0,0.001989033992609216,450,57,48.381258964538574,0.8487940169217294,0.10751390881008573,0.07426875439807445 +13,0.0006656886564370426,0.08703389815338669,6.284277396025041e-07,0.20659642385111915,0.0,0.0,0.0,0.0,0.0006952917674950893,900,113,157.6981496810913,1.3955588467353213,0.17522016631232368,0.09772866989065589,0.0010757716005254123,0.41428317912765056,2.2639515015956634e-05,0.0,0.0,0.0,0.0,0.0,0.0010757716005254123,450,57,47.70015549659729,0.8368448332736367,0.10600034554799398,0.07492752365049041 +14,0.0003420950226265834,0.03358766082648436,3.4995474424948957e-07,0.13486777688066165,0.0,0.0,0.0,0.0,0.0003607326176521989,900,113,157.660076379776,1.3952219148652742,0.17517786264419555,0.0989597013500412,0.0008682416723038639,1.067326461550605,1.39864987777777e-05,0.0,0.0,0.0,0.0,0.0,0.0008682416723038639,450,57,47.50433969497681,0.8334094683329264,0.10556519932217068,0.06998455429269948 +15,0.00032250956939404,0.03796051059134845,2.0489448113398992e-07,0.13379798481861752,0.0,0.0,0.0,0.0,0.00034089527511645834,900,113,158.42128372192383,1.4019582630258747,0.17602364857991537,0.09779412188954585,0.0008600625935489208,1.0569037955788663,1.91862969400389e-05,0.0,0.0,0.0,0.0,0.0,0.0008600625935489208,450,57,48.475810289382935,0.8504528120944375,0.10772402286529541,0.06928659294192728 diff --git a/insurance/realtabformer/mlu-eval.ipynb b/insurance/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3471f289cb7f016cf51d09ea7b3a0c94b159c645 --- /dev/null +++ b/insurance/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2459 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.565090Z", + "iopub.status.busy": "2024-03-22T17:48:32.564211Z", + "iopub.status.idle": "2024-03-22T17:48:32.602921Z", + "shell.execute_reply": "2024-03-22T17:48:32.602187Z" + }, + "papermill": { + "duration": 0.054165, + "end_time": "2024-03-22T17:48:32.605109", + "exception": false, + "start_time": "2024-03-22T17:48:32.550944", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.632778Z", + "iopub.status.busy": "2024-03-22T17:48:32.632342Z", + "iopub.status.idle": "2024-03-22T17:48:32.639689Z", + "shell.execute_reply": "2024-03-22T17:48:32.638818Z" + }, + "papermill": { + "duration": 0.023919, + "end_time": "2024-03-22T17:48:32.641677", + "exception": false, + "start_time": "2024-03-22T17:48:32.617758", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.666219Z", + "iopub.status.busy": "2024-03-22T17:48:32.665872Z", + "iopub.status.idle": "2024-03-22T17:48:32.670288Z", + "shell.execute_reply": "2024-03-22T17:48:32.669459Z" + }, + "papermill": { + "duration": 0.019208, + "end_time": "2024-03-22T17:48:32.672274", + "exception": false, + "start_time": "2024-03-22T17:48:32.653066", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.698128Z", + "iopub.status.busy": "2024-03-22T17:48:32.697757Z", + "iopub.status.idle": "2024-03-22T17:48:32.702899Z", + "shell.execute_reply": "2024-03-22T17:48:32.701664Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.021273, + "end_time": "2024-03-22T17:48:32.705320", + "exception": false, + "start_time": "2024-03-22T17:48:32.684047", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.734913Z", + "iopub.status.busy": "2024-03-22T17:48:32.734569Z", + "iopub.status.idle": "2024-03-22T17:48:32.740963Z", + "shell.execute_reply": "2024-03-22T17:48:32.740031Z" + }, + "papermill": { + "duration": 0.023874, + "end_time": "2024-03-22T17:48:32.743198", + "exception": false, + "start_time": "2024-03-22T17:48:32.719324", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d343101a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.770956Z", + "iopub.status.busy": "2024-03-22T17:48:32.770581Z", + "iopub.status.idle": "2024-03-22T17:48:32.776641Z", + "shell.execute_reply": "2024-03-22T17:48:32.775636Z" + }, + "papermill": { + "duration": 0.022705, + "end_time": "2024-03-22T17:48:32.778873", + "exception": false, + "start_time": "2024-03-22T17:48:32.756168", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/realtabformer/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.012826, + "end_time": "2024-03-22T17:48:32.804618", + "exception": false, + "start_time": "2024-03-22T17:48:32.791792", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.831861Z", + "iopub.status.busy": "2024-03-22T17:48:32.831454Z", + "iopub.status.idle": "2024-03-22T17:48:32.842155Z", + "shell.execute_reply": "2024-03-22T17:48:32.841141Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.026984, + "end_time": "2024-03-22T17:48:32.844341", + "exception": false, + "start_time": "2024-03-22T17:48:32.817357", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/realtabformer/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.873217Z", + "iopub.status.busy": "2024-03-22T17:48:32.872829Z", + "iopub.status.idle": "2024-03-22T17:48:34.907078Z", + "shell.execute_reply": "2024-03-22T17:48:34.905937Z" + }, + "papermill": { + "duration": 2.050961, + "end_time": "2024-03-22T17:48:34.909240", + "exception": false, + "start_time": "2024-03-22T17:48:32.858279", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:34.934797Z", + "iopub.status.busy": "2024-03-22T17:48:34.934314Z", + "iopub.status.idle": "2024-03-22T17:48:34.952680Z", + "shell.execute_reply": "2024-03-22T17:48:34.951911Z" + }, + "papermill": { + "duration": 0.033772, + "end_time": "2024-03-22T17:48:34.955024", + "exception": false, + "start_time": "2024-03-22T17:48:34.921252", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:34.980303Z", + "iopub.status.busy": "2024-03-22T17:48:34.979475Z", + "iopub.status.idle": "2024-03-22T17:48:34.989126Z", + "shell.execute_reply": "2024-03-22T17:48:34.988220Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024507, + "end_time": "2024-03-22T17:48:34.991219", + "exception": false, + "start_time": "2024-03-22T17:48:34.966712", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:35.018864Z", + "iopub.status.busy": "2024-03-22T17:48:35.018499Z", + "iopub.status.idle": "2024-03-22T17:48:35.498861Z", + "shell.execute_reply": "2024-03-22T17:48:35.497941Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.497969, + "end_time": "2024-03-22T17:48:35.501411", + "exception": false, + "start_time": "2024-03-22T17:48:35.003442", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:35.528095Z", + "iopub.status.busy": "2024-03-22T17:48:35.527397Z", + "iopub.status.idle": "2024-03-22T17:48:48.370968Z", + "shell.execute_reply": "2024-03-22T17:48:48.370163Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 12.85894, + "end_time": "2024-03-22T17:48:48.373403", + "exception": false, + "start_time": "2024-03-22T17:48:35.514463", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 17:48:39.992016: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 17:48:39.992132: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 17:48:40.117715: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:48.400981Z", + "iopub.status.busy": "2024-03-22T17:48:48.400390Z", + "iopub.status.idle": "2024-03-22T17:48:48.406841Z", + "shell.execute_reply": "2024-03-22T17:48:48.406076Z" + }, + "papermill": { + "duration": 0.02259, + "end_time": "2024-03-22T17:48:48.408784", + "exception": false, + "start_time": "2024-03-22T17:48:48.386194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:48.433480Z", + "iopub.status.busy": "2024-03-22T17:48:48.432733Z", + "iopub.status.idle": "2024-03-22T17:48:56.990631Z", + "shell.execute_reply": "2024-03-22T17:48:56.989549Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.57279, + "end_time": "2024-03-22T17:48:56.993041", + "exception": false, + "start_time": "2024-03-22T17:48:48.420251", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T17:48:57.020225Z", + "iopub.status.busy": "2024-03-22T17:48:57.019880Z", + "iopub.status.idle": "2024-03-22T17:48:57.026502Z", + "shell.execute_reply": "2024-03-22T17:48:57.025618Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.02276, + "end_time": "2024-03-22T17:48:57.028481", + "exception": false, + "start_time": "2024-03-22T17:48:57.005721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:57.054648Z", + "iopub.status.busy": "2024-03-22T17:48:57.053907Z", + "iopub.status.idle": "2024-03-22T17:48:57.059601Z", + "shell.execute_reply": "2024-03-22T17:48:57.058618Z" + }, + "papermill": { + "duration": 0.021435, + "end_time": "2024-03-22T17:48:57.061565", + "exception": false, + "start_time": "2024-03-22T17:48:57.040130", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:57.087847Z", + "iopub.status.busy": "2024-03-22T17:48:57.087014Z", + "iopub.status.idle": "2024-03-22T17:54:39.984658Z", + "shell.execute_reply": "2024-03-22T17:54:39.983743Z" + }, + "papermill": { + "duration": 342.926299, + "end_time": "2024-03-22T17:54:39.999916", + "exception": false, + "start_time": "2024-03-22T17:48:57.073617", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:54:40.027722Z", + "iopub.status.busy": "2024-03-22T17:54:40.027401Z", + "iopub.status.idle": "2024-03-22T17:54:40.643997Z", + "shell.execute_reply": "2024-03-22T17:54:40.643048Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.633398, + "end_time": "2024-03-22T17:54:40.646155", + "exception": false, + "start_time": "2024-03-22T17:54:40.012757", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:54:40.674361Z", + "iopub.status.busy": "2024-03-22T17:54:40.673611Z", + "iopub.status.idle": "2024-03-22T18:02:02.384439Z", + "shell.execute_reply": "2024-03-22T18:02:02.383434Z" + }, + "papermill": { + "duration": 441.742571, + "end_time": "2024-03-22T18:02:02.401879", + "exception": false, + "start_time": "2024-03-22T17:54:40.659308", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.431631Z", + "iopub.status.busy": "2024-03-22T18:02:02.431317Z", + "iopub.status.idle": "2024-03-22T18:02:02.840156Z", + "shell.execute_reply": "2024-03-22T18:02:02.839186Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.426339, + "end_time": "2024-03-22T18:02:02.842464", + "exception": false, + "start_time": "2024-03-22T18:02:02.416125", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.872250Z", + "iopub.status.busy": "2024-03-22T18:02:02.871890Z", + "iopub.status.idle": "2024-03-22T18:02:02.875980Z", + "shell.execute_reply": "2024-03-22T18:02:02.875132Z" + }, + "papermill": { + "duration": 0.02141, + "end_time": "2024-03-22T18:02:02.877923", + "exception": false, + "start_time": "2024-03-22T18:02:02.856513", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.906802Z", + "iopub.status.busy": "2024-03-22T18:02:02.906111Z", + "iopub.status.idle": "2024-03-22T18:02:02.913535Z", + "shell.execute_reply": "2024-03-22T18:02:02.912635Z" + }, + "papermill": { + "duration": 0.024448, + "end_time": "2024-03-22T18:02:02.915615", + "exception": false, + "start_time": "2024-03-22T18:02:02.891167", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10420892" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.943512Z", + "iopub.status.busy": "2024-03-22T18:02:02.942817Z", + "iopub.status.idle": "2024-03-22T18:02:03.040150Z", + "shell.execute_reply": "2024-03-22T18:02:03.039190Z" + }, + "papermill": { + "duration": 0.114006, + "end_time": "2024-03-22T18:02:03.042749", + "exception": false, + "start_time": "2024-03-22T18:02:02.928743", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 15200] --\n", + "├─Adapter: 1-1 [2, 1071, 15200] --\n", + "│ └─Embedding: 2-1 [2, 1071, 19, 800] (440,800)\n", + "│ └─TensorInductionPoint: 2-2 [19, 1] 19\n", + "│ └─Sequential: 2-3 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 820,224\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 15200] (recursive)\n", + "│ └─Embedding: 2-4 [2, 267, 19, 800] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [19, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,861,692\n", + "Trainable params: 10,420,892\n", + "Non-trainable params: 440,800\n", + "Total mult-adds (M): 43.07\n", + "========================================================================================================================\n", + "Input size (MB): 0.20\n", + "Forward/backward pass size (MB): 632.89\n", + "Params size (MB): 43.45\n", + "Estimated Total Size (MB): 676.54\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:03.074601Z", + "iopub.status.busy": "2024-03-22T18:02:03.074272Z", + "iopub.status.idle": "2024-03-22T19:01:15.975389Z", + "shell.execute_reply": "2024-03-22T19:01:15.974397Z" + }, + "papermill": { + "duration": 3552.935987, + "end_time": "2024-03-22T19:01:15.993965", + "exception": false, + "start_time": "2024-03-22T18:02:03.057978", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.029822943683765413, 'avg_role_model_std_loss': 0.7290258940829034, 'avg_role_model_mean_pred_loss': 0.00741166268384139, 'avg_role_model_g_mag_loss': 4.97963041305542, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.030589072092229292, 'n_size': 900, 'n_batch': 113, 'duration': 167.66863656044006, 'duration_batch': 1.4837932438976997, 'duration_size': 0.18629848506715563, 'avg_pred_std': 0.11875602419990881}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011320239888348927, 'avg_role_model_std_loss': 0.8666478302036187, 'avg_role_model_mean_pred_loss': 0.0016214420898055298, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011320239888348927, 'n_size': 450, 'n_batch': 57, 'duration': 53.24812388420105, 'duration_batch': 0.9341776120035272, 'duration_size': 0.11832916418711345, 'avg_pred_std': 0.05855883448793177}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008769770529793783, 'avg_role_model_std_loss': 0.7716960565807296, 'avg_role_model_mean_pred_loss': 0.00043396698297856246, 'avg_role_model_g_mag_loss': 1.4175501622094049, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008993633385075049, 'n_size': 900, 'n_batch': 113, 'duration': 166.8349289894104, 'duration_batch': 1.4764153007912424, 'duration_size': 0.18537214332156712, 'avg_pred_std': 0.09298099682921857}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006308909958720001, 'avg_role_model_std_loss': 0.1587160864532443, 'avg_role_model_mean_pred_loss': 0.0010557503438670954, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006308909958720001, 'n_size': 450, 'n_batch': 57, 'duration': 54.493409872055054, 'duration_batch': 0.9560247345974571, 'duration_size': 0.12109646638234456, 'avg_pred_std': 0.07172276902322967}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0052478324769375225, 'avg_role_model_std_loss': 0.4535665847330644, 'avg_role_model_mean_pred_loss': 0.00015836110432902994, 'avg_role_model_g_mag_loss': 0.8015708598825667, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005376618540095579, 'n_size': 900, 'n_batch': 113, 'duration': 167.3072214126587, 'duration_batch': 1.4805948797580415, 'duration_size': 0.18589691268073189, 'avg_pred_std': 0.09632732554347115}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005088021330302581, 'avg_role_model_std_loss': 0.6213396075144173, 'avg_role_model_mean_pred_loss': 0.00011936115010066849, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005088021330302581, 'n_size': 450, 'n_batch': 57, 'duration': 52.64925193786621, 'duration_batch': 0.9236710866292318, 'duration_size': 0.11699833763970269, 'avg_pred_std': 0.052176826527309525}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0030236450636746464, 'avg_role_model_std_loss': 0.41981709804675077, 'avg_role_model_mean_pred_loss': 4.2389841936938424e-05, 'avg_role_model_g_mag_loss': 0.5216422769096163, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031053333073052473, 'n_size': 900, 'n_batch': 113, 'duration': 167.31541466712952, 'duration_batch': 1.4806673864347746, 'duration_size': 0.18590601629681058, 'avg_pred_std': 0.09325551584494852}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019911888805735443, 'avg_role_model_std_loss': 0.2196139024221849, 'avg_role_model_mean_pred_loss': 1.918494357696417e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019911888805735443, 'n_size': 450, 'n_batch': 57, 'duration': 53.6285560131073, 'duration_batch': 0.9408518598790754, 'duration_size': 0.11917456891801623, 'avg_pred_std': 0.07260068974523037}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016811677904075219, 'avg_role_model_std_loss': 0.24856906805416656, 'avg_role_model_mean_pred_loss': 1.2625722286744727e-05, 'avg_role_model_g_mag_loss': 0.3810585221648216, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001737628386148976, 'n_size': 900, 'n_batch': 113, 'duration': 167.82943487167358, 'duration_batch': 1.4852162378024212, 'duration_size': 0.1864771498574151, 'avg_pred_std': 0.09434814767631809}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027509150341696416, 'avg_role_model_std_loss': 0.07755351969834473, 'avg_role_model_mean_pred_loss': 2.1412663001758838e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027509150341696416, 'n_size': 450, 'n_batch': 57, 'duration': 53.66714072227478, 'duration_batch': 0.941528784601312, 'duration_size': 0.11926031271616618, 'avg_pred_std': 0.08398696716482702}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001128127839474473, 'avg_role_model_std_loss': 0.21778014314879, 'avg_role_model_mean_pred_loss': 6.985574985683568e-06, 'avg_role_model_g_mag_loss': 0.28518845240275065, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001170292465992841, 'n_size': 900, 'n_batch': 113, 'duration': 160.84526014328003, 'duration_batch': 1.4234093817989384, 'duration_size': 0.1787169557147556, 'avg_pred_std': 0.09777768919310342}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021597746671694847, 'avg_role_model_std_loss': 0.3930031767821128, 'avg_role_model_mean_pred_loss': 3.3223664668092175e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021597746671694847, 'n_size': 450, 'n_batch': 57, 'duration': 49.775447607040405, 'duration_batch': 0.8732534667901826, 'duration_size': 0.11061210579342312, 'avg_pred_std': 0.07324986261511712}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010482495646445184, 'avg_role_model_std_loss': 0.11189875107239877, 'avg_role_model_mean_pred_loss': 8.224311742242863e-06, 'avg_role_model_g_mag_loss': 0.2797618282172415, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010887739290612647, 'n_size': 900, 'n_batch': 113, 'duration': 162.833313703537, 'duration_batch': 1.4410027761374955, 'duration_size': 0.1809259041150411, 'avg_pred_std': 0.09814084033621887}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019042760821240436, 'avg_role_model_std_loss': 0.35190855250582836, 'avg_role_model_mean_pred_loss': 1.6172142232353588e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019042760821240436, 'n_size': 450, 'n_batch': 57, 'duration': 52.383286476135254, 'duration_batch': 0.9190050258971098, 'duration_size': 0.11640730328030056, 'avg_pred_std': 0.06503926100732203}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001563700584617133, 'avg_role_model_std_loss': 0.21518477240972333, 'avg_role_model_mean_pred_loss': 4.5778025960834125e-06, 'avg_role_model_g_mag_loss': 0.30026551425457, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016092181016897989, 'n_size': 900, 'n_batch': 113, 'duration': 165.6556372642517, 'duration_batch': 1.465979090834086, 'duration_size': 0.1840618191825019, 'avg_pred_std': 0.09587243578470914}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003619586681533191, 'avg_role_model_std_loss': 0.5286780524212229, 'avg_role_model_mean_pred_loss': 0.00025007503121890473, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003619586681533191, 'n_size': 450, 'n_batch': 57, 'duration': 52.2312707901001, 'duration_batch': 0.9163380840368438, 'duration_size': 0.11606949064466689, 'avg_pred_std': 0.07745434307460591}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011870296497040222, 'avg_role_model_std_loss': 0.15783826198738787, 'avg_role_model_mean_pred_loss': 2.913096641859988e-06, 'avg_role_model_g_mag_loss': 0.26780749612384375, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012269385414159235, 'n_size': 900, 'n_batch': 113, 'duration': 164.9515302181244, 'duration_batch': 1.4597480550276494, 'duration_size': 0.18327947802013822, 'avg_pred_std': 0.0955763464315539}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.000885065957877992, 'avg_role_model_std_loss': 0.12204710721440885, 'avg_role_model_mean_pred_loss': 1.0928920943189783e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000885065957877992, 'n_size': 450, 'n_batch': 57, 'duration': 50.95037126541138, 'duration_batch': 0.8938661625510768, 'duration_size': 0.11322304725646973, 'avg_pred_std': 0.07468080061912667}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007502347028801321, 'avg_role_model_std_loss': 0.07230353024762727, 'avg_role_model_mean_pred_loss': 2.765409718966213e-06, 'avg_role_model_g_mag_loss': 0.2302845541636149, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007828545368586977, 'n_size': 900, 'n_batch': 113, 'duration': 159.93213367462158, 'duration_batch': 1.4153286165895715, 'duration_size': 0.17770237074957954, 'avg_pred_std': 0.09794058360620937}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002954557936366958, 'avg_role_model_std_loss': 0.5175472071741417, 'avg_role_model_mean_pred_loss': 0.0001666847069735606, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002954557936366958, 'n_size': 450, 'n_batch': 57, 'duration': 48.75002574920654, 'duration_batch': 0.8552636096352025, 'duration_size': 0.10833339055379232, 'avg_pred_std': 0.06763517065790661}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008960025814141975, 'avg_role_model_std_loss': 0.1276295194669632, 'avg_role_model_mean_pred_loss': 9.211538751067532e-06, 'avg_role_model_g_mag_loss': 0.24726980176236896, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009317982033179659, 'n_size': 900, 'n_batch': 113, 'duration': 158.89508271217346, 'duration_batch': 1.406151174444013, 'duration_size': 0.17655009190241497, 'avg_pred_std': 0.09495367640546992}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002206301508348487, 'avg_role_model_std_loss': 0.6127840376083674, 'avg_role_model_mean_pred_loss': 3.638367311774459e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002206301508348487, 'n_size': 450, 'n_batch': 57, 'duration': 48.69274377822876, 'duration_batch': 0.8542586627759432, 'duration_size': 0.1082060972849528, 'avg_pred_std': 0.07679086615609233}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012414162013576263, 'avg_role_model_std_loss': 0.15656802731034372, 'avg_role_model_mean_pred_loss': 7.65507348889812e-06, 'avg_role_model_g_mag_loss': 0.2784161967039108, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012826652981392625, 'n_size': 900, 'n_batch': 113, 'duration': 159.0150740146637, 'duration_batch': 1.4072130443775548, 'duration_size': 0.17668341557184855, 'avg_pred_std': 0.09698942330031268}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0015209214665810578, 'avg_role_model_std_loss': 1.1925622708846475, 'avg_role_model_mean_pred_loss': 1.541826602484448e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015209214665810578, 'n_size': 450, 'n_batch': 57, 'duration': 48.32934379577637, 'duration_batch': 0.8478832244873047, 'duration_size': 0.10739854176839193, 'avg_pred_std': 0.060592792895540856}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000802709650840067, 'avg_role_model_std_loss': 0.1468484333481462, 'avg_role_model_mean_pred_loss': 6.78528227740518e-07, 'avg_role_model_g_mag_loss': 0.21540525201294158, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000833754398206818, 'n_size': 900, 'n_batch': 113, 'duration': 158.784077167511, 'duration_batch': 1.4051688244912477, 'duration_size': 0.17642675240834554, 'avg_pred_std': 0.09867627354981624}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001989033992609216, 'avg_role_model_std_loss': 0.6257210954253898, 'avg_role_model_mean_pred_loss': 2.2637783627021217e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001989033992609216, 'n_size': 450, 'n_batch': 57, 'duration': 48.381258964538574, 'duration_batch': 0.8487940169217294, 'duration_size': 0.10751390881008573, 'avg_pred_std': 0.07426875439807445}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006656886564370426, 'avg_role_model_std_loss': 0.08703389815338669, 'avg_role_model_mean_pred_loss': 6.284277396025041e-07, 'avg_role_model_g_mag_loss': 0.20659642385111915, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006952917674950893, 'n_size': 900, 'n_batch': 113, 'duration': 157.6981496810913, 'duration_batch': 1.3955588467353213, 'duration_size': 0.17522016631232368, 'avg_pred_std': 0.09772866989065589}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010757716005254123, 'avg_role_model_std_loss': 0.41428317912765056, 'avg_role_model_mean_pred_loss': 2.2639515015956634e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010757716005254123, 'n_size': 450, 'n_batch': 57, 'duration': 47.70015549659729, 'duration_batch': 0.8368448332736367, 'duration_size': 0.10600034554799398, 'avg_pred_std': 0.07492752365049041}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0003420950226265834, 'avg_role_model_std_loss': 0.03358766082648436, 'avg_role_model_mean_pred_loss': 3.4995474424948957e-07, 'avg_role_model_g_mag_loss': 0.13486777688066165, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003607326176521989, 'n_size': 900, 'n_batch': 113, 'duration': 157.660076379776, 'duration_batch': 1.3952219148652742, 'duration_size': 0.17517786264419555, 'avg_pred_std': 0.0989597013500412}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008682416723038639, 'avg_role_model_std_loss': 1.067326461550605, 'avg_role_model_mean_pred_loss': 1.39864987777777e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008682416723038639, 'n_size': 450, 'n_batch': 57, 'duration': 47.50433969497681, 'duration_batch': 0.8334094683329264, 'duration_size': 0.10556519932217068, 'avg_pred_std': 0.06998455429269948}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00032250956939404, 'avg_role_model_std_loss': 0.03796051059134845, 'avg_role_model_mean_pred_loss': 2.0489448113398992e-07, 'avg_role_model_g_mag_loss': 0.13379798481861752, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00034089527511645834, 'n_size': 900, 'n_batch': 113, 'duration': 158.42128372192383, 'duration_batch': 1.4019582630258747, 'duration_size': 0.17602364857991537, 'avg_pred_std': 0.09779412188954585}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008600625935489208, 'avg_role_model_std_loss': 1.0569037955788663, 'avg_role_model_mean_pred_loss': 1.91862969400389e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008600625935489208, 'n_size': 450, 'n_batch': 57, 'duration': 48.475810289382935, 'duration_batch': 0.8504528120944375, 'duration_size': 0.10772402286529541, 'avg_pred_std': 0.06928659294192728}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'pred_duration': 5.626660346984863, 'grad_duration': 4.452868223190308, 'total_duration': 10.079528570175171, 'pred_std': 0.1404879093170166, 'std_loss': 0.001446777256205678, 'mean_pred_loss': 4.327647218360653e-07, 'pred_rmse': 0.02408498339354992, 'pred_mae': 0.015106265433132648, 'pred_mape': 1.252137541770935, 'grad_rmse': 0.47066444158554077, 'grad_mae': 0.20095407962799072, 'grad_mape': 9.563034057617188}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'avg_pred_duration': 5.626660346984863, 'avg_grad_duration': 4.452868223190308, 'avg_total_duration': 10.079528570175171, 'avg_pred_std': 0.1404879093170166, 'avg_std_loss': 0.001446777256205678, 'avg_mean_pred_loss': 4.327647218360653e-07}, 'min_metrics': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'pred_duration': 5.626660346984863, 'grad_duration': 4.452868223190308, 'total_duration': 10.079528570175171, 'pred_std': 0.1404879093170166, 'std_loss': 0.001446777256205678, 'mean_pred_loss': 4.327647218360653e-07, 'pred_rmse': 0.02408498339354992, 'pred_mae': 0.015106265433132648, 'pred_mape': 1.252137541770935, 'grad_rmse': 0.47066444158554077, 'grad_mae': 0.20095407962799072, 'grad_mape': 9.563034057617188}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'pred_duration': 5.626660346984863, 'grad_duration': 4.452868223190308, 'total_duration': 10.079528570175171, 'pred_std': 0.1404879093170166, 'std_loss': 0.001446777256205678, 'mean_pred_loss': 4.327647218360653e-07, 'pred_rmse': 0.02408498339354992, 'pred_mae': 0.015106265433132648, 'pred_mape': 1.252137541770935, 'grad_rmse': 0.47066444158554077, 'grad_mae': 0.20095407962799072, 'grad_mape': 9.563034057617188}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.030571Z", + "iopub.status.busy": "2024-03-22T19:01:16.030259Z", + "iopub.status.idle": "2024-03-22T19:01:16.034407Z", + "shell.execute_reply": "2024-03-22T19:01:16.033634Z" + }, + "papermill": { + "duration": 0.025063, + "end_time": "2024-03-22T19:01:16.036311", + "exception": false, + "start_time": "2024-03-22T19:01:16.011248", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.070909Z", + "iopub.status.busy": "2024-03-22T19:01:16.070227Z", + "iopub.status.idle": "2024-03-22T19:01:16.159934Z", + "shell.execute_reply": "2024-03-22T19:01:16.158930Z" + }, + "papermill": { + "duration": 0.110121, + "end_time": "2024-03-22T19:01:16.162741", + "exception": false, + "start_time": "2024-03-22T19:01:16.052620", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.199073Z", + "iopub.status.busy": "2024-03-22T19:01:16.198774Z", + "iopub.status.idle": "2024-03-22T19:01:16.494447Z", + "shell.execute_reply": "2024-03-22T19:01:16.493514Z" + }, + "papermill": { + "duration": 0.315967, + "end_time": "2024-03-22T19:01:16.496397", + "exception": false, + "start_time": "2024-03-22T19:01:16.180430", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCTUlEQVR4nO3de1xUdf4/8NfMwAz3Qa7DKBcvIAqIhjFipbWyYpGGWSixSn5dtV01k9yMfinWbqFZ38zLN7NttbYIdNdq81aImqsgKJfEUEJDQWFAQIY7AzOf3x8HRgcGnBmQAXk/H4/zgDnnc875zMi8/JxzPud8eIwxBkIIGST4pq4AIYQYgkKLEDKoUGgRQgYVCi1CyKBCoUUIGVQotAghgwqFFiFkUDEzdQX6i1qtRmlpKWxtbcHj8UxdHUJIJ4wx1NXVQSqVgs/vvj01ZEKrtLQU7u7upq4GIeQeSkpKMGLEiG6XD5nQsrW1BcB9IHZ2diauDSGks9raWri7u2u+q90ZMqHVcUhoZ2dHoUXIAHav0zd0Ip4QMqhQaBFCBhUKLULIoDJkzmmR3lGpVGhtbTV1NcggZm5uDoFA0OvtUGiRHjHGIJfLUVNTY+qqkAeAvb09JBJJr/pKUmiRHnUElouLC6ysrKhjLjEKYwyNjY2oqKgAALi5uRm9LQqtThRNrbhcVgsAkI1yNHFtTEulUmkCy9FxaH8WpPcsLS0BABUVFXBxcTH6UJFOxHeSXXwb83efxVvf55u6KibXcQ7LysrKxDUhD4qOv6XenB81KrR27twJLy8vWFhYQCaTITMzs8fy+/fvh6+vLywsLBAQEIDDhw9rLd+4cSN8fX1hbW2NYcOGITQ0FBkZGVplqqurER0dDTs7O9jb22PJkiWor683pvo9crYRAQBu1bf0+bYHKzokJH2lL/6WDA6t5ORkxMbGIj4+HtnZ2QgMDERYWJjmWLWztLQ0REVFYcmSJcjJyUFERAQiIiJw8eJFTRkfHx/s2LEDeXl5OH36NLy8vDBz5kzcunVLUyY6Ohq//PILUlJScPDgQZw6dQrLli0z4i33zMWWC62q+hao1DTmByEDDjNQcHAwW7Fihea1SqViUqmUJSQk6CwfGRnJwsPDtebJZDK2fPnybvehUCgYAHbs2DHGGGP5+fkMADt37pymzJEjRxiPx2M3b97Uq94d21QoFD2Wa1Op2cjXDzLPdQdZRW2zXtt+UDU1NbH8/HzW1NRk6qqQB0RPf1P6fkcNamkplUpkZWUhNDRUM4/P5yM0NBTp6ek610lPT9cqDwBhYWHdllcqldi9ezfEYjECAwM127C3t8fkyZM15UJDQ8Hn87scRnZoaWlBbW2t1qQPAZ8HB+v2Q8Q6OkQkxuHxePj2229NXY0+tXHjRkycONHU1TDs8LCyshIqlQqurq5a811dXSGXy3WuI5fL9Sp/8OBB2NjYwMLCAh9++CFSUlLg5OSk2YaLi4tWeTMzMzg4OHS734SEBIjFYs1kyGNpnG3pvBYZ/Pbu3Qt7e/s+297atWuRmpraZ9sz1oC5evjEE08gNzcXaWlpmDVrFiIjI7s9T6aPuLg4KBQKzVRSUqL3uprQopYWGQKUSqVe5WxsbAZE1xeDQsvJyQkCgQDl5eVa88vLyyGRSHSuI5FI9CpvbW2NMWPGYMqUKfjss89gZmaGzz77TLONzgHW1taG6urqbvcrEok0j6Ex9HE0HVcQK+qa9V5nKGCMoVHZZpKJGTgQ+tGjR/Hoo4/C3t4ejo6OePrpp3H16lUAwNSpU7Fu3Tqt8rdu3YK5uTlOnToFACgrK0N4eDgsLS0xcuRIJCYmwsvLC1u3bjXqs8vLy8Pvfvc7WFpawtHREcuWLdO6+n3y5EkEBwfD2toa9vb2eOSRR3D9+nUAwM8//4wnnngCtra2sLOzQ1BQEM6fP9/j/k6ePInFixdDoVCAx+OBx+Nh48aNAAAvLy/89a9/xaJFi2BnZ6e5oLVu3Tr4+PjAysoKo0aNwvr167W6JnQ+PHzxxRcRERGB999/H25ubnB0dMSKFSvu++1eBnUuFQqFCAoKQmpqKiIiIgBwjzFOTU3FypUrda4TEhKC1NRUvPLKK5p5KSkpCAkJ6XFfarUaLS0tmm3U1NQgKysLQUFBAIDjx49DrVZDJpMZ8hb0Qi0t3ZpaVRi/4QeT7Dv/7TBYCfX/c21oaEBsbCwmTJiA+vp6bNiwAXPnzkVubi6io6Px3nvvYdOmTZpL8MnJyZBKpXjssccAAIsWLUJlZSVOnjwJc3NzxMbGGt3yb2hoQFhYGEJCQnDu3DlUVFTgj3/8I1auXIm9e/eira0NERERWLp0Kb7++msolUpkZmZq6hYdHY1Jkybh448/hkAgQG5uLszNzXvc59SpU7F161Zs2LABBQUFALiWUof3338fGzZsQHx8vGaera0t9u7dC6lUiry8PCxduhS2trZ47bXXut3PiRMn4ObmhhMnTuDKlSuYP38+Jk6ciKVLlxr1WenD4B7xsbGxiImJweTJkxEcHIytW7eioaEBixcvBsD9Yw8fPhwJCQkAgNWrV2P69On44IMPEB4ejqSkJJw/fx67d+8GwP2DvvPOO5gzZw7c3NxQWVmJnTt34ubNm3j++ecBAOPGjcOsWbOwdOlS7Nq1C62trVi5ciUWLFgAqVTaV5+FBoXW4Ddv3jyt1//4xz/g7OyM/Px8REZG4pVXXsHp06c1IZWYmIioqCjweDxcvnwZx44dw7lz5zQXf/7+97/D29vbqLokJiaiubkZX3zxBaytrQEAO3bswOzZs7F582aYm5tDoVDg6aefxujRowFwf/MdiouL8Ze//AW+vr4AoFc9hEIhxGIxeDyezqOR3/3ud3j11Ve15r355pua3728vLB27VokJSX1GFrDhg3Djh07IBAI4Ovri/DwcKSmpg6s0Jo/fz5u3bqFDRs2QC6XY+LEiTh69KjmZHtxcbHWQ+mnTp2KxMREvPnmm3jjjTfg7e2Nb7/9Fv7+/gAAgUCAy5cv4/PPP0dlZSUcHR3x8MMP47///S/8/Pw02/nqq6+wcuVKzJgxA3w+H/PmzcO2bdt6+/51otDSzdJcgPy3w0y2b0MUFhZiw4YNyMjIQGVlJdRqNQDu79Pf3x8zZ87EV199hcceewxFRUVIT0/HJ598AgAoKCiAmZkZHnroIc32xowZg2HDhhlV90uXLiEwMFATWADwyCOPQK1Wo6CgANOmTcOLL76IsLAw/P73v0doaCgiIyM19+fFxsbij3/8I/75z38iNDQUzz//vCbcjHX3lfgOycnJ2LZtG65evYr6+nq0tbXd87SKn5+f1u04bm5uyMvL61Xd7sWoew9XrlzZ7eHgyZMnu8x7/vnnNa2mziwsLHDgwIF77tPBwQGJiYkG1dNY1CteNx6PZ9AhminNnj0bnp6e+PTTTyGVSqFWq+Hv76856RwdHY2XX34Z27dvR2JiIgICAhAQEGCy+u7Zswcvv/wyjh49iuTkZLz55ptISUnBlClTsHHjRrzwwgs4dOgQjhw5gvj4eCQlJWHu3LlG7+/uAAW4bkXR0dF46623EBYWBrFYjKSkJHzwwQc9bqfzYSqPx9P8B3G/DJirhwMJtbQGt6qqKhQUFODNN9/EjBkzMG7cONy+fVurzDPPPIPm5mYcPXoUiYmJiI6O1iwbO3Ys2trakJOTo5l35cqVLtvQ17hx4/Dzzz+joaFBM+/MmTPg8/kYO3asZt6kSZMQFxeHtLQ0+Pv7a/0n7ePjgzVr1uDHH3/Es88+iz179txzv0KhECqVSq86pqWlwdPTE//v//0/TJ48Gd7e3poLAQMNhZYOHaFV19yG5lb9/tHJwDFs2DA4Ojpi9+7duHLlCo4fP47Y2FitMtbW1oiIiMD69etx6dIlREVFaZb5+voiNDQUy5YtQ2ZmJnJycrBs2TJYWloade9cdHQ0LCwsEBMTg4sXL+LEiRNYtWoVFi5cCFdXVxQVFSEuLg7p6em4fv06fvzxRxQWFmLcuHFoamrCypUrcfLkSVy/fh1nzpzBuXPntM55dcfLywv19fVITU1FZWUlGhsbuy3r7e2N4uJiJCUl4erVq9i2bRu++eYbg99rf6DQ0sHOwgxCM+6jodbW4MPn85GUlISsrCz4+/tjzZo12LJlS5dy0dHR+Pnnn/HYY4/Bw8NDa9kXX3wBV1dXTJs2DXPnztVcSbOwsDC4PlZWVvjhhx9QXV2Nhx9+GM899xxmzJiBHTt2aJZfvnwZ8+bNg4+PD5YtW4YVK1Zg+fLlEAgEqKqqwqJFi+Dj44PIyEg8+eSTeOutt+6536lTp+Kll17C/Pnz4ezsjPfee6/bsnPmzMGaNWuwcuVKTJw4EWlpaVi/fr3B77U/8JihHWAGqdraWojFYigUCr36bD2y6Thu1jThwJ+n4iEP407ADnbNzc0oKirCyJEjjfqyPkhu3LgBd3d3HDt2DDNmzDB1dQatnv6m9P2ODo6zqibgbCvCzZomamkNUcePH0d9fT0CAgJQVlaG1157DV5eXpg2bZqpqzbk0eFhN+hk/NDW2tqKN954A35+fpg7dy6cnZ01HU2/+uor2NjY6Jzu7qZzvz355JPd1uPdd9/tt3r0N2ppdYNCa2gLCwtDWJjuPmlz5szp9k6Me/VU70t///vf0dTUpHOZg4NDv9Wjv1FodYP6apHu2NrawtbW1tTVwPDhw01dBZOgw8NuuNhRS4uQgYhCqxualhaFFiEDCoVWN+icFiEDE4VWN+4OrSHSlY2QQYFCqxtO7YeHSpUatU1tJq4NIaQDhVY3LMwFsLPgLq7eqqcnmBLDPIgDWwwUFFo96DhErKDzWmQQ6uuBLQDu0VM8Hg81NTV9ul1DUGj1gE7GEzLwUGj1wNmWu6GTQqsdY4CywTQTDWzRZwNbtLS0YO3atRg+fDisra0hk8m0Ht55/fp1zJ49G8OGDYO1tTX8/Pxw+PBhXLt2DU888QQA7vE/PB4PL774olGfR29Qj/geUK/4TlobgXf7/pn8enmjFBBa37tcOxrYovuBLVauXIn8/HwkJSVBKpXim2++waxZs5CXlwdvb2+sWLECSqUSp06dgrW1NfLz82FjYwN3d3f8+9//xrx581BQUAA7OztYWloa9Zn0BoVWD+jwcPCigS10D2xRXFyMPXv2oLi4WDMozNq1a3H06FHs2bMH7777LoqLizFv3jzN46dHjRqlWb/jnkYXF5c+P1+mLwqtHlBodWJuxbV4TLVvA9DAFrrl5eVBpVLBx8dHa35LS4tmINaXX34Zf/rTn/Djjz8iNDQU8+bNw4QJE4za3/1A57R6QKHVCY/HHaKZYjLwMcezZ89GdXU1Pv30U2RkZCAjIwMAtAa2+Ne//oXW1tYBM7BFeno6pk6diuTkZPj4+ODs2bMAuEFSf/nlF4SHh+P48eMYP3680Y9Crq+vh0AgQFZWFnJzczXTpUuX8NFHHwEA/vjHP+K3337DwoULkZeXh8mTJ2P79u199l57i0KrBx3ntCrpnNagQgNbcHQNbDFp0iSoVCpUVFRgzJgxWtPdh5Hu7u546aWXcODAAbz66qv49NNPNdsEoPeAGfcDhVYPOlpaVQ1KtKnu77BIpO/QwBYcXQNb+Pj4IDo6GosWLcKBAwdQVFSEzMxMJCQk4NChQwCAV155BT/88AOKioqQnZ2NEydOaPbn6ekJHo+HgwcP4tatW1pXQPsNGyIUCgUDwBQKhd7rtKnUbOTrB5nnuoOsXNF0H2s3MDU1NbH8/HzW1DT43ntKSgobN24cE4lEbMKECezkyZMMAPvmm280ZQ4fPswAsGnTpnVZv7S0lD355JNMJBIxT09PlpiYyFxcXNiuXbv02n/nfV24cIE98cQTzMLCgjk4OLClS5eyuro6xhhjcrmcRUREMDc3NyYUCpmnpyfbsGEDU6lUrKWlhS1YsIC5u7szoVDIpFIpW7lypd7/Ji+99BJzdHRkAFh8fDxjjDGlUsk2bNjAvLy8mLm5OXNzc2Nz585lFy5cYIwxtnLlSjZ69GgmEomYs7MzW7hwIausrNRs8+2332YSiYTxeDwWExOjVz069PQ3pe931KjQ2rFjB/P09GQikYgFBwezjIyMHsvv27ePjR07lolEIubv788OHTqkWaZUKtlrr73G/P39mZWVFXNzc2MLFy5kN2/e1NqGp6cnA6A1JSQk6F1nY0KLMcYm/y2Fea47yPJu1Bi03oNgMIdWXyspKWEA2LFjx0xdlUGtL0LL4MPD5ORkxMbGIj4+HtnZ2QgMDERYWFi3fVjS0tIQFRWFJUuWICcnBxEREYiIiMDFixcBAI2NjcjOzsb69euRnZ2NAwcOoKCgAHPmzOmyrbfffhtlZWWaadWqVYZW32AuttRXayg6fvw4/vOf/6CoqAhpaWlYsGABDWwxUBialMHBwWzFihWa1yqVikml0m5bPZGRkSw8PFxrnkwmY8uXL+92H5mZmQwAu379umaep6cn+/DDDw2troaxLa2Yf2Qwz3UHWfK5YqP3PVgN5ZbW0aNHmZ+fH7O0tGQuLi4sIiKCXbt2jTHG2Jdffsmsra11TuPHj++3Os6aNavberzzzjv9Vg9D9EVLy6B+WkqlEllZWYiLi9PM4/P5CA0NRXp6us510tPTu5wEDQsL6/EO+I5bDzp3Xtu0aRP++te/wsPDAy+88ALWrFkDMzPdb6GlpQUtLXdaR7W1tfd4d7rRE0yHJhrYYuAyKLQqKyuhUqng6uqqNd/V1RWXL1/WuY5cLtdZXi6X6yzf3NyMdevWISoqSmvAxpdffhkPPfQQHBwckJaWhri4OJSVleF///d/dW4nISFBr1F474X6apHOaGAL0xpQPeJbW1sRGRkJxhg+/vhjrWV3t9YmTJgAoVCI5cuXIyEhASKRqMu24uLitNapra2Fu7u7wXWi0AI9uZX0mb74WzIotJycnCAQCFBeXq41v7y8XKtj2t0kEole5TsC6/r16zh+/Pg9h66XyWRoa2vDtWvXtDrodRCJRDrDzFBDObQ6DnUaGxtNcmMsefA0NjYC6N1htEGhJRQKERQUhNTUVERERAAA1Go1UlNTsXLlSp3rhISEIDU1Fa+88opmXkpKCkJCQjSvOwKrsLAQJ06c0NwD1ZPc3Fzw+Xy4uLgY8hYMNpSf9CAQCGBvb6+5MmxlZWVU50pCGGNobGxERUUF7O3tIRAIjN6WwYeHsbGxiImJweTJkxEcHIytW7eioaEBixcvBsA90mP48OFISEgAAKxevRrTp0/HBx98gPDwcCQlJeH8+fPYvXs3AC6wnnvuOWRnZ+PgwYNQqVSa810ODg4QCoVIT09HRkaG5plC6enpWLNmDf7whz8YfROrvoZySwuApkVs7GNZCLmbvb19t0dl+jI4tObPn49bt25hw4YNkMvlmDhxIo4ePao52V5cXAw+/073r6lTpyIxMRFvvvkm3njjDXh7e+Pbb7+Fv78/AODmzZv4z3/+AwCYOHGi1r5OnDiBxx9/HCKRCElJSdi4cSNaWlowcuRIrFmzpstVyfuhI7TqW9rQqGyDlXBAnQa873g8Htzc3ODi4oLW1lZTV4cMYubm5r1qYXXgsSFylrW2thZisRgKheKe58vuxhjDuA1H0dyqxqm/PAEPR8MekUII0Y++31G6YfoeeDzenUNEGpWHEJOj0NIDdTAlZOCg0NLDUD8ZT8hAQqGlBwotQgYOCi09ONu0DyU2BPtqETLQUGjpgVpahAwcFFp6oNAiZOCg0NKDC4UWIQMGhZYenO96eukQ6YtLyIBFoaUHRxtu2KRWFUNNI93KQogpUWjpQWQmgL0V9ygNuoJIiGlRaOmJesUTMjBQaOmJriASMjBQaOmJQouQgYFCS09D+QmmhAwkFFp6opYWIQMDhZaeKLQIGRgotPREoUXIwEChpae7e8UTQkyHQktPHSfiqxuUaFWpTVwbQoYuCi09DbMSQsDnxvyrqleauDaEDF0UWnri83lwar8Hkc5rEWI6FFoGoFF5CDE9Ci0D0P2HhJieUaG1c+dOeHl5wcLCAjKZDJmZmT2W379/P3x9fWFhYYGAgAAcPnxYs6y1tRXr1q1DQEAArK2tIZVKsWjRIpSWlmpto7q6GtHR0bCzs4O9vT2WLFmC+vp6Y6pvNBfb9mfFU2gRYjIGh1ZycjJiY2MRHx+P7OxsBAYGIiwsDBUVFTrLp6WlISoqCkuWLEFOTg4iIiIQERGBixcvAgAaGxuRnZ2N9evXIzs7GwcOHEBBQQHmzJmjtZ3o6Gj88ssvSElJwcGDB3Hq1CksW7bMiLdsvI7DwwoKLUJMhxkoODiYrVixQvNapVIxqVTKEhISdJaPjIxk4eHhWvNkMhlbvnx5t/vIzMxkANj169cZY4zl5+czAOzcuXOaMkeOHGE8Ho/dvHlT5zaam5uZQqHQTCUlJQwAUygUer/XzvaeKWKe6w6yl/553uhtEEJ0UygUen1HDWppKZVKZGVlITQ0VDOPz+cjNDQU6enpOtdJT0/XKg8AYWFh3ZYHAIVCAR6PB3t7e8027O3tMXnyZE2Z0NBQ8Pl8ZGRk6NxGQkICxGKxZnJ3d9f3bXaLesUTYnoGhVZlZSVUKhVcXV215ru6ukIul+tcRy6XG1S+ubkZ69atQ1RUFOzs7DTbcHFx0SpnZmYGBweHbrcTFxcHhUKhmUpKSvR6jz2hXvGEmJ6ZqStwt9bWVkRGRoIxho8//rhX2xKJRBCJRH1UMw5dPSTE9AxqaTk5OUEgEKC8vFxrfnl5OSQSic51JBKJXuU7Auv69etISUnRtLI6ttH5RH9bWxuqq6u73e/90NHSalSq0NDS1m/7JYTcYVBoCYVCBAUFITU1VTNPrVYjNTUVISEhOtcJCQnRKg8AKSkpWuU7AquwsBDHjh2Do6Njl23U1NQgKytLM+/48eNQq9WQyWSGvIVesRaZwUooAECtLUJMxtAz/ElJSUwkErG9e/ey/Px8tmzZMmZvb8/kcjljjLGFCxey119/XVP+zJkzzMzMjL3//vvs0qVLLD4+npmbm7O8vDzGGGNKpZLNmTOHjRgxguXm5rKysjLN1NLSotnOrFmz2KRJk1hGRgY7ffo08/b2ZlFRUXrXW98rE/cy7b3jzHPdQZZZVNWr7RBCtOn7HTU4tBhjbPv27czDw4MJhUIWHBzMzp49q1k2ffp0FhMTo1V+3759zMfHhwmFQubn58cOHTqkWVZUVMQA6JxOnDihKVdVVcWioqKYjY0Ns7OzY4sXL2Z1dXV617mvQmve/51hnusOskMXSnu1HUKINn2/ozzGhsaQybW1tRCLxVAoFFrnywz1py+zcOSiHG/N8UPMVK++qyAhQ5y+31G699BA1FeLENOi0DIQdXsgxLQotAxEHUwJMS0KLQPR4SEhpkWhZSAKLUJMi0LLQB2hVVnfArV6SFx4JWRAodAykFP7ifg2NUNNU6uJa0PI0EOhZSBzAR8O1twAFxV19Kx4QvobhZYRqNsDIaZDoWUEOhlPiOlQaBmBQosQ06HQMgKFFiGmQ6FlBM05LeoVT0i/o9AyArW0CDEdCi0jUGgRYjoUWkagm6YJMR0KLSN0nNOqaWxFS5vKxLUhZGih0DKC2NIc5gIeAKCqXmni2hAytFBoGYHP52nuQaTzWoT0LwotI9HJeEJMg0LLSNRXixDToNAyErW0CDENCi0jUWgRYhpGhdbOnTvh5eUFCwsLyGQyZGZm9lh+//798PX1hYWFBQICAnD48GGt5QcOHMDMmTPh6OgIHo+H3NzcLtt4/PHHwePxtKaXXnrJmOr3CZf20KJnahHSvwwOreTkZMTGxiI+Ph7Z2dkIDAxEWFgYKioqdJZPS0tDVFQUlixZgpycHERERCAiIgIXL17UlGloaMCjjz6KzZs397jvpUuXoqysTDO99957hla/z1BLixATMXTo6uDgYLZixQrNa5VKxaRSKUtISNBZPjIykoWHh2vNk8lkbPny5V3KFhUVMQAsJyeny7Lp06ez1atXG1pdDX2H3NbX+WtVzHPdQfbo5tQ+2R4hQ52+31GDWlpKpRJZWVkIDQ3VzOPz+QgNDUV6errOddLT07XKA0BYWFi35Xvy1VdfwcnJCf7+/oiLi0NjY2O3ZVtaWlBbW6s19SVnGwsAXEuLMRrggpD+YmZI4crKSqhUKri6umrNd3V1xeXLl3WuI5fLdZaXy+UGVfSFF16Ap6cnpFIpLly4gHXr1qGgoAAHDhzQWT4hIQFvvfWWQfswhJMt95z45lY16lvaYGthft/2RQi5w6DQMqVly5Zpfg8ICICbmxtmzJiBq1evYvTo0V3Kx8XFITY2VvO6trYW7u7ufVYfK6EZbERmqG9pw626FgotQvqJQYeHTk5OEAgEKC8v15pfXl4OiUSicx2JRGJQeX3JZDIAwJUrV3QuF4lEsLOz05r6Gp2MJ6T/GRRaQqEQQUFBSE1N1cxTq9VITU1FSEiIznVCQkK0ygNASkpKt+X11dEtws3NrVfb6Q3qFU9I/zP48DA2NhYxMTGYPHkygoODsXXrVjQ0NGDx4sUAgEWLFmH48OFISEgAAKxevRrTp0/HBx98gPDwcCQlJeH8+fPYvXu3ZpvV1dUoLi5GaWkpAKCgoAAA10qTSCS4evUqEhMT8dRTT8HR0REXLlzAmjVrMG3aNEyYMKHXH4KxqKVFiAkYc2ly+/btzMPDgwmFQhYcHMzOnj2rWTZ9+nQWExOjVX7fvn3Mx8eHCYVC5ufnxw4dOqS1fM+ePQxAlyk+Pp4xxlhxcTGbNm0ac3BwYCKRiI0ZM4b95S9/Maj7Ql93eWCMsfjvLjLPdQfZ5iOX+mybhAxV+n5HeYwNjev1tbW1EIvFUCgUfXZ+a+eJK9jyQwGeDxqBLc8H9sk2CRmq9P2O0r2HvUDntAjpfxRavUDntAjpfxRavUChRUj/o9DqhY7QqmpQQqUeEqcGCTE5Cq1ecLAWgscDVGqG2400wAUh/YFCqxfMBXw4WHH3INIhIiH9g0Krl5w1DwOk0CKkP1Bo9RKdjCekf1Fo9RKFFiH9i0Krlyi0COlfFFq9RL3iCelfFFq9dKelRaPyENIfKLR6iQ4PCelfFFq95EKhRUi/otDqpY5ReWqb29DcqjJxbQh58FFo9ZKdpRmEAu5jrKST8YTcdxRavcTj8ei8FiH9iEKrs4ZK4MxHwPk9eq/iRKFFSL+h0Ors6nEgZQNw6n1Ard85KuqrRUj/odDqbNwcwNIBqL0BFKbotQodHhLSfyi0OjO3ACa+wP2epd8hIoUWIf2HQkuXIG4MR/z6A1BTfM/iFFqE9B8KLV2cxgAjpwFgQPYX9yzecU6LnqlFyP1HodWdyf/D/cz+J6Bq7bEotbQI6T9GhdbOnTvh5eUFCwsLyGQyZGZm9lh+//798PX1hYWFBQICAnD48GGt5QcOHMDMmTPh6OgIHo+H3NzcLttobm7GihUr4OjoCBsbG8ybNw/l5eXGVF8/Y8MBaxegXg4UHOmxqOZWnvoWDJGxbwkxGYNDKzk5GbGxsYiPj0d2djYCAwMRFhaGiooKneXT0tIQFRWFJUuWICcnBxEREYiIiMDFixc1ZRoaGvDoo49i8+bN3e53zZo1+P7777F//3789NNPKC0txbPPPmto9fVnJgQm/YH7/fw/eiza0dJStqlR29x2/+pECAGYgYKDg9mKFSs0r1UqFZNKpSwhIUFn+cjISBYeHq41TyaTseXLl3cpW1RUxACwnJwcrfk1NTXM3Nyc7d+/XzPv0qVLDABLT0/Xq94KhYIBYAqFQq/yjDHGqosYixczFm/HWNXVHov6xx9lnusOssLyOv23TwjR0Pc7alBLS6lUIisrC6GhoZp5fD4foaGhSE9P17lOenq6VnkACAsL67a8LllZWWhtbdXajq+vLzw8PLrdTktLC2pra7Umgw3zAsa07zNrb49F6bwWIf3DoNCqrKyESqWCq6ur1nxXV1fI5XKd68jlcoPKd7cNoVAIe3t7vbeTkJAAsVismdzd3fXen5bJ7d0fcr4E2roPJOoVT0j/eGCvHsbFxUGhUGimkpIS4zbkHQbYSoHGKuDS990Wo5YWIf3DoNBycnKCQCDoctWuvLwcEolE5zoSicSg8t1tQ6lUoqamRu/tiEQi2NnZaU1GEZgBQTHc7z3cRE2hRUj/MCi0hEIhgoKCkJqaqpmnVquRmpqKkJAQneuEhIRolQeAlJSUbsvrEhQUBHNzc63tFBQUoLi42KDtGG3SQoDHB66fBm4V6CxCoUVI/zAzdIXY2FjExMRg8uTJCA4OxtatW9HQ0IDFi7lzP4sWLcLw4cORkJAAAFi9ejWmT5+ODz74AOHh4UhKSsL58+exe/duzTarq6tRXFyM0tJSAFwgAVwLSyKRQCwWY8mSJYiNjYWDgwPs7OywatUqhISEYMqUKb3+EO5JPBzweRIoOMSdkJ+V0KUIndMipJ8Yc2ly+/btzMPDgwmFQhYcHMzOnj2rWTZ9+nQWExOjVX7fvn3Mx8eHCYVC5ufnxw4dOqS1fM+ePQxAlyk+Pl5Tpqmpif35z39mw4YNY1ZWVmzu3LmsrKxM7zob1eXhbr+mcF0fEtwZUzZ2WXzicjnzXHeQzdp6yrjtEzLE6fsd5TE2NLpw19bWQiwWQ6FQGHd+S60GtgVyN1BHfHznSRDt8ktr8dS2/8JGZIb0uN/B1sK8j2pOyNCg73f0gb162Of4fCDoRe53HSfkfVxtMNLJGvUtbfj45NX+rRshQwiFliEmLQT4ZsCNTECep7XITMBH3JO+AIC/ny7CjduNpqghIQ88Ci1D2LgAvk9zv+tobf1+vCumjHKAsk2NLT/ovspICOkdCi1DdTyy5sI+oKVeaxGPx8Ob4ePB4wHf5ZYit6Sm/+tHyAOOQstQI6cBjmMAZR1w8V9dFvsPF+PZSSMAAH87mE+PqiGkj1FoGYrHu+uEvO5H1vwlbCwszPk4f/02jlzU/x5LQsi9UWgZI/AFQCACyn4GbmZ3WSwRW2DZtNEAgE1HLqOlTb+hyAgh90ahZQxrR8Avgvu9m9bW8mmj4GIrQnF1I75Iu95/dSPkAUehZayOEXsu/htoVnRZbC0yw9qZYwEA244XorpB2Z+1I+SBRaFlLI8pgPM4oLWRu5Kow7ygERjnZoe65jZsSy3s5woS8mCi0DIWj3en+8P5fwA6rhIK+Dy8GT4OAPDPs9dxpaK+SxlCiGEotHpjQiRgZglU5AMlGTqLPDLGCTN8XaBSM2w6cqmfK0jIg4dCqzcs7YGAedzvPYzYE/fUOAj4PBy7VIG0K5X9UzdCHlAUWr3VcYj4y7dAY7XOImNcbBAt8wAA/O3QJajU1OGUEGNRaPWW9CFAMgFQtQC5id0WWz3DG7YWZsgvq8WB7Bv9WEFCHiwUWr119wn5rD06T8gDgKONCKt+NwYAsOWHAjQqaVBXQoxBodUXAp4DhLZA1RXg2n+7LRYz1QvuDpaoqGvBJz/91o8VJOTBQaHVF0S2wITnud/PfAS0NukuZibA67O4LhC7T/0GuaK5v2pIyAODQquvTF4CgAdcOQb83xTg1x90FnsqQIIgz2FoalXh/R/pmVuEGIpCq69I/IH5X3IDu96+BiRGAl9Hcb/fhXvmFtfa+nf2DVy82fUWIEJI9yi0+tK4p4GV54BHVnOPZS44DOyUASc3A613DgUneQzDnEApGAPeOXSJnrlFiAEotPqayAb4/dvAn9K4Bwa2NQMn3+UOGQtTNMVemzUWQjM+0n+rwrFLFSasMCGDC4XW/eI8Flj0H+C5fwC2bsDtIuCr54CkaOD2dYwYZoUlj44EACQcvoRWldrEFSZkcKDQup94PMB/HnfIOHUVd8h4+SB3yHhqC/786Ag4WgvxW2UDvjpLz9wiRB9GhdbOnTvh5eUFCwsLyGQyZGZm9lh+//798PX1hYWFBQICAnD48GGt5YwxbNiwAW5ubrC0tERoaCgKC7Uf5eLl5QUej6c1bdq0yZjq9z+RLTDzb8BLpwGvx4C2JuD432D7j8fw3kTu0PCDlF9xpaLOxBUlZOAzOLSSk5MRGxuL+Ph4ZGdnIzAwEGFhYaio0H1eJi0tDVFRUViyZAlycnIQERGBiIgIXLx4UVPmvffew7Zt27Br1y5kZGTA2toaYWFhaG7W7sf09ttvo6ysTDOtWrXK0Oqblss4IOZ7YN5ngI0EqP4NM7L+jCTxDtg2yxHzj3Mor6W+W4T0iBkoODiYrVixQvNapVIxqVTKEhISdJaPjIxk4eHhWvNkMhlbvnw5Y4wxtVrNJBIJ27Jli2Z5TU0NE4lE7Ouvv9bM8/T0ZB9++KHe9WxubmYKhUIzlZSUMABMoVDovY37qknB2NE3GNs4jLF4O1a10Z09tO4rFvbhT0zRpDR17QjpdwqFQq/vqEEtLaVSiaysLISGhmrm8fl8hIaGIj09Xec66enpWuUBICwsTFO+qKgIcrlcq4xYLIZMJuuyzU2bNsHR0RGTJk3Cli1b0NbW/f17CQkJEIvFmsnd3d2Qt3r/WdgBYe9wh4xOY+HAFNhk+U9cltdh+RdZNBgGId0wKLQqKyuhUqng6uqqNd/V1RVyue6hsuRyeY/lO37ea5svv/wykpKScOLECSxfvhzvvvsuXnvttW7rGhcXB4VCoZlKSkr0f6P9yXU88OxugCfA71kanhGeR/pvVXh1389Q0yNsCOnCzNQV0FdsbKzm9wkTJkAoFGL58uVISEiASCTqUl4kEumcPyBJJwKPvgL89wNssfoCp9vG4eCFMrjaWWD90+NNXTtCBhSDWlpOTk4QCAQoLy/Xml9eXg6JRKJzHYlE0mP5jp+GbBMAZDIZ2tracO3aNUPewsA17TXAaSyEzZX4ZvRBAMBnp4vw6Sl6GgQhdzMotIRCIYKCgpCamqqZp1arkZqaipCQEJ3rhISEaJUHgJSUFE35kSNHQiKRaJWpra1FRkZGt9sEgNzcXPD5fLi4uBjyFgYucwvgmR0AePAo+Q7/F1wFAHjn8CV8l3vTtHUjZCAx9Ax/UlISE4lEbO/evSw/P58tW7aM2dvbM7lczhhjbOHChez111/XlD9z5gwzMzNj77//Prt06RKLj49n5ubmLC8vT1Nm06ZNzN7enn333XfswoUL7JlnnmEjR45kTU1NjDHG0tLS2Icffshyc3PZ1atX2ZdffsmcnZ3ZokWL9K63vlcmTO5IHGPxdkz9wTj27jcZzHPdQTbmjUPsdOEtU9eMkPtK3++owaHFGGPbt29nHh4eTCgUsuDgYHb27FnNsunTp7OYmBit8vv27WM+Pj5MKBQyPz8/dujQIa3larWarV+/nrm6ujKRSMRmzJjBCgoKNMuzsrKYTCZjYrGYWVhYsHHjxrF3332XNTc3613nQRNaLQ2MbQ3kgus/q9mfv8xinusOMr8NR9nFmzWmrh0h942+31EeY0PjEQO1tbUQi8VQKBSws7MzdXV6VvRf4POnAQDK6O+w8LgIGUXVcLYV4cCfpsLdwcrEFSSk7+n7HaV7DweikY+1P1QQEB5ejd0LxsFXYotbdS2I2ZOJ2w1KE1ewHw2N/1OJASi0BqrQjYDdCOD2NYjTN2Pv4mBIxRb47VYDlnx+Dk3KB7zzqaoNOPoGsGU0cOl7U9eGDCAUWgOVhR0w+yPu97MfQ6L4GXv/Jxh2FmbILq7Bqq9z0PagPs6muRb4egFwdifQWAV88yeg6qqpa0UGCAqtgcw7FJgYDYAB362Aj4M5PnvxYQjN+Dh2qRzrv/vlwXvq6e3rwGczgSspgJkl4DwOUNYB/1oMtLWYunZkAKDQGujC3gFsXIGqQuCnzXjYywHbFkwEjwd8nVmMD48VPjjBVZIJfPo74NYl7ikY/3ME+MO/AUsHoOxnIGWDqWtIBgAKrYHOchjw9Ifc72c+AkpzMMvfDW/N8QMAbEstxJLPzw/+R9pc2A/sfRporAQkAcDS44B0EiAeDszdxZXJ2AVcOnj/6tDWAvzyDVBZeO+yxGQotAYD33DA71mAqYDvVgJtSiwK8cKGp8dDKODj+OUK/P5/f8I3OTcGX6uLMeBEAnDgj4CqBRj7FLD4KBdWHXzCgJCV3O/f/RmoKe77erS1AMkLgf0vAjsmA5/OADI/BRqr+35fpFeon9Zg0VAJ7AzmTkw//gbw+DoAwK/ldXh138/Iax+KbOZ4V7wzNwDOtjpuFq8rB/L2AT8nAfUV3MAb3jOBMaGAtWN/vhtOaxPw3Qrg4r+511Nf5q6a8gVdy7YpgT2zgJtZwIiHgcVHAIF539RD1QrsiwEKDgECIaBWcf9BAADfHBg7CwiMAsb8HjAT9s0+SRf6fkcptAaTvH8B/17CfZGWn+IeawOgVaXGrpNXse14IVpVDMOszPHXCH88PUHKtSAKjgA/f82NBsR0dZXgASMmA95hgM9MQDKBe779/VRfASS9ANw4xz07/+kPgYcW9bzO7WvArmlAi4Ibpu33b/e+HqpW4F//A1z6DyAQAS8kA65+QN5+7jOT590pa+UI+D8HBC7gDl3v92c0xFBodfJAhBZj3Gg+BYcA6UPAkhRAcOfpQvmltXh1/8+4VKZAAK8Ia13O47GWk+A319zZxoiHgYkvAE4+wJVUoPBHoPyi9n5sJID377lW2OgnuGfc96XyX4DE+YCiBLCwB+b/k2v16SP/O2Bfe7hF/4urp7FUbcCBpcAvB7gW1oKvuSu2d5Nf5MIrbz9Qf9eTSJx9ufAKiNQ+lCVGo9Dq5IEILQCoLeNG82lRcC2NR1bfWVYnR1tuMmrS9sKp6c4jbZotXWERFN0eVt5dt6m4yYVXYQrw20mgteHOMr454BnCtcK8Z3Lr96aF8euPXPcFZT3gMBp4YR/gNMawbRxaC5z7lGv5vHQasJMaXg+1CvjmJe5wmW/OjQ4+dlb35VVtwG8nuAC7fIgbzxIAwANGPc4dPvqGc+NeEqNQaHXywIQWAOR8yZ0LMrPgrrJV/grkJnItp/bDP7VAhJN8GfY0TMUZtT/mTByBjXP8YG91j3MybS3A9TNcuBT+CFR36tRpKwUcRwP2HoC9Z/vP9slOqvt8FMC1EjM+AX6IA5iaG5Uo8gvAysHw99/aDHwWyh26eT7CjS8pMOB5lmo18J+VQO5X3KHp859zo4Prq1nBtfhyvwaK0+7MN7PkDq/9nuUCXkj3iBqCQquTByq0GAO+fBa4erzrMncZ16Lym4sWMxtsPVaIT366CjUDnG1F2PRsAGaMc+26XneqrnLh9esPXJiperjvkW8GiEfcFWSed4Lt4r+Ac3/nyk36AxD+Ye9OaldeAXZP51ps09cBT7yh33pqNXBwNZD9BcATcIPp+kUYX4/qIuDCPuBCElB91wMbza0An1mA/7PchQ5zS+P30UGt4g6tSzK4CxLiEcDk/zGupTkAUWh18kCFFsBd9v+/qVxvcbvh3PmVwCidh3/Zxbexdv/P+O0Wd9j3XNAIbJg9HnYWBl59a6nnWjeKEq7nes11rh41xdw8dfcDjXB43CHt1FV9cxL7wn6uqwR4wKLvgFHTey7PGHDoVeD8ZwCPDzz7KRDwXO/r0bHtsp+582O/fKPdLUNow3Xl8JsLjJkBmOn5GPCWOuDGeS6kis9yvys7jY3JN+cGBJ66kuvfNohRaHXywIUWwLWC6su51lV3h2XtmltVeP+HAnx2pgiMAcOszPGErwum+zjjMW9nOFj38lK+WgXUld0JsZpiLtRutwcbjwfMfMeww7D2eluY9/DevlsJ5PyTu2vgpdOATTdPsmUMOLIOyPwEAA+Y+wkQON+guuiNMeBmdnuAfQvU3rizTGR3p9/dqMe1W5uKG1w4dYRU+UXuUPpuQlvA/WFgeBBw7Yz24emox4GQVVwwDsIrmxRanTyQoWWEc9eq8Zf9P+NaVaNmHo8HTBguxjQfZ0z3ccZEd3uYCUzT77i8thnpV6uQfrUKab9VoqS6CW5iC4x3s8N4qR3Gu9nBTyqGu4MleDweoGwEPn0CuHUZGPUE8IcDAL9T3RkDfnwTSN/BvX5mJ3eI2h/UauDmeeDiASD/Wy7YO1jYcwHW1gwUZ2iHWwexB+Ah4/5j8pgCuIzX/g/qZhaQtoM7x9bRncV5HBCyApgQqX+rbgCg0OqEQusOZZsa569V46dfb+GnX2/hslz7kMPWwgyPjnHCdB9nTPNxhtS+D87HdKO6QYmzv1Uh7Wol0q9W4eqthnuvBMBWZIZx7UE2xbYCM08vAF/VDPxuPTBt7Z2CjAHHNgJntnKvZ38EBL3Y129DP2o1UHKWO3z85VugodOo7DwBIPEH3KdwAeUu0787xe3r3G1O2V9w5/kArvUZvJR7NpsxFzz6GYVWJxRa3ZMrmnGq8BZO/XoL/y2shKKpVWu5t4uNJsCCRzr0fLh2D4qmVmQWVXMtqauVXQKTxwP8pWJMHe2IKaMd4Se1w/WqRuSX1iK/tBa/lCnwq7weyk6P5XlecBJbzHdDBT7+z3MrrLynYaK7GIG/7oDZmQ+4Qk+9z32JTai6QYkCeR0K5TVo+e00JPKfAJEteB4ySP0ew3gvaa8+XzTVANmfA2d3AXWl3DwzS2BSNDDlz9yV3wGKQqsTCi39qNQMF27U4KdfuRDLLalB5zFjhQI+LMz5sBQKYGkugKXQDJaa12btP/l3LROgQdmGjN+qkHdT0WV7vhJbTBnliKmjHSEb6QixVc8XCFpValy9VX8nyEprkV+qwEbVR5grOIMy5oCnWt7FQsExxJr/CwBwZMQrUE5ehknuw+4cWt5HDS1t+LW8Dr+W16FAXo9fy+twWV6HyvqeH69jLuBhvJsdJnkMwyQPezzkMQwjhhlRX1Ur16JL2w7IL7TP5HGHowHPcQ+YtJVwU1/dDtVLFFqdUGgZp6ZRidNXKnGq/VCyvLb3z7Qa5WSNkNGOCBntiCmjHOFk0/vzLowxlN2qhPiLUFjXX8MtMzc4t3Hnj/7WGo2/q8I1ZR2thZjkYc8Fg7s9Jrjbw0akfz8vtZqhrqUNNY1K3G5sxe1GJarrlbh6iwungvI6lFQ3dbu+h4MVfFxtMVZigzEuNpArWpBTfBvZxTU6Q83ZVoRJ7vZ4yLO9viPsYSnUszXGGFB0ijufV/ij7jLWzu0B5nbXTzft19ZO97zY01sUWp1QaPUeYwyKplY0tarQpFShUalCc6tK8/run5pl7a95PCDIcxhCRjlBIra4f5UsuwD8PZR7YgSAmqlv4JTrQuQU30ZOcQ1+KVWgVaX9J8/nAT6utpjkYY+A4fZgYKhpbMXtBi6UFE13wqmmsRWKplaoOjcXdXC2FcFXYssFlKstfCS28HaxgXU3AckYw43bTchur2tO8W38UlqLtk77EvB5GOdmi5FONrC3NIfY0hz2VuawszS/67VQM19zuFlxGcjczXVbqZNzFwXUrTpqogNPwN3OxeNx3UXAa79Cqc9PAFHJmntlu0Oh1QmF1hCSmwj88Ab31IjHYrUWNbeqkF9WqwmFnOIa3KzpvlXUE0tzAYZZcQExzNocno7WGOtqi7HtQdXrbiTt9c27qeBaYtdrkF18GxV1hrV2hWZ8rXCTiC3h7WIDb2crjBW3wt1MAfPGCi7Easu4nx2hVifnLhh07nphqGU/AdKJPRah0OqEQmuIYUzvvkoVtc3IKalBTnEN8stqIRTwMczKHMOshbC3MscwKyHsLe+E07D2VkyvTpgbiTGGUkUzcopvQ65oRm1TK2qauNZfRyvw7kmfFqG5gIeRTtbwdrHFGBcbeLvawNvFFl5OVhCZCbj7LhsquM6ujAFgBvwE99NlHCC07rEe9zW0du7ciS1btkAulyMwMBDbt29HcHBwt+X379+P9evX49q1a/D29sbmzZvx1FNPaZYzxhAfH49PP/0UNTU1eOSRR/Dxxx/D2/tO7+7q6mqsWrUK33//Pfh8PubNm4ePPvoINjb63aBKoUWGGsYY6lvaNIFW29SK242tuHG7EYUV9SisqMeV8jo0dDOyk4DPg5ejFbxdbOHtagOJ2AICHg98Hg98Pg98HleGx+O1z0f7fB4EfNw1n4dAdzFs73EHxn0LreTkZCxatAi7du2CTCbD1q1bsX//fhQUFMDFpWtv5LS0NEybNg0JCQl4+umnkZiYiM2bNyM7Oxv+/v4AgM2bNyMhIQGff/45Ro4cifXr1yMvLw/5+fmwsODOfzz55JMoKyvDJ598gtbWVixevBgPP/wwEhMT9ao3hRYhXXW03ArL63ClgruQwIVZPepa7nVblv6+X/koAkaIeyxz30JLJpPh4Ycfxo4dXO9itVoNd3d3rFq1Cq+//nqX8vPnz0dDQwMOHrzzbO8pU6Zg4sSJ2LVrFxhjkEqlePXVV7F2LdcpUKFQwNXVFXv37sWCBQtw6dIljB8/HufOncPkyZMBAEePHsVTTz2FGzduQCq99w2jFFqE6I8xhvLaFk2IFZbXoapBCcYYVGoGNQPUjHGTGlAxpnOZSs1ta8cLD2GMS89HRfp+Rw14ngegVCqRlZWFuLg4zTw+n4/Q0FCkp6frXCc9PR2xsdonQ8PCwvDtt98CAIqKiiCXyxEaeufha2KxGDKZDOnp6ViwYAHS09Nhb2+vCSwACA0NBZ/PR0ZGBubOndtlvy0tLWhpuXPCsra21pC3SsiQxuPxIBFbQCK2wDQfZ1NXR4tBN5hVVlZCpVLB1VX70Saurq6Qy+U615HL5T2W7/h5rzKdDz3NzMzg4ODQ7X4TEhIgFos1k7u7u57vkhAykD2wo/HExcVBoVBoppKSElNXiRDSBwwKLScnJwgEApSXl2vNLy8vh0Qi0bmORCLpsXzHz3uVqajQvrm0ra0N1dXV3e5XJBLBzs5OayKEDH4GhZZQKERQUBBSU1M189RqNVJTUxESEqJznZCQEK3yAJCSkqIpP3LkSEgkEq0ytbW1yMjI0JQJCQlBTU0NsrKyNGWOHz8OtVoNmUxmyFsghAx2zEBJSUlMJBKxvXv3svz8fLZs2TJmb2/P5HI5Y4yxhQsXstdff11T/syZM8zMzIy9//777NKlSyw+Pp6Zm5uzvLw8TZlNmzYxe3t79t1337ELFy6wZ555ho0cOZI1NTVpysyaNYtNmjSJZWRksNOnTzNvb28WFRWld70VCgUDwBQKhaFvmRDSD/T9jhocWowxtn37dubh4cGEQiELDg5mZ8+e1SybPn06i4mJ0Sq/b98+5uPjw4RCIfPz82OHDh3SWq5Wq9n69euZq6srE4lEbMaMGaygoECrTFVVFYuKimI2NjbMzs6OLV68mNXV1eldZwotQgY2fb+jdBsPIWRAuC/9tAazjmym/lqEDEwd3817taOGTGjV1XFPyKT+WoQMbHV1dRCLu7/lZ8gcHqrVapSWlsLW1vaeT4Gsra2Fu7s7SkpK6FDyLvS5dI8+G90M+VwYY6irq4NUKgW/8+AkdxkyLS0+n48RI0YYtA7179KNPpfu0Wejm76fS08trA4PbI94QsiDiUKLEDKoUGjpIBKJEB8fD5Fo8Ax02R/oc+kefTa63Y/PZciciCeEPBiopUUIGVQotAghgwqFFiFkUKHQIoQMKhRahJBBhUKrk507d8LLywsWFhaQyWTIzMw0dZVMbuPGjeDxeFqTr6+vqavV706dOoXZs2dDKpWCx+NpBmfpwBjDhg0b4ObmBktLS4SGhqKwsNA0le1n9/psXnzxxS5/Q7NmzTJqXxRad0lOTkZsbCzi4+ORnZ2NwMBAhIWFdXnU81Dk5+eHsrIyzXT69GlTV6nfNTQ0IDAwEDt37tS5/L333sO2bduwa9cuZGRkwNraGmFhYWhubu7nmva/e302ADBr1iytv6Gvv/7auJ3d16d6DTLBwcFsxYoVmtcqlYpJpVKWkJBgwlqZXnx8PAsMDDR1NQYUAOybb77RvFar1UwikbAtW7Zo5tXU1DCRSMS+/vprE9TQdDp/NowxFhMTw5555pk+2T61tNp1jOl49/iL9xrTcSgpLCyEVCrFqFGjEB0djeLiYlNXaUC51/idBDh58iRcXFwwduxY/OlPf0JVVZVR26HQamfMmI5DhUwmw969e3H06FF8/PHHKCoqwmOPPaZ5RhnRb/zOoWzWrFn44osvkJqais2bN+Onn37Ck08+CZVKZfC2hsyjaYjxnnzySc3vEyZMgEwmg6enJ/bt24clS5aYsGZksFiwYIHm94CAAEyYMAGjR4/GyZMnMWPGDIO2RS2tdsaM6ThU2dvbw8fHB1euXDF1VQYMfcbvJHeMGjUKTk5ORv0NUWi1M2ZMx6Gqvr4eV69ehZubm6mrMmDoM34nuePGjRuoqqoy6m+IDg/vEhsbi5iYGEyePBnBwcHYunUrGhoasHjxYlNXzaTWrl2L2bNnw9PTE6WlpYiPj4dAIEBUVJSpq9av6uvrtVoGRUVFyM3NhYODAzw8PPDKK6/gb3/7G7y9vTFy5EisX78eUqkUERERpqt0P+nps3FwcMBbb72FefPmQSKR4OrVq3jttdcwZswYhIWFGb6zPrkG+QDpaUzHoWr+/PnMzc2NCYVCNnz4cDZ//nx25coVU1er3504cYIB6DJ1jPOpz/idD6qePpvGxkY2c+ZM5uzszMzNzZmnpydbunSpZoBnQ9HztAghgwqd0yKEDCoUWoSQQYVCixAyqFBoEUIGFQotQsigQqFFCBlUKLQIIYMKhRYhZFCh0CKEDCoUWoSQQYVCixAyqPx/ThKEiYyLfQoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.535119Z", + "iopub.status.busy": "2024-03-22T19:01:16.534818Z", + "iopub.status.idle": "2024-03-22T19:03:19.526880Z", + "shell.execute_reply": "2024-03-22T19:03:19.526064Z" + }, + "papermill": { + "duration": 123.014918, + "end_time": "2024-03-22T19:03:19.529471", + "exception": false, + "start_time": "2024-03-22T19:01:16.514553", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:03:19.568996Z", + "iopub.status.busy": "2024-03-22T19:03:19.568673Z", + "iopub.status.idle": "2024-03-22T19:03:19.589191Z", + "shell.execute_reply": "2024-03-22T19:03:19.588310Z" + }, + "papermill": { + "duration": 0.042336, + "end_time": "2024-03-22T19:03:19.591200", + "exception": false, + "start_time": "2024-03-22T19:03:19.548864", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0109920.0049930.000584.4443540.2009549.5630330.4706644.327647e-075.6215140.0151061.2521370.0240850.1404880.00144710.065868
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.010992 0.004993 0.00058 4.444354 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.200954 9.563033 0.470664 4.327647e-07 5.621514 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.015106 1.252137 0.024085 0.140488 0.001447 \n", + "\n", + " total_duration \n", + "realtabformer 10.065868 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:03:19.626727Z", + "iopub.status.busy": "2024-03-22T19:03:19.626449Z", + "iopub.status.idle": "2024-03-22T19:03:20.096934Z", + "shell.execute_reply": "2024-03-22T19:03:20.096067Z" + }, + "papermill": { + "duration": 0.490882, + "end_time": "2024-03-22T19:03:20.099222", + "exception": false, + "start_time": "2024-03-22T19:03:19.608340", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:03:20.137539Z", + "iopub.status.busy": "2024-03-22T19:03:20.136699Z", + "iopub.status.idle": "2024-03-22T19:05:40.481644Z", + "shell.execute_reply": "2024-03-22T19:05:40.480813Z" + }, + "papermill": { + "duration": 140.366735, + "end_time": "2024-03-22T19:05:40.484194", + "exception": false, + "start_time": "2024-03-22T19:03:20.117459", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:40.523302Z", + "iopub.status.busy": "2024-03-22T19:05:40.522476Z", + "iopub.status.idle": "2024-03-22T19:05:40.548513Z", + "shell.execute_reply": "2024-03-22T19:05:40.547780Z" + }, + "papermill": { + "duration": 0.047483, + "end_time": "2024-03-22T19:05:40.550447", + "exception": false, + "start_time": "2024-03-22T19:05:40.502964", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:40.585958Z", + "iopub.status.busy": "2024-03-22T19:05:40.585710Z", + "iopub.status.idle": "2024-03-22T19:05:40.591049Z", + "shell.execute_reply": "2024-03-22T19:05:40.590252Z" + }, + "papermill": { + "duration": 0.025293, + "end_time": "2024-03-22T19:05:40.592955", + "exception": false, + "start_time": "2024-03-22T19:05:40.567662", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.04078416973075253}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:40.630543Z", + "iopub.status.busy": "2024-03-22T19:05:40.630098Z", + "iopub.status.idle": "2024-03-22T19:05:41.096810Z", + "shell.execute_reply": "2024-03-22T19:05:41.095666Z" + }, + "papermill": { + "duration": 0.490429, + "end_time": "2024-03-22T19:05:41.101208", + "exception": false, + "start_time": "2024-03-22T19:05:40.610779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAh0lEQVR4nO3deXhU1f348fedO0sm22QhkAQSEgiCiiCiIG5gpSqu1IoVbQEVsC3WWqQ/xadF0SrWKoVqS7XVRFqXVr+obS3u4lYBBVQUZRNCWMOSPZPZ7vn9McmQIdtkstxJ+LyeZ55kbs7c+7mTzCfnnnsWTSmlEEIIE1nMDkAIISQRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOkkEYkW3XPPPWia1qn7LCoqQtM0Pv3004jK/+53v2PQoEHous6pp57aqbGI2CGJSLTLAw88wMsvv9wtx3rjjTf4f//v/3H22WdTWFjIAw880C3HFd3PanYAomd54IEHuPrqq5k8eXKXH+udd97BYrHw5JNPYrfbu/x4wjxSI+olampqzA6h05WWluJ0OjstCSmlcLvdnbKvjqitrTU7hJgjiagHami72bRpE9dddx2pqamcc845APz9739n9OjROJ1O0tLSuPbaaykpKQl7/QcffMCUKVPIzc3F4XCQk5PDL37xizY/pJqmUVNTw9NPP42maWiaxowZMwAoLi7mpz/9KUOHDsXpdJKens6UKVPYuXNns/uqra3l5ptvJj09neTkZKZNm0ZZWVnYsQoLC6mpqQkdq6ioCAC/3899993H4MGDcTgc5OXlcdddd+HxeMKOkZeXx2WXXcbrr7/O6aefjtPp5PHHH2fVqlVomsY///lPFi5cSP/+/UlKSuLqq6+moqICj8fDbbfdRt++fUlMTOSGG25osu9I3+sJEyYwfPhw1q1bx3nnnUd8fDx33XVXq+/z8UguzXqwKVOmMGTIEB544AGUUtx///38+te/5pprrmHmzJkcPHiQRx99lPPOO48NGzaQkpICwAsvvEBtbS0/+clPSE9PZ+3atTz66KPs3r2bF154ocXj/e1vf2PmzJmMGTOG2bNnAzB48GAAPvnkE/73v/9x7bXXMmDAAHbu3MmyZcuYMGECmzZtIj4+Pmxft9xyCykpKdxzzz1s3ryZZcuWUVxcHEoSf/vb33jiiSdYu3Ytf/3rXwE466yzAJg5cyZPP/00V199Nbfffjtr1qxh0aJFfP3117z00kthx9m8eTNTp07l5ptvZtasWQwdOjT0s0WLFuF0OrnzzjvZtm0bjz76KDabDYvFQllZGffccw+rV6+mqKiI/Px8FixYEHptpO81wOHDh5k0aRLXXnstP/zhD+nXr187f9PHASV6nLvvvlsBaurUqaFtO3fuVLquq/vvvz+s7MaNG5XVag3bXltb22SfixYtUpqmqeLi4ibHaSwhIUFNnz69yeub2+fHH3+sALV8+fLQtsLCQgWo0aNHK6/XG9r+0EMPKUC98soroW3Tp09XCQkJYfv87LPPFKBmzpwZtn3evHkKUO+8805o28CBAxWgXnvttbCy7777rgLU8OHDw2KYOnWq0jRNTZo0Kaz8uHHj1MCBA0PP2/Nejx8/XgHqz3/+c5P3Rxwll2Y92I9//OPQ9ytWrMAwDK655hoOHToUemRmZjJkyBDefffdUFmn0xn6vqamhkOHDnHWWWehlGLDhg1RxdJ4nz6fj8OHD1NQUEBKSgrr169vUn727NnYbLbQ85/85CdYrVb++9//tnqchp/PnTs3bPvtt98OwKuvvhq2PT8/n4suuqjZfU2bNi0shrFjx6KU4sYbbwwrN3bsWEpKSvD7/UD73msAh8PBDTfc0Op5He/k0qwHy8/PD32/detWlFIMGTKk2bKNP3C7du1iwYIF/Otf/wprlwGoqKiIKha3282iRYsoLCxkz549qEYTfza3z2PjTExMJCsrq8U2pQbFxcVYLBYKCgrCtmdmZpKSkkJxcXHY9sbv0bFyc3PDnrtcLgBycnKabDcMg4qKCtLT09v1XgP0799f7vq1QRJRD9a4FmIYBpqmsXLlSnRdb1I2MTERgEAgwHe/+12OHDnCHXfcwbBhw0hISGDPnj3MmDEDwzCiiuVnP/sZhYWF3HbbbYwbNw6Xy4WmaVx77bVR77M1kXa0bPweHau596m17Q3JNdL3OpIYRJAkol5i8ODBKKXIz8/nhBNOaLHcxo0b2bJlC08//TTTpk0LbX/zzTcjOk5LCeDFF19k+vTpPPLII6FtdXV1lJeXN1t+69atnH/++aHn1dXV7Nu3j0suuaTV4w8cOBDDMNi6dSsnnnhiaPuBAwcoLy9n4MCBEZ1HR0T6XovISRtRL3HVVVeh6zoLFy4MuyyC4H/yw4cPA0f/2zcuo5Ri6dKlER0nISGh2eSi63qT4z766KMEAoFm9/PEE0/g8/lCz5ctW4bf72fSpEmtHr8hUS1ZsiRs++LFiwG49NJL2zqFDov0vRaRkxpRLzF48GB+85vfMH/+fHbu3MnkyZNJSkpix44dvPTSS8yePZt58+YxbNgwBg8ezLx589izZw/Jycn83//9X5O2opaMHj2at956i8WLF5OdnU1+fj5jx47lsssu429/+xsul4uTTjqJjz/+mLfeeov09PRm9+P1erngggu45ppr2Lx5M3/6058455xzuOKKK1o9/siRI5k+fTpPPPEE5eXljB8/nrVr1/L0008zefLksFpWV4n0vRaRk0TUi9x5552ccMIJ/P73v2fhwoVAsOH1wgsvDH3AbTYb//73v7n11ltZtGgRcXFxfO973+OWW25h5MiRbR5j8eLFzJ49m1/96le43W6mT5/O2LFjWbp0Kbqu88wzz1BXV8fZZ5/NW2+91eIdq8cee4xnnnmGBQsW4PP5mDp1Kn/4wx8iavv561//yqBBgygqKuKll14iMzOT+fPnc/fdd7fj3eqYSN5rETlNHVu3FEKIbiZtREII00kiEkKYThKREMJ0koiEEKaTRCSEMJ0kIiGE6Xp0PyLDMNi7dy9JSUmdPsm7EKLjlFJUVVWRnZ2NxdJyvadHJ6K9e/c2GSkthIg9JSUlDBgwoMWf9+hElJSUBARPMjk52eRohBDHqqysJCcnJ/RZbUmPTkQNl2PJycmSiISIYW01nUhjtRDCdJKIhBCmk0QkhDBdj24jioRSCr/f3+IEXaJluq5jtVqla4Tocr06EXm9Xvbt2ycra3ZAfHw8WVlZMvm76FK9NhEZhsGOHTvQdZ3s7Gzsdrv8Z28HpRRer5eDBw+yY8cOhgwZ0mqHNCE6otcmIq/Xi2EY5OTkNFllVETG6XRis9koLi7G6/USFxdndkiil+r1/+Lkv3jHyPsXud1ltWzeX0WdT9oj20v+yoToJBt3V/DfjftYvyuyhQjEUZKIhOgkZbXB5ZGsFgulVXUmR9OzSCI6zuXl5TVZI0xEp7IumIg+2naI97ccMjmankUSkRCdwOs3cHuPtg1Vun2tlBbHkkTUC3i9XrNDOO5V1YUnnlqv36RIeqbjMhF5/UaLD3/AiLisL4Ky0ZgwYQK33HILt9xyCy6Xiz59+vDrX/86tLxxXl4e9913H9OmTSM5OZnZs2cD8OGHH3LuuefidDrJycnh1ltvpaamJrTf0tJSLr/8cpxOJ/n5+TzzzDNRxSeaqqwLJh6X0waAL6Ci/v0fj3ptP6LW/PHdbS3+LL9PApNH9Q89f+L97fgCza9BOSDVyZTTj07M9tRHO8Kq5wC/+O4JUcX49NNPc9NNN7F27Vo+/fRTZs+eTW5uLrNmzQLg4YcfZsGCBaHVTbdv387FF1/Mb37zG5566ikOHjwYSmaFhYUAzJgxg7179/Luu+9is9m49dZbKS0tjSo+Ea6hBpQSb6PW68cXULi9AezW4/J/fbsdl4moJ8jJyeH3v/89mqYxdOhQNm7cyO9///tQIvrOd77D7bffHio/c+ZMrr/+em677TYAhgwZwh/+8AfGjx/PsmXL2LVrFytXrmTt2rWcccYZADz55JOceOKJ3X5uvVFuWjyXj8zCYdV5Y9MBfG4fNV4/rnib2aH1CMdlIppzfkGLP7McMwpk9nmDWyx77IiRG8/O70hYYc4888ywISnjxo3jkUceCQ3ePf3008PKf/7553zxxRdhl1tKqdBQly1btmC1Whk9enTo58OGDSMlJaXTYj6eJcXZSIoLJh2nTafS7cMjl2YRMz0R7dmzhzvuuIOVK1dSW1tLQUEBhYWFTT5onak91eWuKttRCQkJYc+rq6u5+eabufXWW5uUzc3NZcuWLd0V2nFveP9kBmckkOKU2lCkTE1EZWVlnH322Zx//vmsXLmSjIwMtm7dSmpqqplhxYQ1a9aEPV+9ejVDhgxB1/Vmy5922mls2rSJgoLma3vDhg3D7/ezbt260KXZ5s2bKS8v79S4j1c7D9Xg8RtkJemM0IshOw/iZMaCSJmaiH7729+Sk5MTakwFyM/vvMubnmzXrl3MnTuXm2++mfXr1/Poo4/yyCOPtFj+jjvu4Mwzz+SWW25h5syZJCQksGnTJt58800ee+wxhg4dysUXX8zNN9/MsmXLsFqt3HbbbTidzm48q97r0+IySo7UcrXra5JrNkFCHzhjZtPrd9EsU5v0//Wvf3H66aczZcoU+vbty6hRo/jLX/7SYnmPx0NlZWXYo7eaNm0abrebMWPGMGfOHH7+85+HbtM3Z8SIEbz33nts2bKFc889l1GjRrFgwQKys7NDZQoLC8nOzmb8+PFcddVVzJ49m759+3bH6fR67vqBrgmV3+ILGNSU7afyyEGTo+o5TK0Rffvttyxbtoy5c+dy11138cknn3Drrbdit9uZPn16k/KLFi1i4cKFJkTa/Ww2G0uWLGHZsmVNfrZz585mX3PGGWfwxhtvtLjPzMxM/vOf/4Rt+9GPftShOEWQxxfAGnBjp5oDtV5Kytxorq2cmS6JPhKm1ogMw+C0007jgQceYNSoUcyePZtZs2bx5z//udny8+fPp6KiIvQoKSnp5oiFaJ7bGyDOX4lV19Abbr3WSo0oUqYmoqysLE466aSwbSeeeCK7du1qtrzD4QitYSZrmYlY4QsY+A2Fw1+F1aJhrZ/DSdX13qaDzmbqpdnZZ5/N5s2bw7Zt2bKFgQMHmhRRbFi1apXZIYh2aGgfcho16JajNSLNU2VmWD2KqTWiX/ziF6xevZoHHniAbdu28eyzz/LEE08wZ84cM8MSol0axpQlqGo0NHDVDxHySI0oUqYmojPOOIOXXnqJ5557juHDh3PfffexZMkSrr/+ejPDEqJdEh1WLh+ZxWl9gx8nS30isvhk9ZhImd6z+rLLLuOyyy4zOwwhohZn0ynomwS7AlAFlsSM4A/8HjAMkHm/22R6IhKi1/C5AbAnZ5CdEoeuaeCvA7usItMWSURCdNCRGi8HK+vIrqogyaYRl+BiYN9U8HuDyUkSUZukzihEB+08XMPrX+xif1l1cIMtHqz1Q2f8bvMC60GkRiREB3n9BlajLngpplvBaqcWO3j92D218iGLgNSIhOggr9/AFnAH+w/ZgpdhHxfX8PnuCqqr5RZ+JCQRCdFBvkDTRKRswUuzgEcuzSJxfCUipYINiN39UM3Ped2c5cuXk56ejsfjCds+efJkGaAao4KXZp6wRKQ1JCKv9CWKxPF1+RrwwQctz+nTZc69HayRTZI1ZcoUbr31Vv71r38xZcoUILj6xquvvtrqyHphHm/AQFdedF0DqyO40RoHQMArK75G4viqEfUATqeT6667LmyyuL///e/k5uYyYcIE8wITLfL4DXTDG6wR1Scirf4fj+H3tPZSUe/4qhHptmDtxIzjtsOsWbM444wz2LNnD/3796eoqIgZM2aETaYvYse4QekEAgkkVFpDiUi3BWtEhk8SUSSOr0SkaRFfIplp1KhRjBw5kuXLl3PhhRfy1Vdf8eqrr5odlmhBTlo8uHSo1UEPJiKLLfjV8MmlWSSOr0TUg8ycOZMlS5awZ88eJk6cSE5OTtsvEuZpuASrbxvKTk/B6ooj2SZLCkVC2ohi1HXXXcfu3bv5y1/+wo033mh2OKIVWw5UcbC8EkOp0KXZ4Kw0BqYnkOqQy+lISCKKUS6Xi+9///skJiYyefJks8MRLQgYile/2MdnOw6EJaKGSzQC0kYUCbk0i2F79uzh+uuvx+FwmB2KaEHDpGhWw4uu6aFE5MWG4Q+g4UZ+e22TRBSDysrKWLVqFatWreJPf/qT2eGIVngDwURkx4umxYfaiL46WEfdrnL6JNoZInMStUkSUQwaNWoUZWVl/Pa3v2Xo0KFmhyNa0VAjsuEPbtCDd2VtNgd1BC/dCHjBEmdShD2DJKIY1NK6ZSL2eAMGmvJj04IT6DfUiHSrDaVZMBTBdiKbJKLWSH1RiA4IjjPzHl3LrKFGpGsENHuwAdvvNTHCnqHXJyLVjgGnoil5/1rnC9QP72joLFvfFqRbLAQs1mAiMnwmRxn7em0istmCwypqa2X0c0c0vH8N76cI1y8pjvMLXGSlOEOXZQBWi4ahWTEMgm1EolW9to1I13VSUlIoLS0FID4+XsZqtYNSitraWkpLS0lJSUHXdbNDikmueBuuPnbYYw9dlgFYdY2AZsMwFAT8JkbYM5iaiO655x4WLlwYtm3o0KF88803nbL/zMxMgFAyEu2XkpISeh9FC44Z3gGQ4LDSPz2ZpLpqqRFFwPQa0cknn8xbb70Vem61dl5ImqaRlZVF37598fnkOr29bDab1ITaUFpVh+9QGen+AHGNElFynI2TBvSBw+XSRhQB0xOR1WqN+D+ux+MJm7mwsjKy+YB1XZcPlOgSX5RUcOjrPZyjeRjQ/5iZHfT6j1dAElFbTG+s3rp1K9nZ2QwaNIjrr7+eXbt2tVh20aJFuFyu0ENGpAuzeQONJ0U7WiNSSuExdDz+AEouzdpkaiIaO3YsRUVFvPbaayxbtowdO3Zw7rnnUlVV1Wz5+fPnU1FREXqUlJR0c8RChPP6Dayqfr7qRo3VfkPx302HWb+rHJ9PElFbTL00mzRpUuj7ESNGMHbsWAYOHMg///lPbrrppiblHQ6HDAAVMcXrN4gzfE1qRFaLhmEJfrwM6dDYJtMvzRpLSUnhhBNOYNu2bWaHIkREPIGm81VD8EYJlmDfq4DUiNoUU4mourqa7du3k5WVZXYoQkTE5zewqqaJCECzBhOR1IjaZmoimjdvHu+99x47d+7kf//7H9/73vfQdZ2pU6eaGZYQEfOGDfEIH9iq6Q0reUgiaoupbUS7d+9m6tSpHD58mIyMDM455xxWr15NRkaGmWEJEbFzh/QhudaG3RoIa6wG0HSpEUXK1ET0/PPPm3l4ITrs5GwXfKuDz9KkRmQJXZpJP6K2mN6hUYgeTalGQzzC24gG9HHhKnPgsARMCKxnkUQkRJS8foO9hyvoU+cl0WFtkohOy+8L5YlglalU2hJTd82E6EnK3V7+s34nm/dXBhfvPKaNqOH2vQzxaJvUiISIkvfYW/fHTDMTsNhQhoHm9yAjHVsniUiIKHn9DZ0ZLUfXMWvknS1HSNxZxsCMJLJNiK8nkUszIaLkC6hme1U30KzBSzUVCIAhDdatkUQkRJSCE+d7sGg0m4is9YnIUEraidogiUiIKHkDAXTlw2pp2qsawKJbUWjB6WJlcrRWSSISIkqexksJNVcj0i0YFmv92maSiFojjdVCRCm/TwJ9sp2klTmabay26hbqNCuGMiQRtUESkRBRynI5yUq3Q52t2RqRbqlfyUPVyQT6bZBEJERH+OuCX5tJROkJdhyuRBLxgyFLCrVGEpEQUSqtrMNWWU2SYWBtJhHl9UmAnHSo9EgiaoM0VgsRpVWbD7J2214q3L5m75oBoMswj0hIIhIiSp5Ao7tmx44zq2doVgJKbt+3RRKREFHy+g101XQpoQYlR2r576ZDfLmnQpadboMkIiGi5G3oR9TMNLEQvGtmaFbp0BgBSURCREEphc8fQA8tJdRMP6KGRCQdGtskiUiIKAQMBQEPoFq8NLPqFgKaNTjWTGpErZJEJEQUvPUN1QC61XZ0nftGQpdmSkkbURukH5EQUdAtGmfnJZLudqK1cOvepmuhsWYq4EVrtpSAGKoRPfjgg2iaxm233WZ2KEK0yWHVGZkZx4DU+Bb7EDXUiAACspJHq2KiRvTJJ5/w+OOPM2LECLNDESJyLaze0cBmsZCVlkS61y6N1W0wvUZUXV3N9ddfz1/+8hdSU1PNDkeIiNR6/Rwsr6DOF2ixRmSxaEw4MZsT+iVhRWZobI3piWjOnDlceumlTJw4sc2yHo+HysrKsIcQZth1pJa3Nu7i20M1LdaIgKMrechds1aZvtLr+vXr+eSTTyIqv2jRIhYuXNjFUQnRNm/YpGgtjDMD0K0oglPFSmN1y0yrEZWUlPDzn/+cZ555hri4Vn6RjcyfP5+KiorQo6SkpIujFKJ5wRU8PPW9qluuEb2wfj+rvz1CVY27G6PreUyrEa1bt47S0lJOO+200LZAIMD777/PY489hsfjQdfDV4NyOBw4HK1Ug4XoJqFpYq2t14gMS/AjZsjEaK2KKhF9++23DBo0qEMHvuCCC9i4cWPYthtuuIFhw4Zxxx13NElCQsQSjz+AbnjrJ85v+Z+jJbSkkLQRtSaqRFRQUMD48eO56aabuPrqqyO+tGosKSmJ4cOHh21LSEggPT29yXYhYo3HF1xKqKUVPBpY6ucjMqQfUauiaiNav349I0aMYO7cuWRmZnLzzTezdu3azo5NiJhV5w+0uoJHA80aTEQq4AfD6K7wepyoEtGpp57K0qVL2bt3L0899RT79u3jnHPOYfjw4SxevJiDBw9GFcyqVatYsmRJVK8VojudlOViUJqVBIe11RqR3niRRZkutkUdumtmtVq56qqreOGFF/jtb3/Ltm3bmDdvHjk5OUybNo19+/Z1VpxCxJShmUnku3QS7K0notClmUL6ErWiQ4no008/5ac//SlZWVksXryYefPmsX37dt5880327t3LlVde2VlxChFblDq6RFArl2Z9k+NITnBit1pkmEcromqsXrx4MYWFhWzevJlLLrmE5cuXc8kll2CxBPNafn4+RUVF5OXldWasQsSMfUfKSfH6ibNaWhx9DzB2UDrsTQefWy7NWhFVIlq2bBk33ngjM2bMICsrq9kyffv25cknn+xQcELEIn/A4KW13zJqbzmnD8rA1sxcRGF0WzARSY2oRVElojfffJPc3NxQDaiBUoqSkhJyc3Ox2+1Mnz69U4IUIpYEOzN60ACr3dn2C2S8WZuiaiMaPHgwhw4darL9yJEj5OfndzgoIWKZx29gDdShWzQ0W+uJ6JOdR1i1rYwdh2ukRtSKqBKRUqrZ7dXV1VF1bhSiJ/H4A1iNumBnRlt8q2WVAq/S61fykDailrTr0mzu3LkAaJrGggULiI8/+ksIBAKsWbOGU089tVMDFCLW1NX3qtZ1DdqoEVn1xvNWS42oJe1KRBs2bACCNaKNGzditx9d3dJutzNy5EjmzZvXuREKEWM8/gA2ow6rxdJmjchq0epX8kDaiFrRrkT07rvvAsHBqUuXLiU5OblLghIilnl8R9uI2qwRWSyykkcEorprVlhY2NlxCNFj9EuOIy7DSlqNve0aka5haDpGQNY2a03Eieiqq66iqKiI5ORkrrrqqlbLrlixosOBCRGrMl1xZKZZQXO0WSPSLfVLCvmRNqJWRJyIXC4XmqaFvhfiuOarDX5tIxEl2K2kJiWQUKVLjagVESeixpdjcmkmjmeHqz04a6pxKIXexqVZpiuOzBOzYcd2aSNqRVT9iNxuN7W1taHnxcXFLFmyhDfeeKPTAhMiVr3+5X42fLuXSrevzRoRID2rIxBVIrryyitZvnw5AOXl5YwZM4ZHHnmEK6+8kmXLlnVqgELEGo/Xg0UFsEbQjwiAhrFo0kbUoqhnaDz33HMBePHFF8nMzKS4uJjly5fzhz/8oVMDFCLW+D01AFitNtDtrZatrPPxf58dYF1xmfSsbkVUiai2tpakpCQA3njjDa666iosFgtnnnkmxcXFnRqgELHEFzBQnmCzhNURD1rrq5VZNI0avxZ8nazk0aKoElFBQQEvv/wyJSUlvP7661x44YUAlJaWSidH0au5fQFsRi3BOfMT2yzf0LNaIRPotyaqRLRgwQLmzZtHXl4eY8eOZdy4cUCwdjRq1KhODVCIWFLnDWAPuLFaLGiOpDbLWy3BsWYgiag1UfWsvvrqqznnnHPYt28fI0eODG2/4IIL+N73vtdpwQkRa9y+ALZATbCh2p7QZnndoqFCiyxKImpJ1Cu9ZmZmkpmZGbZtzJgxHQ5IiFiWHGfjlAwrKWUOsLd9aaZpGlpobTNpI2pJVJdmNTU1/PrXv+ass86ioKCAQYMGhT0itWzZMkaMGEFycjLJycmMGzeOlStXRhOSEN0iNcHOiWkaWS4nONpORABa/Z01qRG1LKoa0cyZM3nvvff40Y9+RFZWVmjoR3sNGDCABx98kCFDhqCU4umnn+bKK69kw4YNnHzyyVHtU4gu5w3evo+kRgTQx5VAksOKxfAHZ0qL8vPSm0WViFauXMmrr77K2Wef3aGDX3755WHP77//fpYtW8bq1aslEYmYdKTGi6O6gjil0CNMRFeMGgg19eMzlQGa3oUR9kxRJaLU1FTS0tI6NZBAIMALL7xATU1N6C7csTweDx6PJ/S8srKyU2MQoi3vbT5Av+27KejjpG8EjdVAcBWPBgEfWCQRHSuqNqL77ruPBQsWhI03i9bGjRtJTEzE4XDw4x//mJdeeomTTjqp2bKLFi3C5XKFHjk5OR0+vhDt4XbXoCkDu26J+NIMzRJ8gIw3a0FUNaJHHnmE7du3069fP/Ly8rDZbGE/X79+fcT7Gjp0KJ999hkVFRW8+OKLTJ8+nffee6/ZZDR//vzQvNkQrBFJMhLdKVBTDoDVmXR0DFkbVn65nz4llQxOs5MmDdbNiioRTZ48udMCsNvtFBQUADB69Gg++eQTli5dyuOPP96krMPhwOFoeXlfIbpSwFAY7goA7AkpEb+uzh/AHbAQCMhKHi2JKhHdfffdnR1HiGEYYe1AQsSKGq8fe6Aaiwa2diQiPTRvtSEj8FsQdYfG8vJyXnzxRbZv384vf/lL0tLSWL9+Pf369aN///4R7WP+/PlMmjSJ3NxcqqqqePbZZ1m1ahWvv/56tGEJ0WVqPH4c/ipsugUtLvJZSo+u5OGRNqIWRJWIvvjiCyZOnIjL5WLnzp3MmjWLtLQ0VqxYwa5du0JzFbWltLSUadOmsW/fPlwuFyNGjOD111/nu9/9bjRhCdGlgomoOthQ3c5EZDQkIpmlsVlRJaK5c+cyY8YMHnroodB0IACXXHIJ1113XcT7efLJJ6M5vBCmSI23MyxVkeh1tC8R6Rq+hiWFpEbUrKgS0SeffNJsY3L//v3Zv39/h4MSIhalJ9hJT/KDLw4ckU93o1sseGS111ZFlYgcDkeznQm3bNlCRkZGh4MSIib5asFXFxyiER95h95EhxUtPg4bFqkRtSCqDo1XXHEF9957Lz5f8E3VNI1du3Zxxx138P3vf79TAxQiVpTu343bF8BwJIf3lm7D6IGpnDcsm+wUp7QRtSCqRPTII49QXV1NRkYGbreb8ePHU1BQQFJSEvfff39nxyhETPjoiy18VlJOlSWKdf1kJY9WRXVp5nK5ePPNN/noo4/4/PPPqa6u5rTTTmPixImdHZ8QMcEfMKDmEABxKf3avwNZyaNV7U5EhmFQVFTEihUr2LlzJ5qmkZ+fT2ZmJkqpqKcEESKWVdb5ifeXoVs07MntawfdfrCabV8fpsBdzeABcmnWnHZdmimluOKKK5g5cyZ79uzhlFNO4eSTT6a4uJgZM2bINLGi16p0+3D6ynBYLWjxfdr1Wn9AUeEBj196VrekXTWioqIi3n//fd5++23OP//8sJ+98847TJ48meXLlzNt2rRODVIIs1VVV2ELuHE4bJDQvkSkWzQCFiuGIf2IWtKuGtFzzz3HXXfd1SQJAXznO9/hzjvv5Jlnnum04ISIFe7yUgCs8S6wtm/g9dGe1dKPqCXtSkRffPEFF198cYs/nzRpEp9//nmHgxIi1ngqgolIT2x/Pzmr3igRyej7ZrXr0uzIkSP069fyHYN+/fpRVlbW4aCEiDVDEj34Up0k9slq92utodH3SI2oBe1KRIFAAKu15Zfouo7fLxlf9D6Z1mpIjYeMKBKR3jD6XtqIWtKuRKSUYsaMGS1OTibzCIleqzbYh4h23jEDsOkWEpxxOKy69KxuQbsS0fTp09ssI3fMRG/jdrvxHDlEnE0nLj693a93OW38YOwgWOeSGlEL2pWICgsLuyoOIWJW6YE97N1fhSM+kdPs8dHtpGFsmrQRNSuqsWZCHE8abt1rCR2YWcJS/z9fakTNkkQkRBu8lfV9iJKiS0RKKf65YT+f7y7H5/UGV3sVYSQRCdEGf9VBgHaPMWugaRoHawxqvQEC0peoWZKIhGiDUd2BUff1tPpuL0r6EjVLEpEQrQn4Ue5gJ934DiQim27F0HTpS9QCSURCtMJTdRB/wCBgsZOcHMWEaPUahnkEDCV9iZoR9bpmQhwPLO7DDM5IoDouE4ct+o+LVbc0Gm8mNaJjmVojWrRoEWeccQZJSUn07duXyZMns3nzZjNDEiKMzVNO36Q4BuXmdmg/9oaBr4aMwG+OqYnovffeY86cOaxevZo333wTn8/HhRdeSE1NjZlhCXFU7eHgV2fkq3Y0J95uxWa3B1cAkRpRE6Zemr322mthz4uKiujbty/r1q3jvPPOMykqIY4qP3wA3F4ctmScHdjP5SOzIZAFFXukjagZMdVGVFFRAUBaWvP/fTweT9jA2ubWVhOi0yjFvn17KK+qIakgjuEd3Z+s5NGimLlrZhgGt912G2effTbDhzf/K1+0aBEulyv0yMnJ6eYoxXHFW43XUwdoJKS0f9R9EzLerEUxk4jmzJnDl19+yfPPP99imfnz51NRURF6lJSUdGOE4nijag/j8RvUWZNIjo/r0L4+Lynnw2/L2Vfhlp7VzYiJS7NbbrmF//znP7z//vsMGDCgxXIOh6PFuZCE6GzeykMEDEWd3UWyM/KVXZtT4/Fz2K3or8tKHs0xNREppfjZz37GSy+9xKpVq8jPzzczHCHCuCuCY8xUfBo2vWMXDzZrsB9RQPoRNcvURDRnzhyeffZZXnnlFZKSkti/fz8QXEnW6ezIPQohOq6uftS93s7lg5pjtWgYFitGQPoRNcfUNqJly5ZRUVHBhAkTyMrKCj3+8Y9/mBmWEAB4q4KDXe3JHU9EtrCe1dJGdCzTL82EiEkBPxl6HXEZCXgGDuzw7hoSUUB6VjcrJhqrhYg5deU4bRaccUmQ2YGZGevZQit5IG1EzYiZ2/dCxJTaI8Gv8enBYRkdZNMtWG02LBrSs7oZUiMSohmq9hCllXVY4hNINxQWS8eSUU5aPDljBsGmjVIjaobUiIRohrviIN8equH93Ubn7VR6VrdIEpEQzair70OkJ6Z3uDYU0rCSR8DbOfvrRSQRCdGMzrx1D+DxB1j59WG+3FuB4ZdEdCxJREIcy1uL1x2cE8vZSYnIomkUVwSoqvNj+GVp9mNJIhLiWO4j1PkCePREkhMTO2WXVotGwGIHIODzyNpmx5BEJMSxGkbd21y4OjjYtYGmaVhswRH8KhCQButjSCIS4li1h6nzBXBbU0iJ75xEBKBbbYAWHPgakMuzxqQfkRDHULVHOKFfEmlZgzqtRgRgs+oELLbgMA+/F2RGmxBJREIcQ6s9gstpwzUwBzo4/UdjdqsFv2avX8lDakSNyaWZEI0ZAahf2ZX49E7dtcOqo1ntGABy5yyM1IiEaKyugiPVdXiVTlwgjpRO3PX3T+uPpuVAxW5JRMeQGpEQjdUe5kBVHV+V6ZSU1XXqrjVNA2t9w5BcmoWRRCREY7WHcXsDuG0ppCZ0XkN1iB7sS4T0rg4jl2ZCNOKrLMXjN6hNSCE9oXNva32zv5KyXVXkedxkSY0ojNSIhGik+sheAFRCBk673qn7rnT72VulqPUGpI3oGJKIhGhgBHCXHwAgKT2703dvt1qO9iOSEfhhJBEJ0aD2MDVuDwGLjbT0vp2+e7tuIWCx13do7NyG8J5OEpEQDapLqfUGqLWlk+nq/OWs7FYLAc1+tGe1CJHGaiEa1JRycnYyfZMLSHV1bInp5jis9TUiGWvWhKk1ovfff5/LL7+c7OxsNE3j5ZdfNjMccbyr3IvVYiFrQD5xts5tqIaGRNQw1kwSUWOmJqKamhpGjhzJH//4RzPDEAKMAKoyeMeM5AFdcoiGsWZKIYnoGKZemk2aNIlJkyZFXN7j8eDxHP0FVlZWdkVYopczDMXuMjduX4DslDiS4myUl+5mZ/Eh0lNSGOBMpZNmqQ7jctqYef5JWNZ+AH53Fxyh5+pRbUSLFi1i4cKFZochergPth1ifXFwYKumwcD0eJz7P8PlMzhAGjmWrrlQ0DQNzR4ffBLwBx96j/oIdpkeddds/vz5VFRUhB4lJSVmhyR6oDPyUklPtNM32YFSsPNQLdqR7VgtGiecOKJrD251gFb/sfPVdu2xepAelY4dDgcOh8wmJTom3m7lR2cORNM0ymq8bN5ziKyqCvpmuHD2H9alx353y0H6lHoY7NKI97khLrlLj9dT9KgakRDRUkpRWnm0E6FWv4x0aoKdM5MOMTA1DqerL8SndWkce8rcHKyz4PEb0k7UiCQicVwoOeLmmTW7eGnDblTjFTSUgr3rg99nDu/yOJw2Hb8lDr+hwCeJqIGpl2bV1dVs27Yt9HzHjh189tlnpKWlkZuba2Jkorf5ZOcRAFLi7aHaEAAHN0PlvuAqrJld3D4ExNl0fJY4/IFKSUSNmJqIPv30U84///zQ87lz5wIwffp0ioqKTIpK9DYHKuvYdaQWi6ZxWm5qcKNScHg7bP5v8HnOGHB0zhpmrXHaLVRZHFIjOoapiWjChAnh1WQhukBDbWhoZhKuOCsc2go7P4Sq/cECKTmQd063xBJn1SnT4/AHlLQRNdKj7poJ0V5lNV62lVYDcHq2HT57Fsp3BX+oWyF7FOSPB0vnD+lojsOm47c48PsNqRE1IolI9GrristQCoYl19Fn8/NQVxFMQP1PD16O2RO6NZ6GxmpDIYmoEUlEotcKGIo95W5c7hLOUp+CzQBnKpwyBRI6d6mgSA3NTGLY2cOwfPUVeGtMiSEWSSISvZauwY8GHKSi/CNcNluwLejkq6BhmIUZMVk0iEsKPvFWmxZHrJFEJHonnxu2vonlwFekOm2QeQoMndRtbUGtarg7560Bw4AuGtvWk0giEr2LYcD+z6nc9BaJFh8Wiw6Dz4cBZwRHuJpMKcXrW6rI2V/FCRkJ2Hw14EgyOyzTSSISvUflPtjyGnVH9vD17nL8cemMvOAaEvvlmx1ZiKZpFB9x4/TYyAsY2DzVkoiQRCR6AyMAO96DkrWgFN+W+/g2ZRy23NM5q2/s9dBPcFjx6vF4/XUkSDsRIIlI9HQ+N3y5ItQ3aI8jn3eShxGwJvDDYZnhwzliRKLDildPwBeoBY9M7geSiERP5vfAF/8IXpLpNiryLubl7XH4dIOx+amkJ8bmlDEJDis1ejxevwEeqRGBjL4XPVXABxtfDCYhm5Oa4dfxf7sS8PoN+qc4OTPfnH5CkUhw6Hj1BLwBI9jBUkgiEj2QEYCvXg5ejlntMOIHfLjPQoXbh8tp49IRWVgssXdJ1iDJYcNtcwVrRO4ys8OJCXJpJnoWw4Cv/wWHtwWHapwyBZKzOG9IgKo6PxNP7EuCI7b/rF1OG3VWV3BZIfcRs8OJCVIjEj2HUrDlNSj9Biw6h/Mug5TgXTGnXefq0QNIibebHGTb+qc6ufG7p3Fytgt8deCVuaslEYmeQSnY9hbs+xwD2JB4Lss362zc3fPaWHSLhs3uODpftdSKJBGJHkAp+PZd2P0p3oDBKu10Vh0JNkaX1fbgNeSd9fNj10oiiu2LaSGUgh3voXatpqzGy7tqNLviBmLTNS44sR8nZvXMVTA27a3k0B4Y4nOTVX3A7HBMJ4lIxC4jAJtXUlfyGd8equHzuDPYnzSUjCQHFw/PpE+M9hOKRK3Xz06fi8w6P1kNS10fxyQRiZhiGIoKt49AzWH6lLwOlftQmoUNzjM5nHwSYwamMjY/Daves1sV+iXHsdaeQXWFH6oPHPervh6/Zy5ixqFqD1v2V7HrcDV1h0tIrdpMgbGDPpmJYHXgPOUKhtWkUZCRhCveZna4naJvsgOfzUWViqPO4yWuYhekDTI7LNNIIhLdTymo3Ev14d18s30nBw/uJ85XSa6/Eg2FRQOL0xb8YA6dBHHJjO5jdtCdy2HVyU51Un4kl7LaXWQd2iaJyGx//OMf+d3vfsf+/fsZOXIkjz76KGPGjDE7rNhVVxHsVewuo8YbICGlH6QOBHsCtV4/DqsenAkwligFVfugdFOwH5CniuK9Ffjq/KQAqfE20lLjSUqIx5E5FEv2qOA59WKDMhL5fG8+Byu3knngS7RBE4I9xY9Dpieif/zjH8ydO5c///nPjB07liVLlnDRRRexefNm+vbta3Z4sSPgCy4GuP8LvId2cLjaw6FqL9UeP2fkpWLVdUjN4xt3Pz6t6UNuvwwK+iYyMD0e27HtKb46qKug7MhBaqorMTQLSneg4lIgLhXdZsemW8h0xYVesr+ijjibhaQ4W+RJzjCCyefQZji4GX9N8Da11WIBq53M/JM5UmHnpII80vv0C84n7UiOiQnMusNJWcn8b2sOh8sSKKusIq1kNeSfZ3ZYptCUyQuLjR07ljPOOIPHHnsMAMMwyMnJ4Wc/+xl33nlnq6+trKzE5XJRUVFBcnLPvI3booA/OEVE1T78h7ZTt3cTVTW1lLt9VNT6qHT0o9aWhgXFhP4Bkn3BD/mWA1UcqvFRY++D25YKVjvp8VaStDqSLXXkJfiCiYjgChfegHHMgTU81gSI78OEUSdCQh9wpvHi56UcrAlg1QzS4qCPQ5EWp0i1G6Q4FIkWf3A0vL8u+NVXi6o5hM/npdbjp6zWR2ltgJQBJzF0xNjgZchx3Djb4IOtB/Hv38RZdR/gsFkh79zgEkcmzqvdmSL9jJr6l+D1elm3bh3z588PbbNYLEycOJGPP/64SXmPx4PH4wk9r6yMcC6XIztg+9vBywNgf2UdxYePWUGhUT4+oV8iqfVDBUor3ew4VEPDT7X674LFFSf0TSItIVj2UHVdaA0tOLo/rX7fgzISyUhygFKU1XrZfKAq7LiNvx+UYqFvUrBGUlHjYcuBajzWJEoTTuRQ9gmkpWcwNDOJE/olBcdW1R6Bg99QkPgNmYd2c7imirKaw3hqDKiAGsBvtZDXsNKpPR49xUlAObBiYDU8WL3lWPwe4nFj9+2BkqpQPKccqKK81htcBqdedf2j1Go5uoIq8NXeCty+AAFD4dNslMflciQ+j7LkXPolJXFCn5yYnCfIDGcP7oOl4DzY6oE96zi48U1K3nsFQ4/D0B2gaSjNUl9L1BjaL4mU+gb70koPO479O25kSN8k0hKCZQ9Ve9l+sOUpRwZlJJBR3x3iSI2PraVVrZYB4JRrjvYO7yBTE9GhQ4cIBAL069cvbHu/fv345ptvmpRftGgRCxcubP+B/B6oPhh6qqrdGNUtj+/Rar1A/bW6uw6j9ugv+9jqo6oLgF7/y6nzoOqa/rJDr/EAdl9wm9eL8rT8R6RUQnA99oQ+aGmZbLUmk9Q3l7zUBC7sl9h0TFV8Ggw8C8vAs0h2l5NctY+82sOUV7sprwtQYzhQjkTU4IFocSlgtXNq04OCrxZqD0PNoeDX2sPgLmNoTgLK8FMX0KjxW6gO6FT6LFT4LOhx8TAgO9i+YXWy2X2ISp+O25aCx5ZMSryD7BQn5/ZLIi89XpJQI6FZAoZ8F5KzqP5sFR5/NfhrgWb+RhPdYNT/7mvqMKpaWZIovhZU/d9mtQejqpW5j5yJQH3ZGi9GVdNEFFYGQB1bm45ej6obz58/n7lz54aeV1ZWkpOT0/YLU3Jg5LXB7zWNRI+f3Fo/aFqzzRFxiQ6wWgCNRG+A/DpfM+0WwdfGx9nAHnwbE3wBBtf5Qz8Llav/LjHOCrb6soEAQ+oCR/emBcsGv9eIT0wO/uI1jVSlmDayHR9eZwo4U9CA1PpHRDQtuOCgPSE0mDT8jMFZ/2h8E0spFfb+nOOqC46nslpw2vSmbVSiKU2DzFPIuuAk4quqMOoqwe9FKQNlqNCHPi7BDrbg+xnvrf87boEzwQY2vb5sgNxaX4tl4+NtYA+WdfoC5NY0Ldu4DNCpi1Oamoj69OmDruscOBDexf3AgQNkZmY2Ke9wOHA4ouhNa0+AtKMTqCfVPyIRX/+IRMOHNBIOwv63tCrWaxDHxtc3Oa6FkqItcTaduLQUIKXNson1j0i09++4f4RlO4up/6rsdjujR4/m7bffDm0zDIO3336bcePGmRiZEKI7mX5pNnfuXKZPn87pp5/OmDFjWLJkCTU1Ndxwww1mhyaE6CamJ6If/OAHHDx4kAULFrB//35OPfVUXnvttSYN2EKI3sv0fkQd0av7EQnRC0T6GZXbGUII00kiEkKYThKREMJ0pjdWd0RD81bEQz2EEN2q4bPZVlN0j05EVfXd0CPqXS2EME1VVRUul6vFn/fou2aGYbB3716SkpJivvdxg4ZhKSUlJb3qTl9vPK/eeE7QveellKKqqors7GwslpZbgnp0jchisTBgwACzw4hKcnJyr/rjbtAbz6s3nhN033m1VhNqII3VQgjTSSISQphOElE3czgc3H333dHNIhDDeuN59cZzgtg8rx7dWC2E6B2kRiSEMJ0kIiGE6SQRCSFMJ4lICGE6SURd7P777+ess84iPj6elJSUiF6jlGLBggVkZWXhdDqZOHEiW7du7dpA2+nIkSNcf/31JCcnk5KSwk033UR1dSurRAATJkxA07Swx49//ONuirh5f/zjH8nLyyMuLo6xY8eydu3aVsu/8MILDBs2jLi4OE455RT++9//dlOk7dOe8yoqKmrye4mL6+Z5x5XoUgsWLFCLFy9Wc+fOVS6XK6LXPPjgg8rlcqmXX35Zff755+qKK65Q+fn5yu12d22w7XDxxRerkSNHqtWrV6sPPvhAFRQUqKlTp7b6mvHjx6tZs2apffv2hR4VFRXdFHFTzz//vLLb7eqpp55SX331lZo1a5ZKSUlRBw4caLb8Rx99pHRdVw899JDatGmT+tWvfqVsNpvauHFjN0feuvaeV2FhoUpOTg77vezfv79bY5ZE1E0KCwsjSkSGYajMzEz1u9/9LrStvLxcORwO9dxzz3VhhJHbtGmTAtQnn3wS2rZy5UqlaZras2dPi68bP368+vnPf94NEUZmzJgxas6cOaHngUBAZWdnq0WLFjVb/pprrlGXXnpp2LaxY8eqm2++uUvjbK/2nlekf5tdSS7NYsyOHTvYv38/EydODG1zuVyMHTu22dVvzfDxxx+TkpLC6aefHto2ceJELBYLa9asafW1zzzzDH369GH48OHMnz+f2tqWF7rsSg2rDDd+n1tbZRiC5924PMBFF10UM78XiO68AKqrqxk4cCA5OTlceeWVfPXVV90RbkiPHvTaG+3fvx+g2dVvG35mtv3799O3b9+wbVarlbS0tFZjvO666xg4cCDZ2dl88cUX3HHHHWzevJkVK1Z0dchNtHeVYQiedyz/XiC68xo6dChPPfUUI0aMoKKigocffpizzjqLr776qtsGlUuNKAp33nlnk8a9Yx8t/dJjWVef1+zZs7nooos45ZRTuP7661m+fDkvvfQS27dv78SzEO01btw4pk2bxqmnnsr48eNZsWIFGRkZPP74490Wg9SIonD77bczY8aMVssMGjQoqn03rHB74MABsrKyQtsPHDjAqaeeGtU+IxXpeWVmZlJaWhq23e/3c+TIkWZX6G3J2LFjAdi2bRuDBw9ud7wd0d5VhiH4u2lPeTNEc17HstlsjBo1im3btnVFiM2SRBSFjIwMMjIyumTf+fn5ZGZm8vbbb4cST2VlJWvWrOEnP/lJlxyzQaTnNW7cOMrLy1m3bh2jR48G4J133sEwjFByicRnn30GEJZwu0vjVYYnT54MHF1l+JZbbmn2NePGjePtt9/mtttuC2178803Y2pV4mjO61iBQICNGzdyySWXdGGkxzC1qfw4UFxcrDZs2KAWLlyoEhMT1YYNG9SGDRtUVVVVqMzQoUPVihUrQs8ffPBBlZKSol555RX1xRdfqCuvvDImb9+PGjVKrVmzRn344YdqyJAhYbfvd+/erYYOHarWrFmjlFJq27Zt6t5771Wffvqp2rFjh3rllVfUoEGD1HnnnWfWKajnn39eORwOVVRUpDZt2qRmz56tUlJSQreuf/SjH6k777wzVP6jjz5SVqtVPfzww+rrr79Wd999d8zevm/PeS1cuFC9/vrravv27WrdunXq2muvVXFxceqrr77qtpglEXWx6dOnK6DJ49133w2VAVRhYWHouWEY6te//rXq16+fcjgc6oILLlCbN2/u/uBbcfjwYTV16lSVmJiokpOT1Q033BCWXHfs2BF2nrt27VLnnXeeSktLUw6HQxUUFKhf/vKXpvYjUkqpRx99VOXm5iq73a7GjBmjVq9eHfrZ+PHj1fTp08PK//Of/1QnnHCCstvt6uSTT1avvvpqN0ccmfac12233RYq269fP3XJJZeo9evXd2u8Mg2IEMJ0ctdMCGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOkkEQkhTCeJSPQoRUVFYVPu3nPPPWGDgWfMmBEaYyV6DklEolkzZsxocU7pOXPmoGla2Ej9rkgAeXl5LFmyJGzbD37wA7Zs2dLia5YuXUpRUVHo+YQJE8IGqYrYJIlItCgnJ4fnn38et9sd2lZXV8ezzz5Lbm6uKTE5nc4mk7I15nK5Il6kQMQOSUSiRaeddho5OTlhMyiuWLGC3NxcRo0a1aF9N1dTmTx5cqiWNWHCBIqLi/nFL34RmpQNml6aHatxzWzGjBm89957LF26NLSPHTt2UFBQwMMPPxz2us8++wxN07p1Dh5xlCQi0aobb7yRwsLC0POnnnqKG264ocuPu2LFCgYMGMC9997Lvn372LdvX7v3sXTpUsaNG8esWbNC+8jNzW1yTgCFhYWcd955FBQUdNYpiHaQRCRa9cMf/pAPP/yQ4uJiiouL+eijj/jhD3/Y5cdNS0tD13WSkpLIzMyMahZEl8uF3W4nPj4+tA9d15kxYwabN28OrfXl8/l49tlnufHGGzv7NESEZIZG0aqMjAwuvfRSioqKUEpx6aWX0qdPH7PD6pDs7GwuvfRSnnrqKcaMGcO///1vPB4PU6ZMMTu045bUiESbbrzxRoqKinj66ac7rdZgsVg4diosn8/XKfuOxMyZM0MN8YWFhfzgBz8gPj6+244vwkkiEm26+OKL8Xq9+Hw+Lrrook7ZZ0ZGRli7TyAQ4MsvvwwrY7fbCQQCHTpOS/u45JJLSEhIYNmyZbz22mtyWWYyuTQTbdJ1na+//jr0fUsqKipCE+I3SE9PJycnp0nZ73znO8ydO5dXX32VwYMHs3jxYsrLy8PK5OXl8f7773PttdficDiiuiTMy8tjzZo17Ny5k8TERNLS0rBYLKG2ovnz5zNkyJCYmgD/eCQ1IhGR5ORkkpOTWy2zatUqRo0aFfZYuHBhs2VvvPFGpk+fzrRp0xg/fjyDBg3i/PPPDytz7733snPnTgYPHhz1qinz5s1D13VOOukkMjIy2LVrV+hnN910E16vt1vuAorWyZzV4rj1wQcfcMEFF1BSUtJkZVTRvSQRieOOx+Ph4MGDTJ8+nczMTJ555hmzQzruyaWZOO4899xzDBw4kPLych566CGzwxFIjUgIEQOkRiSEMJ0kIiGE6SQRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCm+/9mtOU6DexJtAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:41.147054Z", + "iopub.status.busy": "2024-03-22T19:05:41.146225Z", + "iopub.status.idle": "2024-03-22T19:05:41.582765Z", + "shell.execute_reply": "2024-03-22T19:05:41.581740Z" + }, + "papermill": { + "duration": 0.460683, + "end_time": "2024-03-22T19:05:41.584886", + "exception": false, + "start_time": "2024-03-22T19:05:41.124203", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9qUlEQVR4nO3deXxU5b348c+ZM1smyUxWskBCwiKKrIpQRAWXVsVa0Vat2gri0gXbKpcWub8riq3Ful256qXWW0BfdWlV1LaK1LqgoiCCIC4EiSxhCZB9m/Wc8/tjkiFD9slyJuH7fr3mRWbmmTPfk8l8eZ7nPItiGIaBEEKYyGJ2AEIIIYlICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOmsZgfQHbquc/DgQZKTk1EUxexwhBDHMQyD2tpacnNzsVjarvf060R08OBB8vLyzA5DCNGBkpIShgwZ0ubz/ToRJScnA+GTdLvdJkcjhDheTU0NeXl5ke9qW/p1ImpqjrndbklEQsSxjrpOpLNaCGE6SURCCNNJIhJCmK5f9xF1hmEYhEIhNE0zOxTRD9lsNlRVNTuMAW9AJ6JAIMChQ4doaGgwOxTRTymKwpAhQ0hKSjI7lAFtwCYiXdfZvXs3qqqSm5uL3W6XQY+iSwzD4OjRo+zfv5+RI0dKzagXDdhEFAgE0HWdvLw8XC6X2eGIfiozM5M9e/YQDAYlEfWiAd9Z3d6wciE60pVa9P7KBopKa/EFpT+yq+RbKkQP2b6/mte3H2LLvkqzQ+l3JBEJ0UMqG4IAWC0WjtT6TI6mf5FEdILYs2cPiqKwdevWPjnO+vXrGTt2LDabjVmzZnXrPfuLGl84Ea3fVcZ7O8tMjqZ/kUR0gnr33XdRFIWqqqpeOf78+fOZMGECu3fvZtWqVb3yHvEkENLxBo71DdV4gyZG0/9IIuoHAoGA2SF0WXFxMeeddx5DhgwhJSUlpmP05Xk3DXyNVa0vOvE0BGI/1onohExEgZDe5i2k6Z0uG+xE2VjMmDGDW2+9ldtuu42MjAwuvPBCPv/8cy6++GKSkpLIysrixz/+MWVlx6r/b7zxBmeddRYpKSmkp6fz3e9+l+Li4laPv2fPHs4991wAUlNTURSFOXPmdOk4O3bs4Mwzz8TpdDJmzBjWrVsXObaiKJSXlzN37lwURYnUiNatW8fkyZNxOBzk5ORwxx13RH35Wzvvpprb2rVrmThxIgkJCZx33nkcOXKENWvWcMopp+B2u7n22mujBq7qus7SpUspLCwkISGB8ePH8+KLL0aebzrumjVrOP3003E4HHzwwQcxfV4ANb7weXgSbAAENSPmz/9ENGDHEbXn8Xd2tflcYUYisyYOjtz/03vFBLXWd+UekprAlZOOLcy2Yv3uqOo5wO3fPimmGJ966il+9rOfsX79eqqqqjjvvPO46aab+O///m+8Xi8LFy7kqquu4u233wagvr6e+fPnM27cOOrq6li8eDGXX345W7dubTGEIS8vj5deeonvf//7FBUV4Xa7SUhI6NJxfv3rX/PII48wevRoHn74YS699FJ2795NXl4ehw4dYtSoUdxzzz1cffXVeDweDhw4wMyZM5kzZw5PP/00O3bs4Oabb8bpdHL33Xe3et4Ahw4dAuDuu+/msccew+VycdVVV3HVVVfhcDh49tlnqaur4/LLL+fRRx9l4cKFACxdupS//OUv/PGPf2TkyJG89957/OhHPyIzM5Pp06dH3u+OO+7gwQcfZNiwYaSmpsb0WcGxGlCKy0ZDIERQM/AGNOzWE/L/+i47IRNRfzBy5Ejuv/9+AH73u98xceJEfv/730eeX7FiBXl5eezcuZOTTjqJ73//+1GvX7FiBZmZmXz55ZeMGTMm6jlVVUlLSwNg0KBBUU2nzh7n1ltvjZRdvnw5b7zxBn/+85/5zW9+Q3Z2Noqi4PF4yM7OBuB///d/ycvL47HHHkNRFE4++WQOHjzIwoULWbx4cSTJNT9vOJaIfve73zFt2jQAbrzxRhYtWkRxcTHDhg0D4Ac/+AHvvPMOCxcuxO/38/vf/55///vfTJ06FYBhw4bxwQcf8MQTT0QlonvuuYdvf/vbHX8gHchPc3Hp+BwcVpV/fXmYoDdIfSCEx2Xr9rFPBKYnogMHDrBw4ULWrFlDQ0MDI0aMYOXKlUyaNKnX3nPeuSPafM5y3Pi1W84Z3mbZ48e6zZ1W2J2wopx++umRn7dt28Y777zT6nyn4uJiTjrpJL7++msWL17Mxo0bKSsrQ9fDzYJ9+/a1SETt6exxmr7gAFarlUmTJvHVV1+1edyvvvqKqVOnRg0QnDZtGnV1dezfv5/8/PwW593cuHHjIj9nZWXhcrkiSajpsY8//hiAXbt20dDQ0CLBBAIBJk6cGPVYT/2dJTttJDvDSSfBplLjDeKXplmnmZqIKisrmTZtGueeey5r1qwhMzOTr7/+ultV5M7oSnW5t8p2JDExMfJzXV0dl156KX/4wx9alMvJyQHg0ksvZejQoTz55JPk5uai6zpjxozpcodvTx0nVs3Puzmb7VjNQlGUqPtNjzUlzbq6OgBee+01Bg8eHFXO4XB06v26Y8xgN8MzE0lJkNpQZ5maiP7whz+Ql5fHypUrI48VFvZcrWKgOO2003jppZcoKCjAam35kZWXl1NUVMSTTz7J2WefDdBhx6vdbgeIWh6lK8fZsGED55xzDgChUIjNmzdz6623tvl+p5xyCi+99BKGYURqRevXryc5ObndRdVjMXr0aBwOB/v27YtqhvWmPWX1+EM6Ockq49S9kFsATnufvPdAYGpP2t///ncmTZrElVdeyaBBg5g4cSJPPvmkmSHFpXnz5lFRUcE111zDpk2bKC4uZu3atdxwww1omkZqairp6en86U9/YteuXbz99tvMnz+/3WMOHToURVH45z//ydGjR6mrq+vScR5//HFefvllduzYwbx586isrGTu3Lltvt/Pf/5zSkpK+MUvfsGOHTt49dVXueuuu5g/f36PzwdMTk5mwYIF3H777Tz11FMUFxezZcsWHn30UZ566qkefa8mn+yt5PXth6j+4k3Y8Tp89jcwWr/IIVoyNRF98803LF++nJEjR7J27Vp+9rOf8ctf/rLNPxa/309NTU3U7USQm5vL+vXr0TSN73znO4wdO5bbbruNlJQULBYLFouF559/ns2bNzNmzBhuv/12HnjggXaPOXjwYJYsWcIdd9xBVlYWt956a5eOc99993Hfffcxfvx4PvjgA/7+97+TkZHR7vu9/vrrfPzxx4wfP56f/vSn3HjjjfzXf/1Xt343bfntb3/LnXfeydKlSznllFO46KKLeO2113qtxu1tnOiaWPMNQU2nvrKUmoqjvfJeA5FiGOalbbvdzqRJk/jwww8jj/3yl79k06ZNfPTRRy3K33333SxZsqTF49XV1S128fD5fOzevZvCwkKcTmfPBy9OCJ39O/q/97/BW1/LDfydqoYAJZVelFMv51vfmtaH0cafmpoaPB5Pq9/R5kytEeXk5DB69Oiox0455RT27dvXavlFixZRXV0duZWUlPRFmEJ0yBvQcIZqsKoKatOl1wapEXWWqZ3V06ZNo6ioKOqxnTt3MnTo0FbLOxyOFlc9hDBbUNMJ6QaOUC3WBAVrY5+X4Tsxug56gqk1ottvv50NGzbw+9//nl27dvHss8/ypz/9iXnz5pkZlhBd0tQ/lKDXo1qO1YgUf62ZYfUrpiaiM844g5dffpnnnnuOMWPG8Nvf/pZHHnmE6667zsywhOiSpjlliUYdCgp4Gscu+aVG1Fmmj6z+7ne/y3e/+12zwxAiZkkOK5eOzyGpyAIhsHgGAzuwBGX3mM6SGXlCdJPTpjJiUDLZCeEmmiUpM/xEyA+6TPPoDNNrREIMGEEvAHZ3JrkpTlRFgZAP7LKLTEckEQnRTRX1AY7W+MitrSbZpuBM9DB0UCqEAuHkJImoQ9I0E6Kb9pTXs/azfZRWhifbYnOBNby+EyGveYH1I5KI4tCMGTO47bbbzA4jIt7iiTeBkI5V94WbYqoVrHYasNMQCBHyS4d1Z0giGqD64zrX/VUgpGPTvOHxQ7ZwM+yjvfVs219NXZ1cwu8MSURxZs6cOaxbt45ly5ahKAqKolBcXMyNN94YWX951KhRLFu2rMXrZs2axb333ktubi6jRo0C4MMPP2TChAk4nU4mTZrEK6+80mI7oPbWw24tnj179vTVr6NfCGotE5FhCzfNNL80zTrjxOqsNgzQTNjmRbW1XM6xDcuWLWPnzp2MGTOGe+65BwgvcD9kyBBeeOEF0tPT+fDDD7nlllvIycnhqquuirz2rbfewu128+abbwLhCYeXXnopM2fO5Nlnn2Xv3r0tmlgdrYfdWjyZmZk98EsZOMJNM39UIlKaElFAmmadcWIlIi0I7z/U9+979n+AtXOLZHk8Hux2Oy6XK7LeMxC16kBhYSEfffQRf/vb36ISUWJiIv/3f/8XWfTsj3/8I4qi8OSTT+J0Ohk9ejQHDhzg5ptvjrzmscce63A97NbiEccENB3VCKCqClgb50JawzP1tYDs+NoZJ1Yi6scef/xxVqxYwb59+/B6vQQCASZMmBBVZuzYsZEkBFBUVMS4ceOilq+YPHly1Gs6sx62aJ8/pKPqgXCNqDERKY3/8eghv5mh9RsnViJSbeHaiRnv2w3PP/88CxYs4KGHHmLq1KkkJyfzwAMPsHHjxqhysay/3Jn1sEX7pg5LR9MSSayxRhKRagsnfz0oiagzTqxEpCidbiKZyW63R60lvX79es4880x+/vOfRx5ra/PE5kaNGsVf/vIX/H5/ZPmUTZs2RZXpaD3s1uIR0fLSXOBRoUEFNfx7ttjC/+pBaZp1hlw1i0MFBQVs3LiRPXv2UFZWxsiRI/nkk09Yu3YtO3fu5M4772yRUFpz7bXXous6t9xyC1999RVr167lwQcfBIgsYN/RetitxaPL/KmWmppgjX1Duekp5HqcuG3yu+oMSURxaMGCBaiqyujRo8nMzOTCCy/kiiuu4Oqrr2bKlCmUl5dH1Y7a4na7+cc//sHWrVuZMGEC/+///T8WL14MEOk36mg97NbiaWsFzRPVzsO1HK2qQTeMSNNseE4aQ9MTSXV07mrpic7UNau7q731cGXN6tY988wz3HDDDVRXV0e2mRZt6+jvSNMN/uetrxlT+jLn5oawjr8KMkZC7WH4ZAU4kuDMX5gQeXzo7JrVJ1Yf0Qno6aefZtiwYQwePJht27ZFxghJEuoZTYuiWfUAqqJGakQBbOghDQUvsrhxxyQRDXClpaUsXryY0tJScnJyuPLKK7n33nvNDmvACGjhRGQngKK4In1EXxz14dtXRUaSnZG6Dj28d9tAI4logPvNb37Db37zG7PDGLCaakQ2QuEH1PBVWZvNgY9w0w0tABbpHmiPpGkhuiGg6ShGCJvSOLyhsUakWm0YigXdADQZS9QRSURCdEN4nlng2F5mTTUiVUFT7OEraSFZCaEjAz4R9eOLgiIOdPT3E9Qap3c0DZZt7AtSLRY0izWciHQTJlr3MwM2Edls4WkVDQ0y+1nErmldJ1VVW30+K9nJuSM85KQkRJplAFaLgq5Yw2vna1Ij6oipndWt7WU/atQoduzY0e1jq6pKSkoKR44cAcDlckVGEwvRGbquc/ToUVwuV5vTXzwuG54MOxywR5plAFZVQVNs6LoBWqivQu63TL9qduqpp/Lvf/87cr+tDzwWTctWNCUjIbrKYrGQn5/f/n9ix03vAEh0WBmc7ibZVyc1ok4wPRFZrdZeW+dGURRycnIYNGgQwaC000XX2e32yFSX1hyp9REsqyQ9pOFslojcThujh2RAeZX0EXWC6Yno66+/Jjc3F6fTydSpU1m6dCn5+fmtlvX7/fj9xy6F1tR0bj1gVVXbbOML0R2flVRT9tUBzlL8DBl83MoOauPXy4xVQfsZUzurp0yZwqpVq3jjjTdYvnw5u3fv5uyzz6a2trbV8kuXLsXj8URueXl5fRyxENECWvNF0Y7ViAzDwK+r+EMahjTNOmRqIrr44ou58sorGTduHBdeeCGvv/46VVVV/O1vf2u1/KJFi6iuro7cSkpK+jhiIaIFQjpWo3G96mad1SHd4PUvy9myr4pgUBJRR0xvmjWXkpLCSSedxK5du1p93uFwRBb4EiIeBEI6Tj3YokZktSjolvDXS5cBjR2Kq3FEdXV1FBcXyxKlot/way3Xq4bGhecs4bFsmtSIOmRqIlqwYAHr1q1jz549fPjhh1x++eWoqso111xjZlhCdFowpGM1WiYiAMUaTkRSI+qYqU2z/fv3c80111BeXk5mZiZnnXUWGzZskH2zRL8RiJriET3DXlGbdvKQRNQRUxPR888/b+bbC9FtZ4/MwN1gw27VojqrARRVakSdFVed1UL0N6fmeuAbFYKWFjUiS6RpJuOIOiKJSIjuMIxmUzyi+4iGZHjwVDpwWGQrpo5IIhIiRoGQzsHyajJ8AZIc1haJ6LTCQVCVBFZZiqYjcXX5Xoj+pMob4J9b9lBUWhPevPO4PqKmy/cyxaNjUiMSIkaB4y/dHzdDX7PYMHQdJeRHZjq2TxKREDEKhJoGM1oiW0039/bOCpL2VDI0M5lcE+LrT6RpJkSMgprR6qjqJoo13FQzNA106bBujyQiIWIUXjjfj0Wh1URkbUxEumFIP1EHJBEJEaOApqEaQayWlqOqASyqFQMlvFysLI7WLklEQsTI33wrodZqRKoF3WJt3NtMElF7pLNaiBgVZiSSkZtAWqWj1c5qq2rBp1jRDV0SUQckEQkRoxxPAjnpdvDZWq0RqZbGnTwMnyyg3wFJREJ0R8gX/reVRJSeaMfhSSKJEOiypVB7JBEJEaMjNT5sNXUk6zrWVhJRQUYi5KVDjV8SUQeks1qIGL1bdJSPdx2k2hts9aoZAKpM8+gMSURCxMivNbtqdvw8s0a6YkUz5PJ9RyQRCRGjQEhHNVpuJdSkpKKB178s4/MD1bLtdAckEQkRo0DTOKJWlomF8FUzXbHKgMZOkEQkRAwMwyAY0lAjWwm1Mo6oKRHJgMYOSSISIgaaboDmB4w2m2ZW1YKmWMNzzaRG1C5JRELEINDYUQ2gWm3H9rlvJtI0MwzpI+qAjCMSIgaqRWFaQRLp3gSUNi7d21QlMtfM0AIorZYSEEc1ovvuuw9FUbjtttvMDkWIDjmsKuOznQxJdbU5hqipRgSgyU4e7YqLGtGmTZt44oknGDdunNmhCNF5beze0cRmsZCTlkx6wC6d1R0wvUZUV1fHddddx5NPPklqaqrZ4QjRKQ2BEEerqvEFtTZrRBaLwoxTcjkpKxkrskJje0xPRPPmzeOSSy7hggsu6LCs3++npqYm6iaEGfZVNPDv7fv4pqy+zRoRcGwnD7lq1i7Tt5zesmULmzZt6lT5pUuXsmTJkl6OSoiOBaIWRWtjnhmAasUgvFSsdFa3zbQaUUlJCb/61a945plncDrb+SCbWbRoEdXV1ZFbSUlJL0cpROvCO3j4G0dVt10jemFLKRu+qaC23tuH0fU/ptWINm/ezJEjRzjttNMij2maxnvvvcdjjz2G3+9HVaN3g3I4HDgc7VSDhegjkWVire3XiHRL+Cumy8Jo7YopEX3zzTcMGzasW298/vnns3379qjHbrjhBk4++WQWLlzYIgkJEU/8IQ1VDzQunN/2f46WyJZC0kfUnpgS0YgRI5g+fTo33ngjP/jBDzrdtGouOTmZMWPGRD2WmJhIenp6i8eFiDf+YHgrobZ28GhiaVyPSJdxRO2KqY9oy5YtjBs3jvnz55Odnc1PfvITPv74456OTYi45Qtp7e7g0USxhhORoYVA1/sqvH4npkQ0YcIEli1bxsGDB1mxYgWHDh3irLPOYsyYMTz88MMcPXo0pmDeffddHnnkkZheK0RfGp3jYVialUSHtd0akdp8k0VZLrZN3bpqZrVaueKKK3jhhRf4wx/+wK5du1iwYAF5eXlcf/31HDp0qKfiFCKujMpOptCjkmhvPxFFmmYGMpaoHd1KRJ988gk///nPycnJ4eGHH2bBggUUFxfz5ptvcvDgQS677LKeilOI+GIYx7YIaqdpNsjtxJ2YgN1qkWke7Yips/rhhx9m5cqVFBUVMXPmTJ5++mlmzpyJxRLOa4WFhaxatYqCgoKejFWIuHGoooqUQAin1dLm7HuAKcPS4WA6BL3SNGtHTIlo+fLlzJ07lzlz5pCTk9NqmUGDBvHnP/+5W8EJEY9Cms7LH3/DxINVTBqWia2VtYiiqLZwIpIaUZtiSkRvvvkm+fn5kRpQE8MwKCkpIT8/H7vdzuzZs3skSCHiSXgwox8FsNoTOn6BzDfrUEx9RMOHD6esrKzF4xUVFRQWFnY7KCHimT+kY9V8qBYFxdZ+Itq0p4J3d1Wyu7xeakTtiCkRGYbR6uN1dXUxDW4Uoj/xhzSsui88mNHmaresYUDAUBt38pA+orZ0qWk2f/58ABRFYfHixbhcxz4ETdPYuHEjEyZM6NEAhYg3vsZR1aqqQAc1IqvafN1qqRG1pUuJ6NNPPwXCNaLt27djtx/b3dJutzN+/HgWLFjQsxEKEWf8IQ2b7sNqsXRYI7JalMadPJA+onZ0KRG98847QHhy6rJly3C73b0SlBDxzB881kfUYY3IYpGdPDohpqtmK1eu7Ok4hOg3stxOnJlW0urtHdeIVAVdUdE12dusPZ1ORFdccQWrVq3C7XZzxRVXtFt29erV3Q5MiHiV7XGSnWYFxdFhjUi1NG4pFEL6iNrR6UTk8XhQFCXysxAntGBD+N8OElGi3UpqciKJtarUiNrR6UTUvDkmTTNxIiuv85NQX4fDMFA7aJple5xkn5ILu4ulj6gdMY0j8nq9NDQ0RO7v3buXRx55hH/96189FpgQ8Wrt56V8+s1BarzBDmtEgIys7oSYEtFll13G008/DUBVVRWTJ0/moYce4rLLLmP58uU9GqAQ8cYf8GMxNKydGEcEQNNcNOkjalPMKzSeffbZALz44otkZ2ezd+9enn76af7nf/6nRwMUIt6E/PUAWK02UO3tlq3xBXlp62E2762UkdXtiCkRNTQ0kJycDMC//vUvrrjiCiwWC9/61rfYu3dvjwYoRDwJajqGP9wtYXW4QGl/tzKLolAfUsKvk5082hRTIhoxYgSvvPIKJSUlrF27lu985zsAHDlyRAY5igHNG9Sw6Q2E18xP6rB808hqA1lAvz0xJaLFixezYMECCgoKmDJlClOnTgXCtaOJEyf2aIBCxBNfQMOuebFaLCiO5A7LWy3huWYgiag9MY2s/sEPfsBZZ53FoUOHGD9+fOTx888/n8svv7zHghMi3niDGjatPtxRbU/ssLxqUTAimyxKImpLzDu9Zmdnk52dHfXY5MmTux2QEPHM7bQxNtNKSqUD7B03zRRFQYnsbSZ9RG2JqWlWX1/PnXfeyZlnnsmIESMYNmxY1K2zli9fzrhx43C73bjdbqZOncqaNWtiCUmIPpGaaOeUNIUcTwI4Ok5EAErjlTWpEbUtphrRTTfdxLp16/jxj39MTk5OZOpHVw0ZMoT77ruPkSNHYhgGTz31FJdddhmffvopp556akzHFKLXBcKX7ztTIwLI8CSS7LBi0UPhldJi/L4MZDElojVr1vDaa68xbdq0br35pZdeGnX/3nvvZfny5WzYsEESkYhLFfUBHHXVOA0DtZOJ6HsTh0J94/xMQwdF7cUI+6eYElFqaippaWk9GoimabzwwgvU19dHrsIdz+/34/f7I/dramp6NAYhOrKu6DBZxfsZkZHAoE50VgPhXTyaaEGwSCI6Xkx9RL/97W9ZvHhx1HyzWG3fvp2kpCQcDgc//elPefnllxk9enSrZZcuXYrH44nc8vLyuv3+QnSF11uPYujYVUunm2YolvANZL5ZG2KqET300EMUFxeTlZVFQUEBNpst6vktW7Z0+lijRo1i69atVFdX8+KLLzJ79mzWrVvXajJatGhRZN1sCNeIJBmJvqTVVwFgTUg+NoesA2s+LyWjpIbhaXbSpMO6VTElolmzZvVYAHa7nREjRgBw+umns2nTJpYtW8YTTzzRoqzD4cDhaHt7XyF6k6Yb6N5qAOyJKZ1+nS+k4dUsaJrs5NGWmBLRXXfd1dNxROi6HtUPJES8qA+EsGt1WBSwdSERqZF1q3WZgd+GmAc0VlVV8eKLL1JcXMyvf/1r0tLS2LJlC1lZWQwePLhTx1i0aBEXX3wx+fn51NbW8uyzz/Luu++ydu3aWMMSotfU+0M4QrXYVAuKs/OrlB7bycMvfURtiCkRffbZZ1xwwQV4PB727NnDzTffTFpaGqtXr2bfvn2RtYo6cuTIEa6//noOHTqEx+Nh3LhxrF27lm9/+9uxhCVErwonorpwR3UXE5HelIhklcZWxZSI5s+fz5w5c7j//vsjy4EAzJw5k2uvvbbTx/nzn/8cy9sLYYpUl52TUw2SAo6uJSJVIdi0pZDUiFoVUyLatGlTq53JgwcPprS0tNtBCRGP0hPtpCeHIOgER+eXu1EtFvyy22u7YkpEDoej1cGEO3fuJDMzs9tBCRGXgg0Q9IWnaLg6P6A3yWFFcTmxYZEaURtiGtD4ve99j3vuuYdgMPxLVRSFffv2sXDhQr7//e/3aIBCxIsjpfvxBjV0hzt6tHQHTh+ayjkn55KbkiB9RG2IKRE99NBD1NXVkZmZidfrZfr06YwYMYLk5GTuvffeno5RiLiw/rOdbC2potYSw75+spNHu2Jqmnk8Ht58803Wr1/Ptm3bqKur47TTTuOCCy7o6fiEiAshTYf6MgCcKVldP4Ds5NGuLiciXddZtWoVq1evZs+ePSiKQmFhIdnZ2RiGEfOSIELEsxpfCFeoEtWiYHd3rR+0+Ggdu74qZ4S3juFDpGnWmi41zQzD4Hvf+x433XQTBw4cYOzYsZx66qns3buXOXPmyDKxYsCq8QZJCFbisFpQXBldem1IM6j2gz8kI6vb0qUa0apVq3jvvfd46623OPfcc6Oee/vtt5k1axZPP/00119/fY8GKYTZautqsWleHA4bJHYtEakWBc1iRddlHFFbulQjeu655/jP//zPFkkI4LzzzuOOO+7gmWee6bHghIgX3qojAFhdHrB2beL1sZHVMo6oLV1KRJ999hkXXXRRm89ffPHFbNu2rdtBCRFv/NXhRKQmdX2cnFVtlohk9n2rutQ0q6ioICur7SsGWVlZVFZWdjsoIeLNyCQ/wdQEkjJyuvxaa2T2PVIjakOXEpGmaVitbb9EVVVCIcn4YuDJttZBqgsyY0hEatPse+kjakuXEpFhGMyZM6fNxclkHSExYDWExxDRxStmADbVQmKCE4dVlZHVbehSIpo9e3aHZeSKmRhovF4v/ooynDYVpyu9y6/3JNi4esow2OyRGlEbupSIVq5c2VtxCBG3jhw+wMHSWhyuJE6zu2I7SNPcNOkjalVMc82EOJE0XbpXEruxsoSl8f98qRG1ShKREB0I1DSOIUqOLREZhsHfPi1l2/4qgoFAeLdXEUUSkRAdCNUeBejyHLMmiqJwtF6nIaChyViiVkkiEqIDel03Zt03UhqHvRgylqhVkoiEaI8WwvCGB+m6upGIbKoVXVFlLFEbJBEJ0Q5/7VFCmo5mseN2x7AgWqOmaR6abshYolaYmoiWLl3KGWecQXJyMoMGDWLWrFkUFRWZGZIQUSzecoZnJpI7OA+HLeZtALGqlmbzzaRGdDxTE9G6deuYN28eGzZs4M033yQYDPKd73yH+vp6M8MSIsLmr2JQspNh+fndOo69aeKrLjPwWxN7iu8Bb7zxRtT9VatWMWjQIDZv3sw555xjUlRCNNNQHv43ofO7drTGZbdis9tB8UmNqBWmJqLjVVdXA5CW1r0PXYieUlV+GLwBHDY3Cd04zqXjc0HLgeoD0kfUirhJRLquc9tttzFt2jTGjBnTahm/3x81sba1vdWE6DGGwaFDB6iqrSd5hJPW/yq7QHbyaFPcXDWbN28en3/+Oc8//3ybZZYuXYrH44nc8vLy+jBCccIJ1BHw+wCFxJSuz7pvQeabtSkuEtGtt97KP//5T9555x2GDBnSZrlFixZRXV0duZWUlPRhlOJEYzSU4w/p+KzJuF3Obh1rW0kVH3xTxaFqr4ysboWpTTPDMPjFL37Byy+/zLvvvkthYWG75R0OR5trIQnR0wI1ZWi6gc/uwZ3Q+Z1dW1PvD1HuNRisyk4erTE1Ec2bN49nn32WV199leTkZEpLS4HwBo4JCd3pGhSi+7zV4TlmhisNm9q9xoPNGh5HpMk4olaZ2jRbvnw51dXVzJgxg5ycnMjtr3/9q5lhCQGAr3HWvdrF7YNaY7Uo6BYZR9QW05tmQsSrQG14sqvd3f1EZIsaWS19RMeLm8v3QsQVLUSm6sOZmYh/6NBuH64pEWlSI2qVJCIhWuOrIsFmIcGZDNndWJmxkS2ykwfSR9SKuLh8L0TcaagI/+tKB0Xp9uFsqgWrzYZFQUZWt0JqREK0wmgo40iND4srkXTdwGLpXjLKS3ORN3kYfLldakStkBqREK3wVh/lm7J63tuv99xBZWR1myQRCdEKX+MYIjUpvdu1oYimnTy0QM8cbwCRRCREK3ry0j2AP6Sx5qtyPj9YjR6SRHQ8SURCHC/QQMAbXpwvoYcSkUVR2FutUesLoYdka/bjSSIS4njeCnxBDb+ahDspqUcOabUoaBY7AFrQL3ubHUcSkRDHa5p1b/Pg6eZk1yaKomCxhWfwG5omHdbHkUQkxPEayvEFNbzWFFJcPZOIAFSrDVDCE181aZ41J+OIhDiO0VDBSVnJpOUM67EaEYDNqqJZbOFpHqEAyIo2EZKIhDiO0lCBJ8GGZ2gedHP5j+bsVgshxd44A19qRM1J00yI5nQNGnd2xZXeo4d2WFUUqx0dQK6cRZEakRDN+aqpqPMRMFScmpOUHjz0908bjKLkQfV+SUTHkRqREM01lHO41scXlSollb4ePbSiKGBt7BiSplkUSURCNNdQjjeg4bWlkJrYcx3VEWp4LBEyujqKNM2EaCZYcwR/SKchMYX0xJ69rLWjtIbKfbUU+L3kSI0oitSIhGimruIgAEZiJgl2tUePXeMNcbDWoCGgSR/RcSQRCdFE1/BWHQYgOT23xw9vt1qOjSOSGfhRJBEJ0aShnHqvH81iIy19UI8f3q5a0Cz2xgGNPdsR3t9JIhKiSd0RGgIaDbZ0sj09v6+e3WpBU+zHRlaLCFMT0Xvvvcell15Kbm4uiqLwyiuvmBmOONHVH+HUXDdjR40g29O9LaZb47A21ohkrlkLpiai+vp6xo8fz+OPP25mGEKE1RzEarGQM6QQp61nO6qhKRE1zTWTRNScqZfvL774Yi6++GIzQxAiTNcwag6iALiH9MpbNM01MwwkER2nX40j8vv9+P3HPsCamhoToxH9la4b7K/04g1q5KY4SXbaqDqynz17y0hPSWFIQio9tEp1FE+CjZvOHY3l4/ch5O2Fd+i/+lUiWrp0KUuWLDE7DNHPvb+rjC17wxNbFQWGprtIKN2KJ6hzmDTyLL3TY6EoCordFb6jhcI3tV99BXtNv7pqtmjRIqqrqyO3kpISs0MS/dAZBamkJ9kZ5HZgGLCnrAGlohirReGkU8b17ptbHaA0fu2CDb37Xv1Iv0rHDocDh0NWkxLd47Jb+fG3hqIoCpX1AYoOlJFTW82gTA8Jg0/u1fd+Z+dRMo74Ge5RcAW94HT36vv1F/2qRiRErAzD4EjNsUGESuM20qmJdr6VXMbQVCcJnkHgSuvVOA5Uejnqs+AP6dJP1IypNaK6ujp27doVub979262bt1KWloa+fn5JkYmBpqSCi8vbdlPQYaLWRMGRxIRhgEHt4R/zh7T63Ek2FRCFichvQ6CkoiamJqIPvnkE84999zI/fnz5wMwe/ZsVq1aZVJUYiDatKcCgBSX/VgSAjhaBDWHwruwZvdy/xDgtKkELU5CWo0komZMTUQzZszAkP2dRC87XONjX0UDFkXhtPzU8IOGAeXFUPR6+H7eZHD0zB5m7UmwW6i1OAjphiSiZvpVZ7UQsWiqDY3KTsbjtELZ17DnA6gtDRdIyYOCs/okFqdVpVJ1EtIM6SNqRhKRGNAq6wPsOlIHwKRcO2x9Fqr2hZ9UrZA7EQqng6Xnp3S0xmFTCVkchEK61IiakUQkBrTNeysxDDjZ7SOj6HnwVYcT0OBJ4eaYPbFP42nqrNYNJBE1I4lIDFiabnCgyovHW8KZxidg0yEhFcZeCYk9u1VQZ43KTubkaSdj+eILCNSbEkM8kkQkBixVgR8POUp11Xo8Nlu4L+jUK6BpmoUZMVkUcCaH7wTqTIsj3kgiEgNT0Atfv4nl8BekJtggeyyMurjP+oLa1XR1LlAPug69NLetP5FEJAYWXYfSbdR8+W+SLEEsFhWGnwtDzgjPcDWZYRis3VlLXmktJ2UmYgvWgyPZ7LBMJ4lIDBw1h2DnG/gqDvDV/ipCznTGn38VSVmFZkcWoSgKeyu8JPhtFGg6Nn+dJCIkEYmBQNdg9zoo+RgMg2+qgnyTMhVb/iTOHBR/U4USHVYCqotAyEei9BMBkohEfxf0wuerI2ODDjgKedt9Mpo1kR+dnB09nSNOJDmsBNREgloD+GVxP5BEJPqzkB8++2u4SabaqC64iFeKnQRVnSmFqaQnxeeSMYkOK/Wqi0BIB7/UiECWARH9lRaE7S+Gk5Atgfox1/LSvkQCIZ3BKQl8q9CccUKdkehQCaiJBDQ9PMBSSCIS/ZCuwRevhJtjVjuMu5oPDlmo9gbxJNi4ZFwOFkv8NcmaJDtseG2ecI3IW2l2OHFBmmaif9F1+OrvUL4rPFVj7JXgzuGckRq1vhAXnDKIREd8/1l7Emz4rJ7wtkLeCrPDiQtSIxL9h2HAzjfgyA6wqJQXfBdSwlfFEuwqPzh9CCkuu8lBdmxwagJzv30ap+Z6IOiDgKxdLYlI9A+GAbv+DYe2oQOfJp3N00Uq2/f3vz4W1aJgszuOrVcttSJJRKIfMAz45h3Y/wkBTeddZRLvVoQ7oysb+vEe8gmN62M3SCKK78a0EIYBu9dh7NtAZX2Ad4zT2eccik1VOP+ULE7J6Z+7YHx5sIayAzAy6CWn7rDZ4ZhOEpGIX7oGRWvwlWzlm7J6tjnPoDR5FJnJDi4ak01GnI4T6oyGQIg9QQ/ZvhA5NQfNDsd0kohEXNF1g2pvEK2+nIyStVBzCEOx8GnCtyh3j2by0FSmFKZhVft3r0KW28nH9kzqqkNQd/iE3/X1xD1zETfK6vzsLK1lX3kdvvISUmuLGKHvJiM7CawOEsZ+j5Pr0xiRmYzHZTM73B4xyO0gaPNQazjx+QM4q/dB2jCzwzKNJCLR9wwDag5SV76fHcV7OHq0FGewhvxQDQoGFgUsCbbwF3PUxeB0c3qG2UH3LIdVJTc1gaqKfCob9pFTtksSkdkef/xxHnjgAUpLSxk/fjyPPvookydPNjus+OWrDo8q9lZSH9BITMmC1KFgT6QhEMJhVcMrAcYTw4DaQ3Dky/A4IH8tew9WE/SFSAFSXTbSUl0kJ7pwZI/CkjsxfE4D2LDMJLYdLORozddkH/4cZdiM8EjxE5Dpieivf/0r8+fP549//CNTpkzhkUce4cILL6SoqIhBgwaZHV780ILhzQBLPyNQtpvyOj9ldQHq/CHOKEjFqqqQWsAObxaf1GeQn5XJiEFJDE13YTu+PyXoA181lRVHqa+rQVcsGKoDw5kCzlRUmx2baiHb44y8pLTah9NmIdlp63yS0/Vw8ikrgqNFhOrDl6mtFgtY7WQXnkpFtZ3RIwpIz8gKryftcMfFAmZ9YXSOmw+/zqO8MpHKmlrSSjZA4Tlmh2UKxTB5h8MpU6Zwxhln8NhjjwGg6zp5eXn84he/4I477mj3tTU1NXg8Hqqrq3G7++dl3DZpofASEbWHCJUV4zv4JbX1DVR5g1Q3BKlxZNFgS8OCwYzBGu5g+Eu+83AtZfVB6u0ZeG2pYLWT7rKSrPhwW3wUJAbDiYjwDhcBTT/ujRX81kRwZTBj4imQmAEJaby47QhH6zWsik6aEzIcBmlOg1S7TorDIMkSCs+GD/nC/wYbMOrLCAYDNPhDVDYEOdKgkTJkNKPGTQk3Q07gztkm7399lFDpl5zpex+HzQoFZ4e3ODJxXe2e1NnvqKl/CYFAgM2bN7No0aLIYxaLhQsuuICPPvqoRXm/34/f74/cr6np5FouFbuh+K1w8wAorfGxt/y4HRSa5eOTspJIbZwqcKTGy+6yepqeVRp/Chc3OGlQMmmJ4bJldb7IHlpw7HhK47GHZSaRmewAw6CyIUDR4dqo923+87AUC4OSwzWS6no/Ow/X4bcmcyTxFMpyTyItPZNR2cmclJUcnlvVUAFHdzAiaQfZZfspr6+lsr4cf70O1VAPhKwWCpp2OrW7UFMS0AwHVnSsuh9roApLyI8LL/bgASipjcQz9nAtVQ2B8DY4jeoab0eslmM7qAJfHKzGG9TQdIOgYqPKmU+Fq4BKdz5ZycmclJEXl+sEmWHa8AwsI86Br/1wYDNHt79JybpX0VUnuuoARcFQLI21RIVRWcmkNHbYH6nxs/v4v+NmRg5KJi0xXLasLkDx0baXHBmWmUhm43CIivogXx+pbbcMAGOvOjY6vJtMTURlZWVomkZWVlbU41lZWezYsaNF+aVLl7JkyZKuv1HID3VHI3eNOi96Xdvze5SGANDYVvf60BuOfdjHVx8NnwZq44fj82P4Wn7Ykdf4AXsw/FgggOFv+4/IMBLD+7EnZqCkZfO11U3yoHwKUhP5TlZSyzlVrjQYeiaWoWfi9lbhrj1EQUM5VXVeqnwa9boDw5GEMXwoijMFrHYmtHxTCDZAQznUl4X/bSgHbyWj8hIx9BA+TaE+ZKFOU6kJWqgOWlCdLhiSG+7fsCZQ5C2jJqjitaXgt7lJcTnITUng7KxkCtJdkoSaiawSMPLb4M6hbuu7+EN1EGoAWvkbTfKC3vjZ1/vQa9vZksjVAEbj32adH722nbWPEpKAxrL1AfTalokoqgyAcXxtOnb9qm68aNEi5s+fH7lfU1NDXl5exy9MyYPxPwz/rCgk+UPkN4RAUVrtjnAmOcBqARSSAhqFvmAr/Rbh17qcNrCHf42JQY3hvlDkuUi5xp+SnFawNZbVNEb6tGNHU8Jlwz8ruJLc4Q9eUUg1DK4f34Uvb0IKJKSgAKmNt05RlPCGg/bEyGTS6DOGhMZb84tYhmFE/X7O8vjC86msFhJsass+KtGSokD2WHLOH42rthbdVwOhAIahY+hG5EvvTLSDLfz7dAUa/47bkJBoA5vaWFYjvyHYZlmXywb2cNmEoEZ+fcuyzcsAPbo5pamJKCMjA1VVOXw4eoj74cOHyc7OblHe4XDgcMQwmtaeCGnHFlBPbrx1hqvx1hlNX9LOcBD1f0u74r0GcXx8g9zONkqKjjhtKs60FCClw7JJjbfO6Orf8eBOlu0ppv5XZbfbOf3003nrrbcij+m6zltvvcXUqVNNjEwI0ZdMb5rNnz+f2bNnM2nSJCZPnswjjzxCfX09N9xwg9mhCSH6iOmJ6Oqrr+bo0aMsXryY0tJSJkyYwBtvvNGiA1sIMXCZPo6oOwb0OCIhBoDOfkflcoYQwnSSiIQQppNEJIQwnemd1d3R1L3V6akeQog+1fTd7Kgrul8notrGYeidGl0thDBNbW0tHo+nzef79VUzXdc5ePAgycnJcT/6uEnTtJSSkpIBdaVvIJ7XQDwn6NvzMgyD2tpacnNzsVja7gnq1zUii8XCkCFDzA4jJm63e0D9cTcZiOc1EM8J+u682qsJNZHOaiGE6SQRCSFMJ4mojzkcDu66667YVhGIYwPxvAbiOUF8nle/7qwWQgwMUiMSQphOEpEQwnSSiIQQppNEJIQwnSSiXnbvvfdy5pln4nK5SElJ6dRrDMNg8eLF5OTkkJCQwAUXXMDXX3/du4F2UUVFBddddx1ut5uUlBRuvPFG6ura2SUCmDFjBoqiRN1++tOf9lHErXv88ccpKCjA6XQyZcoUPv7443bLv/DCC5x88sk4nU7Gjh3L66+/3keRdk1XzmvVqlUtPhens4/XHTdEr1q8eLHx8MMPG/Pnzzc8Hk+nXnPfffcZHo/HeOWVV4xt27YZ3/ve94zCwkLD6/X2brBdcNFFFxnjx483NmzYYLz//vvGiBEjjGuuuabd10yfPt24+eabjUOHDkVu1dXVfRRxS88//7xht9uNFStWGF988YVx8803GykpKcbhw4dbLb9+/XpDVVXj/vvvN7788kvjv/7rvwybzWZs3769jyNvX1fPa+XKlYbb7Y76XEpLS/s0ZklEfWTlypWdSkS6rhvZ2dnGAw88EHmsqqrKcDgcxnPPPdeLEXbel19+aQDGpk2bIo+tWbPGUBTFOHDgQJuvmz59uvGrX/2qDyLsnMmTJxvz5s2L3Nc0zcjNzTWWLl3aavmrrrrKuOSSS6IemzJlivGTn/ykV+Psqq6eV2f/NnuTNM3izO7duyktLeWCCy6IPObxeJgyZUqru9+a4aOPPiIlJYVJkyZFHrvggguwWCxs3Lix3dc+88wzZGRkMGbMGBYtWkRDQ9sbXfampl2Gm/+e29tlGMLn3bw8wIUXXhg3nwvEdl4AdXV1DB06lLy8PC677DK++OKLvgg3ol9Peh2ISktLAVrd/bbpObOVlpYyaNCgqMesVitpaWntxnjttdcydOhQcnNz+eyzz1i4cCFFRUWsXr26t0Nuoau7DEP4vOP5c4HYzmvUqFGsWLGCcePGUV1dzYMPPsiZZ57JF1980WeTyqVGFIM77rijRefe8be2PvR41tvndcstt3DhhRcyduxYrrvuOp5++mlefvlliouLe/AsRFdNnTqV66+/ngkTJjB9+nRWr15NZmYmTzzxRJ/FIDWiGPzHf/wHc+bMabfMsGHDYjp20w63hw8fJicnJ/L44cOHmTBhQkzH7KzOnld2djZHjhyJejwUClFRUdHqDr1tmTJlCgC7du1i+PDhXY63O7q6yzCEP5uulDdDLOd1PJvNxsSJE9m1a1dvhNgqSUQxyMzMJDMzs1eOXVhYSHZ2Nm+99VYk8dTU1LBx40Z+9rOf9cp7NunseU2dOpWqqio2b97M6aefDsDbb7+NruuR5NIZW7duBYhKuH2l+S7Ds2bNAo7tMnzrrbe2+pqpU6fy1ltvcdttt0Uee/PNN+NqV+JYzut4mqaxfft2Zs6c2YuRHsfUrvITwN69e41PP/3UWLJkiZGUlGR8+umnxqeffmrU1tZGyowaNcpYvXp15P59991npKSkGK+++qrx2WefGZdddllcXr6fOHGisXHjRuODDz4wRo4cGXX5fv/+/caoUaOMjRs3GoZhGLt27TLuuece45NPPjF2795tvPrqq8awYcOMc845x6xTMJ5//nnD4XAYq1atMr788kvjlltuMVJSUiKXrn/84x8bd9xxR6T8+vXrDavVajz44IPGV199Zdx1111xe/m+K+e1ZMkSY+3atUZxcbGxefNm44c//KHhdDqNL774os9ilkTUy2bPnm0ALW7vvPNOpAxgrFy5MnJf13XjzjvvNLKysgyHw2Gcf/75RlFRUd8H347y8nLjmmuuMZKSkgy3223ccMMNUcl19+7dUee5b98+45xzzjHS0tIMh8NhjBgxwvj1r39t6jgiwzCMRx991MjPzzfsdrsxefJkY8OGDZHnpk+fbsyePTuq/N/+9jfjpJNOMux2u3Hqqacar732Wh9H3DldOa/bbrstUjYrK8uYOXOmsWXLlj6NV5YBEUKYTq6aCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCLRr6xatSpqpcu77747ag7enDlzIlMbRP8hiUi0as6cOW0u5Tpv3jwURYmaINsbCaCgoIBHHnkk6rGrr76anTt3tvmaZcuWsWrVqsj9GTNmRM0NE/FJEpFoU15eHs8//zxerzfymM/n49lnnyU/P9+UmBISElqshdScx+Pp9NrgIn5IIhJtOu2008jLy4tauGz16tXk5+czceLEbh27tZrKrFmzIrWsGTNmsHfvXm6//fbIWkjQsml2vOY1szlz5rBu3TqWLVsWOcbu3bsZMWIEDz74YNTrtm7diqIofbr0hThGEpFo19y5c1m5cmXk/ooVK7jhhht6/X1Xr17NkCFDuOeeezh06BCHDh3q8jGWLVvG1KlTufnmmyPHyM/Pb3FOACtXruScc85hxIgRPXUKogskEYl2/ehHP+KDDz5g79697N27l/Xr1/OjH/2o1983LS0NVVVJTk4mOzs7psXHPB4Pdrsdl8sVOYaqqsyZM4eioqLIFjvBYJBnn32WuXPn9vRpiE6ShdFEuzIzM7nkkktYtWoVhmFwySWXkJGRYXZY3ZKbm8sll1zCihUrmDx5Mv/4xz/w+/1ceeWVZod2wpIakejQ3LlzWbVqFU899VSP1RosFgvHr0ATDAZ75NidcdNNN0U64leuXMnVV1+Ny+Xqs/cX0SQRiQ5ddNFFBAIBgsEgF154YY8cMzMzM6rfR9M0Pv/886gydrsdTdO69T5tHWPmzJkkJiayfPly3njjDWmWmUyaZqJDqqry1VdfRX5uS3V1dWQd6ibp6enk5eW1KHveeecxf/58XnvtNYYPH87DDz9MVVVVVJmCggLee+89fvjDH+JwOGJqEhYUFLBx40b27NlDUlISaWlpWCyWSF/RokWLGDlyZFytO30ikhqR6BS3243b7W63zLvvvsvEiROjbkuWLGm17Ny5c5k9ezbXX38906dPZ9iwYZx77rlRZe655x727NnD8OHDY96sYMGCBaiqyujRo8nMzGTfvn2R52688UYCgUCfXAUU7ZOlYsUJ6/333+f888+npKSkxYaEom9JIhInHL/fz9GjR5k9ezbZ2dk888wzZod0wpOmmTjhPPfccwwdOpSqqiruv/9+s8MRSI1ICBEHpEYkhDCdJCIhhOkkEQkhTCeJSAhhOklEQgjTSSISQphOEpEQwnSSiIQQppNEJIQw3f8H9WVoupmTblwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:41.625103Z", + "iopub.status.busy": "2024-03-22T19:05:41.624806Z", + "iopub.status.idle": "2024-03-22T19:05:41.843809Z", + "shell.execute_reply": "2024-03-22T19:05:41.842964Z" + }, + "papermill": { + "duration": 0.241397, + "end_time": "2024-03-22T19:05:41.845833", + "exception": false, + "start_time": "2024-03-22T19:05:41.604436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6PklEQVR4nO3deVwT19oH8F/CEkB2AUFBEXBHRRERrWIVUVEvFK0WtSLa9taWLuJS6Fu1vvYWqWi9Va++9V617m0R0VJccKFSF0SsrQsu4K4gm7IbQjjvH7yZl8iWKJBk5vl+Pn4kM2cmz5DMw5lzZs4RMcYYCCGE58SaDoAQQtoCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIOhcstuwYQOcnZ1hZGQEb29vnD9/vtGymzdvxvDhw2FlZQUrKyv4+fk1WZ4Qwl86lex+/PFHREREYNmyZbh48SL69++PsWPHIi8vr8HyKSkpCAkJwcmTJ3H27Fk4OTnB398fjx49auPICSGaJtKlgQC8vb3h5eWF9evXAwBqamrg5OSEjz76CJGRkc1uL5fLYWVlhfXr12PWrFkqvWdNTQ0eP34MMzMziESiV4qfENKyGGMoLS1Fx44dIRY3XXfTb6OYXllVVRUyMjIQFRXFLROLxfDz88PZs2dV2kdFRQVkMhmsra0bLSOVSiGVSrnXjx49Qu/evV8+cEJIq3vw4AEcHR2bLKMzya6goAByuRwdOnRQWt6hQwdcv35dpX189tln6NixI/z8/BotEx0djeXLl9db/u9//xsmJibqBU0IaVUVFRV45513YGZm1mxZnUl2r2rlypXYu3cvUlJSYGRk1Gi5qKgoREREcK9LSkrg5OSEoKAgmJubt0WoGiGTyZCcnIwxY8bAwMBA0+GQVySUz7OkpATvvPOOSk1MOpPsbGxsoKenhydPnigtf/LkCezt7ZvcNjY2FitXrsSxY8fQr1+/JstKJBJIJJJ6yw0MDHj9pVEQynEKBd8/T3WOTWd6Yw0NDeHp6Ynjx49zy2pqanD8+HH4+Pg0ut0333yDFStW4PDhwxg0aFBbhEoI0UI6U7MDgIiICISGhmLQoEEYPHgw1q5di/LycoSFhQEAZs2ahU6dOiE6OhoAEBMTg6VLl2L37t1wdnZGbm4uAMDU1BSmpqYaOw5CSNvTqWQ3bdo05OfnY+nSpcjNzYWHhwcOHz7MdVrcv39fqft548aNqKqqwpQpU5T2s2zZMnz55ZdtGTohRMN0KtkBQHh4OMLDwxtcl5KSovT67t27rR8QIUQn6EybHSGEvAqdq9kRQuqrqKhQut+0rFKKM5ezYWVzAabGyncX9OzZU5D3jFKyI4QHrl+/Dk9Pz3rLv2mgbEZGBgYOHNj6QWkZSnaE8EDPnj2RkZHBvb6R8wwRP1/Gmjf7ooeDZb2yQkTJTqBUvewR6iWPrjExMVGqrYnvFUKSWole7v3h0aW9BiPTHpTsBErVyx6hXvIQ/qFkJ1CqXvYI9ZJHF9wpKEe5tLrBddn55dz/+vqNn+btJProatOuVeLTNpTsBIoue3TbnYJyvB6b0my5BXGXmy1zcuFIQSQ8SnYC0VQtAKCagK5RfJZrp3nAza7+o4/llVIkppzFxJE+aGdcf2ALAMjKK8OnP15q8nvBJ5TsBEDVWgBANQFd42ZnCvdOFvWWy2Qy5NoCA7tY8XrUE3VQshOA5moBANUECP9RshOQxmoBANUECP9RsiNEB0nlzyE2eoQ7JTcgNqpfW6+ursbj6sfILMpstA32TkkZxEaPIJU/B9DwH0E+oWQnAM2dGACdHLrmcfk9tOu6Dp83Mw3yvw7/q8n17boCj8s94IkOTZbjA0p2AqDqiQHQyaErOrbrgvI7H+Gf0zzg2kA7bHV1NU7/fhrDXhvW6B+v7LwyfPLjJXR8vUtrh6sVKNkJQHMnBkAnh66R6Bmh5nkndDXvgd7tG+6NvaN/B72sezXaBlvzvBg1z/Mh0Wt8Aio+oWQnAM2dGACdHIT/aPBOQoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCDfEkAJUyOQDgyqPiRsuUV0pxIR+wv/e0yQl3CNFVOpfsNmzYgFWrViE3Nxf9+/fHunXrMHjw4EbL//zzz1iyZAnu3r2Lbt26ISYmBgEBAW0YseZl/1+SioxvbppEfezISm92f+0kOve14Z3m/oDRH6/6dOpb++OPPyIiIgKbNm2Ct7c31q5di7Fjx+LGjRuws7OrV/7MmTMICQlBdHQ0Jk6ciN27dyMoKAgXL16Eu7u7Bo5AM/z72AMAXO1MYWyg12CZGznFWBB3Gaun9EUPh8bnl6BJsrWDan/A6I9XXSLGGNN0EKry9vaGl5cX1q9fDwCoqamBk5MTPvroI0RGRtYrP23aNJSXlyMxMZFbNmTIEHh4eGDTpk0qvWdJSQksLCxQXFwMc3PzljkQLXTpXiGCNp5Dwrwh8OjSXtPhkGYUlVfh6NXcRv+ACeWPlzrnp86k9KqqKmRkZCAqKopbJhaL4efnh7Nnzza4zdmzZxEREaG0bOzYsUhISGj0faRSKaRSKfe6pKQEQO2w5TKZ7BWOQLtVV1dz//P5OPnCzFCEyQMcGl3//PlzAEAXKwl62Jk0uS9d/rzViV1nkl1BQQHkcjk6dFCe1apDhw64fv16g9vk5uY2WD43N7fR94mOjsby5cvrLT969ChMTJr+0uiyB2UAoI9z587h0RVNR0NelVA+z4qKCpXL6kyyaytRUVFKtcGSkhI4OTnB39+f15exf94vAi5fwJAhQ9C/s7WmwyGvSCifp+LKSxU6k+xsbGygp6eHJ0+eKC1/8uQJ7O3tG9zG3t5erfIAIJFIIJHU770yMDBodNYtPlBMn6ivr8/r4xQKoXye6hybztxUbGhoCE9PTxw/fpxbVlNTg+PHj8PHx6fBbXx8fJTKA0BycnKj5Qkh/KUzNTsAiIiIQGhoKAYNGoTBgwdj7dq1KC8vR1hYGABg1qxZ6NSpE6KjowEAn3zyCXx9fbF69WpMmDABe/fuxYULF/D9999r8jAIIRqgU8lu2rRpyM/Px9KlS5GbmwsPDw8cPnyY64S4f/8+xOL/r6wOHToUu3fvxhdffIHPP/8c3bp1Q0JCgqDusSOE1NKpZAcA4eHhCA8Pb3BdSkpKvWVvvvkm3nzzzVaOihCi7XSmzY4QQl4FJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiMoLi7G7MkBePiv2Zg9OQDFxcWaDomQFqdzs4uRluXm5obs7Gzu9Z8ZBbC0tISrqyuysrI0GBkhLYtqdgL2YqKrKzs7G25ubm0cESGth5KdQOXk5CglutdG+cF60iK8NsqPW5adnY2cnBxNhEdIixMxxpimg9BmJSUlsLCwQHFxMczNzTUdTovp27cvrly50mw5d3d3XL58uQ0iIi2lsrISs/8ejoOpf+Bvwwdg2/+sh7GxsabDahXqnJ9UsxMoRXtcu3btGlyvODmo3U63BAUFwcTEBD/t2ILnd//ATzu2wMTEBEFBQZoOTeMo2QmUSCQCAJSXlze4vrKyUqkc0X5BQUE4cOBAg+sOHDgg+IRHyU6gXnvtNe5nCwsLuLi4wMrKCi4uLrCwsGiwHNFelZWVjSY6hQMHDnB/xISIkp1A6enpcT8XFxfj9u3bePr0KW7fvq10n13dckR7ffzxxyqVmzVrFioqKlo5Gu1EyU6gLl261KLliGalpqaqVC4uLg7Xr19v5Wi0EyU7gTIwMGjRckSzSktLVSpna2uLnj17tnI02omSnUDV1NRwP0skEqV1dV/XLUe0l5GRkUrlzMzMYGJi0srRaCedSXZFRUWYMWMGzM3NYWlpiblz56KsrKzJ8h999BF69OgBY2NjdO7cGR9//DE99/l/6tYEpFKp0rq6r1WtMRDNev78eYuW4yOdSXYzZszA1atXkZycjMTERJw6dQrvvfdeo+UfP36Mx48fIzY2FleuXMG2bdtw+PBhzJ07tw2j1l6qdjxQB4VuoGTXPLWfoLh9+zZcXFxaK54GZWZmonfv3khPT8egQYMAAIcPH0ZAQAAePnyIjh07qrSfn3/+GTNnzkR5eTn09RseA0EqlSrVbEpKSuDk5ISCggJePUExevRolRq1hw8fjuPHj7dBRORVdOjQAU+fPm22nJWVFZ48edIGEbWNkpIS2NjYqPQEhdqjnri5ucHX1xdz587FlClTVG4reBVnz56FpaUll+gAwM/PD2KxGGlpaXjjjTdU2o/iF9JYogOA6OhoLF++vN7yo0eP8qqtQ5UTQ1EuKSmplaMhr0rV20kqKip49XmqcxuN2snu4sWL2Lp1KyIiIhAeHo5p06Zh7ty5GDx4sLq7Ullubi7s7OyUlunr68Pa2hq5ubkq7aOgoAArVqxo8tIXAKKiohAREcG9VtTs/P39eVWzW7lypUrlTE1NERAQ0MrRkFf1YrtrU+X49HmWlJSoXFbtZOfh4YF//vOfWL16NQ4ePIht27bhtddeQ/fu3TFnzhy8/fbbsLW1VWlfkZGRiImJabJMZmamuiHWU1JSggkTJqB379748ssvmywrkUjq9U4Ctbdg8Ok2jPz8fJXL8em4Cb9uJ1LnWF551BOpVIp//etfiIqKQlVVFQwNDTF16lTExMTAwcGhyW3z8/NRWFjYZBkXFxfs3LkTCxYsULr0qq6uhpGREX7++ecmL2NLS0sxduxYmJiYIDExUe3Lbr6OetK+fXsUFRU1W87a2rrZz4honlgshiqnskgk4tXtROqcny89UvGFCxewZcsW7N27F+3atcPChQsxd+5cPHz4EMuXL0dgYCDOnz/f5D5sbW1VqgX6+Pjg2bNnyMjIgKenJwDgxIkTqKmpgbe3d6PblZSUYOzYsZBIJDh48GCbtC/qCrFYtY54VcsRzerZs6dKV0FCvaEYAMDUtHr1aubu7s4MDAxYYGAg++WXX5hcLlcq8+DBA6anp6furps0btw4NmDAAJaWlsZ+//131q1bNxYSEsKtf/jwIevRowdLS0tjjDFWXFzMvL29Wd++fVlWVhbLycnh/lVXV6v8vsXFxQwAKy4ubtHj0TRXV1cGoNl/rq6umg6VqKBPnz4qfZ59+vTRdKgtSp3zU+2a3caNGzFnzhzMnj270ctUOzs7/Oc//1F3103atWsXwsPDMXr0aIjFYkyePBnfffcdt14mk+HGjRtc78zFixeRlpYGAPWGF79z5w6cnZ1bND5dY2tr2+iQ7C+WI9rP3t4eV69eVamcUKmd7JKTk9G5c+d6lzeMMTx48ACdO3eGoaEhQkNDWyxIoLbtaPfu3Y2ud3Z2VmqzGDlypEptGEKl6si1fB3hlm9cXV1Vuh/S1dW1DaLRTmo3yLi6uqKgoKDe8qKiInTt2rVFgiKtT9VBOWnwTt2g6h92IVcA1E52jf2yysrKqANAh9DjRfxy9+7dFi3HRypfxiputBWJRFi6dKnS0wRyuRxpaWnw8PBo8QBJ61D1DxP9AdMNNGRX81ROdn/88QeA2prd5cuXYWhoyK0zNDRE//79sXDhwpaPkLQKVTseqINCNzx8+JD72cjISKlGXvd13XJCo3KyO3nyJAAgLCwM//znP3l1g60QqXpjKZ9uQOWzO3fucD+/2PRQ93XdckKjdm/s1q1bWyMO0sZUnSKRplIkfKFSsgsODsa2bdtgbm6O4ODgJsvGx8e3SGCkdTU2heLLliOa1bVrV/z1118qlRMqlZKdhYUFdwtC3Wn2iO7q3Lkzbt68qVI5ov3at2/fouX4SKVkV/fSlS5j+SE8PBzHjh0DUNvBVFVVxa2r+zo8PFwj8RH1qDPEk1DRU94CVbctrm6ie/E1tdnpBnoipnkq1ewGDBig8p30Fy9efKWASNugm1D5pX///io9Lta/f/82iEY7qZTsgoKCWjkM0tbqDoRgYGAAmUzGva57GSv0ARN0xa1bt1q0HB+plOyWLVvW2nGQNta7d28AtbOHOTg44P79+9w6e3t7PHr0CHK5nCtHtFtlZWWLluMjarMTqDNnzgCofdSvbqIDgPv370MulyuVI9rN1NSU+/nFaQXqPvJXt5zQqJTsrK2tuZFOrKysYG1t3eg/ohvoCQp+UTQ16evr13vEz9bWlptRT8hNUipdxn777bcwMzPjfqZhf3Rf3b/wnTt3Vqrd1X0t5JqALunSpQuA2rlZXnz+9cGDB/XKCdErT7jDd3ydcMfPz4/rvbOzs8OMGTNQXl6Odu3aYdeuXcjLywNQO5m24n48or3kcjkcHByanDXOzs4Ojx8/hp6eXhtG1rpadcIdPT095OTk1JvHtbCwEHZ2dlxbD9Fuz549437Oy8vDt99+22w5ot0UNwxbW1tDIpGgpKQE5ubmkEqlKCoqEvzYhC02eKdUKlUa9oloN0dHxxYtRzQrJSUFJSUlMDU1RVFREXJyclBeXo6cnBwUFRXB1NQUJSUlSElJ0XSoGqNyzU4xuY1IJMK///1vpbYcuVyOU6dOCXuaNh0TGBiIAwcOAKhtwB4xYgSePn0KKysrnDp1irscCgwM1GSYREWKJFZWVgZDQ0MEBwfD2NgYlZWViI+PR1lZGVdu9OjRGoxUc1ROdorLHMYYNm3apHTdb2hoCGdnZ2zatKnlIyStou4E2fn5+di3b1+z5Yj2UtwEbmBggNLSUohEIiQlJSEgIAA//PADTE1NIZPJ6j0aKCQqJzvFoH+vv/464uPjYWVl1WpBkdanahKjZKcbFBNk29nZQV9fX6ntXHE7yuPHj1WaSJuv1G6zO3nyJCU6nhk/fjyGDRsGJycnDBs2DOPHj9d0SERNis6HR48eITAwEOfOnUNlZSXOnTuHwMBAPH78WKmcEKndGztnzpwm12/ZsuWlgyFtR3EDuK2tLY4ePcrVBB48eAA9PT3Y2toiPz+fbhTXEd27d0dycjIA4NixY0hMTOTW1R3ppHv37m0em7ZQu2b39OlTpX95eXk4ceIE4uPj6TYFHaKYGT4/P7/e7UJyuZzroBDyDPK6ZNWqVQBqbw17sV1OKpVybeyKckKkds1u//799ZbV1NRg3rx5gp5tXNeomsQo2ekGY2NjeHl5IT09HQAwcOBArjdWMeyal5cXjWf3qsRiMSIiIjBy5EgsXry4JXZJWhnNQcEvitq44mmCF8eVtLCwQEFBAeRyOa+eoFBHiyQ7AMjOzkZ1dXVL7Y60suXLl3M/N/W42PLly/G3v/1NU2ESFaWmpuLu3bsQiUQYP348SktLce/ePXTp0gVmZmY4fPgwiouLkZqaipEjR2o6XI1QO9lFREQovWaMIScnB7/++itCQ0NbLDDSuh49egQAcHBwgKGhodLjYs7OznBwcEBOTg5Xjmg3xefk4eGBy5cvc4MBPHjwAI6OjvDw8MAff/wh6M9T7WT3xx9/KL0Wi8WwtbXF6tWrm+2pJdrD3NwcT548gUwmw7179/Dbb7/h0KFDGD9+PHx9feHg4MCVI9pP0aH04vkJAA8fPuSSX1MDBfCd2snu5MmTrREHaWNBQUFYtWoVCgoKEBQUhMjISHh5eUEikSAoKAiFhYVcOaL9aCrF5rVYmx3RLWPHjuVuQ0hKSkJSUlKj5Yj2U9w0DNS2wU6fPh0VFRUwMTHB7t27uTbYuuUEh+mIwsJCNn36dGZmZsYsLCzYnDlzWGlpqUrb1tTUsHHjxjEAbP/+/Wq9b3FxMQPAiouLXyJq7VVdXc3s7OwYgEb/2dnZserqak2HSlQwatQoBoDp6+szsVis9Dnq6ekxfX19BoCNGjVK06G2KHXOT52Zg2LGjBm4evUqkpOTkZiYiFOnTuG9995Tadu1a9fS6Mov0NPTw8aNGyESiZTmKABq79kSiUTYuHGjYG9T0DWKNrnq6mrY2tpi/vz5eO+99zB//nzY2Nhwd0q8OIqxkOjEZWxmZiYOHz6M9PR0DBo0CACwbt06BAQEIDY2Fh07dmx020uXLmH16tW4cOEC1+jeFKlUqjRreklJCQBAJpMpTTfIB5MmTcLevXuxePFi3Lt3j1veoUMHxMTEYNKkSbw7Zr7q1KkTbt68CUNDQxgZGdXrXVdMj9mpUydefabqHEuLJbuHDx/iv//7v/H999+31C45Z8+ehaWlJZfogNphxcViMdLS0vDGG280uF1FRQWmT5+ODRs2qPwkQHR0tNI9aApHjx6FiYnJyx2AFpNIJFizZg2uXbvGjWfXu3dv6OnpNdqOR7SPYpDVqqoqWFtbw9/fHxKJBFKpFBcuXOAmO3d0dOTV51pRUaFy2RZLdoWFhfjPf/7TKskuNze33jDw+vr6sLa2Rm5ubqPbzZ8/H0OHDlVrAMqoqCilewlLSkrg5OQEf39/Xt+GMW7cOCQnJ2PMmDEwMDDQdDhETc+ePcOOHTsA1N5+0tAtKEDtnCIBAQFtGVqrUlx5qUKjl7GRkZGIiYlpsszLjr918OBBnDhxotEPvTESiaTevJtA7aCIQkgCQjlOvlF11rAuXbrw6vNV51g0muwWLFiA2bNnN1nGxcUF9vb2XNe5QnV1NYqKihq9PD1x4gSys7NhaWmptHzy5MkYPny4oMfiJ/wzfPhwODs7Q09PD3fv3lUayUZPTw/Ozs6oqanB8OHDNRilZmk02dna2tab0LchPj4+ePbsGTIyMuDp6QmgNpnV1NTA29u7wW0iIyPxzjvvKC3r27cvvv32W0yaNOnVgydEi+jp6eHNN9/EqlWrGnzWOTs7G4sWLRJ077rK88YGBwc3uf7Zs2f47bffWm0qxfHjx+PJkyfYtGkTZDIZwsLCMGjQIOzevRtA7bOBo0ePxvbt2zF48OAG9yESibB//361ngrg67yxL5LJZNycBXy6zBEKuVwONzc32NjYoKCggOuQAICuXbuiffv2KCwsxK1bt3iV8Fpl3lgLC4tm18+aNUvV3alt165dCA8Px+jRoyEWizF58mRuxjOg9mS9ceOGWr0zhPCFYtSTPXv2wMvLCydPnuSedX799ddx/vx5DB06lEY9UcXWrVtbM45mWVtbc7W4hjg7Ozc6p62CipVYQnROTk4OAMDd3R16enrw9fVFeXk5fH19oaenB3d3d6VyQqQzT1AQQhqnuGH+ypUrDa5XLFflxnq+Urlmp+rwTTThDiFtT9Eb+/XXXyMhIUFpXU1NDaKjo9G1a1fqjVXFtm3b0KVLFwwYMIAuBwnRMnp6eli9ejWmTJmCoKAgLFq0iJtKcdWqVUhMTERcXByvOifUpXKymzdvHvbs2YM7d+4gLCwMM2fOpGn2CNEiwcHBiIuLw4IFCzBixAhuedeuXREXF9fsHRV8p3Kb3YYNG5CTk4PFixfjl19+gZOTE6ZOnYojR45QTY8QLREcHIysrCwkJycjIiICycnJuHXrluATHaBmB4VEIkFISAiSk5Nx7do19OnTBx988AGcnZ1RVlbWWjESQtSg6I0dMWIE1xtLXqE3ViwWQyQSgTHWajcSE0JIS1Er2UmlUuzZswdjxoxB9+7dcfnyZaxfvx7379+Hqalpa8VICFGDXC7Hb7/9hlOnTrXqU026RuUOig8++AB79+6Fk5MT5syZgz179sDGxqY1YyOEqCk+Ph4LFizgHhdbs2YNnJ2dsXr1asG326mc7DZt2oTOnTvDxcUFv/32G3777bcGy8XHx7dYcIQQ1cXHx2PKlCmYOHEiduzYgYcPH8LR0RHffPMNpkyZIvgeWZWT3axZs2geB0K0lFwux4IFCzBx4kQkJCRALpejsLAQ3t7eSEhIQFBQEBYuXIjAwEDBdliodVMxIUQ71R0IQCwWK7XTicViREVFCX4gAHo2lhAeqDsQQEMdFDQQgI7MLkYIaZriAf/169fjf/7nf+p1UCimHRXyQABUsyOEB4YPHw5bW1tERUXB3d0dqamp2LNnD1JTU+Hu7o7PP/8cdnZ2gh4IgJIdITxRtwNR8QgnPcr5/yjZEcIDqampyMvLQ3R0NK5cuYIRI0YgJCQEI0aMwNWrV/H1118jLy8Pqampmg5VYyjZEcIDio6H8PDwBgcCCA8PVyonRJTsCOGBuiMVNzQQAI1UTMmOEF6oO1JxTU2N0joaqbgWJTtCeEAxUnFiYiKCgoJw7tw5bqTioKAgJCYmIjY2VrBPTwB0nx0hvEEjFTeNkh0hPBIcHIzAwMB688YKuUanQMmOEJ5paN5YQm12BDTYIxEGSnYCFx8fD1dXV4wZMwZr1qzBmDFj4OrqSuMSEt6hZCdg8fHxmDx5MvLy8pSW5+XlYfLkyZTwCK9QshMouVyO999/v8ky8+bNo0tawhuU7AQqJSUF+fn5AOo/LK54nZeXh5SUlLYOjZBWQclOoE6cOMH97OfnpzQkkJ+fX4PlCNFllOwE6t69ewBqR7Y9cOAAvL29YWxsDG9vbxw4cAB9+vRRKkeIrtOZZFdUVIQZM2bA3NwclpaWmDt3LsrKyprd7uzZsxg1ahTatWsHc3NzjBgxApWVlW0QsW6g8c6IUOhMspsxYwauXr2K5ORkJCYm4tSpU9xQ0405e/Ysxo0bB39/f5w/fx7p6ekIDw+HWKwzh91qunTpAgC4evUqAgMDlZ6lDAwMxNWrV5XKEaLzmA64du0aA8DS09O5ZYcOHWIikYg9evSo0e28vb3ZF1988UrvXVxczACw4uLiV9qPtjl27BgDwAAwY2Nj7ucXXx87dkzToZKXUFVVxRISElhVVZWmQ2lV6pyfOvG42NmzZ2FpaYlBgwZxy/z8/CAWi5GWloY33nij3jZ5eXlIS0vDjBkzMHToUGRnZ6Nnz574xz/+gddee63R95JKpZBKpdzrkpISAIBMJoNMJmvBo9KsYcOGwdbWluuRbYidnR2GDRvGq+MWCsVnxvfPTp3j04lkl5ubCzs7O6Vl+vr6sLa2Rm5uboPb3L59GwDw5ZdfIjY2Fh4eHti+fTtGjx6NK1euoFu3bg1uFx0djeXLl9dbfvToUZiYmLzikWiXOXPmICYmBtXV1UrLFa/DwsJw5MgRTYRGXoFcLse1a9fw9OlTXL58Gb179+bt87EVFRUql9VosouMjERMTEyTZTIzM19q34oBDP/+978jLCwMADBgwAAcP34cW7ZsQXR0dIPbRUVFISIigntdUlICJycn+Pv7w9zc/KVi0VYBAQEYOHAgFi9erNTr2qlTJ8TExDRYYybabf/+/fjss8+4qRQBwNnZmbefp+LKSxUaTXYLFizA7Nmzmyzj4uICe3v7eo80VVdXo6ioCPb29g1upxh+unfv3krLe/Xqhfv37zf6fhKJBBKJpN5yAwMDGBgYNBmrLpo6dSomT55MQwLxQHx8PN566y1MnDgRO3bswMOHD+Ho6IhvvvkGb731Fi/HtFPrnGyDNsRXpuiguHDhArfsyJEjTXZQ1NTUsI4dO9broPDw8GBRUVEqvzdfOyheJJQGbb6qrq5mzs7ObNKkSUwulyt9nnK5nE2aNIl17dqVVVdXazrUFqXO+akT92D06tUL48aNw7vvvovz58/j9OnTCA8Px1tvvYWOHTsCAB49eoSePXvi/PnzAGrn0Fy0aBG+++47xMXFISsrC0uWLMH169cxd+5cTR4OIS0uNTUVd+/exeeff17v1iqxWIyoqCjcuXNH0FMp6kQHBQDs2rUL4eHhGD16NMRiMSZPnozvvvuOWy+TyXDjxg2lBstPP/0Uz58/x/z581FUVIT+/fsjOTkZrq6umjgEQlqNYopEd3f3Btcrlgt5KkWdSXbW1tbYvXt3o+udnZ0bfBogMjISkZGRrRkaIRpXdyrFIUOG1FtPUynq0BMUhJDG1Z1KUSaTKY08LZPJaCpF6FDNjrSeusOyt2vXjnpjdZBiKsUpU6bA3Nwcz58/BwCsWbMGRkZGkEqliIuLE/TnSjU7gYuPj4eLi4vSsOwuLi40SrGOYowpPQEE1D4V1FATj9CIGP0WmlRSUgILCwsUFxfz7qZixbDsjdm3bx/v7sviK7lcDgcHB+Tn58PY2FhpZB/Fazs7Ozx+/JhXtTt1zk+q2QmUXC7HzJkzAQCGhoZYvHgxNm7ciMWLF8PQ0BAAMHPmTBqWXUfUHXl69OjRSoOxjh49GgCNPE3JTqCOHj2KyspKGBgYoLS0FF999RUcHBzw1VdfobS0FAYGBqisrMTRo0c1HSpRgWJE6SFDhjQ4GKuih1bII09TshOoNWvWAAA++ugjrianYGhoiA8//FCpHNFuDx48AFA77iNjTKk3ljGGkJAQpXJCRL2xAvXs2TMAgI+PT4Prvb29lcoR7ebk5AQAWLduHWJjY7mBHdasWYMuXbpwz3srygkR1ewESjE2YGRkJDdCjEJNTQ2++OILpXJEu40aNQoAcPPmTVRWVmLjxo3YsmULNm7ciMrKSty8eVOpnBBRb2wz+NobW1lZyY3PFxAQgMjISDx69AidOnXCypUrkZSUBKB2vDBjY2NNhkpUUFVVBWNjY9TU1DTaGysWi1FZWVmv2UKXqXN+0mWsQBkbGyMwMBAHDhxAUlISl9zqCgwMpESnI86cOVOvhq4gEokA1NbYz5w5g5EjR7ZhZNqDLmMFLCEhAYGBgQ2uCwwMREJCQtsGRF6a4gH/nTt31hvV287ODjt37lQqJ0SU7AQuISEBFRUVeP/99+Hh4YH3338fFRUVlOh0jOIBf1dXV2RnZyM5ORkRERFITk5GVlYWXFxclMoJEbXZNYOvbXYvkslkSEpKQkBAAC9HZOY7uVwONzc39O3bFwkJCZDL5dznqaenh6CgIFy5cgW3bt0S7BMU1GZHCA/UHQggMDAQY8aMwa1bt3Dv3j0kJyfj119/FfxAAJTsCOGJ4OBgLFy4EN9++y0SExO55fr6+li4cKHgn3OmZEcIT8THxyM2NhYTJkzganbdunVDcnIyYmNjMWTIEEEnPGqzawa12RFdQG12NOoJIYJAE+40j5IdITxAE+40j5IdITxQd8KdhtCEO5TsCOGFuhPuNDSwA024Q8mOEF5Q3GeXmJiIoKAgnDt3DpWVlTh37hyCgoKQmJiI2NhYXnVOqItuPSGEJ4KDgxEXF4cFCxZgxIgR3PKuXbsiLi5O0LedAJTsCOGV4OBgBAYG4uTJkzh06BDGjx9PU2P+H0p2hPCMnp4efH19UV5eDl9fX0p0/4fa7AghgkDJjqCyshIff/wxvvzyS3z88cdKo9wSwheU7AQuKCgIJiYm2LRpEy5duoRNmzbBxMQEQUFBmg6NkBZFyU7AgoKCcODAgQbXHThwgBIe4RVKdgJVWVnJJbqAgAClGeQDAgIA1CY8uqQlfEHJTqAiIiIA1A7j/csvvyjNIP/LL7/A1dVVqRwhuk5nkl1RURFmzJgBc3NzWFpaYu7cuSgrK2tym9zcXLz99tuwt7dHu3btMHDgQOzbt6+NItZuFy5cAACsXLmywRnkv/rqK6VyhOg6nbnPbsaMGcjJyUFycjJkMhnCwsLw3nvvYffu3Y1uM2vWLDx79gwHDx6EjY0Ndu/ejalTp+LChQsYMGBAG0avfSwtLQEA27dvx8KFC+vNIN+3b1+lcoToPKYDrl27xgCw9PR0btmhQ4eYSCRijx49anS7du3ase3btysts7a2Zps3b1b5vYuLixkAVlxcrH7gWiwpKYkBaPZfUlKSpkMlaqqurmbJycksIiKCJScns+rqak2H1GrUOT91omZ39uxZWFpaYtCgQdwyPz8/iMVipKWl4Y033mhwu6FDh+LHH3/EhAkTYGlpiZ9++gnPnz9vcpJgqVQKqVTKvS4pKQFQO5KvTCZrmQPSAnWfnQQALy8vBAQEICkpCenp6Url+HTcfLd//34sWrQI9+/fB1BbU+/cuTNWrVrV6Hmiy9T5bupEssvNza038a++vj6sra2Rm5vb6HY//fQTpk2bhvbt20NfXx8mJibYv38/3NzcGt0mOjoay5cvr7f86NGjMDExefmD0DIZGRlKr9PT05WSnEJMTAw8PT3bKizyCs6ePYuYmJh6y+/fv49p06bhs88+g4+PjwYiaz0VFRUql9VosouMjGzww6krMzPzpfe/ZMkSPHv2DMeOHYONjQ0SEhIwdepUpKamcm1SL4qKilLqgSwpKYGTkxP8/f15NQfFunXrAAATJ07En3/+iQcPHnDrOnfuDHd3dyQlJeHMmTNYsmSJpsIkKpLL5QgJCWmyzLp16/Dll1/y6llZxZWXKjSa7BYsWIDZs2c3WcbFxQX29vbIy8tTWl5dXY2ioiLY29s3uF12djbWr1+PK1euoE+fPgCA/v37IzU1FRs2bMCmTZsa3E4ikUAikdRbbmBgwKuJaIqLiwEAoaGheOONN+qNkhEXF4ekpCQUFxfz6rj56vjx40r3RM6cOROenp7IyMjAzp07AdTWglJSUjBu3DhNhdni1PputkEb4itTdFBcuHCBW3bkyJEmOyj++usvBoBdu3ZNabm/vz979913VX5vvnZQzJs3jwFgbm5uTC6Xs6qqKpaQkMCqqqqYXC5nrq6uDACbN2+epkMlKhg5ciTXqVRZWan0eVZWVnLrRo4cqelQW5Q656dOJDvGGBs3bhwbMGAAS0tLY7///jvr1q0bCwkJ4dY/fPiQ9ejRg6WlpTHGGKuqqmJubm5s+PDhLC0tjWVlZbHY2FgmEonYr7/+qvL78jXZVVRUcCfAhAkT2KlTp9iePXvYqVOn2IQJE7h1FRUVmg6VqMDc3JwBYCNGjGCMMaVkxxhjw4YNYwCYubm5JsNscbxMdoWFhSwkJISZmpoyc3NzFhYWxkpLS7n1d+7cYQDYyZMnuWU3b95kwcHBzM7OjpmYmLB+/frVuxWlOXxNdowxFhgY2ORtJ4GBgZoOkajIysqKAWBOTk4N1tQdHR0ZAGZlZaXpUFsUL5OdpvA52THWeMKjRKdbJk6cqFJNfeLEiZoOtUWpc36KGGNM7VZBAVFnxnFdVVlZiYiICJw7dw5DhgzBmjVrYGxsrOmwiBrKyspgZmbWbLnS0lKYmpq2QURtQ53zUyfusyOty9jYGN999x2SkpIQEBBAva86yNTUFF5eXg3eK6ng5eXFq0SnLp0ZCIAQ0rTz58/Dy8urwXVeXl44f/58G0ekXSjZEcIj58+fR2lpKSZNmoQuXbpg0qRJKC0tFXyiA+gylhDeMTU1xb59+6hZ4gVUsyOECAIlO0KIIFCyI4QIArXZNUNxG6I6oyvoIplMhoqKCpSUlFAbDw8I5fNUnJeq3C5Mya4ZpaWlAAAnJycNR0IIaUxpaSksLCyaLENPUDSjpqYGjx8/hpmZGUQikabDaTWKcfsePHjA2ydFhEQonydjDKWlpejYsSPE4qZb5ahm1wyxWAxHR0dNh9FmzM3NeX1yCI0QPs/manQK1EFBCBEESnaEEEGgZEcA1A5Hv2zZsgaHpCe6hz7P+qiDghAiCFSzI4QIAiU7QoggULIjhAgCJTsdcPfuXYhEIly6dKlN9nP69Gn07dsXBgYGCAoKeqX3JERbULLTQSkpKRCJRHj27Fmr7D8iIgIeHh64c+cOtm3b1irvQeobOXIkPv30U02HwdG2eF4VJbtWVlVVpekQ1JadnY1Ro0bB0dERlpaWL7WPtjxuxhiqq6vb7P20mS5+39pMq81xJlC+vr7sww8/ZJ988glr3749GzlyJLt8+TIbN24ca9euHbOzs2MzZ85k+fn53DaHDh1iw4YNYxYWFsza2ppNmDCBZWVlcesVc+L+8ccf3M91/4WGhqq1nz179jAfHx8mkUhYnz59WEpKitL6uv+2bt3KGGMsJSWFeXl5MUNDQ2Zvb88+++wzJpPJmjzukydPMgDs8OHDzMPDgxkZGbHXX3+dPXnyhCUlJbGePXsyMzMzFhISwsrLy7l9yeVy9vXXXzNnZ2dmZGTE+vXrx37++WduvWK/SUlJbODAgczAwEBpvmBdFBoaWu93n5WVxebMmcP9Hrp3787Wrl1bb7vAwED21VdfMQcHB+bs7MwYY+z06dOsf//+TCKRME9PT7Z//37uO6TQ1PeyoXju3LnTVr+OVkHJroX5+voyU1NTtmjRInb9+nV27tw5Zmtry6KiolhmZia7ePEiGzNmDHv99de5beLi4ti+ffvYrVu32B9//MEmTZrE+vbty+RyOWNMOdlVV1ezffv2MQDsxo0bLCcnhz179kyt/Tg6OrK4uDh27do19s477zAzMzNWUFDAqqurWU5ODjM3N2dr165lOTk5rKKigj18+JCZmJiwDz74gGVmZrL9+/czGxsbtmzZskaP+/r161xSGjJkCPv999/ZxYsXmZubG/P19WX+/v7s4sWL7NSpU6x9+/Zs5cqV3L6++uor1rNnT3b48GGWnZ3Ntm7dyiQSCZeUFfvt168fO3r0KMvKymKFhYWt/dG2qmfPnjEfHx/27rvvspycHJaTk8OeP3/Oli5dytLT09nt27fZzp07mYmJCfvxxx+57UJDQ5mpqSl7++232ZUrV9iVK1dYcXExs7a2ZjNnzmRXr15lSUlJrHv37krJ7unTp01+LxuKp7q6WhO/mhZDya6F+fr6sgEDBnCvV6xYwfz9/ZXKPHjwgEtWDcnPz2cA2OXLlxljysmOsf8/2Z8+fdpkLI3tp25ikclkzNHRkcXExHDLLCwsuBodY4x9/vnnrEePHqympoZbtmHDBmZqasol0hePu26cx44d45ZFR0czACw7O5tb9ve//52NHTuWMcbY8+fPmYmJCTtz5ozSvubOnctCQkKU9puQkNDk8esaX19f9sknnzRZ5sMPP2STJ0/mXoeGhrIOHTowqVTKLdu4cSNr3749q6ys5JZt3rxZ6TukyvdSlXh0CbXZtQJPT0/u5z///BMnT56Eqakp969nz54AatvGAODWrVsICQmBi4sLzM3N4ezsDAC4f/++Wu+r6n58fHy4n/X19TFo0CBkZmY2ut/MzEz4+PgoDXE1bNgwlJWV4eHDhw0ed139+vXjfu7QoQNMTEzg4uKitCwvLw8AkJWVhYqKCowZM0bpd7Z9+3bu96UwaNCgRmPmiw0bNsDT0xO2trYwNTXF999/X+/z7Nu3LwwNDbnXN27cQL9+/WBkZMQtGzx4sNI2qnwv+YaGeGoF7dq1434uKyvDpEmTEBMTU6+cg4MDAHDT3m3evBkdO3ZETU0N3N3d1W5sbqn9vKy6x11X3ZFyRSJRvZFzRSIRampqANT+vgDg119/RadOnZTKvficZ2Pvxxd79+7FwoULsXr1avj4+MDMzAyrVq1CWlqaUrmX+T2o8r3kG0p2rWzgwIHYt28fnJ2doa9f/9ddWFiIGzduYPPmzRg+fDgA4Pfff29yn4q/4nK5/KX2c+7cOYwYMQIAUF1djYyMDISHhzf6fr169cK+ffvAGONqd6dPn4aZmVmLj/XXu3dvSCQS3L9/H76+vi26b21naGio9JmePn0aQ4cOxQcffMAtU6XW1aNHD+zcuRNSqZT7A5Genq5UprnvZUPx6Dq6jG1lH374IYqKihASEoL09HRkZ2fjyJEjCAsLg1wuh5WVFdq3b4/vv/8eWVlZOHHiBCIiIprcZ5cuXSASiZCYmIj8/HyUlZWptZ8NGzZg//79uH79Oj788EM8ffoUc+bMafT9PvjgAzx48AAfffQRrl+/jgMHDmDZsmWIiIhodnRYdZmZmWHhwoWYP38+fvjhB2RnZ+PixYtYt24dfvjhhxZ9L23j7OyMtLQ03L17FwUFBejWrRsuXLiAI0eO4ObNm1iyZEm9pNWQ6dOno6amBu+99x4yMzNx5MgRxMbGAgD3x6q572VD8Shq37qKkl0r69ixI06fPg25XA5/f3/07dsXn376KSwtLSEWiyEWi7F3715kZGTA3d0d8+fPx6pVq5rcZ6dOnbB8+XJERkaiQ4cOCA8PV2s/K1euxMqVK9G/f3/8/vvvOHjwIGxsbJp8v6SkJJw/fx79+/fH+++/j7lz5+KLL754pd9NY1asWIElS5YgOjoavXr1wrhx4/Drr7+ia9eurfJ+2mLhwoXQ09ND7969YWtri7FjxyI4OBjTpk2Dt7c3CgsLlWp5jTE3N8cvv/yCS5cuwcPDA//1X/+FpUuXAgDXjtfc97KheNRtQ9Y2NMQTIQKwa9cuhIWFobi4GMbGxpoORyOozY4QHtq+fTtcXFzQqVMn/Pnnn/jss88wdepUwSY6gJIdIbyUm5uLpUuXIjc3Fw4ODnjzzTfxj3/8Q9NhaRRdxhJCBIE6KAghgkDJjhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7IjWmD17NkQiETcySocOHTBmzBhs2bJFrecyt23b9tLDyb+K2bNn0wRFWoySHdEq48aNQ05ODu7evYtDhw7h9ddfxyeffIKJEyfSPBPk1Wh06FBC6lDMp/Ci48ePMwBs8+bNjDHGVq9ezdzd3ZmJiQlzdHRk8+bNY6WlpYyx/x/FuO4/xfDx27dvZ56enszU1JR16NCBhYSEsCdPnnDvU1RUxKZPn85sbGyYkZERc3NzY1u2bOHW379/n7355pvMwsKCWVlZsb/97W/cvAzLli2r9766Pi8G31DNjmi9UaNGoX///oiPjwcAiMVifPfdd7h69Sp++OEHnDhxAosXLwYADB06FGvXroW5uTlycnKQk5ODhQsXAgBkMhlWrFiBP//8EwkJCbh79y5mz57Nvc+SJUtw7do1HDp0CJmZmdi4cSM3GoxMJsPYsWNhZmaG1NRUnD59Gqamphg3bhyqqqqwcOFCTJ06lauZ5uTkYOjQoW37iyJN03S2JUShsZodY4xNmzaN9erVq8F1P//8M2vfvj33euvWrczCwqLZ90tPT2cAuFrhpEmTWFhYWINld+zYUW8eDqlUyoyNjdmRI0eajZ9oHtXsiE5gdUZJPnbsGEaPHo1OnTrBzMwMb7/9NgoLC1FRUdHkPjIyMjBp0iR07twZZmZm3EjIinHa5s2bh71798LDwwOLFy/GmTNnuG3//PNPZGVlwczMjJuzwdraGs+fP+ftnA18Q8mO6ITMzEx07doVd+/excSJE9GvXz/s27cPGRkZ2LBhA4CmJ4guLy/H2LFjYW5ujl27diE9PR379+9X2m78+PG4d+8e5s+fj8ePH2P06NHcJXBZWRk8PT1x6dIlpX83b97E9OnTW/noSUugIZ6I1jtx4gQuX76M+fPnIyMjAzU1NVi9ejU3ou5PP/2kVL6huROuX7+OwsJCrFy5Ek5OTgCACxcu1HsvW1tbhIaGIjQ0FMOHD8eiRYsQGxuLgQMH4scff4SdnR3Mzc0bjJNvczbwDdXsiFaRSqXIzc3Fo0ePcPHiRXz99dcIDAzExIkTMWvWLLi5uUEmk2HdunW4ffs2duzYgU2bNintw9nZGWVlZTh+/DgKCgpQUVGBzp07w9DQkNvu4MGDWLFihdJ2S5cuxYEDB5CVlYWrV68iMTERvXr1AgDMmDEDNjY2CAwMRGpqKu7cuYOUlBR8/PHH3HSSzs7O+Ouvv3Djxg0UFBRAJpO1zS+NqEbTjYaEKISGhnK3bejr6zNbW1vm5+fHtmzZwk3GzRhja9asYQ4ODszY2JiNHTuWbd++vd6k4e+//z5r37690q0nu3fvZs7OzkwikTAfHx928ODBehNH9+rVixkbGzNra2sWGBjIbt++ze0zJyeHzZo1i9nY2DCJRMJcXFzYu+++y4qLixljjOXl5bExY8YwU1NTuvVEC9HgnYQQQaDLWEKIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCD8L6jSNtuByeREAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:41.885904Z", + "iopub.status.busy": "2024-03-22T19:05:41.885619Z", + "iopub.status.idle": "2024-03-22T19:05:42.161890Z", + "shell.execute_reply": "2024-03-22T19:05:42.160988Z" + }, + "papermill": { + "duration": 0.298816, + "end_time": "2024-03-22T19:05:42.164011", + "exception": false, + "start_time": "2024-03-22T19:05:41.865195", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXFUlEQVR4nO2dd3xT5f7H3yerezFKKZQpe2+BqwxBQAUc94LIVUHc4EJkqBfcoKLyExVwgIvhQIWriBeRIVO27FGKIKstpbvNPL8/TpMmadImadqk7fN+vfJqcnLGkzT55Pt81yPJsiwjEAgEAUQV6AEIBAKBECKBQBBwhBAJBIKAI4RIIBAEHCFEAoEg4AghEggEAUcIkUAgCDhCiAQCQcDRBHoA5cFisXDhwgWioqKQJCnQwxEIBE7IskxOTg6JiYmoVO7tniotRBcuXCApKSnQwxAIBGVw7tw5GjZs6Pb5Ki1EUVFRgPIio6OjAzwagUDgTHZ2NklJSbbvqjuqtBBZp2PR0dFCiASCIKYs14lwVgsEgoAjhEggEAQcIUQCgSDgVGkfUVVBlmVMJhNmsznQQxEI/IparUaj0ZQ7fUYIUQVjMBi4ePEi+fn5gR6KQFAhhIeHU79+fXQ6nc/nEEJUgVgsFlJSUlCr1SQmJqLT6UTipaDaIMsyBoOBtLQ0UlJSaNGiRalJi6VRI4Row/FU5v5ynDb1o5n7r06Vdl2DwYDFYiEpKYnw8PBKu65AUFmEhYWh1Wr566+/MBgMhIaG+nSeGiFEJrPM4QvZqFWBsUZ8/ZUQCKoC/vh814hvSHxUCACp2foAj0QgELiiZghRtCJE6bl6LBaxaIlAEGzUCCGqExmCJIHJInM13xDo4dRYzpw5gyRJ7N+/v1LOs3XrVjp06IBWq+XWW28t1zVrKhs3bkSSJDIzMyv0OjVCiLRqFbXCldBiao6YngULFf0hnzx5Mp07dyYlJYVPP/20Qq4h8A81QogA6lr9REKIfMJgqHqWZHJyMgMHDqRhw4bExsb6dI7KfN3WxFd/UxX+dzVPiLILAzoOWZbJN5gCcvNmUd/+/fszadIknnzySerUqcOQIUM4dOgQw4YNIzIyknr16nH33XeTnp5uO2bt2rX84x//IDY2ltq1a3PLLbeQnJzs8vxnzpxhwIABAMTFxSFJEuPGjfPqPMeOHaNPnz6EhobSvn17Nm3aZDu3JElcuXKF++67D0mSbBbRpk2b6NmzJyEhIdSvX5/p06c7fPldvW6r5fbLL7/QpUsXwsLCGDhwIKmpqfz888+0adOG6Oho7rrrLofEVYvFwuzZs2natClhYWF06tSJb7/91va89bw///wz3bp1IyQkhC1btpT5v3nllVeIj48nKiqK+++/n+nTp9O5c2fb8+PGjePWW2/l1VdfJTExkVatWgHwxRdf0L17d6KiokhISOCuu+4iNTXV4dxr1qyhZcuWhIWFMWDAAM6cOVPmePxBjQjfA8RHKfkNgbaICoxm2s78JSDXPvLSEMJ1nv/LP/vsMx555BG2bt1KZmYmAwcO5P777+edd96hoKCAadOmMWrUKH777TcA8vLymDx5Mh07diQ3N5eZM2dy2223sX///hIh3qSkJFauXMkdd9zB8ePHiY6OJiwszKvzPPPMM8ybN4+2bdvy9ttvM3z4cFJSUkhKSuLixYu0atWKl156idGjRxMTE8P58+e56aabGDduHJ9//jnHjh3jgQceIDQ0lBdeeMHl6wa4ePEiAC+88ALvvfce4eHhjBo1ilGjRhESEsKyZcvIzc3ltttuY/78+UybNg2A2bNn8+WXX7Jw4UJatGjB5s2b+fe//03dunXp16+f7XrTp09n7ty5NGvWjLi4uFL/J0uXLuXVV1/lgw8+oG/fvqxYsYK33nqLpk2bOuy3fv16oqOjWbdunW2b0Wjk5ZdfplWrVqSmpjJ58mTGjRvHmjVrAKWv1+23387EiRN58MEH2b17N08//XTZHxQ/UHOEqChyliamZh7TokUL3njjDUD5Fe7SpQuvvfaa7fnFixeTlJTEiRMnaNmyJXfccYfD8YsXL6Zu3bocOXKE9u3bOzynVqupVasWAPHx8Q5TJ0/PM2nSJNu+CxYsYO3atXzyySdMnTqVhIQEJEkiJiaGhIQEAD744AOSkpJ47733kCSJ1q1bc+HCBaZNm8bMmTNtImf/uqFYiF555RX69u0LwIQJE5gxYwbJyck0a9YMgH/+859s2LCBadOmodfree211/j111/p3bs3AM2aNWPLli0sWrTIQYheeuklBg8eXPY/BJg/fz4TJkxg/PjxAMycOZP//e9/5ObmOuwXERHBxx9/7FB2cd9999nuN2vWjHfffZcePXqQm5tLZGQkCxYsoHnz5rz11lsAtGrVioMHD/L66697NLbyUHOEKCo4hChMq+bIS0MCdm1v6Natm+3+gQMH2LBhA5GRkSX2S05OpmXLlpw8eZKZM2eyc+dO0tPTsVgsAJw9e7aEEJWGp+exfsEBNBoN3bt35+jRo27Pe/ToUXr37u1QZtO3b19yc3P5+++/adSoUYnXbU/Hjh1t9+vVq0d4eLhNhKzb/vjjDwBOnTpFfn5+CYExGAx06dLFYVv37t1t9y0WmcwCIzFhGtQuEgWPHz/Oo48+6rCtZ8+eNqvUSocOHUrUfu3Zs4cXXniBAwcOcPXqVYf3tW3bthw9epRevXo5HGP/HlckNUaIip3VgfURSZLk1fQokERERNju5+bmMnz4cJe/jvXr1wdg+PDhNG7cmI8++ojExEQsFgvt27f32lnqr/P4iv3rtker1druS5Lk8Ni6zfrltlooP/30Ew0aNHDYLyQkxO310vP0XMoqJE+vI6mW72VBzq8hLy+PIUOGMGTIEJYuXUrdunU5e/YsQ4YMCQpndtX4RviBYPERVVW6du3KypUradKkCRpNyY/NlStXOH78OB999BHXXXcdQJmOV+svtn17FG/Os2PHDq6//noATCYTe/bsYdKkSW6v16ZNG1auXIksyzaraOvWrURFRZXa2N0X2rZtS0hICGfPnnWYhpVFvl55L7IKjDSwyKicypJatWrFrl27uOeee2zbdu3aVeZ5jx07xpUrV5gzZ45twYndu3c77NOmTRtWr17tsG3Hjh0ej7081JiomX2ZhzfRI4HCxIkTycjIYMyYMezatYvk5GR++eUXxo8fj9lsJi4ujtq1a/Phhx9y6tQpfvvtNyZPnlzqORs3bowkSfz444+kpaWRm5vr1Xnef/99vv/+e44dO8bEiRO5evWqgx/EmUcffZRz587x2GOPcezYMVatWsWsWbOYPHmy3+sBo6KimDJlCk899RSfffYZycnJ7N27l/nz5/PZZ5+5Pa7AqAiRRZbJLjSWeP6xxx7jk08+4bPPPuPkyZO88sor/Pnnn2V2dWjUqBE6nY758+dz+vRpVq9ezcsvv+ywz8MPP8zJkyd55plnOH78OMuWLau0/KuaI0RFzuoCo5lcvf9zNao7iYmJbN26FbPZzI033kiHDh148skniY2NRaVSoVKpWLFiBXv27KF9+/Y89dRTvPnmm6Wes0GDBrz44otMnz6devXqMWnSJK/OM2fOHObMmUOnTp3YsmULq1evpk6dOqVeb82aNfzxxx906tSJhx9+mAkTJvD888+X671xx8svv8x//vMfZs+eTZs2bRg6dCg//fRTiQiXFZPZgtFssT3OzC8pRGPHjmXGjBlMmTKFrl27kpKSwrhx48qseq9bty6ffvop33zzDW3btmXOnDnMnTvXYZ9GjRqxcuVKfvjhBzp16sTChQsdghMViSRXYfMgOzubmJgYsrKyPFrFo/2sX8jVm/jt6X40q1vS6epvCgsLSUlJoWnTpj63RxDUHHIKjaSk56GWJMxF08c2CVFo1KXbC4MHDyYhIYEvvviikkbqSGmfc0+/ozXGRwTK9CxXbyI1R18pQiQQeIN1WhYVqqHQZKHQaCa70EitiGLndn5+PgsXLmTIkCGo1WqWL1/Or7/+6pAvVBWpUUJUJyqE0+l5wmEtCEoKDGZuu6E3l86fA0CWQQKs7p9FixZx++23s2bNGl599VUKCwtp1aoVK1euZNCgQYEbuB+oUUIUHyRlHgKBKwqNZt7/7CsSorTo1CpOp+UB0Dw+Eq1aRb169QgLC+PXX38N8Ej9Tw0TImX+GuikRoHAGbPFgt5kIbFhI9rWj0ajVhFSK5c8g4k6MWG2PLjqSo2JmoEo8xAELwVGJVqmU6tszunYcCVhMrMg8AmHFU3QCNGcOXOQJIknn3yywq4RL1qBCIKUAoPiqA61K8OJCdMiIVFgMKM3Vu818YJCiHbt2sWiRYscankqgmAp8xAInCksEpowXbEQadQqIkMV70lmQcmcoupEwIUoNzeXsWPH8tFHH5XZAqG8iDIPQbBiDd07Fybbpmf5xmpdERBwIZo4cSI333xzpYQfrVOzzHwjelP1NnUFVQeLRUZf5CNyFqLoUC0qSUJvMtvEqjoSUCFasWIFe/fuZfbs2R7tr9fryc7Odrh5Q2y4EhYFSM+t/g7AQNKkSRPmzZtneyxJEj/88EPAxhPMFJrMyMhoVCo0aseaMbVKIto6PXNR8lFdCJgQnTt3jieeeIKlS5d6XP4we/ZsYmJibDdrFbGnSJIUNC1jaxoXL15k2LBhgR5GUGJ1VIfp1C6LV2OLFn7IKqi+07OACdGePXtITU2la9euaDQaNBoNmzZt4t1330Wj0Ti0hrAyY8YMsrKybLdz5855fV3RRD8wJCQklOjDU9lUVHP68lLsH3L9dYwM1aBWSRjNFvKqacF2wITohhtu4ODBg+zfv9926969O2PHjmX//v2o1SW7CYaEhBAdHe1w85aAC5EsgyEvMDcvf01zcnIYO3YsERER1K9fn3feeYf+/fv7lGJhPzWzNrf/7rvvGDBgAOHh4XTq1Int27c7HLNlyxauu+46wsLCSEpK4vHHHycvL8/2fFnN4H1tTl/ZuHNUW1FJEjFh1pyi6jk9C1hmdVRUVIn2oREREdSuXdurtqLeYmsZG6ipmTEfXksMzLWfvQA6190HXTF58mS2bt3K6tWrqVevHjNnzmTv3r0OK0aUh+eee465c+fSokULnnvuOcaMGcOpU6fQaDQkJyczdOhQXnnlFRYvXkxaWhqTJk1i0qRJLFmyBCi7GbwVb5rTVzYWWaawyFEdqnPfyjc2TEtGnoGsAiOJsTKqMvoPVTVqVIkHiBC+p+Tk5PDZZ5+xbNkybrjhBgCWLFlCYqL/RHTKlCncfPPNALz44ou0a9eOU6dO0bp1a2bPns3YsWNt1leLFi1499136devHwsWLCA0NLTMZvBWvGlOX9nojRZkWUatkmyBFFdEhGjQqlUYzRZyCk02C6m6EFRCtHHjxgq/RsDLPLThimUSqGt7yOnTpzEajfTs2dO2LSYmxrZGlj+wT2C19r1OTU2ldevWHDhwgD///JOlS5fa9pFlGYvFQkpKCm3atCmzGbwV++b0wYZ1Whaqde2otiIVTc/Sc/Vk5huEEFV1Al7mIUleTY+qM87N6AGH5vMPPfQQjz/+eInjGjVq5FUzeHfN8IOBwjL8Q/bEhitClFNowmxRrKjqQo0TIlHm4RnNmjVDq9Wya9cu2zI7WVlZnDhxwtawviLp2rUrR44c4ZprrnH5/MGDB8tsBl8VsA/dl0WYVk2IRo3eZCa7wEhchK7MY6oKAc+srmysPqL0XANmS/XMyfAHUVFR3HvvvTzzzDNs2LCBw4cPM2HCBFQqVZmN2v3BtGnT2LZtG5MmTWL//v2cPHmSVatW2Vbp8KQZfLAjy3KZETN7JEmyq8ivXtGzGidEdSJ1SBKYLTIZeSK7ujTefvttevfuzS233MKgQYPo27cvbdq0qZT+2x07dmTTpk2cOHGC6667ji5dujBz5kybs9yTZvDBjt5kwSIrEbAQjWdfxdgi31Buocmh0X5Vp0Y1z7fS/ZV1pOcaWPP4dbRN9D4XyVOqW/P8vLw8GjRowFtvvcWECRMCPZwqT2a+gbMZ+YTrNFwT73kP9ZOpORQYzCTGhlEnMvAN0/zRPL/GWUQAdW0hfOEnKo19+/axfPly25pcY8eOBWDkyJEBHln1wJtpmT2xYYpvqDrVntVIIQp45KwKMXfuXDp16sSgQYPIy8vj999/5+jRo0RGRrq9CTyj2FHt3dfQ6ifKN5gwVJMuEjUuagbFkTPRMrZ0unTpwp49e0psLygoYP/+/ZU/oGqEt45qe7RqFZEhGnL1JjILjMRHeXd8MFIjhUis5lE+wsLC3IbVBZ5hNFswW5RFFEO8FCJQrKJcvYnMfKMtElyVqdFTs7TcyrGIqnA8QFBBWJvlh2pUPtWNRYdqkSSJQmPgG6b54/NdM4UoushZnV2xQmTNHM7Pz6/Q6wiqHjb/kA/WECj9rKNClAlNVn5g01Csn2/7THlvqdlTswr2EanVamJjY22tKcLDwyslGVBQuVhkGWRQeVFykZefj2wyoZZVFBb65iIIV5vJMhnIyDIRo6PSP1uyLJOfn09qaiqxsbEuW/d4So0UIvsyD1mWK/QfmJCQoFzLrk+OoHpxObsQk0UmMSbU48/SxaxCJbM/KoRMD5MZnZFlmfSsQiwyGLNC0Pl4nvISGxtr+5z7So0UIqtzr9BoIUdvIjq04iqZJUmifv36xMfHYzRWn7wPQTH3v7URgHl3dqZDg9gy97+Sp+f+77aDBD9O+gdhOt+/hl//fJR1Ry4zsnMij9/Q3Ofz+IpWqy2XJWSlRgpRmE5NVIiGHL2J1Gx9hQqRFbVa7Zd/mCC4kGWZ8zmKvydTL3mUQX/ir2zO55hpXjeCuOjy5V0NaNeQT3de4Ku9l3lqaHu0pfQ0Cmaq5qj9QN1A9yUSVAvs66aveug0PnJBWX2mfYOYcl+/b/Pa1I7QcSXPwNZT6eU+X6CosUIUL9qBCPyAfeg600MhOnQ+C4D2ieUXIo1axS0dlaZyq/cHqOGeH6jBQqSY0MIiEpQHe4vI09qvQxcUIWrnp4LrEZ0bAPDL4Uu2tICqRo0VooCv5iGoFljsLKKrHghRVr6RcxkFALTzg0UE0LVRLA3jwsgzmFl/7LJfzlnZ1FghEmUeAn9gn1ScVVD21OzwRcUaSqoVRky4f4IkkiQxsrPSp2lVFZ2e1Vwhiq7cMg9B9cTeIsryoGvi4fOKo7pdff9YQ1ZGFk3PNh5PJasKtgepuUIUVTllHoLqjddCVOQfat/Avw35WtaLonVCFEazzM+HLvr13JVBDRYi4SMSlB97Z3V2QdnLQR8qCt2380Po3pkRVXh6VmOFyOqsziow2pZ0EQi8RfbCIso3mDidlgv4L2Jmz/COihDtSLnCpayq5fussUIUE6a11eaIEL7AVxwsokIjllJWhjl6MQeLrFjjFdFDKKlWON0bxyHL8OOfVcsqKpcQ5ebmkp2d7XCrKkiSRN1I4bAWlA97H5EsQ06h++lZsX/I/9MyK1U1eua1EKWkpHDzzTcTERFBTEwMcXFxxMXFERsbS1xcXEWMscKwRs6Ew1rgKxanpmCllXnYImYVuHLMTR3qo1ZJHDyfRXLRNLAq4HXR67///W9kWWbx4sXUq1evSvfXsXVqFGUeAh9xbk54Nd9AE1wvcV2cUV1xFlHtyBCua1GHjcfTWL3/Ak8Nbllh1/InXgvRgQMH2LNnD61ataqI8VQqthC+8BEJfMTZInJX5mEwWThxOQfwf+jemZGdExUhOnCBJwe1qBLGgtdTsx49enDu3LmKGEulYyvzEFMzgY84W0SZbrKrT1zOwWiWiQnT0iA2rELHNLhtAqFaFSnpeRwsKrANdry2iD7++GMefvhhzp8/T/v27Uv0qe3YsaPfBlfRiAp8QXkp4SPKc20R2ScyVrSFEhmiYVCbevz450VW7b9Ax4axFXo9f+C1EKWlpZGcnMz48eNt2yRJsrVcNZurTk6OKPMQlJcSFpEbZ/WhIke1P1p/eMLIzg348c+L/PfABZ69qQ1qL/ppBwKvhei+++6jS5cuLF++vBo4q0WZh6B8lIyalW4Rta3AiJk9/VrWJSZMS2qOnp2nr9DnmjqVcl1f8VqI/vrrL1avXl0tFtizTs3Sc/WYLXLQ/2oIgg/n/EVX4XuzRebIRf91ZfQEnUbFTR0SWP7HOVbtvxD0QuS1s3rgwIEcOHCgIsZS6dSK0CFJyofpSp6wigTe40nU7HRaLoVGC+E6NU1ruw7tVwQjOikV+WsOXURvCm6XidcW0fDhw3nqqac4ePAgHTp0KOGsHjFihN8GV9Fo1CpqR4SQnqsnNVtfLZbuFVQM6bl6fvrzIrd1beCw2ILzKqeuLKLDRYWubetHe7X2WXnp1bQWkSEacgpNnMvI55r4qEq7trd4LUQPP/wwAC+99FKJ57x1Vi9YsIAFCxZw5swZANq1a8fMmTMZNmyYt8PymfgoRYiEw1pQGpO/PsDmE2lsOpHG4nE9bNudp2auLCJbj+pKmpZZUakkYsK05OpN5OmD2yLyempmsVjc3ryNmDVs2JA5c+awZ88edu/ezcCBAxk5ciSHDx/2dlg+Y4ucCYe1oBQ2n0gD4LdjjgtllpyalbSIDlWyo9qecJ2yhFWeoewWJYHEKyEyGo1oNBoOHTrkl4sPHz6cm266iRYtWtCyZUteffVVIiMj2bFjh1/O7wkil0jgCXUidS63WyzK35CiTg55BjMGk8X2vCzLtqlZZYXu7QkPUSY91coi0mq1NGrUqEJyhcxmMytWrCAvL4/evXu73Eev1/u92l+UeQg8oVGtcNt9e7+Q1SKKDtNidf/YW0XnMgrIKTShU6toUa98iyn6QkSRRZRfnSwigOeee45nn32WjIwMvwzg4MGDREZGEhISwsMPP8z3339P27ZtXe47e/ZsYmJibLekpKRyX1+UeQg8IdGuLMP+R8uqSRqVRGy4YjXZ5xJZ84daJUQFZBXWiCpiEXntrH7vvfc4deoUiYmJNG7cmIgIx3Dk3r17vTpfq1at2L9/P1lZWXz77bfce++9bNq0yaUYzZgxg8mTJ9seZ2dnl1uMbBX4wlktKAWNXbTrdFoe9aIVS1pGUSKVJBEbpiUjz+AQOTtUQT2qPaWqWEReC9Gtt97q1wHodDpbcmS3bt3YtWsX//d//8eiRYtK7BsSEkJISIhfr2/rSSR8RIJSMNv5pM9cyaN389pAcdRMkiC2aHkg+6nZIVsPosr3D0EF+YhMepBUoPbPckjggxDNmjXLbxd3hcViQa+vPOvEvszDWi8nEDhj3wI2JT2veLtcbBHFFU3NrCF8xVHt31VdvSXC31EzWYbVj0HORRj1BYTF+uW0XguRlT179nD06FFAyf/p0qWL1+eYMWMGw4YNo1GjRuTk5LBs2TI2btzIL7/84uuwvMbqI9KbLGQXmogJ85/KC6oPZjshOp1WLESyTYgo4SNKzdGTnmtArZJoUz9AQmSziPwkRJtehz+/AkkNlw5C0+v8clqvhSg1NZU777yTjRs3EhsbC0BmZiYDBgxgxYoV1K1b16tz3XPPPVy8eJGYmBg6duzIL7/8wuDBg70dls+EatVEhSrZp2k5hUKIBC4x20XK/rpibxEpfxWLyHFqZk1kvKZuJKFadSWN1JEInfIVzzf4YWp24CvYOFu5f8vbfhMh8EGIHnvsMXJycjh8+DBt2rQB4MiRI9x77708/vjjLF++3ONzffLJJ95evkKIjwohp9BEarY+qNPgBZVPVoGRrafSKbD7Iuvt8oSsUzZJgrgIq0WkCJE1fyhQ0zKA8JCiqVl5LaK/tsHqScr9vk9At3HlO58TXgvR2rVr+fXXX20iBNC2bVvef/99brzxRr8OrrKIjwolOS1PRM4EJViwMZmFm5Idtlkc8oiUvypJsjmrrVMzq0VUEYspeorVIiqXj+hKMqy4C8wGaDMCbnjBP4Ozw6cSD+dCV1CSHS0Wi4sjgh+xmofAHRcyC0pss6/qkGX78L3VWe1oEbUPoEVU7jyi/AxY+k8ouAoNusFti0Dl/3won9qAPPHEE1y4ULxu0vnz53nqqae44YYb/Dq4ykKUeQjcketiSuPKIpIkbD6iq/lGruYZOF8kYoGoMbNSrjwikx5WjIWM0xDTCMasAF142cf5gNdC9N5775GdnU2TJk1o3rw5zZs3p2nTpmRnZzN//vyKGGOFI8o8BO7IdbFgor1FZBUlSSrOrM7MN9isoSa1w4kKDVwAxOc8IlmGVZPg7DYIiYaxX0NkfAWMUMFrH1FSUhJ79+7l119/5dixYwC0adOGQYMG+X1wlYUo8xC4o2yLqDh8HxdhjZoZK2UNM0/wOY9o4xw4+DWoNDDqc4hvU/Yx5cCnPCJJkhg8eHClhtkrElHmIXCHayEqvi87hO8Vi8hkkdl5+goA7QJU2mHF6iPK98YiOvAVbJqj3L/5bWg+oAJG5ohPQrR+/XrWr19PampqCQf14sWL/TKwyqTYWS18RAJHXIW97avvi2vNlJy0UK2KQqOFnSlKUXggWn/YY42aGcwWDCYLOk0Z3pgzW2HVROV+3yeh270VO8AivPYRvfjii9x4442sX7+e9PR0rl696nCritQt8hFlF5ooNAZ3lbKgcskpa2pW9DtsLQ2yRs6sCYSBzCECCNMVJ1IWlJXUmH4KvhoLFiO0HQk3VGw5lz1eW0QLFy7k008/5e67766I8QSE6FANIRoVepOFtBw9SbUqJjIgqFoYTBaHJmdWLC6c1dbi/NhwLZeKLOv6MaHUjvRvkba36DQqdGoVBrOFPIOJmHA3jvO8K7DsX0Vh+u4VFqZ3h9dXMhgM9OnTpyLGEjAkSSp2WIsQvqAId9nIspuERsDmJ4LAO6qtRJSVXW3SK5ZQxmmIbQRjloO2YpfFdsZrIbr//vtZtmxZRYwloNgc1iKELyjClaMa3Cc0QnHkDAI/LbMSbsuudjE1k2XFJ3R2O4TEwF3fVGiY3h1eT80KCwv58MMP+fXXX+nYsWOJLOu3337bb4OrTEQukcAZd0LkLqERiivwofJX7XCH1SLKd/V6Ns6Gg98Uhek/g/jWlTw6Ba+F6M8//6Rz584AJZroV+VePqLMQ+CMeyGyv+9kEdn5YALVldEZtxbR/uVKWw+AW96plDC9O7wWog0bNlTEOAKOKPMQOOOZRVQkREVODmvUrFaEjoTo4FiwM9JVT6IzW5QGZwD/eAq63hOAkRVT+d28gxQxNRM446q8A8B+JTOrJkkoFpG1yX7npNigmSGUWNss/aRSQ2YxQttbYeDMwA2uCJ87NFY3RJmHwBn3zuqSFpFVcwa1jefV29pz3TWeNwisaByyq/OuwNJ/QWFmUZh+oVdh+iu5epb/cZY/zlzls/E9/Ca2QoiKqCvKPAROuAt3O/qIlL9WH1GIRs3YXo0remheYbWICgvzYMUjcDWlKEy/wuMw/ZEL2SzZmsKqAxdsuVXbk6/Q55o6fhmjEKIirM7qK7l6zBYZtSo4zGpB4MhxMzVzV/QarCg+IpmBx16CjB12YfrSrTazRWbdkUss2XrGVrIC0KlhDOP7NqV7k1p+G6PXQrR582b69OmDRuN4qMlkYtu2bVx//fV+G1xlUjsiBJWk/MJdydUTHySORkHgcJ/QSPGKL04WUTASrtPwlOZb2mX8TwnTj/681DB9Vr6Rr3af5bNtf9l6KqlVEsPaJzC+b1O6NvK//8trIRowYAAXL14kPt4x6SkrK4sBAwZUyHLUfmH3YrhmMMS6XpBRrZKoExlCao6e1BwhRDWZU6m5PP/DQS5muY+gyrLiF7LvRxSsdMpYQ3/N98qDW+ZBs/4u9zuVmsun21JYuec8BUU1l3HhWu7q1Yh/X9uY+jEVl23ttRC5W/vrypUrJVZ9DRq2vgvr/gPx7eC+tRDqOr+jbpRViAqB4EhGc0VWgRGT2RLwOqbqyv2f7eLMlfxS97FOzop9RBU7Jp9J+Z3rjr4EwJrYMdzU1bFG1GKR2XQijcVbU/j9ZLpte+uEKMb3bcLIzg0qZQUSj4Xo9ttvBxTlHzdunMOKq2azmT///DN4a9Da3Qbb34PUw/DNOLjra1CXfOnxUSEcJvjLPDq9+D8Ajrw0xJasJvAfZYkQKJaQGqlEQmNQkX4Svvo3atnEj+ZerIi4h5uKnsrVm1i5528+23aG00ULRkoSDG5Tj/F9m3Jts1qVauV5/CmOiVEsBFmWiYqKIiys2EzT6XRce+21PPDAA/4foT+ITYK7voIlN0Hyevj5GaXhk9Mbbb/qa7Biv+Lo+asFtKgnlj8KBFYBkp0SGoOGvHSl6X1hJpm1O/P0+UdobVDG+v6GUyzcmGxrcRIVqmF09yTu7dMkYJ0nPBaiJUuWANCkSROmTJkSvNMwdyR2gTs+VhK5di+GWs2hzySHXWxlHkFsEZnshEgVtPOB6o81cFZcaxZE/wtjobL8z9UzENuYkwMWof8ymXy9iat5Bt785TgAzepEMK5vE+7o2tCWaxQovNbxWbNmVT0RstL6ZhjyqnL/f8/D0R8dnq4KZR72Sx+rg+nDX8OwWkRBNzWzWGDVo3BupxKmH/sNupgEQGnWduSi0tQ/qVYYv07uxz29mwRchMAHIbp8+TJ33303iYmJaDQa1Gq1wy3oufZR6D4BkGHl/XB+r+2pulWgzMNk15pX5DoFDouTRRQ0/4qNr8GhlUVh+i+gbqvifkQGE0dsa63FBJVF7bUUjhs3jrNnz/Kf//yH+vXrB5dJ6gmSBMPegMy/4NSvsPxOuH89xCYVZ1cHsRCZxdQsKJCdfERB8Z/Yvww2v6ncH/5/0KwfYL/IoslmEbWtHxydAax4LURbtmzh999/t7UCqZKoNfDPJbBkGFw+BMtGwX2/2E3N9G7TFAKNvY8o+EZXcyi2iIJkapbyO6x+XLl/3dPQ5d+2p6yRVaNZZv+5TCDwq4s44/XULCkpyaHor8oSGq1E0iITIPUIfHMvdcOVt8NgspBdUI61wisQe4uoGvwXggJfPs+yzUekPA7oj1baieKm9+1uhwHPOzwdbtdAP6UoVN+2fnDlyXktRPPmzWP69OmcOXOmAoZTycQ0hLtWgDYckn8jdN10YkKVX49gdVjbW0TV4gchwJy8nEPP19bzxfYzAC6b5buipEVUEaPzgLx0pel9YRY07Am3flAil0CrVjksI1QrQke96OBKhvVaiEaPHs3GjRtp3rw5UVFR1KpVy+FW5bCG9ZFgzxIeDfkZCF6HtdlsL0QBHEg14dnvD5KWo+c/qw4DkFNo9Oi44jwi5XFApmb2Yfq4JqU2vY+0i4y1rR8ddG4Hr31E8+bNq4BhBJjWN8OQ1+CXGTyg/5S9qlhSczoFelQuMQv18Sv2Fia4r7h3JuAJjRYL/PCIEqYPLaqmj3DfkiNcpyZDmZUFTVN/e7wWonvvrZyVHyudax+BjGRUuz5mnvZ91p7vCl0aBnpUJTDbhe8tQpTKjfNb6KkQYZuaKX8r3cLY8Coc/q4oTP8l1G1Z6u4RdqVAbYNQiHzS8eTkZJ5//nnGjBlDamoqAD///DOHDx/26+AqFUmCoa9zKqY3YZKBwfufgMyzgR5VCRx9RAEcSBVlz18Z/HPBNg7+neXy+ewypmaaImdQQH1E+5bC73OV+8PfhaZlt94JDyl2WAdb6B58EKJNmzbRoUMHdu7cyXfffUdubi4ABw4cYNasyluitkJQa9ja+U2OWhoRacqApaMUJ2AQYTKLqFl5uGPBdnb/dZUxH+1w+XxpPiKVVJy7ZXGKmlWajyhlM/zXGqafAl3GenSY1UcUolHRtE7wVUZ4LUTTp0/nlVdeYd26deh0xWs4DRw4kB07XP9z3TF79mx69OhBVFQU8fHx3HrrrRw/ftzbIfmV2LhajDc8Q4aqFqQdha/vBbNnDszKwCyiZn7BXT/q7FKmZmqVZMvdKuEjqgwhSjsBX/0bLKaiMP1zHh9qDeG3rh+NRh1sFbo+CNHBgwe57bbbSmyPj48nPT3dxRHu2bRpExMnTmTHjh2sW7cOo9HIjTfeSF5enrfD8hvxUaFcojYzQp9XwvqnN8CaKUEzDzKJPKIKpTQfkUqSbIIjO03NKlyHbNX0WZDUC25d4JWH3OojCsZpGfggRLGxsVy8eLHE9n379tGgQQOvzrV27VrGjRtHu3bt6NSpE59++ilnz55lz5493g7Lb1jLPLblN4Q7PkEJ638K2+YHbEz2mIWPqEJxnppp7Jw/apVk8wVV6tTMWAjLxyhlSXFN4M5loPWug2ivZrXQqVXc2K5exYyxnHgtRHfeeSfTpk3j0qVLSJKExWJh69atTJkyhXvuKd8ibVlZij/GXT6SXq8nOzvb4eZvrK1AcgpNFDYfAkNnK0+smwlHVvv9et5iX/Qqpmb+xzmjPiq0ONpUqkVUUQOyhun//sOjML07RvdoxOGXhjCgVeWva+8JXgvRa6+9RuvWrUlKSiI3N5e2bdty/fXX06dPH55//vmyT+AGi8XCk08+Sd++fWnfvr3LfWbPnk1MTIztlpTkuv90eYgK0RCqVd6W1Gw99HoYejwAyPDdg/B34Kw1ECUe/sb5PSw0OfZcjwotXkJaJRVPwUokNFZU2GzDK0Vhei2MXlpmmL40tEHoG7Li9ch0Oh0fffQRycnJ/Pjjj3z55ZccO3aML774olxtQCZOnMihQ4dYsWKF231mzJhBVlaW7Xbu3Dmfr+cOSZLsVn0tLArrz4EWN4KpAJaPhqt/+f26niKmZhWLfeY6OFpEapVkFzVTtlk7ZlbIzGzfl/D7W8r9Ee9C0+sq4CLBgc8dkRo1akSjRo38MohJkybx448/snnzZho2dJ9EGBIS4tAru6KIjwrhbEZ+cZmHWgP/XAyLh8Hlg0q1/oT/KaZyJeNoEQkl8jfOmdbOQmR9/52LXv3uIzq9Cf77hHL/+meg813+PX+Q4ZEQTZ48mZdffpmIiAgmT55c6r5vv/22xxeXZZnHHnuM77//no0bN9K0aVOPj61IXPYlColSqvU/vgHSjilh/bHfgFrr5iwVg0horFjsM9cBIkPsp2YSsm1qZv1bAQmNacfhq7uVMH37O7wK01dVPBKiffv2YTQabffd4W2a+8SJE1m2bBmrVq0iKiqKS5cuAUqjfvvm/JWN25axMQ2UZXqX3KSE9X96WmlAVYnp/fYWkSjx8D9OMzOinSwi69vvbI36zSLKTVPWptcXhelHflCpn69A4ZEQbdiwweX98rJgwQIA+vfv77B9yZIljBs3zm/X8Rbr4oouV/NI7Az//ESpet77GdRuDn2fqLSxCYvId/6+WvYyQc4WkX1phBI1K5qSFe3m1wUWjQWwwhqmb+pTmL6qEtCu2cEafq4bVcZqHq2GwZDZsHaaEtaPawJtR1bK2Jy/KALPKDCY+cfrLn5EnT6DJieTyD7SpPiInEs8/DQ1s4Xpd0ForDLt9yFMX1XxSIisiyt6wnfffefzYIKF+LKECODahyEjGf74UAnrRzeAht0rfGwm0Y/IJzLyDR7t5zzddRYiq+A4LydU7qnZby/D4e+LwvRfQp0W5TtfFcOj8L197k50dDTr169n9+7dtuf37NnD+vXrbYswVnWs4fu0sro0DpkNLYaAqVBpwl8JYX0RNfMNt0svOW13jprZZ1YreUSOFpHsD4to7xewpSjIM2J+tQ7Tu8Mji8i6uCLAtGnTGDVqFAsXLrTlDZnNZh599FGio4OzjsVbrFOzK3kGTGaL+yJBa1h/yVC4dNDWhJ+w2Aobm/AR+YZboXB6E83OQuRkETknNFpnyj77iE5vhB+fVO5fPxU6j/HtPFUcrxMaFy9ezJQpUxySF9VqNZMnT2bx4sV+HVygqB2hQ62SkGVFjEolJBLGfAVR9ZWw/jfuq/VPXs7halnnKwMRNfMNT4WihI/IwSKyK/Eo2lauotfUY/DVPUVh+n/CgGd9OEn1wGshMplMHDt2rMT2Y8eOYakmjlSVSqJOpNLixGXkzJmYBkqOkTZC+YX7aXKJX9qvd59j8DubeeDz3a7P4SGixMM3PF2M0rkVr71FpAiRcr/cCY25qUrTe30WJF0LI9+vEWF6d3gdNRs/fjwTJkwgOTmZnj17ArBz507mzJnD+PHj/T7AQBEfFcrlbH1RLpEHvq/6nZRp2ooxsPdzqNUc/vEkAL8du8yM7w4CsPfsVQoMZsJ0vpXDiBIP78jTm1i59296Na3t0f7OUzOt2rn63rHEwycfkbGgqJr+bI0L07vDayGaO3cuCQkJvPXWW7Z2IPXr1+eZZ57h6aef9vsAA4VHkTNnWg1V6tJ+ngq/zoK4JuyN6sejS/faPuAWGY5dyqZLozifxuXoTBVKVBYzVx1m5d6/qR2hc/m88ztYqrPa3kdk+3962RjNYoHvH4bzu4vC9N9ChGciWZ3xWohUKhVTp05l6tSptjYc1cVJbY/Py0/3egiuJMMfi7B89xBvW2ZSaGzCgFZ1MZgtbD11hcMXfBcis0MbEJ9OUaP46eAFwANfXxHOeVpau/XA1A5RMxz+euys/u0lOPKDEqa/cynUucaz46o55eoLEB0dXS1FCEop8/CEobPRNxuMylzIO5Y5DE4s5P2xXenUMBaAwxd874NtcnBW+3yaGkOhsXS/pbN8mJ1216rc5RH5kNC493PY8o5yf8R8aPIPDw6qGfiUWf3tt9/y9ddfc/bsWQwGx1+avXv3+mVggaZuaWUeZZClt3DvlQd4xXKS9qozLJTmoDbfQPsGiq/p0HnfG7p527M6V28iLUcflA3TgwHnd9DZIrI3dFxFzWS750oleQP8+JRyv9+0Ghumd4fXFtG7777L+PHjqVevHvv27aNnz57Url2b06dPM2zYsIoYY0DwyUcEFBrNPPj5bvZfNjFV9yymiPqor5yAr++hfb1wAI5fysHo/NPrId72rL7+jQ0MmLuRoxf9382yOuLsI7Kfcil5RD4kNKYWdWuwmKDDv6D/DP8OuhrgtRB98MEHfPjhh8yfPx+dTsfUqVNZt24djz/+uK3Va3Ug3gcfkdkiM/nr/exMySAqRMPc+4ah+ffXSlg/ZRNJ258jKlSNwWzh5OVcn8blbdQso8g3svlEmk/Xq2lYnIXI7r5jz2rr/kX7ubOI7MP0jXrX+DC9O7wWorNnz9KnTx8AwsLCyMnJAeDuu+9m+fLl/h1dALFW4Kfl6D2aAsmyzIv/Pcyag5fQqVUsuqebsqJm/Y7wryUgqZD2fcEToWsAuOph7ZMzjuuaee4k0mmCt01oMFHSIiq+r5JcZFaXFjWzD9PXaqa0etVUfGO/qojXn86EhAQyMjIApUujdS2zlJSUoK2m9wVrQqPBbCGroOx1zT7YmMzn2/9CkuDt0Z3o09yucrrlEBj6OgD3F37GTaodHke8dp/J4N7Ff3A6TbGgHHwYXrzdwdyvOJhwziOy1xf7PKKSCY1OJ7JY4PuHlDB9WFxR03sRpneH15/OgQMHsnq1sprF+PHjeeqppxg8eDCjR492ud5ZVSVEoyY2XOnOV5af6Ovd53jzF2VhyFm3tOWWjokld+r1oNKIH3hbu4DIdM+c+v9cuJ1NJ9J4+Eulab995q83UbOabhGFe5hAWsIispucORS9Fv0euF1gcf2LcGRVcdN7EaYvFa+jZh9++KGtlGPixInUrl2bbdu2MWLECB566CG/DzCQxEeFkJlvJDVbT8t6US73WX+0OGv6kf7NGde3lHa3Q15jx969XGv8g7YbH4LWG5ReRh7w99UCwLvqe3sLNaTGC5GGfIO5xHZny7SEj6hE1KzoOOv+1hPY69Cez2DrPOX+yPehSV+fx11T8OrTaTKZeOWVV2wtXUFZ5+zdd9/lsccec1iCujpQr8hPdD7TdWe/vWevMnGZkjV9R9eGTB3SqvQTqtS8EfEMhyxN0OkzlJagBZkejcX6efemH5H9r7uuhk/NIkJ8s4jscSzxcFNrlvybXZh+OnQa7duAaxhefTo1Gg1vvPEGJpP7ZXmrE9a8n50pGSWeO5Way32f7qLQaKF/q7rMuaODR9m1BnUYEwxTKAyrB+kn4Ou7wVS249r6wfem6NVgKvYn1cSpmckuRSJc55nxX9JH5FjiUWpCY+pRJUwvm6HjaOg/vRyjr1l4/em84YYb2LRpU0WMJejoW+Rw3nbqisM053J2Ifcu/oPMfCOdGsbwwdiuHjuDm9WJ5DK1+LrlW6CLhJTN8NNTHtdrmLxIaNTbCVFNdFbn6YunYp74iGRZdlhJF5zC95Jk8xkVF70qf8P0V2DpKNBnQ6M+Sua0CNN7jNc+omHDhjF9+nQOHjxIt27diIhwzNgdMWKE3wYXaLo3iUOnUXEpu5DT6Xk0rxtJVoGRexf/wfnMAprWiWDxuB4e/9oCXNeiDqsPXGDlhVrc888lyoKN+75UqvWvc79Uk1VyvMkjsreIauJ3ItdQbLlrPKjBsMjFTmgrzlEzV+H7UPRc+8dEyDqr/B/vFGF6b/FaiB599FHA9fplkiRhNpd0CFZVQrVqujWKY/vpK2w7lU6D2DAe/Hw3xy7lUDcqhM/v60ntSO8+cNe1qAvAn39nktlwMLHD3oA1U5QoS1wTaF96f3D7X+yynNV6U/X5X/hCnr5YiDyxN80WVxaRm8ZoVovIYuZt7QLirh5UwvRjv4HwWuUee03Da3vdYrG4vVUnEbLyjxbK9GzzyXRb1nRkiIZPx/cgqVa41+dLiAmlZb1IZBm2nroCPR+AaxVx5/uH4dwfrg8s+uB7YxHZT82qUYqXx+TaCZEnSmSR5RIpEY4WEVhrYK0W0Z3ZS7hJ/QdmlVbpK1S7eTlHXTOpeY4DL+nTXElCW3fksi1r+sN7utEu0YNmaW6wWkW/n1TKLjY0epzf1T3ArFcycTNSShxjtX686VltPzWrgTrkZBG5fgfst7uq/1M5Tc0cLKI9nzIy7xsADnV7FRr38cOoayYeT80KCgpYv349t9xyCwAzZsxAry9O9FOr1bz88suEhlavTnMdGsQQFaIhR29ynTXtA9e1qMMnW1L4/WQ6siwz/vO9hPMIX4ek0z4/RWnCP+F/iqlfhOzKIirjOvZTs+qU9e4pDkLkwcu3F+5iHKdm1ihanctbYafi03vHeAdtGo+kU7lGW7Px2CL67LPPWLRoke3xe++9x7Zt29i3bx/79u3jyy+/tK3cWp3QqFXc2C4BgJnusqa9pFfT2ujUKs5nFnA6PQ+AfEK5Tz9FWR8tXanWtw/ru3ZWex41M5plUrOV3kqHzmex56+SKQnVjVy7qJknMmx0Xm8aF85qoKV0jl67nwLZzMaQAfyf+fbyL7BYw/FYiJYuXcqDDz7osG3ZsmVs2LCBDRs28Oabb/L111/7fYDBwGu3t+f3qQMYX1rWtBeE6dT0aKpYO1tOptu2pxKnNOG3hvV/fArnr5A3jdHsheiBz3fT87X1HLuUzS3zt3DHgu1k+lh4W1VwtIjKliJXUzN7fVFJErGWDBbr3kRryoXGfXkv6klA8r55vsABj4Xo1KlTdOjQwfY4NDQUlV33up49e3LkyBH/ji5ICNGofXJMl4azn8hGQgf4p1Ktz/4veVSt1PXJLhIay/qd17voTvjfAxds9zPKubRRsJPrZdTM4EqI7ARGJ+t5InUmDaV0ciIaw+gvMRZ5N1TC21ouPH77MjMzHXxCaWlpNGnSxPbYYrE4PC8oneuKonHbk6+UfLLljTDsDQCmar/iFtV2ZGDtoYvs+euqbbcyndUuna/FXyyfFwWsInjiI7LfXppFJGHh9jMv0sxwnKtyJJu6vw/htYp7VpdoOivwBo+FqGHDhhw6dMjt83/++ScNGzb0y6BqAm0SoqkTqSPPqRDTNoXo+QBcOxGAt7QL6cIJPv7dMZpWprPaWHo6RXX3a3iSR2RvYBpN7n1E0zQraH11I0a0PGiYTHZ446LjZYf9BL7hsRDddNNNzJw5k8LCks3kCwoKePHFF7n55pv9OrjqjEol8Y9rSkbfHCrEb3yZdeZuhEhGPtK9RepZx4UtrZp1/FIOU745wLkMx+JcvYsokLPPozpj76x2ZxLZV9u7nprBGPV6Htb8CMAX8c+wS27tvuhV4BMeC9Gzzz5LRkYGrVq14s0332TVqlWsWrWKN954g1atWnH16lWefbbmLpnrC1Y/kT0OfhuVmseNEzloaUJtKYcl2jeIprjFrEWWWXPwIkPmbebbPX/zyNI9DudyFY6u7tMxezyziErPI6p1cQsva5YAsDXpQXZFD1LOV6Jndc15XysCj4WoXr16bNu2jTZt2jB9+nRuu+02brvtNmbMmEHbtm3ZsmUL9erVq8ixVjusfiJ70nKL/WwWi0wBoUwwPMMFuRbNVRdZqJ2HFuULJgOr9xc7n0849cF2aRFJru9XR/IMZfuIzKUIUUvpHO23PoZGsrDS/A/+SLq/xEqvXi0nJHCLV77+pk2bsnbtWtLS0tixYwc7duwgLS2NtWvX0qxZs4oaY7UlPjqU1gmODdd2nC52XhuL6p5SieM+w1Ry5VD6qI/wmuZjQEaWZdLthCvEqcLepUVE1XdW601mXv7xSMmIoxO5nmRWu3FW1yWTxbo30Zjy2GlpzQzjA6jVKpt4O7eKrarvZbDgU9CxVq1a9OzZk549e1KrlijwKw/OVtGagxdt9+2boB2TGzHJ+DhmWeJfms08ql4F4CBEzj2HXBW9Vofvy+fb/uKTLSnc/YmburwiPIma2U/NDEXO6lD0fKSbS0MpnfyopjxomIwBrVNjNMfjhUVUPkT2Q4AZ2FqZzjavG4FaJXHofDZnryhOZ+epwu90YZZpHABTtV+TeG4NV3KLfUolhaj02qmqWvZx7qrrjpnOeNLN0j4vy2i2IGFhnvYDOqtOkyFHcqj/R2QRCTi2irUJmNVZLZSoXAghCjC9m9fmw7u7sWRcT65tpliXPxVZRc5RnGZ1IvjSPJiPTDcB0HXfs7QwFCeROjc/K8tZXUV1yOOMHXtrx91LdZ6aTdcsZ6h6F3pZw4OGyeijm9ieV6uK3z9ZWER+JaBCtHnzZoYPH05iYiKSJPHDDz8EcjgB48Z2CTSqHc5NHeoDxdMz59onq8Uz23QX/zN3Q20x8JHuLRpJlx2et1JWPyJnIcopNDJ60Xa+2H7G15dSKXjqj3HIQXejuvYWUaPTK3hI8xMAzxgfZrfcukQ/opKN0bwbk8A1ARWivLw8OnXqxPvvvx/IYQQNQ9oloJLg4PkszmXkY3SyaKxdBi2oeMI4kfNhrRzC+s4N8l1bRMX3nR24n2xJYWdKBv9ZddhPr6hi8DRUbvHA5LPuc73qAF0PvwbAW8Z/strSp+hajtd17yMSQlQeAipEw4YN45VXXqlW66GVhzqRIVzbTOl/tObgxRI+IrXdt6KAUN6t9wrn5do0V11kkXYe4WpHC8h1QqP7qVluYdVYFMHT77zDWpSlOKtbSWd5X/suKtnMSvN1zDfbfR5L9CMqOh/WPKISuwl8oEr5iPR6PdnZ2Q636ob99MzZRxTm1AD+dGEkEwzPkCOH0Vt9hEm58x2+cWU5q62/5huOpfL8DwddZhYHI7586d2F72tZrvKJbi5RUgEXYrsx3fiAwxUcpmbOjdEQFpG/qFJCNHv2bGJiYmy3pKSkQA/J71inZwf+zuJMenF0aFCbelzb1HHJ4tQcvS2sb5JV9C/8FX6fa3vedaOvYqxfzfGf7uLLHWf5fPtffnsdFYmnESoHZ7UrHTLk8bZlDg2ldJIt9fmpzRu2anorDv2I7BqjWUtDRK2Zf6hSQjRjxgyysrJst3PnzgV6SH6nblQIPZsq0bNV+88D0KR2OB/f273EFzCtaCnsTZZOvGC6V9n42ytw8FvAtbN6/7lM2/2aHDWLCVXBdw/SnmQy5EjGG6eSo4ou9VpK1Mx6bse/wiIqH16v4hFIQkJCCAmp/su03NyhPjtOZ7DxuJI57G5NMvsC2S/Ng+lXJ5fBWd/AD49ATEOXU7OfD12ye+ReiWRZDt5IUBnDevG/h4kL1zkIrXPU7DntCjj2IwZZwwOGpzkr13PdBsR+gUUXeUS2WrMq9ZMefIi3LwgZ0j4BSSrOI7IKUVkJiP+t9zC0vgXMBlg+htr6v0vdv7QOj498ude7QVcipfX+SUnPY8nWM7y97oTb/t5j1b8yyvgDADPkR9kjK0uFO0cpoWSr2GIfkai+9ycBFaLc3Fz279/P/v37AUhJSWH//v2cPXs2kMMKOPFRofRsUlw6o1U7hozdodVo4fYPIbELFGTwYu6LxJDrdv/SdG3t4UvunwwwpbmI7P1iDj6yotfaT3WAFzWfKg8GPM+PluKVN8pqFatWSUSHagH4O7MAEAmN/iKgQrR79266dOlCly5dAJg8eTJdunRh5syZgRxWUHBzx/q2+1aLqKy8GI1K4lCaie9avYUc3ZBG8gUW6d6xVes7U9YCjcGKyiE73PE12M9i9U7LKbWSzvKe9l00koWf1QPg+imOtWZlNM9XSRK9i5aX2nIynVy9icKi5nPCIiofARWi/v37I8tyidunn34ayGEFBUOLpmdgPzUr/RgZmVvmb2Hyz5fY1P19cgnjWtVR5mg/wpU/yHl55aqC/Xe+x6vrWbAx2fbYXhDs0xHizFdYrHuTKKmA7ea2vKF7BCTJwcr8apcrS7z4fGqVRLfGcYRqVaTm6Hl8+T4KjRaSaoX5vad5TUP4iIKU+KhQehRNz7Qaz3xE9k9vz63H0/JTmGQVd6h/Z5L6hxL7P7Z8r0OHwqqCve2Rnqvn9bXFnStdvZowCnlN/yoNpCskW+rzsPFJDLIyxTKXsSqKo0WkLEPeqyiN4rdjqQC8NKK924CCwDPEuxfEjOysrKFWN1KJFJalGfbPmy0yG8wdmWkaD8AU7TeMUG112D85LY/tp1007w92SpkGmZ3eJBUW/k/7Pq0tyVyRoxhvnGqrpvek+4Cr1rr2rVuGtktgQOt4LwYvcIUQoiBmTI9GvDumC88MUaI6ZfmI7L9YJouMwWRhmfkGFpmUXuJvahfRXXLse23fs8eZPX9ddenADTSlOYZNTn6eGZpl3Kjeg76o6f1ZWWm7YpHlEqLlCvvwvbXEpn+reCQJwrRqZg5v68MrEDgjhCiIUakkRnRKJCFGWca7rO+N0W6Hn+warM0xjWGtuQchkokPdW/TWCqOiJV2zjsWbOOF1cFXAFta+N5erP+tXscDmjUAvKp9zBamt+7nyay0YVwYOrUKlQTN6iqW1DXxkXw6viffPNybxNgwH1+FwB4hRFWIMvOI7BZPtGZdA8ioeNL4KPstzagl5bJY+6ZdWL/0cy7dGXypFKUFqKwr4fZX7beF6d80jmK9+jqH/WTZs+r8OpEh7Hj2BnY+O4imdSJs2/u1rEv7BjHeD17gEiFEVQhPvjjuKCSEBwxT+FuuQ3PVRT7UvY0Oo8uQdbBT2tTMbLHQuihMr5ZkvjFdz/vmkSX2s5QiRC3rRdIwLowZw1oDUCtCR92o6p/RH0iEEFUhylsblkYs9xmeIVsOo5fqGLO1H1GgN5Z53Kr95/nz78zyXdyPlFZ6osq9zGLdG0RKhWwzt+VZ0/2A5EJ0HKdmm57pb7sfptOwZdpAHurX3K/jFrhHCFEVwh+R9hNyEhONTxSF9bfQ4vjCMo95YsV+Rry3tcT2q3kG5v16wtZju7Jwq0OGPFqsn0CilGEL0ztX01uxyI4Rtvoxxb6e7IKyxVngX4QQVSHKMzWz53dLR/5TFNbvmvwBI1VbfDrP1JV/Mu/Xk9z6QUmRqkhcOqstZlh5P5EZh7kiRzHOOJXsojA9lLQmrcmzVuybzmXmGxBULkKIqhCunNW+VhYsN9/AQtMtALyh/ZAeTmF9T9hZlIPksDptJeDqNet/fhaOr8Gi0vGA4WnOyY6LfTqLuLNFZO93yhIWUaUjhKgK4Tw1iw7VlMtv9LrpTo7E9idEMrFI9zZNpItlHxQEODur/61eR8guZYp5uNfr7JVblnkOWZY5UzSllCRHv1MVTDav8gghqkI4/6o3qh3OyyPb+Xw+GRXLEp9zCOvHkuPF8YFh66nibPD+qn3F1fQD/8OFhje5PMZ5rNmFJu5YsA0ozph+qJ+yWvHEAcJJXdlUqcZoNR37X+qh7RJ49qY2NKodzs+HLrEt2bdSjSyzlgcMU/g+ZCbNVJdYpHuHuw0zMKD106jLx4nLOaz44xwxYVqyCoz0b1WXTSeUhnFtpL94TzsftSTztakfF/UjaOnGnCktB8tqYU0d0ppbOzegVb0ot/sKKgZhEVUh7L9MC+/uRqPaSsW3fcsQbykwmEqE9d1V66/af56/ruTZHldG44sh8zazeGsK7/x6gsVbU7hnsbLMdDxX+UT3pi1M/5xpAu+sP2lLaPQG6/pxapVEm/rRYtXWACCEqApRJ9J1Ut2o7knMH9PFoRjTU/L0Sj+dE3ISjxqfxCSruF29hcfV35fY94kV++n35kbb48qYmrkyZMIpZLHuTRKlDE5ZEh3C9O7qx3KKlkpSqyRiwrTEiwTFoEIIURXi4f7NublDfT4Y29Vhu1atYninRJrXjXRzpHvyjcV9r7dYOvC86T4AJmu/5VYfw/oViVJN/x7tVWdIl6MZb3zGIUxvXXDAGWuTtHCtmt+nDWDTMwOEGAURQoiqEJEhGt4f29W29pmr573lgN2qHgArzANZaBoOwOtuwvq7zmR4fR1fcOXXeU6zlMHqvehlpZreOUy/oWjBAXuub1m3+IEE0aFawnRq4sJ1fh+zwDeEEFUjIkP9E3t43TSaNeaetmp957D+vxZuJznNfS9sfzHv15MOj+9W/48Jmp8BmGx8xKMwfat6UdzZw/X6dzHhweGQFwghqlb4YhG5QkbFZOMj7Lc0J07KZYn2jRJh/eOXPA/z+4LJbOH/1hcLUX/VPl7QfAbAG8bR/GS51qPz3NKxPgNbxxNRtEquvRs6TghR0CCEqBoR5SeLCJRq/fuLqvWbqi6zSPcOOoozjiu6Wfx3+4p9PfZh+q9M/fnAPMLj89SLCSVUq+bGdgklnosNE1OzYEEIUTWia6M4v54vnRjGG6bawvqvaz/EGiu7kqe3RaLKgyzL7D17lZxCIyazhYnL9rJkawpTv/0TgHpksLgoTL/V3I7nTPfhTeJAbtEYR3RS2u7a15TFRQghChaEEFUjkmqF8393dvbrOU/KDW1h/dvUW3lSsxKA574/5Jfz/3TwIrd/sI0R721l7eFL/PTnRV787xFACdN/optLfSmDk5YGPGJ8EpOXObjWhQeua1GHsb0aMXHANbbnxNQseBBCVM2w7yLoL+zD+k9qvvNrWH/1fqWrZEp6nkP/bNdheu9fW7hW8Q1p1Cpeva0D91/XzPbcDW2UiFstYRkFHCFE1YxmdrlEnZJiHZ6zb/7lLSvMA1lQFNZ/Q7uIntJRn8/ljmkrD9ruP6/5ksHqvRTKWh4wPM3fsm8rZYQXOaldcU18JBum9GfDlP4+nVvgP4QQVTMiQzTs/c9gDr84hFUT+zo817h2BGfm3My4Pk1oVCucd8d04eVb29MuMdqjc79hGs1P5p7oJDOLdO/Q1C6sn1PoW+sMV3nQ96h/4T7NWkAJ0++TW/h0boCwUoQIFAsyJkxM0QKNEKJqSK0IHRGlhPJfGNGOzVMHMKJTIndf25jbujTw6LxKWP9R9lmuIU7KZbH2DeLIBqDDC//zy9JDA1T7mKX5HIDXjXeyxsMw/RM3uBarMG3pQiQIDoQQCRzquZY90KvUffUUNR6z1C0R1r9azgZpbaUztqb3K0z9WWAe7vGx7vw8HRqKlTaqAkKIqjm3Fq0WO7ZXI7f7mO2UqE/zsgtn04lRnMdyOD1Vx3lDuwhfS2D3/nUVKA7TR0h6tpjbFTnHPQ/TG0yurbFwneh0UxUQQlTNmXNHR5bd34tZw903UPOlF/YpuSGPGJ/AKKu5Vb2Np4rC+t6QmW/gSp7BVk2fIF3lpKWBki7gQZj+8YHFoXhDEK5IK/AcIUTVnFCtmj7X1EGncf+vdvajOEfb3LHVLqz/hOY73p77okfryVuZ8d1BVFh4Vzufdqq/SPMgTN+mvuJYlySYfGPxyq16FxZRaREzQXAhhEjAnT0acW2zWjx/cxsAvnrQMwcxwFfmAXxgUkouXpIW8fmKpR4fu/30Ff6j+YJB6n1FYfopZYbpVz7Sm2lDW7N+cj+H7QaThZFF09DnbmrD/566nm3TB3o8FkFgkWRvfsKCjOzsbGJiYsjKyiI62rMQtMAzDv6dxetrj5GjN/H5+J50eul/bveVsDBfO59b1DvJlCNY33cpwwf2K9UKe+m/RzDvWMiLWqWQ9RHDE/xsKd1RDnBmzs0Oj5tM/wmAhf/uSq+mtTmZmkvPprU8eYmCSsDT76gQIoFHHPw7i+Hvuc+oDsHACt0rdFGd4oylHuv6LOWBoT1c7qs3mXlk5mt8pH0LtSQzx3gnCz0sZHUWopT0PA6ez2J4x/qlrgArCAyefkfF1EzgER0axpRaPqJHx/1FYf0mqst03vYoP+87U2K/cxn5zP7kK+YXVdMvNw1goRdhemea1olgRKdEIUJVHCFEAo9pW790q/OKXVi/h+oEhu8eKdF0+vnPf+HhC88SIen53dy+aMVZz0Skd7Pavg5dEOQEhRC9//77NGnShNDQUHr16sUff/wR6CEJXDBreNsy9zklN1Sa2ctqRqq3cebb5ygs6ou9YssRpmbMIkG6yglLAyYan/AoTL/o7m78q1tD3h7dqdyvQRCcBNxH9NVXX3HPPfewcOFCevXqxbx58/jmm284fvw48fGlR1CEjygwpGYX0vO19aXu8y/1Rt7UfgjAZMPDtBx0Hy02PMQN6n2kydHcZniZv+W6bo+vFaGjVoSO+/o25a5SkjEFwU2VcVb36tWLHj168N577wFgsVhISkriscceY/r06aUeK4QocBQazSz/46ytd5ArntGsYKJmNQZZTbKcSBvVOQplLXca/sN++Rq3x6198jpaJ4j/Z3XA0+9oQPPfDQYDe/bsYcaMGbZtKpWKQYMGsX379hL76/V69Hq97XF2dnaljFNQklCtmvF9mzK+b1Oy8o1czTfw6pqjrDty2bbPXNMoGkuXuUW9kzbSOQCeNE4sIULThrbmoeublViDXlBzCKgQpaenYzabqVfPcUmYevXqcexYyWVsZs+ezYsvvlhZwxN4SEy4lphwLR/d0x2AfIOJ5384xHd7z/O08RESpKt0V53gNeMY1lp6AkpxrSd1bYKaQZWqCJwxYwaTJ0+2Pc7OziYpyfVSMYLAEa7T8Paozrw9qjMAeQXD2HvsENM6deNZsZyzwAUBFaI6deqgVqu5fPmyw/bLly+TkFBy1YWQkBBCQsTqnFWNiLBQunbpHuhhCIKYgIbvdTod3bp1Y/364giMxWJh/fr19O7dO4AjEwgElUnAp2aTJ0/m3nvvpXv37vTs2ZN58+aRl5fH+PHjAz00gUBQSQRciEaPHk1aWhozZ87k0qVLdO7cmbVr15ZwYAsEgupLwPOIyoPIIxIIghtR9CoQCKoMQogEAkHAEUIkEAgCTsCd1eXB6t4SpR4CQXBi/W6W5Yqu0kKUk5MDILKrBYIgJycnh5gY92vMVemomcVi4cKFC0RFRQVFsaS15OTcuXMiimeHeF/cU93fG1mWycnJITExEZXKvSeoSltEKpWKhg0bBnoYJYiOjq6WH6ryIt4X91Tn96Y0S8iKcFYLBIKAI4RIIBAEHCFEfiQkJIRZs2aJDgFOiPfFPeK9UajSzmqBQFA9EBaRQCAIOEKIBAJBwBFCJBAIAo4QIoFAEHCEEJWTjIwMxo4dS3R0NLGxsUyYMIHc3NxSj/nwww/p378/0dHRSJJEZmZm5Qy2AvF2td5vvvmG1q1bExoaSocOHVizZk0ljbTy8ea9OXz4MHfccQdNmjRBkiTmzZtXeQMNIEKIysnYsWM5fPgw69at48cff2Tz5s08+OCDpR6Tn5/P0KFDefbZZytplBXLV199xeTJk5k1axZ79+6lU6dODBkyhNTUVJf7b9u2jTFjxjBhwgT27dvHrbfeyq233sqhQ4cqeeQVj7fvTX5+Ps2aNWPOnDkuF5CotsgCnzly5IgMyLt27bJt+/nnn2VJkuTz58+XefyGDRtkQL569WoFjrLi6dmzpzxx4kTbY7PZLCcmJsqzZ892uf+oUaPkm2++2WFbr1695IceeqhCxxkIvH1v7GncuLH8zjvvVODoggdhEZWD7du3ExsbS/fuxUvlDBo0CJVKxc6dOwM4ssrDulrvoEGDbNtKW60XlPfNfn+AIUOGuN2/quLLe1NTEUJUDi5dukR8fLzDNo1GQ61atbh06VKARlW5lLZar7v34NKlS17tX1Xx5b2pqQghcsH06dORJKnUm6slsQUCgW9U6TYgFcXTTz/NuHHjSt2nWbNmJCQklHA6mkwmMjIyaoyj0dvVegESEhK82r+q4st7U1MRFpEL6tatS+vWrUu96XQ6evfuTWZmJnv27LEd+9tvv2GxWOjVq1cAX0Hl4ctqvb1793bYH2DdunXVbnVfsZKxFwTaW17VGTp0qNylSxd5586d8pYtW+QWLVrIY8aMsT3/999/y61atZJ37txp23bx4kV537598kcffSQD8ubNm+V9+/bJV65cCcRLKDcrVqyQQ0JC5E8//VQ+cuSI/OCDD8qxsbHypUuXZFmW5bvvvluePn26bf+tW7fKGo1Gnjt3rnz06FF51qxZslarlQ8ePBiol1BhePve6PV6ed++ffK+ffvk+vXry1OmTJH37dsnnzx5MlAvoVIQQlROrly5Io8ZM0aOjIyUo6Oj5fHjx8s5OTm251NSUmRA3rBhg23brFmzZKDEbcmSJZX/AvzE/Pnz5UaNGsk6nU7u2bOnvGPHDttz/fr1k++9916H/b/++mu5ZcuWsk6nk9u1ayf/9NNPlTziysOb98b6eXG+9evXr/IHXomINiACgSDgCB+RQCAIOEKIBAJBwBFCJBAIAo4QIoFAEHCEEAkEgoAjhEggEAQcIUQCgSDgCCESCAQBRwiRwO+MGzfOZceCoUOHBnpogiBFVN8LKoShQ4eyZMkSh23uVjM1Go1otVqHbQaDAZ1O5/V1fT1OEFiERSSoEEJCQkhISHC4xcXFASBJEgsWLGDEiBFERETw6quv8sILL9C5c2c+/vhjmjZtSmhoKABnz55l5MiRREZGEh0dzahRoxzaarg7TlC1EEIkCAgvvPACt912GwcPHuS+++4D4NSpU6xcuZLvvvuO/fv3Y7FYGDlyJBkZGWzatIl169Zx+vRpRo8e7XAu5+MEVQ8xNRNUCD/++CORkZEO25599lnbyiV33XUX48ePd3jeYDDw+eefU7duXUDpUXTw4EFSUlJISkoC4PPPP6ddu3bs2rWLHj16uDxOUPUQQiSoEAYMGMCCBQscttWqVct2337BASuNGzd2EJOjR4+SlJRkEyGAtm3bEhsby9GjR21C5HycoOohhEhQIURERHDNNdeU+rwn2zy9lqBqI3xEgqClTZs2nDt3jnPnztm2HTlyhMzMTNq2bRvAkQn8jbCIBBWCXq8vsWSORqOhTp06Hp9j0KBBdOjQgbFjxzJv3jxMJhOPPvoo/fr1czm1E1RdhEUkqBDWrl1L/fr1HW7/+Mc/vDqHJEmsWrWKuLg4rr/+egYNGkSzZs346quvKmjUgkAhWsUKBIKAIywigUAQcIQQCQSCgCOESCAQBBwhRAKBIOAIIRIIBAFHCJFAIAg4QogEAkHAEUIkEAgCjhAigUAQcIQQCQSCgCOESCAQBBwhRAKBIOD8P6Z6FThJrrjWAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019827, + "end_time": "2024-03-22T19:05:42.203827", + "exception": false, + "start_time": "2024-03-22T19:05:42.184000", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4633.875758, + "end_time": "2024-03-22T19:05:44.947278", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/realtabformer/42/mlu-eval.ipynb", + "output_path": "eval/insurance/realtabformer/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/realtabformer/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "realtabformer" + }, + "start_time": "2024-03-22T17:48:31.071520", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/realtabformer/model.pt b/insurance/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..031fb800914ee9436f6ed2979a5e24d56a5cd830 --- /dev/null +++ b/insurance/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd1a49d8056abc7e60ed44b5a73d928fd175a32a62b6f2d0610e0c0ecf1c526 +size 43505805 diff --git a/insurance/realtabformer/params.json b/insurance/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..d184ca1b8785b216e6ed568fc89d0f80099a8059 --- /dev/null +++ b/insurance/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/eval.csv b/insurance/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..e82ccd77108e3f05e0cebfcc66ec9cef5163dd65 --- /dev/null +++ b/insurance/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.005787722477074393,0.5310536811323393,0.019623511412314006,6.455421447753906,0.09179345518350601,0.9963166117668152,0.1400327980518341,9.540074643155094e-06,2.324864387512207,0.09191911667585373,4.1357741355896,0.14008395373821259,0.03438407555222511,1.4360357522964478,8.780285835266113 diff --git a/insurance/tab_ddpm_concat/history.csv b/insurance/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..528956c40e3a85a196cb5fa0ef12a0ec33f50b02 --- /dev/null +++ b/insurance/tab_ddpm_concat/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03592884845203823,7.2996755396477075,0.009430593892273626,0.007427950899711706,0.0,0.0,0.0,0.0,0.09437693562669058,900,113,140.96822214126587,1.2475063906306714,0.15663135793473987,0.05326718922737425,0.014524129554629325,4.156348642464237,0.0009876720450387843,0.0,0.0,0.0,0.0,0.0,0.014524129554629325,450,57,44.241886377334595,0.7761734452163964,0.09831530306074354,0.03724088621113384 +1,0.014371616853814986,6.227466808899376,0.00030516517503158583,6.165503883999514e-05,0.0,0.0,0.0,0.0,0.2792855604737997,900,113,141.62208795547485,1.2532928137652641,0.15735787550608318,0.03787480419155507,0.01340457120962027,5.29680332468604,0.0002864654652569243,0.0,0.0,0.0,0.0,0.0,0.01340457120962027,450,57,45.41609525680542,0.7967736009965863,0.10092465612623426,0.030410112535352248 +2,0.01384979708948069,7.497149070959221,0.00021502181714296655,1.949448532994009e-05,0.0,0.0,0.0,0.0,0.11678517651847667,900,113,143.15943694114685,1.2668976720455474,0.15906604104571873,0.033135497126629394,0.014283642331914355,5.459189098905386,0.0005690211627043279,0.0,0.0,0.0,0.0,0.0,0.014283642331914355,450,57,46.33096122741699,0.8128238811827543,0.1029576916164822,0.026298031341611294 +3,0.013210438237422042,6.6640938428891605,0.00014334700532385921,4.62662972798474e-06,0.0,0.0,0.0,0.0,0.2998970259560479,900,113,146.3652093410492,1.2952673393013203,0.16262801037894356,0.03572286798132468,0.013382001433314548,4.643842804434571,0.00048086737289648024,0.0,0.0,0.0,0.0,0.0,0.013382001433314548,450,57,47.94945430755615,0.8412184966237921,0.10655434290568033,0.029577646339148805 +4,0.013310110395153363,6.041971506328766,0.00019507275757620039,0.0,0.0,0.0,0.0,0.0,0.013310110395153363,900,113,143.80060958862305,1.2725717662710003,0.15977845509847005,0.038453474558428326,0.013661402131223844,4.1234811326556935,0.00045455218989546767,0.0,0.0,0.0,0.0,0.0,0.013661402131223844,450,57,45.394694089889526,0.7963981419278864,0.10087709797753228,0.0333288926100195 +5,0.013588898373353812,7.486738423658624,0.0002779256184064716,0.00022828346642199903,0.0,0.0,0.0,0.0,0.08210628287142349,900,113,141.604829788208,1.2531400866213098,0.15733869976467557,0.03336982285620364,0.015309163269638602,11.10999310352408,0.0006529700210521443,0.0,0.0,0.0,0.0,0.0,0.015309163269638602,450,57,45.059311866760254,0.7905142432764957,0.10013180414835612,0.01890142163948009 +6,0.01360072170642929,9.74690950918578,0.0002635270801760223,9.69009121440144e-06,0.0,0.0,0.0,0.0,3.451531191302153,900,113,140.73290538787842,1.2454239414856496,0.15636989487542047,0.029921204000052097,0.013079761469271034,5.622179901249935,0.00037308515279698264,0.0,0.0,0.0,0.0,0.0,0.013079761469271034,450,57,43.94645428657532,0.7709904260802687,0.09765878730350071,0.024666702521866875 +7,0.013433132627978921,7.019836858438262,0.0001345649655801632,2.2781116680966485e-05,0.0,0.0,0.0,0.0,0.16974145690082676,900,113,139.68711066246033,1.2361691209067285,0.15520790073606702,0.034294621425524224,0.013359390444432696,5.3989552207959495,0.00037170510294370413,0.0,0.0,0.0,0.0,0.0,0.013359390444432696,450,57,44.08630442619324,0.7734439373016357,0.09796956539154053,0.028002551381002393 +8,0.013708434053179291,8.164382093871072,0.00011137482917280241,0.0002664083573553297,0.0,0.0,0.0,0.0,0.02415950643726521,900,113,140.14124727249146,1.2401880289601013,0.15571249696943495,0.03384085310688984,0.013566718476617503,6.5259337021869435,0.00033163626410593374,0.0,0.0,0.0,0.0,0.0,0.013566718476617503,450,57,43.685551166534424,0.766413178360253,0.09707900259229872,0.027984272067745525 +9,0.013571654115286139,8.742722461416001,0.0003018974633938696,1.6721362351543374e-05,0.0,0.0,0.0,0.0,0.027267425186518167,900,113,140.09695625305176,1.2397960730358564,0.15566328472561305,0.030262669457732577,0.01325964125830473,5.029873872576148,0.00043549182230303055,0.0,0.0,0.0,0.0,0.0,0.01325964125830473,450,57,43.787155866622925,0.7681957169582969,0.09730479081471761,0.025762035086620273 +10,0.014379269344628685,7.366559921767701,0.00037252366738006987,0.0003263944470220142,0.0,0.0,0.0,0.0,0.017774089858867227,900,113,142.94084358215332,1.2649632175411798,0.1588231595357259,0.035403397502954556,0.013996612014921589,7.5169032518093255,0.0005061832475970126,0.0,0.0,0.0,0.0,0.0,0.013996612014921589,450,57,46.45759129524231,0.8150454613200405,0.10323909176720514,0.02128785624773356 +11,0.013456141208298505,8.885624297614187,0.0001408637665720865,6.312208974526988e-06,0.0,0.0,0.0,0.0,0.12043976681565659,900,113,140.75263118743896,1.2455985060835306,0.15639181243048775,0.03045645008374632,0.015452116089096914,10.887882433234672,0.0007382755149755995,0.0,0.0,0.0,0.0,0.0,0.015452116089096914,450,57,44.30102300643921,0.7772109299375299,0.09844671779208714,0.015029358677566051 +12,0.013939871930827697,10.302565499742524,0.00030740466281282784,1.2590549886226655e-06,0.0,0.0,0.0,0.0,0.05802687374461028,900,113,140.6320457458496,1.244531378281855,0.15625782860649956,0.026154642290048366,0.013364347005262971,4.240572321283827,0.0003598822188279074,0.0,0.0,0.0,0.0,0.0,0.013364347005262971,450,57,44.313255310058594,0.777425531755414,0.0984739006890191,0.03320887988727344 +13,0.013616377720609307,7.797813521520289,0.0002454437854082967,8.744494782553778e-05,0.0,0.0,0.0,0.0,0.013847984937537047,900,113,141.74561762809753,1.2543859967088278,0.15749513069788615,0.03314163033084004,0.014651317608594481,6.794933559461737,0.0006282155579800827,0.0,0.0,0.0,0.0,0.0,0.014651317608594481,450,57,47.70662569999695,0.8369583456139815,0.10601472377777099,0.02225242922768781 +14,0.013695524584295021,8.25407733499283,0.00019777481913188744,1.3750431502962278e-05,0.0,0.0,0.0,0.0,0.021342603873668445,900,113,142.06192708015442,1.2571851953995967,0.157846585644616,0.03127054079328623,0.013534503092782365,4.2766512755933626,0.00026410735567626234,0.0,0.0,0.0,0.0,0.0,0.013534503092782365,450,57,44.758880853652954,0.7852435237482974,0.09946417967478434,0.03563740865833927 +15,0.013962997518893745,8.13189057305249,0.00029005304642092415,2.4200141843822267e-05,0.0,0.0,0.0,0.0,0.015241746428526111,900,113,141.51434302330017,1.252339318790267,0.15723815891477796,0.03115090290166899,0.013381186781658066,5.749990332730752,0.0003508059210490602,0.0,0.0,0.0,0.0,0.0,0.013381186781658066,450,57,44.66229581832886,0.7835490494443659,0.09924954626295301,0.027999498015433028 diff --git a/insurance/tab_ddpm_concat/mlu-eval.ipynb b/insurance/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..578468b7c10239d71c2c32fde23760115b439a2e --- /dev/null +++ b/insurance/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2413 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.416336Z", + "iopub.status.busy": "2024-03-23T02:43:21.415927Z", + "iopub.status.idle": "2024-03-23T02:43:21.449367Z", + "shell.execute_reply": "2024-03-23T02:43:21.448672Z" + }, + "papermill": { + "duration": 0.049244, + "end_time": "2024-03-23T02:43:21.451376", + "exception": false, + "start_time": "2024-03-23T02:43:21.402132", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.477099Z", + "iopub.status.busy": "2024-03-23T02:43:21.476294Z", + "iopub.status.idle": "2024-03-23T02:43:21.483018Z", + "shell.execute_reply": "2024-03-23T02:43:21.482186Z" + }, + "papermill": { + "duration": 0.021722, + "end_time": "2024-03-23T02:43:21.485015", + "exception": false, + "start_time": "2024-03-23T02:43:21.463293", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.508289Z", + "iopub.status.busy": "2024-03-23T02:43:21.507988Z", + "iopub.status.idle": "2024-03-23T02:43:21.511920Z", + "shell.execute_reply": "2024-03-23T02:43:21.511189Z" + }, + "papermill": { + "duration": 0.017649, + "end_time": "2024-03-23T02:43:21.513722", + "exception": false, + "start_time": "2024-03-23T02:43:21.496073", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.536975Z", + "iopub.status.busy": "2024-03-23T02:43:21.536705Z", + "iopub.status.idle": "2024-03-23T02:43:21.540551Z", + "shell.execute_reply": "2024-03-23T02:43:21.539745Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017667, + "end_time": "2024-03-23T02:43:21.542525", + "exception": false, + "start_time": "2024-03-23T02:43:21.524858", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.565424Z", + "iopub.status.busy": "2024-03-23T02:43:21.565183Z", + "iopub.status.idle": "2024-03-23T02:43:21.570712Z", + "shell.execute_reply": "2024-03-23T02:43:21.569865Z" + }, + "papermill": { + "duration": 0.019222, + "end_time": "2024-03-23T02:43:21.572651", + "exception": false, + "start_time": "2024-03-23T02:43:21.553429", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "04e127fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.597281Z", + "iopub.status.busy": "2024-03-23T02:43:21.597006Z", + "iopub.status.idle": "2024-03-23T02:43:21.601689Z", + "shell.execute_reply": "2024-03-23T02:43:21.600943Z" + }, + "papermill": { + "duration": 0.019065, + "end_time": "2024-03-23T02:43:21.603536", + "exception": false, + "start_time": "2024-03-23T02:43:21.584471", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tab_ddpm_concat/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01087, + "end_time": "2024-03-23T02:43:21.625568", + "exception": false, + "start_time": "2024-03-23T02:43:21.614698", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.648463Z", + "iopub.status.busy": "2024-03-23T02:43:21.648209Z", + "iopub.status.idle": "2024-03-23T02:43:21.657226Z", + "shell.execute_reply": "2024-03-23T02:43:21.656431Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022549, + "end_time": "2024-03-23T02:43:21.659004", + "exception": false, + "start_time": "2024-03-23T02:43:21.636455", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tab_ddpm_concat/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:21.682468Z", + "iopub.status.busy": "2024-03-23T02:43:21.682207Z", + "iopub.status.idle": "2024-03-23T02:43:23.669777Z", + "shell.execute_reply": "2024-03-23T02:43:23.668713Z" + }, + "papermill": { + "duration": 2.001732, + "end_time": "2024-03-23T02:43:23.672143", + "exception": false, + "start_time": "2024-03-23T02:43:21.670411", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:23.699630Z", + "iopub.status.busy": "2024-03-23T02:43:23.699077Z", + "iopub.status.idle": "2024-03-23T02:43:23.713239Z", + "shell.execute_reply": "2024-03-23T02:43:23.712333Z" + }, + "papermill": { + "duration": 0.030141, + "end_time": "2024-03-23T02:43:23.715333", + "exception": false, + "start_time": "2024-03-23T02:43:23.685192", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:23.744201Z", + "iopub.status.busy": "2024-03-23T02:43:23.743897Z", + "iopub.status.idle": "2024-03-23T02:43:23.751617Z", + "shell.execute_reply": "2024-03-23T02:43:23.750857Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022187, + "end_time": "2024-03-23T02:43:23.753794", + "exception": false, + "start_time": "2024-03-23T02:43:23.731607", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:23.778747Z", + "iopub.status.busy": "2024-03-23T02:43:23.778484Z", + "iopub.status.idle": "2024-03-23T02:43:23.873632Z", + "shell.execute_reply": "2024-03-23T02:43:23.872697Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.109645, + "end_time": "2024-03-23T02:43:23.875814", + "exception": false, + "start_time": "2024-03-23T02:43:23.766169", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:23.901793Z", + "iopub.status.busy": "2024-03-23T02:43:23.901508Z", + "iopub.status.idle": "2024-03-23T02:43:28.468900Z", + "shell.execute_reply": "2024-03-23T02:43:28.467933Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.583327, + "end_time": "2024-03-23T02:43:28.471403", + "exception": false, + "start_time": "2024-03-23T02:43:23.888076", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-23 02:43:26.147756: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-23 02:43:26.147815: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-23 02:43:26.149466: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:28.496516Z", + "iopub.status.busy": "2024-03-23T02:43:28.495935Z", + "iopub.status.idle": "2024-03-23T02:43:28.502497Z", + "shell.execute_reply": "2024-03-23T02:43:28.501772Z" + }, + "papermill": { + "duration": 0.021058, + "end_time": "2024-03-23T02:43:28.504307", + "exception": false, + "start_time": "2024-03-23T02:43:28.483249", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:28.529659Z", + "iopub.status.busy": "2024-03-23T02:43:28.529381Z", + "iopub.status.idle": "2024-03-23T02:43:36.715273Z", + "shell.execute_reply": "2024-03-23T02:43:36.714194Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.200979, + "end_time": "2024-03-23T02:43:36.717518", + "exception": false, + "start_time": "2024-03-23T02:43:28.516539", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-23T02:43:36.745175Z", + "iopub.status.busy": "2024-03-23T02:43:36.744856Z", + "iopub.status.idle": "2024-03-23T02:43:36.751481Z", + "shell.execute_reply": "2024-03-23T02:43:36.750608Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022855, + "end_time": "2024-03-23T02:43:36.753275", + "exception": false, + "start_time": "2024-03-23T02:43:36.730420", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:36.777638Z", + "iopub.status.busy": "2024-03-23T02:43:36.777388Z", + "iopub.status.idle": "2024-03-23T02:43:36.782020Z", + "shell.execute_reply": "2024-03-23T02:43:36.781186Z" + }, + "papermill": { + "duration": 0.018973, + "end_time": "2024-03-23T02:43:36.783868", + "exception": false, + "start_time": "2024-03-23T02:43:36.764895", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:36.808459Z", + "iopub.status.busy": "2024-03-23T02:43:36.808193Z", + "iopub.status.idle": "2024-03-23T02:43:37.246567Z", + "shell.execute_reply": "2024-03-23T02:43:37.245650Z" + }, + "papermill": { + "duration": 0.453098, + "end_time": "2024-03-23T02:43:37.248594", + "exception": false, + "start_time": "2024-03-23T02:43:36.795496", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:37.277295Z", + "iopub.status.busy": "2024-03-23T02:43:37.276943Z", + "iopub.status.idle": "2024-03-23T02:43:37.593531Z", + "shell.execute_reply": "2024-03-23T02:43:37.592656Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.33287, + "end_time": "2024-03-23T02:43:37.595617", + "exception": false, + "start_time": "2024-03-23T02:43:37.262747", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:37.623311Z", + "iopub.status.busy": "2024-03-23T02:43:37.622970Z", + "iopub.status.idle": "2024-03-23T02:43:37.724473Z", + "shell.execute_reply": "2024-03-23T02:43:37.723596Z" + }, + "papermill": { + "duration": 0.117602, + "end_time": "2024-03-23T02:43:37.726479", + "exception": false, + "start_time": "2024-03-23T02:43:37.608877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-23T02:43:37.753954Z", + "iopub.status.busy": "2024-03-23T02:43:37.753678Z", + "iopub.status.idle": "2024-03-23T02:43:38.163608Z", + "shell.execute_reply": "2024-03-23T02:43:38.162714Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.426011, + "end_time": "2024-03-23T02:43:38.165654", + "exception": false, + "start_time": "2024-03-23T02:43:37.739643", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:38.194496Z", + "iopub.status.busy": "2024-03-23T02:43:38.193448Z", + "iopub.status.idle": "2024-03-23T02:43:38.198024Z", + "shell.execute_reply": "2024-03-23T02:43:38.197317Z" + }, + "papermill": { + "duration": 0.020725, + "end_time": "2024-03-23T02:43:38.199881", + "exception": false, + "start_time": "2024-03-23T02:43:38.179156", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:38.226177Z", + "iopub.status.busy": "2024-03-23T02:43:38.225477Z", + "iopub.status.idle": "2024-03-23T02:43:38.232402Z", + "shell.execute_reply": "2024-03-23T02:43:38.231573Z" + }, + "papermill": { + "duration": 0.022101, + "end_time": "2024-03-23T02:43:38.234226", + "exception": false, + "start_time": "2024-03-23T02:43:38.212125", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9613961" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:38.261068Z", + "iopub.status.busy": "2024-03-23T02:43:38.260433Z", + "iopub.status.idle": "2024-03-23T02:43:38.349346Z", + "shell.execute_reply": "2024-03-23T02:43:38.348499Z" + }, + "papermill": { + "duration": 0.104146, + "end_time": "2024-03-23T02:43:38.351163", + "exception": false, + "start_time": "2024-03-23T02:43:38.247017", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 12] --\n", + "├─Adapter: 1-1 [2, 1071, 12] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 13,312\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,613,961\n", + "Trainable params: 9,613,961\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.08\n", + "========================================================================================================================\n", + "Input size (MB): 0.13\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.46\n", + "Estimated Total Size (MB): 346.07\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T02:43:38.381420Z", + "iopub.status.busy": "2024-03-23T02:43:38.381120Z", + "iopub.status.idle": "2024-03-23T03:35:44.027515Z", + "shell.execute_reply": "2024-03-23T03:35:44.026454Z" + }, + "papermill": { + "duration": 3125.685603, + "end_time": "2024-03-23T03:35:44.051068", + "exception": false, + "start_time": "2024-03-23T02:43:38.365465", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03592884845203823, 'avg_role_model_std_loss': 7.2996755396477075, 'avg_role_model_mean_pred_loss': 0.009430593892273626, 'avg_role_model_g_mag_loss': 0.007427950899711706, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.09437693562669058, 'n_size': 900, 'n_batch': 113, 'duration': 140.96822214126587, 'duration_batch': 1.2475063906306714, 'duration_size': 0.15663135793473987, 'avg_pred_std': 0.05326718922737425}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014524129554629325, 'avg_role_model_std_loss': 4.156348642464237, 'avg_role_model_mean_pred_loss': 0.0009876720450387843, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014524129554629325, 'n_size': 450, 'n_batch': 57, 'duration': 44.241886377334595, 'duration_batch': 0.7761734452163964, 'duration_size': 0.09831530306074354, 'avg_pred_std': 0.03724088621113384}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014371616853814986, 'avg_role_model_std_loss': 6.227466808899376, 'avg_role_model_mean_pred_loss': 0.00030516517503158583, 'avg_role_model_g_mag_loss': 6.165503883999514e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.2792855604737997, 'n_size': 900, 'n_batch': 113, 'duration': 141.62208795547485, 'duration_batch': 1.2532928137652641, 'duration_size': 0.15735787550608318, 'avg_pred_std': 0.03787480419155507}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01340457120962027, 'avg_role_model_std_loss': 5.29680332468604, 'avg_role_model_mean_pred_loss': 0.0002864654652569243, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01340457120962027, 'n_size': 450, 'n_batch': 57, 'duration': 45.41609525680542, 'duration_batch': 0.7967736009965863, 'duration_size': 0.10092465612623426, 'avg_pred_std': 0.030410112535352248}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01384979708948069, 'avg_role_model_std_loss': 7.497149070959221, 'avg_role_model_mean_pred_loss': 0.00021502181714296655, 'avg_role_model_g_mag_loss': 1.949448532994009e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.11678517651847667, 'n_size': 900, 'n_batch': 113, 'duration': 143.15943694114685, 'duration_batch': 1.2668976720455474, 'duration_size': 0.15906604104571873, 'avg_pred_std': 0.033135497126629394}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014283642331914355, 'avg_role_model_std_loss': 5.459189098905386, 'avg_role_model_mean_pred_loss': 0.0005690211627043279, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014283642331914355, 'n_size': 450, 'n_batch': 57, 'duration': 46.33096122741699, 'duration_batch': 0.8128238811827543, 'duration_size': 0.1029576916164822, 'avg_pred_std': 0.026298031341611294}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013210438237422042, 'avg_role_model_std_loss': 6.6640938428891605, 'avg_role_model_mean_pred_loss': 0.00014334700532385921, 'avg_role_model_g_mag_loss': 4.62662972798474e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.2998970259560479, 'n_size': 900, 'n_batch': 113, 'duration': 146.3652093410492, 'duration_batch': 1.2952673393013203, 'duration_size': 0.16262801037894356, 'avg_pred_std': 0.03572286798132468}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013382001433314548, 'avg_role_model_std_loss': 4.643842804434571, 'avg_role_model_mean_pred_loss': 0.00048086737289648024, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013382001433314548, 'n_size': 450, 'n_batch': 57, 'duration': 47.94945430755615, 'duration_batch': 0.8412184966237921, 'duration_size': 0.10655434290568033, 'avg_pred_std': 0.029577646339148805}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013310110395153363, 'avg_role_model_std_loss': 6.041971506328766, 'avg_role_model_mean_pred_loss': 0.00019507275757620039, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013310110395153363, 'n_size': 900, 'n_batch': 113, 'duration': 143.80060958862305, 'duration_batch': 1.2725717662710003, 'duration_size': 0.15977845509847005, 'avg_pred_std': 0.038453474558428326}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013661402131223844, 'avg_role_model_std_loss': 4.1234811326556935, 'avg_role_model_mean_pred_loss': 0.00045455218989546767, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013661402131223844, 'n_size': 450, 'n_batch': 57, 'duration': 45.394694089889526, 'duration_batch': 0.7963981419278864, 'duration_size': 0.10087709797753228, 'avg_pred_std': 0.0333288926100195}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013588898373353812, 'avg_role_model_std_loss': 7.486738423658624, 'avg_role_model_mean_pred_loss': 0.0002779256184064716, 'avg_role_model_g_mag_loss': 0.00022828346642199903, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.08210628287142349, 'n_size': 900, 'n_batch': 113, 'duration': 141.604829788208, 'duration_batch': 1.2531400866213098, 'duration_size': 0.15733869976467557, 'avg_pred_std': 0.03336982285620364}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.015309163269638602, 'avg_role_model_std_loss': 11.10999310352408, 'avg_role_model_mean_pred_loss': 0.0006529700210521443, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015309163269638602, 'n_size': 450, 'n_batch': 57, 'duration': 45.059311866760254, 'duration_batch': 0.7905142432764957, 'duration_size': 0.10013180414835612, 'avg_pred_std': 0.01890142163948009}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01360072170642929, 'avg_role_model_std_loss': 9.74690950918578, 'avg_role_model_mean_pred_loss': 0.0002635270801760223, 'avg_role_model_g_mag_loss': 9.69009121440144e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 3.451531191302153, 'n_size': 900, 'n_batch': 113, 'duration': 140.73290538787842, 'duration_batch': 1.2454239414856496, 'duration_size': 0.15636989487542047, 'avg_pred_std': 0.029921204000052097}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013079761469271034, 'avg_role_model_std_loss': 5.622179901249935, 'avg_role_model_mean_pred_loss': 0.00037308515279698264, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013079761469271034, 'n_size': 450, 'n_batch': 57, 'duration': 43.94645428657532, 'duration_batch': 0.7709904260802687, 'duration_size': 0.09765878730350071, 'avg_pred_std': 0.024666702521866875}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013433132627978921, 'avg_role_model_std_loss': 7.019836858438262, 'avg_role_model_mean_pred_loss': 0.0001345649655801632, 'avg_role_model_g_mag_loss': 2.2781116680966485e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.16974145690082676, 'n_size': 900, 'n_batch': 113, 'duration': 139.68711066246033, 'duration_batch': 1.2361691209067285, 'duration_size': 0.15520790073606702, 'avg_pred_std': 0.034294621425524224}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013359390444432696, 'avg_role_model_std_loss': 5.3989552207959495, 'avg_role_model_mean_pred_loss': 0.00037170510294370413, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013359390444432696, 'n_size': 450, 'n_batch': 57, 'duration': 44.08630442619324, 'duration_batch': 0.7734439373016357, 'duration_size': 0.09796956539154053, 'avg_pred_std': 0.028002551381002393}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013708434053179291, 'avg_role_model_std_loss': 8.164382093871072, 'avg_role_model_mean_pred_loss': 0.00011137482917280241, 'avg_role_model_g_mag_loss': 0.0002664083573553297, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02415950643726521, 'n_size': 900, 'n_batch': 113, 'duration': 140.14124727249146, 'duration_batch': 1.2401880289601013, 'duration_size': 0.15571249696943495, 'avg_pred_std': 0.03384085310688984}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013566718476617503, 'avg_role_model_std_loss': 6.5259337021869435, 'avg_role_model_mean_pred_loss': 0.00033163626410593374, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013566718476617503, 'n_size': 450, 'n_batch': 57, 'duration': 43.685551166534424, 'duration_batch': 0.766413178360253, 'duration_size': 0.09707900259229872, 'avg_pred_std': 0.027984272067745525}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013571654115286139, 'avg_role_model_std_loss': 8.742722461416001, 'avg_role_model_mean_pred_loss': 0.0003018974633938696, 'avg_role_model_g_mag_loss': 1.6721362351543374e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.027267425186518167, 'n_size': 900, 'n_batch': 113, 'duration': 140.09695625305176, 'duration_batch': 1.2397960730358564, 'duration_size': 0.15566328472561305, 'avg_pred_std': 0.030262669457732577}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01325964125830473, 'avg_role_model_std_loss': 5.029873872576148, 'avg_role_model_mean_pred_loss': 0.00043549182230303055, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01325964125830473, 'n_size': 450, 'n_batch': 57, 'duration': 43.787155866622925, 'duration_batch': 0.7681957169582969, 'duration_size': 0.09730479081471761, 'avg_pred_std': 0.025762035086620273}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014379269344628685, 'avg_role_model_std_loss': 7.366559921767701, 'avg_role_model_mean_pred_loss': 0.00037252366738006987, 'avg_role_model_g_mag_loss': 0.0003263944470220142, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.017774089858867227, 'n_size': 900, 'n_batch': 113, 'duration': 142.94084358215332, 'duration_batch': 1.2649632175411798, 'duration_size': 0.1588231595357259, 'avg_pred_std': 0.035403397502954556}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013996612014921589, 'avg_role_model_std_loss': 7.5169032518093255, 'avg_role_model_mean_pred_loss': 0.0005061832475970126, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013996612014921589, 'n_size': 450, 'n_batch': 57, 'duration': 46.45759129524231, 'duration_batch': 0.8150454613200405, 'duration_size': 0.10323909176720514, 'avg_pred_std': 0.02128785624773356}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013456141208298505, 'avg_role_model_std_loss': 8.885624297614187, 'avg_role_model_mean_pred_loss': 0.0001408637665720865, 'avg_role_model_g_mag_loss': 6.312208974526988e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.12043976681565659, 'n_size': 900, 'n_batch': 113, 'duration': 140.75263118743896, 'duration_batch': 1.2455985060835306, 'duration_size': 0.15639181243048775, 'avg_pred_std': 0.03045645008374632}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.015452116089096914, 'avg_role_model_std_loss': 10.887882433234672, 'avg_role_model_mean_pred_loss': 0.0007382755149755995, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015452116089096914, 'n_size': 450, 'n_batch': 57, 'duration': 44.30102300643921, 'duration_batch': 0.7772109299375299, 'duration_size': 0.09844671779208714, 'avg_pred_std': 0.015029358677566051}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013939871930827697, 'avg_role_model_std_loss': 10.302565499742524, 'avg_role_model_mean_pred_loss': 0.00030740466281282784, 'avg_role_model_g_mag_loss': 1.2590549886226655e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.05802687374461028, 'n_size': 900, 'n_batch': 113, 'duration': 140.6320457458496, 'duration_batch': 1.244531378281855, 'duration_size': 0.15625782860649956, 'avg_pred_std': 0.026154642290048366}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013364347005262971, 'avg_role_model_std_loss': 4.240572321283827, 'avg_role_model_mean_pred_loss': 0.0003598822188279074, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013364347005262971, 'n_size': 450, 'n_batch': 57, 'duration': 44.313255310058594, 'duration_batch': 0.777425531755414, 'duration_size': 0.0984739006890191, 'avg_pred_std': 0.03320887988727344}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013616377720609307, 'avg_role_model_std_loss': 7.797813521520289, 'avg_role_model_mean_pred_loss': 0.0002454437854082967, 'avg_role_model_g_mag_loss': 8.744494782553778e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013847984937537047, 'n_size': 900, 'n_batch': 113, 'duration': 141.74561762809753, 'duration_batch': 1.2543859967088278, 'duration_size': 0.15749513069788615, 'avg_pred_std': 0.03314163033084004}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014651317608594481, 'avg_role_model_std_loss': 6.794933559461737, 'avg_role_model_mean_pred_loss': 0.0006282155579800827, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014651317608594481, 'n_size': 450, 'n_batch': 57, 'duration': 47.70662569999695, 'duration_batch': 0.8369583456139815, 'duration_size': 0.10601472377777099, 'avg_pred_std': 0.02225242922768781}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013695524584295021, 'avg_role_model_std_loss': 8.25407733499283, 'avg_role_model_mean_pred_loss': 0.00019777481913188744, 'avg_role_model_g_mag_loss': 1.3750431502962278e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.021342603873668445, 'n_size': 900, 'n_batch': 113, 'duration': 142.06192708015442, 'duration_batch': 1.2571851953995967, 'duration_size': 0.157846585644616, 'avg_pred_std': 0.03127054079328623}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013534503092782365, 'avg_role_model_std_loss': 4.2766512755933626, 'avg_role_model_mean_pred_loss': 0.00026410735567626234, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013534503092782365, 'n_size': 450, 'n_batch': 57, 'duration': 44.758880853652954, 'duration_batch': 0.7852435237482974, 'duration_size': 0.09946417967478434, 'avg_pred_std': 0.03563740865833927}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013962997518893745, 'avg_role_model_std_loss': 8.13189057305249, 'avg_role_model_mean_pred_loss': 0.00029005304642092415, 'avg_role_model_g_mag_loss': 2.4200141843822267e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015241746428526111, 'n_size': 900, 'n_batch': 113, 'duration': 141.51434302330017, 'duration_batch': 1.252339318790267, 'duration_size': 0.15723815891477796, 'avg_pred_std': 0.03115090290166899}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013381186781658066, 'avg_role_model_std_loss': 5.749990332730752, 'avg_role_model_mean_pred_loss': 0.0003508059210490602, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013381186781658066, 'n_size': 450, 'n_batch': 57, 'duration': 44.66229581832886, 'duration_batch': 0.7835490494443659, 'duration_size': 0.09924954626295301, 'avg_pred_std': 0.027999498015433028}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.019623516283574557, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006512481076056464, 'pred_duration': 2.325235366821289, 'grad_duration': 6.460307598114014, 'total_duration': 8.785542964935303, 'pred_std': 0.03438406065106392, 'std_loss': 1.4360365867614746, 'mean_pred_loss': 9.540035534882918e-06, 'pred_rmse': 0.14008395373821259, 'pred_mae': 0.09191913157701492, 'pred_mape': 4.1357741355896, 'grad_rmse': 0.1400328129529953, 'grad_mae': 0.0917934775352478, 'grad_mape': 0.9963167309761047}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.019623516283574557, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006512481076056464, 'avg_pred_duration': 2.325235366821289, 'avg_grad_duration': 6.460307598114014, 'avg_total_duration': 8.785542964935303, 'avg_pred_std': 0.03438406065106392, 'avg_std_loss': 1.4360365867614746, 'avg_mean_pred_loss': 9.540035534882918e-06}, 'min_metrics': {'avg_loss': 0.019623516283574557, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006512481076056464, 'pred_duration': 2.325235366821289, 'grad_duration': 6.460307598114014, 'total_duration': 8.785542964935303, 'pred_std': 0.03438406065106392, 'std_loss': 1.4360365867614746, 'mean_pred_loss': 9.540035534882918e-06, 'pred_rmse': 0.14008395373821259, 'pred_mae': 0.09191913157701492, 'pred_mape': 4.1357741355896, 'grad_rmse': 0.1400328129529953, 'grad_mae': 0.0917934775352478, 'grad_mape': 0.9963167309761047}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.019623516283574557, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006512481076056464, 'pred_duration': 2.325235366821289, 'grad_duration': 6.460307598114014, 'total_duration': 8.785542964935303, 'pred_std': 0.03438406065106392, 'std_loss': 1.4360365867614746, 'mean_pred_loss': 9.540035534882918e-06, 'pred_rmse': 0.14008395373821259, 'pred_mae': 0.09191913157701492, 'pred_mape': 4.1357741355896, 'grad_rmse': 0.1400328129529953, 'grad_mae': 0.0917934775352478, 'grad_mape': 0.9963167309761047}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:35:44.087866Z", + "iopub.status.busy": "2024-03-23T03:35:44.087216Z", + "iopub.status.idle": "2024-03-23T03:35:44.091777Z", + "shell.execute_reply": "2024-03-23T03:35:44.090708Z" + }, + "papermill": { + "duration": 0.025693, + "end_time": "2024-03-23T03:35:44.094065", + "exception": false, + "start_time": "2024-03-23T03:35:44.068372", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:35:44.127678Z", + "iopub.status.busy": "2024-03-23T03:35:44.127393Z", + "iopub.status.idle": "2024-03-23T03:35:44.224252Z", + "shell.execute_reply": "2024-03-23T03:35:44.223005Z" + }, + "papermill": { + "duration": 0.118083, + "end_time": "2024-03-23T03:35:44.228259", + "exception": false, + "start_time": "2024-03-23T03:35:44.110176", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:35:44.273179Z", + "iopub.status.busy": "2024-03-23T03:35:44.272668Z", + "iopub.status.idle": "2024-03-23T03:35:44.602463Z", + "shell.execute_reply": "2024-03-23T03:35:44.601470Z" + }, + "papermill": { + "duration": 0.3522, + "end_time": "2024-03-23T03:35:44.604985", + "exception": false, + "start_time": "2024-03-23T03:35:44.252785", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAESCAYAAAAv/mqQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4UElEQVR4nO3deXhTZdo/8O9JmqR7WpZuUEqVskPZpLQKqFQLIlKHUeRlLDoCL77AiIzjTFWKy2hVZESFl+VlpINaQR3B+aGyWCgIlKVAsWyVQmlBmrYsTZouSZrz/P5IT9rQJM3eJffnus7V9uQ5OU/S9O5z7mc5HGOMgRBCPEDU3hUghHgPCjiEEI+hgEMI8RgKOIQQj6GAQwjxGAo4hBCPoYBDCPEYn/augC14nsf169cRFBQEjuPauzqEkBYYY6ipqUFUVBREIuttmE4RcK5fv47o6Oj2rgYhxIqrV6+id+/eVst0ioATFBQEwPCCgoOD27k2hJCWVCoVoqOjjX+n1nSKgCNcRgUHB1PAIaSDsiXdQUljQojH2BVw1q5di+HDhxtbGomJifjxxx8tls/KygLHcSabr6+v05UmhHROdl1S9e7dG++++y7i4uLAGMO//vUvTJ8+HadOncKQIUPMHhMcHIyioiLjz9TLRIj3sivgTJs2zeTnt99+G2vXrsWRI0csBhyO4xAREeF4DQkhXYbDORy9Xo8tW7agtrYWiYmJFsup1WrExMQgOjoa06dPx9mzZ9t8bo1GA5VKZbIRQjo/uwNOYWEhAgMDIZPJsGDBAmzbtg2DBw82W3bAgAH49NNP8d133+Hzzz8Hz/NISkrCtWvXrJ4jMzMTcrncuNEYHEK6Bs7eFf+0Wi3KysqgVCrxzTffYOPGjdi/f7/FoNOSTqfDoEGDMGvWLLz11lsWy2k0Gmg0GuPPQj+/UqmkbnEPu3qrDiH+EgT5Stq7KqSDUqlUkMvlNv192j0ORyqVol+/fgCA0aNH4/jx4/joo4+wfv36No+VSCQYOXIkiouLrZaTyWSQyWT2Vo24WLmyHvd/kItRfULw9YKk9q4O6QKcHofD87xJa8QavV6PwsJCREZGOnta4gGXq2qh5xl+rVC3d1VIF2FXCyc9PR1TpkxBnz59UFNTg+zsbOTm5mLXrl0AgLS0NPTq1QuZmZkAgDfffBPjxo1Dv379UF1djRUrVqC0tBRz5851/SshLqes1wEAVA066HkGsYiGNBDn2BVwKisrkZaWhvLycsjlcgwfPhy7du3CQw89BAAoKyszmS16+/ZtzJs3DwqFAqGhoRg9ejQOHz5sU76HtL/qOkPAYQyoadAhxF/azjUinZ3dSeP2YE9SirjO2txLeG/nBQBA7kv3o2+PgHauEemI7Pn7pLlUxKLqem2L73XtWBPSVVDAIRapWgSZ6jqtlZKE2IYCDrFIyOEAzQlkQpxBAYdYpDRp4VDAIc6jgEMsahlkKOAQV6CAQywyaeHUUw6HOI8CDrGoZcBRUguHuAAFHGKWTs9DrWk0/kzd4sQVKOAQs1R3BBjqFieuQAGHmHVni4ZaOMQVKOAQs+4cd0M5HOIKFHCIWUKA6RFoWJeoul6HTjDtjnRwFHCIWUILp293fwCAnmcmSWRCHEEBh5glJInDg30h8xE17aPLKuIcCjjELGW9oTUj95cgxF/StI8CDnEOBRxiljCyWO4nQYifYeEtauEQZ1HAIWYJrZkQPwnkTS0cmt5AnEUBh5gl9FIZWjhNAYdaOMRJFHCIWcYWDuVwiAtRwCFmCSOLg/0kxsXTaXoDcZZdAWft2rUYPnw4goODERwcjMTERPz4449Wj/n6668xcOBA+Pr6YtiwYfjhhx+cqjDxjOYcjhRyuqQiLmJXwOnduzfeffddnDhxAvn5+XjwwQcxffp0nD171mz5w4cPY9asWXjuuedw6tQppKamIjU1FWfOnHFJ5Yl7MMaaczgtLqloPhVxll0BZ9q0aXjkkUcQFxeH/v374+2330ZgYCCOHDlitvxHH32EyZMn4y9/+QsGDRqEt956C6NGjcLq1autnkej0UClUplsxHMadDy0eh6AoZdK6Ban+VTEWQ7ncPR6PbZs2YLa2lokJiaaLZOXl4fk5GSTfSkpKcjLy7P63JmZmZDL5cYtOjra0WoSBwjd3z4iDv5ScYsWDuVwiHPsDjiFhYUIDAyETCbDggULsG3bNot30lQoFAgPDzfZFx4eDoVCYfUc6enpUCqVxu3q1av2VpM4QcjVhPhLwHEc5XCIy9h1q18AGDBgAAoKCqBUKvHNN99gzpw52L9/v0tv3yuTySCTyVz2fMQ+yhY9VABMcjiMMXAc3WOcOMbugCOVStGvXz8AwOjRo3H8+HF89NFHWL9+fauyERERqKioMNlXUVGBiIgIB6tLPMHYwjEGHEMOR9vIo0HHw08qbre6kc7N6XE4PM9Do9GYfSwxMRE5OTkm+/bs2WMx50M6BpVx0J8h0ARIxfARGVo1lMchzrCrhZOeno4pU6agT58+qKmpQXZ2NnJzc7Fr1y4AQFpaGnr16oXMzEwAwAsvvICJEydi5cqVmDp1KrZs2YL8/Hxs2LDB9a+EuEzLiZsAwHEcQvwluKHWorpOh0i5X3tWj3RidgWcyspKpKWloby8HHK5HMOHD8euXbvw0EMPAQDKysogEjU3mpKSkpCdnY3XXnsNr7zyCuLi4rB9+3YMHTrUta+CuJSQwxECjvC9EHAIcZRdAeef//yn1cdzc3Nb7XviiSfwxBNP2FUp0r6q61oHHMPlVS2UdElFnEBzqUgrLSduCmjGOHEFCjikFbOXVDS9gbgABRzSivkWDq36R5xHAYe0Yj6HI6yJQzkc4jgKOKSV5ksqqXGfcbQxtXCIEyjgEBM8z6BqMN8tDlDAIc6hgENM1DQ0QrjBZutucUoaE+dQwCEmhFHG/lIxpD7NHw+hW1xJy4wSJ1DAISZa3h6mJVr1j7gCBRxiQsjRBN8ZcJoSyHVaPTSNeo/Xi3QNFHCIiWozY3AAIMjXB8IyOHS7GOIoCjjEhLlRxgAgEjWv/EdrGxNHUcAhJoSkcEiLMTgC43wqauEQB1HAISaMLZw7LqkM+2h6A3EOBRxiwty0BkHzjHHqGieOoYBDTFjK4QCge4wTp1HAISYs9VIBtCYOcR4FHGJC1eKe4ncy5nBoxjhxEAUcYsK2HA61cIhj7Ao4mZmZuOeeexAUFISwsDCkpqaiqKjI6jFZWVngOM5k8/X1darSxH3MLb4loBwOcZZdAWf//v1YuHAhjhw5gj179kCn0+Hhhx9GbW2t1eOCg4NRXl5u3EpLS52qNHEPTaMe9TrDtIU7pzYAtCYOcZ5dd23YuXOnyc9ZWVkICwvDiRMnMGHCBIvHcRxHd9vsBISWi4gDgmStPxrCglyUwyGOciqHo1QqAQDdunWzWk6tViMmJgbR0dGYPn06zp49a7W8RqOBSqUy2Yj7KVtM3BSJWt8/nFo4xFkOBxye57FkyRLce++9Vm9sN2DAAHz66af47rvv8Pnnn4PneSQlJeHatWsWj8nMzIRcLjdu0dHRjlaT2MHS0hQCYX9NQyMa9bzH6kW6DocDzsKFC3HmzBls2bLFarnExESkpaVhxIgRmDhxIr799lv07NkT69evt3hMeno6lEqlcbt69aqj1SR2sNZDded+VUOjR+pEuha7cjiCRYsWYceOHThw4AB69+5t17ESiQQjR45EcXGxxTIymQwymcyRqhEnVBvnUbUegwMAPmIRgmQ+qNE0orpOi24B5ssRYoldLRzGGBYtWoRt27Zh7969iI2NtfuEer0ehYWFiIyMtPtY4l7WpjUI6IZ4xBl2BZyFCxfi888/R3Z2NoKCgqBQKKBQKFBfX28sk5aWhvT0dOPPb775Jnbv3o3Lly/j5MmT+MMf/oDS0lLMnTvXda+CuETz0hSWA45xLA4ljokD7LqkWrt2LQDg/vvvN9m/adMmPPPMMwCAsrIyiETNcez27duYN28eFAoFQkNDMXr0aBw+fBiDBw92rubE5Wxp4YRQ1zhxgl0Bhwn3D7EiNzfX5OcPP/wQH374oV2VIu3D2sRNgZy6xokTaC4VMRJaOOZGGQtoPhVxBgUcYiQEEZtyOJQ0Jg6ggEOMVPbkcGjVP+IACjjEqDmHY3l8DXWLE2dQwCEADB0CtvVSUQ6HOI4CDgEAqDWN0POGXkhrvVRC64dyOMQRFHAIgOYAIvMRwVcitliuecY45XCI/SjgEABtT9wUCJdUynodeL7tcVmEtEQBhwBosXi6lcspoHmMDs+AGg3NGCf2oYBDALSYKd5GC8dXIoZf0yUXzaci9qKAQwC0nEfV9pITxjwOzacidqKAQwDYnsNpWYa6xom9KOAQAM2tlbZyOC3L0OA/Yi8KOASAbdMaBML0BiV1jRM7UcAhAFpM3LSnhUOXVMROFHAIANsW3xLQfCriKIcWUSedA2MMjY2N0Ov1bZb1YY3oFSRGqAxoaGiwWjYiQIxeQWLwOm2bZUnXIJFIIBZbHoFuK47ZsoxfO1OpVJDL5VAqlQgODm7v6nQKWq0W5eXlqKurs6m8QtmARp4hLEgGqY/1hm+tphG363Twk4jQPZDuruENOI5D7969ERgY2Ooxe/4+qYXTBfE8j5KSEojFYkRFRUEqlYLjWt9Js6XGihrwjCG2ewCkVuZSAUBNvRYSZQP8JGL06R7gyqqTDogxhqqqKly7dg1xcXFOtXQo4HRBWq0WPM8jOjoa/v7+bZbnGQMTN4AD4O/vBx+x9RZOI8TgankwsRi+vr4uqjXpyHr27IkrV65Ap9M5FXDsShpnZmbinnvuQVBQEMLCwpCamoqioqI2j/v6668xcOBA+Pr6YtiwYfjhhx8crjCxXcu7Z1jTchKm2Mw9xe8kbnpePU3e9BpttZBtZVfA2b9/PxYuXIgjR45gz5490Ol0ePjhh1FbW2vxmMOHD2PWrFl47rnncOrUKaSmpiI1NRVnzpxxuvLENYTAIeY4mz5YQlDSM2bTnTwIETiVNK6qqkJYWBj279+PCRMmmC0zc+ZM1NbWYseOHcZ948aNw4gRI7Bu3Tqzx2g0Gmg0GuPPKpUK0dHRlDS2UUNDA0pKShAbG2vTJU+tphGXqtSQikUYGNn2+6vnGc5eVwIAhkTJbWoVkc7N2mfKnqSxU+NwlErDh65bt24Wy+Tl5SE5OdlkX0pKCvLy8iwek5mZCblcbtyio6OdqSZpg77pf46tgUPENTexu+JlFcdx2L59e3tXw6Vef/11jBgxor2r4XjA4XkeS5Yswb333ouhQ4daLKdQKBAeHm6yLzw8HAqFwuIx6enpUCqVxu3q1auOVpPYwHhJZWPA4Tiu+bKK591WL2+WlZWFkJAQlz3fSy+9hJycHJc9n6Mc7qVauHAhzpw5g4MHD7qyPgAAmUwGmYzGd3iKvQEHAHw4Do3omi2czkSr1UIqbXtJkcDAQLNjaDzNoRbOokWLsGPHDuzbtw+9e/e2WjYiIgIVFRUm+yoqKhAREeHIqYmDGGOo0zaa3dQNjWjQ6aFt5C2WuXPT6nk06PSoaWi7rL1pwp07d+K+++5DSEgIunfvjkcffRSXLl0CACQlJeGvf/2rSfmqqipIJBIcOHAAAFBeXo6pU6fCz88PsbGxyM7ORt++fbFq1SqH3rvCwkI8+OCD8PPzQ/fu3TF//nyo1Wrj47m5uRg7diwCAgIQEhKCe++9F6WlpQCA06dP44EHHkBQUBCCg4MxevRo5OfnWz1fbm4unn32WSiVSnBNifzXX38dANC3b1+89dZbSEtLQ3BwMObPnw8A+Otf/4r+/fvD398fd911F5YtWwadrnnqyZ2XVM888wxSU1PxwQcfIDIyEt27d8fChQtNjnEHu+8tvnjxYmzbtg25ubmIjY1t85jExETk5ORgyZIlxn179uxBYmKi3ZUljqvX6TE4Y1e7nPvcmynwl9r+UautrcXSpUsxfPhwqNVqZGRk4PHHH0dBQQFmz56N999/H++++64xj7R161ZERUVh/PjxAIC0tDTcuHEDubm5kEgkWLp0KSorKx2qe21tLVJSUpCYmIjjx4+jsrISc+fOxaJFi5CVlYXGxkakpqZi3rx5+PLLL6HVanHs2DFj3WbPno2RI0di7dq1EIvFKCgogERifb5aUlISVq1ahYyMDOOwk5atkw8++AAZGRlYvny5cV9QUBCysrIQFRWFwsJCzJs3D0FBQXj55Zctnmffvn2IjIzEvn37UFxcjJkzZ2LEiBGYN2+eQ++VLewKOAsXLkR2dja+++47BAUFGfMwcrkcfn5+AAy/7F69eiEzMxMA8MILL2DixIlYuXIlpk6dii1btiA/Px8bNmxw8UshXcWMGTNMfv7000/Rs2dPnDt3Dk8++SSWLFmCgwcPGgNMdnY2Zs2aBY7jcOHCBfz00084fvw4xowZAwDYuHEj4uLiHKpLdnY2GhoasHnzZgQEGEZVr169GtOmTcN7770HiUQCpVKJRx99FHfffTcAYNCgQcbjy8rK8Je//AUDBw4EAJvqIZVKIZfLwXGc2SuBBx98EH/+859N9r322mvG7/v27YuXXnoJW7ZssRpwQkNDsXr1aojFYgwcOBBTp05FTk5Oxwk4a9euBQDcf//9Jvs3bdqEZ555BoDhDW454CwpKQnZ2dl47bXX8MorryAuLg7bt2+3mmgmrucnEePcmylmHyu9UYcajQ69QvwQGtB2PgAAyqsbcLNWgx6BMkTIrXe9+7UxVeJOFy9eREZGBo4ePYobN26Ab0pMl5WVYejQoXj44YfxxRdfYPz48SgpKUFeXh7Wr18PACgqKoKPjw9GjRplfL5+/fohNDTUrjoIzp8/j/j4eGOwAYB7770XPM+jqKgIEyZMwDPPPIOUlBQ89NBDSE5OxpNPPonIyEgAwNKlSzF37lx89tlnSE5OxhNPPGEMTI4SAmlLW7duxccff4xLly5BrVajsbGxzS7qIUOGmIwajoyMRGFhoVN1a4tdORzWNNDrzk0INoDh+jMrK8vkuCeeeAJFRUXQaDQ4c+YMHnnkEVfUndiB4zj4S33MbtKme1EFysw/bm4L8vWBr0QMqVjUZll7R6lOmzYNt27dwv/93//h6NGjOHr0KABDghQwXKZ888030Ol0yM7OxrBhwzBs2DCXv2e22rRpE/Ly8pCUlIStW7eif//+OHLkCABD7uTs2bOYOnUq9u7di8GDB2Pbtm1Ona9l8AMMQ09mz56NRx55BDt27MCpU6fw6quvGt8vS+68tOM4zhjc3YXWwyFodKCXquVoY1e6efMmioqK8Nprr2HSpEkYNGgQbt++bVJm+vTpaGhowM6dO5GdnY3Zs2cbHxswYAAaGxtx6tQp477i4uJWz2GrQYMG4fTp0yaj6Q8dOgSRSIQBAwYY940cORLp6ek4fPgwhg4diuzsbONj/fv3x4svvojdu3fjd7/7HTZt2tTmeaVSqU3LigCG0fwxMTF49dVXMWbMGMTFxRmT1h0NBRxi98C/lmUbXdwtHhoaiu7du2PDhg0oLi7G3r17sXTpUpMyAQEBSE1NxbJly3D+/HnMmjXL+NjAgQORnJyM+fPn49ixYzh16hTmz58PPz8/h+YDzZ49G76+vpgzZw7OnDmDffv2YfHixXj66acRHh6OkpISpKenIy8vD6Wlpdi9ezcuXryIQYMGob6+HosWLUJubi5KS0tx6NAhHD9+3CTHY0nfvn2hVquRk5ODGzduWF1mJC4uDmVlZdiyZQsuXbqEjz/+2OlWlLtQwPFyjLEW43Bs/zg0D/xzbcARiUTYsmULTpw4gaFDh+LFF1/EihUrWpWbPXs2Tp8+jfHjx6NPnz4mj23evBnh4eGYMGECHn/8cWOPjSMz2/39/bFr1y7cunUL99xzD37/+99j0qRJWL16tfHxCxcuYMaMGejfvz/mz5+PhQsX4r//+78hFotx8+ZNpKWloX///njyyScxZcoUvPHGG22eNykpCQsWLMDMmTPRs2dPvP/++xbLPvbYY3jxxRexaNEijBgxAocPH8ayZcvsfq2eQAtwdUH2zKXieYYzDsyLqtM2orhSDYlYhEE2zL9qT9euXUN0dDR++uknTJo0qb2r0ym5ai4VrYfj5YRLIg4c7JmD6eOmFo4r7N27F2q1GsOGDUN5eTlefvll9O3b1+IEY+I5dEnl5Vrmb+zJcQgtIZ4xk/V0OgKdTodXXnkFQ4YMweOPP46ePXsaBwF+8cUXxmH+d25DhgzxWB2nTJlisR7vvPOOx+rhadTC8XKOzKMCABHHgQMHBgY9YxCh4yxRkZKSgpQU82OOHnvsMSQkJJh9rK0RwK60ceNG1NfXm33M2uoLnR0FHC/naMARZow38oaks51j+9pNUFAQgoKC2rsa6NWrV3tXoV3QJZWXczTgtDzG1V3jpOuigOPlXBFwOmLimHRMFHC8nLCAlg8FHOIBFHC8nBAsRA4EHB9a9Y/YiQKOlxMCjo8Dw/6phUPsRQHHyzkycVPQVZPGXXER9Y6CAo6Xc2TipoBaOO7j6kXUAcPSMRzHobq62qXPaw8KOF6OeqmIJ1HA8RaMAdraVhuvqQWnq4O4sc7s49Y2cWMdOF0deE0bZWkRdZctoq7RaPDSSy+hV69eCAgIQEJCAnJzc43HlpaWYtq0aQgNDUVAQACGDBmCH374AVeuXMEDDzwAwLAECMdxJgvneQqNNPYWujrgnahWuwc78ZTBAGxaZ++V64A0oO1yTWgRdcuLqC9atAjnzp3Dli1bEBUVhW3btmHy5MkoLCxEXFwcFi5cCK1WiwMHDiAgIADnzp1DYGAgoqOj8e9//xszZsxAUVERgoODjeuQexIFHNLh0CLq5hdRLysrw6ZNm1BWVoaoKMM/j5deegk7d+7Epk2b8M4776CsrAwzZswwLrl61113GY8X5miFhYW5PD9kK7sDzoEDB7BixQqcOHEC5eXl2LZtG1JTUy2Wz83NNTblWiovL6d7U3mSxN/Q0mhB06jHrxVqiDgOQ6LsX9OmUc/jvKIGADAkKhgiS13rEn+7npcWUTevsLAQer0e/fv3N9mv0WjQvXt3AMCf/vQnPP/889i9ezeSk5MxY8YMDB8+3KHzuYPdOZza2lrEx8djzZo1dh1XVFSE8vJy4xYWFmbvqYkzOM5wWdNi0/v4g0n8IZIFtHrMlk3sGwgmMTyH3sffcllaRN0li6ir1WqIxWKcOHECBQUFxu38+fP46KOPAABz587F5cuX8fTTT6OwsBBjxozBJ5984rLX6jTmBABs27ZtVsvs27ePAWC3b992+DxKpZIBYEql0uHn8Cb19fXs3LlzrL6+3mo5Vb2Wnb56mxUpVA6f68xv1ez01dusXtvo8HO0dOPGDQaAHThwwLjv559/NvmsqdVqFhAQwP7zn/+wwYMHs3fffddY9vz58wwAy8/PN+67ePEiA8A+/PBDm+rQ8lwbNmxgoaGhTK1WGx///vvvmUgkYgqFwuzx48aNY4sXLzb72FNPPcWmTZvWZh2++OILFhgYaLKvqKio1XvTlr/97W9s2LBhjDHGDh06xACwGzdu2Hy8wNpnyp6/T4/1Uo0YMQKRkZF46KGHcOjQIatlNRoNVCqVyUZcz5kucYGru8ZpEXUDc4uo9+/fH7Nnz0ZaWhq+/fZblJSU4NixY8jMzMT3338PAFiyZAl27dqFkpISnDx5Evv27TOeLyYmBhzHYceOHaiqqjLpafMYu0NdC7ChhXPhwgW2bt06lp+fzw4dOsSeffZZ5uPjw06cOGHxmOXLlzMArTZq4djG1hbOjZoGdvrqbXblhtpqOWt+rVCx01dvM2Wd1uHnuNOePXvYoEGDmEwmY8OHD2e5ubmtPms//PADA8AmTJjQ6vjr16+zKVOmMJlMxmJiYlh2djYLCwtj69ats+n8d57rl19+YQ888ADz9fVl3bp1Y/PmzWM1NTWMMcYUCgVLTU1lkZGRTCqVspiYGJaRkcH0ej3TaDTsqaeeYtHR0UwqlbKoqCi2aNGiNn8vggULFrDu3bszAGz58uWMMca0Wi3LyMhgffv2ZRKJhEVGRrLHH3+c/fLLL4wxxhYtWsTuvvtuJpPJWM+ePdnTTz9t0qJ58803WUREBOM4js2ZM8emejDmuhaOU4uocxzXZtLYnIkTJ6JPnz747LPPzD6u0Wig0WiMP6tUKkRHR9Mi6jaydRH1SlUDFKoGdPOXonc3+xK7gstVaqg1jYgO9bf5rp2eRouoO69TL6I+duxYHDx40OLjMpkMMpnMgzXyTsZLKrHjl1Q+TbeW6UjzqWgR9Y6rXUYaFxQUGLsNSfsxBhwHchuCjji9gRZR77jsbuGo1WoUFxcbfy4pKUFBQQG6deuGPn36ID09Hb/99hs2b94MAFi1ahViY2MxZMgQNDQ0YOPGjdi7dy92797tuldBHOLMTHGBu2756wxaRL3jsjvg5OfnmwzkE3oQ5syZg6ysLJSXl6OsrMz4uFarxZ///Gf89ttv8Pf3x/Dhw/HTTz+ZHQxIPMuZmeICY8DRd5yAYw0tot6+6M6bXZCQ4Ovbt6/V+TK/VtSgQadHbI8ABPk69t/9Vq0W127XIchXgtgets+XIp1LfX09rly54nTSmGaLd0HCpUFdXZ3Vcq4Yh0PLjHoHYZS3WOzc/YBo8mYXJBaLERISYpwh7e/vb3bQm06rAWMMjVoNGvhGh87VqGsEa9RCy0RoaPBcDoR4Ds/zqKqqgr+/P3x8nAsZFHC6KGFirKVlGRhjqKhuAAD41Po6tIg6AOj0PCpVGog4gFN7frkD4hkikQh9+vRxaLR2SxRwuiiO4xAZGYmwsDDodLpWj9+s1WDetjyAA/YsmehwwLlVq8Xz3x0GOGD3kolOXZ6RjksqlUIkcj4DQwGnixOLxWavu+uUOvxWo0eIvwT+/o63THr4SPFbjR4AoIMYAb4dc7Qx6RgoaeyllPWGVo/cz7m8i9RHhACpIaBV17VuSRHSEgUcLyUEhxAnAw4AhPgbWjXV9RRwiHUUcLyU0MIJdkHAEVpJ1XVap5+LdG0UcLyU0BoRWifOCPE3BBwltXBIGyjgeKnmHI7z/QZCwKEcDmkLBRwvpWy6/Anxc76FI296Dgo4pC0UcLyUq3qpgBYtnHrK4RDrKOB4KSGHI/d3QcBpClpKauGQNlDA8VLuaeFQwCHWUcDxUkoXjsNpzuHQJRWxjgKOl3LpJRW1cIiNKOB4IcaY8ZLKFb1UxnE4lMMhbaCA44XUmkbj4lsuyeH4NU9t6AQLSJJ2RAHHCwmtG6mPCL4S5z8CQgtHzzOoNY4t5EW8AwUcLyQM0JP7SZxeUAkAfCViyHxEJs9NiDl2B5wDBw5g2rRpiIqKAsdx2L59e5vH5ObmYtSoUZDJZOjXrx+ysrIcqCpxFVW963qoBDSfitjC7oBTW1uL+Ph4rFmzxqbyJSUlmDp1Kh544AEUFBRgyZIlmDt3Lnbt2mV3ZYlrVLtwDI4ghKY3EBvYPXNvypQpmDJlis3l161bh9jYWKxcuRIAMGjQIBw8eBAffvihxZuVEfcy9lC5oEtcIKfpDcQGbs/h5OXlITk52WRfSkoK8vLyLB6j0WigUqlMNuI6zTkc1y0H2rwmDrVwiGVuDzgKhQLh4eEm+8LDw6FSqSze6jQzMxNyudy4RUdHu7uaXsWV0xoExvlUlMMhVnTIXqr09HQolUrjdvXq1fauUpeibLrsceUlVfOaOHRJRSxz+10bIiIiUFFRYbKvoqICwcHBFm9DK5PJIJPJ3F01r+WWFo4/JY1J29zewklMTEROTo7Jvj179iAxMdHdpyYWGBdQd2XS2I/mU5G22R1w1Go1CgoKUFBQAMDQ7V1QUICysjIAhsuhtLQ0Y/kFCxbg8uXLePnll3HhwgX87//+L7766iu8+OKLrnkFxG6uXEBdQPOpiC3sDjj5+fkYOXIkRo4cCQBYunQpRo4ciYyMDABAeXm5MfgAQGxsLL7//nvs2bMH8fHxWLlyJTZu3Ehd4u3IlbeIETTPp6IcDrHM7hzO/fffb3WCnrlRxPfffz9OnTpl76mIm7gnh0Pd4qRtHbKXiriPTs8bJ1i64hYxgpY5HJoxTiyhgONlVC2SusG+ruukFFo42kYeDTreZc9LuhYKOF5GuJwKkvnAR+y6X3+gzAdikWHmOeVxiCUUcLxMtRt6qACA4zhjEpryOMQSCjhexh0TNwVyShyTNlDA8TLKOtf3UAma51PRJRUxjwKOl3FnC4emN5C2UMDxMtUeaOHQ9AZiCQUcL9M86M91Y3AElMMhbaGA42WELmv3tHAMQYxyOMQSCjheRuXWHA61cIh1FHC8jFtzOBRwSBso4HgZpRtuESOgNXFIWyjgeBnjLWLc2C2upGVGiQUUcLwIY8wtS1MIqFuctIUCjhdp0PHQNhpmcrtyaQqBkMOp0+qhadS7/PlJ50cBx4sIXeJiEYcAqdjlzx/kK4Fwq3K6XQwxhwKOF2mZMOaEyOBCYhGHYF9a25hYRgHHi7izS1xg7BqnFg4xw6GAs2bNGvTt2xe+vr5ISEjAsWPHLJbNysoCx3Emm6+vr8MVJo5TurGHSkBr4hBr7A44W7duxdKlS7F8+XKcPHkS8fHxSElJQWVlpcVjgoODUV5ebtxKS0udqjRxjDuXphDIjTPGqWuctGZ3wPnHP/6BefPm4dlnn8XgwYOxbt06+Pv749NPP7V4DMdxiIiIMG533muceIY7B/0J6B7jxBq7Ao5Wq8WJEyeQnJzc/AQiEZKTk5GXl2fxOLVajZiYGERHR2P69Ok4e/as1fNoNBqoVCqTjTjPnRM3BTS9gVhjV8C5ceMG9Hp9qxZKeHg4FAqF2WMGDBiATz/9FN999x0+//xz8DyPpKQkXLt2zeJ5MjMzIZfLjVt0dLQ91SQWNOdwXD8GR9A8+I8uqUhrHrm3eFpaGkaMGIGJEyfi22+/Rc+ePbF+/XqLx6Snp0OpVBq3q1evuruaXsETvVRyWvWPWGHXjYl69OgBsViMiooKk/0VFRWIiIiw6TkkEglGjhyJ4uJii2VkMhlkMpk9VSM2oBwOaW92tXCkUilGjx6NnJwc4z6e55GTk4PExESbnkOv16OwsBCRkZH21ZQ4zZ3zqASUwyHW2H3rxaVLl2LOnDkYM2YMxo4di1WrVqG2thbPPvssACAtLQ29evVCZmYmAODNN9/EuHHj0K9fP1RXV2PFihUoLS3F3LlzXftKSJvcuYC6oHngH+VwSGt2B5yZM2eiqqoKGRkZUCgUGDFiBHbu3GlMJJeVlUEkam443b59G/PmzYNCoUBoaChGjx6Nw4cPY/Dgwa57FcQmHsnh+FEOh1jGsU5w53mVSgW5XA6lUong4OD2rk6nxPMMd7/6AxgDjr06CWFB7hntfUOtwZi//wQAKH57iktvJ0w6Jnv+PunT4CVqGhoh/Gtxbwun+blVDY1uOw/pnCjgeAkhp+InEUPm4/qlKQQSsQiBMsOVOk1vIHeigOMlPJEwFtDaxsQSCjhewhMJY4EQ1GhNHHInCjjt4Mjlm5i/OR95l2567JyeGIMjoK5xYgkFHACXq9T458ES3K517x8IYwz/PFiC2RuPYve5CszfnI9LVWq3nlNQ7cmAQ13jxAKvDzg7zygw7ZODeGvHOUz+6AB+vljllvPUa/VYsrUAb+04Bz3PEOovQY2mEfM250PV4P4/THfecfNOdI9xYonXBhyeZ1i5uwgLPj+BWq0efhIxKlQaPP3PY3jz/51Dg851dx24eqsOv1t7GN8VXIdYxCHj0cHY/eJERMp9cbmqFku3FoDn3TscSugx8kwLh+ZTEfO8MuCoGnSYtzkfn+w1TCD9472xOPbqJDw9LgYA8OmhEkxffQjny51fh+fAr1V49JODOF+uQvcAKb6Ym4A/3heLnkEybHh6DGQ+Ivx0vhKrci46fS5rmnup3Lc0haB5PhXlcIgprws4xZU1SF19CDkXKiHzEeEfT8YjY9pgBPlK8FbqUGx65h70CJSiqKIG01cfwsafLzvU+mCM4X9zizFn0zEo63WIjw7Bjj/dh3F3dTeWGdZbjszfDQMAfJxzETvPlLvsdd5JuLwJ9mQOh1o45A5eFXB2n1Ugdc1hXL5Riyi5L/79fBJ+N6q3SZkHBoZh55IJSB4UBq2ex9+/P4+nPz2KcmW9zedRaxrxP1+cxPs7i8AYMHNMNLbOH4dIuV+rsr8b1RvP3RcLAFj61WkUKWqce5EWeGJpCgHlcIglXhFweJ7hH3t+xfzPTkCtaURCbDf8Z/F9GNpLbrZ8j0AZ/i9tDN55fBj8JGIcKr6Jyat+xve/tN0CuVylxuNrDuHHMwpIxBzefnwo3p0xDL4Sy6N706cMRNLd3VGn1WP+Z/luGb/i0W5xyuEQC7p8wFE16DD/s3x83JQjeSapLz6fm4AegdYX+OI4Dv+V0Aff/+k+DO8th7Jeh4XZJ7H0qwLUWOhV+ulcBaavPoSLlWqEBcmwZX4iZifEtHnTOR+xCKv/axR6h/qh9GYdFm85Bb2Lk8ieHGks5ImqajSoVDW4/Xyk8+jSAae4Uo3UNYfw0/lKSH1EWPlEPF5/bAgkdsxgvqtnIP79fBIWP9gPIg749uRvmPLRzzh+5ZaxDM8zfLjnV8zdnI8aTSPGxIRix+L7MDom1ObzdAuQYsPTY+ArEeHAr1VYsavIrtfaFk+ONI4M8YWvRAS1phGT/rEfXxwtdXsvHOkcumzA2XOuAqlrDuFyVS0i5b74ZkEiZozu3faBZkjEIvz54QH46r8T0TvUD9du12Pm+jx8sKsIt2q1mLc5Hx81taDSEmOQPW8cwoLtX/5hcFQwVvw+HgCwbv8l/L/T1x2q7500jXrUN3XzCwlddwr2leDfzydheG85ahoa8eq2M3hyfR4uVrgnP0U6jy63Hg7PM3y89yJW/WQIAGNju2HNf41CzyDXrJFc06DD8v+cxbcnfwMASMUiaPU8pD4ivPP4MPzewaDW0rs/XsC6/ZfgKxHh2+fvxeAo59YAqqxpwNi3c8BxwKW3H4FI5Pr7ipuj5xn+dfgKPthdhDqtHhIxh+cn3o3/eaCf1ZxWR1d6sxabDl3B+XIVRvQJwfh+PTGmb2infk3OsOfvs0sFnJoGHV7ceho/nTcs8j4nMQavPTrYrksoW+345Tpe3XYGynodeoX4Yd0fRmNYb/NJaHvpeYZns47jwK9V6B3qh/8sug/dAhxvmRRX1iD5Hwcg95Pg9PKHXVJHe/xWXY+M7WeQc8Fwd9a7egTg7ceHIfHu7m0c2XEwxpBfehsbf76M3ecqcOdfjcxHhLGx3TA+rgfu69cTgyKD2szddRVeGXAadHpM++QgLlaqIfUR4e3UoXhijHvvZ6VQNiDnQgWmDI10KiCYo6zT4bE1B1F6sw5Jd3fH5j+OdWj1PMYYdp2twILPT6BPN38cePkBl9bTnnr8eEaB1/9zFpU1GgDAk2N645VHBnlkMKKjdHoeP55R4J8/X8bpa0rj/vsH9MSkQeE4VXYbBy/eML4mQY9AGe7r1x33xfXE+LgeCHfgEruz8MqAAwAf7vkVW49fxfqnRyM+OsRzFXSTXytqkLrmEOq0evzx3lhkTLNtHWhtI4/jV27hp/MV2HuhEqU36wAAo/qE4Nv/udedVW6TqkGH93dewOdHygAA3QOkyJg2GI/FR3WoFoGyXoetx8uQdegKrisNPW1SHxFmjOqFP94bi7jwIGNZxhguVqrx88Ub+PliFY5evmXMmQniwgIxvin4JNzVDf5Su5cT77C8NuDwPIOyXodQF7c22tPOM+VY8PlJAMDKJ+ItJr5vqjXILarC3guVOPBrFWo0zct7SsUijLu7OxY/2A/39O3mkXq3Jf/KLaR/W4iLlYbZ8uPjeuDt1GHo092/XetVdrMOmw6X4KvjV1GrNQSNHoFSpCX2xeyEPujexnAKwJCkP1lajYPFVTh48QZ++U1pcgnmI+LQLywQg6OCMTRKjiFRwRgcFYwgX/f3ILqD2wPOmjVrsGLFCigUCsTHx+OTTz7B2LFjLZb/+uuvsWzZMly5cgVxcXF477338Mgjj9h8Pm9fRP0fu4vw8d5iSH1E+GZBIob3DgFjDEUVNcg5X4m9Fypxsuy2yYe6R6AMDw7siQcHhmN8XA8EyDref1RtI48NBy7h473F0Dby8JWIsCS5P567L9Yk78YYg55naOQZeNb0lTfs0/MMesbQqGcQiTj4S8Twk4oh8xHZ3GJijOFE6W1s/LkEu88pIPTgDwgPwnPjY/FYfJRTCeHbtVocvnQTB4ur8PPFG7h22/yo9Zju/hgSFYwhUXIMjgrGkKhgty1270puDThbt25FWloa1q1bh4SEBKxatQpff/01ioqKEBYW1qr84cOHMWHCBGRmZuLRRx9FdnY23nvvPZw8eRJDhw517Qu6tBeovQkIHzSOA8ABnKjF9y2+ciLTfWAw/NWa+QpYeAx3nMuer2jxvMYfWu3neR6r9xXj9LVqdPOXYGR0CAqv3cbtWg04MIjAIAKP6FBfDIsKwuDIYESHyiACAxhveB7GN71mcdP7IQJELb5vtU/4amNdW/1o5mNl9qPGUK6sx2d5pTjfNK1DIuaaijM0mrzNXIujWn7fzPDuMnAcg0wsgq8PB6mPCL4+IsjEhu9lPhx8fUSQND1eeqsWZTfU4ACIwGNwZBAeGNADA8ICwAm/Z8YbzsT45vcUrMV7J7b8nrbYz8DhVl0jym7VouymGmU3a3HtVm3T77Kp7k2/Uw4Mcj8xokP90DvEF5FyQ+tKx3PQ6Zlh4w15Ji3PQavnodMD2qbHtDygbWTQ8Qwcx8FHLIJEzEEiFsFHxEEsEsFHLIJP0z5Jq30c5D16YdQ463k/twachIQE3HPPPVi9ejUAwx9DdHQ0Fi9ejL/97W+tys+cORO1tbXYsWOHcd+4ceMwYsQIrFu3zqZz2vyCNj0ClB6y5+UQQqwo8EvEiL/utFrGnoBjVztbq9XixIkTSE9PN+4TiURITk5GXl6e2WPy8vKwdOlSk30pKSnYvn27xfNoNBpoNM1Zf5XKxmUiokYCYkmL/0C447+T0DLhzX9vdwul6aulVlFbX1v8lwZg2pIw81iDjsdv1fWQ+IgR7CdDkJ8UYrG4uR6t/rO22GdswfEAr2963U1feeG/dst9+uZ9phWxUMWW+zlzBcy8xtbH8szwHxuc8N+ea3qUNTdchSM4gLujxcXAgQcHnsHspmcAD4BnnOF7nkEsEqF7kC8kPk1/Dne+fy1bySatYqHSLd6rlu8hY5Yfu/N5zZxDD6BWy6NWo0eNlkedVg8RB4g5QMyxVt+L0LSv6auwj+MYwACeMTA0/UkwBh6Gr2AMfNM+k8cZgG6xZn5fjrMr4Ny4cQN6vd54l01BeHg4Lly4YPYYhUJhtrxCobB4nszMTLzxxhv2VM0g5W37j+lEfAHc3d6VcDMRAGeGaHIAxE1bZycGENy0RbZzXVylQ05tSE9Ph1KpNG5Xr15t7yoRQlzArhZOjx49IBaLUVFRYbK/oqICERERZo+JiIiwqzwAyGQyyGSumYpACOk47GrhSKVSjB49Gjk5OcZ9PM8jJycHiYmJZo9JTEw0KQ8Ae/bssVieENJ12T04Y+nSpZgzZw7GjBmDsWPHYtWqVaitrcWzzz4LAEhLS0OvXr2QmZkJAHjhhRcwceJErFy5ElOnTsWWLVuQn5+PDRs2uPaVEEI6PLsDzsyZM1FVVYWMjAwoFAqMGDECO3fuNCaGy8rKIBI1N5ySkpKQnZ2N1157Da+88gri4uKwfft2m8fgEEK6ji41tYEQ4nn2/H12yF4qQkjX1PEm2JghNMJsHgBICPEY4e/SloulThFwamoMc2yio927vg0hxHE1NTWQy60vQtcpcjg8z+P69esICrK+ippKpUJ0dDSuXr1KuZ470HtjHr0vltn63jDGUFNTg6ioKJMOI3M6RQtHJBKhd2/b1woODg6mD48F9N6YR++LZba8N221bASUNCaEeAwFHEKIx3SpgCOTybB8+XKah2UGvTfm0ftimTvem06RNCaEdA1dqoVDCOnYKOAQQjyGAg4hxGMo4BBCPIYCDiHEY7pUwFmzZg369u0LX19fJCQk4NixY+1dpXb1+uuvg+M4k23gwIHtXa12ceDAAUybNg1RUYZbCt951xDGGDIyMhAZGQk/Pz8kJyfj4sWL7VNZD2rrfXnmmWdafYYmT57s8Pm6TMDZunUrli5diuXLl+PkyZOIj49HSkoKKisr27tq7WrIkCEoLy83bgcPHmzvKrWL2tpaxMfHY82aNWYff//99/Hxxx9j3bp1OHr0KAICApCSkoKGhgYP19Sz2npfAGDy5Mkmn6Evv/zS8ROyLmLs2LFs4cKFxp/1ej2LiopimZmZ7Vir9rV8+XIWHx/f3tXocACwbdu2GX/meZ5FRESwFStWGPdVV1czmUzGvvzyy3aoYfu4831hjLE5c+aw6dOnu+wcXaKFI9ygLzk52bivrRv0eYuLFy8iKioKd911F2bPno2ysrL2rlKHU1JSAoVCYfL5kcvlSEhI8PrPDwDk5uYiLCwMAwYMwPPPP4+bN286/FxdIuBYu0GftRvudXUJCQnIysrCzp07sXbtWpSUlGD8+PHG9YWIgfAZoc9Pa5MnT8bmzZuRk5OD9957D/v378eUKVOg1+sder5OsTwFccyUKVOM3w8fPhwJCQmIiYnBV199heeee64da0Y6i6eeesr4/bBhwzB8+HDcfffdyM3NxaRJk+x+vi7RwnHkBn3eKCQkBP3790dxcXF7V6VDET4j9Plp21133YUePXo4/BnqEgHHkRv0eSO1Wo1Lly4hMrKr3KnaNWJjYxEREWHy+VGpVDh69Ch9fu5w7do13Lx50+HPUJe5pGrrBn3e6KWXXsK0adMQExOD69evY/ny5RCLxZg1a1Z7V83j1Gq1yX/lkpISFBQUoFu3bujTpw+WLFmCv//974iLi0NsbCyWLVuGqKgopKamtl+lPcDa+9KtWze88cYbmDFjBiIiInDp0iW8/PLL6NevH1JSUhw7ocv6uzqATz75hPXp04dJpVI2duxYduTIkfauUruaOXMmi4yMZFKplPXq1YvNnDmTFRcXt3e12sW+ffsYgFbbnDlzGGOGrvFly5ax8PBwJpPJ2KRJk1hRUVH7VtoDrL0vdXV17OGHH2Y9e/ZkEomExcTEsHnz5jGFQuHw+Wg9HEKIx3SJHA4hpHOggEMI8RgKOIQQj6GAQwjxGAo4hBCPoYBDCPEYCjiEEI+hgEMI8RgKOIQQj6GAQwjxGAo4hBCP+f8+fqvA/ReZGAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:35:44.647935Z", + "iopub.status.busy": "2024-03-23T03:35:44.647646Z", + "iopub.status.idle": "2024-03-23T03:37:44.837644Z", + "shell.execute_reply": "2024-03-23T03:37:44.836611Z" + }, + "papermill": { + "duration": 120.211589, + "end_time": "2024-03-23T03:37:44.840418", + "exception": false, + "start_time": "2024-03-23T03:35:44.628829", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:37:44.878032Z", + "iopub.status.busy": "2024-03-23T03:37:44.877126Z", + "iopub.status.idle": "2024-03-23T03:37:44.897480Z", + "shell.execute_reply": "2024-03-23T03:37:44.896521Z" + }, + "papermill": { + "duration": 0.041068, + "end_time": "2024-03-23T03:37:44.899482", + "exception": false, + "start_time": "2024-03-23T03:37:44.858414", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0057880.5310540.0196246.4554210.0917930.9963170.1400330.000012.3248640.0919194.1357740.1400840.0343841.4360368.780286
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.005788 0.531054 0.019624 6.455421 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.091793 0.996317 0.140033 0.00001 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 2.324864 0.091919 4.135774 0.140084 0.034384 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 1.436036 8.780286 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:37:44.934076Z", + "iopub.status.busy": "2024-03-23T03:37:44.933591Z", + "iopub.status.idle": "2024-03-23T03:37:45.289624Z", + "shell.execute_reply": "2024-03-23T03:37:45.288736Z" + }, + "papermill": { + "duration": 0.375632, + "end_time": "2024-03-23T03:37:45.291662", + "exception": false, + "start_time": "2024-03-23T03:37:44.916030", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:37:45.331457Z", + "iopub.status.busy": "2024-03-23T03:37:45.330721Z", + "iopub.status.idle": "2024-03-23T03:39:53.403787Z", + "shell.execute_reply": "2024-03-23T03:39:53.402854Z" + }, + "papermill": { + "duration": 128.096926, + "end_time": "2024-03-23T03:39:53.406736", + "exception": false, + "start_time": "2024-03-23T03:37:45.309810", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:39:53.448532Z", + "iopub.status.busy": "2024-03-23T03:39:53.448107Z", + "iopub.status.idle": "2024-03-23T03:39:53.477479Z", + "shell.execute_reply": "2024-03-23T03:39:53.476588Z" + }, + "papermill": { + "duration": 0.050432, + "end_time": "2024-03-23T03:39:53.479608", + "exception": false, + "start_time": "2024-03-23T03:39:53.429176", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:39:53.515188Z", + "iopub.status.busy": "2024-03-23T03:39:53.514910Z", + "iopub.status.idle": "2024-03-23T03:39:53.520273Z", + "shell.execute_reply": "2024-03-23T03:39:53.519406Z" + }, + "papermill": { + "duration": 0.025524, + "end_time": "2024-03-23T03:39:53.522187", + "exception": false, + "start_time": "2024-03-23T03:39:53.496663", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.040662367926644426}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:39:53.559534Z", + "iopub.status.busy": "2024-03-23T03:39:53.558799Z", + "iopub.status.idle": "2024-03-23T03:39:53.970683Z", + "shell.execute_reply": "2024-03-23T03:39:53.969499Z" + }, + "papermill": { + "duration": 0.433224, + "end_time": "2024-03-23T03:39:53.972960", + "exception": false, + "start_time": "2024-03-23T03:39:53.539736", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAE8CAYAAABU7q18AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFz0lEQVR4nO3deVwTd/4/8NckIQdXADkCyqUoWC8oKsWtRatf0bpWate11FbwqN2tfm3L2l9rd+vZ/dJLq61Wd/tdoW6r7fa71l5Wa6nHWq/iXWtVKILKrRDuQJLP748hAyEJlzkgvJ+PRx5hZj4zeQ+Yt5+Z+RwcY4yBEEKckMjRARBCiK1QgiOEOC1KcIQQp0UJjhDitCjBEUKcFiU4QojTogRHCHFalOAIIU6LEhwhxGlRgiMAgLCwMPz2t7+16WdwHIfVq1d3WG716tXgOM6msZC+gRKckzh27BhWr16NyspKR4dCeoH33nsPmZmZjg7D5ijBOYljx45hzZo1lOBIp1CCI4SQXo4SnBNYvXo1XnjhBQBAeHg4OI4Dx3G4fv06MjIy8OCDD8Lf3x8ymQz33HMPtm7davFY3377LaKjoyGXy3HPPfdg9+7dXY5Ho9Hg+eefh5+fHzw8PPDwww/j5s2bZssePXoUY8aMgVwux6BBg/C3v/3NbDmO47B06VJ89NFHiIyMhFwuR2xsLI4cOWLyu+A4DlevXsUTTzwBpVIJPz8/vPLKK2CM4caNG5g5cyY8PT2hUqmwfv36Lp8fAHzzzTdISEiAh4cHPD09MWbMGOzcudOozKefforY2FgoFAr4+vriiSeewK1bt4zKpKamwt3dHbdu3UJSUhLc3d3h5+eH5cuXQ6fTGZXV6/XYtGkTRowYAblcDj8/P0ydOhXZ2dlCmc78vcPCwnDp0iUcPnxY+LcyYcKEbv0eejxGer3z58+z5ORkBoC9/fbb7J///Cf75z//yWpqatiYMWNYamoqe/vtt9m7777LpkyZwgCwzZs3Gx0jNDSUDRkyhHl5ebGXXnqJbdiwgY0YMYKJRCL27bffdimeJ554ggFgjz/+ONu8eTObNWsWGzlyJAPAVq1aJZS7cOECUygULCQkhKWnp7N169axgIAAoWxrANjw4cOZr68vW7t2LXv99ddZaGgoUygU7OLFi0K5VatWMQAsOjqaJScns/fee49Nnz6dAWAbNmxgkZGR7I9//CN777332G9+8xsGgB0+fLhL55eRkcE4jmPDhw9nf/3rX9mWLVvYokWL2JNPPmlUBgAbM2YMe/vtt9lLL73EFAoFCwsLYxUVFUK5lJQUJpfL2bBhw9iCBQvY1q1b2aOPPsoAsPfee8/oc1NTUxkANm3aNLZx40b21ltvsZkzZ7J3331XKNOZv/dnn33GBgwYwKKiooR/K139G/cWlOCcxJtvvskAsLy8PKP1dXV1JmUTExPZwIEDjdaFhoYyAOzf//63sE6tVrPAwEAWExPT6TjOnTvHALBnnnnGaP3jjz9ukuCSkpKYXC5n+fn5wrqff/6ZicViswkOAMvOzhbW5efnM7lczh555BFhnSHBLV68WFin1WrZgAEDGMdx7LXXXhPWV1RUMIVCwVJSUjp9fpWVlczDw4PFxcWx+vp6o216vZ4xxlhjYyPz9/dnw4cPNyrz1VdfMQBs5cqVwrqUlBQGgK1du9boWDExMSw2NlZY/v777xkAtmzZMpOYDJ/LWOf/3sOGDWMJCQmdOOPejS5RnZxCoRB+VqvVKC8vR0JCAn799Veo1WqjskFBQXjkkUeEZU9PT8ybNw9nz55FcXFxpz5v7969AIBly5YZrX/uueeMlnU6Hfbv34+kpCSEhIQI64cOHYrExESzx46Pj0dsbKywHBISgpkzZ2L//v0ml3OLFi0SfhaLxRg9ejQYY1i4cKGw3svLC5GRkfj11187dW4AcODAAVRXV+Oll16CXC432mZo2pKdnY3S0lI888wzRmWmT5+OqKgofP311ybH/cMf/mC0PH78eKO4/v3vf4PjOKxatcpk39ZNarry9+4LKME5uR9++AGTJ0+Gm5sbvLy84Ofnh5dffhkATP7BR0REmLQ/GzJkCADg+vXrnfq8/Px8iEQiDBo0yGh9ZGSk0XJZWRnq6+sxePBgk2O0LWtgruyQIUNQV1eHsrIyo/WtkyYAKJVKyOVy+Pr6mqyvqKiwfEJt5ObmAgCGDx9usUx+fj4A8+cRFRUlbDcw3E9rzdvb2yiu3NxcBAUFwcfHp934uvL37gskjg6A2E5ubi4mTZqEqKgobNiwAcHBwZBKpdi7dy/efvtt6PV6R4doM2KxuFPrAIA5eNR+S3F1VV/+e1tCCc5JmGv5/+WXX0Kj0eCLL74wqtEcPHjQ7DFycnLAGDM61tWrVwHwT946IzQ0FHq9Hrm5uUY1mCtXrhiV8/Pzg0KhwLVr10yO0basgbmyV69ehaurq0kNyFYMNdOffvoJERERZsuEhoYC4M/jwQcfNNp25coVYXtXP3f//v24c+eOxVpcV/7efaWnCF2iOgk3NzcAMGroa6gZtK6hqNVqZGRkmD1GYWEhPvvsM2G5qqoKO3bsQHR0NFQqVafimDZtGgDgnXfeMVq/ceNGo2WxWIzExETs2bMHBQUFwvrLly9j//79Zo99/PhxnDlzRli+ceMGPv/8c0yZMsVqtaCOTJkyBR4eHkhPT0dDQ4PRNsPvefTo0fD398e2bdug0WiE7d988w0uX76M6dOnd/lzH330UTDGsGbNGpNths/tyt/bzc2tTzQKpxqckzDcfP/zn/+Mxx57DC4uLnjggQcglUoxY8YMPP3006ipqcH7778Pf39/FBUVmRxjyJAhWLhwIX788UcEBARg+/btKCkpsZgQzYmOjkZycjLee+89qNVqjBs3DllZWcjJyTEpu2bNGuzbtw/jx4/HM888A61Wi3fffRfDhg3DhQsXTMoPHz4ciYmJWLZsGWQyGd577z3hOPbi6emJt99+G4sWLcKYMWPw+OOPw9vbG+fPn0ddXR0++OADuLi44PXXX8f8+fORkJCA5ORklJSUYNOmTQgLC8Pzzz/f5c+dOHEinnzySbzzzju4du0apk6dCr1ej//85z+YOHEili5diilTpnT67x0bG4utW7fi1VdfRUREBPz9/U1qm07BcQ9wibWtW7eO9e/fn4lEIqHJyBdffMFGjhzJ5HI5CwsLY6+//jrbvn27SZOS0NBQNn36dLZ//342cuRIJpPJWFRUFPv000+7HEd9fT1btmwZ69evH3Nzc2MzZsxgN27cMGkmwhhjhw8fZrGxsUwqlbKBAweybdu2CU09WgPAlixZwj788EM2ePBgJpPJWExMDDt48KBROcO+ZWVlRutTUlKYm5ubSawJCQls2LBhXT7HL774go0bN44pFArm6enJxo4dy3bt2mVU5pNPPmExMTFMJpMxHx8fNnfuXHbz5s1OxWXud6DVatmbb77JoqKimFQqZX5+fmzatGns9OnTRnF15u9dXFzMpk+fzjw8PBgAp20ywjFG86KSno/jOCxZsgSbN292dCikF6F7cIQQp0X34EinddTYV6FQQKlU2ika6ysrKzNpMNyaVCrtsB0a6VkowZFOCwwMbHd7SkpKrx6CZ8yYMSaNcFtLSEjAoUOH7BcQuWuU4EinHThwoN3tQUFBNvtse9wq/uijj1BfX29xu7e3t81jINZFDxkIIU6LHjIQQpwWXaKaodfrUVhYCA8Pjz7TpYWQ3oQxhurqagQFBUEkslxPowRnRmFhIYKDgx0dBiGkAzdu3MCAAQMsbqcEZ4aHhwcA/pfn6enp4GgIIW1VVVUhODhY+K5aQgnODMNlqaenJyU4Qnqwjm4h0UMGQojTogRHCHFalOAIIU6L7sF1E2MMWq223b6LxDyxWAyJREJNcIjNUYLrhsbGRhQVFaGurs7RofRarq6uCAwMhFQqdXQoxIlRgusivV6PvLw8iMViBAUFQSqVUk2kCxhjaGxsRFlZGfLy8jB48OB2G2oScjcowXVRY2Mj9Ho9goOD4erq6uhweiWFQgEXFxfk5+ejsbHRZH7RvuRKcTV+Ka7CpKEBcJfR19Ha6L/ObqJax92h3x9v78Ui/FpWi3MFlY4OxSnRvzJCeoCbFXQ/1xYowRHSA+hp0DKboARHbCIsLMxkLlRiWdxAGgrdFhya4I4cOYIZM2YgKCgIHMdhz549RttTU1PBcZzRa+rUqR0ed8uWLQgLC4NcLkdcXBxOnTplozMg5O74esggdxFDqXBxdChOyaEJrra2FqNGjcKWLVsslpk6dSqKioqE165du9o95ieffIK0tDSsWrUKZ86cwahRo5CYmIjS0lJrh+/0GhsbHR2C03vyvlD8ccIg+LrLHB2KU3Jogps2bRpeffVVPPLIIxbLyGQyqFQq4dXRuPgbNmzAU089hfnz5+Oee+7Btm3b4Orqiu3bt1vcR6PRoKqqyujVHY1avcWXVqfvdNmmTpTtjgkTJmDp0qVYunQplEolfH198corrwjzHYSFhWHdunWYN28ePD09sXjxYgDA0aNHMX78eCgUCgQHB2PZsmWora0VjltaWooZM2ZAoVAgPDwcH330Ubfi64sqahuRff0OLhWqHR2KU+rxDW8OHToEf39/eHt748EHH8Srr76Kfv36mS3b2NiI06dPY8WKFcI6kUiEyZMn4/jx4xY/Iz09HWvWrLnrWLcczLG4LdzXDUkx/YXlvx/JRZPO/J3lAd4KzB7dMuDm9h/yUN9o3CXs+f8a0q0YP/jgAyxcuBCnTp1CdnY2Fi9ejJCQEDz11FMAgLfeegsrV67EqlWrAAC5ubmYOnUqXn31VWzfvh1lZWVCkszIyADA30ooLCzEwYMH4eLigmXLllGNuZNu12rwn2vl6O+lwLCg3jvlYk/VoxPc1KlTMWvWLISHhyM3Nxcvv/wypk2bhuPHj0MsFpuULy8vh06nQ0BAgNH6gIAA/PLLLxY/Z8WKFUhLSxOWDYPpOaPg4GC8/fbb4DgOkZGRuHjxIt5++20hwT344IP405/+JJRftGgR5s6di+eeew4AMHjwYLzzzjtISEjA1q1bUVBQgG+++QanTp3CmDFjAAD/+Mc/MHToULufW2+j1enx5fkiAIBG171aOWlfj05wjz32mPDziBEjMHLkSAwaNAiHDh3CpEmTrPY5MpkMMtnd3wNZMjHC4jZRm95cix8YZLFs255fC34TfjdhGbnvvvuMupbFx8dj/fr1wqABo0ePNip//vx5XLhwweiykzEmdFm7evUqJBIJYmNjhe1RUVHw8vKyWszOStuqbQhNbmcbPTrBtTVw4ED4+voiJyfHbILz9fWFWCxGSUmJ0fqSkhKoVCqbxyeVdP6Wpq3K3i03Nzej5ZqaGjz99NNYtmyZSdmQkBBcvXrVXqE5HZ1RgnNgIE6sV7WDu3nzJm7fvm1xhnWpVIrY2FhkZWUJ6/R6PbKyshAfH2+vMHu0kydPGi2fOHECgwcPNnvJDwD33nsvfv75Z0RERJi8pFIpoqKioNVqcfr0aWGfK1euoLKy0pan4RRa1+D0lOFswqEJrqamBufOncO5c+cAAHl5eTh37hwKCgpQU1ODF154ASdOnMD169eRlZWFmTNnIiIiAomJicIxJk2ahM2bNwvLaWlpeP/99/HBBx/g8uXL+OMf/4ja2lrMnz/f3qfXIxUUFCAtLQ1XrlzBrl278O677+LZZ5+1WP7FF1/EsWPHsHTpUpw7dw7Xrl3D559/jqVLlwIAIiMjMXXqVDz99NM4efIkTp8+jUWLFkGhUNjrlHqt1jU4HXVlsAmHXqJmZ2dj4sSJwrLhRn9KSgq2bt2KCxcu4IMPPkBlZSWCgoIwZcoUrFu3zuh+WW5uLsrLy4XlOXPmoKysDCtXrkRxcTGio6Oxb98+kwcPfdW8efNQX1+PsWPHQiwW49lnnxWag5gzcuRIHD58GH/+858xfvx4MMYwaNAgzJkzRyiTkZGBRYsWISEhAQEBAXj11Vfxyiuv2ON0ejUd1eBsjmN0d9NEVVUVlEol1Gq1yaxaDQ0NyMvLQ3h4eK8b5mfChAmIjo7uEV2oevPv0VqK1Q3YdaoAAJA8NgQqZd/8PXRHe9/R1nrVQwZCnImeMUglInjKJZTcbIQSHCEOEuSlaLdpEbl7lOD6kEOHDjk6BNKGVqfHT4VV0OkZ7g3xouHvrYwSHCEOpNUzHPyF79Y2aoASEjElOGuiBEeIgxTcrsOJvNvCso4x+kJaWa9q6EuIM6lqaMKtinphWU/dUa2OEhwhDqJt07hXRy22rI4SHCEOomtTZaPeDNZHCY4QB2k7QpKeEpzVUYIjxEG0bWtwdIlqdfTQhhAHMVyS+nnIMH6wLzzlNPGMtVGCI8RB9Iwf3DTc1w2h/dw63oF0GSU4a2AM0DXZ/3PFLqbD/1qwY8cOPP/88ygsLDQajSUpKQkeHh745z//aasoiQUJQ/zwwGBfGuzShijBWYOuCfjPevt/7vg/ARJpp4rOnj0by5YtwxdffIHZs2cD4GfD+vrrr/Htt9/aMkrSDo7jkFtWg/pGHcJ8XeFBl6lWRQ8Z+giFQoHHH39cmAkLAD788EOEhIRgwoQJjguM4MSvt/Hd5RLcqaV5aK2NanDWIHbha1OO+NwueOqppzBmzBjcunUL/fv3R2ZmJlJTU6mDt4Oc/PU2Sqs1KKvWAKB2cLZACc4aOK7Tl4qOFBMTg1GjRmHHjh2YMmUKLl26hK+//trRYfVZhep6XC+vE5ZpVF/rowTXxyxatAgbN27ErVu3MHnyZKed/7U3aNvQl6ZGtT66B9fHPP7447h58ybef/99LFiwwNHh9Gltu2pRDc76KMH1MUqlEo8++ijc3d2RlJTk6HD6NENne1HzPVC6B2d9lOD6oFu3bmHu3LlG7eGI/RkSmouET3BUg7M+ugfXh1RUVODQoUM4dOgQ3nvvPUeH0+dpdXxCiwvvB6XCBX4e9B+OtTm0BnfkyBHMmDEDQUFB4DgOe/bsEbY1NTXhxRdfxIgRI+Dm5oagoCDMmzcPhYWF7R5z9erV4DjO6BUVFWXjM+kdYmJikJqaitdffx2RkZGODqfPM9Tggr0ViPB3h1JBjXytzaE1uNraWowaNQoLFizArFmzjLbV1dXhzJkzeOWVVzBq1ChUVFTg2WefxcMPP4zs7Ox2jzts2DB89913wrJEQhVVALh+/bqjQyCtLLw/HFo9g0RE7RBtxaHf/GnTpmHatGlmtymVShw4cMBo3ebNmzF27FgUFBQgJCTE4nElEglUKpVVYyXE2kQiDlIRh5KqBtypbUQ/dyn8PWh+VGvqVQ8Z1Go1OI6Dl5dXu+WuXbuGoKAgDBw4EHPnzkVBQUG75TUaDaqqqoxeHWF0Q/iu0O+vxc+FVdj3UzFySmocHYrT6TUJrqGhAS+++CKSk5Ph6elpsVxcXBwyMzOxb98+bN26FXl5eRg/fjyqq6st7pOeng6lUim82mv86uLC3yepq6uzWIZ0zPD7M/w++xqtTo+vLxRh/6ViMPDJnlqJWF+vuDnV1NSE3//+92CMYevWre2WbX3JO3LkSMTFxSE0NBT/+te/sHDhQrP7rFixAmlpacJyVVWVxSQnFovh5eWF0lJ+LktXV1fqy9kFjDHU1dWhtLQUXl5eEIvFjg7JIZp0DFdL+P90Y0O9AdCIvrbQ4xOcIbnl5+fj+++/b7f2Zo6XlxeGDBmCnJwci2VkMlmX2oQZ7u8ZkhzpOi8vrz59n9QwXLmI44TJnmlOBuvr0QnOkNyuXbuGgwcPol+/fl0+Rk1NDXJzc/Hkk09aLS6O4xAYGAh/f380NTlgoMtezsXFpc/W3AwMbeAkYo56MtiQQxNcTU2NUc0qLy8P586dg4+PDwIDA/G73/0OZ86cwVdffQWdTofi4mIAgI+PD6RSfvSOSZMm4ZFHHsHSpUsBAMuXL8eMGTMQGhqKwsJCrFq1CmKxGMnJyVaPXywW9/kvKukejZavwckkIohF1JPBVhya4LKzszFx4kRh2XAfLCUlBatXr8YXX3wBAIiOjjba7+DBg8Igjbm5uSgvLxe23bx5E8nJybh9+zb8/Pxw//3348SJE/Dz87PtyRDSBT/k8P9mGWvpi0oJzvocmuAmTJjQbnOBzjQlaNt49eOPP77bsAixOTcZX/PnOCCsnytch6uoJ4MN9Oh7cIQ4q9hQH9Q36TA61Af93GXo5079UG2BEhwhDuDnIcMjMQMcHYbTowRHiJ0xxqDTM4hF/GAQ1Q1NKKnSQO4iwgBvV0eH51R6TU8GQpxFeU0j3v0+B+//51cAQJG6AV+eL8Tx3NsOjsz5UIIjxM4M7d3EIv7rZxhMhJ6iWh8lOELszNCLwTBMUktDX4eF5LQowRFiZ4ZeDIYGvoZ36otqfZTgCLEzw2QzbWtwNISU9VGCI8TOWu7BtanBUV9Uq6MER4idCffgxG3vwVGCszZqB0eInXnIXDDI3x3+zbNoeSokmDTUHzIJDdxgbZTgCLGzkH6uCOnX0qDXVSrByAFejgvIidElKiHEaVENjhAH0+r0KKxsgJ4xhPm6OTocp0I1OELsLPv6HWz67hq++7kEANCg1ePfZ25iz7lbDo7M+VCCI8TOdHoGPTPMpQWIhXZwNC+DtVGCI8TODCnM0AdV1OpbSP1RrYsSHCF2Zkhihtkmxa2mnaTuWtZFCY4Qe2vOYYb5dEWtEpyeOtxbFSU4QuzMcJvNkNZEIk6ozVENzroowRFiZ4bHC61rbmLqrmUT1A6OEDvzUkgR4uMKb1epsO7+wb7gOA4yCdU5rIkSHCF2NmKAEiMGKI3WxYR4Oyga5+bQ/y6OHDmCGTNmICgoCBzHYc+ePUbbGWNYuXIlAgMDoVAoMHnyZFy7dq3D427ZsgVhYWGQy+WIi4vDqVOnbHQGhJCezKEJrra2FqNGjcKWLVvMbn/jjTfwzjvvYNu2bTh58iTc3NyQmJiIhoYGi8f85JNPkJaWhlWrVuHMmTMYNWoUEhMTUVpaaqvTIOSulVY14MadOmi0OkeH4lxYDwGAffbZZ8KyXq9nKpWKvfnmm8K6yspKJpPJ2K5duyweZ+zYsWzJkiXCsk6nY0FBQSw9Pb3TsajVagaAqdXqrp0EIZ3w/S8lbMvBayz7+m1h3fajv7IN315htyrqHBhZ79HZ72iPvaOZl5eH4uJiTJ48WVinVCoRFxeH48ePm92nsbERp0+fNtpHJBJh8uTJFvcBAI1Gg6qqKqMXIbai1TFomvRGk8zQqL620WMTXHFxMQAgICDAaH1AQICwra3y8nLodLou7QMA6enpUCqVwis4OPguoyfEMkNPBlFLKxGh0S911bKuHpvg7GnFihVQq9XC68aNG44OiTgx1qarFkDt4GylxyY4lUoFACgpKTFaX1JSImxry9fXF2KxuEv7AIBMJoOnp6fRixBbYW26agGAuPmbSPnNunpsggsPD4dKpUJWVpawrqqqCidPnkR8fLzZfaRSKWJjY4320ev1yMrKsrgPIfbWtqsWQJeotuLQhr41NTXIyckRlvPy8nDu3Dn4+PggJCQEzz33HF599VUMHjwY4eHheOWVVxAUFISkpCRhn0mTJuGRRx7B0qVLAQBpaWlISUnB6NGjMXbsWGzcuBG1tbWYP3++vU+PELOoq5b9ODTBZWdnY+LEicJyWloaACAlJQWZmZn4f//v/6G2thaLFy9GZWUl7r//fuzbtw9yuVzYJzc3F+Xl5cLynDlzUFZWhpUrV6K4uBjR0dHYt2+fyYMHQhzF21WKIC853GQts2gN6++JkH6u8GueaYtYB8cY1YnbqqqqglKphFqtpvtxhPRAnf2Oduse3K+//trtwAghxF66leAiIiIwceJEfPjhh+12myKEdE5VQxNKqhpQo9E6OhSn0q0Ed+bMGYwcORJpaWlQqVR4+umnqUM7IZ309YUivH/kV+SUVgvrjufexs6TBbhcRL1orKlbCS46OhqbNm1CYWEhtm/fjqKiItx///0YPnw4NmzYgLKyMmvHSYjTqGvUokajNe6qRU9RbeKu2sFJJBLMmjULn376KV5//XXk5ORg+fLlCA4Oxrx581BUVGStOAlxGoYUZtSTQUwJzhbuKsFlZ2fjmWeeQWBgIDZs2IDly5cjNzcXBw4cQGFhIWbOnGmtOAlxGsxMX1QJdba3iW61g9uwYQMyMjJw5coVPPTQQ9ixYwceeughiJoneAwPD0dmZibCwsKsGSshTkFvtqsWJThb6FaC27p1KxYsWIDU1FQEBgaaLePv749//OMfdxUcIc6ImemqZbgHp6UEZ1XdSnAHDhxASEiIUGMzYIzhxo0bCAkJgVQqRUpKilWCJMSZtAyX1JLiJHQPzia6dQ9u0KBBRt2jDO7cuYPw8PC7DooQZ+btKoWvhwzSVjNoqZQKxIX7YJCfmwMjcz7dqsFZ6t1VU1Nj1E+UEGJq+kjT2zr9vRTo76VwQDTOrUsJztAZnuM4rFy5Eq6ursI2nU6HkydPIjo62qoBEkJId3UpwZ09exYAX4O7ePEipNKWiWulUilGjRqF5cuXWzdCQvqARq0eNRotRBzg1WpCaHJ3upTgDh48CACYP38+Nm3aRCNtENINn2bfQI1Gi+kjA+Hvwd/SuVFRhy/OFSJQKcdjY0McHKHz6NY9uIyMDGvHQUifoa5vQnWDFvpWXbUMDX2pmYh1dTrBzZo1C5mZmfD09MSsWbPaLbt79+67DowQZ9e6J4OI+qLaRKcTnFKpFFpeK5VKmwVEiLPTm2npa2gHRzU46+p0gmt9WUqXqIR0nyGHicx01dJTgrOqbjX0ra+vR11dnbCcn5+PjRs34ttvv7VaYIQ4K3NdtSTNvYKoBmdd3UpwM2fOxI4dOwAAlZWVGDt2LNavX4+ZM2di69atVg2QEGdjrqtWy3hwerP7kO7p9oi+48ePBwD83//9H1QqFfLz87Fjxw688847Vg2QEGfj5eoCL1cXiFo9ZZC5iBAT4oWYEG8HRuZ8utVMpK6uDh4eHgCAb7/9FrNmzYJIJMJ9992H/Px8qwZIiLOZGxdqsk7uIsaESH8HROPcuj3pzJ49e3Djxg3s378fU6ZMAQCUlpZS419CSI/RrQS3cuVKLF++HGFhYYiLi0N8fDwAvjYXExNj1QDDwsLAcZzJa8mSJWbLZ2ZmmpSlAQBIb1Dd0ITKukZ6kmpF3bpE/d3vfof7778fRUVFGDVqlLB+0qRJeOSRR6wWHAD8+OOP0Ol0wvJPP/2E//qv/8Ls2bMt7uPp6YkrV64Iy61HTiXEkRhj+PBkATgAv4sdALlLy+z2249eh54xLBofDg+5i+OCdCLdSnAAoFKpoFKpjNaNHTv2rgNqy8/Pz2j5tddew6BBg5CQkGBxH47jTGIjpCdgDCiv1gg/tyYRc2jUMtCDVOvpVoKrra3Fa6+9hqysLJSWlkLf5i9iq5nvGxsb8eGHHyItLa3dWllNTQ1CQ0Oh1+tx77334n/+538wbNgwi+U1Gg00Go2wXFVFc1MS22id09r+ExYJw5ZThrOWbiW4RYsW4fDhw3jyyScRGBhot0vAPXv2oLKyEqmpqRbLREZGYvv27Rg5ciTUajXeeustjBs3DpcuXcKAAQPM7pOeno41a9bYKGpCWuhbVdvafm1oZi3r45il4Xnb4eXlha+//hq/+c1vbBGTRYmJiZBKpfjyyy87vU9TUxOGDh2K5ORkrFu3zmwZczW44OBgqNVqeipMrKpRq8eWgzkAgKUPRsBF3PKcb/vRPKjrm/DY2GAEKml03/ZUVVVBqVR2+B3tVg3O29sbPj4+3Q6uO/Lz8/Hdd991eaQSFxcXxMTEICcnx2IZmUwGmUx2tyES0qHWNThRmyqc0OFeRzU4a+lWM5F169Zh5cqVRv1RbS0jIwP+/v6YPn16l/bT6XS4ePGixekNCbGn1tdLbW/s0JBJ1tetGtz69euRm5uLgIAAhIWFwcXF+JH2mTNnrBKcgV6vR0ZGBlJSUiCRGIc8b9489O/fH+np6QCAtWvX4r777kNERAQqKyvx5ptvIj8/H4sWLbJqTIR0l4dcAj1jJvfghgR4oL+XAu7ybjduIG106zeZlJRk5TDa991336GgoAALFiww2VZQUGA0P2tFRQWeeuopFBcXw9vbG7GxsTh27Bjuuecee4ZMiFkKqRiLxg80u21suH1v+/QF3XrI4Ow6ewOTEOIYnf2OduseHMAPk/S///u/WLFiBe7cuQOAvzS9detWdw9JSJ/WqNWjVqNFo5bawVlLty5RL1y4gMmTJ0OpVOL69et46qmn4OPjg927d6OgoEAYK44QYqy6oQlfXyiCi1iER2ON22Ue+LkEV0uqMTHKH9HBXo4J0Ml0qwaXlpaG1NRUXLt2zagj+0MPPYQjR45YLThCnI1Wx1CkbkBxVYPJNrGIBr20tm4luB9//BFPP/20yfr+/fujuLj4roMixFkZbni3bQMHtCQ4agdnPd1KcDKZzGx/zatXr5p0jieEtDA09DXXu5G6allftxLcww8/jLVr16KpqQkAP3pHQUEBXnzxRTz66KNWDZAQZ8KEGbVMtwmXqNSwwWq6leDWr1+Pmpoa+Pn5ob6+HgkJCYiIiICHhwf++te/WjtGQpyGoVUWZ9KPgWa3t4VuPUVVKpU4cOAAfvjhB5w/fx41NTW49957MXnyZGvHR4hTMeQuc5eoQg2O7sFZTZcTnF6vR2ZmJnbv3o3r16+D4ziEh4dDpVKBMUaj5xLSDo7jJ5iRtRrJ18DPQ4Z7gjwR6EVD7FtLl3oyMMYwY8YM7N27F6NGjUJUVBQYY7h8+TIuXryIhx9+GHv27LFhuPZBPRkI6dlsMlxSZmYmjhw5gqysLEycONFo2/fff4+kpCTs2LED8+bN617UhBBiRV16yLBr1y68/PLLJskNAB588EG89NJL+Oijj6wWHCF9CWMMjVo9NFqd6cameqD0F/6ddFqXEtyFCxcwdepUi9unTZuG8+fP33VQhDirkqoG/N/pm8i6XGKy7WpJDbYczMEX5wqNN+j1wIVPgEufAec+Mp2thljUpQR3584dBAQEWNweEBCAioqKuw6KEGdV36jDjTt1KFJb7qqlb5vAKq8DVUX8zzVlwB3bTOrkjLqU4HQ6ncmAk62JxWJotdq7DooQZ2VIXu121WrbDq68zXD7t3NtEpsz6tJDBsYYUlNTLc5f0HriFkKIKUPqaq+rlsnM9hV5/LtqBFB8EVAX2C5AJ9OlBJeSktJhGXqCSohlTKjBmW4TmeuL2tQA1PHjLaJ/LJ/g6u4Aeh0gMm1LR4x1KcFlZGTYKg5C+gTD7TVzXbXEnJlL1JrmhxFyT8BDBUikgLaRT3LuNLBFR7o9oi8hpOva66plmFrE6CFDTSn/7h7A7+Tmzy/XltouSCdC0/cQYmdiESc8UGhN4SLG4AB3KFp346ppHl/RQ8W/u/YD1DeBemqt0BmU4Aixo0iVByJVHma3echd8NuRQcYr627z727Nl6NyJf/eoLZRhM6FLlEJ6akYa3nAoGieUpASXJdQDY6QHoQxBj3jn7JyTXWAtrnplcKLf6cE1yU9uga3evVqcBxn9IqKimp3n08//RRRUVGQy+UYMWIE9u7da6doCenY9fJafH7uFn68fsdkm0arw8bvruGdrGv8k1TDfTa5JyB2af7ZkOCq+C5cpF09OsEBwLBhw1BUVCS8jh49arHssWPHkJycjIULF+Ls2bNISkpCUlISfvrpJztGTIhl6vom/FpWixJzs2q1erSq05u5PAUAmQff/o3pAY3pvCjEWI9PcBKJBCqVSnj5+vpaLLtp0yZMnToVL7zwAoYOHYp169bh3nvvxebNm+0YMSGW6dsZsrz1k1WdngH1fIKrk3jieO5t1Gi0fFMRWfP4Z3SZ2qEen+CuXbuGoKAgDBw4EHPnzkVBgeVuKsePHzcZNj0xMRHHjx9v9zM0Gg2qqqqMXoTYQsu0gabbOI4znnimuQaXU+2CE7/extFrZXxBWfNT2MYaG0fb+/XoBBcXF4fMzEzs27cPW7duRV5eHsaPH4/q6mqz5YuLi01GOwkICOhwrtb09HQolUrhFRwcbLVzIKQ11s60gUCrEUVa1eA8fAIQ4uMKdT0/ix1k7vx7Y61NY3UGPTrBTZs2DbNnz8bIkSORmJiIvXv3orKyEv/617+s+jkrVqyAWq0WXjdu3LDq8QkxaOnJYD7DtUw8oxceMohcfVBwp66lC5fUjX/XmP+PnrToVc1EvLy8MGTIEOTk5JjdrlKpUFJiPJBgSUkJVCpVu8eVyWQWR0ghxJpa+qKaZ3jQoNfUADotwIkgdfcGUI/6xuaRfqV0idpZPboG11ZNTQ1yc3MRGBhodnt8fDyysrKM1h04cADx8fH2CI+QDrU3HhwAhPRzxSB/d0gbmx8gyD1x6Bp/qVrd0DzWoqEGR5eoHerRNbjly5djxowZCA0NRWFhIVatWgWxWIzk5GQA/NBM/fv3R3p6OgDg2WefRUJCAtavX4/p06fj448/RnZ2Nv7+97878jQIEcSF+2BsmA+Yhe2Jw5qvNoqah/5XeKO4tE2TEsM9OA3V4DrSoxPczZs3kZycjNu3b8PPzw/3338/Tpw4AT8/vl9eQUEBRKKWSui4ceOwc+dO/OUvf8HLL7+MwYMHY8+ePRg+fLijToEQI3yD9U4UbL7/ppN5Ga3W6xlEdInaaT06wX388cftbj906JDJutmzZ2P27Nk2iogQO2lOcFqZUlgVE+LF1/wMl6haDaBraunlQEz06ARHiLO5XFSFvPJaDPRzQ5TKdMLif5++iZsV9UgWF8MfQJOUT3ASEYcJkc1jwXEyQCzhH0JoqgFXH5PjEF6veshASG9XWq3BleJqlFWbn7+EAdDr9UBDJQBA48InOBdJq68qx7V6kkoPGtpDCY4QO2IdPEUViwCJXsPPxQCgUcInMp2eQV3fBK2uuYO98CSV7sO1hxIcIXbUYTs4kQhyrZovJ/NAE+PvIjVq9dh+NA93ahv5gkJvhjqbxtvbUYIjxI4YDF21LNTgOA5ybRXfXk7hjWAfBf77wQi4SvlhzJuE3gyGBEc1uPZQgiPEjgxDuJnrbA/wl6hybRWfBhXe4DgOErEIrjK+JqfTNSc4F1f+ne7BtYsSHCF2JAyXZPEeHH+JqtfzNTgDiTDrfdt7cJTg2kPNRAixo/aGSwIAHzcpOJc6yFzEgGs/5JbV4FpJNYrV/EMHHV2idgklOELsaFKUPxKG+Ak1srZiQ7yAfB2gkwFuvigt0uByUcuoISYjilANrl10iUqIHUnEIshdxJCILXz1Gir5BrwiCSD3arkkbabVmUlwzFLPVkI1OEJ6ktrmeVBdvQGRCE3N7d4UUjEG+7vD2625W5YhwTE90FQPSF0dEGzPRwmOEDs6d6MSpVUNGBroiWAf06R0JS8PVdfvQD4gCCMANDXX2GJDvTEmrFWXLJEYcFHwya2xlhKcBXSJSogdFdypw6XCKlTWNZndLqkvh1bPUCfhk5nhktTF3CUt9WboECU4QuyoozkZpPX8xDL1Uj7BGS5RRRxQ36iDRqtrVZjmZugIJThC7EjfXoLTNUGq4YdJqpPxYx4aElz29QpsO5yLE7+2mjCanqR2iO7BEWJHLX1RzWS4mlJw0KNJpECjiL+nlhTTH006PU7nVyD7egV0rZ+q0iVqhyjBEWJHhmZsInPXTtXFEHEcaqW+0Bp6ZIlFcGluWgK0aiYC0CVqJ1CCI8SO2h0uqeomRBxQI/MH0xu3bRMLXbVaJzi6RO0IJThC7MjicEmMAZUFfENg31Ao3KQAgO9/4afBdJXyX1XzCY4uUS2hBEeIHT0cHQStnkEmaXONWl8BaGrg6SrHjPvHAGIXMMZw8SY/dNJvInwBoM09OLpE7QglOELsyHAvzUTFdf7dI1CYRKZJx4Snrobx4IzvwTXX4JrqAb2Ob/xLjFAzEUJ6gvJr/Hu/QcIqQ5s3sYiDr7sMQwM9ENrPrWUfFwXANX+FqRZnFtXgCLGjU3l3UNuoxagBXvBpvs8GrQaozAcAqN3C8Ol/foVYxOG3I4MAADKJCCqlHFOVgcYH4zi+Fqep5hOc3HSWrr6uR9fg0tPTMWbMGHh4eMDf3x9JSUm4cuVKu/tkZmY2T67b8pLL5XaKmJD2XSmuwrmCStRqtC0rb+fwl5iu/QA3P1Q3aFHToBVqcCb361qjJ6nt6tEJ7vDhw1iyZAlOnDiBAwcOoKmpCVOmTEFtbft/TE9PTxQVFQmv/Px8O0VMSPv05kY2KrrAv/tFCslMq2doaGpOcC5iMMag1emNu2oBNPBlB3r0Jeq+ffuMljMzM+Hv74/Tp0/jgQcesLgfx3FQqVSd/hyNRgONpmWeyqqqqq4HS0gnGB4aGNq1oe4O/4CB44DAUUad6tX1fId8hYsY5TWN+PBEPtxkYix+oOU+HdXg2teja3BtqdVqAICPT/szedfU1CA0NBTBwcGYOXMmLl261G759PR0KJVK4RUcHGy1mAlpzTDkuNDQt+g8/+4dDii8IBZxQvIbHOCBpxMGYtJQf2EE4CZdmyogJbh29ZoEp9fr8dxzz+E3v/kNhg8fbrFcZGQktm/fjs8//xwffvgh9Ho9xo0bh5s3b1rcZ8WKFVCr1cLrxo0btjgFQoSGviIR+Ptuxc2Xp0ExQhlDLa5Jq4erVAIPuQvEYj7B6dpe49Ilart69CVqa0uWLMFPP/2Eo0ePtlsuPj4e8fHxwvK4ceMwdOhQ/O1vf8O6devM7iOTySCTyawaLyHm6Fp31Sq/yk/cLHMH+kUIZVzEHBqajGtrhhqcTs/AGGuZlYtqcO3qFQlu6dKl+Oqrr3DkyBEMGDCgS/u6uLggJiYGOTk5NoqOkM4T7sFxXMvDBdVIo973fh4yuMkkOPBzMUL7uWF0mDckrbZr9QwuYkpwndGjL1EZY1i6dCk+++wzfP/99wgPD+/yMXQ6HS5evIjAwMCOCxNiY3PjQpE6LgyeonqgIo9fqRphVGZmdH/Murc/ymsacTq/AiKOM5qFS0f9UTutR9fglixZgp07d+Lzzz+Hh4cHiouLAQBKpRIKhQIAMG/ePPTv3x/p6ekAgLVr1+K+++5DREQEKisr8eabbyI/Px+LFi1y2HkQYqBUNE8ak3+avyHnFQy4mj40u1PbCABwl0mE7l0ijoOesTYd7pvvwemaAG0jIJHaNP7epkcnuK1btwIAJkyYYLQ+IyMDqampAICCggKIWlXfKyoq8NRTT6G4uBje3t6IjY3FsWPHcM8999grbELaxxhQfJH/uU3tzaC8mk9w/dxbEtbgAD6ZGU2pKpHyfVd1TXwtTtJ+C4O+hmOMJlVsq6qqCkqlEmq1Gp6e1P2FWIdez3A0pxyK+mLce/tLiCUuwLhlgMT4AddPt9Q48DM/TNKYMB/cP9i3/QOf2ArUVwIxT/A1wj6gs9/RHn0PjhBnomMMp/MrcOuXU/zAl35RJskNMJ5BK8irE90M6UGDRT36EpUQZ6JnDCK9Fr51ueA4d4uXp2G+rvD1kMFFxBmPHgK+FshxaGkmAlCCawclOELsRK8HvOuvQ6xvBCf3BLxCzZaTScR48j7TbR+eyEdZtQaP3jsAIf1aTfQs9eDfNdTFsC26RCXETvSMwa/2KgCAU42wPDmqBYbuXdrWo/oCgFzJvzeo7zpGZ0MJjhA70TVUwavhFkRcc4LrIoml7lqU4CyiBEeInYhKLgFgqJEHmm371hGJuZm1AEDhxb83VN5VfM6IEhwh9sAYuBK+7dttjyHdOoRY1EENrrGOb+xLBJTgCLEH9Q0omioxMtQPE+9P6NYhDP1Rm3Rt7sG5KFqam9BlqhFKcITYQ9F5iDgObgNGwM+7e43HLdbgALoPZwE1EyHE1hrrgNJf+J8DR3X7MH4eMmi0bvA09GdtTa4EakopwbVBCY4QW7v5I6DXok7mi/NlMnjUqDG8v7LLh4kN9UZsqLf5jYYHDfUV3Y/TCdElKiG21FQP3DoNALjdLxYn8u7gwk0b1LJc+/HvdeXWP3YvRgmOEFvKPcjPe+ruhwYlP2qvRNS1Br6d4ubHv9eWWf/YvRhdohJiK7dOt0wqMzgR2nr+R3E3E9zp/AocyylHpMoDU4a1mTXOtXnEEU0N0NQAuNBcwAAlOEKshzH+HljVLX6+hTK+WxbC7ge8gqGr5S9NDT0Suorj+Ea+Zp+iusgBmQc/y31tWZ8ZNqkjlOAIuRt6PVB1k09m5VeNn2JyHBA6jk9waGm/1np+ha6w2JPBwM2vOcGVUoJrRgmOkK7SNQGVBUDZFeD2Nb4ZiIFIDHioAM8gQDUKcPdr2U3fZtLnLmq3HRwAeAYCd34F1LeA/rHd+gxnQwmOkI7odUBVIVCZD1Tk85egel3Ldhc5P+2fbyTgE84PIW6GYRpAl25eohpqfhZrcMrmGeeqbnXr+M6IEhwh5uh1fA2t9Gc+sbXt4ynzAHwHA75DAK8QvubWgZEDlAj3dYNM0r1LVEMNTtu2q5aBRxB/WVxfyT9skLl363OcCSU4QlrT1ACFZ4Gic/zPBi4KwDuUH6TSOwxQeHd5PDc3mQRusu5/5Tq8B+ci5+/D1ZTyl6qBI7v9Wc6CEhwhjPGXoLey+Vqb4fJT6gYERfO1NPeALic0a3OTSRDkJYevu+k8DgLfIXyCK79KCQ6U4Ehfpm3kL0ELzwLVxS3rlf35m/R+UZ269OysS4VqNDTpMMjPHV6uXZ+/1M9DhjljQjooFAVcPwrcyeMffkhd2y/v5HpFT4YtW7YgLCwMcrkccXFxOHXqVLvlP/30U0RFRUEul2PEiBHYu3evnSIlPZ6uCbidC/yyFzi+GbjyDZ/cRBJ+EpjYVODeeUDAMKsmNwA4k1+BI1fLUVnXZNXjGnHzBTwCAL2Wr5H2cT2+BvfJJ58gLS0N27ZtQ1xcHDZu3IjExERcuXIF/v7+JuWPHTuG5ORkpKen47e//S127tyJpKQknDlzBsOHD3fAGRC70Wn5iVca1Px7UwOgbX411QN1t4HacoC1ukmv8AKCYvjkJnWzeGhrqNZoAQAe8rv72jXp9GAMkJp7WMFxQMg44NJnwI1TgP89fNLro3r8xM9xcXEYM2YMNm/eDADQ6/UIDg7Gf//3f+Oll14yKT9nzhzU1tbiq6++Etbdd999iI6OxrZt2zr1mU4/8TNj/Je+sZZPBg2V/JM3w8+aaoATAWIZ/6WXuQNSd/7JocyD/1nqxh9H1wjom/iaka6JXza8cyJALAFELvyAjGIp/y6Rt/wskrTc22KMv//FdMbHaftu+DxtQ3PMza/GWv4YHZG58/eqfIfwDwzscG+tRqPF+0d+BccBz0yIMJ+cOuHgL6W4cFONiVF+GDnAy3whxoDzu/gmLTJ3IGIy4DMIkHT9srin6ux3tEfX4BobG3H69GmsWLFCWCcSiTB58mQcP37c7D7Hjx9HWlqa0brExETs2bPH4udoNBpoNBphuaqqk9OvFV0Abp6y/KUyWs86Xt+dfTq1vs02nZa/hOlQDV/rsSVOBIhEfI8AZqH5Q1eIJYDcC5B5No90K+efLkoUgNyTb4Qr87TbA4ODv5Qir7wWNc21Nz8PWbeTGwC4SsXQM4bDV8pw/kYlAGB0mA+GBrb6knMcMPRhPsnVlgOX9vDrDSP/ciIAXMvvwMEPT0yMWWS1Q/XoBFdeXg6dToeAgACj9QEBAfjll1/M7lNcXGy2fHFxsdnyAJCeno41a9Z0PcCmeqCmF4/eYOi/KPdqfilbXkBLLU9TDTRW880mGmta3kVivnYmNrykzS8Xfj2aa3g6Lf+ubWh+1/DvjPFJzVy7Lo4zPqZIYnx8sQtfw5R7Gsft4tqjvrD1TTqo6/l7biKOQ1x4v7s63sgBXvi5qAqVdU0or+Hb5jU06UwLytyBe1OAguNAySW+httUz7/6kB6d4OxlxYoVRrW+qqoqBAd3oi+ffxTg3nwf0OhL1ernrq635rFMvujNyyIRf5lpocW9XTDWcpnJ9Hyy5MTNNTpJ83LPSVTddd/AfogJ8YJEJIK7TAKF9O4eXCik/KTQZTUaaHUMjAFebhb+jhIpMDCBfzXW8f9Z6TTNtfvm/1x69h2qu9ajE5yvry/EYjFKSkqM1peUlEClUpndR6VSdak8AMhkMshk7bQtsqR1bYd0DcfxX0Anui9kjo+b9c9PIhYhUKno2k5S1z7ZZKRHNxORSqWIjY1FVlaWsE6v1yMrKwvx8fFm94mPjzcqDwAHDhywWJ4Q4rx6dA0OANLS0pCSkoLRo0dj7Nix2LhxI2prazF//nwAwLx589C/f3+kp6cDAJ599lkkJCRg/fr1mD59Oj7++GNkZ2fj73//uyNPgxDiAD0+wc2ZMwdlZWVYuXIliouLER0djX379gkPEgoKCiBqNb7WuHHjsHPnTvzlL3/Byy+/jMGDB2PPnj3UBo6QPqjHt4NzBKdvB0dIL9fZ72iPvgdHCCF3gxIcIcRpUYIjhDitHv+QwREMtyU73WWLEGJXhu9mR48QKMGZUV1dDQCd681ACHGY6upqKJWWG9vTU1Qz9Ho9CgsL4eHhAa6XdBcydC+7ceOG0zz5dcZzAui8rIExhurqagQFBRk1E2uLanBmiEQiDBgwwNFhdIunp6dTfWkA5zwngM7rbrVXczOghwyEEKdFCY4Q4rQowTkJmUyGVatWdW9UlB7KGc8JoPOyJ3rIQAhxWlSDI4Q4LUpwhBCnRQmOEOK0KMERQpwWJbhe6q9//SvGjRsHV1dXeHl5dWofxhhWrlyJwMBAKBQKTJ48GdeuXbNtoF10584dzJ07F56envDy8sLChQtRU1PT7j4TJkwAx3FGrz/84Q92iti8LVu2ICwsDHK5HHFxcTh16lS75T/99FNERUVBLpdjxIgR2Lt3r50i7ZqunFdmZqbJ30Uul9sxWkpwvVZjYyNmz56NP/7xj53e54033sA777yDbdu24eTJk3Bzc0NiYiIaGhpsGGnXzJ07F5cuXcKBAwfw1Vdf4ciRI1i8eHGH+z311FMoKioSXm+88YYdojXvk08+QVpaGlatWoUzZ85g1KhRSExMRGlpqdnyx44dQ3JyMhYuXIizZ88iKSkJSUlJ+Omnn+wcefu6el4A36uh9d8lPz/fjhEDYKRXy8jIYEqlssNyer2eqVQq9uabbwrrKisrmUwmY7t27bJhhJ33888/MwDsxx9/FNZ98803jOM4duvWLYv7JSQksGeffdYOEXbO2LFj2ZIlS4RlnU7HgoKCWHp6utnyv//979n06dON1sXFxbGnn37apnF2VVfPq7P/Nm2JanB9RF5eHoqLizF58mRhnVKpRFxcHI4fP+7AyFocP34cXl5eGD16tLBu8uTJEIlEOHnyZLv7fvTRR/D19cXw4cOxYsUK1NXV2TpcsxobG3H69Gmj37NIJMLkyZMt/p6PHz9uVB4AEhMTe8zfBejeeQFATU0NQkNDERwcjJkzZ+LSpUv2CFdAne37iOLiYgAQJusxCAgIELY5WnFxMfz9/Y3WSSQS+Pj4tBvj448/jtDQUAQFBeHChQt48cUXceXKFezevdvWIZsoLy+HTqcz+3v+5ZdfzO5TXFzco/8uQPfOKzIyEtu3b8fIkSOhVqvx1ltvYdy4cbh06ZLdBrOgGlwP8tJLL5nclG37svSPqSez9XktXrwYiYmJGDFiBObOnYsdO3bgs88+Q25urhXPgnRVfHw85s2bh+joaCQkJGD37t3w8/PD3/72N7vFQDW4HuRPf/oTUlNT2y0zcODAbh1bpVIBAEpKShAYGCisLykpQXR0dLeO2VmdPS+VSmVyw1qr1eLOnTtC/J0RFxcHAMjJycGgQYO6HO/d8PX1hVgsRklJidH6kpISi+egUqm6VN4RunNebbm4uCAmJgY5OTm2CNEsSnA9iJ+fH/z8/Gxy7PDwcKhUKmRlZQkJraqqCidPnuzSk9ju6Ox5xcfHo7KyEqdPn0ZsbCwA4Pvvv4derxeSVmecO3cOAIwSub1IpVLExsYiKysLSUlJAPgBVLOysrB06VKz+8THxyMrKwvPPfecsO7AgQOIj4+3Q8Sd053zakun0+HixYt46KGHbBhpGw59xEG6LT8/n509e5atWbOGubu7s7Nnz7KzZ8+y6upqoUxkZCTbvXu3sPzaa68xLy8v9vnnn7MLFy6wmTNnsvDwcFZfX++IUzBr6tSpLCYmhp08eZIdPXqUDR48mCUnJwvbb968ySIjI9nJkycZY4zl5OSwtWvXsuzsbJaXl8c+//xzNnDgQPbAAw846hTYxx9/zGQyGcvMzGQ///wzW7x4MfPy8mLFxcWMMcaefPJJ9tJLLwnlf/jhByaRSNhbb73FLl++zFatWsVcXFzYxYsXHXUKZnX1vNasWcP279/PcnNz2enTp9ljjz3G5HI5u3Tpkt1ipgTXS6WkpDAAJq+DBw8KZQCwjIwMYVmv17NXXnmFBQQEMJlMxiZNmsSuXLli/+Dbcfv2bZacnMzc3d2Zp6cnmz9/vlHSzsvLMzrPgoIC9sADDzAfHx8mk8lYREQEe+GFF5harXbQGfDeffddFhISwqRSKRs7diw7ceKEsC0hIYGlpKQYlf/Xv/7FhgwZwqRSKRs2bBj7+uuv7Rxx53TlvJ577jmhbEBAAHvooYfYmTNn7BovDZdECHFa9BSVEOK0KMERQpwWJThCiNOiBEcIcVqU4AghTosSHCHEaVGCI4Q4LUpwhBCnRQmO9HmZmZlGw76vXr3aaACC1NRUof8l6V0owRG7SU1NtThfwpIlS8BxnNGoI7ZILGFhYdi4caPRujlz5uDq1asW99m0aRMyMzOF5QkTJhh1jCc9FyU4YlfBwcH4+OOPUV9fL6xraGjAzp07ERIS4pCYFAqFyUCbrSmVyk5P7EN6FkpwxK7uvfdeBAcHG422u3v3boSEhCAmJuaujm2uZpWUlCTUCidMmID8/Hw8//zzwkCbgOklaluta5Kpqak4fPgwNm3aJBwjLy8PEREReOutt4z2O3fuHDiOs+v4Z8QYJThidwsWLEBGRoawvH37dsyfP9/mn7t7924MGDAAa9euFWZ56qpNmzYhPj7eaBavkJAQk3MCgIyMDDzwwAOIiIiw1imQLqIER+zuiSeewNGjR5Gfn4/8/Hz88MMPeOKJJ2z+uT4+PhCLxfDw8IBKperWiLlKpRJSqRSurq7CMcRiMVJTU3HlyhVhntCmpibs3LkTCxYssPZpkC6gEX2J3fn5+WH69OnIzMwEYwzTp0+Hr6+vo8O6K0FBQZg+fTq2b9+OsWPH4ssvv4RGo8Hs2bMdHVqfRjU44hALFixAZmYmPvjgA6vVckQiEdoOb9jU1GSVY3fGokWLhAcoGRkZmDNnDlxdXe32+cQUJTjiEFOnTkVjYyOampqQmJholWP6+fkZ3VfT6XQms8NLpVLodLq7+hxLx3jooYfg5uaGrVu3Yt++fXR52gPQJSpxCLFYjMuXLws/W6JWq4VJZAz69euH4OBgk7IPPvgg0tLS8PXXX2PQoEHYsGEDKisrjcqEhYXhyJEjeOyxxyCTybp1aRwWFoaTJ0/i+vXrcHd3h4+PD0QikXAvbsWKFRg8eHCPmjSmr6IaHHEYT09PeHp6tlvm0KFDiImJMXqtWbPGbNkFCxYgJSUF8+bNQ0JCAgYOHIiJEycalVm7di2uX7+OQYMGdXsGs+XLl0MsFuOee+6Bn58fCgoKhG0LFy5EY2OjXZ4Kk47RnAyEWNF//vMfTJo0CTdu3DCZBZ7YHyU4QqxAo9GgrKwMKSkpUKlU+OijjxwdEgFdohJiFbt27UJoaCgqKyvxxhtvODoc0oxqcIQQp0U1OEKI06IERwhxWpTgCCFOixIcIcRpUYIjhDgtSnCEEKdFCY4Q4rQowRFCnNb/ByMdqKwdFZwgAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:39:54.013488Z", + "iopub.status.busy": "2024-03-23T03:39:54.013180Z", + "iopub.status.idle": "2024-03-23T03:39:54.383834Z", + "shell.execute_reply": "2024-03-23T03:39:54.382764Z" + }, + "papermill": { + "duration": 0.393386, + "end_time": "2024-03-23T03:39:54.386070", + "exception": false, + "start_time": "2024-03-23T03:39:53.992684", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCz0lEQVR4nO3de1xUdf4/8NeZGWaG6yAwMKAgKIimCIZJmIkm39DKNFsz10000y66v4wss/LatmyZpW2u7bZfJbfUatdsu3wto9K8i5e8hkJcVBgUlTvM9fP74zAHBma4DHNzfD8fj/MY5pzPOfM5wrz9fM7nxjHGGAghxAOJXJ0BQghxFApwhBCPRQGOEOKxKMARQjwWBThCiMeiAEcI8VgU4AghHosCHCHEY0lcnQF3ZDQaUVZWBn9/f3Ac5+rsEELaYIyhtrYWEREREImsl9MowFlQVlaGyMhIV2eDENKJixcvok+fPlaPU4CzwN/fHwD/jxcQEODi3BBC2qqpqUFkZKTwXbWGApwFpmppQEAABThC3Fhnj5CokYEQ4rEowBFCPBYFOEKIx6IARwjxWBTgCCEeiwIcIcRjUYAjxIXy1bX44sRl1Gn0rs6KR6IAR4gLfXOqHL9drceJ0ipXZ8UjUYAjxA1cutHg6ix4JApwhLgBI61t5xAuDXB79uzBxIkTERERAY7jsGPHDrPjs2bNAsdxZtv48eM7ve769esRHR0NuVyOlJQUHD582EF3QIh9pPQLcnUWPJJLA1x9fT0SExOxfv16q2nGjx+P8vJyYdu6dWuH1/zkk0+QlZWF5cuX49ixY0hMTERGRgauXLli7+wT0mMh/jLIvcRQeHu5OiseyaWD7SdMmIAJEyZ0mEYmk0GlUnX5mm+//Tbmzp2L2bNnAwDef/99fP3119i4cSNeeumlHuWXEHt77M6+rs6CR3P7Z3A//fQTQkNDER8fj6effhrXrl2zmlar1eLo0aNIT08X9olEIqSnp+PAgQNWz9NoNKipqTHbCHGGG/Va5BVfx5myaldnxSO5dYAbP348Nm/ejNzcXLzxxhvYvXs3JkyYAIPBYDF9ZWUlDAYDwsLCzPaHhYVBrVZb/Zzs7GwoFApho8kuibNcq9fg5wuVOHOZ/lN1BLeeD+7RRx8Vfk5ISMDQoUPRv39//PTTTxg3bpzdPmfJkiXIysoS3psm0yPEkfQGI778pRwAoDEYXZwbz+TWJbi2+vXrh5CQEBQUFFg8HhISArFYjIqKCrP9FRUVHT7Hk8lkwuSWNMklcRZ9q74hjFE/EUe4qQLcpUuXcO3aNYSHh1s8LpVKkZycjNzcXGGf0WhEbm4uUlNTnZVNQrrEYBbgXJgRD+bSAFdXV4cTJ07gxIkTAICioiKcOHECpaWlqKurwwsvvICDBw+iuLgYubm5mDRpEmJjY5GRkSFcY9y4cXjvvfeE91lZWfjggw/w4Ycf4ty5c3j66adRX18vtKoS4i5al+CMFOEcwqXP4PLy8jB27Fjhvek5WGZmJjZs2ICTJ0/iww8/RFVVFSIiInDvvffitddeg0wmE84pLCxEZWWl8H7atGm4evUqli1bBrVajaSkJOzcubNdwwMhrta6BGegoQwOwTGq/LdTU1MDhUKB6upqeh5HHOZqrQYfHSwBAPjKxJg3ur+Lc3Tz6Op39KZ6BkeIJ2ldanswsbcLc+K53LqbCCGezMgYpBIRAuQSqBRyV2fHI1GAI8RFIgK9MX9srKuz4dEowBHiQnqDEafLamAwMtweFdjpQsakeyjAEeJCeiPDj7/yM90k9lFAIqYAZ08U4AhxkdJrDThY1DJ5hIEx+kLaGbWiEuIiNU06XL7RKLw30nBUu6MAR4iL6Nt07jVQl1S7owBHiIsY2hTZaDSD/VGAI8RF2s6QZKQAZ3cU4AhxEX3bEhxVUe2OGm0IcRFTlVTpL8PdcSEIkNPCM/ZGAY4QFzEygOOAmBBf9A32dXV2PBIFOEJcJG2AEqPjQmiySweiAEeIC3Ech8KrdWjUGhAd4gN/qqbaFTUyEOJiB3+7hu/PVeB6vdbVWfE4VIIjxEUO/XYNV2o1uFqrAUD94ByBAhwhLlJW3YjiygbhPa3LYH9URSXERdp29KWlUe2PAhwhLtJ2qBaV4OyPAhwhLmIabC9qnuSSnsHZHwU4QlzEFNC8JHyAoxKc/VEjAyEuojfwAS0lJhgKby8o/WWdnEG6y6UluD179mDixImIiIgAx3HYsWOHcEyn02Hx4sVISEiAr68vIiIiMHPmTJSVlXV4zRUrVoDjOLNt4MCBDr4TQrrPVIKL7OWN2FA/KLypk6+9ubQEV19fj8TERDz++OOYMmWK2bGGhgYcO3YMS5cuRWJiIm7cuIFnn30WDz74IPLy8jq87uDBg/H9998L7yUSKqgS9zNnVAz0RgaJiNZhcBSXfvMnTJiACRMmWDymUCiwa9cus33vvfceRowYgdLSUkRFRVm9rkQigUqlsmteCbE3kYiDVMShoqYJ1+u1CPaTItSf1ke1p5uqkaG6uhocxyEwMLDDdBcuXEBERAT69euHGTNmoLS0tMP0Go0GNTU1ZhshznK2rAY7T6tRUFHn6qx4nJsmwDU1NWHx4sWYPn06AgICrKZLSUlBTk4Odu7ciQ0bNqCoqAh33303amtrrZ6TnZ0NhUIhbJGRkY64BUIEeoMRX58sx7dn1GDgn8VRLxH7uykCnE6nwyOPPALGGDZs2NBh2gkTJmDq1KkYOnQoMjIy8M0336Cqqgqffvqp1XOWLFmC6upqYbt48aK9b4EQMzoDw/mKWpwtq4FExH8NaUZf+3P7p++m4FZSUoIffvihw9KbJYGBgRgwYAAKCgqsppHJZJDJqImeOI9punIRxwmLPdOaDPbn1iU4U3C7cOECvv/+ewQHB3f7GnV1dSgsLER4eLgDckiIbUx94CRijkYyOJBLA1xdXR1OnDiBEydOAACKiopw4sQJlJaWQqfT4Xe/+x3y8vLw8ccfw2AwQK1WQ61WQ6ttmTdr3LhxeO+994T3ixYtwu7du1FcXIz9+/fjoYceglgsxvTp0519e4RYpdHzJTiZRASxiEYyOIpLq6h5eXkYO3as8D4rKwsAkJmZiRUrVuC///0vACApKcnsvB9//BFjxowBABQWFqKyslI4dunSJUyfPh3Xrl2DUqnEqFGjcPDgQSiVSsfeDCHdsK+A/5tlrGUsKgU4+3NpgBszZgxYB7/Ujo6ZFBcXm73ftm1bT7NFiMP5ysQA+EVnooN94DNERSMZHMDtGxkI8UTJfYPQqDNgeN8gBPvJEOxHjVyOQAGOEBdQ+svw0LA+rs6Gx6MAR4iTMcZgMDKIRfxkELVNOlTUaCD3EqFPLx9XZ8+juHU3EUI8UWWdFn/9oQAf/PwbAKC8uglf/lKGA4XXXJwzz0MBjhAnM/V3EzePYDBNJkKtqPZHAY4QJzONYjBNk9TS0ddlWfJYFOAIcTLTKAZTB1/TK41FtT8KcIQ4mWmxmbYluK70+yTdQwGOECdreQbXpgRHY1HtjgIcIU4mPIMTt30GRwHO3qgfHCFO5i/zQv9QP4Q2r6IV4C3BuEGhkEnELs6Z56EAR4iTRQX7ICq4pUOvj1SCoX0CXZchD0ZVVEKIx6ISHCEupjcYUVbVBCNjiA7xdXV2PAqV4Ahxsrzi61j3/QV8f7YCANCkN+I/xy5hx4nLLs6Z56EAR4iTGYwMRmZaSwsQC/3gaF0Ge6MAR4iTmUKYaQyqqNW3kMaj2hcFOEKczBTEmgtuQgkOoOFa9kYBjhBna45hHGfe0RcAjDTg3q4owBHiZKbHbKawJhJxQmmOSnD2RQGOECczNS+0LrmJabiWQ1A/OEKcLNBbiqggH/TykQr7RsWFgOM4yCRU5rAnCnCEOFlCHwUS+ijM9g2L6uWi3Hg2l/53sWfPHkycOBERERHgOA47duwwO84Yw7JlyxAeHg5vb2+kp6fjwoULnV53/fr1iI6OhlwuR0pKCg4fPuygOyCEuDOXBrj6+nokJiZi/fr1Fo+/+eabePfdd/H+++/j0KFD8PX1RUZGBpqamqxe85NPPkFWVhaWL1+OY8eOITExERkZGbhy5YqjboOQHrtS04SL1xug0RtcnRXPwtwEAPb5558L741GI1OpVGz16tXCvqqqKiaTydjWrVutXmfEiBFs/vz5wnuDwcAiIiJYdnZ2l/NSXV3NALDq6uru3QQhXfDDrxVs/Y8XWF7xNWHfxr2/sbe/y2eXbzS4MGc3j65+R932iWZRURHUajXS09OFfQqFAikpKThw4IDFc7RaLY4ePWp2jkgkQnp6utVzAECj0aCmpsZsI8RR9AYGjc5otsgMzerrGG4b4NRqNQAgLCzMbH9YWJhwrK3KykoYDIZunQMA2dnZUCgUwhYZGdnD3BNinWkkg6ill4jQ6ZeGatmX2wY4Z1qyZAmqq6uF7eLFi67OEvFgrM1QLYD6wTmK2wY4lUoFAKioqDDbX1FRIRxrKyQkBGKxuFvnAIBMJkNAQIDZRoijsDZDtQBA3PxNpPhmX24b4GJiYqBSqZCbmyvsq6mpwaFDh5CammrxHKlUiuTkZLNzjEYjcnNzrZ5DiLO1HaoFUBXVUVza0beurg4FBQXC+6KiIpw4cQJBQUGIiorCwoUL8ac//QlxcXGIiYnB0qVLERERgcmTJwvnjBs3Dg899BAWLFgAAMjKykJmZiaGDx+OESNGYO3ataivr8fs2bOdfXuEWERDtZzHpQEuLy8PY8eOFd5nZWUBADIzM5GTk4MXX3wR9fX1mDdvHqqqqjBq1Cjs3LkTcrlcOKewsBCVlZXC+2nTpuHq1atYtmwZ1Go1kpKSsHPnznYND4S4Si8fKSIC5fCVtayiNbh3AKKCfaBsXmmL2AfHGJWJ26qpqYFCoUB1dbXHPY8zGAzQ6XSuzgYhHfLy8oJYbH0Zxa5+R20qwf3222/o16+fLacSF2GMQa1Wo6qqytVZIaRLAgMDoVKpzBpjusumABcbG4u0tDTMmTMHv/vd78yqjMQ9mYJbaGgofHx8evRHQ+zPYDTCyPjncmLRrf27YYyhoaFBGF4ZHh5u87VsCnDHjh3Dpk2bkJWVhQULFmDatGmYM2cORowYYXNGiOMYDAYhuAUHB7s6O7e8qgYtdAYGf7kEci++GlbdqEOT3gA/mQRyGU3y4+3tDQC4cuUKQkNDO6yudsSmbiJJSUlYt24dysrKsHHjRpSXl2PUqFEYMmQI3n77bVy9etWmzBDHMD1z8/Hx6SQlcQYja98dxFRmowfiLUx/rz15ZtyjfnASiQRTpkzBZ599hjfeeAMFBQVYtGgRIiMjMXPmTJSXl/fk8sTOqFrqxoQIRyHOxB5/rz0KcHl5eXjmmWcQHh6Ot99+G4sWLUJhYSF27dqFsrIyTJo0qccZJMTztA9iVIJzDJsC3Ntvv42EhASMHDkSZWVl2Lx5M0pKSvCnP/0JMTExuPvuu5GTk4Njx47ZO7+EdGrWrFlmncHtYcyYMVi4cGGHaaKjo7F27dpOr2UpiFEBzjFsCnAbNmzA73//e5SUlGDHjh144IEHIBKZXyo0NBT/+7//a5dMkltXVwLLTcfCUC3cQo8PVqxYgaSkJKd8lk3NNbt27UJUVFS7oMYYw8WLFxEVFQWpVIrMzEy7ZJIQT0dVVMewqQTXv39/s+FRJtevX0dMTEyPM0UIwFc1d+/ejXXr1oHjOHAch8LCQsyZMwcxMTHw9vZGfHw81q1bZ/H8lStXQqlUIiAgAE899RS0Wm2XPre+vh4zZ86En58fwsPDsWbNmnZprly5gokTJ8Lb2xsxMTH4+OOP26XhOA4bNmzAhAkT4O3tjX79+uHf//43xCIOEhGHkpIScByHTz/9FBnjxiI6rBfuuXskzp8/jyNHjmD48OHw8/PDhAkTutUzYePGjRg8eDBkMhnCw8OFcdoAUFpaikmTJsHPzw8BAQF45JFHzGbfMZWu/vWvfyE6OhoKhQKPPvooamtrhTRGoxFvvvkmYmNjIZPJEBUVhddff104vnjxYgwYMAA+Pj7o168fli5dKrSE5uTkYOXKlfjll1+E32lOTk6X763bbJkumOM4VlFR0W5/cXEx8/HxseWSbsXTpixvbGxkZ8+eZY2NjWb7NTqD1U2nN3Q5rbaLaburqqqKpaamsrlz57Ly8nJWXl7Ompqa2LJly9iRI0fYb7/9xj766CPm4+PDPvnkE+G8zMxM5ufnx6ZNm8ZOnz7NvvrqK6ZUKtnLL7/cpc99+umnWVRUFPv+++/ZyZMn2QMPPMD8/f3Zs88+K6SZMGECS0xMZAcOHGB5eXls5MiRzNvbm73zzjtCGgAsODiYffDBByw/P5+9+uqrTCwWs7NnzzLGGCsqKmIA2MCBA9nOnTvZ2bNn2Z133smSk5PZmDFj2N69e9mxY8dYbGwse+qpp7qU97/97W9MLpeztWvXsvz8fHb48GEhTwaDgSUlJbFRo0axvLw8dvDgQZacnMzS0tKE85cvX878/PzYlClT2KlTp9iePXuYSqUy+7d78cUXWa9evVhOTg4rKChgP//8M/vggw+E46+99hrbt28fKyoqYv/9739ZWFgYe+ONNxhjjDU0NLDnn3+eDR48WPidNjRYnqbd2t8tY13/jnYrwD333HPsueeeYyKRiD355JPC++eee479v//3/1hKSgobOXJkdy7plm6VAPf2d/lWt8+PXTJL+9fc81bTfnqk1Czthp8KLKazRVpamllgsWT+/Pns4YcfFt5nZmayoKAgVl9f35KnDRuYn58fMxg6DrS1tbVMKpWyTz/9VNh37do15u3tLeQjPz+fAWCHDx8W0pw7d44BaBfg2gamlJQU9vTTTzPGWgLcP//5T+H41q1bGQCWm5sr7MvOzmbx8fEd5tskIiKCvfLKKxaPfffdd0wsFrPS0pbf15kzZ8zuZfny5czHx4fV1NQIaV544QWWkpLCGGOspqaGyWQys4DWmdWrV7Pk5GTh/fLly1liYmKn59kjwHXrGdzx48dNpT6cOnUKUmnLwrVSqRSJiYlYtGhRz4uVhHRg/fr12LhxI0pLS9HY2AitVtvuoXViYqJZx+bU1FTU1dXh4sWL6Nu3r9VrFxYWQqvVIiUlRdgXFBSE+Ph44f25c+cgkUiQnJws7Bs4cCACAwPbXa/tPISpqak4ceKE2b6hQ4fCyBiMRoaQECUAICEhQTgeFhbWpVXhrly5grKyMowbN87i8XPnziEyMtJsSv7bbrsNgYGBOHfuHO644w4AfGuwv7+/kCY8PFz4/HPnzkGj0Vj9DIBf2e7dd99FYWEh6urqoNfrXTZpRbcC3I8//ggAmD17NtatW+dxM23cauaPjbV6rO1wyHmj+1tN27YB8PG7HPccdtu2bVi0aBHWrFmD1NRU+Pv7Y/Xq1Th06JDDPtPe9AaGyjoNdM2rznh5eUGrN6K6UYcGnUHYZ8JxHIxGo8VrtWYa3tRTrT+77ed39hkHDhzAjBkzsHLlSmRkZEChUGDbtm0Wn2M6g02NDJs2baLg5gGkEpHVTSIWdTmtVxfT2pRHqRQGQ8taofv27cPIkSPxzDPPYNiwYYiNjUVhYWG783755Rc0NjYK7w8ePAg/P79OFxTq378/vLy8zALmjRs3cP78eeH9wIEDodfrcfToUWFffn6+xZlaDh482O597IB4fmLLVk2mpv8ketKK6u/vj+joaLMZrVsbNGgQLl68aLbmyNmzZ1FVVYXbbrutS58RFxcHb29vq5+xf/9+9O3bF6+88gqGDx+OuLg4lJSUmKVp+zt1pC6X4KZMmYKcnBwEBARgypQpHabdvn17jzNGCMBXlw4dOoTi4mL4+fkhLi4OmzdvxrfffouYmBj861//wpEjR9q13mu1WsyZMwevvvoqiouLsXz5cixYsKBd16a2/Pz8MGfOHLzwwgsIDg5GaGgoXnnlFbPz4uPjMX78eDz55JPYsGEDJBIJFi5caLF089lnn2H48OEYNWoUPv74Yxw+fBir393AH2y9qpbt/0RmVqxYgaeeegqhoaGYMGECamtrsW/fPvzxj39Eeno6EhISMGPGDKxduxZ6vR7PPPMM0tLSMHz48C5dXy6XY/HixXjxxRchlUpx11134erVqzhz5gzmzJmDuLg4lJaWYtu2bbjjjjvw9ddf4/PPPze7RnR0tDB7d58+feDv7w+ZzDETfXb5v1WFQiGMDWu9xJ6ljRB7WbRoEcRiMW677TYolUpkZGRgypQpmDZtGlJSUnDt2jU888wz7c4bN24c4uLiMHr0aEybNg0PPvggVqxY0aXPXL16Ne6++25MnDgR6enpGDVqlNnzNoCvxURERCAtLQ1TpkzBvHnzEBoa2u5aK1euxLZt2zB06FBs3rwZW7duRfzAgRY+lf9u9XQkQ2ZmJtauXYu//e1vGDx4MB544AFcuHCB/wSOwxdffIFevXph9OjRSE9PR79+/fDJJ5906zOWLl2K559/HsuWLcOgQYMwbdo04Rndgw8+iOeeew4LFixAUlIS9u/fj6VLl5qd//DDD2P8+PEYO3YslEoltm7d2rOb7gDN6GuBp83o29TUhKKiIsTExNDcfU7EcRw+//zzdsPGrtQ2gTEgyFcqVO91BiOu12sh4jiatrxZR3+3Xf2O2vRgpLGxEQ0NDcL7kpISrF27Ft99950tlyPklsRZ+JnRWAa7sinATZo0CZs3bwYAVFVVYcSIEVizZg0mTZqEDRs22DWDhNhTaWkp/Pz8rG6lpaWOz0RHMayT+NZR3n/++We7ZtMT2Dyj7zvvvAMA+Pe//w2VSoXjx4/jP//5D5YtW4ann37arpkkxF4iIiLa9UNre9xerD39EYs4Po61KsKJOA4+UnGnjQ0d5b13797dzaLHsynANTQ0CB0Bv/vuO0yZMgUikQh33nlnuyZhQtyJRCJBbKz1/n/OEOzX/hmbSMTBX+5lIbU5V+f9ZmNTFTU2NhY7duzAxYsX8e233+Lee+8FwPek9oSH8oQQz2BTgFu2bBkWLVqE6OhopKSkCMNRvvvuOwwbNsyuGYyOjhZmHWi9zZ8/32L6nJycdmmp5ZDcDAxGI/QGo9WqLek+m6qov/vd7zBq1CiUl5cjMTFR2D9u3Dg89NBDdsscABw5csSs1/Pp06fxP//zP5g6darVcwICApCfny+8p7UIiLtgjOF6PT9tUy8fKUStxsRV1vH7Q/ykENPfrF3YvD6ZSqWCSqUy2+eIZQOVSqXZ+7/85S/o378/0tLSrJ7DcVy7vBHiLvRGyyU0DnwjKhXg7MemAFdfX4+//OUvyM3NxZUrV9oNBP7tt9/skrm2tFotPvroI2RlZXVYKqurq0Pfvn1hNBpx++23489//jMGDx5sNb1Go4FGoxHe19TU2DXfhHSJKcIRu7EpwD3xxBPYvXs3HnvsMYSHhzutCrhjxw5UVVVh1qxZVtPEx8dj48aNGDp0KKqrq/HWW29h5MiROHPmDPr06WPxnOzsbKxcudJBuSakhVn8avO14cCBUVdfu7JpqFZgYCC+/vpr3HXXXY7Ik1UZGRmQSqX48ssvu3yOTqfDoEGDMH36dLz22msW01gqwUVGRtJQLTcwZswYJCUldWm1KmfoaX6MjOFqLf+3pvSXQdSqcFBZq4GBMfTykdo8+4onscdQLZtKcL169UJQUJAtp9qspKQE33//fbdnKvHy8sKwYcNQUFBgNY1MJnPYbAbE9bRardnkrC7VeoqktseEKiqV4ezFpv8mXnvtNSxbtsxsPKqjbdq0CaGhobj//vu7dZ7BYMCpU6cQHh7uoJwRR7F10RnTuqivv/46IiIihNl49+/fj6SkJMjlcgwfPhw7duwAx3FmowNOnz6NCRMmwM/PD2FhYXjssceEBZYs5ae4uNhu90tro9qfTSW4NWvWoLCwEGFhYYiOjm43A6i9F3w2Go3YtGkTMjMzIZGYZ3nmzJno3bs3srOzAQCrVq3CnXfeidjYWFRVVWH16tUoKSnBE088Ydc83dQYAww613y22KvLa4CuW7cO58+fx5AhQ7Bq1SoAfO2hT58++OyzzxAcHIz9+/dj3rx5CA8PxyOPPCKcm5ubi4CAAOzatQsAX6WZOHEi7rvvPmzZsgUlJSXt1lutqqrCPffcgyeeeALvvPMOGhsbsXjxYjzyyCP44YcfLOanbSt/V/DV0vZRTO4lhsHIzLqOkJ6xKcDZe9Xwznz//fcoLS3F448/3u5YaWmp2WSEN27cwNy5c6FWq9GrVy8kJydj//79XZ6x9JZg0AE/u2YKadz9PCDpWnVRoVBAKpXCx8fHrNtP6wahmJgYHDhwAJ9++qlZgPP19cU///lPoWr6/vvvg+M4fPDBB5DL5bjttttw+fJlzJ07Vzjnvffew7Bhw/DnP/9Z2Ldx40ZERkbi/PnzGDBggMX8dIdIZH06JF+Zzb22iBU2/YsuX77c3vno0L333mu1d/dPP/1k9v6dd94RJgIgnqkri84kJCSYPXfLz8/H0KFDzR5Wt+23+csvv+DHH3+En59fu88sLCzEgAED7HsjxOFs/i+jqqoK//73v1FYWIgXXngBQUFBOHbsGMLCwmhWA3cn9uJLUq767B7o6qIzvr6+3b52XV0dJk6ciDfeeKPdMWc8wzUyBsb4GryIRjLYhU0B7uTJk0hPT4dCoUBxcTHmzp2LoKAgbN++HaWlpcJcccRNcVyXq4mu1tGiMyaWFp1pKz4+Hh999BE0Go3QYn7kyBGzNLfffjv+85//IDo6ut2zXmv56S6DkV89iwOHXr7mv4OaRh00eiP85RL4SKm6ag82taJmZWVh1qxZuHDhglmR/7777sOePXvsljlCWi86U1lZibi4OOTl5eHbb7/F+fPnsXTp0naBypLf//73MBqNmDdvHs6dO4dvv/0Wb731FoCWscrz58/H9evXMX36dBw5cgSFhYX49ttvMXv2bCGotc1PV5bza40xQGdgwpKBrZnyQa2o9mNTgDty5AiefPLJdvt79+4NtVrd40wRYmLrojNtBQQE4Msvv8SJEyeQlJSEV155BcuWLQMA4T/piIgI7Nu3DwaDAffeey8SEhKwcOFCBAYGCg1ZbfNj8wzAFmqgLdOWE7vpcN17K5RKJTt27BhjjDE/Pz9WWFjIGGPsu+++Y3369LHlkm6lurqaAWDV1dWuzopdNDY2srNnz7LGxkZXZ8WtfPTRR8zLy4s1NDQ47TO1egNTVzeyipr2v4vqBi1TVzey2kat0/Ljzjr6u+3qd9SmEtyDDz6IVatWQafj+1JxHIfS0lIsXrwYDz/8sB3DLyH2s3nzZuzduxdFRUXYsWOH0MfNXivCd4elJgR7LP5MzNkU4NasWYO6ujoolUo0NjYiLS0NsbGx8Pf3x+uvv27vPBJiF2q1Gn/4wx8waNAgPPfcc5g6dSr+8Y9/uCg37UMcVVHtz6amGoVCgV27dmHfvn345ZdfUFdXh9tvvx3p6en2zh8hdvPiiy/ixRdfdG0mOopeVISzu24HOKPRiJycHGzfvh3FxcXgOA4xMTFQqVRgjNHsuYR0hDP1c2t/yEvEQe4lhpeYvkP20q0qKmMMDz74IJ544glcvnwZCQkJGDx4MEpKSjBr1iy7T1dOiKfxEosQ6i+3uLKWzEsMhbcXvKkPnN10618yJycHe/bsQW5uLsaOHWt27IcffsDkyZOxefNmzJw5066ZJPbR3T5bhLiSPf5euxXgtm7dipdffrldcAOAe+65By+99BI+/vhjCnBuRiqVQiQSoaysDEqlElKplB4luCHGWmbzbTdUy2gEDFpALAVEnj0ZJmMMWq0WV69ehUgk6tFcft0KcCdPnsSbb75p9fiECRPw7rvv2pwZ4hgikQgxMTEoLy9HWVmZq7NzSzMYGTR6A0Qc/7ytNZ3BiEatAWIRZz6zCGOArh4wGgBOBMj8nZxr1/Dx8UFUVJTZbEHd1a0Ad/36dYSFhVk9HhYWhhs3bticGeI4UqkUUVFR0Ov1PRpLSXrm8o0G7D1bgSBfKR5MMp+UouRaPfb/egWhATLcNzCi5UDVRSB/f8v7AROAXlFOyrFriMViSCSSHtc0uhXgDAaD1UHIpkzp9foeZYg4Dsdx8PLyajdBKXEeTqJHo1EMPefVbp0BqcyARqMYjUax+bGaIsDYavbs2mIgnKZu6opuBTjGGGbNmmV1/YLWC7cQQtozPWOzVDCRNPcdMbZdN/VGEf+qSgDUp4BqG8e/3oK6FeAyMzM7TUMNDIRYx5qnCrHUD840VbmhdYDTNQEN1/mfeyfzAa7hOv88TiRufxFiplsBbtOmTY7KByG3BNNUSJyFoVri5mKd2cr3dRX8qzwA8Ffx8/jptXyQ8+v+ehC3Gs9ubybEzZhil6Uqqqmx0Nh6Qri6K/yrXxh/km8o/77+iuMy6UGoyzQhTiYWcRBbqKN6e4kRF+YH79bdR+qa51f0b17kxicYqL4ENFJvha6gAEeIE8Wr/BGvstyPzV/uhQeGRpjvbLjGv/o2V0flCv61qdpBOfQsVEUlxF0x1tLA4B3Ev1KA6xa3DnArVqwQVhA3bQMHDuzwnM8++wwDBw6EXC5HQkICvvnmGyfllpCeY4zBYGR8a6uuAdA3d73yDuRfKcB1i1sHOAAYPHgwysvLhW3v3r1W0+7fvx/Tp0/HnDlzcPz4cUyePBmTJ0/G6dOnnZhjQqwrrqzHFycu40jx9XbHNHoD1n5/Ae/mXuBbUk3P2eQBLcstCgGuhh+fSjrk9gFOIpFApVIJW0hIiNW069atw/jx4/HCCy9g0KBBeO2113D77bfjvffec2KOCbGuulGH367Wo6Kmqd0xcaumVYPRQvUU4MehisQAMwKaGkdn96bn9gHuwoULiIiIQL9+/TBjxowOVzE6cOBAu1mFMzIycODAAUdnk5AuMXUBsdgPTtQmwDXyAa5BEoADhddQp9HzXUVkAXwiqqZ2yq0DXEpKCnJycrBz505s2LABRUVFuPvuu1FbW2sxvVqtbjcZQFhYWKdLGWo0GtTU1JhthDhCy3RI7Y9xXEv3EUOrBoaCWi8c/O0a9l64yic0zSairXNwbm9+bt1NZMKECcLPQ4cORUpKCvr27YtPP/0Uc+bMsdvnZGdnY+XKlXa7HiHWmIZqWZskQyziYDAyfjxqcwnOPygMUVIfVDfyq9hB5se/ausdnd2bnluX4NoKDAzEgAEDUFBQYPG4SqVCRUWF2b6KigqoVKoOr7tkyRJUV1cL28WLF+2WZ0JaaxnJYDnCCSU4g1FoZBD5BKH0ekPLEC6pL/+qsVyTIS1uqgBXV1eHwsJChIeHWzyempqK3Nxcs327du1Campqh9eVyWQICAgw2whxhJaxqJaZGhqMmjrAoAc4EaR+vQAAjdrmefykVEXtKrcOcIsWLcLu3btRXFyM/fv346GHHoJYLMb06dMB8DOXLFmyREj/7LPPYufOnVizZg1+/fVXrFixAnl5eViwYIGrboEQM0ZhNhHLIS4q2Af9Q/0g1TY3IMgD8NMFvqpa29Q816KpBEdV1E659TO4S5cuYfr06bh27RqUSiVGjRqFgwcPQqnkh62UlpaaTWc8cuRIbNmyBa+++ipefvllxMXFYceOHRgyZIirboEQMykxQRgRHWR16dOMwc2PU8p/4V+9e0F9pU2XEtMzOA2V4Drj1gFu27ZtHR7/6aef2u2bOnUqpk6d6qAcEdIz/IicLiRsfv5mkAWa7TYaGURURe0yt66iEnLLag5weplC2DUsKpAv+ZmqqHoNYNA5P283EbcuwRHiac6V16Cosh79lL4YqGrfmPWfo5dw6UYjpovVCAWgk/IBTiLiMCa+eS44TgaIJXwjhKYW8Alqdx3CoxIcIU50pVaDfHUtrtZaXr+EoXnB46YqAIDGiw9wXpJWX1WOa9WSSg0NHaEAR4gTsU5aUcUiQGLU8GsxANBK+EBmMDJUN+qgNzQPsBdaUuk5XEcowBHiRJ32gxOJINdX8+lk/tAx/imSVm/Exr1FuF6v5RMKoxkaLF+IAKAAR4hTMZiGalkpwXEc5Poavr+cdy9EBnnjj/fEwkfKT2OuE0YzmAIcleA6QgGOECcyTeFmabA9wFdR5foaPgx69wLHcZCIRfCR8SU5g6E5wHn58K/0DK5DFOAIcSJhuiSrz+D4KqrRyJfgTEyLQuuNbZ/BUYDrCHUTIcSJOpouCQCCfKXgvBog8xIDPsEovFqHCxW1UFfzjQ4GqqJ2CwU4Qpxo3MBQpA1QCiWytpKjAoESA2CQAb4huFKuwbnylllD2s0oQiW4DlEVlRAnkohFkHuJIRFb+eo1VfEdeEUSQB7YUiVtpjdYCHDM2shWQiU4QtxJffM6qD69AJEIuuZ+b95SMeJC/dDLt3nxGVOAY0ZA1whIfVyQWfdHAY4QJzpxsQpXapowKDwAkUHtg1J+URFqiq9D3icCCQB0zSW25L69cEd0qyFZIjHg5c0HN209BTgrqIpKiBOVXm/AmbIaVDVYHiQvaayE3sjQIOGDmalK6mWpSkujGTpFAY4QJ+psTQZpI7+wTKOUD3CmKqqI42f01egNrRLT2gydoQBHiBMZOwpwBh2kGn6apAYZP6mrKcDlFd/A+7sLcfC3VgtGU0tqp+gZHCFO1DIW1UKEq7sCDkboRN7QivhnapOH9YbOYMTRkhvIK74BQ+tWVaqidooCHCFOZOrGJrJUd6pVQ8RxqJeGQG8akSUWwau5awnQqpsIQFXULqAAR4gTdThdUs0liDigThYKZjTv2yYWhmq1DnBURe0MBThCnMjqdEmMAVWlfEfgkL7w9pUCAH74lV/n10fKf1UtBziqolpDAY4QJ3owKQJ6I4NM0qaO2ngD0NQhwEeOiaPuAMReYIzh1CV+6qS7YkMAoM0zOKqidoYCHCFOZHqW1s6NYv7VPxwQ86MVdAYmtLqa5oMzfwbXXILTNQJGA9/5l5ihbiKEuIPKC/xrcH9hl6nPm1jEIcRPhkHh/ugb7Ntyjpc3wDV/hakUZ5FbB7js7Gzccccd8Pf3R2hoKCZPnoz8/PwOz8nJyWlee7Jlk8vlTsoxIR07XHQdP+ZfaZl6HOCX/6sqAQBU+0bjnz//hk37itCk46ujMokIKoUc44eEY0RMq+FaHEcNDZ1w6wC3e/duzJ8/HwcPHsSuXbug0+lw7733or6+419mQEAAysvLha2kpMRJOSakY/nqGpworUK9Rt+y81oBX8X0CQZ8laht0qOuSS+U4No9r2uNAlyH3PoZ3M6dO83e5+TkIDQ0FEePHsXo0aOtnsdxHFQqlaOzR0i3GS3NbFR+kn9VxgvBTG9kaNI1BzgvMRhjMBgZDIxBJmn1rI0mvuyQW5fg2qqurgYABAV1vNBtXV0d+vbti8jISEyaNAlnzpzpML1Go0FNTY3ZRogjmBoNTP3a0HCdb2DgOCA80WxQfXUjPyDf20uMyjot/vpDAT7cX2x+QSrBdeimCXBGoxELFy7EXXfdhSFDhlhNFx8fj40bN+KLL77ARx99BKPRiJEjR+LSpUtWz8nOzoZCoRC2yMhIR9wCIcKU40JH3/Jf+NdeMYB3IMQiTgh+cWH+eDKtH8YNChVmANYZ2hQBKcB16KYJcPPnz8fp06exbdu2DtOlpqZi5syZSEpKQlpaGrZv3w6lUom///3vVs9ZsmQJqqurhe3ixYv2zj4hAFo6+opE4J+7qZurpxHDhDSmUpxOb4SPVAJ/uRfEYj7AGdrWcamK2iG3fgZnsmDBAnz11VfYs2cP+vTp061zvby8MGzYMBQUFFhNI5PJIJPJeppNQjplaD1Uq/I8v3CzzA8IjhXSeIk5NOnMS2umEpzByMAYa1mVi0pwHXLrEhxjDAsWLMDnn3+OH374ATExMd2+hsFgwKlTpxAeHu6AHBLSPcIzOI5raVxQDTUbfa/0l0GlkGPXWTX2nL+KBq0eklbHaTxq17l1CW7+/PnYsmULvvjiC/j7+0OtVgMAFAoFvL29AQAzZ85E7969kZ2dDQBYtWoV7rzzTsTGxqKqqgqrV69GSUkJnnjiCZfdByEmM1L6wmhkCBA1AjeK+J2qBLM0k5J6Q6M34G8/FqKyTosRMUFmq3AZjAzCgAgaj9ohtw5wGzZsAACMGTPGbP+mTZswa9YsAEBpaSlErf53u3HjBubOnQu1Wo1evXohOTkZ+/fvx2233easbBNilcK7edGYkqP8A7nASMCnfa8AU0dgP5lEGN4l4jgYGWtTgmt+BmfQAXotIJE6NP83G7cOcKwLy6H99NNPZu/feecdvPPOOw7KESF2wBigPsX/3Kb0ZlJZywe4YL+WgBUXxgczsyVVJVJ+7KpBx5fiJB13obrVuHWAI8STGI0Mewsq4d2oxu31lRBLvADlwHbpTl+uxvfn+GmSQv1bhhnel2DlObLUF2is4p/DWSgN3srcupGBEE9iYAxHS27g8q+H+dqJciAgad9637qzb0RgF8ZRU0ODVVSCI8RJjIxBZNQjpKEQHOdntXoaHeKDEH8ZvESc+ewh4EuBHIeWbiIABbgOUIAjxEmMRqBXYzHERi04eQAQ2NdiOplEjMfubH/so4MluFqrwcO390FUcKuFnqX+/KuGhhi2RVVUQpzEyBiU9ecBAJwqwfriqFaYhnfpW8/qCwByBf/aVN3jPHoaCnCEOImhqQaBTZch4poDXDdJrA3XogBnFQU4QpxEVHEGAEOdPNym1k6JpZW1AMA7kH9tqupR/jwRBThCnIExcBV837dr/gNsuoRY1EkJTtvAd/YlAgpwhDhD9UV466owtK8SY0el2XQJ03hUnaHNMzgv75buJlRNNUMBjhBnKP8FIo6Db58EKHsF2HQJqyU4gJ7DWUHdRAhxNG0DcOVX/ufwRJsvo/SXQaP3RYBpPGtrcgVQd4UCXBsU4AhxtEtHAKMeDbIQ/HJVBv+6agzprej2ZZL79kJy316WD5oaGhpv2J5PD0RVVEIcSdcIXD4KALgWnIyDRddx8pIDSlk+wfxrQ6X9r30TowBHiCMV/sive+qnRJOCn7VXIupeB98u8VXyr/VX7X/tmxhVUQlxlMtHWxaVicuAvpH/UWxjgDtacgP7CyoRr/LHvYPbLIvpE8K/auoAXRPgRYudAxTgCLEfxvhnYDWX+fUWrvLDshA9CgiMhKGer5qaRiR0F8fxnXwttqJ6yQGZP6Cp5UtxgbQyHEABjpCeMRqBmkt8MKs8b96KyXFA35F8gENL/7XW6yt0h9WRDCa+yuYAd4UCXDMKcIR0l0EHVJUCV/OBaxf4biAmIjHgrwICIgBVIuCnbDnN2GbR527qsB8cAASEA9d/A6ovA72TbfoMT0MBjpDOGA1ATRlQVQLcKOGroEZDy3EvOb/sX0g8EBTDTyFugWkZQC8bq6imkp/VEpyieUnNmss2Xd8TUYAjxBKjgS+hXTnLB7a2Yzxl/kBIHBAyAAiM4ktunRjaR4GYEF/IJLZVUU0lOH3boVom/hF8tbixim9skPnZ9DmehAIcIa1p6oCy40D5Cf5nEy9voFdffpLKXtGAd69uz+fmK5PAV2b7V67TZ3Becv45XN0VvqoaPtTmz/IUFOAIYYyvgl7O40ttpuqn1BeISOJLaX5h3Q5o9uYrkyAiUI4Qv/brOAhCBvABrvI8BThQgCO3Mr2Wr4KWHQdq1S37Fb35h/TKgV2qenbVmbJqNOkM6K/0Q6BP99cvVfrLMO2OqE4SDQSK9wLXi/jGD6lPx+k93E0xkmH9+vWIjo6GXC5HSkoKDh8+3GH6zz77DAMHDoRcLkdCQgK++eYbJ+WUuD2DDrhWCPz6DXDgPSD///jgJpLwi8AkzwJunwmEDbZrcAOAYyU3sOd8JaoadHa9rhnfEMA/DDDq+RLpLc7tS3CffPIJsrKy8P777yMlJQVr165FRkYG8vPzERoa2i79/v37MX36dGRnZ+OBBx7Ali1bMHnyZBw7dgxDhgxxwR0QpzHo+YVXmqr5V10ToG/edI1AwzWgvhJgrR7SewcCEcP44Cb1tXppe6jV6AEA/vKefe10BiMYA6SWGis4DogaCZz5HLh4GAi9jQ96tyiOdWX5eBdKSUnBHXfcgffeew8AYDQaERkZiT/+8Y946aWX2qWfNm0a6uvr8dVXXwn77rzzTiQlJeH999/v0mfW1NRAoVCguroaAQG2zd3l1hjjv/Taej4YNFXxLW+mnzW1ACcCxDL+Sy/zA6R+fMuhzJ//WerLX8egBYw6vmRk0PHvTa+cCBBLAJEXPyGjWMq/SuQtP4skLc+2GOOffzGD+XXavpo+T9/UnOfmTVvPX6MzMj/+WVXIAL7BwAnP1uo0enyw5zdwHPDMmFjLwakLfvz1Ck5eqsbYgUoM7RNoORFjwC9b+S4tMj8gNh0I6g9Iul8tdldd/Y66dQlOq9Xi6NGjWLJkibBPJBIhPT0dBw4csHjOgQMHkJWVZbYvIyMDO3bssPo5Go0GGo1GeF9T08Xl18pPApcOW/9Sme1nne+35Zwu7W9zzKDnqzCdquNLPY7EiQCRiB8RwKx0f+gOsQSQBwKygOaZbuV866LEG5AH8J1wZQFOazD48dcrKKqsR11z6U3pL7M5uAGAj1QMI2PYnX8Vv1ysAgAMjw7CoPBWX3KOAwY9yAe5+krgzA5+v2nmX04EgGv5N3Bx40k7dzxht0u5dYCrrKyEwWBAWFiY2f6wsDD8+uuvFs9Rq9UW06vVaovpASA7OxsrV67sfgZ1jUDdTTx7g2n8ojyweVO0bEBLKU9TC2hr+W4T2rqWV5GYL52JTZu0efPi96O5hGfQ86/6puZXDf/KGB/ULPXr4jjza4ok5tcXe/ElTHmAeb69fNzqC9uoM6C6kX/mJuI4pMQE9+h6Q/sE4mx5DaoadKis4/vmNekM7RPK/IDbM4HSA0DFGb6Eq2vkt1uIWwc4Z1myZIlZqa+mpgaRkV0Yyxc6EPBrfg5o9qVq9XN399vzWu2+6M3vRSK+mmmlx71TMNZSzWRGPlhy4uYSnaT5vfsEKlvd2S8Yw6ICIRGJ4CeTwFvas4YLbym/KPTVOg30BgbGgEBfK79HiRTol8Zv2gb+PyuDprl03/yfi3s/oeoxtw5wISEhEIvFqKioMNtfUVEBlUpl8RyVStWt9AAgk8kgk3XQt8ia1qUd0j0cx38BPei5kCVBvva/P4lYhHCFd/dOkvrckl1G3LqbiFQqRXJyMnJzc4V9RqMRubm5SE1NtXhOamqqWXoA2LVrl9X0hBDP5dYlOADIyspCZmYmhg8fjhEjRmDt2rWor6/H7NmzAQAzZ85E7969kZ2dDQB49tlnkZaWhjVr1uD+++/Htm3bkJeXh3/84x+uvA1CiAu4fYCbNm0arl69imXLlkGtViMpKQk7d+4UGhJKS0shajW/1siRI7Flyxa8+uqrePnllxEXF4cdO3ZQHzhCbkFu3w/OFTy+HxwhN7mufkfd+hkcIYT0BAU4QojHogBHCPFYbt/I4Aqmx5JdHrJFCHEq03ezsyYECnAW1NbWAkDXRjMQQlymtrYWCoX1zvbUimqB0WhEWVkZ/P39wd0kw4VMw8suXrzoMS2/nnhPAN2XPTDGUFtbi4iICLNuYm1RCc4CkUiEPn36uDobNgkICPCoLw3gmfcE0H31VEclNxNqZCCEeCwKcIQQj0UBzkPIZDIsX77ctllR3JQn3hNA9+VM1MhACPFYVIIjhHgsCnCEEI9FAY4Q4rEowBFCPBYFuJvU66+/jpEjR8LHxweBgYFdOocxhmXLliE8PBze3t5IT0/HhQsXHJvRbrp+/TpmzJiBgIAABAYGYs6cOairq+vwnDFjxoDjOLPtqaeeclKOLVu/fj2io6Mhl8uRkpKCw4cPd5j+s88+w8CBAyGXy5GQkIBvvvnGSTntnu7cV05OTrvfi1wud2JuKcDdtLRaLaZOnYqnn366y+e8+eabePfdd/H+++/j0KFD8PX1RUZGBpqamhyY0+6ZMWMGzpw5g127duGrr77Cnj17MG/evE7Pmzt3LsrLy4XtzTffdEJuLfvkk0+QlZWF5cuX49ixY0hMTERGRgauXLliMf3+/fsxffp0zJkzB8ePH8fkyZMxefJknD592sk571h37wvgRzW0/r2UlJQ4MccAGLmpbdq0iSkUik7TGY1GplKp2OrVq4V9VVVVTCaTsa1btzowh1139uxZBoAdOXJE2Pd///d/jOM4dvnyZavnpaWlsWeffdYJOeyaESNGsPnz5wvvDQYDi4iIYNnZ2RbTP/LII+z+++8325eSksKefPJJh+azu7p7X13923QkKsHdIoqKiqBWq5Geni7sUygUSElJwYEDB1yYsxYHDhxAYGAghg8fLuxLT0+HSCTCoUOHOjz3448/RkhICIYMGYIlS5agoaHB0dm1SKvV4ujRo2b/ziKRCOnp6Vb/nQ8cOGCWHgAyMjLc5vcC2HZfAFBXV4e+ffsiMjISkyZNwpkzZ5yRXQENtr9FqNVqABAW6zEJCwsTjrmaWq1GaGio2T6JRIKgoKAO8/j73/8effv2RUREBE6ePInFixcjPz8f27dvd3SW26msrITBYLD47/zrr79aPEetVrv17wWw7b7i4+OxceNGDB06FNXV1XjrrbcwcuRInDlzxmmTWVAJzo289NJL7R7Ktt2s/TG5M0ff17x585CRkYGEhATMmDEDmzdvxueff47CwkI73gXprtTUVMycORNJSUlIS0vD9u3boVQq8fe//91peaASnBt5/vnnMWvWrA7T9OvXz6Zrq1QqAEBFRQXCw8OF/RUVFUhKSrLpml3V1ftSqVTtHljr9Xpcv35dyH9XpKSkAAAKCgrQv3//bue3J0JCQiAWi1FRUWG2v6Kiwuo9qFSqbqV3BVvuqy0vLy8MGzYMBQUFjsiiRRTg3IhSqYRSqXTItWNiYqBSqZCbmysEtJqaGhw6dKhbLbG26Op9paamoqqqCkePHkVycjIA4IcffoDRaBSCVlecOHECAMwCubNIpVIkJycjNzcXkydPBsBPoJqbm4sFCxZYPCc1NRW5ublYuHChsG/Xrl1ITU11Qo67xpb7astgMODUqVO47777HJjTNlzaxEFsVlJSwo4fP85WrlzJ/Pz82PHjx9nx48dZbW2tkCY+Pp5t375deP+Xv/yFBQYGsi+++IKdPHmSTZo0icXExLDGxkZX3IJF48ePZ8OGDWOHDh1ie/fuZXFxcWz69OnC8UuXLrH4+Hh26NAhxhhjBQUFbNWqVSwvL48VFRWxL774gvXr14+NHj3aVbfAtm3bxmQyGcvJyWFnz55l8+bNY4GBgUytVjPGGHvsscfYSy+9JKTft28fk0gk7K233mLnzp1jy5cvZ15eXuzUqVOuugWLuntfK1euZN9++y0rLCxkR48eZY8++iiTy+XszJkzTsszBbibVGZmJgPQbvvxxx+FNADYpk2bhPdGo5EtXbqUhYWFMZlMxsaNG8fy8/Odn/kOXLt2jU2fPp35+fmxgIAANnv2bLOgXVRUZHafpaWlbPTo0SwoKIjJZDIWGxvLXnjhBVZdXe2iO+D99a9/ZVFRUUwqlbIRI0awgwcPCsfS0tJYZmamWfpPP/2UDRgwgEmlUjZ48GD29ddfOznHXdOd+1q4cKGQNiwsjN13333s2LFjTs0vTZdECPFY1IpKCPFYFOAIIR6LAhwhxGNRgCOEeCwKcIQQj0UBjhDisSjAEUI8FgU4csvLyckxmxV5xYoVZuNzZ82aJQxPIjcXCnDEaWbNmmV1OvH58+eD4zizQfmOCCzR0dFYu3at2b5p06bh/PnzVs9Zt24dcnJyhPdjxowxGzdK3BcFOOJUkZGR2LZtGxobG4V9TU1N2LJlC6KiolySJ29v73bz0LWmUCi6vO4FcS8U4IhT3X777YiMjDSbjHL79u2IiorCsGHDenRtSyWryZMnC6XCMWPGoKSkBM8995wwDx3QvoraVuuS5KxZs7B7926sW7dOuEZRURFiY2Px1ltvmZ134sQJcBzn1OmBiDkKcMTpHn/8cWzatEl4v3HjRsyePdvhn7t9+3b06dMHq1atEhZB6a5169YhNTXVbJGbqKiodvcEAJs2bcLo0aMRGxtrr1sg3UQBjjjdH/7wB+zduxclJSUoKSnBvn378Ic//MHhnxsUFASxWAx/f3+oVCqbJpRUKBSQSqXw8fERriEWizFr1izk5+cLy+jpdDps2bIFjz/+uL1vg3QDTXhJnE6pVOL+++9HTk4OGGO4//77ERIS4ups9UhERATuv/9+bNy4ESNGjMCXX34JjUaDqVOnujprtzQqwRGXePzxx5GTk4MPP/zQbqUckUiEtrN/6XQ6u1y7K5544gmhAWXTpk2YNm0afHx8nPb5pD0KcMQlxo8fD61WC51Oh4yMDLtcU6lUmj1XMxgM7RZPlkqlMBgMPfoca9e477774Ovriw0bNmDnzp1UPXUDVEUlLiEWi3Hu3DnhZ2uqq6uFNRZMgoODERkZ2S7tPffcg6ysLHz99dfo378/3n77bVRVVZmliY6Oxp49e/Doo49CJpPZVDWOjo7GoUOHUFxcDD8/PwQFBUEkEgnP4pYsWYK4uDi3WlPhVkUlOOIyAQEBCAgI6DDNTz/9hGHDhpltK1eutJj28ccfR2ZmJmbOnIm0tDT069cPY8eONUuzatUqFBcXo3///jYv8LNo0SKIxWLcdtttUCqVKC0tFY7NmTMHWq3WKa3CpHM0ZTkhdvTzzz9j3LhxuHjxYrtFkonzUYAjxA40Gg2uXr2KzMxMqFQqfPzxx67OEgFVUQmxi61bt6Jv376oqqrCm2++6erskGZUgiOEeCwqwRFCPBYFOEKIx6IARwjxWBTgCCEeiwIcIcRjUYAjhHgsCnCEEI9FAY4Q4rEowBFCPNb/BxZMXID8AaxOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:39:54.426106Z", + "iopub.status.busy": "2024-03-23T03:39:54.425239Z", + "iopub.status.idle": "2024-03-23T03:39:54.654904Z", + "shell.execute_reply": "2024-03-23T03:39:54.654132Z" + }, + "papermill": { + "duration": 0.251927, + "end_time": "2024-03-23T03:39:54.656807", + "exception": false, + "start_time": "2024-03-23T03:39:54.404880", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2u0lEQVR4nO3deVwTV7sH8B9hCWBYRFYtGHe0LlRExLeIFtyoFsStlrpSbW3pIqAV+yp6fStVoVqrr97aqrV1qxapl6o1VVGuRUGsVhRREdQqERE1bIaQnPuHN/MSCRAUCMk838/HT5kzZ5JnSHh6Zs6Zc0wYYwyEEGLkBPoOgBBCWgIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvGFyy27BhA8RiMSwtLeHr64uMjIw6627evBn+/v5o27Yt2rZti6CgoHrrE0KMl0Eluz179iAqKgpxcXE4d+4c+vXrh5EjR6KoqEhr/dTUVEyZMgXHjx9Heno63N3dMWLECNy5c6eFIyeE6JuJIU0E4OvrCx8fH6xfvx4AoFKp4O7ujg8//BALFy5s8HilUom2bdti/fr1mDZtmk7vqVKpcPfuXdjY2MDExOSF4ieENC3GGEpLS9G+fXsIBPW33cxaKKYXVlVVhaysLMTGxnJlAoEAQUFBSE9P1+k1KioqoFAo4ODgUGcduVwOuVzObd+5cwe9evV6/sAJIc3u9u3beOmll+qtYzDJrri4GEqlEi4uLhrlLi4uuHLlik6v8emnn6J9+/YICgqqs058fDyWLVtWq/zbb7+FtbV144ImhDSriooKvPPOO7CxsWmwrsEkuxf1xRdfYPfu3UhNTYWlpWWd9WJjYxEVFcVty2QyuLu7IzQ0FLa2ti0Rql4oFApIJBIMHz4c5ubm+g6HvCC+fJ4ymQzvvPOOTreYDCbZOTo6wtTUFPfu3dMov3fvHlxdXes9NiEhAV988QV+//139O3bt966QqEQQqGwVrm5ublRf2nU+HKefGHsn2djzs1gemMtLCzg7e2No0ePcmUqlQpHjx6Fn59fncetWrUKy5cvx+HDhzFgwICWCJUQ0goZTMsOAKKiojB9+nQMGDAAAwcOxNq1a1FeXo6ZM2cCAKZNm4YOHTogPj4eALBy5UosWbIEO3fuhFgshlQqBQCIRCKIRCK9nQchpOUZVLKbPHky7t+/jyVLlkAqlcLLywuHDx/mOi1u3bql0f28ceNGVFVVYcKECRqvExcXh6VLl7Zk6IQQPTOoZAcAkZGRiIyM1LovNTVVY7ugoKD5AyKEGASDuWdHCCEvwuBadoSQ2ioqKjTGm5ZVyvHHxTy0dTwLkZXm6AJPT09ejhmlZEeIEbhy5Qq8vb1rla/SUjcrKwv9+/dv/qBaGUp2hBgBT09PZGVlcdu5hY8QtfcivpzYBz3c7GvV5SNKdoQYAWtra43WmuDmAwjTKtGzdz94dWynx8haD+qgIFi2bBksLCwQGhoKCwsLrc8GE2LoqGXHc9qeKVy6dCmWLl0KA5r9i5fyi8tRLq/Wui/vfjn3XzOzuv/M2wjN0MmxTbPE19pQsuOxhh6eNjExoYTXSuUXl2NYQmqD9aL3XWywzvGYobxIeJTseErXS9Vly5YhLi6umaMhjaVu0a2d7IWuzrUffSyvlCMlNR1jhvqhjVXtiS0A4HpRGT7Zc77O1qGxoWTHU7o+Lrd06VJKdq1YV2cRenewq1WuUCggdQL6d2xr1LOeNAYlOwIA2LRpE4RCIeRyOd577z19h0NIk6PeWAKFQoFZs2ahbdu2mDVrFhQKhb5DIqTJUcuOYNGiRRgTMo57vCjll/36Dok0QK58AoHlHeTLciGwrH3Prrq6Gner7yKnJKfO3th8WRkElncgVz4BUPtS2NgY1Opi+iCTyWBnZ4fHjx8b1bTsjVkpjb4irc//5GRgUUZEk7zWioHfYWzPgU3yWi2tMX+f1LLjqWHDhuH48eM61SOtT/s2HVGe/yG+muyFLlp6Y6urq3Hqf0/hH6/+o86WXV5RGT7ecx7th3Vs7nBbBUp2PHX48GGta21oq0daH6GpJVRPOqCTbQ/0aqe9NzbfLB89HXrW2RurevIYqif3ITStewEqY0IdFDxlYWGB+fPn11tn/vz5sLCwaKGICGlelOx4bNWqVXUmvPnz52PVKm0TBBFimCjZ8dyqVasgl8sRs3g5bPqPQczi5ZDL5ZToiNGhe3YEFhYWCI+Yi71VryA8YhBduhKjRC07QggvULIjhPACJTtCCC/QPTueqG+iR4AmeyTGj5IdD+g60SNAkz0S40XJjgcamugRoMkeifGjZMcjdU30CNBkj8T4UbLjgYamAwJoSiBi/CjZ8cDd8pto0+lrLMpouO6/D/+73v1tOgF3y73gDZcmio6QlkHJjgcamg4IoCmBiPGjZMcDKpU5VE86oLzUFSpb7ZeflZVy3H3YHpWlrnV2UCiflPFqSiBiXCjZ8UBeURkAYGFSQ8NKzPDD9cwGX6+NkL42xPAY3Ld2w4YNWL16NaRSKfr164evv/4aAwfWPaX03r17sXjxYhQUFKBbt25YuXIlgoODWzBi/RvxsisAoIuzCFbmplrr5BY+RvS+i0ic0Ac93OrufKBBxa1DpUIJAMi+81jr/vJKOc7eB1xvPqx3KBGfGFSy27NnD6KiorBp0yb4+vpi7dq1GDlyJHJzc+Hs7Fyr/h9//IEpU6YgPj4eY8aMwc6dOxEaGopz586hd+/eejgD/XBoY4E3B3rUW6e6+unYuS5ObeocnkJaD91a69RSr8mgFtzx9fWFj48P1q9fDwBQqVRwd3fHhx9+iIULF9aqP3nyZJSXlyMlJYUrGzRoELy8vLBp0yat7yGXyyGXy7ltmUwGd3d3FBcXG9WCOxUVFcjNzeW2rxY+xvz9l7F6XC90r9Gy69GjB6ytrfURIqlHSXkVfs8pQmenNlpb61elj7Fgfw5WjeuJ7q71tdRNIW5nuC11mUwGR0dH41pwp6qqCllZWYiNjeXKBAIBgoKCkJ6ervWY9PR0REVFaZSNHDkSycnJdb5PfHw8li1bVqv8yJEjRvVHn5eXh+jo6FrlU7/X3E5MTESXLl1aKCrSGCIARUXa9z1t+Jmh6PpFCKX1v87lJo6rJVVUVOhc12CSXXFxMZRKJVxcNMd3ubi44MqVK1qPkUqlWutLpXV/+rGxsRoJUt2yGzFihNG17F599VVuu6xSjt/SMjHS3weiGvd4qGVnmC7cKgEunsWgQYPQz8NB3+E0G5lMpnNdg0l2LUUoFGpddcvc3NyoHqOys7PT6NhRKBQofVQC/8GDjOo8+Uo9VtLMzMyoP8/GnJvBzGfn6OgIU1NT3Lt3T6P83r17cHV11XqMq6tro+oTQoyXwSQ7CwsLeHt74+jRo1yZSqXC0aNH4efnp/UYPz8/jfoAIJFI6qxPCDFeBnUZGxUVhenTp2PAgAEYOHAg1q5di/LycsycORMAMG3aNHTo0AHx8fEAgI8//hgBAQFITEzE66+/jt27d+Ps2bP45ptv9HkahBA9MKhkN3nyZNy/fx9LliyBVCqFl5cXDh8+zHVC3Lp1CwLBfxqrgwcPxs6dO/HPf/4TixYtQrdu3ZCcnMyrMXaEkKcMKtkBQGRkJCIjI7XuS01NrVU2ceJETJw4sZmjIoS0dgZzz44QQl4EJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjxMiUlZVh3uypuPvdB5g3eyrKysr0HVKrYKbvAAghTWfgwIHIzMzktlMlN2FjYwMfHx9kZGToMTL9o5YdIUbi2URXU2ZmJgYOHNjCEbUulOwIMQJFRUV1Jjq1zMxMnDx5EhUVFS0UVetCyY4QIzBx4kSd6gUEBODKlSvNHE3rRMmOECNQUFCgUz03Nzd4eno2bzCtFCU7QoyArpemCoUC1tbWzRxN60TJjhAj0L59+yatZ4wMJtmVlJQgPDwctra2sLe3R0RERL3jh0pKSvDhhx+iR48esLKygoeHBz766CM8fvy4BaMmpGVUVVU1aT1jZDDJLjw8HJcuXYJEIkFKSgpOnjyJOXPm1Fn/7t27uHv3LhISEpCdnY1t27bh8OHDiIiIaMGoCWkZMpmsSesZIxPGGGvMATdu3EDnzp2bKx6tcnJy0KtXL2RmZmLAgAEAgMOHDyM4OBh///23zk3zvXv34u2330Z5eTnMzLSPp5bL5ZDL5dy2TCaDu7s7iouLYWtr++In00opFApIJBIMHz4c5ubm+g6HNJKzszMePXrUYD17e3sUFRU1f0AtRCaTwdHREY8fP27w77PRT1B07doVAQEBiIiIwIQJE2BpafncgeoqPT0d9vb2XKIDgKCgIAgEApw5cwbjxo3T6XXUv5C6Eh0AxMfHY9myZbXKjxw5wosbuxKJRN8hkOeg6yNhZWVlOHjwYDNH03IaM2aw0cnu3Llz2Lp1K6KiohAZGYnJkycjIiKiWUdnS6VSODs7a5SZmZnBwcEBUqlUp9coLi7G8uXL6730BYDY2FhERUVx2+qW3YgRI6hlR1qt6upqjW1bW1soFAqYm5trXLpWV1cjODi4pcNrNo25LG90svPy8sJXX32FxMREHDhwANu2bcOrr76K7t27Y9asWZg6dSqcnJx0eq2FCxdi5cqV9dbJyclpbIi1yGQyvP766+jVqxeWLl1ab12hUAihUFir3NzcnBdJgC/naezUSaCysrLWPmP6fBtzLs/dQWFmZoawsDDs3bsXK1euxPXr1xETEwN3d3dMmzYNhYWFDb5GdHQ0cnJy6v3XuXNnuLq61rrPUF1djZKSEri6utb7HqWlpRg1ahRsbGywf/9+o/qgCSG6e+5ZT86ePYstW7Zg9+7daNOmDWJiYhAREYG///4by5YtQ0hISIOzLDg5OenUCvTz88OjR4+QlZUFb29vAMCxY8egUqng6+tb53EymQwjR46EUCjEgQMHWuT+IiH6MGbMGKSkpOhUj68a3bL78ssv0adPHwwePBh3797F9u3bcfPmTfzrX/9Cp06d4O/vj23btuHcuXNNFmTPnj0xatQozJ49GxkZGTh16hQiIyPx5ptvcj2xd+7cgaenJ5dgZTIZRowYgfLycnz33XeQyWSQSqWQSqVQKpVNFhshxDA0umW3ceNGzJo1CzNmzICbm5vWOs7Ozvjuu+9eOLiaduzYgcjISAQGBkIgEGD8+PFYt24dt1+hUCA3N5frnTl37hzOnDkD4GkPck35+fkQi8VNGh8h+vTkyZMmrWeMGp3sJBIJPDw8IBBoNgoZY7h9+zY8PDxgYWGB6dOnN1mQAODg4ICdO3fWuV8sFqPmkMGhQ4eikUMICTFYVlZWTVrPGDX6MrZLly4oLi6uVV5SUoJOnTo1SVCEkMahZNewRie7ulpLZWVl1AFAiJ5oG2LyIvWMkc6XseqBtiYmJliyZInG0wRKpRJnzpyBl5dXkwdICGnY/fv3m7SeMdI52f35558AnrbsLl68CAsLC26fhYUF+vXrh5iYmKaPkBDSoJpJzMTEROMKrOY2JTsdHD9+HAAwc+ZMfPXVV0b96BQhhqakpIT7+dlbTTW3a9bjm0b3xm7durU54iCEvABtjzi+SD1jpFOyCwsLw7Zt22Bra4uwsLB66yYlJTVJYIQQ3bm4uOg0KYaLi0sLRNM66ZTs7OzsYGJiwv1MCGld1H+fTVXPGOmU7GpeutJlLCGtz7OD/F+0njHi75kTYkTc3d2btJ4x0qll98orr+jc/G3KCQAIIbp5/fXX8csvv+hUj690SnahoaHNHAYh5EXoOtX6wYMHMXv27GaOpnXSKdnFxcU1dxyEkBegXoPC3NwcCoWi1n51ua5rVRij5568kxDSeohEIgBPpzoLDg6GhYUF8vLy0KVLF1RVVXEtP3U9PtIp2Tk4OODq1atwdHRE27Zt671/x+cR2oToS2hoKJKTk2FmZoaLFy/i9u3bAICLFy/Cw8MDZmZmqK6u5vUtKZ2S3Zo1a2BjY8P9zOexOoS0Rh07dgTwdG0WdaJTu3XrVq16fNToRbL5RiaTwc7OTqdFeA2ZQqHAwYMHERwcTIsSGSClUgk3N7d6H/R3dnbG3bt3YWpq2oKRNa/G/H02epydqamp1hXFHzx4YFS/REIMjVwuB/D0tpObmxvatGkDNzc3ODg4AOD3lOzAc3RQ1NUQlMvlGtM+EUJaTmpqKmQyGUQikcZ98/LycgBPOyZkMhlSU1MRGBiorzD1Sudkp17cxsTEBN9++61Gr45SqcTJkyfh6enZ9BESQhqUmpoK4OkQFAsLC4SFhcHKygqVlZVISkrihpxQstPBmjVrADxt2W3atEnjktXCwgJisRibNm1q+ggJIQ2qqqoC8HQ8XWlpKUxMTLh7sN9//z1EIhEUCgVXj490Tnb5+fkAgGHDhiEpKQlt27ZttqAIIY2Tk5MD4GknhJmZmcbayGZmZnBycsLdu3e5enzU6A6K48ePU6IjpJVRdz7cuXMHISEhOH36NCorK3H69GmEhITg7t27GvX4qNEdFLNmzap3/5YtW547GELI8+nevTskEgkA4Pfff0dKSgq3r+byid27d2/x2FqLRrfsHj58qPGvqKgIx44dQ1JSEh49etQMIRJCGrJ69WoAT4eGPXtfTi6Xc/fY1fX4qNEtu/3799cqU6lUmDt3Lrp06dIkQRFCGsfKygo+Pj7IzMwEAPTv35/rjVVPu+bj48PrRbKbZCIAgUCAqKgoDB06FAsWLGiKlySENIJSqcT9+/e5pwmenVfSzs4OxcXFUCqVvB3832QzFefl5aG6urqpXo4Q0ghpaWkoKCiATCbDqFGj0Lt3b7Rr1w69e/fGqFGjIJPJkJ+fj7S0NH2HqjeNbtlFRUVpbDPGUFhYiF9//RXTp09vssAIIbq7c+cOAEAsFuPIkSNQqVQAnj7GefnyZYjFYuTn53P1+KjRye7PP//U2BYIBHByckJiYmKDPbWEkOahngBAPR62JpVKxZXXN1GAsWt0sjt+/HhzxEEIeQE1x76amJhoPMNec5vPY2RpdTFCjEB6ejr387PzTdbcrlmPbwwm2ZWUlCA8PBy2trawt7dHRESEzvPpM8YwevRomJiYIDk5uXkDJUQP/vrrL+7nZ2cfEgqFWuvxjcGsQREeHo7CwkJIJBIoFArMnDkTc+bMwc6dOxs8du3atTS7MjFqpaWl3M+BgYEYMWIErl27hm7duuHIkSP49ddfa9XjG4NIdjk5OTh8+DAyMzMxYMAAAMDXX3+N4OBgJCQkoH379nUee/78eSQmJuLs2bNwc3Nr8L3kcjk3CSLwdCZU4OlMvtpWbTIW6nMz5nM0Zk5OTgCetuqys7O55AY87aG1sLBAVVUVnJycjOozbsy5NFmy+/vvv/Ff//Vf+Oabb5rqJTnp6emwt7fnEh0ABAUFQSAQ4MyZMxg3bpzW4yoqKvDWW29hw4YNcHV11em94uPjsWzZslrlR44cgbW19fOdgAFRP19JDIu646GqqgoPHz7EG2+8AVdXV0ilUqSmpnKPkLVt21bnNWYNQUVFhc51myzZPXjwAN99912zJDupVApnZ2eNMjMzMzg4OEAqldZ53Lx58zB48GCEhITo/F6xsbEaYwllMhnc3d0xYsQIo1+DQiKRYPjw4bQGhQGytLREUlISgKff2QMHDmitN2fOHLz22mstGVqzUl956UKvl7ELFy7EypUr663zvPNvHThwAMeOHas1LrAhQqFQ44aumrm5OS+SAF/O09gEBQXBycmpwQV3goKCjOpxscZ8V/Wa7KKjozFjxox663Tu3Bmurq61Fvmprq5GSUlJnZenx44dQ15eHuzt7TXKx48fD39/f24aa0KMgampKTZt2oTx48fXWWfjxo1GlegaS6/JzsnJibuxWh8/Pz88evQIWVlZ8Pb2BvA0malUKvj6+mo9ZuHChXjnnXc0yvr06YM1a9Zg7NixLx48Ia2UpaWlxiSd6tlP+E7nZBcWFlbv/uacy65nz54YNWoUZs+ejU2bNkGhUCAyMhJvvvkm1xN7584dBAYGYvv27Rg4cCBcXV21tvo8PDzQqVOnZouVEH1QKpWIjo7G2LFj8fPPP+PEiRM4dOgQRo8ejYCAAIwfPx4xMTEICQnhbetO52RnZ2fX4P5p06a9cEB12bFjByIjIxEYGAiBQIDx48dzK54BT2+w5+bmNqp3hhBjoZ71ZNeuXTA3N0dAQADKy8sREBAAc3NzxMbGYvDgwUhLS8PQoUP1Ha5e6Jzstm7d2pxxNMjBwaHeAcRisbjONW3VGtpPiKEqLCwEAPTu3VvrfnW5uh4fGczjYoSQuqkHzGdnZ2vdry7XZWC9sdK5Zafr9E204A4hLc/f3x9isRgrVqyo9fy3SqVCfHw8OnXqBH9/f/0E2AronOy2bduGjh074pVXXqHLQUJaGVNTUyQmJmLChAkIDQ3F/PnzuaUUV69ejZSUFOzbt4+3nRNAI5Ld3LlzsWvXLuTn52PmzJl4++234eDg0JyxEUIaISwsDPv27UN0dDSGDBnClXfq1An79u1rcESFsdP5nt2GDRtQWFiIBQsW4H/+53/g7u6OSZMm4bfffqOWHiGtRFhYGK5fvw6JRIKoqChIJBJcu3aN94kOaGQHhVAoxJQpUyCRSHD58mW8/PLLeP/99yEWi3WeW44Q0rxMTU0REBCAIUOGICAggNeXrjU9d2+sQCDgpntWKpVNGRMhhDS5RiU7uVyOXbt2Yfjw4ejevTsuXryI9evX49atWxCJRM0VIyGkEZRKJU6cOIGTJ0/ixIkT1Bj5fzp3ULz//vvYvXs33N3dMWvWLOzatQuOjo7NGRshpJGSkpIQHR2NgoICAMCXX34JsViMxMRE3t+30znZbdq0CR4eHujcuTNOnDiBEydOaK2nnlOLENKykpKSMGHCBIwZMwY//PAD/v77b7z00ktYtWoVJkyYwPseWZ2T3bRp02gdB0JaKfVEAGPGjEFycjKUSiUePHgAX19fJCcnIzQ0lCYC0LXitm3bmjEMQsiLqDkRgEAg0LhPJxAIaCIA0LOxhBiFmhMBaOugoIkADGR1MUJI/dQP+K9fvx7//d//XauDYs6cORr1+IhadoQYAX9/fzg5OSE2Nha9e/dGWloadu3ahbS0NPTu3RuLFi2Cs7MzrycCoGRHiJGo2YGofoSTHuX8D0p2hBiBtLQ0FBUVIT4+HtnZ2RgyZAimTJmCIUOG4NKlS1ixYgWKioqQlpam71D1hpIdIUZA3fEQGRmpdSKAyMhIjXp8RMmOECNQc6ZibRMB0EzFlOwIMQo1ZypWqVQa+2im4qco2RFiBNQzFaekpCA0NBSnT5/mZioODQ1FSkoKEhISePv0BEDj7AgxGjRTcf0o2RFiRMLCwhASEoLjx49zi2QPGzaM1y06NUp2hBgZdQeFepFsSnRP0T07QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvGEyyKykpQXh4OGxtbWFvb4+IiAiUlZU1eFx6ejpee+01tGnTBra2thgyZAgqKytbIGJCSGtiMMkuPDwcly5dgkQiQUpKCk6ePMlNNV2X9PR0jBo1CiNGjEBGRgYyMzMRGRkJgcBgTpsQ0kQM4gmKnJwcHD58GJmZmRgwYAAA4Ouvv0ZwcDASEhLQvn17rcfNmzcPH330ERYuXMiV9ejRo0ViJoS0LgaR7NLT02Fvb88lOgAICgqCQCDAmTNnMG7cuFrHFBUV4cyZMwgPD8fgwYORl5cHT09PfP7553j11VfrfC+5XA65XM5ty2QyAIBCoYBCoWjCs2pd1OdmzOfIF0qlEqmpqTh58iSEQiGGDh1qtI+MNeb7ahDJTiqVwtnZWaPMzMwMDg4OkEqlWo+5ceMGAGDp0qVISEiAl5cXtm/fjsDAQGRnZ6Nbt25aj4uPj8eyZctqlR85cgTW1tYveCatn0Qi0XcI5AWkp6dj69atKCoqAvB0dTFnZ2fMnDkTfn5+eo6u6VVUVOhcV6/JbuHChVi5cmW9dXJycp7rtdUTGL777ruYOXMmAOCVV17B0aNHsWXLFsTHx2s9LjY2FlFRUdy2TCaDu7s7RowYAVtb2+eKxRAoFApIJBIMHz4c5ubm+g6HPIf9+/dj1apVCA4ORkxMDKRSKVxdXZGQkIBVq1Zh9+7dWq+CDJn6yksXek120dHRmDFjRr11OnfuDFdXV+7/VGrV1dUoKSmBq6ur1uPU00/36tVLo7xnz564detWne8nFAohFAprlZubm/MiCfDlPI2NUqnEp59+ijFjxiA5ORlKpRIHDx7EP/7xDwwZMgShoaFYuHAhxo8fb1SXtI35ruo12Tk5OcHJyanBen5+fnj06BGysrLg7e0NADh27BhUKhV8fX21HiMWi9G+fXvk5uZqlF+9ehWjR49+8eAJaUXS0tJQUFCAXbt2QSAQQKlUcvsEAgFiY2MxePBgpKWlYejQofoLVI8MYgxGz549MWrUKMyePRsZGRk4deoUIiMj8eabb3I9sXfu3IGnpycyMjIAPF1Dc/78+Vi3bh327duH69evY/Hixbhy5QoiIiL0eTqENDn1qmG9e/fWul9dzufVxQyigwIAduzYgcjISAQGBkIgEGD8+PFYt24dt1+hUCA3N1fjhuUnn3yCJ0+eYN68eSgpKUG/fv0gkUjQpUsXfZwCIc2m5upigwYNqrWfVhcDTBgtGV4vmUwGOzs7PH782Og7KA4ePIjg4GC6Z2eAlEolunbtij59+mjcswsODoapqSlCQ0ORnZ2Na9euGdU9u8b8fRpMy44QUjf16mITJkxASEgIhg8fjmvXruHmzZuQSCT49ddfsW/fPqNKdI1FyY4QIxEWFoaYmBh8+eWXSElJ4cpNTU0RExPD+9XFDKKDghDSsKSkJKxevRoWFhYa5RYWFli9ejWSkpL0FFnrQMmOECOgVCrx3nvv1Vtn7ty5GkNS+IaSHSFGIDU1Fffv3wcABAYGIi0tDbt27UJaWhoCAwMBPH1ePDU1VY9R6hclO0KMwLFjxwAAgwYNQlJSEp48eYLMzEw8efIESUlJ3HAUdT0+og4KQozA7du3ATwdPNy9e3cUFBQAeDoRgFgsxmuvvYbTp09z9fiIWnaEGAF3d3cAwLfffovevXtrXMb27t0bW7Zs0ajHR5TsCDECAQEB3M+MMaifFaj587P1+IYuYwkxAjUHCx87dgy//vort21lZaW1Ht9Qy44QI/DsFGg1mZiY6FTP2FGyI8QIqB/wj4+PrzWrt7OzM1asWKFRj48o2RFiBPz9/SEWi/HHH3/g2rVrkEgkiIqKgkQiwdWrV5Geno5OnTrB399f36HqDSU7QoyAeiKAlJQUjB8/HkKhED4+PhAKhRg/fjxSUlKQkJDA63t21EFBiJEICwvDvn37EBUVhSFDhnDlYrEY+/bto4kA9B0AIaRp1eyQIP9ByY4QI5GUlIQJEyagT58+GoOK+/TpgwkTJtCsJ/oOgBDy4pRKJaKjo7nVxXx9fWFlZQVfX18kJydjzJgxiImJoVlPCCGGTb262KJFiyAQaP5Zq1cXy8/PR1pamp4i1D9KdoQYAVpdrGGU7AgxAjVXF9OGVhejZEeIUVAPKl6xYgVUKpXGPpVKhfj4eBpUrO8ACCEvruag4tDQUJw+fRqVlZU4ffo0QkNDaVAxaFAxIUZDPag4OjpaY1Bxp06daFAxKNkRYlTCwsIQEhKC48eP49ChQxg9ejSGDRvG6xadGiU7QoyMqakpAgICUF5ejoCAAEp0/4/u2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBcMJtmVlJQgPDwctra2sLe3R0REBMrKyuo9RiqVYurUqXB1dUWbNm3Qv39//Pzzzy0UMSGkNTGYZBceHo5Lly5BIpEgJSUFJ0+exJw5c+o9Ztq0acjNzcWBAwdw8eJFhIWFYdKkSfjzzz9bKGpCSKvBDMDly5cZAJaZmcmVHTp0iJmYmLA7d+7UeVybNm3Y9u3bNcocHBzY5s2bdX7vx48fMwDs8ePHjQ/cgFRVVbHk5GRWVVWl71DIC5LL5SwhIYEFBwezhIQEJpfL9R1Ss2nM36dBPEGRnp4Oe3t7DBgwgCsLCgqCQCDAmTNnMG7cOK3HDR48GHv27MHrr78Oe3t7/PTTT3jy5AmGDh1a53vJ5XLI5XJuWyaTAQAUCgUUCkXTnFArpD43Yz5HPli4cCHWrl3LzXxy8OBBLFiwAJ988gm++OILPUfX9BrzfTWIZCeVSmst/GtmZgYHBwdIpdI6j/vpp58wefJktGvXDmZmZrC2tsb+/fvRtWvXOo+Jj4/HsmXLapUfOXIE1tbWz38SBkIikeg7BPKctm3bhuTk5FrlKpUKX375JW7cuIEZM2a0eFzNqaKiQue6ek12CxcuxMqVK+utk5OT89yvv3jxYjx69Ai///47HB0dkZycjEmTJnGLkGgTGxuLqKgoblsmk8Hd3R0jRoyAra3tc8fS2ikUCkgkEgwfPhzm5ub6Doc0UlVVVZ1XOGq//PILfvzxR1hYWLRQVM1PfeWlC70mu+jo6Ab/T9O5c2e4urqiqKhIo7y6uholJSVwdXXVelxeXh7Wr1+P7OxsvPzyywCAfv36IS0tDRs2bMCmTZu0HicUCiEUCmuVm5ub8yIJ8OU8jc3atWvBGOO23377bXh7eyMrKws//vgjAIAxhn//+9+YP3++vsJsco35ruo12Tk5OcHJyanBen5+fnj06BGysrLg7e0NADh27BhUKhV8fX21HqNu3j67+IipqWmtmVwJMXQ1l0msrKyEqakpDh48iA8++ACbN2+GlZUVV8+Ykl1jGMTQk549e2LUqFGYPXs2MjIycOrUKURGRuLNN99E+/btAQB37tyBp6cnMjIyAACenp7o2rUr3n33XWRkZCAvLw+JiYmQSCQIDQ3V49kQ0vRyc3MBAP3794elpaXGPktLS3h5eWnU4yODSHYAsGPHDnh6eiIwMBDBwcF49dVX8c0333D7FQoFcnNzuRadubk5Dh48CCcnJ4wdOxZ9+/bF9u3b8f333yM4OFhfp0FIs1Dfh7ty5Qqqq6s19lVXV+Pq1asa9fjIIHpjAcDBwQE7d+6sc79YLNa4ZwEA3bp1oycmCC/4+PggJSUFFRUV6NChA5YuXQpLS0t8++23WLp0KdcI8PHx0XOk+mMwyY4QUrddu3bBxsYGAFBUVIT333+/znp8ZTCXsYSQuolEogZbbT4+PhCJRC0UUetDyY4QI5GRkVFnwvPx8eE67/iKkh0hRiQjIwOlpaUYO3YsOnbsiLFjx6K0tJT3iQ6ge3aEGB2RSISff/4ZBw8eRHBwMA0S/3/UsiOE8AIlO0IIL1CyI4TwAt2za4B6oHJjZlcwRAqFAhUVFZDJZHSPxwjw5fNU/10++0CBNpTsGlBaWgoAcHd313MkhJC6lJaWws7Ort46JkyXlMhjKpUKd+/ehY2NDUxMTPQdTrNRz9t3+/Zto563jy/48nkyxlBaWor27dvXmuHoWdSya4BAIMBLL72k7zBajK2trVH/cfANHz7Phlp0atRBQQjhBUp2hBBeoGRHADydjj4uLk7rlPTE8NDnWRt1UBBCeIFadoQQXqBkRwjhBUp2hBBeoGTXxGbMmNHkq5cNHToUn3zySb11xGIx1q5d26TvS4gxoWRXD12SDDEsS5cu5ZYVbG1a2/ettcXzoijZEWJEqqqq9B1Cq0XJrg4zZszAiRMn8NVXX8HExAQmJibIy8tDREQEOnXqBCsrK/To0QNfffWV1uOXLVsGJycn2Nra4r333tP5S1heXo5p06ZBJBLBzc0NiYmJteoUFRVh7NixsLKyQqdOnbBjx45adUxMTLBx40aMHj0aVlZW6Ny5M/bt28ftLygogImJCX766Sf4+/vDysoKPj4+uHr1KjIzMzFgwACIRCKMHj0a9+/f1/G3BmzZsgUvv/wyhEIh3NzcEBkZye27desWQkJCIBKJYGtri0mTJuHevXvcfnWr64cffoBYLIadnR3efPNNbjIG4OmzyqtWrULXrl0hFArh4eGBzz//nNv/6aefonv37rC2tkbnzp2xePFiKBQKAMC2bduwbNkyXLhwgftMt23bpvO5Nafn/b6pb5t8/vnnaN++PXr06AEA+OOPP+Dl5QVLS0sMGDAAycnJMDExwfnz57ljs7OzMXr0aIhEIri4uGDq1KkoLi6uM56CgoKW+nU0D0a0evToEfPz82OzZ89mhYWFrLCwkD158oQtWbKEZWZmshs3brAff/yRWVtbsz179nDHTZ8+nYlEIjZ58mSWnZ3NUlJSmJOTE1u0aJFO7zt37lzm4eHBfv/9d/bXX3+xMWPGMBsbG/bxxx9zdUaPHs369evH0tPT2dmzZ9ngwYOZlZUVW7NmDVcHAGvXrh3bvHkzy83NZf/85z+Zqakpu3z5MmOMsfz8fAaAeXp6ssOHD7PLly+zQYMGMW9vbzZ06FD2v//7v+zcuXOsa9eu7L333tMp9n//+9/M0tKSrV27luXm5rKMjAwuJqVSyby8vNirr77Kzp49y06fPs28vb1ZQEAAd3xcXBwTiUQsLCyMXbx4kZ08eZK5urpq/O4WLFjA2rZty7Zt28auX7/O0tLS2ObNm7n9y5cvZ6dOnWL5+fnswIEDzMXFha1cuZIxxlhFRQWLjo5mL7/8MveZVlRU6HRuze1Fv29Tp05l2dnZLDs7mz1+/Jg5ODiwt99+m126dIkdPHiQde/enQFgf/75J2OMsYcPHzInJycWGxvLcnJy2Llz59jw4cPZsGHD6oynurpaH7+aJkPJrh4BAQEaSUabDz74gI0fP57bnj59OnNwcGDl5eVc2caNG5lIJGJKpbLe1yotLWUWFhbsp59+4soePHjArKysuDhyc3MZAJaRkcHVycnJYQBqJbtnk5Svry+bO3cuY+w/ye7bb7/l9u/atYsBYEePHuXK4uPjWY8ePeqNW619+/bss88+07rvyJEjzNTUlN26dYsru3Tpksa5xMXFMWtrayaTybg68+fPZ76+vowxxmQyGRMKhRrJrSGrV69m3t7e3HZcXBzr16+fzse3pOf9vrm4uDC5XM6Vbdy4kbVr145VVlZyZZs3b9ZIdsuXL2cjRozQeO3bt28zACw3N1fneAwJzXrSSBs2bMCWLVtw69YtVFZWoqqqqtYN7379+sHa2prb9vPzQ1lZGW7fvo2OHTvW+dp5eXmoqqqCr68vV+bg4MBdmgBATk4OzMzM4O3tzZV5enrC3t6+1uv5+fnV2q55GQMAffv25X52cXEBAPTp00ejrKioqM6Y1YqKinD37l0EBgZq3Z+TkwN3d3eNeQF79eoFe3t75OTkcEsAisVibrFnAHBzc+PePycnB3K5vM73AIA9e/Zg3bp1yMvLQ1lZGaqrqw161g9dvm99+vSBhYUFt52bm4u+ffvC0tKSKxs4cKDGMRcuXMDx48e1riObl5eH7t27N+2JtAJ0z64Rdu/ejZiYGERERODIkSM4f/48Zs6cadA3hWvOYquer+/ZMpVK1eDrWFlZNXk8z75/Q++Rnp6O8PBwBAcHIyUlBX/++Sc+++wzg/18dP2+tWnTptGvXVZWhrFjx+L8+fMa/65du4YhQ4Y01Sm0KpTs6mFhYQGlUsltnzp1CoMHD8b777+PV155BV27dkVeXl6t4y5cuIDKykpu+/Tp0xCJRA3OdtylSxeYm5vjzJkzXNnDhw9x9epVbtvT0xPV1dXIysriynJzc/Ho0aNar3f69Ola2z179qw3hudlY2MDsViMo0ePat3fs2dP3L59G7dv3+bKLl++jEePHqFXr146vUe3bt1gZWVV53v88ccf6NixIz777DMMGDAA3bp1w82bNzXqPPuZtibP+317Vo8ePXDx4kXI5XKuLDMzU6NO//79cenSJYjFYnTt2lXjnzp5tubf1fOgZFcPsViMM2fOoKCgAMXFxejWrRvOnj2L3377DVevXsXixYtrfYmAp93/ERERuHz5Mg4ePIi4uDhERkY2OJOqSCRCREQE5s+fj2PHjiE7OxszZszQOK5Hjx4YNWoU3n33XZw5cwZZWVl45513tLZ69u7diy1btuDq1auIi4tDRkaGRu9oU1u6dCkSExOxbt06XLt2DefOncPXX38NAAgKCkKfPn0QHh6Oc+fOISMjA9OmTUNAQAAGDBig0+tbWlri008/xYIFC7B9+3bk5eXh9OnT+O677wA8TYa3bt3C7t27kZeXh3Xr1mH//v0aryEWi5Gfn4/z58+juLhYIyHo2/N+35711ltvQaVSYc6cOcjJycFvv/2GhIQEAP9pvX/wwQcoKSnBlClTkJmZiby8PPz222+YOXMml+CejUeXFn6rpu+bhq1Zbm4uGzRoELOysmIA2JUrV9iMGTOYnZ0ds7e3Z3PnzmULFy7UuOE9ffp0FhISwpYsWcLatWvHRCIRmz17Nnvy5IlO71laWsrefvttZm1tzVxcXNiqVatq3SguLCxkr7/+OhMKhczDw4Nt376ddezYsVYHxYYNG9jw4cOZUChkYrFYoxdP3UGhvmHNGGPHjx9nANjDhw+5sq1btzI7Ozudf2ebNm1iPXr0YObm5szNzY19+OGH3L6bN2+yN954g7Vp04bZ2NiwiRMnMqlUyu3X1nmwZs0a1rFjR25bqVSyf/3rX6xjx47M3NyceXh4sBUrVnD758+fz/3eJ0+ezNasWaMR/5MnT9j48eOZvb09A8C2bt2q87k1txf5vj3r1KlTrG/fvszCwoJ5e3uznTt3cq+pdvXqVTZu3Dhmb2/PrKysmKenJ/vkk0+YSqXSGk9+fn4z/waaF03xZKRMTEywf//+Jn90jRimHTt2YObMmXj8+HGT3V81NNQbS4gR2r59Ozp37owOHTrgwoUL+PTTTzFp0iTeJjqAkl2LunXrVr034y9fvgwPD48WjKhxtA1TUDt06BD8/f1bMBpSH6lUiiVLlkAqlcLNzQ0TJ07UeNKEj+gytgVVV1fX+8iNWCyGmVnr/f/P9evX69zXoUMHXrcaSOtHyY4Qwgs09IQQwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOxIqzFjxgxuVlxzc3O4uLhg+PDh2LJlS6Oey9y2bZvWKa+aW3MstkSaDiU70qqMGjUKhYWFKCgowKFDhzBs2DB8/PHHGDNmDKqrq/UdHjFk+nwwl5Ca6nqo/ejRowwAN0NxYmIi6927N7O2tmYvvfQSmzt3ListLWWM/Wcyg5r/4uLiGGOMbd++nXl7ezORSMRcXFzYlClT2L1797j3KSkpYW+99RZzdHRklpaWrGvXrmzLli3c/lu3brGJEycyOzs71rZtW/bGG29wD8fHxcXVet/jx483y++JPB9q2ZFW77XXXkO/fv2QlJQEABAIBFi3bh0uXbqE77//HseOHcOCBQsAAIMHD8batWtha2uLwsJCFBYWIiYmBgCgUCiwfPlyXLhwAcnJySgoKMCMGTO491m8eDEuX76MQ4cOIScnBxs3boSjoyN37MiRI2FjY4O0tDScOnUKIpEIo0aNQlVVFWJiYjBp0iSuZVpYWIjBgwe37C+K1E/f2ZYQtbpadowxNnnyZNazZ0+t+/bu3cvatWvHbes6LVVmZiYDwLUKx44dy2bOnKm17g8//MB69OjBTX/EGGNyuZxZWVmx3377rcH4if5Ry44YBMYYN/Hk77//jsDAQHTo0AE2NjaYOnUqHjx4gIqKinpfIysrC2PHjoWHhwdsbGwQEBAA4OkEDQAwd+5c7N69G15eXliwYAH++OMP7tgLFy7g+vXrsLGxgUgkgkgkgoODA548eaLT7MFE/yjZEYOQk5ODTp06oaCgAGPGjEHfvn3x888/IysrCxs2bABQ/wLR5eXlGDlyJGxtbbFjxw5kZmZysxirjxs9ejRu3ryJefPmcYsHqS+By8rK4O3tXWvNhqtXr+Ktt95q5rMnTaH1TrFByP87duwYLl68iHnz5iErKwsqlQqJiYncdPU//fSTRn1taydcuXIFDx48wBdffMGtBXL27Nla7+Xk5ITp06dj+vTp8Pf3x/z585GQkID+/ftjz549cHZ2rnO1MmNbs8HYUMuOtCpyuRxSqRR37tzBuXPnsGLFCoSEhGDMmDGYNm0aunbtCoVCga+//ho3btzADz/8gE2bNmm8hlgsRllZGY4ePYri4mJUVFTAw8MDFhYW3HEHDhzA8uXLNY5bsmQJfvnlF1y/fh2XLl1CSkoKt0BReHg4HB0dERISgrS0NOTn5yM1NRUfffQR/v77b+59//rrL+Tm5qK4uBgKhaJlfmlEN/q+aUiI2vTp07lhG2ZmZszJyYkFBQWxLVu2aCww/uWXXzI3NzdmZWXFRo4cybZv315r7Yz33nuPtWvXTmPoyc6dO5lYLGZCoZD5+fmxAwcO1Fo4umfPnszKyoo5ODiwkJAQduPGDe41CwsL2bRp05ijoyMTCoWsc+fObPbs2ezx48eMMcaKiorY8OHDmUgkoqEnrRDNZ0cI4QW6jCWE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPDC/wEUSDK9s4ie3wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:39:54.696305Z", + "iopub.status.busy": "2024-03-23T03:39:54.695984Z", + "iopub.status.idle": "2024-03-23T03:39:54.983368Z", + "shell.execute_reply": "2024-03-23T03:39:54.982462Z" + }, + "papermill": { + "duration": 0.309516, + "end_time": "2024-03-23T03:39:54.985337", + "exception": false, + "start_time": "2024-03-23T03:39:54.675821", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAEmCAYAAAD8/yLTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+jklEQVR4nO3deVgT1/oH8G8IJGENKEpAQVARN9xAKF5btKYFi1Xb3lu0i2ite29V6q6I1rZYt4tbtfbWrVUBu2h/tfWqtLVVEBV3QaqIBcWggoSdQHJ+f1BGwprEhAB5P88zD+TMmZkzjHk9c87MOTzGGAMhhJgoM2MXgBBCjImCICHEpFEQJISYNAqChBCTRkGQEGLSKAgSQkwaBUFCiEmjIEgIMWnmxi5Aa6VSqZCdnQ1bW1vweDxjF4cQUgNjDIWFhXBxcYGZWeN1PQqCOsrOzoarq6uxi0EIaURWVhY6d+7caB4KgjqytbUFUPVHtrOzM3JpCCE1FRQUwNXVlfueNoaCoI6qb4Ht7OwoCBLSQmnSVEUdI4QQk0ZBkBBi0igIEkJMGgVBQohJoyBICDFpFAQJISaNgmAroFIxhMddwrbf0o1dFELaHHpOsBU4czsX3124BwCYMaybkUtDSNtCNcFWoEShNHYRCGmzKAgSQkwaBUFCiEmjIEgIMWkUBAkhJo2CICHEpFEQJISYNAqChBCTRkGQEGLSKAgSQkwaBUFCiEmjIEgIMWkUBAkhJo2CICHEpFEQbAWYsQtASBtGQZAQYtIoCBJCTBoFQUKISaMgSAgxaS0iCG7duhXu7u4QiUTw9/fH2bNnG81/8OBB9OzZEyKRCN7e3vjpp5/U1jPGsHz5cjg7O8PS0hJSqRQ3b96sd1/l5eUYMGAAeDweLl26pK9TIoS0EkYPgrGxsQgPD0dkZCQuXLiA/v37IygoCA8ePKg3f0JCAsaPH4/Jkyfj4sWLGDt2LMaOHYtr165xedasWYNNmzZh+/btSEpKgrW1NYKCglBWVlZnfwsWLICLi4vBzo8Q0sIxI/Pz82OzZs3iPiuVSubi4sKioqLqzf/666+zkJAQtTR/f382bdo0xhhjKpWKSSQStnbtWm59fn4+EwqF7MCBA2rb/fTTT6xnz57s+vXrDAC7ePGixuWWy+UMAJPL5Rpvo6tj12Wsy8IfWZeFPxr8WIS0Bdp8P41aE1QoFEhOToZUKuXSzMzMIJVKkZiYWO82iYmJavkBICgoiMufkZEBmUymlkcsFsPf319tnzk5OZgyZQq++uorWFlZNVnW8vJyFBQUqC2EkNbPqEHw0aNHUCqVcHJyUkt3cnKCTCardxuZTNZo/uqfjeVhjGHixImYPn06fH19NSprVFQUxGIxt7i6umq0HSGkZTN6m6AxbN68GYWFhVi8eLHG2yxevBhyuZxbsrKyDFhCQkhzMWoQdHR0BJ/PR05Ojlp6Tk4OJBJJvdtIJJJG81f/bCzPL7/8gsTERAiFQpibm6N79+4AAF9fX4SFhdV7XKFQCDs7O7WFENL6GTUICgQC+Pj4ID4+nktTqVSIj49HQEBAvdsEBASo5QeA48ePc/k9PDwgkUjU8hQUFCApKYnLs2nTJly+fBmXLl3CpUuXuEdsYmNj8fHHH+v1HAkhLZu5sQsQHh6OsLAw+Pr6ws/PD9HR0SguLsakSZMAABMmTECnTp0QFRUFAJg9ezYCAwOxfv16hISEICYmBufPn8eOHTsAADweD3PmzMFHH30ET09PeHh4ICIiAi4uLhg7diwAwM3NTa0MNjY2AIBu3bqhc+fOzXTmhJCWwOhBMDQ0FA8fPsTy5cshk8kwYMAAHD16lOvYyMzMhJnZkwrrkCFDsH//fixbtgxLliyBp6cnDh06hL59+3J5FixYgOLiYkydOhX5+fkYOnQojh49CpFI1OznRwhp2XiMMRqpSQcFBQUQi8WQy+UGbx88dl2GqV8lAwDurA4x6LEIaQu0+X6aZO8wIYRUoyBICDFpFAQJISaNgiAhxKRRECSEmDQKgoQQk0ZBkBBi0igIEkJMGgVBQohJoyBICDFpFAQJISaNgiAhxKRRECSEmDQKgoQQk0ZBsBWgsc4IMRwKgoQQk0ZBkBBi0igIEkJMGgVBQohJoyBICDFpFAQJISaNgiAhxKRRECSEmDQKgoQQk0ZBkBBi0syfZuOioiKoVCq1tKZmeyeEkJZE65pgRkYGQkJCYG1tDbFYDAcHBzg4OMDe3h4ODg6GKCMhhBiM1jXBt956C4wx7Ny5E05OTuDxeIYoFyGENAutg+Dly5eRnJwMLy8vQ5SHEEKalda3w4MHD0ZWVpYhykIawGgsLUIMRuua4H//+19Mnz4d9+7dQ9++fWFhYaG2vl+/fnorHCGEGJrWQfDhw4dIT0/HpEmTuDQejwfGGHg8HpRKpV4LSAghhqR1EHznnXcwcOBAHDhwgDpGCCGtntZB8K+//sIPP/yA7t27G6I8hBDSrLTuGHn++edx+fJlQ5SFEEKandY1wZdffhlz587F1atX4e3tXadjZPTo0XorHCGEGBzTEo/Ha3AxMzPTdneMMca2bNnCunTpwoRCIfPz82NJSUmN5o+Li2NeXl5MKBSyvn37siNHjqitV6lULCIigkkkEiYSidiIESPYn3/+qZbn5ZdfZq6urkwoFDKJRMLeeustdu/ePY3LLJfLGQAml8s1P1Ed/Xz1Puuy8EfWZeGPBj8WIW2BNt9PrW+HVSpVg4suPcOxsbEIDw9HZGQkLly4gP79+yMoKAgPHjyoN39CQgLGjx+PyZMn4+LFixg7dizGjh2La9eucXnWrFmDTZs2Yfv27UhKSoK1tTWCgoJQVlbG5Rk+fDji4uKQlpaGb7/9Funp6fjnP/+pdfkJIa2cNtFVoVAwPp/Prl69qnOErs3Pz4/NmjWL+6xUKpmLiwuLioqqN//rr7/OQkJC1NL8/f3ZtGnTGGNVtUCJRMLWrl3Lrc/Pz2dCoZAdOHCgwXIcPnyY8Xg8plAoNCo31QQJabkMVhO0sLCAm5ub3p4FVCgUSE5OhlQq5dLMzMwglUqRmJhY7zaJiYlq+QEgKCiIy5+RkQGZTKaWRywWw9/fv8F95uXlYd++fRgyZEidNs5q5eXlKCgoUFsIIa2f1rfDS5cuxZIlS5CXl/fUB3/06BGUSiWcnJzU0p2cnCCTyerdRiaTNZq/+qcm+1y4cCGsra3Rvn17ZGZm4vDhww2WNSoqCmKxmFtcXV01O0k9oEcxCTEcrYPgli1b8Pvvv8PFxQVeXl4YNGiQ2tKazJ8/HxcvXsSxY8fA5/MxYcIEsAZe1F28eDHkcjm30PvThLQNWj8iM3bsWL0d3NHREXw+Hzk5OWrpOTk5kEgk9W4jkUgazV/9MycnB87Ozmp5BgwYUOf4jo6O6NGjB3r16gVXV1ecOXMGAQEBdY4rFAohFAq1PkdCSMumdRCMjIzU28EFAgF8fHwQHx/PBVeVSoX4+Hi899579W4TEBCA+Ph4zJkzh0s7fvw4F7g8PDwgkUgQHx/PBb2CggIkJSVhxowZDZaleoTs8vLypz8xQkirofPw+snJyUhNTQUA9OnTBwMHDtRpP+Hh4QgLC4Ovry/8/PwQHR2N4uJiboCGCRMmoFOnToiKigIAzJ49G4GBgVi/fj1CQkIQExOD8+fPY8eOHQCqBnOYM2cOPvroI3h6esLDwwMRERFwcXHhAm1SUhLOnTuHoUOHwsHBAenp6YiIiEC3bt3qrQUaGw2lRYgBadv1nJOTw4YPH854PB5zcHBgDg4OjMfjseeff549ePBAl95stnnzZubm5sYEAgHz8/NjZ86c4dYFBgaysLAwtfxxcXGsR48eTCAQsD59+jT4sLSTkxMTCoVsxIgRLC0tjVt/5coVNnz4cNauXTsmFAqZu7s7mz59Ort7967GZaZHZAhpubT5fvIY066eERoaitu3b2Pv3r3o1asXACAlJQVhYWHo3r07Dhw4YIBQ3fIUFBRALBZDLpcbfHKpo9dkmP51MgDgzuoQgx6LkLZAm++n1rfDR48exYkTJ7gACAC9e/fG1q1b8eKLL2pfWkIIMSKdXpur74FiCwuLOtNvEkJIS6fTUFqzZ89GdnY2l3bv3j3MnTsXI0aM0GvhCCHE0LS+Hd6yZQtGjx4Nd3d37q2JrKws9O3bF19//bXeC9gWKZVKVFRUaJyfr6pAJ1s+AKgNAkGIKRMIBDAz07oeV4fWHSMAwBjDiRMncOPGDQBAr1696rzP29bp0jHCGINMJkN+fr5WxypVKJFbrAAAdHaw1LaohLRJZmZm8PDwgEAgqLNOm++nTkGQ6BYE79+/j/z8fHTs2BFWVlYaz89SWFaB7PxSAICXxLA90YS0BiqVCtnZ2dygLrW/SwbtHQaA+Ph4xMfH48GDB3U6Q3bu3KnLLts8pVLJBcD27dtrtW0544NnXjVyj0gkMkTxCGl1OnTogOzsbFRWVjY4+pMmtA6CK1euxIcffghfX184OzvTbHMaqm4DtLKyMnJJCGkbqm+DlUpl8wbB7du3Y/fu3Xj77bd1Pqgpo/80CNEPfX2XtO5aUSgUGDJkiF4OTgghxqZ1EHz33Xexf/9+Q5SFtCETJ07U67BrADBs2DC10YPq4+7ujujoaL0elxiHJtdbH7S+HS4rK8OOHTtw4sQJ9OvXr869+IYNG/RWONIyDBs2DAMGDKDg0oasWLEChw4dwqVLl4xdFKPTOgheuXKFG6ev5gxvALV3GQ49xURap4qKiqfqtGgOWt8O//rrrw0uv/zyiyHKSIxo4sSJOHnyJDZu3Agejwcej4f09HRMnjwZHh4esLS0hJeXFzZu3Fjv9itXrkSHDh1gZ2eH6dOnQ6FQaHTc4uJiTJgwATY2NnB2dsb69evr5Hnw4AFefvllWFpawsPDA/v27auTh8fjYdu2bRg5ciQsLS3RtWtXfPPNN9z6O3fugMfjIS4uDs8++ywsLS0xePBg/Pnnnzh37hx8fX1hY2ODkSNH4uHDhxr+1aoeFevTpw+EQiGcnZ3VBgnOzMzEmDFjYGNjAzs7O7z++utqo6WvWLECAwYMwFdffQV3d3eIxWKMGzcOhYWFXB6VSoU1a9age/fuEAqFcHNzw8cff8ytX7hwIXr06AErKyt07doVERER3BMKu3fvxsqVK3H58mXumu7evbvJc7px4waGDh0KkUiE3r1748SJE+DxeDh06JDa3zI2NhaBgYEQiUTYt28fcnNzMX78eHTq1AlWVlbw9vauM9qUJtfbYAw4pFebpu14gqWlpSwlJYWVlpZyaSqVihWXVzS5ZOeXsKTbj1jS7Uca5ddkUalUGpU7Pz+fBQQEsClTprD79++z+/fvs7KyMrZ8+XJ27tw5dvv2bfb1118zKysrFhsby20XFhbGbGxsWGhoKLt27Rr78ccfWYcOHdiSJUs0Ou6MGTOYm5sbO3HiBLty5QobNWoUs7W1ZbNnz+byjBw5kvXv358lJiay8+fPsyFDhjBLS0v2n//8h8sDgLVv35598cUXLC0tjS1btozx+XyWkpLCGGMsIyODAWA9e/ZkR48eZSkpKeyZZ55hPj4+bNiwYezUqVPswoULrHv37mz69Okalf2zzz5jIpGIRUdHs7S0NHb27FmuTEqlkg0YMIANHTqUnT9/np05c4b5+PiwwMBAbvvIyEhmY2PDXn31VXb16lX2+++/M4lEova3W7BgAXNwcGC7d+9mt27dYn/88Qf74osvuPWrVq1ip0+fZhkZGeyHH35gTk5O7NNPP2WMMVZSUsI++OAD1qdPH+6alpSUNHpOlZWVzMvLi73wwgvs0qVL7I8//mB+fn4MAPv+++/V/pbu7u7s22+/Zbdv32bZ2dns7t27bO3atezixYssPT2dbdq0ifH5fJaUlKTV9a6tvu9UNW2+nxQEdaSPIFhcXsENltrcS3F5hcbnGhgY2Og/RsYYmzVrFnvttde4z2FhYaxdu3asuLiYS9u2bRuzsbFhSqWy0X0VFhYygUDA4uLiuLTc3FxmaWnJlSMtLY0BYGfPnuXypKamMgB1gmDt4OXv789mzJjBGHvyxf3vf//LrT9w4AADwOLj47m0qKgo5uXl1Wi5q7m4uLClS5fWu+7YsWOMz+ezzMxMLu369etq5xIZGcmsrKxYQUEBl2f+/PnM39+fMcZYQUEBEwqFakGvKWvXrmU+Pj7c58jISNa/f3+Nt//555+Zubk5u3//Ppd2/PjxeoNgdHR0k/sLCQlhH3zwAWNMs+tdH30FQZ2H1yembevWrdi5cycyMzNRWloKhUJRZyKr/v37qz0cHhAQgKKiImRlZaFLly4N7js9PR0KhQL+/v5cWrt27eDl5cV9Tk1Nhbm5OXx8fLi0nj17wt7evs7+ak+ZEBAQUKdDoF+/ftzv1dO1ent7q6U9ePCgwTJXe/DgAbKzsxscUSk1NRWurq5qU7b27t0b9vb2SE1NxeDBgwFU9XLb2tpyeZydnbnjp6amory8vNFRm2JjY7Fp0yakp6ejqKgIlZWVTzX4b1paGlxdXdUmQPPz86s3r6+vr9pnpVKJTz75BHFxcbh37x4UCgXKy8u5fxuaXG9DoiBoRJYWfKR8GNRkPnlpBbLySgAAfTuJ9XZsXcXExGDevHlYv349AgICYGtri7Vr1yIpKUkvZTOGmo331R18tdM0GS/T0lI/A1zU7kyoefymjpGYmIg333wTK1euRFBQEMRiMWJiYpqtnc3a2lrt89q1a7Fx40ZER0fD29sb1tbWmDNnjsbtw4amdcfI77//jsrKyjrplZWV+P333/VSKFPB4/FgJTDXYOFDZFG1aJa/6UWbnnyBQAClUsl9Pn36NIYMGYKZM2di4MCB6N69O9LT0+tsd/nyZZSWlnKfz5w5AxsbmyYnru/WrRssLCzUgurjx4/x559/cp979uyJyspKJCcnc2lpaWn1jtBz5syZOp9rjoyuT7a2tnB3d0d8fHy963v16oWsrCy1eatTUlKQn5+P3r17a3QMT09PWFpaNniMhIQEdOnSBUuXLoWvry88PT3x119/qeWpfU2b4uXlhaysLLUOnHPnzmm07enTpzFmzBi89dZb6N+/P7p27ap2LTW53oakdU1w+PDhuH//Pjp27KiWLpfLMXz4cK3+sKR1cHd3R1JSEu7cuQMbGxt4enpi7969+N///gcPDw989dVXOHfuHDw8PNS2UygUmDx5MpYtW4Y7d+4gMjIS7733XpNjwNnY2GDy5MmYP38+2rdvj44dO2Lp0qVq23l5eSE4OBjTpk3Dtm3bYG5ujjlz5tRbSzp48CB8fX0xdOhQ7Nu3D2fPnsWXX36pnz9OPVasWIHp06ejY8eOGDlyJAoLC3H69Gn8+9//hlQqhbe3N958801ER0ejsrISM2fORGBgYJ3byIaIRCIsXLgQCxYsgEAgwD/+8Q88fPgQ169fx+TJk+Hp6YnMzEzExMRg8ODBOHLkCL7//nu1fbi7uyMjIwOXLl1C586dYWtr2+i82i+88AK6deuGsLAwrFmzBoWFhVi2bBmAph+N8/T0xDfffIOEhAQ4ODhgw4YNyMnJ4YK+JtfbkLQ+CmOs3pPOzc2tUw0mbcO8efPA5/PRu3dvdOjQAUFBQXj11VcRGhoKf39/5ObmYubMmXW2GzFiBDw9PfHcc88hNDQUo0ePxooVKzQ65tq1a/Hss8/i5ZdfhlQqxdChQ9Xa/wBg165dcHFxQWBgIF599VVMnTq1zn/OQNVjOjExMejXrx/27t2LAwcOaFzr0kVYWBiio6Px2WefoU+fPhg1ahRu3rwJoCpgHD58GA4ODnjuuecglUrRtWtXxMbGanWMiIgIfPDBB1i+fDl69eqF0NBQrs1w9OjRmDt3Lt577z0MGDAACQkJiIiIUNv+tddeQ3BwMIYPH44OHTo0OUEan8/HoUOHUFRUhMGDB+Pdd9/F0qVLATQ9stGyZcswaNAgBAUFYdiwYZBIJHXeJtLkehuKxuMJvvrqqwCAw4cPIzg4WO1/DaVSiStXrsDLywtHjx41TElbGG3HEywrK0NGRgY8PDy0Hg5LXqrAX7lVbYL9OtvrUlyTxePx8P333+v9FT5SdZs7dOhQ3Lp1C926dWv24zf2nTLIeIJicVWDPGMMtra2arcdAoEAzzzzDKZMmaLNORBCWpHvv/+eaw65desWZs+ejX/84x9GCYD6pHEQ3LVrF4CqtoR58+bRrS/RWWZmZqO3oykpKXBzc2vGEmnHxsamwXU///wznn322WYsjX7s27cP06ZNq3ddly5dcP36dRQWFmLhwoXIzMyEo6MjpFJp877ZYSA0vL6O6HZYd5WVlbhz506D693d3WFu3nKf3rp161aD6zp16qS3x2SaU2FhoVrPb00WFhaNPtdpLM1+O1wtJycH8+bN44bXrx1DqXeYNMXc3Bzdu3c3djF01prL3hBbW1u1h7NNidZBcOLEicjMzERERAQNr08IafW0DoKnTp3CH3/8UecVKWJA1GBBiMFo/Zygq6trnVtgQghprbQOgtHR0Vi0aFGjDduEENJaaH07HBoaipKSEnTr1g1WVlZ1XvTOy8vTW+EIIcTQtA6CNM8E0Za7uzvmzJnDTZpDb3GQlkTrIBgWFmaIchATcv/+fTg4OBi7GIQA0KFNEKgaBHHZsmUYP34899L2zz//jOvXr+u1cKRtkkgkjY5Y0hwYY/UOCUdMj9ZB8OTJk/D29kZSUhK+++47FBUVAagaOy4yMlLvBSQtQ2FhId58801YW1vD2dkZ//nPf3SeF7a+yXm+++47DB8+HFZWVujfvz8SExPVtjl16hQ3EZKrqyvef/99FBcXc+u/+uor+Pr6wtbWFhKJBG+88YbaSNC//fYbeDwefv75Z/j4+EAoFOLUqVM6/S1I26J1EFy0aBE++ugjHD9+HAKBgEt//vnn6wxeqamtW7fC3d0dIpEI/v7+OHv2bKP5Dx48iJ49e0IkEsHb2xs//fST2nrGGJYvXw5nZ2dYWlpCKpVyQxkBVV+8mrOldevWDZGRkc0/0i1jgKJYo4VXUQJeRYnG+ZtctHzMKTw8HKdPn8YPP/yA48eP448//sCFCxf09qdYunQp5s2bh0uXLqFHjx4YP348V1NLT09HcHAwXnvtNVy5cgWxsbE4deqU2gxuFRUVWLVqFS5fvoxDhw7hzp07mDhxYp3jLFq0CKtXr0ZqaqrakPrEdGndJnj16lXs37+/TnrHjh3x6NEjrQsQGxuL8PBwbN++Hf7+/oiOjkZQUBDS0tLqHRsuISEB48ePR1RUFEaNGoX9+/dj7NixuHDhAvr27QsAWLNmDTZt2oQ9e/bAw8MDERERCAoKQkpKCkQiEW7cuAGVSoXPP/8c3bt3x7Vr1zBlyhQUFxdj3bp1Wp+DzipKgE9cmswmBuDdZC4tLckGBJoNglFYWIg9e/Zg//793LwW1WP56cu8efMQEhICoGr8vz59+uDWrVvo2bMnoqKi8Oabb3K1Tk9PT2zatAmBgYHYtm0bRCIR3nnnHW5fXbt2xaZNmzB48GAUFRWpDXjw4Ycf4oUXXtBbuUnrp3VN0N7eHvfv36+TfvHiRXTq1EnrAmzYsAFTpkzBpEmT0Lt3b2zfvh1WVlbYuXNnvfk3btyI4OBgzJ8/H7169cKqVaswaNAgbNmyBUBVLTA6OhrLli3DmDFjuIE0s7OzuVuw4OBg7Nq1Cy+++CK6du2K0aNHY968efjuu++0Lr8puH37NioqKtQm1hGLxXqdCKdmrczZ2RkAuNvZy5cvY/fu3bCxseGWoKAgqFQqZGRkAACSk5Px8ssvw83NDba2tggMDARQNWJNTZqO3kxMh9Y1wXHjxmHhwoU4ePAgN/nL6dOnMW/ePEyYMEGrfSkUCiQnJ2Px4sVcmpmZGaRSaZ02oWqJiYkIDw9XSwsKCuICXEZGBmQyGaRSKbdeLBbD398fiYmJGDduXL37lcvlaNeuXYNlLS8vR3l5Ofe5oKCgyfNrkoVVVY2sCfISBTIfV83V4a2niZZgYdV0nmZU30RH1RMLFRUVYdq0aXj//ffrbOfm5obi4mIEBQUhKCgI+/btQ4cOHZCZmYmgoKA6TRw0BBypTesg+Mknn2DWrFlwdXWFUqlE7969oVQq8cYbb3BzDmjq0aNHUCqV3BSH1ZycnHDjxo16t5HJZPXml8lk3PrqtIby1Hbr1i1s3ry50VvhqKgorFy5svET0haPp9ktaaUFmMXfA1VoeAurT127doWFhQXOnTvHjfMnl8vx559/4rnnnjP48QcNGoSUlJQGR2+5evUqcnNzsXr1am4Sp/Pnzxu8XKRt0Pp2WCAQ4IsvvkB6ejp+/PFHfP3117hx4wa++uor8Pm6T+NoLPfu3UNwcDD+9a9/NToy9uLFiyGXy7ml5mxhbZ2trS3CwsIwf/58/Prrr9yEPmZmZs0yitDChQuRkJCA9957D5cuXcLNmzdx+PBhrmPEzc0NAoEAmzdvxu3bt/HDDz9g1apVBi8XaRt0HrnSzc3tqUf/dXR0BJ/PrzOYY05OjtokzzVJJJJG81f/zMnJ4dqWqj/XHvkmOzsbw4cPx5AhQ7Bjx45GyyoUCo3+bJsxbdiwAdOnT8eoUaNgZ2eHBQsWICsrS+sBYnXRr18/nDx5EkuXLsWzzz4Lxhi6deuG0NBQAECHDh2we/duLFmyBJs2bcKgQYOwbt06jB492uBlI62fRkEwPDwcq1atgrW1dZ32uNo2bNig8cEFAgF8fHwQHx/PvUKlUqkQHx+v9vhDTQEBAYiPj1d7Pu348eMICAgAAHh4eEAikSA+Pp4LegUFBUhKSsKMGTO4be7du4fhw4fDx8cHu3btarbp/VorW1tb7Nu3j/tcXFyMlStXYurUqU1uW3uwjZqjELm7u9cZlcje3r5O2uDBg3Hs2LEGjzF+/HiMHz++weMMGzaMRj8i9dIoCF68eBEVFRXc7w3R5dYoPDwcYWFh8PX1hZ+fH6Kjo1FcXIxJkyYBACZMmIBOnTohKioKADB79mwEBgZi/fr1CAkJQUxMDM6fP8/V5Hg8HubMmYOPPvoInp6e3CMyLi4uXKC9d+8ehg0bhi5dumDdunV4+PAhV56GaqCm7uLFi7hx4wb8/Pwgl8vx4YcfAgDGjBlj5JIR8pRYC7B582bm5ubGBAIB8/PzY2fOnOHWBQYGsrCwMLX8cXFxrEePHkwgELA+ffqwI0eOqK1XqVQsIiKCOTk5MaFQyEaMGMHS0tK49bt27WKoGqq0zqIpuVzOADC5XK5R/tLSUpaSksJKS0s1Pka1/OJydjnrMbuc9VjrbfXlwoULbNCgQcza2po5ODgwqVTKrly5wn7//XdmbW3d4EKIoTT2ndLm+0kTLemoWSdaKlHgr7yWOdFSaWkp7t271+D6tjgfB2kZmnWipeqJ1zVBDxybFktLSwp0pFXTqDdALBZzi52dHeLj49Wew0pOTkZ8fDw3QTshhLQWGtUEqydeB6qe2Xr99dexfft27rlApVKJmTNnanRbaOqq34IghDwdfbXkad0m2KFDB5w6darOe6NpaWkYMmQIcnNz9VKwlk7bNkGVSoWbN2+Cz+ejQ4cOEAgEGvemF5YqkC0vAwB4Seg/GkIYY3j48CFKSkrg6elZ50UNg06+XllZiRs3btQJgtUjs5D6mZmZwcPDA/fv30d2dtPvC9dUqlAit7jqHVhBqaUhikdIq8Pj8dC5c+enflNN6yA4adIkTJ48Genp6dyoIklJSVi9ejX3bB+pn0AggJubGyorK6FUKjXe7uSfD/DhrykAgPgPhhmodIS0LhYWFnp5VVfrILhu3TpIJBKsX7+eG1LL2dkZ8+fPxwcffPDUBWrreDweLCws6szS1xglzwL3CquCZnO8pkaIKdE6CJqZmWHBggVYsGABN5wUdYgQQlornQdQACj4EUJaP52C4DfffIO4uDhkZmbWGbRSn/NOEEKIoWk9dMqmTZswadIkODk54eLFi/Dz80P79u1x+/ZtjBw50hBlJIQQg9E6CH722WfYsWMHNm/eDIFAgAULFuD48eN4//33IZfLDVFGk0dvdxNiOFoHwczMTAwZMgRA1XujhYWFAIC3334bBw4c0G/pCCHEwLQOghKJBHl5eQCqRpeunms4IyODBq0khLQ6WgfB559/Hj/88AOAqgen586dixdeeAGhoaF45ZVX9F5AQggxJK17h3fs2MG9Hjdr1iy0b98eCQkJGD16NKZNm6b3AhJCiCFpFQQrKyvxySef4J133kHnzp0BVM1D3NBcvoQQ0tJpdTtsbm6ONWvWoLKy0lDlIfVohlktCTFZWrcJjhgxAidPnjREWQghpNlp3SY4cuRILFq0CFevXoWPjw+sra3V1tNcr4SQ1kTrIDhz5kwA9c8vzOPxtBoiihBCjE3rIEgDpxJC2hKt2wQJIaQt0bgmWFpaivj4eIwaNQoAsHjxYpSXl3Pr+Xw+Vq1aRYN+EkJaFY2D4J49e3DkyBEuCG7ZsgV9+vSBpWXVnBc3btyAi4sL5s6da5iSEkKIAWh8O7xv3z5MnTpVLW3//v349ddf8euvv2Lt2rWIi4vTewEJIcSQNA6Ct27dgre3N/dZJBLBzOzJ5n5+fkhJSdFv6QgAGkqLEEPS+HY4Pz9frQ3w4cOHautVKpXaekIIaQ00rgl27twZ165da3D9lStXuPeJCSGktdA4CL700ktYvnw5ysrK6qwrLS3FypUrERISotfCEUKIoWl8O7xkyRLExcXBy8sL7733Hnr06AEASEtLw5YtW1BZWYklS5YYrKDEOL5Nvovzfz3GR2P7gm9GIzmQtkfjIOjk5ISEhATMmDEDixYt4kaR5vF4eOGFF/DZZ5/BycnJYAUlxvHBwcsAgCHd2uPl/i5GLg0h+qfVa3MeHh44evQo8vLycOvWLQBA9+7d0a5dO4MUjrQc+aUVxi4CIQah07zD7dq1g5+fn77LQlowuhMmbRW9O9yKVSpV+OVGDvJLFAY/lhmN7EraKAqCrdjnv9/GO7vP45/bEw1+LKoJkrbK6EFw69atcHd3h0gkgr+/P86ePdto/oMHD6Jnz54QiUTw9vbGTz/9pLaeMYbly5fD2dkZlpaWkEqluHnzplqejz/+GEOGDIGVlRXs7e31fUrN5v8uZwMAbj0oMvixeKAoSNomowbB2NhYhIeHIzIyEhcuXED//v0RFBSEBw8e1Js/ISEB48ePx+TJk3Hx4kWMHTsWY8eOVXuIe82aNdi0aRO2b9+OpKQkWFtbIygoSO35RoVCgX/961+YMWOGwc/RkHjNeItKd8OkrTJqENywYQOmTJmCSZMmoXfv3ti+fTusrKywc+fOevNv3LgRwcHBmD9/Pnr16oVVq1Zh0KBB2LJlC4CqWmB0dDSWLVuGMWPGoF+/fti7dy+ys7Nx6NAhbj8rV67E3Llz1d6FJo2jNkHSVhktCCoUCiQnJ0MqlT4pjJkZpFIpEhPrb+NKTExUyw8AQUFBXP6MjAzIZDK1PGKxGP7+/g3uU1Pl5eUoKChQW4ytOdvpzIzecEKIYRjtn/ajR4+gVCrrPGDt5OQEmUxW7zYymazR/NU/tdmnpqKioiAWi7nF1dX1qfanD81ZOaOaIGmr6P93DS1evBhyuZxbsrKymu3YDDSWFiGGYrQg6OjoCD6fj5ycHLX0nJwcSCSSereRSCSN5q/+qc0+NSUUCmFnZ6e2GFuarLDZjkU1QdJWGS0ICgQC+Pj4ID4+nktTqVSIj49HQEBAvdsEBASo5QeA48ePc/k9PDwgkUjU8hQUFCApKanBfbZmFcrmqyFSECRtlU6vzelLeHg4wsLC4OvrCz8/P0RHR6O4uBiTJk0CAEyYMAGdOnVCVFQUAGD27NkIDAzE+vXrERISgpiYGJw/fx47duwAUPXIyJw5c/DRRx/B09MTHh4eiIiIgIuLC8aOHcsdNzMzE3l5ecjMzIRSqcSlS5cAVL0HbWNj06x/g9aCHpYmbZVRg2BoaCgePnyI5cuXQyaTYcCAATh69CjXsZGZmak2hP+QIUOwf/9+LFu2DEuWLIGnpycOHTqEvn37cnkWLFiA4uJiTJ06Ffn5+Rg6dCiOHj2qNgve8uXLsWfPHu7zwIEDAQC//vorhg0bZuCzbp2a85lEQpoTjzGawUIXBQUFEIvFkMvlBm8f/PFKNt7bfxEAcGf1k4Fr3Rcd4X6vma5P1cf4/G0fBPV5unZVQpqLNt9P6h0mDar5/yO1CZK2ioIgaZCqxj0CtQmStoqCIGmQimqCxARQECSc2s3DNYMgxUDSVlEQbEEOns/CjK+TUVahbPZjz9yXjP4rj6Gg7Mkw+jVjIvUOk7aKgmALMv+bK/j5mgxfn/lLLX3DsT+b3Db5rzydj/u4WIGfrspQUFaJ+X9PrATUvh3WefeEtGgUBFsgeY1JjcoqlLj9qLjJbV7blgiZvO6c0Jp4bVsC9/v/rj955VC9Y4SiIGmbKAi2cHNjL2mc9+7jEq33X6KobDDIUpsgMQUUBFu4n69pPgSYolKFf21PwJenMjTe5tj1nAbXMdWT36kmSNoqo742R55Qqp7+xZ3wuMuQFZTh3J3HmDzUQ6Nt+I009qnVBJ+6dIS0TFQTbCGCon9/6n3ICp60CRaVV2q0jbmGQdCMekZIG0VBsIWoOWPcndwSrDl6Awnpj3Te36c/39AoX+M1wSe/UwgkbRXdDrdA1VNpfvZbus77OHen8Udmbj0oxJ85RRBZNPz/II2tQUwBBcE26kYTo05LN1Tdfk95tuG2Qz00UxLS4tHtsIm7dq/hWfNqtglSPCRtFQVBE5d4O7fBdSq6HSYmgIJgC9BS295aaLEI0SsKgq1U9Imm3yduyOFL9zTKRzVBYgooCLYAusSa6BM3dT7e/G+uaJSPOkaIKaAg2AIYKtYcu17/K3ea3n6rdYxQQCRtFAXBViapkY6M2qZ+lYxSRd2xCTWdr7iltlUSok8UBFsAbYJN6I4zWu1boVQ1namWa/fkYIzR7TAxCRQEjeRRUTkYY2CMocSAI0k39lpcQ0ZtPoVvku9SxwgxCfTGiBEcPJ+F+d9cwbTnuuKGrBAn/3xosGPpOu7B/G+u4MCUZ/RbGEJaIAqCRvDhjykAgM9/v63T9hkajDRd7ZWtCfhp9rM61QjDdp7lfqf2QWIwFaVASS5Q/KjqZ0ne3z+rl0fqaZ0HA+P26e3wFASbmUrFdApINQ1f95vGedNyCvFnTiF6OdtpfRxd2hOJiVNWAKWPawW12oGtVrCr0HJE9ELNBxrWBAXBZvSoqBwjN/6B/JKKpjPrEQ0KTXSiUgHl8ifBSi2o1QxsNdLL5Lody8wCsGr/99IOsHas8blGupUjYNNRr6dJQbAZfXkqAw8Ly41ahh/+HqaLmBjGqmpcNYNYcW6toFYd2GrU1JgunXY8wNLhSfCydvw7gNUOajXShbZG+9+agqCJef/ARWMXgehDpQIordV2Vlyr7az2UqnbbIQQ2KoHMa6WVjuw/Z1uaQ+Y8fV6uoZEQZBohLpFDEilAsry6watBjsKcoHyhodAaxRf8CRYVQcxtVvPdnVrauZCvZ5uS0NBsBlR05wJYAxQFNXfIdBQR0FpnvrUfprimQGW7aBZW9rfQU1gTY3EtVAQNAGVGr4m1xwYY+C1pi9hZXkD7WaNBDWlju2+QnHdW0zr2gGtxiKyB8zofYenRUHQBFT+/f7bih+uG7Uc2fmlGL3lNN7wc0X4i17NXwCVEijNV+/NrBnE6uv9VDQ+TUGDzEXqDf9NdRJYtgPMBXo9XaIZCoLNoFShRNbjEqPdhdx5VIwBrvbYnXDHOAX429r/peFRUTk2/XLr6YMgY0B5YcOdAPX1fpY+hk6tmzx+A7eXjXQSCKyf7vxIs6Eg2AzGbj2NtBwdaxR6MCf20lMNwgroZyitvGJFwysryhp+Q6Ch3k+Vjs9biuwbbjer06bWruo2lW4726wWEQS3bt2KtWvXQiaToX///ti8eTP8/PwazH/w4EFERETgzp078PT0xKeffoqXXnqJW88YQ2RkJL744gvk5+fjH//4B7Zt2wZPT08uT15eHv7973/j//7v/2BmZobXXnsNGzduhI2NjX5P7lY8+jw8BhczWzxmtngMGzxmNiiEFVgzjl9xJ1fLp/JrKSzTIuAoK5+8NVDjYdoXHl3Ec+YP4cArBL7eqR7UKjR/FVCNhVX9t50NtadZOgB8C92ORdokHjPyS6GxsbGYMGECtm/fDn9/f0RHR+PgwYNIS0tDx451nwxPSEjAc889h6ioKIwaNQr79+/Hp59+igsXLqBv374AgE8//RRRUVHYs2cPPDw8EBERgatXryIlJQUikQgAMHLkSNy/fx+ff/45KioqMGnSJAwePBj79+/XqNwFBQUQi8WQy+Wws2vklbQDbwBpR+okVzIz5MOGC4z5zAZ5zBb5sEUes8Fj2NZIq/ophw1UzT7wD4MdStDNphzfh/Ws+4ZAfR0FZfm6HcrMvO5DtDUWhagdMkqE8OzSBWY2jlXtaAIrvZ4taRs0/n6iBQRBf39/DB48GFu2bAEAqFQquLq64t///jcWLVpUJ39oaCiKi4vx448/cmnPPPMMBgwYgO3bt4MxBhcXF3zwwQeYN28eAEAul8PJyQm7d+/GuHHjkJqait69e+PcuXPw9fUFABw9ehQvvfQS7t69CxcXlybLrfEf+eQanDzxf3DgFcIBRXDgFcKGp9tDqyrGQwGs1AJjPmzxmD0JpnnMFvlcjbPqZ2WNCr8I5WiHQjjwCtGOVwgHFKI9r6Dqc3X63+VszyuAPYpgwdNxqC/urYGqW8zv0kqRU2mNPGaLpf8cWrc9TWjX6OMbU/aex/GUHES96o3xfm66lakZVChVKC6vRLFCWfWzvBIlCiWKyitRoqhEUbkSJeWV4Jvx4GJviU72lujkYIn21oLW1XPegmkTBI16O6xQKJCcnIzFixdzaWZmZpBKpUhMTKx3m8TERISHh6ulBQUF4dChQwCAjIwMyGQySKVSbr1YLIa/vz8SExMxbtw4JCYmwt7enguAACCVSmFmZoakpCS88sordY5bXl6O8vInjz4UFGj2sOq7d57HiYo+amkCVMD+70DjwCuCA2r+rD9NzCuBGY/BHsWw52l361jALFECEcQohiWvkXa5RhQxEYr4dpDzxJDz7FDAs0OBmR33+81CIXKUNsiDLRw7OKOQZwMVjw8oARRWLTfKnrSL/nHS9u/fSv5e7jZZhuoJ5Rd/dxV7jNzJU5OKMRSXK1GiqERxuVLngSdEFmZwEVtCYE7tj43x7iTG2n/119v+jBoEHz16BKVSCScnJ7V0Jycn3Lhxo95tZDJZvfllMhm3vjqtsTy1b7XNzc3Rrl07Lk9tUVFRWLlypYZn9kR9A8YoYIEHcMAD5qBxZyUfStijCPa8Iq7GVh0kn6T9HVj/TrNHEfg8BjteKexQyu2rnJnjMaraKHOZLR7DFnk1fzJb5MIOj5ktV+ssh+aPb6Q/AKoCW8OqA5qunnb75iDgm8FayIeVwBw2QnNYCflVPwV8WAvNUaFkuPe4BPfyS5FTUI6yChVuazFMmqkSW+q3TbdFdIy0BosXL1argRYUFMDV1bXJ7Va/1g8RoyqR/Ndj3H5UjIRbjzDS2xlCczO0txYgLacQOQVl+C3tIe7Lq26Tg/o4QSYvw80HRahQqvAvX1dUKlXIKSjHm/5uyMwrwd3HpXCyE6GgrAIFANo5WuP8wyJc/CsfzvYi8AAoKirxz762KMjLQUdBBZwlzsgqs8SVB5XIfFyKD170wi/Jd+He3hqveXXA8RQZCu8V4DVPR1QoGXioGpmagUHAb/pd0PvyUnS0E4HfyC1daYUSlha6vVeqYgwlikrYCFtWxwaPBy6wWQvNYS2oCnza1OjKK5W4n1+G+/IyKGleg0a1qSDo6OgIPp+PnJwctfScnBxIJJJ6t5FIJI3mr/6Zk5MDZ2dntTwDBgzg8jx48EBtH5WVlcjLy2vwuEKhEEKh9u9QtrMWoJ21AK7tqhrww1/oobZ+pLdzfZvpWU/uNw8Az9VYM2t4d+73VwZ2xisDm6E4pA6hOR/ujtZwd6TnC5ubURsfBAIBfHx8EB8fz6WpVCrEx8cjICCg3m0CAgLU8gPA8ePHufweHh6QSCRqeQoKCpCUlMTlCQgIQH5+PpKTk7k8v/zyC1QqFfz9/fV2foSQVoAZWUxMDBMKhWz37t0sJSWFTZ06ldnb2zOZTMYYY+ztt99mixYt4vKfPn2amZubs3Xr1rHU1FQWGRnJLCws2NWrV7k8q1evZvb29uzw4cPsypUrbMyYMczDw4OVlpZyeYKDg9nAgQNZUlISO3XqFPP09GTjx4/XuNxyuZwBYHK5XA9/BUKIPmnz/TR6EGSMsc2bNzM3NzcmEAiYn58fO3PmDLcuMDCQhYWFqeWPi4tjPXr0YAKBgPXp04cdOXJEbb1KpWIRERHMycmJCYVCNmLECJaWlqaWJzc3l40fP57Z2NgwOzs7NmnSJFZYWKhxmSkIEtJyafP9NPpzgq2VNs8hEUKalzbfT3ogiRBi0igIEkJMGgVBQohJo4eldVTdlKrp63OEkOZT/b3UpMuDgqCOCgurXtvS5K0RQohxFBYWQiwWN5qHeod1pFKpkJ2dDVtb2xY/8kf1K35ZWVltqiebzqt1ac7zYoyhsLAQLi4uMGtiQFyqCerIzMwMnTt3NnYxtGJnZ9emvlTV6Lxal+Y6r6ZqgNWoY4QQYtIoCBJCTBoFQRMgFAoRGRmp0yg4LRmdV+vSUs+LOkYIISaNaoKEEJNGQZAQYtIoCBJCTBoFQUKISaMg2Apt3boV7u7uEIlE8Pf3x9mzZxvNf/DgQfTs2RMikQje3t746aef1NYzxrB8+XI4OzvD0tISUqkUN2/eNOQp1Eub8/riiy/w7LPPwsHBAQ4ODpBKpXXyT5w4ETweT20JDg429GnUS5tz2717d51yi0QitTyt8ZoNGzasznnxeDyEhIRweYxyzQw1sisxjJiYGCYQCNjOnTvZ9evX2ZQpU5i9vT3LycmpN//p06cZn89na9asYSkpKWzZsmX1TkcgFovZoUOH2OXLl9no0aPrTEdgaNqe1xtvvMG2bt3KLl68yFJTU9nEiROZWCxmd+/e5fKEhYWx4OBgdv/+fW7Jy8trrlPiaHtuu3btYnZ2dmrlrp5uolprvGa5ublq53Tt2jXG5/PZrl27uDzGuGYUBFsZPz8/NmvWLO6zUqlkLi4uLCoqqt78r7/+OgsJCVFL8/f3Z9OmTWOMVU1FIJFI2Nq1a7n1+fn5TCgUsgMHDhjgDOqn7XnVVllZyWxtbdmePXu4tLCwMDZmzBh9F1Vr2p7brl27mFgsbnB/beWa/ec//2G2trasqKiISzPGNaPb4VZEoVAgOTkZUqmUSzMzM4NUKkViYmK92yQmJqrlB4CgoCAuf0ZGBmQymVoesVgMf3//Bvepb7qcV20lJSWoqKhAu3bt1NJ/++03dOzYEV5eXpgxYwZyc3P1Wvam6HpuRUVF6NKlC1xdXTFmzBhcv36dW9dWrtmXX36JcePGwdpafZrR5r5mFARbkUePHkGpVMLJyUkt3cnJCTKZrN5tZDJZo/mrf2qzT33T5bxqW7hwIVxcXNS+lMHBwdi7dy/i4+Px6aef4uTJkxg5ciSUSqVey98YXc7Ny8sLO3fuxOHDh/H1119DpVJhyJAhuHv3LoC2cc3Onj2La9eu4d1331VLN8Y1o1FkSKu3evVqxMTE4LffflPrQBg3bhz3u7e3N/r164du3brht99+w4gRI4xRVI0EBASozbs9ZMgQ9OrVC59//jlWrVplxJLpz5dffglvb2/4+fmppRvjmlFNsBVxdHQEn89HTk6OWnpOTg4kEkm920gkkkbzV//UZp/6pst5VVu3bh1Wr16NY8eOoV+/fo3m7dq1KxwdHXHr1q2nLrOmnubcqllYWGDgwIFcuVv7NSsuLkZMTAwmT57c5HGa45pREGxFBAIBfHx8EB8fz6WpVCrEx8er1RxqCggIUMsPAMePH+fye3h4QCKRqOUpKChAUlJSg/vUN13OCwDWrFmDVatW4ejRo/D19W3yOHfv3kVubi6cnZ31Um5N6HpuNSmVSly9epUrd2u+ZkDVI1vl5eV46623mjxOs1yzZu2GIU8tJiaGCYVCtnv3bpaSksKmTp3K7O3tuUco3n77bbZo0SIu/+nTp5m5uTlbt24dS01NZZGRkfU+ImNvb88OHz7Mrly5wsaMGWOUxy20Oa/Vq1czgUDAvvnmG7XHKQoLCxljjBUWFrJ58+axxMRElpGRwU6cOMEGDRrEPD09WVlZWbOdly7ntnLlSva///2Ppaens+TkZDZu3DgmEonY9evX1c6/tV2zakOHDmWhoaF10o11zSgItkKbN29mbm5uTCAQMD8/P3bmzBluXWBgIAsLC1PLHxcXx3r06MEEAgHr06cPO3LkiNp6lUrFIiIimJOTExMKhWzEiBEsLS2tOU5FjTbn1aVLFwagzhIZGckYY6ykpIS9+OKLrEOHDszCwoJ16dKFTZkypc7zds1Fm3ObM2cOl9fJyYm99NJL7MKFC2r7a43XjDHGbty4wQCwY8eO1dmXsa4ZDaVFCDFp1CZICDFpFAQJISaNgiAhxKRRECSEmDQKgoQQk0ZBkBBi0igIEkJMGgVBQohJoyBI2oz6hmY35pD6pHWgobRImxIcHIxdu3appQmFwnrzVlRUwMLCQi1NoVBAIBBofVxdtyPGRzVB0qYIhUJIJBK1xcHBAQDA4/Gwbds2jB49GtbW1vj444+xYsUKDBgwAP/973/h4eHBjUeYmZmJMWPGwMbGBnZ2dnj99dfVho1qaDvS+lAQJCZlxYoVeOWVV3D16lW88847AIBbt27h22+/xXfffYdLly5BpVJhzJgxyMvLw8mTJ3H8+HHcvn0boaGhavuqvR1pneh2mLQpP/74I2xsbNTSlixZgiVLlgAA3njjDUyaNEltvUKhwN69e9GhQwcAVeMtXr16FRkZGXB1dQUA7N27F3369MG5c+cwePDgercjrRMFQdKmDB8+HNu2bVNLqzn5Un2Dr3bp0kUtkKWmpsLV1ZULgADQu3dv2NvbIzU1lQuCtbcjrRMFQdKmWFtbo3v37o2u1yRN02OR1o/aBAmppVevXsjKykJWVhaXlpKSgvz8fPTu3duIJSOGQDVB0qaUl5fXmfLR3Nwcjo6OGu9DKpXC29sbb775JqKjo1FZWYmZM2ciMDBQo7lMSOtCNUHSphw9ehTOzs5qy9ChQ7XaB4/Hw+HDh+Hg4IDnnnsOUqkUXbt2RWxsrIFKTYyJhtcnhJg0qgkSQkwaBUFCiEmjIEgIMWkUBAkhJo2CICHEpFEQJISYNAqChBCTRkGQEGLSKAgSQkwaBUFCiEmjIEgIMWkUBAkhJu3/AQXsLLdVwORdAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019191, + "end_time": "2024-03-23T03:39:55.023918", + "exception": false, + "start_time": "2024-03-23T03:39:55.004727", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3397.729204, + "end_time": "2024-03-23T03:39:57.766335", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tab_ddpm_concat/3/mlu-eval.ipynb", + "output_path": "eval/insurance/tab_ddpm_concat/3/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/tab_ddpm_concat/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-23T02:43:20.037131", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/model.pt b/insurance/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..892fb1f0ec3c1bc45f187e56af50d3736e3b9fd7 --- /dev/null +++ b/insurance/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb00eb0a3fc2be866c2dbee35c72985295e305f44a2cf6fffdd10153f525a448 +size 38514197 diff --git a/insurance/tab_ddpm_concat/params.json b/insurance/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..1aee4f1b59ccfb97ad613287b0d1578db4e2edd8 --- /dev/null +++ b/insurance/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "tab_ddpm_concat", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tvae/eval.csv b/insurance/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..cc1959a62ea8533c7828accec999aca21aef94ee --- /dev/null +++ b/insurance/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.005972265340193796,0.004282069878915665,0.00043351158954291826,6.498488903045654,0.0045961132273077965,0.3815285265445709,0.008567946031689644,1.2653150349706266e-07,2.329435348510742,0.012678780592978,0.9071698188781738,0.020820939913392067,0.14727535843849182,2.352470396260742e-08,8.827924251556396 diff --git a/insurance/tvae/history.csv b/insurance/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..df00da9f90994d0589a2b9a8ed9f0847c18ffbc5 --- /dev/null +++ b/insurance/tvae/history.csv @@ -0,0 +1,13 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03318693633812169,0.5761553976163866,0.009452911170182633,0.5073339222537147,0.0,0.0,0.0,0.0,0.03357576689010279,900,113,140.07320928573608,1.2395859228826203,0.1556368992063734,0.1377185063222341,0.011029714790444511,0.10201651341138354,0.0008927303574836721,0.0,0.0,0.0,0.0,0.0,0.011029714790444511,450,57,44.65146517753601,0.7833590382023862,0.09922547817230225,0.09806239102934405 +1,0.008319700596491909,1.3899801381037156,0.00032640089475284936,0.08519980923169189,0.0,0.0,0.0,0.0,0.008420131139250265,900,113,141.50769209861755,1.252280461049713,0.15723076899846394,0.08328526641810889,0.0042552733277746784,0.797521198754859,4.676108655203848e-05,0.0,0.0,0.0,0.0,0.0,0.0042552733277746784,450,57,44.46026659011841,0.7800046770196212,0.09880059242248534,0.05082482615845245 +2,0.0030213647201243372,0.4467241249097572,2.9881127188979347e-05,0.03057058709777064,0.0,0.0,0.0,0.0,0.003060984404421308,900,113,141.14445447921753,1.249065968842633,0.15682717164357504,0.09463177754881635,0.004963098757434637,0.23140327160492807,0.0003732240848752491,0.0,0.0,0.0,0.0,0.0,0.004963098757434637,450,57,44.70943355560303,0.7843760272912812,0.09935429679022895,0.07549247669317481 +3,0.0017553741914970386,0.2178003895808945,4.943996762005542e-06,0.02547209452009863,0.0,0.0,0.0,0.0,0.001773931053143719,900,113,141.30070185661316,1.2504486889965767,0.15700077984068128,0.09433466191939284,0.002269126343291848,0.25996375590649135,0.000101374774749564,0.0,0.0,0.0,0.0,0.0,0.002269126343291848,450,57,44.660250663757324,0.7835131695396022,0.09924500147501628,0.06720550957119517 +4,0.0011990225297955073,0.19844268139087864,6.492888718708172e-06,0.02050420185758008,0.0,0.0,0.0,0.0,0.0012119675411183077,900,113,141.14717173576355,1.2490900153607394,0.15683019081751506,0.09565485499601449,0.0021288806346031683,0.3498717648343612,6.43633937075372e-05,0.0,0.0,0.0,0.0,0.0,0.0021288806346031683,450,57,44.87625551223755,0.7873027282848692,0.09972501224941678,0.06606765008090358 +5,0.0010727717719338317,0.1799432011435814,1.986284879775273e-06,0.019064169105970197,0.0,0.0,0.0,0.0,0.0010836002508100744,900,113,144.71740865707397,1.2806850323634864,0.1607971207300822,0.0960029606352997,0.0025687832964791193,0.24147355027886128,0.00013711076116941693,0.0,0.0,0.0,0.0,0.0,0.0025687832964791193,450,57,44.92388701438904,0.7881383686734919,0.09983086003197564,0.06918113802053165 +6,0.0008831181887621319,0.1315059885362297,1.4708352961886883e-06,0.01531516697154277,0.0,0.0,0.0,0.0,0.0008934507078892138,900,113,141.82023167610168,1.255046298018599,0.15757803519566854,0.09242146539674924,0.0017530873860091055,0.304231248577451,7.560512728054956e-05,0.0,0.0,0.0,0.0,0.0,0.0017530873860091055,450,57,44.82837462425232,0.7864627127061811,0.09961861027611627,0.06672564117858808 +7,0.0005385009764318562,0.07839578661780303,7.117195005359047e-07,0.012112246143321197,0.0,0.0,0.0,0.0,0.0005440399935145655,900,113,141.7691547870636,1.2545942901510052,0.15752128309673732,0.09718591164368971,0.0023827175304500592,2.154619756286098,0.00013736589136319527,0.0,0.0,0.0,0.0,0.0,0.0023827175304500592,450,57,46.83988165855408,0.8217523097991943,0.10408862590789796,0.06465611442723584 +8,0.0004240491726927252,0.046745863687660795,1.4920491366082138e-07,0.010413902575771013,0.0,0.0,0.0,0.0,0.0004284947737728039,900,113,147.48687982559204,1.3051936267751507,0.1638743109173245,0.0963774272529161,0.0019166796955202396,0.40897835704338553,4.469346806295368e-05,0.0,0.0,0.0,0.0,0.0,0.0019166796955202396,450,57,45.08631491661072,0.7909879809931705,0.1001918109258016,0.0743747265813382 +9,0.0005618307212070148,0.06497306443851963,5.931063866807492e-07,0.012489941705846124,0.0,0.0,0.0,0.0,0.0005679361823422369,900,113,142.84351229667664,1.2641018787316516,0.15871501366297405,0.09641832253376467,0.0017379091629603257,0.4798262932258854,2.2975261447157275e-05,0.0,0.0,0.0,0.0,0.0,0.0017379091629603257,450,57,46.893903970718384,0.822700069661726,0.10420867549048529,0.06686989114856706 +10,0.00031843584507846066,0.034923846742066104,2.1399565589458585e-07,0.009167301410602199,0.0,0.0,0.0,0.0,0.00032186484126011945,900,113,144.4654381275177,1.2784552046682982,0.16051715347501966,0.10122920503526663,0.0017598816551940722,0.5481770304940762,4.9227021056914934e-05,0.0,0.0,0.0,0.0,0.0,0.0017598816551940722,450,57,45.89448928833008,0.8051664787426329,0.10198775397406684,0.0716182768922871 +11,0.0002415686168306921,0.022000115441331264,3.5452563263274956e-08,0.007707212487649586,0.0,0.0,0.0,0.0,0.00024431025462238017,900,113,141.62232375144958,1.253294900455306,0.15735813750161065,0.09990070720689487,0.001438923770741288,0.40616633773342614,1.6548902490660263e-05,0.0,0.0,0.0,0.0,0.0,0.001438923770741288,450,57,44.89892363548279,0.7877004146575928,0.09977538585662842,0.07267187253098216 diff --git a/insurance/tvae/mlu-eval.ipynb b/insurance/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..89990342d4c5f4d886302275e909ebb67a91fc46 --- /dev/null +++ b/insurance/tvae/mlu-eval.ipynb @@ -0,0 +1,2350 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:15.832582Z", + "iopub.status.busy": "2024-03-22T21:10:15.832310Z", + "iopub.status.idle": "2024-03-22T21:10:15.864039Z", + "shell.execute_reply": "2024-03-22T21:10:15.863191Z" + }, + "papermill": { + "duration": 0.045887, + "end_time": "2024-03-22T21:10:15.865957", + "exception": false, + "start_time": "2024-03-22T21:10:15.820070", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:15.890459Z", + "iopub.status.busy": "2024-03-22T21:10:15.890140Z", + "iopub.status.idle": "2024-03-22T21:10:15.896569Z", + "shell.execute_reply": "2024-03-22T21:10:15.895747Z" + }, + "papermill": { + "duration": 0.020964, + "end_time": "2024-03-22T21:10:15.898544", + "exception": false, + "start_time": "2024-03-22T21:10:15.877580", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:15.921400Z", + "iopub.status.busy": "2024-03-22T21:10:15.921136Z", + "iopub.status.idle": "2024-03-22T21:10:15.925068Z", + "shell.execute_reply": "2024-03-22T21:10:15.924234Z" + }, + "papermill": { + "duration": 0.017692, + "end_time": "2024-03-22T21:10:15.927063", + "exception": false, + "start_time": "2024-03-22T21:10:15.909371", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:15.949835Z", + "iopub.status.busy": "2024-03-22T21:10:15.949596Z", + "iopub.status.idle": "2024-03-22T21:10:15.953396Z", + "shell.execute_reply": "2024-03-22T21:10:15.952525Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017386, + "end_time": "2024-03-22T21:10:15.955228", + "exception": false, + "start_time": "2024-03-22T21:10:15.937842", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:15.978266Z", + "iopub.status.busy": "2024-03-22T21:10:15.978022Z", + "iopub.status.idle": "2024-03-22T21:10:15.983576Z", + "shell.execute_reply": "2024-03-22T21:10:15.982717Z" + }, + "papermill": { + "duration": 0.019282, + "end_time": "2024-03-22T21:10:15.985371", + "exception": false, + "start_time": "2024-03-22T21:10:15.966089", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f9755450", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:16.010429Z", + "iopub.status.busy": "2024-03-22T21:10:16.009783Z", + "iopub.status.idle": "2024-03-22T21:10:16.014775Z", + "shell.execute_reply": "2024-03-22T21:10:16.013957Z" + }, + "papermill": { + "duration": 0.019757, + "end_time": "2024-03-22T21:10:16.016754", + "exception": false, + "start_time": "2024-03-22T21:10:15.996997", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tvae/2\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01075, + "end_time": "2024-03-22T21:10:16.039486", + "exception": false, + "start_time": "2024-03-22T21:10:16.028736", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:16.062440Z", + "iopub.status.busy": "2024-03-22T21:10:16.062197Z", + "iopub.status.idle": "2024-03-22T21:10:16.071048Z", + "shell.execute_reply": "2024-03-22T21:10:16.070279Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.02251, + "end_time": "2024-03-22T21:10:16.072890", + "exception": false, + "start_time": "2024-03-22T21:10:16.050380", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tvae/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:16.096624Z", + "iopub.status.busy": "2024-03-22T21:10:16.096378Z", + "iopub.status.idle": "2024-03-22T21:10:18.041844Z", + "shell.execute_reply": "2024-03-22T21:10:18.040779Z" + }, + "papermill": { + "duration": 1.959483, + "end_time": "2024-03-22T21:10:18.044052", + "exception": false, + "start_time": "2024-03-22T21:10:16.084569", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:18.075401Z", + "iopub.status.busy": "2024-03-22T21:10:18.074949Z", + "iopub.status.idle": "2024-03-22T21:10:18.088157Z", + "shell.execute_reply": "2024-03-22T21:10:18.087265Z" + }, + "papermill": { + "duration": 0.033106, + "end_time": "2024-03-22T21:10:18.090200", + "exception": false, + "start_time": "2024-03-22T21:10:18.057094", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:18.114579Z", + "iopub.status.busy": "2024-03-22T21:10:18.114289Z", + "iopub.status.idle": "2024-03-22T21:10:18.121731Z", + "shell.execute_reply": "2024-03-22T21:10:18.120914Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.02171, + "end_time": "2024-03-22T21:10:18.123701", + "exception": false, + "start_time": "2024-03-22T21:10:18.101991", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:18.154002Z", + "iopub.status.busy": "2024-03-22T21:10:18.153706Z", + "iopub.status.idle": "2024-03-22T21:10:18.249740Z", + "shell.execute_reply": "2024-03-22T21:10:18.248699Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.11742, + "end_time": "2024-03-22T21:10:18.252569", + "exception": false, + "start_time": "2024-03-22T21:10:18.135149", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:18.283575Z", + "iopub.status.busy": "2024-03-22T21:10:18.283205Z", + "iopub.status.idle": "2024-03-22T21:10:22.825266Z", + "shell.execute_reply": "2024-03-22T21:10:22.824451Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.558, + "end_time": "2024-03-22T21:10:22.827618", + "exception": false, + "start_time": "2024-03-22T21:10:18.269618", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 21:10:20.494705: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 21:10:20.494767: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 21:10:20.496499: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:22.852459Z", + "iopub.status.busy": "2024-03-22T21:10:22.851926Z", + "iopub.status.idle": "2024-03-22T21:10:22.858466Z", + "shell.execute_reply": "2024-03-22T21:10:22.857786Z" + }, + "papermill": { + "duration": 0.02084, + "end_time": "2024-03-22T21:10:22.860310", + "exception": false, + "start_time": "2024-03-22T21:10:22.839470", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:22.885949Z", + "iopub.status.busy": "2024-03-22T21:10:22.885609Z", + "iopub.status.idle": "2024-03-22T21:10:30.941034Z", + "shell.execute_reply": "2024-03-22T21:10:30.939890Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.070949, + "end_time": "2024-03-22T21:10:30.943402", + "exception": false, + "start_time": "2024-03-22T21:10:22.872453", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T21:10:30.970225Z", + "iopub.status.busy": "2024-03-22T21:10:30.969883Z", + "iopub.status.idle": "2024-03-22T21:10:30.976588Z", + "shell.execute_reply": "2024-03-22T21:10:30.975784Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022337, + "end_time": "2024-03-22T21:10:30.978455", + "exception": false, + "start_time": "2024-03-22T21:10:30.956118", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:31.002771Z", + "iopub.status.busy": "2024-03-22T21:10:31.002497Z", + "iopub.status.idle": "2024-03-22T21:10:31.006952Z", + "shell.execute_reply": "2024-03-22T21:10:31.006137Z" + }, + "papermill": { + "duration": 0.018837, + "end_time": "2024-03-22T21:10:31.008840", + "exception": false, + "start_time": "2024-03-22T21:10:30.990003", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:31.033115Z", + "iopub.status.busy": "2024-03-22T21:10:31.032802Z", + "iopub.status.idle": "2024-03-22T21:10:31.509372Z", + "shell.execute_reply": "2024-03-22T21:10:31.508461Z" + }, + "papermill": { + "duration": 0.490925, + "end_time": "2024-03-22T21:10:31.511295", + "exception": false, + "start_time": "2024-03-22T21:10:31.020370", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:31.538862Z", + "iopub.status.busy": "2024-03-22T21:10:31.538167Z", + "iopub.status.idle": "2024-03-22T21:10:31.860807Z", + "shell.execute_reply": "2024-03-22T21:10:31.859916Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.338706, + "end_time": "2024-03-22T21:10:31.862904", + "exception": false, + "start_time": "2024-03-22T21:10:31.524198", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:31.890343Z", + "iopub.status.busy": "2024-03-22T21:10:31.890059Z", + "iopub.status.idle": "2024-03-22T21:10:31.993523Z", + "shell.execute_reply": "2024-03-22T21:10:31.992637Z" + }, + "papermill": { + "duration": 0.119468, + "end_time": "2024-03-22T21:10:31.995534", + "exception": false, + "start_time": "2024-03-22T21:10:31.876066", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T21:10:32.023805Z", + "iopub.status.busy": "2024-03-22T21:10:32.023522Z", + "iopub.status.idle": "2024-03-22T21:10:32.441608Z", + "shell.execute_reply": "2024-03-22T21:10:32.440687Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.43467, + "end_time": "2024-03-22T21:10:32.443626", + "exception": false, + "start_time": "2024-03-22T21:10:32.008956", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:32.472096Z", + "iopub.status.busy": "2024-03-22T21:10:32.471754Z", + "iopub.status.idle": "2024-03-22T21:10:32.475764Z", + "shell.execute_reply": "2024-03-22T21:10:32.474946Z" + }, + "papermill": { + "duration": 0.020448, + "end_time": "2024-03-22T21:10:32.477589", + "exception": false, + "start_time": "2024-03-22T21:10:32.457141", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:32.503675Z", + "iopub.status.busy": "2024-03-22T21:10:32.503407Z", + "iopub.status.idle": "2024-03-22T21:10:32.510124Z", + "shell.execute_reply": "2024-03-22T21:10:32.509290Z" + }, + "papermill": { + "duration": 0.021921, + "end_time": "2024-03-22T21:10:32.512042", + "exception": false, + "start_time": "2024-03-22T21:10:32.490121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9638537" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:32.538436Z", + "iopub.status.busy": "2024-03-22T21:10:32.538171Z", + "iopub.status.idle": "2024-03-22T21:10:32.629566Z", + "shell.execute_reply": "2024-03-22T21:10:32.628706Z" + }, + "papermill": { + "duration": 0.107145, + "end_time": "2024-03-22T21:10:32.631711", + "exception": false, + "start_time": "2024-03-22T21:10:32.524566", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 36] --\n", + "├─Adapter: 1-1 [2, 1071, 36] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 37,888\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 36] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,638,537\n", + "Trainable params: 9,638,537\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.18\n", + "========================================================================================================================\n", + "Input size (MB): 0.39\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.55\n", + "Estimated Total Size (MB): 346.43\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:32.661713Z", + "iopub.status.busy": "2024-03-22T21:10:32.661417Z", + "iopub.status.idle": "2024-03-22T21:50:16.858505Z", + "shell.execute_reply": "2024-03-22T21:50:16.857483Z" + }, + "papermill": { + "duration": 2384.230615, + "end_time": "2024-03-22T21:50:16.876809", + "exception": false, + "start_time": "2024-03-22T21:10:32.646194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03318693633812169, 'avg_role_model_std_loss': 0.5761553976163866, 'avg_role_model_mean_pred_loss': 0.009452911170182633, 'avg_role_model_g_mag_loss': 0.5073339222537147, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03357576689010279, 'n_size': 900, 'n_batch': 113, 'duration': 140.07320928573608, 'duration_batch': 1.2395859228826203, 'duration_size': 0.1556368992063734, 'avg_pred_std': 0.1377185063222341}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011029714790444511, 'avg_role_model_std_loss': 0.10201651341138354, 'avg_role_model_mean_pred_loss': 0.0008927303574836721, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011029714790444511, 'n_size': 450, 'n_batch': 57, 'duration': 44.65146517753601, 'duration_batch': 0.7833590382023862, 'duration_size': 0.09922547817230225, 'avg_pred_std': 0.09806239102934405}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008319700596491909, 'avg_role_model_std_loss': 1.3899801381037156, 'avg_role_model_mean_pred_loss': 0.00032640089475284936, 'avg_role_model_g_mag_loss': 0.08519980923169189, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008420131139250265, 'n_size': 900, 'n_batch': 113, 'duration': 141.50769209861755, 'duration_batch': 1.252280461049713, 'duration_size': 0.15723076899846394, 'avg_pred_std': 0.08328526641810889}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0042552733277746784, 'avg_role_model_std_loss': 0.797521198754859, 'avg_role_model_mean_pred_loss': 4.676108655203848e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0042552733277746784, 'n_size': 450, 'n_batch': 57, 'duration': 44.46026659011841, 'duration_batch': 0.7800046770196212, 'duration_size': 0.09880059242248534, 'avg_pred_std': 0.05082482615845245}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0030213647201243372, 'avg_role_model_std_loss': 0.4467241249097572, 'avg_role_model_mean_pred_loss': 2.9881127188979347e-05, 'avg_role_model_g_mag_loss': 0.03057058709777064, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003060984404421308, 'n_size': 900, 'n_batch': 113, 'duration': 141.14445447921753, 'duration_batch': 1.249065968842633, 'duration_size': 0.15682717164357504, 'avg_pred_std': 0.09463177754881635}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004963098757434637, 'avg_role_model_std_loss': 0.23140327160492807, 'avg_role_model_mean_pred_loss': 0.0003732240848752491, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004963098757434637, 'n_size': 450, 'n_batch': 57, 'duration': 44.70943355560303, 'duration_batch': 0.7843760272912812, 'duration_size': 0.09935429679022895, 'avg_pred_std': 0.07549247669317481}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017553741914970386, 'avg_role_model_std_loss': 0.2178003895808945, 'avg_role_model_mean_pred_loss': 4.943996762005542e-06, 'avg_role_model_g_mag_loss': 0.02547209452009863, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001773931053143719, 'n_size': 900, 'n_batch': 113, 'duration': 141.30070185661316, 'duration_batch': 1.2504486889965767, 'duration_size': 0.15700077984068128, 'avg_pred_std': 0.09433466191939284}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002269126343291848, 'avg_role_model_std_loss': 0.25996375590649135, 'avg_role_model_mean_pred_loss': 0.000101374774749564, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002269126343291848, 'n_size': 450, 'n_batch': 57, 'duration': 44.660250663757324, 'duration_batch': 0.7835131695396022, 'duration_size': 0.09924500147501628, 'avg_pred_std': 0.06720550957119517}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011990225297955073, 'avg_role_model_std_loss': 0.19844268139087864, 'avg_role_model_mean_pred_loss': 6.492888718708172e-06, 'avg_role_model_g_mag_loss': 0.02050420185758008, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012119675411183077, 'n_size': 900, 'n_batch': 113, 'duration': 141.14717173576355, 'duration_batch': 1.2490900153607394, 'duration_size': 0.15683019081751506, 'avg_pred_std': 0.09565485499601449}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021288806346031683, 'avg_role_model_std_loss': 0.3498717648343612, 'avg_role_model_mean_pred_loss': 6.43633937075372e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021288806346031683, 'n_size': 450, 'n_batch': 57, 'duration': 44.87625551223755, 'duration_batch': 0.7873027282848692, 'duration_size': 0.09972501224941678, 'avg_pred_std': 0.06606765008090358}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010727717719338317, 'avg_role_model_std_loss': 0.1799432011435814, 'avg_role_model_mean_pred_loss': 1.986284879775273e-06, 'avg_role_model_g_mag_loss': 0.019064169105970197, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010836002508100744, 'n_size': 900, 'n_batch': 113, 'duration': 144.71740865707397, 'duration_batch': 1.2806850323634864, 'duration_size': 0.1607971207300822, 'avg_pred_std': 0.0960029606352997}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025687832964791193, 'avg_role_model_std_loss': 0.24147355027886128, 'avg_role_model_mean_pred_loss': 0.00013711076116941693, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025687832964791193, 'n_size': 450, 'n_batch': 57, 'duration': 44.92388701438904, 'duration_batch': 0.7881383686734919, 'duration_size': 0.09983086003197564, 'avg_pred_std': 0.06918113802053165}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008831181887621319, 'avg_role_model_std_loss': 0.1315059885362297, 'avg_role_model_mean_pred_loss': 1.4708352961886883e-06, 'avg_role_model_g_mag_loss': 0.01531516697154277, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008934507078892138, 'n_size': 900, 'n_batch': 113, 'duration': 141.82023167610168, 'duration_batch': 1.255046298018599, 'duration_size': 0.15757803519566854, 'avg_pred_std': 0.09242146539674924}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017530873860091055, 'avg_role_model_std_loss': 0.304231248577451, 'avg_role_model_mean_pred_loss': 7.560512728054956e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017530873860091055, 'n_size': 450, 'n_batch': 57, 'duration': 44.82837462425232, 'duration_batch': 0.7864627127061811, 'duration_size': 0.09961861027611627, 'avg_pred_std': 0.06672564117858808}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005385009764318562, 'avg_role_model_std_loss': 0.07839578661780303, 'avg_role_model_mean_pred_loss': 7.117195005359047e-07, 'avg_role_model_g_mag_loss': 0.012112246143321197, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005440399935145655, 'n_size': 900, 'n_batch': 113, 'duration': 141.7691547870636, 'duration_batch': 1.2545942901510052, 'duration_size': 0.15752128309673732, 'avg_pred_std': 0.09718591164368971}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023827175304500592, 'avg_role_model_std_loss': 2.154619756286098, 'avg_role_model_mean_pred_loss': 0.00013736589136319527, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023827175304500592, 'n_size': 450, 'n_batch': 57, 'duration': 46.83988165855408, 'duration_batch': 0.8217523097991943, 'duration_size': 0.10408862590789796, 'avg_pred_std': 0.06465611442723584}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004240491726927252, 'avg_role_model_std_loss': 0.046745863687660795, 'avg_role_model_mean_pred_loss': 1.4920491366082138e-07, 'avg_role_model_g_mag_loss': 0.010413902575771013, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004284947737728039, 'n_size': 900, 'n_batch': 113, 'duration': 147.48687982559204, 'duration_batch': 1.3051936267751507, 'duration_size': 0.1638743109173245, 'avg_pred_std': 0.0963774272529161}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019166796955202396, 'avg_role_model_std_loss': 0.40897835704338553, 'avg_role_model_mean_pred_loss': 4.469346806295368e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019166796955202396, 'n_size': 450, 'n_batch': 57, 'duration': 45.08631491661072, 'duration_batch': 0.7909879809931705, 'duration_size': 0.1001918109258016, 'avg_pred_std': 0.0743747265813382}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005618307212070148, 'avg_role_model_std_loss': 0.06497306443851963, 'avg_role_model_mean_pred_loss': 5.931063866807492e-07, 'avg_role_model_g_mag_loss': 0.012489941705846124, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005679361823422369, 'n_size': 900, 'n_batch': 113, 'duration': 142.84351229667664, 'duration_batch': 1.2641018787316516, 'duration_size': 0.15871501366297405, 'avg_pred_std': 0.09641832253376467}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017379091629603257, 'avg_role_model_std_loss': 0.4798262932258854, 'avg_role_model_mean_pred_loss': 2.2975261447157275e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017379091629603257, 'n_size': 450, 'n_batch': 57, 'duration': 46.893903970718384, 'duration_batch': 0.822700069661726, 'duration_size': 0.10420867549048529, 'avg_pred_std': 0.06686989114856706}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00031843584507846066, 'avg_role_model_std_loss': 0.034923846742066104, 'avg_role_model_mean_pred_loss': 2.1399565589458585e-07, 'avg_role_model_g_mag_loss': 0.009167301410602199, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00032186484126011945, 'n_size': 900, 'n_batch': 113, 'duration': 144.4654381275177, 'duration_batch': 1.2784552046682982, 'duration_size': 0.16051715347501966, 'avg_pred_std': 0.10122920503526663}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017598816551940722, 'avg_role_model_std_loss': 0.5481770304940762, 'avg_role_model_mean_pred_loss': 4.9227021056914934e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017598816551940722, 'n_size': 450, 'n_batch': 57, 'duration': 45.89448928833008, 'duration_batch': 0.8051664787426329, 'duration_size': 0.10198775397406684, 'avg_pred_std': 0.0716182768922871}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0002415686168306921, 'avg_role_model_std_loss': 0.022000115441331264, 'avg_role_model_mean_pred_loss': 3.5452563263274956e-08, 'avg_role_model_g_mag_loss': 0.007707212487649586, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00024431025462238017, 'n_size': 900, 'n_batch': 113, 'duration': 141.62232375144958, 'duration_batch': 1.253294900455306, 'duration_size': 0.15735813750161065, 'avg_pred_std': 0.09990070720689487}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001438923770741288, 'avg_role_model_std_loss': 0.40616633773342614, 'avg_role_model_mean_pred_loss': 1.6548902490660263e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001438923770741288, 'n_size': 450, 'n_batch': 57, 'duration': 44.89892363548279, 'duration_batch': 0.7877004146575928, 'duration_size': 0.09977538585662842, 'avg_pred_std': 0.07267187253098216}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0004335116477921561, 'avg_g_mag_loss': 0.01439704919715214, 'avg_g_cos_loss': 0.004639643085016966, 'pred_duration': 2.322702407836914, 'grad_duration': 6.45476222038269, 'total_duration': 8.777464628219604, 'pred_std': 0.14727535843849182, 'std_loss': 2.352470396260742e-08, 'mean_pred_loss': 1.2653148928620794e-07, 'pred_rmse': 0.020820941776037216, 'pred_mae': 0.01267878245562315, 'pred_mape': 0.9071698188781738, 'grad_rmse': 0.008567946963012218, 'grad_mae': 0.004596114624291658, 'grad_mape': 0.3815285265445709}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0004335116477921561, 'avg_g_mag_loss': 0.01439704919715214, 'avg_g_cos_loss': 0.004639643085016966, 'avg_pred_duration': 2.322702407836914, 'avg_grad_duration': 6.45476222038269, 'avg_total_duration': 8.777464628219604, 'avg_pred_std': 0.14727535843849182, 'avg_std_loss': 2.352470396260742e-08, 'avg_mean_pred_loss': 1.2653148928620794e-07}, 'min_metrics': {'avg_loss': 0.0004335116477921561, 'avg_g_mag_loss': 0.01439704919715214, 'avg_g_cos_loss': 0.004639643085016966, 'pred_duration': 2.322702407836914, 'grad_duration': 6.45476222038269, 'total_duration': 8.777464628219604, 'pred_std': 0.14727535843849182, 'std_loss': 2.352470396260742e-08, 'mean_pred_loss': 1.2653148928620794e-07, 'pred_rmse': 0.020820941776037216, 'pred_mae': 0.01267878245562315, 'pred_mape': 0.9071698188781738, 'grad_rmse': 0.008567946963012218, 'grad_mae': 0.004596114624291658, 'grad_mape': 0.3815285265445709}, 'model_metrics': {'tvae': {'avg_loss': 0.0004335116477921561, 'avg_g_mag_loss': 0.01439704919715214, 'avg_g_cos_loss': 0.004639643085016966, 'pred_duration': 2.322702407836914, 'grad_duration': 6.45476222038269, 'total_duration': 8.777464628219604, 'pred_std': 0.14727535843849182, 'std_loss': 2.352470396260742e-08, 'mean_pred_loss': 1.2653148928620794e-07, 'pred_rmse': 0.020820941776037216, 'pred_mae': 0.01267878245562315, 'pred_mape': 0.9071698188781738, 'grad_rmse': 0.008567946963012218, 'grad_mae': 0.004596114624291658, 'grad_mape': 0.3815285265445709}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:50:16.910779Z", + "iopub.status.busy": "2024-03-22T21:50:16.910443Z", + "iopub.status.idle": "2024-03-22T21:50:16.914868Z", + "shell.execute_reply": "2024-03-22T21:50:16.914003Z" + }, + "papermill": { + "duration": 0.023507, + "end_time": "2024-03-22T21:50:16.916813", + "exception": false, + "start_time": "2024-03-22T21:50:16.893306", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:50:16.949358Z", + "iopub.status.busy": "2024-03-22T21:50:16.948812Z", + "iopub.status.idle": "2024-03-22T21:50:17.027456Z", + "shell.execute_reply": "2024-03-22T21:50:17.026679Z" + }, + "papermill": { + "duration": 0.097375, + "end_time": "2024-03-22T21:50:17.029677", + "exception": false, + "start_time": "2024-03-22T21:50:16.932302", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:50:17.063805Z", + "iopub.status.busy": "2024-03-22T21:50:17.063515Z", + "iopub.status.idle": "2024-03-22T21:50:17.349423Z", + "shell.execute_reply": "2024-03-22T21:50:17.348532Z" + }, + "papermill": { + "duration": 0.305267, + "end_time": "2024-03-22T21:50:17.351502", + "exception": false, + "start_time": "2024-03-22T21:50:17.046235", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAEWCAYAAAAzXsGAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDYklEQVR4nO3de1xUdf4/8Nfch/sIyGUMQU3CFPHKhJXWylcyWqN1w1hXzbW0/WIXqc1oU7rshpWlm/rLbF2tLdbLd62+i2Yhaq2CYICrpvJVQzBhQCTul7m9f3+MHBkYYAaG+/v5eJzHMOe8zzmfmZx3n/M5n/P5iIiIwBhjA4S4rwvAGGP24KTFGBtQOGkxxgYUTlqMsQGFkxZjbEDhpMUYG1A4aTHGBhROWoyxAYWTFmNsQOGkxRgbULqUtLZs2YKgoCAolUpoNBpkZ2d3GL93716EhIRAqVQiNDQUBw4csNj+6quvIiQkBC4uLhg2bBgiIyORlZVlERMUFASRSGSxrFu3rivFZ4wNYFJ7d9i9ezcSEhKwdetWaDQabNy4EVFRUcjPz4ePj0+b+IyMDMTFxSE5ORkPPfQQUlJSEBMTg9zcXEyYMAEAEBwcjM2bN2P06NFoaGjAhg0bMGfOHFy6dAnDhw8XjvX666/jySefFN67ubnZXG6TyYTi4mK4ublBJBLZ+7EZYz2MiFBTUwO1Wg2xuIP6FNkpPDyc4uPjhfdGo5HUajUlJydbjY+NjaXo6GiLdRqNhlasWNHuOaqqqggAHTp0SFgXGBhIGzZssLe4gqtXrxIAXnjhpZ8vV69e7fC3bFdNS6fTIScnB4mJicI6sViMyMhIZGZmWt0nMzMTCQkJFuuioqLwxRdftHuObdu2wcPDA2FhYRbb1q1bhzfeeAMjR47Eb37zG6xatQpSqfWP0NTUhKamJuE93RzM4urVq3B3d+/0szLGeld1dTUCAgI6vYKyK2mVl5fDaDTC19fXYr2vry8uXLhgdR+tVms1XqvVWqxLTU3FY489hvr6evj7+yMtLQ3e3t7C9meeeQZTpkyBp6cnMjIykJiYiJKSErz33ntWz5ucnIzXXnutzXp3d3dOWoz1Y50139jdptVT7r//fpw6dQrl5eX46KOPEBsbi6ysLKGdrGVtbeLEiZDL5VixYgWSk5OhUCjaHC8xMdFin+Yszhgb2Oy6e+jt7Q2JRILS0lKL9aWlpfDz87O6j5+fn03xLi4uuP3223HXXXdh+/btkEql2L59e7tl0Wg0MBgMuHLlitXtCoVCqFVx7YqxwcOupCWXyzF16lSkp6cL60wmE9LT0xEREWF1n4iICIt4AEhLS2s3vuVxW7ZJtXbq1CmIxWKrdywZY4OX3ZeHCQkJWLJkCaZNm4bw8HBs3LgRdXV1WLp0KQBg8eLFGDFiBJKTkwEAzz77LGbNmoV3330X0dHR2LVrF77//nts27YNAFBXV4c///nPmDdvHvz9/VFeXo4tW7bg2rVrePTRRwGYG/OzsrJw//33w83NDZmZmVi1ahV++9vfYtiwYY76LlgHjEYj9Hp9XxeDDWAymQwSiaTbx7E7aS1YsADXr1/H2rVrodVqMWnSJBw8eFBobC8qKrLoYzFjxgykpKTglVdewcsvv4yxY8fiiy++EPpoSSQSXLhwAR9//DHKy8vh5eWF6dOn49///jfGjx8PwHypt2vXLrz66qtoamrCqFGjsGrVqjZ3JZnjERG0Wi0qKyv7uihsEFCpVPDz8+tWX0kR0dCY2KK6uhoeHh6oqqri9i07lJSUoLKyEj4+PnB2duaOuaxLiAj19fUoKyuDSqWCv79/mxhbf6P95u5hf1HdqMf54mqYCIgY49XXxelTRqNRSFheXkP7u2Dd5+TkBAAoKyuDj49Ply8V+YHpVr6/UoEF207gT/vP9XVR+lxzG5azs3Mfl4QNFs3/lrrTPspJq5URKvOXeq2yoY9L0n/wJSFzFEf8W+Kk1cqIYeYqbGW9HnVNhj4uDWOsNU5arbgqpPBwkgHg2hZj/REnLStGqMy1rZ9+ru/jkrCBSiQStTsowED16quvYtKkSX1dDE5a1jRfIl77mWtabODauXMnVCqVw473wgsvtHm6pS9wlwcrhJoWXx6yIUCn00Eul3ca5+rqCldX114oUce4pmXFbVzTsoqIUK8z9Mlibx/ogwcP4p577oFKpYKXlxceeughXL58GYD5KY3Vq1dbxF+/fh0ymQzfffcdAHOn2ujoaDg5OWHUqFFISUlBUFAQNm7c2KXv7syZM/jFL34BJycneHl5Yfny5aitrRW2Hz16FOHh4XBxcYFKpcLdd9+NwsJCAMB//vMf4RE2d3d3TJ06Fd9//32H5zt69CiWLl2KqqoqYXjyV199FYB56PI33ngDixcvhru7O5YvXw4AWL16NYKDg+Hs7IzRo0djzZo1Fl0TWl8ePv7444iJicH69evh7+8PLy8vxMfH9/jjXlzTsqK5psUN8ZYa9EbcufbrPjn3udej4Cy3/Z9rXV0dEhISMHHiRNTW1mLt2rV45JFHcOrUKSxcuBBvv/021q1bJ9yC3717N9RqNe69914A5mdoy8vLcfToUchkMiQkJKCsrKxLZa+rq0NUVBQiIiJw8uRJlJWV4YknnsDKlSuxc+dOGAwGxMTE4Mknn8Q//vEP6HQ6ZGdnC2VbuHAhJk+ejA8++AASiQSnTp2CTCbr8JwzZszAxo0bsXbtWuTn5wOARS1p/fr1WLt2LZKSkoR1bm5u2LlzJ9RqNc6cOYMnn3wSbm5uePHFF9s9z5EjR+Dv748jR47g0qVLWLBgASZNmmQxLLqjcdKygtu0Br758+dbvP/b3/6G4cOH49y5c4iNjcVzzz2HY8eOCUkqJSUFcXFxEIlEuHDhAg4dOoSTJ09i2rRpAIC//vWvGDt2bJfKkpKSgsbGRnzyySdwcXEBAGzevBm//OUv8dZbb0Emk6GqqgoPPfQQxowZAwAYN26csH9RURH+8Ic/ICQkBABsKodcLoeHhwdEIpHVYaN+8Ytf4Pnnn7dY98orrwh/BwUF4YUXXsCuXbs6TFrDhg3D5s2bIZFIEBISgujoaKSnp3PS6m3NNa2ymiY0GYxQSLv/ZPpg4CST4NzrUX12bntcvHgRa9euRVZWFsrLy2EymQCYE8CECRMwZ84cfPbZZ7j33ntRUFCAzMxMfPjhhwCA/Px8SKVSTJkyRTje7bff3uURRc6fP4+wsDAhYQHA3XffDZPJhPz8fMycOROPP/44oqKi8F//9V+IjIxEbGys8HxeQkICnnjiCfz9739HZGQkHn30USG5dVVzMm5p9+7deP/993H58mXU1tbCYDB0+pzu+PHjLR7H8ff3x5kzZ7pVts5wm5YVni5yKGXmr6aksrGPS9N/iEQiOMulfbLY25P6l7/8JSoqKvDRRx8hKytLmJJOp9MBMF9y/c///A/0ej1SUlIQGhqK0NBQh39nttqxYwcyMzMxY8YM7N69G8HBwThx4gQAc1vSDz/8gOjoaBw+fBh33nknPv/8826dr2UCBczDPy1cuBAPPvggUlNTkZeXhz/+8Y/C99We1pepIpFI+B9ET+GkZYVIJOJ2rQHsxo0byM/PxyuvvILZs2dj3Lhx+Pnnny1iHn74YTQ2NuLgwYNISUnBwoULhW133HEHDAYD8vLyhHWXLl1qcwxbjRs3Dv/5z39QV1cnrDt+/DjEYjHuuOMOYd3kyZORmJiIjIwMTJgwASkpKcK24OBgrFq1Ct988w1+9atfYceOHZ2eVy6Xw2g02lTGjIwMBAYG4o9//COmTZuGsWPHCjcC+htOWu0YMezmM4jcrjXgDBs2DF5eXti2bRsuXbqEw4cPtxl7zcXFBTExMVizZg3Onz+PuLg4YVtISAgiIyOxfPlyZGdnIy8vD8uXL4eTk1OXnp1buHAhlEollixZgrNnz+LIkSN4+umnsWjRIvj6+qKgoACJiYnIzMxEYWEhvvnmG1y8eBHjxo1DQ0MDVq5ciaNHj6KwsBDHjx/HyZMnLdq82hMUFITa2lqkp6ejvLwc9fXtd5YeO3YsioqKsGvXLly+fBnvv/9+t2tzPYWTVju4r9bAJRaLsWvXLuTk5GDChAlYtWoV3nnnnTZxCxcuxH/+8x/ce++9GDlypMW2Tz75BL6+vpg5cyYeeeQR4U6aUqm0uzzOzs74+uuvUVFRgenTp+PXv/41Zs+ejc2bNwvbL1y4gPnz5yM4OBjLly9HfHw8VqxYAYlEghs3bmDx4sUIDg5GbGws5s6da3WmqdZmzJiBp556CgsWLMDw4cPx9ttvtxs7b948rFq1CitXrsSkSZOQkZGBNWvW2P1ZewMPAtiOLUcu4Z2v8zF/ym14Nzas0/jBqLGxEQUFBRg1alSXfqyDyU8//YSAgAAcOnQIs2fP7uviDFgd/ZviQQC76VabFj9/OBQdPnwYtbW1CA0NRUlJCV588UUEBQVh5syZfV20IY8vD9sh9NXiy8MhSa/X4+WXX8b48ePxyCOPYPjw4UJH088++0x4pKX10jyvQW+YO3duu+V48803e60cvY1rWu1ormmVVDbCaCJIxDwQ3lASFRWFqCjrfdLmzZsHjUZjdVtnPdUd6a9//SsaGqz/T9XT07PXytHbOGm1w9ddCalYBIOJUFbTCH8Pp74uEusn3Nzc4Obm1tfFwIgRI/q6CH2iS5eHW7ZsQVBQEJRKJTQaDbKzszuM37t3L0JCQqBUKhEaGooDBw5YbH/11VcREhICFxcXDBs2DJGRkUJnwGYVFRVYuHAh3N3doVKpsGzZMosHTh1NIhbBz8PcUMjdHhjrP+xOWrt370ZCQgKSkpKQm5uLsLAwREVFtfswaUZGBuLi4rBs2TLk5eUhJiYGMTExOHv2rBATHByMzZs348yZMzh27BiCgoIwZ84cXL9+XYhZuHAhfvjhB6SlpSE1NRXfffed8HR6T+EOpoz1Q2Sn8PBwio+PF94bjUZSq9WUnJxsNT42Npaio6Mt1mk0GlqxYkW756iqqiIAdOjQISIiOnfuHAGgkydPCjFfffUViUQiunbtmtVjNDY2UlVVlbBcvXqVAFBVVZXNn3XV7jwKXJ1Kmw9ftHmfwaShoYHOnTtHDQ0NfV0UNkh09G+q+Xff2W/UrpqWTqdDTk4OIiMjhXVisRiRkZHIzMy0uk9mZqZFPGBu5GwvXqfTYdu2bfDw8EBYWJhwDJVKZfGQZ2RkJMRicZvLyGbJycnw8PAQloCAAHs+KgDgNq5pMdbv2JW0ysvLYTQa4evra7He19cXWq3W6j5ardam+NTUVLi6ukKpVGLDhg1IS0uDt7e3cAwfHx+LeKlUCk9Pz3bPm5iYiKqqKmG5evWqPR8VAHAbP8rDWL/Tb/pp3X///Th16hQyMjLwwAMPIDY2tsuDrgGAQqGAu7u7xWIv7qvFumowTmzRX9iVtLy9vSGRSFBaWmqxvrS01OpAYwDg5+dnU7yLiwtuv/123HXXXdi+fTukUim2b98uHKN1AjMYDKioqGj3vI4gNMT/3GD3cL+M9TVHT2wBmIdxFolEqKysdOhx7WFX0pLL5Zg6darFjBwmkwnp6emIiIiwuk9ERESbGTzS0tLajW953KamJuEYlZWVyMnJEbYfPnwYJpOp3U5+juCvMnd5aNAb8XN9z457zRizkb2t/7t27SKFQkE7d+6kc+fO0fLly0mlUpFWqyUiokWLFtFLL70kxB8/fpykUimtX7+ezp8/T0lJSSSTyejMmTNERFRbW0uJiYmUmZlJV65coe+//56WLl1KCoWCzp49KxzngQceoMmTJ1NWVhYdO3aMxo4dS3FxcTaX29Y7E61N/1MaBa5OpdNXK+3abzBoc6fHZCJqqu2bxWSyq+xfffUV3X333eTh4UGenp4UHR1Nly5dIiKiiIgIevHFFy3iy8rKSCqV0rfffktERMXFxfTggw+SUqmkoKAg+uyzzygwMJA2bNhg0/kB0Oeffy68P336NN1///2kVCrJ09OTnnzySaqpqRG2HzlyhKZPn07Ozs7k4eFBM2bMoCtXrhAR0alTp+i+++4jV1dXcnNzoylTpljcSbfmyJEjBMBiSUpKIiLznfXnn3+e1Go1OTs7U3h4OB05ckTY98qVK/TQQw+RSqUiZ2dnuvPOO2n//v1UUFDQ5phLliyx6fto5oi7h3b3iF+wYAGuX7+OtWvXQqvVYtKkSTh48KDQ2F5UVASx+FYFbsaMGUhJScErr7yCl19+GWPHjsUXX3yBCRMmAAAkEgkuXLiAjz/+GOXl5fDy8sL06dPx73//2+I5rs8++wwrV67E7NmzIRaLMX/+fLz//vtdStT2GDHMCWU1Tfjp53qE3ubR4+fr1/T1wJvqvjn3y8WA3KXzuJt4Yov2J7ZYuXIlzp07h127dkGtVuPzzz/HAw88gDNnzmDs2LGIj4+HTqfDd999BxcXF5w7dw6urq4ICAjAP//5T8yfPx/5+flwd3eHk1PvPynSpcd4Vq5ciZUrV1rddvTo0TbrHn30UTz66KNW45VKJfbt29fpOT09PS1GcuwtI1ROyCuq5Mb4AYYntrA+sUVRURF27NiBoqIiqNXm/wG98MILOHjwIHbs2IE333wTRUVFmD9/vjD89OjRo4X9m59p9PHxcXh7ma342cNONN9B/Im7PQAyZ3ONp6/ObQee2MK6M2fOwGg0Ijg42GJ9U1MTvLy8AADPPPMMfv/73+Obb75BZGQk5s+fj4kTJ3bpfD2h33R56K+4g2kLIpH5Eq0vFp7YwiETW9TW1kIikSAnJwenTp0SlvPnz+Mvf/kLAOCJJ57Ajz/+iEWLFuHMmTOYNm0aNm3a5LDP2l2ctDrBcyAOPDyxhZm1iS0mT54Mo9GIsrIy3H777RZLy8vIgIAAPPXUU9i3bx+ef/55fPTRR8IxAdg8YUZP4KTViRGqm73iuaY1YPDEFmbWJrYIDg7GwoULsXjxYuzbtw8FBQXIzs5GcnIy9u/fDwB47rnn8PXXX6OgoAC5ubk4cuSIcL7AwECIRCKkpqbi+vXrPTrSSrvsul85gHW1y0NNo54CV6dS4OpUqmnU91Dp+qeB/MB0WloajRs3jhQKBU2cOJGOHj3aphvCgQMHCADNnDmzzf7FxcU0d+5cUigUFBgYSCkpKeTj40Nbt2616fytz9VRlwetVksxMTHk7+9PcrmcAgMDae3atWQ0GqmpqYkee+wxCggIILlcTmq1mlauXGnzf5OnnnqKvLy8LLo86HQ6Wrt2LQUFBZFMJiN/f3965JFH6PTp00REtHLlShozZgwpFAoaPnw4LVq0iMrLy4Vjvv766+Tn50cikahPujzwxBY2CHvtG1Q16PH1czNxh1/fD/7WW3hii1t4YgvH4IkteskIlROqGvS4Vlk/pJLWUMYTW/Rf3KZlA26MH3p4Yov+i2taNuCJW4centii/+KkZYPbuKbFWuCJLfoWXx7aYKiPFT9E7tWwXuCIf0uctGwwVNu0mi916ut5lm3mGM3/lrpzGc2XhzZormmV1TShyWCEQirp4xL1DolEApVKJYxu4Ozs3KXOlYwREerr61FWVgaVSgWJpOu/IU5aNvB0kUMpE6NRb0JJZSOCvG0fImWga360oztDXzPWTKVSdXu0YU5aNhCJRBihcsLl63W4VtkwpJKWSCSCv78/fHx8oNfz6K2s62QyWbdqWM04adloxDBnc9IaYu1azSQSiUP+wTHWXdwQbyPuq8VY/8BJy0bcV4ux/oGTlo1u9dXi2/+M9SVOWjbiiVsZ6x+6lLS2bNmCoKAgKJVKaDQaZGdndxi/d+9ehISEQKlUIjQ0FAcOHBC26fV6rF69GqGhoXBxcYFarcbixYtRXGw5FnlQUBBEIpHFsm7duq4Uv0uaLw9LKhthNHEPccb6it1Ja/fu3UhISEBSUhJyc3MRFhaGqKiodvvxZGRkIC4uDsuWLUNeXh5iYmIQExODs2fPAjD3kM3NzcWaNWuQm5uLffv2IT8/H/PmzWtzrNdffx0lJSXC8vTTT9tb/C7zcVNCKhbBYCKU1TT22nkZY63YNewgEYWHh1N8fLzw3mg0klqtpuTkZKvxsbGxFB0dbbFOo9HQihUr2j1HdnY2AaDCwkJhnT0TZVrT1ZFLW7rnrXQKXJ1KJwtudPkYjDHrbP2N2lXT0ul0yMnJQWRkpLBOLBYjMjISmZmZVvfJzMy0iAfMw360Fw8AVVVVEIlEbeZVW7duHby8vDB58mS88847MBgM7R6jqakJ1dXVFkt3DfUHpxnrD+zqXFpeXg6j0SjMJt3M19cXFy5csLqPVqu1Gq/Vaq3GNzY2YvXq1YiLi7MYcvWZZ57BlClT4OnpiYyMDCQmJqKkpATvvfee1eMkJyfjtddes+fjdco8yUUFz4HIWB/qVz3i9Xo9YmNjQUT44IMPLLa1nE1l4sSJkMvlWLFiBZKTk6FQKNocKzEx0WKf6upqBAQEdKt8PHErY33PrqTl7e0NiUSC0tJSi/WlpaXtPgTp5+dnU3xzwiosLMThw4c7nXxCo9HAYDDgypUrFnPHNVMoFFaTWXfwxK2M9T272rTkcjmmTp2K9PR0YZ3JZEJ6ejoiIiKs7hMREWERDwBpaWkW8c0J6+LFizh06JAwPXdHTp06BbFYDB8fH3s+QrfcGleLO5gy1lfsvjxMSEjAkiVLMG3aNISHh2Pjxo2oq6vD0qVLAQCLFy/GiBEjkJycDAB49tlnMWvWLLz77ruIjo7Grl278P3332Pbtm0AzAnr17/+NXJzc5Gamgqj0Si0d3l6ekIulyMzMxNZWVm4//774ebmhszMTKxatQq//e1vMWzYMEd9F51q2RBPRDy2FGN9oSu3Jjdt2kQjR44kuVxO4eHhdOLECWHbrFmz2kzguGfPHgoODia5XE7jx4+n/fv3C9sKCgoIgNXlyJEjRESUk5NDGo2GPDw8SKlU0rhx4+jNN9+kxsZGm8vsiC4PjXqDMHFreY3t52aMdY4na22lO5O1thT+50Moq2nC/668GxNvUzmugIwNcbb+RvnZQzsN1fHiGesvOGnZiTuYMta3OGnZiftqMda3OGnZiftqMda3OGnZidu0GOtbnLTsZH7+kGtajPUVTlp2aq5pVTXoUdvU/igTjLGewUnLTq4KKTyczFN68yUiY72Pk1YX8CQXjPUdTlpdwI3xjPUdTlpdwBO3MtZ3OGl1AU/cyljf4aTVBfwoD2N9h5NWF3CbFmN9h5NWFzTXtMpqmtBkMPZxaRgbWjhpdYGnixxKmfmrK6nkiVsZ602ctLpAJBJxuxZjfYSTVhfdNuzmM4jcrsVYr+Kk1UXCuFpc02KsV3HS6iKhgylPJ8ZYr+Kk1UXcwZSxvtGlpLVlyxYEBQVBqVRCo9EgOzu7w/i9e/ciJCQESqUSoaGhOHDggLBNr9dj9erVCA0NhYuLC9RqNRYvXozi4mKLY1RUVGDhwoVwd3eHSqXCsmXLUFtb25XiOwQ3xDPWN+xOWrt370ZCQgKSkpKQm5uLsLAwREVFoayszGp8RkYG4uLisGzZMuTl5SEmJgYxMTE4e/YsAKC+vh65ublYs2YNcnNzsW/fPuTn52PevHkWx1m4cCF++OEHpKWlITU1Fd999x2WL1/ehY/sGM1tWtqqRhhNQ2IWNsb6B3snVAwPD6f4+HjhvdFoJLVaTcnJyVbjY2NjKTo62mKdRqOhFStWtHuO7OxsAkCFhYVERHTu3DkCQCdPnhRivvrqKxKJRHTt2jWrx2hsbKSqqiphuXr1arcna23JYDTRmMT9FLg6la79XO+QYzI2lNk6WatdNS2dToecnBxERkYK68RiMSIjI5GZmWl1n8zMTIt4AIiKimo3HgCqqqogEomgUqmEY6hUKkybNk2IiYyMhFgsRlZWltVjJCcnw8PDQ1gCAgJs/Zg2kYhF8FcpAfAlImO9ya6kVV5eDqPRCF9fX4v1vr6+0Gq1VvfRarV2xTc2NmL16tWIi4sTZpnVarXw8fGxiJNKpfD09Gz3OImJiaiqqhKWq1ev2vQZ7SG0a3FjPGO9RtrXBWhJr9cjNjYWRIQPPvigW8dSKBRQKBQOKpl15kkuKrimxVgvsitpeXt7QyKRoLS01GJ9aWkp/Pz8rO7j5+dnU3xzwiosLMThw4eFWlbzMVo39BsMBlRUVLR73t7AE7cy1vvsujyUy+WYOnUq0tPThXUmkwnp6emIiIiwuk9ERIRFPACkpaVZxDcnrIsXL+LQoUPw8vJqc4zKykrk5OQI6w4fPgyTyQSNRmPPR3AonriVsT5gbwv/rl27SKFQ0M6dO+ncuXO0fPlyUqlUpNVqiYho0aJF9NJLLwnxx48fJ6lUSuvXr6fz589TUlISyWQyOnPmDBER6XQ6mjdvHt1222106tQpKikpEZampibhOA888ABNnjyZsrKy6NixYzR27FiKi4uzudy23pmwx7GL1ylwdSr9Yv0Rhx2TsaHK1t+o3UmLiGjTpk00cuRIksvlFB4eTidOnBC2zZo1i5YsWWIRv2fPHgoODia5XE7jx4+n/fv3C9sKCgoIgNXlyJEjQtyNGzcoLi6OXF1dyd3dnZYuXUo1NTU2l7knklbB9VoKXJ1Kd7xygEwmk8OOy9hQZOtvVEREQ6JnZHV1NTw8PFBVVWXRXtYdTQYj7njlIAAg55VIeLn2bMM/Y4OZrb9RfvawGxRSCXzczImK27UY6x2ctLqJx4tnrHdx0uomfnCasd7FSaubuK8WY72Lk1Y3cV8txnoXJ61u4jYtxnoXJ61uMj9/yDUtxnoLJ61uaq5pVTXoUdtk6OPSMDb4cdLqJleFFB5OMgB8ichYb+Ck5QC3uj3wzDyM9TROWg7AjfGM9R5OWg4gzIHIjfGM9ThOWg5wG3cwZazXcNJyAJ64lbHew0nLAbivFmO9h5OWAzQ3xF+vaUKj3tjHpWFscOOk5QDDnGVwkkkAACVVjX1cGsYGN05aDiASibjbA2O9hJOWg3AHU8Z6ByctB+GaFmO9o0tJa8uWLQgKCoJSqYRGo0F2dnaH8Xv37kVISAiUSiVCQ0Nx4MABi+379u3DnDlz4OXlBZFIhFOnTrU5xn333QeRSGSxPPXUU10pfo/gDqaM9Q67k9bu3buRkJCApKQk5ObmIiwsDFFRUW1mgG6WkZGBuLg4LFu2DHl5eYiJiUFMTAzOnj0rxNTV1eGee+7BW2+91eG5n3zySZSUlAjL22+/bW/xewz31WKsl9g7N1l4eDjFx8cL741GI6nVakpOTrYaHxsbS9HR0RbrNBoNrVixok1s8xyIeXl5bbbNmjWLnn32WXuLK+iJeQ9bOllwgwJXp9Ld69J75PiMDXa2/kbtqmnpdDrk5OQgMjJSWCcWixEZGYnMzEyr+2RmZlrEA0BUVFS78R357LPP4O3tjQkTJiAxMRH19e03ejc1NaG6utpi6UnNbVraqkYYTUNiKknG+oTUnuDy8nIYjUb4+vparPf19cWFCxes7qPVaq3Ga7Vauwr6m9/8BoGBgVCr1Th9+jRWr16N/Px87Nu3z2p8cnIyXnvtNbvO0R0+bkpIxSIYTITS6kaob7ZxMcYcy66k1ZeWL18u/B0aGgp/f3/Mnj0bly9fxpgxY9rEJyYmIiEhQXhfXV2NgICAHiufRCyCv0qJqxUNuFbZwEmLsR5i1+Wht7c3JBIJSktLLdaXlpbCz8/P6j5+fn52xdtKo9EAAC5dumR1u0KhgLu7u8XS04S+WtwYz1iPsStpyeVyTJ06Fenp6cI6k8mE9PR0REREWN0nIiLCIh4A0tLS2o23VXO3CH9//24dx5H4wWnGep7dl4cJCQlYsmQJpk2bhvDwcGzcuBF1dXVYunQpAGDx4sUYMWIEkpOTAQDPPvssZs2ahXfffRfR0dHYtWsXvv/+e2zbtk04ZkVFBYqKilBcXAwAyM/PB2Cupfn5+eHy5ctISUnBgw8+CC8vL5w+fRqrVq3CzJkzMXHixG5/CY7CE7cy1gu6cmty06ZNNHLkSJLL5RQeHk4nTpwQts2aNYuWLFliEb9nzx4KDg4muVxO48ePp/3791ts37FjBwFosyQlJRERUVFREc2cOZM8PT1JoVDQ7bffTn/4wx/s6r7Q010eiIh2ZxdR4OpUWrQ9q8fOwdhgZetvVEREQ+L+fHV1NTw8PFBVVdVj7VvHL5Vj4V+zMGa4C9Kfv69HzsHYYGXrb5SfPXSgWw9NN2CI/L+AsV7HScuB/FVKAECj3oSKOl0fl4axwYmTlgMppBL4uCkA8B1ExnoKJy0H4yFqGOtZnLQcrGW7FmPM8ThpORj31WKsZ3HScrDbuKbFWI/ipOVgXNNirGdx0nIw4fnDn3mCC8Z6AictB2uuaVU3GlDTqO/j0jA2+HDScjBXhRQqZxkAbtdirCdw0uoBPK4WYz2Hk1YP4L5ajPUcTlo9gHvFM9ZzOGn1AJ64lbGew0mrB/DErYz1HE5a1pScBq6e7PLuPFY8Yz2Hk1ZreZ8CH94LHFwNdHEgv+Y2res1TWjUGx1ZOsaGPE5arY2NAiQK4FoOUHSiS4cY5iyDk0wCACipanRk6Rgb8jhpteY6HAh7zPx35uYuHUIkEvEdRMZ6CCctayJWml8v7AduXO7SIW711eJnEBlzpC4lrS1btiAoKAhKpRIajQbZ2dkdxu/duxchISFQKpUIDQ3FgQMHLLbv27cPc+bMgZeXF0QikTARa0uNjY2Ij4+Hl5cXXF1dMX/+/DYzVzvM8GAg+AEABGRu6dIhuKbFWM+wO2nt3r0bCQkJSEpKQm5uLsLCwhAVFYWysjKr8RkZGYiLi8OyZcuQl5eHmJgYxMTE4OzZs0JMXV0d7rnnHrz11lvtnnfVqlX417/+hb179+Lbb79FcXExfvWrX9lbfNs117ZOpQB1N+zenftqMdZD7J1QMTw8nOLj44X3RqOR1Go1JScnW42PjY2l6Ohoi3UajYZWrFjRJragoIAAUF5ensX6yspKkslktHfvXmHd+fPnCQBlZmZaPW9jYyNVVVUJy9WrV+2brNVkItp6L1GSO9HRt2zbp4Uv8n6iwNWpFLs1w+59GRuKbJ2s1a6alk6nQ05ODiIjI4V1YrEYkZGRyMzMtLpPZmamRTwAREVFtRtvTU5ODvR6vcVxQkJCMHLkyHaPk5ycDA8PD2EJCAiw+XwAAJEIiHja/Hf2NkBv311Afv6QsZ5hV9IqLy+H0WiEr6+vxXpfX19otVqr+2i1Wrvi2zuGXC6HSqWy+TiJiYmoqqoSlqtXr9p8PsH4GMD9NqDuOnBmj127NrdpaasaYTTxxK2MOcqgvXuoUCjg7u5usdhNIgPuesr8d8ZmwGSyeVcfNyWkYhEMJkJpNffVYsxR7Epa3t7ekEgkbe7alZaWws/Pz+o+fn5+dsW3dwydTofKyspuHadLpiwG5G5AeT5w6ZDNu0nEImHGab5EZMxx7EpacrkcU6dORXp6urDOZDIhPT0dERERVveJiIiwiAeAtLS0duOtmTp1KmQymcVx8vPzUVRUZNdxukTpAUxdYv47c5Ndu/JggIw5ntTeHRISErBkyRJMmzYN4eHh2LhxI+rq6rB06VIAwOLFizFixAgkJycDAJ599lnMmjUL7777LqKjo7Fr1y58//332LZtm3DMiooKFBUVobi4GIA5IQHmGpafnx88PDywbNkyJCQkwNPTE+7u7nj66acRERGBu+66q9tfQqc0TwEnPgAKvjM/TO0/0abdzA9OV3BNizFH6sqtyU2bNtHIkSNJLpdTeHg4nThxQtg2a9YsWrJkiUX8nj17KDg4mORyOY0fP572799vsX3Hjh0EoM2SlJQkxDQ0NNB///d/07Bhw8jZ2ZkeeeQRKikpsbnMtt5Obdfe35m7P/zzSZt3efebfApcnUov/fN0187J2BBi629URNTFoQwGmOrqanh4eKCqqqprjfLFecC2+wCxFHj2NOAxotNd9py8ihf/eRozg4fjk9+F239OxoYQW3+jg/buocOpJwOB9wAmA5C11aZdbj3Kw88fMuYonLTsMeNmZ9Ocj4Gmmk7DW3YwHSIVWsZ6HCcte4ydA3iNBZqqgNy/dxquVjnBRS5Bo96E/WdKeqGAjA1+nLTsIRYDEfHmv098ABgNHYbLpWIsnzkGAJB84AKPYsqYA3DSslfYY4CzN1BVBJz/stPw5TNHw99DiWuVDdh+rKAXCsjY4MZJy14yJyD8SfPfGZs7HUfeSS7B6gdCAAD/78gllPEjPYx1Cyetrpj+BCBVAsW5QFHno1XMC1NjUoAKdToj1n+T3wsFZGzw4qTVFS7et8aRz+j80R6xWIS1v7wTALA35yecvVbVk6VjbFDjpNVVd91skM//Cii/1Gn4lJHDMC9MDSLgjdRz3AWCsS7ipNVVw4OB4LkACDhh2zjyq+eGQCEVI6ugAl//YPt4YoyxWzhpdceMluPIl3caPkLlhBUzRwMA3jxwAU0G7gLBmL04aXVH4N2A/yTA0Aic3G7TLitmjYGPmwJFFfXYefxKjxaPscGIk1Z3iES3Hu05+ZFN48i7KKR48WYXiE2HL6G8tqknS8jYoMNJq7vufPjWOPKnd9u0y68mj0DoCA/UNhnwXtr/9XABGRtcOGl1l0QG3PV789+ZW2waR75lF4hd2UU4X1LdkyVkbFDhpOUIUxYDCveb48in2bTL9CBPRIf6w0TAn/ZzFwjGbMVJyxGU7ubEBdjU2bTZS3NDIJeKcfzSDaSftz5DN2PMEictR9E8BYgkwJV/A8WnbNolwNMZT9wzCgDw5wPnoTPYPkUZY0MVJy1HUQUAE35l/jtzs827/ff9t8PbVYGC8jp8knmlZ8rG2CDCScuRIm52Nj27D6j6yaZdXBVS/CEqGADwfvpFVNTpeqp0jA0KXUpaW7ZsQVBQEJRKJTQaDbKzszuM37t3L0JCQqBUKhEaGooDBw5YbCcirF27Fv7+/nByckJkZCQuXrxoERMUFASRSGSxrFu3rivF7znqSUDQvQAZbR5HHgB+PTUAd/q7o7rRgI2HuAsEYx2xO2nt3r0bCQkJSEpKQm5uLsLCwhAVFYWyMusNyRkZGYiLi8OyZcuQl5eHmJgYxMTE4OzZs0LM22+/jffffx9bt25FVlYWXFxcEBUVhcZGy86ar7/+OkpKSoTl6aeftrf4Pa+5tpXzMdBoW1cGiViENQ+Zu0B8llWEi6Wdjz/P2JBl79xk4eHhFB8fL7w3Go2kVqspOTnZanxsbCxFR0dbrNNoNLRixQoiIjKZTOTn50fvvPOOsL2yspIUCgX94x//ENYFBgbShg0b7C2uoNvzHtrKaCTaNM08R+LxTXbtuvyTkxS4OpUWb8/qocIx1n/Z+hu1q6al0+mQk5ODyMhIYZ1YLEZkZCQyM60PhpeZmWkRDwBRUVFCfEFBAbRarUWMh4cHNBpNm2OuW7cOXl5emDx5Mt555x0YDO2P0d7U1ITq6mqLpVe0HEc+a2un48i39PKD4yCTiPDt/13Hkfwyc0fVsgtAzk7g86eAv0wC3h0HXDnWI0VnbCCQ2hNcXl4Oo9EIX19fi/W+vr64cOGC1X20Wq3VeK1WK2xvXtdeDAA888wzmDJlCjw9PZGRkYHExESUlJTgvffes3re5ORkvPbaa/Z8PMeZ+BiQ/gZQdRU49wUQ+mubdgt0l2DNxCoUnz4K5Z4NIPkliBp+bhv490eARz68dbeSsSHErqTVlxISEoS/J06cCLlcjhUrViA5ORkKhaJNfGJiosU+1dXVCAgI6JWyQqY0jyN/NNnc2XTCfPPD1a3V3QCuZgFXTwBFJ4DiPCw26gAZACOABgBSJ+C2aUCAxrzkfgxcSAX+ZylQXWyu1Vk7NmODlF1Jy9vbGxKJBKWlpRbrS0tL4efnZ3UfPz+/DuObX0tLS+Hv728RM2nSpHbLotFoYDAYcOXKFdxxxx1ttisUCqvJrNdMfwI4tgEoOQUUHjcPY1Pxozk5NSepcit3Cl2Go8h1Ij7+yQ/58vHY/OzjULm53Np++2zgYCKQ/SHwzR/NXSui/gyIJb320RjrS3a1acnlckydOhXp6enCOpPJhPT0dERERFjdJyIiwiIeANLS0oT4UaNGwc/PzyKmuroaWVlZ7R4TAE6dOgWxWAwfHx97PkLvcfEGwuLMf+9bAawfC2yaAnz530DuJ7cSlvcd5keAYj4Ans4FXrgI9fL/wTHvx3CsIQh/OXrF8rhiCTD3LWDOn8zvsz4A9j4O6Bt665Mx1rfsbeHftWsXKRQK2rlzJ507d46WL19OKpWKtFotEREtWrSIXnrpJSH++PHjJJVKaf369XT+/HlKSkoimUxGZ86cEWLWrVtHKpWKvvzySzp9+jQ9/PDDNGrUKGpoaCAiooyMDNqwYQOdOnWKLl++TJ9++ikNHz6cFi9ebHO5e+3uYUvX/48oycN8JzHJneh1b6K/ziH6Zg3RhQNEdTfa3fW7/yujwNWpNCZxP10qq7EedHqv+ZhJ7ubjdnA8xvo7W3+jdictIqJNmzbRyJEjSS6XU3h4OJ04cULYNmvWLFqyZIlF/J49eyg4OJjkcjmNHz+e9u/fb7HdZDLRmjVryNfXlxQKBc2ePZvy8/OF7Tk5OaTRaMjDw4OUSiWNGzeO3nzzTWpsbLS5zH2StIiIzqcSHfsLUWEmka7Brl2X7cymwNWp9Lsd2e0H/fgd0ZsB5sT1/lSiioLulZexPmLrb1RENDTGRKmuroaHhweqqqrg7u7e18WxyY/XazFnw3cwmAh/XxaOe8cOtx5Ydh749NdA9U+Aiw+wcA+gnty7hWWsm2z9jfKzh/3Y6OGuWBwRBAD4U+p5GIztjALhMw54Ig3wnQDUlQE7ooGLh3qvoIz1Ik5a/dyzs8dC5SxDfmkNdp282n6guxpY+hUw+j5AXwekxAK5f++1cjLWWzhp9XMezjKsijSPAvFe2v+hulHffrDSHfjNXnPnVjIC/7sSOLoOGBotAGyI4KQ1APxGMxK3+7iiok6HzYc7mc1aKgce2Qrc+7z5/dFkc/IydpDsGBtAOGkNADKJGK9EjwMA7DhegNf/dQ4F5XXt7yASAbPXAtHvASIxkPcp8I/HgKbaXioxYz2Hk9YAcd8dPnhooj/0RsLfjhfg/vVHseRv2Th8oRQmUzuXf9OXAY+lmB8FunQI2PkgUFNqPZaxAYK7PAwgJhPhu4vX8UlmIY7klwlNVSM9nbHorkA8Ou02qJzlbXf8KcfcMF9fDqhGAr/dB3iP7d3CM9YJW3+jnLQGqMIbdfj0RCF2n7yK6kbz8DdKmRgxk0ZgcUQQ7lS3+ow3LgOf/dr8/KPTMCBuFzDyrj4oeQsGHVBZBFRcNpev+fXnAsBluPlO6Oj7gNvCzW11bFDjpNXKYEtazRp0Rnx56ho+ziy0mPR1etAwLI4IwgMT/CCT3GwFqCsHUhYA174HJApg/kfmGbJ7ktEAVBaak2XLxFRxGai8ar7L2RmZs/mB8zH3A6PvN/dL662RLYx6c+fd4jygOBeoLgGGBZlrql63A97B5u4mPNJGt3HSamWwJq1mRITvC3/GxxlXcPCsFoab7Vw+bgr8RjMSvwkfCR93JaCrB/65DMi/OU6/WAbIXQC5KyB3bvG3i3mROVu+txYncwakSqD6WtvkVFkImDoYCFHmDHiONi9eYwDPMYDnKKCiAPjxqHmpL7fcx9X3Vi1s9H3mpOEIJpO53NdyzQnqWi6gPQ0YGjveT+ZiLrv3WMBr7K2E5nU7oHB1TNmGAE5arQz2pNVSaXUjUrKKkJJdhOs1TQAAqViEuaH+WBIRiKkB7hB9/TKQvQ1AL/znlyrbJqbmVze/jmspJhNQ9gNw+Yg5gRVmAIZWI1p433GzFnYfEHQPoHDrvExE5kEamxNUcZ55vsomKyPcKtzNk5aop5inivv5ClB+Cbhx0ZxcO6otuo+4WSNrTmi3m189Asyj3DIBJ61WhlLSaqYzmHDwBy0+ybiC7wtvjYB6p787lswIxLwQNziZ6sy1L10toKu7udQC+vpbf+uaY1q817faR99gTkCeYwCv0a0Sk7/jfqCGJvPAiT8eNSey4jxYJF6xFBgx7VYSGzEVkMiA2jJzbMtaVOsaHGC+0+o/0Zyg1JOBEVPMn6G98ht05iR24yJQfvHm682EVn+j/c/RnMidPM2dghXu7bx6tF0vdxmUl6OctFoZikmrpbPXqvD3zEJ8ceoamm7OZO2mlGJSgArj/N1xp787xvm7Y/Rwl1ttYANBfYV5Vu/mJPZzgeV2uRug9DA/TN6aWAr43GlOTOop5tfh4wCJgwb0ra8AblxqkcxuLhU/AqZudPYVScy1ydZJTSwByHRzoRZ/t1gAK+tbx5I5Mbr7A27qm6/+5stwt5t/y5SO+Y5a4KTVylBPWs1+rtNhb85VfJJZiJ9+bjtwoFwixlhfV4tEdqe/OzycZX1Q2i74+Qrw47fAj0fMrw0VNzeIzI3mLROU74Qe+fF1qvnmxM8FQEOl+ZK0sbqD16pb75sTT19z8ryVxCySW4tXZ0+7aoSctFrhpGXJaCKcvVaFcyXVOF9SjXPF1bigrUFtk/VGc7WH0pzA1OZENs7fHYGezhCL+/FliskElJ4xPwngF2qukQxkROZL8faSGsj8BETzApHle1Fn72+ug8h87OoSoKb45muJeU6CmpLOb0w0kyjMTQZu/kD0evN/gw5w0mqFk1bnTCbCTz833EpkN1+t1cgAwFkuwR1+bkIS83KRw0kugbNMYn6VS+AklwrvFVIxRIOwLWZIIQIafr6ZxFomtVavrdsLf58B+I7v8NCctFrhpNV11Y16XCipEWpk57XVyNfWCG1jthKLACeZOZE5ycVwlklvJbcWic5ZLoWrQgp3JynclTK4O8luvkrhppTBXSmFu5NsYLW9DTUGHVCrvZXIxs4xt5N1gJNWK5y0HMtgNOHKjTqcK6nBueJqXCytQXWjHg16I+p1RjTobr7qjdDZmdxs5SSTtEpsUosE17x+mLMcapUSapUTvFzkXNvrpzhptcJJq+8YjCY06C0T2a3EZrCyzYDaRgOqGw2oadSjusGA6kY9qhv0qG40tNvuZgu5VAx/DyXUHk7wV5lf1Srz3yNUTvD3UMJNOUBuOgwytv5GB8xkrWzgkkrEcJOIHZYMDEYTapsMrZJZ2+RW3aBHVYMe5XU6lFQ24HptE3QGEwpv1KPwRn27x3dTSIVE5u/hhBE3X/1VSvi6K4XLWaWs99vpdAYTGnRG1OkMqNcZUK8zoq7JiAa9AU4yKbxc5fB0kWOYsxyS/nyTpBs4abEBRyoRQ+Ustz6iRQd0BhNKqxtRXNmA4qoGFFc2ouTma3FlA0qqGlHVoEdNkwH5pTXIL63p9JgiEaCUmtvjlFIxlC0SWvOrUiYW2uyULbbJpWI03qxZ1jUZWyQjY4uEZLi53rxOb7TtwkgkAjycZPB0kcPrZhJrmdDMfyvg6SyHp6s5RikbGBP+dilpbdmyBe+88w60Wi3CwsKwadMmhIeHtxu/d+9erFmzBleuXMHYsWPx1ltv4cEHHxS2ExGSkpLw0UcfobKyEnfffTc++OADjB17a/iUiooKPP300/jXv/4FsViM+fPn4y9/+QtcXfnZLmYbuVSMAE9nBHg6txtT12SwSGTFVY0ouZnkSiobcb22CY16o5A8iGC+vNXb8OC3A8klYjgrJHCRm29mOMkkqNMZUFGnQ2W9HkRAZb0elfV6/Hi9gwEjW3CWS4SE5qqQQiYRQyYRQyEVQyYRQS41v5dLxZDffJW1epW3jJOIIZOKoZCIEXqbh8Nq2na3ae3evRuLFy/G1q1bodFosHHjRuzduxf5+flWZ3vOyMjAzJkzkZycjIceeggpKSl46623kJubiwkTJgAA3nrrLSQnJ+Pjjz/GqFGjsGbNGpw5cwbnzp2DUmnu/Dd37lyUlJTgww8/hF6vx9KlSzF9+nSkpKTYVG5u02KOpDea0Kg3olFvfm3QG82vuua/W63XG9GoM6Lx5uVdg96IJoMJTjIxnOVSOMslcFHcfL2ZiFwUEmGbs1xq8b6jO6cGowmVDXpU1OmE5UadDj+3+vtGnQ4VdU2oqNPZXIPrqv9deTcm3qbqMKbHGuI1Gg2mT5+OzZs3AwBMJhMCAgLw9NNP46WXXmoTv2DBAtTV1SE1NVVYd9ddd2HSpEnYunUriAhqtRrPP/88XnjhBQBAVVUVfH19sXPnTjz22GM4f/487rzzTpw8eRLTpk0DABw8eBAPPvggfvrpJ6jVbZ/yb2pqQlNTk8UXEhAQwEmLsVaICLVNBoskV6czQm8wQWc0QW80QXfzb53h1nu9kdBk8d5KnNEEvYHw/347BWOGd3xVZHPFwp4ZYJuamkgikdDnn39usX7x4sU0b948q/sEBATQhg0bLNatXbuWJk6cSEREly9fJgCUl5dnETNz5kx65plniIho+/btpFKpLLbr9XqSSCS0b98+q+dNSkoimJ+ktVh6fYZpxphNbJ1h2q7eeeXl5TAajfD19bVY7+vrC61Wa3UfrVbbYXzza2cxrS89pVIpPD092z1vYmIiqqqqhOXq1Q7mDGSMDRiD9u6hQqGAQqHo62IwxhzMrpqWt7c3JBIJSkstZ3QpLS2Fn5+f1X38/Pw6jG9+7SymrKzMYrvBYEBFRUW752WMDU52JS25XI6pU6ciPT1dWGcymZCeno6IiAir+0RERFjEA0BaWpoQP2rUKPj5+VnEVFdXIysrS4iJiIhAZWUlcnJyhJjDhw/DZDJBo9HY8xEYYwOdvY1lu3btIoVCQTt37qRz587R8uXLSaVSkVarJSKiRYsW0UsvvSTEHz9+nKRSKa1fv57Onz9PSUlJJJPJ6MyZM0LMunXrSKVS0ZdffkmnT5+mhx9+mEaNGkUNDQ1CzAMPPECTJ0+mrKwsOnbsGI0dO5bi4uJsLretjXyMsb5h62/U7qRFRLRp0yYaOXIkyeVyCg8PpxMnTgjbZs2aRUuWLLGI37NnDwUHB5NcLqfx48fT/v37LbabTCZas2YN+fr6kkKhoNmzZ1N+fr5FzI0bNyguLo5cXV3J3d2dli5dSjU1NTaXmZMWY/2brb9RfmCaMdYv8APTrTTn5upqK7OtMMb6XPNvs7N61JBJWjU15odfAwIC+rgkjLGO1NTUwMPDo93tQ+by0GQyobi4GG5ubp0OJdL8yM/Vq1cH5KUkl79vcfm7hohQU1MDtVoNcQdTzg2ZmpZYLMZtt91m1z7u7u4D8h9dMy5/3+Ly26+jGlYzHmSbMTagcNJijA0onLSsUCgUSEpKGrDPLnL5+xaXv2cNmYZ4xtjgwDUtxtiAwkmLMTagcNJijA0onLQYYwMKJy3G2IAyZJPWli1bEBQUBKVSCY1Gg+zs7A7j9+7di5CQECiVSoSGhuLAgQO9VFJLycnJmD59Otzc3ODj44OYmBjk5+d3uM/OnTshEoksluap2Xrbq6++2qYsISEhHe7TX757AAgKCmpTfpFIhPj4eKvxff3df/fdd/jlL38JtVoNkUiEL774wmI7EWHt2rXw9/eHk5MTIiMjcfHixU6Pa+/vx5GGZNLavXs3EhISkJSUhNzcXISFhSEqKqrNkM7NMjIyEBcXh2XLliEvLw8xMTGIiYnB2bNne7nkwLfffov4+HicOHECaWlp0Ov1mDNnDurqOp6Q093dHSUlJcJSWFjYSyVua/z48RZlOXbsWLux/em7B4CTJ09alD0tLQ0A8Oijj7a7T19+93V1dQgLC8OWLVusbn/77bfx/vvvY+vWrcjKyoKLiwuioqLQ2NjY7jHt/f04XI+O6tVPhYeHU3x8vPDeaDSSWq2m5ORkq/GxsbEUHR1tsU6j0dCKFSt6tJy2KCsrIwD07bffthuzY8cO8vDw6L1CdSApKYnCwsJsju/P3z0R0bPPPktjxowhk8lkdXt/+u4BWEz/ZzKZyM/Pj9555x1hXWVlJSkUCvrHP/7R7nHs/f042pCrael0OuTk5CAyMlJYJxaLERkZiczMTKv7ZGZmWsQDQFRUVLvxvamqqgoA4Onp2WFcbW0tAgMDERAQgIcffhg//PBDbxTPqosXL0KtVmP06NFYuHAhioqK2o3tz9+9TqfDp59+it/97ncdjhzSn777lgoKCqDVai2+Xw8PD2g0mna/3678fhxtyCWtnpi7sa+YTCY899xzuPvuuzFhwoR24+644w787W9/w5dffolPP/0UJpMJM2bMwE8//dSLpTXTaDTYuXMnDh48iA8++AAFBQW49957hfHOWuuv3z0AfPHFF6isrMTjjz/ebkx/+u5bs2XO0da68vtxtCEzNM1gFB8fj7Nnz3bYJgSYZzNqOVvSjBkzMG7cOHz44Yd44403erqYFubOnSv8PXHiRGg0GgQGBmLPnj1YtmxZr5alu7Zv3465c+dCrVa3G9OfvvvBYsjVtHpi7sa+sHLlSqSmpuLIkSN2jxMmk8kwefJkXLp0qYdKZzuVSoXg4OB2y9Ifv3sAKCwsxKFDh/DEE0/YtV9/+u5tmXO0ta78fhxtyCWtnpi7sTcREVauXInPP/8chw8fxqhRo+w+htFoxJkzZ+Dv798DJbRPbW0tLl++3G5Z+tN339KOHTvg4+OD6Ohou/brT9+9LXOOttaV34/D9Upzfz/TE3M39pbf//735OHhQUePHqWSkhJhqa+vF2Jal/+1116jr7/+mi5fvkw5OTn02GOPkVKppB9++KHXy//888/T0aNHqaCggI4fP06RkZHk7e1NZWVlVsven777ZkajkUaOHEmrV69us62/ffc1NTWUl5dHeXl5BIDee+89ysvLo8LCQiKybc7RX/ziF7Rp0ybhfWe/n542JJMWkePnbuwtAKwuO3bsEGJal/+5554TPquvry89+OCDlJub2/uFJ6IFCxaQv78/yeVyGjFiBC1YsIAuXbokbO/P332zr7/+mgC0mZuTqP9990eOHLH676W5jLbMORoYGEhJSUkW6zr6/fQ0Hk+LMTagDLk2LcbYwMZJizE2oHDSYowNKJy0GGMDCictxtiAwkmLMTagcNJijA0onLQYYwMKJy3G2IDCSYsxNqBw0mKMDSj/H70OFaZtu6PrAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:50:17.386909Z", + "iopub.status.busy": "2024-03-22T21:50:17.386632Z", + "iopub.status.idle": "2024-03-22T21:52:21.290770Z", + "shell.execute_reply": "2024-03-22T21:52:21.289942Z" + }, + "papermill": { + "duration": 123.924454, + "end_time": "2024-03-22T21:52:21.293166", + "exception": false, + "start_time": "2024-03-22T21:50:17.368712", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:52:21.330010Z", + "iopub.status.busy": "2024-03-22T21:52:21.329650Z", + "iopub.status.idle": "2024-03-22T21:52:21.350473Z", + "shell.execute_reply": "2024-03-22T21:52:21.349615Z" + }, + "papermill": { + "duration": 0.041919, + "end_time": "2024-03-22T21:52:21.352348", + "exception": false, + "start_time": "2024-03-22T21:52:21.310429", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0059720.0042820.0004346.4984890.0045960.3815290.0085681.265315e-072.3294350.0126790.907170.0208210.1472752.352470e-088.827924
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.005972 0.004282 0.000434 6.498489 0.004596 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.381529 0.008568 1.265315e-07 2.329435 0.012679 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.90717 0.020821 0.147275 2.352470e-08 8.827924 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:52:21.385859Z", + "iopub.status.busy": "2024-03-22T21:52:21.385567Z", + "iopub.status.idle": "2024-03-22T21:52:21.761468Z", + "shell.execute_reply": "2024-03-22T21:52:21.760449Z" + }, + "papermill": { + "duration": 0.395721, + "end_time": "2024-03-22T21:52:21.764104", + "exception": false, + "start_time": "2024-03-22T21:52:21.368383", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:52:21.799759Z", + "iopub.status.busy": "2024-03-22T21:52:21.799434Z", + "iopub.status.idle": "2024-03-22T21:54:37.075871Z", + "shell.execute_reply": "2024-03-22T21:54:37.075091Z" + }, + "papermill": { + "duration": 135.297049, + "end_time": "2024-03-22T21:54:37.078379", + "exception": false, + "start_time": "2024-03-22T21:52:21.781330", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:37.114646Z", + "iopub.status.busy": "2024-03-22T21:54:37.114322Z", + "iopub.status.idle": "2024-03-22T21:54:37.141219Z", + "shell.execute_reply": "2024-03-22T21:54:37.140476Z" + }, + "papermill": { + "duration": 0.047164, + "end_time": "2024-03-22T21:54:37.143160", + "exception": false, + "start_time": "2024-03-22T21:54:37.095996", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:37.176920Z", + "iopub.status.busy": "2024-03-22T21:54:37.176646Z", + "iopub.status.idle": "2024-03-22T21:54:37.182095Z", + "shell.execute_reply": "2024-03-22T21:54:37.181299Z" + }, + "papermill": { + "duration": 0.024429, + "end_time": "2024-03-22T21:54:37.184016", + "exception": false, + "start_time": "2024-03-22T21:54:37.159587", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.042389741490617215}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:37.219070Z", + "iopub.status.busy": "2024-03-22T21:54:37.218743Z", + "iopub.status.idle": "2024-03-22T21:54:37.650044Z", + "shell.execute_reply": "2024-03-22T21:54:37.649128Z" + }, + "papermill": { + "duration": 0.451484, + "end_time": "2024-03-22T21:54:37.652371", + "exception": false, + "start_time": "2024-03-22T21:54:37.200887", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7kklEQVR4nO3deXxU5b0/8M/ZZkkmM5N9IyGRVWQVgSIiqFxFrMKlKi4VEBVvC1ctpVfT168othbaumCtpcurJPKqihtqr9aFiwIuBWSTzQYIgYQsZJ9k9plznt8fkwyZZJJMJpOcmeT7fr3mRXLmmTPfEzLfPMt5nodjjDEQQoiKeLUDIIQQSkSEENVRIiKEqI4SESFEdZSICCGqo0RECFEdJSJCiOooERFCVEeJiBCiOkpEhBDVUSIiEfX111/jqaeeQlNTk9qhkBhCiYhE1Ndff43169dTIiK9QomIEKI6SkQkYp566in87Gc/AwDk5+eD4zhwHAeDwYDrrruuU3lFUZCdnY3bb7/df+zZZ5/F1VdfjeTkZOj1ekydOhVvv/120Pf7+9//jqlTp0Kv1yMpKQl33XUXysvL++fiSL/iaBkQEilHjx7Fxo0b8frrr+OFF15ASkoKAKCkpARPP/00KioqkJGR4S+/Z88ezJkzB2+99ZY/GeXk5OC2227DuHHj4Ha7sW3bNuzfvx8ffPABbrnlFv9rn3nmGfziF7/AnXfeiTlz5qC2thYvvfQSDAYDDh8+DLPZPKDXTvqIERJBv/vd7xgAVlpa6j9WXFzMALCXXnopoOyPf/xjZjAYmN1u9x9r/zVjjLndbjZ+/Hh2/fXX+4+dO3eOCYLAnnnmmYCyx44dY6IodjpOoh81zUi/Gz16NCZPnow33njDf0yWZbz99tu49dZbodfr/cfbf93Y2AiLxYLZs2fj0KFD/uPbt2+Hoii48847UVdX539kZGRg1KhR+PzzzwfmwkjEiGoHQIaGJUuW4Oc//zkqKiqQnZ2NXbt2oaamBkuWLAko98EHH+BXv/oVjhw5ApfL5T/OcZz/69OnT4MxhlGjRgV9L0mS+uciSL+hREQGxJIlS1BQUIC33noLjz32GN58802YTCbMnz/fX+aLL77AbbfdhmuvvRZ//OMfkZmZCUmSUFhYiNdee81fTlEUcByHjz76CIIgdHovg8EwINdEIocSEYmo9jWX9vLz8zF9+nS88cYbWL16NbZv345FixZBq9X6y7zzzjvQ6XT45JNPAo4XFhYGnGvEiBFgjCE/Px+jR4/unwshA4r6iEhExcfHA0DQGxqXLFmCvXv3YsuWLairq+vULBMEARzHQZZl/7Fz587hvffeCyi3ePFiCIKA9evXg3UY9GWMob6+PjIXQwYMDd+TiPrmm28wffp0LFiwAHfddRckScKtt96K+Ph4XLhwAbm5uTAYDJAkCdXV1QH9OZ999hluuOEGzJ49G/fccw9qamrw8ssvIyMjA0ePHg1IOhs3bkRBQQGuvvpqLFq0CAkJCSgtLcW7776LlStXYu3atWpcPgmXiiN2ZJD65S9/ybKzsxnP852G8mfNmsUAsAcffDDoa//2t7+xUaNGMa1Wy8aOHcsKCwvZk08+yYL9qr7zzjvsmmuuYfHx8Sw+Pp6NHTuWrVq1ihUXF/fXpZF+QjUiQojqqI+IEKI6SkSEENVRIiKEqI4SESFEdZSICCGqo0RECFFdTE/xUBQFlZWVSEhI6HJqASFEPYwxtLS0ICsrCzzfdb0nphNRZWUlcnJy1A6DENKD8vJyDBs2rMvnYzoRJSQkAPBdpNFoVDkaQkhHzc3NyMnJ8X9WuxLTiaitOWY0GikRERLFeuo6oc5qQojqKBERQlRHiYgQorqY7iMKBWMMXq83YLEtEhpBECCKIt0aQfrdoE5EbrcbVVVVsNvtaocSs+Li4pCZmQmNRqN2KGQQG7SJSFEUlJaWQhAEZGVlQaPR0F/2XmCMwe12o7a2FqWlpRg1alS3N6QR0heDNhG53W4oioKcnBzExcWpHU5M0uv1kCQJ58+fh9vthk6nUzukqPfv6macqGjGnDGpSDFoe34BATAEOqvpr3jf0M8vdIrCsPO7GpQ12PHF6Vq1w4kp9FtGSITUWV1wexUAQLXF1WmHEdI1SkSEREit9dLOtE6PDJubRmpDRYloiMvLy8OmTZvUDmNQsDq9/q/zUuLglRUVo4ktg7azmpCBNj0/CROHmaEwhngtfbR6g35ag4Db7ab7fKIAx3HQawS1w4hJqjfNKioq8MMf/hDJycnQ6/WYMGECDhw40K/v6fYqXT46Vqe7K+sJoWw45s6di9WrV2P16tUwmUxISUnBL37xC3/nZ15eHn75y19i6dKlMBqNWLlyJQDgyy+/xOzZs6HX65GTk4NHHnkENpvNf96amhrceuut0Ov1yM/Px6uvvhpWfCQ01FkdOlVrRI2NjZg1axauu+46fPTRR0hNTcXp06eRmJjYr+/78udnunwuPyUei6Zk+7//y54SeOTgv1DDEvW446pLC7Nt+aoUjg4dlD/5j9FhxfjKK6/ggQcewP79+3HgwAGsXLkSubm5eOihhwAAzz77LNatW4cnn3wSAFBSUoL58+fjV7/6FbZs2YLa2lp/MissLAQALF++HJWVlfj8888hSRIeeeQR1NTUhBUf6ezj41XQMzsyLEexz5qKtGEjMX98htphxQRVE9FvfvMb5OTk+D8oAJCfn69iRNEjJycHL7zwAjiOw5gxY3Ds2DG88MIL/kR0/fXX46c//am//IMPPoh7770Xjz32GABg1KhR+P3vf485c+Zg8+bNKCsrw0cffYT9+/dj2rRpAIC//e1vuPzyywf82gYjj6zgu6oWjKj/HKN0FchrYmhMH652WDFD1UT0j3/8AzfddBPuuOMO7N69G9nZ2fjxj3/s/7B15HK54HJdGiJtbm4O631XXTeyy+f4DrNAVl47osuyHWeMrJgVuST6ve99L2BKysyZM/Hcc8/5J+9eddVVAeW//fZbHD16NKC5xRjzT3U5deoURFHE1KlT/c+PHTsWZrM5YjEPZQ6PDDCGJNcF6AwCJNkKZqsDQEsZh0LVRHT27Fls3rwZa9aswc9//nN88803eOSRR6DRaLBs2bJO5Tds2ID169f3+X01YuhdY/1Vtq/i4+MDvrdarXj44YfxyCOPdCqbm5uLU6dODVRoQ5LTLUMj26BnLoiCwXfQSndXh0rVRKQoCq666ir8+te/BgBMmTIFx48fx5/+9KegiaigoABr1qzxf9+2Hu5gtG/fvoDv9+7di1GjRkEQgo/KXHnllTh58iRGjgxe2xs7diy8Xi8OHjzob5oVFxejqakponEPVQ6PDK1shShwEFur1byzXuWoYoeqo2aZmZkYN25cwLHLL78cZWVlQctrtVr/+tSDfZ3qsrIyrFmzBsXFxXj99dfx0ksv4dFHH+2y/OOPP46vv/4aq1evxpEjR3D69Gm8//77WL16NQBgzJgxmD9/Ph5++GHs27cPBw8exIMPPgi9Xj9QlzSoOTwyNF4rJIGHKPg+VpyrhUbOQqRqIpo1axaKi4sDjp06dQrDh1Mn39KlS+FwODB9+nSsWrUKjz76qH+YPpiJEydi9+7dOHXqFGbPno0pU6Zg3bp1yMrK8pcpLCxEVlYW5syZg8WLF2PlypVIS0sbiMsZ9BxuGTpvC0Seg9A6UVjy2uEK8xaOoUbVptlPfvITXH311fj1r3+NO++8E/v378df/vIX/OUvf1EzrKggSRI2bdqEzZs3d3ru3LlzQV8zbdo0fPrpp12eMyMjAx988EHAsfvuu69PcRIfX9OsBZLAQTAPg1FngV7nhkI1opComoimTZuGd999FwUFBXj66aeRn5+PTZs24d5771UzLEJ67Xv5yfBa4sE3xgHGLFyRdQEQRUBDkxdCofpP6fvf/z6+//3vqx0GIX3C8xw0ihPgeSA+xXdQdgOK4jtGuqV6IiKd7dq1S+0QSDg8Dt+/+taZAYwBXiegoRVCe0KpmpAI2F/agDOVNbC5vIDGgOI6F74514AzVXVqhxYTKBEREgFnLzaitskKt6wAUhzcnBZehcHjpB1kQkGJiJAIUNy+ZhkvCICoBa/xbTQgu2zdvYy0okRESAQoLl/Nh9fE+SYhir4bRWWXQ82wYgYlIkIigHl8iUjU+uYAcq0d1IqHmmahoERESB8pCgNz+xKOoPUlIF7jqxG1NdlI9ygREdJHblmBqPiWpxFaa0K81JqIPE7V4ooldB8RIX3klhVIihM8BwitTTO9Xg+DVoQk0pZCoaBEREgfJWhFLLwiEcoFM9BaE8pJNSMn2wSk0rbToRhaTTPGAK974B+9mPi4detWJCcnB6xECQCLFi2iCapRiuM4aJgbOlEARN+wPYTWXVVkj3qBxZChVSOSPcAXzw38+87+KSCGtt3PHXfcgUceeQT/+Mc/cMcddwDw7b7x4YcfdjuznqhMdvv+bft/FlprQl5X8PIkwNCqEcUAvV6Pe+65J2BDgb///e/Izc3F3Llz1QuMdKnK4sDJshpcbHb6a0SNLuBQWSP2nalSObrYMLRqRILkq52o8b698NBDD2HatGmoqKhAdnY2ioqKsHz58oDF9En0qLe6cbHeAl5yI721ScZJOri8CmQ31YhCMbQSEceF3ERS05QpUzBp0iRs3boVN954I06cOIEPP/xQ7bBIF9yyAoG5IPCcv0YktU7xgNcFxhj9EenB0EpEMeTBBx/Epk2bUFFRgXnz5g3aTQIGA7dXgai4WxORr29IbN0CXFA88MoKJJG2ou4O9RFFqXvuuQcXLlzAX//6V6xYsULtcEg3XB4ZguIJSESSpm1TAgavx61ecDGCElGUMplM+MEPfgCDwYBFixapHQ7phsftAsAgcJeaZryo8TfHPHR3dY8oEUWxiooK3HvvvdBq6aa4aCa7fYlGEASAb+3t4Diw1o5r6rDuGfURRaHGxkbs2rULu3btwh//+Ee1wyE98LbOsOckXcA+5Hq9HoJbBidT06wnlIii0JQpU9DY2Ijf/OY3GDNmjNrhkB7cMMoMNJshxCcEHJ88PBWwC4CWRsx6QokoCnW1bxmJTjp4AUkAtB12zW1rplGNqEeq9hE99dRT4Dgu4DF27Fg1QyKk97ytndFCh768tvlmindg44lBqteIrrjiCvzf//2f/3tRjGxItPd439DPr2cHz15ESoMNmWYJ7W+XPVZtg6e2ESnpFuSmqhZeTFA9EYmiiIyMjIifV5J80yrsdjv0en0PpUlX7HZfR2zbz5MEYozhbGU93E1OpPGBd+27FAEerwK3m4bve6J6Ijp9+jSysrKg0+kwc+ZMbNiwAbm5uUHLulyugOUxmpubuzyvIAgwm82oqakBAMTFxdFt9r3AGIPdbkdNTQ3MZrNvaJp04lUYuNbVGUWtLuA5rrVppnhpKZCeqJqIZsyYgaKiIowZMwZVVVVYv349Zs+ejePHjyMhIaFT+Q0bNmD9+vUhn7+tptWWjEjvmc3mfqmxDhZt0zs4AKImsObNib5apOKlzuqeqJqIbr75Zv/XEydOxIwZMzB8+HC8+eabeOCBBzqVLygowJo1a/zfNzc3dzsHi+M4ZGZmIi0tDR4P/VXqLUmSqCbUA7dXgcDc4HkOnBhYI+IpEYVM9aZZe2azGaNHj8aZM2eCPq/VasO6y1gQBPpAkX7hlhUIihtiu3lmbXhqmoUsqqZ4WK1WlJSUIDMzU+1QCAlJsJn3bbjWJWcUWi62R6omorVr12L37t04d+4cvv76a/znf/4nBEHA3XffrWZYhITM5fXViHwTXgMTkU6rhV4SoOXoPqKeqNo0u3DhAu6++27U19cjNTUV11xzDfbu3YvUVLrpgsSG4clxyM5LAOdUOt3QOCozEWg2Ayl0+0hPVE1E27ZtU/PtCekzSeAhcR7fFI8OndW0k0fooqqPiJCYw1jnHTza8K03gdJcsx5F1agZIbHmTHUjNPU2mPQSTB2aZpUtXtReaAKfoMWEqSoFGCMoERHSBxV1jdA0OQCeh6nDbi0yJ8LuliE6aYpHT6hpRkgfeFtXX+RFbcCiaAAgSG19RNQ06wklIkL6QG5dj5qXOt9oK7YlIoU6q3tCiYiQPvCvVy3pOj0ntnZeM9nr69QmXaJEREgftC2MH7RG1DodiSkywJQBjSvWUCIipA8UjwMAIGg637TYViNSGKB4aCeP7lAiIqQPFG9b0yxIjUiUIIkCdBIPmTqsu0XD94T0wbxRieAkE7RxcZ2e04g8rrosHfC6AFDTrDuUiAjpA4MoA1oR0HTurAYACJIvEVGNqFvUNCOkL7ytfT8d55m18U/zoCH87lAiIiRMDreMf1fUodLi6LyVUKtvq2w4eqEJjVbbAEcXWygRERImq8uLc9WNqGpydJ7w2qrFDdjcMjxuapp1hxIRIWFqWyZW4Lkua0Rc6/wzmdZM7xYlIkLC1LZwvsDznVZnbONPRNRH1C1KRISEye1tVyPqIhFBoJ08QkGJiJAw+RKRBzyHS6sxdtC2pZCXElG3KBEREiaXxwOBeSCG0DRjtKVQt+iGRkLC5Gmd8NpdZ7UkacEEDjyjnTy6Q4mIkDBNytRByTZBFCVACP5RujIvBeCSgMTebww6lFAiIiRMcXzb9I5utgtqWz6WFkfrFvURERKutukdXTTLANAUjxBFTSLauHEjOI7DY489pnYohITkVGU9KpocsCtdNyxKG104UWlBycWmgQssBkVFIvrmm2/w5z//GRMnTlQ7FEJCdrqyFmUNdthkocsydi+HZqcXdgft5NEd1ROR1WrFvffei7/+9a9ITEzstqzL5UJzc3PAgxC1tC0TK3S1BAgArm2VRplGzbqjeiJatWoVbrnlFsybN6/Hshs2bIDJZPI/cnJyBiBCQoJrW/5V1HTdR3RpAX26obE7qiaibdu24dChQ9iwYUNI5QsKCmCxWPyP8vLyfo6QkOAYY1C8bYmo61EzXqLO6lCoNnxfXl6ORx99FDt27IBO13XVtj2tVgutlu7HIOqTFQa+db3q7hKR0DrFg1Ei6pZqiejgwYOoqanBlVde6T8myzL27NmDP/zhD3C5XBCErjsBCVGTq3XCKwdA0naTiIS2phklou6olohuuOEGHDt2LODY/fffj7Fjx+Lxxx+nJESimturQFSc4HkOnNRNIpIkCBwgQh7A6GKPaokoISEB48ePDzgWHx+P5OTkTscJiTYJOhGz8gxAc0LX61UDyE4yIjs/2fcNYwDHDVCEsSWszuqzZ89GOg5CYooo8DCKMow6CQiy3bRf2xQPAFBoCL8rYdWIRo4ciTlz5uCBBx7A7bffHnJnc0927doVkfMQMiBad3ntrkbkn+IB+EbO2icm4hdWjejQoUOYOHEi1qxZg4yMDDz88MPYv39/pGMjJGpdtDhQUdeERru720TklBm+u2jDyapmMJr42qWwEtHkyZPx4osvorKyElu2bEFVVRWuueYajB8/Hs8//zxqa2sjHSchUaWywYKyeivqrK7ua0QA6h0KLA4PLaDfjT7d0CiKIhYvXoy33noLv/nNb3DmzBmsXbsWOTk5WLp0KaqqqiIVJyFRxeP0Nct4Qey2uSUJPGTO1wNCy8V2rU+J6MCBA/jxj3+MzMxMPP/881i7di1KSkqwY8cOVFZWYuHChZGKk5Co4nX7EhEn6bsdCeM5gPG+RCTTcrFdCquz+vnnn0dhYSGKi4uxYMECbN26FQsWLADP+/Jafn4+ioqKkJeXF8lYCYkaXpcdEgC+uxEzABzH+WpMHkBunZtGOgsrEW3evBkrVqzA8uXLkZmZGbRMWloa/va3v/UpOEKildflgARA6Oau6jYc1Yh6FFYi2rFjB3Jzc/01oDaMMZSXlyM3NxcajQbLli2LSJCERJu2pll388z8+LbdXqmPqCth9RGNGDECdXV1nY43NDQgPz+/z0EREu0Ulx0AIGrieizLiRJ4DlBovlmXwqoRMcaCHrdarRG7uZGQaDYzzwDACL0xoceys8dkgqttBuJo/mRXepWI1qxZA8DXAbdu3TrExV36ayDLMvbt24fJkydHNEBCopFJlAG9BOhCqBHRTh496lUiOnz4MABfjejYsWPQaC5ts6vRaDBp0iSsXbs2shESEo08vqYZpJ4TkX+aB80161KvEtHnn38OwLdcx4svvgij0dgvQRESzZweGZWVtTA4nEjrZgmQNqfqHFCqm2EytyAzr//ji0Vh9REVFhZGOg5CYkaL04vSqlqYFDvSQqgRNTgAZvdAsjsGILrYFHIiWrx4MYqKimA0GrF48eJuy27fvr3PgRESrZweGZLshCjwITXNOFECA42adSfkRGQymXx3ibZ+TchQ5fJ4ISpOiBoBCGH4XhBFKAAUuqGxSyEnovbNMWqakaHM5XSAA4PIc6HViGjd6h6FdUOjw+GA3W73f3/+/Hls2rQJn376acQCIyRaeRwtAFrnmfE93xvUtpMH1Yi6FlYiWrhwIbZu3QoAaGpqwvTp0/Hcc89h4cKF2Lx5c0QDJCTaeJy+P8JcCM0yAOBFqhH1JOwVGmfPng0AePvtt5GRkYHz589j69at+P3vfx/RAAmJNh5na41IGx9Sed5/QyPNNetKWMP3drsdCQm+W9s//fRTLF68GDzP43vf+x7Onz8f0QAJiTYT0jVQGowQkpNCKn/5sCRwdUngDYZ+jix2hVUjGjlyJN577z2Ul5fjk08+wY033ggAqKmpoZscyaCXwLlg0kswGEL7XRdEDXiOo22nuxFWIlq3bh3Wrl2LvLw8zJgxAzNnzgTgqx1NmTIlogESEnVcVt+/2hBrODzNNetJWIno9ttvR1lZGQ4cOICPP/7Yf/yGG27ACy+8EPJ5Nm/ejIkTJ8JoNMJoNGLmzJn46KOPwgmJkAFTUlGNi81OeMXQ+ojqHApO17SgtKapfwOLYWHv9JqRkYGMjIyAY9OnT+/VOYYNG4aNGzdi1KhRYIzhlVdewcKFC3H48GFcccUV4YZGSL/xyApKK6oR77YhURNaInLKHOqsbsRpFNrttQthJSKbzYaNGzdi586dqKmpgaIoAc+HuhPsrbfeGvD9M888g82bN2Pv3r2UiEhUsrtlSLIDPAdI+tD6iPi2+4gUBVBkQFBtp/eoFdZP5MEHH8Tu3btx3333ITMz0z/1oy9kWcZbb70Fm83m73PqyOVyweW6tAB5c3Nzn9+XkN6wu9yQZDskkQen7XlRNAAQJd99RApjvn4iSkSdhPUT+eijj/Dhhx9i1qxZfQ7g2LFjmDlzJpxOJwwGA959912MGzcuaNkNGzZg/fr1fX5PQsLlsFl90zsEEZBCa5pJogQGDgqDb+QshKVDhpqwOqsTExORlBTaPRQ9GTNmDI4cOYJ9+/bhRz/6EZYtW4aTJ08GLVtQUACLxeJ/lJeXRyQGQkLltlsAtN7MyIf28REEDgovttaIaHG0YMJKRL/85S+xbt26gPlm4dJoNBg5ciSmTp2KDRs2YNKkSXjxxReDltVqtf4RtrYHIQPJbfMlIk4XWrMMACSeh8KJUBjNN+tKWE2z5557DiUlJUhPT0deXh4kKXDL3UOHDoUdkKIoAf1AhEQTj60RACDozSG/RhQ4KO22ndb0UH4oCisRLVq0KCJvXlBQgJtvvhm5ubloaWnBa6+9hl27duGTTz6JyPkJibTLExV4Mo3gMtJDfo3Ic5g1OhOCQwIHapoFE1YievLJJyPy5jU1NVi6dCmqqqpgMpkwceJEfPLJJ/iP//iPiJyfkEiLk62+3TvMqSG/huM4SBoN4OR8w/ekk7DHEZuamvD222+jpKQEP/vZz5CUlIRDhw4hPT0d2dnZIZ2DtqQmMcfp6yNCL5pmAGiaRw/CSkRHjx7FvHnzYDKZcO7cOTz00ENISkrC9u3bUVZW5l+riJDBRJEVlFdUQs97kSQl9OrDU1zrgNDYgvQ8B0Lv5h46who1W7NmDZYvX47Tp08H7Oy6YMEC7NmzJ2LBERJN7A4bKustOFtrA9/LGlFVixe1VjccLmf/BBfjwkpE33zzDR5++OFOx7Ozs1FdXd3noAiJRnZLAwCA0xrAS70c+2q9m5qG74MLKxFptdqg0ytOnTqF1NTQO/EIiSWOlnoAAKfv/S42bQvoy15apTGYsBLRbbfdhqeffhoejy+7cxyHsrIyPP744/jBD34Q0QAJiRbu5loAAB/X+1kFXOtysbRudXBhJaLnnnsOVqsVqampcDgcmDNnDkaOHImEhAQ888wzkY6RkKjgbfElIsGQ0uvXts3Al6lpFlRYo2Ymkwk7duzAV199hW+//RZWqxVXXnkl5s2bF+n4CIkastWXiCRjWq9f21Yjoj6i4HqdiBRFQVFREbZv345z586B4zjk5+cjIyMDjLGILAlCSNRhDIrN10cUlxj6XdVteEpE3epVImKM4bbbbsM///lPTJo0CRMmTABjDN999x2WL1+O7du347333uunUAlRkdOC0ak6OL06aNIzei7fwZX5aeC9ieATpZ4LD0G9SkRFRUXYs2cPdu7cieuuuy7guc8++wyLFi3C1q1bsXTp0ogGSYjq7PXQiQJ0phRA2/tpq1qNBhB4gNEUj2B61Vn9+uuv4+c//3mnJAQA119/PZ544gm8+uqrEQuOkKhh9zXLEJcc3usFmuLRnV4loqNHj2L+/PldPn/zzTfj22+/7XNQhESbprpKlDfaUeUNb5PEimYPSuusqGxoiXBkg0OvmmYNDQ1IT++6oy49PR2NjY19DoqQaNNUV42qRge8Vg0yw3h9vYPB1uyCSWNHVsSji329qhHJsgxR7Dp3CYIAr5fWWyGDDGPwtg7da029HzEDLt1HRDc0BtfrUbPly5dDq9UGfZ5WViSDkscOj9MOgENcL9Yhak8QfR3clIiC61UiWrZsWY9laMSMDDr2erg8MlxiPIbFx4V1CqFtOWWZWgzB9CoRFRYW9lcchEQtxVYPl1eBU2uGKS68+4D8NSLFTbu9BhHWXDNChhJHcy0UBrg0Zhg04S1qKvqbZgrAlB5KDz2UiAjpgbOpdbJrfBJ4PryaTFvTzL/JIglAe98S0oNErgVTcsywjh0d9jnSTfFIyU2EwKH1pkZdTy8ZUigREdIdRQbvtEAnCdClhXMHkY8kCpC0Wl9tiGpEnVDTjJDuOJp8fTqCBGj7uLOwf5oHjZx1pGoi2rBhA6ZNm4aEhASkpaVh0aJFKC4uVjMkQgI5GlDeaEeJTYtmV/gJxO1VUNLgQmmdDUym5WI7UjUR7d69G6tWrcLevXuxY8cOeDwe3HjjjbDZbGqGRcgl9npcbHbiuyYRLk/4o10KYzjf6EZ1sxOyh5pmHanaR/Txxx8HfF9UVIS0tDQcPHgQ1157rUpREXKJu6UOHpnBIZlh1If/cZEEHgrne70se6hztoOo+nlYLL5dNJOSgi9O7nK5AqaRBNtJhJBIclpqfF/oE6EVhbDPw3OAwvs+bl6vG8EnSQ1dUdNZrSgKHnvsMcyaNQvjx48PWmbDhg0wmUz+R05OzgBHSYYaV3MdAEBK6Ns2WRzHXdpSyEN9RB1FTSJatWoVjh8/jm3btnVZpqCgABaLxf8oLy8fwAjJkONxwOOwAoBvZcY+4njaZLErUdE0W716NT744APs2bMHw4YN67KcVqvtcuY/IRFnr4fTK8MtxMMYH9/387Xu9ko1os5UrRExxrB69Wq8++67+Oyzz5Cfn69mOIQEsjfA5VHglEww6vu+6D1Pu712SdUa0apVq/Daa6/h/fffR0JCAqqrqwH49k3T6/VqhkYIYK/HmIwEDEsZDSktvCVi25s+Mg1inBmSPmp6RKKGqj+RzZs3w2KxYO7cucjMzPQ/3njjDTXDIsTH0QCe45CQmA6dFP6IWRuDXg+dJECgnTw6UbVGxBhT8+0J6Z69wfdvGHvdB0U7eXQpKjqrCYk6ioLmxhrUNNnhtogYF+YuQu2db/KAa7DDkGBHhFLboEGNVUKCcTbB5nDhos2Lkua+N8sAoLzJg4omByxWe0TON5hQIiIkGEcjnF4FTtEEc3xkbhnxL45Gy4B0QomIkGBaF8x3SiaY9b3fYjoYXvKdR6Hh+04oERESjL0BTo8Mh2iGOcwF8zuSWjurZbqzuhNKRIQEodjq4PIqcEimsHfu6EjUUI2oK5SICAnC1bpzh0eTiARtZAaXBcm3TjXz0kakHVEiIqQjrwtuews4ABpjCrgI7UEmaXyJSPFQIuqI7iMipCN7PUx6CdPG5OCKqy6L2GlzUs3IzjZBEHnaZLEDSkSEdNR6R7UQn4wEXWT6hwBAr48D2pp5sgcQIzMaNxhQ04yQjuz1vn/jInA7dXuCdKkWJFPzrD1KRIR05GjAmRor9lUzWOyRG2p3ehWUN8sob7QDNHIWgBIRIR0otnrU21w4aYlcswwAPLKCkgYPKhodYF5nRM8d6ygREdKeoviH7r3axD7t3NGRRuQh8xIYAC+NnAWgRERIe84mOFxuKJwAvTE5YkP3AKAReMi8r4Pa43JE7LyDASUiQtqz18PhluEUzUg26CJ6ao7jwIm+CbSym5pm7VEiIqQ9Wx0cHhl2KRGJ8ZEfXm9LRB5KRAEoERHSnr0ODrcMh2RGcn8kotYF9L0eSkTtUSIipD17PRTG4OinGhHfOt9MdlNndXt0ZzUhbRgDbHWYNMyMy6+cAjEu8olo8mVpkEQTtLRJTQBKRIS0cVp8Uy84HpqEZN+G9RGWmJDgm+bBaE2i9qhpRkgba43v3/hkgI/MOtWdtPYR0Z3VgSgREdLGWo3TNS348qKEiqb+uc/nooOhosmBOktzv5w/VqmaiPbs2YNbb70VWVlZ4DgO7733nprhkKGupRrNDg/K3Eb01wIdFc0yyhrsqLO09NM7xCZVE5HNZsOkSZPw8ssvqxkGIQBjcDZWwC0z2LXJSDFEZueOjkSNr5ea0X1EAVTtrL755ptx8803qxkCIT6ORrQ0W8A4HvFJ2dCI/fM3WqPTwwZA8dAUj/ZiatTM5XLB5bp0/0VzM7WzSYQ0laHZ4YFVk4ZhKcZ+extJZwAAMI8LUBSAp25aIMY6qzds2ACTyeR/5OTkqB0SGSRY03lYHB40azMxLDGu395Hq/ed26sogJdqRW1iKhEVFBTAYrH4H+Xl5WqHRAYDRUFLVQlcXgX2uCxkm/vvbkOtJEHmNfAqDKBpHn4x1TTTarXQavunE5EMYc0XwHtsMCUYkJY7ut/6hwBAJ/Hw8lp4vW4wjw0cIrwcbYyKqURESL+oPgaDVsS4CVNw+disfn2rOI2IifkZ0Np5qhG1o2oislqtOHPmjP/70tJSHDlyBElJScjNzVUxMjJkOC1Qqk/4+iiyJkd0IbRgBJ5DkskMyA0ALRfrp2oiOnDgAK677jr/92vWrAEALFu2DEVFRSpFRYYKr9uFM7vfgFzXgNFjxkEyZg/MG7fOwAcN4fupmojmzp0LxpiaIZChSFFw8fR+lB36FF57MxROxNnEazBmgDY8rLAyoMkBk60FhgF5x+hHfURkaGkoRfn+91FRcQEMANMmIHv6YuSNiNyOrj051SBD12BHTjMlojaUiMjQwBhQ8hmqju/GhXo7vLwO0ohZmDLjeugGeCRWo4sHAHgc1gF932hGiYgMfrIX+Pf/ouncUZyvt+OiYRwyr5yP6SMz+71zOhhJFw8ZgNdlG/D3jlaUiMjg5nEAx98Bmsqh0WhQnX0j4nMnYfrINFWSEABo9AlwAJCdVCNqQ4mIDF72Bl8SstUBogZxk3+A+YYcSAKvWhICAE28CQ4AiouWAmlDiYgMPooMlO0Fzn8Nt8cNi6xF6rT7AEMaIrtTWXi0Bt+kWtnt8q3UKEZ+bexYQ4mIDBour4ya8hKIZz4GbHVweRWUyGkoNl2Dm53xyI+SIao4fRxkToJHlgG3FRCT1A5JdZSIyKBwoqwWJfs+QGrzCQCAh9fjXOLVqDeOQIpRh8Q4SeUIL0mK10CblwWdp8mXiOIoEVEiIrHPVofh595ES3MptCIPZ8p42LNnY5guDt9LjMPIVAP4ftiRI1xaUYA2MQlosgIu6rAGKBGRWNd4Hjj+DgyyCxNGDEPchNvAJQ/czYlh07a2E92UiABKRCSGVZedQkLxO4gXAZiGIX78DwBN/y1qFknVDhHeRjsMjQ0w0/p+lIhIbHI2VKBs1ytgXhdGjJ2MlElLACF6+oF6UtIigG90IKOhBma1g4kCMbVCIyEAwGz1KP2sELLHBW/CMJiuuiOmkhAA306yANzWepUjiQ6UiEhscbXg4pevoMnSBLsmBSNvWA5JE3urdupNqQAAxdbgmwc3xFEiIrHD44Tj0Osor6yCUzQiYfo9SE8yqx1VWAymZDBwcLvdgIt2o6FERGKD7IVy7G2UlJyFg9OhYeR/4qpRsdvLa47XwiUa4fTIvlrREEeJiEQ/RQa+ex8NFWfQ5OFwNnMB5k0ZG1X3BvWWSS/BozVDYUBzw0W1w1EdJSIS3WQvcOJdoPYUkhP00E66HddMGQ9TFN0pHQ6O46AxZQAArPUXVI5GfTR8T6KXxwHHkbeBxvPQa7Xgxi/GtOQRakcVMRPGjkGcUAydYFE7FNVRIiJRx+GWUXb2O7hPfAC7pQ7xcXG44j/uiY07pnshOTMPOCf4linxugAx9kb/IoUSEYkKFrsH52oaUH2uGErlEZgdZQAAl2jAxZyFGGUajkH3MdUmAHoz4GjyTVVJHa12RKqhREQGDGMMNreMRqsT9ppSjNQ0QnBbAEcjLpwtg93SBGNr2TitCG3OlUieeBNSzEZVFzLrNxyHC3wWbBfLYYw/jkxKROp6+eWX8bvf/Q7V1dWYNGkSXnrpJUyfPl3tsKKXrc73F9TRCHBAE4woRzoalDiMyzQiNaH3dYdGmxvVzU54ZAWSwMMcJyEtQQchAiNTFrsHJystOH/uFLT1xUh2nIUkO+AaZkKcxvcrmCJ5AJ0IY2IqknKvQPxl04fE8hjVmuHw2L6At/QIMqcsuLTn2RCjeiJ64403sGbNGvzpT3/CjBkzsGnTJtx0000oLi5GWlqa2uFFD68LqDkJVB0FmivhVRTUW92oaXHB6vICANyaNLgmzwBGTQF0RjTY3KhodOCy1HjEay/9VzOPA/bmBsQzO+BsBjgOnx+rR51XD4eUCIX3jUhpRB7DEvW4LMWAK7KMvR4uv1DfjG9PnISruhjJ9rMYLtvAAdBKPKQ4A7yp44DkTECfiHSdGen6xCH3QRw56nIcOmoGrE2oOfE50ibfrHZIquCYyjsczpgxA9OmTcMf/vAHAICiKMjJycF///d/44knnuj2tc3NzTCZTLBYLDAajd2WjTmy13fHbXMF0HAWqD0FKF64vDLONzpxzpuCFikZHFOQ4KnDML4ecRKPNKMOeo0IGNJwwqLB6To3eMhIFD0wcA7wLgvcTgcYgGnDE/1NntI6K+xuGaLAw8EbUAcjLJwJdikJ2ngT7vjeSHC8CChe1DRZYRA80MMFzuMA89jhddnhtDWD9zoRz7sBjx2WFhtOVjWDg+++mWRzApKGT4CYMQ5IzAN4Qc2fcNT4175/AcffgSRwyJg4DxmXXw1Rn6B2WBER6mdU1RqR2+3GwYMHUVBQ4D/G8zzmzZuHf/3rX53Ku1wuuFwu//fNzSHeGt9QCpTsDDqnx+724liFBRxjaHuWw6WvM01a5Cb69qFyemV8W9546cXtXgMA6Qla5Ke07lnllXGwzFeW6/C+DECqQYMRqQYADLLC8M25wDlHonJpX/TEuNaycclAyhX49HQ8PEIcUgwajMsyYWxGAuLhBGr/DdR8B1guAC0XkWJ1QvFcqjG132ldEXSw69IQb0wEGEO+2elbbN5ta42xGXZXIxrtpyDYOHAHvmi9ZIazpQ3+626rJCmtB9KNWlyW4ltrx6gTkZOejKTccYjLvBxIugwQVK+ER52pU2fgq9pzwMWDKD+8AxcO70BuejKyUswAx6PZKeO76hYAwWukOUlxyDL7apJWlxcnKrv+XGSb9RiWqAcA2N0yjlV0fetAhkmH4Um+ZVW8CoPYsUY84U5AF5kKgKq/FXV1dZBlGenp6QHH09PT8e9//7tT+Q0bNmD9+vW9fyOvC7DWBn/O7YVi7fo/g4k6QOduPY8Mxd7U9ftIWsAp+76WFTCnb5eGYFVOptEC7kufYua2BzzvAaBwIuxSEryJ+Rgx9RogIQNajsNMsQHDEuOQlqBt14lrAIZd5Xs4m4GWKqTbG5CueGD3AhavBBviwOtNMJiTkGxMCN7/47YDtlpw9jrE2+oQb6v1HfM6ASbDrfCAXoFNEeHh9PAIOnh5HTyCHpD0YGnJuOzyXECKAyfFYZioBQZjR3MEaUQe18y/E8cP58J65kuI9loIstM3mgaAOTxQrN380dXEAaIvucDlhdLS3e+zHpBa12xyd18WvA7Q+P6wQlEAvsP9z0zp4cpCp2rTrLKyEtnZ2fj6668xc+ZM//H/+Z//we7du7Fv376A8sFqRDk5OT03zdw2wFrj+zrgQ8HBLTPUtriCPOX7Jl4rwqT37bLglRXU2dxg4C79beI4/+v0ogij3te/IjOGBpsn4HztT6+RBBh1vrIMQL3N3VrMV4pJceBEPcBx0Ig8EnTRdSexV1bglhV4W6tCOlGAJHCDc3RrADHGYLPbIHqs0HEywBS4PF7UW51dvsagE2Fs7QN0eRXUWd1dlk3QXfq9a/+7H/S8WgGm1t9nhV2q/fqZhvW4/EpMNM1SUlIgCAIuXgyca3Px4kVkZGR0Kq/VaqENZ3tgTTyQlB/8KQDZKaGdRgSQEeJAjgAg1RRaWQ5ASpTsMBEqUeAhCjRDKNI4joMh3gDg0i+EFkBWamiv1wLIDnGMRwMgO8Tz9vf/tKq/SRqNBlOnTsXOnTv9xxRFwc6dOwNqSISQwU31nsM1a9Zg2bJluOqqqzB9+nRs2rQJNpsN999/v9qhEUIGiOqJaMmSJaitrcW6detQXV2NyZMn4+OPP+7UgU0IGbxUv4+oLwb1fUSEDAKhfkapt5EQojpKRIQQ1VEiIoSoTvXO6r5o694KeaoHIWRAtX02e+qKjulE1NLim0KRkxO7uzkQMhS0tLTAZOr6Dt+YHjVTFAWVlZVISEiImakFbdNSysvLB9VI32C8rsF4TcDAXhdjDC0tLcjKygLfca5aOzFdI+J5HsOGDVM7jLAYjcZB9cvdZjBe12C8JmDgrqu7mlAb6qwmhKiOEhEhRHWUiAaYVqvFk08+Gd4qAlFsMF7XYLwmIDqvK6Y7qwkhgwPViAghqqNERAhRHSUiQojqKBERQlRHiaifPfPMM7j66qsRFxcHs9kc0msYY1i3bh0yMzOh1+sxb948nD59un8D7aWGhgbce++9MBqNMJvNeOCBB2C1Wrt9zdy5c8FxXMDjv/7rvwYo4uBefvll5OXlQafTYcaMGdi/f3+35d966y2MHTsWOp0OEyZMwD//+c8BirR3enNdRUVFnf5fdLoB3uiSkX61bt069vzzz7M1a9Ywk8kU0ms2btzITCYTe++999i3337LbrvtNpafn88cDkf/BtsL8+fPZ5MmTWJ79+5lX3zxBRs5ciS7++67u33NnDlz2EMPPcSqqqr8D4vFMkARd7Zt2zam0WjYli1b2IkTJ9hDDz3EzGYzu3jxYtDyX331FRMEgf32t79lJ0+eZP/v//0/JkkSO3bs2ABH3r3eXldhYSEzGo0B/y/V1dUDGjMlogFSWFgYUiJSFIVlZGSw3/3ud/5jTU1NTKvVstdff70fIwzdyZMnGQD2zTff+I999NFHjOM4VlFR0eXr5syZwx599NEBiDA006dPZ6tWrfJ/L8syy8rKYhs2bAha/s4772S33HJLwLEZM2awhx9+uF/j7K3eXleov5v9iZpmUaa0tBTV1dWYN2+e/5jJZMKMGTOC7n6rhn/9618wm8246qqr/MfmzZsHnuc77UXX0auvvoqUlBSMHz8eBQUFsNvt3ZbvL227DLf/OXe3yzDgu+725QHgpptuipr/FyC86wIAq9WK4cOHIycnBwsXLsSJEycGIly/mJ70OhhVV1cDQNDdb9ueU1t1dTXS0gI3zxJFEUlJSd3GeM8992D48OHIysrC0aNH8fjjj6O4uBjbt2/v75A76e0uw4DvuqP5/wUI77rGjBmDLVu2YOLEibBYLHj22Wdx9dVX48SJEwM2qZxqRGF44oknOnXudXx09Z8ezfr7ulauXImbbroJEyZMwL333outW7fi3XffRUlJSQSvgvTWzJkzsXTpUkyePBlz5szB9u3bkZqaij//+c8DFgPViMLw05/+FMuXL++2zGWXXRbWudt2uL148SIyMzP9xy9evIjJkyeHdc5QhXpdGRkZqKmpCTju9XrR0NAQdIfersyYMQMAcObMGYwYMaLX8fZFb3cZBnz/N70pr4ZwrqsjSZIwZcoUnDlzpj9CDIoSURhSU1ORmhriXr29lJ+fj4yMDOzcudOfeJqbm7Fv3z786Ec/6pf3bBPqdc2cORNNTU04ePAgpk6dCgD47LPPoCiKP7mE4siRIwAQkHAHSvtdhhctWgTg0i7Dq1evDvqamTNnYufOnXjsscf8x3bs2BFVuxKHc10dybKMY8eOYcGCBf0YaQeqdpUPAefPn2eHDx9m69evZwaDgR0+fJgdPnyYtbS0+MuMGTOGbd++3f/9xo0bmdlsZu+//z47evQoW7hwYVQO30+ZMoXt27ePffnll2zUqFEBw/cXLlxgY8aMYfv27WOMMXbmzBn29NNPswMHDrDS0lL2/vvvs8suu4xde+21al0C27ZtG9NqtayoqIidPHmSrVy5kpnNZv/Q9X333ceeeOIJf/mvvvqKiaLInn32Wfbdd9+xJ598MmqH73tzXevXr2effPIJKykpYQcPHmR33XUX0+l07MSJEwMWMyWifrZs2TIGoNPj888/95cBwAoLC/3fK4rCfvGLX7D09HSm1WrZDTfcwIqLiwc++G7U19ezu+++mxkMBmY0Gtn9998fkFxLS0sDrrOsrIxde+21LCkpiWm1WjZy5Ej2s5/9TNX7iBhj7KWXXmK5ublMo9Gw6dOns7179/qfmzNnDlu2bFlA+TfffJONHj2aaTQadsUVV7APP/xwgCMOTW+u67HHHvOXTU9PZwsWLGCHDh0a0HhpGRBCiOpo1IwQojpKRIQQ1VEiIoSojhIRIUR1lIgIIaqjREQIUR0lIkKI6igREUJUR4mIxJSioqKAJXefeuqpgMnAy5cv98+xIrGDEhEJavny5V2uKb1q1SpwHBcwU78/EkBeXh42bdoUcGzJkiU4depUl6958cUXUVRU5P9+7ty5AZNUSXSiRES6lJOTg23btsHhcPiPOZ1OvPbaa8jNzVUlJr1e32lRtvZMJlPImxSQ6EGJiHTpyiuvRE5OTsAKitu3b0dubi6mTJnSp3MHq6ksWrTIX8uaO3cuzp8/j5/85Cf+RdmAzk2zjtrXzJYvX47du3fjxRdf9J+jtLQUI0eOxLPPPhvwuiNHjoDjuAFdg4dcQomIdGvFihUoLCz0f79lyxbcf//9/f6+27dvx7Bhw/D000+jqqoKVVVVvT7Hiy++iJkzZ+Khhx7ynyM3N7fTNQFAYWEhrr32WowcOTJSl0B6gRIR6dYPf/hDfPnllzh//jzOnz+Pr776Cj/84Q/7/X2TkpIgCAISEhKQkZER1iqIJpMJGo0GcXFx/nMIgoDly5ejuLjYv9eXx+PBa6+9hhUrVkT6MkiIaIVG0q3U1FTccsstKCoqAmMMt9xyC1JSUtQOq0+ysrJwyy23YMuWLZg+fTr+93//Fy6XC3fccYfaoQ1ZVCMiPVqxYgWKiorwyiuvRKzWwPM8Oi6F5fF4InLuUDz44IP+jvjCwkIsWbIEcXFxA/b+JBAlItKj+fPnw+12w+Px4KabborIOVNTUwP6fWRZxvHjxwPKaDQayLLcp/fp6hwLFixAfHw8Nm/ejI8//piaZSqjphnpkSAI+O677/xfd8VisfgXxG+TnJyMnJycTmWvv/56rFmzBh9++CFGjBiB559/Hk1NTQFl8vLysGfPHtx1113QarVhNQnz8vKwb98+nDt3DgaDAUlJSeB53t9XVFBQgFGjRkXVAvhDEdWISEiMRiOMRmO3ZXbt2oUpU6YEPNavXx+07IoVK7Bs2TIsXboUc+bMwWWXXYbrrrsuoMzTTz+Nc+fOYcSIEWHvmrJ27VoIgoBx48YhNTUVZWVl/uceeOABuN3uARkFJN2jNavJkPXFF1/ghhtuQHl5eaedUcnAokREhhyXy4Xa2losW7YMGRkZePXVV9UOacijphkZcl5//XUMHz4cTU1N+O1vf6t2OARUIyKERAGqERFCVEeJiBCiOkpEhBDVUSIihKiOEhEhRHWUiAghqqNERAhRHSUiQojq/j+37vJVjE1ynAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:37.687630Z", + "iopub.status.busy": "2024-03-22T21:54:37.687344Z", + "iopub.status.idle": "2024-03-22T21:54:38.041620Z", + "shell.execute_reply": "2024-03-22T21:54:38.040668Z" + }, + "papermill": { + "duration": 0.374366, + "end_time": "2024-03-22T21:54:38.043763", + "exception": false, + "start_time": "2024-03-22T21:54:37.669397", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6EklEQVR4nO3deXzU9Z348dd37skxk4OcJJBwKnJroXiiUg+sYt1Vq7SCWnUrbktZqtL9lRa3NtoKlbYu7XYXKFsoVUt1eyi1aLGiUkRArnKEKwIh5M5k7vl+f39MMpB7MpnkO0nez8djHiQzn/nOe0Lmnc/9UTRN0xBCCB0Z9A5ACCEkEQkhdCeJSAihO0lEQgjdSSISQuhOEpEQQneSiIQQupNEJITQnUnvAHpCVVXOnDlDamoqiqLoHY4QohVN02hoaCA/Px+DoeN6T79ORGfOnKGwsFDvMIQQXSgrK6OgoKDDx/t1IkpNTQXCb9LhcOgcjRCitfr6egoLCyOf1Y7060TU3BxzOBySiIRIYF11nUhntRBCd5KIhBC6k0QkhNBdv+4jioamaQSDQUKhkN6hDGhGoxGTySTTKERMBnQi8vv9nD17FrfbrXcog0JSUhJ5eXlYLBa9QxH9zIBNRKqqcvz4cYxGI/n5+VgsFvlr3Us0TcPv93P+/HmOHz/O6NGjO528JkRrAzYR+f1+VFWlsLCQpKQkvcMZ8Ox2O2azmZMnT+L3+7HZbHqHpIt/lNez/3Q9143NYkiKVe9w+o0B/2dL/jL3ncH+s1ZVjS0HKzhV7eZvR87rHU6/Mrh/c4SIo0qXD39QBaC8zoecSxE9SURCxMl5ly/ytTcQotEvI7XRkkQkRJy4vMHI10VDkgiGVB2j6V8GbGd1fzZz5kwmT57Miy++qHcoohumFWcwsSANVdNItspHqzt0rxGdPn2aL33pS2RmZmK325kwYQIfffSR3mEJ0W2KomC3GCUJxUDXRFRTU8NVV12F2WzmjTfe4MCBAyxfvpz09PRefV1/UO3w1ro63VnZQBRlu2v+/Pls3bqVlStXoigKiqJQUFDAqlWrWpTbtWsXBoOBkydPArBixQomTJhAcnIyhYWFPP7447hcrhbPee+997jmmmuw2+0UFhbyta99jcbGxm7HKKIjndXR0zV1P//88xQWFrJmzZrIfcXFxb3+ui+9c7TDx4qHJHPnlKGR7//r3VICofZ/oQrS7dx9xYWN2VZvO46nVQflNz43pluxrVy5ksOHDzN+/HieeeYZAJYvX86GDRv46le/Gim3fv16rrrqKoYPHw6Eh85//OMfU1xczLFjx3j88cd58skn+c///E8ASktLueWWW/je977H6tWrOX/+PE888QRPPPFEi5+/iN2b+85i19zk1n3CdlcW2QWjuGV8rt5h9Qu61oj+7//+jyuuuIK7776b7OxspkyZwi9+8Qs9Q9Kd0+nEYrGQlJREbm4uubm5zJ07l23btnHq1CkgPGt848aNzJ07N/K8hQsXcv3111NUVMQNN9zA9773PV5++eXI4yUlJcydO5eFCxcyevRorrzySn784x+zbt06vF5vn7/PgSYQUjl4toG6/X8htXIXReWb8QUCeofVb+haIzp27BirVq1i0aJFfOtb32LHjh187Wtfw2KxMG/evDblfT4fPt+FIdL6+vqYXnfB9aM6fMzQahXIo9eO7LBs6xUjD13VO7W5yZMnc+mll7Jhwwaefvpptm7dSkVFBXfffXekzF/+8hdKSkr4xz/+QX19PcFgEK/Xi9vtJikpiT179vDJJ5+wfv36yHM0TYsshbn00kt7JfbBwhMIgaaR4fsUW4oRc8iF1lgJyFbG0dC1RqSqKlOnTuX73/8+U6ZM4dFHH+WRRx7hZz/7WbvlS0pKcDqdkVus+1VbTIYObyajIeqy5ijKxsvcuXPZsGEDABs2bOCWW24hMzMTgBMnTvD5z3+eiRMn8tvf/padO3fy0ksvAeGlLgAul4vHHnuM3bt3R2579uzhyJEjjBzZcbIV0fH6Q1hCjdg134XfIZfMro6WrokoLy+PcePGtbjv0ksvjTRBWluyZAl1dXWRW1lZWV+E2ecsFkubbUvuv/9+9u3bx86dO3n11VdbNMt27tyJqqosX76cz372s4wZM4YzZ860eP7UqVM5cOAAo0aNanOT1fI95wmEsIZcmIwKpqZqtcFbpXNU/Yeuieiqq67i0KFDLe47fPhwpAO2NavVGtmfeiDvU11UVMT27ds5ceIElZWVqKpKUVERV155JQ8//DChUIg77rgjUn7UqFEEAgF+8pOfcOzYMf73f/+3Ta3yqaee4v333+eJJ55g9+7dHDlyhNdff50nnniir9/egOQJhLAEXZiNF2rViq9BRs6ipGsi+sY3vsGHH37I97//fY4ePcqGDRv4r//6LxYsWKBnWLpbvHgxRqORcePGkZWVFakhzp07lz179vCFL3wBu90eKT9p0iRWrFjB888/z/jx41m/fj0lJSUtrjlx4kS2bt3K4cOHueaaa5gyZQpLly4lPz+/T9/bQOXxh7AFGzAZFIxNi3/NQTe+GKZwDEaKpnPK/sMf/sCSJUs4cuQIxcXFLFq0iEceeSSq59bX1+N0Oqmrq2tTO/J6vRw/fpzi4uJBuyVFXxvMP/P3Sys5/9FrTDQcp3jkJew/uJ+ALYMxty8iyTJ4Jzh29hm9mO4/oc9//vN8/vOf1zsMIXrks8WZBOuSMdQkgSOfy/I/BZMJBnES6g7dl3gIMRAYDAoW1YvJYIDkIeE7Q35QpWkWDUlEQsRLwBP+1960REnTICiTRaMhiUiIOPj78WqOnqmg0RcESwqHKn3sOFHN0bOVeofWL0giEiIOjp2r4XytC39IBXMSfsVKUNUIeOUEmWhIIhIiDlR/uFlmMBrBZMVgCY8ahnyyu0E0JBEJEQeqL1zzMViSwosQTeF5XiGfR8+w+g1JRELEgRYIJyKTNRkAxRI+wkoNSNMsGpKIhOghVdXQ/OGEY7SGE5DBEq4RNTfZROckESWgmTNnsnDhQr3DiEi0eBKNP6RiUsPb0xibakIGc1MiCsjwfTQkEQ1Qzdt/iN7nD6mYVS8GBYxNTTO73U6K1USqSY4UioYkogTT3p7VpaWlPPzwwxQXF2O32xk7diwrV65s87w777yTZ599lvz8fMaOHQvA+++/z+TJk7HZbFxxxRW89tprKIrC7t27I8/dt28ft956KykpKeTk5PDlL3+ZysrKDuM5ceJEX/04+oVUq4k5l6UzqTANmmpChVlpTBjq5JIsOXY6GoNrIYymQUiH7TuN5rbbOXagvT2r09PTKSgo4JVXXiEzM5P333+fRx99lLy8PO65557Ic7ds2YLD4eCtt94CwgsOb7/9dmbPns2GDRs4efJkmyZWbW0tN9xwA1/5ylf40Y9+hMfj4amnnuKee+7h7bffbjeerKysOPxQBg5FUbBofjAZwdS02NfYtMeTHr9v/dDgSkShAPxted+/7jX/BqboNh9rvWd1s2XLlkW+Li4u5oMPPuDll19ukYiSk5P57//+78hGZz/72c9QFIVf/OIX2Gw2xo0bx+nTp1vsbvDTn/6UKVOm8P3vfz9y3+rVqyksLOTw4cOMGTOm3XhEK6GmpnDz/7OxqSYU9LVfXrQwuBJRP/bSSy+xevVqTp06hcfjwe/3M3ny5BZlJkyY0GK3xUOHDjFx4sQWW3JMmzatxXP27NnDO++8Q0pKSpvXLC0tZcyY7p1CMhidrfNQc6qCzKCXnKYaUY0Pjp+qIWBTmH65zgH2A4MrERnN4dqJHq/bAxs3bmTx4sUsX76cGTNmkJqayg9/+EO2b9/eolxycnK3r+1yubj99tt5/vnn2zyWl5cXc8yDSZXLz7mqOgxmPzlNTTLFbMMXVAn5pUYUjcGViBQl6iaSnlrvWb1t2zauvPJKHn/88ch9paWlXV5n7Nix/OpXv8Ln82G1hpsKO3bsaFFm6tSp/Pa3v6WoqAiTqf1fh/b20BYX+EMqRs2H0aBE+ojMTUs8CPrQNA0lyj7CwUpGzRJQ6z2rR48ezUcffcTmzZs5fPgw3/72t9sklPbcf//9qKrKo48+ysGDB9m8eTMvvPACQOSDsWDBAqqrq7nvvvvYsWMHpaWlbN68mQcffDCSfNrbQ1tc4A+qmFR/UyIKJ3xTUxPZqAbanB4s2pJElIBa71l98803c9ddd3Hvvfcyffp0qqqqWtSOOuJwOPj973/P7t27mTx5Mv/+7//O0qVLASL9Rvn5+Wzbto1QKMRNN93EhAkTWLhwIWlpaRia9l7uaA9tEeYLhDCqgRaJyGxp3lNcIxiQOV1d0X3P6p6QPau7b/369Tz44IPU1dW12IA/Hgbrz/wve8tI/vtKCtPtFMxZGu4T1DQ+XPMkmqZx2d3/jsORrneYuug3e1aL3rVu3TpGjBjB0KFD2bNnT2SOULyT0GAW8oeXcRiNRjA0faQUBc1ogaBPOqyjIIlogCsvL2fp0qWUl5eTl5fH3XffzbPPPqt3WANKsGmFvWK2tZi4arfbMfpDKCFpmnVFEtEA9+STT/Lkk0/qHcaAduPoNKhPw5ic2uL+ycOzwG0Eq4yYdUUSkRA9ZCMIZiNYWzV3m5tpUiPqkq6jZt/97ncjCymbb5dccomeIQnRfc0ndRhbLXBtXm+mBvs2nn5I9xrRZZddxl/+8pfI9x1NqotVPx4U7HcG689657FzDKluJC/NzMXTZfeWNxI4X8OQnDqGyTrhTumeiEwmU68spjSbw8sq3G63jBD1Ebc73Gnb/LMfDDRN49iZKvy1XrINLWft+1QjgaCK3y+bo3VF90R05MgR8vPzsdlszJgxg5KSEoYNG9ZuWZ/Ph893YSi0vr6+w+sajUbS0tKoqKgAICkpSabZ9xJN03C73VRUVJCWlhYexh4kgqqG0rQ7o8nacu6U0tQ0U4OyFUhXdE1E06dPZ+3atYwdO5azZ8+ybNkyrrnmGvbt20dqamqb8iUlJS22w+hKc02rORmJ3pWWljbotgppXt6hACZLy5q3YgrXDNWgdFZ3RddEdOutt0a+njhxItOnT2f48OG8/PLLPPzww23KL1myhEWLFkW+r6+vp7CwsMPrK4pCXl4e2dnZBALyV6k3mc3mQVUTauYPqhg1PwaDgmJqWSMySCKKmu5Ns4ulpaUxZswYjh492u7jVqs1soq8O4xG46D8kIje5w+pGFU/povWmTUzSNMsagm16NXlclFaWir74Ih+o72V982Upi1nVNkutku6JqLFixezdetWTpw4wfvvv88XvvAFjEYj9913n55hCRE1XzBcIzIqbRORzWrFbjZiVWQeUVd0bZp9+umn3HfffVRVVZGVlcXVV1/Nhx9+KJuzi35jeGYSQ4tSUbxqmwmNo/PSoT4Nhsj0ka7omog2btyo58sL0WNmowGzEggv8WjVWS0neUQvofqIhOh3NK3tCR7NDE0TO2WtWZcSatRMiP7maHkNlqpGnHYzzlZNszMNQc5/Wosh1coEOcmjU5KIhOiB05U1WGo9YDDgbHVaS0gx4faHMHlliUdXpGkmRA8Em3ZfNJisbU7zNZqb+4ikadYVSURC9EAoEK7tGMxtJ9qamhORKp3VXZFEJEQPRParNrc9LMDU1HmthYLhTm3RIUlEQvRA88b47daImpYjaWoINDnbrDOSiIToATXgAcBoaTtpsblGpGqgBuQkj85IIhKiB9Rgc9OsnRqRyYzZZMRmNhCSDutOyfC9ED0wa3Q6itmJNSmpzWMWk4ErRuRA0AdI06wzkoiE6IEUUwisJrB0cLKt0RxORFIj6pQ0zYToiWBT30/rdWbNIss8ZAi/M5KIhIiRxx/iH6crOVPnaXuUUJM9Zxv55NNaalyNfRxd/yKJSIgYuXxBTpTXcLbW03bBa5MGPzT6QwT80jTrjCQiIWLUvE2s0aB0WCNSmtafhWTP9E5JIhIiRs0b5xsNhja7MzaLJCLpI+qUJCIhYuQPXlQj6iARYZSTPKIhiUiIGIUTUQCDwoXdGFtpPlIoKImoU5KIhIiRLxDAqAUwRdE00+RIoU7JhEYhYhRoWvDaWWe12WxFMyoYNDnJozOSiISI0aQ8G+pQJyaTGYztf5SmFg0BJQPSu38w6GAiiUiIGCUZmpd3dHJcUPP2sbI5WqcSpo/oueeeQ1EUFi5cqHcoQkSneXlHB80yQJZ4RCkhEtGOHTv4+c9/zsSJE/UORYioHT5TxelaD26144bF8Rof+8/UUXqutu8C64d0T0Qul4u5c+fyi1/8gvT0dL3DESJqR86c51S1m8aQscMy7qBCvTeI2yMneXRG90S0YMECbrvtNmbNmtVlWZ/PR319fYubEHpp3ibW2NEWIIDSvEtjSEbNOqP7kdMff/wxO3bsiKp8SUkJy5Yt6+WohIhO8/avJkvHfUQXNtCXCY2d0a1GVFZWxte//nXWr1+PzdbxX5SLLVmyhLq6usitrKysl6MUon2apqEGmxNRx6NmBrN0VkdDtxrRzp07qaioYOrUqZH7QqEQ7777Lj/96U/x+XwYjS3b3larFatV5mMI/YVUDUPTftWdJSJj0xIPTRJRp3RLRDfeeCN79+5tcd+DDz7IJZdcwlNPPdUmCQmRSHxNC14VwGztJBEZm5tmkog6o1siSk1NZfz48S3uS05OJjMzs839QiQaf1DFpHoxGBQUcyeJyGzGqICJUB9G1//IzGohYpBqM3FVUQrUp3a8XzUwNMPB0OLM8DeaBorSRxH2LzElomPHjjFixIh4x8Jf//rXuF9TiN5gMhpwmEJgM0M7x01HNC/xAFCDLb8XETGNmo0aNYrrr7+eX/3qV3i9MlFLDFJNp7x2ViOKLPEAGTnrREyJ6OOPP2bixIksWrSI3NxcHnvsMf7+97/HOzYhEta5Og+nK2upcfs7TUTekMbBc40cOFuPJgtfOxRTIpo8eTIrV67kzJkzrF69mrNnz3L11Vczfvx4VqxYwfnz5+MdpxAJ5Ux1HaeqXFS6fJ3XiIAqj0qdJyAb6HeiRxMaTSYTd911F6+88grPP/88R48eZfHixRQWFvLAAw9w9uzZeMUpREIJeMPNMoPR1Gm/j9loIKSEu2Jlu9iO9SgRffTRRzz++OPk5eWxYsUKFi9eTGlpKW+99RZnzpxhzpw58YpTiIQS9IcTkWK2dzoSZlBAM4QTUUi2i+1QTKNmK1asYM2aNRw6dIjZs2ezbt06Zs+ejcEQzmvFxcWsXbuWoqKieMYqRMII+tyYAUNnI2aAoijhGlMAQk1r00RbMSWiVatW8dBDDzF//nzy8vLaLZOdnc3//M//9Cg4IRJV0OfBDBg7mVXdTJEaUZdiSkRvvfUWw4YNi9SAmmmaRllZGcOGDcNisTBv3ry4BClEomlumnW2zizC0Hzaq/QRdSSmPqKRI0dSWVnZ5v7q6mqKi4t7HJQQiU71uQEwWZK6LKuYzBgUUGUeUYdiqhFpmtbu/S6XK+otPYToz2YUpQAO7I7ULsteMzYP5Xw9JMlC7o50KxEtWrQICHfALV26lKSkC38NQqEQ27dvZ/LkyXENUIhE5DSFwG4GWxQ1IjnJo0vdSkS7du0CwjWivXv3YrFcOGbXYrEwadIkFi9eHN8IhUhEgXDTDHPXiSiyzEOV7WI70q1E9M477wDhfYNWrlyJw+HolaCESGTeQIgzZ86T4vGS3ckWIM0OV3pQy+txpjWQV9T78fVHMfURrVmzJt5xCNFvNHiDHD97HqfqJjuKGlG1BzR3ALPb0wfR9U9RJ6K77rqLtWvX4nA4uOuuuzotu2nTph4HJkSi8gZCmENeTEZDVE0zxWRGQ0bNOhN1InI6neFZok1fCzFY+QJBTKoXk8UIUQzfG00mVECVCY0dijoRXdwck6aZGMx8Xg8KGiaDEl2NSPat7lJMExo9Hg9utzvy/cmTJ3nxxRf585//HLfAhEhUAU8D0LTOzND13KDmkzykRtSxmBLRnDlzWLduHQC1tbVMmzaN5cuXM2fOHFatWhXXAIVINAFv+I+wEkWzDMBgkhpRV2LeofGaa64B4NVXXyU3N5eTJ0+ybt06fvzjH8c1QCESTcDbVCOyJkdV3hCZ0ChrzToS0/C92+0mNTU8tf3Pf/4zd911FwaDgc9+9rOcPHkyrgEKkWgm5FhQqx0YMzOiKn9pQQZKZQaGlJRejqz/innz/Ndee42ysjI2b97MTTfdBEBFRYVMchQDXqriw2k3k5IS3e+60WTBoCiyeX4nYkpES5cuZfHixRQVFTF9+nRmzJgBhGtHU6ZMiWuAQiQcnyv8rzXKGo5B1pp1JaZE9M///M+cOnWKjz76iDfffDNy/4033siPfvSjqK+zatUqJk6ciMPhwOFwMGPGDN54441YQhKiz5SeLudcvZegKbo+okqPypGKBo5X1PZuYP1YzCe95ubmkpub2+K+adOmdesaBQUFPPfcc4wePRpN0/jlL3/JnDlz2LVrF5dddlmsoQnRawIhleOny0n2N5JuiS4ReUMKlS4/SRZVTnvtQEyJqLGxkeeee44tW7ZQUVGBqqotHj927FhU17n99ttbfP/ss8+yatUqPvzwQ0lEIiG5/SHMIQ8GBcz26PqIDM3ziFQV1BAY5aT31mL6iXzlK19h69atfPnLXyYvLy+y9KMnQqEQr7zyCo2NjZE+p9Z8Ph8+34UNyOvr63v8ukJ0h9vnxxxyYzYZUKxdb4oGYDKH5xGpmhbuJ5JE1EZMP5E33niDP/7xj1x11VU9DmDv3r3MmDEDr9dLSkoKv/vd7xg3bly7ZUtKSli2bFmPX1OIWHkaXeHlHUYTmKNrmplNZjQUVI3wyFkUW4cMNjF1Vqenp5OREd0ciq6MHTuW3bt3s337dr761a8yb948Dhw40G7ZJUuWUFdXF7mVlZXFJQYhouV31wFNkxkN0X18jEYF1WBqqhHJ5mjtiSkR/cd//AdLly5tsd4sVhaLhVGjRnH55ZdTUlLCpEmTWLlyZbtlrVZrZISt+SZEX/I3hhORYouuWQZgNhhQFROqJuvNOhJT02z58uWUlpaSk5NDUVERZnPLI3c//vjjmANSVbVFP5AQiSTQWAOA0Z4W9XNMRgX1omOnLV2UH4xiSkR33nlnXF58yZIl3HrrrQwbNoyGhgY2bNjAX//6VzZv3hyX6wsRb5emqwTyHCi5OVE/x2RQuGpMHkaPGQVpmrUnpkT0ne98Jy4vXlFRwQMPPMDZs2dxOp1MnDiRzZs387nPfS4u1xci3pJCrvDpHWlZUT9HURTMFgt4lfDwvWgj5nHE2tpaXn31VUpLS/nmN79JRkYGH3/8MTk5OQwdOjSqa8iR1KLf8Yb7iOhG0wyQZR5diCkRffLJJ8yaNQun08mJEyd45JFHyMjIYNOmTZw6dSqyV5EQA4kaUik7fQa7IUiGObVbH55D5z0YaxrIKfIQfTf34BHTqNmiRYuYP38+R44caXGy6+zZs3n33XfjFpwQicTtaeRMVR3Hzjdi6GaN6GxDkPMuPx6ft3eC6+diSkQ7duzgsccea3P/0KFDKS8v73FQQiQid101AIo1BYO5m2NfTbOpZfi+fTElIqvV2u7yisOHD5OVFX0nnhD9iaehCgDF3v1TbJo30A8FZZfG9sSUiO644w6eeeYZAoFwdlcUhVOnTvHUU0/xT//0T3ENUIhE4a8/D4AhqfurCpSm7WJl3+r2xZSIli9fjsvlIisrC4/Hw3XXXceoUaNITU3l2WefjXeMQiSEYEM4ERlThnT7uc0r8EPSNGtXTKNmTqeTt956i23btrFnzx5cLhdTp05l1qxZ8Y5PiIQRcoUTkdmR3e3nNteIpI+ofd1ORKqqsnbtWjZt2sSJEydQFIXi4mJyc3PRNC0uW4IIkXA0DbUx3EeUlB79rOpmBklEnepWItI0jTvuuIM//elPTJo0iQkTJqBpGgcPHmT+/Pls2rSJ1157rZdCFUJH3jrGZNnwBm1YcnK7Lt/K1OJsDMF0DOnmrgsPQt1KRGvXruXdd99ly5YtXH/99S0ee/vtt7nzzjtZt24dDzzwQFyDFEJ37ipsJiM25xCwdn/ZqtViAaMBNFni0Z5udVb/+te/5lvf+labJARwww038PTTT7N+/fq4BSdEwnCHm2UkZcb2fKMs8ehMtxLRJ598wi233NLh47feeit79uzpcVBCJJrayjOU1bg5G4ztkMTT9QGOV7o4U90Q58gGhm41zaqrq8nJ6bijLicnh5qamh4HJUSiqa0s52yNh6DLQl4Mz6/yaDTW+3Ba3OTHPbr+r1s1olAohMnUce4yGo0Eg7LfihhgNI1g09C91dn9ETO4MI9IJjS2r9ujZvPnz8dqtbb7uOysKAakgJuA1w0oJHVjH6KLGU3hDm5JRO3rViKaN29el2VkxEwMOO4qfIEQPlMyBclJMV3C2LydckhaDO3pViJas2ZNb8UhRMJSG6vwBVW81jScSbHNA4rUiFS/nPbajpjWmgkxmHjqz6Nq4LOkkWKJbVNTU6RppoKmdlF68JFEJEQXvLVNi12TMzAYYqvJNDfNIocsihbk7FshupCuNDClMA3XJWNivkaOM5khw9IxKjRNarR19ZRBRRKREJ1RQxi8ddjMRmzZscwgCjObjJit1nBtSGpEbUjTTIjOeGrDfTpGM1h7eLJwZJmHjJy1pmsiKikp4TOf+QypqalkZ2dz5513cujQIT1DEqIlTzVlNW5KG63U+2JPIP6gSmm1j+OVjWgh2S62NV0T0datW1mwYAEffvghb731FoFAgJtuuonGxkY9wxLiAncV5+q9HKw14QvEPtqlahona/yU13sJBaRp1pqufURvvvlmi+/Xrl1LdnY2O3fu5Nprr9UpKiEu8DdUEghpeMxpOOyxf1zMRgOqEn5+KBSQztlWEurnUVcXPkUzI6P9zcl9Pl+LZSTtnSQiRDx56yrCX9jTsZqMMV/HoIBqCH/cgkE/7S+SGrwSprNaVVUWLlzIVVddxfjx49stU1JSgtPpjNwKCwv7OEox2PjqKwEwp/bsmCxFUS4cKRSQPqLWEiYRLViwgH379rFx48YOyyxZsoS6urrIraysrA8jFINOwEPA4wII78zYQ4pBDlnsSEI0zZ544gn+8Ic/8O6771JQUNBhOavV2uHKfyHizl2FNxjCb0zGkZzc8+s1nfYqNaK2dK0RaZrGE088we9+9zvefvttiouL9QxHiJbc1fgCKl6zE4e955veG+S01w7pWiNasGABGzZs4PXXXyc1NZXy8nIgfG6a3W7XMzQhwF3F2NxUCoaMwZwd2xaxF5s2KhtTUhpme8L0iCQMXX8iq1atoq6ujpkzZ5KXlxe5/eY3v9EzLCHCPNUYFIXU9Bxs5thHzJql2O3YzEaMcpJHG7rWiDRN0/Plheicuzr8bwxn3bdLTvLoUEJ0VguRcFSV+poKKmrd+OtMjIvxFKGLnawNoFS7SUl1E6fUNmBIY1WI9nhrafT4ONcYpLS+580ygLLaAKdrPdS53HG53kAiiUiI9nhq8AZVvCYnacnxmTIS2RxNtgFpQxKREO1p2jDfa3aSZu/+EdPtMZjD11Fl+L4NSURCtMddjTcQwmNKIy3GDfNbMzd1VodkZnUbkoiEaIfaWIkvqOIxO2M+uaM1k0VqRB2RRCREO3xNJ3cELOmkWuMzuGw0h/ep1oJyEGlrkoiEaC3ow+9uQAEsjiEocTqDzGwJJyI1IImoNZlHJERr7iqcdjOfGVvIZVeMiNtlC7PSGDrUidFkkEMWW5FEJERrTTOqjcmZpNri0z8EYLcnQXMzLxQAU3xG4wYCaZoJ0Zq7KvxvUhymU1/MaL5QCwpJ8+xikoiEaM1TzdEKF9vLNerc8Rtq9wZVyupDlNW4QUbOWpBEJEQramMVVY0+DtTFr1kGEAiplFYHOF3jQQt643rt/k4SkRAXU9XI0H3Qmt6jkztas5gMhAxmNCAoI2ctSCIS4mLeWjw+P6pixO7IjNvQPYDFaCBkCHdQB3yeuF13IJBEJMTF3FV4/CG8pjQyU2xxvbSiKCim8ALakF+aZheTRCTExRor8QRCuM3ppCfHf3i9OREFJBG1IIlIiIu5K/H4Q3jMaWT2RiJq2kA/GJBEdDFJREJczF2Fqml4eqlGZGhabxbyS2f1xWRmtRDNNA0aK5lUkMalU6dgSop/Ipo8IhuzyYlVDqlpQRKREM28deGlF4oBS2pm+MD6OEtPTQ0v89BkT6KLSdNMiGauivC/yZlgiM8+1W009RHJzOqWJBEJ0cxVzpGKBt47Z+Z0be/M8znn0Thd66Gyrr5Xrt9f6ZqI3n33XW6//Xby8/NRFIXXXntNz3DEYNdQTr0nwCm/g97aoON0fYhT1W4q6xp66RX6J10TUWNjI5MmTeKll17SMwwhQNPw1pzGH9JwWzMZkhKfkztaM1nCvdSazCNqQdfO6ltvvZVbb71VzxCECPPU0FBfh6YYSM4YisXUO3+jLTY7jYAakCUeF+tXo2Y+nw+f78L8i/p6aWeLOKk9Rb0ngMuSTcEQR6+9jNmWAoAW8IGqgkG6aaGfdVaXlJTgdDojt8LCQr1DEgOEVnuSOk+AemseBelJvfY6Vnv42kFVhaDUipr1q0S0ZMkS6urqIreysjK9QxIDgarScLYUX1DFnZTP0LTem21oNZsJGSwEVQ1kmUdEv2qaWa1WrNbe6UQUg1j9pxgCjThTU8geNqbX+ocAbGYDQYOVYNCPFmhEIc7b0fZT/SoRCdEryveSYjUxbsIULr0kv1dfKsliYmJxLla3QWpEF9E1EblcLo4ePRr5/vjx4+zevZuMjAyGDRumY2Ri0PDWoZbvD/dR5E+O60Zo7TEaFDKcaRCqBtkuNkLXRPTRRx9x/fXXR75ftGgRAPPmzWPt2rU6RSUGi6Dfx9GtvyFUWc2YseMwO4b2zQs3rcBHhvAjdE1EM2fORNM0PUMQg5Gqcu7I3zn18Z8JuutRFRPH0q9mbB8deHjapUGtB2djAyl98oqJT/qIxOBSfZyyv7/O6dOfogGaNZWh0+6iaGT8TnTtyuHqELZqN4X1koiaSSISg4OmQenbnN23lU+r3AQNNswjr2LK9Buw9fFIrMWWDEDA4+rT101kkojEwBcKwj9+T+2JTzhZ5eZcyjjypt7CtFF5vd453R6zLZkQEPQ19vlrJypJRGJgC3hg32+htgyLxUL50JtIHjaJaaOydUlCABZ7Kh4g5JUaUTNJRGLgcleHk1BjJZgsJE3+J25JKcRsNOiWhAAsyU48gOqTrUCaSSISA48aglMfwsn38Qf81IWsZH3my5CSTXxPKouNNSW8qDbk94V3ajTFf2/s/kYSkRgwfMEQFWWlmI6+CY2V+IIqpaFsDjmv5lZvMsUJMkSVZE8ipJgJhELgd4EpQ++QdCeJSAwI+0+dp3T7H8iq3w9AwGDnRPqVVDlGMsRhIz3JrHOEF2QkW7AW5WML1IYTUZIkIklEov9rrGT4iZdpqD+O1WTAO2Q87qHXUGBL4rPpSYzKSsHQCydyxMpqMmJNz4BaF/ikwxokEYn+ruYk7PstKSEfE0YWkDThDpTMvpucGDNrUzvRL4kIJBGJfqz81GFSD/2WZBPgLCB5/D+Bpfc2NYunco+JYI2blJpq0mR/P0lEon/yVp/m1F9/iRb0MfKSyQyZdC8YE6cfqCulDUYMNR5yqytI0zuYBNCvdmgUAkBrrOL422sIBXwEUwtwXnF3v0pCQPgkWcDvqtI5ksQgiUj0L74Gzr33S2rranFbhjDqxvmYLf1v1067MwsAtbE6vA5ukJNEJPqPgBfPx7+m7MxZvCYHqdPuJycjTe+oYpLizERDwe/3g09Oo5FEJPqHUBB176uUlh7Do9ioHvUFrhjdf3t505Kt+EwOvIFQuFY0yEkiEolPDcHB16k+fZTagMKxvNnMmnJJQs0N6i6n3UzAmoaqQX31Ob3D0Z0kIpHYQkHY/zs4f5jMVDvWSf/M1VPG40ygmdKxUBQFizMXAFfVpzpHoz8ZvheJK+DBs/tVqDmJ3WpFGX8Xn8kcqXdUcTPhkrEkGQ9hM9bpHYruJBGJhOPxhzh17CD+/X/AXVdJclISl33u/v4xY7obMvOK4IQxvE1J0Aem/jf6Fy+SiERCqHMHOFFRTfmJQ6hndpPmOQWAz5TCucI5jHYOZ8B9TK2pYE8DT214qUrWGL0j0o0kItFnNE2j0R+ixuXFXXGcUZYajP468NTw6bFTuOtqcTSVTbKasBZOJXPizQxJc+i6kVmvURQ+NeTTeK4MR/I+8iQR6eull17ihz/8IeXl5UyaNImf/OQnTJs2Te+wEldjZfgvqKcGFKjFQRk5VKtJjMtzkJXa/bpDTaOf8novgZCK2WggLclMdqoNYxxGpurcAQ6cqePkicNYqw6R6TmGOeTBV+AkyRL+FRxiDoDNhCM9i4xhl5E8Ytqg2B6j3DKcQOPfCB7fTd6U2RfOPBtkdE9Ev/nNb1i0aBE/+9nPmD59Oi+++CI333wzhw4dIjs7W+/wEkfQBxUH4OwnUH+GoKpS5fJT0eDD5QsC4Ldk45s8HUZPAZuD6kY/p2s8jMhKJtl64b9aC3hw11eTrLnBWw+Kwjt7q6gM2vGY01EN4REpi8lAQbqdEUNSuCzf0e3h8k+r6tmz/wC+8kNkuo8xPNSIAljNBsxJKQSzxkFmHtjTybGlkWNPH3QfxFGjL+XjT9LAVUvF/nfInnyr3iHpQtF0PuFw+vTpfOYzn+GnP/0pAKqqUlhYyL/+67/y9NNPd/rc+vp6nE4ndXV1OByOTsv2O6FgeMZt/WmoPgbnD4MaxBcMcbLGy4ngEBrMmSiaSmqgkgJDFUlmA9kOG3aLCVKy2V9n4UilHwMh0k0BUhQPBl8dfq8HDfjM8PRIk+d4pQu3P4TJaMBjSKESB3WKE7c5A2uyk7s/OwrFYAI1SEWtixRjADs+lIAHLeAm6HPjbazHEPSSbPBDwE1dQyMHztajEJ43k5mWSsbwCZhyx0F6ERiMev6EE8YH2z+Afb/FbFTInTiL3EuvxGRP1TusuIj2M6prjcjv97Nz506WLFkSuc9gMDBr1iw++OCDNuV9Ph8+ny/yfX19lFPjq49D6ZZ21/S4/UH2nq5D0TSaH1W48HWe08qw9PA5VN5giD1lNReefNFzAHJSrRQPaTqzKhhi56lwWaXV62pAVoqFkVkpgEZI1dhxouWaI5N64Vz09KSmskmZMOQy/nwkmYAxiSEpFsblO7kkN5VkvHD+H1BxEOo+hYZzDHF5UQMXakwXn7SuGm24bdkkO9JB0yhO84Y3m/c3NsVYj9tXQ437MMZGBeWjvzW9ZY1jx6sj77u5kqQ23ZHjsDJiSHivHYfNRGFOJhnDxpGUdylkjACj7pXwhHP55dPZdv4EnNtJ2a63+HTXWwzLySR/SBooBuq9IQ6WNwDt10gLM5LITwvXJF2+IPvPdPy5GJpmpyDdDoDbH2Lv6Y6nDuQ6bQzPCG+rElQ1TK1rxBPuAVt8KgC6/lZUVlYSCoXIyclpcX9OTg7/+Mc/2pQvKSlh2bJl3X+hoA9c59t/zB9EdXX8n6GZbGDzN10nhOqu7fh1zFbwhsJfh1Q0b/iUhvaqnJrFCv4Ln2LN727xeABQFRNucwbB9GJGXn41pOZiVRRmmKopSE8iO9V6USduChRcEb5566HhLDnuanLUAO4g1AXNNJKEwe4kJS2DTEdq+/0/fjc0nkdxV5LcWEly4/nwfUEvaCH8qgHsKo2qiYBiJ2C0ETTYCBjtYLajZWcy4tJhYE5CMSdRYLLCQOxojiOLycDVt9zDvl3DcB19D5P7PMaQNzyaBmieAKqrkz+6liQwhZMLviBqQ2e/z3YwN+3Z5O+8LAYbWMJ/WFFVMLSa/6ypXbyz6OnaNDtz5gxDhw7l/fffZ8aMGZH7n3zySbZu3cr27dtblG+vRlRYWNh108zfCK6K8NctPhQK/pDG+QZfOw+Fv0m2mnDaw6csBEMqlY1+NJQLf5sUJfI8u8mEwx7uXwlpGtWNgRbXu/jyFrMRhy1cVgOqGv1NxcKlNHMSiskOioLFZCDVllgziYMhFX9IJdhUFbKZjJiNysAc3epDmqbR6G7EFHBhU0KgqfgCQapc3g6fk2Iz4WjqA/QFVSpd/g7Lptou/N5d/Lvf7nWtRpxNv8+qdqH2G+Es6HL7lX7RNBsyZAhGo5Fz51qutTl37hy5ubltylutVqyxHA9sSYaM4vYfAoYOie4yJiA3yoEcI5DljK6sAgxJkBMmomUyGjAZZYVQvCmKQkpyCnDhF8IK5GdF93wrMDTKMR4LMDTK6/b2/7Suv0kWi4XLL7+cLVu2RO5TVZUtW7a0qCEJIQY23XsOFy1axLx587jiiiuYNm0aL774Io2NjTz44IN6hyaE6CO6J6J7772X8+fPs3TpUsrLy5k8eTJvvvlmmw5sIcTApfs8op4Y0POIhBgAov2MSm+jEEJ3koiEELqTRCSE0J3undU90dy9FfVSDyFEn2r+bHbVFd2vE1FDQ3gJRWFh/z3NQYjBoKGhAaez4xm+/XrUTFVVzpw5Q2pqar9ZWtC8LKWsrGxAjfQNxPc1EN8T9O370jSNhoYG8vPzMbReq3aRfl0jMhgMFBQU6B1GTBwOx4D65W42EN/XQHxP0Hfvq7OaUDPprBZC6E4SkRBCd5KI+pjVauU73/lObLsIJLCB+L4G4nuCxHxf/bqzWggxMEiNSAihO0lEQgjdSSISQuhOEpEQQneSiHrZs88+y5VXXklSUhJpaWlRPUfTNJYuXUpeXh52u51Zs2Zx5MiR3g20m6qrq5k7dy4Oh4O0tDQefvhhXC5Xp8+ZOXMmiqK0uP3Lv/xLH0XcvpdeeomioiJsNhvTp0/n73//e6flX3nlFS655BJsNhsTJkzgT3/6Ux9F2j3deV9r165t8/9is/XxQZea6FVLly7VVqxYoS1atEhzOp1RPee5557TnE6n9tprr2l79uzR7rjjDq24uFjzeDy9G2w33HLLLdqkSZO0Dz/8UPvb3/6mjRo1Srvvvvs6fc51112nPfLII9rZs2cjt7q6uj6KuK2NGzdqFotFW716tbZ//37tkUce0dLS0rRz5861W37btm2a0WjUfvCDH2gHDhzQ/t//+3+a2WzW9u7d28eRd66772vNmjWaw+Fo8f9SXl7epzFLIuoja9asiSoRqaqq5ebmaj/84Q8j99XW1mpWq1X79a9/3YsRRu/AgQMaoO3YsSNy3xtvvKEpiqKdPn26w+ddd9112te//vU+iDA606ZN0xYsWBD5PhQKafn5+VpJSUm75e+55x7ttttua3Hf9OnTtccee6xX4+yu7r6vaH83e5M0zRLM8ePHKS8vZ9asWZH7nE4n06dPb/f0Wz188MEHpKWlccUVV0TumzVrFgaDoc1ZdK2tX7+eIUOGMH78eJYsWYLb7e60fG9pPmX44p9zZ6cMQ/h9X1we4Oabb06Y/xeI7X0BuFwuhg8fTmFhIXPmzGH//v19EW5Ev170OhCVl5cDtHv6bfNjeisvLyc7u+XhWSaTiYyMjE5jvP/++xk+fDj5+fl88sknPPXUUxw6dIhNmzb1dshtdPeUYQi/70T+f4HY3tfYsWNZvXo1EydOpK6ujhdeeIErr7yS/fv399micqkRxeDpp59u07nX+tbRf3oi6+339eijj3LzzTczYcIE5s6dy7p16/jd735HaWlpHN+F6K4ZM2bwwAMPMHnyZK677jo2bdpEVlYWP//5z/ssBqkRxeDf/u3fmD9/fqdlRowYEdO1m0+4PXfuHHl5eZH7z507x+TJk2O6ZrSifV+5ublUVFS0uD8YDFJdXd3uCb0dmT59OgBHjx5l5MiR3Y63J7p7yjCE/2+6U14Psbyv1sxmM1OmTOHo0aO9EWK7JBHFICsri6ysKM/q7abi4mJyc3PZsmVLJPHU19ezfft2vvrVr/bKazaL9n3NmDGD2tpadu7cyeWXXw7A22+/jaqqkeQSjd27dwO0SLh95eJThu+8807gwinDTzzxRLvPmTFjBlu2bGHhwoWR+956662EOpU4lvfVWigUYu/evcyePbsXI21F167yQeDkyZParl27tGXLlmkpKSnarl27tF27dmkNDQ2RMmPHjtU2bdoU+f65557T0tLStNdff1375JNPtDlz5iTk8P2UKVO07du3a++99542evToFsP3n376qTZ27Fht+/btmqZp2tGjR7VnnnlG++ijj7Tjx49rr7/+ujZixAjt2muv1estaBs3btSsVqu2du1a7cCBA9qjjz6qpaWlRYauv/zlL2tPP/10pPy2bds0k8mkvfDCC9rBgwe173znOwk7fN+d97Vs2TJt8+bNWmlpqbZz507ti1/8omaz2bT9+/f3WcySiHrZvHnzNKDN7Z133omUAbQ1a9ZEvldVVfv2t7+t5eTkaFarVbvxxhu1Q4cO9X3wnaiqqtLuu+8+LSUlRXM4HNqDDz7YIrkeP368xfs8deqUdu2112oZGRma1WrVRo0apX3zm9/UdR6RpmnaT37yE23YsGGaxWLRpk2bpn344YeRx6677jpt3rx5Lcq//PLL2pgxYzSLxaJddtll2h//+Mc+jjg63XlfCxcujJTNycnRZs+erX388cd9Gq9sAyKE0J2MmgkhdCeJSAihO0lEQgjdSSISQuhOEpEQQneSiIQQupNEJITQnSQi0a+sXbu2xU6X3/3ud1uswZs/f35kaYPoPyQRiXbNnz+/w61cFyxYgKIoLRbI9kYCKCoq4sUXX2xx37333svhw4c7fM7KlStZu3Zt5PuZM2e2WBsmEpMkItGhwsJCNm7ciMfjidzn9XrZsGEDw4YN0yUmu93eZi+kizmdzqj3BheJQxKR6NDUqVMpLCxssXHZpk2bGDZsGFOmTOnRtdurqdx5552RWtbMmTM5efIk3/jGNyJ7IUHbpllrF9fM5s+fz9atW1m5cmXkGsePH2fUqFG88MILLZ63e/duFEXp060vxAWSiESnHnroIdasWRP5fvXq1Tz44IO9/rqbNm2ioKCAZ555hrNnz3L27NluX2PlypXMmDGDRx55JHKNYcOGtXlPAGvWrOHaa69l1KhR8XoLohskEYlOfelLX+K9997j5MmTnDx5km3btvGlL32p1183IyMDo9FIamoqubm5MW0+5nQ6sVgsJCUlRa5hNBqZP38+hw4dihyxEwgE2LBhAw899FC834aIkmyMJjqVlZXFbbfdxtq1a9E0jdtuu40hQ4boHVaP5Ofnc9ttt7F69WqmTZvG73//e3w+H3fffbfeoQ1aUiMSXXrooYdYu3Ytv/zlL+NWazAYDLTegSYQCMTl2tH4yle+EumIX7NmDffeey9JSUl99vqiJUlEoku33HILfr+fQCDAzTffHJdrZmVltej3CYVC7Nu3r0UZi8VCKBTq0et0dI3Zs2eTnJzMqlWrePPNN6VZpjNpmokuGY1GDh48GPm6I3V1dZF9qJtlZmZSWFjYpuwNN9zAokWL+OMf/8jIkSNZsWIFtbW1LcoUFRXx7rvv8sUvfhGr1RpTk7CoqIjt27dz4sQJUlJSyMjIwGAwRPqKlixZwujRoxNq3+nBSGpEIioOhwOHw9Fpmb/+9a9MmTKlxW3ZsmXtln3ooYeYN28eDzzwANdddx0jRozg+uuvb1HmmWee4cSJE4wcOTLmwwoWL16M0Whk3LhxZGVlcerUqchjDz/8MH6/v09GAUXnZKtYMWj97W9/48Ybb6SsrKzNgYSib0kiEoOOz+fj/PnzzJs3j9zcXNavX693SIOeNM3EoPPrX/+a4cOHU1tbyw9+8AO9wxFIjUgIkQCkRiSE0J0kIiGE7iQRCSF0J4lICKE7SURCCN1JIhJC6E4SkRBCd5KIhBC6k0QkhNDd/wdS+FLce/7raQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:38.081795Z", + "iopub.status.busy": "2024-03-22T21:54:38.081474Z", + "iopub.status.idle": "2024-03-22T21:54:38.300460Z", + "shell.execute_reply": "2024-03-22T21:54:38.299522Z" + }, + "papermill": { + "duration": 0.24051, + "end_time": "2024-03-22T21:54:38.302646", + "exception": false, + "start_time": "2024-03-22T21:54:38.062136", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4KUlEQVR4nO3deVxU1f8/8NcM+44IiAsyLK65oKiouaai4gK5ZIipSFn2wUrUlL6p+e1bpOJSavmxPmouqKWIhiuhFCmi4hJupAbKKgLJsDkMM+f3B7+5H0YGmFFglvt+Ph4+nLn33Jn39TrvOXPOuecIGGMMhBBi4ITaDoAQQloCJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL+hdstu6dStEIhHMzc3h6+uLS5cu1Vv2+++/x9ChQ9GqVSu0atUKo0ePbrA8IcRw6VWyO3jwIMLDw7Fq1SpcvXoVvXv3xtixY1FQUKCyfGJiIoKCgnDu3DkkJyfD1dUVfn5+yMnJaeHICSHaJtCniQB8fX3Rv39/bNmyBQAgl8vh6uqKhQsXYvny5Y0eL5PJ0KpVK2zZsgWzZ89W6z3lcjlyc3NhY2MDgUDwUvETQpoWYwylpaVo164dhMKG627GLRTTS6uqqkJqaioiIiK4bUKhEKNHj0ZycrJar1FRUQGpVAoHB4d6y0gkEkgkEu55Tk4Ounfv/uKBE0KaXVZWFjp06NBgGb1JdoWFhZDJZGjTpo3S9jZt2uDu3btqvcayZcvQrl07jB49ut4ykZGRWL16dZ3tP/zwAywtLTULmhDSrCoqKvD222/Dxsam0bJ6k+xe1ldffYUDBw4gMTER5ubm9ZaLiIhAeHg491wsFsPV1RWBgYGwtbVtiVC1QiqVIj4+HmPGjIGJiYm2wyEviS/XUywW4+2331ariUlvkp2joyOMjIzw+PFjpe2PHz+Gi4tLg8dGRUXhq6++wq+//opevXo1WNbMzAxmZmZ1tpuYmBj0fxoFvpwnXxj69dTk3PSmN9bU1BQ+Pj5ISEjgtsnlciQkJGDQoEH1Hrd27Vp8/vnnOHXqFPr169cSoRJCdJDe1OwAIDw8HHPmzEG/fv0wYMAAbNq0CeXl5QgJCQEAzJ49G+3bt0dkZCQAYM2aNVi5ciWio6MhEomQn58PALC2toa1tbXWzoMQ0vL0KtnNmDEDT548wcqVK5Gfnw9vb2+cOnWK67R49OiRUvfzd999h6qqKkybNk3pdVatWoXPPvusJUMnhGiZXiU7AAgLC0NYWJjKfYmJiUrPMzMzmz8gQohe0Js2O0IIeRl6V7MjhNRVUVGhNN60rFKCC2kP0MrxCqwtlEcXdO3alZdjRinZEWIA7t69Cx8fnzrb16oom5qair59+zZ/UDqGkh1PqVsT4GstQN907doVqamp3PP0vKcI/zkNG6b3RJe29nXK8hElO55StybA11qAvrG0tFS6TsKHRTBLqkS3Hr3h7dZai5HpDkp2PKVuTYCvtQBieCjZ8RTVBPRfRmE5yiXVKvc9eFLO/W1sXP/H3MrMGO6OVs0Sn66hZMcTDX0wAPpw6JuMwnKMjEpstNziQ2mNljm3ZAQvriklOx5Q94MB0IdDXyi+uDbN8IaXc91bH8srJYhLTMbEEYNgZVF3YgsAuF9Qho8OXm/wS9CQULLjgcY+GAB9OPSVl7M1erS3q7NdKpUi3wno69bKoGc90QQlOx6p74MB0IeDGD66XYwQwgtUs+MBiewZhOY5yBCnQ2iu+mdsdXU1cqtzcaf4Tr0dFBniMgjNcyCRPQOguoZIWkZj15SuZ12U7Hggt/whrNw34xM1lsz99tS3De63cgdyy73hgzYNliPNS91rStfzvyjZ8UA7KzeUZyzE1zO84VlPB0V1dTXO/3Eerw55td6awIOCMnx48DrajXRrznCJGhq7pnQ966JkxwNmRuaQP2sPd9su6N66/g6KDOMMdHPoVm8HhfxZCeTPnsDMqP4Fi0jLaOya0vWsizooCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxA89nxQKVUBgC4mVNSb5nySgmuPAFcHv7T4OpihOgrvUt2W7duxbp165Cfn4/evXtj8+bNGDBgQL3lf/75Z6xYsQKZmZno1KkT1qxZA39//xaMWPse/P8ktTymsTVhjbHn/uVGX8/KTO/+2xicxr7A6MurLr36X3vw4EGEh4dj27Zt8PX1xaZNmzB27Fikp6fD2dm5TvkLFy4gKCgIkZGRmDhxIqKjoxEYGIirV6+iR48eWjgD7fB7xQUA4OlsDQsTI5Vl0vNKsPhQGtZP64kubetffMXKzJgWyNYB6n2B0ZdXbQLGGNN2EOry9fVF//79sWXLFgCAXC6Hq6srFi5ciOXLl9cpP2PGDJSXlyMuLo7bNnDgQHh7e2Pbtm1qvadYLIadnR1KSkpga2vbNCeig64/LELgdxcRu2AgvN1aazsc0oji8iqcuZVf7xcYX768NPl86k1Kr6qqQmpqKiIiIrhtQqEQo0ePRnJysspjkpOTER4errRt7NixiI2Nrfd9JBIJJBIJ91wsFgOomdNfKpW+xBnoturqau5vQz5PQ2FjKsDUPm3r3f/s2TMAgFsrM3RxtmzwtfT5emsSu94ku8LCQshkMrRpo7zkW5s2bXD37l2Vx+Tn56ssn5+fX+/7REZGYvXq1XW2nzlzBpaWDf+n0WdZZQBgjIsXLyLnprajIS+LL9ezoqJC7bJ6k+xaSkREhFJtUCwWw9XVFX5+fgb9M/bGo2Ig7QoGDhyI3h0dtB0OeUl8uZ6KX17q0Jtk5+joCCMjIzx+/Fhp++PHj+Hi4qLyGBcXF43KA4CZmRnMzOr2XpmYmNS7JJ0hUKwtamxsbNDnyRd8uZ6anJveDCo2NTWFj48PEhISuG1yuRwJCQkYNGiQymMGDRqkVB4A4uPj6y1PCDFcelOzA4Dw8HDMmTMH/fr1w4ABA7Bp0yaUl5cjJCQEADB79my0b98ekZGRAIAPP/wQw4cPx/r16zFhwgQcOHAAV65cwfbt27V5GoQQLdCrZDdjxgw8efIEK1euRH5+Pry9vXHq1CmuE+LRo0cQCv9bWR08eDCio6Px6aef4pNPPkGnTp0QGxvLqzF2hJAaepXsACAsLAxhYWEq9yUmJtbZNn36dEyfPr2ZoyKE6Dq9abMjhJCXQcmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIZDIZriT/gfLbv+FK8h+QyWTaDomQJkfJjudiYmLg5eWFd4ICUfjLOrwTFAgvLy/ExMRoOzRCmhQlOx6LiYnBtGnT0L17d7w2biLMOvbCa+Mmonv37pg2bRolPGJQ9G7dWNI0SktLsXDhQjg4OODEiRPc9rOP/gQAtG7dGh988AHGjBkDGxsbbYVJSJOhmh1P7d+/H7m5uSgqKlK5v6ioCDk5Odi/f38LR0ZeVmVlJSJXfIzHB1cgcsXHqKys1HZIOoGSHSEGJDAwEJaWlvhpzw48y7yGn/bsgKWlJQIDA7UdmtZRsuOp2NjYJi1HtC8wMBBHjx5Vue/o0aO8T3gCxhjTdhC6TCwWw87ODiUlJbC1tdV2OE3G1tYWpaWljZazsbGBWCxugYjIy6isrISlpWWj5SoqKmBhYdECEbUMTT6fVLPjqbKysiYtR7Trgw8+UKvc7NmzUVFR0czR6CZKdoQYgKSkJLXKHTp0CHfv3m3maHQTJTueUvcnuSH9dDdk6jRJAICTkxO6du3azNHoJkp2PGViYtKk5Yh2mZubq1XOxsZGrbY9Q6Q3ya64uBjBwcGwtbWFvb09QkNDG2xPKi4uxsKFC9GlSxdYWFigY8eO+OCDD1BSUtKCUesudcde0Rgt/fDs2bMmLWeI9CbZBQcH49atW4iPj0dcXBx+//13zJ8/v97yubm5yM3NRVRUFG7evIldu3bh1KlTCA0NbcGodVd5eXmTliPaRcmucRoPPfn777/h4eHRXPGodOfOHXTv3h2XL19Gv379AACnTp2Cv78/srOz0a5dO7Ve5+eff8asWbNQXl4OY2PVd8pJJBJIJBLuuVgshqurKwoLCw2q/crU1FTtslVVVc0YCWkKbdq0wT///NNouVatWuHx48ctEFHLEIvFcHR0VGvoicb3xnp5eWH48OEIDQ3FtGnT1G4reBnJycmwt7fnEh0AjB49GkKhECkpKXj99dfVeh3FP0h9iQ4AIiMjsXr16jrbz5w5w9u2jtr3zhLdpO5wkoqKCoO6npoMo9G4Znf9+nXs3LkT+/fvR1VVFWbMmIHQ0FAMGDBA40DV9eWXX+LHH39Eenq60nZnZ2esXr0aCxYsaPQ1CgsL4ePjg1mzZuGLL76otxxfanaWlpaorq5utJyxsTFvx2XpE77W1Ju1Zuft7Y2vv/4a69evx7Fjx7Br1y4MGTIEnTt3xrx58/DWW2/ByclJrddavnw51qxZ02CZO3fuaBpiHWKxGBMmTED37t3x2WefNVjWzMwMZmZmdbabmJgYVM+khYWFWsMVLCwsDOq8iWH1sGtyLi99u5hEIsG3336LiIgIVFVVwdTUFG+88QbWrFmDtm3bNnjskydP6p11Q8HDwwN79+7F4sWLldokqqurYW5ujp9//rnBn7GlpaUYO3YsLC0tERcXp/HPbkO9XczJyQmFhYWNlnN0dMSTJ09aICLyMoRCIdT5KAsEAsjl8haIqGVo8vl84fnsrly5gh07duDAgQOwsrLCkiVLEBoaiuzsbKxevRoBAQG4dOlSg6/h5OSkVi1w0KBBePr0KVJTU+Hj4wMAOHv2LORyOXx9fes9TiwWY+zYsTAzM8OxY8dapH1RX6j7U8aQfvIYsq5du6r1K4ivA4oBAExD69evZz169GAmJiYsICCA/fLLL0wmkymVycrKYkZGRpq+dIPGjRvH+vTpw1JSUtgff/zBOnXqxIKCgrj92dnZrEuXLiwlJYUxxlhJSQnz9fVlPXv2ZPfv32d5eXncn+rqarXft6SkhAFgJSUlTXo+2mZqasoANPrH1NRU26ESNbzyyitqXc9XXnlF26E2KU0+nxrX7L777jvMmzcPc+fOrfdnqrOzM/7zn/9o+tIN2rdvH8LCwjBq1CgIhUJMnToV33zzDbdfKpUiPT2da0y/evUqUlJSANT0INeWkZEBkUjUpPHpG2traxQXF6tVjug+FxcX3Lp1S61yfKVxsouPj0fHjh0hFCqPR2aMISsrCx07doSpqSnmzJnTZEECgIODA6Kjo+vdLxKJlNosRowYoVYbBl85ODgoJTtzc3NYWVmhvLxcaeCpg4ODNsIjGvL09ERCQoJa5fhK4zsoPD09VTZsFxcXw93dvUmCIs3v+drus2fPUFRUVGeE/fPliG5S94udzxUAjZNdff9YZWVl1AGgR7Kyspq0HNGuzMzMJi1niNT+GRseHg6gput65cqVSncTyGQypKSkwNvbu8kDJM1DIBA0aTmiXTSLTePUTnbXrl0DUFOzS0tLUxqxbWpqit69e2PJkiVNHyFpFh4eHrh586Za5Yjuy87O5h6bm5srNUfUfl67HN+onezOnTsHAAgJCcHXX39tUANs+eidd97BsWPH1CpHdF9GRgb3+Pl219rPa5fjG43b7Hbu3EmJzgAkJyc3aTlCdJ1aNbspU6Zg165dsLW1xZQpUxosGxMT0ySBkeb16NGjJi1HtMvd3R1//vmnWuX4Sq1kZ2dnxzVU29nZNWtApGV06NChScsR7WrdunWTljNEaiW7nTt3qnxM9FerVq24x4qpshQN2atWrUJBQUGdckR31Z6WrCnKGaIXngiA6LcbN25wj0tKSpTmBKw9XrJ2OaK71F342pAWyNaUWsmuT58+ao+3unr16ksFRFrGw4cPucfP3/pX+1rXLkd0V+/evdW6Xax3794tEI1uUivZBQYGNnMYpKWJRCKcP38ejo6OdSYEkEgkaN26NYqKing/YYK+uHfvXpOWM0RqJbtVq1Y1dxykhc2ZMwf79u1DYWFhnZodAG5S1aae0IE0D1oas3F6s5QiaVrDhw/nHj8/c23t57XLEd1Veyqu55cVqN0Gy+cpu9RKdg4ODtxMJ61atYKDg0O9f4h+SEpKatJyRLsUTU3GxsZ1Zv92cnLiVtTjc5OUWj9jN27cCBsbG+4x3Ryu/9RpzFaUGzVqVDNHQ16Wm5sbgJq1WZ6//7X2zDWKcnykVrKr3W4zd+7c5oqFtKDLly9zjy0sLJTacmo/r12O6K6hQ4fCycmpwcWRnJ2dMXTo0BaMSrdo3GZnZGTEDTitraioCEZGRk0SFGl+5eXl3ONRo0YhKSkJ+/fvR1JSklJNrnY5otsUA4YdHBzQtm1bWFlZoW3btlzz0vMTBPCNxoOK65u8UyKRaLRQL9Gu2vMRVldXY/369Xjw4AE8PT2VFs+uXY7orsTERIjF4jpriyi+rKytrSEWi5GYmMjbZgm1k51icRuBQIAffvhBqVdHJpPh999/5/cybXqmTZs23ONTp05xj9PS0uotR3RXYmIigJoZw01NTTFlyhSuOSImJgZlZWVcOUp2jdi4cSOAmprdtm3blH6ympqaQiQSYdu2bU0fIWkW6g4WpkHF+kGxvq+JiQlKS0shEAhw4sQJ+Pv748cff4S1tTWkUimv1wFWO9kpJv0bOXIkYmJi6AZxPTdkyJAmLUe0S7FAtrOzM4yNjSGTybh9iuEoubm5ai2kbag07qA4d+4cJToD8Msvv3CPnx9KVPt57XJEdyk6H3JychAQEICLFy+isrISFy9eREBAAHJzc5XK8ZHGHRTz5s1rcP+OHTteOBjScu7fv889fr7Tqfbz2uWI7urcuTPi4+MBAL/++ivi4uK4fbVnOuncuXOLx6YrNK7Z/fPPP0p/CgoKcPbsWcTExODp06fNECJpDlZWVtzj56f9qd0DW7sc0V3r1q0DUDM07Pl2OYlEwrWxK8rxkcY1uyNHjtTZJpfLsWDBAl6vNq5vJk+ejNjYWAgEAjx58gTJyck4efIkxo8fj0GDBsHGxgaMMUyePFnboRI1WFhYoH///twg8L59+3K9sYpp1/r378/r+eyaZCIAoVCI8PBwrseW6L6SkhIANT9ZHRwccPr0aXh6euL06dNwcHDgfsoqyhHdJpPJ8OTJE27ZhKtXr+L8+fNcorOzs0NhYaFSxwXfNNlMxQ8ePFAajEp0m+JmcWdnZxQUFCAqKkppv2L78zeVE92UlJSEzMxMCAQCjB8/HqWlpXj48CHc3NxgY2ODU6dOoaSkBElJSRgxYoS2w9UKjZNdeHi40nPGGPLy8nD8+HGa+0yPtG/fHgDw5MkT+Pv7w9TUlLuDoqqqCidPnlQqR3RbTk4OAMDb2xtpaWncZABZWVno0KEDvL29ce3aNa4cH2mc7K5du6b0XCgUwsnJCevXr2+0p5bojqFDh0IkEsHR0RG3b99GZmYmgJo7KNzd3eHj44OioiJe3ziuTxQTADz/+QSA7OxsLvk1NFGAodM42Z07d6454iAtzMjICOvXr8e0adMwYcIELFq0CPfu3UOnTp0QHx+P48eP49ChQzS5g56gpRQbR6uL8diUKVNw6NAhLFq0SGlclpubGw4dOtToguhEdygGDQM17a0zZ85ERUUFLC0tER0dzc1UVLsc3+jNtOzFxcUIDg6Gra0t7O3tERoayt3c3BjGGMaPHw+BQIDY2NjmDVTPXLx4sU47TnZ2Ni5evKiliMiLOHPmDICaW8MKCwuxadMmbN++HZs2bUJRURE3U7GiHB/pTc0uODgYeXl5iI+Ph1QqRUhICObPn4/o6OhGj920aRPNrqzCxx9/jHXr1tVZcIcxxg0+Xbt2rTZCIxpStMlVV1ejTZs2mDlzJsrLy2FlZYXo6Gg8fvxYqRwfCVh9E9TpkDt37qB79+64fPky+vXrB6BmWiJ/f39kZ2ejXbt29R57/fp1TJw4EVeuXEHbtm1x5MiRBufhl0gkSqumi8ViuLq6orCwELa2tk12TtpWVVUFW1vbOovt1CYUCiEWi2meQj0wduxYnDt3Dqampmjbtq3Ser8ikQi5ubmoqqrCyJEjcfr0aS1G2rTEYjEcHR1RUlLS6OezyWp22dnZ+N///V9s3769qV6Sk5ycDHt7ey7RAcDo0aMhFAqRkpKC119/XeVxFRUVmDlzJrZu3QoXFxe13isyMhKrV6+us/3MmTMGNZFlbGxsg4kOqLkzJiwsjNeLtOiLDh06AKj5EnNwcICfnx/MzMwgkUhw5coVrre9Q4cOOHHihBYjbVoVFRVql22yZFdUVIT//Oc/zZLs8vPz4ezsrLTN2NgYDg4OyM/Pr/e4RYsWYfDgwQgICFD7vSIiIpTGEipqdn5+fgZVs/vhhx+4x61bt8asWbPw7NkzmJubY+/evdy6sYWFhfD399dWmERNT58+xZ49ewDUDD9RNQQFqJmC35Cup1gsVrusVtvsli9fjjVr1jRY5kXn3zp27BjOnj1b70Wvj5mZWZ11N4GaSRFNTExeKBZdpOiUMDY2ho2NDb7++mtun0gkQklJCaqrq5GTk2NQ522o1F01zM3NzaCupybnotVkt3jx4kZXK/Pw8ICLi0udRX6qq6tRXFxc78/Ts2fP4sGDB7C3t1faPnXqVAwdOpSbxpqvFDNjVFdX45VXXsGePXuQnZ2NDh064KuvvuJ+9vB5Zlt9ohgkbmRkhMzMTKV7YI2MjCASiSCXy3k9SFyryc7JyUmtey8HDRqEp0+fIjU1FT4+PgBqkplcLoevr6/KY5YvX463335baVvPnj2xceNGTJo06eWD13OKG8aBmuUS09LSYG5ujrS0NKXlE2uXI7rLyMgI06dPx7p16+Ds7Izg4GCuN3bfvn148OABli5dyutB4mr3xjY2wPTp06f47bffmm1WhfHjx+Px48fYtm0bN/SkX79+3NCTnJwcjBo1Crt378aAAQNUvoZAIGi0N/Z5YrEYdnZ2avX26JP33nsP//73v1XuEwgE3Kwn7777Lq0togdkMhm8vLzg6OiIwsJCrmYOAO7u7mjdujWKiopw7949g0p4mnw+1R5UbGdn1+AfNzc3zJ49+6WDr8++ffvQtWtXroF1yJAhSp0hUqkU6enpGvXO8Nmrr77KPW5oDGLtckR3KWY92bx5M+7fv4/4+HiEh4cjPj4e9+7dwzfffIOMjAwkJSVpO1StUftn7M6dO5szjkY5ODg0OIBYJBLVu6atgh4MKWwxrq6u3OOGpmWvXY7orry8PABAjx49YGRkhOHDh6O8vBzDhw+HkZERevTooVSOj/TmdjHStBQN2p6enioX3PH09IS7uzuvG7T1Sdu2bQEAN2/eVLlfsV1Rjo/UrtmpO30TLbijH2rPeuLv7w93d3f89ddf6Ny5MzIyMnDixAma9USPKL68vvzyyzr3f8vlckRGRvL+y0vtDgqhUAg3Nzf06dOnwZ+Dqtao0GeG2kGhEBMTg8WLF9dp0I6KiqJZT/RMTEwMpk2bhokTJ2Lp0qXIyclB+/btsW7dOsTFxRnkTDYafT6Zmt5//33WqlUr5u3tzb7++mtWVFSk7qF6raSkhAFgJSUl2g6l2VRXV7P4+HgWHh7O4uPjWXV1tbZDIi/o8OHDTCQSMQDcH3d3d3b48GFth9YsNPl8ajQRgEQiQUxMDHbs2IELFy5gwoQJCA0NhZ+fn8HOKmLoNTsFqVSKEydOwN/f36BG2PORTCbDuXPnuNXiRo4cabDNEc0y9ASouZUqKCgI8fHxuH37Nl555RW8//77EIlEas8tRwhpXore2GHDhnG9seQlemOFQiE3+JTPy7MRQvSDRslOIpFg//79GDNmDDp37oy0tDRs2bIFjx49grW1dXPFSAjRgEwmw2+//Ybff/+9We9q0jdqDz15//33ceDAAbi6umLevHnYv38/HB0dmzM2QoiGnu9d37BhA0QiEdavX29wPbGaUjvZbdu2DR07doSHhwd+++03/PbbbyrLxcTENFlwhBD11R56UnsWm7Vr12LatGkGOfREE2onu9mzZxtsjysh+k4mk2Hx4sWYOHEiYmNjIZPJUFRUBF9fX8TGxiIwMBBLlixBQEAAbzss1E52u3btasYwCCEvQzERwP79+yEUCpXa6YRCISIiIjB48GAkJSVhxIgR2gtUi+jeWEIMQO2JAFR1UNBEAHq0lCIhpH6KG/y3bNmCf//733U6KObPn69Ujo+oZkeIARg6dCicnJwQERGBHj16ICkpCfv370dSUhJ69OiBTz75BM7OzryeCICSHSEGonYHouIuUA3uBjV4lOwIMQBJSUkoKChAZGQkbt68iWHDhiEoKAjDhg3DrVu38OWXX6KgoIDXMxVTsiPEACg6HsLCwlROyx4WFqZUjo8o2RFiAGrPVKxqIgCaqZiSHSEGofZMxXK5XGkfzVRcg5IdIQZAMc1+XFwcAgMDcfHiRVRWVuLixYsIDAxEXFwcoqKieHv3BEDj7AgxGFOmTMGhQ4ewePFiDBs2jNvu7u7O+/tiAUp2hBiUKVOmICAggDczFWuCkh0hBkbVurGE2uwIITxByY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbygN8muuLgYwcHBsLW1hb29PUJDQ1FWVtboccnJyXjttddgZWUFW1tbDBs2DJWVlS0Qsf6gdUYJH+hNsgsODsatW7cQHx+PuLg4/P7779xU0/VJTk7GuHHj4Ofnh0uXLuHy5csICwuDUKg3p93sYmJi4OnpiTFjxmDDhg0YM2YMPD09aUlMYniYHrh9+zYDwC5fvsxtO3nyJBMIBCwnJ6fe43x9fdmnn376Uu9dUlLCALCSkpKXeh1ddPjwYQag3j+HDx/WdojkBVVVVbHY2FhWVVWl7VCalSafT724XSw5ORn29vbo168ft2306NEQCoVISUnB66+/XueYgoICpKSkIDg4GIMHD8aDBw/QtWtXfPHFFxgyZEi97yWRSCCRSLjnYrEYACCVSiGVSpvwrLRLJpMhJCQEQM1Se7WnBVI8DwkJgb+/P91upIcU/1cN6f+sKpqcn14ku/z8fDg7OyttMzY2hoODA/Lz81Ue8/fffwMAPvvsM0RFRcHb2xu7d+/GqFGjcPPmTXTq1EnlcZGRkVi9enWd7WfOnIGlpeVLnonuuHbtGpfI+/TpAx8fH5iamqKqqgqpqalITU2FWCzGV199hT59+mg5WqIJmUyG27dv459//kFaWhq6d+9usF9YFRUVapfVarJbvnw51qxZ02CZO3fuvNBrK2oq7777LleD6dOnDxISErBjxw5ERkaqPC4iIgLh4eHcc7FYDFdXV/j5+cHW1vaFYtFFBw4cAAC4urqisLAQ27dv5/a5ubnB1dUVWVlZSE9Px//8z/9oK0yioSNHjmDZsmXcUooAIBKJsGbNGpW/gPSd4gtbHVpNdosXL8bcuXMbLOPh4QEXFxcUFBQoba+urkZxcTFcXFxUHqeYfrp79+5K27t164ZHjx7V+35mZmYwMzOrs93ExAQmJiYNxqpPsrKyuL8nTZqEvXv3Ijs7Gx06dMDatWvxyy+/cPsN6bwNWUxMDN58801MnDgRe/bsUbqeb775pkHOaafR/80WaEN8aYoOiitXrnDbTp8+3WAHhVwuZ+3atavTQeHt7c0iIiLUfm9D7aAICgpiAJiNjQ2TSqVKDdpSqZRZW1szACwoKEjboRI1VFdXM5FIxCZNmsRkMpnS9ZTJZGzSpEnM3d2dVVdXazvUJqXJ51MvxmB069YN48aNwzvvvINLly7h/PnzCAsLw5tvvol27doBAHJyctC1a1dcunQJQM0amkuXLsU333yDQ4cO4f79+1ixYgXu3r2L0NBQbZ6OTlC0w5WWlqqcxlsxhpHa6/RDUlISMjMz8cknn9QZWiUUChEREYGMjAxeL6WoFx0UALBv3z6EhYVh1KhREAqFmDp1Kr755htuv1QqRXp6ulKD5UcffYRnz55h0aJFKC4uRu/evREfHw9PT09tnIJOUXxJAMCJEydw/Phx7nntxZZrlyO6S7FEYo8ePVTuV2zn81KKepPsHBwcEB0dXe9+kUikcvXz5cuXY/ny5c0Zml5q37499/j5f7faz2uXI7qr9lKKAwcOrLOfllLUozsoSNMaOnQonJycAAAWFhZK+xTPnZ2deb30nj6pvZSiVCpVuv1PKpXSUorQo5odaXqKn6sN1eyIflAspTht2jTY2tri2bNnAIANGzbA3NwcEokEhw4dMtjxduqgmh1PJSUlccN5FB8MBcXzgoICXjdo6yPGmNIdQEDNXUH0BUY1O97KycnhHjs7OyM4OBjl5eWwsrLCvn37uERYuxzRXTKZDO+99x4AwNzcXGlmH8XzBQsWICAggLe1O6rZ8ZSiV87GxgY5OTlYs2YN/P39sWbNGuTk5MDGxkapHNFtiYmJePLkCQBg1KhRSEpKwv79+5GUlIRRo0YBqKmpJyYmajFK7aJkx1PXr18HUHNrmKpxWR07dlQqR3Tb2bNnAQADBw7E0aNH4evrCwsLC/j6+uLo0aNcD62iHB9RsuOp8vJyADVDElQNKr5165ZSOaLbFLf/BQcHgzGm1BvLGENQUJBSOT6iNjueGjp0KGJjY+Hm5oY///wTw4YN4/aJRCK4ubnh4cOHvB6qoE9cXV0BAJs3b0ZUVBQePnwIoKY31s3NjbvfW1GOjwSMumkaJBaLYWdnh5KSEoOa9aSqqgoWFhaQy+WYMGEC/Pz8cO/ePXTq1AlnzpzB8ePHIRQKUVlZCVNTU22HSxqRkJCA0aNHA6jpcFq9ejXMzMwgkUiwatUqrsPp119/5drwDIEmn09Kdo0w1GQHAB9//DHWrVtXZ/JOIyMjyGQyLF26FGvXrtVihERdtb+8LCwslHpjFc8N8ctLk88ntdnx2Nq1a7F06VKle2EVKNHplwsXLih9YdWmuL5yuRwXLlxoybB0CiU7nlu7di0qKioQFRUFf39/REVFoaKighKdnlEMEdq7d2+dWb2dnZ2xd+9epXJ8RB0UBKampvjggw/g5eUFf39/mqxTDylu8Pf09MSDBw9w7tw5nDx5EuPHj8fIkSO5qc9oIgBCiF6rPRGAQCDA8OHDMWzYMAwfPhwCgYAmAgDV7AgxCLUnAggICMCYMWNw7949PHz4EPHx8Th+/DjvJwKgZEeIgZgyZQqWLFmCjRs3Ii4ujttubGyMJUuWGNz6E5qiZEeIgYiJiUFUVBQmTJjA1ew6deqE+Ph4REVFYeDAgbxOeDTOrhGGPM6uNqlUihMnTlAHhZ6SyWTw8vJCz549ERsbC5lMxl1PIyMjBAYG4ubNm7h3755B/ZSlcXaE8AwtuNM4SnaEGABacKdxlOwIMQC1F9xRhRbcoWRHiEGoPc7u+dvG5HI5jbMDJTtCDIJinF1cXJzK+Qnj4uIQFRVlUJ0TmqKhJ4QYiClTpuDQoUNYvHix0vyE7u7uOHToEK+HnQCU7AgxKFOmTEFAQECde2P5XKNToGRHiIExMjLC8OHDUV5ejuHDh1Oi+/+ozY5AJpMprVkgk8m0HRIhTY6SHc/FxMTAw8MDY8aMwYYNGzBmzBh4eHggJiZG26ER0qQo2fFYTEwMpk6dWmfFqaysLEydOpUSHjEolOx4SiaTISQkBADw/O3RiuchISH0k5YYDEp2PJWQkACxWNxgGbFYjISEhBaKiJDmRcmOp3bt2sU9dnR0xLZt27Bz505s27YNjo6OKssRos/0JtkVFxcjODgYtra2sLe3R2hoKMrKyho8Jj8/H2+99RZcXFxgZWWFvn374vDhwy0UsW5LTk4GULP+RF5eHubNm4dWrVph3rx5yMvL46Z5UpQjRN/pTbILDg7GrVu3EB8fj7i4OPz++++YP39+g8fMnj0b6enpOHbsGNLS0jBlyhS88cYbuHbtWgtFrbsUXxT29vYqpwRq1aqVUjlC9J1eJLs7d+7g1KlT+OGHH+Dr64shQ4Zg8+bNOHDgAHJzc+s97sKFC1i4cCEGDBgADw8PfPrpp7C3t0dqamoLRq+bFMvtFRQUYPLkyUr3Uk6ePJlbQf75ZfmI7qNxk6rpxR0UycnJsLe3R79+/bhto0ePhlAoREpKCl5//XWVxw0ePBgHDx7EhAkTYG9vj59++gnPnj3DiBEj6n0viUQCiUTCPVc04kulUkil0qY5IR0wa9YsfPLJJwCAkydP4vjx49y+2jW9WbNmGdR5G7ojR45g6dKlePToEQBgw4YN6NixI9atW1fv50SfafJ/Uy+SXX5+fp0ahrGxMRwcHJCfn1/vcT/99BNmzJiB1q1bw9jYGJaWljhy5Ai8vLzqPSYyMhKrV6+us/3MmTOwtLR88ZPQMbX/DVRNCVS73IkTJ1osLvLikpOTsWbNmjrbHz16hBkzZmDZsmUYNGiQFiJrPhUVFWqX1WqyW758ucqLU9udO3de+PVXrFiBp0+f4tdff4WjoyNiY2PxxhtvICkpCT179lR5TEREBMLDw7nnYrEYrq6u8PPzM7g1KMLDw7Fhw4YG9wcGBrZcQOSFyWQyBAUFNVhm8+bN+OyzzwzqXtnGhk/VptVkt3jxYsydO7fBMh4eHnBxceHakBSqq6tRXFwMFxcXlcc9ePAAW7Zswc2bN/HKK68AAHr37o2kpCRs3boV27ZtU3mcmZkZzMzM6mw3MTExuIVo1q9fDyMjI0RFRSkNLBYIBFiyZAnWrl2rxeiIJhISElBZWck9nzVrFnx8fJCamoq9e/cCqKkFJSYmYty4cdoKs8lp9JlkeuD27dsMALty5Qq37fTp00wgELCcnByVx/z5558MALt9+7bSdj8/P/bOO++o/d4lJSUMACspKXmx4PWARCJhUVFRzN/fn0VFRTGJRKLtkIiGRowYwQAwAKyyspJVVVWx2NhYVlVVxSorK7l9I0aM0HaoTUqTz6deJDvGGBs3bhzr06cPS0lJYX/88Qfr1KkTCwoK4vZnZ2ezLl26sJSUFMYYY1VVVczLy4sNHTqUpaSksPv377OoqCgmEAjY8ePH1X5fPiQ7xpjSh4PoH1tbWwaADRs2jDFW93q++uqrDACztbXVZphNTpPPp14MPQGAffv2oWvXrhg1ahT8/f0xZMgQbN++ndsvlUqRnp7ONViamJjgxIkTcHJywqRJk9CrVy/s3r0bP/74I/z9/bV1GoQ0C0U7XEZGhsoOp4cPHyqV4yO96I0FAAcHB0RHR9e7XyQS1bmhvVOnTnTHBOGFV199FXFxccjKysLkyZOxbNkybtzkmjVrkJ2dzZXjKwF7PkMQJZqsOK7PpFIpt4K8oXXE8EFZWRlsbGwaLVdaWgpra+sWiKhlaPL51JufsYSQ+llbW6N///4Nlunfv79BJTpNUbIjxEBcunSp3oTXv39/XLp0qYUj0i2U7AgxIJcuXUJpaSkmTZoENzc3TJo0CaWlpbxPdIAedVAQQtRjbW2Nw4cPUxvsc6hmRwjhBUp2hBBeoGRHCOEFarNrhGIYoiazK+gjqVSKiooKiMViauMxAHy5norPpTrDhSnZNaK0tBQA4OrqquVICCH1KS0thZ2dXYNl6A6KRsjlcuTm5sLGxgYCgUDb4TQbxbx9WVlZBn2nCF/w5XoyxlBaWop27drVWUvleVSza4RQKESHDh20HUaLsbW1NegPB9/w4Xo2VqNToA4KQggvULIjhPACJTsCoGY6+lWrVqmckp7oH7qedVEHBSGEF6hmRwjhBUp2hBBeoGRHCOEFSnaEEF6gZGeARowYgY8++kjbYRAN6dp107V4XhYlO0IMSFVVlbZD0F3Ntnot0Yo5c+Zwq78r/rRv3559++23SuWuXr3KBAIBy8zMZIwxtn79etajRw9maWnJOnTowBYsWMBKS0uVjklKSmJDhgxh5ubmrEOHDmzhwoWsrKysxc7NkKm6bvfv32fz5s1jIpGImZubs86dO7NNmzbVOS4gIID93//9H2vbti0TiUSMMcbOnz/PevfuzczMzJiPjw87cuQIA8CuXbvGHZuWlsbGjRvHrKysmLOzM5s1axZ78uRJvfFkZGS01D9Hs6BkZ2CePn3KBg0axN555x2Wl5fH8vLy2JIlS9iQIUOUyi1evFhp28aNG9nZs2dZRkYGS0hIYF26dGELFizg9t+/f59ZWVmxjRs3sr/++oudP3+e9enTh82dO7fFzs2Qqbpuz549YytXrmSXL19mf//9N9u7dy+ztLRkBw8e5I6bM2cOs7a2Zm+99Ra7efMmu3nzJispKWEODg5s1qxZ7NatW+zEiROsc+fOSsnun3/+YU5OTiwiIoLduXOHXb16lY0ZM4aNHDmy3niqq6u18U/TZCjZGaDhw4ezDz/8kHt+7do1JhAI2MOHDxljjMlkMta+fXv23Xff1fsaP//8M2vdujX3PDQ0lM2fP1+pTFJSEhMKhayysrJpT4Cnnr9uqvzrX/9iU6dO5Z7PmTOHtWnThkkkEm7bd999x1q3bq10Xb7//nulZPf5558zPz8/pdfOyspiAFh6erra8egTarPjAW9vb3Tr1g3R0dEAgN9++w0FBQWYPn06V+bXX3/FqFGj0L59e9jY2OCtt95CUVERKioqAAA3btzArl27YG1tzf0ZO3Ys5HI5MjIytHJefLB161b4+PjAyckJ1tbW2L59Ox49eqRUpmfPnjA1NeWep6eno1evXjA3N+e2DRgwQOmYGzdu4Ny5c0rXs2vXrgCABw8eNOMZaQ8lO54IDg7mkl10dDTGjRuH1q1bAwAyMzMxceJE9OrVC4cPH0Zqaiq2bt0K4L8N3mVlZXj33Xdx/fp17s+NGzdw7949eHp6auekDNyBAwewZMkShIaG4syZM7h+/TpCQkLqdEJYWVlp/NplZWWYNGmS0vW8fv067t27h2HDhjXVKegUms/OAJmamkImkyltmzlzJj799FOkpqbi0KFD2LZtG7cvNTUVcrkc69ev5yZA/Omnn5SO79u3L27fvg0vL6/mPwGeev66nT9/HoMHD8b777/PbVOn1tWlSxfs3bsXEomEmwjg8uXLSmX69u2Lw4cPQyQSwdhYdRpQ9f9In1HNzgCJRCKkpKQgMzMThYWFkMvlEIlEGDx4MEJDQyGTyTB58mSuvJeXF6RSKTZv3oy///4be/bsUUqGALBs2TJcuHABYWFhXA3g6NGjCAsLa+nTM1jPX7dOnTrhypUrOH36NP766y+sWLGiTtJSZebMmZDL5Zg/fz7u3LmD06dPIyoqCgC42bb/9a9/obi4GEFBQbh8+TIePHiA06dPIyQkhEtwqv4f6TVtNxqSppeens4GDhzILCwslIYMfPvttwwAmz17dp1jNmzYwNq2bcssLCzY2LFj2e7duxkA9s8//3BlLl26xMaMGcOsra2ZlZUV69WrF/viiy9a6KwM3/PX7e7du2zu3LnMzs6O2dvbswULFrDly5ez3r17c8cohp487/z586xXr17M1NSU+fj4sOjoaO41Ff766y/2+uuvM3t7e2ZhYcG6du3KPvroIyaXy1XGo+9DT2iKJ0J4YN++fQgJCUFJSQksLCy0HY5WUJsdIQZo9+7d8PDwQPv27XHjxg0sW7YMb7zxBm8THUDJjhCDlJ+fj5UrVyI/Px9t27bF9OnT8cUXX2g7LK2in7GEEF6g3lhCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTuiM+bOnQuBQACBQAATExO0adMGY8aMwY4dOzS6L3PXrl2wt7dvvkDrMXfuXAQGBrb4+xL1ULIjOmXcuHHIy8tDZmYmTp48iZEjR+LDDz/ExIkTUV1dre3wiD7T7q25hPxXfTe1JyQkMADs+++/Z4w1vF7GuXPn6qydsGrVKsYYY7t372Y+Pj7M2tqatWnThgUFBbHHjx9z71NcXMxmzpzJHB0dmbm5OfPy8mI7duzg9j969IhNnz6d2dnZsVatWrHJkydzN8evWrWqzvueO3euWf6dyIuhmh3Rea+99hp69+6NmJgYAIBQKMQ333yDW7du4ccff8TZs2fx8ccfAwAGDx6MTZs2wdbWFnl5ecjLy8OSJUsAAFKpFJ9//jlu3LiB2NhYZGZmYu7cudz7rFixArdv38bJkydx584dfPfdd3B0dOSOHTt2LGxsbJCUlITz58/D2toa48aNQ1VVFZYsWYI33niDq5nm5eVh8ODBLfsPRRqm7WxLiEJ9NTvGGJsxYwbr1q2byn3Pr5exc+dOZmdn1+j7Xb58mQHgaoWTJk1iISEhKsvu2bOHdenShZv+iDHGJBIJs7CwYKdPn240fqJ9VLMjeoExxk082dh6GfVJTU3FpEmT0LFjR9jY2GD48OEAwK3psGDBAhw4cADe3t74+OOPceHCBe7YGzdu4P79+7CxseHWbHBwcMCzZ88Mds0GQ0PJjuiFO3fuwN3dXa31MlQpLy/H2LFjYWtri3379uHy5cs4cuSI0nHjx4/Hw4cPsWjRIuTm5mLUqFHcT+CysjL4+PjUWbPhr7/+wsyZM5v57ElToCmeiM47e/Ys0tLSsGjRIrXWy1C1dsLdu3dRVFSEr776Cq6urgCAK1eu1HkvJycnzJkzB3PmzMHQoUOxdOlSREVFoW/fvjh48CCcnZ1ha2urMk5DW7PB0FDNjugUiUSC/Px85OTk4OrVq/jyyy8REBCAiRMnYvbs2WqtlyESiVBWVoaEhAQUFhaioqICHTt2hKmpKXfcsWPH8Pnnnysdt3LlShw9ehT379/HrVu3EBcXh27dugGoWZ3N0dERAQEBSEpKQkZGBhITE/HBBx8gOzube98///wT6enpKCwshFQqbZl/NKIebTcaEqIwZ84cbtiGsbExc3JyYqNHj2Y7duxgMpmMK6fOehnvvfcea926tdLQk+joaCYSiZiZmRkbNGgQO3bsWJ2Fo7t168YsLCyYg4MDCwgIYH///Tf3mnl5eWz27NnM0dGRmZmZMQ8PD/bOO++wkpISxhhjBQUF3BodoKEnOocm7ySE8AL9jCWE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPDC/wMw+OT4VT+fNAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:38.340848Z", + "iopub.status.busy": "2024-03-22T21:54:38.340537Z", + "iopub.status.idle": "2024-03-22T21:54:38.594405Z", + "shell.execute_reply": "2024-03-22T21:54:38.593496Z" + }, + "papermill": { + "duration": 0.275425, + "end_time": "2024-03-22T21:54:38.596747", + "exception": false, + "start_time": "2024-03-22T21:54:38.321322", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABfZklEQVR4nO2dd3xTVf/H35lddEGhLVAos+whAoIiIFVQlKGPIvKgIAKKiIqAgMpwgQiIA0Hwh+BAxfEIKkOsbBC0lL1nGW2hLXS3Wff3R5o0aZM2SUeacN6vV14k9557cxLaT7/nfJdMkiQJgUAg8ELk7p6AQCAQVBZC4AQCgdciBE4gEHgtQuAEAoHXIgROIBB4LULgBAKB1yIETiAQeC1C4AQCgdeidPcEqiMGg4GrV68SGBiITCZz93QEAkExJEkiKyuLunXrIpfbt9OEwNng6tWrREVFuXsaAoGgDC5dukT9+vXtnhcCZ4PAwEDA+OUFBQW5eTYCgaA4mZmZREVFmX9X7SEEzgamZWlQUJAQOIGgGlPWFpJwMggEAq9FCJxAIPBahMAJBAKvRezBCTwevV6PVqt19zQEFYhCoUCpVJY7TEsInMCjyc7O5vLly4i6rd6Hv78/kZGRqNVql+8hBE7gsej1ei5fvoy/vz+1a9cWQdlegiRJaDQarl+/zvnz52nWrFmpwbylIQTuFkaSJGatO4paKee1/q3cPR2n0Wq1SJJE7dq18fPzc/d0BBWIn58fKpWKixcvotFo8PX1dek+wslwC3MhLZdVey6yfMd58rV6d0/HZYTl5p24arVZ3aMC5iHwUA5dvml+rtUb3DcRgaCSEAJXHiQJjv4P8m66eyYucehyhvm5Vi826QXehxC48rBpOvwwAv54zd0zcQlhwQncxYgRIxg0aFClv48QuPLQaiAgg4Sv4Uycu2fjFDq9gSNXMs2vNTohcFVFr169eOmll9w9jVsCIXDlocEd0HWs8fmvL0JBlnvn4wRnrmeTZ+FY0AgLTuAknhBcLQSuvPSZASENIeMS/Dnb3bNxmEOXMqxee8MSVZIkcjU6tzwcDTQeMWIE27Zt48MPP0QmkyGTyahfvz5LliyxGpeQkIBcLufixYsALFy4kLZt2xIQEEBUVBTjxo0jOzvb6pqdO3fSo0cP/Pz8iIqKYsKECeTk5Dg0r6SkJPr374+fnx+NGjVi9erVREdHs2jRIvMYmUzGkiVLGDBgAAEBAbzzzjvo9XpGjRpFo0aN8PPzIyYmhg8//NDq3nq9nokTJxISEkKtWrWYMmVKlQVmizi48qIOgAEfwZcD4Z/l0HowRN/p7lmVyaErN61ea3We72TI0+ppNWOTW9772Jt98VeX/ev04YcfcurUKdq0acObb74JwIIFC1i9ejXPPfecedw333zDnXfeScOGDQFjyMRHH31Eo0aNOHfuHOPGjWPKlCl8+umnAJw9e5Z+/frx9ttvs2LFCq5fv8748eMZP348X3zxRZnzevLJJ0lNTWXr1q2oVComTpzItWvXSoybNWsWc+fOZdGiRSiVSgwGA/Xr1+eHH36gVq1a7N69mzFjxhAZGcljjz1m/nwrV65kxYoVtGzZkgULFvC///2Pe+65p+wvtpwIgasIGveC256C/atg3Xh4dheo/d09q1Kx9KCCWKJWFcHBwajVavz9/YmIiABg2LBhLFiwgMTERBo0aIDBYOC7777j9ddfN19nuWcXHR3N22+/zbPPPmsWuDlz5jBs2DDzuGbNmvHRRx/Rs2dPlixZUmqg7IkTJ/jzzz/5559/uP322wH4/PPPadasWYmxTzzxBCNHjrQ6Nnt20cqlUaNG7NmzhzVr1pgFbtGiRUybNo2HH34YgKVLl7JpU9X8IRICV1Hc9xac3gzp52DLO9D3HXfPyC4FOj3Hk4wOBn+1glyN3iucDH4qBcfe7Ou293aVDh060LJlS1avXs3UqVPZtm0b165d49FHHzWP+fPPP5kzZw4nTpwgMzMTnU5Hfn4+ubm5+Pv7c/DgQQ4dOsQ333xjvkaSJAwGA+fPn6dly5Z23//kyZMolUpuu+0287GmTZsSGhpaYqxJAC1ZvHgxK1asIDExkby8PDQaDR06dAAgIyODpKQkunbtah6vVCq5/fbbq2SZKvbgKgrfYHhokfH535/CpX/cOp3SOJGUhVYvEeqvokFNo6XpDXtwMpkMf7XSLY/yZlMMGzaM1atXA7B69Wr69etHrVq1ALhw4QIPPvgg7dq146effiI+Pp7FixcDoNFoAGPRgbFjx3LgwAHz4+DBg5w+fZomTZqUa26WBAQEWL3+7rvvmDRpEqNGjeKPP/7gwIEDjBw50jwvdyMEriJp3hfaPQ6SAdY+D7oCd8/IJqb4t3b1Q/BRGn8EvEHgPAW1Wo1eb50a98QTT3DkyBHi4+P58ccfGTZsmPlcfHw8BoOBBQsWcMcdd9C8eXOuXr1qdf1tt93GsWPHaNq0aYlHWdU4YmJi0Ol0JCQkmI+dOXOGGzdulPlZdu3aRffu3Rk3bhwdO3akadOmnD171nw+ODiYyMhI9u7daz6m0+mIj48v894VgRC4iqbfHAioA6knYds8d8/GJqb9t3b1g1EphMBVNdHR0ezdu5cLFy6QmpqKwWAgOjqa7t27M2rUKPR6PQMGDDCPb9q0KVqtlo8//phz587x1VdfsXTpUqt7vvrqq+zevZvx48dz4MABTp8+zdq1axk/fnyZ82nRogWxsbGMGTOGffv2kZCQwJgxY/Dz8yvTMm3WrBn//vsvmzZt4tSpU7zxxhv884/16uXFF19k7ty5/PLLL5w4cYJx48Zx8+ZNx7+wciAErqLxrwn9Fxif7/wAkg66dz42KBK4ELPAaUSqVpUxadIkFAoFrVq1onbt2iQmJgLGZerBgwcZPHiwVXWU9u3bs3DhQt577z3atGnDN998w5w5c6zu2a5dO7Zt28apU6fo0aMHHTt2ZMaMGdStW9ehOX355ZeEh4dz9913M3jwYEaPHk1gYGCZVTzGjh3Lww8/zJAhQ+jatStpaWmMGzfOaswrr7zC8OHDeeqpp+jWrRuBgYEMHjzYoXmVF5kkKgWWIDMzk+DgYDIyMlzvqrXmKTj2C0S0hdFbQKGq0Dm6Sq5GR5uZmzBIsG96Hyb/eIhtp64z/9H2/KeT/f6S1ZH8/HzOnz9Po0aNXC6nI7DN5cuXiYqK4s8//6RPnz5umUNp/7+O/o4KC66yeOB98AuF5MOwa5G7Z2PmyJVMDBJEBPlSJ8hXLFEFAPz111+sW7eO8+fPs3v3bh5//HGio6O5++673T21ciEErrKoUQfuL9yD2zYPrh1373wKKXIwBAOgVhr3WITAeS87duygRo0adh9gTLuaPn06rVu3ZvDgwdSuXdsc9OvJiDi4yqTto3DkJzi10ehVHbUZ5K7HS1UEBy0cDEDRHpwXxMEJbHP77bdz4MCBUsf07duXvn3dE0NYmQiBq0xkMnjwA1i8G67EG+Pjur/g1ikdtggRAQuBExac1+Ln50fTpk3dPQ23IJaolU1Q3aKshr/ehrSzpY+vRDJytVxIywUsl6iFe3BekIsqEBRHCFxV0HG4MV9Vlw/rXgCDe6wlU4J9w1r+hPgbgz/Vwskg8GKEwFUFMhk89BGoAuDiLvj3/9wyDcv4NxMqhXAyCLwXIXBVRWhDiJ1lfL55Jty4WOVTOHjpJgDt6gWbj4k9OIE3IwSunBgrNji4f9X5GWjQDbQ5xgrAVRxjffiKtQcVivbghBdV4I0IgSsHccdTePDjnfyccMWxC+RyGPAJKH3h3BZjL4cq4lpWPkkZ+chl0MaGBSeWqJ6DrUq7v/zyi9vmU50RAlcOTqZkcfRqJku3nXXcigtrCr0Lu3Bteg0ykypvghaYSpQ3rVODAJ+i6KAiJ4PwonoqSUlJ3H///e6eRrVECFw5+O8dDQn0UXLmWjZ/HEtx/MJuz0O9TlCQAb+9XCVL1UPF4t9MmJwMYg/Oc4mIiMDHx8etc5AkCZ1O59Y52EIIXDkI8lUxvJuxZv6SrWccr1AqV8DAxSBXwakNxmyHSqZ4BoMJtdKYWeEVe3CSBJoc9zyc/COVlZXFsGHDCAgIIDIykg8++MDldoKWS9QLFy4gk8n4+eef6d27N/7+/rRv3549e/ZYXVNWg5qvvvqK22+/ncDAQCIiInjiiSesejRs3boVmUzGhg0b6NSpEz4+PuzcudPpuVc2IpOhnDx9VyP+b+d5Dl7OYPfZNO5sGubYhXVaQs9XYcvbsH4yNOoJNWpXyhwlSbJwMIRYnfOqMBFtLrzrWHmgCmf6VWMDIgeZOHEiu3btYt26dYSHhzNjxgz2799vLvVdXl577TXmz59Ps2bNeO211xg6dChnzpxBqVQ61KBGq9Xy1ltvERMTw7Vr15g4cSIjRoxg/fr1Vu8zdepU5s+fT+PGjW2WOHc3QuDKSVgNHx7vHMWqPRf5dOsZxwUO4K6X4NhaSDkMGybDoysrZY6Xb+SRnqNBpZDRMjLQ6pxaVPStcrKysli1ahWrV682lyL64osvHK7d5giTJk2if//+gLEpTOvWrTlz5gwtWrRwqEHN008/bb5X48aN+eijj+jcuTPZ2dnmBH2AN998k3vvvbfC5l3RCIGrAEbf3Zhv9iay60waBy/dpH1UiGMXKlQw8BNYfg8c/R+0fhhaDSj7OicxBfi2iAjCR2md7G/2onpDqpbK32hJueu9HeTcuXNotVq6dOliPhYcHExMTEyFTaddu3bm55GRkQBcu3aNFi1aONSgJj4+nlmzZnHw4EFu3LiBoTD7JjExkVatWpmvs9WEpjohBK4CqB/qz4AOdfl5/xU+3XqGz4Y78Z9et4PRktuxAH5/BaLvMlYFrkCKl0iyxORFLfAGC04mc2qZ6M1YljkylR03iZSpQc2ECRNKXNegQQNycnLM1UW++eYbc9Xhvn37lmgmU7wJTXVDOBkqiOd6GjsXbTqawumULOcuvnsKhMVAzjXYNL3C53awFIFTmZPtvUDgPITGjRujUqmsehdkZGRw6tSpKnn/shrUnDhxgrS0NObOnUuPHj1o0aKFzSbQnoAQuAqiWXgg97UKB2DJNicrhqh8jV5VZHDwW2N/1QrCYJA4csXYA7W4gwG8zMngIQQGBvLUU08xefJktmzZwtGjRxk1ahRyubzc7QcdoawGNQ0aNECtVpub3Kxbt4633nqr0udVGQiBq0DG9TbW3Fp34CqXb+Q6d3FUZ2N8HBjTuPIzK2RO51JzyC7Q4auS06xOjRLnRTUR97Bw4UK6devGgw8+SGxsLHfeeSctW7askt4SZTWoqV27NitXruSHH36gVatWzJ07l/nz51f6vCoDsQdXgXSICqF7k1rsPpvG8u3nmD2wjXM36P0anPgdbpyHzTOKGkmXA9P+W5u6wSgVJf+eFXlRvcDJ4EEEBgZabfLn5OQwe/ZsxowZU+a1Fy5csHptGX8ZHR1dIh4zJCSkxLHOnTvzxx9/2H2PoUOHMnToULvv06tXryrpTF9ehAVXwYzrZbTivvvnEqnZTjZ+VvsbvaoA8V/AuW3lno+tEkmWmLyoBWIPrkpJSEjg22+/5ezZs+zfv9/c6HngwIFunpl3IQSugrmzaS3a1Q+mQGfgi13nnb9B9F1w+yjj83UvGKPky0FpDgYQyfbuZP78+bRv357Y2FhycnLYsWMHx48fL7NBjMBxxBK1gpHJZIzr1YRnv97Pl3su8mzPJgT6OtmZ6N7ZcPoPuHkR4t6C++e6NBet3sCxqyYHg22BE3tw7qFjx47Ex8eXOJ6Xl1dmgxiB4wiBqwTuaxVBk9oBnL2ew9d/J/JcrybO3cAn0Lj/9vUjsHcptB4EDe5weh6nUrIo0BkI9FUSXct2vJLIZKhe3MoNYioDty9RFy9eTHR0NL6+vnTt2pV9+/bZHXv06FEeeeQRoqOjkclkVjWxTMyaNQuZTGb1aNGiRSV+gpLI5TKeK9yL+7+d58nX6p2/SdNY6PBfQIK140Gb7/QtDlkk2MvltsMPisJEnCjcWc3whM1ugfNUxP+rWwXu+++/Z+LEicycOZP9+/fTvn17+vbtazeoMDc3l8aNGzN37lwiIiLs3rd169YkJSWZH+6ocjCwQ13qhfiRml3AD/GXXbtJ37ehRgSknYZtzi9T7ZVIssQU6AugdVMzHFdRKAoroRSLrhd4B7m5xlCr8jSfdusSdeHChYwePZqRI0cCsHTpUn7//XdWrFjB1KlTS4zv3LkznTt3BrB53oRSqSxVAKsClULO6B6NmPXrMZZtP8vQzlE2wzRKxS/U2Ff1u6Gw6yNoOQDq3ebw5QcLi1xa9mAojtpiTlq9hI8HbVoolUr8/f25fv06KpUKudztCxJBBSBJErm5uVy7do2QkBDzHzJXcNuPs0ajIT4+nmnTppmPyeVyYmNjS9SucpbTp09Tt25dfH196datG3PmzKFBgwZ2xxcUFFBQUBTSkZlZMUG2Qzo34OO/znApPY/fDiUxqGM952/S4gFo84ixZtza8TBmKyjVZV6Wr9VzsjBlrF0pyf8qS4HTGcC9dROdQiaTERkZyfnz57l4seqb+Agql5CQkHIbKm4TuNTUVPR6PeHh4VbHw8PDOXHihMv37dq1KytXriQmJoakpCRmz55Njx49OHLkCIGBgTavmTNnDrNnz3b5Pe3hp1Yw8s5o5v9xiiVbzzKgfV27e2Glcv88OLcVrh2FnQuhl33r1cSxpEz0BomwGmrqBtuPjlfIZSjkMvQGySMdDWq1mmbNmollqpehUqnKZbmZ8KAFiWNY1qZv164dXbt2pWHDhqxZs4ZRo0bZvGbatGlMnDjR/DozM5OoqKgKmc/wbtEs3XaOkylZ/HXiGrGtwsu+qDgBYfDA+/Dj07D9fWjxIESUniVxyNQisH5ImfmNKoVR4Kp7sO/iLWeIrhVA/3aRVsflcnmVpDgJPA+3bVqEhYWhUChISbHuZZCSklKh+2chISE0b96cM2fO2B3j4+NDUFCQ1aOiCPZTMewO4/J4sTNlzYvT+mGjsBl0sPZ50Jde//6QnRLltvCEYN9L6bm8v+kkM9YecfdUBB6E2wROrVbTqVMn4uLizMcMBgNxcXF069atwt4nOzubs2fPmov+uYNRdzVCrZSTkHiTv8+lu3YTmQz6LwDfYEg6AHs+KXV4WRkMlnhCZ61cjTHUJiu/+jU2EVRf3Op2mjhxIsuXL2fVqlUcP36c5557jpycHLNX9cknn7RyQmg0Gg4cOMCBAwfQaDRcuXKFAwcOWFlnkyZNYtu2bVy4cIHdu3czePBgFApFicThqqROoC+PdqoPwKdb7VuSZRIYAf0Kw0W2vAupp20Oy8rXci7VmOJVWoiICU8I9jXNTaM3eGy8nqDqcavADRkyhPnz5zNjxgw6dOjAgQMH2Lhxo9nxkJiYSFJSUd/Qq1ev0rFjRzp27EhSUhLz58+nY8eOPPPMM+Yxly9fZujQocTExPDYY49Rq1Yt/v77b2rXrpyGLo4y9u4myGWw43QqhwuXjy7RfqgxCFhfYFyqGkoGER++koEkQb0QP8JqlO0W9YSEe72FqFXneQqqF253Mpi6+dhi69atVq9tlYIpznfffVdRU6tQGtTyZ0D7uvxy4CpLtp3h02GdXLuRTAYPLoJP74BLe2HfcrjjWashh53YfwPPKHqpswhCLtDp8VOX38Mm8H5EZGQVYkrf2nAkmbPXs12/UUgU3Pum8XncbEi3rlpSVomk4niCk0FnsT+Yr62+8xRUL4TAVSExEYHEtqyDJMFnzpY1L06nkRDdw9gLdN0LVo2HnXEwAPh4wB6czmqJ6kJur+CWRAhcFWOy4v6XcIWkjDzXbySXw4CPQOkHF3ZA/EoA0rILuHzDeN82paRoWWKy4DTVuHWgpcAJC07gKELgqphODUPp2qgmWr3E8u0uFMS0pGZj6DPD+PyPNyDjMocKO9g3Dgsg2M+xJGWzwFVnC05vvQcnEDiCEDg3YGpO8+2+RNJzypli1HUs1O8Cmiz47WUOmzMYHLPewDNaBwoLTuAK5RK47OxsMjMzrR6Csrm7WRit6waRp9WzcveF8t1MrjC2HFT4wOk/8D/xI+C4gwE8o6qvpZNBWHACR3Fa4M6fP0///v0JCAggODiY0NBQQkNDCQkJITQ0tDLm6HUYy5obrbhVuy+QXVDO6PzazaHXqwA8mrqY2tx0yoJTKz0rTERYcAJHcToO7r///S+SJLFixQrCw8OrpFGtN9KvTQSNwwI4l5rDt3sTGX134/LdsPsEtId/IfjaId5Wf0HryCEOX+oJgb7CghO4gtMCd/DgQeLj44mJiamM+dwyKOQyxvZszKs/HWb5jnM82b0hPspyBK8qVOxr9xZdNj9MX/k/cOZXaD3YoUtVHpCLKiw4gSs4vUTt3Lkzly5dqoy53HIM7lifiCBfrmUV8FP8lXLfb1d2BJ/qC/tq/j4JctIcus4TclFFHJzAFZy24D7//HOeffZZrly5Qps2bUrUS2/Xrl2FTc7bUSvljL67MW/9dozPtp/lsdvrO1/W3ILDVzL4WzeIp0IOEZJ9BjZOhUeWlz0PD3MyCAtO4ChOC9z169c5e/asueIHGDfNJUlCJpOh14u/rs4wtEsUn/x1motpuaw/ksyA9nVduo8kSRy6nIEWJal9PiBk3UA4vAbaPAwx95d6rSkXVVOd9+CEBSdwAafNhaeffpqOHTuyZ88ezp07x/nz563+FTiHv1rJiO6NAFiy9azLBTEvpuWSkadFrZDToO1d0P0F44nfXoa8m6Ve62mBvsKCEziK0xbcxYsXWbdunWhOW4E81b0hy7af5XhSJltPXqd3izpO38OUf9qybpBxT63XNDj+G6SfhT9eh4H2C2SKPTiBt+K0BXfPPfdw8ODBypjLLUuIv5onuhrLmrtaENNUQaS9Kf5N5WcMAEYGCV/B2b/sXmv2olbnXFTLMBFhwQkcxGkL7qGHHuLll1/m8OHDtG3btoSTYcCAARU2uVuJZ3o0ZtXui/xz4Qb/XEinc3RNp64/bKtEUsNu0GUM7PsM1r0I43aDT8nOYpXtZNhx+jpv/3acuY+0pWMD14LBi9eDEwgcwWmBe/ZZY3HFN998s8Q54WRwnfAgXx7pVI9v913i0y1n+GJkF4ev1Rskjly1U+Syzww4tQFuJsKfs6H//BLXm5wMBZUkcL8dTOJkShYbjySXQ+CEF1XgPE4vUQ0Gg92HELfyYSprvuXkdY5ddTyv98y1bHI1evzVCprUrmF90qcGDPjY+Pyf5XBxd4nr1YUBxpWVbH8zz1hQIK0chQVENRGBKzglcFqtFqVSyZEjonVbZRAdFsADbY3dv5Y4URDT5GBoUy8Yha3G0o17wW1PGZ+vfR40uVanK7tkeUaeFqBclVOEBSdwBacETqVS0aBBA2GpVSLP9WoCwO+HrnKhsDNWWRwqFLj2pSXY3/cWBNaF9HOw9V2rU0Ve1MpxMmTkGYsJlM+CE15UgfM4vUR97bXXmD59OunpLvb3FJRK67rB9IqpjUGCz7Y7Fldo08FQHN9geGiR8fmexXD5X/Opooq+lWMZZRZacGnZBS7fQ1hwAldwWuA++eQTtm/fTt26dYmJieG2226zegjKz/OFBTF/ir9MSmZ+qWM1OgPHk7IAB4pcNu8L7YaAZDAuVXVGwansQN8KWaKKPTiBCzjtRR00aFAlTENgSefomnSODuWfCzf4fMc5Xuvfyu7YE8mZaPQGQvxVNKjpX/bN+801xsRdPwHb34d7Xq/UQF+d3mCud5er0ZOv1eOrcr5qil5YcAIXcFrgZs6cWRnzEBRjXK+mjFz5D9/sTeT53k0J8VfbHHewcHnatl6wY7X5/GtC/wWw5knYsRBaDkClMOa/VobAZeZbF/NMy9FQL8TP6ftoRSaDwAVcLl0RHx/P119/zddff01CQkJFzkkA9IqpTcvIIHI1elbtvmh33KHCHgztnShRTquBxoekh7XP4yMzCkZFOBkupeda7eWZlqcm0rNdW6aKXFSBKzgtcNeuXeOee+6hc+fOTJgwgQkTJtCpUyf69OnD9evXK2OOtyQymczsUV25+zy5GttlzQ852cXezAPzwS8Ukg9R96ixpFJ5nQw/xV+mx7wtvPnbUfOx4gKXmuOao8EqF1UrLDiBYzgtcC+88AJZWVkcPXqU9PR00tPTOXLkCJmZmUyYMKEy5njL8kCbCBrW8udGrpZv95UsMpqr0XH6msnBEOLczWvUgfvnARCe8CFNZZfL5WS4cjOPmeuMwnY6Jdt8vFIsuGpc1klQvXBa4DZu3Minn35Ky5YtzcdatWrF4sWL2bBhQ4VO7lZHqZAz9m6jFff5jnMlLKyjVzMxSFAn0IeIYF/n36Dto9CsL3KDhvdVy9DrtGVfYwNJknj1x0NmZ0K+hYVVQuBc9KRaWnAancHlslKCWwuXUrWKJ9iDMQjYYBB/WSuaRzrVo06gD0kZ+fySYF3W/KC5B2qIazeXyeDBDzCoA+koP8Pj+t9dus3XexPZeSbV/DqvFIFzNdhXV2x/sDo3yBFUH1wql/Tiiy9y9epV87ErV67w8ssv06dPnwqdnAB8lAqe6WEsiLl021mrcIkSJZJcIbgemXfPAmCC7DtIczxFDCAxLZc5648D8FBhNWJLgcssYcG5tgdn+blBlEwSOIZLgb6ZmZlER0fTpEkTmjRpQqNGjcjMzOTjjz+ujDne8jzRtSHBfirOpeaw6Wiy+bgpRatdVEi57q9v/1926lvjK9MirRsPDlriBoPEpB8PkqvRc0fjmowtbH2YpynpRQ30MUYkpbm4B6ctNqd8ESoicACnBS4qKor9+/fz+++/89JLL/HSSy+xfv169u/fT/369Stjjrc8NXyUPNWtIWAsiClJEhm5Wi6kGZPm29YrhwUHqFQKpupGkyP5ILu4G+JXOHTdF7svsO98OgFqBe//pz0BhSJmtQeXaxS4RrUDANeXqMKC8y6SM/KZ/MNBjlzJqNT3cTrQF4whDPfeey/33ntvRc9HYIcRdzZi+Y7zHLmSyY7TqcgLg3qjavpRM8B2ELCjqBVyLkt1eE/3OG+qVsHmmdDsPghpYPeas9ezmbfxBADT+7ckqqY/yRnGtDJbe3CNwwI4dDnDZSdD8Rg9YcF5Nmv+vcQP8ZfRGSQ+GNKh0t7HJYGLi4sjLi6Oa9eulXAsrFjh2F9/gXPUDFAztEsDVuw6z6dbz9CjWW2gHA4GC0y5qF/p72VGoxMoL++FX1+E//5sdEQUQ6c38MqagxToDPRoFsYTXYxC6FeYgqU3SGj1BlQKuVngGoUZ69S57EUtFsIiLDjPxlQpp6xc6/Li9BJ19uzZ3HfffcTFxZGamsqNGzesHoLKY/TdjVApZPx9Lp01/xrj4srlYChEIZehkMuQkJNx7weg9DXmqx74xub4ZTvOceDSTQJ9lLz3SDtzipivuujHyWTFmQWucImaXaCzWsI6SvElqrDgPJsLaUaBSy1HhRlHcNqCW7p0KStXrmT48OGVMR9BKUQG+zG4Yz3W/HuZi4X7bxVhwYGx6KXeIJEX1Ah6T4fNM2DjdGjSB4IizeNOJmexaPNpAGY81Iq6FnmlaoUcuQwMEuRr9AT5qswCVz/UD6Vchs4gkZ6jsbrOEYo7GYQF59kkpht/fq9nVa7AOW3BaTQaunfvXhlzETjA2J5NzKtGmcxYxbciMHfW0ktwx/NQ9zYoyDD2VS0MqtXqDUxccwCN3kCfFnX4Tydrp5JMJjMvU00WnClMJMRPZd4rdGWZqi/cgzMVLHbFChRUD7ILdKQWetNv5GorteG40wL3zDPPsHr16sqYi8ABmtSuwf1tIszPa/i4tI1aAqvOWgqlseWgXGVsWHPkJwAWbznD0auZBPupmPNwW5vVS/zURQKnN0hkFWY3BFsInCueVFM1EdPnzRUC57FcTLOuVJ3mYmykIzj925Gfn8+yZcv4888/adeuXYmshoULF1bY5AS2eeW+GM5ey+G/haEjFYGpJpz5r2l4K+g5Bba8AxumcNzvNj75y9iz9c2BrakTZDs1zFTrLU+jtwryDfJTUauGyYJz/gdabyFwmfk6JnybQE6BjqFd7Ht6BdWTxDTrniCpWRoig50voeUITgvcoUOH6NChA0CJ5jMO1SMTlJsmtWuw6eW7K/SeNqv63vUyHFsHKYdJ+X4COsNz3N8mggGFGQu2sFyimvbfAtQKVAo5tQJ8ANeCfU216gIsLNZpPx8WAueBXEy3Frjr2flAxWy1FMdpgduyZUtlzEPgZsydtSz3QxQqGPgJhmX30Eu7g//438G0QZNL/UNmWqLmWwhcsJ/Ryi/PEtVkwQVU0JJc4D6KL1Er09HgcsFLgXdh5WSwIEHXkKW6/gC8rV5JLXnpnb6KlqgGs8AFFQpcLZOTwQULzpRsX1F7jgL3YYoA8CncFjE5HCjItneJywiBEwBFP2wai5aQ+Vo9r/xwkA91D5OsboBv/nXY9Fqp9/G1sUQ1W3A1yuNkMC1Rne/nIKhemASufWEO9fWsAtDrYNVD8OPTkJNWYe8lBE4AWLYOLLLg5m86ybnrOQQHBhLw6FJABgdXw+nNdu/jpzLex5bAmS04J50MBoNkilQRS1QPp0Cn52pGHgCdo0OBQoH7ezFc3Q+n/wS9693XiuN2gVu8eDHR0dH4+vrStWtX9u3bZ3fs0aNHeeSRR4iOjkYmk7Fo0aJy31NgRKWw7qy173w6/7frPABzH2lLYLM74Y5xxsG/vgj5mTbvY3Iy5GtsCFyNQieDkxacZZCvWKJ6Npdv5CFJ4K9W0CIiCADlzTPw1zvGAf3etQosLy9OC9z27dvR6Ur2B9DpdGzfvt2pe33//fdMnDiRmTNnsn//ftq3b0/fvn25du2azfG5ubk0btyYuXPnEhERUSH3FBhRWbQOzNXomPzjQSQJHu1Un3tahBsH3fM6hDaCzCvwp+3uapZxcJl2nAzO7sFZpmnZs+B0egMfxZ0m/qJoSF6dMYWINKjpT+1AH2QYGJX2AegLoMk90GFYhb6f0wLXu3dvm13tMzIy6N27t1P3WrhwIaNHj2bkyJG0atWKpUuX4u/vbzdhv3Pnzrz//vs8/vjj+Pj4VMg9BUYsA33nbjjBxbRc6gb78sZDFj1Z1f4woLDm378r4HzJP2il7cGZlqhZBTqnWv9ZOj7sWXC7z6axcPMp5m444fB9BVWPKQc1ulYAtQN9GK7YTDvDMVDXgIc+tFncoTw4LXCSJNkME0hLSyMgIMDh+2g0GuLj44mNjS2ajFxObGwse/bscXZalXbPWwW10vh/uvXkdb7cY2xT+N5/2hHkW6w8faMecPvTxufrXgCNtVfVvES1FDh/4z2CfFUoCnOtbuQ43v/ByoJT23YymEo15WpEhkN1xuRgaFjLnzr6ZF5VfgeA9p5ZpZbnchWHNzQefvhhwBjMO2LECCsLSq/Xc+jQIadyVFNTU9Hr9YSHh1sdDw8P58QJ1/4Ku3rPgoICCgqKNr4zM23vL3kzpj24DUeMFYOHdW1gLslUgtjZcOoPuHEB/nob+s0xn7IpcIUWnFwuo2aAmutZBaRmFzjcKMdUKkkuK1oCF8e0r1es6IigmmFKsm9Q048amyYgkxWw19CCus2GElUJ7+ewBRccHExwcDCSJBEYGGh+HRwcTEREBGPGjOHrr7+uhClWPnPmzLH6PFFRlfFVV29MAgfGIprTH2hpf7BvkHE5AfD3Ekjcaz5l3oOzcDKY4uDA0pPq+D6cqaOWUiE3L4GLY/LMim5b1RvTErXrjd+Rnd9GPmqmaMdw3QmL3hkctuC++OILAKKjo5k0aZJTy1FbhIWFoVAoSElJsTqekpJi14FQWfecNm0aEydONL/OzMy85UTOUuAsy4/bpVmscUP4wDew9nl4dieofEvdgwNcqihiCvJVymXmeL3imNK/hL5VX/QGicvpeUSQRuMEo9W/OuBJLuZHVFo2g9N7cDNnziy3uAGo1Wo6depEXFyc+ZjBYCAuLo5u3bpV6T19fHwICgqyetxqRNfyB+CZuxpxR+Najl3U9x2oEQ5pp2Hbe4BlLqqhVIFzJlTEFCailMvwsWPBFS1RhcJVV5Iz89Ho9cxV/x9yTRbUu509tR8DKq/wpdMCl5KSwvDhw6lbty5KpRKFQmH1cIaJEyeyfPlyVq1axfHjx3nuuefIyclh5MiRADz55JNMmzbNPF6j0XDgwAEOHDiARqPhypUrHDhwgDNnzjh8T4Ftnr6rEb+9cBev9S9laVocv1B48APj810fwtUE8xI1p0BHVn5RqSQTrgT76i2WqHYtuML7CYGrvlxMzWGQfBe95AdAoYaBiwkLMv5hrSwLzumoyREjRpCYmMgbb7xBZGRkuSqIDBkyhOvXrzNjxgySk5Pp0KEDGzduNDsJEhMTkcuLfqCvXr1Kx44dza/nz5/P/Pnz6dmzJ1u3bnXongLbqBRy14pntugPbR4x1oxbOx7/u78HrGvtWwlcDecripiCj5Vymf09OLFErfakJCUyU/Wl8UXPV6FOC2oHngKqkcDt3LmTHTt2mEsmlZfx48czfvx4m+dMomUiOjraoU3k0u4pqATunwfntkLKEZqcWg504Vqm8QfWv7BUkglXlqgmC05VzIIzVUCRJIlUsUSt9rRMeItQWTZX/ZpT984XAagdaPyDdy2rgI/jThMe7MuA9nXt/iFzFpf6ogpPlcCKgDCjyAH1Di8mRpZoritnab2Ba15UU6CvopgFpyy07nM0enOhThEmUk05+gst0v9CKynY1+5NYykuoHZhAYYz17JZsPkUU348VKGxvk4L3KJFi5g6dSoXLlyouFkIPJ82j0BMf+QGLfNUy1BgDLgtLnCueFGL9uCsvajKwqBhy9QvYcFVQ3LTYf0kAJboHyIw+jbzKZMFd76wjWBYDTU+yoqrGOP0EnXIkCHk5ubSpEkT/P39S5Qst5XGJbgFkMmg/wL0F3bSvuAczyjW85n+IasYOLDcg3N8z0VnsQenthS4wiVqqoXDQuibNRqdweo7cwsbp0LOdc5I9flEN5j1hR57gNo1rIO9HQ3+dhSnBc5eBQ+BgKBIcnq9SdCmF5mo/JHNhk4E+1k7d0xL1Mx8ncO/fOZAX7kcGUXrF2Xh3p6w4Gwz+9ej/PDvZTa82IOomv42x2TkaZm59ggDO9ajd0ydip/EyY1w6HskmZxJBWPQylTUDy2aS1ig2mp4RFDF9mZwWuCeeuqpCp2AwMvo8ATb1v8fPRWHeE+1jDW+y6xOB/sZ81H1BokbuRrC7TSvsURnioNTyMxtA6FoiWrZlUkIHFy9mUdksC97zqaRXaDj6NUMuwK360wqvxy4SnJmfsULXH5h20kgpfUzHPi3KXWDfa32Uf3VSgLUCnIKc4gjK9iCc8l2PXv2LK+//jpDhw41lyHasGEDR48erdDJCTwPP7WSadpnyJZ86Sw/RZ+sdVbn5XIZoYXJ946GimgtMhksw5LkMpPAWVpw5Zq+x7N6byLd5/7F7F+PmeMQC0rpO2oKwckuKFkCrdz88QZkXYWaTfi74VgAGtQqKbSmfTio+CWq0wK3bds22rZty969e/n555/JzjbWUT948CAzZ9quESa4dVAp5FyT12aubigAfa58akzKt8DUXctRR4PeYolqacGZKpNYCuWt7uGfu+E4ACt3XzCLliNNsnMLKrgKy9ktsH+V8fnATzh/0yik0bVKZkFZCpzbLbipU6fy9ttvs3nzZtTqovXzPffcw99//12hkxN4Jn4qBd/o+/C3oSUqQ76xrJKF8BTFwjnmaDAH+ipkVntwJoFLFxacGcuPXyRw9i04039LhVpwBdnw6wTj886joWH3oioiNiy4sBoWFpwDWxbO4LTAHT58mMGDB5c4XqdOHVJTUytkUgLPxletQELOq9rR6BS+xsKYpr/mWDSfcXCJapmqJbP4iTVZc5Z5jGIPrgjT9+ZIcdEKraMX9ybcTITgBhBrXNWZqog0rFm6Bef2JWpISAhJSUkljickJFCvXr0KmZTAszEl3F+UIjjfrrBKy6bXIeMK4Hywr2U1EbmsdAtO6FtJSrXgCm2+HI2uYpb3F3fDvs+Mzwd8CD6BQFGp8oa29uBqVCOBe/zxx3n11VdJTk5GJpNhMBjYtWsXkyZN4sknn6zQyQk8Ez8LL1lmu1FQvzNosuC3l0CSnE7XKgoTsVyggqIwk8F6iSoUrjil7cGZvi5JMpa4KhfaPFhbmCLZcbixxwKQla81/1/bFLhCCy7YT4W/umKbCjktcO+++y4tWrQgKiqK7OxsWrVqxd1330337t15/fXXK3RyAs/E16LqbnCALwxcbKwecfoPOLTGHOzraEUR6zARSwvO6FSwdjJUxCfwYGx8/tK8qJbfV055HQ1b3oX0sxAYCfe9bT5sKlNeM0BNYPES+BRZbfVCKjYGDlyIg1Or1Sxfvpw33niDI0eOkJ2dTceOHWnWrFmFT07gmZh6o0JhqlZgDPSaatyb2TCFyD6/Ao7vwRUtUeVWeYpymYzsAp057xWEBWeLUi04i+e5Gh1gu5lTmVyJhz2fGJ8/uAj8QsynTA4GW9YbwJ1Nw3i2ZxPubhbm2nuXgsv2YIMGDWjQoOKbRAg8H8slqjkXtfsEOPoLJB+i4+F3gCcd34OzKHhpiVwmKyGSt7rA2fr0pXtRi65w2YLTFcAvz4NkgLaPQUw/q9PmRjN2go1VCjlT72/h2nuXgUMCN3HiRN566y0CAgKsSnvbYuHChRUyMYHnYip66a9WFKViKVTGpery3tRK3Mj98hh259zl0P3Mgb4KmdmxAEbxNO3tKOUydAbplg8TsUVpXlTLrytH42KoyI4FcP04+IdBv7klTl8s9KA2sBEDV9k4JHAJCQlotVrzc3uUp/ilwHswpeIUryRCZDu4ayJsn8ebqi+4L68lWr3Bql6cLUzhDgq5HJVCzt3Na7P91HVC/FXmpP2wGj4kZ+bf8oG+tijNgrMkx5VYuOTDRoED6D8fAkqWuzdZcNF2lqiViUMCt2XLFpvPBQJb+NkTOIC7JyEd/5Xa148zQ/UVN3IHUyew9NAAUzURU4HLnoUCB0Ue1LBANcmZ+RgkOHs9mzX/XmJMj8Zmh8atTKlxcBZ/D5yOhdNr4ZdxYNBBy4eg1SCbw8rag6tM3FxHReCNmASueKkkAJQ+yAYuRo+cwYpdFBxdX+b9LKuJFMe0RDXFUhkkif8s2c1n284xcc1BVz+Cx2LLgi1wIA4OXMhm2P0RJB8C3xB4YIHNrvQFOj1XM/IAaGAjyLeycciCMzV9doSff/7Z5ckIvAO7S1QT9Tvxo3ogQzT/o862qdD+HiuvW3F0FgUvLZGkIk+sKd1HkuBGrnE7ZfdZkVkDkF/aHpylBeeMwF0/CVuN+217Y6bgc9OHDoElh11Kz0OSIECtIKyGuuSASsYhC86yKXJQUBBxcXH8+++/5vPx8fHExcURHOxC0xKB1xFSWC0krJTl4a81R3DOEIFPXgpsfqPU+1lmMhTHFEtnme5jQquXGP3lv2TmV05TYU+hdAuuiBxHl6gGvTGgV6/hYs27GLK3Ia+sOWBzaGJ6kYPBHXv0DllwpqbPAK+++iqPPfYYS5cuNbcJ1Ov1jBs37pbsJyooycO31ScjT8t/OtW3OyY4MIhXtWP4wedN2P8ltB5sjnwvjr0wEShaotoT083HUli85QzT7neiHaKX4bAF56gXde9ncHkfelUgw5IfB2SFlppUQsQupJYeIlLZOL0Ht2LFCiZNmmTVA1WhUDBx4kRWrFhRoZMTeCY1A9S8cl8MDUsJC6gZoOYfqQUJEY8aD6x70ViFwgY6i2T74piWqLVKWf5UVku66oLBIjbGdhycY5aZQ3Fw6eeMAdvAAtl/uWyoCYBGb7AZ12h2MIR5iMDpdDpOnDhR4viJEycwGBxzRwsEpnzUtWGjjVUnMhIhbrbNsSYvqikGztJGSCtliWrC4MXBcV/sOk/HtzZz7Gqm3TGlpmphGehbhgVnMMC6CaDL46T/bXyaeRf1QvzMe63JFr1wTVwspYpIVeC0wI0cOZJRo0axcOFCdu7cyc6dO1mwYAHPPPOM6B4vcBiTxZWcp4QBHxkP7ltmrEZRDNMenKq4k4GiMBHLihTqYpae3nv1jdm/HiMjT8uUn+x7jB1JtgcHwkTiv4ALO9Ap/HjmxnAUcjkfDe1I/VBjDmmKTYFzX4gIuJCqNX/+fCIiIliwYIG5bFJkZCSTJ0/mlVdeqfAJCrwTq6q+TXrDbU8a9+LWjofndoHKD43OwOlrWVy+YQwzKB4mkpWvNWc5WO7B+akVaPJurfxUTSlWWoHOYHN/DGwXyLTJzUuweQYA72mHcEkKZ/J9zenUMJSIIF+OXs0kJdN6K0BvkLh0w8METi6XM2XKFKZMmUJmptEsFs4FgbOUqOp739tw+k9IP8vZNa/xUvrDnEzOskqkr+Fj/eNqst4C1ApzehgYU8Qy8oo8p968RDVhEjhbWi5Jxj0ym/1GLS6w62SQJPj1RdBkc1TRis/zY+nepBbP9mwCQHhhNZDkDGsLLikjD61eQqWQERlc8ZVCHKFcgb5BQUFC3AQuYVqimmvC+QYjPWjMY44+tQL51Xg0egOBvkruaFyT53s34YF2kVb3KHIw+FiVUUoq9oumv4UEzh720rWswkTsORkOfgtn49DJ1LyQ+zShAb58MKSDeU/UVGa8+BLVtDyNCvW3yiGuSlyqJvLjjz+yZs0aEhMT0WisPSf79++vkIkJvBuTBXczV4tOb0ACXj9cjzv0dzJYsYvlwSspePov6tcOsRs/ZSpVXjNATWm/P7eAvpktXUungVopR6s3IEmmdK2SgddlholkJRsbNwPvax7hnFSXFY+2s2r3aBK44k4Gd++/gQsW3EcffcTIkSMJDw8nISGBLl26UKtWLc6dO8f9999fGXMUeCGh/mpzZs+lG3k8vfIfvv/3Em/pniRPXZM6+eeIOvJpqcGhJu9gWA21lQVXnFthD86WpzTIV4lv4bK0tGBfE9nFLThJgt8mQn4Gx2jM5/oHGHlnNPe0sG7mbW+JavaguqGKiAmnBe7TTz9l2bJlfPzxx6jVaqZMmcLmzZuZMGECGRkZlTFHgReikMsI9TdacS98u58dp1PxUyl4/8ne+A38wDho50JIOlTmvWoGqG2lQZq5VZeoNXyU+BQWH7XnSZVK24M7+jOc/B0dSl4uGENMZKjNum1lLVEbuCnIF1wQuMTERLp37w6An58fWVlZAAwfPpxvv/22Ymcn8GpMy9QjV4zOqvmPtqdPy3BoPQhaDjBWqVj7vLFqhQXFxaxmgA8ymcyuyN0KFpylM8ZEoK+qyIKzs0dnXdFXX+SQyUmF9ZMB+Fg7kEuqRnz8REebjorwIKMH+0au1kpILxYG+Ua7KcgXXBC4iIgI0tPTAWNVX1Mv1PPnz4taXAKnMAmcCau9mgfmg1+osVrF7o9KvY8pidveMrX48TyN3uuyGyQJtpy4ZuVMqOGjxLdMC876da5p3IYpkJvGCUMUn+oHMntAa5rUrmHzHsF+KnwKC5teKwwVkSSpqNClm4J8wQWBu+eee1i3bh1gDPp9+eWXuffeexkyZIjNfqkCgT1qFRM4U5I+AIHh0O894/Otc43VK+xgEkp7jobix+987y86v/MnSYVlfLyFkSv/sXod6Ks0W1yOWHBQWFHkxO9w5Cd0yJmkHcv97RuUmlcsk8nMjWNMjobUbA25Gj0yGUTVdE+ICLjgRV22bJk5Jev555+nVq1a7N69mwEDBjB27NgKn6DAeymeP2rakzPT7jE48hOc3mRcqj69CeQll0imopZGh0TJVURxC84UP7fnbBoP32b/F9fTqeHriAVn/X3lZaYh/TYRGbBM9yAZoa15Z3CbMiuBhAf5cjEt1yxwpioidYP9bMffVRFOWXA6nY63336b5ORk87HHH3+cjz76iBdeeAG1uurrPQk8l5oBRdkHKoUMf3WxXwSZDB78AHyC4PI/sHepzfvUKsOCs/fL6e2+h0AfJT6FtfkcLVsetH0Wsuxkzhoi+cTwCB8Pvc1mq7/imB0NhZ5UUxURdzoYwEmBUyqVzJs3D53OxeYUAoEFlkvUYD+1bSEKrgf3vWV8HvcWpJ0tMcS0tLW/B2f7/T01w2H94SQe/nQXlwvToOwR6Fu0N1Zq2fJC7pYfJPTUGgzImKwdy+PdmtMhKsShORVfol50Y5lyS5zeg+vTpw/btm2rjLkIbjEsnQxW+2/Fue0paNQTdHnGahaStTVisjDsLaLsCZ/eQ51i477Zz/7Em7z2vyOljjMuUUu34ExfQQ1ymaP6HICdNR9hv9ScOkGO97MILxYqklgNYuDAhT24+++/n6lTp3L48GE6depEQID1BxgwYECFTU7g3VjuwYXYK28OxqXqgI/g025wcScxfj8CbcynAgtzVO0J2Y7T1xn86S7e/087mtYpqqvt6fFxx5Psl0gCk5OhjD24wj3LV5XfUU+WRo5/fdbXeQauptv9g2GL4rFwF6pBFgO4IHDjxo0DbPc/lclk6PUuNo8V3HLUstiDK9WCAwiNhthZsGEKnU8voi5zuUoYNdRK5KY6cXZ+I3M0ehISbzJ+dQIbX7rbfNzTw5qulRHqYgwTKcOLKsEd8mMMV/4JwD9tZ1OQafR6lpYdUpyIYOP/ZZGTwQP34AAMBoPdhxA3gTPULLYHVyadR0PUHaj0puWURKBv0d9oeRkJ3Vn51nvHnm7BlUWQr6pML6pCn8dc5XIAvtH14VxgJ7PwO9NCwdT6MSWzgMx8rdlT7W4LTrQNFLiNUAurrUwLDkAuh4GL0cl96Kk4xH8U261aE5ZlcRQvmOnNhTDBuAdXVhzc7eeWEC1P4apUkzm6oeRqdGbvsjNNYkx7cBqdgYOXbgJGJ5IjHtjKxOElal5eHnFxcTz44IMATJs2jYKCIhNZoVDw1ltv4etbehNfgcCEUiEnxF/FzVytldiVSlhTDjR5lttPf8gbyq+YrOphPlW8km9x1Err856+RC2LwLLi4C79Q/vL3wAwXfsM2fiTo9GbIwmd2YNTK+XUClCTlqNh3/nCTCc3W2/ghAW3atUqPvvsM/PrTz75hN27d5OQkEBCQgJff/01S5YsqZRJCrwXU6hIcPEg31I42mA4Bw2NCZbl8lz2YrMr0JRYbg9V8VLmXr5EreFjUU2keJiINh/WPo8MiZ/0Pdhq6AAY+zKYcnedLeFmsuL2njMKXLSbPajghMB98803jBkzxurY6tWr2bJlC1u2bOH9999nzZo1FT5BgXdjip+qXUoP1eJIciVTtGPQSAo65u0xZjtgu62gJcUFzpP0be+5NNb8e8mpawJ9VWbRL1Euafs8SD1JrqoWb2qHmw/nFOjNySDO9jE1/V8eKFyiutvBAE4I3JkzZ2jbtq35ta+vL3KLGvldunTh2LFjFTs7gdcztV9LXo5tTu8WtZ267qTUgE90hbnPG6ZATmqZFlnxJawnVRkZsuxvpvx4yCwejmDpRbXqjXr1AOxcBMDW5lPJoCiJ3rgHVz4LzlTZxN0OBnBC4G7evGm153b9+nWio6PNrw0Gg9V5Z1i8eDHR0dH4+vrStWtX9u3bV+r4H374gRYtWuDr60vbtm1Zv3691fkRI0YUls8pevTr18+luQkql7b1g3kxtplT+Yomy2KJfgDX/JtCbhpsmGJuQGOiTrFWgsX34CwzGb7++yJrD1xxdvpVzvlU271jbaGQyyzi4AotOJ3GmNcr6aH1YM7V6g0UiX+ORl9UYcRZCy7Iev/d3UG+4ITA1a9fnyNH7EdOHzp0iPr1nU9c/v7775k4cSIzZ85k//79tG/fnr59+3Lt2jWb43fv3s3QoUMZNWoUCQkJDBo0iEGDBpWYW79+/UhKSjI/RK0670OLkm0tZ4FMAUd+ort2j9X54kuk4gJnymS4cjOP1385wovfHcBgkNhzNo2buSWbGFcH5m4o2ZO4NIri4AotuF2LIOUI+NWE+983i1mAj3FcefbgTLFwJjzKgnvggQeYMWMG+fklex/m5eUxe/Zs+vfv7/QEFi5cyOjRoxk5ciStWrVi6dKl+Pv7s2LFCpvjP/zwQ/r168fkyZNp2bIlb731FrfddhuffPKJ1TgfHx8iIiLMj9DQUKfnJqj+aOq0gztfBGCKfhlBFFk4xQ2Q4mEiJgsuK7+ooOaafy8xdPnf9Fu0o5JmXD6Kt+YrC5NlnK81wLXjsG2e8cQD70ONom2BgMJskJwCnYUX1TmFs+zTEKBWlCiH5Q4cFrjp06eTnp5OTEwM77//PmvXrmXt2rXMmzePmJgYbty4wfTp0516c41GQ3x8PLGxsUUTksuJjY1lz549Nq/Zs2eP1XiAvn37lhi/detW6tSpQ0xMDM899xxpaWl251FQUEBmZqbVQ+AZBPqqoOerENac2tzkDeXX5nPFf0HVxZbBekli15lUKzGb+vNhwHaXdk+iU0PjH3STk0Gr1cAv48CghZgHoM0jQFFxKVNLxlyN3hw+4+oeHBiXp846KSoDhwUuPDyc3bt307JlS6ZOncrgwYMZPHgw06ZNo1WrVuzcuZPw8PCyb2RBamoqer2+xHXh4eFWJZksSU5OLnN8v379+PLLL4mLi+O9995j27Zt3H///XYzLebMmUNwcLD5ERUV5dTnELiPQF8lqHxhwCcYJBmPKrfTU17Y5b0sC06CYZ/vraKZVi3DujYAMIeJ9M/5Ba7uB59g6L/QbN6ak+0tLTizF9W594ywEjj3L0/ByVzURo0asXHjRtLT0zlz5gwATZs2pWbNmpUyOVd5/PHHzc/btm1Lu3btaNKkCVu3bqVPnz4lxk+bNo2JEyeaX2dmZgqR8xCCTKlaDbqyytCPkYoNvKv6nL4F7yHD+ufy5/1X6GhR/sdTyiW5EpBsyurwVclpJEtiRIExoJe+70BQUX9ZU7K9eYlq4UV11gIL8VehVsrR6AzVIsgXXEzVqlmzJl26dKFLly7lErewsDAUCgUpKSlWx1NSUoiIiLB5TUREhFPjARo3bkxYWJhZlIvj4+NjbmItmll7FkEWqUDztI9y0VCHerI0piq/tWmBvLH2qPl5doFn1DV0RYdNn91HIeM91TJ80EDj3tDxv1bjiltw+VoDusI3dHaBKZPJzFZcdQjyBTfnoqrVajp16kRcXJz5mMFgIC4ujm7dutm8plu3blbjATZv3mx3PMDly5dJS0sjMjLS7hiBZ2KZ65iHL1N1owH4rzKOVgWltxzMzPcMgdMZHKvGa4nJ+qpz4iu6yE+Sg6+x5FQx1Tdpp8mLCsZlKjhXTcTE7Q1DUchl3N6wejj13J5sP3HiRJYvX86qVas4fvw4zz33HDk5OYwcORKAJ598kmnTppnHv/jii2zcuJEFCxZw4sQJZs2axb///sv48eMByM7OZvLkyfz9999cuHCBuLg4Bg4cSNOmTenbt69bPqOgYsmzyKu0rCYSEx7IHkNrvtYZtyHG3vwAP+w7C5zdRHcXLuib0fq6cZGaf78LwPv6JyCkQcmBpjQ3pcKcCWKybF3xEbz/aHv+fS2WZuGBZQ+uAtwucEOGDGH+/PnMmDGDDh06cODAATZu3Gh2JCQmJpKUlGQe3717d1avXs2yZcto3749P/74I7/88gtt2hgLICoUCg4dOsSAAQNo3rw5o0aNolOnTuzYsQMfH8fTgQTVF8uwDss+Dsue7MSjneozVzeUK1ItwvVJvKL8we59PGQLziULTg7w6wTk2lz2GlqwSntPqXuOMlnRd5md77oFp5DLCK0G4SEmnC54WRmMHz/ebIEVZ+vWrSWOPfroozz66KM2x/v5+bFp06aKnJ6gmpGZV7S0tNwIb1grgPcfbc8P8Zd5TTuKlep5PK3YyHp9V/ZLzUvcR2ejWXJ1xBULLuriT3BuK5LSjyk5Y5CQU6Az4FessY9l5ZAAHyWZ+bpyWXDVDbdbcAKBs1hacPbYaujAdv97kcsk5qmWEaIuGSK04YjtUKTqhrMWXDjptDw0FwBDr+lclIwOOFuNZySLxHqzBWcWOM9XOCFwAo+jeGVee6wOGcs1KYSm8qu8IP+5ZFtCD8G55jgS76j+D5UuG+p1QtH9efPemq3GM5JFH1mTJ9Vc8NLlGVcfhMAJPA5HBS5HEcTrWqOz6inW0kZ+vjKnVWk4U7duoHwXsYoEDHIVDFwMcoVFZy09BTo93+5LNLcctAzq9Vdb71i5sgdX3RACJ/A4gh2s/iuTyfjD0Jlf9XegxMDbLEGF46Eh8RfTXZ1iheKowIWRwSzVlwCcazUe6rQEsOiNauDrvxOZ9vNh7l24HcAq79QyVATEHpxA4BZmD2jNPS3q8NWoLjbPm36h29cPBmCmdgRZ8mCac5FnFescfp9v913i7d+OmePC3IWjAjdb9QWhsmyOGhqS2PIZ83FLCy4h8QZgHWpjoqQF5+qMqw9C4AQeR90QP1aM6EyPZraLZCbMuJf412MJLSyDnk4Qmxq8DMALyv/RXOZYZdwf4y/z+c7zLPrzVMVM3EUcEbh+8n30V+xDKymYrB0LiiIr17I3qmUJqYxcrdUS1ZSuVYTnK5wQOIHX4a9WUquGj9USK7PJQPap70At0zNP9RkKHG9xeeaa40UmK4OyBC6ELN5SfQHAEv1DHJOirTygPha9US3Ltp9MyTI7GWQYSxxZIiw4gaAaY/n72TAsgFW1JpAp+dNBfo5RivV2ryuOzk0RwfM3nWTh5lNlelHfUH1FbVkGpwz1zGXcLT+7ZWcty9LlJ5MzLfovgH8xC06EiQgE1RjrIGB/Cnzr8JbOmGz+ivJHGsuuumtqZXIjR8MnW87wUdxpbubaj/vrLU/gEcVO9JKMKdqxaDAuTS09oOYlqs5AvqZI4E4kZxU5GWQyavgIC04g8BhuWJQdrx/qj59ayQ/6nmzXt8VHpmWuajkyyg6idYclY2k12mvaHEgu76r+D4D/0z/AAamp+ZzllM1ly7V6K+fCyeSsoi72lHQyeIEBJwRO4L0kpuean/uqFPirFICMadpnyJZ86SI/yXDFZvdNsBQsraepP9muijJN+Q2RsnTOG8JZqPtPseuLbmAqepmvM5CeUyT6J1OyLBrMYCNMxPMVTgicwGspvrQz5WFeoTZzdUMBeFX5HfVlthscVTVavYHrWcaeC5YClZRRsiJKd/kRnlBuAeBV7RjysS4kYSlNRb1R9Val2LPydVb3tqytV/wenooQOIHX8sp9zakZoObNga0BrBLNv9H3Ya+hBQGyAuYqlwPuLy0y+NNddH7nT04mZ3E82X5fEH/yC+cMq3T3sk9qWWKMzIYFV6AzkJxh3bTG9D4yZNzesKZVWXeRySAQVGNa1w0m/vVYnuwWDVC4RDUiIWeKdgx5kpq7FEcZothq9z5V9Wt+5IpRbNYeuMK764/bHTdZ+T0N5Ne5LIUxT/e4zTGW2mSy4LLydaTlGAXuzqa1ALiQmmMeH+yvomfzothCIXACQTXH0pIpXiroohTBfJ2x7NZryq+JwHbntQtpOQz7/G+2nbpeeRMthr2WfbfLTvCU4g8ApmmfIQc/m+Os9uAKhf3SjVwkydh8p1tjo8AVT6y3DJ72An0TAie4dSgucABf6O8nwdCUIFke76hWYGupejEtl11n0nhqxT6SMvKqYKa2xcUHDe+pliOXSXyv68UOQzuHrvctDBNJTDM6XeoE+tIiIsjm+PqhfiWOeTJC4AS3DLbKJRmQM1k7hgJJSR9FAgPlu0q9R7c5f7H2wJXKmiJgfzfwZeVPNJEnkSKF8I5uWKn3kFstUY2f+2KacTkaHuRDTIR1SXGTxVg3xK/EMU9GCJzgliEy2PZy7oxUnw91DwMwS/UlYWSUep+Z646Wer4iyMiz9gC3k51ltOI3AF7TjiKTsrpWlQz0NTXZCavhQ/1QP3P9Nyiy1upafEe2CmR6GkLgBLcMXRvVZNZDrWyeW6Z/kCOGaEJl2cwuzOu0h6IS1m4nLLymf59L42JaUQyfCh3zVMtQyCR+0XfnT0OnMu9nacH5qqwt11B/NTKZjObhNUpcF+RXJHopmfYb9ngKQuAEtwwymYx7WoTbPKdDyRTtGLSSgv6Kfdwvt9/xvjICYBdvOWt+npB40+rc88pfaCG/RKoUxGztkw7dT2YjVcuEqSlMjMU+nGm0TCZDUaiO7eqHODj76osQOMEthVRKvNsxKZol+ocAeFP1BSFk2RynrIQkTXt3bCFL5HnFWgBmaEdwA8eakpdmwdUMMAb0trDch7MQxL3T+/Dr+LtoGen5DdCFwAkEFnyiG8wpQz1qyzKZofrK5hhFBQtcvlZv0zurQM/7qqWoZHo26juz3tDV4XvKbOzBmTDVybN0NFh+orAaPrQtLBbq6QiBE9xSBPuVLHee8Ma95urAGlRM0Y5FL8l4WLGT3vKEEuMreoXaZ8E2/rlwo8TxMYrfaSu/wE0pgDe0I3Am5NhWsr0Jk8BZWnDeEBJiCyFwgluKEH81X4zozOdP3k7LyCBG3dWI0AC1lcVzQGrK5/oHAHhX9X8Ekmt1j4q04CRJ4srNktZbE9kVXlL+BMCb2uFcJ9Sp+5YqcIV7cCH+asKDjDms3hASYotq0fhZIKhKereoA0BsqyKHQ3ELZqHuUe6Vx9NYnsw05TdM1402n6tIL+qecyWzJ+QYmKdaho9MyxZ9e3429CjXe1guUVtFBtHOYvkZExFESuZ1YcEJBLcSBah5VTsGgCeUW7hTfth8Ti6XYaigKr//2liajlBsopP8NFmSH9O1z2Bamvqq5NzfJoLRPRqVeV/LMucxEYG0qx9M/3aRfD/2Dquy5b1japvHeCNC4AQCbO9u/SO1YJXuXgDmKj/HH2Nc2Jlr2TSevp5nVv3r8vtl5Gk5mZxFaLEWiA1kKUxWfg/Au7onSKKW+dw3z9zBkv92oldMnTLvb1kw01elYN34u1j8xG0EFiuJNPLORhyceR99W0e4/FmqM0LgBAKwu38/T/c4l6UwouTXzcJj4s/jKWw6mkxqdoHti0vhjnfj6Ltou1VArwwDc5XL8ZNp2KVvzbf6e6ynWDhHR6p8ONMs2pbjxVsQAicQYF80cvBjqta4/zZSuYnOshNW58d+Fc/tb/+JVl926XNLTKXDd5xONR8bqthCd8UxciUfpuqKlqbF5+iIk0Ond399u+qAEDiBgNIDMHYa2vKdrhcA76mW4YOmxJisfNeaQ5uEri6pTFOuBuB93WNckkpmXJjmaCijyxY4Z8F5M0LgBAIHeFc3jGQplMbyZF4uDN+oCIx9IyRjOIosj38NzVml72tzrMmCc0S8KjoY2VMRAicQUHZ+aSYBTNeOAmC04jfayc5anS9Pc+hH5DvopThIgaTiVe1oDHZ+LVVKxwTugbYRdG1U0+X5eBNC4AQCHIvk/8twG//T34lCJvG+6jPUFJU0euyzPSTbaA5TnLTsAuZsKCpHXpsbzFB9CcAHukc4K9Wzed3ADnWJCTeGcpQmcJPua86nwzohFxYcIAROIACs9+Aa1PS3O+5N7XBSpSBi5Jd5XrnW6twdc+LKfJ9XfjjIZ9vOFb6SeFv1BcGyXA4ZGrFc39/udR8+3tFsZepsCFz867H89UpPnu/dtMS5WxkhcAIB1hbcj891o72dZPMbBDFDOwKAcYq1tJRdtDr/5Z4Ldt/jZq6GrSeL+jr0l++lr+JfNJKCydqx6ClZcdhRatXwoXHtGl7Ry7QiEQInEACNwoqKP9YJ9GXt+Lv4c+LdNseuN3Rlg74zKpme91WfoaTIgzpjrf1qv29YnKtJJrNVKwH4VD+Qk1IDh+fas3lt2tSzqOUmNM0uIhdVIABqBqjZMqmXVd+GpnUCiQkP5GRK8bpwMmZoR9JNfow28guMUfzOp/qBZb7Hrwevmp/PVH1JmCyTE4YoFusGlXpdSLFsB7VSzm8v9KBAp+fH+Mv0aFrbzpUCYcEJBIU0CgsgPMjX6pi9mLPrhJir676o/JkmstIb0WQXFFl598r/ZaBiN3pJxmTtWLRl2Bn3tbJdhdhHqWBY14Y0qGV/z/BWRwicQFAKpQXV/s9wF3/pO+Aj0zJPtQw5xmwGW97Ua4X9DYLI5m3VCsDYB+Kw1LjMOXhrKaOqQAicQFAKpUecyZiuHUWm5Ecn+WlGKDYBRm/qqWLLWtPm/+vKbwiX3eSsIZJFukcqZ9ICM0LgBIJSGF8YdtGjWRh3Nq1V4nwytZijewKAycrvaSBLAeCF1UWVgL/dl0jv+VvpIT/EY8ptGCQZU7RjKEDt0BxaRHpnKaOqQAicQFAKD99Wn22Te7FyZBe+fNp2T4Rv9fewS98aP5mG95TLkWEgLaeowsi0nw8TQB5zVJ8DsFLfl3gpxu57PtS+LgBt6gUxpV8M/72jYQV+olsL4UUVCMqgYa2ymyxP1T3DJvlUuimO8YThL77JjiUrX2uuv/aq8jvqy1JJNNTmfd1jdu/Uv20kHw/tyPxH2+GjdD0uTmBEWHACQQVwSQpnnm4IANOUq6lLKnvOpnE9q4CusuM8qdwMwFTdaPLwtXufj4Z2BBDiVkFUC4FbvHgx0dHR+Pr60rVrV/bt21fq+B9++IEWLVrg6+tL27ZtWb9+vdV5SZKYMWMGkZGR+Pn5ERsby+nTpyvzIwhuESwDbIuzSn8f/xiaU0OWz7uq/2Pn6ev0eOd33lMtA2C17h52G9rYvf74m/1EFZAKxu0C9/333zNx4kRmzpzJ/v37ad++PX379uXatWs2x+/evZuhQ4cyatQoEhISGDRoEIMGDeLIkSPmMfPmzeOjjz5i6dKl7N27l4CAAPr27Ut+ftnJ0AJBaXz1dFeah9egbrAvUTX9+OX5O83nJOS8qh1DgaSil+IgOfu+5hXlD0TLU7gq1TQ7IyxpERFIoI+SRUM64KcWVltFI5MkB6rnVSJdu3alc+fOfPLJJwAYDAaioqJ44YUXmDp1aonxQ4YMIScnh99++8187I477qBDhw4sXboUSZKoW7cur7zyCpMmTQIgIyOD8PBwVq5cyeOPP17mnDIzMwkODiYjI4OgIM/v7i2oXKKn/m71eqziV6apvrU6NkIzma2GjubXdYN9WTikA10b1RT5oy7g6O+oWy04jUZDfHw8sbGx5mNyuZzY2Fj27Nlj85o9e/ZYjQfo27evefz58+dJTk62GhMcHEzXrl3t3rOgoIDMzEyrh0DgKF+M6EzP5rWZ9592AHyuf4BDhqLOVz/pe5jFbXSPRvz2wl1sn9KbOxrXEuJWybjVi5qamoperyc83DoVJTw8nBMnTti8Jjk52eb45ORk83nTMXtjijNnzhxmz57t0mcQCHq3qGPutfqf2+rTePp6JmvHslb9BjepwZva4Yy9uzGPdKpP83AR01aViDARYNq0aUycONH8OjMzk6ioKDfOSOCpyOUyLsw11nW7mdSXQF9/DoZ6Z0s+T8CtAhcWFoZCoSAlJcXqeEpKChERtn8oIiIiSh1v+jclJYXIyEirMR06dLB5Tx8fH3x8fFz9GAKBTUIiy84zFVQubt2DU6vVdOrUibi4okqoBoOBuLg4unXrZvOabt26WY0H2Lx5s3l8o0aNiIiIsBqTmZnJ3r177d5TIBB4KZKb+e677yQfHx9p5cqV0rFjx6QxY8ZIISEhUnJysiRJkjR8+HBp6tSp5vG7du2SlEqlNH/+fOn48ePSzJkzJZVKJR0+fNg8Zu7cuVJISIi0du1a6dChQ9LAgQOlRo0aSXl5eQ7NKSMjQwKkjIyMiv2wAoGgQnD0d9Tte3BDhgzh+vXrzJgxg+TkZDp06MDGjRvNToLExETk8iJDs3v37qxevZrXX3+d6dOn06xZM3755RfatCkKoJwyZQo5OTmMGTOGmzdvctddd7Fx40Z8fe1HkAsEAu/D7XFw1RERBycQVG88Ig5OIBAIKhMhcAKBwGsRAicQCLwWtzsZqiOmbUmRsiUQVE9Mv5tluRCEwNkgK8tYT19kMwgE1ZusrCyCg2036QbhRbWJwWDg6tWrBAYGmpOhTelbly5dEp5VC8T3Yh/x3dinvN+NJElkZWVRt25dqzCy4ggLzgZyuZz69evbPBcUFCR+WG0gvhf7iO/GPuX5bkqz3EwIJ4NAIPBahMAJBAKvRQicg/j4+DBz5kxRdaQY4nuxj/hu7FNV341wMggEAq9FWHACgcBrEQInEAi8FiFwAoHAaxECJxAIvBYhcHZIT09n2LBhBAUFERISwqhRo8jOzi51/AsvvEBMTAx+fn40aNCACRMmkJGRUYWzrhqc/W4Ali1bRq9evQgKCkImk3Hz5s2qmWwls3jxYqKjo/H19aVr167s27ev1PE//PADLVq0wNfXl7Zt27J+/foqmmnV4sz3cvToUR555BGio6ORyWQsWrSowuYhBM4Ow4YN4+jRo2zevJnffvuN7du3M2bMGLvjr169ytWrV5k/fz5Hjhxh5cqVbNy4kVGjRlXhrKsGZ78bgNzcXPr168f06dOraJaVz/fff8/EiROZOXMm+/fvp3379vTt25dr167ZHL97926GDh3KqFGjSEhIYNCgQQwaNIgjR45U8cwrF2e/l9zcXBo3bszcuXPtNptymcqtnO6ZHDt2TAKkf/75x3xsw4YNkkwmk65cueLwfdasWSOp1WpJq9VWxjTdQnm/my1btkiAdOPGjUqcZdXQpUsX6fnnnze/1uv1Ut26daU5c+bYHP/YY49J/fv3tzrWtWtXaezYsZU6z6rG2e/FkoYNG0offPBBhc1FWHA22LNnDyEhIdx+++3mY7Gxscjlcvbu3evwfUzllJVK70n5rajvxtPRaDTEx8cTGxtrPiaXy4mNjWXPnj02r9mzZ4/VeIC+ffvaHe+JuPK9VCZC4GyQnJxMnTp1rI4plUpq1qxJcnKyQ/dITU3lrbfeKnPp5mlUxHfjDaSmpqLX683NkUyEh4fb/R6Sk5OdGu+JuPK9VCa3lMBNnToVmUxW6uPEiRPlfp/MzEz69+9Pq1atmDVrVvknXgVU1XcjEFQl3rN2coBXXnmFESNGlDqmcePGRERElNgQ1el0pKenl7kJmpWVRb9+/QgMDOR///sfKpWqvNOuEqriu/EmwsLCUCgUpKSkWB1PSUmx+z1EREQ4Nd4TceV7qUxuKYGrXbs2tWvXLnNct27duHnzJvHx8XTq1AmAv/76C4PBQNeuXe1el5mZSd++ffHx8WHdunUe1Ye1sr8bb0OtVtOpUyfi4uIYNGgQYCyUGhcXx/jx421e061bN+Li4njppZfMxzZv3ky3bt2qYMZVgyvfS6VSYe4KL6Nfv35Sx44dpb1790o7d+6UmjVrJg0dOtR8/vLly1JMTIy0d+9eSZKMnba7du0qtW3bVjpz5oyUlJRkfuh0Ond9jErB2e9GkiQpKSlJSkhIkJYvXy4B0vbt26WEhAQpLS3NHR+hQvjuu+8kHx8faeXKldKxY8ekMWPGSCEhIVJycrIkSZI0fPhwaerUqebxu3btkpRKpTR//nzp+PHj0syZMyWVSiUdPnzYXR+hUnD2eykoKJASEhKkhIQEKTIyUpo0aZKUkJAgnT59utxzEQJnh7S0NGno0KFSjRo1pKCgIGnkyJFSVlaW+fz58+clQNqyZYskSUXhD7Ye58+fd8+HqCSc/W4kSZJmzpxp87v54osvqv4DVCAff/yx1KBBA0mtVktdunSR/v77b/O5nj17Sk899ZTV+DVr1kjNmzeX1Gq11Lp1a+n333+v4hlXDc58L6afl+KPnj17lnseolySQCDwWm4pL6pAILi1EAInEAi8FiFwAoHAaxECJxAIvBYhcAKBwGsRAicQCLwWIXACgcBrEQInEAi8FiFwAo9hxIgRNquc9OvXz91TE1RTbqlke4Hn069fP7744gurY/a6o2u12hLVXDQaDWq12un3dfU6gXsRFpzAo/Dx8SEiIsLqERoaCoBMJmPJkiUMGDCAgIAA3nnnHWbNmkWHDh34/PPPadSokbnCS2JiIgMHDqRGjRoEBQXx2GOPWZX4sXedwLMQAifwKmbNmsXgwYM5fPgwTz/9NABnzpzhp59+4ueff+bAgQMYDAYGDhxIeno627ZtY/PmzZw7d44hQ4ZY3av4dQLPQyxRBR7Fb7/9Ro0aNayOTZ8+3dyt64knnmDkyJFW5zUaDV9++aW53t3mzZs5fPgw58+fJyoqCoAvv/yS1q1b888//9C5c2eb1wk8DyFwAo+id+/eLFmyxOpYzZo1zc8tm+GYaNiwoZVIHT9+nKioKLO4AbRq1YqQkBCOHz9uFrji1wk8DyFwAo8iICCApk2blnrekWOOvpfAsxF7cIJbjpYtW3Lp0iUuXbpkPnbs2DFu3rxJq1at3DgzQUUjLDiBR1FQUFCi/ZxSqSQsLMzhe8TGxtK2bVuGDRvGokWL0Ol0jBs3jp49e9pc4go8F2HBCTyKjRs3EhkZafW46667nLqHTCZj7dq1hIaGcvfddxMbG0vjxo35/vvvK2nWAnchSpYLBAKvRVhwAoHAaxECJxAIvBYhcAKBwGsRAicQCLwWIXACgcBrEQInEAi8FiFwAoHAaxECJxAIvBYhcAKBwGsRAicQCLwWIXACgcBrEQInEAi8lv8HGLLKWg/LVGoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018729, + "end_time": "2024-03-22T21:54:38.634759", + "exception": false, + "start_time": "2024-03-22T21:54:38.616030", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 2666.929317, + "end_time": "2024-03-22T21:54:41.375544", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tvae/2/mlu-eval.ipynb", + "output_path": "eval/insurance/tvae/2/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/tvae/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "tvae" + }, + "start_time": "2024-03-22T21:10:14.446227", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tvae/model.pt b/insurance/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fe024c9141e2839399e94ffa00c792f0da3fec3 --- /dev/null +++ b/insurance/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86800e68cd854a15a18fb0b685da25ff930b8062fcf84265ce348c33e151e6c1 +size 38612117 diff --git a/insurance/tvae/params.json b/insurance/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..b285a7a8bff6eddb1779941210fa348cbd2a5598 --- /dev/null +++ b/insurance/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/lct_gan/eval.csv b/treatment/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..7abeb3dbc3058e5f74a0b93e300522340b3be5f7 --- /dev/null +++ b/treatment/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0,0.00044872478942392847,0.004746775878468595,11.582196950912476,0.09063062071800232,1.6706515550613403,0.14325737953186035,3.869256761390716e-05,6.415107011795044,0.04724160209298134,4645954.5,0.06889685243368149,0.23633873462677002,0.0008537429966963828,17.99730396270752 diff --git a/treatment/lct_gan/history.csv b/treatment/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..f403f0312531ce2d3f7424576d5f69a36cfb8a6a --- /dev/null +++ b/treatment/lct_gan/history.csv @@ -0,0 +1,8 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.08288680652415173,14.884825719568891,0.014639717982716328,0.007401900000145865,0.0,0.0,0.0,0.0,0.406318084973221,900,225,398.05458784103394,1.7691315015157063,0.4422828753789266,0.04329945017169747,0.01646603072894332,1.1389748531219954,0.001041307360675295,0.0,0.0,0.0,0.0,0.0,0.01646603072894332,450,113,104.75376605987549,0.9270244784059778,0.23278614679972331,0.1397827780404771 +1,0.014255170632645281,0.24843680457298686,0.000636702568035041,0.15559853479266167,0.0,0.0,0.0,0.0,0.014447312605981198,900,225,406.69940519332886,1.8075529119703504,0.4518882279925876,0.22602122453765736,0.011084747278914115,0.774564493527634,0.00036442376687365014,0.0,0.0,0.0,0.0,0.0,0.011084747278914115,450,113,105.70433187484741,0.9354365652641364,0.23489851527743869,0.13189282911609182 +2,0.009175471019561883,0.14389958513339807,0.0003093209020328993,0.16885624952562567,0.0,0.0,0.0,0.0,0.00928754332613001,900,225,406.6975419521332,1.8075446308983696,0.4518861577245924,0.23166270198714403,0.00785602382393045,1.3146459211567783,0.0001525110592525784,0.0,0.0,0.0,0.0,0.0,0.00785602382393045,450,113,105.51167917251587,0.9337316740930608,0.23447039816114637,0.11703323267914861 +3,0.006946129192502769,0.16631651740127382,0.00011629207969720338,0.1469966934973167,0.0,0.0,0.0,0.0,0.007030756754486194,900,225,404.7738826274872,1.798995033899943,0.44974875847498574,0.23086450531949393,0.006420711345431553,1.1872994360646385,8.72488180067034e-05,0.0,0.0,0.0,0.0,0.0,0.006420711345431553,450,113,103.53712010383606,0.9162577000339475,0.2300824891196357,0.1300600932704221 +4,0.005421485699508695,0.08202415918292068,0.00010085892813725515,0.13527608269825578,0.0,0.0,0.0,0.0,0.005492081102662875,900,225,403.6879127025604,1.7941685009002686,0.44854212522506715,0.23655629260775943,0.009256900170618995,2.8233697297467697,0.0006465031184101571,0.0,0.0,0.0,0.0,0.0,0.009256900170618995,450,113,101.8208520412445,0.9010694870906594,0.22626856009165447,0.13276277751503135 +5,0.0043073366452492535,0.19139563928810466,8.40804457109845e-05,0.12489897313269062,0.0,0.0,0.0,0.0,0.0043637448957360905,900,225,398.688072681427,1.7719469896952311,0.4429867474238078,0.23179641341906973,0.0055422348428186925,1.4432227016979617,5.823207058893942e-05,0.0,0.0,0.0,0.0,0.0,0.0055422348428186925,450,113,101.09947466850281,0.8946856165354231,0.22466549926333956,0.11363168490392674 +6,0.003263339866756117,0.3763446019548343,2.3110432702086737e-05,0.10734513330583771,0.0,0.0,0.0,0.0,0.0033083394544084713,900,225,398.69307565689087,1.7719692251417372,0.4429923062854343,0.24115597604735134,0.00555163197664519,1.1181977761929087,0.00010318647899446903,0.0,0.0,0.0,0.0,0.0,0.00555163197664519,450,113,101.25119686126709,0.8960282908076733,0.22500265969170463,0.11773095095579861 diff --git a/treatment/lct_gan/mlu-eval.ipynb b/treatment/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6593d32aeff0cef891570f145ac20eee93665eb7 --- /dev/null +++ b/treatment/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.603965Z", + "iopub.status.busy": "2024-03-22T16:53:54.603594Z", + "iopub.status.idle": "2024-03-22T16:53:54.637506Z", + "shell.execute_reply": "2024-03-22T16:53:54.636558Z" + }, + "papermill": { + "duration": 0.049204, + "end_time": "2024-03-22T16:53:54.639737", + "exception": false, + "start_time": "2024-03-22T16:53:54.590533", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.666476Z", + "iopub.status.busy": "2024-03-22T16:53:54.666093Z", + "iopub.status.idle": "2024-03-22T16:53:54.673364Z", + "shell.execute_reply": "2024-03-22T16:53:54.672349Z" + }, + "papermill": { + "duration": 0.023236, + "end_time": "2024-03-22T16:53:54.675564", + "exception": false, + "start_time": "2024-03-22T16:53:54.652328", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.700593Z", + "iopub.status.busy": "2024-03-22T16:53:54.700315Z", + "iopub.status.idle": "2024-03-22T16:53:54.704453Z", + "shell.execute_reply": "2024-03-22T16:53:54.703667Z" + }, + "papermill": { + "duration": 0.019169, + "end_time": "2024-03-22T16:53:54.706517", + "exception": false, + "start_time": "2024-03-22T16:53:54.687348", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.731010Z", + "iopub.status.busy": "2024-03-22T16:53:54.730726Z", + "iopub.status.idle": "2024-03-22T16:53:54.734735Z", + "shell.execute_reply": "2024-03-22T16:53:54.733853Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018617, + "end_time": "2024-03-22T16:53:54.736705", + "exception": false, + "start_time": "2024-03-22T16:53:54.718088", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.760821Z", + "iopub.status.busy": "2024-03-22T16:53:54.760582Z", + "iopub.status.idle": "2024-03-22T16:53:54.766024Z", + "shell.execute_reply": "2024-03-22T16:53:54.765223Z" + }, + "papermill": { + "duration": 0.019584, + "end_time": "2024-03-22T16:53:54.768060", + "exception": false, + "start_time": "2024-03-22T16:53:54.748476", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fe93b2cc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.793738Z", + "iopub.status.busy": "2024-03-22T16:53:54.793435Z", + "iopub.status.idle": "2024-03-22T16:53:54.798469Z", + "shell.execute_reply": "2024-03-22T16:53:54.797700Z" + }, + "papermill": { + "duration": 0.020228, + "end_time": "2024-03-22T16:53:54.800426", + "exception": false, + "start_time": "2024-03-22T16:53:54.780198", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/lct_gan/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011272, + "end_time": "2024-03-22T16:53:54.823078", + "exception": false, + "start_time": "2024-03-22T16:53:54.811806", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.847715Z", + "iopub.status.busy": "2024-03-22T16:53:54.847006Z", + "iopub.status.idle": "2024-03-22T16:53:54.856170Z", + "shell.execute_reply": "2024-03-22T16:53:54.855346Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023787, + "end_time": "2024-03-22T16:53:54.858208", + "exception": false, + "start_time": "2024-03-22T16:53:54.834421", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/lct_gan/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:54.882551Z", + "iopub.status.busy": "2024-03-22T16:53:54.881844Z", + "iopub.status.idle": "2024-03-22T16:53:56.921479Z", + "shell.execute_reply": "2024-03-22T16:53:56.920577Z" + }, + "papermill": { + "duration": 2.054095, + "end_time": "2024-03-22T16:53:56.923643", + "exception": false, + "start_time": "2024-03-22T16:53:54.869548", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:56.951621Z", + "iopub.status.busy": "2024-03-22T16:53:56.951153Z", + "iopub.status.idle": "2024-03-22T16:53:56.967827Z", + "shell.execute_reply": "2024-03-22T16:53:56.967069Z" + }, + "papermill": { + "duration": 0.033246, + "end_time": "2024-03-22T16:53:56.969900", + "exception": false, + "start_time": "2024-03-22T16:53:56.936654", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:56.994398Z", + "iopub.status.busy": "2024-03-22T16:53:56.994089Z", + "iopub.status.idle": "2024-03-22T16:53:57.002129Z", + "shell.execute_reply": "2024-03-22T16:53:57.001365Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022576, + "end_time": "2024-03-22T16:53:57.004084", + "exception": false, + "start_time": "2024-03-22T16:53:56.981508", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:57.027681Z", + "iopub.status.busy": "2024-03-22T16:53:57.027415Z", + "iopub.status.idle": "2024-03-22T16:53:57.124714Z", + "shell.execute_reply": "2024-03-22T16:53:57.123657Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.111825, + "end_time": "2024-03-22T16:53:57.127116", + "exception": false, + "start_time": "2024-03-22T16:53:57.015291", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:53:57.155124Z", + "iopub.status.busy": "2024-03-22T16:53:57.154827Z", + "iopub.status.idle": "2024-03-22T16:54:01.862723Z", + "shell.execute_reply": "2024-03-22T16:54:01.861696Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.725452, + "end_time": "2024-03-22T16:54:01.865848", + "exception": false, + "start_time": "2024-03-22T16:53:57.140396", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 16:53:59.415157: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 16:53:59.415216: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 16:53:59.416897: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:01.897085Z", + "iopub.status.busy": "2024-03-22T16:54:01.896382Z", + "iopub.status.idle": "2024-03-22T16:54:01.903562Z", + "shell.execute_reply": "2024-03-22T16:54:01.902506Z" + }, + "papermill": { + "duration": 0.022721, + "end_time": "2024-03-22T16:54:01.905669", + "exception": false, + "start_time": "2024-03-22T16:54:01.882948", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:01.933894Z", + "iopub.status.busy": "2024-03-22T16:54:01.933539Z", + "iopub.status.idle": "2024-03-22T16:54:24.491400Z", + "shell.execute_reply": "2024-03-22T16:54:24.490297Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.574641, + "end_time": "2024-03-22T16:54:24.494065", + "exception": false, + "start_time": "2024-03-22T16:54:01.919424", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T16:54:24.524116Z", + "iopub.status.busy": "2024-03-22T16:54:24.523692Z", + "iopub.status.idle": "2024-03-22T16:54:24.530799Z", + "shell.execute_reply": "2024-03-22T16:54:24.529968Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024897, + "end_time": "2024-03-22T16:54:24.532947", + "exception": false, + "start_time": "2024-03-22T16:54:24.508050", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:24.559222Z", + "iopub.status.busy": "2024-03-22T16:54:24.558898Z", + "iopub.status.idle": "2024-03-22T16:54:24.563980Z", + "shell.execute_reply": "2024-03-22T16:54:24.563114Z" + }, + "papermill": { + "duration": 0.020558, + "end_time": "2024-03-22T16:54:24.565917", + "exception": false, + "start_time": "2024-03-22T16:54:24.545359", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:24.591748Z", + "iopub.status.busy": "2024-03-22T16:54:24.591466Z", + "iopub.status.idle": "2024-03-22T17:18:56.244286Z", + "shell.execute_reply": "2024-03-22T17:18:56.243405Z" + }, + "papermill": { + "duration": 1471.680855, + "end_time": "2024-03-22T17:18:56.259077", + "exception": false, + "start_time": "2024-03-22T16:54:24.578222", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:18:56.287603Z", + "iopub.status.busy": "2024-03-22T17:18:56.287251Z", + "iopub.status.idle": "2024-03-22T17:18:56.608589Z", + "shell.execute_reply": "2024-03-22T17:18:56.607712Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.338263, + "end_time": "2024-03-22T17:18:56.610783", + "exception": false, + "start_time": "2024-03-22T17:18:56.272520", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:18:56.639191Z", + "iopub.status.busy": "2024-03-22T17:18:56.638880Z", + "iopub.status.idle": "2024-03-22T17:50:01.925295Z", + "shell.execute_reply": "2024-03-22T17:50:01.924278Z" + }, + "papermill": { + "duration": 1865.316192, + "end_time": "2024-03-22T17:50:01.940639", + "exception": false, + "start_time": "2024-03-22T17:18:56.624447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T17:50:01.971379Z", + "iopub.status.busy": "2024-03-22T17:50:01.970531Z", + "iopub.status.idle": "2024-03-22T17:50:02.499249Z", + "shell.execute_reply": "2024-03-22T17:50:02.498338Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.546432, + "end_time": "2024-03-22T17:50:02.501385", + "exception": false, + "start_time": "2024-03-22T17:50:01.954953", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:50:02.532791Z", + "iopub.status.busy": "2024-03-22T17:50:02.532446Z", + "iopub.status.idle": "2024-03-22T17:50:02.536685Z", + "shell.execute_reply": "2024-03-22T17:50:02.535782Z" + }, + "papermill": { + "duration": 0.023513, + "end_time": "2024-03-22T17:50:02.539641", + "exception": false, + "start_time": "2024-03-22T17:50:02.516128", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:50:02.568171Z", + "iopub.status.busy": "2024-03-22T17:50:02.567868Z", + "iopub.status.idle": "2024-03-22T17:50:02.574874Z", + "shell.execute_reply": "2024-03-22T17:50:02.573995Z" + }, + "papermill": { + "duration": 0.023652, + "end_time": "2024-03-22T17:50:02.576882", + "exception": false, + "start_time": "2024-03-22T17:50:02.553230", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18680833" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:50:02.605328Z", + "iopub.status.busy": "2024-03-22T17:50:02.605063Z", + "iopub.status.idle": "2024-03-22T17:50:02.699743Z", + "shell.execute_reply": "2024-03-22T17:50:02.698806Z" + }, + "papermill": { + "duration": 0.111362, + "end_time": "2024-03-22T17:50:02.702059", + "exception": false, + "start_time": "2024-03-22T17:50:02.590697", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 75] --\n", + "├─Adapter: 1-1 [2, 2648, 75] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 77,824\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 75] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,680,833\n", + "Trainable params: 18,680,833\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.97\n", + "========================================================================================================================\n", + "Input size (MB): 1.99\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.72\n", + "Estimated Total Size (MB): 1156.19\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:50:02.735226Z", + "iopub.status.busy": "2024-03-22T17:50:02.734892Z", + "iopub.status.idle": "2024-03-22T19:00:38.391656Z", + "shell.execute_reply": "2024-03-22T19:00:38.390642Z" + }, + "papermill": { + "duration": 4235.677039, + "end_time": "2024-03-22T19:00:38.394746", + "exception": false, + "start_time": "2024-03-22T17:50:02.717707", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.08288680652415173, 'avg_role_model_std_loss': 14.884825719568891, 'avg_role_model_mean_pred_loss': 0.014639717982716328, 'avg_role_model_g_mag_loss': 0.007401900000145865, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.406318084973221, 'n_size': 900, 'n_batch': 225, 'duration': 398.05458784103394, 'duration_batch': 1.7691315015157063, 'duration_size': 0.4422828753789266, 'avg_pred_std': 0.04329945017169747}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01646603072894332, 'avg_role_model_std_loss': 1.1389748531219954, 'avg_role_model_mean_pred_loss': 0.001041307360675295, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01646603072894332, 'n_size': 450, 'n_batch': 113, 'duration': 104.75376605987549, 'duration_batch': 0.9270244784059778, 'duration_size': 0.23278614679972331, 'avg_pred_std': 0.1397827780404771}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014255170632645281, 'avg_role_model_std_loss': 0.24843680457298686, 'avg_role_model_mean_pred_loss': 0.000636702568035041, 'avg_role_model_g_mag_loss': 0.15559853479266167, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014447312605981198, 'n_size': 900, 'n_batch': 225, 'duration': 406.69940519332886, 'duration_batch': 1.8075529119703504, 'duration_size': 0.4518882279925876, 'avg_pred_std': 0.22602122453765736}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011084747278914115, 'avg_role_model_std_loss': 0.774564493527634, 'avg_role_model_mean_pred_loss': 0.00036442376687365014, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011084747278914115, 'n_size': 450, 'n_batch': 113, 'duration': 105.70433187484741, 'duration_batch': 0.9354365652641364, 'duration_size': 0.23489851527743869, 'avg_pred_std': 0.13189282911609182}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009175471019561883, 'avg_role_model_std_loss': 0.14389958513339807, 'avg_role_model_mean_pred_loss': 0.0003093209020328993, 'avg_role_model_g_mag_loss': 0.16885624952562567, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00928754332613001, 'n_size': 900, 'n_batch': 225, 'duration': 406.6975419521332, 'duration_batch': 1.8075446308983696, 'duration_size': 0.4518861577245924, 'avg_pred_std': 0.23166270198714403}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00785602382393045, 'avg_role_model_std_loss': 1.3146459211567783, 'avg_role_model_mean_pred_loss': 0.0001525110592525784, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00785602382393045, 'n_size': 450, 'n_batch': 113, 'duration': 105.51167917251587, 'duration_batch': 0.9337316740930608, 'duration_size': 0.23447039816114637, 'avg_pred_std': 0.11703323267914861}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006946129192502769, 'avg_role_model_std_loss': 0.16631651740127382, 'avg_role_model_mean_pred_loss': 0.00011629207969720338, 'avg_role_model_g_mag_loss': 0.1469966934973167, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007030756754486194, 'n_size': 900, 'n_batch': 225, 'duration': 404.7738826274872, 'duration_batch': 1.798995033899943, 'duration_size': 0.44974875847498574, 'avg_pred_std': 0.23086450531949393}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006420711345431553, 'avg_role_model_std_loss': 1.1872994360646385, 'avg_role_model_mean_pred_loss': 8.72488180067034e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006420711345431553, 'n_size': 450, 'n_batch': 113, 'duration': 103.53712010383606, 'duration_batch': 0.9162577000339475, 'duration_size': 0.2300824891196357, 'avg_pred_std': 0.1300600932704221}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005421485699508695, 'avg_role_model_std_loss': 0.08202415918292068, 'avg_role_model_mean_pred_loss': 0.00010085892813725515, 'avg_role_model_g_mag_loss': 0.13527608269825578, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005492081102662875, 'n_size': 900, 'n_batch': 225, 'duration': 403.6879127025604, 'duration_batch': 1.7941685009002686, 'duration_size': 0.44854212522506715, 'avg_pred_std': 0.23655629260775943}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009256900170618995, 'avg_role_model_std_loss': 2.8233697297467697, 'avg_role_model_mean_pred_loss': 0.0006465031184101571, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009256900170618995, 'n_size': 450, 'n_batch': 113, 'duration': 101.8208520412445, 'duration_batch': 0.9010694870906594, 'duration_size': 0.22626856009165447, 'avg_pred_std': 0.13276277751503135}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0043073366452492535, 'avg_role_model_std_loss': 0.19139563928810466, 'avg_role_model_mean_pred_loss': 8.40804457109845e-05, 'avg_role_model_g_mag_loss': 0.12489897313269062, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0043637448957360905, 'n_size': 900, 'n_batch': 225, 'duration': 398.688072681427, 'duration_batch': 1.7719469896952311, 'duration_size': 0.4429867474238078, 'avg_pred_std': 0.23179641341906973}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0055422348428186925, 'avg_role_model_std_loss': 1.4432227016979617, 'avg_role_model_mean_pred_loss': 5.823207058893942e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0055422348428186925, 'n_size': 450, 'n_batch': 113, 'duration': 101.09947466850281, 'duration_batch': 0.8946856165354231, 'duration_size': 0.22466549926333956, 'avg_pred_std': 0.11363168490392674}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003263339866756117, 'avg_role_model_std_loss': 0.3763446019548343, 'avg_role_model_mean_pred_loss': 2.3110432702086737e-05, 'avg_role_model_g_mag_loss': 0.10734513330583771, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033083394544084713, 'n_size': 900, 'n_batch': 225, 'duration': 398.69307565689087, 'duration_batch': 1.7719692251417372, 'duration_size': 0.4429923062854343, 'avg_pred_std': 0.24115597604735134}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00555163197664519, 'avg_role_model_std_loss': 1.1181977761929087, 'avg_role_model_mean_pred_loss': 0.00010318647899446903, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00555163197664519, 'n_size': 450, 'n_batch': 113, 'duration': 101.25119686126709, 'duration_batch': 0.8960282908076733, 'duration_size': 0.22500265969170463, 'avg_pred_std': 0.11773095095579861}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0027152151927152266, 'avg_role_model_std_loss': 0.07267851859859407, 'avg_role_model_mean_pred_loss': 1.4569837801249912e-05, 'avg_role_model_g_mag_loss': 0.09808282882389095, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027543007831668043, 'n_size': 900, 'n_batch': 225, 'duration': 400.2191047668457, 'duration_batch': 1.7787515767415365, 'duration_size': 0.4446878941853841, 'avg_pred_std': 0.23774532583390182}\n", + "Time out: 3949.559079647064/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0047467758956138695, 'avg_g_mag_loss': 0.016803590516231294, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.370237112045288, 'grad_duration': 11.638083219528198, 'total_duration': 18.008320331573486, 'pred_std': 0.23633873462677002, 'std_loss': 0.0008537429966963828, 'mean_pred_loss': 3.869256761390716e-05, 'pred_rmse': 0.06889685243368149, 'pred_mae': 0.04724160581827164, 'pred_mape': 4645955.0, 'grad_rmse': 0.14325737953186035, 'grad_mae': 0.09063062816858292, 'grad_mape': 1.6706515550613403}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0047467758956138695, 'avg_g_mag_loss': 0.016803590516231294, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 6.370237112045288, 'avg_grad_duration': 11.638083219528198, 'avg_total_duration': 18.008320331573486, 'avg_pred_std': 0.23633873462677002, 'avg_std_loss': 0.0008537429966963828, 'avg_mean_pred_loss': 3.869256761390716e-05}, 'min_metrics': {'avg_loss': 0.0047467758956138695, 'avg_g_mag_loss': 0.016803590516231294, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.370237112045288, 'grad_duration': 11.638083219528198, 'total_duration': 18.008320331573486, 'pred_std': 0.23633873462677002, 'std_loss': 0.0008537429966963828, 'mean_pred_loss': 3.869256761390716e-05, 'pred_rmse': 0.06889685243368149, 'pred_mae': 0.04724160581827164, 'pred_mape': 4645955.0, 'grad_rmse': 0.14325737953186035, 'grad_mae': 0.09063062816858292, 'grad_mape': 1.6706515550613403}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0047467758956138695, 'avg_g_mag_loss': 0.016803590516231294, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.370237112045288, 'grad_duration': 11.638083219528198, 'total_duration': 18.008320331573486, 'pred_std': 0.23633873462677002, 'std_loss': 0.0008537429966963828, 'mean_pred_loss': 3.869256761390716e-05, 'pred_rmse': 0.06889685243368149, 'pred_mae': 0.04724160581827164, 'pred_mape': 4645955.0, 'grad_rmse': 0.14325737953186035, 'grad_mae': 0.09063062816858292, 'grad_mape': 1.6706515550613403}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:00:38.430630Z", + "iopub.status.busy": "2024-03-22T19:00:38.429858Z", + "iopub.status.idle": "2024-03-22T19:00:38.434341Z", + "shell.execute_reply": "2024-03-22T19:00:38.433435Z" + }, + "papermill": { + "duration": 0.024595, + "end_time": "2024-03-22T19:00:38.436245", + "exception": false, + "start_time": "2024-03-22T19:00:38.411650", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:00:38.469769Z", + "iopub.status.busy": "2024-03-22T19:00:38.469456Z", + "iopub.status.idle": "2024-03-22T19:00:38.596113Z", + "shell.execute_reply": "2024-03-22T19:00:38.595324Z" + }, + "papermill": { + "duration": 0.146222, + "end_time": "2024-03-22T19:00:38.598627", + "exception": false, + "start_time": "2024-03-22T19:00:38.452405", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:00:38.635319Z", + "iopub.status.busy": "2024-03-22T19:00:38.634452Z", + "iopub.status.idle": "2024-03-22T19:00:38.905876Z", + "shell.execute_reply": "2024-03-22T19:00:38.904973Z" + }, + "papermill": { + "duration": 0.291967, + "end_time": "2024-03-22T19:00:38.907984", + "exception": false, + "start_time": "2024-03-22T19:00:38.616017", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAESCAYAAAAv/mqQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApwElEQVR4nO3deXRU9d0/8PedPZNlspHVQFgSdgglJAXcTU0tauHBmnpSgzkVHi1RNKXFuCTWnhpU6smjcKTSR3iq5gm/56l4nlIbpIHYQ40GE1EsEAGBRGCysEySmWSWe7+/PyYzycQscyeTubN8Xufck5k7d/nMMPPm3u+993s5xhgDIYT4gEzqAgghoYMChxDiMxQ4hBCfocAhhPgMBQ4hxGcocAghPkOBQwjxGYXUBbhDEARcunQJkZGR4DhO6nIIIUMwxtDT04OUlBTIZGNvwwRE4Fy6dAlpaWlSl0EIGUNbWxtuuOGGMacJiMCJjIwEYH9DUVFREldDCBmqu7sbaWlpzt/pWAIicBy7UVFRURQ4hPgpd5o7qNGYEOIzFDiEEJ+hwCGE+ExAtOEQz/E8D6vVKnUZJMCpVKpxD3m7gwInSDHGoNfrcf36dalLIUFAJpNh+vTpUKlUE1oOBU6QcoRNQkICtFotnTBJPOY48fby5cuYOnXqhL5LQRU457uMOKXvQe70WMSETyyJAxnP886wiYuLk7ocEgSmTJmCS5cuwWazQalUerwcj3bKduzYgfT0dGg0GuTm5qKxsdGt+WpqasBxHFavXu3Jasf1yDtNeOSdJjS3XpuU5QcKR5uNVquVuBISLBy7UjzPT2g5ogNn7969KC0tRUVFBZqbm7F48WLk5+ejo6NjzPnOnz+PzZs346abbvK42PHMTrKf6XhK3zNp6wgktBtFvMVb3yXRgfPqq69i/fr1KC4uxrx587Bz505otVq89dZbo87D8zwKCwvxm9/8BjNmzJhQwWPJTLQHztftFDiE+CNRgWOxWNDU1IS8vLzBBchkyMvLQ0NDw6jzvfDCC0hISMDPf/5zt9ZjNpvR3d3tMrhj9kDgtNAWDiF+SVTgdHV1ged5JCYmuoxPTEyEXq8fcZ4jR47gP//zP7Fr1y6311NZWQmdTucc3L1S3LFL9U2nEVZecHt9hAzFcRzef/99qcvwqueffx5ZWVlSlzG5Zxr39PTgwQcfxK5duxAfH+/2fGVlZTAYDM6hra3NrflSo8OgVclh4QVcuGL0tGxCJLdnzx5ER0d7bXmbN29GXV2d15bnKVGHxePj4yGXy9He3u4yvr29HUlJSd+Z/uzZszh//jzuuece5zhBsG95KBQKtLS0YObMmd+ZT61WQ61WiykNACCTcchMjMSxtuto0fdiVsL4l8sTEsgsFotbJ+NFREQgIiLCBxWNTdQWjkqlwtKlS12SUhAE1NXVYfny5d+Zfs6cOTh+/DiOHTvmHO69917cdtttOHbs2KR0qjXYjuNeu0+oYIzBZLFJMoi9uWttbS1uvPFGREdHIy4uDnfffTfOnj0LAFixYgW2bNniMn1nZyeUSiX+8Y9/AAAuX76MVatWISwsDNOnT0d1dTXS09NRVVXl0Wd3/Phx3H777QgLC0NcXBw2bNiA3t5e5+v19fXIyclBeHg4oqOjsXLlSly4cAEA8MUXX+C2225DZGQkoqKisHTpUnz22Wdjrq++vh7FxcUwGAzgOA4cx+H5558HAKSnp+O3v/0tioqKEBUVhQ0bNgAAtmzZgszMTGi1WsyYMQPPPfecyyUtw3epHnroIaxevRrbtm1DcnIy4uLisHHjxkm/DEb0iX+lpaVYt24dsrOzkZOTg6qqKhiNRhQXFwMAioqKkJqaisrKSmg0GixYsMBlfsdm4vDx3pI50I7TQkeqXPRZecwrPyDJuk+8kA+tyv2vmtFoRGlpKRYtWoTe3l6Ul5djzZo1OHbsGAoLC/Hyyy9j69atzkO1e/fuRUpKivOUi6KiInR1daG+vh5KpRKlpaXjnrYxVi35+flYvnw5jh49io6ODjz88MMoKSnBnj17YLPZsHr1aqxfvx7//d//DYvFgsbGRmdthYWFWLJkCd544w3I5XIcO3Zs3BPnVqxYgaqqKpSXl6OlpQUAXLZOtm3bhvLyclRUVDjHRUZGYs+ePUhJScHx48exfv16REZG4te//vWo6zl8+DCSk5Nx+PBhnDlzBgUFBcjKysL69es9+qzcITpwCgoK0NnZifLycuj1emRlZaG2ttbZkNza2uqVi7w8Ndt5aLx3nCmJv1q7dq3L87feegtTpkzBiRMncP/99+OJJ57AkSNHnAFTXV2NBx54ABzH4dSpU/j73/+Oo0ePIjs7GwDwxz/+ERkZGR7VUl1djf7+fvzpT39CeHg4AGD79u2455578NJLL0GpVMJgMODuu+92Ng/MnTvXOX9rayt+9atfYc6cOQDgVh0qlQo6nQ4cx43YVHH77bfjl7/8pcu4Z5991vk4PT0dmzdvRk1NzZiBExMTg+3bt0Mul2POnDlYtWoV6urq/CtwAKCkpAQlJSUjvlZfXz/mvHv27PFklW7LTLL/T3D+ihH9Vh4apXxS1xcowpRynHghX7J1i3H69GmUl5fj008/RVdXl7Pdr7W1FQsWLMCdd96Jd999FzfddBPOnTuHhoYG/OEPfwAAtLS0QKFQ4Hvf+55zebNmzUJMTIxHtZ88eRKLFy92hg0ArFy5EoIgoKWlBTfffDMeeugh5Ofn4wc/+AHy8vJw//33Izk5GYB9j+Dhhx/G22+/jby8PPzkJz8Zsd1SDEeQDrV371689tprOHv2LHp7e2Gz2cbtHXP+/PmQywf/bZKTk3H8+PEJ1TaeoOsPZ0qEGjFaJRgDznTQVo4Dx3HQqhSSDGLPUr3nnntw9epV7Nq1C59++ik+/fRTAPYGUsC+m/K///u/sFqtqK6uxsKFC7Fw4UKvf2bu2r17NxoaGrBixQrs3bsXmZmZ+OSTTwDY207+9a9/YdWqVTh06BDmzZuHffv2TWh9Q8MPABoaGlBYWIgf/ehH2L9/Pz7//HM888wzzs9rNMN37TiOc4b7ZAm6wOE4znk+Dp0AGHiuXLmClpYWPPvss7jjjjswd+5cXLvmem3cj3/8Y/T396O2thbV1dUoLCx0vjZ79mzYbDZ8/vnnznFnzpz5zjLcNXfuXHzxxRcwGgdPs/jnP/8JmUyG2bNnO8ctWbIEZWVl+Pjjj7FgwQJUV1c7X8vMzMSTTz6JDz/8EP/2b/+G3bt3j7telUrl9nVLH3/8MaZNm4ZnnnkG2dnZyMjIcDZa+5ugCxxgyJEqajgOODExMYiLi8Obb76JM2fO4NChQygtLXWZJjw8HKtXr8Zzzz2HkydP4oEHHnC+NmfOHOTl5WHDhg1obGzE559/jg0bNiAsLMyj64EKCwuh0Wiwbt06fPXVVzh8+DAee+wxPPjgg0hMTMS5c+dQVlaGhoYGXLhwAR9++CFOnz6NuXPnoq+vDyUlJaivr8eFCxfwz3/+E0ePHnVp4xlNeno6ent7UVdXh66uLphMplGnzcjIQGtrK2pqanD27Fm89tprE96KmixBGTiZtIUTsGQyGWpqatDU1IQFCxbgySefxCuvvPKd6QoLC/HFF1/gpptuwtSpU11e+9Of/oTExETcfPPNWLNmjfOIjUajEV2PVqvFgQMHcPXqVSxbtgz33Xcf7rjjDmzfvt35+qlTp7B27VpkZmZiw4YN2LhxI/793/8dcrkcV65cQVFRETIzM3H//ffjrrvuwm9+85tx17tixQo88sgjKCgowJQpU/Dyyy+POu29996LJ598EiUlJcjKysLHH3+M5557TvR79QWOiT1JQgLd3d3Q6XQwGAxu3Sbms/NXcd/OBiTrNGgou8MHFfqX/v5+nDt3DtOnT/foRxZsvv32W6SlpeHvf/877rgj9L4P3jDWd0rM7zOoOuByyBjYpbps6IehzwpdmOcdBpHAc+jQIfT29mLhwoW4fPkyfv3rXyM9PR0333yz1KWFvKDcpdKFKZGss6fwaWrHCTlWqxVPP/005s+fjzVr1mDKlCnOkwDfffdd52n+w4f58+f7rMa77rpr1DpefPFFn9Xha0G5hQPYrxy/bOhHS3sPstNjpS6H+FB+fj7y80c+5+jee+9Fbm7uiK9NpOtMsf74xz+ir69vxNdiY4P3+xq8gZMYifqWTmo4Ji4iIyPdugf2ZEtNTZW6BEkE5S4VMNj7HwUOIf4jaAPHcfLf1+09oq9WJoRMjqANnFkJEeA44JrJis5es9TlEEIQxIGjUcqRHme/5uRrPV1TRYg/CNrAAegSB0L8TVAHzuAlDtT7H3FfMHai7i+COnAGt3Bol4oEFm93og7Y+6riOA7Xr1/36nLFCO7AGeiM63R7DwSBjlQRIrWgDpxpceFQyWUwWXhcvD7yWZ0hgzHAYpRmoE7UvdaJutlsxubNm5Gamorw8HDk5ua69LJ54cIF3HPPPYiJiUF4eDjmz5+PDz74AOfPn8dtt90GwN4FCMdxeOihhzz6PCYiaM80BgClXIYZU8JxSt+DFn0P0mK1UpckHasJeDFFmnU/fQlQhY8/3QDqRH30TtRLSkpw4sQJ1NTUICUlBfv27cMPf/hDHD9+HBkZGdi4cSMsFgv+8Y9/IDw8HCdOnEBERATS0tLw5z//GWvXrkVLSwuioqIQFhbm0WcyEUEdOAAwJynSHjjtPciblzj+DERy1In6yJ2ot7a2Yvfu3WhtbUVKiv0/j82bN6O2tha7d+/Giy++iNbWVqxdu9bZ5eqMGTOc8zuu0UpISPB6+5C7gj5wqDOuAUqtfUtDqnWLQJ2oj+z48ePgeR6ZmZku481mM+Li4gAAjz/+OB599FF8+OGHyMvLw9q1a7Fo0SKP1jcZgroNBxh625gQDxyOs+/WSDFQJ+pe6US9t7cXcrkcTU1NLjeXPHnyJP7jP/4DAPDwww/jm2++wYMPPojjx48jOzsbr7/+utfe60QFfeA4LuI829kLKz+5PdKTiaNO1O1G6kR9yZIl4HkeHR0dmDVrlsswdNcrLS0NjzzyCN577z388pe/xK5du5zLBOB25+yTIegDJzU6DOEqOaw8w/ku4/gzEElRJ+p2I3WinpmZicLCQhQVFeG9997DuXPn0NjYiMrKSvz1r38FADzxxBM4cOAAzp07h+bmZhw+fNi5vmnTpoHjOOzfvx+dnZ0uR9p8hgUAg8HAADCDweDR/D/efoRN27Kf/eWLi16uzD/19fWxEydOsL6+PqlL8cjBgwfZ3LlzmVqtZosWLWL19fUMANu3b59zmg8++IABYDfffPN35r906RK76667mFqtZtOmTWPV1dUsISGB7dy50631D1/Xl19+yW677Tam0WhYbGwsW79+Pevp6WGMMabX69nq1atZcnIyU6lUbNq0aay8vJzxPM/MZjP76U9/ytLS0phKpWIpKSmspKTE7X+XRx55hMXFxTEArKKigjHGmMViYeXl5Sw9PZ0plUqWnJzM1qxZw7788kvGGGMlJSVs5syZTK1WsylTprAHH3yQdXV1OZf5wgsvsKSkJMZxHFu3bp1bdTA29ndKzO8zKDtRH+6pP3+JmqNtePz2WSi9c/b4MwQ46kTdFXWiPnHUiboIjnacU6F+pCpEUCfq/ivo23AA1864SPCjTtT9V0ht4Vy4akKfhUeYSj7OHCSQUSfq/iskAic+QoXYcBWuGi0409GLhTfopC6JSIQ6UZdWSOxScRyHzET7tSih1BlXABwPIAHCW9+lkAgcAJiTZG89D4V2HMeugclkkrgSEiwcZ3nL5RNrjgiJXSogtI5UyeVyREdHO6+Q1mq1Hp30RggACIKAzs5OaLVaKBQTi4yQCRxHZ1xfh0DgAHCe6u5ptwyEDCWTyTB16tQJ/8cVMoGTMbCFo+/uh8FkhU7ruyMSUuA4DsnJyUhISIDVapW6HBLgVCoVZLKJt8CETOBEaZRI0WlwydCPrzt6sCxE7jcul8snvN9NiLeETKMxQH3jECK1kAocOuOYEGmFVuAk0hYOIVIKqcDJHHInTjopjhDfC6nAmZUQARkHXDdZ0dljlrocQkKOR4GzY8cOpKenQ6PRIDc3F42NjaNO+9577yE7OxvR0dEIDw9HVlYW3n77bY8LngiNUo70OHtn2KF0iQMh/kJ04OzduxelpaWoqKhAc3MzFi9ejPz8/FFPMIuNjcUzzzyDhoYGfPnllyguLkZxcTEOHDgw4eI9kUntOIRIRnTgvPrqq1i/fj2Ki4sxb9487Ny5E1qtFm+99daI0996661Ys2YN5s6di5kzZ2LTpk1YtGgRjhw5MuHiPUFHqgiRjqjAsVgsaGpqQl5e3uACZDLk5eWhoaFh3PkZY6irq3Pez2c0ZrMZ3d3dLoO3OAKnpV2CDqQJCXGiAqerqws8zyMx0fUOlomJidDr9aPOZzAYEBERAZVKhVWrVuH111/HD37wg1Gnr6yshE6ncw5paWliyhyTY5fqdHsPBIGOVBHiSz45ShUZGYljx47h6NGj+N3vfofS0lKXG7APV1ZWBoPB4Bza2tq8Vkt6nBYquQwmC49vr43c4xohZHKIupYqPj4ecrkc7e3tLuPb29tdbsQ1nEwmw6xZswAAWVlZOHnyJCorK3HrrbeOOL1arYZarRZTmtsUchlmJkTg5OVutLT3YGqcuNvQEkI8J2oLR6VSYenSpairq3OOEwQBdXV1WL58udvLEQQBZrN058HMHuj9jxqOCfEt0VeLl5aWYt26dcjOzkZOTg6qqqpgNBpRXFwMACgqKkJqaioqKysB2NtjsrOzMXPmTJjNZnzwwQd4++238cYbb3j3nYgwOykKwCU6NE6Ij4kOnIKCAnR2dqK8vBx6vR5ZWVmora11NiS3tra69JthNBrxi1/8At9++y3CwsIwZ84cvPPOOygoKPDeuxDJ2RkXbeEQ4lMhcefN4b69ZsKNLx2GUs7hxAs/hFIeUld4EOJVYn6fIflLS40OQ7hKDivPcK7LKHU5hISMkAwcjuOoMy5CJBCSgQMM9o1D7TiE+E7oBg5t4RDic6EbOLSFQ4jPhWzgONpwLlw1wWSxSVwNIaEhZAMnPkKNuHAVGAPOdNCV44T4QsgGDkCdcRHiayEdONQZFyG+RYED6oyLEF8J6cBx7FJ9TbtUhPhEiAeO/SJOfXc/DCarxNUQEvxCOnAiNUqkRocBoNvGEOILIR04wOBWDgUOIZOPAieJ2nEI8ZWQD5w5SYP3GyeETK6QD5zMIddUBUBfZIQEtJAPnJlTIiDjgOsmKzp6pOvYnZBQEPKBo1HKkR4fDoAucSBksoV84ADUVQUhvkKBA7qIkxBfocDB4JEq2sIhZHJR4GDIuTjtvRAEOlJFyGShwAEwLVYLlUKGPiuPtmsmqcshJGhR4ABQyGWYNWXgEgdqxyFk0lDgDKDOuAiZfBQ4A5xHqqgzLkImDQXOgDl0ESchk44CZ4DjSNXZzl5YbILE1RASnChwBqToNIhQK2ATGM51GaUuh5CgRIEzgOM46oyLkElGgTPEbGrHIWRSUeAMMXikigKHkMlAgTMEnYtDyOSiwBnC0U1F61UTTBabxNUQEnwocIaIi1AjPkIFxoAzHXQCICHeRoEzjKMd5xQ1HBPidRQ4w9DtfwmZPBQ4w8ym28YQMmkocIahI1WETB4KnGEyEuxnG7d3m3HdZJG4GkKCCwXOMJEaJVKjwwDYuxwlhHiPR4GzY8cOpKenQ6PRIDc3F42NjaNOu2vXLtx0002IiYlBTEwM8vLyxpzeHzjbcfTdEldCSHARHTh79+5FaWkpKioq0NzcjMWLFyM/Px8dHR0jTl9fX48HHngAhw8fRkNDA9LS0nDnnXfi4sWLEy5+stAlDoRMDo6JvKF2bm4uli1bhu3btwMABEFAWloaHnvsMTz11FPjzs/zPGJiYrB9+3YUFRWNOI3ZbIbZPHjb3e7ubqSlpcFgMCAqKkpMuR7Z9/m3eHLvF8hJj8X/e2T5pK+PkEDW3d0NnU7n1u9T1BaOxWJBU1MT8vLyBhcgkyEvLw8NDQ1uLcNkMsFqtSI2NnbUaSorK6HT6ZxDWlqamDInbHai/UNrae+ByDwmhIxBVOB0dXWB53kkJia6jE9MTIRer3drGVu2bEFKSopLaA1XVlYGg8HgHNra2sSUOWEzpoRDLuNg6LOio8c8/gyEELcofLmyrVu3oqamBvX19dBoNKNOp1aroVarfViZK41SjvQ4Lc52GtGi70Fi1Oi1EkLcJ2oLJz4+HnK5HO3t7S7j29vbkZSUNOa827Ztw9atW/Hhhx9i0aJF4iv1scEjVdRwTIi3iAoclUqFpUuXoq6uzjlOEATU1dVh+fLRG1dffvll/Pa3v0VtbS2ys7M9r9aH6EgVId4nepeqtLQU69atQ3Z2NnJyclBVVQWj0Yji4mIAQFFREVJTU1FZWQkAeOmll1BeXo7q6mqkp6c723oiIiIQERHhxbfiXY6+cegSB0K8R3TgFBQUoLOzE+Xl5dDr9cjKykJtba2zIbm1tRUy2eCG0xtvvAGLxYL77rvPZTkVFRV4/vnnJ1b9JBp6TZUgMMhknMQVERL4RJ+HIwUxx/m9hRcY5pbXwmIT8NGvbsW0uHCfrJeQQDNp5+GEErmMc17ISQ3HhHgHBc4YHO04FDiEeAcFzhgyqTMuQryKAmcMdKSKEO+iwBmD40jVN51GWGyCxNUQEvgocMaQrNMgUq2ATWA412WUuhxCAh4Fzhg4jqN2HEK8iAJnHM5LHKj3P0ImjAJnHLMTHefiUP/GhEwUBc44Mum2MYR4DQXOOByHxluvmmCy2CSuhpDARoEzjrgINeIj7J2BnabbxhAyIRQ4bpidNNCOQ7tVhEwIBY4bMumaKkK8ggLHDXSJAyHeQYHjhkzq35gQr6DAcYNjl6qjx4xrRovE1RASuChw3BChVuCGmDAAtFtFyERQ4LiJ2nEImTgKHDc52nFOUTsOIR6jwHETbeEQMnEUOG4aei5OANzoghC/RIHjppkJ4ZDLOHT329DebZa6HEICEgWOm9QKOabH2+9NRZc4EOIZChwRnO041HBMiEcocERwtOPQkSpCPEOBI4LjqnE6UkWIZyhwRHBs4Zzu6AEv0JEqQsSiwBFhWlw41AoZ+q0C2q6apC6HkIBDgSOCXMYhI5E64yLEUxQ4ImXSkSpCPEaBI5Lj0Pgp2sIhRDQKHJGct42hLRxCRKPAEcmxhXOuywizjZe4GkICCwWOSMk6DSI1CtgEhnNdRqnLISSgUOCIxHGccyuH+jgmRBwKHA/Q7X8J8QwFjgcGt3DoTpyEiEGB4wFnZ1zt3RJXQkhgocDxwOyBXaq2q30wmm0SV0NI4KDA8UBsuApTItUAgNMdtFtFiLs8CpwdO3YgPT0dGo0Gubm5aGxsHHXaf/3rX1i7di3S09PBcRyqqqo8rdWvUGdchIgnOnD27t2L0tJSVFRUoLm5GYsXL0Z+fj46OjpGnN5kMmHGjBnYunUrkpKSJlywvxhsx6HAIcRdogPn1Vdfxfr161FcXIx58+Zh586d0Gq1eOutt0acftmyZXjllVfw05/+FGq12q11mM1mdHd3uwz+hjrjIkQ8UYFjsVjQ1NSEvLy8wQXIZMjLy0NDQ4PXiqqsrIROp3MOaWlpXlu2t1B3o4SIJypwurq6wPM8EhMTXcYnJiZCr9d7raiysjIYDAbn0NbW5rVle0vGQOB09phx1WiRuBpCAoNfHqVSq9WIiopyGfxNhFqBtNgwALRbRYi7RAVOfHw85HI52tvbXca3t7cHVYOwu+j2v4SIIypwVCoVli5dirq6Ouc4QRBQV1eH5cuXe704f5dJF3ESIopC7AylpaVYt24dsrOzkZOTg6qqKhiNRhQXFwMAioqKkJqaisrKSgD2huYTJ044H1+8eBHHjh1DREQEZs2a5cW34nuz6SJOQkQRHTgFBQXo7OxEeXk59Ho9srKyUFtb62xIbm1thUw2uOF06dIlLFmyxPl827Zt2LZtG2655RbU19dP/B1IaOiRKsYYOI6TuCJC/BvHGPP7Gyx1d3dDp9PBYDD4VQOy2cZjXvkB8AJDQ9ntSNaFSV0SIT4n5vfpl0epAoVaIceM+HAA1I5DiDsocCaIOuMixH0UOBNEnXER4j4KnAnKpHNxCHEbBc4EDT00zgt+3/5OiKQocCZoaqwWGqUMZpuA1qsmqcshxK9R4EyQXMYhI4HOOCbEHRQ4XkDtOIS4hwLHCxydcVHvf4SMjQLHCzKpf2NC3EKB4wWOI1XfdBlhtvESV0OI/6LA8YKkKA0iNQrwAsM3nUapyyHEb1HgeAHHcZhDlzgQMi4KHC+hzrgIGR8FjpdQZ1yEjI8Cx0voxniEjI8Cx0scgdN2tQ+9ZpvE1RDinyhwvCQ2XIUpkfY7i56mrRxCRkSB40V0pIqQsVHgeFEmdcZFyJgocLyIboxHyNgocLzI0b8xHakiZGQUOF6UkWC/aryzx4yrRovE1RDifyhwvChcrUBarP3eVHTGMSHfRYHjZbMT7TcCo3YcQr6LAsfLqDMuQkZHgeNl1BkXIaOjwPGy2UOOVAXAbdsJ8SkKHC+bER8BhYxDT78Nlw39UpdDiF+hwPEylUKG6fHhAKgdh5DhKHAmgbNvHGrHIcQFBc4kmE194xAyIgqcSZBJV40TMiIKnEng2MI53d4LXqAjVYQ4BFfgXG8FDBcBq7RHh9JitdAoZTDbBFy4QreNIcRBIXUBXvWXTcDZQ/bHqghAGwto4wFtnH0Ijx9lXBygiQZk3slfuYxDRkIkjl80oEXfgxlTIryyXEICXXAFDmOATAEINsDSax+ut7o3LycDwmKHBVPcYDiNNE6lHXVxs5PsgfOL6mZEhykRo1UhJlyFGO3QxwPPw1WIHfKaLkwJhTy4Nj4JAYItcIret4dOvwEwXRkcjF2uz13GXQXMBoAJgKnLPnS1uLc+pXYggIZvNcXh4TAVzMpruMZrYOpTw9inQfsVDc4xDYzQwAwlAG7URevClM4wsgeTyuV5bLhrcEVrlVBSSBE/x7EAOP++u7sbOp0OBoMBUVFR3l+BzQL0XXUvnEwDj/mJ9XcjQA6zLAx9nAYmhKGXqdEtqNHNq2CCBr1MAxPs4WQaCCmjyzg1ehFmf87UMEEDlSYMMVq1c0sqVqtC9EA4RWtVCFfLoVHIoVbKnH/VCjk0SjnUCpn978BrSjkHjhs9EAOGIABWE2DtA6xGwGIaeG4aeGy0T6OOHDZE2f8qVFK/A78n5vcZXFs4nlKogMgk++AOxuy7a8augRC6MhhELuF0BbAYAUvPwF+j/YsOQAYeYUIvwtCL2KHLlnv+NqxMDpNRDaPRNaSMsAeVhSlwDQpYIYcNClgHHluZAjbIYYH9r23gdciVYHIVZHIlIFdCJldBplCCU6ghUyghU6igUKogU6ihUCqhUKqhUKqgVKmhUKqhUquhVKqgUSkHA00hg3rgr1Iug1IGqJgZKtYPJd8PJeuHku+Dgu+D3NY/ekhYhofIKIFi6/P8AwUAuXowhDRRg0E0WkCNNk6hAYIhwCeIAscTHDf4RYqdLm5egR8MH4txoK3JONjm5BhvHvZ8+OuW3oFpjM4flZLjoYMJOpjG2lsThwGwDQwesjGZS6BZoYAcAsJgRhgskHG+2ci2ytSwysJgk4fBJteAl4eBV2gBjoOKN0FpM0Jp64XCZoSCHwgq3gyYzPb/UCZCphwnnAbGq8L9K5iipwFzfuS1xXkUODt27MArr7wCvV6PxYsX4/XXX0dOTs6o0//P//wPnnvuOZw/fx4ZGRl46aWX8KMfee9NBBSZ3P4/pcaLu4YuITY8uIZsXfFmgLfaB8E6+Ji3AIIVjLdAsFkh2CwDgxXMZgHj7a+Btzmn5ZyDDbKBv3JmhZx9N5kUnAAFBGhgHfNtmJkSJqhhghp9TI0+2Hcv+9jAOKgGHmuGPFajb2B6x2MTG/g7ZHw/VGAizgKRg0c4+hCJPkRwfYhAHyK5PkTC5HwewQ28Pmya4c/t/0ZW+25731VR/7RS65mWh0gpA2fv3r0oLS3Fzp07kZubi6qqKuTn56OlpQUJCQnfmf7jjz/GAw88gMrKStx9992orq7G6tWr0dzcjAULFnjlTYQ8L4UYB/se3QT26uy7m4JtlFAbCCzeYj+aqAwDlOH2o31KLZSQQSsIUPIM4bwACy/AyjNYbQKsjse8/fHQ12yCAIvLdMOnHXhsE2ATmH3eIcu08AIEgcEqMPCCABvPwA97bhAYrgoMNkEAzzteY7Dx9mXaBp4PP9GTg4Bw9LsEVCRnGjWwwjnvnUPGvLCZa+6egfu9UIuD6Ebj3NxcLFu2DNu3bwcACIKAtLQ0PPbYY3jqqae+M31BQQGMRiP279/vHPf9738fWVlZ2Llzp1vrnPRGY0K8hLHB8LENCaSRnlt5wTne8dzGM/CMgeeHLsd1OpvAwA9b7uDfgfH8KOOd848yfuj0PMP3Z8Th+Xvnj/meJ63R2GKxoKmpCWVlZc5xMpkMeXl5aGhoGHGehoYGlJaWuozLz8/H+++/P+p6zGYzzGaz83l3d7eYMgmRDMdxUMo5KCe0mRi8RJ240dXVBZ7nkZiY6DI+MTERer1+xHn0er2o6QGgsrISOp3OOaSlpYkpkxDip/zyTLGysjIYDAbn0NbWJnVJhBAvELVLFR8fD7lcjvb2dpfx7e3tSEoa+RyWpKQkUdMDgFqthlqtFlMaISQAiNrCUalUWLp0Kerq6pzjBEFAXV0dli9fPuI8y5cvd5keAA4ePDjq9ISQ4CX6sHhpaSnWrVuH7Oxs5OTkoKqqCkajEcXFxQCAoqIipKamorKyEgCwadMm3HLLLfj973+PVatWoaamBp999hnefPNN774TQojfEx04BQUF6OzsRHl5OfR6PbKyslBbW+tsGG5tbYVsSDcPK1asQHV1NZ599lk8/fTTyMjIwPvvv0/n4BASgujiTULIhATdxZuOTKTzcQjxP47fpTvbLgEROD099s7I6XwcQvxXT08PdDrdmNMExC6VIAi4dOkSIiMjx+yjpbu7G2lpaWhra6Ndr2HosxkZfS6jc/ezYYyhp6cHKSkpLu23IwmILRyZTIYbbrjB7emjoqLoyzMK+mxGRp/L6Nz5bMbbsnHwyzONCSHBiQKHEOIzQRU4arUaFRUVdFnECOizGRl9LqObjM8mIBqNCSHBIai2cAgh/o0ChxDiMxQ4hBCfocAhhPgMBQ4hxGeCKnB27NiB9PR0aDQa5ObmorGxUeqSJFVZWYlly5YhMjISCQkJWL16NVpa3LxveojZunUrOI7DE088IXUpkrt48SJ+9rOfIS4uDmFhYVi4cCE+++wzryw7aALHcb+siooKNDc3Y/HixcjPz0dHR4fUpUnmo48+wsaNG/HJJ5/g4MGDsFqtuPPOO2E0GqUuza8cPXoUf/jDH7Bo0SKpS5HctWvXsHLlSiiVSvztb3/DiRMn8Pvf/x4xMTHeWQELEjk5OWzjxo3O5zzPs5SUFFZZWSlhVf6lo6ODAWAfffSR1KX4jZ6eHpaRkcEOHjzIbrnlFrZp0yapS5LUli1b2I033jhpyw+KLRzH/bLy8vKc48a7X1YoMhgMAIDY2FiJK/EfGzduxKpVq1y+O6Hs//7v/5CdnY2f/OQnSEhIwJIlS7Br1y6vLT8oAseT+2WFGkEQ8MQTT2DlypXUveuAmpoaNDc3O/vfJsA333yDN954AxkZGThw4AAeffRRPP744/iv//ovryw/ILqnIBO3ceNGfPXVVzhy5IjUpfiFtrY2bNq0CQcPHoRGo5G6HL8hCAKys7Px4osvAgCWLFmCr776Cjt37sS6desmvPyg2MLx5H5ZoaSkpAT79+/H4cOHRfUrFMyamprQ0dGB733ve1AoFFAoFPjoo4/w2muvQaFQgOd5qUuURHJyMubNm+cybu7cuWhtbfXK8oMicDy5X1YoYIyhpKQE+/btw6FDhzB9+nSpS/Ibd9xxB44fP45jx445h+zsbBQWFuLYsWOQy0Pz5uArV678zqkTX3/9NaZNm+adFUxac7SP1dTUMLVazfbs2cNOnDjBNmzYwKKjo5ler5e6NMk8+uijTKfTsfr6enb58mXnYDKZpC7NL9FRKsYaGxuZQqFgv/vd79jp06fZu+++y7RaLXvnnXe8svygCRzGGHv99dfZ1KlTmUqlYjk5OeyTTz6RuiRJARhx2L17t9Sl+SUKHLu//OUvbMGCBUytVrM5c+awN99802vLpv5wCCE+ExRtOISQwECBQwjxGQocQojPUOAQQnyGAocQ4jMUOIQQn6HAIYT4DAUOIcRnKHAIIT5DgUMI8RkKHEKIz/x/qw1LwXi+ny0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:00:38.945124Z", + "iopub.status.busy": "2024-03-22T19:00:38.944791Z", + "iopub.status.idle": "2024-03-22T19:05:25.592731Z", + "shell.execute_reply": "2024-03-22T19:05:25.591881Z" + }, + "papermill": { + "duration": 286.669422, + "end_time": "2024-03-22T19:05:25.595196", + "exception": false, + "start_time": "2024-03-22T19:00:38.925774", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:25.632815Z", + "iopub.status.busy": "2024-03-22T19:05:25.631976Z", + "iopub.status.idle": "2024-03-22T19:05:25.653483Z", + "shell.execute_reply": "2024-03-22T19:05:25.652522Z" + }, + "papermill": { + "duration": 0.042389, + "end_time": "2024-03-22T19:05:25.655469", + "exception": false, + "start_time": "2024-03-22T19:05:25.613080", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.00.0004490.00474711.5821970.0906311.6706520.1432570.0000396.4151070.0472424645954.50.0688970.2363390.00085417.997304
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.0 0.000449 0.004747 11.582197 0.090631 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 1.670652 0.143257 0.000039 6.415107 0.047242 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 4645954.5 0.068897 0.236339 0.000854 17.997304 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:25.690857Z", + "iopub.status.busy": "2024-03-22T19:05:25.690259Z", + "iopub.status.idle": "2024-03-22T19:05:26.216854Z", + "shell.execute_reply": "2024-03-22T19:05:26.215856Z" + }, + "papermill": { + "duration": 0.546735, + "end_time": "2024-03-22T19:05:26.219100", + "exception": false, + "start_time": "2024-03-22T19:05:25.672365", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:26.257147Z", + "iopub.status.busy": "2024-03-22T19:05:26.256380Z", + "iopub.status.idle": "2024-03-22T19:10:48.212600Z", + "shell.execute_reply": "2024-03-22T19:10:48.211612Z" + }, + "papermill": { + "duration": 321.97834, + "end_time": "2024-03-22T19:10:48.215223", + "exception": false, + "start_time": "2024-03-22T19:05:26.236883", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:10:48.252524Z", + "iopub.status.busy": "2024-03-22T19:10:48.252211Z", + "iopub.status.idle": "2024-03-22T19:10:48.278190Z", + "shell.execute_reply": "2024-03-22T19:10:48.277470Z" + }, + "papermill": { + "duration": 0.046837, + "end_time": "2024-03-22T19:10:48.280187", + "exception": false, + "start_time": "2024-03-22T19:10:48.233350", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:10:48.314828Z", + "iopub.status.busy": "2024-03-22T19:10:48.314536Z", + "iopub.status.idle": "2024-03-22T19:10:48.319916Z", + "shell.execute_reply": "2024-03-22T19:10:48.319062Z" + }, + "papermill": { + "duration": 0.025269, + "end_time": "2024-03-22T19:10:48.322225", + "exception": false, + "start_time": "2024-03-22T19:10:48.296956", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.41919846044793607}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:10:48.360763Z", + "iopub.status.busy": "2024-03-22T19:10:48.359857Z", + "iopub.status.idle": "2024-03-22T19:10:48.794117Z", + "shell.execute_reply": "2024-03-22T19:10:48.793175Z" + }, + "papermill": { + "duration": 0.455983, + "end_time": "2024-03-22T19:10:48.796356", + "exception": false, + "start_time": "2024-03-22T19:10:48.340373", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAE8CAYAAACRnDftAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABM+UlEQVR4nO3de3yU1Z348c8z90ySmdzvCeEmcicgSLAavFREVqF2XautgIi1u9jqUtuVbn9YabvpukVkWxdtLURXEatV7EoVKQpWQS1XuYkCIQFyh2Qmc789vz+eZELIhVzmkknO+/WaVzLPnHnmzEC+c855vuccSZZlGUEQhBijinYFBEEQ+kIEL0EQYpIIXoIgxCQRvARBiEkieAmCEJNE8BIEISaJ4CUIQkwSwUsQhJgkgpcgCDFJBC8h5MrKypAkidOnT0e7KsIgJoKXMCBs3LiRp59+OtrVEGKICF7CgCCCl9BbIngJghCTRPASIuKdd96hpKSExMRETCYT06dPZ+PGjQDMnj2bLVu2UFFRgSRJSJJEYWFhj88dCAT42c9+Rk5ODkajkeuvv56jR49SWFjI4sWLg+UuXLjAo48+ysSJE0lISMBkMjF37lwOHjzY7nw7duxAkiT++Mc/8stf/pK8vDwMBgM33ngjJ06cCMXHIYSAJtoVEAa/srIylixZwvjx41mxYgVJSUns37+fd999l3vuuYd///d/x2KxcPbsWdasWQNAQkJCj8+/YsUKnnzySW677TbmzJnDwYMHmTNnDi6Xq125U6dOsXnzZu68806GDx9ObW0tzz33HCUlJRw9epScnJx25X/1q1+hUql49NFHsVgsPPnkk3z729/m008/7f+HIvSfLAghtmHDBhmQy8vL5aamJjkxMVG++uqrZafT2a5cIBAI/j5v3jx52LBhvX6tmpoaWaPRyAsWLGh3/Gc/+5kMyIsWLQoec7lcst/vb1euvLxc1uv18qpVq4LHPvjgAxmQx44dK7vd7uDxtWvXyoB86NChXtdTCD3RbRTCatu2bTQ3N/PYY49hMBjaPSZJUr/Pv337dnw+H//yL//S7vj3v//9DmX1ej0qlfJf3u/3c/78eRISEhgzZgz79u3rUP6+++5Dp9MF71977bWA0oITok8ELyGsTp48CcCECRPCcv6KigoARo0a1e54SkoKycnJ7Y4FAgHWrFnD6NGj0ev1pKWlkZ6ezueff47FYulw7oKCgnb3W8/X2NgYyrcg9JEIXsKQ8R//8R8sX76c6667jpdeeomtW7eybds2xo8fTyAQ6FBerVZ3eh5ZrJw+IIgBeyGsRo4cCcDhw4c7tI4u1tcu5LBhwwA4ceIEw4cPDx4/f/58hxbS66+/zvXXX88f/vCHdsebmppIS0vr0+sL0SNaXkJY3XzzzSQmJlJaWtrh6t/FLZj4+PhOu26Xc+ONN6LRaFi3bl2747/97W87lFWr1R1aTa+99hrnzp3r9esK0SdaXkJYmUwm1qxZw9KlS5k+fTr33HMPycnJHDx4EIfDwQsvvADAtGnTePXVV1m+fDnTp08nISGB22677bLnz8zM5OGHH2b16tXcfvvt3HLLLRw8eJB33nmHtLS0di26f/iHf2DVqlXcd999zJo1i0OHDvHyyy8zYsSIsL1/IYyifLVTGIQuTpVo9ec//1meNWuWHBcXJ5tMJnnGjBnyK6+8EnzcZrPJ99xzj5yUlCQDvUqb8Pl88v/7f/9PzsrKkuPi4uQbbrhBPnbsmJyamip/73vfC5ZzuVzyD3/4Qzk7O1uOi4uTr7nmGnn37t1ySUmJXFJSEizXmirx2muvtXud8vJyGZA3bNjQ249ECANJlsXoozD4NDU1kZyczC9+8Qv+/d//PdrVEcJAjHkJMc/pdHY41jrJe/bs2ZGtjBAxYsxLGLDq6+vx+/1dPq7T6UhJSeHVV1+lrKyMW2+9lYSEBD766CNeeeUVbr75Zq655poI1liIJBG8hAFr+vTpwSTUzpSUlLBjxw4mTZqERqPhySefxGq1Bgfxf/GLX0SwtkKkiTEvYcD6+OOPO+0StkpOTmbatGkRrJEwkIjgJQhCTBID9oIgxKQhN+YVCASoqqoiMTExJKsaCIIQWrIs09zcTE5OTnAVkM4MueBVVVVFfn5+tKshCMJlnDlzhry8vC4fH3LBKzExEVA+GJPJFOXaCIJwKavVSn5+fvBvtStDLni1dhVNJpMIXoIwgF1uWEcM2AuCEJNE8BIEISaJ4CUIQkyK6pjXunXrWLduHadPnwZg/PjxrFy5krlz53ZavqysjPvuu6/dMb1e32GRu/6SZRmfz9ftvDqhc2q1Go1GI9JQhLCLavDKy8vjV7/6FaNHj0aWZV544QXmz5/P/v37GT9+fKfPMZlMHD9+PHg/1H8kHo+H6upqHA5HSM87lBiNRrKzs9vtvCMIoRbV4HXpSpm//OUvWbduHZ988kmXwUuSJLKyssJSn0AgQHl5OWq1mpycHHQ6nWhB9IIsy3g8Hurr6ykvL2f06NHdJhkKQn8MmFQJv9/Pa6+9ht1up7i4uMtyNpuNYcOGEQgEmDp1Kv/xH//RZaADcLvduN3u4H2r1dplWY/HQyAQID8/H6PR2Lc3MsTFxcWh1WqpqKjA4/F02KtxUAj4oXwnWKshfwakjY52jYakqH8tHjp0iISEBPR6Pd/73vd48803GTduXKdlx4wZw/r163nrrbd46aWXCAQCzJo1i7Nnz3Z5/tLSUsxmc/DWk+x60Vron0H/+VV+ApWfQlMlHHkTHBeiXaMhKeqrSng8HiorK7FYLLz++us8//zz7Ny5s8sAdjGv18vYsWO5++67+fnPf95pmc5aXvn5+Vgslg5Jqi6Xi/LycoYPHz44WwwRMqg/R58Hdv9G+dkqezJceWv06jTIWK1WzGZzp3+jF4v6V6ROp2PUqFFMmzaN0tJSJk+ezNq1a3v0XK1WS1FRESdOnOiyjF6vD2bTi6x6od8ajiuBKy4ZptyjHKs/pnQlhYiKevC6VCAQaNdS6o7f7+fQoUNkZ2eHuVbCxQoLC4NrxA8551u+KDPHQVIB6OKVYNbU9YqvQnhEdcB+xYoVzJ07l4KCApqbm9m4cSM7duxg69atACxcuJDc3FxKS0sBWLVqFTNnzmTUqFE0NTXxX//1X1RUVLB06dJovg1hqJBlaGwJUsnDQZIgZQTUHFLGv1LE/o+RFNXgVVdXx8KFC6mursZsNjNp0iS2bt3K17/+dQAqKyvbDf42NjbywAMPUFNTE1wCeNeuXT0aHxPa83g8Ig+rt2y14HWCRgemHOWYOVcJXtaq6NZtCIpq8PrDH/7Q7eM7duxod3/NmjWsWbMmjDXqmscX6PIxlQQatapHZSUJtD0oq9P0rkc/e/ZsJkyYAMD//u//otVq+ed//mdWrVqFJEkUFhZy//3389VXX7F582buuOMOysrK+Oijj1ixYgV79uwhLS2Nb3zjG5SWlhIfHw8oXzD3338/f/3rX8nKyhram1q0BihTLqjULb/ntT0WCMBgv9I6gAyYPK+B7pkPur4oMDwtngVFucH7v/vwJF5/5xdx85LjuPOqtnSN9R+X4/R0HOz9169f0es6vvDCC9x///189tln7Nmzh+9+97sUFBTwwAMPAPDrX/+alStX8vjjjwNw8uRJbrnlFn7xi1+wfv166uvreeihh3jooYfYsGEDAIsXL6aqqooPPvgArVbLD37wA+rq6npdt0GhuUb5mXhRknR8mtIS83nA0QAJGdGp2xAkgtcgkp+fz5o1a5AkiTFjxnDo0CHWrFkTDF433HADP/zhD4Plly5dyre//W0eeeQRAEaPHs1///d/U1JSwrp166isrOSdd97hs88+Y/r06YDSWh47dmzE39uAYGsJXgkXBS9Jgvh0sJwDe70IXhEkglcPLbt+VJePqS6ZQfTd60Z2WfbS2UZLrhnen2q1M3PmzHbTmYqLi1m9enVwgvlVV13VrvzBgwf5/PPPefnll4PHZFkOTpP68ssv0Wg07bYXu/LKK0lKSgpZnWOG3wf2BuX3xEump10cvISIEcGrh3ozBhWusv3VOo7Vymaz8eCDD/KDH/ygQ9mCggK+/PLLSFVt4HM0KLlcWgMeTSJaWW77oohPV362BjchIkTwGkQ+/fTTdvc/+eQTRo8ejVqt7rT81KlTOXr0KKNGdd6qvPLKK/H5fOzduzfYbTx+/DhNTU0hrXdMsDfgl2UONmjYueMkM0ekUjwyVXksPq2ljGh5RZK4NDKIVFZWsnz5co4fP84rr7zCb37zGx5++OEuy//bv/0bu3bt4qGHHuLAgQN89dVXvPXWWzz00EOAMpf0lltu4cEHH+TTTz9l7969LF26lLi4uEi9pYHDcZ6qJidn3HEMSzUyY3hK22PGliDmsohM+wgSwWsQWbhwIU6nkxkzZrBs2TIefvhhvvvd73ZZftKkSezcuZMvv/ySa6+9lqKiIlauXElOTk6wzIYNG8jJyaGkpIQ77riD7373u2RkDL1BaZ+tgWqLC6cmifE5ZtQXD3TqEkCtUZJYXZboVXKIEd3GQUSr1fL000+zbt26Do+1rlZ7qenTp/Pee+91ec6srCzefvvtdsfuvffeftUzFjXUVeEPyGgS07giMwFQLm64fQEMWrUy19FWD85GMKZc5mxCKIiWlyBcTiCA5XwtAPm5+UiSRHmDnef/Vs7WIy3pE4Yk5aezKSpVHIpEy0sQLkN2NdHsdBOQ1AzLUdIk4vVqbG4f7gt+fP4AmrhkpbCzMYo1HVpE8BokLp1KJYTOhYYavH4ZryGJTLOywm56gh6jTo3D46eu2U2OCF4RJ7qNgnAZKZKdSXlmriwsCA7US5JElllZaLHG6oK4JKWwqyk6lRyCRPAShMuQ3BbidRqyMjPbHc8yKcGr1uJqG/NyWZSrjkLYieAlCJfTOghvSG53uF3LS5+oHPR7wRfafUSFzongJQjd8PgCHCs/S63VhWxov4R4RqISvJocXtyyCrQtybvu5khXc0gSA/aC0I0Gm5vz5+twSD4y49q3vOJ0akZnJhCv1+APyErry+sEl1WsLhEBIngJQjfqGy2oAx7ijVrQd9y85R8mtc1GwGAGWx24u94bVAgd0W0UhG5YG5XJ1gZjorLoYHdax71E8IoIEbwEoRt2y3kA9ImpXZZxef2ct7nbWmZizCsiohq81q1bx6RJk4L7KRYXF/POO+90+5zXXnuNK6+8EoPBwMSJE/nLX/4SodoKQ5HbpuyGHWfqfL5itcXJuh0neXP/ubaWl0u0vCIhqsErLy+PX/3qV+zdu5c9e/Zwww03MH/+fI4cOdJp+V27dnH33Xdz//33s3//fhYsWMCCBQs4fPhw+Copy8r65NG49TBf6MUXXyQ1NbXDfpcLFiwYkpOoQ8Xl9RNwKKtEGM2dt7xMBi0ANrcPn1aZsC1aXpER1QH72267rd39X/7yl6xbt45PPvmE8ePHdyi/du1abrnlFn70ox8B8POf/5xt27bx29/+lmeffTY8lfR74W+rw3Puy7n2h5cfZwHuvPNOfvCDH/DnP/+ZO++8E1B2/dmyZUu3K0YI3WtyeNH7m9GpVejikzstY9Sp0WlUeHwBmjGSDErwkuWOa34LITVgxrz8fj+bNm3CbrdTXFzcaZndu3dz0003tTs2Z84cdu/e3eV53W43Vqu13W2wiYuL45577gnu+APw0ksvUVBQwOzZs6NXsRiXZTbw9eF6xmYnKlcSOyFJEqY4pfXV5NcrBwM+kagaAVFPlTh06BDFxcW4XC4SEhJ48803u9xEtqamhsxLpmhkZmZSU1PT5flLS0t54okn+l5BtVZpAUWDWtvjog888ADTp0/n3Llz5ObmUlZWxuLFi9ttyCH0ntZnQ6vTgL7z4AWQFKelodlNkysAWgN4XeC2tSWtCmER9eA1ZswYDhw4gMVi4fXXX2fRokXs3LkzZLtgr1ixguXLlwfvW61W8vPzu3nGJSSpR123aCsqKmLy5Mm8+OKL3HzzzRw5coQtW7ZEu1qxzedWxh6hbTC+E0nGlpaX06usqup1gccGpEegkkNX1IOXTqcLbgAxbdo0/v73v7N27Vqee+65DmWzsrKora1td6y2tpasrKwOZVvp9Xr0en1oKz1ALV26lKeffppz585x00039S5ICx18dPgUuQ12stKSievmC8zc0m20tgYvewN47JGq5pA1YMa8WgUCgQ5XzVoVFxezffv2dse2bdvW5RjZUHPPPfdw9uxZfv/737NkyZJoVyemybJMeVUNNVYXgdariF3IMhmYnG9mVEYC6Fq2l/PYIlDLoS2qLa8VK1Ywd+5cCgoKaG5uZuPGjezYsYOtW7cCyoYSubm5lJaWAvDwww9TUlLC6tWrmTdvHps2bWLPnj387ne/i+bbGDDMZjPf/OY32bJlCwsWLIh2dWKa0+tHctuQAENC1+NdABkmAze0LI+DQwSvSIlq8Kqrq2PhwoVUV1djNpuZNGkSW7du5etf/zqgbOWlUrU1DmfNmsXGjRv56U9/yk9+8hNGjx7N5s2bmTBhQrTewoBz7tw5vv3tbw+ZrnK4WJ0+dH4bOo0KdVz3waud1rEx0W0Mu6gGrz/84Q/dPt7Z0sZ33nlnMJdJaNPY2MiOHTvYsWMH//M//xPt6sQ8i9OL3m9Hr1F1O1jfyuX1Y3V5Mavi0INytVEIq6gP2AuhUVRURGNjI//5n//JmDFjol2dmGdxetH57Og16h4Frzf3n6PG4uIbw9UUgmh5RYAIXoNEV/syCn1jdXrR+e3o43rW8ko0aKixgDXQ0l0XY15hN+CuNgrCQGD3+JTgpVF1uo7XpRJb5jhaWrPsfW5lapkQNqLl1QlZbKDQL4Ph87t9Qhq+BiMqSepxywvA6lGBSqNMEfLYlJ20hbAQLa+LaLXKt6fD4YhyTWJb6+fX+nnGIsljR6tWodbpQXP5K7emluDV7PaBviUvTIx7hZVoeV1ErVaTlJREXV0dAEajUcwN7AVZlnE4HNTV1ZGUlIRarY52lfqudTXUHnQZoa3b2OzyQUK8suOQuOIYViJ4XaJ1qlFrABN6LykpqdspWwNdrdXF0c9PckWTk9yUy3cZoa3baPf4CGiMSpfGK1rw4SSC1yUkSSI7O5uMjAy8XjHg2ltarTa2W1zABbuH+voGMj1ecnsw3gUQp1UzpSCJRL2GgDNOBK8IEMGrC2q1Oub/CIW+aXYpVxp1PUxQBeVL7/oxLdudnWod8xLBK5zEgL0gXMLmbsnxUvc8eLXTOjnbKwbsw0m0vAThEs0uH/HBllfPBuyhbYpQXEBLIoiWV5iJlpcgXMLm9qHz9a7bCPDJqfO8/EklXzT4lANizCusRPAShEvYHC60AWevg1eCviXXK9CycKEIXmElgpcgXMQfkFF5lK3LdDo9aAw9fm58S/Cy+FuDlxMCgZDXUVCIMS9BuIhaJbF0Rjp+TTLquMRebV/W2vKy+lpmFsgy+JxtA/hCSImWlyBcQvLY0ahUSL0YrIe2lpfN07KLECitLyEsRPAShEu17nit737t+kvF65W8QI8vgE/dsu2ZmN8YNiJ4CcJFjlRZ2HXsNPXNbmUnoF7QqVXKID/gllpbXmLQPlzEmJcgXKTW6sJx4QIunb9XOV6gZNlPG5aMWiWhaUoAJyLXK4yi2vIqLS1l+vTpJCYmkpGRwYIFCzh+/Hi3zykrK0OSpHY3g6HnV4QEoTvtpwb1ruUFMHNEKtMLU9DHtTxXZNmHTVSD186dO1m2bBmffPIJ27Ztw+v1cvPNN2O3d/8PbjKZqK6uDt4qKioiVGNhsGsXvHrZbWxHZ1R+igH7sIlqt/Hdd99td7+srIyMjAz27t3Ldddd1+XzJEmK6SVXhIHL5vKi9TvQaRL6NK+xdYqQ3q/DDGLAPowG1IC9xWIBICUlpdtyNpuNYcOGkZ+fz/z58zly5EiXZd1uN1artd1NEDrj9QfwuhyoZL+ydn0fWl5Hqiy8/Ekln9d6Wk4qxrzCZcAEr0AgwCOPPMI111zT7SayY8aMYf369bz11lu89NJLBAIBZs2axdmzZzstX1paitlsDt7y8/PD9RaEGGdr6TKqVRJqfTyoe98xac31srZm2YsB+7AZMMFr2bJlHD58mE2bNnVbrri4mIULFzJlyhRKSkp44403SE9P57nnnuu0/IoVK7BYLMHbmTNnwlF9YRBw+fwYcaJTq5D6shQOEK9rmd/ob8myFy2vsBkQqRIPPfQQb7/9Nh9++CF5eXm9eq5Wq6WoqIgTJ050+rher0evv/wGCoKQbY7jO0XJBI6Z+7aOF5fMb1TTNr9RNWDaCYNGVD9RWZZ56KGHePPNN3n//fcZPnx4r8/h9/s5dOgQ2dnZYaihMNRIHqXb2NcrjUadkmXvCGjxt+4AJ1pfYRHVlteyZcvYuHEjb731FomJidTU1ABgNpuJi1OmVyxcuJDc3FxKS0sBWLVqFTNnzmTUqFE0NTXxX//1X1RUVLB06dKovQ9hEGnd8aePLS+9RoVaJeEPqPCo9MTJbiV49SFnTOheVIPXunXrAJg9e3a74xs2bGDx4sUAVFZWorqoyd3Y2MgDDzxATU0NycnJTJs2jV27djFu3LhIVVsYpHYcryPhywqu0Hgx9THYSJKEUaem2eXDKxmU4CXSJcIiqsGrJzsr79ixo939NWvWsGbNmjDVSBjKzjY6yWpqxJ8ig65vLS+AooJkArKMti4B7BaRqBomA2LAXhAGApu7Nbte1+duI8C0YcktJ0wAO2LMK0zEJRBBQElQdbk9aP0uJUE1FGNUwV2ERPAKB9HyEgSUBFWt34laBWq1BrTGPp/L5fVjdXrR+zTKFCHRbQwLEbwEgYu6jGoVkj6hV8s/X+pYtZUdx+uZrvXxNRAD9mEiuo2CQN92ye6KsSXL3hZozbIXLa9wEMFLEABfIEA8zv4vhUNboqpNTBEKK9FtFARgUl4SE91JyBXx/W55tU4RapbF/o3hJFpegtBCcjejkqQQdBuVlpfdryMgy0q3sQc5jULviOAlCK08LVOD+tltbJ0i5FMb8PgDSuAS414hJ4KXIACv7TnD56fOKsGmnzlerVOEZEmNl4t2zxZCSox5CUOe1x/gbKOT7KZGVOaEXu8a1JmigmRkWUZzNgG8lpaNOFL7X1khSAQvYcizuXyoAx50kq9fy+FcLDhF6EICWMT8xnAQ3UZhyGt2+ZRNN9QqJI0eNLrQnbw1U18kqoacaHkJQ16z24veb2uZ09j/LiO0TREy+LWYQLS8wkC0vIQhz9ba8gpBgmqrY9VWXv60kqMNPuWAyPUKuT4Fr1OnToW6HoIQNcq8Rkefd8nuTGuiql0kqoZNn4LXqFGjuP7663nppZdwuVyhrpMgRJxRdrZ0G/uXoNoqTnvJFCGxBVrI9Sl47du3j0mTJrF8+XKysrJ48MEH+eyzz0JdN0GIiBvHZvIPYxJIT9D3awXViwWnCAVadq4SLa+Q61PwmjJlCmvXrqWqqor169dTXV3N1772NSZMmMBTTz1FfX19qOspCGEleWxIkhSybmPbLkIXTRESQqpfA/YajYY77riD1157jf/8z//kxIkTPProo+Tn57Nw4UKqq6u7fX5paSnTp08nMTGRjIwMFixYwPHjxy/7uq+99hpXXnklBoOBiRMn8pe//KU/b0MQwG1VfoboamPrFCGvOk7J2vc6xPzGEOtX8NqzZw//8i//QnZ2Nk899RSPPvooJ0+eZNu2bVRVVTF//vxun79z506WLVvGJ598wrZt2/B6vdx8883Y7V3nxOzatYu7776b+++/n/3797NgwQIWLFjA4cOH+/NWhCGq0e7h5d2n+OqMsu1eqMa8WqcI+VQGfH4ZAn7we0JybkEhyT3ZwucSTz31FBs2bOD48ePceuutLF26lFtvvbXdFmVnz56lsLAQn8/X4/PW19eTkZHBzp07ue666zotc9ddd2G323n77beDx2bOnMmUKVN49tlnL/saVqsVs9mMxWLBZArNt6wQuyrO23n7s+N8reFVJhekwnU/6tcqqhfbV9lIICAz8dTv0UsBuPpBMKaE5NyDWU//RvuUpLpu3TqWLFnC4sWLu9ypOiMjgz/84Q+9Oq/FYgEgJaXrf+Ddu3ezfPnydsfmzJnD5s2bOy3vdrtxu93B+1artVd1Ega3ZpcPvd/WtoJqiAIXwNSClilC1YngElOEQq1PwWvbtm0UFBS0a2mBsg/jmTNnKCgoQKfTsWjRoh6fMxAI8Mgjj3DNNdcwYcKELsvV1NSQmZnZ7lhmZmZwt+1LlZaW8sQTT/S4HsLQ0uzyofPZ0KlDlybRgTauJXiJK46h1Kcxr5EjR9LQ0NDh+IULFxg+fHifKrJs2TIOHz7Mpk2b+vT8rqxYsQKLxRK8nTlzJqTnF2Kbze1D77eHNMerlcvrp9bqojkgElXDoU8tr66GyWw2GwaDodfne+ihh3j77bf58MMPycvL67ZsVlYWtbW17Y7V1taSlZXVaXm9Xo9er+91nYShweb2ovPb0BlCN6+xVesuQtf6fFxlQHQbQ6xXwat1rEmSJFauXInR2La3nd/v59NPP2XKlCk9Pp8sy3z/+9/nzTffZMeOHT1qtRUXF7N9+3YeeeSR4LFt27ZRXFzc49cVhFbNLh+pvtZdg0IbvNqmCLV8eYqVJUKqV8Fr//79gBJ0Dh06hE7XtnSITqdj8uTJPProoz0+37Jly9i4cSNvvfUWiYmJwXErs9lMXFwcAAsXLiQ3N5fS0lIAHn74YUpKSli9ejXz5s1j06ZN7Nmzh9/97ne9eSuCAIBWrcIoO9Br1GAIbfBqnSJkD4jVVMOhV8Hrgw8+AOC+++5j7dq1/U41WLduHQCzZ89ud3zDhg0sXrwYgMrKynYXBmbNmsXGjRv56U9/yk9+8hNGjx7N5s2bux3kF4Su3D2jANlrAI8v5GNebVOExJhXOPRpzGvDhg0hefGepJjt2LGjw7E777yTO++8MyR1EIY4vw/JYwf6v2vQpVqnCDllHX5ZRi2CV0j1OHjdcccdlJWVYTKZuOOOO7ot+8Ybb/S7YoIQEZ5m5adK07bqaYjoNSo0Kgmv2oDXH0AtVpYIqR4HL7PZrExcbfldEGLdsWorXxw7xvQmB3k5uSFNUAXlwlacTo3HHYfXH8AgWl4h1ePgdXFXMVTdRkGIpkaHB5u1UZk4HeLB+lZThyWDNw7dKRX4vcpNrQ3Law01fRrzcjqdyLIcTJWoqKjgzTffZNy4cdx8880hraAghEuzy4fOb1euNIYpu35qQbKymsRZnTI52+sAtei5hEKfMuznz5/Piy++CEBTUxMzZsxg9erVzJ8/P3gFURAGOqvTi95nC0t2fTuSpEwRApEuEUJ9Xkn12muvBeD1118nKyuLiooKXnzxRf77v/87pBUUhHBpbXmFI0G1VesUIVtruoRIVA2ZPgUvh8NBYqLyTfXee+9xxx13oFKpmDlzJhUVFSGtoCCEQyAgKytK+EK75dmlvqhpZuOnlXzV6FcOiJZXyPR5A47Nmzdz5swZtm7dGhznqqurE2tkCTHB5vERkGUMgZblcMI0YB8fzPUSa9mHWp+C18qVK3n00UcpLCzk6quvDs4rfO+99ygqKgppBQUhHDy+AOlGSFD7kJDAEJ5B9LiW4GUTW6CFXJ+uNv7jP/4jX/va16iurmby5MnB4zfeeCPf+MY3QlY5QQiXtAQ935mcBJ4k0BpAE56VR+J1LZOzg2NeIniFSp+CFyhL01y6DM2MGTP6XSFBiBiXsnJvuFpdAEZ9W7fRHxBThEKpT8HLbrfzq1/9iu3bt1NXV0cgEGj3uNhRW4gJEQheOrUKrfqiKUIieIVMn4LX0qVL2blzJ/feey/Z2dnBaUOCECv+fLCK+MovuUrtxWxICtvrKFOENHhVrVOExNXGUOlT8HrnnXfYsmUL11xzTajrIwgR0dDsRt18ASkZCGPwAphakIQq3YeuUiUG7EOoT8ErOTm52x1+BGEga83xKvC35HiFsdsIUFSQDB49VKnB61KmCanUYX3NoaBPqRI///nPWblyJQ6H+BYRYk9rjlecv1nZNSjMwQsAjaFt1QrRdQyJPrW8Vq9ezcmTJ8nMzKSwsBCttv0s+X379oWkcoIQDlanF3XAjVHlVcZrwxy8XF4/FqeXeL+GBJVX6TrqE8L6mkNBn4LXggULQlwNQYgcq9OH3tesrCahjQON7vJP6ofjNc28/0UdN9pkJiUhxr1CpE/B6/HHHw91PQQhYqyui1aTiECXMb4l18sl6wCfSFQNkT6NeYGyFM7zzz/PihUruHDhAqB0F8+dO9fjc3z44Yfcdttt5OTkIEkSmzdv7rb8jh07kCSpw62r3bIFoTNatYp0rUuZuhOB4BWnu2QLNDHmFRJ9anl9/vnn3HTTTZjNZk6fPs0DDzxASkoKb7zxBpWVlcG1vi7HbrczefJklixZctl18S92/PjxdhPAMzIyev0ehKFr2rBk8BjhbFxkWl66S7dAEy2vUOhT8Fq+fDmLFy/mySefDC6NA3Drrbdyzz339Pg8c+fOZe7cub1+/YyMDJKSknr9PEEIcjUpP8Oc4wVgbGl5OTGIKUIh1Kdu49///ncefPDBDsdzc3Mj0oWbMmUK2dnZfP3rX+fjjz/utqzb7cZqtba7CUOXLMvKlnvB4BX+lpdOo0wR8rVMERItr9DoU/DS6/WdBoEvv/yS9PT0fleqK9nZ2Tz77LP86U9/4k9/+hP5+fnMnj2729SM0tJSzGZz8Jafnx+2+gkDX12zm3U7TnDkZMuimcbIJFsbdRq8KoOy2YcYsA+JPgWv22+/nVWrVuH1egFl/lZlZSX/9m//xje/+c2QVvBiY8aM4cEHH2TatGnMmjWL9evXM2vWLNasWdPlc1asWIHFYgnezpw5E7b6CQNfk8NLwGVTdvGRIpSgirKLUNGoXOUKp2h5hUSfgtfq1aux2Wykp6fjdDopKSlh1KhRJCYm8stf/jLUdezWjBkzOHHiRJeP6/V6TCZTu9uQFQhA1X4o/xu4hmb3ucnhIc5nQa9VK6unRmiazpT8JCYOy1Jyy0TwCok+DdibzWa2bdvGxx9/zMGDB7HZbEydOpWbbrop1PW7rAMHDpCdnR3x1405sgxH34T6L5X7Vftg6iKIS4pqtSKtyenF4LNi0KggLjmyL65r2ZHb61L+PcRqLP3S6+AVCAQoKyvjjTfe4PTp00iSxPDhw8nKykKW5V4tj2Oz2dq1msrLyzlw4AApKSkUFBSwYsUKzp07F0y9ePrppxk+fDjjx4/H5XLx/PPP8/777/Pee+/19m0MPXXHlMClUivb2rub4attMOnOaNcsoiwOLwafBYNBDXGRW1zA5fXT5FCR6PEpq6t6nW3BTOiTXnUbZVnm9ttvZ+nSpZw7d46JEycyfvx4KioqWLx4ca+XgN6zZw9FRUXBde+XL19OUVERK1euBKC6uprKyspgeY/Hww9/+EMmTpxISUkJBw8e5K9//Ss33nhjr153yJFlqNyl/F5QDJPvVr71z58AW1106xZhTU6P0vLSqiPa8vqytplX9pyjwtKycKdIVO23XrW8ysrK+PDDD9m+fTvXX399u8fef/99FixYwIsvvsjChQt7dL7Zs2crl627eb2L/fjHP+bHP/5xb6osAFirwFYPag3kXaXM50sfA3VfwLl9MOaWaNcwItw+P3a3H4PXikEb2W6jsXUXIS5OVE2N2OsPRr1qeb3yyiv85Cc/6RC4AG644QYee+wxXn755ZBVTgiR+i+Un6mj23Zuzp7c9tgly3gPVl6/zIg0I2kaBxpVpIOX0k5wiF2EQqZXwevzzz/nllu6/paeO3cuBw8e7HelhBCSZag/rvyeMRaHx0d5gx1XQr4SyLxOaBoaGwUn6DXMH2dmcrZR6TZH8GJF/EXBS0YWwSsEetVtvHDhApmZmV0+npmZSWNjY78rJYSQ47yy0YRKQ6Wcxf99fBqPL4BBq+ZuUwFJluPQWA4pw6Nd08hwtvz/NJgjuppp6y5CLgz4A1Y0IlG133rV8vL7/Wg0Xcc7tVqNz+frd6WEEGppVXkSsvnLsQY8vgAqScLl9fNBXQJ+WYbG09GtY4S4fX5kh7ICSqTTJLRqFXqtCp/agMcXEAP2IdCrlpcsyyxevBi9vvMNOt1ud0gqJYRQk3K19it3Ck6Pn9QEHd+cmscrn1Vyzp5BrdVFjlQLHjvo4qNc2fDa9NkZUmqOcL3RR0Kkc7yARH3bFCGj1x7x1x9sehW8Fi1adNkyPb3SKESALAeDV3LOSPIa4picn0S8XsPMEamcqNOjb8oCLNB0BjKujG59w8jnD9Dk8JLuuoDOpAJjWsTrMCU/GbUhB0P9cdHyCoFeBa8NGzaEqx5CODguKJOAVRpy8kdyZ6EmmJoyIdfMhFwzHB+hTBmynhvUwavJ6SUgyyQGmtCq1RAf+TSFiXlmiMuCJjFFKBT6vJKqEAOaq5SfiVlKjhd0nAFhzlV+WqsiWLHIu2D3oAr4SJIcSEhRaXkByuwGECtLhIAIXoNZcw2yLHPKncgFu6fThOAmTTo1Vhdyc7Wyn+Agdd7mweBrwqhVgdYQlfE9t89PrVNFs7tlB6FuErSFyxPBazBrrsbu8fNxrZZXPqvs8LfiD8i8/HkzX13wY3e6wVYbnXpGwHm7mzhvk7JuvTEtKpOiz1xw8srB85xucChfFD5XxOswmIjgNVgFAmCrxeL0YtOlk59iRKVq/werVkkMS4unWZ9Bo8MD1uooVTb86puV4GXUqcEYnWk5CXoNsqTBKbfsc+q2RaUeg4UIXoOVowH8Phrd4NKYKUjpfAWDwtR47No0rE4v2AbnLkyyLDMqI4F8g0PJdI+PznhXgqFlFyHilC68RwSv/hDBa7BqGe+qDZhBkshLjuu0WF5yHA5dKja3D791cAYvSZK4dnQ6MzNltGpV1FpeRq0aSQK3Kg5vQASv/hLBa7Cy1eLw+LFo0tBrVaTGd74rtDlOCwlZBGSwXRjEg/YBv5I6AlFrealUEvE6DR61Ucmy94hE1f4QwWuwstdjc/twaFPIMhm6XCRSkiQy0tPxqfRYHW6wN0S4ouF3we7BZW0AOQBqLeijtxR4vF6DV21UNuIQY179IoLXYGVvoNntw6FNJstk6LZobooRhy6VZpdvUI57vXO4mj/u2KtclIhPj+ryywmGi1tezVGrx2DQpzXshQHO4wCPncJUI3MmjicxvvucphHpCWRcOYa0JuegW1nVH5A5b/OQ6z1PnFENCV2vihIJ47IT8ZBLYs1R0W3sJxG8BiOH0vXTGJMZlnH5CcgJeg0JucPA+jk0D66W13mbG39AxhRoRK9VQUJGVOszKiMRdDlwQSO6jf0kuo2Dkb1e+dmbgenWFomtdlBlfldZlETQTMmqTAuKcvACQJeg/BRXG/slqsHrww8/5LbbbiMnJwdJkti8efNln7Njxw6mTp2KXq9n1KhRHda5FwD7eRodHg436Tjb2LM5dA1yAqcb3VSdt7Yt2DcI1FicaPxOkjVuZawrPrrBy+sPUO1SY3F6lY1vfWIZqb6KavCy2+1MnjyZZ555pkfly8vLmTdvHtdffz0HDhzgkUceYenSpWzdujXMNY0x9nou2D0cOK+m8nzPgpfF5eeE3UC9zd3WchsEqppcxHvOk6jXKAsQajpPGYkUi9PLpr21HKtvCVpi3KvPojrmNXfuXObOndvj8s8++yzDhw9n9erVAIwdO5aPPvqINWvWMGfOnHBVM/Y4GnB6/DiMKaQldr5w5KWyTAYOa1NJsDfgsVSjSx8T5kqGn8Pjw+L0kuM9T0K8ZkB0GRNbsuydGPAFAmjczWCM3P6Rg0lMjXnt3r27w67cc+bMYffu3V0+x+12Y7Va290GNY8d2WPH4fXj1CZ1mZx6qXi9BikxExmwNgyO5XFUksRNYzOZlOxRdguKcpcRQK9Ro9eqLkqXEONefRVTwaumpqbDBiCZmZlYrVaczs5XpiwtLcVsNgdv+fn5kahq9NgbcHsDOFQJqDQ6ko097yYlpGQrpzg/OIKXQatmYp6ZsfEtXbPErOhWqIXJoMWrNuIWWfb9ElPBqy9WrFiBxWIJ3s6cORPtKoWXvQGHx49Dm0xKgq7DShLdScrIA8DVfAG8g2S5Fq9L2UEJIDE7unVpkXhxoqpbJKr2VUzleWVlZVFb237NqdraWkwmE3FxnU881uv1XW4YMig5GnB4fDi1yaQl9O59Z6Ym8ZU6AZvLjmyrRUoeFqZKhp/L6+dotZVCVR0poOzRqOt8ZY1IMxm01ImWV7/FVMuruLiY7du3tzu2bds2iouLo1SjAchej9OrtLzSEnp3ZS0jUY9Tn4oMuJtiO1n1bKOTncfr+eTAYeXAAGl1AZjiNLjV8bh9ftHy6oeotrxsNhsnTpwI3i8vL+fAgQOkpKRQUFDAihUrOHfuHC+++CIA3/ve9/jtb3/Lj3/8Y5YsWcL777/PH//4R7Zs2RKttzCwyDLYGxiVkUDmhMlozL2bgKxVqygpGkdcdTOS+3yYKhkZZ1ry24Zpm5QDppzoVeYS+SlGdFcUkFlhAPcgv4AURlFtee3Zs4eioiKKiooAWL58OUVFRaxcuRKA6upqKisrg+WHDx/Oli1b2LZtG5MnT2b16tU8//zzIk2ilccOXieSpMKUkoVR1/vvJmNKrpKJHuNLQp+5oKwRn0HLMjgDqOWVkWhg4oh8TAat0vIaRDMaIimqLa/Zs2d3uilEq86y52fPns3+/fvDWKsY1jKnEUOSsvRLX7TmQtkblKWkVTE1sgBAs8vLeZsHfcBOstYDkmbAXGkM0iUqGf8Bv/Klo0+Ido1iTuz9zxS6Zm/ggt3NvvNqjlX3rTvi0pg4Vu/iQEUDAXtsdh1P1SuD4MO1jS35XWl9D+ZhUmvzUO/R4gsERNexj0TwGkzs9VhdPk454qi19i3VQadRU+1LxOn101R/NsQVjIxTDUri50h9k3IgqSB6lenCls+r2V8XwOHxg0sEr74QwWswCeZ4pfQ6TaKVSiWhT1K6WE3150JZu4jw+QNUt6wkkUtLN9o88BKTTXFaPOp4JV1CtLz6RASvwUKWke31LTleSaT3cE5jZ+JTlCtz9guxl2mvUat44NoRfHNSCvG+ltUxzHnRrVQnEg0a3OqElkRVEbz6QgSvwcJjw+t24vWDS5dESg/nNHYmKT0XiN1cL61aRYG6UbmKZ0wdkIPhiQYNHk0Cbq/oNvaVCF6DRUury6UxYTbGKVt89VFadj4g4XNY8ThiJ4kyEJDbrl5bWqaBDcBWF0BSnA6POh6XaHn1mQheg4X9fHBOY0+XwelKgtGIHJeEDDTUxM5c0KPVVl7YdZrD5yzQ1JIfmDTwxrsAkoxa3OoEXKLl1WcieA0W9nr8fhm3PrXPg/UXi0vOJlGvQeWInYUJj1VbaXR4cTttbWvxJxdGtU5dMcdpcWuUMa+A2wZ+X7SrFHNiamK20A17PfkpRnLHTsaXdvlNNy5nytgrkMprQWUJQeXCz+LwcrZRWRZpjP6CMt4Vnwb6xCjXrHNGnZpZV+ZTGEhSDnialZVehR4TLa/BQJaDSzerEjPQafr/zyq1bshhj42t0A6ebQKgMM1Igr2ly5gyPHoVugxJkphWmEJqajoqSQJXbHxJDCQieA0GLouymYNKHbpv75ZpQr7mBnxeT2jOGSZef4AjVcq40eRcM1woVx5IGRHFWvWQIUn5OYg2PYkUEbwGg5ZpQZ/Vynx6uik059Sb+OK8lz3lDVScqbx8+Sg6XtOMy+vHFKelMM6lTHZWaQZkcurFml1ezrn1NDk94GyKdnVijgheg4G9HpvbT53fRLMrRAO/koQcn44MnK8duFccZVlm/5kmACbnmVFdaFliKalgwM1nvFTFeQd/O+NTZgS4mqJdnZgjgtdg0LJ6qkOb3K/M+kuZW5JVrfUDN9NekiS+PjaTKzITmZBrhoYvlQfSRke3Yj1gjtPi0piURFXRbew1EbwGA3t925zGEAavlAyl2+VvrsXhGbiX8rPMBuZNysbgt4O1SllqJhaCl1EJXi5vgICjSazr1UsieMW6QABvcz1uXwCHNrnHW531hCE5G6NOjdFznrMXerZ5bST5A5f8sZ//SvlpyhmwKRIXS9RrCOjNyrLbLgd4O98BS+icCF6xztWEw+kmIGnQJ6Zi0KpDd25jGiajDk3ATXX9wEpWDQRkNv29kg++qFOy1AHqW7uMV0SvYr0gSRLmRCMetRGnxy/GvXpJBK9Yd9FKEqHsMgKg1hCfrOR71Ved6XbV20g7Wm2lzurmi5qWuZfuZmiqUH6Pod2+U+P1uDUmpVsuxr16RQSvWGerQyVJqBIzyEg0hPz0yRn5pMTrmJbiGjBDMm6fn10nlbW6ZgxPUVqbtUeVMSNzXkxlqqcm6HBpTDi9fpEu0UsDIng988wzFBYWYjAYuPrqq/nss8+6LFtWVoYkSe1uBkPo/2hjhq2WTJOBG6ZNZOaIlJCfXpucx5jMRIZrG3u1gW047TpxHrvbT5JRy5T8JOVg7SHlZ9aEqNWrL0amJzD1yhHkJxvbNscVeiTqwevVV19l+fLlPP744+zbt4/JkyczZ84c6uq6npZiMpmorq4O3ioqKiJY4wGmdZefxEwkKQzBpTXR03pO2ZAjys41OYNTgW68MhO1SoLmWrDVKzMM0q+MbgV7KSVeR15ugdJ6bN1AReiRqAevp556igceeID77ruPcePG8eyzz2I0Glm/fn2Xz5EkiaysrOAtMzMzgjUeQDwOAk6LMhaVEKbPID4dNHpcLheff/lV2+B4FHh8Af56tBZZhvE5JgpSW3bArtqn/EwbDdrOd04f0OLTlJ+O8yJdoheiGrw8Hg979+7lpptuCh5TqVTcdNNN7N69u8vn2Ww2hg0bRn5+PvPnz+fIkSNdlnW73Vit1na3QcNWQ4PNzUfn/HxwMkwTeyUJzHkcr23myNEjfFVrC8/r9ECjw4PD4yfRoOG6K9KVg14n1Lbsip07LWp164+zLh1nrR5sDpFp3xtRDV4NDQ34/f4OLafMzExqajpfgnjMmDGsX7+et956i5deeolAIMCsWbM4e7bznW5KS0sxm83BW37+wJ7v1iu2OuxuPxZNqrIyQbiY80hL0JPoruFYTfSCf6bJwHdmFnDb5Jy2lJDqz5W1sBLSB/xcxq4crmrmuEVLk8MDMbrdXDREvdvYW8XFxSxcuJApU6ZQUlLCG2+8QXp6Os8991yn5VesWIHFYgnezpwZuPP0eq25Brvbh12XRqYpxGkSFzPnkZagw+Su5twFBxanN3yv1Qmfv22sLdGgJdPUcoEm4Idze5Xfc6cprcQYlJagx6lNVrZBE+NePRbV4JWWloZaraa2tv3W8rW1tWRl9WyHY61WS1FRESdOnOj0cb1ej8lkancbLALNNdg9PuzaVDLDkCYRlJiDXq8nTefD6L3AkXORW3uqzuqibNdpTtV30l2tOaQsB6SLh8zYusp4sUyTAYc2CZvbp+xULvRIVIOXTqdj2rRpbN++PXgsEAiwfft2iouLe3QOv9/PoUOHyM7ODlc1ByavC6elnoAMvvgMkoxhXEFBrYGkYWSa9CS5znLwrEXZsivMaiwuXt93lmaXjz0Vje2TZAN+qNil/F5QPOBXkOhOeqLS8nL7AnistZd/ggAMgG7j8uXL+f3vf88LL7zAsWPH+Od//mfsdjv33XcfAAsXLmTFihXB8qtWreK9997j1KlT7Nu3j+985ztUVFSwdOnSaL2F6LCew+5WdgtKSUoOT5rExVJGkGLUke2vwuX1c7Q6vGNfx2uaeX3vGdzeALlJccyfktP+PVbtb2t15UwJa13CzaBVozUpiz/aG2sHREpKLIj6GvZ33XUX9fX1rFy5kpqaGqZMmcK7774bHMSvrKxEpWqLsY2NjTzwwAPU1NSQnJzMtGnT2LVrF+PGjYvWW4gOaxXNLh82fTZZpggk6aaMQJIkRhuaOISXQJgu6bt9fj4+0cDBM0rXtDDNyLyJOe2XtnbboPxD5ffCr8V0q6tVUmoW/tNaHE4XyY6G4Eq2QtckeSBNWIsAq9WK2WzGYrHE9vjXwU3UVRzjSMJMhk+6jsK0+PC/5ifPEnBcwD76dhLzQz/G5A/IvLDrdPCCwFWFyVwzMq1jZv+x/4Oaw5CYCVMXgyrqHYh+21txgYa/rWe45gJXlHwLsidFu0pR09O/0dj/Vx+KZBms58hINHD99KLIBC6A9CtQSRKJ1q+ChzosS9NLLq8/OJalVknkpxgxx2n55tQ8rh2d3jFw1X2hBC5JgtFzBkXgAhiTZeLqyeMZlZHQtm2b0K2odxuFPrA3gM+jdJfiI9i9yBgHlZ9CwwnwualzyLz9eTVzJ2aRbe55ZrvL6+f0eTun6u2crLNx51X5ZJmVru/XRqWhUUud7/jtbIQv31F+z78azLmheFcDQoJeA+n50HAAbCJ49YQIXrHIUond40Odkochki2PhEwwpoDjAjR8yae1qVicXl7fc5aZI1OZnJfU6bZrXn+A6iYX55qcnG10UNXkajdmdqrBFgxecbou1iPzOuHz18DrAlM2DL8uLG8xqhJbrpjbapWrqaoQrs02CIngFYsaT1Neb+dos5YpBc1ckRmhVUMlScmnKv8Qzu1jzuR7CcjVnKq389FXDXx66jxZ5jgS9BryU+IYn2MGoMnh5U/72s+ASEvQMTwtgVEZCZdPsPU64fM/KnP/9Ikw4ZuD8g+71mfEcsGLAQcFtlplRVihSyJ4xZpAAO/509jcPixJucEWS8RkT4aKj8Fahc5Rw+2TczhabeWz8gs0ObycuWi56NbglRqvIy1RT1q8jpykOApT4zH3NC/N2QiHXle6yloDTPqnmFjiuS8c3gAn3Mlk+85Q0HRGBK/LEMEr1thqsDY341Np0afkYjJEOE1AnwAZY5VB84pdSBP/kfE5ZsZlm2iweahvduPw+Ei+aC19lUri3pnDevc6sgx1R+HLd5XxPX0CTPqWModxkMo2G/jMkEVyYwWO+nKMBVdHu0oDmghesaaxgiaHF6s+h4LUhOjUoaAYao9Aw1fQdAaS8pEkifREfWi2XrOcg/Kd0NiyTps5F8bNB4O5/+cewAxaNXHpI6DxU5qqT2GU5ZidrxkJg+M68xAin/8Ki9NLkyGPwtQIpUhcKj5N6T6C0jLyh2CidsAP9cfhwCuw70UlcKk0MPxamPLtQR+4WuXkDcOv0mJpbhYpE5chWl6xxG3DUV+J2xegOb6Q3OQoLrw3/Dplg1d7A3y1DcbM7X0rIRAA61kld6vuiHIlEUBSQeZ4KLwmptajD4Xh6Yl8os9F6zyNp+5LdKYhNme3F0TwiiXnv+KC3Y1dl0Z2VmbnuVCRoouHK/8BDr0G1QeVwDXq68ok7u54HMouPw1fwYWTbQELlHGtzPGQMxXiksJa/YEqJV6HL2UkgXOnaaw8SuaokmhXacASwSuW1B8n22zAXzgNdevGE9GUOhKumAPH34WqA0pXL+8qZVFAnVHpCnrsyhpVzbVK0Lp0yRetAVJHKQmwycMHTcZ8X0mSRPqw8cTV70TrqAWXFQwxPI0tjETwihUuKzSeRqNSUXjlNDAao10jRU4R6E3wxdtKWsNX2y7/nPg0SBmhrDlvyhvyAetSM67IQ+0Yi2StgvovIH9GtKs0IIngFStqDinpA0n5Spb7QJI6Eq7+nrIk8/mvwFanJJaqNMqGGMZUZSMPc55Sf12ULjTECI1aBVkTwVqldMnzpourjp0QwSsWBAL4zh3g2DkL6qTrGBOQlS2/BhKNHvKnKzdQAq34g+u7jHH4v/or52vOkmqpQp00eOZxhopor8eCuqPU1dfS5NXwd1s6Ay1udUoErn6RNXo+tqRwst7OucMfRrs6A5IIXgNdIID/9MdUNzmpTpzItBFh2lxWGFAkScJ8xbUA1J88gMcq1ra/lAheA13VPqqqzuKUtTjSJzM2W1x5GirGXTEal3k4Hp+f8s/ejnZ1BhwRvAYyZxP24+9T1eSk0jyDWVfmDryxLiFstGoVhVfNRUaiseIw1ScORrtKA4oIXgOV34v38z/x1bnzWLUZGAunMTojSnMZhagZXjgC3fBZyEDFx6/RUFsV7SoNGAMieD3zzDMUFhZiMBi4+uqr+eyzz7ot/9prr3HllVdiMBiYOHEif/nLXyJU0wjxuuDzV3GcP4ctoKUqbw43T8gWY11D1KSvzUOdlIfsc1HzURnY6qNdpQEh6sHr1VdfZfny5Tz++OPs27ePyZMnM2fOHOrq6jotv2vXLu6++27uv/9+9u/fz4IFC1iwYAGHDx+OcM3DQJaR675A/vvz0HQGc2ICedfdy9yrxmDUiayWoUqn0zHpliXEJ2cw2hSAfWVQ+Slej+uyzx3Mor570NVXX8306dP57W9/Cyibzubn5/P973+fxx57rEP5u+66C7vdzttvtw1gzpw5kylTpvDss89e9vUGzO5BsozsdeJx2XA11mKrr8BV/QXWxjqGp8aTlJoJ47+h7JAjCKDMCz26ObhU0KE6NxcMBSRkjsCUlk1iUioJCSb0Wg0alRSzLfWe/o1G9evc4/Gwd+/edpvKqlQqbrrpJnbv3t3pc3bv3s3y5cvbHZszZw6bN2/utLzb7cbtdgfvW6093yzV4vDy54PnaI3usgyZF/6O2XYKkMk2GyhMMQIyLq+P/ZWNKIVlJGTlCYCMTGainhFpRpBlvP4A+8trkQN+Lt18x6/SckA9ntlX3Q4aHYIQpDPC5Luh5nM8Jz/CYS9HZzuKp+EoDUBrMoVfpcVsjGNifoqyQodKzaenGwnIMhJShxS8RIOWcRddxd5b0YSvi41vpdSRzLjpH4P339h3Frvb12lZs1HH7ZPbVoP1+gMhXUwgqsGroaEBv98f3GC2VWZmJl988UWnz6mpqem0fE1N52sflZaW8sQTT/Spfn5ZpsHmaXcsydYE9pYxB40BDMpUF8nnB2dTsFyH5qzOC62n8gfw+/3Kr5IWrz4JKTETbdpI0oaNZURmCnSykYUgIEmQPRld5kQmji6noeIozQ1n8TXX43Pa8PoDqANedLIE7ua259nPg6z8v7z0/6bs04C97f+5bL9AwN95h8xnSGt3v9HhxersfD23Dq8T4j7eoB9IWbFiRbuWmtVqJT8/v0fPTdBr+ObUvHbfVGrnHCTv15AkCaNOC3HKMswaGUaO9bY01S9qsksSSBIGrRr0SktKLcOYqSrUOiNxBn2nO+4IQrdUKoyZIynIHNl2LBBA9jnxuOzIfj9oJGVlDznAiJFOArKM/+JuRAudRoKEthVw80a4aGt4tY84mrj2V7xvmZCFv4tAp1G3b+JpQpzmE9XglZaWhlqtpra2tt3x2tpasrKyOn1OVlZWr8rr9Xr0+r4tTazTqChIvXT1hoJOy6qBjB6um6cCBtjUamEwUKmQdPHoO5n4np7U89Nk9WL9x9ykni+I2WED4X6K6le+Tqdj2rRpbN++PXgsEAiwfft2iouLO31OcXFxu/IA27Zt67K8IAiDU9S7jcuXL2fRokVcddVVzJgxg6effhq73c59990HwMKFC8nNzaW0tBSAhx9+mJKSElavXs28efPYtGkTe/bs4Xe/+10034YgCBEW9eB11113UV9fz8qVK6mpqWHKlCm8++67wUH5yspKVBctVjdr1iw2btzIT3/6U37yk58wevRoNm/ezIQJE6L1FgRBiIKo53lF2oDJ8xIEoVM9/RsVl7kEQYhJIngJghCTRPASBCEmRX3APtJah/h6M01IEITIaf3bvNxw/JALXs3NypSJnmbZC4IQHc3NzZjN5i4fH3JXGwOBAFVVVSQmJsbsrPv+aJ0edebMGXG19SLic+lcND4XWZZpbm4mJyenXZrUpYZcy0ulUpGXlxftakSdyWQSf6SdEJ9L5yL9uXTX4molBuwFQYhJIngJghCTRPAaYvR6PY8//nifV9oYrMTn0rmB/LkMuQF7QRAGB9HyEgQhJongJQhCTBLBSxCEmCSClyAIMUkEr0FI7EDeud58LmVlZUiS1O5mMBgiWNvw+/DDD7ntttvIyclBkqQutw+82I4dO5g6dSp6vZ5Ro0ZRVlYW9np2RQSvQUbsQN653n4uoGSVV1dXB28VFRURrHH42e12Jk+ezDPPPNOj8uXl5cybN4/rr7+eAwcO8Mgjj7B06VK2bt0a5pp2QRYGlRkzZsjLli0L3vf7/XJOTo5cWlraafl/+qd/kufNm9fu2NVXXy0/+OCDYa1npPX2c9mwYYNsNpsjVLvoA+Q333yz2zI//vGP5fHjx7c7dtddd8lz5swJY826Jlpeg0jrDuQ33XRT8FhPdiC/uDwoO5B3VT4W9eVzAbDZbAwbNoz8/Hzmz5/PkSNHIlHdAWug/V8RwWsQ6W4H8q52FO/tDuSxqC+fy5gxY1i/fj1vvfUWL730EoFAgFmzZnH27NlIVHlA6ur/itVqxel0Rrw+Q25VCUHoieLi4nZ7gc6aNYuxY8fy3HPP8fOf/zyKNRNaiZbXIBKJHchjUV8+l0tptVqKioo4ceJEOKoYE7r6v2IymYiL6/nO2aEigtcgInYg71xfPpdL+f1+Dh06RHZ2driqOeANuP8rUblMIITNpk2bZL1eL5eVlclHjx6Vv/vd78pJSUlyTU2NLMuyfO+998qPPfZYsPzHH38sazQa+de//rV87Ngx+fHHH5e1Wq186NChaL2FsOjt5/LEE0/IW7dulU+ePCnv3btX/ta3viUbDAb5yJEj0XoLIdfc3Czv379f3r9/vwzITz31lLx//365oqJClmVZfuyxx+R77703WP7UqVOy0WiUf/SjH8nHjh2Tn3nmGVmtVsvvvvtuVOovgtcg9Jvf/EYuKCiQdTqdPGPGDPmTTz4JPlZSUiIvWrSoXfk//vGP8hVXXCHrdDp5/Pjx8pYtWyJc48jozefyyCOPBMtmZmbKt956q7xv374o1Dp8PvjgAxnocGv9HBYtWiSXlJR0eM6UKVNknU4njxgxQt6wYUPE691KLIkjCEJMEmNegiDEJBG8BEGISSJ4CYIQk0TwEgQhJongJQhCTBLBSxCEmCSClyAIMUkEL0EQYpIIXsKgV1ZWRlJSUvD+z372M6ZMmRK8v3jxYhYsWBDxegn9I4KXEDKLFy9GkiS+973vdXhs2bJlSJLE4sWL25UPddAoLCzk6aefbnfsrrvu4ssvv+zyOWvXrm23Fvvs2bN55JFHQlovIfRE8BJCKj8/n02bNrVbnM7lcrFx40YKCgqiUqe4uDgyMjK6fNxsNrdrmQmxQQQvIaSmTp1Kfn4+b7zxRvDYG2+8QUFBAUVFRf06d2ctogULFgRbc7Nnz6aiooJ//dd/De74Ax27jZe6uAW4ePFidu7cydq1a4PnKC8vZ9SoUfz6179u97wDBw4gSdKQXuMrmkTwEkJuyZIlbNiwIXh//fr13HfffWF/3TfeeIO8vDxWrVoV3PGnt9auXUtxcTEPPPBA8BwFBQUd3hPAhg0buO666xg1alSo3oLQCyJ4CSH3ne98h48++oiKigoqKir4+OOP+c53vhP2101JSUGtVpOYmEhWVlafVoM1m83odDqMRmPwHGq1msWLF3P8+PHgXo9er5eNGzeyZMmSUL8NoYfEGvZCyKWnpzNv3jzKysqQZZl58+aRlpYW7Wr1S05ODvPmzWP9+vXMmDGD//u//8PtdnPnnXdGu2pDlmh5CWGxZMkSysrKeOGFF0LWOlGpVFy6/JzX6w3JuXti6dKlwYsRGzZs4K677sJoNEbs9YX2RPASwuKWW27B4/Hg9XqZM2dOSM6Znp7ebhzL7/d32Nlbp9Ph9/v79TpdnePWW28lPj6edevW8e6774ouY5SJbqMQFmq1mmPHjgV/74rFYuHAgQPtjqWmppKfn9+h7A033MDy5cvZsmULI0eO5KmnnqKpqaldmcLCQj788EO+9a1vodfr+9RdLSws5NNPP+X06dMkJCSQkpKCSqUKjn2tWLGC0aNHD6pNSmKRaHkJYWMymTCZTN2W2bFjB0VFRe1uTzzxRKdllyxZwqJFi1i4cCElJSWMGDGC66+/vl2ZVatWcfr0aUaOHEl6enqf6v3oo4+iVqsZN24c6enpVFZWBh+7//778Xg8Ebl6KnRPrGEvCL3wt7/9jRtvvJEzZ8502D1aiCwRvAShB9xuN/X19SxatIisrCxefvnlaFdpyBPdRkHogVdeeYVhw4bR1NTEk08+Ge3qCIiWlyAIMUq0vARBiEkieAmCEJNE8BIEISaJ4CUIQkwSwUsQhJgkgpcgCDFJBC9BEGKSCF6CIMSk/w/ata3AuNeaNAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:10:48.834451Z", + "iopub.status.busy": "2024-03-22T19:10:48.834145Z", + "iopub.status.idle": "2024-03-22T19:10:49.238845Z", + "shell.execute_reply": "2024-03-22T19:10:49.237854Z" + }, + "papermill": { + "duration": 0.426045, + "end_time": "2024-03-22T19:10:49.240918", + "exception": false, + "start_time": "2024-03-22T19:10:48.814873", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLRUlEQVR4nO2deXxU9bn/32f2TDIz2feEgGyCkAQUDVZBpVKlFq6ttbYKiFi9pfdVLtda03t/WPVl8d5btLT1hW0toFepFRdstVUpCrSAC6vsyJKEJSskM5l9O78/TjJZSEKWWTLJ9/16ndfMnPmeM89MMp95vs95vs8jybIsIxAIBHGGKtYGCAQCQX8Q4iUQCOISIV4CgSAuEeIlEAjiEiFeAoEgLhHiJRAI4hIhXgKBIC4R4iUQCOISTawNiDbBYJDz589jMpmQJCnW5ggEgk7IskxzczO5ubmoVN37V8NOvM6fP09BQUGszRAIBJfhzJkz5Ofnd/v8sBMvk8kEKB+M2WyOsTUCgaAzNpuNgoKC0He1O4adeLVOFc1msxAvgWAQc7mwjgjYCwSCuESIl0AgiEuEeAkEgrgkpjGv1atXs3r1aioqKgCYOHEiy5cv57bbbuty/Lp167j//vs77NPr9bjd7rDaJcsyfr+fQCAQ1vMK+oZWq0WtVsfaDMEgJabilZ+fzzPPPMOYMWOQZZmXXnqJuXPnsnfvXiZOnNjlMWazmWPHjoUehztXy+v1Ul1djdPpDOt5BX1HkiTy8/NJSkqKtSmCQUhMxeuOO+7o8Pjpp59m9erVfPLJJ92KlyRJZGdnR8SeYDDI6dOnUavV5ObmotPpRCJrjJBlmfr6es6ePcuYMWOEBya4hEGTKhEIBNiwYQMOh4OysrJux9ntdkaMGEEwGGTKlCn8/Oc/71boADweDx6PJ/TYZrN1O9br9RIMBikoKMBoNPbvjQjCRkZGBhUVFfh8vsElXsEAnN4KtmoomAbpY2Jt0bAk5gH7AwcOkJSUhF6v5+GHH+btt99mwoQJXY4dN24ca9as4Z133uGVV14hGAwyffp0zp492+35V6xYgcViCW29ya7vaUmCIHoMWq+36hOo+hSaquDQ2+C8GGuLhiVSrBtweL1eqqqqsFqtvPHGG7z44ots3bq1WwFrj8/n48orr+See+7hqaee6nJMV55XQUEBVqv1kiRVt9vN6dOnGTlyJAaDYWBvTDBgBuXfw++Fnb9WblvJKYbxt8fOpiGGzWbDYrF0+R1tT8xdDJ1Ox+jRo5k6dSorVqyguLiYVatW9epYrVZLaWkpJ06c6HaMXq8PZdOLrHrBgGk4pghXQgqUfFfZV39EmUoKokrMxaszwWCwg6fUE4FAgAMHDpCTkxNhqwY/M2fOZOnSpbE2Y+hzoeWHMmsCJBeCLlERs6bK2No1DImpeJWXl7Nt2zYqKio4cOAA5eXlbNmyhe9973sAzJ8/n/Ly8tD4J598kg8//JBTp06xZ88e7r33XiorK1m8eHGs3kLcsWXLFiRJoqmpKdamxB+yDI0tIpUyEiQJUkcpj5uqYmfXMCWmVxvr6uqYP38+1dXVWCwWJk+ezAcffMBXv/pVAKqqqjoEzxsbG3nwwQepqakhJSWFqVOnsmPHjl7FxwSCAWOvBZ8LNDow5yr7LHlQcwBs52Nr2zAkpp7XH/7wByoqKvB4PNTV1fH3v/89JFygeAnr1q0LPX7uueeorKzE4/FQU1PDe++9R2lpaVRs9fqD3W7+QLDXY329HDsQPB4PP/nJTygoKECv1zN69OjQZ33TTTcBkJKSgiRJLFy48LLna25u5nvf+x6JiYnk5OTw3HPPXTJN/b//+z+uvvpqTCYT2dnZfPe736Wuri70fKvHt3nzZq6++mqMRiPTp0/vkHA86GkVKHMeqFpSN8z5bc8FB/Z3E/SNQZPnNdh5/uPuLwqMTE9kXmle6PHvtp3EF+j6Im5+SgJ3Xd2WrrFm+2lc3kuDvf/+1bH9tnX+/Pns3LmTX/3qVxQXF3P69GkaGhooKCjgzTff5Jvf/CbHjh3DbDaTkJBw2fMtW7aM7du38+c//5msrCyWL1/Onj17KCkpCY3x+Xw89dRTjBs3jrq6OpYtW8bChQv561//2uFc//mf/8nKlSvJyMjg4YcfZtGiRWzfvr3f7zWqNNcot6Z2SdKJ6Yon5veCswGSMmNj2zBEiNcQ4/jx47z++uts2rSJWbNmATBq1KjQ86mpqQBkZmaSnJx82fM1Nzfz0ksvsX79em655RYA1q5dS25ubodxixYtCt0fNWoUv/rVr7jmmmuw2+0dlvc8/fTTzJgxA4DHHnuMOXPm4Ha7B08qRE/YW8QrqZ14SRIkZoD1HDjqhXhFESFevWTJTaO7fU7VKZfy+zde0e3YznmXi64fORCzLmHfvn2o1eqQQAyUU6dO4fP5mDZtWmifxWJh3LhxHcbt3r2bn/3sZ+zfv5/GxkaCLVOoqqqqDjHJyZMnh+63XiWuq6ujsLAwLPZGjIAfHA3KfVOn5WntxUsQNYR49RKdpvfhwUiN7Q29mQaGG4fDwezZs5k9ezavvvoqGRkZVFVVMXv2bLxeb4exWq02dL81gz4YD7EiZ4OSy6U14NWY0Mpy2wqAxAzltlXcBFFh0OV5CQbGpEmTCAaDbN26tcvndTodQK/L/YwaNQqtVsvnn38e2me1Wjl+/Hjo8dGjR7lw4QLPPPMMN9xwA+PHj+8QrB8SOBoIyDJ7GjQ8v+Ukn5xqtyQoMb1ljPC8ookQryFGUVERCxYsYNGiRWzcuJHTp0+zZcsWXn/9dQBGjBiBJEm8++671NfXY7fbezyfyWRiwYIF/PjHP+bjjz/m0KFDPPDAA6hUqpDnUVhYiE6n49e//jWnTp3iz3/+c7fLteIW5wXON7k440lgRJqRaSNT254zpim3bqvItI8iQryGIKtXr+Zb3/oWP/jBDxg/fjwPPvggDocDgLy8PJ544gkee+wxsrKy+OEPf3jZ8z377LOUlZXx9a9/nVmzZnH99ddz5ZVXhoLsGRkZrFu3jg0bNjBhwgSeeeYZfvGLX0T0PUYbv72BaqsblyaZibkW1O0DnbokUGuUJFa3NXZGDjNivjA72vS06HNQLgQehDgcDvLy8li5ciUPPPBAxF5nMP09av7+a05XVnKu4Ot886szkCQJWZbx+IMYtGr4/EWw18Pkb0Na9xdsBJentwuzRcBecFn27t3L0aNHmTZtGlarlSeffBKAuXPnxtiyKBEMYr1QC0BBXgGSJHG6wcHfD9eSadYztyQPDMmKeLmaYmrqcEKI1zCncypDZw4fPgzAL37xC44dO4ZOp2Pq1Kn84x//ID09PVpmxhTZ3USzy0NQUjMiV0mTSNSrsXv8eC4G8AeCaBJSlMGuxhhaOrwQ4jXMyc3NZd++fT0+X1hYyO7du6Nn1CDjYkMNvoCMz5BMlkWpsJuRpMeoU+P0Bqhr9pArxCvqCPEa5mg0GkaP7j4BVwCpkoPJ+RaakwpDgXpJksi2GDhV76DG5iY3KVkZ7G6KmZ3DDXG1USC4DJLHSqJOQ3ZWVof92WblIkKt1a3EvEC52ji8roHFDCFeAsHlaA3CG1I67M62KOJVY3OD3qTsDPjAH94+ooKuEeIlEPSA1x/kyOmz1NrcyIaOl+0zTYp4NTl9eGQVaFuWZnmao23msETEvASCHmiwe7hwoQ6n5CcroaPnlaBTMyYriUS9hkBQVrwvnwvcNlFdIgoI8RIIeqC+0Yo66CXRqAX9pQmTX5/crjSQwQL2OvB03xtUED7EtFEg6AFbo7LY2mA0KUUHe6I17iXEKyoI8RoiDLbuQYPNnv7isF4AQG9K63aM2xfggt3T5pmJmFdUiKl4rV69msmTJ4f6KZaVlfG3v/2tx2M2bNjA+PHjMRgMTJo06ZIyw4L+07n2lgA8dqX0TYI5tcvnq60uVm85ydt7z7V5Xm7heUWDmIpXfn4+zzzzDLt372bXrl3cfPPNzJ07l0OHDnU5fseOHdxzzz088MAD7N27l3nz5jFv3jwOHjwYOSNlWalPHoutl/lCCxcuZOvWraxatQpJkpAkiZMnT/LAAw8wcuRIEhISGDdu3CXNfBcuXMi8efN4+umnyc3NDVVH3bFjByUlJRgMBq6++mo2btyIJEkdMvEPHjzIbbfdRlJSEllZWdx33300NDR0a09FRUVY/hzRxO0LEHQqVSKMlq49L7NBKa5o9/jxa1vKXQvPKyrENGB/xx13dHj89NNPs3r1aj755BMmTpx4yfhVq1bxta99jR//+McAPPXUU2zatInf/OY3vPDCC5ExMuCDf6yMzLkvxw3/cfk4C8rncvz4ca666qrQoumUlBTy8/PZsGEDaWlp7Nixg+9///vk5OTw7W9/O3Ts5s2bMZvNbNq0CVBW9N9xxx3cfvvtrF+/nsrKykumf01NTdx8880sXryY5557DpfLxU9+8hO+/e1v89FHH3VpT0ZGRpg+lOjR5PShDzSjU6vQJaZ0OcaoU6PTqPD6gzRjJAUU8ZLlS2t+C8LKoLnaGAgE2LBhAw6Hg7Kysi7H7Ny5k2XLlnXYN3v2bDZu3NjteT0eT4cO3Dbb0HPpLRYLOp0Oo9FIdnZbffUnnngidH/kyJHs3LmT119/vYN4JSYm8uKLL4YqrL7wwgtIksTvf/97DAYDEyZM4Ny5czz44IOhY37zm99QWlrKz3/+89C+NWvWUFBQwPHjxxk7dmyX9sQb2RYDXx2px2czKVcSu0CSJMwJWhqaPTQF9Ip4Bf1Koqo2+iW5hxMxF68DBw5QVlaG2+0mKSmJt99+u9sqBzU1NWR1WqKRlZVFTU1Nt+dfsWJFhy9xn1FrFQ8oFqi1lx/TA88//zxr1qyhqqoKl8uF1+vt0K4MlLLRrcIFcOzYMSZPntyhflb75hsA+/fv5+OPP+7QFaiVkydPMnZs/9u2DTa0fjtanQb0XYsXQHKreLmDoDWAzw0euxCvCBNz8Ro3bhz79u3DarXyxhtvsGDBArZu3Rq2Ltjl5eUdvDWbzUZBQUEPR3RCkno1dRtsvPbaazzyyCOsXLmSsrIyTCYT//u//8unn37aYVxiYmKfz22327njjjv47//+70uea+0INCTwe5TYI7QF47sg2aj8yDS5fEpVVZ8bvHYg/qbK8UTMxUun04WqGkydOpXPP/+cVatW8dvf/vaSsdnZ2dTW1nbYV1tb2+PURK/Xo9frw2v0IESn03VoqrF9+3amT5/OD37wg9C+kydPXvY848aN45VXXsHj8YQ+t/bNNwCmTJnCm2++SVFRERpN1/9Cne2JR/558BR5DQ6y01NI6OEHzJKgiJetVbwcDeB1RMvMYcugy/MKBoMdYlTtKSsrY/PmzR32bdq0qdsY2XCiqKiITz/9lIqKChoaGhgzZgy7du3igw8+4Pjx4/y///f/LhGhrvjud79LMBjk+9//PkeOHOGDDz4I1aNvbbixZMkSLl68yD333MPnn3/OyZMn+eCDD7j//vtDgtXZnrhob9YOWZY5fb6GGpuboPbS6XF7ss0GigssjM5MAl2LJ+vtubGJYODEVLzKy8vZtm0bFRUVHDhwgPLycrZs2cL3vvc9QGlbX15eHhr/ox/9iPfff5+VK1dy9OhRfvazn7Fr165eNZEY6jzyyCOo1WomTJhARkYGs2fP5s477+Tuu+/m2muv5cKFCx28sO4wm8385S9/Yd++fZSUlPCf//mfLF++HCAUB8vNzWX79u0EAgFuvfVWJk2axNKlS0lOTkalUnVpT1VVVeTefARw+QJIHjsSYEjqPt4FkGk2cPP4LCbmWoR4RRM5hixatEgeMWKErNPp5IyMDPmWW26RP/zww9DzM2bMkBcsWNDhmNdff10eO3asrNPp5IkTJ8rvvfden17TarXKgGy1Wi95zuVyyYcPH5ZdLle/3s9Q5ZVXXpG1Wq3sdDqj+rqx/HtUN7nkP21YL+9e92NZPvrX3h9Y9aksf/RzWT60MXLGDXF6+o62J6Yxrz/84Q89Pr9ly5ZL9t11113cddddEbJIAPDyyy8zatQo8vLy2L9/fyiHKxbduGOF1eVDH3Cg16h6DNa34vYFsLl9WFQJ6EG52iiIKDEP2AsGHzU1NSxfvpyamhpycnK46667ePrpp2NtVlSxunzo/A70GnWvxOvtveeosbr5l5FqikAE7KOAEC/BJTz66KM8+uijsTYjpthcPnQBB/qE3nleJoOGGivYgi1XtkXMK+IMuquNAsFgwOH1K+KlUXVZx6szppY1jtZAi3j5PcrSMkHEEJ5XF8iigcKgIJZ/h29clY6/wYhKknrteQHYvCpQaZQlQl47JHS9JlIwcITn1Q6tVvn1dDqdMbZEAG0letRqddRfW/I60KpVqHV60Fw+ydncIl7NHj/oW/LCRNwrogjPqx1qtZrk5GTq6uoAMBqNocRMQXQJBoPU19djNBq7zeKPKK3VUHsxZYS2aWOz2w9JiUrHIXHFMaII8epE61KjVgETxA6VSkVhYWHUf0BqbW4Of3GSsU0u8lIvP2WEtmmjw+snqDEqUxqf8OAjiRCvTkiSRE5ODpmZmfh8IuAaS3Q6XShjP5pcdHipr28gy+sjrxfxLoAErZqSwmRMeg1BV4IQryggxKsb1Gp1TGItgtjT7FauNOp6maAKyo/eTeNa2p2dao15CfGKJCJgLxB0wu5pyfFS9168OtC6vtEnAvaRRHheAkEnmt1+EkOeV+8C9tC2RCghqMUEwvOKMMLzEgg6Yff40fn7Nm0E+OTUBV79pIqjDX5lh4h5RRQhXgJBJ+xON9qgq8/ilaRvyfUKthQuFOIVUYR4CQTtCARlVF6ldZlOpweN4TJHtJHYIl7WQKt4uSDOijDGEyLmJRC0Q62SWDwtg4AmBXWCqU/ty1o9L5u/pXGKLIPf1RbAF4QV4XkJBJ2QvA40KhVSH4L10OZ52b0tXYRA8b4EEUGIl0DQmdaO1/qea9d3JlGv5AV6/UH86pbCjWJ9Y8QQ4iUQtOPQeSs7jlRQ3+xROgH1AZ1apQT5AY/U6nmJoH2kEDEvgaAdtTY3zosXcesCfcrxAiXLfuqIFNQqCU1TErgQuV4RJKae14oVK7jmmmswmUxkZmYyb948jh071uMx69atQ5KkDlv77s4CwUDouDSob54XwHWj0rimKBV9QsuxIss+YsRUvLZu3cqSJUv45JNP2LRpEz6fj1tvvRWHo+c/uNlsprq6OrRVVlZGyWLBUKeDePVx2tgBnVG5FQH7iBHTaeP777/f4fG6devIzMxk9+7d3Hjjjd0eJ0lSj12yBYL+Ynf70Aac6DRJ/VrX2LpESB/QYQERsI8ggypgb7VaAUhNTe1xnN1uZ8SIERQUFDB37lwOHTrU7ViPx4PNZuuwCQRd4QsE8bmdqOSAUru+H57XofNWXv2kii9qvS0nFTGvSDFoxCsYDLJ06VKuv/56rrrqqm7HjRs3jjVr1vDOO+/wyiuvEAwGmT59OmfPnu1y/IoVK7BYLKGtoKAgUm9BEOfYW6aMapWEWp8I6r5PTFpzvWytWfYiYB8xBo14LVmyhIMHD/Laa6/1OK6srIz58+dTUlLCjBkzeOutt8jIyOC3v/1tl+PLy8uxWq2h7cyZM5EwXzAEcPsDGHGhU6uQ+lMKB0jUtaxvDLRk2QvPK2IMilSJH/7wh7z77rts27aN/Pz8Ph2r1WopLS3lxIkTXT6v1+vR6y/fQEEgyLEkcG9pCsEjlv7V8aLT+kY1besbY1ARdqgT009UlmV++MMf8vbbb/PRRx8xcuTIPp8jEAhw4MABcnJyImChYLgheZVpY3+vNBp1Spa9M6gl0Nq5TXhfESGmnteSJUtYv34977zzDiaTiZqaGgAsFgsJCcryivnz55OXl8eKFSsAePLJJ7nuuusYPXo0TU1N/O///i+VlZUsXrw4Zu9DMIRo7fjTT89Lr1GhVkkEgiq8Kj0JskcRr37kjAl6JqbitXr1agBmzpzZYf/atWtZuHAhAFVVVR2aMDQ2NvLggw9SU1NDSkoKU6dOZceOHUyYMCFaZguGKFuO1ZF0vJKxGh/mfoqNJEkYdWqa3X58kkERL5EuERFiKl696Yi8ZcuWDo+fe+45nnvuuQhZJBjOnG10kd3USCBVBl3/PC+A0sIUgrKMti4JHFaRqBohBkXAXiAYDNg9rdn1un5PGwGmjkhpOWESOBAxrwghLoEIBCgJqm6PF23ArSSohiNGFeoiJMQrEgjPSyBASVDVBlyoVaBWa0Br7Pe53L4ANpcPvV+jLBES08aIIMRLIKDdlFGtQtIn9an8c2eOVNvYcqyea7R+vgIiYB8hxLRRIKB/XbK7w9iSZW8PtmbZC88rEgjxEggAfzBIIq6Bl8KhLVHVLpYIRRQxbRQIgMn5yUzyJCNXJg7Y82pdItQsi/6NkUR4XgJBC5KnGZUkhWHaqHhejoCOoCwr08Ze5DQK+oYQL4GgFW/L0qABThtblwj51Qa8gaAiXCLuFXaEeAkEwIZdZ/ji1FlFbAaY49W6REiW1Pho1z1bEFZEzEsw7PEFgpxtdJHT1IjKktTnrkFdUVqYgizLaM4mgc/a0ogjbeDGCkII8RIMe+xuP+qgF53kH1A5nPaElghdTAKrWN8YCcS0UTDsaXb7laYbahWSRg8aXfhO3pqpLxJVw47wvATDnmaPD33A3rKmceBTRmhbImQIaDGD8LwigPC8BMMee6vnFYYE1VaOVNt49dMqDjf4lR0i1yvs9Eu8Tp06FW47BIKYoaxrdPa7S3ZXtCaqOkSiasTol3iNHj2am266iVdeeQW32x1umwSCqGOUXS3TxoElqLaSoO20REi0QAs7/RKvPXv2MHnyZJYtW0Z2djYPPfQQn332WbhtEwiiwi1XZvH1cUlkJOkHVEG1PaElQsGWzlXC8wo7/RKvkpISVq1axfnz51mzZg3V1dV85Stf4aqrruLZZ5+lvr4+3HYKBBFF8tqRJCls08a2LkLtlggJwsqAAvYajYY777yTDRs28N///d+cOHGCRx55hIKCAubPn091dXWPx69YsYJrrrkGk8lEZmYm8+bN49ixY5d93Q0bNjB+/HgMBgOTJk3ir3/960DehkAAHptyG6arja1LhHzqBCVr3+cU6xvDzIDEa9euXfzgBz8gJyeHZ599lkceeYSTJ0+yadMmzp8/z9y5c3s8fuvWrSxZsoRPPvmETZs24fP5uPXWW3E4us+J2bFjB/fccw8PPPAAe/fuZd68ecybN4+DBw8O5K0IhimNDi+v7jzFl2eUtnvhinm1LhHyqwz4AzIEAxDwhuXcAgVJ7k0Ln048++yzrF27lmPHjnH77bezePFibr/99g4tys6ePUtRURF+v7/X562vryczM5OtW7dy4403djnm7rvvxuFw8O6774b2XXfddZSUlPDCCy9c9jVsNhsWiwWr1YrZHJ5fWUH8UnnBwbufHeMrDX+iuDANbvzxgKqotmdPVSPBoMykU79HLwXh2ofAmBqWcw9levsd7VeS6urVq1m0aBELFy7stlN1ZmYmf/jDH/p0XqvVCkBqavd/4J07d7Js2bIO+2bPns3GjRu7HO/xePB4PKHHNputTzYJhjbNbj/6gL2tgmqYhAtgSmHLEqFqE7jFEqFw0y/x2rRpE4WFhR08LVD6MJ45c4bCwkJ0Oh0LFizo9TmDwSBLly7l+uuv56qrrup2XE1NDVlZWR32ZWVlhbptd2bFihU88cQTvbZDMLxodvvR+e3o1OFLk7gEbUKLeIkrjuGkXzGvK664goaGhkv2X7x4kZEjR/bLkCVLlnDw4EFee+21fh3fHeXl5Vit1tB25syZsJ5fEN/YPX70AUdYc7xacfsC1NrcNAdFomok6Jfn1V2YzG63YzAY+ny+H/7wh7z77rts27aN/Pz8HsdmZ2dTW1vbYV9tbS3Z2dldjtfr9ej1+j7bJBge2D0+dAE7OkP41jW20tpF6Aa/n6sNiGljmOmTeLXGmiRJYvny5RiNbb3tAoEAn376KSUlJb0+nyzL/Nu//Rtvv/02W7Zs6ZXXVlZWxubNm1m6dGlo36ZNmygrK+v16woErTS7/aT5W7sGhVe82pYItfx4isoSYaVP4rV3715AEZ0DBw6g07WVDtHpdBQXF/PII4/0+nxLlixh/fr1vPPOO5hMplDcymKxkJCQAMD8+fPJy8tjxYoVAPzoRz9ixowZrFy5kjlz5vDaa6+xa9cufve73/XlrQgEAGjVKoyyE71GDYbwilfrEiFHUFRTjQR9Eq+PP/4YgPvvv59Vq1YNONVg9erVAMycObPD/rVr17Jw4UIAqqqqOlwYmD59OuvXr+e//uu/+OlPf8qYMWPYuHFjj0F+gaA77plWiOwzgNcf9phX2xIhEfOKBP2Kea1duzYsL96bFLMtW7Zcsu+uu+7irrvuCosNgmFOwI/kdQAD7xrUmdYlQi5ZR0CWUQvxCiu9Fq8777yTdevWYTabufPOO3sc+9Zbbw3YMIEgKniblVuVpq3qaZjQa1RoVBI+tQFfIIhaVJYIK70WL4vFoixcbbkvEMQ7R6ptHD1yhGuanOTn5oU1QRWUC1sJOjVeTwK+QBCD8LzCSq/Fq/1UMVzTRoEgljQ6vdhtjcrC6TAH61uZMiIFfAnoTqkg4FM2tTYirzXc6FfMy+VyIctyKFWisrKSt99+mwkTJnDrrbeG1UCBIFI0u/3oAg7lSmOEsuunFKYo1STO6pTF2T4nqMXMJRz0K8N+7ty5vPzyywA0NTUxbdo0Vq5cydy5c0NXEAWCwY7N5UPvt0cku74DkqQsEQKRLhFG+l1J9YYbbgDgjTfeIDs7m8rKSl5++WV+9atfhdVAgSBStHpekUhQbaV1iZC9NV1CJKqGjX6Jl9PpxGRSfqk+/PBD7rzzTlQqFddddx2VlZVhNVAgiATBoKxUlPCHt+VZZ47WNLP+0yq+bAwoO4TnFTb63YBj48aNnDlzhg8++CAU56qrqxM1sgRxgd3rJyjLGIIt5XAiFLBPDOV6iVr24aZf4rV8+XIeeeQRioqKuPbaa0PrCj/88ENKS0vDaqBAEAm8/iAZRkhS+5GQwBCZIHpCi3jZRQu0sNOvq43f+ta3+MpXvkJ1dTXFxcWh/bfccgv/8i//EjbjBIJIkZ6k597iZPAmg9YAmshUHknUtSzODsW8hHiFi36JFyilaTqXoZk2bdqADRIIooZbqdwbKa8LwKhvmzYGgmKJUDjpl3g5HA6eeeYZNm/eTF1dHcFgsMPzoqO2IC6Ignjp1Cq06nZLhIR4hY1+idfixYvZunUr9913Hzk5OaFlQwJBvPDn/edJrDrO1WofFkNyxF5HWSKkwadqXSIkrjaGi36J19/+9jfee+89rr/++nDbIxBEhYZmD+rmi0gpQATFC2BKYTKqDD+6KpUI2IeRfolXSkpKjx1+BILBTGuOV2GgJccrgtNGgNLCFPDq4bwafG5lmZBKHdHXHA70K1XiqaeeYvny5Tid4ldEEH+05nglBJqVrkERFi8ANIa2qhVi6hgW+uV5rVy5kpMnT5KVlUVRURFabcdV8nv27AmLcQJBJLC5fKiDHowqnxKvjbB4uX0BrC4fiQENSSqfMnXUJ0X0NYcD/RKvefPmhdkMgSB62Fx+9P5mpZqENgE0ussfNACO1TTz0dE6brHLTE5GxL3CRL/E6/HHHw+3HQJB1LC521WTiMKUMbEl18st6wC/SFQNE/2KeYFSCufFF1+kvLycixcvAsp08dy5c70+x7Zt27jjjjvIzc1FkiQ2btzY4/gtW7YgSdIlW3fdsgWCrtCqVWRo3crSnSiIV4KuUws0EfMKC/3yvL744gtmzZqFxWKhoqKCBx98kNTUVN566y2qqqpCtb4uh8PhoLi4mEWLFl22Ln57jh071mEBeGZmZp/fg2D4MnVECniNcDYhOp6XrnMLNOF5hYN+ideyZctYuHAh//M//xMqjQNw++23893vfrfX57ntttu47bbb+vz6mZmZJCcn9/k4gSCEu0m5jXCOF4CxxfNyYRBLhMJIv6aNn3/+OQ899NAl+/Py8qIyhSspKSEnJ4evfvWrbN++vcexHo8Hm83WYRMMX2RZVlruhcQr8p6XTqMsEfK3LBESnld46Jd46fX6LkXg+PHjZGRkDNio7sjJyeGFF17gzTff5M0336SgoICZM2f2mJqxYsUKLBZLaCsoKIiYfYLBT12zh9VbTnDoZEvRTGN0kq2NOg0+lUFp9iEC9mGhX+L1jW98gyeffBKfzwco67eqqqr4yU9+wje/+c2wGtiecePG8dBDDzF16lSmT5/OmjVrmD59Os8991y3x5SXl2O1WkPbmTNnImafYPDT5PQRdNuVLj5SlBJUUboIlY7OU65wCs8rLPRLvFauXIndbicjIwOXy8WMGTMYPXo0JpOJp59+Otw29si0adM4ceJEt8/r9XrMZnOHbdgSDML5vXD6H+AentPnJqeXBL8VvVatVE+N0jKdkoJkJo3IVnLLhHiFhX4F7C0WC5s2bWL79u3s378fu93OlClTmDVrVrjtuyz79u0jJycn6q8bd8gyHH4b6o8rj8/vgSkLICE5pmZFmyaXD4PfhkGjgoSU6L64rqUjt8+t/D1ENZYB0WfxCgaDrFu3jrfeeouKigokSWLkyJFkZ2cjy3KfyuPY7fYOXtPp06fZt28fqampFBYWUl5ezrlz50KpF7/85S8ZOXIkEydOxO128+KLL/LRRx/x4Ycf9vVtDD/qjijCpVIrbe09zfDlJph8V6wtiypWpw+D34rBoIaE6BUXcPsCNDlVmLx+pbqqz9UmZoJ+0adpoyzLfOMb32Dx4sWcO3eOSZMmMXHiRCorK1m4cGGfS0Dv2rWL0tLSUN37ZcuWUVpayvLlywGorq6mqqoqNN7r9fIf//EfTJo0iRkzZrB//37+/ve/c8stt/TpdYcdsgxVO5T7hWVQfI/yq3/hBNjrYmtblGlyeRXPS6uOqud1vLaZP+46R6W1pXCnSFQdMH3yvNatW8e2bdvYvHkzN910U4fnPvroI+bNm8fLL7/M/Pnze3W+mTNnKpete3i99jz66KM8+uijfTFZAGA7D/Z6UGsg/2plPV/GOKg7Cuf2wLivxdrCqODxB3B4Ahh8Ngza6E4bja1dhGifqJoWtdcfivTJ8/rjH//IT3/600uEC+Dmm2/mscce49VXXw2bcYIwUX9UuU0b09a5Oae47blOZbyHKr6AzKh0I+kaJxpVtMVL8ROcootQ2OiTeH3xxRd87Wvd/0rfdttt7N+/f8BGCcKILEP9MeV+5pU4vX5ONzhwJxUoQuZzQdPwaBScpNcwd4KF4hyjMm2O4sWKxHbiJSML8QoDfZo2Xrx4kaysrG6fz8rKorGxccBGCcKI84LSaEKloUrO5i/bK/D6gxi0au4xF5JsPQaNpyF1ZKwtjQ6ulv9PgyWq1Uxbuwi5MRAI2tCIRNUB0yfPKxAIoNF0r3dqtRq/3z9gowRhpMWr8ibl8NcjDXj9QVSShNsX4OO6JAKyDI0VsbUxSnj8AWSnUgEl2mkSWrUKvVaFX23A6w+KgH0Y6JPnJcsyCxcuRK/vukGnx+MJi1GCMNKkXK390pOKyxsgLUnHN6fk88fPqjjnyKTW5iZXqgWvA3SJMTY2srz22RlSaw5xk9FPUrRzvACTvm2JkNHniPrrDzX6JF4LFiy47JjeXmkURAFZDolXSu4V5DckUFyQTKJew3Wj0jhRp0fflA1YoekMZI6Prb0RxB8I0uT0keG+iM6sAmN61G0oKUhBbcjFUH9MeF5hoE/itXbt2kjZIYgEzovKImCVhtyCK7irSBNKTbkqz8JVeRY4NkpZMmQ7N6TFq8nlIyjLmIJNaNVqSIx+msKkfAskZEOTWCIUDvpdSVUQBzSfV25N2UqOF1y6AsKSp9zazkfRsOhz0eFFFfSTLDmRkGLieQHK6gYQlSXCgBCvoUxzDbIsc8pj4qLD22VCcJMmgxqbG7m5WuknOES5YPdi8Ddh1KpAa4hJfM/jD1DrUtHsaekg1EOCtuDyCPEayjRX4/AG2F6r5Y+fVV3yXQkEZV79opkvLwZwuDxgr42NnVHggsNDgq9JqVtvTI/JougzF138cf8FKhqcyg+F3x11G4YSQryGKsEg2GuxunzYdRkUpBpRqTp+YdUqiRHpiTTrM2l0esFWHSNjI099syJeRp0ajLFZlpOk1yBLGlxyS59Tjz0mdgwVhHgNVZwNEPDT6AG3xkJhatcVDIrSEnFo07G5fGAfml2YZFlmdGYSBQankumeGJt4V5KhpYsQCcoU3ivEayAI8RqqtMS7aoMWkCTyUxK6HJafkoBTl4bd4ydgG5riJUkSN4zJ4LosGa1aFTPPy6hVI0ngUSXgCwrxGihCvIYq9lqc3gBWTTp6rYq0xK67QlsStJCUTVAG+8UhHLQPBpTUEYiZ56VSSSTqNHjVRiXL3isSVQeCEK+hiqMeu8ePU5tKttnQbZFISZLIzMjAr9Jjc3rA0RBlQyPPRYcXt60B5CCotaCPXSnwRL0Gn9qoNOIQMa8BIcRrqOJooNnjx6lNIdts6HFoXqoRpy6NZrd/SMa9/nawmte37FYuSiRmxLT8cpKhvefVHDM7hgL9qmEvGOR4neB1UJRmZPakiZgSe85pGpWRROb4caQ3uYZcZdVAUOaC3Uue7wIJRjUkdV8VJRpMyDHhJQ9TzWExbRwgQryGIk5l6qcxpjAi8/ILkJP0GpLyRoDtC2geWp7XBbuHQFDGHGxEr1VBUmZM7RmdaQJdLlzUiGnjABHTxqGIo1657UtgutUjsdcOqczv81YlETRLsinLgmIsXgDokpRbcbVxQMRUvLZt28Ydd9xBbm4ukiSxcePGyx6zZcsWpkyZgl6vZ/To0ZfUuRcAjgs0Or0cbNJxtrF3a+ga5CQqGj2cv2BrK9g3BKixutAEXKRoPEqsKzG24uULBKl2q7G6fErjW78oI9VfYipeDoeD4uJinn/++V6NP336NHPmzOGmm25i3759LF26lMWLF/PBBx9E2NI4w1HPRYeXfRfUVF3onXhZ3QFOOAzU2z1tntsQ4HyTm0TvBUx6jVKAUNN1yki0sLp8vLa7liP1LaIl4l79JqYxr9tuu43bbrut1+NfeOEFRo4cycqVKwG48sor+ec//8lzzz3H7NmzI2Vm/OFswOUN4DSmkm7qunBkZ7LNBg5q00hyNOC1VqPLGBdhIyOP0+vH6vKR67tAUqJmUEwZTS1Z9i4M+INBNJ5mMEavf+RQIq5iXjt37rykK/fs2bPZuXNnt8d4PB5sNluHbUjjdSB7HTh9AVza5G6TUzuTqNcgmbKQAVvD0CiPo5IkZl2ZxeQUr9ItKMZTRgC9Ro1eq2qXLiHiXv0lrsSrpqbmkgYgWVlZ2Gw2XK6uK1OuWLECi8US2goKCqJhauxwNODxBXGqklBpdKQYez9NSkrNUU5xYWiIl0GrZlK+hSsTW6ZmpuzYGtSC2aDFpzbiEVn2AyKuxKs/lJeXY7VaQ9uZM2dibVJkcTTg9AZwalNITdJdUkmiJ5Iz8wFwN18E3xAp1+JzKx2UAEw5sbWlBVP7RFWPSFTtL3GV55WdnU1tbceaU7W1tZjNZhISul54rNfru20YMiRxNuD0+nFpU0hP6tv7zkpL5kt1Ena3A9lei5QyIkJGRh63L8DhahtFqjpSQenRqOu6ska0MRu01AnPa8DEledVVlbG5s2bO+zbtGkTZWVlMbJoEOKox+VTPK/0pL5dWcs06XHp05ABT1N8J6uebXSx9Vg9n+w7qOwYJF4XgDlBg0ediMcfEJ7XAIip52W32zlx4kTo8enTp9m3bx+pqakUFhZSXl7OuXPnePnllwF4+OGH+c1vfsOjjz7KokWL+Oijj3j99dd57733YvUWBheyDI4GRmcmkXVVMRpL3xYga9UqZpROIKG6GclzIUJGRoczLfltI7RNyg5zbuyM6URBqhHd2EKyKg3gGeIXkCJITD2vXbt2UVpaSmlpKQDLli2jtLSU5cuXA1BdXU1VVVVo/MiRI3nvvffYtGkTxcXFrFy5khdffFGkSbTidYDPhSSpMKdmY9T1/bfJmJqnZKLHeUnoMxeVGvGZtJTBGUSeV6bJwKRRBZgNWsXzGkIrGqJJTD2vmTNndtkUopWusudnzpzJ3r17I2hVHNOyphFDslL6pT+05kI5GpRS0qq4iiwA0Oz2ccHuRR90kKL1gqQZNFcaQ+hMSsZ/MKD86OiTYm1R3BF//5mC7nE0cNHhYc8FNUeq+zcdcWvMHKl3s6+ygaAjPqeOp+qVIPhIbWNLfld6/8U8QtTavdR7tfiDQTF17CdCvIYSjnpsbj+nnAnU2vqX6qDTqKn2m3D5AjTVnw2zgdHhVIOS+HmFvknZkVwYO2O64b0vqtlbF8TpDYBbiFd/EOI1lAjleKX2OU2iFZVKQp+sTLGa6s+F07qo4A8EqW6pJJFHyzTaMvgSk80JWrzqRCVdQnhe/UKI11BBlpEd9S05Xslk9HJNY1ckpipX5hwX4y/TXqNW8eANo/jm5FQS/S3VMSz5sTWqC0wGDR51UkuiqhCv/iDEa6jgtePzuPAFwK1LJrWXaxq7IjkjD4jfXC+tWkWhulG5imdMG5TBcJNBg1eThMcnpo39RYjXUKHF63JrzFiMCUqLr36SnlMASPidNrzO+EmiDAbltqvX1pZlYIPQ6wJITtDhVSfiFp5XvxHiNVRwXAitaextGZzuSDIakROSkYGGmvhZC3q42sZLOyo4eM4KTS35gcmDL94FkGzU4lEn4RaeV78R4jVUcNQTCMh49Gn9Dta3JyElB5Neg8oZP4UJj1TbaHT68LjsbbX4U4pialN3WBK0eDRKzCvosUPAH2uT4o64Wpgt6AFHPQWpRvKuLMaffvmmG5ej5MqxSKdrQWUNg3GRx+r0cbZRKYs0Tn9RiXclpoPeFGPLusaoUzN9fAFFwWRlh7dZqfQq6DXC8xoKyHKodLPKlIlOM/A/q9TakMMRH63Q9p9tAqAo3UiSo2XKmDoydgZdBkmSmFqUSlpaBipJAnd8/EgMJoR4DQXcVqWZg0odvl/vlmVC/uYG/D5veM4ZIXyBIIfOK3Gj4jwLXDytPJE6KoZW9RJDsnI7hJqeRAshXkOBlmVBn9XKfFrRFJ5z6s0cveBj1+kGKs9UXX58DDlW04zbF8CcoKUowa0sdlZpBmVyanua3T7OefQ0ubzgaoq1OXGHEK+hgKMeuydAXcBMsztMgV9JQk7MQAYu1A7eK46yLLP3TBMAxfkWVBdbSiwlFw669Yydqbzg5B9n/MqKAHdTrM2JO4R4DQVaqqc6tSkDyqzvjKUlWdVWP3gz7SVJ4qtXZjE2y8RVeRZoOK48kT4mtob1AkuCFrfGrCSqimljnxHiNRRw1LetaQyjeKVmKtOuQHMtTu/gvZSfbTEwZ3IOhoADbOeVUjPxIF5GRbzcviBBZ5Oo69VHhHjFO8EgvuZ6PP4gTm1Kr1ud9QZDSg5GnRqj9wJnL/aueW00CQQ7fdkvfKncmnMHbYpEe0x6DUG9RSm77XaCr+sOWIKuEeIV77ibcLo8BCUNelMaBq06fOc2pmM26tAEPVTXD65k1WBQ5rXPq/j4aJ2SpQ5Q3zplHBs7w/qAJElYTEa8aiMub0DEvfqIEK94p10liXBOGQFQa0hMUfK96s+f6bHqbbQ5XG2jzubhaE3L2ktPMzRVKvfjqNt3WqIej8asTMtF3KtPCPGKd+x1qCQJlSmTTJMh7KdPySwgNVHH1FT3oAnJePwBdpxUanVNG5mqeJu1h5WYkSU/rjLV05J0uDVmXL6ASJfoI4NCvJ5//nmKioowGAxce+21fPbZZ92OXbduHZIkddgMhvB/aeMGey1ZZgM3T53EdaNSw356bUo+47JMjNQ29qmBbSTZceICDk+AZKOWkoJkZWftAeU2+6qY2dUfrshIYsr4URSkGNua4wp6RczF609/+hPLli3j8ccfZ8+ePRQXFzN79mzq6rpflmI2m6murg5tlZWVUbR4kNHa5ceUhSRFQFxaEz1t55SGHDHmXJMrtBTolvFZqFUSNNeCvV5ZYZAxPrYG9pHURB35eYWK99jaQEXQK2IuXs8++ywPPvgg999/PxMmTOCFF17AaDSyZs2abo+RJIns7OzQlpWVFUWLBxFeJ0GXVYlFJUXoM0jMAI0et9vNF8e/bAuOxwCvP8jfD9ciyzAx10xhWksH7PN7lNv0MaDtunP6oCYxXbl1XhDpEn0gpuLl9XrZvXs3s2bNCu1TqVTMmjWLnTt3dnuc3W5nxIgRFBQUMHfuXA4dOtTtWI/Hg81m67ANGew1NNg9/PNcgI9PRmhhrySBJZ9jtc0cOnyIL2vtkXmdXtDo9OL0BjAZNNw4NkPZ6XNBbUtX7LypMbNtIJx16zhr82J3ikz7vhBT8WpoaCAQCFziOWVlZVFT03UJ4nHjxrFmzRreeecdXnnlFYLBINOnT+fs2a473axYsQKLxRLaCgoG93q3PmGvw+EJYNWkKZUJIoUln/QkPSZPDUdqYif+WWYD915XyB3FuW0pIdVfKLWwkjIG/VrG7jh4vpljVi1NTi/Eabu5WBDzaWNfKSsrY/78+ZSUlDBjxgzeeustMjIy+O1vf9vl+PLycqxWa2g7c2bwrtPrM801ODx+HLp0ssxhTpNojyWf9CQdZk815y46sbp8kXutLvAH2mJtJoOWLHPLBZpgAM7tVu7nTVW8xDgkPUmPS5uitEETca9eE1PxSk9PR61WU1vbsbV8bW0t2dm963Cs1WopLS3lxIkTXT6v1+sxm80dtqFCsLkGh9ePQ5tGVgTSJEKYctHr9aTr/Bh9Fzl0Lnq1p+psbtbtqOBUfRfT1ZoDSjkgXSJkxddVxvZkmQ04tcnYPX6lU7mgV8RUvHQ6HVOnTmXz5s2hfcFgkM2bN1NWVtarcwQCAQ4cOEBOTk6kzByc+Ny4rPUEZfAnZpJsjGAFBbUGkkeQZdaT7D7L/rNWpWVXhKmxunljz1ma3X52VTZ2TJINBqByh3K/sGzQV5DoiQyT4nl5/EG8ttrLHyAABsG0cdmyZfz+97/npZde4siRI/zrv/4rDoeD+++/H4D58+dTXl4eGv/kk0/y4YcfcurUKfbs2cO9995LZWUlixcvjtVbiA22czg8Sreg1OSUyKRJtCd1FKlGHTmB87h9AQ5XRzb2daymmTd2n8HjC5KXnMDcktyO7/H83javK7ckorZEGoNWjdasFH90NNYOipSUeCDmNezvvvtu6uvrWb58OTU1NZSUlPD++++HgvhVVVWoVG0a29jYyIMPPkhNTQ0pKSlMnTqVHTt2MGHChFi9hdhgO0+z249dn0O2OQpJuqmjkCSJMYYmDuAjGKFL+h5/gO0nGth/RpmaFqUbmTMpt2Npa48dTm9T7hd9Ja69rlaS07IJVGhxutykOBtClWwF3SPJg2nBWhSw2WxYLBasVmt8x7/2v0Zd5REOJV3HyMk3UpSeGPnX/OQFgs6LOMZ8A1NB+GNMgaDMSzsqQhcEri5K4for0i/N7D/yF6g5CKYsmLIQVDGfQAyY3ZUXafjHGkZqLjJ2xncgZ3KsTYoZvf2Oxv9ffTgiy2A7R6bJwE3XlEZHuAAyxqKSJEy2L0O7LilL00fcvkAolqVWSRSkGrEkaPnmlHxuGJNxqXDVHVWES5JgzOwhIVwA47LNXFs8kdGZSW1t2wQ9EvNpo6AfOBrA71WmS4lRnF5kToCqT6HhBPg91Dll3v2imtsmZZNj6X1mu9sXoOKCg1P1Dk7W2bnr6gKyLcrU9yuj09Gopa47frsa4fjflPsF14IlLxzvalCQpNdARgE07AO7EK/eIMQrHrFW4fD6UafmY4im55GUBcZUcF6EhuN8WpuG1eXjjV1nue6KNIrzk7tsu+YLBKlucnOuycXZRifnm9wdYmanGuwh8UrQdVOPzOeCLzaAzw3mHBh5Y0TeYkwxtVwxt9cqV1NVYazNNgQR4hWPNFZwut7B4WYtJYXNjM2KUtVQSVLyqU5vg3N7mF18H0G5mlP1Dv75ZQOfnrpAtiWBJL2GgtQEJuZaAGhy+nhzT8cVEOlJOkamJzE6M+nyCbY+F3zxurL2T2+Cq745JL/YtX4j1os+DDgptNcqFWEF3SLEK94IBvFdqMDu8WNNzgt5LFEjpxgqt4PtPDpnDd8ozuVwtY3PTl+kyenjTLty0a3ilZaoI92kJz1RR25yAkVpiVh6m5fmaoQDbyhTZa0BJn87Lko89wenL8gJTwo5/jMUNp0R4nUZhHjFG/YabM3N+FVa9Kl5mA1RThPQJ0HmlUrQvHIH0qRvMTHXwoQcMw12L/XNHpxePyntaumrVBL3XTeib68jy1B3GI6/r8T39Ekw+TvKGsYhSo7FwGeGbFIaK3HWn8ZYeG2sTRrUCPGKNxoraXL6sOlzKUxLio0NhWVQewgavoSmM5BcgCRJZJj04Wm9Zj0Hp7dCY0udNkseTJgLBsvAzz2IMWjVJGSMgsZPaao+hVGW43a9ZjQYGteZhxHyhS+xunw0GfIpSotSikRnEtOV6SMonlEgDAu1gwGoPwb7/gh7XlaES6WBkTdAyfeGvHC1kps/goBKi7W5WaRMXAbhecUTHjvO+io8/iDNiUXkpcSw8N7IG5UGr44G+HITjLut715CMAi2s0ruVt0h5UoigKSCrIlQdH1c1aMPByMzTHyiz0PrqsBbdxydeZit2e0DQrziiQtfctHhwaFLJyc7q+tcqGihS4TxX4cDG6B6vyJco7+qLOLuCa9T6fLT8CVcPNkmWKDEtbImQu4USEiOqPmDldREHf7UKwieq6Cx6jBZo2fE2qRBixCveKL+GDkWA4GiqahbG0/EkrQrYOxsOPY+nN+nTPXyr1aKAuqMylTQ61BqVDXXKqLVueSL1gBpo5UE2JSRQyZjvr9IkkTGiIkk1G9F66wFtw0McbyMLYII8YoX3DZorECjUlE0fioYjbG2SCG3FPRmOPquktbw5abLH5OYDqmjlJrz5vxhL1idmTY2H7XzSiTbeag/CgXTYm3SoESIV7xQc0BJH0guULLcBxNpV8C1DyslmS98CfY6JbFUpVEaYhjTlEYelnzFfl2MLjTECRq1CrInge28MiXPv0ZcdewCIV7xQDCI/9w+jpyzok6+kXFBWWn5NZjQ6KHgGmUDRWjFF67/ZE4g8OXfuVBzljTredTJQ2cdZ7gQ/no8UHeYuvpamnwaPrdnMNh0q0uEcA0IWaNnuzWVk/UOzh3cFmtzBiVCvAY7wSCBiu1UN7moNk1i6qgINZcVDCokScIy9gYA6k/uw2sTte07I8RrsHN+D+fPn8Ula3FmFHNljrjyNFyYMHYMbstIvP4Apz97N9bmDDqEeA1mXE04jn3E+SYXVZZpTB+fN/hiXYKIoVWrKLr6NmQkGisPUn1if6xNGlQI8RqsBHz4vniTL89dwKbNxFg0lTGZMVrLKIgZI4tGoRs5HRmo3L6BhtrzsTZp0DAoxOv555+nqKgIg8HAtddey2effdbj+A0bNjB+/HgMBgOTJk3ir3/9a5QsjRI+N3zxJ5wXzmEPajmfP5tbr8oRsa5hyuSvzEGdnI/sd1Pzz3Vgr4+1SYOCmIvXn/70J5YtW8bjjz/Onj17KC4uZvbs2dTV1XU5fseOHdxzzz088MAD7N27l3nz5jFv3jwOHjwYZcsjgCwj1x1F/vxFaDqDxZRE/o33cdvV4zDqRFbLcEWn0zH5a4tITMlkjDkIe9ZB1af4vO7LHjuUiXn3oGuvvZZrrrmG3/zmN4DSdLagoIB/+7d/47HHHrtk/N13343D4eDdd9sCmNdddx0lJSW88MILl329QdM9SJaRfS68bjvuxlrs9ZW4q49ia6xjZFoiyWlZMPFflA45AgEo60IPbwyVCjpQ5+GioZCkrFGY03MwJaeRlGRGr9WgUUlx66n39jsa059zr9fL7t27OzSVValUzJo1i507d3Z5zM6dO1m2bFmHfbNnz2bjxo1djvd4PHg8ntBjm633zVKtTh9/3n+OVnWXZci6+DkW+ylAJsdioCjVCMi4fX72VjWiDJaRkJUDABmZLJOeUelGkGV8gSB7T9ciBwN0br4TUGnZp57IzKu/ARodAkEInRGK74GaL/Ce/CdOx2l09sN4Gw7TALQmUwRUWizGBCYVpCoVOlRqPq1oJCjLSEiXpOCZDFomtLuKvbuyCX83jW+ltCuYNutbocdv7TmLw+PvcqzFqOMbxW3VYH2BYFiLCcRUvBoaGggEAqEGs61kZWVx9OjRLo+pqanpcnxNTde1j1asWMETTzzRL/sCskyD3dthX7K9CRwtMQeNAQzKUhfJHwBXU2jcJe6szgetpwoECQQCyl1Ji0+fjGTKQpt+BekjrmRUVip00chCIECSIKcYXdYkJo05TUPlYZobzuJvrsfvsuMLBFEHfehkCTzNbcc5LoCs/F92/t+U/RpwtP2fy46LBANdT8j8hvQOjxudPmyuruu5XfI6YZ7jDflASnl5eQdPzWazUVBQ0Ktjk/Qavjklv8Mvldo1G8n3FSRJwqjTQoJShlkjwxVX+lpc9XYuuySBJGHQqkGveFJqGcZNUaHWGUkw6LvsuCMQ9IhKhTHrCgqzrmjbFwwi+1143Q7kQAA0klLZQw4y6goXQVkm0H4a0YJOI0FSWwXc/FFu2hyvjoqjSeh4xftrV2UT6EboNOqOLp4mzGk+MRWv9PR01Go1tbW1HfbX1taSnZ3d5THZ2dl9Gq/X69Hr+1eaWKdRUZjWuXpDYZdj1UBmL+vmqYBBtrRaMBRQqZB0iei7WPiekdz702T3of5jXnLvC2Je0kB4gMT0J1+n0zF16lQ2b94c2hcMBtm8eTNlZWVdHlNWVtZhPMCmTZu6HS8QCIYmMZ82Llu2jAULFnD11Vczbdo0fvnLX+JwOLj//vsBmD9/Pnl5eaxYsQKAH/3oR8yYMYOVK1cyZ84cXnvtNXbt2sXvfve7WL4NgUAQZWIuXnfffTf19fUsX76cmpoaSkpKeP/990NB+aqqKlTtitVNnz6d9evX81//9V/89Kc/ZcyYMWzcuJGrrroqVm9BIBDEgJjneUWbQZPnJRAIuqS331FxmUsgEMQlQrwEAkFcIsRLIBDEJTEP2Eeb1hBfX5YJCQSC6NH63bxcOH7YiVdzs7JkordZ9gKBIDY0NzdjsVi6fX7YXW0MBoOcP38ek8kUt6vuB0Lr8qgzZ86Iq63tEJ9L18Tic5FlmebmZnJzczukSXVm2HleKpWK/Pz8WJsRc8xms/iSdoH4XLom2p9LTx5XKyJgLxAI4hIhXgKBIC4R4jXM0Ov1PP744/2utDFUEZ9L1wzmz2XYBewFAsHQQHheAoEgLhHiJRAI4hIhXgKBIC4R4iUQCOISIV5DENGBvGv68rmsW7cOSZI6bAaDIYrWRp5t27Zxxx13kJubiyRJ3bYPbM+WLVuYMmUKer2e0aNHs27duojb2R1CvIYYogN51/T1cwElq7y6ujq0VVZWRtHiyONwOCguLub555/v1fjTp08zZ84cbrrpJvbt28fSpUtZvHgxH3zwQYQt7QZZMKSYNm2avGTJktDjQCAg5+bmyitWrOhy/Le//W15zpw5HfZde+218kMPPRRRO6NNXz+XtWvXyhaLJUrWxR5Afvvtt3sc8+ijj8oTJ07ssO/uu++WZ8+eHUHLukd4XkOI1g7ks2bNCu3rTQfy9uNB6UDe3fh4pD+fC4DdbmfEiBEUFBQwd+5cDh06FA1zBy2D7X9FiNcQoqcO5N11FO9rB/J4pD+fy7hx41izZg3vvPMOr7zyCsFgkOnTp3P27NlomDwo6e5/xWaz4XK5om7PsKsqIRD0hrKysg69QKdPn86VV17Jb3/7W5566qkYWiZoRXheQ4hodCCPR/rzuXRGq9VSWlrKiRMnImFiXNDd/4rZbCYhofeds8OFEK8hhOhA3jX9+Vw6EwgEOHDgADk5OZEyc9Az6P5XYnKZQBAxXnvtNVmv18vr1q2TDx8+LH//+9+Xk5OT5ZqaGlmWZfm+++6TH3vssdD47du3yxqNRv7FL34hHzlyRH788cdlrVYrHzhwIFZvISL09XN54okn5A8++EA+efKkvHv3bvk73/mObDAY5EOHDsXqLYSd5uZmee/evfLevXtlQH722WflvXv3ypWVlbIsy/Jjjz0m33fffaHxp06dko1Go/zjH/9YPnLkiPz888/LarVafv/992NivxCvIcivf/1rubCwUNbpdPK0adPkTz75JPTcjBkz5AULFnQY//rrr8tjx46VdTqdPHHiRPm9996LssXRoS+fy9KlS0Njs7Ky5Ntvv13es2dPDKyOHB9//LEMXLK1fg4LFiyQZ8yYcckxJSUlsk6nk0eNGiWvXbs26na3IkriCASCuETEvAQCQVwixEsgEMQlQrwEAkFcIsRLIBDEJUK8BAJBXCLESyAQxCVCvAQCQVwixEsw5Fm3bh3Jycmhxz/72c8oKSkJPV64cCHz5s2Lul2CgSHESxA2Fi5ciCRJPPzww5c8t2TJEiRJYuHChR3Gh1s0ioqK+OUvf9lh3913383x48e7PWbVqlUdyhnPnDmTpUuXhtUuQfgR4iUIKwUFBbz22msd6ju53W7Wr19PYWFhTGxKSEggMzOz2+ctFksHz0wQHwjxEoSVKVOmUFBQwFtvvRXa99Zbb1FYWEhpaemAzt2VRzRv3ryQNzdz5kwqKyv593//91DTDLh02tiZ9h7gwoUL2bp1K6tWrQqd4/Tp04wePZpf/OIXHY7bt28fkiQN6zI5sUSIlyDsLFq0iLVr14Yer1mzhvvvvz/ir/vWW2+Rn5/Pk08+GWqa0VdWrVpFWVkZDz74YOgchYWFl7wngLVr13LjjTcyevTocL0FQR8Q4iUIO/feey///Oc/qayspLKyku3bt3PvvfdG/HVTU1NRq9WYTCays7P7VVDRYrGg0+kwGo2hc6jVahYuXMixY8dC7dJ8Ph/r169n0aJF4X4bgl4iykALwk5GRgZz5sxh3bp1yLLMnDlzSE9Pj7VZAyI3N5c5c+awZs0apk2bxl/+8hc8Hg933XVXrE0btgjPSxARFi1axLp163jppZfC5p2oVCo6V3Dy+XxhOXdvWLx4cehixNq1a7n77rsxGo1Re31BR4R4CSLC1772NbxeLz6fj9mzZ4flnBkZGR3iWIFA4JLmuDqdjkAgMKDX6e4ct99+O4mJiaxevZr3339fTBljjJg2CiKCWq3myJEjofvdYbVa2bdvX4d9aWlpFBQUXDL25ptvZtmyZbz33ntcccUVPPvsszQ1NXUYU1RUxLZt2/jOd76DXq/v13S1qKiITz/9lIqKCpKSkkhNTUWlUoViX+Xl5YwZM2ZI1fmPR4TnJYgYZrMZs9nc45gtW7ZQWlraYXviiSe6HLto0SIWLFjA/PnzmTFjBqNGjeKmm27qMObJJ5+koqKCK664goyMjH7Z/cgjj6BWq5kwYQIZGRlUVVWFnnvggQfwer1RuXoq6BlRBlog6AP/+Mc/uOWWWzhz5swlDVgF0UWIl0DQCzweD/X19SxYsIDs7GxeffXVWJs07BHTRoGgF/zxj39kxIgRNDU18T//8z+xNkeA8LwEAkGcIjwvgUAQlwjxEggEcYkQL4FAEJcI8RIIBHGJEC+BQBCXCPESCARxiRAvgUAQlwjxEggEcYkQL4FAEJf8f/XmbGAWpzDIAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:10:49.281017Z", + "iopub.status.busy": "2024-03-22T19:10:49.280683Z", + "iopub.status.idle": "2024-03-22T19:10:49.490180Z", + "shell.execute_reply": "2024-03-22T19:10:49.489281Z" + }, + "papermill": { + "duration": 0.232063, + "end_time": "2024-03-22T19:10:49.492171", + "exception": false, + "start_time": "2024-03-22T19:10:49.260108", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiVElEQVR4nO3de1xUdf4/8BfDZRC5eOEqoXgp0byQ8MPFdMviUq2urlv6VRNE13ZVNoulzN2CNR+FbUJY+ZVHJrqapmuZ3UxFEtcUU/GyaIorSphcFI17DgPz+f3hl6lxZmBQhpnPzOv5ePjAOedzzrzPHHidcz4z8zkOQggBIiLJKCxdABHRnWB4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRScnJ0gV0NY1Gg7KyMnh4eMDBwcHS5RDRbYQQqKurQ58+faBQGD+/srvwKisrQ1BQkKXLIKJ2XL58Gffcc4/R+XYXXh4eHgBuvTCenp4WrsZ81Go19uzZg5iYGDg7O1u6HOoE9rJPa2trERQUpP1bNcbuwqv1UtHT09Pmw8vNzQ2enp42/YtuT+xtn7bXrcMOeyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikZHcflbBVjY2NOHfunPZx/U8qHCosRk/vY3DvptRpGxISAjc3t64ukahTMbxsxLlz5xAWFqY3/R8G2hYUFGDUqFHmL4rIjBheNiIkJAQFBQXax0Xl1UjaVoiMp4ZjcEAPvbZEsmN42Qg3NzedsynF99ehPPAThgwbidB+vS1YGZF5sMOeiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxJFUiazQ7TdUAYzfVMVeb6jC8CKyQsZuqALo31TFXm+owvAiskK331AFMH5TFXu9oQrDi8gK3X5DFYA3VbkdO+yJSEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKRk8fBatWoVgoOD4erqitGjR+PIkSNttq+ursbChQsREBAApVKJ++67Dzt37uyiaonIWlh0MMKtW7ciKSkJWVlZGD16NDIzMxEbG4uioiL4+vrqtW9qakJ0dDR8fX3x0UcfITAwEN9//z169OjR9cUTkUVZNLwyMjIwb948JCQkAACysrLw5ZdfIjs7Gy+99JJe++zsbNy4cQOHDh2Cs7MzACA4OLgrSyYym0tVDWhQNRudX3ytQfvTycnwn253pRP6e3c3S33WxmLh1dTUhIKCAixZskQ7TaFQICoqCvn5+QaX+eyzzxAZGYmFCxfi008/hY+PD2bMmIHFixfD0dHR4DIqlQoqlUr7uLa2FgCgVquhVqs7cYusS3Nzs/anLW+nrSi53oDozIMmtf3LR4Vtzs957kEE95Y3wEz9fbVYeFVVVaGlpQV+fn460/38/PRu+dTq4sWL+PrrrzFz5kzs3LkTFy5cwIIFC6BWq5GammpwmbS0NCxdulRv+p49e2z6dlGX6wHACYcPH8aV05auhtrTur9mDWqBXzdhsI1aA9xQAb2UgLOB3urKnxyw8YIjdufuR5C7Wcs1q8bGRpPaSXUDDo1GA19fX7z33ntwdHREWFgYrly5gjfffNNoeC1ZsgRJSUnax7W1tQgKCkJMTAw8PT27qnSzKLnegAZVi8F5qooaoPAsfAcNRz9/L6Pr6K50lPoobSvOlNViReFh/D76Qdzfx/DvpVqtRk5ODqKjo7XdJrevY+OFwxg7dqzRdcig9eqoPRYLL29vbzg6OqKyslJnemVlJfz9/Q0uExAQAGdnZ51LxCFDhqCiogJNTU1wcXHRW0apVEKpVOpNd3Z2NvgLIItLVaZdZrz4ydl22+xLfthu+kmsVWsflpOTU7u/l8Z+dzuyDmtmau0WCy8XFxeEhYUhNzcXkydPBnDrzCo3NxeJiYkGl3nwwQexefNmaDQaKBS3zpvPnz+PgIAAg8Fly1o7djOnhWKQr/41QsNPKnyRl48JD0eiezf98AaAC1fr8dzWk212EhNZK4teNiYlJSE+Ph7h4eGIiIhAZmYmGhoatO8+xsXFITAwEGlpaQCA+fPn491338WiRYvw5z//Gf/973/x+uuv49lnn7XkZljUIF93DAvUvyxUq9Wo8AFG9esp9VGYyBiLhte0adNw7do1pKSkoKKiAqGhodi1a5e2E7+0tFR7hgUAQUFB2L17N55//nmMGDECgYGBWLRoERYvXmypTSAiC7F4h31iYqLRy8S8vDy9aZGRkTh8+LCZqyIia2fxrwcREd0JhhcRSYnhRURSYngRkZQs3mFPRICq5SYUrldwqbYIClfD3+1pbm5GWXMZzt44a/CL2Zdq66FwvQJVy00Axr9VYSsYXkRWoKzhe3Tv/w7+2vZwdgCA/931v0bnde8PlDWEIgx+RtvYCoYXkRXo070fGi79GSunhWKggW9MALfOvA5+cxAPjn3Q4JlX8dV6LNp6En3G9zN3uVaB4UVkBZSOrtDcDER/z8EY2tvwJZ9arcYlp0sY0muIwW9NaG7WQHPzGpSOruYu1yqww56IpMQzL0m118HbXucuYH8dvGRbGF6SMrWDt63OXcC+OnjJtjC8JNVeB297nbuA/XXwkm1heEmqvQ7e9jp3Afvr4CXbwg57IpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIil1OLwuXrxojjqIiDqkw+E1aNAgjB8/Hh988AFu3rxpjpqIiNrV4fA6fvw4RowYgaSkJPj7++OPf/wjjhwx4a4BRESdqMPhFRoaipUrV6KsrAzZ2dkoLy/H2LFjMWzYMGRkZODatWvmqJOISMcdd9g7OTlhypQp2LZtG9544w1cuHABycnJCAoKQlxcHMrLyzuzTiIiHXccXseOHcOCBQsQEBCAjIwMJCcno7i4GDk5OSgrK8OkSZM6s04iIh0dHkk1IyMD69atQ1FREZ544gls2LABTzzxBBSKWznYv39/rF+/HsHBwZ1dKxGRVofDa/Xq1ZgzZw5mz56NgIAAg218fX2xdu3auy6OiMiYDodXTk4O+vbtqz3TaiWEwOXLl9G3b1+4uLggPj6+04okIrpdh/u8Bg4ciKqqKr3pN27cQP/+/TulKCKi9nQ4vIQQBqfX19fD1ZV3oSGirmHyZWNSUhIAwMHBASkpKXBzc9POa2lpwbfffovQ0NBOL5CIyBCTw+vEiRMAbp15FRYWwsXFRTvPxcUFI0eORHJycudXSERkgMnhtW/fPgBAQkICVq5cCU9PT7MVRUTUng6/27hu3Tpz1EFE1CEmhdeUKVOwfv16eHp6YsqUKW223b59e6cURkTUFpPCy8vLCw4ODtr/ExFZmknh9ctLRV42EpE14DDQRCQlk868HnjgAe1lY3uOHz9+VwUREZnCpPCaPHmyWYtYtWoV3nzzTVRUVGDkyJF45513EBER0e5yW7ZswfTp0zFp0iTs2LHDrDUSkXUxKbxSU1PNVsDWrVuRlJSErKwsjB49GpmZmYiNjUVRURF8fX2NLldSUoLk5GSMGzfObLURkfWyeJ9XRkYG5s2bh4SEBAwdOhRZWVlwc3NDdna20WVaWlowc+ZMLF26FAMGDOjCaonIWpgUXr169dKOJNGzZ0/06tXL6L+OaGpqQkFBAaKion4uSKFAVFQU8vPzjS736quvwtfXF3Pnzu3Q8xGR7TDpsvGtt96Ch4eH9v+mdt63p6qqCi0tLfDz89OZ7ufnh3Pnzhlc5ptvvsHatWtx8uRJk55DpVJBpVJpH9fW1gIA1Go11Gr1nRVuBZqbm7U/DW1H67S2trG9dVDXMWVftLdPbWV/mlq7SeH1y4EFZ8+efUcFdYa6ujrMmjULa9asgbe3t0nLpKWlYenSpXrT9+zZozMyhmwu1wOAE7755ht87268XU5Ozl2vg8yvI/vC2D61lf3Z2NhoUrsOf7fR0dER5eXlep3p169fh6+vL1paWkxel7e3NxwdHVFZWakzvbKyEv7+/nrti4uLUVJSgokTJ2qnaTQaALfuZlRUVISBAwfqLLNkyRLtcD7ArTOvoKAgxMTESP3l8jNltVhReBhjx47F/X30t0OtViMnJwfR0dFwdna+o3VQ1zFlX7S3T21lf7ZeHbWnw+FlbDBClUqlM0yOKVxcXBAWFobc3FztxzE0Gg1yc3ORmJio1z4kJASFhYU6015++WXU1dVh5cqVCAoK0ltGqVRCqVTqTXd2djb6Ry0DJycn7c+2tqOt7TR1HWR+HdkXxvaprexPU2s3ObzefvttALcGI3z//ffh7v7zeWlLSwv+/e9/IyQkpINl3hrkMD4+HuHh4YiIiEBmZiYaGhqQkJAAAIiLi0NgYCDS0tLg6uqKYcOG6Szfo0cPANCbTkS2zeTweuuttwDcOvPKysqCo6Ojdp6LiwuCg4ORlZXV4QKmTZuGa9euISUlBRUVFQgNDcWuXbu0nfilpaV6N/sgIjI5vC5dugQAGD9+PLZv346ePXt2WhGJiYkGLxMBIC8vr81l169f32l1EJE8Otzn1TqiKhGRJXU4vObMmdPm/LY+GU9E1Fk6HF4//vijzmO1Wo3Tp0+juroajzzySKcVRkTUlg6H1yeffKI3TaPRYP78+XqfsSIiMpdOeRtPoVAgKSlJ+44kEZG5ddpnEIqLi7XfrSIiMrcOXzb+8qs2wK3PfZWXl+PLL7/U+Q4kEZE5dTi8Wu+c3UqhUMDHxwfp6entvhNJRNRZ+DkvIpISv3dDRFJieBGRlBheRCQlhhcRSanTwuuHH37AM88801mrIyJqU6eF1/Xr17F27drOWh0RUZt42UhEUmJ4EZGUGF5EJCWTP2E/ZcqUNudXV1ffbS1ERCYzOby8vLzanR8XF3fXBRERmcLk8Fq3bp056yAi6hD2eRGRlEw+8zJ1uBvegIOIuoLJ4bV+/Xr069cPDzzwAIQQ5qyJiKhdJofX/Pnz8eGHH+LSpUtISEjA008/jV69epmzNiIio0zu81q1ahXKy8vx4osv4vPPP0dQUBCmTp2K3bt380yMiLpchzrslUolpk+fjpycHHz33Xe4//77sWDBAgQHB6O+vt5cNRIR6bnjdxsVCgUcHBwghEBLS0tn1kRE1K4OhZdKpcKHH36I6Oho3HfffSgsLMS7776L0tJSuLu7m6tGIiI9JnfYL1iwAFu2bEFQUBDmzJmDDz/8EN7e3uasjYjIKJPDKysrC3379sWAAQOwf/9+7N+/32C77du3d1pxRPbiJ/WtrpfTV2qMtmn4SYVj1wD/739E925KvfkXrtpXv7PJ4RUXFwcHBwdz1kJkt4r/L3he2l7YTksnbLxwtM0W3ZUdvqOhlDr0IVWyHu0dqds7SgP2d6S2ZjH3+wMABvq6o5uzo8E2ReU1+MtHhUh/cjgGBxgeKKG70gn9vbubrU5rYh8RbYNMO1K3f5QG7OdIbc16dXfB/0T0bbNNc3MzAGCgT3cMC2x7lBd7wN9aSbV3pDblKA3Y15GabAvDS1LtHal5lCZbxyFxiEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEpWEV6rVq1CcHAwXF1dMXr0aBw5csRo2zVr1mDcuHHo2bMnevbsiaioqDbbE5Ftsnh4bd26FUlJSUhNTcXx48cxcuRIxMbG4urVqwbb5+XlYfr06di3bx/y8/MRFBSEmJgYXLlypYsrJyJLsnh4ZWRkYN68eUhISMDQoUORlZUFNzc3ozev3bRpExYsWIDQ0FCEhITg/fffh0ajQW5ubhdXTkSWZNEvZjc1NaGgoABLlizRTlMoFIiKikJ+fr5J62hsbIRarTZ6D0mVSgWVSqV9XFtbCwBQq9VQq9V3Ub11a/1idnNzs01vpz2xl31q6rZZNLyqqqrQ0tICPz8/nel+fn44d+6cSetYvHgx+vTpg6ioKIPz09LSsHTpUr3pe/bsgZubW8eLlsTlegBwwuHDh3HltKWroc5gL/u0sbHRpHZSD4mzfPlybNmyBXl5eXB1dTXYZsmSJUhKStI+rq2t1faTeXp6dlWpXe5U6Q2g8Bh+9atfYWRf3tncFtjLPm29OmqPRcPL29sbjo6OqKys1JleWVkJf3//NpddsWIFli9fjr1792LEiBFG2ymVSiiV+sMgOzs7w9nZ+c4Kl4CTk5P2py1vpz2xl31q6rZZtMPexcUFYWFhOp3trZ3vkZGRRpf7xz/+gWXLlmHXrl0IDw/vilKJyMpY/LIxKSkJ8fHxCA8PR0REBDIzM9HQ0ICEhAQAt+5aFBgYiLS0NADAG2+8gZSUFGzevBnBwcGoqKgAALi7u/PGt0R2xOLhNW3aNFy7dg0pKSmoqKhAaGgodu3ape3ELy0thULx8wni6tWr0dTUhCeffFJnPampqfj73//elaUTkQVZPLwAIDExEYmJiQbn5eXl6TwuKSkxf0FEZPUs/iFVIqI7wfAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSklWE16pVqxAcHAxXV1eMHj0aR44cabP9tm3bEBISAldXVwwfPhw7d+7sokqJyFpYPLy2bt2KpKQkpKam4vjx4xg5ciRiY2Nx9epVg+0PHTqE6dOnY+7cuThx4gQmT56MyZMn4/Tp011cORFZksXDKyMjA/PmzUNCQgKGDh2KrKwsuLm5ITs722D7lStX4rHHHsMLL7yAIUOGYNmyZRg1ahTefffdLq6ciCzJouHV1NSEgoICREVFaacpFApERUUhPz/f4DL5+fk67QEgNjbWaHsisk1OlnzyqqoqtLS0wM/PT2e6n58fzp07Z3CZiooKg+0rKioMtlepVFCpVNrHtbW1AAC1Wg21Wn035VuVxsZGFBUVaR+fL6+BquICTp90QVOll07bwYMHw83NratLpA64fX8Cxvepre1PU/8uLRpeXSEtLQ1Lly7Vm75nzx6b2uHFxcX4y1/+ojd91j/126anp2PgwIFdUBXdKWP7E9Dfp7a2PxsbG01qZ9Hw8vb2hqOjIyorK3WmV1ZWwt/f3+Ay/v7+HWq/ZMkSJCUlaR/X1tYiKCgIMTEx8PT0vMstsB6NjY0YO3as9nH9TyrsPnAUseP+H9y7KXXa2tqR2hbdvj8B4/vU1vZn69VReywaXi4uLggLC0Nubi4mT54MANBoNMjNzUViYqLBZSIjI5Gbm4vnnntOOy0nJweRkZEG2yuVSiiVSr3pzs7OcHZ2vuttsBZeXl6IiIjQPlar1airvoFxY35lU9tpL27fn4D97FNTt83il41JSUmIj49HeHg4IiIikJmZiYaGBiQkJAAA4uLiEBgYiLS0NADAokWL8NBDDyE9PR2/+c1vsGXLFhw7dgzvvfeeJTeDiLqYxcNr2rRpuHbtGlJSUlBRUYHQ0FDs2rVL2ylfWloKheLnN0XHjBmDzZs34+WXX8Zf//pX3HvvvdixYweGDRtmqU0gIgtwEEIISxfRlWpra+Hl5YWamhqb6vO6nVqtxs6dO/HEE0/Y9CWGPbGXfWrq36jFP6RKRHQnGF5EJCWGFxFJieFFRFKy+LuNXa31/QlTPwgnK7VajcbGRtTW1tp05649sZd92vq32d57iXYXXnV1dQCAoKAgC1dCRG2pq6uDl5eX0fl291EJjUaDsrIyeHh4wMHBwdLlmE3r16AuX75s0x8JsSf2sk+FEKirq0OfPn10PuN5O7s781IoFLjnnnssXUaX8fT0tOlfdHtkD/u0rTOuVuywJyIpMbyISEoMLxulVCqRmppqcEQNkhP3qS6767AnItvAMy8ikhLDi4ikxPAiIikxvCzs4Ycf1hnSmohMw/CSRF5eHhwcHFBdXW3pUqgDrO3gZG313A2GF5GVa2pqsnQJVonhZUVUKhUWL16MoKAgKJVKDBo0CGvXrkVJSQnGjx8PAOjZsyccHBwwe/bsdtdXV1eHmTNnonv37ggICMBbb72ld+TduHEjwsPD4eHhAX9/f8yYMQNXr17Vzm8948vNzUV4eDjc3NwwZswYvRuikr7Zs2dj//79WLlyJRwcHODg4IDi4mLMnTsX/fv3R7du3TB48GCsXLlSb7nJkyfjtddeQ58+fTB48GAAwKFDhxAaGgpXV1eEh4djx44dcHBwwMmTJ7XLnj59Go8//jjc3d3h5+eHWbNmoaqqymg9JSUlXfVydD5BFvXQQw+JRYsWCSGEmDp1qggKChLbt28XxcXFYu/evWLLli2iublZfPzxxwKAKCoqEuXl5aK6urrddf/hD38Q/fr1E3v37hWFhYXid7/7nfDw8NA+nxBCrF27VuzcuVMUFxeL/Px8ERkZKR5//HHt/H379gkAYvTo0SIvL0+cOXNGjBs3TowZM6azXwqbU11dLSIjI8W8efNEeXm5KC8vFzdv3hQpKSni6NGj4uLFi+KDDz4Qbm5uYuvWrdrl4uPjhbu7u5g1a5Y4ffq0OH36tKipqRG9evUSTz/9tDhz5ozYuXOnuO+++wQAceLECSGEED/++KPw8fERS5YsEWfPnhXHjx8X0dHRYvz48UbraW5utsRL0ykYXhbWGl5FRUUCgMjJyTHYrjVEfvzxR5PWW1tbK5ydncW2bdu006qrq4Wbm5tOeN3u6NGjAoCoq6vTed69e/dq23z55ZcCgPjpp59MqsWe/fLgZMzChQvF73//e+3j+Ph44efnJ1QqlXba6tWrRe/evXVe8zVr1uiE17Jly0RMTIzOui9fvqw96Jlajyx42WglTp48CUdHRzz00EOdsr6LFy9CrVbr3LjUy8tLewnSqqCgABMnTkTfvn3h4eGhff7S0lKddiNGjND+PyAgAAB0Li/JdKtWrUJYWBh8fHzg7u6O9957T+/1Hj58OFxcXLSPi4qKMGLECLi6umqn3X5T2lOnTmHfvn1wd3fX/gsJCQEAFBcXm3GLLMPuhsSxVt26devy52xoaEBsbCxiY2OxadMm+Pj4oLS0FLGxsXqdxL8cubN1HDSNRtOl9dqCLVu2IDk5Genp6YiMjISHhwfefPNNfPvttzrtunfv3uF119fXY+LEiXjjjTf05rUecGwJw8tKDB8+HBqNBvv370dUVJTe/NajcEtLi0nrGzBgAJydnXH06FH07dsXAFBTU4Pz58/j17/+NQDg3LlzuH79OpYvX64dWfbYsWOdsTn0f1xcXHT22cGDBzFmzBgsWLBAO82Us6LBgwfjgw8+gEql0n4x++jRozptRo0ahY8//hjBwcFwcjL8p317PTLjZaOVCA4ORnx8PObMmYMdO3bg0qVLyMvLw7/+9S8AQL9+/eDg4IAvvvgC165dQ319fZvr8/DwQHx8PF544QXs27cPZ86cwdy5c6FQKLRnTn379oWLiwveeecdXLx4EZ999hmWLVtm9m21J8HBwfj2229RUlKCqqoq3HvvvTh27Bh2796N8+fP45VXXtELIUNmzJgBjUaDZ555BmfPnsXu3buxYsUKAD+fCS9cuBA3btzA9OnTcfToURQXF2P37t1ISEjQBtbt9ch89szwsiKrV6/Gk08+iQULFiAkJATz5s1DQ0MDACAwMBBLly7FSy+9BD8/PyQmJra7voyMDERGRmLChAmIiorCgw8+iCFDhmj7TXx8fLB+/Xps27YNQ4cOxfLly7V/ENQ5kpOT4ejoiKFDh8LHxwexsbGYMmUKpk2bhtGjR+P69es6Z2HGeHp64vPPP8fJkycRGhqKv/3tb0hJSQEA7f7s06cPDh48iJaWFsTExGD48OF47rnn0KNHD+1wyrfXc3tfm0w4JI4daWhoQGBgINLT0zF37lxLl0N3adOmTUhISEBNTY1F+kwtjX1eNuzEiRM4d+4cIiIiUFNTg1dffRUAMGnSJAtXRndiw4YNGDBgAAIDA3Hq1CksXrwYU6dOtcvgAhhe0iotLcXQoUONzv/uu+8AACtWrEBRURFcXFwQFhaGAwcOwNvbu6vKpE5UUVGBlJQUVFRUICAgAE899RRee+01S5dlMbxslFRzc3ObX+1o6x0nIlvA8CIiKfHdRiKSEsOLiKTE8CIiKTG8iEhKDC8yq9mzZ2sHvnN2doafnx+io6ORnZ3doa+mrF+/Hj169DBfoUa0DgxI1ofhRWb32GOPoby8HCUlJfjqq68wfvx4LFq0CBMmTEBzc7OlyyNZWXIwMbJ98fHxYtKkSXrTc3NzBQCxZs0aIYQQ6enpYtiwYcLNzU3cc889Yv78+XoDIv7yX2pqqhBCiA0bNoiwsDDh7u4u/Pz8xPTp00VlZaX2eW7cuCFmzJghvL29haurqxg0aJDIzs7Wzi8tLRVPPfWU8PLyEj179hS//e1vxaVLl4QQQqSmpuo97759+8zyOlHH8cyLLOKRRx7ByJEjsX37dgCAQqHA22+/jTNnzuCf//wnvv76a7z44osAgDFjxiAzMxOenp4oLy9HeXk5kpOTAQBqtRrLli3DqVOnsGPHDpSUlOiM7//KK6/gu+++w1dffYWzZ89i9erV2m8YqNVqxMbGwsPDAwcOHMDBgwfh7u6Oxx57DE1NTUhOTsbUqVO1Z47l5eUYM2ZM175QZJyl05Nsm7EzLyGEmDZtmhgyZIjBedu2bRO9e/fWPl63bp3w8vJq9/luH8Z64sSJIiEhwWDbjRs3isGDBwuNRqOdplKpRLdu3cTu3bvbrZ8si2deZDFCCO1YVHv37sWjjz6KwMBAeHh4YNasWbh+/ToaGxvbXEd7w1jPnz8fW7ZsQWhoKF588UUcOnRIu+ypU6dw4cIFeHh4aIdN7tWrF27evGmTwybbGoYXWczZs2fRv39/lJSUYMKECRgxYgQ+/vhjFBQUYNWqVQDavmdh6zDWnp6e2LRpE44ePYpPPvlEZ7nHH38c33//PZ5//nmUlZXh0Ucf1V5y1tfXIywsDCdPntT5d/78ecyYMcPMW093i9/cJYv4+uuvUVhYiOeffx4FBQXQaDRIT0/XDprXOoJsK0PDF5s6jLWPjw/i4+MRHx+PcePG4YUXXsCKFSswatQobN26Fb6+vvD09DRYpy0Nm2xreOZFZqdSqVBRUYErV67g+PHjeP311zFp0iRMmDABcXFxGDRoENRqtXY46o0bNyIrK0tnHcHBwaivr0dubi6qqqrQ2Nho0jDWKSkp+PTTT3HhwgWcOXMGX3zxBYYMGQIAmDlzJry9vTFp0iQcOHBAO/T2s88+ix9++EH7vP/5z39QVFSEqqoqqNXqrnnRqH2W7nQj2xYfH6/9mIGTk5Pw8fERUVFRIjs7W7S0tGjbZWRkiICAANGtWzcRGxsrNmzYoHefyj/96U+id+/eOh+V2Lx5swgODhZKpVJERkaKzz77TO9ehkOGDBHdunUTvXr1EpMmTRIXL17UrrO8vFzExcUJb29voVQqxYABA8S8efNETU2NEEKIq1eviujoaOHu7s6PSlgZDolDRFLiZSMRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCSl/w9LXnAVqq2zjAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:10:49.532570Z", + "iopub.status.busy": "2024-03-22T19:10:49.531730Z", + "iopub.status.idle": "2024-03-22T19:10:49.808277Z", + "shell.execute_reply": "2024-03-22T19:10:49.807332Z" + }, + "papermill": { + "duration": 0.2992, + "end_time": "2024-03-22T19:10:49.810292", + "exception": false, + "start_time": "2024-03-22T19:10:49.511092", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAEmCAYAAAAKtqdLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUdklEQVR4nO3dd3gU1dfA8e+2VNIoaRAIvXdIDIgUgzRp6gsC0kEREX/GQpEiIEUpooggIIKCAioKSlGI9E5ooQskBIE0QnrbMu8fSzZZUtgNSTab3M/z7MNm5s7uWZKczNy59x6ZJEkSgiAIVkpu6QAEQRCehkhigiBYNZHEBEGwaiKJCYJg1UQSEwTBqokkJgiCVRNJTBAEqyaSmCAIVk1p6QBKmk6n4969ezg5OSGTySwdjiAIj5EkiaSkJLy9vZHLn3yeVe6S2L179/Dx8bF0GIIgPMGdO3eoVq3aE9uVuyTm5OQE6P+DnJ2dLRyNIAiPS0xMxMfHx/C7+iTlLollXUI6OzuLJCYIpZip3T2iY18QBKsmkpggCFZNJDFBEKxauesTM4UkSWg0GrRaraVDEUophUKBUqkUw3RKAZHEHpOZmcn9+/dJTU21dChCKefg4ICXlxc2NjaWDqVcE0ksB51OR1hYGAqFAm9vb2xsbMRfWiEXSZLIzMwkJiaGsLAw6tata9KgzPJMo9WRrtFRwbboU45IYjlkZmai0+nw8fHBwcHB0uEIpZi9vT0qlYrbt2+TmZmJnZ2dpUMq1RbsusqB6zGsHtYG38qORfraIonlQfxVFUwhfk5Ms/XMf6w5HAbAlfuJRZ7ExHdBEIRic+G/eCZvDQVgQuc69GjqVeTvIZKYIAjFIjopnTd+CCFToyOwoTtBXesVy/uIJFZGdOrUif/973+WDqPM2L9/PzKZjPj4eEuHYpUyNTrGbzjD/YR0aldx5POBLZDLi+cmmUhi5Yz45RRKwsztlzh9+yFOdkpWD2uDk52q2N5LJDGhzMrMzLR0COXShuO3+elkBDIZfPlqS2pVqVCs7yeS2BNIkkRqpsYij8IWZ8/IyGDSpEn4+Phga2tLnTp1+PbbbwkPD6dz584AuLm5IZPJGDFixBNfLykpiSFDhuDo6IiXlxeff/55rsvXH374gTZt2uDk5ISnpyeDBw8mOjrasD/rDDA4OJg2bdrg4OBAu3btuHbtmsmf65NPPsHd3R0nJyfGjBnD5MmTadGihWH/iBEj6NevH3PnzsXb25v69eubFBvAzp07qVevHvb29nTu3Jnw8HCT4xKynbj1gI+3XwLgw24N6NzAvdjfUwyxeII0tZZGM/6yyHtfnt0NBxvzv0XDhg3j2LFjfPnllzRv3pywsDBiY2Px8fHh119/5eWXX+batWs4Oztjb2//xNcLCgriyJEjbN++HQ8PD2bMmMGZM2eMEoharWbOnDnUr1+f6OhogoKCGDFiBDt37jR6rY8++ojFixdTpUoVxo0bx6hRozhy5MgTY9i4cSNz587l66+/pn379mzatInFixdTs2ZNo3bBwcE4OzuzZ88ek2O7c+cOL730Em+99Ravv/46p0+f5r333ntiTIKxu/FpjN94Bo1Oondzb8Z1rFUi7yuSWBlz/fp1tmzZwp49ewgMDASgVq3sH6aKFSsC4O7ujqur6xNfLykpifXr1/Pjjz/y/PPPA/Ddd9/h7e1t1G7UqFGG57Vq1eLLL7+kbdu2JCcnU6FC9uXE3Llz6dixIwCTJ0+mV69epKenP3Gw6LJlyxg9ejQjR44EYMaMGfz9998kJycbtXN0dGTNmjVGU4GeFNuKFSuoXbs2ixcvBqB+/fqEhoby6aefPvH/R9BLy9Tyxg+neZCSSSMvZz57uVmJzXYRSewJ7FUKLs/uZrH3Nte5c+dQKBSGRPG0bt26hVqtxs/Pz7DNxcXFcKmWJSQkhI8//pjz58/z8OFDdDodABERETRq1MjQrlmzZobnXl76MUPR0dFUr169wDiuXbvG+PHjjbb5+fnxzz//GG1r2rRprrmMT4rtypUr+Pv7Gx0TEBBQYDxCNkmSmLz1AhfvJlLR0YZVw1pjb2P+z25hWbRP7ODBg/Tu3Rtvb29kMhm///57ge23bt1K165dqVKlCs7OzgQEBPDXX8V7qSeTyXCwUVrkUZi/ZKZcHha1lJQUunXrhrOzMxs3buTUqVP89ttvQO7OdZUq+y5V1ufLSipFwdHReDS4ObEJhbPq4C22nbuHUi7j6yGtqOZWslP2LJrEUlJSaN68OcuXLzep/cGDB+natSs7d+4kJCSEzp0707t3b86ePVvMkVqPpk2botPpOHDgQJ77s85STF1mqFatWqhUKk6dOmXYlpCQwPXr1w1fX716lQcPHrBgwQI6dOhAgwYNcnWcP6369esbxQDk+jovpsTWsGFDTp48abTt+PHjTx90OXDgegyf7r4KwIzejXimVqW8G6bGQXpCscRg0cvJHj160KNHD5PbL1261OjrefPmsW3bNv744w9atmxZxNFZJ19fX4YPH86oUaMMHfu3b98mOjqaAQMGUKNGDWQyGX/++Sc9e/bE3t7eqM/qcU5OTgwfPpwPPviAihUr4u7uzsyZM5HL5YYzqerVq2NjY8OyZcsYN24cFy9eZM6cOUX6ud5++23Gjh1LmzZtaNeuHZs3b+bChQtG/X15MSW2cePGsXjxYj744APGjBlDSEgI69atK9L4y6Lw2BTe/vEMOgkGtvFh6DM18m+860MIPwz9V0KtTkUah1UPsdDpdCQlJRk6q/OSkZFBYmKi0aOsW7FiBa+88grjx4+nQYMGjB07lpSUFACqVq3KrFmzmDx5Mh4eHkyYMOGJr7dkyRICAgJ48cUXCQwMpH379jRs2NDQGV+lShXWrVvHzz//TKNGjViwYAGLFi0q0s80ZMgQpkyZwvvvv0+rVq0ICwtjxIgRT7whYEps1atX59dff+X333+nefPmrFy5knnz5hVp/GVNUrqaMd+fJjFdQ6vqrszu1zj/7o/rf0Poz5AcBbbFUJxHKiUA6bfffjPrmE8//VRyc3OToqKi8m0zc+ZMCcj1SEhIyNU2LS1Nunz5spSWlmZu+OVKcnKy5OLiIq1Zs8aicQQGBkqvvfaaxd6/vP68aLU6afS6U1KNSX9KfnP3SFEJBXz+9ERJWtxIkmY6S9LuqSa9fkJCQr6/o3mx2ruTP/74I7NmzWLbtm24u+c/oG7KlCkEBQUZvs6qaSeY7uzZs1y9ehU/Pz8SEhKYPXs2AH379i2xGFJTU1m5ciXdunVDoVDw008/sXfvXqPxYELJWBr8L3uvRGGjlPPN0Da4OxdwNvzPJ5D4H7jWgM5TiyUeq0ximzZtYsyYMfz888+GsVD5sbW1xdbWtoQisz6PD4F43OXLlwFYtGgR165dw8bGhtatW3Po0CEqV65cZHE0btyY27dv57nvm2++4aWXXmLnzp3MnTuX9PR06tevz6+//vrE779QtHZfvM+Xwf8CML9/U1r4uObf+M4pOPGN/nnvpWBTtOuIZbG6JPbTTz8xatQoNm3aRK9evSwdjtXz9vbm3LlzBe6vXr06ISEhxRrHzp07UavVee7z8PDA3t6evXv3FmsMQsHCYlMI2nIegFHta/Jy62r5N9Zkwva3AQmaD4LaXYotLosmseTkZG7cuGH4OiwsjHPnzlGxYkWqV6/OlClTuHv3Lt9//z2gv4QcPnw4X3zxBf7+/kRGRgL6sVEuLi4W+QzWTqlUUqdOHUuHQY0aBdzZEkqFVQdvkpqpxb9mRab2bFBw4yNLIeYKOFSGbsV7k8SidydPnz5Ny5YtDcMjgoKCaNmyJTNmzADg/v37REREGNqvWrUKjUbDW2+9hZeXl+HxzjvvWCR+QSgvElLV/Hb2LgDvvVAfpaKA1BFzDQ4u1D/v8Sk45D96oChY9EysU6dOBa7U8PhYnf379xdvQIIg5GnL6Tukq3U08HSira9b/g11Otg+EbSZUPcFaPJyscdm1ePEBEEoflqdxA/H9TddRrTzLXg6XMhauHMcVI7QawmUwCRwkcQEQSjQgevRRMSl4mynpG+Lqvk3TLgLez7WPw+cCa4lM5RJJDFBEAq07qj+LGxgW5/8V6eQJNjxHmQmQbW20HZMicUnkpiAr6+v0bxUU1YUEcqHWzHJHLweg0wGQ5/xzb/h5d/h+i6Qq6DPMpCX3FI8VjdOTCh+9+/fx82tgM5bodzI6gvrUt+d6pXyWWIn7SHs/FD/vEMQuDcsoej0RBITcvH09LR0CEiShFarRakUP6KWkpKh4ZfT/wEwrJ1v/g3/ng4p0VC5HnQo+WW9xeXkk0gSZKZY5mFGoRBTinmYKuflZHh4ODKZjK1bt9K5c2ccHBxo3rw5x44dMzrm8OHDdOjQAXt7e3x8fJg4caJh5QwwvZDIrl27aN26Nba2thw+fNjs2IWis/XsXZIyNNSq7EiHOvlMMbt1AM7+oH/eZxkoS36Kn/gz9yTqVJjn/eR2xWHqPZPnm5lSzONpfPTRRyxatIi6devy0UcfMWjQIG7cuIFSqeTmzZt0796dTz75hLVr1xITE8OECROYMGEC3333HWB6IZHJkyezaNEiatWqJS5pLUiSJL4/Gg7A0IAaeRe+VafBH48GmrcdA9WfKbkAcxBJrAwwtZjH03j//fcNc1VnzZpF48aNuXHjBg0aNGD+/PkMGTLEcNZXt25dvvzySzp27MiKFSuws7MzuZDI7Nmz6dq1a5HFLRTOsVsP+Dc6GQcbRf5zJPcvgIdh4OQNz88s2QBzEEnsSVQO+jMiS723CUwt5vE08ivw0aBBA86fP8+FCxfYuHGjoY0kSeh0OsLCwmjYsKHJhUTatGlTZDELhbf+0VnYy62q4ZxX9e775+HoMv3zXovBrhgWOzSRSGJPIpMV2xIi1qSgAh/Jycm88cYbTJw4Mddx1atXNxTr6NatGxs3bqRKlSpERETQrVu3XMU6Hi/0IZS8u/Fp7LkcBcCwgDwm5ms1+hUqJC007g8NepZwhMZEEisDchbzyCp9llXM47nnniv292/VqhWXL1/OdzWM0NBQQ7GOrAUpT58+XexxCYWz8fhtdBK0q12Juh5OuRsc/1p/JmbnCj0+K/H4HifuTpYBOYt57Nu3j0uXLjF69GijYh7FadKkSRw9epQJEyZw7tw5/v33X7Zt22ZYvz9nsY5bt26xffv2Ii8kIhSNdLWWTafuADA8r2EVcbdg36OldV74BCrkv6pySRFJrIx4UjGP4tSsWTMOHDjA9evX6dChg2E5pawbCyVRSEQoGn9euE9cSiZVXe15vsFjCUqS4I//gSYNaj4HLV+zSIyPk0kFrYVTBiUmJuLi4kJCQgLOzsadkenp6YSFhVGzZs0S+eUvTikpKVStWpXFixczevRoS4dTJpWlnxfQ34zp89URQu8m8GH3+ozv9Fj3wNmNsG08KO3gzaNQqXaxxFHQ72heRJ9YGVEainkI1u3snXhC7yZgo5TzatvqxjuTo+GvR4U+Ok0ptgRWGCKJlSF5FfO4cuVKgQWKk5OTSzBCoTTLGtzap7k3FR1tjHfumgTp8eDZDAKeXKu0JIkkVka0bNkyz2IeaWlpBRYCEQSAmKQMdoTeB2B4gK/xzmu74NJWkCn0U4sUpSttlK5ohCJnb29fKgqBCKXbTycjUGslWlV3pWm1HEV30hP164QBBLwF3i0sEl9BxN3JPJSzex1CIZWVnxO1VsfGE/old3INqwieDYl3wc1X3xdWCokklkPWqPTU1FQLRyJYg6yfk5yzGazR35eiiErMoHIFW3o08creEXEcTq3RP+/9BdiYNg2upInLyRwUCgWurq6GJWIcHBxKZLCoYF0kSSI1NZXo6GhcXV1RKEpuFdPikDVPcrB/dWyUj85rNBn6qkVI0OI1qNXJUuE9kUWT2MGDB1m4cCEhISHcv3+f3377jX79+hV4zP79+wkKCuLSpUv4+Pgwbdo0RowYUWQxZS0ImHOtK0HIi6ura6lYQPJpXL6XyMnwOJRyGUP8cwyrOPw5xF4DxyrwQumeXWHRJJaSkkLz5s0ZNWoUL7300hPbh4WF0atXL8aNG8fGjRsJDg5mzJgxeHl50a1btyKJSSaT4eXlhbu7O2q1ukheUyh7VCqV1Z+BAfxwPByAbk088XB+NGA3+iocfDSjosdnxV789mlZNIn16NGjwDFMj1u5ciU1a9Zk8eLFADRs2JDDhw/z+eefF1kSy6JQKMrED6kg5Cc+NdNQ1dswrEKn069QoVNDvR76VSpKOavq2D927BiBgYFG27p165ZrqeScMjIySExMNHoIggA/n/6PdLWOhl7O2VW9T38L/50EGyf9OmFW0CdsVUksMjISDw8Po20eHh4kJiaSlpaW5zHz58/HxcXF8MhaCkYQyrOcVb2HB9TQ38BK+A/2fqxvEDgTXAoolFuKWFUSK4wpU6aQkJBgeNy5c8fSIQmCxWVV9XaxV+mrehuK3yaDjz+0sZ5FA6xqiIWnpydRUVFG26KionB2dsbe3j7PY2xtbbG1LfkKLIJQmuWq6n3xV7i+W1/8tveXILee8xvriRQICAggODjYaNuePXsICAiwUESCYH1yVvV+zb8GpMZlF7997n1wb2DZAM1k0SSWnJzMuXPnDBOUw8LCOHfuHBEREYD+UnDYsGGG9uPGjePWrVt8+OGHXL16la+//potW7bw7rvvWiJ8QbBKuap6/z0NUmOhSgN41vp+lyyaxE6fPk3Lli1p2bIloK+dmLUqKMD9+/cNCQ2gZs2a7Nixgz179tC8eXMWL17MmjVrinx4hSCUVTmreg9v5ws398G5jYDMYsVvn5ZF+8Q6depU4CTadevW5XnM2bNnizEqQSi7clb1fraGA6x4VPzWbyz4+BV8cCllVX1igiAUXq6q3gfmQ/xtcK4Kz8+wbHBPQSQxQSgnjt3UV/V2tFEwoGosHFuu39FrCdjmUZrNSjzV5WRycrKhgGoWUxb2FwSh5K0/Fg7AKy09cdw9ASQdNHkZ6ne3bGBPyewzsaxJ2I6Ojri4uODm5oabmxuurq64ubkVR4yCIDylnFW937bfDZGhYO8G3T+1cGRPz+wzsddeew1Jkli7di0eHh5ldr0tSZI4cuMBqZkaXmhs3cutCMKGR1W9X6qRTuXTn+s3dpsHFapYNrAiYHYSO3/+PCEhIdSvX7844ik1/rhwn4k/naWqqz1dGrijVIjuQ8E6pau1bDoZAUh8pPsGNOn6RQ6bD7J0aEXC7N/Mtm3blov5hy808sDNQWV0Gi4I1ujPC/d5mKrm9QpHqRRzApT28OJSq1ihwhRmn4mtWbOGcePGcffuXZo0aZJrffFmzZoVWXCWZKdSMMS/Bl/tu8HaI2H0aOr15IMEoZSRJIn1R8OpQjzv8b1+Y5ePoGJNywZWhMxOYjExMdy8eZORI0catslkMiRJQiaTodVqizRASxoaUIOVB25yKvwhF/6Lp1k1V0uHJAhmyarq/bXN99hqksCrBfi/aemwipTZl5OjRo2iZcuWHDt2jFu3bhEWFmb0b1ni4WzHi830Z2DfHQm3bDCCUAjfHw2nq/w0PeXHS23x26dl9qe5ffs227dvLzcFWUc/W4vfz93jj/P3mNyjQfY65IJQykUnpXMg9Ca7VN/pN7SfCF5lo7snJ7PPxLp06cL58+eLI5ZSqWk1F9r6uqHRSfxw7LalwxEEk206eYcg2U94yh5CxVrQcZKlQyoWZp+J9e7dm3fffZfQ0FCaNm2aq2O/T58+RRZcaTGqfU1OhT9k44nbTOhSBztV7gIiZyIesmL/Tab1akiNSo4WiFIQsqm1Oi4e280q5V79ht5fgCrvhUOtnUwysxa7vIAVH62hYz8xMREXFxcSEhJMniKl0erouHA/d+PTWPBSU171q56rje/kHQC0rO7Kb+PbF2nMgmCunWfDqPdbT+rI76FtMRRFv68sHZLJzP0dNftyUqfT5fso7QmssJQKOSPb+wKw9khYgcsH3YvPu2CJIJSk1L2fUUd+jxRVRRTdSnfx26dlVhJTq9UolUouXrxYXPGUWgPa+uBoo+B6VDKHb8Tm285WKWpVCpZ18+IJ+iZvBkDd7TP9HMkyzKwkplKpqF69epk94yqIs52K/2ujL/e29nBYvu1slWJ6kmBBOi2qnf9DJdNy3rE9rq1fsXRExc7s37iPPvqIqVOnEhcXVxzxlGoj2vkik8G+azHcjEnOs01enf6CUFJSj6ygeuplEiV7pJ6LyszUooKYfXfyq6++4saNG3h7e1OjRg0cHY3vxJ05c6bIgittfCs78nwDD/ZeiWLdkXDm9GuSq404ExMsJj4C1f5PAFjvOJIJjRpaOKCSYXYS69evXzGEYT1GPevL3itR/BLyH++9UA9XBxuj/bYqkcQEC5AkpD+DUGnTOKmrj3unN8rsMlmPMzuJzZw5szjisBoBtSrRwNOJq5FJ/HTyDm92qm10t1J07AsWEfoLsht7yJCUzJW/yaYWPpaOqMQU+rQhJCSEDRs2sGHDhqeqPrR8+XJ8fX2xs7PD39+fkydPFth+6dKl1K9fH3t7e3x8fHj33XdJT08v9PsXSJJAk2m0SSaTMaq9fgWAn05GoNNJpKmzb3TYiTMxoaSlPIDd+tH4yzT98fd7Rl/Vu5ww+0wsOjqaV199lf379+Pq6gpAfHw8nTt3ZtOmTVSpYvpKkZs3byYoKIiVK1fi7+/P0qVL6datG9euXcPd3T1X+x9//JHJkyezdu1a2rVrx/Xr1xkxYgQymYwlS5aY+1EKFhcGf/4PKteDnguNdr3Y3IvZf14mIi6Vk+Fx1Kqc3S+osKLy70IZ8ddUSH3AVZ0Pq3S92etfw9IRlSizf+PefvttkpKSuHTpEnFxccTFxXHx4kUSExOZOHGiWa+1ZMkSxo4dy8iRI2nUqBErV67EwcGBtWvX5tn+6NGjtG/fnsGDB+Pr68sLL7zAoEGDnnj2Vijxt+HWfji5GiJOGO1ysFHSu7l+dYstp++QnKEx7Pvj/D2uRSYVfTyCkJcbe+HCJiRkTFaP5bkG3vqq3uWI2Uls9+7dfP311zRsmH3no1GjRixfvpxdu3aZ/DqZmZmEhIQQGBiYHYxcTmBgIMeOHcvzmHbt2hESEmJIWrdu3WLnzp307NnT3I/xZLU6QYvXAAm2vw2aDKPdr7TW9znsCo0kMtH4cnbqb6FFH48gPC4zBf58F4ANUg/OSXUYFuBr2ZgswOzLSZ1Ol2vSN+gHwj5evq0gsbGxaLVaPDw8jLZ7eHhw9erVPI8ZPHgwsbGxPPvss0iShEajYdy4cUydOjXf98nIyCAjIzsBJSYmmhwjL8yBf/+C2Gtw+HPoNNmwq1V1V2pXceRmTApbThkv1x2fmvn4KwlC0ds3D+IjSLbzYn78K/qq3nUqWzqqEleopXjeeecd7t27Z9h29+5d3n33XZ5//vkiDe5x+/fvZ968eXz99decOXOGrVu3smPHDubMyX9u2Pz583FxcTE8fHzMuGvjUBF6fKZ/fnARRGcnV5lMxoBHI/h3hN43OixTa3oyF4RCuRsCx78GYJ7sdVKxY1hADeTy8jGsIiezk9hXX31FYmIivr6+1K5dm9q1a1OzZk0SExNZtmyZya9TuXJlFAoFUVHGRTiioqLw9My7RNr06dMZOnQoY8aMoWnTpvTv35958+Yxf/78fM8Cp0yZQkJCguFhdpGTxv2hXg/QqeGPiZDjffq3qopCLkOt1Q+xcHPQn6GqNWYtDCIIZklNS0Pa/jZIOmJ8+/Ljw/o42ih4uXU1S4dmEWZfTvr4+HDmzBn27t1ruOxr2LChUd+WKWxsbGjdujXBwcGGAbQ6nY7g4GAmTJiQ5zGpqam5lgJSKPS3kvNbWcLW1hZbW1uzYjMik0GvxRB+GO6cgNPfgt9YANyd7Ohcvwp7r0QD4OZgw8NUNZlaHTqdxIztF2la1YWBbXMv3SMI5tLqJL45eJO04IW8p7iEZF+RBQwDMnipVTWc7HJ385QHhVpsWyaT0bVrV7p27fpUbx4UFMTw4cNp06YNfn5+LF26lJSUFEMRkmHDhlG1alXmz58P6BdkXLJkCS1btsTf358bN24wffp0evfubUhmxcKlKgTOhJ3vw95ZUL8HuOj/6r3S2ic7iTnaQGwKao2OozcfsOF4BAAD2viUm9HTQvG4E5dK0JZzxN6+zG6bXwH4xn4Mv13T9/cOb1e+hlXkVKgkFhwcTHBwMNHR0bku4/IbHpGXgQMHEhMTw4wZM4iMjKRFixbs3r3b0NkfERFhdOY1bdo0ZDIZ06ZN4+7du1SpUoXevXszd+7cwnwM87QZDaE/68/GdrwHgzaBTEaXBu5UcrThQUombo+mIGVodaRmZg+7iErMwNNFrM0vmE+SJH49c5ePt18iJSOTLbbfYitTc0jXjAX3mgPQvk4l6rg7WThSyzE7ic2aNYvZs2fTpk0bvLy8nvoMY8KECflePu7fv9/oa6VSycyZMy0z9Ukuh95fwspn4fpuuPQbNHkJG6WcQX7V+WrfDRp4OrH3ShSZGh1J6dlJ7EZ0skhigtkepmQy9bdQdl2MBGBylZO0TboMKgdSOy6EPx8AlMthFTmZncRWrlzJunXrGDp0aHHEU7q5N4Dn3of982HXh/qxZA4VebdrPTrWr4JvJUe+2ncD0K8Am+Xf6CSerVv+bn0LhXfgegwf/Hye6KQMlHIZHz3nxoiz6/Q7u0yjW8AzLLS7w38P0+ja0KPA1yrrzE5imZmZtGvXrjhisQ7Pvqs/C4u5Cn9Ph37LUchltPWtaHQJeele9ni0G9F5rz0mCI9Ly9SyYNcV1j+qrFXHvQJLB7agyeG3ICMRvFuB/zgAwyKd5Z3ZQyzGjBnDjz/+WByxWAelrb4AKTI4twFu7jPsslHk/d8pkphgiot3E3hx2SFDAhvRzpc/336WJgkH4MofIFdCny9BXn4md5vC7DOx9PR0Vq1axd69e2nWrFmu0ftFPhG7NPLx0w+zOLlKP0n8zWNg44Aix0DDb4e3IfRuAkv3/pvvKrCCAPqhEysP3OTzPdfR6CTcnWxZ+H/N6VivCqTFw84P9A3bvwOeTS0aa2lkdhK7cOECLVq0AMhVMKRcDSN4fgZc3QEPw/V9ZC/MMfr8DbyceaZWJZbu/ZfY5EziUzNzLaAolG/pai1nIh6y5O/rnL79EICeTT2Z26+pfrgOwN6ZkBwJlerAcx9aMNrSy+wktm/fvic3Kg9snaDXEvhpIBz7Cpq8DN4t+GPCsySkqanqqi9U6u1ix72EdG5EJ9PGt6KFgxZKg4t3E/h6/w32XI4yzPaoYKtkVp/GvNSqavYfw/DDELJO/7z3F6ASd7jzUqhxYsIj9bvrk9fFX2H7BBi7j6bVXIya1HavIJKYAMDtBynM2HaJA9djDNs8nG1pX7sy73ath0/FHEvoqNNh+6OlrVqPAN9nSzZYKyKS2NPq/inc/AciQ+HYcnj2f0a767hX4NC/saJzX2DZPzc4cD0GuQz6NPfm9edq09DLKe9umIOfQdxNqOAJgbNKPlgrIpYhfVoVqkC3efrn++fDg5tGu+s+Gkl9IyaZn05GMHPbRdRilYty6d8o/WKZSwa0YOmrLWnk7Zx3AosMhSNf6J/3WgT2riUXpBUSZ2JFofkguLBZvxLsH+/A8D8M9f7quFcA4FpkEvuv6S8jalRypGP9KtSuUsFSEQslTJIkbsWkANDQyzn/hjqt/jJSp4GGvfUPoUBmn4kdPHgQjUaTa7tGo+HgwYNFEpTVkcngxaWgtIfwQ3B2g2FXVhK7n5C9+uvsPy/z/OIDHPo35vFXEsqo2ORMkjI0yGRQo6Dlo098A/fOgK0L9FiYfzvBwOwk1rlz5zyrfyckJNC5c+ciCcoqVawJXT7SP//7I0jSr5NW0dGGio55D6248F9CSUUnWNitR2MFq7nZ518l/uFt+OfRAp8vzAZnrxKKzrqZncQkScrzOv7Bgwe5qoGXO/5vglcLSE/Qz618pE6Oy8aeTT3p37IqANGJxVRqTih1bsXqLyVrVc6nC0GS9Ovlq1OhxrPQclgJRmfdTO4Te+mllwD9gNYRI0YYLTSo1Wq5cOFC+Z5TCaBQ6qckreoEl3+HqzuhQU/qeFTgZLj+7LVPc2+iEjP47exdopMyCnw5oezIOhOrVSWfP/QXtsDNYFDY6seEidJ/JjM5ibm46Mc/SZKEk5MT9vb2hn02NjY888wzjB07tugjtDZezaD9RH1hkR3vge+zRmdizaq5cv5OPIBIYuVIVqd+rbxu5qTEwu5HRWg6fgiV65RgZNbP5CT23XffAeDr68v7778vLh0L0nESXN4GcbcgeBb1608BoIqTLV4udtyL15/FxogkVmYdv/WA3RcjmdS9AfY2ihyXk3n83uyeAmlx4N5YPz9SMIvZQywssiChtVHZ6y8J1veGU2sIaPwKE7vUoUV1V2QyGe5O+ukj0Unp+fYxCtbrfkIar646DkCrGm50b+xJRFwqkMfl5L97IHQLyOT6rghF+Vwn/2mYfeEdFRXF0KFD8fb2RqlUolAojB7CIzWfg5b6hSPlf04kqIsvXRroF69zd9afiaWrdSRl5B6uIli3r/65YXienK7hzsNUtDoJBxsFns455j9mJBuK3+L/JlRrXcKRlg1mn4mNGDGCiIgIpk+fXiTLU5dpL8yB639B7HU4tBg664v82qkUONkpSUrXEJ2YgXM5rVJTVp171OcJoNbqDP1hNSs7Gv++/PMJJNwB1+rZw3MEs5mdxA4fPsyhQ4cMy/EIBbB3g54L4efhcGgJNOoHHo0Aff9YUrqGmKQMw4BYwfpptDr+jcqeJ5uu1ua4M5nj+/zfaTixUv/8xc/BRvQxF5bZl5M+Pj751ngU8tCoL9TvpS++u/1t/bQSwN1Jf0kZnSTGipUlKRlaowrw6ersMzFDp74mU/+zgATNXoU65tVsFYyZncSWLl3K5MmTCQ8PL4ZwyiCZTD+J19YZ7p6GU2sADJ37MUkZXLqXwMsrjhLyaGE8wXrFpWYafZ2u0XIr9rExYke+gOjL4FApe/EAodDMTmIDBw5k//791K5dGycnJypWrGj0MNfy5cvx9fXFzs4Of39/Tp48WWD7+Ph43nrrLby8vLC1taVevXrs3LnT7PctUc7eEPix/vneWRB/hyqGM7EMvt53k5DbD9lw/LblYhSeWlRiOp0X7TfalmF0JlYBYq7rl9kB/TJOjpVKOMqyx+w+saVLlxbZm2/evJmgoCBWrlyJv78/S5cupVu3bly7dg13d/dc7TMzM+natSvu7u788ssvVK1aldu3b+Pq6lpkMRWb1iP1xXcjjsGOINyr6qua341PMyySd0usxW/V/rkanWtbdFI6D1L0Z2c1K9vDjxNBmwl1ukLTV0o6xDJJJlmwg8vf35+2bdvy1VdfAaDT6fDx8eHtt99m8uTJudqvXLmShQsXcvXq1VwFSkyVmJiIi4sLCQkJODsXsCRKcYi5DivbgzaTU60/4/+OVMPBRkFqpr6frIKtktCPXxB3fK3QuTvx9Ft+JNf22lUcuRmTgoezLSe63oYdQaByhLeO6+9KCrmY+ztaqAlaN2/eZNq0aQwaNIjoaP1fn127dnHp0iWTXyMzM5OQkBACA7M7NeVyOYGBgRw7dizPY7Zv305AQABvvfUWHh4eNGnShHnz5qHVavN9n4yMDBITE40eFlOlnqHYQ4uLC3Aj0ZDAAJIzNGIqkpXKK4FB9sTv1m7psOfRQPHnZ4gEVoTMTmIHDhygadOmnDhxgq1bt5KcrL8EOn/+vFmj+WNjY9FqtXh4GFcv9vDwIDIyMs9jbt26xS+//IJWq2Xnzp1Mnz6dxYsX88knn+T7PvPnz8fFxcXw8PGxcMHR9u+AeyNUGXFMU23MtfumWMa6TNFf50hMSFsJmUlQtY2+3J9QZMxOYpMnT+aTTz5hz5492Nhkr5PVpUsXjh8/XqTBPU6n0+Hu7s6qVato3bo1AwcO5KOPPmLlypX5HjNlyhQSEhIMjzt37hRrjE+ktIE+y5CQ8bLiEB3kF7BVymlfR9/BK2pUlj3d5adolHjoUfHbZaL4bREzO4mFhobSv3//XNvd3d2JjY01+XUqV66MQqEgKirKaHtUVBSenp55HuPl5UW9evWMpjc1bNiQyMhIMjMz8zzG1tYWZ2dno4fFVWsD/m8AME/5LV1qOdLEW79KyM1Hd7KEssGZZGar1um/eDbIMNhZKDpmJzFXV1fu37+fa/vZs2epWrWqya9jY2ND69atCQ4ONmzT6XQEBwcTEBCQ5zHt27fnxo0b6HTZgwmvX7+Ol5eX0VmhNZB1mU4kVfCRx/CO4mfDevuPn4mduxMvxo+Vchma/Ptkpyh/wl0Wj9q1NnR4rwSjKj/MTmKvvvoqkyZNIjIyEplMhk6n48iRI7z//vsMG2beapRBQUGsXr2a9evXc+XKFd58801SUlIYOXIkAMOGDWPKlCmG9m+++SZxcXG88847XL9+nR07djBv3jzeeustcz+G5dlW4GKrjwGoH76BpnJ9laSb0clIkkR4bArpai39lh/h5RVHSc0UE8VLm7RMLTqdREKaOs/9AfJLDFLqi03L+y4TxW+LidnjxLKSho+PD1qtlkaNGqHVahk8eDDTpk0z67UGDhxITEwMM2bMIDIykhYtWrB7925DZ39ERATyHCtc+vj48Ndff/Huu+/SrFkzqlatyjvvvMOkSZPM/RilQmCf10C9H1noz9Q9PhUlk7iXkM6C3Vf55sAthgXUMLRNTNPgYCOKU5UW0YnpPL/4AM/Wrcx7L9TLtd+WTOYp9bMz/lB1p3fN9iUdYrlR6HFiERERXLx4keTkZFq2bEndunWLOrZiYdFxYnlJiYWv2kJaHMtkQ1ic1ivPZnuDOoqJ4qXItnN3eWfTOSpXsGHla615ZaXxsKAPlZsYr9xOpOTG/Jrr+GJEJ8sEaoXM/R0t9J/26tWrU726GOvy1BwrQ/f58NsbvCFt4VdZK8Kl3FVutp+/RwNPJ3o2FRVwSoPL9/XjDeNT1cSnGl9ONpKF87riTwCmq0dS2yPvG1VC0TApiQUFBTFnzhwcHR0JCgoqsO2SJUuKJLBypdlAuLAZm5v/sEC1hkGZHyE91l35ZfC/AJyc+jzuzqJvxdKu3NdX89boJO7Gpxm2K9CyQLUapUzHDq0fe3Rt6JpfcRChSJiUxM6ePYtarTY8z4+YLlNIMhm8+DnqZf48wxUGKA6wWZt3Dc8TYXH0bu5dwgEKj7tyP3vmR1hs9rCYkYrdNJOHkSA58LF6BKCfeiQUH5OS2L59+/J8LhQhN1/CmwdR9+w8PlJu5B9tC2Jwy9Xs7Z/O8mIzsaKuJcUkZRgVeQl/oE9iPrIo3lP+DMBczRDiFW7IdZJh+IxQPERxu1LEtt14zulq4SxLZZZqfb7tMjS6fPcJxS/nWRjA7QepgMQ85bfYyzI5qm3EFm0nFv1fc5YMaIGrg3WNYbQ2Jp2JZRXONcXWrVsLHUx5V7VSBSbo3uBX2VR6Kk7STXuKv3Rtc7VLStdgpxJTVyzl8SR2Jy6Vl+WH6KC4SLqkYqpmNCCjbwvTB38LhWfSmVjOCdTOzs4EBwdz+vRpw/6QkBCCg4MNBXaFwlHIZWRWbsQ32hcBmK36DmdyT0NKERWSLOrxJOaqi2e66gcAlmpezvPuslB8TDoTyyqcCzBp0iQGDBjAypUrDXMYtVot48ePLx3jrqxcXQ8nlkX2p4f8JLXl95mi2sQU9WijNskiiZUYjVbHmYh4mlVzMZz9Zg2v8Hax415COjNU3+MqS+GuXV122b0M8XmP4BeKh9l9YmvXruX99983moStUCgICgpi7dq1RRpceTSxSx0ysGGqegwAgxTB+MmuGLURZ2IlZ9Hf1xnwzTHe23Ie0Fcvypqk365OZbrIz9BHcQytJCPu+UXY2NhaMtxyyewkptFouHr1aq7tV69eNZqYLRROXQ8nBrSpxgmpIT9qugAwX7UGW7JX6UgR8yhLzNrDYQDsCNUvenAjOhmtTsLNQUXTynI+Uen/cK/R9sTGpzXNqrlaKtRyy+wR+yNHjmT06NHcvHkTPz8/AE6cOMGCBQsME7eFpzMswJeD12NRPTeH9MOh1E6/zwTl7yzWDAD0HftCyVApZORYfJfL9/SXkg29nGl/+2u8ZXHc1rnzueYV+jmomPFiI1zsVbzUSnTqlxSzk9iiRYvw9PRk8eLFhiV5vLy8+OCDD3jvPbHUSFFoUtWF41OfByA0/WOaHn6LcYo/2KF9hqtSdVIy8l/6RShaKqWcnFksqz8s0Ok2ta/+BMBUzWjSscXFXoWdSsGM3mLNsJJk9uWkXC7nww8/5O7du8THxxMfH8/du3f58MMPjfrJhKKRUKM7u7RtUcm0LFCtQo6OlAwN9xPSRBHjEmCjMP4VuXI/ERUaXvnvU2RI/Kx5jiO6ptgq5WLYi4U81WDXUrNSahkml8NM9QgSJQdayG8xQvEXy/ffIGD+P6w5FGbp8Mo8VY4kJkkSV+4n8qZiO87JN9HaV2KuZggALvaFq74lPL1CJbFffvmFAQMG8Mwzz9CqVSujh1C0FDIZ0bgxXzMIgPeUW6iQdg+AuTv1dy3FGVnxsVFm/4rcjU/DPSOcCcrfAEgPnE88TgC4OogkZilmJ7Evv/ySkSNH4uHhwdmzZ/Hz86NSpUrcunWLHj16FEeM5ZpCrp8juUnbmRO6BjjKMpir/BaQsFcpWHPoFv7zgo0mIQtFJ+fl5OW78SxQrcFGpoW63bBv8X88+vbgai+mFlmK2Uns66+/ZtWqVSxbtgwbGxs+/PBD9uzZw8SJE0lISCiOGMu1rIneEnKmqMeQIanoqLhAX/kRalRy4JMdV4hOymD2H6bX/BRMl/NM7MLvn9NGfp10uT30WoxcITdcRjqLy0mLMTuJRURE0K5dOwDs7e1JStKvqzR06FB++umnoo1OQCnPXq3iluTNFxp9pamZqu+p75S9kkJMcgZpj+6iHb/1gBO3HpRsoGWUSqH///fiAW+o9VOLztSZCK76+qVujyZ3i8tJyzE7iXl6ehIXFwfoV3fNqjUZFhYm+maKgUJuvOTOKu2LXNH5UFGWzKCH2fU2L95N5LmF+wiPTeHVVccZuOo4mWK1i6em79iXmKNai5MsjRBdXaQ22dPAspKX6Ni3HLOTWJcuXdi+fTugH/j67rvv0rVrVwYOHJhnPUrh6Ty+bJgGJZPVY9FKMp5J3ktH+XnDvpikDDot2m/4Wq0VScxcOy7c580NISSl6+c/2ijl9JKfIFBxlkxJwWT1WBpWzV7nzXAmJpKYxZg92HXVqlWG6UVvvfUWlSpV4ujRo/Tp04c33nijyAMs73wqOuTadl6qw3fa7oxR7mKu6lteyPiMVHIvWa0TZ8ZmW3skjJDbD+lQtwqD/avjRjLTHxW//VrblySnOlR0zO7Eb+HjSvDVaJpUEyu4WIpZZ2IajYZPPvmEyMhIw7ZXX32VL7/8krffftvqCthaA2c7FYc+7IzbY30uizX/xx1dFarJYg2riT5OJ3KYSXQ6ieikdHQ6Cc2js9eTYfo+xYHxq6giS+RfXVW+1vSloZeT0bETutThzPSudK7vXuJxC3pmJTGlUslnn32GRlO0c/eWL1+Or68vdnZ2+Pv7c/LkSZOO27RpEzKZjH79+hVpPKWNT0UHpvRsCED7OpV4/blapGHHR5pRgH5d9xayG7mO04ks9kSSJFFr6k785gbz5sYQsv7HToTFId3cR/uk3egkGZPUY8lERXMfV6PjZTKZ0ZmZUPLM7hN7/vnnOXDgQJEFsHnzZoKCgpg5cyZnzpyhefPmdOvWjejo6AKPCw8P5/3336dDhw5FFktp9n+tq7Hn3edYN9KPWpX1hScO6przq/ZZ5DKJ+arVqDD+4yIuJ58sZ/Xuvy5FGf7PHiYkoN32DgA/aAM5I9WjRxNP3niutkXiFPJndp9Yjx49mDx5MqGhobRu3RpHR+NKLn369DHr9ZYsWcLYsWMNK2CsXLmSHTt2sHbtWiZPnpznMVqtliFDhjBr1iwOHTpEfHy8uR/D6shkMup66C9l2tepjKONgpRMLZ+oX6OT/DwN5Xd4XfEny7X9DMfoJNDqJGSAXC4Kizzu2M0HDFp93Ghb1mpS/1P+ijLxNvEqdxamDwT0l472NmJ+ZGljdhIbP348kHd9SZlMhlZr+goLmZmZhISEMGXKFMM2uVxOYGAgx44dy/e42bNn4+7uzujRozl06FCB75GRkUFGRvZ4qsTExAJaWwefig6ETO/Knxfu8/7P55mlHsaXNsuZqNzKbl1bbkr6ZWDUWh1dPz9ARQcbfh4XICokPebxBAZwPyGNxrIwxip2ALC92nskX9HfXPEU9T5LJbMvJ3U6Xb4PcxIYQGxsLFqtFg8PD6PtHh4eRjcPcjp8+DDffvstq1evNuk95s+fb1QjwMfHx6wYSys7lQIPZ/0qott17dinbY6tTMM81bfI0J9OXItM4lZMCqdvPxSd/CZKSk3jM9UqFDKJYMWzXHVuD0Dn+lWoVEGs2loaWVXJtqSkJIYOHcrq1aupXLmyScdMmTKFhIQEw+POnTvFHGXJ8XKxf/RMxjT1KFIkW/zlVxmk0NcGTc2xDpZWZDGTjFbsorH8NvGSI5NShnAtUj8jpWX13DVAhdLB5MvJtLQ0goODefFFfSWeKVOmGF2mKRQK5syZg52d6afclStXRqFQEBUVZbQ9KioKT0/PXO1v3rxJeHg4vXv3NmzLGrOmVCq5du0atWsbd7za2tpia1s2/4LWruJIvxbe/H7uHnepwkLNQD5Wfc9k5Y8Ea1uyI/Seoe2QNcf5eVw7C0Zb+tWQRfKu8hcA5mpeIxYXYm8/BDAMvRBKH5PPxNavX88333xj+Pqrr77i6NGjnD17lrNnz7JhwwZWrFhh1pvb2NjQunVrgoODDdt0Oh3BwcEEBATkat+gQQNCQ0M5d+6c4dGnTx86d+7MuXPnysyloqlkMhlz+zc1fP299gXO6urgLEtjjuo7dj5aFx7gVPhDS4RoRfTFb+1kag5rG3PNs7fR3iuPzsiE0sfkJLZx40Zef/11o20//vgj+/btY9++fSxcuJAtW7aYHUBQUBCrV69m/fr1XLlyhTfffJOUlBTD3cphw4YZOv7t7Oxo0qSJ0cPV1RUnJyeaNGlSLgfb5uyr1yFnsnoMaknBC4oQustPWS6wUiyvS+v/UxygveLSo+K3Y3imtnF3xfAA3xKKTjCXyUnsxo0bNG2a/Vffzs4OuTz7cD8/Py5fvmx2AAMHDmTRokXMmDGDFi1acO7cOXbv3m3o7I+IiDCs5S/k9vhQsGtSdVZo9WcRs1XrcCbZAlGVXgt2XaXl7L+JeJBq2FaFeD5SbgRgieYVIiQP/HwrGvb3bu7Ns3VN64MVSp7JfWLx8fFGfWAxMTFG+3U6ndF+c0yYMIEJEybkuW///v0FHrtu3bpCvWdZ8fgqFwDLNf3oJT9Bbfl9Jit/YqpmrAUiK30kSWLlgZsAfBH8r2H7zEfFby/qfPlW2xMAtxyj8K/et/5hOWWZyWdi1apV4+LFi/nuv3DhAtWqVSuSoATT2akUrB/lZ7QtAxsmq/WJa7ByH8/Is8+Qw2JTuP2gfK4CO2TNCcPzX8/8B0CgPIQXFcfRSHImqceiRT+YVSaDQX76PtY3O4lR+qWZyUmsZ8+ezJgxg/T09Fz70tLSmDVrFr169SrS4ATTdKxXxfC8RxP9Xd1TUgM2aPRl3+Yp9cV3Vx+8RedF++m4cH+5XKbn6E3jhSIrkMoc1XcArNH24pJU07BPJZczu28T/n73Ofq3FDUkSzOTk9jUqVOJi4ujfv36LFy4kG3btrFt2zY+++wz6tevz8OHD5k6dWpxxioUYPngVgzxr86yQS0N2z7VDCJScqOWPJKJyq2GwiIA4WJNfj5UbsZLFke4zoOlmpeM9inkMlQKOfU8nMRMh1LO5CTm4eHB0aNHadiwIZMnT6Z///7079+fKVOm0KhRIw4fPpxr5L1Qcno182Ju/6YoFXLD5U8SDkxX6+/yvqH4k4ay24b2A1flnnJTVj1Izsh1R7K17BrDlHsAmKIZQzrGYwmVCpG4rIVZcydr1qzJ7t27iYuL48YN/dIvderUoWLFik84UihJ779Qn2ZVXXhz4xn26NqwQ+tHL8VJFqhW81LmLLQoiEvJtHSYJSLk9kP+b+VRqrllLy5pg5pPVfppa5s1nTima5zrOKWYMG81CjXtqGLFivj5+eHn5ycSWCmkkMvo0dTL8PXH6hEkSA40l99ihGK3BSMreTeik9BJEBGXPaTiLeU26sjvESO5ME8zOM/jlHKrmpFXronvVDkQg6uhUvX7yp/xkemneSVnFO3ilqWRWmt8GVlPdoc3FdsAmKkeTgIV8jxOIS4nrYZIYuXEFm0njmobYS/LZK5yLSDRZOZfXLxbtmuF5pzzKEfHp6rV2Mi07NG2ZqfO36jtGx1rGZ6rxOWk1RBJrAz7ZVzO+acypmpGky6peE4Rykty/Tpsqw/dskxwxSziQSpL/r5GdFL2AOyhij20lN8gSbJnunoEYJyohvjVMDzPaxCxUDqZvSiiYD3a+Br3V4ZLXizVvMxk1SamqzZwIKM5KkXZHKA8dO0JbueYWuRNLB8qNwHwqeZVIqlk1L65j6vRHUnRJ2Y9xHeqnFmj7cklXQ3cZMnMUP3wqDhs2ZMzgYHEJ6q1OMoyOKWrx0bt87naV3K0wVYpz3GEWH/NWogzsXJGg5JJ6rFss5lOX8VRkpKPAU2feJy1WH3wFn+GGi8Y0Ed+jC6Kc2RI+sLDUo6/3YP9q3MnLpU5/ZpQqYIt/VtWRSdJuDqUvxVRrJVIYuXIgQ860XHhfi5KtVij7ckbyh30vrOIB3EvU6lipSe/gBXIOSsBwI1EZqrWA/CVpp+h/kCWef2NE/jnA1sUa3xC0Sub1xJCLlVd7alRKbsy1eeaV4jQVcFFHcX2JW9y+V7ZXKlhmmojlWRJXNNVY6XWvEpcgnUQSayceHz6Xzq2TNWMAWC44m+O7t8J6JeruXI/kdRM6xtD9vjUomflobysOIROkjFFPQa1uPAok0QSKyeykthXg7MniB/WNeUX7XPIZRJ97nwKmkz2X4uhxxeHePHLwxaKtHDCY1OoPXWn4Wt70pmnXAPAeu0LnJHq5TrGRil+/MsC8V0sJ2SPxkS92MwbVweVYfsn6iHESs64p93iyPppjFynX9L6lpWtcjHhpzNGX7+r/JXq8hjuSpVYpBmQ5zH2KlEItywQSaycyDl2c0Q7X8PzeJyYpR4GQJuIb6ktu1vCkRWNO3FphudNZbcYrdCflU1TjyIFe8O+YQHZA1p9K2VPChesl0hi5UTONbHeeb4udd2z5wz+oQsgWNsSW5mGBarVhuK71uLwv7EkpKkBUKLhU9VqFDKJbdp27NNlXz5/9nIzPu7dmA2j/WlfpxJf5lh7TbBeIomVEzn79WUyGZtef4aJXeoY9k5XjyRZsqOt/DpDFPoSeuuPhlvFkj2vfZu97PRYxU4ayW/zUKrAbPVQw/aPezdiQFsf5HIZz9atzMYxzxjdrRWsl0hi5cTjdycrVbDl3a7Znd33qMxnmoEATFJuwpMHzNx+iVdWHC3JMJ+Kr+w+/1P+CsAc9Ws8wMWwz9FW3Jksq0pFElu+fDm+vr7Y2dnh7+/PyZMn8227evVqOnTogJubG25ubgQGBhbYXtDLa4nlx7dt0HYlRFcXp0fFd0EqtR38VyMTab/gH34J0Rf8kKFjgWoNtjI1B7VN2arrYNTeyU6V18sIZYDFk9jmzZsJCgpi5syZnDlzhubNm9OtWzeio6PzbL9//34GDRrEvn37OHbsGD4+PrzwwgvcvWudHdIlJb81Gba8kb3Shb747lgyJQVdFWfoKddfpqWrtSUQoXle/z6Eu/FpvP/zeQAGKA7wjPwKqZItUzWjefwTO9uJM7GyyuJJbMmSJYwdO5aRI0fSqFEjVq5ciYODA2vXrs2z/caNGxk/fjwtWrSgQYMGrFmzBp1OR3BwcAlHbl3k+RS78KtZkcCG2bUR/pWq8bW2LwCzVOtwIZmle/9FerxKrwXFp2YardRahYeG4reLNa/wn+Se6xhxJlZ2WTSJZWZmEhISQmBgoGGbXC4nMDCQY8eOmfQaqampqNVqsUz2ExRUsKdjPePq1l9r+vKvripVZIlMVf7IygM3mffYnERLCn1sIcdZqvU4y1I5r6vFOm13w/bWNdwMzyuIM7Eyy6JJLDY2Fq1Wm6tKkoeHB5GRkSa9xqRJk/D29jZKhDllZGSQmJho9BCMyR9bADATFZPUY9FJMgYq99NOfpHVh8IsFF1uf1+KMjzvJj9FT8VJNJL+Ujir+C3AG8/VwreSA5Ur2FLNzT6vlxLKAKv+87RgwQI2bdrE/v37sbOzy7PN/PnzmTVrVglHVvrkdzmZ374zUj1+0AYyXLmHecpv6Z65gOjEdNyd8/5/Lkk/HNeXnnMmhdmPit9+o32RK5J+IOvwgBq4OtjwfEMPOtXXX1qW1XXTBAufiVWuXBmFQkFUVJTR9qioKDw9PQs8dtGiRSxYsIC///6bZs2a5dtuypQpJCQkGB537twpktitTUGXk4p8di7UDOSeVBFfeRT/U24lrJTdqZyk3ISHLJ5bOk++zFH8dlbfJrzbtR4KuQwbpVzMkSzjLPrdtbGxoXXr1kad8lmd9AEBAfke99lnnzFnzhx2795NmzZtCnwPW1tbnJ2djR7lUUFJLOe+za8/Y3iejAPT1KMAGKPYgSrmQnGFZzY/2RWGKPU/N1PUY8lALGJYXln8T1RQUBCrV69m/fr1XLlyhTfffJOUlBRGjtRXrh42bBhTpkwxtP/000+ZPn06a9euxdfXl8jISCIjI0lOTrbUR7AKsnwHWRhfTtb1cOLs9K6Gr//RteIP7TMoZTrqHp8KWsst0XM6PI6OC/dhSybzVfoVKn7SdOaE1NDQ5ua8npYKT7AQiyexgQMHsmjRImbMmEGLFi04d+4cu3fvNnT2R0REcP9+9nLDK1asIDMzk1deeQUvLy/DY9GiRZb6CFahwMvJxzr23RxtuDSrm+HrWerhxEuOOD28TMK+pcUU4ZMNXqMv/jFB+Tu15feJllyZrxlk1EZUKSp/SkXH/oQJE5gwYUKe+/bv32/0dXh4ePEHVAYV9KudM8FlPc05TScWF+ZqhrBQtQq7w5/yu8yfps1aUrtK3oVni0umRkcDWQTjFH8AMEM9gsRHxW9rVnbk+1F+JRqPUDpY/ExMKBl5TTvKkt+dy9XDsvsbf9Z25Ii2MbZkUnn/JJ5fvJ+LdxO4GZNcYgNh5ehYoFqNSqblL20bduvaGvbte78TPhXF0jrlkUhi5URBl5M5k1jOdl0bebDGkMhkTNGMIV1S8aziEq8oDvLissM8v/gAi/6+VjxBP5KcoUGSJIYr/qKF/CaJkj0zchS/vTqne4HHC2WbSGLlREGXkwUNoQpslD0QOULyYInmFQCmKTdQGf3I+eX7bhZFiHm6FZNMk5l/8cGaP3hfuQWABZrBRKGfobHtrfbYiRVayzWRxMqJgga75rzUzOsu5vkZLxief6vtyUWdL66yFEMptOL044kIQKJ3xEIcZRmc0DXgJ21nw/7mPq7FHoNQuokkVk6YejmZFxcHFdc/6QGAFgWT1GPRSHJ6K47zvDykKMPMU1/5EToqLpAhKZmiHmMofuvlYvnZA4LliSRWThQ8TsyoYZ5slHJ+GK2/+3dJqskabS8A5qi+owKpfBn8Lw+LeBXY8RtD2Hr4PDNV3wPwpeYlbknehv05F3UUyq9SMcRCKH6F6dh/XFvf7JVClmpeorv8JL7yKD5QbmbmHgcu/BfPmuFt838BMyzYdZWdoZEsUW2goiyZKzofVmlfBODcjK7EJmeU+BAPoXQSZ2LlhKnTjgpip1JwdU536rpXIB1bpjwqvjtUsZfWsmscvfmgCCLVW3ngJh3l53lJcRidJGOyeixqlBz6sDOuDjbUcXcqcNiIUH6IJFZOmDrt6ElpwU6lYNc7+qWfj+kas1nTCblMYoFqDZrMdDTap6+UdCM6GQfSmav6FoDvtN05L9Whc/0qYiyYkItIYuWEvIDv9JM69h+nzDEmY55mMDGSC3Xldxmv3MaGR8vkPI3eyw7znvJnqsli+U+qzGLN/wHw3UgxIl/ITSSxcsLUjn1zL9ESqMBM9XAAxiu2cfTYYW4/KPySPT8cv019zTVGKnYD8JF6NKnYGa2uIQg5iSRWThTYsf+Uk6Z36vzZo22NjUzLuMSldF74T6Ffa/bv55ivWo1cJrFV+ywHdM2p4mSLf61KTxWjUHaJJFZOmDp30tx0NjygBvriuyNIkuxpJb/Ba4o9hZ5P+briTxrK7/BAcmKO+jUATn2U99LjggAiiZUbBSWnp1295sbcHkRSiU81rwLwoXIzC37aS3RiulmvI8VcY6JyKwCz1UN5iLO4jBSeSCSxcqKgRCUzcZxYfscqFXJ2vdOBjdrnOaWrRwVZOv5XPmHomhMmv46k03Jy2TBsZRoOaJuxTdee7RPai8tI4YlEEisnCr6czNHO7AtKvYZezrSqUYkp6jFkSEq6KM5RL/Zvk479Yu+/fDT9A/zlV0mVbPlIM5rF/9eCZtVcCxWLUL6IJFZOFHw5WTSDRje9/gw3pGos1/QDYKbqe37450yBx2i0On7ce5zJyh8BWKQZQONGTXi5dbUiiUko+0QSKydM7th/inymUsi5Na8nK7R9uKarRmVZIvb7ZhZ4zOd7rzNbtQ5nWRrndLVYp+3GN0MLLv4iCDmJJFZOFMW0I1PI5TIWvdqGKeox6CQZrygOEnrw9zzbanUSNw78RDfFadSSgsnq1zk8WdyJFMwjkpjwVJeTeR3at0VVzkj1WK/Vr0Pmsvd9yEzN1S42Jpo5qnUArNT2xr1ua7xdRaVuwTwiiZUTBd2dLI4KQeELerFIM4C7UiWqy2NYNXtMrjYHv34Td1k8N3VefKXpJwp9CIUiklg5Yfq0o6J73ZA5/Q3Fd0crdhJz/bhh37qNP/B/sqzit2M4OeNF895YEB4pFUls+fLl+Pr6Ymdnh7+/PydPniyw/c8//0yDBg2ws7OjadOm7Ny5s4QitV4F94kVz5I2dioF7XsMZpu2HQqZRPSG19GqM0lITOS5a58AsFHzPJ/87w1cHFTFEoNQ9lk8iW3evJmgoCBmzpzJmTNnaN68Od26dSM6OjrP9kePHmXQoEGMHj2as2fP0q9fP/r168fFixdLOHLrUlC/19OME3tS/hvToZZ+9L1Ugcby2xzdMIsNn71FLXkkUZIrB33fpp6Hk1nvKQg5WTyJLVmyhLFjxzJy5EgaNWrEypUrcXBwYO3atXm2/+KLL+jevTsffPABDRs2ZM6cObRq1YqvvvqqhCO3Miau7CpR9DUkH+BimAfpF/4Nbyj+BGC6eiTfjOlc0KGC8EQWTWKZmZmEhIQQGJh9W10ulxMYGMixY8fyPObYsWNG7QG6deuWb/uMjAwSExONHuWRj1v+iwnaKLN/DMydt23qedtWXQcOaptiK1OjlOnYqfVj2ezp5r2ZIOTBokksNjYWrVaLh4eH0XYPDw8iIyPzPCYyMtKs9vPnz8fFxcXw8PHxKZrgrcQPo/0Y5Fedt7vUybeNt6s9r7b1YXhADZNrOI7vVBtPZzve6FjbxEhkTNWMJkmyJ06qQPPXV2GrFPUihadX5guFTJkyhaCgIMPXiYmJ5SqRdahbhQ51qzyx3YKXm5n1uh92b8AH3eqbdFPg6pzuXPgvgWbVXPjf6sr8r2sDGvjUNOv9BCE/Fk1ilStXRqFQEBUVZbQ9KioKT0/PPI/x9PQ0q72trS22trZFE7BgxNS7mnYqBX419ZWSVo7vXZwhCeWQRS8nbWxsaN26NcHBwYZtOp2O4OBgAgIC8jwmICDAqD3Anj178m0vCELZZvHLyaCgIIYPH06bNm3w8/Nj6dKlpKSkMHLkSACGDRtG1apVmT9/PgDvvPMOHTt2ZPHixfTq1YtNmzZx+vRpVq1aZcmPIQiChVg8iQ0cOJCYmBhmzJhBZGQkLVq0YPfu3YbO+4iICOQ5SvW0a9eOH3/8kWnTpjF16lTq1q3L77//TpMmTSz1EQRBsCCZVNjF0K1UYmIiLi4uJCQk4OzsbOlwBEF4jLm/oxYf7CoIgvA0RBITBMGqiSQmCIJVs3jHfknL6gIsr9OPBKG0y/rdNLW7vtwlsaSkJIByNWpfEKxRUlISLi4uT2xX7u5O6nQ67t27h5OTU7Gto1WQrGlPd+7cKZd3R8XnL7+f39TPLkkSSUlJeHt7Gw2vyk+5OxOTy+VUq2b5cmDOzs7l7oc4J/H5y+/nN+Wzm3IGlkV07AuCYNVEEhMEwaqJJFbCbG1tmTlzZrldWUN8/vL7+Yvrs5e7jn1BEMoWcSYmCIJVE0lMEASrJpKYIAhWTSQxQRCsmkhiJSAuLo4hQ4bg7OyMq6sro0ePJjk5ucD2b7/9NvXr18fe3p7q1aszceJEEhISSjDqwivPFd3N+eyrV6+mQ4cOuLm54ebmRmBg4BP/r0o7c7/3WTZt2oRMJqNfv37mv6kkFLvu3btLzZs3l44fPy4dOnRIqlOnjjRo0KB824eGhkovvfSStH37dunGjRtScHCwVLduXenll18uwagLZ9OmTZKNjY20du1a6dKlS9LYsWMlV1dXKSoqKs/2R44ckRQKhfTZZ59Jly9flqZNmyapVCopNDS0hCN/euZ+9sGDB0vLly+Xzp49K125ckUaMWKE5OLiIv33338lHHnRMPfzZwkLC5OqVq0qdejQQerbt6/Z7yuSWDG7fPmyBEinTp0ybNu1a5ckk8mku3fvmvw6W7ZskWxsbCS1Wl0cYRYZPz8/6a233jJ8rdVqJW9vb2n+/Pl5th8wYIDUq1cvo23+/v7SG2+8UaxxFgdzP/vjNBqN5OTkJK1fv764QixWhfn8Go1GateunbRmzRpp+PDhhUpi4nKymB07dgxXV1fatGlj2BYYGIhcLufEiRMmv07WUr1KZemd7loSFd1Lq8J89selpqaiVqupWLFicYVZbAr7+WfPno27uzujR48u9HuX3t+IMiIyMhJ3d3ejbUqlkooVK+ZbtfxxsbGxzJkzh9dff704QiwyBVV0v3r1ap7HmFvRvbQqzGd/3KRJk/D29s6V1K1BYT7/4cOH+fbbbzl37txTvbc4EyukyZMnI5PJCnyY+sNbkMTERHr16kWjRo34+OOPnz5woVRasGABmzZt4rfffsPOzs7S4RS7pKQkhg4dyurVq6lcufJTvZY4Eyuk9957jxEjRhTYplatWnh6ehIdHW20XaPREBcXl2/V8ixJSUl0794dJycnfvvtN1Qq1dOGXaxKoqJ7aVWYz55l0aJFLFiwgL1799KsWbPiDLPYmPv5b968SXh4OL17Z1eE1+l0gP5K5dq1a9SuXdu0Ny9MB55guqyO/dOnTxu2/fXXX0/s2E9ISJCeeeYZqWPHjlJKSkpJhFok/Pz8pAkTJhi+1mq1UtWqVQvs2H/xxReNtgUEBFhtx745n12SJOnTTz+VnJ2dpWPHjpVEiMXKnM+flpYmhYaGGj369u0rdenSRQoNDZUyMjJMfl+RxEpA9+7dpZYtW0onTpyQDh8+LNWtW9doiMV///0n1a9fXzpx4oQkSfoE5u/vLzVt2lS6ceOGdP/+fcNDo9FY6mOYZNOmTZKtra20bt066fLly9Lrr78uubq6SpGRkZIkSdLQoUOlyZMnG9ofOXJEUiqV0qJFi6QrV65IM2fOtOohFuZ89gULFkg2NjbSL7/8YvQ9TkpKstRHeCrmfv7HFfbupEhiJeDBgwfSoEGDpAoVKkjOzs7SyJEjjX5Qw8LCJEDat2+fJEmStG/fPgnI8xEWFmaZD2GGZcuWSdWrV5dsbGwkPz8/6fjx44Z9HTt2lIYPH27UfsuWLVK9evUkGxsbqXHjxtKOHTtKOOKiY85nr1GjRp7f45kzZ5Z84EXE3O99ToVNYmIpHkEQrJq4OykIglUTSUwQBKsmkpggCFZNJDFBEKyaSGKCIFg1kcQEQbBqIokJgmDVRBITBMGqiSQmlAojRozIcyWQ7t27Wzo0oZQTq1gIpUb37t357rvvjLblVy1arVbnWtUjMzMTGxsbs9+3sMcJpYM4ExNKDVtbWzw9PY0ebm5uAMhkMlasWEGfPn1wdHRk7ty5fPzxx7Ro0YI1a9ZQs2ZNwzpcERER9O3blwoVKuDs7MyAAQOMlojJ7zjBOokkJliNjz/+mP79+xMaGsqoUaMAuHHjBr/++itbt27l3Llz6HQ6+vbtS1xcHAcOHGDPnj3cunWLgQMHGr3W48cJ1ktcTgqlxp9//kmFChWMtk2dOpWpU6cCMHjwYEaOHGm0PzMzk++//54qVaoAsGfPHkJDQwkLC8PHxweA77//nsaNG3Pq1Cnatm2b53GC9RJJTCg1OnfuzIoVK4y25SyakbPYSpYaNWoYJaIrV67g4+NjSGAAjRo1wtXVlStXrhiS2OPHCdZLJDGh1HB0dKROnToF7jdlm6nvJZQNok9MKFMaNmzInTt3uHPnjmHb5cuXiY+Pp1GjRhaMTCgu4kxMKDUyMjJylWpTKpVmVcMJDAykadOmDBkyhKVLl6LRaBg/fjwdO3bM83JUsH7iTEwoNXbv3o2Xl5fR49lnnzXrNWQyGdu2bcPNzY3nnnuOwMBAatWqxebNm4spasHSxPLUgiBYNXEmJgiCVRNJTBAEqyaSmCAIVk0kMUEQrJpIYoIgWDWRxARBsGoiiQmCYNVEEhMEwaqJJCYIglUTSUwQBKsmkpggCFZNJDFBEKza/wOOCky5TrG20gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019261, + "end_time": "2024-03-22T19:10:49.848721", + "exception": false, + "start_time": "2024-03-22T19:10:49.829460", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 8219.40624, + "end_time": "2024-03-22T19:10:52.592795", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/lct_gan/42/mlu-eval.ipynb", + "output_path": "eval/treatment/lct_gan/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/lct_gan/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T16:53:53.186555", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/lct_gan/model.pt b/treatment/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f1e9ed20c9f26ecaeba13e8709e31b240056ce7 --- /dev/null +++ b/treatment/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7605c6a45b313a0595da3f1d70c51c9d4cd98069d6756acf793194c058040638 +size 74778241 diff --git a/treatment/lct_gan/params.json b/treatment/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..c818c9a697a558e26da078791e3837ca33efd4f4 --- /dev/null +++ b/treatment/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 0.2, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/realtabformer/eval.csv b/treatment/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..159b5167c64160dba67749ce01c9f06e7e303548 --- /dev/null +++ b/treatment/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.0,0.3809523773193367,0.005029816041935322,6.139832019805908,0.46518340706825256,7.454497337341309,0.9615514278411865,0.00015098779113031924,27.904475212097168,0.043616220355033875,2446463.5,0.07092119008302689,0.24172629415988922,0.00011316310701658949,34.044307231903076 diff --git a/treatment/realtabformer/history.csv b/treatment/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..fb22adf4e466e4ee0835c3a5abe661b4a4f72bd5 --- /dev/null +++ b/treatment/realtabformer/history.csv @@ -0,0 +1,5 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03997775016380672,4.1784445257359195,0.003654398211962573,1.125379170803628,0.0,0.0,0.0,0.0,0.04061377539565386,900,450,674.5398631095886,1.4989774735768635,0.7494887367884318,0.13649817022379668,0.014926234884779453,5.134242437605607,0.0010615593935528929,0.0,0.0,0.0,0.0,0.0,0.014926234884779453,450,225,201.71023845672607,0.8964899486965603,0.44824497434828015,0.10115677190537907 +1,0.013389052229801714,1.3475211419678794,0.0008060232592924967,0.8510899391982396,0.0,0.0,0.0,0.0,0.013631856909132883,900,450,668.8134686946869,1.4862521526548598,0.7431260763274299,0.1980984934745033,0.008773729122132234,3.148535844511999,0.0007421919603760864,0.0,0.0,0.0,0.0,0.0,0.008773729122132234,450,225,197.29309678077698,0.8768582079145644,0.4384291039572822,0.0931849179521747 +2,0.007238608380309618,1.7181790947342477,0.0001944728850099263,0.5850073061873101,0.0,0.0,0.0,0.0,0.0073909809483623376,900,450,671.3675940036774,1.4919279866748385,0.7459639933374193,0.18768813919843738,0.011670945049314128,4.705433707986743,0.0008353222444575463,0.0,0.0,0.0,0.0,0.0,0.011670945049314128,450,225,201.83496594429016,0.8970442930857341,0.44852214654286704,0.08814679903484034 +3,0.007349486502970738,1.3100048318551674,0.0001732898741345901,0.552599703557272,0.0,0.0,0.0,0.0,0.007496255471258072,900,450,675.7493937015533,1.5016653193367853,0.7508326596683926,0.1919305363571362,0.008832458678805387,4.634036721558722,0.000772223847405924,0.0,0.0,0.0,0.0,0.0,0.008832458678805387,450,225,201.21838569641113,0.8943039364284939,0.44715196821424696,0.07392007318481268 diff --git a/treatment/realtabformer/mlu-eval.ipynb b/treatment/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..cccaef42e91621965476aae815909e7076903807 --- /dev/null +++ b/treatment/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2312 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.509609Z", + "iopub.status.busy": "2024-03-22T21:16:01.509278Z", + "iopub.status.idle": "2024-03-22T21:16:01.542820Z", + "shell.execute_reply": "2024-03-22T21:16:01.541972Z" + }, + "papermill": { + "duration": 0.048611, + "end_time": "2024-03-22T21:16:01.545010", + "exception": false, + "start_time": "2024-03-22T21:16:01.496399", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.569755Z", + "iopub.status.busy": "2024-03-22T21:16:01.569411Z", + "iopub.status.idle": "2024-03-22T21:16:01.576059Z", + "shell.execute_reply": "2024-03-22T21:16:01.575231Z" + }, + "papermill": { + "duration": 0.020997, + "end_time": "2024-03-22T21:16:01.577917", + "exception": false, + "start_time": "2024-03-22T21:16:01.556920", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.601394Z", + "iopub.status.busy": "2024-03-22T21:16:01.601121Z", + "iopub.status.idle": "2024-03-22T21:16:01.605121Z", + "shell.execute_reply": "2024-03-22T21:16:01.604363Z" + }, + "papermill": { + "duration": 0.018064, + "end_time": "2024-03-22T21:16:01.607047", + "exception": false, + "start_time": "2024-03-22T21:16:01.588983", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.630333Z", + "iopub.status.busy": "2024-03-22T21:16:01.629817Z", + "iopub.status.idle": "2024-03-22T21:16:01.633637Z", + "shell.execute_reply": "2024-03-22T21:16:01.632807Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017561, + "end_time": "2024-03-22T21:16:01.635544", + "exception": false, + "start_time": "2024-03-22T21:16:01.617983", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.658463Z", + "iopub.status.busy": "2024-03-22T21:16:01.658206Z", + "iopub.status.idle": "2024-03-22T21:16:01.663633Z", + "shell.execute_reply": "2024-03-22T21:16:01.662844Z" + }, + "papermill": { + "duration": 0.019036, + "end_time": "2024-03-22T21:16:01.665456", + "exception": false, + "start_time": "2024-03-22T21:16:01.646420", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "153f3577", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.691381Z", + "iopub.status.busy": "2024-03-22T21:16:01.690621Z", + "iopub.status.idle": "2024-03-22T21:16:01.696401Z", + "shell.execute_reply": "2024-03-22T21:16:01.695551Z" + }, + "papermill": { + "duration": 0.021042, + "end_time": "2024-03-22T21:16:01.698287", + "exception": false, + "start_time": "2024-03-22T21:16:01.677245", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/realtabformer/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011356, + "end_time": "2024-03-22T21:16:01.720851", + "exception": false, + "start_time": "2024-03-22T21:16:01.709495", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.744904Z", + "iopub.status.busy": "2024-03-22T21:16:01.744611Z", + "iopub.status.idle": "2024-03-22T21:16:01.754648Z", + "shell.execute_reply": "2024-03-22T21:16:01.753746Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024601, + "end_time": "2024-03-22T21:16:01.756631", + "exception": false, + "start_time": "2024-03-22T21:16:01.732030", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/realtabformer/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:01.781822Z", + "iopub.status.busy": "2024-03-22T21:16:01.781532Z", + "iopub.status.idle": "2024-03-22T21:16:03.880213Z", + "shell.execute_reply": "2024-03-22T21:16:03.879298Z" + }, + "papermill": { + "duration": 2.113569, + "end_time": "2024-03-22T21:16:03.882203", + "exception": false, + "start_time": "2024-03-22T21:16:01.768634", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:03.909111Z", + "iopub.status.busy": "2024-03-22T21:16:03.908710Z", + "iopub.status.idle": "2024-03-22T21:16:03.923589Z", + "shell.execute_reply": "2024-03-22T21:16:03.922838Z" + }, + "papermill": { + "duration": 0.030682, + "end_time": "2024-03-22T21:16:03.925451", + "exception": false, + "start_time": "2024-03-22T21:16:03.894769", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:03.949182Z", + "iopub.status.busy": "2024-03-22T21:16:03.948882Z", + "iopub.status.idle": "2024-03-22T21:16:03.956011Z", + "shell.execute_reply": "2024-03-22T21:16:03.955324Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021402, + "end_time": "2024-03-22T21:16:03.957981", + "exception": false, + "start_time": "2024-03-22T21:16:03.936579", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:03.981722Z", + "iopub.status.busy": "2024-03-22T21:16:03.981467Z", + "iopub.status.idle": "2024-03-22T21:16:04.073372Z", + "shell.execute_reply": "2024-03-22T21:16:04.072644Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.106321, + "end_time": "2024-03-22T21:16:04.075491", + "exception": false, + "start_time": "2024-03-22T21:16:03.969170", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:04.102547Z", + "iopub.status.busy": "2024-03-22T21:16:04.102266Z", + "iopub.status.idle": "2024-03-22T21:16:08.861893Z", + "shell.execute_reply": "2024-03-22T21:16:08.861116Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.775879, + "end_time": "2024-03-22T21:16:08.864333", + "exception": false, + "start_time": "2024-03-22T21:16:04.088454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 21:16:06.437912: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 21:16:06.437991: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 21:16:06.439715: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:08.889960Z", + "iopub.status.busy": "2024-03-22T21:16:08.889374Z", + "iopub.status.idle": "2024-03-22T21:16:08.895447Z", + "shell.execute_reply": "2024-03-22T21:16:08.894744Z" + }, + "papermill": { + "duration": 0.020825, + "end_time": "2024-03-22T21:16:08.897377", + "exception": false, + "start_time": "2024-03-22T21:16:08.876552", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:08.923043Z", + "iopub.status.busy": "2024-03-22T21:16:08.922708Z", + "iopub.status.idle": "2024-03-22T21:16:31.891605Z", + "shell.execute_reply": "2024-03-22T21:16:31.890521Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.984464, + "end_time": "2024-03-22T21:16:31.894136", + "exception": false, + "start_time": "2024-03-22T21:16:08.909672", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T21:16:31.924230Z", + "iopub.status.busy": "2024-03-22T21:16:31.923854Z", + "iopub.status.idle": "2024-03-22T21:16:31.930601Z", + "shell.execute_reply": "2024-03-22T21:16:31.929661Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024535, + "end_time": "2024-03-22T21:16:31.932760", + "exception": false, + "start_time": "2024-03-22T21:16:31.908225", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:31.959846Z", + "iopub.status.busy": "2024-03-22T21:16:31.959527Z", + "iopub.status.idle": "2024-03-22T21:16:31.964480Z", + "shell.execute_reply": "2024-03-22T21:16:31.963643Z" + }, + "papermill": { + "duration": 0.020899, + "end_time": "2024-03-22T21:16:31.966551", + "exception": false, + "start_time": "2024-03-22T21:16:31.945652", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:31.994153Z", + "iopub.status.busy": "2024-03-22T21:16:31.993837Z", + "iopub.status.idle": "2024-03-22T21:16:41.497048Z", + "shell.execute_reply": "2024-03-22T21:16:41.496077Z" + }, + "papermill": { + "duration": 9.519677, + "end_time": "2024-03-22T21:16:41.499124", + "exception": false, + "start_time": "2024-03-22T21:16:31.979447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:41.528542Z", + "iopub.status.busy": "2024-03-22T21:16:41.528144Z", + "iopub.status.idle": "2024-03-22T21:16:41.866503Z", + "shell.execute_reply": "2024-03-22T21:16:41.865498Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.355412, + "end_time": "2024-03-22T21:16:41.868649", + "exception": false, + "start_time": "2024-03-22T21:16:41.513237", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:41.899593Z", + "iopub.status.busy": "2024-03-22T21:16:41.899282Z", + "iopub.status.idle": "2024-03-22T21:16:42.010321Z", + "shell.execute_reply": "2024-03-22T21:16:42.009337Z" + }, + "papermill": { + "duration": 0.129193, + "end_time": "2024-03-22T21:16:42.012558", + "exception": false, + "start_time": "2024-03-22T21:16:41.883365", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T21:16:42.042357Z", + "iopub.status.busy": "2024-03-22T21:16:42.042043Z", + "iopub.status.idle": "2024-03-22T21:16:42.588076Z", + "shell.execute_reply": "2024-03-22T21:16:42.587088Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.563354, + "end_time": "2024-03-22T21:16:42.590230", + "exception": false, + "start_time": "2024-03-22T21:16:42.026876", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:42.621551Z", + "iopub.status.busy": "2024-03-22T21:16:42.620716Z", + "iopub.status.idle": "2024-03-22T21:16:42.625070Z", + "shell.execute_reply": "2024-03-22T21:16:42.624257Z" + }, + "papermill": { + "duration": 0.02226, + "end_time": "2024-03-22T21:16:42.626963", + "exception": false, + "start_time": "2024-03-22T21:16:42.604703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:42.654211Z", + "iopub.status.busy": "2024-03-22T21:16:42.653945Z", + "iopub.status.idle": "2024-03-22T21:16:42.660740Z", + "shell.execute_reply": "2024-03-22T21:16:42.659851Z" + }, + "papermill": { + "duration": 0.022498, + "end_time": "2024-03-22T21:16:42.662580", + "exception": false, + "start_time": "2024-03-22T21:16:42.640082", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19390534" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:42.690063Z", + "iopub.status.busy": "2024-03-22T21:16:42.689798Z", + "iopub.status.idle": "2024-03-22T21:16:42.844098Z", + "shell.execute_reply": "2024-03-22T21:16:42.843156Z" + }, + "papermill": { + "duration": 0.170368, + "end_time": "2024-03-22T21:16:42.846290", + "exception": false, + "start_time": "2024-03-22T21:16:42.675922", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 52992] --\n", + "├─Adapter: 1-1 [2, 2648, 52992] --\n", + "│ └─Embedding: 2-1 [2, 2648, 69, 768] (215,808)\n", + "│ └─TensorInductionPoint: 2-2 [69, 1] 69\n", + "│ └─Sequential: 2-3 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 787,456\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 52992] (recursive)\n", + "│ └─Embedding: 2-4 [2, 661, 69, 768] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [69, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-7 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 19,606,342\n", + "Trainable params: 19,390,534\n", + "Non-trainable params: 215,808\n", + "Total mult-adds (M): 77.67\n", + "========================================================================================================================\n", + "Input size (MB): 1.83\n", + "Forward/backward pass size (MB): 3885.09\n", + "Params size (MB): 78.43\n", + "Estimated Total Size (MB): 3965.34\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:16:42.877991Z", + "iopub.status.busy": "2024-03-22T21:16:42.877640Z", + "iopub.status.idle": "2024-03-22T22:34:44.654907Z", + "shell.execute_reply": "2024-03-22T22:34:44.654090Z" + }, + "papermill": { + "duration": 4681.795472, + "end_time": "2024-03-22T22:34:44.657060", + "exception": false, + "start_time": "2024-03-22T21:16:42.861588", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03997775016380672, 'avg_role_model_std_loss': 4.1784445257359195, 'avg_role_model_mean_pred_loss': 0.003654398211962573, 'avg_role_model_g_mag_loss': 1.125379170803628, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.04061377539565386, 'n_size': 900, 'n_batch': 450, 'duration': 674.5398631095886, 'duration_batch': 1.4989774735768635, 'duration_size': 0.7494887367884318, 'avg_pred_std': 0.13649817022379668}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014926234884779453, 'avg_role_model_std_loss': 5.134242437605607, 'avg_role_model_mean_pred_loss': 0.0010615593935528929, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014926234884779453, 'n_size': 450, 'n_batch': 225, 'duration': 201.71023845672607, 'duration_batch': 0.8964899486965603, 'duration_size': 0.44824497434828015, 'avg_pred_std': 0.10115677190537907}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013389052229801714, 'avg_role_model_std_loss': 1.3475211419678794, 'avg_role_model_mean_pred_loss': 0.0008060232592924967, 'avg_role_model_g_mag_loss': 0.8510899391982396, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013631856909132883, 'n_size': 900, 'n_batch': 450, 'duration': 668.8134686946869, 'duration_batch': 1.4862521526548598, 'duration_size': 0.7431260763274299, 'avg_pred_std': 0.1980984934745033}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008773729122132234, 'avg_role_model_std_loss': 3.148535844511999, 'avg_role_model_mean_pred_loss': 0.0007421919603760864, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008773729122132234, 'n_size': 450, 'n_batch': 225, 'duration': 197.29309678077698, 'duration_batch': 0.8768582079145644, 'duration_size': 0.4384291039572822, 'avg_pred_std': 0.0931849179521747}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007238608380309618, 'avg_role_model_std_loss': 1.7181790947342477, 'avg_role_model_mean_pred_loss': 0.0001944728850099263, 'avg_role_model_g_mag_loss': 0.5850073061873101, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0073909809483623376, 'n_size': 900, 'n_batch': 450, 'duration': 671.3675940036774, 'duration_batch': 1.4919279866748385, 'duration_size': 0.7459639933374193, 'avg_pred_std': 0.18768813919843738}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011670945049314128, 'avg_role_model_std_loss': 4.705433707986743, 'avg_role_model_mean_pred_loss': 0.0008353222444575463, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011670945049314128, 'n_size': 450, 'n_batch': 225, 'duration': 201.83496594429016, 'duration_batch': 0.8970442930857341, 'duration_size': 0.44852214654286704, 'avg_pred_std': 0.08814679903484034}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007349486502970738, 'avg_role_model_std_loss': 1.3100048318551674, 'avg_role_model_mean_pred_loss': 0.0001732898741345901, 'avg_role_model_g_mag_loss': 0.552599703557272, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007496255471258072, 'n_size': 900, 'n_batch': 450, 'duration': 675.7493937015533, 'duration_batch': 1.5016653193367853, 'duration_size': 0.7508326596683926, 'avg_pred_std': 0.1919305363571362}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008832458678805387, 'avg_role_model_std_loss': 4.634036721558722, 'avg_role_model_mean_pred_loss': 0.000772223847405924, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008832458678805387, 'n_size': 450, 'n_batch': 225, 'duration': 201.21838569641113, 'duration_batch': 0.8943039364284939, 'duration_size': 0.44715196821424696, 'avg_pred_std': 0.07392007318481268}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005864556821421445, 'avg_role_model_std_loss': 1.186049147437506, 'avg_role_model_mean_pred_loss': 0.00015830445035028238, 'avg_role_model_g_mag_loss': 0.5711013979733394, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006005265417448634, 'n_size': 900, 'n_batch': 450, 'duration': 674.45822930336, 'duration_batch': 1.4987960651185777, 'duration_size': 0.7493980325592888, 'avg_pred_std': 0.18369348015795975}\n", + "Time out: 4172.159170627594/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 525, 'role_model_metrics': {'avg_loss': 0.005029816059507788, 'avg_g_mag_loss': 0.2514285687037889, 'avg_g_cos_loss': 0.0, 'pred_duration': 27.87242317199707, 'grad_duration': 6.086328506469727, 'total_duration': 33.9587516784668, 'pred_std': 0.24172629415988922, 'std_loss': 0.00011316310701658949, 'mean_pred_loss': 0.00015098780568223447, 'pred_rmse': 0.07092119008302689, 'pred_mae': 0.043616220355033875, 'pred_mape': 2446463.25, 'grad_rmse': 0.9615513682365417, 'grad_mae': 0.46518340706825256, 'grad_mape': 7.454497337341309}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.005029816059507788, 'avg_g_mag_loss': 0.2514285687037889, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 27.87242317199707, 'avg_grad_duration': 6.086328506469727, 'avg_total_duration': 33.9587516784668, 'avg_pred_std': 0.24172629415988922, 'avg_std_loss': 0.00011316310701658949, 'avg_mean_pred_loss': 0.00015098780568223447}, 'min_metrics': {'avg_loss': 0.005029816059507788, 'avg_g_mag_loss': 0.2514285687037889, 'avg_g_cos_loss': 0.0, 'pred_duration': 27.87242317199707, 'grad_duration': 6.086328506469727, 'total_duration': 33.9587516784668, 'pred_std': 0.24172629415988922, 'std_loss': 0.00011316310701658949, 'mean_pred_loss': 0.00015098780568223447, 'pred_rmse': 0.07092119008302689, 'pred_mae': 0.043616220355033875, 'pred_mape': 2446463.25, 'grad_rmse': 0.9615513682365417, 'grad_mae': 0.46518340706825256, 'grad_mape': 7.454497337341309}, 'model_metrics': {'realtabformer': {'avg_loss': 0.005029816059507788, 'avg_g_mag_loss': 0.2514285687037889, 'avg_g_cos_loss': 0.0, 'pred_duration': 27.87242317199707, 'grad_duration': 6.086328506469727, 'total_duration': 33.9587516784668, 'pred_std': 0.24172629415988922, 'std_loss': 0.00011316310701658949, 'mean_pred_loss': 0.00015098780568223447, 'pred_rmse': 0.07092119008302689, 'pred_mae': 0.043616220355033875, 'pred_mape': 2446463.25, 'grad_rmse': 0.9615513682365417, 'grad_mae': 0.46518340706825256, 'grad_mape': 7.454497337341309}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:34:44.690729Z", + "iopub.status.busy": "2024-03-22T22:34:44.690390Z", + "iopub.status.idle": "2024-03-22T22:34:44.694856Z", + "shell.execute_reply": "2024-03-22T22:34:44.693981Z" + }, + "papermill": { + "duration": 0.023889, + "end_time": "2024-03-22T22:34:44.696895", + "exception": false, + "start_time": "2024-03-22T22:34:44.673006", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:34:44.729578Z", + "iopub.status.busy": "2024-03-22T22:34:44.729280Z", + "iopub.status.idle": "2024-03-22T22:34:44.859492Z", + "shell.execute_reply": "2024-03-22T22:34:44.858686Z" + }, + "papermill": { + "duration": 0.149537, + "end_time": "2024-03-22T22:34:44.861809", + "exception": false, + "start_time": "2024-03-22T22:34:44.712272", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:34:44.896759Z", + "iopub.status.busy": "2024-03-22T22:34:44.896419Z", + "iopub.status.idle": "2024-03-22T22:34:45.193385Z", + "shell.execute_reply": "2024-03-22T22:34:45.192492Z" + }, + "papermill": { + "duration": 0.316689, + "end_time": "2024-03-22T22:34:45.195432", + "exception": false, + "start_time": "2024-03-22T22:34:44.878743", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDBklEQVR4nO3deVxU9f4/8NfMwACyDYKssrghLoiGMmKlmdzISMMsinDJa2p9oY1bGaXSrZuY2S9v6m1PbiqhVnYLzURETdmUxUCQ3FgUAZEY9m3m8/vjMKMDg86weGbg/Xw8ziPnnM855z3TzJtzPuezCBhjDIQQYiCEfAdACCG6oKRFCDEolLQIIQaFkhYhxKBQ0iKEGBRKWoQQg0JJixBiUIz4DuBuUSgUKCsrg6WlJQQCAd/hEEI6YYyhrq4Ozs7OEAq7v54aNEmrrKwMrq6ufIdBCLmD0tJSDB8+vNvtgyZpWVpaAuA+ECsrK56jIYR0VltbC1dXV9VvtTuDJmkpbwmtrKwoaRGix+5UfUMV8YQQg0JJixBiUChpEUIMyqCp0yK9I5fL0dbWxncYxIAZGxtDJBL1+jiUtMhtMcZQXl6OmpoavkMhA4BEIoGjo2Ov2kpS0iK3pUxY9vb2GDJkCDXMJT3CGENjYyMqKysBAE5OTj0+FiWtThhjOF38F9rkCswYZcd3OLySy+WqhGVra8t3OMTAmZmZAQAqKythb2/f41tFqojvZPepUjz5WSre31+AwT4StbIOa8iQITxHQgYK5XepN/WjPUpa27Ztg4eHB0xNTSGVSpGRkXHb8nv37oWXlxdMTU3h7e2NAwcOdFv2+eefh0AgwObNm9XWV1dXIywsDFZWVpBIJFi+fDnq6+t7Ev5tBU5whNhIiLNltcgprenz4xsiuiUkfaUvvks6J63du3cjMjIS0dHRyMrKgo+PDwIDA1X3qp2lpKQgNDQUy5cvR3Z2NoKDgxEcHIy8vLwuZfft24e0tDQ4Ozt32RYWFoazZ88iMTERCQkJOH78OFauXKlr+HdkYy7Go5O4++0dacV9fnxCSC8xHfn5+bHw8HDVa7lczpydnVlMTIzG8iEhISwoKEhtnVQqZatWrVJbd+XKFebi4sLy8vKYu7s7+/jjj1Xb8vPzGQB26tQp1bpff/2VCQQCdvXqVa3ilslkDACTyWR3LJtVXM3cVyewMW8fYNX1LVodfyBqampi+fn5rKmpie9QyABxu++Utr9Rna60WltbkZmZiYCAANU6oVCIgIAApKamatwnNTVVrTwABAYGqpVXKBRYvHgxXn/9dUyYMEHjMSQSCaZOnapaFxAQAKFQiPT0dI3nbWlpQW1trdqircmuEkx0sUJruwJ7M0u13o8QJYFAgJ9++onvMPrUO++8g8mTJ/Mdhm63h1VVVZDL5XBwcFBb7+DggPLyco37lJeX37H8Bx98ACMjI7z00kvdHsPe3l5tnZGREYYOHdrteWNiYmBtba1adBmWRiAQYPF0dwDAzrQSKBSDu0KeGKbY2FhIJJI+O95rr72GpKSkPjteT/H+9DAzMxP//ve/ERsb26cVvlFRUZDJZKqltFS3K6Z5Ps6wNDVCSXUjjp+/3mdxEaJvWltbtSpnYWGhF01fdEpadnZ2EIlEqKioUFtfUVEBR0dHjfs4Ojretvzvv/+OyspKuLm5wcjICEZGRiguLsY//vEPeHh4qI7RuaK/vb0d1dXV3Z7XxMRENQxNT4ajGSI2whO+3EBkO9NKdNp3oGKMobG1nZeF6dj85ODBg7jvvvsgkUhga2uLRx99FBcvXgQAzJgxA6tXr1Yrf/36dRgbG+P48eMAgGvXriEoKAhmZmYYMWIE4uLi4OHh0eWptrZyc3Px4IMPwszMDLa2tli5cqXa0++jR4/Cz88P5ubmkEgkuPfee1FczD0IOnPmDGbPng1LS0tYWVnB19cXp0+fvu35jh49imXLlkEmk0EgEEAgEOCdd94BAHh4eOC9997DkiVLYGVlpXqgtXr1anh6emLIkCEYOXIk1q5dq9Y0ofPt4bPPPovg4GBs2rQJTk5OsLW1RXh4eL9399KpcalYLIavry+SkpIQHBwMgKuPSkpKQkREhMZ9/P39kZSUhFdeeUW1LjExEf7+/gCAxYsXa6zzWrx4MZYtW6Y6Rk1NDTIzM+Hr6wsAOHLkCBQKBaRSqS5vQSeLprtj+8kiHDlXgSt/NWK4zeBur9TUJsf4db/xcu78dwMxRKz917WhoQGRkZGYNGkS6uvrsW7dOixYsAA5OTkICwvDxo0bsWHDBtXV/e7du+Hs7Iz7778fALBkyRJUVVXh6NGjMDY2RmRkZLdPyLWJJTAwEP7+/jh16hQqKyvx3HPPISIiArGxsWhvb0dwcDBWrFiB7777Dq2trcjIyFDFFhYWhilTpuDTTz+FSCRCTk4OjI2Nb3vOGTNmYPPmzVi3bh0KCwsBcFdKSps2bcK6desQHR2tWmdpaYnY2Fg4OzsjNzcXK1asgKWlJd54441uz5OcnAwnJyckJyfjwoULeOqppzB58mSsWLGiR5+VNnRuER8ZGYmlS5di6tSp8PPzw+bNm9HQ0KBKMEuWLIGLiwtiYmIAAC+//DJmzZqFjz76CEFBQYiPj8fp06fxxRdfAABsbW27XHIaGxvD0dERY8eOBQCMGzcODz/8MFasWIHPPvsMbW1tiIiIwNNPP62xeURfGTXMAveOtsXJCzfwXUYJXg/06rdzkb61cOFCtdfffPMNhg0bhvz8fISEhOCVV17BiRMnVEkqLi4OoaGhEAgEOHfuHA4fPoxTp06pHv589dVXGDNmTI9iiYuLQ3NzM7799luYm5sDALZu3Yp58+bhgw8+gLGxMWQyGR599FGMGjUKAPedVyopKcHrr78OLy/u+6dNHGKxGNbW1hAIBBrvRh588EH84x//UFu3Zs0a1b89PDzw2muvIT4+/rZJy8bGBlu3boVIJIKXlxeCgoKQlJSkX0nrqaeewvXr17Fu3TqUl5dj8uTJOHjwoKqyvaSkRG1Q+hkzZiAuLg5r1qzBW2+9hTFjxuCnn37CxIkTdTrvrl27EBERgTlz5kAoFGLhwoX45JNPdA1fZ4uk7jh54QZ2nyrFy3M8ITbivRqQN2bGIuS/G8jbuXVx/vx5rFu3Dunp6aiqqoJCoQDAfT8nTpyIhx56CLt27cL999+Py5cvIzU1FZ9//jkAoLCwEEZGRrjnnntUxxs9ejRsbGx6FHtBQQF8fHxUCQsA7r33XigUChQWFmLmzJl49tlnERgYiL/97W8ICAhASEiIqn9eZGQknnvuOezYsQMBAQF48sknVcmtp259Eq+0e/dufPLJJ7h48SLq6+vR3t5+x2qVCRMmqHXHcXJyQm5ubq9iu5Me/QIjIiJQXFyMlpYWpKenq92iHT16FLGxsWrln3zySRQWFqKlpQV5eXl45JFHbnv8oqIitdtJABg6dCji4uJQV1cHmUyGb775Ru1yt78EjHeAg5UJqupbcfCs5ieVg4VAIMAQsREvi64PaebNm4fq6mp8+eWXSE9PVzWNUVY6h4WF4fvvv0dbWxvi4uLg7e0Nb2/vPv/MtLV9+3akpqZixowZ2L17Nzw9PZGWlgaAq0s6e/YsgoKCcOTIEYwfPx779u3r1fluTaAA16woLCwMjzzyCBISEpCdnY233377jpX0nW9TBQKB6g9Efxm8lw1aMhYJEernBgDYmUot5A3BjRs3UFhYiDVr1mDOnDkYN24c/vrrL7Uyjz32GJqbm3Hw4EHExcUhLCxMtW3s2LFob29Hdna2at2FCxe6HENb48aNw5kzZ9DQ0KBad/LkSQiFQlUVCABMmTIFUVFRSElJwcSJExEXF6fa5unpiVdffRWHDh3C448/ju3bt9/xvGKxGHK5XKsYU1JS4O7ujrfffhtTp07FmDFjVA8C9A0lLS2E+rlBJBQgo6ga58q1b6RK+GFjYwNbW1t88cUXuHDhAo4cOYLIyEi1Mubm5ggODsbatWtRUFCA0NBQ1TYvLy8EBARg5cqVyMjIQHZ2NlauXAkzM7MeNcsJCwuDqakpli5diry8PCQnJ+PFF1/E4sWL4eDggMuXLyMqKgqpqakoLi7GoUOHcP78eYwbNw5NTU2IiIjA0aNHUVxcjJMnT+LUqVNqdV7d8fDwQH19PZKSklBVVYXGxsZuy44ZMwYlJSWIj4/HxYsX8cknn/T6aq6/UNLSgoOVKR4az9XZ7aT+iHpPKBQiPj4emZmZmDhxIl599VV8+OGHXcqFhYXhzJkzuP/+++Hm5qa27dtvv4WDgwNmzpyJBQsWqJ6kmZqa6hzPkCFD8Ntvv6G6uhrTpk3DE088gTlz5mDr1q2q7efOncPChQvh6emJlStXIjw8HKtWrYJIJMKNGzewZMkSeHp6IiQkBHPnzsU///nPO553xowZeP755/HUU09h2LBh2LhxY7dl58+fj1dffRURERGYPHkyUlJSsHbtWp3f690gYLo2gDFQtbW1sLa2hkwm69EUYikXqvDMV+kwF4uQ/nYALEwG/lBkzc3NuHz5MkaMGNGjH+tAcuXKFbi6uuLw4cOYM2cO3+EYrNt9p7T9jdKVlpb8R9li5DBzNLTKsS/7Kt/hkH525MgR/Pzzz7h8+TJSUlLw9NNPw8PDAzNnzuQ7tEGPkpaW1PojphYP+gECB7q2tja89dZbmDBhAhYsWIBhw4apGpru2rULFhYWGhdNHf77y9y5c7uNY/369Xctjrtt4N/j9KHH7xmOjQcLUVhRh1NFf8FvxFC+QyL9JDAwEIGBmtukzZ8/v9ueGHdqqd6XvvrqKzQ1NWncNnTowP1uUtLSgbWZMR6b7Iz4U6XYmVZMSWuQsrS0hKWlJd9hwMXFhe8QeEG3hzpa1HGL+GveNVyva+E5GkIGH0paOproYo0pbhK0yRn2nKYBAgm52yhp9cAiKXe1FZdeAjkNEEjIXUVJqweCJjlBMsQYV2uakHyuZ8OVEEJ6hpJWD5gai/DUVG74Zpqxh5C7i5JWDz0jdYNAABz78zqKbzTceQcyqAzEiS30BSWtHnK3NcfMMcMAALvSaThmon/6emILgBt6SiAQoKampk+PqwtKWr2gbCG/53Qpmtu0GwKEENI7lLR6YbaXPVwkZqhpbMP+P67xHU7/YwxobeBnoYkt+mxii5aWFrz22mtwcXGBubk5pFIpjh49qtq3uLgY8+bNg42NDczNzTFhwgQcOHAARUVFmD17NgBu+B+BQIBnn322R59Hb1CL+F4QCQV4RuqGD38rxI60YizsmL1nwGprBNb335j8t/VWGSA2v3O5DjSxRfcTW0RERCA/Px/x8fFwdnbGvn378PDDDyM3NxdjxoxBeHg4Wltbcfz4cZibmyM/Px8WFhZwdXXFDz/8gIULF6KwsBBWVlYwMzPr0WfSG5S0eumpaa7YfPhP5JTWIO+qDBNdrPkOiYAmtuhuYouSkhJs374dJSUlqklhXnvtNRw8eBDbt2/H+vXrUVJSgoULF6qGnx45cqRqf2WfRnt7+z6vL9MWJa1esrMwwdyJTvj5TBl2phVjw8JJfIfUf4yHcFc8fJ1bBzSxhWa5ubmQy+Xw9PRUW9/S0qKaFeull17CCy+8gEOHDiEgIAALFy7EpEn6873uUZ3Wtm3b4OHhAVNTU0ilUmRkZNy2/N69e+Hl5QVTU1N4e3vjwIEDatvfeecdeHl5wdzcHDY2NggICFBNRKDk4eGhujdXLhs2bOhJ+H1usT9XIf9TzlXImvp3okpeCQTcLRofC01s0ScTW9TX10MkEiEzMxM5OTmqpaCgAP/+978BAM899xwuXbqExYsXIzc3F1OnTsWWLVv67L32GtNRfHw8E4vF7JtvvmFnz55lK1asYBKJhFVUVGgsf/LkSSYSidjGjRtZfn4+W7NmDTM2Nma5ubmqMrt27WKJiYns4sWLLC8vjy1fvpxZWVmxyspKVRl3d3f27rvvsmvXrqmW+vp6reOWyWQMAJPJZLq+5TtSKBQs8ONjzH11Avv690t9fny+NDU1sfz8fNbU1MR3KDqpqqpiANjx48dV637//XcGgO3bt48xxlh9fT0zNzdnP//8Mxs/fjzbsGGDqmxBQQEDwE6fPq1ad/78eQaAffzxx1rFcOu5vvjiC2ZjY6P2fd2/fz8TCoWsvLxc4/7Tp09nL774osZtTz/9NJs3b94dY9i1axezsLBQW1dYWNjls7mTN998k3l7ezPGuN8zAFZVVaX1/re63XdK29+ozknLz8+PhYeHq17L5XLm7OzMYmJiNJYPCQlhQUFBauukUilbtWpVt+dQBn/48GHVOnd3d62/MLc7Zn8kLcYY+za1iLmvTmCzNyUzhULRL+e42ww1acnlcmZra8sWLVrEzp8/z5KSkti0adPUEgljjIWFhTEfHx8mEAhYcXGx2jECAgLYPffcw9LT01lWVhabPXs2MzMzY5s3b9YqhlvP1dDQwJycnNjChQtZbm4uO3LkCBs5ciRbunQpY4yxS5cusTfffJOlpKSwoqIi9ttvvzFbW1v2n//8hzU2NrLw8HCWnJzMioqK2IkTJ9ioUaPYG2+8cccYlAnm8OHD7Pr166yhoUH1vj08PNgPP/zALl26xNLT09n69etZQkICY4yxl19+mR08eJBdunSJZWZmMqlUykJCQhhjjF25coUJBAIWGxvLKisrWV1dnVafh9JdT1otLS1MJBKp/Y9njLElS5aw+fPna9zH1dW1S7JZt24dmzRpUrfn+PDDD5m1tTW7fv26ar27uztzcHBgQ4cOZZMnT2YbN25kbW1t3cba3NzMZDKZaiktLe3XpFXX3MbGr/2Vua9OYCfPX7/zDgbAUJMWY4wlJiaycePGMRMTEzZp0iR29OjRLknrwIEDDACbOXNml/3LysrY3LlzmYmJCXN3d2dxcXHM3t6effbZZ1qdv/O5/vjjDzZ79mxmamrKhg4dylasWKH6wZeXl7Pg4GDm5OTExGIxc3d3Z+vWrWNyuZy1tLSwp59+mrm6ujKxWMycnZ1ZRESE1v9Pnn/+eWZra8sAsOjoaMYYY62trWzdunXMw8ODGRsbMycnJ7ZgwQL2xx9/MMYYi4iIYKNGjWImJiZs2LBhbPHixWpXVu+++y5zdHRkAoFAlXi1ddeT1tWrVxkAlpKSorb+9ddfZ35+fhr3MTY2ZnFxcWrrtm3bxuzt7dXW/fLLL8zc3JwJBALm7OzMMjIy1LZ/9NFHLDk5mZ05c4Z9+umnTCKRsFdffbXbWKOjoxmALkt/JS3GGFuzL5e5r05gz+84fefCBsCQk1ZfU/7Ru/Xqn+iuL5KW3jw9nD17NnJyclBVVYUvv/wSISEhSE9Ph729PQCozVs3adIkiMVirFq1CjExMTAxMelyvKioKLV9amtr4erq2q/vYdF0d+xIK8ah/AqUy5rhaD24Z7AxZEeOHEF9fT28vb1x7do1vPHGGzSxhZ7Q6emhnZ0dRCIRKioq1NZXVFSotQW5laOjo1blzc3NMXr0aEyfPh1ff/01jIyM8PXXX3cbi1QqRXt7O4qKijRuNzExgZWVldrS38Y6WsLPYyjkCobvMqg/oiGjiS30l05XWmKxGL6+vkhKSkJwcDAAQKFQICkpCRERERr38ff3R1JSEl555RXVusTERPj7+9/2XAqFAi0t3Q9nnJOTA6FQqLoS0xeL/N2RUVSN+FMliHhwNIxF1FPKENHEFvpL59vDyMhILF26FFOnToWfnx82b96MhoYGLFu2DADX/cHFxQUxMTEAgJdffhmzZs3CRx99hKCgIMTHx+P06dP44osvAHBdHN5//33Mnz8fTk5OqKqqwrZt23D16lU8+eSTAIDU1FSkp6er+l+lpqbi1VdfxaJFi3rc4K+/PDzBEXYWYlTUtuBwfgXmejvxHRLpYzSxBc96Upm2ZcsW5ubmxsRiMfPz82NpaWmqbbNmzeryRGHPnj3M09OTicViNmHCBLZ//361irkFCxYwZ2dnJhaLmZOTE5s/f75aRbzysau1tTUzNTVl48aNY+vXr2fNzc1ax9zfTR5u9eHBc8x9dQIL/SK138/Vn5SVpo2NjXyHQgaIxsbGXlfECxgbHLOOajvldl+4WtOE+z84AgUDDkfOwmh7i349X3+Ry+X4888/YW9vr+riQUhv3LhxA5WVlfD09IRIJFLbpu1vVG+eHg4kLhIzPOjlgMMFFdiVXozoeXevcrYviUQiSCQS1egGQ4YMUY08QIguGGNobGxEZWUlJBJJl4SlC0pa/WSxvzsOF1Tg+8wreD1wLIaIDfOjVj7l7emwLITcSiKRdNvSQFuG+UsyAPePtoO77RAU32jEzzlleNrPje+QekQgEMDJyQn29vZoaxvAncFJvzM2Nu7VFZYSJa1+IhQKsEjqjvcPFODb1GI8Nc3VoG+tRCJRn3zhCOktakTUj57wHQ6xkRD512qRXVrDdziEDAiUtPqRjbkY8yZxo0PupPkRCekTlLT6mXKAwIQ/rqG6oZXnaAgxfJS0+pnPcGt4u1ijtV2BvadL+Q6HEINHSaufCQQCLJrOPTnclV4ChWJQtOUlpN9Q0roL5vu4wNLUCCXVjTh+/jrf4RBi0Chp3QVmYhGe9OXG8qIKeUJ6h5LWXRLWcYuYdK4SpdWNPEdDiOGipHWXjBpmgXtH24Ix0ACBhPQCJa27aPF0rvnDntOlaGmX8xwNIYaJktZdFDDOAQ5WJqiqb8XBvHK+wyHEIFHSuouMREI848ddbVGFPCE9Q0nrLnvazxUioQCniv7CufJavsMhxOBQ0rrLHKxMETjBAQBdbRHSE5S0eLCoo0J+X9ZV1DXTGFWE6IKSFg/8R9pi1DBzNLTK8VP2Vb7DIcSgUNLiAdcfkbva2pFWjEEytwghfaJHSWvbtm3w8PCAqakppFIpMjIyblt+79698PLygqmpKby9vXHgwAG17e+88w68vLxgbm4OGxsbBAQEID09Xa1MdXU1wsLCYGVlBYlEguXLl6O+vr4n4euFx+8ZDjNjEf6sqMepor/4DocQg6Fz0tq9ezciIyMRHR2NrKws+Pj4IDAwsNuJD1JSUhAaGorly5cjOzsbwcHBCA4ORl5enqqMp6cntm7ditzcXJw4cQIeHh546KGHcP36zc7FYWFhOHv2LBITE5GQkIDjx49j5cqVPXjL+sHazBjBU7gBAndQhTwh2tN1skU/Pz8WHh6uei2Xy5mzszOLiYnRWD4kJIQFBQWprZNKpWzVqlXdnkM5aePhw4cZY4zl5+czAOzUqVOqMr/++isTCATs6tWrWsV9Nydr1Vbe1RrmvjqBjX5rP6uo7Tp5JSGDiba/UZ2utFpbW5GZmYmAgADVOqFQiICAAKSmpmrcJzU1Va08AAQGBnZbvrW1FV988QWsra3h4+OjOoZEIsHUqVNV5QICAiAUCrvcRiq1tLSgtrZWbdE3E5ytMcVNgjY5w55TNEAgIdrQKWlVVVVBLpfDwcFBbb2DgwPKyzV3SykvL9eqfEJCAiwsLGBqaoqPP/4YiYmJsLOzUx3D3t5erbyRkRGGDh3a7XljYmJgbW2tWlxdXXV5q3eNsj9iXHoJ5DRAICF3pDdPD2fPno2cnBykpKTg4YcfRkhISK8mCI2KioJMJlMtpaX6eSXziLcTbIYYo0zWjCPnaEJUQu5Ep6RlZ2cHkUiEiooKtfUVFRXdzhrr6OioVXlzc3OMHj0a06dPx9dffw0jIyN8/fXXqmN0TmDt7e2orq7u9rwmJiawsrJSW/SRqbEIIdO4q0CqkCfkznRKWmKxGL6+vkhKSlKtUygUSEpKgr+/v8Z9/P391coDQGJiYrflbz1uS0uL6hg1NTXIzMxUbT9y5AgUCgWkUqkub0Evhfm5QyAAjv95HUVVDXyHQ4h+07WGPz4+npmYmLDY2FiWn5/PVq5cySQSCSsvL2eMMbZ48WL25ptvqsqfPHmSGRkZsU2bNrGCggIWHR3NjI2NWW5uLmOMsfr6ehYVFcVSU1NZUVERO336NFu2bBkzMTFheXl5quM8/PDDbMqUKSw9PZ2dOHGCjRkzhoWGhmodtz4+PbzV0m/SmfvqBPb+/ny+QyGEF9r+RnVOWowxtmXLFubm5sbEYjHz8/NjaWlpqm2zZs1iS5cuVSu/Z88e5unpycRiMZswYQLbv3+/altTUxNbsGABc3Z2ZmKxmDk5ObH58+ezjIwMtWPcuHGDhYaGMgsLC2ZlZcWWLVvG6urqtI5Z35PW4fxy5r46gfn88zfW1NrOdziE3HXa/kYFjA2OPiS1tbWwtraGTCbTy/otuYJh5sZkXK1pwqYnffCE73C+QyLkrtL2N6o3Tw8HO5FQgGek3OQXVCFPSPcoaemRp6a5wlgkwJnSGuRekfEdDiF6iZKWHrGzMMEj3k4AaIBAQrpDSUvPKFvI/+/MVcgaaYBAQjqjpKVnfN1t4OVoieY2Bb7PusJ3OIToHUpaeubWAQJ30QCBhHRBSUsPBU9xgYWJES5VNSDl4g2+wyFEr1DS0kMWJkZ4/B4XAMCOVKqQJ+RWlLT0lPIWMbGgAuWyZp6jIUR/UNLSU54OlvAbMRRyBcN3GSV8h0OI3qCkpceUzR++yyhBm1zBczSE6AdKWnoscIIj7CxMUFnXgsT8ijvvQMggQElLj4mNhHhaOUAgVcgTAoCSlt4LlbpBKABSL93Ahco6vsMhhHeUtPSci8QMc8ZxE4PsTKMKeUIoaRkAZYX8D5lX0NjaznM0hPCLkpYBuG+0Hdxth6CupR3/yynjOxxCeEVJywAIhQIsknJXWztSqT8iGdwoaRmIJ3yHw8RIiPxrtcgureE7HEJ4Q0nLQNiYizHPxxkAsJOaP5BBjJKWAVH2R0z44xqqG1p5joYQfvQoaW3btg0eHh4wNTWFVCpFRkbGbcvv3bsXXl5eMDU1hbe3Nw4cOKDa1tbWhtWrV8Pb2xvm5uZwdnbGkiVLUFamXuHs4eEBgUCgtmzYsKEn4Rssn+HW8HaxRqtcgb2nS/kOhxBe6Jy0du/ejcjISERHRyMrKws+Pj4IDAzsMm29UkpKCkJDQ7F8+XJkZ2cjODgYwcHByMvLAwA0NjYiKysLa9euRVZWFn788UcUFhZi/vz5XY717rvv4tq1a6rlxRdf1DV8gyYQCFTNH3amF0OhoAp5MgjpOqGin58fCw8PV72Wy+XM2dmZxcTEaCwfEhLCgoKC1NZJpVK2atWqbs+RkZHBALDi4mLVOnd3d/bxxx9rHWdzczOTyWSqpbS0VK8na9VWY0s7844+yNxXJ7Aj5yr4DoeQPqPtZK06XWm1trYiMzMTAQEBqnVCoRABAQFITU3VuE9qaqpaeQAIDAzstjwAyGQyCAQCSCQStfUbNmyAra0tpkyZgg8//BDt7d03tIyJiYG1tbVqcXV11eId6j8zsQhP+HLvhSrkyWCkU9KqqqqCXC6Hg4OD2noHBweUl5dr3Ke8vFyn8s3NzVi9ejVCQ0PVZpl96aWXEB8fj+TkZKxatQrr16/HG2+80W2sUVFRkMlkqqW0dODUAYVN5yZ1PVJYidLqRp6jIeTuMuI7gFu1tbUhJCQEjDF8+umnatsiIyNV/540aRLEYjFWrVqFmJgYmJiYdDmWiYmJxvUDwahhFrhvtB1OXKjCdxkleONhL75DIuSu0elKy87ODiKRCBUV6mM7VVRUwNHRUeM+jo6OWpVXJqzi4mIkJiaqXWVpIpVK0d7ejqKiIl3ewoChbP6w+1QpWtrlPEdDyN2jU9ISi8Xw9fVFUlKSap1CoUBSUhL8/f017uPv769WHgASExPVyisT1vnz53H48GHY2treMZacnBwIhULY29vr8hYGjIBx9nC0MsWNhlYczNN8q03IgKRrDX98fDwzMTFhsbGxLD8/n61cuZJJJBJWXl7OGGNs8eLF7M0331SVP3nyJDMyMmKbNm1iBQUFLDo6mhkbG7Pc3FzGGGOtra1s/vz5bPjw4SwnJ4ddu3ZNtbS0tDDGGEtJSWEff/wxy8nJYRcvXmQ7d+5kw4YNY0uWLNE6bm2fTBiSzYl/MvfVCWzhf07yHQohvabtb1TnpMUYY1u2bGFubm5MLBYzPz8/lpaWpto2a9YstnTpUrXye/bsYZ6enkwsFrMJEyaw/fv3q7ZdvnyZAdC4JCcnM8YYy8zMZFKplFlbWzNTU1M2btw4tn79etbc3Kx1zAMxaZXLmtioqP3MfXUCyy8bOO+LDE7a/kYFjA2OIQNqa2thbW0NmUx2x/oyQxK+Kwv7c68hTOqG9xd48x0OIT2m7W+U+h4aOGXzh33ZV1HX3MZzNIT0P0paBs5/pC1GDTNHY6sc+7Kv8h0OIf2OkpaBU+uPmEYDBJKBj5LWAPC473CYGYvwZ0U9Mi5X8x0OIf2KktYAYGVqjOAp3ACBO9KoPyIZ2ChpDRDKFvIH88pRWdfMczSE9B9KWgPEBGdr3OMmQbuCYc+pgdM5nJDOKGkNIIv9uautuPQStMsVPEdDSP+gpDWAzJ3oBJshxiiTNePIOc0jyRJi6ChpDSCmxiKETOMGCKQKeTJQUdIaYML83CEQAL+fr0JRVQPf4RDS5yhpDTButkPwgOcwAMCudLraIgMPJa0BSNn8Yc/pK2huowECycBCSWsAemCsPVwkZpA1teGXM2V33oEQA0JJawASCQWq0R92ppfwHA0hfYuS1gAVMtUVYpEQZ0pr8MeVGr7DIaTPUNIaoOwsTDDXm5s8ZCc1fyADCCWtAUw5ZM3PZ8oga6QBAsnAQElrAPN1t4GXoyWa2xT4PusK3+EQ0icoaQ1gAoFA1R9xZ1oxFAoaIJAYvh4lrW3btsHDwwOmpqaQSqXIyMi4bfm9e/fCy8sLpqam8Pb2xoEDB1Tb2trasHr1anh7e8Pc3BzOzs5YsmQJysrUH9VXV1cjLCwMVlZWkEgkWL58Oerr63sS/qASPNkFFiZGuFzVgJSLN/gOh5Be0zlp7d69G5GRkYiOjkZWVhZ8fHwQGBiIykrNHXRTUlIQGhqK5cuXIzs7G8HBwQgODkZeXh4AoLGxEVlZWVi7di2ysrLw448/orCwEPPnz1c7TlhYGM6ePYvExEQkJCTg+PHjWLlyZQ/e8uBibmKEx+9xAQDsSCviNxhC+oKuc5P5+fmx8PBw1Wu5XM6cnZ1ZTEyMxvIhISEsKChIbZ1UKmWrVq3q9hwZGRkMACsuLmaMMZafn88AsFOnTqnK/Prrr0wgELCrV69qFfdAnPdQW4Xltcx9dQIbGbWfldU08h0OIRpp+xvV6UqrtbUVmZmZCAgIUK0TCoUICAhAamqqxn1SU1PVygNAYGBgt+UBQCaTQSAQQCKRqI4hkUgwdepUVZmAgAAIhUKkp6drPEZLSwtqa2vVlsHK08ES0hFDIVcwfJdBAwQSw6ZT0qqqqoJcLoeDg4PaegcHB5SXl2vcp7y8XKfyzc3NWL16NUJDQ1UTNpaXl8Pe3l6tnJGREYYOHdrtcWJiYmBtba1aXF1dtXqPA5WyP+J3GSVoowECiQHTq6eHbW1tCAkJAWMMn376aa+OFRUVBZlMplpKSwf3FUbgBEfYWZjgel0LDp2t4DscQnpMp6RlZ2cHkUiEigr1L31FRQUcHR017uPo6KhVeWXCKi4uRmJiotq02I6Ojl0q+tvb21FdXd3teU1MTGBlZaW2DGZiIyFC/birTWohTwyZTklLLBbD19cXSUlJqnUKhQJJSUnw9/fXuI+/v79aeQBITExUK69MWOfPn8fhw4dha2vb5Rg1NTXIzMxUrTty5AgUCgWkUqkub2FQC/Vzg1AApF66gQuVdXyHQ0jP6FrDHx8fz0xMTFhsbCzLz89nK1euZBKJhJWXlzPGGFu8eDF78803VeVPnjzJjIyM2KZNm1hBQQGLjo5mxsbGLDc3lzHGWGtrK5s/fz4bPnw4y8nJYdeuXVMtLS0tquM8/PDDbMqUKSw9PZ2dOHGCjRkzhoWGhmod92B+enir5/57irmvTmDR/8vjOxRC1Gj7G9U5aTHG2JYtW5ibmxsTi8XMz8+PpaWlqbbNmjWLLV26VK38nj17mKenJxOLxWzChAls//79qm2XL19mADQuycnJqnI3btxgoaGhzMLCgllZWbFly5axuro6rWOmpMU5VljJ3FcnsInrDrL65ja+wyFERdvfqIAxNij6dtTW1sLa2hoymWxQ128pFAwPfnQURTcaEfO4N0L93PgOiRAA2v9G9erpIel/QqFA1fxhR2oxBsnfLDKAUNIahJ7wHQ4TIyHyr9Uiq6SG73AI0QklrUFIMkSMeT7OAKj5AzE8lLQGKeUAgfv/uIbqhlaeoyFEe5S0BikfVwkmDbdGq1yBPacHd28BYlgoaQ1ii6Tc1dau9GLIaYBAYiAoaQ1i83ycYWVqhNLqJhz/8zrf4RCiFUpag5iZWIQnp1J/RGJYKGkNcmFSrnHpkcJKlFY38hwNIXdGSWuQGznMAveNtgNjQFwGzUZN9B8lLaJqIb/7VCla2uU8R0PI7VHSIggYZw9HK1NUN7TiYJ7mkWAJ0ReUtAiMREI801G3tSOVKuSJfqOkRQAAT09zhZFQgNPFfyG/bPBOAkL0HyUtAgCwtzJF4ARu6Oqd6XS1RfQXJS2ioqyQ/yn7Kuqa23iOhhDNKGkRlekjh2K0vQUaW+XYl32V73AI0YiSFlERCARYdEuFPA0QSPQRJS2i5nHf4TAzFuF8ZT3SL1fzHQ4hXVDSImqsTI0RPMUFAPVHJPqJkhbpYtF07hbxYF45KuuaeY6GEHU9Slrbtm2Dh4cHTE1NIZVKkZGRcdvye/fuhZeXF0xNTeHt7Y0DBw6obf/xxx/x0EMPwdbWFgKBADk5OV2O8cADD0AgEKgtzz//fE/CJ3cwwdka97hJ0K5g2J1BAwQS/aJz0tq9ezciIyMRHR2NrKws+Pj4IDAwsMu09UopKSkIDQ3F8uXLkZ2djeDgYAQHByMvL09VpqGhAffddx8++OCD2557xYoVuHbtmmrZuHGjruETLS3255o/xGWUoF2u4DkaQm6h64SKfn5+LDw8XPVaLpczZ2dnFhMTo7F8SEgICwoKUlsnlUrZqlWrupRVTtyanZ3dZdusWbPYyy+/rGu4KjpN1lpX2ePzDBRNre1syruHmPvqBHYw7xrf4ZBBQNvfqE5XWq2trcjMzERAQIBqnVAoREBAAFJTUzXuk5qaqlYeAAIDA7stfzu7du2CnZ0dJk6ciKioKDQ2dj/+U0tLC2pra9UWrdRVAJ9MBr7/O/BXkc4xDhSmxiKE0ACBRA/plLSqqqogl8vh4OCgtt7BwQHl5ZpHBygvL9epfHeeeeYZ7Ny5E8nJyYiKisKOHTuwaNGibsvHxMTA2tpatbi6ump3ootJQGsDkPcDsHUa8NvbQOPgfPQfJnWDQAD8fr4Kl6sa+A6HEAAG9PRw5cqVCAwMhLe3N8LCwvDtt99i3759uHjxosbyUVFRkMlkqqW0VMsK5cnPAKuOAyMfAOStQOpW4JMpQMoWoL2l796QAXAdOgQPeA4DAOyiqy2iJ3RKWnZ2dhCJRKioqFBbX1FRAUdHR437ODo66lReW1KpFABw4cIFjdtNTExgZWWltmjNaRKw+Ccg7AfAfjzQXAMcWsNdeeV+DwyiluLKCvm9mVfQ3EYDBBL+6ZS0xGIxfH19kZSUpFqnUCiQlJQEf39/jfv4+/urlQeAxMTEbstrS9kswsnJqVfH6ZZAAIwJAJ4/AczfClg4AjXFwA/LgS8fBIpO9s959cwsT3sMtzGDrKkNv5wp4zscQnS/PYyMjMSXX36J//73vygoKMALL7yAhoYGLFu2DACwZMkSREVFqcq//PLLOHjwID766COcO3cO77zzDk6fPo2IiAhVmerqauTk5CA/Px8AUFhYiJycHFW918WLF/Hee+8hMzMTRUVF+Pnnn7FkyRLMnDkTkyZN6tUHcEdCEXDPYuClLGD224CxOVCWBcQ+Anz3DFB1vn/PzzORUKAaIJAq5Ile6MmjyS1btjA3NzcmFouZn58fS0tLU22bNWsWW7p0qVr5PXv2ME9PTyYWi9mECRPY/v371bZv376dAeiyREdHM8YYKykpYTNnzmRDhw5lJiYmbPTo0ez111/XrvlCB52aPNxObTljP7/M2Ds2jEVbcf9NiBzQzSSu1zWzMW8dYO6rE1hOyV98h0MGKG1/owLGBkcFTW1tLaytrSGTyXSr3+rO9UIgMRr481futdgSuO9lYHo4IB7S++PrmVfis/FTThme9B2OD5/04TscMgBp+xs1mKeHemfYWOCZeGBpAuA0GWitA478C9jiC2TvAhQDq9JaWSH/85kyFFyj4ZgJfyhp9daI+4EVycDjXwHWrkBdGfC//wM+nwVcPMJ3dH3mHjcbeLtYo6VdgaBPfsebP/xBnakJL+j2sC+1NQMZnwPHPwJaZNy6UXOAv70LOE7sn3PeRddkTXgvIR8HcrkHJEPEIrwwaxSeu38kzMQinqMjhk7b3yglrf7QWA0c/xDI+BJQtAEQAJPDgAffBqyc+/fcd8Hpomq8t78AZ0prAABO1qZ4PXAsgie7QCgU8BscMViUtDq5q0lLqfoScPifQP5P3GsjM2BGBHDvy4CJ5d2JoZ8oFAy//FGGjQcLcbWmCQDg7WKNt4PGYfpIW56jI4aIklYnvCQtpdIMrkV9aTr32nwY8EAUcM9SQGR0d2PpY81tcnxz8jL+k3wR9S3tAICHxjsg6pFxGGFnznN0xJBQ0uqE16QFcF1/Cn4BDkdzV2AAYOcJBPwTGDuXa4FvwKrqW7D58J/4LqMUcgWDkVCARdPd8fKcMbAxF/MdHjEAlLQ64T1pKbW3ApnbgaMbgKaO0SPc7wMeeg9wuYe/uPrI+Yo6rD9QgOTC6wAAK1MjvDRnDBb7u8PEiCrrSfcoaXWiN0lLqVkGnPgYSPsUaO9oOuD9JPDgWsDGnd/Y+sDv56/j/f0FOFdeBwBwGzoEUXO98PBERwgM/KqS9A9KWp3oXdJSqinlGqX+Ec+9FokB6Srg/n8AZjb8xtZLcgXD95ml2HToT1yv44b1meZhg7eDxmOyq4Tf4IjeoaTVid4mLaWyHCBxLXD5OPfazAaY+QYw7TnAyLDrhBpa2vH5sYv44vdLaG7jxpt/bLIz3njYCy4SM56jI/qCklYnep+0AK6y/nwikLgOuF7ArbPxAOZEAxMWGHxl/TVZEzb99id+zL4CxgCxkRDL7xuB/3tgFCxNjfkOj/CMklYnBpG0lOTtQM4uIPl9oL5jAMXh04CH/gW4Tec3tj6Qd1WGf+3PR9ol7kGErbkYr/7NE09Pc4WRyAB7ljXLgLJswGo4YDvK4P+48IWSVicGlbSUWuq54Z5PfgK0dYzR7vUo10zCbjS/sfUSYwyHCyoRc6AAlzrGnx9jb4G3gsbhAc9h+l1Z39YElKRxt/KXj3EJi3VMs2bpBHjcz/VJHTGTu1ImWqGk1YlBJi2lunIgeT2QvYP7cQiNgKl/B2atBszt+I6uV9rkCuxKK8bmpPOoaWwDANw/xg5vB42Dl6Oe/H+StwFXs24mqdJ0bv6AW1m7AfXlmtePuP9mIrMefvfiNjCUtDox6KSlVFnAjeF1/jfutYkVcN8rwPT/A4wNu0Jb1tiGrcnn8d+UYrTKFRAKgJCproh8yBP2lqZ3NxiFAqjIu5mkilOA1nr1MpZOwIhZwMhZ3BWV9XDuCqw0Ayj6ndv3aiagaFffz2YEV37ETC6RWarPVDWYUdLqZEAkLaVLx7huQeV/cK+thgMPrgEmPQUIDbBO6BYlNxrxwcFz2J97DcBdGkmCMa6XwqWjXLIp+h1ovKFexsyGSzIjZ3HJynb0neuuWuqB0jTg8u/cMW+9jVSy8+y4CutIYuaDt98mJa1OBlTSArirgdy9wJH3AFnH9GiOk7iW9SMf4DW0vnC6qBr/2l+AnI6RJBytuJEkFkzpo5Ekasu45H/5OLfUXlHfbmwOuM/gksnIWYCDd+//IDTLgOLUm1di5bngRha/hf2Em7eTHvcafFs9XVDS6mTAJS2ltiYg/TPg9/8HtHSMKDr6b9wYXg7j+Y2tlxhj+OWPa/jg13OqkSQmulhhTdB43UeSaKzmkoUyUd3oNCGJSAwM97uZpJzv6f/2cY3V3K2n8uquMr9TAQE3nZ3ySszNHzAdQN/dTihpdTJgk5ZSww3g2AfA6a+5ehSBEJiyiJtByLJ3c0zyrblNju0ni/Cf5Auo6xhJ4m/jHRA11wsjh1lo3qmlHihJvXnL1/mqRiDkhslWJinX6fyP7V9/HSg+wd1OakqsAhHgPPnmraTbdEA8cEbSoKTVyYBPWko3LgKH3wEKfuZeGw8BZrwEzHgRMOnmB24gbtS3YPPh84jLKOk6koQJA66c4n7sl44BV093rQQf5sXVR42YaRi3XrXXgKITQNFxLpH9dVl9u9AYcPG92bxiuB9gfJcfWvShfk1a27Ztw4cffojy8nL4+Phgy5Yt8PPz67b83r17sXbtWhQVFWHMmDH44IMP8Mgjj6i2//jjj/jss8+QmZmJ6upqZGdnY/LkyWrHaG5uxj/+8Q/Ex8ejpaUFgYGB+M9//gMHB+2evgyapKVUks5V1l/J4F5bOHBjeE1ZbPBjeF2orEPM/rO4/mcGZgjPYqZxPvyEf8JI0WnMeolbR5LqSFSG/qSuprSjPqyjYl9Zl6kkMgFc/W42r3CZalBdwPotae3evRtLlizBZ599BqlUis2bN2Pv3r0oLCyEvb19l/IpKSmYOXMmYmJi8OijjyIuLg4ffPABsrKyMHEiN276jh07cPnyZTg7O2PFihUak9YLL7yA/fv3IzY2FtbW1oiIiIBQKMTJk9rN9DzokhbAPRXL/x935aX8Kz3Mi6vvGvOQYbXcZgy4fu7mlVTRiZvj8He4AQlaXO+F0+RACEbOGtgNOxkD/iq6Wal/+XeunditjMy4W8gR9wMeMwHnKXr9B6vfkpZUKsW0adOwdetWAIBCoYCrqytefPFFvPnmm13KP/XUU2hoaEBCQoJq3fTp0zF58mR89tlnamWLioowYsSILklLJpNh2LBhiIuLwxNPPAEAOHfuHMaNG4fU1FRMn961a0tLSwtaWlpUr2tra+Hq6jq4kpZSeytX13XsA6DpL26dx/1ctyDnybyGdlt/FXPtpJRP+JRdmpRMrME87kWOaBI2/OmA9Hp7AAJMdbfBmkcH2UgSjAE3Ltz8rIpOAI1V6mXEFtwTUeWVmOMkbgZ1PaFt0tIp7ba2tiIzM1Nt2nuhUIiAgACkpqZq3Cc1NRWRkZFq6wIDA/HTTz9pfd7MzEy0tbUhICBAtc7Lywtubm7dJq2YmBj885//1PocA5qRGJj+AuATCpz4f0DaZ9xf6C9mcW27HlwLSFz5jhKor7zZoPPSMaCmWH27kWnHlUPHLZ+TDwQiI0wB8E1LOz4/fglfHL+I08V/IXjbSTw22RmvB47FcJuBN3luFwIBYDeGW6Yt55JYZcHNK7GiE0BzDXD+ELcAgKk14H7vzYp9+/EG0c5Pp6RVVVUFuVzepR7JwcEB586d07hPeXm5xvLl5eUay3d3DLFYDIlEovVxoqKi1JKl8kprUDOTcLeG054Dkt4DcvcAf+wGzv4ETH8euC+SK3O3NNV0PPLvSFLKkS2UBCJg+NSbdVKufoCRicZDmZsYIfJvnnjGzw2bDhXih6wr+F9OGX7NKx+cI0kIBFyTF4fx3PhsCgVQkXuzPqw4hWs3VniAWwDAbCjgcd/NJDZsrF5WIejvDW4vmZiYwMRE8xd80JO4AQu/BPz/Dzi0lvsSn/w3kLWD68849e/9U4Hb2sj121Pe8mlqIe7offNKyt1f51mLHK1NselJHzw7wwPv7y9A6qUb+PToRew5VWrYI0n0llAIOPlwy4wIbiSR8jM368NK0rjhvwt+vvnk2dz+ln6TM4GhI/UiiemUtOzs7CASiVBRoV63UFFRAUdHzW2BHB0ddSrf3TFaW1tRU1OjdrWl63FIJ85TgKW/AH/+xo3hVVUIHFzNTTg7JxoY/1jvvqSqjsYdSUpTR2Pb0R198Wb1aTeWiS7WiFshVRtJYs1PefhvShHeemQcHhir5yNJ9DeREddcwsUXuO/Vm/+vijrqxEozgIZKIO8HbgEAS+eO/1cdiYynYcF7VBHv5+eHLVu2AOAq4t3c3BAREdFtRXxjYyN++eUX1boZM2Zg0qRJOlfEf/fdd1i4cCEAoLCwEF5eXt3WaXU2KJ8e6kLeDmR/CyTHcF9WgGv3E/g+d1umDa06Gjvf7L93l0Y9aJMrEJdegs2H/8Rft4wk8dYj4zDOib4LGrU1c23dlLeTpRkdEw/fQuLGPZVUJrJeTkTcr00eli5dis8//xx+fn7YvHkz9uzZg3PnzsHBwQFLliyBi4sLYmJiAHBNHmbNmoUNGzYgKCgI8fHxWL9+vVqTh+rqapSUlKCsrExVZuzYsXB0dFRdSb3wwgs4cOAAYmNjYWVlhRdffFF1/L78QAa9ljogZQu3tDVy68Y/xl152Y5SL8sY15j18rGO5febMwwpmQ292fhxxAO8DpIna2rDtuQLiD1ZxP9IEoZGeWuvbCdWltW18e7QUeq3kxZdm0DdTr82Lt26dauqcenkyZPxySefQCqVAgAeeOABeHh4IDY2VlV+7969WLNmjapx6caNG9Ual8bGxmLZsmVdzhMdHY133nkHwM3Gpd99951a41Jtbw8paemo9hpwdD2QvbNjDC9j7qmU77PcePbKW77aq+r7KR+rK2/5HCbq3RMpTSNJPD9rFFb050gSA01LfcdAiMe4RHbtTNf6yWFeN5tXjHyAe1p5G9SNpxNKWj1Ukc/Vd11I1Lxd2dFYecvncg8gMoyndJnF1XgvoR9HkhhMmmq4vp7Kiv2KXPXti34ERs+57SEoaXVCSauXLiZzyasij+torBz8Th86GvdCdyNJvP3IePiPGrxjW/VaY3VHv8nfueF4lv92x87dlLQ6oaTVR+Ttet0VpKea2+SITSnCtiM6jCRB+hQlrU4oaRFt3HYkCXPD6XxsiChpdUJJi+jiQmUd1h84hyPnuOYfVqZGePHBMVgywx0mRlRZ3x8oaXVCSYv0xMkLVfjX/gIUXONGhXUbOgRvzvXC3ImOg7txaj+gpNUJJS3SU3IFww+ZV7DpUCEq67iRQ6a62+DtoHGY4qbnAwkaEEpanVDSIr3VcMtIEs1tXJuk+T7OeOPhQTKSRD+jpNUJJS3SV8plzaqRJBgDxEZC/P3eEZgxyhYKxrhFAcgZA2MMCsZdrd1uG2OsowxullPbxq1njEHesU2h0FBOwzblv+XslmMoum5jHeftvO3WmOQK9XKMoeOc3PvqXE7OGBQKht2r/O/YZYqSVieUtEhfy7sqU40kQW7vl4j74D28b1rED7wGN4TcJcqRJJIKKvHl75dQ29wOkRAQCgQQCAQQCbh/C4UCCAWASCjodtvNf99uGyBS7n+bbUKBACIhupYT3IxD4zbhzX8rt3Gxdt0m6Niv6/Fvlr11m7Ok72ZAp6RFSC8IBAIEjHdAwHgDnzTDgOhXT1ZCCLkDSlqEEINCSYsQYlAoaRFCDAolLUKIQaGkRQgxKJS0CCEGZdC001I2/K+treU5EkKIJsrf5p066QyapFVXVwcANMs0IXqurq4O1tbdd/kZNH0PFQoFysrKYGlpecdxkGpra+Hq6orS0lLqp3gL+ly6R5+NZrp8Lowx1NXVwdnZGcLbzOA0aK60hEIhhg/XbWJQKysr+gJqQJ9L9+iz0Uzbz+V2V1hKVBFPCDEolLQIIQaFkpYGJiYmiI6OhomJCd+h6BX6XLpHn41m/fG5DJqKeELIwEBXWoQQg0JJixBiUChpEUIMCiUtQohBoaRFCDEolLQ62bZtGzw8PGBqagqpVIqMjAy+Q+Ld8ePHMW/ePDg7O0MgEOCnn37iOyS9EBMTg2nTpsHS0hL29vYIDg5GYWEh32HphU8//RSTJk1StYT39/fHr7/+2ifHpqR1i927dyMyMhLR0dHIysqCj48PAgMDUVlZyXdovGpoaICPjw+2bdvGdyh65dixYwgPD0daWhoSExPR1taGhx56CA0NDXyHxrvhw4djw4YNyMzMxOnTp/Hggw/isccew9mzZ3t/cEZU/Pz8WHh4uOq1XC5nzs7OLCYmhseo9AsAtm/fPr7D0EuVlZUMADt27BjfoeglGxsb9tVXX/X6OHSl1aG1tRWZmZkICAhQrRMKhQgICEBqaiqPkRFDIZPJAABDhw7lORL9IpfLER8fj4aGBvj7+/f6eINmlIc7qaqqglwuh4OD+qSbDg4OOHfuHE9REUOhUCjwyiuv4N5778XEiRP5Dkcv5Obmwt/fH83NzbCwsMC+ffswfvz4Xh+XkhYhfSA8PBx5eXk4ceIE36HojbFjxyInJwcymQzff/89li5dimPHjvU6cVHS6mBnZweRSISKigq19RUVFXB0dOQpKmIIIiIikJCQgOPHj+s8ZttAJhaLMXr0aACAr68vTp06hX//+9/4/PPPe3VcqtPqIBaL4evri6SkJNU6hUKBpKSkPrkPJwMPYwwRERHYt28fjhw5ghEjRvAdkl5TKBRoaWnp9XHoSusWkZGRWLp0KaZOnQo/Pz9s3rwZDQ0NWLZsGd+h8aq+vh4XLlxQvb58+TJycnIwdOhQuLm58RgZv8LDwxEXF4f//e9/sLS0RHl5OQBu9E0zMzOeo+NXVFQU5s6dCzc3N9TV1SEuLg5Hjx7Fb7/91vuD9/5B5sCyZcsW5ubmxsRiMfPz82NpaWl8h8S75ORkBqDLsnTpUr5D45WmzwQA2759O9+h8e7vf/87c3d3Z2KxmA0bNozNmTOHHTp0qE+OTeNpEUIMCtVpEUIMCiUtQohBoaRFCDEolLQIIQaFkhYhxKBQ0iKEGBRKWoQQg0JJixBiUChpEUIMCiUtQohBoaRFCDEo/x+b9UeaOCBi5gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:34:45.230997Z", + "iopub.status.busy": "2024-03-22T22:34:45.230208Z", + "iopub.status.idle": "2024-03-22T22:43:15.124159Z", + "shell.execute_reply": "2024-03-22T22:43:15.123146Z" + }, + "papermill": { + "duration": 509.915174, + "end_time": "2024-03-22T22:43:15.127171", + "exception": false, + "start_time": "2024-03-22T22:34:45.211997", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:43:15.161923Z", + "iopub.status.busy": "2024-03-22T22:43:15.161610Z", + "iopub.status.idle": "2024-03-22T22:43:15.183186Z", + "shell.execute_reply": "2024-03-22T22:43:15.182334Z" + }, + "papermill": { + "duration": 0.040817, + "end_time": "2024-03-22T22:43:15.185177", + "exception": false, + "start_time": "2024-03-22T22:43:15.144360", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.00.3809520.005036.1398320.4651837.4544970.9615510.00015127.9044750.0436162446463.50.0709210.2417260.00011334.044307
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.0 0.380952 0.00503 6.139832 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.465183 7.454497 0.961551 0.000151 27.904475 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.043616 2446463.5 0.070921 0.241726 0.000113 \n", + "\n", + " total_duration \n", + "realtabformer 34.044307 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:43:15.219860Z", + "iopub.status.busy": "2024-03-22T22:43:15.219538Z", + "iopub.status.idle": "2024-03-22T22:43:15.647783Z", + "shell.execute_reply": "2024-03-22T22:43:15.646798Z" + }, + "papermill": { + "duration": 0.448182, + "end_time": "2024-03-22T22:43:15.649866", + "exception": false, + "start_time": "2024-03-22T22:43:15.201684", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:43:15.685806Z", + "iopub.status.busy": "2024-03-22T22:43:15.685033Z", + "iopub.status.idle": "2024-03-22T22:52:22.374570Z", + "shell.execute_reply": "2024-03-22T22:52:22.373559Z" + }, + "papermill": { + "duration": 546.710655, + "end_time": "2024-03-22T22:52:22.377660", + "exception": false, + "start_time": "2024-03-22T22:43:15.667005", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:52:22.414647Z", + "iopub.status.busy": "2024-03-22T22:52:22.414333Z", + "iopub.status.idle": "2024-03-22T22:52:22.441063Z", + "shell.execute_reply": "2024-03-22T22:52:22.440149Z" + }, + "papermill": { + "duration": 0.047944, + "end_time": "2024-03-22T22:52:22.443151", + "exception": false, + "start_time": "2024-03-22T22:52:22.395207", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:52:22.476264Z", + "iopub.status.busy": "2024-03-22T22:52:22.475900Z", + "iopub.status.idle": "2024-03-22T22:52:22.481296Z", + "shell.execute_reply": "2024-03-22T22:52:22.480405Z" + }, + "papermill": { + "duration": 0.024382, + "end_time": "2024-03-22T22:52:22.483457", + "exception": false, + "start_time": "2024-03-22T22:52:22.459075", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.41001513860391026}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:52:22.519004Z", + "iopub.status.busy": "2024-03-22T22:52:22.518705Z", + "iopub.status.idle": "2024-03-22T22:52:22.913927Z", + "shell.execute_reply": "2024-03-22T22:52:22.912866Z" + }, + "papermill": { + "duration": 0.416011, + "end_time": "2024-03-22T22:52:22.916171", + "exception": false, + "start_time": "2024-03-22T22:52:22.500160", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGv0lEQVR4nO2deXhU5dn/P2f2mcwSsi8kJBAEEQVEQdzApSpaK7WurQW0iG2l1lLfVrx+Ra1taftWq7WW17aaaNX2rS1q+9q6r3UBZFEUZM9C9pBlMvt2fn+czJCQBCaTSc4k83yua65kzjxzzp2TzDf3cz/3fT+SLMsyAoFAoCIatQ0QCAQCIUQCgUB1hBAJBALVEUIkEAhURwiRQCBQHSFEAoFAdYQQCQQC1RFCJBAIVEcIkUAgUB0hRIJBueeee5AkKannrKqqQpIkPvroo7jG//d//zeTJ09Gq9Uye/bspNoiSB2EEAmGxM9+9jOef/75UbnWK6+8wg9+8APOOussKisr+dnPfjYq1xWMPjq1DRCMLX72s59x1VVXsWTJkhG/1htvvIFGo+Gxxx7DYDCM+PUE6iE8onGC2+1W24Sk09LSgtlsTpoIybKM1+tNyrmGg8fjUduElEMI0RgkGrvZuXMnX/3qV5kwYQJnn302AE899RRz587FbDaTlZXFddddR11dXZ/3v/vuu1x99dWUlpZiNBopKSnhe9/73nE/pJIk4Xa7eeKJJ5AkCUmSWL58OQA1NTV8+9vfZtq0aZjNZrKzs7n66quprq4e8Fwej4dbbrmF7Oxs7HY7S5cupaOjo8+1KisrcbvdsWtVVVUBEAqFuO+++5gyZQpGo5GysjLuuusu/H5/n2uUlZXxxS9+kZdffpnTTjsNs9nMo48+yltvvYUkSfz1r3/l3nvvpbi4GJvNxlVXXUVXVxd+v5/bb7+dvLw8rFYrN954Y79zx3uvFy1axMyZM9myZQvnnnsuFouFu+6665j3OR0RU7MxzNVXX83UqVP52c9+hizL/PSnP+VHP/oR11xzDStWrKC1tZWHH36Yc889l23btpGZmQnAs88+i8fj4Vvf+hbZ2dls2rSJhx9+mEOHDvHss88Oer0//elPrFixgnnz5rFy5UoApkyZAsDmzZt5//33ue6665g4cSLV1dWsX7+eRYsWsXPnTiwWS59zrVq1iszMTO655x52797N+vXrqampiYnEn/70J37/+9+zadMm/vjHPwJw5plnArBixQqeeOIJrrrqKr7//e+zceNG1q1bx65du3juuef6XGf37t1cf/313HLLLdx8881MmzYt9tq6deswm83ceeed7Nu3j4cffhi9Xo9Go6Gjo4N77rmHDz/8kKqqKsrLy1m7dm3svfHea4DDhw+zePFirrvuOm644Qby8/OH+JtOA2TBmOPuu++WAfn666+PHauurpa1Wq3805/+tM/YHTt2yDqdrs9xj8fT75zr1q2TJUmSa2pq+l2nNxkZGfKyZcv6vX+gc37wwQcyID/55JOxY5WVlTIgz507Vw4EArHjv/zlL2VAfuGFF2LHli1bJmdkZPQ55/bt22VAXrFiRZ/jd9xxhwzIb7zxRuzYpEmTZEB+6aWX+ox98803ZUCeOXNmHxuuv/56WZIkefHixX3GL1iwQJ40aVLs+VDu9cKFC2VA/p//+Z9+90dwBDE1G8N885vfjH2/YcMGIpEI11xzDW1tbbFHQUEBU6dO5c0334yNNZvNse/dbjdtbW2ceeaZyLLMtm3bErKl9zmDwSCHDx+moqKCzMxMtm7d2m/8ypUr0ev1seff+ta30Ol0/Otf/zrmdaKvr169us/x73//+wC8+OKLfY6Xl5dz8cUXD3iupUuX9rFh/vz5yLLMTTfd1Gfc/PnzqaurIxQKAUO71wBGo5Ebb7zxmD9XuiOmZmOY8vLy2Pd79+5FlmWmTp064NjeH7ja2lrWrl3LP/7xjz5xGYCurq6EbPF6vaxbt47Kykrq6+uRezX+HOicR9tptVopLCwcNKYUpaamBo1GQ0VFRZ/jBQUFZGZmUlNT0+d473t0NKWlpX2eOxwOAEpKSvodj0QidHV1kZ2dPaR7DVBcXCxW/Y6DEKIxTG8vJBKJIEkS//73v9Fqtf3GWq1WAMLhMF/4whdob2/nhz/8IdOnTycjI4P6+nqWL19OJBJJyJbvfOc7VFZWcvvtt7NgwQIcDgeSJHHdddclfM5jEW+iZe97dDQD3adjHY+Ka7z3Oh4bBApCiMYJU6ZMQZZlysvLOeGEEwYdt2PHDvbs2cMTTzzB0qVLY8dfffXVuK4zmAD87W9/Y9myZdx///2xYz6fj87OzgHH7927l/POOy/23OVy0djYyKWXXnrM60+aNIlIJMLevXs58cQTY8ebm5vp7Oxk0qRJcf0cwyHeey2IHxEjGidceeWVaLVa7r333j7TIlD+kx8+fBg48t++9xhZlnnooYfiuk5GRsaA4qLVavtd9+GHHyYcDg94nt///vcEg8HY8/Xr1xMKhVi8ePExrx8VqgcffLDP8QceeACAyy677Hg/wrCJ914L4kd4ROOEKVOm8JOf/IQ1a9ZQXV3NkiVLsNlsHDx4kOeee46VK1dyxx13MH36dKZMmcIdd9xBfX09drudv//97/1iRYMxd+5cXnvtNR544AGKioooLy9n/vz5fPGLX+RPf/oTDoeDGTNm8MEHH/Daa6+RnZ094HkCgQAXXHAB11xzDbt37+Z3v/sdZ599Nl/60peOef1Zs2axbNkyfv/739PZ2cnChQvZtGkTTzzxBEuWLOnjZY0U8d5rQfwIIRpH3HnnnZxwwgn8+te/5t577wWUwOtFF10U+4Dr9Xr++c9/ctttt7Fu3TpMJhNf/vKXWbVqFbNmzTruNR544AFWrlzJ//t//w+v18uyZcuYP38+Dz30EFqtlqeffhqfz8dZZ53Fa6+9NuiK1W9/+1uefvpp1q5dSzAY5Prrr+c3v/lNXLGfP/7xj0yePJmqqiqee+45CgoKWLNmDXffffcQ7tbwiOdeC+JHko/2LQUCgWCUETEigUCgOkKIBAKB6gghEggEqiOESCAQqI4QIoFAoDpCiAQCgeqkTB7Rz3/+c9asWcN3v/vdflmzgxGJRGhoaMBmsyW9ybtAIBg+sizT3d1NUVERGs3gfk9KCNHmzZt59NFHOeWUU4b0voaGhn6V0gKBIPWoq6tj4sSJg76uuhC5XC6+9rWv8Yc//IGf/OQnQ3qvzWYDlB/SbrePhHkCgWAYOJ1OSkpKYp/VwVBdiG699VYuu+wyLrzwwuMKkd/v79M7uLu7GwC73S6ESCBIYY4XOlFViP7yl7+wdetWNm/eHNf4devWxep6BALB+EG1VbO6ujq++93v8vTTT2MymeJ6z5o1a+jq6oo9jt4xQSAQjE1UK3p9/vnn+fKXv9ynw104HEaSJDQaDX6/f9BOeVGcTicOh4Ouri4xNRMIUpB4P6OqTc0uuOACduzY0efYjTfeyPTp0/nhD394XBGKF1mWCYVCgzboEgyOVqtFp9OJ1AjBiKOaENlsNmbOnNnnWEZGBtnZ2f2OJ0ogEKCxsVHsrDkMLBYLhYWFovm7YERRfdVspIhEIhw8eBCtVktRUREGg0H8Zx8CsiwTCARobW3l4MGDTJ069ZgJaQLBcEgpIXrrrbeSdq5AIEAkEqGkpKTfLqOC+DCbzej1empqaggEAnEvKowHAqEI7+5tpcMTZH55FiVZ4m9oJBn3/+LEf/Hhka737509rXxyqIu6dg//+LiBbl/w+G8SJEx6/pUJBMfA6QvyWYMTAL1WIhCKsL2uU12jxjlCiASCo9jd1E1Elpk4wcwlMwtix0R795FDCFGaU1ZWFne3g3QhK8NAeU4GJxbaKc+xcvmsQm44Y5JY7BhBUipYLRCkAlNyrUzJPbJtdEXesQs2BcNHeETjgEAgoLYJAsGwSEshCoQigz5C4UjcY4NxjE2ERYsWsWrVKlatWoXD4SAnJ4cf/ehHsRhFWVkZ9913H0uXLsVut7Ny5UoA/vOf/3DOOedgNpspKSnhtttuw+12x87b0tLC5Zdfjtlspry8nKeffjoh+8YzLd0+urzBPvEgXzDMe/va+MfHDSJONEKk5dTskTf3DfpaeU4GS+YUx57//p39BMMD//FNnGDm6tOONGZ7/L2DeAN9S0m+94UTErLxiSee4Bvf+AabNm3io48+YuXKlZSWlnLzzTcD8Ktf/Yq1a9fGdjfdv38/l1xyCT/5yU94/PHHaW1tjYlZZWUlAMuXL6ehoYE333wTvV7PbbfdRktLS0L2jVfe2dNGXbuHi08qYEaRUhul00hsqekgHJFxekM4LHqVrRx/pKUQjQVKSkr49a9/jSRJTJs2jR07dvDrX/86JkTnn38+3//+92PjV6xYwde+9jVuv/12AKZOncpvfvMbFi5cyPr166mtreXf//43mzZt4vTTTwfgscce48QTTxz1ny1VkWWZ1m6l31WO9UhJi06rIcdqpNnpo8npE0I0AqSlEN16XsWgr2mOWhhZee6UQccevYhy01nlwzGrD2eccUafVZoFCxZw//33x4p3TzvttD7jP/74Yz755JM+0y1ZlmOlLnv27EGn0zF37tzY69OnTyczMzNpNo913IEwvmAYSVJWznqTb1eEqM3lZxoieJ1s0lKIDLr4Q2MjNXa4ZGRk9Hnucrm45ZZbuO222/qNLS0tZc+ePaNl2pilrccbmmAxoNP2/V1GhemwWywMjARpKURjgY0bN/Z5/uGHHzJ16tRB26Oceuqp7Ny5k4qKgb296dOnEwqF2LJlS2xqtnv3bjo7O5Nq91jmsDs6LTP2ey0qRB1CiEaEtFw1GwvU1tayevVqdu/ezZ///Gcefvhhvvvd7w46/oc//CHvv/8+q1atYvv27ezdu5cXXniBVatWATBt2jQuueQSbrnlFjZu3MiWLVtYsWIFZrN5tH6klKe1WxGZbGv/lidRIer0BAlHxMpZshFClKIsXboUr9fLvHnzuPXWW/nud78bW6YfiFNOOYW3336bPXv2cM455zBnzhzWrl1LUVFRbExlZSVFRUUsXLiQK6+8kpUrV5KXlzcaP86YoL3H28kZQIisRh0GnQadVsLlC422aeMe1VrFJoNjtaH0+XwcPHiQ8vLyMde+YtGiRcyePTslSi/G8n0cKnuau2lz+ZlZ7MBu6r8y5gmEMOu1otRjCKR8q1iBINU4Id/GCfmDr4hZDOLjMlKIqZlAIFAdIfEpSDI7VQrio8MdoNsXIstqwGoc+GPR2OVlc3UHNqOO86aL2FoyER6RQADsbu7m71sP8cH+wwMP8LsIezrZ3+Kitl1sxpBshEckEKAsywM4zAOUb9Rvhb2vkhMMUtpRSr1mAbIsi6B1EhEekUAAdHmVpfvMo+vI3G2w7zWQIxi1Goq7P8HmrsHlF0v4yUQIkUDAEY8o82iPqPYDiIQhuwKpdB5GvYZi57bYeEFyEEIkSHsCoQienvYt9t5CFPRB6+fK95POhJIzMOh12PzNeLtaVbB0/CKESJD2RLcKMug0mPS9avnadkM4BBk5YC8Co5WQfRIA4eadapg6bhFCJEh7ovEeu+motZvD+5WvudNjPV/knBPQSqDvqhlNE8c9YtVMkPZkZRj4wox8tL2bUUUi0FHdM2By7PDJM2eh8XyApHFCOAha0SQtGQghEqQ9NpOemcWOvge7GyHkB70JbIWxw1rLBDDawd8NXXV9REqQOOk1NZNlCAVG/zGEuuInn3yS7Oxs/H5/n+NLlizh61//erLviGAwnPXKV0cJ9N52W5Igs1T5vqt+9O0ap6SXRxQOwrv3j/51z/k+6Pq3lhiIq6++mttuu41//OMfXH311YCy+8aLL77IK6+8MpJWpi0H29xoJYk8u/FIsDoqRPaiPmOD4QgfNOnIbOjkpAkNDNymTjBU0ssjGgOYzWa++tWvxnbeAHjqqacoLS1l0aJF6hk2jnlrdwt/33oo1jgfAGeD8tVe3GesTiOx32vD7Q8T6GwcRSvHN+nlEWn1ineixnWHwM0338zpp59OfX09xcXFVFVVsXz5clFSMALIshxrdBbrQeRzKg9J6hMfApAkCY09H5okgl4nZn83GEUz/eGSXkIkSXFPkdRkzpw5zJo1iyeffJKLLrqIzz77jBdffFFts8Yl3mCYUERGksAaXb7vblK+ZuQM+PeSYbHg1Tvwh0LgahFClATSS4jGECtWrODBBx+kvr6eCy+8kJKSkuO/STBkunu8oQyD7sjyvbtn00lr/oDvsRp1ePRZBEJNSi1a9uBbTgniQ8SIUpSvfvWrHDp0iD/84Q/cdNNNapszbolmVVt7JzO6eoQoY+CeQ1ajDq9+AoFwBDxtI21iWiCEKEVxOBx85StfwWq1smTJErXNGbdEPSJbbyFy94iLNXfA91iMWrz6TIKhyJGxgmEhhCiFqa+v52tf+xpGY/99tgTJwe1Xil0zol0Zw0HwtivfD+IRZRh0hExZStWH5/CQ8sQEAyNiRClIR0cHb731Fm+99Ra/+93v1DZnXDOtwEamRX9ki2l3myIsejMYMgZ8zwn5VqZdOBfefVPJvg64RMB6mAghSkHmzJlDR0cHv/jFL5g2bZra5oxrcm1Gcm29PM5YoDovVuh6NJIkgVYH5gngaVfESwjRsBBClIJUV1erbUL64u7pMzTItKwP5ixFiLztQPmImjXeETEiQVqzu6mbmsNuQuGIcsDToXy1ZB3zff/a0cibtUF8oTB4O0fWyDRg3AvRGN7INiUYz/cvEpH596eNbNharyzFgxJ8huMKUbPTR1PARCAUAV/nyBqaBoxbIdLrlXR9j0ds/TIcovcvej/HE55gGFkGjSRh1muV3tS+LuVF87GFKMOgw6ezEQxHhEeUBMZtjEir1ZKZmUlLixJ8tFgsolZrCMiyjMfjoaWlhczMTLTa8Vdn7u7pzGgx9Oxn7+kEOaLUBh4n+GwxamnX2QkEejwiWR40uC04PqoK0fr161m/fn0sOHvSSSexdu1aFi9enJTzFxQUAMTESDB0MjMzY/dxvBETImOPyEbzhyxZxxWVDGOPR+Tt6XEV9Ay63C84PqoK0cSJE/n5z3/O1KlTkWWZJ554giuuuIJt27Zx0kknDfv8kiRRWFhIXl4ewaDY/mWo6PX6cekJRYnu3BHbYtrTI0THmZaBMjWTJR1ejUU54O0UQjQMVBWiyy+/vM/zn/70p6xfv54PP/wwKUIURavVjusPlCAxjkzNokIUX6BaeY/y9+SWrIBLmZ45io/5HsHgpEyMKBwO8+yzz+J2u1mwYMGAY/x+f58Wqk6nc7TME4xD3IFo5f1RU7N4PCKjDoNOQ9BgB1wiYD1MVBeiHTt2sGDBAnw+H1arleeee44ZM2YMOHbdunXce++9o2yhYLwys8hBvt1EjrUnszo6NbNkH/e9ZdkWbj2vAqqb4GCDWMIfJqov30+bNo3t27ezceNGvvWtb7Fs2TJ27hx487o1a9bQ1dUVe9TV1Y2ytYLxRJ7dxEk9YkTQBwG38kIcU7PYCqypZ/cPn/DOh4PqHpHBYKCiogKAuXPnsnnzZh566CEeffTRfmONRqOoRBeMDN6ejGpDBuiG8DcWXeb3CyEaDqoL0dFEIpF+W+kIBMlGlmV2NjrJMOgoybKgjU6tzBPiPse/djTi6ezmsmAYs88pcomGgapCtGbNGhYvXkxpaSnd3d0888wzvPXWW7z88stqmiVIA/yhCK981gzAqvMr0EaDzSbH4G86itZuPx1eLYFQBLM+BEEvGCwjYO34R1UhamlpYenSpTQ2NuJwODjllFN4+eWX+cIXvqCmWYI0IJpDZNBp0Gs1R4LN5sy4z2E2aGmXdAS0ZiCkTM+EECWEqkL02GOPqXl5QRoTzSGKJTNGa8yG4BFFc4m8GivQqQSsbeMzC32kUX3VTCBQg2gOkSWWQ9SpfDVlxn2OI0JkVg6IgHXCCCESpCV9elVHIkdEZChTM73iTXmkntIOIUQJI4RIkJZ4entEgW6lBYhGC4b4W77GyjzoiQuJXKKEEUIkSEuiHpHVqDsyLTPaQRP/R8Ji0GLQaYgY7coB4RElTMrlEQkEo8Gc0kxKsszk2UzgOqQcHEKgGqAiz8rUfBt0mWHrf4RHNAyEEAnSkny7SSntAGjrVL4OIT4Evcs8ejyigEuJNw3BqxIoiDsmEMSW7jMTe7/BqsSXZFmJNwmGjPCIBGlHOCKzq9GJ1aijNMuCJoGs6igvbK/H5Q9xlcaMMeICvyuh86Q7wiMSpB2eQIhXdzbzj48blNKwBLKqozQ7fbQ4/fijuUQBV7LMTCuEEAnSjuiKmcWgRYqEFS8GEvJkzD3dHX3RlrF+IUSJIIRIkHbEOjMadUfiQ1o96IdeJ2bRK7lEvmhSo4gRJYQQIkHa4enlEfWZliXQwiOa1OjR9KzACY8oIYQQCdKOI72qdUeEKMEVM3NUiIjWmwmPKBGEEAnSjmjlfUbvrOoEhSi6A4hbFsHq4SCESJB2uAPRgldtQu0/emMxaDHqNYT1VuWA8IgSQuQRCdKO08smUJFrpdBhgqZO5WACS/cAJxXZmVnsUJrv/wcI+SEcVILfgrgRQiRIOwodZgodPVOpYWZVx8o8dEbQ6iAcUryiOHYCERxBTM0E6UvQpzxg+NnQkqRU74OIEyWA8IgEaUUgFGFPczdWo44yY088x2ABnSGh84UjMv/4uB5PIMy1ugx0tIsl/AQQQiRIK5y+IK/ubMak1/Ktk5TVs+F4QxoJ6ju8BMMy/hyL8oESAeshI6ZmgrTC4++1YjbMpXtQYkSmnuxqv7YnM1tkVw8ZIUSCtOJI0/xe5R0JrphFieYSeSWRXZ0oQogEaUW0V7XV2Ku8Y5iB6liZR7R3tQhWDxkhRIK0whWrMxt+VnWUWJmH8IgSRgiRIK3wRMs7DJphZ1VHiXpErqhH5O9WujUK4kYIkSCtiJZ3WCU/REJK/k8ShMio1xDR9QhRJKRkWAviRizfC9KKM6dk01lop1Dbrhww2pR+08Pg1NIJzJ3Uk0ndalREKOACvWmY1qYPwiMSpBVFmWZmFNmxy9GujJnDPqfUu4+RsWeDRpFLNCQSEqIDBw4k2w6BYHQZRp/qY2LoqcIXK2dDIiEhqqio4LzzzuOpp57C5/Ml2yaBYETwBcN8Wt9FzWF30gLVAN5AmA1bD/HnTbXIhp6WsWLlbEgkJERbt27llFNOYfXq1RQUFHDLLbewadOmZNsmECSVDk+AV3c28+rO5qQt3QPotBI1hz00dfkIxrKr3cM+bzqRkBDNnj2bhx56iIaGBh5//HEaGxs5++yzmTlzJg888ACtra3JtlMgGDZ99rtPUlY1gF6rwaBTPkqizCMxhhWs1ul0XHnllTz77LP84he/YN++fdxxxx2UlJSwdOlSGhsbk2WnQDBsoi1iLQYJ/D371CdpM0RzT72ZV4r2rhZTs6EwLCH66KOP+Pa3v01hYSEPPPAAd9xxB/v37+fVV1+loaGBK664Ill2CgTDJlpn5sCjJBxqdEeCy8PELMo8hkVCeUQPPPAAlZWV7N69m0svvZQnn3ySSy+9FI1G0bXy8nKqqqooKytLpq0CwbCIVt7bJY9ywORIaAuhgbAMVOYhy0k7/3gnISFav349N910E8uXL6ewsHDAMXl5eTz22GPDMk4gSCZRj8gazSFK4tJ9dGrmlnuEKJpdLZIa4yIhIXr11VcpLS2NeUBRZFmmrq6O0tJSDAYDy5YtS4qRAkEyiAarMyLJS2aMYjHoMOm1RCSd0r9aZFcPiYSEaMqUKTQ2NpKXl9fneHt7O+Xl5YTD4aQYJxAkk4XTcunyBJnQ+qlyIEmBaoCzKrI5e2qO8qTNpgiRvxsycpJ2jfFMQsFqeZDKYpfLhckk/gMIUpPinvIOc6hnaT2JU7M+ZR4iu3rIDMkjWr16NaDc9LVr12KxWGKvhcNhNm7cyOzZs5NqoECQdJLUEG1QjNHNFoUQxcuQhGjbtm2A4hHt2LEDg+HIzgcGg4FZs2Zxxx13JNdCgSAJuPwhqtvc2PUypYHoqllm0s7f5Q3y+q5mZBm+ktlT5iGyq+NmSEL05ptvAnDjjTfy0EMPYbfbR8QogSDZtHb7eXVnMxMNLkpBCSInMZAsSVBz2INGkpDzrEggsquHQELB6srKymTbIRCMKNGs6kypx0tJ8rTM0rN8H5FlAloLRhBTsyEQtxBdeeWVVFVVYbfbufLKK485dsOGDXGdc926dWzYsIHPP/8cs9nMmWeeyS9+8QumTZsWr1kCQVxEhchO8qdlALqeerNAKIJX6hEiEayOm7iFyOFwxFYGHI7k/Dd5++23ufXWWzn99NMJhULcddddXHTRRezcuZOMjIykXEMgAPBEW8QSzSFKfqDaYtASCEXwSGYyQWRXD4G4haj3dCxZU7OXXnqpz/Oqqiry8vLYsmUL5557blKuIRCAEqyGXsmMyW6IhiJEnZ4gHtmoHBDZ1XGTUIzI6/Uiy3Js+b6mpobnnnuOGTNmcNFFFyVsTFeX0pohKytrwNf9fj9+/5Gm5E6nM+FrCdKL6H5mlrBLyZ5L8tQMwNyz0aInrBHZ1UMkoYTGK664gieffBKAzs5O5s2bx/33388VV1zB+vXrEzIkEolw++23c9ZZZzFz5swBx6xbtw6HwxF7lJSUJHQtQfrh8odBljGFe1ayRkCILHotJr1W2UlI9K4eEgl3aDznnHMA+Nvf/kZBQQE1NTU8+eST/OY3v0nIkFtvvZVPP/2Uv/zlL4OOWbNmDV1dXbFHXV1dQtcSpB8Xzchn8XQHFm04KVsIDcQFJ+bxrUVTmFWSKbKrh0hCUzOPx4PNpij+K6+8wpVXXolGo+GMM86gpqZmyOdbtWoV//d//8c777zDxIkTBx1nNBoxGo2JmCxIc0qyLKBtB40GjHbQJn8nrb67eYjs6qGQcPP8559/nrq6Ol5++eVYXKilpWVISY6yLLNq1Sqee+453njjDcrLyxMxRyCID2+H8tU8YeSvZRDZ1UMhISFau3Ytd9xxB2VlZcyfP58FCxYAinc0Z86cuM9z66238tRTT/HMM89gs9loamqiqakJr9ebiFkCwYB0eYLsONRFS3OTcmCEhKjN5efvWw7xf580gKEnRiSyq+MiIf/0qquu4uyzz6axsZFZs2bFjl9wwQV8+ctfjvs80cD2okWL+hyvrKxk+fLliZgmEPSjocvLa7uaOcNXR56VEROiiCxT2+4hw6iFAjE1GwoJT5QLCgooKCjoc2zevHlDOsdg7UQEgmQSzaq2yT3pHiMkRJae5XtvIIJsyOipNxNCFA8JCZHb7ebnP/85r7/+Oi0tLUQikT6vi51gBamEO6As3WeEuwHdiAmRuVe9mU/KwAwiuzpOEhKiFStW8Pbbb/P1r3+dwsLCvqsFAkGK4faH0EX8GKUgihBljsh1tBoJo16DPxjBg1ERIpFdHRcJCdG///1vXnzxRc4666xk2yMQJB23P4Qp1IVBq1ESDbX6EbuWRa/FH4zgjYjs6qGQ0KrZhAkTBi3DEAhSDUWInOi1mhFfuj8SJwqL7OohkJAQ3XfffaxduxaPx5NsewSCpOMOhDGFnMq20CMsRGaDFrNBS1iWRXb1EEhoanb//fezf/9+8vPzKSsrQ6/v6+pu3bo1KcYJBMNFlmW+NKsIaZcGg3/kheiLp/SKmXZEhUgkNR6PhIRoyZIlSTZDIBgZJElSyjtMPghIIy5EfXfz6MmuFrlExyUhIbr77ruTbYdAMLKMZnlHFJFdHTcJxYhAaf/xxz/+kTVr1tDe3g4oU7L6+vqkGScQDJcWp49Pq5txuaJ7mY2sEDV0evn7lkO8urNZFL4OgYQ8ok8++YQLL7wQh8NBdXU1N998M1lZWWzYsIHa2tpYryKBQG0Otrn5ZOcezB4f1okFoDMc/03DIBxRyjyyrQYoEsHqeEnII1q9ejXLly9n7969fXZ2vfTSS3nnnXeSZpxAMFxc/hDmYKeyYmbJHvHrmQ1KdrUnEO5Vgd+TXS0YlISEaPPmzdxyyy39jhcXF9PU1DRsowSCZOHyhzCHOjHqNGAZ+X3oo2UevmCYiL7HIwr3ZFcLBiUhITIajQP2i96zZw+5ubnDNkogSBbdvhCmYE9W9Wh4RHotkqQ4QD65J7saxPTsOCQkRF/60pf48Y9/TDAYBJQly9raWn74wx/yla98JakGCgTDIeoRKVOzka8G0GgkTPpe0zORXR0XCQnR/fffj8vlIjc3F6/Xy8KFC6moqMBms/HTn/402TYKBAkRDEfw+QOKRzRKMSJQthWCnjIPkV0dFwmtmjkcDl599VXee+89Pv74Y1wuF6eeeioXXnhhsu0TCBLG7Q9hDHWjk2S0euMR72SEMeuVMo9QRBYtY+NkyEIUiUSoqqpiw4YNVFdXI0kS5eXlFBQUIMuyaAkiSBkyjDqWnGDEGLEhWbJHrSfQVXMnHvkcdIlcongY0tRMlmW+9KUvsWLFCurr6zn55JM56aSTqKmpYfny5UNqEysQjDR6rYZCvYesDMOoTcvg6DIPkV0dD0PyiKqqqnjnnXd4/fXXOe+88/q89sYbb7BkyRKefPJJli5dmlQjBYKE8RxWvo6iEPVBZFfHxZA8oj//+c/cdddd/UQI4Pzzz+fOO+/k6aefTppxgsHp9AR4a3cLb+5uod0dUNuclORAq4tD9XV4g+FRFaKaw27+tuUQb+9pFcHqOBmSEH3yySdccsklg76+ePFiPv7442EbJTg+22o72VbbyfbaTv68qZYWp09tk1KOT+u7qK6rpcsbHFUh8oci1LV7aHb6RHZ1nAxJiNrb28nPzx/09fz8fDo6OoZtlOD4LJqWy/nT8yhwmAiEIvz70ybCEfGH3hufqwttJIhBP3IN8wciml3dp0ujyK4+JkMSonA4jE43eFhJq9USCoWGbZTg+EgBF7M0B7iyxEWGQUO7O8DOhv7Z7ulMuLsZAL01e0S2mB6MaB6ROxBS+mOL7OrjMqTfjizLLF++fND95/1+ofgjiSzLfHKoiwpTFxmf/x1CfozAFyjgBfkMNle3M7PYLlIogEAogsbTBoAxs+A4o5NLhlH5WPmDEYLhCHqjTfGG/N2QMfL1bmORIQnRsmXLjjtGrJiNHM1OP2/vPISr+VnOmGhEm5ENfieloUbK3ds4IJ1GXbuX0myL2qaqjtMXxBI8jE4jYbAPHk4YCYw6DTqNRCgi4/GHcRis4G4THtExGJIQVVZWjpQdgjjY3dxNofMT8oxBtBlFMPdGaD+A9rPnOE2zh7KyM8mzD+ytphtObxBLoEOpurfmjeq1JUkiw6ijyxvEFQjhENnVxyXhDo2C0UWWZQ40dVDo2kF2hhHKFypNvnKnwYRJFNuNzJL2xQou0x2nx4851IFRr1VlOmQ16pQyj3BE5BLFwehF8ATDorXbj659DwY5iCOrRBEgUMoWSuZDRw00fgxl5454F8KxwDR7iECBFY3eAKbMUb/+1af1KvPwiOzq4yE8ojHCwTY3ea7dOCwGtEWz+tZNZU0GcyahgI+9u7azva5TNTtTBXPgMA6zHltWoSr7zvdZMBAe0XERQjRGaGxtw+5vxGE2QMHMvi9KEuSdiMsXYs+OTXxU3Y6c7slz7lbl6yjHhwZEZFcfFyFEY4BwRCbQvAcAW14JmBz9B+XNwGbWk+Wvw+Px0OEJjrKVqcXegwdodvoImtSpMTvU4eFvWw7x2s5mkV0dB0KIxgBajcSSEg/TC2xYCqcNPCgjF60lC4dRg93XQG17+m4HHghFaKir5kCbm3DG6C7dRwmFZeraPTQ6fSK7Og6EEI0FwkEMzlomWAxIOScMPEaSIGsyDrOeTF8dNYfTd6nY6ezAEHaj1WowZRaqYkM0qdHtF9nV8SCEaCzgrIdISAl6Zhxjc4JeQnSo3UMkTWvPPId7Nvk0Z6m2gphhPFJvFo7Ionf1cRBClOJEIjLvbt5CTbuboK3k2CtAmaVYTAYsETcaXwdt7vScBvg7GgGQbKNb2tEbs16Lpud35Q6EjgSshRANiBCiFKfdE8DTUk2z0492wqRjD9YZ0GSWYjPqyPTXc9iVnn2Kgp2KR6RzqDMtg2h2dU/xqz8EJrvyghCiAREJjSlOS6cba6CFDKMWzYTS478hs5TynP1MKYhgKLSPvIEpSNipbPJpyipS1Y4Mo45uXwi3PwzGnt+Fr0tVm1IV4RGlOJ0tNWjkMKYMe3z7cjkmYtJrMbga0nOpOOAm7OkEwJo9UVVTrEYdFoNWiRHFPCLRqmUghEeU4vhaDmAGjLnl8WUI24tAo1WyeL0do7KpYErR3cSJhXY8OgeWLHU9wi+eUngkw7qjJ/fLJ4RoIIRHlMJEIjKhzgYAbHnHiQ9F0erBVkiz08crH3zEnuY0i0l0N2LUaZmQX4pRp24BcN8yj6hH1JWenupxEEKUwrS7/Zi8LWg1ErbcOOJDURwT8QXDeFtrqD2cZomNXT1L9/Zide04mqgQhUMQTLPfSRyIqVkK4+nuwIwPs1GPxjaEDOHMUmwmPfbDjRzq8o6cgamGLNPedBCXy4M2koXaUtTa7eftPa3otRJXzC5WSj0CbmV6Fi37EABCiFKaUl0XJWUTCJnzlClXvNiLsJp0mEIddHY58QXD6dGnyHOYLqeLhu4QEa9FdSGSJKhr9xy59yaHIkR+J6BeakEqIqZmqUx3IxIS+glD/EjpzRhsuZj0GqyBFhq70mSrIWc9vlAYlyGHCVaz2tZg7Snz8AXDBEKRIytnImDdD1WF6J133uHyyy+nqKgISZJ4/vnn1TQn9ehW8mFIJEPYXoTNqMfqb6GxM02mZ131+AJhXMZ8HOYheJAjhEmvxaBTPmIuf0jkEh0DVYXI7XYza9YsHnnkETXNSEnaun1s/nQn+1pdYEsgMc9ehM2kwxZooSFNPKJQZx3+UIRuQz7Z1tToUmk3KV5Rty94pH2LXwjR0agaI1q8eDGLFy9W04SUpa21iZDfi1drSqznco8QZXa1IevTYAYe9OLrbEEGwrYiLIbUCH9aTTraXAG6faEjQiSmZv1Ijd9WnPj9/j57pzmd4/cX6mytA8DgKFASFIdKRh5mk4n5JVqkKanhHYwonXV4AiG8+kzs9gEax6mEzahMEbt9IbCL7OrBGFP/KtetW4fD4Yg9SkpK1DZpxPC2KUJkzk7wZ9RokOyFSEjgbEiiZSlKZw2+YASnsYgca+psqWQz6cgwapWk+GiwOuCBUHoWJA/GmBKiNWvW0NXVFXvU1dWpbdKIEInIhLoU8XDkDUNs7UpsSXbW4wuGk2Fa6tJZQ0mWmQsWnM7cstHb5/54zCvPYuW5UzhjcjbozUcapImAdR/GlBAZjUbsdnufx3ik3e3D5GtDq5Gw5w5HiIrxBEK8t/UTnvqwJnkGphoBN7hakZCwFUzGblJ/xSxKv+2/zT0i6e0YfWNSmDElROlCW3M9GjmEyWxGGs7mgPYijDotOm8bbo8Xp2+cNtTvrFW+ZuSkfsayEKIBUTVY7XK52LdvX+z5wYMH2b59O1lZWZSWDqG2apxh9rUSNOowZhYNb08uow2txYHF0IU10EpjZyn2gtTxFpJGZy3uQIhPgzb0tR2cWpo6U7NIRGbDtnq6fUGun1eKyZypvODrVNOslENVj+ijjz5izpw5zJkzB4DVq1czZ84c1q5dq6ZZqjNJ38XJxQ5OqJg6/JPZCrGZ9FgDLTSM17qzjmq6fSE+905IuU0DNBqJwy4/nZ6g4pEKj2hAVPWIFi1aJDYCHIhupedyNNg8LOzF2IyfYO1uobFzHCY2etrB044rINNlL2aqzaS2Rf2wmnR4AmG6fSHyhBANiIgRpRjBYIBwd4vyJBnN3x3FSoa1v5lWp0+peRpPHN4PQIsmh7DGQJ499YQoGjxXkhozlYM+J0TG+UrmEBBClGIcqD7I5gOt7G7v9Uc7HKwFGA0GrBo/+pCTZuc484ra9xOWZQ5JiveYZ0+dHKIo0bq3Tk9A2VZIowM5IpbweyGEKMXobq1DBiIZBcMLVEfR6sCaT67VyOkTPFgM46gdSCgAnbV4AiEOm0qxGLTYjKlXLBAVoi5vUPmdRgPWYnoWQwhRiuE9fAgAS04SG787iinJsnB6lpfsFMo6HjadNRAJ0yVb8ekc5NtN/fN2UoBMiyJETm9P+kQsTtSpjkEpiBCiFCISkQlHM6rz4+xRHQ/2HlFz1ifvnKlA2x4AXBml6LSalJyWgeIRWQxazNFC3NgSvvCIoqSeH5vGtHe7MPo70GokHLlJ9Ih6Vt/C3S00tHaSZbfG9mYfs0TCMSGafvLpVNhLCKXoFtuZFgO3LJxy5IDwiPohPKIUor2pFpAxZDiQTEksXzHZwWRnd2MXr3y4jQOtqZVrkxAd1RD0KZnUjhJ0Ws3YaYdr7tniyXNYXTtSCCFEKUR3tPXHhBHYGNBejM2ow+YfJ4mNrZ8DIOecAJox9mccLdvxdii7egiEEKUSBVIHOVYDmfkjUN5iL8Zq0mENNFPfMcaFqNe07IPuHP70QTV7U3z/tq21HVS+d5CNBw6DwapU4csyeNvVNi0lEEKUQhRrOpiaZ6OoZHLyT96T2GgPtNDlCSg5LWOV9oM90zILe3wTaHMFUnK1rDfhiEynJ0iHJ6As4Ue9InebuoalCEKIUoWAW0lwkySwJiGj+mis+eh0BiboQ5hDndSM5Y0Xmz4GwJ05nQ5vGI0kMXGC+rt2HIsjSY09S/iWHiHyCCECIUQpQ1ezkpgnm7NAPwJlChotOIrJtBiw+xqoaR+jQhRwQ5vSseGgXlmJKsw0pXygeoJFadfb7gko9ZWWbOUFEbAGhBClDHv27ebjQ13s9lhH7iKZk3CY9dj9jdS1ewin6HL3MWn+TCmPsBWw36N4QeU5Kd6DCJhg0SNJ4A9G8ATCYmp2FEKIUgR/h5JsmJEzgn24M0vJMGqZbXdyzdyJaFI7rNIfWYZGZVoWzDuZuh6vblK2RU2r4kKn1cSmZ+3uwBGPyNshil8RQpQSBEPhWIP7CQVJzKg+GnsRklbPRItMrsaZ8gHefnRUKx6EVs9BzSSCYRmHWU/uGClbycromZ65A8rWQlq9IkKi5kwIUSrQ2taMNuxHr9NhzUpCD6LB0GjB0eNxRdurjiUOfaR8LTgFh93GjCI7M4sdY0ZQ8+0mChwm9FpN35UzV4u6hqUAYzzPf3zQ2XAAAK2jSKmWH0kyJ0H7QZpqPmdLezHzJ2el1PY7g+Jph8P7lA/wxNPIt5i4+KQRWF0cQc6YnK3s5hHFWgDORnA1Q/4M9QxLAYRHlAK4mg8CYMkrG/mLTVCmfu31e9nb1Mm+FtfIXzMZ1G1UvmZXgCVLXVuShS1f+epqVteOFEAIkcrIskygXZkmZRaOQCLj0dgKwWAhxwQ2fxN7m7tTv12vtwMaPwFALpnH+/vbaO32H+dNqUsoHFFWLK09QtTdpATi0xghRCoj+12clBmiONNMdmH5yF9QkiC7gqwMAzn+OtpcAVpS/UNd/Z6yZJ81mUORHDYeaOevH9WNyba3L2yv55E399PQ6YWMPJA0EPSCP7VLVEYaIUQqo+luYILFQGlJKTrTKOXDZFeg02io0DYB8FlDCrcsdbdB86fK9+XnsPGgUpt1YqENg27s/flqJImILNPS7VPigRk9MaM0n56Nvd/keKOrZ9vszFHcx21CGWi0TDT6MAU72dXYTTCcgt6FLMPeV5WvOVOpC2VS1+5Bq5E4rWxsxonybMrCQGxqGS3nie7ckqYIIVKZ6v2f0+EJELKO4LL90eiMkDkJu1lHabiWQCjC540pODVo3a3kDml0RCZfwH/2KVnIM4vtKbWt9FDIPVqIoltGdY2z7plDRAiRirhcThoP1bC7qZuQfQQzqgci70QkJE7W15NjNWA1pVgmR8ANe19Rvi+dz452iaYuHwadhnnl2cd+bwoT3e6o3R1UvFBHrza+kRT0SkcJIUQq0lT9OSCjtedjsmaO7sVzTgCNjokGNzfMNKVWvZYsw56XFDHKyKE7//SYN3RWRQ7WMdzmNsOgxWLQEpFl2lx+pQpfZ4BwENzpm9gohEhFnA1Kcy9LfsXoX1xvguwpaCQJqfmz0b/+sTi0GVr3KJngJ16O1WzizCnZTMq2cEqxQ23rhoUkSRQ4FK+oodOndJeMbm6QxtMzIUQqIUci+JqVXUqzJp6gjhEFJytfmz4hFPCzubqdj6pV7hh4eD/sf0P5fvJ5YCtAkiTmlE7gy3OK0Yy5St3+TM6xMr3ARnZP7RmOYuVrdOEiDRm7Pu4Yp621ASnQjaTVkTtRBY8IIGuKUnzp66Jx7xb+05CDRpIonmCm0KFCo7HOWvhsA8gywbyTeddVwoJAGHPPppBjpabseJw80cHJE3t5drH6vxplWjpOfs6hIDwilWir3gmAPmsSOoNKtV4aDRTPBWCi+zNOyLMSkWVe/KQRpy84ura07YNP/grhEC7rJP7adSIfH3Ly4o7G1M/8Hi6OiUolfsCTtvlEQohUItC0CwB7icrFjoWngM6I5G7jwpzDTLDo6faF2LDl0Oj0tZZlqP0QPv0bkVCAA5FcnuyaQ4tb8YTOqsgeN55Qb2RZ5rDLr7QE0WiV3C6A9gOq2qUWQojUwOfkFLubkydmUjJ1lrq26M1QMh8AY917XDmnAJtJR4cnyDObavmsoWvkPBL3Ydj+NKG9r9PY6eH1rkL+ET4Lv6xlUraFG86YpM4UcRTYeLCdJz+o4cMDPa1iJ/SU97QfVM8oFRFCpAZte5GQsOaWjv6y/UBMPF3ZqNDbgb1pI9eeXkKhw4Q/GOHdvW19WspGktBeNuxuJ7jzRdj8R+is47AX3pDn8mnGmVjNRi46KZ8vzyke08v0x6M0S+kqWXPYo9zTrB4h6jqk1J6lGeP3N53ChJt2oAXIna62KQo6A0y9CD57Dmo/xJY1matPK2F7XQehsIxOq/y/CkdkHnlzHzaTjgyDDqNeg1GnQavRIAFFmWZmFCk71AZCEd7b30Y4LBOWZcJBP5qOg5gPf4rJWUOBw8ikrAzImUrW6eej/9zNBQU2Tiy0K43DxjkFdqXhvy8YptHpozgzS2mU5m5T9mwrVNlTHmWEEI0yXW0N7N7xGdlWE5PPPJGUiX7kTYe2GdC8Ez79O9rZNzB3Um6fIV3eYGx/rti2OL2QISZEEVlm54FDOHz1TPDWkOk7hEZWdjWNAG2GiUyacxlklqAHrjltbNaOJYpGI1GWbeHzpm6q29wUZ5ohbwYcfAdadgkhEowshz7fTDgi026ayBSjTW1z+jLtUqX3j7MRtv1JeZ47LbacPMGi5+ZzJ9PhDuAPhfEFI/hDSm+dSCRCvjGgCFlnDYb2ar7kaUCSQGOWkCwGNJZ8NHnTMZbMxZaVm5bL1L0pz83g86Zu9jR3c+aUbKS8ExUh6qhRssoNKZTtPsIIIRpFwqEgruqtAORUnK6yNQOg1cPJ18Cnf1diFZ89pywtF5wM9mIkoxWrVos1I6J8UOR2CLaDq0lp/h9wx06lAUqyM5RGbBPKFEGz5qe9+PRmco4Vg05DpydIY5ePoswssBUojdKadkDpGWqbOGoIIRpFanduJOJ3g9FGScXJapszMAYLzLoeat9XltW7DimPeJA0YM1VWppklikiNhKbRY4TDDoNU3Kt7Gp0srfFRVGmGYpPhc//BfVbYeI8JdcrDRBCNErIkTBtu94FwD71LHT6FG5jodVB+blQOBuaPlFacbiaIdSTVyRJyrK/OQvMExRPx16ofNWm8M+Vgpw6KZMT8q1Myu6ZhuXNgP1vKtuPH96reJJpgBCiUaJu50ZCrsPIOiNTTjlTbXPiw2SHsrOVByh7cEXCitiIKVZSyLOZyOsdKtTqoWg21HwA1e9C9tS08IrG/0+YAshBH207XgXANm0RJnPq70w6IBqtstQvRGhE8AXDSh/ukvlK8zpX65E2ueMcIUSjgHTgLU7O1ZGXX8i0UxepbY4gBdnd1E3V+9VK9wO9GSb1eM0H3uyzCDBeEUI00rTsgoZt6LUappz1FQwGEUMR9EerAW8gzEc1HbQ4fVB8mpLgGPDA5y+O++2GhBCNIG2HdlP/wf8iI0Pp/COFjQLBUUzJtTIlz0o4IvN/nzTiCgEzrgCNTunRtOflcS1GKSFEjzzyCGVlZZhMJubPn8+mTZvUNml4yDL1uzay/7XHqW1zUStNhPJFalslSGEkSeKiGfnYzXq6vEE2bD1EpyYTpl+mxOQatsGufxxZuRxnqC5E//u//8vq1au5++672bp1K7NmzeLiiy+mpWVs9u/taKrm43//gdr3nyUSDkPOCeSfcU1arHwIhodJr+WqUydiM+k47Arw9MZadgSLYdpiJUereSds/oOy6204pLa5SUWSVe46NX/+fE4//XR++9vfAhCJRCgpKeE73/kOd9555zHf63Q6cTgcdHV1YbfbR8PcGJFQCE3Yq8zhPW3sO7CPzrrPCXa3AiBLGswV53DymZei02lH1TbB2KbbF+RfOxpp6PSxYEo2Z0zOhs5aAp++gL+7A4NOg96UgSarDBylYMkCcyboLaBNrVXNeD+jquYRBQIBtmzZwpo1a2LHNBoNF154IR988EG/8X6/H7//yPbITqczvgu1H1D6IMsynzc66fJG3Vu5Z96taLEky8wrz4rNxXc3Oenw+I/MzWUZGZDkMFI4wLzyLKSestVQk5OgJ4gsadAWnMSkUy+goGDikO6HQABgM+m5em4JOxudnJDfk2SUWcrnpdexa/Ob5HfvxBiuRyPVo9W8r2yAAEzNt2E1GUBnoLk7SG2nD9AgSxqQNMi9SqynFVjJNCs9s1u7fRxo8xwxoJeOTS+w4RhsD7lTrlVyzZKAqkLU1tZGOBwmPz+/z/H8/Hw+//zzfuPXrVvHvffeO/QLhQJKTgYgu51EvAO3QZUAfL1uScCF7O8/J49KVygCepMFLNnYLRloLcUUlE3HZhtd70ww/tBoJGYetWOJP6KlO+80muyzyPA34/DVkxFowxzsxBB2EZFlkCMQ9BHxewl5PIOcHXD5IKIIkez2EXENkiJg9UN4ECGSk7cP25jKrF6zZg2rV6+OPXc6nZSUxLExYWYJzLoOJIncigCOcI8HJEkgST2tSCUkjQS2no6AkkShN0RWJIKEMubIOC1GswWdOSMW+xnFfVoFacrpZVmcXpZFJCLjD1UQCEcIhSMEwzIyMkaTBHIAwkFsPh+lngBSJIwsR/qtuJkyDKBXXJ+MQJhS98D/nE02A2gHmeolsTuAqkKUk5ODVqulublvw/Dm5mYKCgr6jTcajRiNCTSaN2TEOuDlTIj/bfYU69IhEIDiLZkNWswMFHtUioytGWCNc0PcjJ6Hmqi6lGMwGJg7dy6vv/567FgkEuH1119nwYIFKlomEAhGE9WnZqtXr2bZsmWcdtppzJs3jwcffBC3282NN96otmkCgWCUUF2Irr32WlpbW1m7di1NTU3Mnj2bl156qV8AWyAQjF9UzyMaDmrmEQkEguMT72dUpPsKBALVEUIkEAhURwiRQCBQHdWD1cMhGt6Ku9RDIBCMKtHP5vFC0WNaiLq7uwHiy64WCASq0d3djcPhGPT1Mb1qFolEaGhowGaz9ZRfpBfREpe6ujqxahgn4p4lRqL3TZZluru7KSoqQnOMVjhj2iPSaDRMnCgq3O12u/hQDRFxzxIjkft2LE8oighWCwQC1RFCJBAIVEcI0RjGaDRy9913J9aRIE0R9ywxRvq+jelgtUAgGB8Ij0ggEKiOECKBQKA6QogEAoHqCCESCASqI4QohRnqDrjPPvss06dPx2QycfLJJ/Ovf/1rlCxNLYZy36qqqmIbI0QfJpNpFK1Vn3feeYfLL7+coqIiJEni+eefP+573nrrLU499VSMRiMVFRVUVVUNywYhRCnKUHfAff/997n++uv5xje+wbZt21iyZAlLlizh008/HWXL1SWRnYPtdjuNjY2xR01NzSharD5ut5tZs2bxyCOPxDX+4MGDXHbZZZx33nls376d22+/nRUrVvDyyy8nboQsSEnmzZsn33rrrbHn4XBYLioqktetWzfg+GuuuUa+7LLL+hybP3++fMstt4yonanGUO9bZWWl7HA4Rsm61AeQn3vuuWOO+cEPfiCfdNJJfY5de+218sUXX5zwdYVHlIJEd8C98MILY8eOtQMuwAcffNBnPMDFF1886PjxSCL3DcDlcjFp0iRKSkq44oor+Oyzz0bD3DHLSPytCSFKQY61A25TU9OA72lqahrS+PFIIvdt2rRpPP7447zwwgs89dRTRCIRzjzzTA4dOjQaJo9JBvtbczqdeL3ehM45pqvvBYLhsmDBgj576J155pmceOKJPProo9x3330qWpZeCI8oBRnqDrgABQUFQxo/Hknkvh2NXq9nzpw57Nu3byRMHBcM9rdmt9sxm80JnVMIUQqSyA64CxYs6DMe4NVXX02rHXOTsXNwOBxmx44dFBYWjpSZY54R+VtLOMwtGFH+8pe/yEajUa6qqpJ37twpr1y5Us7MzJSbmppkWZblr3/96/Kdd94ZG//ee+/JOp1O/tWvfiXv2rVLvvvuu2W9Xi/v2LFDrR9BFYZ63+6991755Zdflvfv3y9v2bJFvu6662STySR/9tlnav0Io053d7e8bds2edu2bTIgP/DAA/K2bdvkmpoaWZZl+c4775S//vWvx8YfOHBAtlgs8n/913/Ju3btkh955BFZq9XKL730UsI2CCFKYR5++GG5tLRUNhgM8rx58+QPP/ww9trChQvlZcuW9Rn/17/+VT7hhBNkg8Egn3TSSfKLL744yhanBkO5b7fffntsbH5+vnzppZfKW7duVcFq9XjzzTdloN8jep+WLVsmL1y4sN97Zs+eLRsMBnny5MlyZWXlsGwQbUAEAoHqiBiRQCBQHSFEAoFAdYQQCQQC1RFCJBAIVEcIkUAgUB0hRAKBQHWEEAkEAtURQiQQCFRHCJFgTFFVVUVmZmbs+T333MPs2bNjz5cvX86SJUtG3S7B8BBCJBiQ5cuXI0kS3/zmN/u9duuttyJJEsuXL+8zPtkCUFZWxoMPPtjn2LXXXsuePXsGfc9DDz3Up3/yokWLuP3225NqlyD5CCESDEpJSQl/+ctf+jS78vl8PPPMM5SWlqpik9lsJi8vb9DXHQ5HH49JMDYQQiQYlFNPPZWSkhI2bNgQO7ZhwwZKS0uZM2fOsM49kKeyZMmSmJe1aNEiampq+N73vhfbXQP6T82Oprdntnz5ct5++20eeuih2DkOHjxIRUUFv/rVr/q8b/v27UiSJPoQqYQQIsExuemmm6isrIw9f/zxx7nxxhtH/LobNmxg4sSJ/PjHP47trjFUHnroIRYsWMDNN98cO0dpaWm/nwmgsrKSc889l4qKimT9CIIhIIRIcExuuOEG/vOf/1BTU0NNTQ3vvfceN9xww4hfNysrC61Wi81mo6CgIKFOkw6HA4PBgMViiZ1Dq9WyfPlydu/eHdvvLBgM8swzz3DTTTcl+8cQxInoWS04Jrm5uVx22WVUVVUhyzKXXXYZOTk5aps1LIqKirjssst4/PHHmTdvHv/85z/x+/1cffXVapuWtgiPSHBcbrrpJqqqqnjiiSeS5jVoNBqOboUVDAaTcu54WLFiRSwQX1lZybXXXovFYhm16wv6IoRIcFwuueQSAoEAwWCQiy++OCnnzM3N7RP3CYfD/XalNRgMhMPhYV1nsHNceumlZGRksH79el566SUxLVMZMTUTHBetVsuuXbti3w9GV1cX27dv73MsOzubkpKSfmPPP/98Vq9ezYsvvsiUKVN44IEH6Ozs7DOmrKyMd955h+uuuw6j0ZjQlLCsrIyNGzdSXV2N1WolKysLjUYTixWtWbOGqVOnptUmA6mI8IgEcWG327Hb7ccc89ZbbzFnzpw+j3vvvXfAsTfddBPLli1j6dKlLFy4kMmTJ3Peeef1GfPjH/+Y6upqpkyZQm5ubkJ233HHHWi1WmbMmEFubi61tbWx177xjW8QCARGZRVQcGxEz2pB2vLuu+9ywQUXUFdX12/nUsHoIoRIkHb4/X5aW1tZtmwZBQUFPP3002qblPaIqZkg7fjzn//MpEmT6Ozs5Je//KXa5ggQHpFAIEgBhEckEAhURwiRQCBQHSFEAoFAdYQQCQQC1RFCJBAIVEcIkUAgUB0hRAKBQHWEEAkEAtX5/0J+lT85FfF/AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:52:22.954710Z", + "iopub.status.busy": "2024-03-22T22:52:22.954391Z", + "iopub.status.idle": "2024-03-22T22:52:23.361755Z", + "shell.execute_reply": "2024-03-22T22:52:23.360832Z" + }, + "papermill": { + "duration": 0.43022, + "end_time": "2024-03-22T22:52:23.364049", + "exception": false, + "start_time": "2024-03-22T22:52:22.933829", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEoCAYAAAD8JfzQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABD1klEQVR4nO29eXhU5d3//zqzz2SW7HtC2AVZRaGgLW6VR611qdW2jwquXbBXlR9VeBas1hbbqoWn+tjWFqhftT61orW1itQqKCJFEEXZJJAFkpCELJOZyazn/P44M0NCEpJMljPJ3K/rmmuSM2f5zPaez33fn0VSFEVBIBAINESntQECgUAghEggEGiOECKBQKA5QogEAoHmCCESCASaI4RIIBBojhAigUCgOUKIBAKB5gghEggEmiOESCAQaI5BawNiPPLII6xcuZIf/OAHrFmzpk/HyLJMTU0NDocDSZKG1kCBQNBvFEWhra2NwsJCdLqe/Z6kEKKdO3fym9/8hhkzZvTruJqaGkpKSobIKoFAMFhUV1dTXFzc4+OaC5HH4+Hf//3fefrpp3n44Yf7dazD4QDUJ+l0OofCPIFAMADcbjclJSXx72pPaC5ES5cu5corr+TSSy/tVYgCgQCBQCD+f1tbGwBOp1MIkUCQxPQ2daKpEL3wwgvs3r2bnTt39mn/1atX8+CDDw6xVQKBYLjRbNWsurqaH/zgBzz33HNYLJY+HbNy5UpaW1vjt+rq6iG2UiAQDAeSVoXRXnnlFa699lr0en18WyQSQZIkdDodgUCg02Pd4Xa7cblctLa2iqGZQJCE9PU7qtnQ7JJLLmHv3r2dtt16662cddZZ3H///b2KkEAgGD1oJkQOh4Np06Z12paWlkZWVlaX7QKBYHQjIqsFAoHmaL5835F33nlHaxMEAgCCYZl3P2+g2Rdi3thMSjJtWps0qhEekUDQDVsPNfDJsVaqm3y8+nENbf6Q1iaNaoQQCQSn4faH+KzGDYBRLxEMy+ypbtHWqFGOECKB4DQO1rUhKwrFGVb+bVp+fJtoATh0JNUckUCQDGSmmRibncaEXDtjs+1cNbOA4gybqPAwhAghEghOY3yOnfE59vj/E3LPnLApGDhiaCYQCDRHCJFA0IH6Nj+t7aFO80H+UIRthxt59eMaMU80RIihmUDQga2HGqlu8rHo7HymFqq5UQadxK7KZiKygrs9jMtm1NjK0YfwiASCKIqi0NCm1rvKtpvi2w16Hdl2MwB1br8mto12hBAJBFG8wQj+UARJUlfOOpLnVIWo0RPo7lDBABFCJBBEaYx6Qxk2EwZ9569GTJhOeoPDblcqIIRIIIhy0hsblpm7PBYTomYhREOCECKBIEpDmyoyWXZTl8diQtTiCxGRxcrZYCOESCCI0hT1drK7ESK72YDJoMOgl/D4w8Nt2qhHLN8LBFHOLcug0RMg19m1hrokSdx6fhlWo16kegwBQogEgiiT8hxMyus5ncNmEl+XoUIMzQQCgeYIiRcIUFfD2vxhMu0m7Obuvxa1re3srGjGYTZw0Vm5w2zh6EZ4RAIBcPBEGy/tPsb28pPd7xDwEPG1UF7voarJN7zGpQDCIxIIUJflAVzWbvLIju+GzzeTHQpR2lzKcd18FEURk9aDiPCIBAKgtV1duk8/PaHV2wiH/wGKjFmvo6jtExzeSjwBsYQ/mAghEgg45RGln+4RVW0HOQJZE5BK52I26ihyfxTfXzA4CCESpDzBsIwvGAHA2VGIQn5oOKD+PWYBlHwBk9GAI3CC9tYGDSwdvQghEqQ8sVZBJoMOi7FDq/PGgxAJQ1o2OAvBbCfsHANA5MQ+LUwdtQghEqQ8sfkep+W0tZuT5ep9zlkQnZhWsiehl8DYWjmcJo56xKqZIOXJTDPx5al56HUdVsFkGZorojuMi2+ePm0mOt92JJ0bIiHQi2qNg4EQIkHK47AYmVbk6ryxrRbCATBawFEQ36y3ZYDZCYE2aK3uJFKCxBFDM4GgO9zH1XtXCeg6fE0kCdJL1b9bjw+/XaMU4REJUp6jjV70kkSu03xqsjomRM7CTvuGIjLb6wyk17RwdkYNegSDgfCIBCnPOwfreWn3sXjhfADcNeq9s6jTvgadRHm7A28gQrCldhitHN0IIRKkNIqixAudOS3RiWe/W71JUqf5IVDrEumceYBEqN2tzhUJBowQIkFK0x6KEJYVJAnsseX7tjr1Pi0bDF2rNabZbLQbXQTCMnjqh9Ha0YsQIkFK0xb1htJMhlPL996ouNjzuj3GbjbgM2YSDMtqLppgwAghEqQ0sahqe8dgxpiXk9Z9zSG72UC7MYNgRAafEKLBQAiRIKWJeUSOjkIU83LsOd0eYzPraTemExIe0aAhhEiQ0ngDarJrWqwqYyQE7U3q3z14RGkmA2FLppr14TsJimgvNFBEHJEgpZmc7yDdZjzVYtrbqAqL0QqmtG6PmZRnZ/Klc+Ddt9Xo66AHzD0X3Rf0jhAiQUqT4zCT4+jQ2TU+UZ0bT3Q9HUmSQG8Aawb4mlTxEkI0IMTQTCDoiDdaZ6iHYVknrJnqfWwoJ0gYIUSClOZgXRuVJ72EI7K6wdes3tsyz3jc3/fW8nZVCH84Au0tQ2tkCiCESJCyyLLC65/WsnH3cXUpHtTJZ+hViE64/dQFLWoskb9laA1NAYQQCVIWXyiCooBOkrAa9Wptan+r+qD1zEKUZjLgNzgIRWThEQ0CQogEKYs3WpnRZor2s29vAUVWi531MvlsM+sJGJynPCKxhD8gNBWip556ihkzZuB0OnE6ncyfP5/XX39dS5MEKURciMzRYh6xSWdbZo8rZjHSzDGPSIFwEEKi6eJA0FSIiouLeeSRR9i1axcffvghF198MVdffTWfffaZlmYJUoRY5454i2lfVIh6GZaBOjRTJAPtOpu6QQzPBoSmQnTVVVdxxRVXMHHiRCZNmsRPfvIT7HY7H3zwgZZmCVKEU0OzmBD1baJaPUb1orySXd0gJqwHRNIENEYiEV588UW8Xi/z58/vdp9AIEAgcKp4ldvtHi7zBKMQbzCWeX/a0KwvHpHZgMmgI2RyAh7hEQ0QzYVo7969zJ8/H7/fj91u5+WXX2bq1Knd7rt69WoefPDBYbZQMFqZVugiz2kh2x6NrI4NzWxZvR5blmVj6UUToKIOjtYIj2iAaL5qNnnyZPbs2cOOHTv47ne/y+LFi9m3r/vmdStXrqS1tTV+q66uHmZrBaOJXKeFs6NiRMgPQa/6QB+GZlJsMtsS7f7hF975QNDcIzKZTEyYMAGAOXPmsHPnTtauXctvfvObLvuazWbMZnOX7QLBgGmPRlSb0sDQj89YbJk/IIRoIGguRKcjy3KneSCBYChQFIV9tW7STAZKMm3oY0Mra0afz/H3vbX4Wtq4MhTB6nersUS9LPsLukdTIVq5ciWXX345paWltLW18fzzz/POO++wadMmLc0SpACBsMybn50A4O6LJ6CPTTZbXD0fdBoNbQGa2/UEwzJWYxhC7WCyDYG1ox9Nhai+vp5bbrmF2tpaXC4XM2bMYNOmTXz5y1/W0ixBChCLITIZdBj1ulOTzdb0Pp/DatLTJBkI6q1AWB2eCSFKCE2F6Pe//72WlxekMLEYongwYyzHrB8eUSyWqF1nB1rUCWtH/iBamTpovmomEGhBLIbIFo8halHvLel9PscpIbKqG8SEdcIIIRKkJJ1qVcvyKRHpz9DMqHpTPilaUlYIUcIIIRKkJL6OHlGwTS0BotODqe8lX+NpHkTnhUQsUcIIIRKkJDGPyG42nBqWmZ2g6/tXwmbSYzLokM1OdYPwiBIm6eKIBILhYHZpOiWZVnIdFvAcUzf2Y6IaYEKunYl5Dmi1wu73hEc0AIQQCVKSPKdFTe0AaGxR7/sxPwQd0zyiHlHQo8439cOrEqiIV0wgiC/dpyd2vMmuzi8pijrfJOg3wiMSpBwRWWF/rRu72UBppg1dAlHVMf6y5zieQJjrdVbMsgcCnoTOk+oIj0iQcviCYTbvO8GrH9eoqWEJRFXHOOH2U+8OEIjFEgU9g2VmSiGESJByxFbMbCY9khxRvRhIyJOxRqs7+mMlYwNCiBJBCJEg5YhXZjQbTs0P6Y1g7H+emM2oxhL5Y0GNYo4oIYQQCVIOXwePqNOwLIESHrGgRp8uugInPKKEEEIkSDlO1ao2nBKiBFfMrDEhIpZvJjyiRBBCJEg5Ypn3aR2jqhMUolgHEK8iJqsHghAiQcrhDcYSXvUJlf/oiM2kx2zUETFG2woJjyghRByRIOU4ryyDCTl2ClwWqGtRNyawdA9wdqGTaUUutfj+e0A4AJGQOvkt6DNCiAQpR4HLSoErOpQaYFR1PM3DYAa9ASJh1SvqQycQwSnE0EyQuoT86g0GHg0tSWr2Poh5ogQQHpEgpQiGZQ6daMNuNlBmjs7nmGxgMCV0vois8OrHx/EFI9xoSMNAk1jCTwAhRIKUwu0PsXnfCSxGPd89W109G4g3pJPgeHM7oYhCINumfqHEhHW/EUMzQUrhC3RYMRvg0j2oc0SWaHR1QB+NzBbR1f1GCJEgpThVNL9DekeCK2YxYrFE7ZKIrk4UIUSClCJWq9pu7pDeMcCJ6niaR6x2tZis7jdCiAQphSeeZzbwqOoY8TQP4REljBAiQUrhi6V3mHQDjqqOEfOIPDGPKNCmVmsU9BkhRIKUIpbeYZcCIIfV+J9BECKzUYdsiAqRHFYjrAV9RizfC1KKBeOzaClwUqBvUjeYHWq96QFwTmkGc8ZEI6kbzKoIBT1gtAzQ2tQhJYQoEokQCoW0NkOQBGRaJDItJmhwq1UVzbng9/e4v8lkQtdLVw6pYx0js0MVokAbpGUPltmjnlEtRIqiUFdXR0tLi9amCJKNcASc80BvgqNHe9xNp9MxduxYTKY+Rl6b7OBtFCtn/SQhITpy5Ajjxo0bbFsGnZgI5ebmYrPZOv9yCVIOWVYIhiPodBKmsBfCfjClqbdu95epqamhtraW0tLSHj8/7cEIr39aSyAs8w17GhKIlbN+kpAQTZgwgYULF3L77bdz/fXXY7Ek31g4EonERSgrK0trcwRJQDAs0xYOolMknCY96IxgtZ1xLicnJ4eamhrC4TBGY/elPQx6icqTPgBCLhsmgKB3CJ7B6CWhVbPdu3czY8YMli1bRn5+Pt/+9rf517/+Ndi2DYjYnJDN1v+C6ILRiRxdUtfrAFldPUM680R1bEgWiUR63Meo12EyqF8lkeaRGAkJ0axZs1i7di01NTWsW7eO2tpaLrjgAqZNm8bjjz9OQ0PDYNuZMGI4JogREyKdJIESFZZeVsz6+vmxRvPN2qVY7WoxNOsPA4ojMhgMXHfddbz44ov87Gc/4/DhwyxfvpySkhJuueUWamtrB8tOgWDAyHLUI0IGBTWGSBqcUDqrSPMYEAN6Fz788EO+973vUVBQwOOPP87y5cspLy9n8+bN1NTUcPXVVw+WnQLBgInqkCpEoA7LBsljtnWX5iGiq/tMQkL0+OOPM336dBYsWEBNTQ3PPPMMlZWVPPzww4wdO5YvfvGLbNiwgd27dw+2vYIEqaioQJIk9uzZMyzn2bZtG9OnT8doNHLNNdcM6JqDRXxoRt+GZf0hNjTzKlEhEtHV/SKhVbOnnnqK2267jSVLllBQUNDtPrm5ufz+978fkHGCoeOdd97hoosuorm5mfT09EE//7Jly5g1axavv/46drt90M+fCJGoS6RToh7RIAqRzWTAYtQjSwa1frWIru4XCQnR5s2bKS0t7RJxqigK1dXVlJaWYjKZWLx48aAYmeoEg8G+B9QlCeXl5XznO9+huLg44XMM9vN2WIxEZAVDuF3d0GHFTFEUIpEIBkNiMb7nT8jigonRSOpGEV3dXxIamo0fP57GxsYu25uamhg7duyAjRpqgmG5x1s4Ivd531Af9k2ECy+8kLvvvpt77rmH7OxsFi1axKeffsrll1+O3W4nLy+Pm2++udN78MYbb3DBBReQnp5OVlYWX/nKVygvL+/2/BUVFVx00UUAZGRkIEkSS5Ys6dd5Dhw4wIIFC7BYLEybNo0tW7bEzy1JEidPnuS2225DkiQ2bNgAwJYtW5g7dy5ms5mCggJWrFhBOBw+4/N+5513kCSJTZs2MXv2bKxWKxdffDH19fW8/vrrTJkyBafTybe+9S18Pl/8XLIss3r1asaOHYvVamXmzJm8+spGrCY9OiXCO+++j2Rx8PrrrzNnzhzMZjPvvfdeQu8XnLa6Zop6gGLCus8kJP9KD5NwHo8nKYMbT+fJtw/3+NjY7DSumV0U//+3W8sJRbp/vsUZVr5+bkn8/3XbjtIe7Bxvcu+XJyVk4x/+8Ae++93vsm3bNlpaWrj44ou54447+OUvf0l7ezv3338/N9xwA//85z8B8Hq9LFu2jBkzZuDxeFi1ahXXXnste/bs6eK5lpSU8NJLL/G1r32NgwcP4nQ6sVqt/TrPD3/4Q9asWcPUqVN5/PHHueqqqzh69CglJSXU1tYyefJkHnroIW688UZcLhfHjx/niiuuYMmSJTzzzDMcOHCAO++8E4vFwo9+9KNunzcQX3n90Y9+xBNPPIHNZuOGG27ghhtuwGw28/zzz+PxeLj22mv51a9+xf333w/A6tWrefbZZ/n1r3/NxIkT2bp1KzfddBM5OTksnHNW/HorVqzg0UcfZdy4cWRkZCT0XnXBHGu2KISor/RLiJYtWwao6r9q1apOwYKRSIQdO3Ywa9asQTUwVZk4cSI///nPAXj44YeZPXs2P/3pT+OPr1u3jpKSEg4dOsSkSZP42te+1un4devWkZOTw759+5g2bVqnx/R6PZmZarZ4bm5upzmivp7n7rvvju/71FNP8cYbb/D73/+e++67j/z8fCRJwuVykZ+fD8D//u//UlJSwhNPPIEkSZx11lnU1NRw//33s2rVqrjIdXzecEqIHn74Yc4//3wAbr/9dlauXEl5eXk81ej666/n7bff5v777ycQCPDTn/6Uf/zjH8yfPx+AMWVjeWfrVv73qV+z8LePx8//0EMP8eUvf7n3N6QXWttDvLX/BIoCX0uPpoyI6Oo+0y8h+uijjwDVI9q7d2+n8bvJZGLmzJksX758cC0cApZeNKHHx3Snrebe9aXxPe57+srvbecP3rB0zpw58b8//vhj3n777W4nfcvLy5k0aRKff/45q1atYseOHTQ2NiLL6rCwqqqqixCdib6eJ/YFBzWe7Nxzz2X//v09nnf//v3Mnz+/0xDm/PPPx+PxcOzYMUpLS7s8747MmDEj/ndeXh42m61TvmNeXl48uv/w4cP4fL4uAhMMBpk+Y6b6T9SOc889t+cXox9IElSe9KGTJJRcu5pvJqKr+0y/hOjtt98G4NZbb2Xt2rU4nc4hMWqoiYXja7lvb6SlnUrE9Hg8XHXVVfzsZz/rsl9s1fKqq65izJgxPP300xQWFiLLMtOmTSMYDPbruoN1nkTp+Lw70jHPS5KkLnlfkiTFRdPjUYdEr732GkVF6jDbHwzjCURwWqLvUXSiuqfr9RdbdPleVhSCehtmEEOzfpDQHNH69esH2w7BGTjnnHN46aWXKCsr63ZV5+TJkxw8eJCnn36aL37xiwC9Trx2l0PVn/N88MEHfOlLXwIgHA6za9cu7r777h6vN2XKFF566SUURYl7Rdu2bcPhcAxoZa07pk6ditlspqqqioULFwLgCYTxBsKkSQFQfNBLjaH+YojmmwXDMu1SVIjEZHWf6bMQXXfddWzYsAGn08l11113xn03btzYp3OuXr2ajRs3cuDAAaxWKwsWLOBnP/sZkydP7qtZKcHSpUt5+umn+eY3v8l9991HZmYmhw8f5oUXXuB3v/sdGRkZZGVl8dvf/paCggKqqqpYsWLFGc85ZswYJEnib3/7G1dccQVWq7Vf53nyySeZOHEiU6ZM4Ze//CXNzc3cdtttPV7ve9/7HmvWrOH73/8+d999NwcPHuSBBx5g2bJlvRYe6y8Oh4Ply5dz7733IssyF1xwAcdPnOTdbe+R67Bw57euHbTUjo7YTHqCYRmfZCUdTkVXi3zHXunzu+FyueK/ZC6X64y3vrJlyxaWLl3KBx98wObNmwmFQlx22WV4vWKSryOFhYVs27aNSCTCZZddxvTp07nnnntIT09Hp9Oh0+l44YUX2LVrF9OmTePee+/lF7/4xRnPWVRUxIMPPsiKFSvIy8vj7rvv7td5HnnkER555BFmzpzJe++9x6uvvkp2ds8xM0VFRfz973/nX//6FzNnzuQ73/kOt99+O//1X/81oNemJ3784x/z3//936xevZopU6bwtau/wlub3mBsWdT7GiIhAvApZnWDiK7uM5LS01q8BjQ0NJCbm8uWLVvibn9HAoEAgcCpN9btdlNSUkJra2uX+Sq/38/Ro0cZO3bsiAgpEAwtTd4AoYhCJm6MRMCWAYbePxf9+Ry9+nEN5fUeLpmSy4yj61QRmntnSgc1ut1uXC5Xt9/RjiT0s9De3t4peKyyspI1a9bw5ptvJnK6OK2tanuX2NLy6axevbqT51VSUtLtfgLB6URkQAFdrPyHNPhVkm1GPRajXs11NTvUjQGxctYXEhKiq6++mmeeeQaAlpYW5s6dy2OPPcbVV1/NU089lZAhsixzzz33cP755/e43Lxy5UpaW1vjt+rq6oSuJUg9XFYDToteDc+QGNQ8sxiXTMnluxeOZ2ZJuoiu7icJV2iMrar8+c9/Jj8/n8rKSp555hn+53/+JyFDli5dyqeffsoLL7zQ4z5msxmn09npJhD0BZNBj1WvqPE9g1j+oyOdu3mI6Or+kJB/6vP5cDhU1/PNN9/kuuuuQ6fT8YUvfIHKysp+n+/uu+/mb3/7G1u3bh30pVyBII4SzWsbAm+oCyYRXd0fEvKIJkyYwCuvvEJ1dTWbNm3isssuA6C+vr5fXoqiKNx99928/PLL/POf/xwRCbOCkUc4IuMLhgmFYkI0NF20Gj0BXtp1jL99UgOm6ByRiK7uEwkJ0apVq1i+fDllZWXMmzcvHu7/5ptvMnv27D6fZ+nSpTz77LM8//zzOBwO6urqqKuro729PRGzBIJuCUUU2vxhArHo8F4K5ieKrChUNfmoaWkXQ7N+ktBPw/XXX88FF1xAbW0tM2fOjG+/5JJLuPbaa/t8ntjE9oUXXthp+/r16+NlKQSCgRLv3hGvzDg0HpHNpJ63PSijmKL9zcRkdZ9I+B3Jz8+PZ1bHmDt3br/OkUQhTIJRjKwoHZbupSGbI7J2yDfzS2lYQURX95GEhMjr9fLII4/w1ltvUV9fH082jHHkyJFBMU4gGAzU7h0yktq6Y8iESK+TMBt1BEIyPsyqEMWiq0XJ2DOSkBDdcccdbNmyhZtvvpmCggLRO0yQ1ESi3pAkRUVoCNI7YtiMegIhmXZZJ2pX94OEhOj111/ntddeixeqEgwuF154IbNmzWLNmjVamwIknz39RVYUVYh0DPnSvc1koNkXUit1mkXt6r6S0E9DRkZGj2kYguRguOoHjQRkWUFSIuqHfQhSOzpiNemxmvREFEVEV/eDhIToxz/+MatWreqUbzYiUBQIB4f/1o9J+SVLlrBlyxbWrl2LJElIkkR5eTm33357vBD85MmTWbt2bZfjrrnmGn7yk59QWFgYL6Xy/vvvM2vWLCwWC+eeey6vvPJKl75kZyrM3509FRUVA34rhgtFUUi3GbGbpFNDsyHkKzMK+M7C8ZyV7zy1hC+CGnsloZ+Hxx57jPLycvLy8igrK+tSLS9pGytGQvDuY8N/3S/+f2DoW1uctWvXcujQIaZNm8ZDDz0EqB5ocXExL774IllZWbz//vvcddddFBQUcMMNN8SPfeutt3A6nWzevBlQM5+vuuoqrrjiCp5//nkqKyu55557Ol2vt8L83dmTk5MzCC/K8CBJEiaDHqTB72XW0/XixKKrRSxRryQkRMnSuXM04nK5MJlM2Gy2TuERDz74YPzvsWPHsn37dv70pz91EqK0tDR+97vfxasv/vrXv0aSJJ5++mksFgtTp07l+PHj3HnnnfFjnnjiiV4L83dnz4hDHtoYom4R0dV9JqF35YEHHhhsO4YHvVH1TrS47gB58sknWbduHVVVVbS3txMMBrt0TJk+fXqnhgYHDx5kxowZneronB7r1ZfC/COZUEQmHI5gkiPoh2FoVtPSzvbykzitRr6cLaKr+0rCPw8tLS38+c9/pry8nB/+8IdkZmaye/du8vLy4gXLkw5J6vMQKZl44YUXWL58OY899hjz58/H4XDwi1/8gh07dnTaL5FC8H0pzD+SCYRl2tv9OGQFvWlol+5BbWtd1eQjy26CQjFZ3VcSEqJPPvmESy+9FJfLRUVFBXfeeSeZmZls3LiRqqqqeK0iQWKYTKZORe23bdvGggUL+N73vhff1lMX145MnjyZZ599lkAggNmsli/duXNnp316K8zfnT0jCVlWl+51EsMyLLPGysUGI2CKJoAHRXR1byT087Bs2TKWLFnC559/3sntv+KKK9i6deugGZeqlJWVsWPHDioqKmhsbGTixIl8+OGHbNq0iUOHDvHf//3fXQSlO771rW8hyzJ33XUX+/fvZ9OmTTz66KPAqUnVpUuX0tTUxDe/+U127txJeXk5mzZt4tZbb42Lz+n2nB5Jn8zEY4iGS4iiaR7+UATZGPWIIqJ2dW8kJEQ7d+7k29/+dpftRUVF1NXVDdioVGf58uXo9XqmTp1KTk4OixYt4rrrruPGG29k3rx5nDx5spN31BNOp5O//vWv7Nmzh1mzZvGf//mfrFq1CiD+A9JbYf7u7Kmqqhq6Jz/IRKIxRBLSsAmRJKkOkF+JRleDGJ71QkLvjNlsxu12d9l+6NChEbW0m6xMmjSJ7du3d9q2fv36Lv3kVq9eHf97w4YN3Z5rwYIFfPzxx/H/n3vuOYxGY7yzKqhtns/UAqo7e0YKEUXBqISHzSPS6SQsRj3twQi+YASbiK7uEwl5RF/96ld56KGHCIVCgOrmV1VVcf/993fpnS7QlmeeeYb33nuPo0eP8sorr8RjhKxWq9amDTmyoqDE54iGxyOCU22F2oMREV3dRxISosceewyPx0NOTg7t7e0sXLiQCRMm4HA4+MlPfjLYNgoGQF1dHTfddBNTpkzh3nvv5etf/zq//e1vtTZrWJBlBQlZ9YYkachXzGJYjWqaR1hWRMnYPpLQT4TL5WLz5s1s27aNjz/+GI/HwznnnMOll1462PYJBsh9993Hfffdp7UZmqDTSaSbJXSKHklnGLZVq+vnFJ+KsG4VsUR9od9CJMsyGzZsYOPGjVRUVCBJEmPHjiU/P79TX/NkQRRfS110koRJkkGX+LAskc9P5zQPEV3dF/rlqyqKwle/+lXuuOMOjh8/zvTp0zn77LOprKxkyZIl/SoTO9TE8t9GXGKuYHCRB1YwP1bFQK9PMCJb1K7uE/16dzZs2MDWrVt56623uOiiizo99s9//pNrrrmGZ555hltuuWVQjUwEvV5Peno69fX1ANhstqTz1gZCOCITCEdQAItBj0E/PPMfI4lgKAIBLwYlgk4Kg+Lv1/GyLNPQ0IDNZusx2LM7Kk962VnRTI7DzMJcMVndF/olRH/84x/5j//4jy4iBHDxxRezYsUKnnvuuaQQIiCepBkTo9GEPxQhGFYDCyVJLcil140eoR0MfMEwumAbRp2E3uJOKM9Mp9NRWlrarx+xQFimusmn1soujk1Wi+jqM9EvIfrkk0/4+c9/3uPjl19+ecKdXocCSZIoKCggNzc3HmowWlAUhYN1bRxu8NDYFsBlM/HVmYVCjDrw+q7D5B/dTEGOg/QL7gJ9/4dnJpMpHtjZV2LR1fEqjXAqulqUjO2Wfr0zTU1N5OXl9fh4Xl4ezc3NAzZqsNHr9YmP8ZOVQBuzTMeYUmLhD4dM1HkiHGkKMr3YpbVlSYPf3QCBNtKsOVjSulYXGCpicUTeYFitvCBqV/dKv4QoEomccays1+sJh8MDNkrQPYqi8MmxViZYWkk78BKEA5iBL5PPX5QvsLOiiWlFzlE1F5YowbCMzqdWmTSnD28dpTSz+h0JhGRCERmjiK7ulX4JkaIoLFmyJJ7JfTqBgEjsG0pOuANs2XcMz4kX+UKxGX1aFgTclIZrGev9iCPSuVQ3tVOaZdPaVM1x+0PYQicx6CRMzp69+KHAbNBh0EmEZQVfIILLZAdvo5iwPgP9EqLFixf3uk+yTFSPRg6eaKPA/Qm55hD6tEKYcys0HUH/2cucqztEWdkCcp3d/0ikGu72ELZgM2aDDuy5w3ptSZJIMxtobQ/hCYZxiejqXumXEJ2edCkYPhRF4UhdMxM9e8nKscDYhWqRt5zJkDGGIiopkg6DsUxrU5MCty+ANdyM2WbQZDhkNxsIRmTCEVnEEvWBYSzgKxgIDW0BDE2HMCkhXJklqgCBuhxcMg+aK6H2Yyj70oisQjnYTHaGCebb0RlNYEkf9ut//dwOaR4+EV3dGyIKboRwtNFLrucgLpsJfeHMzvEomePAmk446Ofz/XvYU92imZ3JgjV4EpfViCOzQJPYnU4LBsIj6hUhRCOE2oZGnIFaXFYT5E/r/KAkQe4UPP4wh/b+iw8rmkSOnbdBvR/m+aFuEaVAekUI0QggIisETxwCwJFbApZuYoVyp+KwGskMVOPz+Wj2ja4Azv7y+dEjnHD7CVmyNLn+sWYff951jH/sO9GhFIinX802UwkhRCMAvU7imhIfZ+U7sBVM7n6ntBz0tkxcZh1Ofw1VTamb7BsMy9RUV3Ck0UskbXiX7mOEIwrVTT5q3f6u0dWCLgghGglEQpjcVWTYTEjZPfQZkyTIHIfLaiTdX03lydRdKna7mzFFvOj1Oizp2rREigU1egMdoqtBDM96QAjRSMB9XC1nYbZD2hlqgncQomNNPmQ5NYcBvpPH1T+smZqtIKaZT+WbRWTllFcUECtn3SGEKMmRZYV3d+6isslLyFFy5hWg9FJsFhM22YvO30yjNzWHAYHmWgAkh3Ytsq1GvVonm2jOWWzCWghRtwghSnKafEF89RWccAfQZ4w5884GE7r0UhxmA+mB45z0BIfHyCQj1KJ6RAaXdp1q1ejqaPJrIAyWaLNFIUTdIgIak5z6Fi/2YD1pZj26jNLeD0gvZWx2OePzZUwFzqE3MAmJuNXeepbMQk3tSDMbaPOH8QYiYI6+F/5WTW1KVoRHlOS01FeiUyJY0pxgy+z9AFcxFqMek6cmNZeKg14ivhYA7FnFmppiNxuwmfTqHFHcI+raD1AgPKKkx19/BCtgzhnbtwhhZ6FaiTDggfbmvonXaKKtjikFTnwGF7ZMbT3Cr8woOBVh3RyN/fILIeoO4RElMbKsEG6pAcCR28v8UAy9ERwFnHD7eXP7hxw6kWJzEm21mA16MvJKMRu0LYbXOc0j5hG1pqan2gtCiJKYJm8AS3s9ep2EI6cP80MxXMX4QxHaGyqpOpligY2t0aV7Z5G2dpxOTIgiYQil2HvSB8TQLInxtTVjxY/VbETn6EeEcHopDosR58lajrW2D52ByYai0FR3FI/Hh17ORGspamgLsOVQA0a9xNWzitRUj6BXHZ7F0j4EgBCipKbU0EpJWQZha6465OorzkLsFgOWcDMtrW78oQgW4yir2d0dvpO0uj3UtIWR222aC5EkQXWT79Rrb3GpQhRwA9qFFiQjYmiWzLTVIiFhzOjnV8poxeTIwWLUYQ/WU9vav35eIxb3cfzhCB5TNhl2q9bWYI+mecRbP8VWzsSEdRc0FaKtW7dy1VVXUVhYiCRJvPLKK1qak3y0qfEwJBIh7CzEYTZiD9RT25Iiw7PW4/iDETzmPFzWfniQQ4TFqMdkUL9inkBYxBKdAU2FyOv1MnPmTJ588kktzUhKGtv87Px0H4cbPOBIIDDPWYjDYsARrKcmRTyicEs1gbBMmymPLHtyVKl0WlSvqM0fOlW+JSCE6HQ0nSO6/PLLufzyy7U0IWlpbKgjHGinXW9JrOZyVIjSWxtRjCkwAg+142+pRwEijkJspuSY/rRbDDR6grT5w6eESAzNupAc71YfCQQCnVoWud2j9w11N1QDYHLlJ9QqmbRcrBYL80r0SOOTwzsYUlqq8QXDtBvTcTqTp8mkw6wOEdv8YXCK6OqeGFE/latXr8blcsVvJSUlWps0ZLQ3qkJkzUrwOep0SM4CJCRw1wyiZUlKSyX+kIzbXEi2PXlaKjksBtLMejUoPjZZHfRBODUTkntiRAnRypUraW1tjd+qq6u1NmlIkGWFcKsqHq7cAYitU51bUtzH8Ycig2Fa8tJSSUmmlUvmn8ecsgytrYkzd2wmd31pPF8YlwVG66kCaWLCuhMjSojMZjNOp7PTbTTS5PVj8Tei10k4cwYiREX4gmG27f6EZz+oHDwDk42gFzwNSEg48sfhtGi/YhajS/tva1Qk25uH35gkZkQJUarQeOI4OiWMxWpFGkhzQGchZoMeQ3sjXl87bv8oLajfUqXep2Unf8SyEKJu0XSy2uPxcPjw4fj/R48eZc+ePWRmZlJa2o/cqlGG1d9AyGzAnF44sJ5cZgd6mwubqRV7sIHallKc+cnjLQwaLVV4g2E+DTkwVjVzTmnyDM1kWWHjR8dp84f45txSLNZ09QF/i5ZmJR2aekQffvghs2fPZvbs2QAsW7aM2bNns2rVKi3N0pwxxlamF7mYNGHiwE/mKMBhMWIP1lMzWvPOmito84c50J6RdE0DdDqJk54ALb6Q6pEKj6hbNPWILrzwQtEIsDva1JrLscnmAeEswmH+BHtbPbUtozCw0dcEviY8QYVWZxETHRatLeqC3WLAF4zQ5g+TK4SoW8QcUZIRCgWJtNWr/wxG8XdXkRphHThBg9uv5jyNJk6WA1CvyyaiM5HrTD4hik2eq0GN6epGvxvkUb6S2Q+EECUZRyqOsvNIAwebOnxoB4I9H7PJhF0XwBh2c8I9yryipnIiisIxSfUec53JE0MUI5b31uILqm2FdAZQZLGE3wEhRElGW0M1CiCn5Q9sojqG3gD2PHLsZs7L8GEzjaJyIOEgtFThC4Y5aSnFZtLjMCdfskBMiFrbQ+p7GpuwFsOzOEKIkoz2k8cAsGUPYuF3VxElmTbOy2wnK4mijgdMSyXIEVoVO36DizynpWvcThKQblOFyN0eDZ+IzxO1aGNQEiKEKImQZYVILKI6r481qvuCMypq7uODd85koPEQAJ60Ugx6XVIOy0D1iGwmPdZYIm58CV94RDGSz49NYZraPJgDzeh1Eq6cQfSIoqtvkbZ6ahpayHTa473ZRyxyJC5EZ00/jwnOEsJJ2mI73Wbi2wvHn9ogPKIuCI8oiWiqqwIUTGkuJMsgpq9YnGBxcrC2lTc/+IgjDckVa5MQzRUQ8quR1K4SDHrdyCmHa422ePKd1NaOJEIIURLRFiv9kTEEjQGdRTjMBhyBURLY2HAAACV7EuhG2Mc4lrbT3qx29RAIIUom8qVmsu0m0vOGIL3FWYTdYsAePMHx5hEuRB2GZdvbsvl/2yv4PMn7t+2uamb9tqPsOHISTHY1C19RoL1Ja9OSAiFESUSRrpmJuQ4KS8YN/smjgY3OYD2tvqAa0zJSaToaHZbZOOTPoNETTMrVso5EZIUWX4hmX1Bdwo95Rd5GbQ1LEoQQJQtBrxrgJklgH4SI6tOx52EwmMgwhrGGW6gcyY0X6z4GwJt+Fs3tEXSSRHGG9l07zsSpoMboEr4tKkQ+IUQghChpaD2hBuYp1kwwDkGagk4PriLSbSac/hoqm0aoEAW90KhWbDhqVFeiCtItST9RnWFTy/U2+YJqfqUtS31ATFgDQoiShkOHD/LxsVYO+uxDd5H0MbisRpyBWqqbfESSdLn7jJz4TE2PcORT7lO9oLHZSV6DCMiwGZEkCIRkfMGIGJqdhhCiJCHQrAYbpmUPYR3u9FLSzHpmOd3cMKcYXXJPq3RFUaBWHZaFcqdTHfXqxmTZtLSqTxj0uvjwrMkbPOURtTeL5FeEECUFoXAkXuA+I38QI6pPx1mIpDdSbFPI0bmTfoK3C80VqgehN3JUN4ZQRMFlNZIzQtJWMtOiwzNvUG0tpDeqIiRyzoQQJQMNjSfQRwIYDQbsmYNQg6gndHpwRT2uWHnVkcSxD9X7/Bm4nA6mFjqZVuQaMYKa57SQ77Jg1Os6r5x56rU1LAkY4XH+o4OWmiMA6F2Farb8UJI+BpqOUld5gF1NRcwbl5lU7Xd6xNcEJw+rX+Dic8mzWVh09hCsLg4hXxiXpXbziGHPB3cteE5A3lTtDEsChEeUBHhOHAXAlls29BfLUId+Tcc/5/O6Fg7Xe4b+moNB9Q71PmsC2DK1tWWwcOSp954T2tqRBAgh0hhFUQg2qcOk9IIhCGQ8HUcBmGxkW8ARqOPzE23JX663vRlqPwFAKZnL++WNNLQFejkoeQlHZHXF0h4VorY6dSI+hRFCpDFKwMPZ6WGK0q1kFYwd+gtKEmRNIDPNRHagmkZPkPpk/1JXbFOX7DPHcUzOZseRJv70YfWILHv7lz3HefLtcmpa2iEtFyQdhNohkNwpKkONECKN0bXVkGEzUVpSisEyTPEwWRMw6HRM0NcB8FlNEpcs9TbCiU/Vv8d+kR1H1dysKQUOTIaR9/HVSRKyolDf5lfnA9Oic0YpPjwbee/kaKM12jY7fRj7uGWUgU5PsdmPJdTC/to2QpEk9C4UBT7frN5nT6Q6nE51kw+9TuLcspE5T5TrUBcG4kPLWDpPrHNLiiKESGMqyg/Q7AsStg/hsv3pGMyQPgan1UBppIpgWOZAbRIODRoOqrFDOgPyuEt477AahTytyJlUbaX7Q87pQhRrGdU6yqpn9hMhRBri8bipPVbJwbo2ws4hjKjujtwpSEhMNx4n227CbkmySI6gFz5/U/27dB57myTqWv2YDDrmjs0687FJTKzdUZM3pHqhrg5lfOUk9EqHCSFEGlJXcQBQ0DvzsNjTh/fi2ZNAZ6DY5OWmaZbkytdSFDj0hipGadm05Z0X94bOn5CNfQSXuU0z6bGZ9MiKQqMnoGbhG0wQCYE3dQMbhRBpiLtGLe5ly5sw/Bc3WiBrPDpJQjrx2fBf/0wc2wkNh9RI8ClXYbdaWDA+izFZNmYUubS2bkBIkkS+S/WKalr8anXJWHODFB6eCSHSCEWW8Z9Qu5RmFk/Sxoj86ep93SeEgwF2VjTxYYXGFQNPlkP5P9W/x10EjnwkSWJ2aQbXzi5CN+IydbsyLtvOWfkOsqK5Z7iK1PvYwkUKMnJ93BFOY0MNUrANSW8gp1gDjwggc7yafOlvpfbzXbxXk41OkijKsFLg0qDQWEsVfLYRFIVQ7nTe9ZQwPxjBGm0KOVJyynpjerGL6cUdPLt4/l+lOiwdJc+zPwiPSCMaK/YBYMwcg8GkUa6XTgdFcwAo9n7GpFw7sqLw2ie1uP2h4bWl8TB88ieIhPHYx/Cn1il8fMzNa3trkz/ye6C4itVM/KAvZeOJhBBpRLBuPwDOEo2THQtmgMGM5G3k0uyTZNiMtPnDbNx1bHjqWisKVH0An/4ZORzkiJzDM62zqfeqntD5E7JGjSfUEUVROOkJqCVBdHo1tgug6YimdmmFECIt8LuZ4fQyvTidkokztbXFaIWSeQCYq7dx3ex8HBYDzb4Qz/+ris9qWofOI/GehD3PEf78LWpbfLzVWsCrkfMJKHrGZNm46QtjtBkiDgM7jjbxzPZKPjgSLRWbEU3vaTqqnVEaIoRICxo/R0LCnlM6/Mv23VF8ntqosL0ZZ90ObjyvhAKXhUBI5t3PGzuVlJUHobxsxNtEaN9rsPN30FLNyXb4pzKHT9MWYLeauezsPK6dXTSil+l7ozRTrSpZedKnvqaZUSFqPabmnqUYo/edTmIidXvRA+ScpbUpKgYTTLwMPnsZqj7AkTmOr59bwp7qZsIRBYNe/b2KyApPvn0Yh8VAmsmA2ajDbNCh1+mQgMJ0K1ML1Q61wbDMtvJGIhGFiKIQCQXQNR/FevJTLO5K8l1mxmSmQfZEMs+7GOMBL5fkO5hS4FQLh41y8p1qwX9/KEKt209ReqZaKM3bqPZsK9DYUx5mhBANM62NNRzc+xlZdgvjFkwhaWY/cs+CxqlwYh98+hL6WTcxZ0xOp11a20Px/lzxtjgdUCAuRLKisO/IMVz+42S0V5LuP4ZOUbuaykCjqZgxs6+E9BKMwA3njszcsUTR6STKsmwcqGujotFLUboVcqfC0a1Qv18IkWBoOXZgJxFZoclSzHizQ2tzOjP5CrX2j7sWPvp/6v85k+PLyRk2I3d+aRzN3iCBcAR/SCYQVmvryLJMnjmoCllLJaamCr7qq0GSQGeVkGwmdLY8dLlnYS6ZgyMzJyWXqTsyNieNA3VtHDrRxoLxWUi5U1Qhaq5Uo8pNSRTtPsQIIRpGIuEQnordAGRPOE9ja7pBb4TpN8CnL6lzFZ+9rC4t508HZxGS2Y5dr8eeJqtfFKUJQk3gqVOL/we98VPpgJKsNLUQW0aZKmj2vJQXn46My7ZjMuho8YWobfVTmJ4Jjny1UFrdXij9gtYmDhtCiIaRqn07kANeMDsomTBda3O6x2SDmd+EqvfVZfXWY+qtL0g6sOeoJU3Sy1QRG4pmkaMEk0HH+Bw7+2vdfF7voTDdCkXnwIG/w/HdUDxXjfVKAYQQDROKHKFx/7sAOCeej8GYxGUs9AYY+yUomAV1n6ilODwnIByNK5IkddnfmgnWDNXTcRao9/okfl5JyDlj0pmUZ2dMVnQYljsVyt9W24+f/Fz1JFMAIUTDRPW+HYQ9J1EMZsbPWKC1OX3D4oSyC9QbqD245IgqNmKINSjkOizkdpwq1BuhcBZUboeKdyFrYkp4RaP/GSYBSshP497NADgmX4jFmvydSbtFp1eX+oUIDQn+UEStw10yTy1e52k4VSZ3lCOEaBiQjrzD9BwDuXkFTD7nQq3NESQhB+va2PB+hVr9wGiFMVGv+cjbnRYBRitCiIaa+v1Q8xFGvY7x538Nk0nMoQi6otdBezDCh5XN1Lv9UHSuGuAY9MGB10Z9uyEhRENI47GDHN/+fygoUDrvVGKjQHAa43PsjM+1E5EV/vZJLZ4wMPVq0BnUGk2HNo1qMRJCNBQoCsf376D8H+uoavRQJRXD2Au1tkqQxEiSxGVT83BajbS2h9i4+xgtunQ460p1Tq7mI9j/6qmVy1FGUgjRk08+SVlZGRaLhXnz5vGvf/1La5MSprmugo9ff5qq919EjkQgexJ5X7ghJVY+BAPDYtRz/TnFOCwGTnqCPLejir2hIph8uRqjdWIf7Hxa7XobCWtt7qCi+fL9//3f/7Fs2TJ+/etfM2/ePNasWcOiRYs4ePAgubm5WpvXI3I4jC7Sro7hfY0cPnKYluoDhNoaAFAkHdYJX2T6giswGPQaWysYKbhsRm48r4S/762lpsWPNxiGcTPBmkHw078QaDmJ6dNXMVreQpdZBq5SsGWCNR2MNtCPzFVNSdG4/N28efM477zzeOKJJwCQZZmSkhK+//3vs2LFik77BgIBAoFT7ZHdbjclJSW0trbidDp7vkjTEbUOsqJwoNZNa3vMvVWi4271JZAUhbljM+Nj8YN1bpp9gVNjc0VBASQlghQJMndsJlI0bfVAnZtmXwhF0qHPP5sx51xCfn7xgF8fQWoiywr7at1MyjvV0faTqgb273ybvLZ9mCMedBLodZLaAAGYmOfAbjGBwcSJthBVLX5AhyLpQNKhdEixnpxvJ92q1sxuaPNzpNF36uIddOysfAeunnrIzbhRjTU7A263G5fL1et3VFOPKBgMsmvXLlauXBnfptPpuPTSS9m+fXuX/VevXs2DDz7Y/wuFg2pMBqB43cjt3ZdBlQD8HV6SoAcl0HVMHpOusAxGiw1sWThtaehtReSXnYXDceY3RyDoDZ1OYtppHUsCsp623HOpc84kLXACl/84acFGrKEWTBEPsqKAIkPIjxxoJ+zz9XB2wOMHWRUixetH9vQQImAPQKQHIVIGrw+bpkLU2NhIJBIhLy+v0/a8vDwOHDjQZf+VK1eybNmy+P8xj6hX0ktg5jdAksiZEMQViXpAkgSSFC1FKiHpJHBEKwJKEgXtYTJlGQl1n1P76TFbbRisafG5n2Hs0ypIUc4ry+S8skxkWSEQnkAwIhOOyIQiCgoKZosEShAiIRx+P6W+IJIcQVHkLituljQTGFXXJy0YodTb/Y+zxWECfQ9DvUGsDqD5HFF/MJvNmM0JFJo3pcUr4GVn9P0wZ5JV6RAIQPWWrCY9Vrqbe1STjO1pYO9jQ9y06E1LNF3Kyc7ORq/Xc+JE584FJ06cID8/XyOrBALBcKOpEJlMJubMmcNbb70V3ybLMm+99Rbz58/X0DKBQDCcaD40W7ZsGYsXL+bcc89l7ty5rFmzBq/Xy6233qq1aQKBYJjQXIhuvPFGGhoaWLVqFXV1dcyaNYs33nijywS2QCAYvWgeRzQQ+hqjIBAItKGv31GRdyAQCDRHCJFAINAczeeIBkJsVOl2uzW2RCAQdEfsu9nbDNCIFqK2tjaAvkVXCwQCzWhra8PlcvX4+IierJZlmZqaGhwORzT9IrWIpbhUV1eLyfo+Il6zxEj0dVMUhba2NgoLC9GdoRTOiPaIdDodxcUiw93pdIovVT8Rr1liJPK6nckTiiEmqwUCgeYIIRIIBJojhGgEYzabeeCBBxKrSJCiiNcsMYb6dRvRk9UCgWB0IDwigUCgOUKIBAKB5gghEggEmiOESCAQaI4QoiSmv40nX3zxRc466ywsFgvTp0/n73//+zBZmlz053XbsGFDvDFC7GaxWIbRWu3ZunUrV111FYWFhUiSxCuvvNLrMe+88w7nnHMOZrOZCRMmsGHDhgHZIIQoSYk1nnzggQfYvXs3M2fOZNGiRdTX13e7//vvv883v/lNbr/9dj766COuueYarrnmGj799NNhtlxb+vu6gRotXFtbG79VVlYOo8Xa4/V6mTlzJk8++WSf9j969ChXXnklF110EXv27OGee+7hjjvuYNOmTYkboQiSkrlz5ypLly6N/x+JRJTCwkJl9erV3e5/ww03KFdeeWWnbfPmzVO+/e1vD6mdyUZ/X7f169crLpdrmKxLfgDl5ZdfPuM+9913n3L22Wd32nbjjTcqixYtSvi6wiNKQmKNJy+99NL4tjM1ngTYvn17p/0BFi1a1OP+o5FEXjcAj8fDmDFjKCkp4eqrr+azzz4bDnNHLEPxWRNClIScqfFkXV1dt8fU1dX1a//RSCKv2+TJk1m3bh1/+ctfePbZZ5FlmQULFnDs2LHhMHlE0tNnze12097entA5R3T2vUAwUObPn9+pddWCBQuYMmUKv/nNb/jxj3+soWWphfCIkpBEGk/m5+enfKPKwWjYaTQamT17NocPHx4KE0cFPX3WnE4nVqs1oXMKIUpCEmk8OX/+/E77A2zevDmlGlUORsPOSCTC3r17KSgoGCozRzxD8llLeJpbMKS88MILitlsVjZs2KDs27dPueuuu5T09HSlrq5OURRFufnmm5UVK1bE99+2bZtiMBiURx99VNm/f7/ywAMPKEajUdm7d69WT0ET+vu6Pfjgg8qmTZuU8vJyZdeuXco3vvENxWKxKJ999plWT2HYaWtrUz766CPlo48+UgDl8ccfVz766COlsrJSURRFWbFihXLzzTfH9z9y5Ihis9mUH/7wh8r+/fuVJ598UtHr9cobb7yRsA1CiJKYX/3qV0ppaaliMpmUuXPnKh988EH8sYULFyqLFy/utP+f/vQnZdKkSYrJZFLOPvts5bXXXhtmi5OD/rxu99xzT3zfvLw85YorrlB2796tgdXa8fbbbytAl1vsdVq8eLGycOHCLsfMmjVLMZlMyrhx45T169cPyAZRBkQgEGiOmCMSCASaI4RIIBBojhAigUCgOUKIBAKB5gghEggEmiOESCAQaI4QIoFAoDlCiAQCgeYIIRKMKDZs2EB6enr8/x/96EfMmjUr/v+SJUu45pprht0uwcAQQiToliVLliBJEt/5zne6PLZ06VIkSWLJkiWd9h9sASgrK2PNmjWdtt14440cOnSox2PWrl3bqX7yhRdeyD333DOodgkGHyFEgh4pKSnhhRde6FTsyu/38/zzz1NaWqqJTVarldzc3B4fd7lcnTwmwchACJGgR8455xxKSkrYuHFjfNvGjRspLS1l9uzZAzp3d57KNddcE/eyLrzwQiorK7n33nvj3TWg69DsdDp6ZkuWLGHLli2sXbs2fo6jR48yYcIEHn300U7H7dmzB0mSRB0ijRBCJDgjt912G+vXr4//v27dOm699dYhv+7GjRspLi7moYceinfX6C9r165l/vz53HnnnfFzlJaWdnlOAOvXr+dLX/oSEyZMGKynIOgHQogEZ+Smm27ivffeo7KyksrKSrZt28ZNN9005NfNzMxEr9fjcDjIz89PqNKky+XCZDJhs9ni59Dr9SxZsoSDBw/G+52FQiGef/55brvttsF+GoI+ImpWC85ITk4OV155JRs2bEBRFK688kqys7O1NmtAFBYWcuWVV7Ju3Trmzp3LX//6VwKBAF//+te1Ni1lER6RoFduu+02NmzYwB/+8IdB8xp0Oh2nl8IKhUKDcu6+cMcdd8Qn4tevX8+NN96IzWYbtusLOiOESNAr//Zv/0YwGCQUCrFo0aJBOWdOTk6neZ9IJNKlK63JZCISiQzoOj2d44orriAtLY2nnnqKN954QwzLNEYMzQS9otfr2b9/f/zvnmhtbWXPnj2dtmVlZVFSUtJl34svvphly5bx2muvMX78eB5//HFaWlo67VNWVsbWrVv5xje+gdlsTmhIWFZWxo4dO6ioqMBut5OZmYlOp4vPFa1cuZKJEyemVJOBZER4RII+4XQ6cTqdZ9znnXfeYfbs2Z1uDz74YLf73nbbbSxevJhbbrmFhQsXMm7cOC666KJO+zz00ENUVFQwfvx4cnJyErJ7+fLl6PV6pk6dSk5ODlVVVfHHbr/9doLB4LCsAgrOjKhZLUhZ3n33XS655BKqq6u7dC4VDC9CiAQpRyAQoKGhgcWLF5Ofn89zzz2ntUkpjxiaCVKOP/7xj4wZM4aWlhZ+/vOfa22OAOERCQSCJEB4RAKBQHOEEAkEAs0RQiQQCDRHCJFAINAcIUQCgUBzhBAJBALNEUIkEAg0RwiRQCDQnP8fltDyz7pMw0sAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:52:23.400992Z", + "iopub.status.busy": "2024-03-22T22:52:23.400664Z", + "iopub.status.idle": "2024-03-22T22:52:23.570840Z", + "shell.execute_reply": "2024-03-22T22:52:23.569887Z" + }, + "papermill": { + "duration": 0.191211, + "end_time": "2024-03-22T22:52:23.573134", + "exception": false, + "start_time": "2024-03-22T22:52:23.381923", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuJ0lEQVR4nO3de1hTZ54H8C/hEgy3qshFpMZLK6ACCsViddUOF9fW1bXTobYVjA47o9KxprSWTgtL7RodldJpGVkd0dqbrFXbbqUIRekUxapQHbRAC16wchG8EIEaQvLuHy6nxiSQIOHkhN/nefLgOXnPyfvyhq/nvDl5jx1jjIEQQgRGxHcFCCGkLyi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAc+K7AQNNqtaivr4ebmxvs7Oz4rg4h5B6MMdy6dQsjR46ESNTD8RWzAu+99x4bPXo0E4vFLCIign333XdGy86aNYsB0HvMmzfPpNe6fPmywe3pQQ96WNfj8uXLPf4t837klZubC7lcjuzsbEybNg2ZmZmIjY1FdXU1vLy89Mrv378fnZ2d3PK1a9cQEhKCp59+2qTXc3NzAwBcvnwZ7u7u/dMIK6RWq1FQUICYmBg4OjryXR3SDwZLnyqVSvj7+3N/q8bwHl4ZGRlITEyETCYDAGRnZ+PgwYPIycnBq6++qld+2LBhOst79uyBRCIxOby6TxXd3d1tPrwkEgnc3d1t+o0+mAy2Pu1tWIfX8Ors7ERZWRlSUlK4dSKRCFFRUSgtLTVpHzt27MAzzzwDFxcXg8+rVCqoVCpuWalUArjzRlCr1fdRe+vW3TZbbuNgM1j61NT28RpeLS0t0Gg08Pb21lnv7e2NqqqqXrc/ceIEzp49ix07dhgto1AokJ6erre+oKAAEonE/EoLTGFhId9VIP3M1vu0o6PDpHK8nzbejx07dmDy5MmIiIgwWiYlJQVyuZxb7j6fjomJsfnTxsLCQkRHRw+KU4zBYLD0affZUW94DS9PT0/Y29ujqalJZ31TUxN8fHx63La9vR179uzBm2++2WM5sVgMsVist97R0dGm3wDdBks7BxNb71NT28brRapOTk4ICwtDUVERt06r1aKoqAiRkZE9brt3716oVCo8//zzlq4mIcQK8X7aKJfLkZCQgPDwcERERCAzMxPt7e3cp4/x8fHw8/ODQqHQ2W7Hjh1YuHAhhg8fzke1CSE84z284uLi0NzcjNTUVDQ2NiI0NBT5+fncIH5dXZ3eVbbV1dUoKSlBQUEBH1UmxOI6Ojr0PrRq+0WFYxW1GOp5Cq5Dfh0KCQgIGBQfPt3LjrHBdQMOpVIJDw8PtLa22vyAfV5eHubNm2fT4yO2qry8HGFhYSaVLSsrw9SpUy1co4Fj6t8o70dehBB9AQEBKCsr01lX3XAT8r0VyHh6Mib4PqBTdjCi8CLECkkkEr2jKdGlaxB/+wsCJ4UgdDSN9dKUOIQQQaLwIoQIEoUXIUSQKLwIIYJEA/Y24t7rgoxdEwQM3uuCiG2h8LIRVVVVBq8L+ouBsrZ2XRAZnCi8bMS91wUZuyaouywhQkfhZSPuvS6Irgkito4G7AkhgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESTewysrKwtSqRTOzs6YNm0aTpw40WP5mzdvYtWqVfD19YVYLMbDDz+MvLy8AaotIcRa8PrdxtzcXMjlcmRnZ2PatGnIzMxEbGwsqqur4eXlpVe+s7MT0dHR8PLywqeffgo/Pz9cunQJDzzwwMBXnhDCK17DKyMjA4mJidwNZrOzs3Hw4EHk5OTg1Vdf1Sufk5OD69ev49ixY9ztvKRS6UBWmRCLudDSjnZVl9Hna5vbuZ8ODob/dF3EDhjj6WKR+lkb3sKrs7MTZWVlSElJ4daJRCJERUWhtLTU4DZffPEFIiMjsWrVKnz++ecYMWIEnn32Waxduxb29vYGt1GpVFCpVNyyUqkEcOe+hmq1uh9bZF26urq4n7bcTltx8Vo7ojOPmlT2pU8reny+8MXHIB0u3AAz9f3KW3i1tLRAo9Fwd8bu5u3trXen4G7nz5/H4cOH8dxzzyEvLw81NTVYuXIl1Go10tLSDG6jUCiQnp6ut76goMCmZxO93AYADjh+/DiunOW7NqQ33f21ZLwG3kMM3wdarQWuq4BhYsDRwGh10y92+KDGHoeKvoG/q0Wra1EdHR0mlRPUfF5arRZeXl7Ytm0b7O3tERYWhitXrmDTpk1GwyslJQVyuZxbViqV8Pf3R0xMjODvmH3xWjvaVRqDz6kaW4GKSniNn4zRPh5G9+Eithf0/9K24ly9EpsrjuOp6McwcaTh96VarUZhYSGio6MN3gX9XL0SH9Qcx4wZM4zuQwi6z456w1t4eXp6wt7eHk1NTTrrm5qa4OPjY3AbX19fODo66pwiBgYGorGxEZ2dnXByctLbRiwWQywW6613dHQ0+AYQigstpp1mvHKgstcyR5JnD5pxEmvVPYbl4ODQ6/vS2HvXnH1YM1Przlt4OTk5ISwsDEVFRVi4cCGAO0dWRUVFSEpKMrjNY489ho8//hharRYi0Z3j5h9//BG+vr4Gg8uWdQ/sZsaFYryX/jlC+y8qfFlciidnR8JliH54A0DN1Ta8mHu6x0FiQqwVr6eNcrkcCQkJCA8PR0REBDIzM9He3s59+hgfHw8/Pz8oFAoAwIoVK/Dee+9h9erVeOGFF/DTTz9h/fr1+NOf/sRnM3g13ssVk/z0TwvVajUaRwBTRw8V9P/ChBjDa3jFxcWhubkZqampaGxsRGhoKPLz87lB/Lq6Ou4ICwD8/f1x6NAhrFmzBsHBwfDz88Pq1auxdu1avppACOEJ7wP2SUlJRk8Ti4uL9dZFRkbi+PHjFq4VIcTa8f71IEII6QsKL0KIIFF4EUIEicKLECJIFF6EEEHi/dNGQgig0tyGyPkKLiirIXI2/MXErq4u1HfVo/J6pcFZJS4o2yByvgKV5jYA418JsxUUXoRYgfr2S3AZ8y5e63kuTgDA3/L/ZvQ5lzFAfXsowuBttIytoPAixAqMdBmN9gsv4J24UIwz8HUv4M6R19GSo3hsxmMGj7xqr7Zhde5pjJwz2tLVtQoUXoRYAbG9M7S3/TDGfQKChhs+5VOr1bjgcAGBwwINfuVLe7sV2tvNENs7W7q6VoEG7AkhgkRHXgLV2wBvb4O7wOAb4CW2hcJLoEwd4O1pcBcYXAO8xLZQeAlUbwO8vQ3uAoNvgJfYFgovgeptgLe3wV1g8A3wEttCA/aEEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBZRXhlZWVBKpXC2dkZ06ZNw4kTxq+83LVrF+zs7HQezs70UT8hgw3v4ZWbmwu5XI60tDSUl5cjJCQEsbGxuHr1qtFt3N3d0dDQwD0uXbo0gDUmhFgD3sMrIyMDiYmJkMlkCAoKQnZ2NiQSCXJycoxuY2dnBx8fH+7RfZ9HQsjgwesV9p2dnSgrK0NKSgq3TiQSISoqCqWlpUa3a2trw+jRo6HVajF16lSsX78eEydONFhWpVJBpVJxy0qlEsCdK9DVanU/tWTgdXV1cT8NtaN7XU9t7G0fZOCY0he99amt9Kepdec1vFpaWqDRaPSOnLy9vVFVVWVwmwkTJiAnJwfBwcFobW3F5s2bMX36dJw7dw6jRo3SK69QKJCenq63vqCgABKJpH8awoPLbQDggJKSElwyPHcdAKCwsPC+90Esz5y+MNanttKfHR0dJpUT3HcbIyMjERkZyS1Pnz4dgYGB+O///m+sW7dOr3xKSgrkcjm3rFQq4e/vj5iYGLi7uw9InS3hXL0SmyuOY8aMGZg4Ur8darUahYWFiI6ONvrdxt72QQaOKX3RW5/aSn92nx31htfw8vT0hL29PZqamnTWNzU1wcfHx6R9ODo6YsqUKaipqTH4vFgshlgsNridsT9qIeieKcLBwaHHdvTUTlP3QSzPnL4w1qe20p+m1p3XAXsnJyeEhYWhqKiIW6fValFUVKRzdNUTjUaDiooK+Pr6WqqahBArxPtpo1wuR0JCAsLDwxEREYHMzEy0t7dDJpMBAOLj4+Hn5weFQgEAePPNN/Hoo49i/PjxuHnzJjZt2oRLly7h97//PZ/NIIQMMN7DKy4uDs3NzUhNTUVjYyNCQ0ORn5/PDeLX1dVBJPr1APHGjRtITExEY2Mjhg4dirCwMBw7dgxBQUF8NYEQwgPewwsAkpKSkJSUZPC54uJineW3334bb7/99gDUihBizXi/SJUQQvqCwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIFF6EEEEyO7zOnz9viXoQQohZzA6v8ePHY86cOfjwww9x+/ZtS9SJEEJ6ZXZ4lZeXIzg4GHK5HD4+PvjDH/7Q430WCSHEEswOr9DQULzzzjuor69HTk4OGhoaMGPGDEyaNAkZGRlobm62RD0JIURHnwfsHRwcsGjRIuzduxcbN25ETU0NkpOT4e/vj/j4eDQ0NPRnPQkhREefw+vUqVNYuXIlfH19kZGRgeTkZNTW1qKwsBD19fVYsGBBf9aTEEJ0mD2TakZGBnbu3Inq6mrMmzcPu3fvxrx587ipmseMGYNdu3ZBKpX2d10JIYRjdnht3boVy5Ytw9KlS43escfLyws7duy478oRQogxZp82FhYWYu3atXrBxRhDXV0dgDu3NEtISDB5n1lZWZBKpXB2dsa0adNM/vRyz549sLOzw8KFC01+LUKIbTA7vMaNG4eWlha99devX8eYMWPMrkBubi7kcjnS0tJQXl6OkJAQxMbG4urVqz1ud/HiRSQnJ2PmzJlmvyYhRPjMDi/GmMH1bW1tcHZ2NrsCGRkZSExMhEwmQ1BQELKzsyGRSJCTk2N0G41Gg+eeew7p6ekYO3as2a9JCBE+k8e85HI5AMDOzg6pqamQSCTccxqNBt999x1CQ0PNevHOzk6UlZUhJSWFWycSiRAVFYXS0lKj27355pvw8vLC8uXL8e2335r1moQQ22ByeH3//fcA7hx5VVRUwMnJiXvOyckJISEhSE5ONuvFW1paoNFouBvMdvP29kZVVZXBbUpKSrBjxw6cPn3apNdQqVRQqVTcslKpBACo1Wqo1Wqz6mtNurq6uJ+G2tG9rqc29rYPMnBM6Yve+tRW+tPUupscXkeOHAEAyGQyvPPOO3B3d+9bze7DrVu3sGTJEmzfvh2enp4mbaNQKJCenq63vqCgQOfoUWgutwGAA0pKSnDJ1Xi5wsLC+94HsTxz+sJYn9pKf3Z0dJhUzuxLJXbu3Gl2ZYzx9PSEvb09mpqadNY3NTXBx8dHr3xtbS0uXryI+fPnc+u0Wi2AO1f8V1dXY9y4cTrbpKSkcKe8wJ0jL39/f8TExPASwP3lXL0SmyuOY8aMGZg4Ur8darUahYWFiI6OhqOjY5/2QQaOKX3RW5/aSn92nx31xqTwWrRoEXbt2gV3d3csWrSox7L79+836YWBO6ebYWFhKCoq4i530Gq1KCoqQlJSkl75gIAAVFRU6Kx7/fXXcevWLbzzzjvw9/fX20YsFkMsFuutd3R0NPpHLQQODg7cz57a0VM7Td0HsTxz+sJYn9pKf5pad5PCy8PDA3Z2dty/+5NcLkdCQgLCw8MRERGBzMxMtLe3QyaTAQDi4+Ph5+cHhUIBZ2dnTJo0SWf7Bx54AAD01hNCbJtJ4XX3qWJ/njYCQFxcHJqbm5GamorGxkaEhoYiPz+fG8Svq6vjvnpECCHdzB7zsoSkpCSDp4kAUFxc3OO2u3bt6v8KEUKsnknhNWXKFO60sTfl5eX3VSFCCDGFSeFF3x0khFgbk8IrLS3N0vUghBCz0Eg4IUSQTDryGjZsGH788Ud4enpi6NChPY5/Xb9+vd8qRwghxpgUXm+//Tbc3Ny4f5s6eE8IIZZiUnjdPbHg0qVLLVUXQggxmdljXvb29gYnCrx27Rrs7e37pVKEENKbfpuMUKVS6UyTQwghlmTyFfZ//etfAdyZjPDvf/87XF1/nXNDo9HgH//4BwICAvq/hoQQYoDJ4fX2228DuHPklZ2drXOK6OTkBKlUiuzs7P6vISGEGGByeF24cAEAMGfOHOzfvx9Dhw61WKUIIaQ3Zn8xu3tGVUII4ZPZ4bVs2bIen+/prj+EENJfzA6vGzdu6Cyr1WqcPXsWN2/exOOPP95vFSOEkJ6YHV4HDhzQW6fVarFixQq9+eMJIcRS+uWL2SKRCHK5nPtEkhBCLK3fZpWora3l7htHCCGWZvZp4923EQPuXPfV0NCAgwcP6nwHkhBCLMns8Oq+c3Y3kUiEESNGYMuWLb1+EkkIIf3FKq7zysrKwqZNm9DY2IiQkBC8++67iIiIMFh2//79WL9+PWpqaqBWq/HQQw/hpZdewpIlS/q9XoQQ68X7TKq5ubmQy+VIS0tDeXk5QkJCEBsba3DmCuDOxIh//vOfUVpain/+85+QyWSQyWQ4dOjQANecEMIn3sMrIyMDiYmJkMlkCAoKQnZ2NiQSidGLXWfPno1///d/R2BgIMaNG4fVq1cjODgYJSUlA1xzQgifeL1vY2dnJ8rKypCSksKtE4lEiIqKQmlpaa/bM8Zw+PBhVFdXY+PGjQbLqFQqqFQqblmpVAK4c3GtWq2+zxbwp/uT3a6uLoPt6F7XUxt72wcZOKb0RW99aiv9aWrdeQ2vlpYWaDQa7u7Y3by9vVFVVWV0u9bWVvj5+UGlUsHe3h5/+9vfEB0dbbCsQqFAenq63vqCggJIJJL7awCPLrcBgANKSkpwydV4ucLCwvveB7E8c/rCWJ/aSn92dHSYVK7fwuvnn3/Gm2++iW3btvXXLo1yc3PD6dOn0dbWhqKiIsjlcowdOxazZ8/WK5uSkqJzeYdSqYS/vz9iYmLg7u5u8bpayrl6JTZXHMeMGTMwcaR+O9RqNQoLCxEdHQ1HR8c+7YMMHFP6orc+tZX+7D476k2/hde1a9ewY8cOs8LL09MT9vb2aGpq0lnf1NQEHx8fo9uJRCKMHz8eABAaGorKykooFAqD4SUWiyEWi/XWOzo6Gv2jFgIHBwfuZ0/t6Kmdpu6DWJ45fWGsT22lP02tO68D9k5OTggLC0NRURG3TqvVoqioCJGRkSbvR6vV6oxrEUJsH69jXsCdK/YTEhIQHh6OiIgIZGZmor29HTKZDAAQHx8PPz8/KBQKAHfGsMLDwzFu3DioVCrk5eXhgw8+wNatW/lsBiFkgPEeXnFxcWhubkZqaioaGxsRGhqK/Px8bhC/rq4OItGvB4jt7e1YuXIlfv75ZwwZMgQBAQH48MMPERcXx1cTCCE8MDm8Fi1a1OPzN2/e7HMlkpKSkJSUZPC54uJineW33noLb731Vp9fixBiG0wOLw8Pj16fj4+Pv+8KEUKIKUwOr507d1qyHoQQYhbevx5ECCF9YfKRl6nT3dANOAgx3y9qDQDg7JVWo2Xaf1HhVDPgc+kGXIboX7tYc7XNYvWzRiaH165duzB69GhMmTIFjDFL1omQQaf2/4Pn1f0VvZR0wAc1J3ss4SLm/SKCAWFyK1esWIFPPvkEFy5cgEwmw/PPP49hw4ZZsm6EDBoxE+98o2SclyuGONobLFPd0IqXPq3Alt9OxgRfwx+guYgdMMbTxWL1tCYmh1dWVhYyMjKwf/9+5OTkICUlBU888QSWL1+OmJgY2NnZWbKehNi0YS5OeCbiwR7LdM8aMW6ECyb59fzp/2Bg1oC9WCzG4sWLUVhYiB9++AETJ07EypUrIZVK0dY2uM63CSH86vOnjSKRCHZ2dmCMQaPR9GedCCGkV2aFl0qlwieffILo6Gg8/PDDqKiowHvvvYe6ujq4ugp4AiFCiOCYPOa1cuVK7NmzB/7+/li2bBk++eQTeHp6WrJuhBBilMnhlZ2djQcffBBjx47FN998g2+++cZguf379/db5QghxBiTwys+Pp4+USSEWA2zLlIlhBBrQd9tJIQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEySrCKysrC1KpFM7Ozpg2bRpOnDhhtOz27dsxc+ZMDB06FEOHDkVUVFSP5Qkhton38MrNzYVcLkdaWhrKy8sREhKC2NhYXL161WD54uJiLF68GEeOHEFpaSn8/f0RExODK1euDHDNCSF84j28MjIykJiYCJlMhqCgIGRnZ0MikRidTvqjjz7CypUrERoaioCAAPz973/n7rJNCBk8eJ0vtrOzE2VlZUhJSeHWiUQiREVFobS01KR9dHR0QK1WG53VVaVSQaVScctKpRIAoFaroVar76P2/OqemK6rq8tgO7rX9dTG3vZBrMtg6S9T28ZreLW0tECj0XB3x+7m7e2Nqqoqk/axdu1ajBw5ElFRUQafVygUSE9P11tfUFAAiURifqWtxOU2AHBASUkJLvUwG1FhYeF974NYh+7+On78OK6c5bs2ltPR0WFSOUHP1L9hwwbs2bMHxcXFcHZ2NlgmJSUFcrmcW1Yqldw4mbu7+0BVtd+dq1dic8VxzJgxAxNH6rdDrVajsLAQ0dHRcHR07NM+iHU5U3cdqDiFRx99FCEP2u79I7rPjnrDa3h5enrC3t4eTU1NOuubmprg4+PT47abN2/Ghg0b8PXXXyM4ONhoObFYDLFY/zZRjo6ORv+ohcDBwYH72VM7emqnqfsg1mGw9JepbeN1wN7JyQlhYWE6g+3dg++RkZFGt/vLX/6CdevWIT8/H+Hh4QNRVUKIleH9tFEulyMhIQHh4eGIiIhAZmYm2tvbIZPJANyZR8zPzw8KhQIAsHHjRqSmpuLjjz+GVCpFY2MjAMDV1ZWmoiZkEOE9vOLi4tDc3IzU1FQ0NjYiNDQU+fn53CB+XV0dRKJfDxC3bt2Kzs5O/Pa3v9XZT1paGv7zP/9zIKtOCOER7+EFAElJSUhKSjL4XHFxsc7yxYsXLV8hQojV4/0iVUII6QurOPIi5vtFfedemWevtBp8vv0XFU41Az6XbsBliP6nrQBQc5VuFEyEi8JLoGr/P3he3V/RQykHfFBzstd9uYjpbUCEh961AhUz8c51cOO8XDHE0V7v+eqGVrz0aQW2/HYyJvh6GN2Pi9gBYzxdLFZPQiyFwkughrk44ZmIB40+3/09uHEjXDDJz3h4ESJUNGBPCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQeA+vrKwsSKVSODs7Y9q0aThx4oTRsufOncNTTz0FqVQKOzs7ZGZmDlxFCSFWhdfwys3NhVwuR1paGsrLyxESEoLY2FhcvXrVYPmOjg6MHTsWGzZs6PWO2oQQ28ZreGVkZCAxMREymQxBQUHIzs6GRCJBTk6OwfKPPPIINm3ahGeeeQZiseGbShBCBgfepoHu7OxEWVkZUlJSuHUikQhRUVEoLS3tt9dRqVRQqVTcslKpBACo1Wqo1ep+ex1r0z0NdFdXl023czAZLH1qatt4C6+WlhZoNBruztjdvL29UVVV1W+vo1AokJ6erre+oKAAEomk317H2lxuAwAHHD9+HFfO8l0b0h8GS592dHSYVM7mb8CRkpICuVzOLSuVSvj7+yMmJgbu7u481syyztRdBypO4dFHH0XIg8P4rg7pB4OlT7vPjnrDW3h5enrC3t4eTU1NOuubmpr6dTBeLBYbHB9zdHSEo6Njv72OtXFwcOB+2nI7B5PB0qemto23AXsnJyeEhYWhqKiIW6fValFUVITIyEi+qkUIEQheTxvlcjkSEhIQHh6OiIgIZGZmor29HTKZDAAQHx8PPz8/KBQKAHcG+X/44Qfu31euXMHp06fh6uqK8ePH89YOQsjA4zW84uLi0NzcjNTUVDQ2NiI0NBT5+fncIH5dXR1Eol8PDuvr6zFlyhRuefPmzdi8eTNmzZqF4uLiga4+IYRHvA/YJyUlISkpyeBz9waSVCoFY2wAakUIsXa8fz2IEEL6gsKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQbKK8MrKyoJUKoWzszOmTZuGEydO9Fh+7969CAgIgLOzMyZPnoy8vLwBqikhxFrwHl65ubmQy+VIS0tDeXk5QkJCEBsbi6tXrxosf+zYMSxevBjLly/H999/j4ULF2LhwoU4e9aG739OCNHDe3hlZGQgMTERMpkMQUFByM7OhkQiQU5OjsHy77zzDubOnYuXX34ZgYGBWLduHaZOnYr33ntvgGtOCOETr+HV2dmJsrIyREVFcetEIhGioqJQWlpqcJvS0lKd8gAQGxtrtDwhxDbxet/GlpYWaDQa7iaz3by9vVFVVWVwm8bGRoPlGxsbDZZXqVRQqVTcslKpBACo1Wqo1er7qb5V6ejoQHV1Nbf8Y0MrVI01OHvaCZ1NHjplJ0yYAIlEMtBVJGa4tz8B431qa/1p6t8l7zedtTSFQoH09HS99QUFBTbV4bW1tXjppZf01i95X7/sli1bMG7cuAGoFekrY/0J6PeprfVnR0eHSeV4DS9PT0/Y29ujqalJZ31TUxN8fHwMbuPj42NW+ZSUFMjlcm5ZqVTC398fMTExcHd3v88WWI+Ojg7MmDGDW277RYVD355E7MxH4DpErFPW1v6ntkX39idgvE9trT+7z456w2t4OTk5ISwsDEVFRVi4cCEAQKvVoqioCElJSQa3iYyMRFFREV588UVuXWFhISIjIw2WF4vFEIvFeusdHR3h6Oh4322wFh4eHoiIiOCW1Wo1bt28jpnTH7Wpdg4W9/YnMHj61NS28X7aKJfLkZCQgPDwcERERCAzMxPt7e2QyWQAgPj4ePj5+UGhUAAAVq9ejVmzZmHLli144oknsGfPHpw6dQrbtm3jsxmEkAHGe3jFxcWhubkZqampaGxsRGhoKPLz87lB+bq6OohEv34oOn36dHz88cd4/fXX8dprr+Ghhx7CZ599hkmTJvHVBEIID+wYY4zvSgwkpVIJDw8PtLa22tSY173UajXy8vIwb948mz7FGEwGS5+a+jfK+0WqhBDSFxRehBBBovAihAgShRchRJB4/7RxoHV/PmHqhXBCpVar0dHRAaVSadODu4PJYOnT7r/N3j5LHHThdevWLQCAv78/zzUhhPTk1q1b8PDwMPr8oLtUQqvVor6+Hm5ubrCzs+O7OhbT/TWoy5cv2/QlIYPJYOlTxhhu3bqFkSNH6lzjea9Bd+QlEokwatQovqsxYNzd3W36jT4YDYY+7emIqxsN2BNCBInCixAiSBReNkosFiMtLc3gjBpEmKhPdQ26AXtCiG2gIy9CiCBReBFCBInCixAiSBRePLl48SLs7Oxw+vTpAdnP0aNHMXnyZDg6OnJTbhMiZBReVqK4uBh2dna4efOmRfYvl8sRGhqKCxcuYNeuXRZ5DaJv9uzZOvdb4Ju11ed+UHj1QWdnJ99VMFttbS0ef/xxjBo1Cg888ECf9jGQ7WaMoaura8Bez5oJ8f02IBjp1axZs9iqVavY6tWr2fDhw9ns2bNZRUUFmzt3LnNxcWFeXl7s+eefZ83Nzdw2X331FXvssceYh4cHGzZsGHviiSdYTU0N9/yFCxcYAPb9999z/777kZCQYNZ+PvnkExYZGcnEYjGbOHEiKy4u1nn+7sfOnTsZY4wVFxezRx55hDk5OTEfHx+2du1aplare2z3kSNHGACWn5/PQkNDmbOzM5szZw5rampieXl5LCAggLm5ubHFixez9vZ2bl8ajYatX7+eSaVS5uzszIKDg9nevXu557v3m5eXx6ZOncocHR3ZkSNH+rMbB1xCQoLe776mpoYtW7aM+z08/PDDLDMzU2+7BQsWsLfeeov5+voyqVTKGGPs6NGjLCQkhInFYhYWFsYOHDjAvYe69fS+NFSfCxcuDNSvo99ReJlg1qxZzNXVlb388susqqqKHT9+nI0YMYKlpKSwyspKVl5ezqKjo9mcOXO4bT799FO2b98+9tNPP7Hvv/+ezZ8/n02ePJlpNBrGmG54dXV1sX379jEArLq6mjU0NLCbN2+atZ9Ro0axTz/9lP3www/s97//PXNzc2MtLS2sq6uLNTQ0MHd3d5aZmckaGhpYR0cH+/nnn5lEImErV65klZWV7MCBA8zT05OlpaUZbXdVVRUXMo8++igrKSlh5eXlbPz48WzWrFksJiaGlZeXs3/84x9s+PDhbMOGDdy+3nrrLRYQEMDy8/NZbW0t27lzJxOLxVzIdu83ODiYFRQUsJqaGnbt2jVLd61F3bx5k0VGRrLExETW0NDAGhoa2O3bt1lqaio7efIkO3/+PPvwww+ZRCJhubm53HYJCQnM1dWVLVmyhJ09e5adPXuWtba2smHDhrHnn3+enTt3juXl5bGHH35YJ7xu3LjR4/vSUH26urr4+NX0CwovE8yaNYtNmTKFW163bh2LiYnRKXP58mUufAxpbm5mAFhFRQVjTDe8GPv1j/fGjRs91sXYfu4OCrVazUaNGsU2btzIrfPw8OCOuBhj7LXXXmMTJkxgWq2WW5eVlcVcXV25YLy33XfX8+uvv+bWKRQKBoDV1tZy6/7whz+w2NhYxhhjt2/fZhKJhB07dkxnX8uXL2eLFy/W2e9nn33WY/uFZtasWWz16tU9llm1ahV76qmnuOWEhATm7e3NVCoVt27r1q1s+PDh7JdffuHWbd++Xec9ZMr70pT6CAWNeZkoLCyM+/eZM2dw5MgRuLq6co+AgAAAd8aWAOCnn37C4sWLMXbsWLi7u0MqlQK4cys3c5i6n7tvuuvg4IDw8HBUVlYa3W9lZSUiIyN1pgV67LHH0NbWhp9//tlgu+8WHBzM/dvb2xsSiQRjx47VWXf16lUAQE1NDTo6OhAdHa3zO9u9ezf3++oWHh5utM62IisrC2FhYRgxYgRcXV2xbds2vf6cPHkynJycuOXq6moEBwfD2dmZW3fvTWlNeV/akkE3JU5fubi4cP9ua2vD/PnzsXHjRr1yvr6+AID58+dj9OjR2L59O0aOHAmtVotJkyaZPfjaX/vpq7vbfbe7Z/K0s7PTm9nTzs4OWq0WwJ3fFwAcPHgQfn5+OuXu/Z6esdezFXv27EFycjK2bNmCyMhIuLm5YdOmTfjuu+90yvXl92DK+9KWUHj1wdSpU7Fv3z5IpVI4OOj/Cq9du4bq6mps374dM2fOBACUlJT0uM/u/2U1Gk2f9nP8+HH8y7/8CwCgq6sLZWVlSEpKMvp6gYGB2LdvHxhj3NHX0aNH4ebm1u/znQUFBUEsFqOurg6zZs3q131bOycnJ50+PXr0KKZPn46VK1dy60w5KpowYQI+/PBDqFQqLvBPnjypU6a396Wh+ggZnTb2wapVq3D9+nUsXrwYJ0+eRG1tLQ4dOgSZTAaNRoOhQ4di+PDh2LZtG2pqanD48GHI5fIe9zl69GjY2dnhyy+/RHNzM9ra2szaT1ZWFg4cOICqqiqsWrUKN27cwLJly4y+3sqVK3H58mW88MILqKqqwueff460tDTI5fIeZ6/sCzc3NyQnJ2PNmjV4//33UVtbi/Lycrz77rt4//33+/W1rI1UKsV3332HixcvoqWlBQ899BBOnTqFQ4cO4ccff8Qbb7yhF0KGPPvss9BqtfiP//gPVFZW4tChQ9i8eTMAcP/59Pa+NFSf7qNjIaLw6oORI0fi6NGj0Gg0iImJweTJk/Hiiy/igQcegEgkgkgkwp49e1BWVoZJkyZhzZo12LRpU4/79PPzQ3p6Ol599VV4e3sjKSnJrP1s2LABGzZsQEhICEpKSvDFF1/A09Ozx9fLy8vDiRMnEBISgj/+8Y9Yvnw5Xn/99fv63Rizbt06vPHGG1AoFAgMDMTcuXNx8OBBjBkzxiKvZy2Sk5Nhb2+PoKAgjBgxArGxsVi0aBHi4uIwbdo0XLt2TecozBh3d3f87//+L06fPo3Q0FD8+c9/RmpqKgBw42C9vS8N1cfcMVhrQlPiECJQH330EWQyGVpbWzFkyBC+qzPgaMyLEIHYvXs3xo4dCz8/P5w5cwZr167F7373u0EZXACFFyGC0djYiNTUVDQ2NsLX1xdPP/00/uu//ovvavGGThsJIYJEA/aEEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9iUUuXLoWdnR335W1vb29ER0cjJyfHrK+m7Nq1q88zwN6PpUuX0pz/VorCi1jc3Llz0dDQgIsXL+Krr77CnDlzsHr1ajz55JM01TPpO15nEyM2r3tK43sVFRUxAGz79u2MMca2bNnCJk2axCQSCRs1ahRbsWIFu3XrFmPs14kK7350z/i6e/duFhYWxlxdXZm3tzdbvHgxa2pq4l7n+vXr7Nlnn2Wenp7M2dmZjR8/nuXk5HDP19XVsaeffpp5eHiwoUOHsn/7t3/jpkZOS0vTe12hT01tS+jIi/Di8ccfR0hICPbv3w8AEIlE+Otf/4pz587h/fffx+HDh/HKK68AAKZPn47MzEy4u7ujoaEBDQ0NSE5OBgCo1WqsW7cOZ86cwWeffYaLFy9i6dKl3Ou88cYb+OGHH/DVV1+hsrISW7du5b6wrlarERsbCzc3N3z77bc4evQoXF1dMXfuXHR2diI5ORm/+93vuCPHhoYGTJ8+fWB/UcQ4vtOT2DZjR16MMRYXF8cCAwMNPrd37142fPhwbnnnzp3Mw8Oj19c7efIkA8Adtc2fP5/JZDKDZT/44AO9qbBVKhUbMmQIO3ToUK/1J/yiIy/CG3bXRIhff/01fvOb38DPzw9ubm5YsmQJrl27ho6Ojh73UVZWhvnz5+PBBx+Em5sbN9lh91QvK1aswJ49exAaGopXXnkFx44d47Y9c+YMampq4Obmxk2bPGzYMNy+fdsmp022NRRehDeVlZUYM2YMLl68iCeffBLBwcHYt28fysrKkJWVBaDnexa2t7cjNjYW7u7u+Oijj3Dy5EkcOHBAZ7t//dd/xaVLl7BmzRrU19fjN7/5DXfK2dbWhrCwMJw+fVrn8eOPP+LZZ5+1cOvJ/aJZJQgvDh8+jIqKCqxZswZlZWXQarXYsmULN2ne//zP/+iUNzR9cVVVFa5du4YNGzbA398fAHDq1Cm91xoxYgQSEhKQkJCAmTNn4uWXX8bmzZsxdepU5ObmwsvLC+7u7gbraUvTJtsaOvIiFqdSqdDY2IgrV66gvLwc69evx4IFC/Dkk08iPj4e48ePh1qtxrvvvovz58/jgw8+QHZ2ts4+pFIp2traUFRUhJaWFnR0dODBBx+Ek5MTt90XX3yBdevW6WyXmpqKzz//HDU1NTh37hy+/PJLBAYGAgCee+45eHp6YsGCBfj2229x4cIFFBcX409/+hN3ByWpVIp//vOfqK6uRktLC9Rq9cD80kjv+B50I7bt7rs0Ozg4sBEjRrCoqCiWk5PD3R+SMcYyMjKYr68vGzJkCIuNjWW7d+/Wu4/lH//4RzZ8+HCdSyU+/vhjJpVKmVgsZpGRkeyLL77Qu5dhYGAgGzJkCBs2bBhbsGABO3/+PLfPhoYGFh8fzzw9PZlYLGZjx45liYmJrLW1lTHG2NWrV1l0dDRzdXWlSyWsDM3nRQgRJDptJIQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBGk/wM1oS3eBraFvAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:52:23.611709Z", + "iopub.status.busy": "2024-03-22T22:52:23.611355Z", + "iopub.status.idle": "2024-03-22T22:52:23.833610Z", + "shell.execute_reply": "2024-03-22T22:52:23.832552Z" + }, + "papermill": { + "duration": 0.244314, + "end_time": "2024-03-22T22:52:23.835761", + "exception": false, + "start_time": "2024-03-22T22:52:23.591447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABgMElEQVR4nO2dd3xT1fvH3zezu6wORtl7L0FAUQQEBbciyk8BUb8qiMhXBVS2DBfyFRUniCLiQGWDgIACglj23quMFkr3yri/P26TJk3SJG3StOW8X6+8uNxx7nOb5JPnPOc5z5FkWZYRCASCAKIKtAECgUAghEggEAQcIUQCgSDgCCESCAQBRwiRQCAIOEKIBAJBwBFCJBAIAo4QIoFAEHA0gTagJJjNZi5evEh4eDiSJAXaHIFAUAhZlklPT6dGjRqoVK79nnItRBcvXiQuLi7QZggEAjecP3+eWrVquTxeroUoPDwcUB4yIiIiwNYIBILCpKWlERcXZ/2uuqJcC5GlOxYRESGESCAow7gLnYhgtUAgCDgBFSKTycT48eOpV68ewcHBNGjQgKlTpyIKAggENxYB7Zq9/fbbzJ07lwULFtCiRQv+/fdfhg4dSmRkJCNHjgykaQKBoBQJqBBt27aN++67j379+gFQt25dvv/+e/755x+f3UOWZYxGIyaTyWdtCgRlAa1Wi1qtDrQZPiGgQtS1a1c+//xzjh07RuPGjdm7dy9btmxh1qxZPmk/Ly+PS5cukZWV5ZP2BIKyhCRJ1KpVi7CwsECbUmICKkRjx44lLS2Npk2bolarMZlMTJs2jUGDBjk9Pzc3l9zcXOv/09LSXLZtNps5ffo0arWaGjVqoNPpRNKjoMIgyzJJSUlcuHCBRo0ala5nlJsOujDw4fcpoEL0448/8t1337Fo0SJatGjBnj17GDVqFDVq1GDw4MEO58+YMYPJkyd71HZeXh5ms5m4uDhCQkJ8bbpAEHCioqI4c+YMBoOh9IRIlmHJ05CXCff8D6o28EmzARWiV199lbFjxzJw4EAAWrVqxdmzZ5kxY4ZTIRo3bhyjR4+2/t+SLFUURaWVCwTlmYB4+IeXw7E1oNKCyeCzZgMqRFlZWQ5CoVarMZvNTs/X6/Xo9frSME0gEBQmJw1Wj1G2u70E0U191nRAheiee+5h2rRp1K5dmxYtWrB7925mzZrFU089FUizBAKBMzZOg/SLULkedH/Fp00HtN8yZ84cHn74YV544QWaNWvGK6+8wn/+8x+mTp0aSLMqJGfOnEGSJPbs2VMq7WzdupVWrVqh1Wq5//77S3TPG5VNmzYhSRIpKSmBNgUSdsGOz5Tt/rNAG+zT5gPqEYWHhzN79mxmz54dSDNuSDZt2kSPHj24fv06lSpV8nn7o0ePpm3btqxevbpCDC/f0JiMsGIUIEOrAdDgDp/fQkRyywF5eXmBNsFrTp48yR133EGtWrWKLXSl+dyWxFdfUx7fOwf++Rwu7YWgSOgzzS+3uKGESJZlsvKMpf7ydu7c7bffzogRIxg1ahTVqlWjT58+HDhwgLvuuouwsDBiYmJ44oknuHr1qvWaNWvWcMstt1CpUiWqVq1K//79OXnypNP2z5w5Q48ePQCoXLkykiQxZMgQr9o5cuQIXbt2JSgoiJYtW7J582Zr25Ikce3aNZ566ikkSeLrr78GYPPmzXTq1Am9Xk/16tUZO3as3Zff2XNbuidr166lXbt2BAcHc8cdd5CYmMjq1atp1qwZERERPP7443aJq2azmRkzZljnMbZp04aff/7ZetzS7urVq+nQoQN6vZ4tW7a4fW/eeustoqOjCQ8P5+mnn2bs2LG0bdvWenzIkCHcf//9TJs2jRo1atCkSRMAvv32Wzp27Eh4eDixsbE8/vjjJCYm2rW9atUqGjduTHBwMD169ODMmTNu7fE7qRfgj7eU7d5TICzaL7cp12VAvCXbYKL5hLWlft9DU/oQovPuT71gwQKef/55tm7dSkpKCnfccQdPP/00H3zwAdnZ2YwZM4YBAwbwxx9/AJCZmcno0aNp3bo1GRkZTJgwgQceeIA9e/Y4jEzGxcWxZMkSHnroIY4ePUpERATBwcFetfPqq68ye/ZsmjdvzqxZs7jnnns4ffo0cXFxXLp0iSZNmjBlyhQeffRRIiMjSUhI4O6772bIkCF88803HDlyhGeeeYagoCAmTZrk9LkBLl26BMCkSZP46KOPCAkJYcCAAQwYMAC9Xs+iRYvIyMjggQceYM6cOYwZo4zqzJgxg4ULF/Lpp5/SqFEj/vzzT/7v//6PqKgobrvtNuv9xo4dy3vvvUf9+vWpXLlyke/Jd999x7Rp0/jkk0/o1q0bixcv5v3336devXp2523YsIGIiAjWrVtn3WcwGJg6dSpNmjQhMTGR0aNHM2TIEFatWgUoNbUefPBBhg8fzrPPPsu///7Lf//7X/cfFH+zegwYMiHuZmj3pN9uc0MJUXmiUaNGvPPOO4DyK9yuXTumT59uPT5v3jzi4uKs02Meeughu+vnzZtHVFQUhw4domXLlnbH1Go1VapUASA6Otqu6+RpOyNGjLCeO3fuXNasWcNXX33Fa6+9RmxsLJIkERkZSWxsLACffPIJcXFxfPTRR0iSRNOmTbl48SJjxoxhwoQJVpGzfW4oEKK33nqLbt26ATBs2DDGjRvHyZMnqV+/PgAPP/wwGzduZMyYMeTm5jJ9+nTWr19Ply5dAKhfvz5btmzhs88+sxOiKVOm0Lt3b/dvCMrgyrBhwxg6dCgAEyZM4PfffycjI8PuvNDQUL788kt0Op11n+1IcP369fnwww+56aabyMjIICwsjLlz59KgQQPef/99AJo0acL+/ft5++23PbLNLxxeAUdWgEoD98wGP+bk3VBCFKxVc2hKn4Dc11s6dOhg3d67dy8bN250GvQ9efIkjRs35vjx40yYMIEdO3Zw9epVay7WuXPnHISoKDxtx/IFB9BoNHTs2JHDhw+7bPfw4cN06dLFLgmvW7duZGRkcOHCBWrXru3w3La0bt3auh0TE0NISIhVhCz7LJOlT5w4QVZWloPA5OXl0a5dO7t9HTt2dGlzYY4ePcoLL7xgt69Tp05Wr9RCq1at7EQIID4+nkmTJrF3716uX79u93dt3rw5hw8fpnPnznbX2P6NS53cdFj9mrLddSREN/Pr7W4oIZIkyesuUqAIDQ21bmdkZHDPPfc4/XWsXr06oORk1alThy+++IIaNWpgNptp2bKl18FSX7VTXGyf2xatVmvdliTJ7v+WfZYvt8VDWblyJTVr1rQ7r3BCrKv7lYTCbWZmZtKnTx/69OnDd999R1RUFOfOnaNPnz5lN5i9cTqkJUDlutD9Vb/frnx8K29w2rdvz5IlS6hbty4ajeNbdu3aNY4ePcoXX3zBrbfeCuA28Gr5xbYtj+JNO9u3b6d79+4AGI1G4uPjGTFihMv7NWvWjCVLliDLstUr2rp1K+Hh4UUWVS8OzZs3R6/Xc+7cObtuWElp0qQJO3fu5MknC2IlO3fudHvdkSNHuHbtGjNnzrROSfr333/tzmnWrBnLli2z27d9+3YfWF0MLu6BHZ8q2/3eB53/52reUKNm5ZXhw4eTnJzMY489xs6dOzl58iRr165l6NChmEwmKleuTNWqVfn88885ceIEf/zxh92cPGfUqVMHSZJYsWIFSUlJZGRkeNXOxx9/zK+//sqRI0cYPnw4169fLzIj/oUXXuD8+fO8+OKLHDlyhKVLlzJx4kRGjx7t8/mA4eHhvPLKK7z88sssWLCAkydPsmvXLubMmcOCBQuK3e6LL77IV199xYIFCzh+/DhvvfUW+/btczvnq3bt2uh0OubMmcOpU6dYtmyZQ9Luc889x/Hjx3n11Vc5evQoixYtso42lipmEyx/CWQztHwIGvYqldsKISoH1KhRg61bt2Iymbjzzjtp1aoVo0aNolKlSqhUKlQqFYsXLyY+Pp6WLVvy8ssv8+677xbZZs2aNZk8eTJjx44lJiaGESNGeNXOzJkzmTlzJm3atGHLli0sW7aMatWqFXm/VatW8c8//9CmTRuee+45hg0bxptvvlmiv40rpk6dyvjx45kxYwbNmjWjb9++rFy50mGEyxsGDRrEuHHjeOWVV2jfvj2nT59myJAhBAUFFXldVFQUX3/9NT/99BPNmzdn5syZvPfee3bn1K5dmyVLlvDbb7/Rpk0bPv30U7vBiVLjny/g0h7QR0KfGaV2W0kuxwWi09LSiIyMJDU11WEVj5ycHE6fPk29evXcflAEguLSu3dvYmNj+fbbb0v93j7/jKcmwMedIS8d+n8AHUs+57Oo76gtIkYkEHhIVlYWn376KX369EGtVvP999+zfv16u3yhcs2aMYoI1eoE7YeU6q1F10wgyKdFixaEhYU5fX333XdIksSqVavo3r07HTp0YPny5SxZsoRevUonjuJXjqxSag2VQs6QM4RHJBDks2rVKgwG58W+YmJiCA4OZv369X67vyzLJGfmEaxTl26aSW4GrMofou8yAmJalN698xFCJBDkU6dOnYDePy3HSEJKNjq1iiax4aVXgXHTDEi7AJVqw21jSueehRBds1IkKT2XU0kZmM3ldnxA4EdSspTkxjyTmVyj8yqlPufSXtg+V9nuN6tUcoacIYSoFLmakUtGrpEsg1hjTWCPySyTnlNQiSAj1/clSRwwm2D5KJBN0OIBaOTZnDt/IISolJBlGaPFExIOkaAQ6TkGzDaZNBk5pSBE/86Di7tAHwF9Z/r/fkUghKiUMJllm7pEQokE9qRkKUHy8CBlDl1GrtFOmHxO2iVYn780V88JEB7rv3t5gBCiUsJgEuIjcI7JbCY9vysWG6FHo1JhlmWy8vzYhbfkDNXs6JPExZJyQwiRLMtk5BjILI1+twuMLpZI8jV169a1qwEuSRK//fZbqdxbUDzSspUqnnqNmiCtmjC9Mpjtt+7ZsbVwaClI6vycoVJcJdYFN8Tw/dWMPC6lZhOm11A/KjCF3G09otL0jS5duuS28qAgsKRmK92ySiFaJEkiLEhDSnaefwLWeZmwMn8poC4vQGwr39+jGNwQHlFEkKK3mbkmjKZSGhYtRKDuGxsbG/BFKf1VmL4iYDSZraNlkcFKfCg83yPKyjP6/nOzaSaknoPIOLh9nG/bLgE3hBDptYrLK8tm0tNTlV+F0nzJMgYvcofS09MZNGgQoaGhVK9enQ8++IDbb7+dUaNGef3stl0zS2H7X375hR49ehASEkKbNm34+++/7a7ZsmULt956K8HBwcTFxTFy5EgyMzOtx90Vgi9uYfobkbQcAzIyQfmfUQCtRkWQRtn2aTjh8n74+2Nl++73QOf7onDF5YbomgFEBGnJzUqn8mz/lrx0yusXMdrEHd1J0ujRo9m6dSvLli0jJiaGCRMmsGvXLrvVIkrCG2+8wXvvvUejRo144403eOyxxzhx4gQajYaTJ0/St29f3nrrLebNm0dSUhIjRoxgxIgRzJ8/H3BfCN6CN4Xpb1Qso2WVgu0rToYFacjJMJGeayQyROfsUu8wmwtyhprfB036lrxNH3LDCFFksIakAN7fbtSsCCVKT09nwYIFLFq0iJ49ewIwf/58atSo4TNbXnnlFfr16wfA5MmTadGiBSdOnKBp06bMmDGDQYMGWb2vRo0a8eGHH3Lbbbcxd+5cgoKC3BaCt+BNYfobEYPJbPV4IkMKCZFeoyTA5hjtqloWm/h5kPAv6MKhbwAL8rvghhGiIK0arT6U/YMPU7tKiLU/XipoQzCa0j069dSpUxgMBjp16mTdFxkZaV0fyxfYFqK31LxOTEykadOm7N27l3379vHdd99Zz5FlGbPZzOnTp2nWrJnbQvAWvClMfyOSlm1ARllcQa+xH7kK1WuQJIk8k5k8oxl9MRZgsJJ+2T5nKKJ68dvyEzeMEEmSRESIjqtmmTSTjshSnFMjexkj8jeFC9EDdoXn//Of/zBy5EiH62rXru1VIXh/FKavSKTYjJYVRq2SCNGpycw1kp5rLJkQrRkHuWlQoz3cNKz47fiRG0aIACKCtVzNyCUtP51eVUqzm+2zqoumfv36aLVadu7caV1iJzU1lWPHjlmL1fuT9u3bc+jQIRo2bOj0+P79+90Wghe4x65bFuw8BhSu15CZayQjx0i1sGKOfB5fBwd/AUlVZnKGnHFDjJpZCNWp0ahUmMwyWaWY3Ggs5A0VJUnh4eEMHjyYV199lY0bN3Lw4EGGDRuGSqUqlbIQY8aMYdu2bYwYMYI9e/Zw/Phxli5dal2hw5NC8AL3pOYHqUN0GnQa51/DMGvaSTGne+Rlwcr8xQ9ufgGqtymWraXBDSVEkiRZc4pSS2NSYT4GL3NBZs2aRZcuXejfvz+9evWiW7duNGvWrFRqb7du3ZrNmzdz7Ngxbr31Vtq1a8eECROswXJPCsEL3GPtlhURqwzWqtGoJEyyTHZxpntsfhtSzkFErTKVM+SMG654flq2gTPXMtGqVTQtpeJTyZl5XLieZf1/naohLt1xZ2RmZlKzZk3ef/99hg0rm318gefkGU0cuawMXjSrHoFW7dofOHctk5RsA9ERQcRG2P8QFVk8/8pB+Kw7mI3w2GJocpfPn8MTRPF8F4TpNagkCYPJTLbBVColOR3mmbmR/t27d3PkyBE6depEamoqU6ZMAeC+++7zk4WC0sQypSNUrylShID86R4GZd6Z6++xPZacIbMRmvYPmAh5ww0nRCqVRHiQhtRsA6nZhtIRomLMvH/vvfc4evQoOp2ODh068Ndff3H48GHuusv1h8qy1LKgbOMqidEZYXotkE12nhGj2YzGk6L2u76GC/+ALgzueqdkxpYSN5wQgTKnJzXbQFq2keqR/r9f4RiRO1lq164d8fHxDvuzs7PZs2eP7wwTlDq5BhPZBhMSEhEeCJFOo0KvUZNrNJGZa3TfpU+/AusnKdt3jIfImiU3uhS4IYUoPEhJFss1msgxmKxzfPyFr2oRBQcHuxxWF5QPCrplarfdMgvhQRpyM0yk53ggRGtfh5xUqN4WOj1TQmtLjwo/auYsFq9Wqaw1X9KynS8f40ssMaJSW5VBUGYpKonRFdb6RIVSThw+2yfWw4Gfy3zOkDMqrBBZsoezsrKcHo8Izh/G97MQybJsjRFp1UKIbmRyDIoHLiEREeS5EIXqNUhI5BnN5NrMnrZksqvVajBkw8r/Kgc6Pwc12vnUdn9TYbtmarWaSpUqWctThISE2HkkOszIxjyyjJCWoXaZVFZSjCYzJkMuACpJjWw0kZoBV66nEx2uJ7g0F9ITBJSrGbnIxjxC9RqMhjyMXvwG6lVGsvNMJKdmUjlUh9lsJikpiZCQEDQaDfwxA66fgYia0ON1vz2Dv6jQ34LYWKUguG2tHFtS03PJNZrJS9Fa3V9fYzCZSUzLRSWBTq0ix2jGYk2CSqJ6pP+TFAVlgytpORhMMsZQLXkp3n3e0nMMpGYbSdeqqJo/3UOlUlG7dm2kpCOw9X/KiXe9A/pwX5vudyq0EEmSRPXq1YmOjna6lPC//57n080naVMrklmP+seVjT9znUkb91KnSgjVwvXEn71ud/zn57pSOdQH9WYEZZoTiRm88cu/aNUqljzfhVC9d9UfDl9KY9KqXYTqNPzyQlc0ahU6nU6JrVhyhpr0g2b9/WG+36nQQmRBrVYr/ehC3N68JuNXHOPSkWTeMqmo4gdBuJJlIiHdRL0YPSl5Egnp9qn6zy3ez9Lh3Xx+X0HZYtWhMySkm7izeTWqRnrvsbSuoyfDqCIhPZejV3PpUCe/0Fz8Aji/HbShcHf5yBlyRoUNVntCXJUQmlePwCzD+sNX/HKPxHQlPhQdrsdZqHrv+RS/3FdQdpBlmRX7LgFwT5viFbhTqyS6NawKwF/H80v8ZSTCuvHK9h1vQGStEtsaKAIuRAkJCfzf//0fVatWJTg4mFatWpVqWYk+LZQ40u8HL/ul/StpOQBERehRiUGzG5J9F1I5l5xFsFZNz2bRxW7n1kZRAPx1/KqyY+0bSs5QbGvo9B9fmBowAipE169fp1u3bmi1WlavXs2hQ4d4//33S7W+cZ+WMQD8efyqX9Y9s3hEMeFBIo/oBmXFvosA3NEsukRTim5pWA2APedTyDy8Dvb/mJ8z9D9Ql+8oS0Ctf/vtt4mLi7MWZQeoV69eqdrQJCacOlVDOHsti01Hk+jX2rdlNJPS8rtmEc67ZoKKjdkss9LSLWtdsrrjcVVCqFctlItXr8PKscrOTs9CzfYlNTPgBNQjWrZsGR07duSRRx4hOjqadu3a8cUXX7g8Pzc3l7S0NLtXSZEkydo9+/OY78vrX0lXumYxEUEIh+jGY/f561xMzSFMr+H2JlElbu/WRtUYrvmN0IyzEF4derzhAysDT0CF6NSpU8ydO5dGjRqxdu1ann/+eUaOHMmCBQucnj9jxgwiIyOtL0up0pJSt6pSW/laZp6bM71DlmUS02yC1UKJbjiW71W8od7NY3wyp7FvTCrPqZcr/7nrHQjytDZI2SagQmQ2m2nfvj3Tp0+nXbt2PPvsszzzzDN8+umnTs8fN24cqamp1tf58+d9YodtSU5fkp5rJNugDNdHhweJrtkNhskss3K/IkT9fdHlN5vpdGAKOsnEOlN7zsf0LHmbZYSAClH16tXtlp8BaNasGefOnXN6vl6vJyIiwu7lC8L0yi+Vr9cat3hD4UEagnVq0TW7wfjndDJJ6blEBGmsI14lYs93aC5sJ0fSM8kwmL9OXCt5m2WEgApRt27dOHr0qN2+Y8eOUadOnVK1Iyw/y9XnQpQfH4oOz0/JF0p0Q2EZLevbMrbkcxkzkuD3NwH4p87zJBBVkE9UAQioEL388sts376d6dOnc+LECRYtWsTnn3/O8OHDS9WOUD97RDH5tYaFDt04GE1mVh9QctP6l3C0DFBEKCcFYloRdruyosrWE1cxlaH18kpCiYQoIyOjRKNYN910E7/++ivff/89LVu2ZOrUqcyePZtBgwaVxCyvCbd4RD5e2aOwRySJKNENw7aT10jOzKNKqI6uDaqWrLFTm2HfYkCCe/5H67iqRARpSMsxsu9Cii/MDThe5xGdPn2aESNGsGnTJnJycqz7Letzm0zeLXvSv39/+vcP7EQ9S7A622DCZJZR+ygF+oo1hyh/hr3QoRsGS7fsrpaxaDysxOgUQw6seFnZvulpqNUBDdC1QTXWHLzMX8ev0q526SUA+wuvhej//u//kGWZefPmERMTUyGGpC1dM1C6Z5Ee1BL2BNt5ZiBiRDcKeUYza3zVLdsyC5JPQlgs9Bxv3X1rY4sQJTGyZ6OS3aMM4LUQ7d27l/j4eJo0aeIPewKCXqNGp1aRl78MsM+EKH+emcUjEjJ0Y/DX8STScoxEhevpVK9K8RtKOgZbPlC273obggpWeri1oTIKt/tcCuk5BsK9qPhYFvHaZ7zpppt8lr9TlvBHwLqwRyQcohsDy0z7fq2qF7+bL8tKl8yUB43uhOb2a9rVrhpCnaohGM0y208ll9TkgOO1R/Tll1/y3HPPkZCQQMuWLa21oS20bt3aZ8aVJmFBGq5nGXwrRGkF0ztAdM1uBHIMJtYdUkrK3NOmBEmMexbB2S2gCYa733P6K3Zro2qcvXaOLceT6N08pvj3KgN4LURJSUmcPHmSoUOHWvdJklTsYHVZITR/VrSvRs4yco1k5lmyqi2jZoKKzqajiWTkGqkRGUS7uGIGkTOvWXOG6DEOKjvPq7ulYRQLt58rKAtSjvFaiJ566inatWvH999/X2GC1aBkP4PvumYWbyhMryHUUg+7YvypBEWwPL9b1r9NDVTF7ZatGw/ZyRDTEm5+weVpXRpURa2SOHU1kwvXs6hVOaR49ysDeC1EZ8+eZdmyZRVuob9QF2tHFZfC8SEQeUQVnaw8I38cVpZGKPbcstN/wZ7vAAn6zwa16yB0ZLCWNrUi2XUuhS3HrzKwU+3i3bMM4HWw+o477mDv3r3+sCWgWBex81HXzFqZ0UaIRIXGis2Gw4lkG0zUrhJCq5rFWMvcmAsrRinbHZ+CuJvcXuJQtbGc4rVHdM899/Dyyy+zf/9+WrVq5RCsvvfee31mXGkS7uMZ+Enp9tM7QIyaVXSW71WSGPu3rl68kMWW2XDtBITFQM8JHl3SvXE1/rfhOFvyp3v4Khm3tPFaiJ577jkApkyZ4nCsQgSrfSREFo9IdM1uDNJzDGzKL6xXrAL5V0/AX+8p231nQHAljy5rU6sS4XoNqdkGDiSk0ibOs+vKGl53zcxms8tXeRUhKJjm4esYka1HpAr4UgUCf7Hu0BXyjGYaRIXSNNbL5YJkGVbm5ww17AUtHvT4Uo1aRZcGhVb3KId49dUwGAxoNBoOHDjgL3sCRpivg9U2taoLEB5RRaWgW1bD+27Zvh/g9J9KzlC/973uw9/aSCmqX57jRF4JkVarpXbt2uXa83GFz4PV6Y7BahEjqpikZOVZRcDrJMasZFibv1b97WOgcl2v728JWO86d93npWxKC687C2+88Qavv/46ycnlP63cFl8P3yelOQlW+6RlQVlj7cHLGM0yTWPDaRjtZbds3XjIugbRzaHLiGLdv07VEOKqBGMwyfx9snxWbfQ6WP3RRx9x4sQJatSoQZ06dQgNDbU7vmvXLp8ZV5r4MkaUlWckPb+daLvheyFFFZFir+J6ZivsXqhsu8kZKgpJkujZNIavt51h6Z6Ecjndw2shuv/++/1gRuAJ1/tu+N4SHwrWqq1dPhBds4rI1Yxctp5QumVeJTHa5gx1GAq1O5fIjoc71OLrbWf4/dAVUrLyqBSiK1F7pY3XQjRx4kR/2BFwfNk1Kxgxs19CSOhQxWP1gcuYZWhdK5I6VUPdX2Bh64dw9RiERkGvkn+nWtSIoGlsOEcup7Ns70We7FK3xG2WJsUeUI6Pj2fhwoUsXLiQ3bt3+9KmgODLUbOCHKIgu/0VZV6eoIAVNkmMHnPtJPz5rrLddyYEl7zCoiRJPNJRWefvp38vlLi90sZrjygxMZGBAweyadMmKlWqBEBKSgo9evRg8eLFREX5YNmUAGARohyDGaPJXKLyntZ5ZnZD96JrVtG4kpbDP2eUQZt+nlZilGVYORpMudDgDmj5kM/sub9tDWauPsz+hFQOX0qjWfXys/ii19+2F198kfT0dA4ePEhycjLJyckcOHCAtLQ0Ro4c6Q8bS4VQm1hOZm7J0hMSXXlEonNWoVi57xKyDO1rV6JmpWDPLtr/E5zaBJqgYuUMFUXVMD09myqB6vLmFXktRGvWrOGTTz6hWbNm1n3Nmzfn448/ZvXq1T41rjTRaVTWtafScw0lakt4RDcGlgL5Ho+WZSXDmnHKdvdXoUp9n9s04KZaAPy2J4E8o9nn7fuLYk3xKDzRFZRkR7O5jD+4XPQaUOEexIk8WUfKsoxQTCEhKqfzEQVOuHA9i13nUpAkuLuVh/Gh9RMh6ypENYWu/uk9dG8URXS4nuTMPP44csUv9/AHxSoD8tJLL3Hx4kXrvoSEBF5++WV69iyja3GnXYIfn4S93xd5WqibIfydZ5JpNWkt324/W2Q71mWERLC6wrIyP3eoU90qdkmrLjn7N+z6RtnuPxs0/hle16hVPNhe8Yp+LEfdM6+F6KOPPiItLY26devSoEEDGjRoQL169UhLS2POnDn+sLHk7P8JDi2FtW8oZThdYAlYp7uY5vHyD3vIyjMx/rei59oV1Kou1DXzxmZBmcarJEZjXkHOUPsnoU4X/xkGPNJREaJNRxOtI7hlHa9HzeLi4ti1axfr16/nyJEjADRr1oxevXr53DifcfPzijeUeAjWTYD7P3Z6Wpi1JpHzYLUn65fnGEyk5QtZlPCIKiRnrmayPyEVtUrirpax7i/Y9iEkHYGQatBrst/taxAVRoc6lYk/e51fdiXw/O0N/H7PkuK1EIHyherduze9e/f2tT3+Qa2Fe/4HX/WGPQuhzUCod6vDaQW5RM6D1XqN2ul+WyxZ1XqNiogg+z+v0KGKwcr9ijfUtUFVqobpiz45+ZRNztAMCCnBOmde8EiHWsSfvc5P8ed57rb6Zf5HsFhCtGHDBjZs2EBiYqJDgHrevHk+McznxHVSUunj5yvrRT2/FTT2H6ICIXLuEek98IgKAtVBDm9+2f4oCDxluadJjLIMK/8Lxhyofzu0esT/xuXTr3V1Ji8/xKmkTHadu06HOqUjgMXF6xjR5MmTufPOO9mwYQNXr17l+vXrdq8yTa+JEBoN147D1v85HA51UwrEVoju/WgLIxY5TvAtCFQ7/lKKSa/lnxOJ6Ry5nI5WLdGnhZtu2YElcPIPUOuh36xSdYnDg7Tc1UqxrzzkFHntEX366ad8/fXXPPHEE/6wx78EV1bc4yXD4M/3lEp41QpWI7HWrc5zIUTagq7Zvgup7LuQysu9M2gQFWbdb+sRFUboUPln+V6lW3Zro6iiJ5ZmX7fPGapa+nGaAR3j+GVXAiv2XWLCPc0J0RWrA1QqeO0R5eXl0bVrV3/YUjq0fEhJrTflKuU5bXKLLHWrXY2aOeuarTlw2e7/Fo8oyolHJHSofCPLsjWJ0W23bP1kyEyEak2gW2BmHHSuV4XaVULIyDWyev9l9xcEEK+F6Omnn2bRokX+sKV0kCQltV4TpJTn3Pej9ZC7mkTOhGhVfuDSgsUjKpxVbb23oNxy+FI6J5My0WlURdf8ObdDiUUC9P/AIRZZWkiSxMMdlKH8n+LPB8QGT/HaV8vJyeHzzz9n/fr1tG7d2iHLetasWT4zzm9UqQ+3vQYbpsDacdCoN4RUcVuTyNmo2cGLaZy9lmktAWFdRijcsWsmMqvLNxZvqEeTKMKDXBQxMxkKcoba/R/U7VY6xrngoQ61+GD9MbafSubctSxqVy2bq8F67RHt27ePtm3bolKpOHDgALt377a+9uzZ4wcT/USXFyGqmVKmc52yhpS7YHWQ1vmfa7VN98xaAsSZRyQotyjdsvzlpIuaaf/3R0q+WkhV6D21lKxzTc1KwdzSUCmu/3MZ9oq89og2btzoDztKH40O7pkN8/rA7m+h7eOEBTUCXHfNXC1e98PO8zzZpQ4hOo3NUtOOHpGbqW6CMsy+C6mcS84iWKumZ7No5ycln4ZNbyvbd04rtZwhdzzSMY6/jl/l5/gLvNSrcZlchPHGXmmr9s3QfrCyvXwU4Rolf8iVEBWe8Nq8egRR4XpOX81kwtKDAKRkKcmQlUMcXXehQ+UXS7esZ7No56NPsgyrXgFjNtS9VUmaLSPc2TyGiCANF1Nz2HaybC45dGMLEUCvSUq5zqtHiTv8FeA6RlRYiKIj9Mx5rB0qCX6Ov8CPOwtcX6e/OsIlKpeYzbJ1kqvLbtnBX+HEelDrlEmtZWhgIkir5r62NYGyOxFWCFFIFegzHYBqu/5HHemydQWOwhQWIr1Gxc31qzK6d2MA3vhtv/WYMyHyoIKIoAyy+/x1LqbmEKbXcHsTJxVIs1NgzVhl+9ZX7HLTygqWibBrD14mNatk9bb8gRAiUFLv69+OZMplqmY+eUaT06JSpkIejS5/FO2F2xtya6NqGEwFx53N7TELj6hcYklivLN5DEFaJ/MNN0yBjCtQtRHcMqp0jfOQVjUjaRobTp7RzLK9CYE2xwGvhejPP//EaHT0GIxGI3/++adPjCp1JAn6zUJW6+mu3s+9qm1Ou2dmJx4RgEolOeSVOPOIhAyVP0xm2TrJtb+zVVzP74R/8+dXBjBnyB32OUVlr3vmtRD16NHD6Sqvqamp9OjRwydGBYSqDZBuexWA8dpvyUpNcjjFVEhJipoEK0JEFYN/TieTlJ5LZLCWWxoW6pZZc4ZkaDvIaUWHssQD7WqiUUnsu5DKkctpgTbHDq+FSJZlp92Oa9euOaz6Wu7o+hKnqEWUlEbIn285HHb0iArc9MJ/EWcTXGWhROUOy2hZnxYxjvWotn8CVw5AcJUykTPkjqphemvqQVmbCOtxHtGDDz4IKC7ekCFD0OsLXFCTycS+ffvK9xw0AI2O2UHP82HOG1Q+sgjODVaG+PMxFip5YvfBLCQ8ToXIt9YK/IzRZLYmqzpUYrx+FjbOULbvfAtCq5aydcVjQMc41h68wq+7ExjTt6lHxf5KA4+tiIyMJDIyElmWCQ8Pt/4/MjKS2NhYnn32WRYuXFhsQ2bOnIkkSYwaNarYbfiCU6FtWGy8XfnP8lFKmc98TIXi1953zZxLkfCUyibbTl4jOTOPqqE6utS3ERrbnKE6t0DbxwNnpJfc1jiKKGtx/cRAm2PFY49o/nxlEl/dunV55ZVXfNoN27lzJ5999hmtW7f2WZvFJVSnYabxMR4I2Ys+6TD8PQdu/S/gOOqlt5nyUVh3vBm+l+UylXYiyMfSLevbMtZ+wc1DS+H47/k5Qx+UqzdPKa5fk882n+Ln+PP09aTUbSngtV82ceJEn4pQRkYGgwYN4osvvqBy5ZIvvVtSwoM0pBDO7mZK4JrN7yip+zjLI7KJERX6LHozfC+G9cseeUaztcSLXRJjTiqsHqNs3/IyRDUOgHUl45EOytLUG48mWatFBBqvhejKlSs88cQT1KhRA41Gg1qttnt5y/Dhw+nXr59Hxfdzc3NJS0uze/kay8TXA1X7Qr3uSpnPlf8FWXYQDNv+te0qrq6m8rjSG5HoWPb463gSaTlGosP1dKpnM2dsw1TIuAxVGsAtowNnYAloGB1Gu9qVMJllft1VNnKKvJ70OmTIEM6dO8f48eOpXr16iYpyL168mF27drFz506Pzp8xYwaTJ/t3FQRr3eo8E/T7AOZ2hZMb4MASTOZadue6ihF5O6lQeERlD8tM+7tbVS94Py/Ew84vle3+H4DWg/XMyigDOsax+1wKP/57nme7B764vtdCtGXLFv766y/atm1bohufP3+el156iXXr1hEU5NkbOm7cOEaPLvgVSktLIy4urkR2FMZaHC3HCNWaK/GhTdNhzTj0YZ/YnWsrRLbvo6s31ZXgCB0qW+QYTKw7pKySeo8lidFkhOUvATK0Hgj1bwucgT6gf+vqTF5+kJNJmew+n0L72oENi3jdNYuLi/PJKE98fDyJiYm0b98ejUaDRqNh8+bNfPjhh2g0Gkwmx5U09Ho9ERERdi9fE6YrVKXxllFK6n5mIgPT7FcosRMim/1qL4VIeERli01HE8nINVKzUjDt4vK/oDvmwpX9St3zPtMCa6APCA/ScndLRWR/+jfwdYq8FqLZs2czduxYzpw5U6Ib9+zZk/3797Nnzx7rq2PHjgwaNIg9e/YUK97kCxzKxWr0St0ioE/2KtpLx6znugpWex8jEkJUllie3y3r17o6KpUEKedgozIxmt5TIbRaAK3zHQ/nT4RdvvcS2XnOl9AqLbzumj366KNkZWXRoEEDQkJCHErFOpv+4Yzw8HBatmxpty80NJSqVas67C9NrFUabeea1b1FSeHf8x3TtV/RP28aRjQuY0QqF0rkSm5EsLrskJVn5I/DSn5N/9bV83OGXgVDFtTpppR/rSDcXK8qcVWCOZ+czeoDl3iwfS33F/kJr4Vo9uzZfjCj7OCybnXvqaTuXU5T1XmGqVfzmemeQnlEtqNmLoRIJDSWedYfTiTbYKJ2lRBa1YyEw8vh2BpQactdzpA7VCqJh9vH8cH6Y/z074XyJUSDBw/2hx0AbNq0yW9te4rFI3JYUii0Kl8GD+O/WR8wSrOElebO6Gy7jyXqmhXf3uw8E5/9eZI7m8fSvIbvY2Y3GivyV3G9p011pNx0m5yhURDVJHCG+YmHOtRk9oZj/H3qWkCL6xdrosnJkyd58803eeyxx0hMVNzY1atXc/DgQZ8aFwjCilhk8XdND/42NSdYymOqZj46tXPFcTV8X5IYUWJaDk98tYO1B+3Xp5rzx3Fmrz/O3R/+5bYNQdGk5xjYdEyputC/dQ3YOA3SLyqrvuRn11c0alUOoVuD/OL6uwI3EdZrIdq8eTOtWrVix44d/PLLL2RkZACwd+9eJk6c6HMDS5uwIlbyMMoybxifIlfW0EO9l/DTK63HbKXH2+H7HIOJpxfsZP7W0y7tmrT8IH8dv8p/vo2323/4Utkq51CeWXfoCnlGMw2iQmlqOg47PlMO9JsF2uDAGudHLNUbl8RfcKgwUVp4LURjx47lrbfeYt26deh0BUvu3nHHHWzfvt2nxgWCMGuMyHEUwSzDKbkGc033AhC9ZYKS8o+9+Lgevnd+z8X/nGf94UQmLz/k0q6r6XlO95eV2dMVgeX53bJ7W0cjWeoMtRoADcpxnS0P6NMilvAgDQkp2Ww7eS0gNnj9Kd6/fz8PPPCAw/7o6GiuXi2bKwR4g6Vrlmcyk2u0FyPLXLMf9Y+QoK6JJitRKROKvUfkMkbkYtzsYkq2W7sKl6m1oLWZjJmVZyTHENhh2PJKSlYefx1XPr+PsxYu74OgStZ65hUZpbi+Mp/uxwDlFHktRJUqVeLSpUsO+3fv3k3NmjV9YlQgCbVZKqZw98wiRB8P7kKNQXOVnTu/ggv/2p3navje1fh9arb7YuaFJ9xa0NkIUfMJa2k96feAudflmbUHL2M0y9wSnUPUP+8qO3tPgTAnxfIrIJaJsGsPXvbo8+hrvBaigQMHMmbMGC5fvowkSZjNZrZu3corr7zCk08+6Q8bSxW1SiJEp4yGFe6eWWI8apWEVP82aPMYIMPyl1DJBW+eq+F7VzGitBz3b3zha+dtOc3ag5ftPCJQPDmD2bHwv6BoLHPLpmgXgCET4m6Gdk8E2KrSo3WtSJrEhJNrNFu7qKWJ10I0ffp0mjZtSlxcHBkZGTRv3pzu3bvTtWtX3nzzTX/YWOpYh/Bz7QXC4pVYR8XufEtJ+b9ygIanC4rCuXKIXDkqnvwC2QrRgYRUpqw4xH++jUercbyZ5FAdSVAUVzNy2XriKneqdlL/2mZQaZRsetWNE3+TJMkatA7ElA+v/9I6nY4vvviCkydPsmLFChYuXMiRI0f49ttvAzYtw9eEuwhYOwhRaDVrreJmRz6mJsrQr7eZ1Z51zQq2k/KXtQbsc5kExWL1gcsEy9lMC/pW2dHtJYhuFlijAsD9+cX1915I5ejl9FK9d7Elv3bt2tx9990MGDCARo0a+dKmgFMwzaOQR2Tpmtl2vdr9H9TphsaUzRTt14DsdWZ1WrbzBR1tsYv72DTvzCMSeMeKvRcZrfmZKPNVqFwXur8aaJMCQrUwPXc0tRTXL12vyKPM6tGjRzN16lRCQ0PtynA4Y9asWT4xLJBYc4lceER2Ho8kQf8PMH/SlZ7q3fQ17eS01NNpu67yFrM9GOmy7ZrZCp1O7fhb4mp0TuDIlbQcMs/GM0S7RtnR7/0KnTPkjkc6xvH7ofzi+nc1dYhB+guPhGj37t0YDAbrtisCXVzJV9jVJLLB4pU45AlFNeF442docnQuk7Vf8zw344y4KsVPn7cdvre9u7MPipi65jmr9l5guuZL1JIMLR+Ghu4rhVZkbm8SRbUwPVczcvnjSCJ9WpROTWuPhGjjxo1OtysqYe66Zk5iQMcaP4vu8C/UU11haM5C4C6Hc168oyFpOQb6t6rO8EW7uO7FGuS2XTNbHSytX6yKimnH57RWnSZXE47+BsgZcoc2v7j+53+e4qd/L5SaEIlPsRNcdc0so+LOgtGyJog3jU8B0C93JSTEO5wTqtcw/YFWdG1YjeUv3uKVTfYekU3XTGRWF5tL504wMH0BALm3T4DwGDdX3Bg8kr809cajiaVWXN8jj8iyuKIn/PLLL8U2pqwQ6mK+mWWBRY0TIZKAreZW/GK6hQfVW5Syos9sArXzP3GtyiFEh+tJtBkBKwrb1CBbj8jVxFuBe7KXv0p1KYdj2mY07vp0oM0pMzSKCadtXCX2nE/ht90JPNu9gd/v6dHPqe1iihEREWzYsIF//y3IJo6Pj2fDhg1ERkb6zdDSJNwyA9+mJpEsy9Y8IGejYpZd0wyDSJfC4PJ+2PFpkfdx1o6h8CqO+dgGq20vUzvJdRExIg84sor6SX9gkNUc6zT1hsoZ8oQBHZVM6x//vVAq9bI8+uvPnz/f+oqJiWHAgAGcPn2aX375hV9++YVTp04xcOBAqlWrGCU0Q/Mzq22rNNqOnhe1Ssc1Ivk2bJjyn43TIcX1MKizZpqNX8OiHecc9ptddM2cIUbN3JCbgXGFUtbjK3M/unTpHmCDyh7921QnSKviRGIGe86n+P1+Xv8MzJs3j1deecUueVGtVjN69GjmzZtXxJXlh7AgpfytrRDZzvVyNrveVhw2h/SB2l2UqQKrXnXpojgbZTSaZV7/db/Dfrs0IpvLXE2GFRTBphloMi5y3hzFztpPUzVMH2iLyhwRQVruyi+u/+O//q9T5LUQGY1Gjhw54rD/yJEjmCvIHKcwvTOPyCaPx8lfzU5TVCroP1spL3psNRxZ4fQ+rnoDlrluttiNmtnsd+Y2C20qgkt7YbsyYXm8cSh92tYPsEFlF0vQesXei34vru+1EA0dOpRhw4Yxa9YstmzZwpYtW3j//fd5+umnGTp0qD9sLHXC9IpHlOnKI3KzgKJaJUF0U+g2Utmx6jXIdUyZd5WB3TQ23GGf3aiZzXXOZuWXRx2SZdmjciiFMZrMJGc6r9XkgNkEy0eBbGK56Wa2Su1KbXi6PHJz/arUqhxMeq7RoTKor/FaiN577z1ee+013n//fbp370737t2ZNWsWr776Ku+++64/bCx1LAmNtnWrbYXAmRDZ1yPK/1/3V5UpA+kX4Q/HtbBcCZEzTC7yiCpKxY9Jyw7SdeYf/LDTMT5WFOOXHuDm6RvYfe66+5N3fgUXd5GrDmWK4QlubRRFZIjW/XU3KCqVxMP5XpG/6xR5LUQqlYrXXnuNhIQEUlJSSElJISEhgddee63CTHp11jUzmdzEiCQn29pgpcwowD+fQcIuu2tclQXJcuIG255qq4POu2blT50W/H0WgLfXHPXquj3nU8kzmVmw7UzRJ6Zdshaxm6t5giQqF6ziKnDJwx1qIUmw7eQ1zidn+e0+JRqz9Ndqq4HGtmtm+VK784hsfSK74w17KlMHZDOsGKUsXZzP2WvO31hnc8/su2AVr2tmwVUBOFek59dyWn3gctF1ndaMgbx0sqLb8b/UW9BpVPRqJhIY3VGrcghdG1QF4Od4/wWtiyVEP//8MwMGDODmm2+mffv2dq+KQGi+R2Q0y+QalQC8JVgsSe7n1Dl0ufpMh6BIJVC68wuH86tHBtn931m9bJPdpNeC/RWla2bBWyFKyy+hUmRBr6Nr4NBSkNT8EPNfZFT0aBJFeJDolnmCpXrjz34sru+1EH344YcMHTqUmJgYdu/eTadOnahatSqnTp3irrsc51eVR+zKxeZ3z5yWALHBfsnpQueEx0Cvycr2H29BqvLLYplK8txt9pmr2U6WMpJdBKsPXEx1cq5TE8sF3giRLMt23Wenw8x5mbDqFeX8LsOZfzIMyF8uSOARfVsWFNf/+5R/iut7LUSffPIJn3/+OXPmzEGn0/Haa6+xbt06Ro4cSWqq45eiPKJSSQVJjfkBa6clQGxwWzy//WCI6wx5GdZF+374z81Mf6AVT3apY3dqlsHkEOcxuRi+X7nPsX74mgOO+8oL3uRFZeaZrB6hWiWx93yKY0GvTTMh9TxE1uZAw+c5l5xFsFZNz2bRPrS6YhOkVXNPG0W4/VWnyGshOnfuHF27dgUgODiY9HTljX/iiSf4/vvvfWtdALGWAsn/xbWkSDmbZ1YYp6Nh1twijZJXdGQlLWpE8njn2g5dPVmGHIN9TpY3HvGYJQUJkfFnkzl+pXSr7ZUEb1x/S7dMq5bo6ayg1+X98PfHyna/91h2OAWAns2iCdF5NM1SkI9lysfqA/4pru+1EMXGxpKcnAwoVRota5mdPn26XI7WuKJgBr4iRJYJr667Zi6C1bbENIeuLyrbq151mltkIctJ98yCp3/lS6nZPDT3b3p/8KeHVwQebzwiS3pFeJCWR29Svii/7k4gz2i2yxmi+X3QuA+r9iu5MKJb5j1takXSKDqMXKOZFft8X1zfayG64447WLZsGaAkN7788sv07t2bRx991Ol6Z+WVgoUW8z0i2fOuWZGx7O6vQaU6kJYAG2e4PM0yhC/LMq/9vNfumKeCf+aq/4Zb/YU3v2WWEbOIIA23NY4iKlzPtcw8/jiSCPHzIeFf0IVD37dJTMshISUblQTdG1eMOZGliSRJdhNhfY3XQvT555/zxhtvADB8+HDmzZtHs2bNmDJlCnPnzvW5gYGicNfMMinelbdjPyO+CCXShRTkFu2YCxf3OD3NIkTbTyU7vPGeflcr+uRXy3B9eJAWjVrFQ+2V5Lvft++B9fmDA70mQkR1a1C/QVSY6JYVk/vb1bTG4o75uLvvlRAZjUbeeustLl8uSPceOHAgH374IS+++KLdEtTlHcvIWXrhYLUH2dBuz2nUC1o8WJBbZHYcrrd0zZxNX/DYa6jYOmTTNVPeK8tyOHec/QBy06BGe+ioFKvbfyENgFY1K0apmkAQFe6/4vpeCZFGo+Gdd97BaHS/6kR5JyzIedfMVWXWIofvndF3Bugj4eJu2Pmlw2HLJENPCuu7oqLlGBUmLV+IIvLzgRpEhfF07An6q7djRg33/A9UyuinxSNqKYSoRFgmwv66O8Fl7azi4HXXrGfPnmzevNlnBpRVCgerLR6RxsWUedsyIB4MrEF4LPSaoGxvmEoMyXaHM4sQIk8+ABdTsjl1NcMDQ8ovllEzi0dEXhajcj8D4GftPcixraznHkgQQuQLejSNplqYjqsZefxzOtn9BR7idWf5rrvuYuzYsezfv58OHToQGhpqd/zee+/1mXGBxHHUzBKsdnGBtx4RQIenYM/3kPAvE7Xf8IJhlPWQpWuW42Te2YlE9wLTdeYfntlQjrEdNQNg89uEZSdwUa7KpPR7aXDuOs2rR/L3qatcSs1BkqBFjYo3Jak00apVzHiwNXFVgmka67u/pddC9MILLwDO1y+TJAmTyb91S0qLwnWrzW4yq21xNbLm5ESl+/BZd+5W/0NPUzwbzB2Aortmb/52wLP2KzjpOTYe0ZWD8PdHAKyp8wpZx4J4buEuUrMM5OV7kI2jw63vq6D49G7u+zl6XnfNzGazy1dFESGwqVud56PMalfEtoSuIwCYrF1ACDn59y15jKiiUxAjUis5Q2YjNLuHVncMBJSlufNMZmIjgniwfU3efaR1AK0VFIX4eXCBpWtmcf9dLq6Yj0cJja64bQwXtnxHLekqozRLmG4cxOZjSWTmGh1WEhEUYPGI2iT+Bhf+UXKG7nqHmyKq8L+BbcnINdK1QTXqVg2pMIt/VlQ8FqLs7Gw2bNhA//79ARg3bhy5uQVL4ajVaqZOnUpQUJCrJsoVoYUSGotaXLEw3hQ8A0AXypuGoXyte5en1Kv5zdSNP4/Bn8eSvGvnBiM9x0gUKbQ58oGy4443IULJmr6vbc0AWibwFo+7ZgsWLOCzzz6z/v+jjz5i27Zt7N69m927d7Nw4cIKldAY7mLUzGVCo82210IEbDK3Y4WpMxrJzHTtV6ioGPW//UlatoE3tQvRGtKhelvo9EygTRIUE4+F6LvvvuPZZ5+127do0SI2btzIxo0beffdd/nxxx99bmCgKBysditEdqNmxbvnFMOTZBBCW9VJBqnXF6+RcsLhS2kM/26XRyOArmietZP71NuQJZVdzpCg/OGxEJ04cYJWrQryMoKCglDZjGV36tSJQ4cO+da6AOI4xaPozGq7PKJiKlEilfmlspIJ/KrmB6LxoA5zOeX+j7eycv8lhi3YWbwG8rJ4xaB46Gmth0GNtr4zTlDqeCxEKSkpdjGhpKQk6tata/2/2Wy2O17esc0jUlZ59WOMCJh6Xwu6NaxKh4f+yx5zAyKkbCZov/G6nbLEDzvPMWvdMafHLJUvXZXLdYd58zvESYlclKtgvG1csW0UlA08FqJatWpx4IDr/JV9+/ZRq1YtnxhVFrAIkTm/NpB10qtHFRq9v98TXery3dM3UzcqgtcNwzDKKvqrd3C7arf3jZURxizZz4cbjnPQSRVJC57Ud3Ig8TDS33MAmGgYQnhE5eKaKCgjeCxEd999NxMmTCAnJ8fhWHZ2NpMnT6Zfv35e3XzGjBncdNNNhIeHEx0dzf3338/Ro96t4uAvQnRqq7ik5xqso2auMqttv05eD9/bEKrXcE7XkHkmpezuW9r5BOP4Ny9PpBeRgqDTKH9Qj51IsxmWj0IyG1lr6shf6k7WNgTlF4/fwddff53k5GSaNGnCu+++y9KlS1m6dCnvvPMOTZo04fr167z++ute3Xzz5s0MHz6c7du3s27dOgwGA3feeSeZmZleP4ivkSSJMJ1lCN9UkEfkgciUNGclOlzPbONDXJCrUUu6ykuaX0vUXqApqlqARUQ89ox2fwPnt2PShjLJMFgUwK8geJxHFBMTw7Zt23j++ecZO3astTiXJEn07t2bTz75hJgY71K/16xZY/f/r7/+mujoaOLj4+nevbtXbfmDsCAN6flJhQWjZu5dIk+mgRRFlVAdp64GMcEwhHm693havZLfTN04ItcuUbtlEV1+OQMlruamXEBGIqxTJgqfa/Myl7ZUpUGQyMmtCHj1LtarV481a9aQnJzMiRMnAGjYsCFVqlTxiTGW4vuu2svNzbULiKelpfnkvq4ItQlYW4XIhcZ4Pfu+CCxe1x/m9lyv05fKZ9cwXfslD+VNQi7ZUnQBoagCbdp8IfKoO7v2DchJheptOFHnMdiyV3hEFYRifaqrVKlCp06d6NSpk89EyGw2M2rUKLp160bLli2dnjNjxgwiIyOtr7i4OJ/c2xW2I2fuMqvtgtUlVCLbe5zvPIk8dSjtVSd4XF3+Z9QXLnOrz++aufUiT/4B+38ESVmEIDW/Xly48IgqBGXm53X48OEcOHCAxYsXuzxn3LhxpKamWl/nz/t3Pe4CITL4tkKjG2yFyBgWy/GWowAYo1lMVHnMLcrXnvs/3kq9cav4dvtZ6yFLjKhI8TZkw4rRynanZ6Fm+4J61cHCI6oIlAkhGjFiBCtWrGDjxo1FpgDo9XrrMtelsdx1gRCZ3OYRFXv2vRNshUwC5I5Ps9dcnwgpiwnab0vWeACw+EB7zqcAMN6mjIlViGz+Zg6LA/z1Plw/DeE1oIdSLz3dOvNeeEQVgYAKkSzLjBgxgl9//ZU//viDevXqBdIcB2ynebgtA1KS2feFsL1ekiRqR4XzumEYJlniHvV2blPtLeLq8oWzGJHBZCNEiUdgy2xl++53IEj58Smozig8oopAQIVo+PDhLFy4kEWLFhEeHs7ly5e5fPky2dnZgTTLSrhN3eqCUrHuY0QlHb639YhUklKT+aBcj/mmvgBM1cwjiPKTxS7LrpdA0jkRIssacpjNsOJlMBug8V3QtL/1HMvUmzBR6KxCEFAhmjt3Lqmpqdx+++1Ur17d+vrhhx8CaZaVMKejZp7EiEp2X9sC/ZbRuGe712eW8RES5KrUViUxsgS5RbIs88uuC0VmPPuamauPON2vcxKstnpEe76Dc9tAGwp3v2un9um59it4CMo3Ae+aOXsNGTIkkGZZCXUyauZJhcaSds1sC/Rbvnu3N4kiiyAmGQYD8Ix6JY2l4gXr/zx+ldE/7qXfh1tKZKenyMh89ucpp8ecBauNJjNkJMHvbyo7erwOlexHSC1VEYRHVDEoE8Hqsop1Bn6O0YMKjbbbJeyaOREyi2e0ztyRtaaOaCUT07VfIRWjbtGRS/7Nv/IGZ9MzTGZZEaGcFIhtBZ2fA+Dn+AvM2XCc01czrV0z4RFVDMS7WARheqW+jdI1U/Z5kiNU0sxqtZ2o5d/XZt8kw2C6qQ7QUXWMgeqNfG/qWaL7+Zuipnjo8/uhtudIZ/+CfYsBCfr/D9QajlxO45WflCD9/zYct66qEqYXweqKgPCIisDyIbdPaHR1tu8yq23FzuIJ2XpZl6jKLOMjAIzVfE81vIv1lKXyzZZRM0t6hJ48Km14TTnY6Rmopaxq8vXWM9ZrjDYrRwqPqGIghKgIQm08IrO7BRZ9mVltO2qmcmwfYIHpTvab6xIpZfFmGcotOpCQ6lBru6gZZJaumUWIXtAsQ5tyCsKrKzWogeuZefy6OwFwHLUME0JUIRBCVATh+R5RZq6xYIFFlxUaC/BlZrXVIyp0jgk14wxPY5Il7ldv41bVPo/blxxa8w2yLNN/zhaenPcP5zwseGZ5VlmGBlICz6uXKgf6zoQgZVXWxTvPk2s006JGBF0aVLW7PlwEqysEQoiKwC5Y7bZrVoBPu2aS5V/HRg/I9Vlg6gPAW5p56Mkr2Y1LyF/Hr1q3d50rmIriKocIYPOxJG5/dyOJ6TlM085DJ5lIi7sDmt8HKIHrb/8+A8CQrnWteUcWhEdUMRBCVATWrlmeEaOpFDOrC03xUNp3fu77xke4JFehjiqRFwNctygtf/4XwKgf9nh0zemrmZy5lsXD6j+5WXWYLFnP+ZsngySxfO9FFu04y8XUHFQS3NOmhjWmBIrgB2tFwfyKgBCiIrB0zWS5YH0zl8P3ttu+7JpZPCIX52YSbM0tela9gkbSBbft+ytYbXbh+LipMkRl0nhd8x0As40PkhVai4xcI6N+2MP4pQcBJTYXpFWjtRnuD9NrxMKJFQQhREUQpFVZu1mp+XObPPF2Sjx8b3cPx1Gzwqw1d2SdqQM6ycS0YuYW+YKE68WbmvO6ZhFVpAwOm2szz3QXRpNMek5BxQPAqsRam9wGMc+s4iCEqAgkSbJm7roTIl+sa2bB9h6WTWdtBmlV+bPPJSYaBpMp6+mkOsoA9WaP7yXLMqeSMoqM43hCeo6Bt9c4n8ZRlEt0s+oQj2j+xCxLyqIBaDCazWTnmezOszy+bYxIZFVXHIQQucFBiPy4rpn1etsYkWQZNXNs0yzDqF6NAbhINWYZHwZgnGYRVT3MLfrfhuPc8f5mHpq7rUQ2J6R47w3pMDBN8xUAC0292C03AsBokskx2Ht1lj+JbSa2CFRXHIQQucHyYbcIketgdcF2yYfvbdp10r4FWZYZ2q0u60ffRphew9emvhw016GSlMkb2u+cnv/nsSTeWnnYum/2+uMA7DqXgsHkny5dUobzSgHPq5fRQHWJRLkS7xofte43mMxkG+w9IsvfVCs8ogqJECI3WCa+WkaEPFtgsWT3tBs1K6Its6x4TA2jw5CkgtwisyzxoHoL3VT77c5/9ed9PDnvH5ftmVxFmz2gqJ7daz875jjVky7xgkbJGZpseJJ0QqzHXvx+N6ev2q/kYvkz2AmR8IgqDEKI3GD51bVUBPSoDEhJu2ZOEhqdeVlmJ9/+fXIDvjH1Bhxzi36Odz+iVjrITNN8hV4ystHUhpXmznZHc41mFu04a7fP0kXV2QarhUdUYRBC5IbCc5lKpWvmxCNy3jVzfv37xgFclitTT3XF6nX4G28e+UHVX3RVHyJb1jHeOBRnyQkOHpF11Ex0zSoiQojcEKqz/7C7rNBo82Uq8fC92jMhckU6IdbcoufVy2ggJXh0XUnM9nTQrRLp1vjV/4wPckGOdtrdvZ5lsPu/tWsmgtUVEiFEbij8Yfek2+XbGJHrUbOiWGO+iQ2mdujy6xZ5rBR+Zpzme6pK6Rwxx/Gl6W4AQnXus6MlJ8FqkUdUcRBC5IbC7r8nhdF8ua6ZZcttkw46IzHROIQsWU9n1RGl7Kob/DUZ1kJn6TCPajYBWHOGAPQeTNOwDt+LGFGFRAiRGxyEyNWK0z6MEak8jBG544IcxQfGhwCQf3+TKgSuMqMOA9O0Ss7Qd8ae7JIbFxzzYCax0+F70TWrMAghckNoISHybIHFkt3TWRkQ17PNima+qS+HzHWQsq87zS2yxZ/Ttp5Vr6Ch6iJJciRv2+QMQcFqr0VhMCo5TiJYXTERQuSGwqNmrhdY9GFmtdMyIEVf4yoCZETD64ZhmGWJh9R/0UV1sES2FYe60iVe1PwGwBTDE6QRZndc64FHZFm1Q2RWV0yEELnBsWsWmOH7krS5R27IQlMvIBB1i2TlnpKBP02tWG7u4nCGswL6rrALVguPqMIghMgNhbtmniw5XfLZ97btWkbNSsa7xkdJlCvRQHWJ5zXLStia59yn2sot6oPkyFreND6FsyexnVHvDttzhUdUcRBC5AZPR81sKWmsxVfBalvSCWGy4UlAyS2qL10sWYMeEEkG47ULAfjQ+CDn5Bin52k8KXuZj20WgogRVRyEELmh8Ifdk8zqEldodDJ874uh9ZXmzmw0tUEvGZmmmYf7kmWe4yxNaazme6pJaRw11+ILUz+X13rzZLnGgom5hZNNBeUXIURuKOz+u/aIbGsI+VCILAmNPhnRkhhvHEq2rKOL+hAPqf7yRaNO6Sgd4THNRkDJGTIUsYSeN3KYayyYlV/SQQFB2UEIkRs8DVbb4svvh8rDrpmnhc0uyNHMzs8tekO7kMqFcotSsw3WpZO8wdY+LUYlmxtYZOxBvNykyGuLsr3wc9esFOy1bYKyjxAiN+g1Krv5ZR5NevVlYTQPSsV6y1emuzhsjqOKlME4zffW/SeTMmgz+Xc6z9jAmUKTTt1hqyXPqFfQWJVAkhzB28bH3F5blO7ViLQXns71qzLtgZb8+B/H0TdB+UUIkRskSbIbOXM96bUAX2ZWWxr2ZSfEiIY3DMMAGKDZzM2qQwA88LFSpTEpPZfb39tULM+otnSFkfmribxl+D9SC+UMOcNZORMLNSoFOewb1LkOnepV8do2QdlFCJEH2HbPXC6waLucUImFyLbdou9rwVvJ2CU3ZqGxJwDTNF+hw+BQFbEogXBlxVuaeQRJBv4ytWSpuZtHVznTu2phegA61hWCcyMghh08wDa72pMYUUl7UZJd18w3bTrjHeNA+qj/pYHqEs+pl/Oh6UG7497K0L2qv+mu3k9uETlDznAWI5p0b3MOXUzjv3c2QZahfe1KXlojKE8Ij8gDbLtmLie92myXPEZk0661DIjvSSOUKYYnABiuWUo96ZLdcU88IouIaAwpjNd+A8Ac4/2clWM9tsPZffq3rsFrfZuiVkmMvaspd7bwvD1B+UMIkQd41jUr2C5518w2FcBygxI16ZLl5i5sNrVGLxl4q1BuUVE69O32syzdk8Bt725i8vKDRO94mygpjePmmnxu6u+VDbb3qRamZ/K9Lbx8CkF5R3TNPCBM775rZjfptaSZ1U6meJQ0AO4aiTeNQ1mneo1u6oM8YNrCr+ZbAddCdCAhlfG/HbD+f++236mkVzKo3zA8RR7eFSzr3TyGgxfTaFMrkt+GdxOrt96ACI/IAzwRIltK2jWTnE3xKEY7daqGuD8JOC/H8KFRiQ+9qV1IJdIBePjTbeQUCmADXEnLsW5rbHKGFhtv5x+5mVc2Vg7RMqpXY3a83pMlz3cVInSDIoTIA0K9FSJfDt/n4+4L6sx7+eFZz3NtvjD146i5FlWldMbm5xYdvJjGj/+eL/Jez6hX0VR1nmtyODM9yBkqzMqRivcVExHk1ZwzQcVCvPMeYDvNw1X8xzbg6o/h++K0GBvpmIPjCkN+3SKAgZpNdJKURRiz8hw9Isswf5x0hZGaXwAlZyiFcK/s61K/KjVEprQAIUQeYVv3xlW3y1aIpBL+VZ1lVhfXyxrRo6HH58bLTVhk7AHANO08tBgdFl1MzzHw4ve7UXKG5hMs5bHV1IJfzbd4dI9m1SOs2yrx6RPkIz4KHmDXNXPpERVsl7RrZp+l7WSnE2QXWT8No91nNtvytvExkuQIGqkSeFa9wiG7+p/TyQD0V23nNvU+cmWNxzlDS57vyuqXbrX+39/F+gXlByFEHhDmUUKj77pm9sFq382+//k59zGjVMKYmp9bNFLzK6GZ5+yOT1h6kAgymaD9FoCPjfdzWq5eZJu/vtCVMzP70aFOZbv9Ii4tsCCEyAPC9AXL3bgSIlvHoeSF0Wzasu4r+be2Y90qnJp+N8NuqVfkecvMXfnL1BK9ZKDRzgnW6PTFlGwSUrJ5VfMD0VIKJ83V+dR0j8t2grVqTs+4m3a1K7s8RyAAIUQeEaYvyItxLUSy23M8xb4eUf6/xWyrsH6pVBLj+zfnzMx+LHiqk6ureNP4FLmyllvVBzj4uzI8/97ao7SXjjFIvQGA1w1Pu8wZWvJ8Fw5P7VvkaJ8YqhdYKBNC9PHHH1O3bl2CgoLo3Lkz//zzT6BNsiPUxiNy5ZnIvowR+alrVpjbGkcx9T7nWcxn5Vg+ND4AQPS2yaQlJ7Js91mmab9CJcn8ZOzODhc5Q8en3UWHOu4nqzbyMn4lqLgEXIh++OEHRo8ezcSJE9m1axdt2rShT58+JCYmBto0K+FeekQlzqx2cr3fArs2Cnd3K/v5XJ+b+nPcXJMoKY0Vs57lKfVqmqnOkyyHMd34uN25U+9vyZmZ/Tgzs5/b5YF+eaErw26px8u9Gxd5nuDGIeBCNGvWLJ555hmGDh1K8+bN+fTTTwkJCWHevHmBNs2KrUfkKhAt28WIfOcRFewrUZMe8cmgDpycfrf1/7a5RY9rNvK6Vkl0nGb4P66jDMOH6tTsHt+bJ26u4/F92teuzPj+zUXxe4GVgApRXl4e8fHx9OrVy7pPpVLRq1cv/v77b4fzc3NzSUtLs3uVBhHBWsL0GkJ0aoJ0zv9k0eF6n90vRFf0WvChTo67mhfmrptY+Ghhj2+n3JTFxtut///b1Jwl+XPRVo68hYNT+lI5VFfkPQQCdwT0J+nq1auYTCZiYuyXmYmJieHIkSMO58+YMYPJkyeXlnlWtGoVi57pjMkso9c4F4noiCAWPNXJJ7/yjaLDeOLmOtbiYABBWjUv3N4Ag8nMozfF8fqvB3ipZyPr8ZkPteLlH/Zyc/0qnEzKZPoDrQBlQmnT2HCXI1e3N4lS7LcR0hE9GvLRxhNMe6Alb/x6gBnGx+mh3kM42bxpHMqXT95Er+bOlwYSCIqDJHtadd0PXLx4kZo1a7Jt2za6dCnIcXnttdfYvHkzO3bssDs/NzeX3Nxc6//T0tKIi4sjNTWViIgIbnRSsw1EBmuRZdmr7mFSei7hQRqCtIrIyrLM1Yw8osL1ZOUZ2XMuBU3uNXSygbYtW/rLfEEFJC0tjcjISLff0YB6RNWqVUOtVnPlyhW7/VeuXCE21rEQll6vR6/3XReoohEZrATVvY1RRRXqVkqSZN0XotPQtWE1oJpPbBQInBHQGJFOp6NDhw5s2LDBus9sNrNhwwY7D0kgEFRsAj5sMXr0aAYPHkzHjh3p1KkTs2fPJjMzk6FDhwbaNIFAUEoEXIgeffRRkpKSmDBhApcvX6Zt27asWbPGIYAtEAgqLgENVpcUTwNhAoEgMHj6HQ14QqNAIBAIIRIIBAFHCJFAIAg4AQ9WlwRLeKu0pnoIBALvsHw33YWiy7UQpacry97ExcUF2BKBQFAU6enpREZGujxerkfNzGYzFy9eJDw8vMwV2bJMPzl//nyFGNETz1O2KavPI8sy6enp1KhRA1URqyWUa49IpVJRq1atQJtRJBEREWXqg1FSxPOUbcri8xTlCVkQwWqBQBBwhBAJBIKAI4TIT+j1eiZOnFhhqgWI5ynblPfnKdfBaoFAUDEQHpFAIAg4QogEAkHAEUIkEAgCjhAigUAQcIQQFZPk5GQGDRpEREQElSpVYtiwYWRkZBR5/osvvkiTJk0IDg6mdu3ajBw5ktTUVLvzJElyeC1evNgvz+DtCrs//fQTTZs2JSgoiFatWrFq1Sq747IsM2HCBKpXr05wcDC9evXi+PHjfrHdGd48zxdffMGtt95K5cqVqVy5Mr169XI4f8iQIQ7vRd++ff39GFa8eZ6vv/7awdagoCC7cwL9/hSJLCgWffv2ldu0aSNv375d/uuvv+SGDRvKjz32mMvz9+/fLz/44IPysmXL5BMnTsgbNmyQGzVqJD/00EN25wHy/Pnz5UuXLllf2dnZPrd/8eLFsk6nk+fNmycfPHhQfuaZZ+RKlSrJV65ccXr+1q1bZbVaLb/zzjvyoUOH5DfffFPWarXy/v37refMnDlTjoyMlH/77Td579698r333ivXq1fPL/aX9Hkef/xx+eOPP5Z3794tHz58WB4yZIgcGRkpX7hwwXrO4MGD5b59+9q9F8nJyX5/luI8z/z58+WIiAg7Wy9fvmx3TiDfH3cIISoGhw4dkgF5586d1n2rV6+WJUmSExISPG7nxx9/lHU6nWwwGKz7APnXX3/1pblO6dSpkzx8+HDr/00mk1yjRg15xowZTs8fMGCA3K9fP7t9nTt3lv/zn//IsizLZrNZjo2Nld99913r8ZSUFFmv18vff/+9H57AHm+fpzBGo1EODw+XFyxYYN03ePBg+b777vO1qR7h7fPMnz9fjoyMdNleoN8fd4iuWTH4+++/qVSpEh07drTu69WrFyqVymEttqKwlM/UaOyn/A0fPpxq1arRqVMn5s2b57aEgrd4u8IuKM9sez5Anz59rOefPn2ay5cv250TGRlJ586dXbbpK4rzPIXJysrCYDBQpUoVu/2bNm0iOjqaJk2a8Pzzz3Pt2jWf2u6M4j5PRkYGderUIS4ujvvuu4+DBw9ajwXy/fEEIUTF4PLly0RHR9vt02g0VKlShcuXL3vUxtWrV5k6dSrPPvus3f4pU6bw448/sm7dOh566CFeeOEF5syZ4zPbLfd2tcKuK/svX75c5PmWf71p01cU53kKM2bMGGrUqGH3Re3bty/ffPMNGzZs4O2332bz5s3cddddmEwmn9pfmOI8T5MmTZg3bx5Lly5l4cKFmM1munbtyoULF4DAvj+eUK5n3/uasWPH8vbbbxd5zuHDh0t8n7S0NPr160fz5s2ZNGmS3bHx48dbt9u1a0dmZibvvvsuI0eOLPF9Bc6ZOXMmixcvZtOmTXYB3oEDB1q3W7VqRevWrWnQoAGbNm2iZ8+egTDVJV26dLFbC7Br1640a9aMzz77jKlTpwbQMs8QHpEN//3vfzl8+HCRr/r16xMbG0tiYqLdtUajkeTkZKcr1NqSnp5O3759CQ8P59dff0Wr1RZ5fufOnblw4YLdUtslxdsVdgFiY2OLPN/yrzdt+oriPI+F9957j5kzZ/L777/TunXrIs+tX78+1apV48SJEyW2uShK8jwWtFot7dq1s9oayPfHE4QQ2RAVFUXTpk2LfOl0Orp06UJKSgrx8fHWa//44w/MZjOdO3d22X5aWhp33nknOp2OZcuWOQyvOmPPnj1UrlzZp5MZi7PCbpcuXezOB1i3bp31/Hr16hEbG2t3TlpaGjt27PD7qr3FXTH4nXfeYerUqaxZs8Yu3ueKCxcucO3aNapXr+4Tu13hixWQTSYT+/fvt9oayPfHIwIdLS+v9O3bV27Xrp28Y8cOecuWLXKjRo3shu8vXLggN2nSRN6xY4csy7Kcmpoqd+7cWW7VqpV84sQJu2FWo9Eoy7IsL1u2TP7iiy/k/fv3y8ePH5c/+eQTOSQkRJ4wYYLP7V+8eLGs1+vlr7/+Wj506JD87LPPypUqVbIO+T7xxBPy2LFjredv3bpV1mg08nvvvScfPnxYnjhxotPh+0qVKslLly6V9+3bJ993332lOnzvzfPMnDlT1ul08s8//2z3XqSnp8uyLMvp6enyK6+8Iv/999/y6dOn5fXr18vt27eXGzVqJOfk5JS555k8ebK8du1a+eTJk3J8fLw8cOBAOSgoSD548KDdMwfq/XGHEKJicu3aNfmxxx6Tw8LC5IiICHno0KHWD7Esy/Lp06dlQN64caMsy7K8ceNGGXD6On36tCzLSgpA27Zt5bCwMDk0NFRu06aN/Omnn8omk8kvzzBnzhy5du3ask6nkzt16iRv377deuy2226TBw8ebHf+jz/+KDdu3FjW6XRyixYt5JUrV9odN5vN8vjx4+WYmBhZr9fLPXv2lI8ePeoX253hzfPUqVPH6XsxceJEWZZlOSsrS77zzjvlqKgoWavVynXq1JGfeeYZh9ycsvI8o0aNsp4bExMj33333fKuXbvs2gv0+1MUogyIQCAIOCJGJBAIAo4QIoFAEHCEEAkEgoAjhEggEAQcIUQCgSDgCCESCAQBRwiRQCAIOEKIBAJBwBFCJPA5zkqslnaZVUH5QpQBEfiFvn37Mn/+fLt9ribuGgwGhyoEeXl56HQ6r+9b3OsEgUV4RAK/oNfriY2NtXtVrlwZUBYImDt3Lvfeey+hoaFMmzaNSZMm0bZtW7788kvq1atnrUxw7tw57rvvPsLCwoiIiGDAgAF2pSxcXScoXwghEgSESZMm8cADD7B//36eeuopAE6cOMGSJUv45Zdf2LNnD2azmfvuu4/k5GQ2b97MunXrOHXqFI8++qhdW4WvE5Q/RNdM4BdWrFhBWFiY3b7XX3+d119/HYDHH3+coUOH2h3Py8vjm2++ISoqClDqHe3fv5/Tp08TFxcHwDfffEOLFi3YuXMnN910k9PrBOUPIUQCv9CjRw/mzp1rt8+2ML2zQmR16tSxE5PDhw8TFxdnFSGA5s2bU6lSJQ4fPmwVosLXCcofQogEfiE0NJSGDRsWedyTfZ7eS1C+ETEiQZmlWbNmnD9/nvPnz1v3HTp0iJSUFJo3bx5AywS+RnhEAr+Qm5vrsEyNRqOhWrVqHrfRq1cvWrVqxaBBg5g9ezZGo5EXXniB2267zaMa04Lyg/CIBH5hzZo1VK9e3e51yy23eNWGJEksXbqUypUr0717d3r16kX9+vX54Ycf/GS1IFCIUrECgSDgCI9IIBAEHCFEAoEg4AghEggEAUcIkUAgCDhCiAQCQcARQiQQCAKOECKBQBBwhBAJBIKAI4RIIBAEHCFEAoEg4AghEggEAUcIkUAgCDj/D3FdD0L2Dy0dAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018036, + "end_time": "2024-03-22T22:52:23.872486", + "exception": false, + "start_time": "2024-03-22T22:52:23.854450", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5786.510322, + "end_time": "2024-03-22T22:52:26.614799", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/realtabformer/4/mlu-eval.ipynb", + "output_path": "eval/treatment/realtabformer/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/realtabformer/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "realtabformer" + }, + "start_time": "2024-03-22T21:16:00.104477", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/realtabformer/model.pt b/treatment/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a74fdcb2f7bfb883057694fb42b95c22ae8da74 --- /dev/null +++ b/treatment/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca83017195316a2f5210cdd5f311db787ee889cbbb92e8fd4dad4f7a666bb2b7 +size 78481207 diff --git a/treatment/realtabformer/params.json b/treatment/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..1e1f2cf6337c15d995e431b71250bf5cc2a95f0a --- /dev/null +++ b/treatment/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 0.2, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/eval.csv b/treatment/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..d616bf1bee44661b59d7c4f777f4cc012c9da831 --- /dev/null +++ b/treatment/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.00041383573696726845,0.0073552794379970025,0.004348100851493655,11.387780904769897,0.032667260617017746,0.8455857038497925,0.05075574293732643,5.348016566131264e-05,6.293136358261108,0.041026778519153595,7420750.5,0.06594013422727585,0.23463046550750732,0.0012389702023938298,17.680917263031006 diff --git a/treatment/tab_ddpm_concat/history.csv b/treatment/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d781dbb982f76817bde317321ae4970ce135ed0 --- /dev/null +++ b/treatment/tab_ddpm_concat/history.csv @@ -0,0 +1,8 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.0695428019142936,28.1492625863294,0.012808751637893953,0.003821606215917402,0.0,0.0,0.0,0.0,0.3170081154054949,900,225,389.3622844219208,1.7304990418752035,0.4326247604688009,0.12492922414161714,0.009141866957506769,3.069006191681122,0.00027733774792486793,0.0,0.0,0.0,0.0,0.0,0.009141866957506769,450,113,97.50238251686096,0.8628529426270881,0.21667196114857992,0.09880975214535895 +1,0.010684975302428939,0.2685297108324734,0.0004185057601977565,0.004228238970455196,0.0,0.0,0.0,0.0,0.013845855410165515,900,225,393.82890224456787,1.7503506766425239,0.43758766916063097,0.2299761895918184,0.010794387987136922,3.2538266676528425,0.0005808094720025207,0.0,0.0,0.0,0.0,0.0,0.010794387987136922,450,113,102.51186180114746,0.9071846177092696,0.22780413733588326,0.09256724538113505 +2,0.008766504123186173,0.18838597702945958,0.0010075531042212967,0.004310787094161949,0.0,0.0,0.0,0.0,0.018823233489600473,900,225,399.25857162475586,1.7744825405544704,0.4436206351386176,0.23340816134845632,0.01156254135871633,4.513857759436349,0.0010884478876808565,0.0,0.0,0.0,0.0,0.0,0.01156254135871633,450,113,101.14439296722412,0.8950831236037533,0.22476531770494249,0.10850579791203765 +3,0.00745334779791051,0.18830440280404173,0.00034807503856150413,0.0012704849093238915,0.0,0.0,0.0,0.0,0.012097826507647874,900,225,397.8769977092743,1.768342212041219,0.4420855530103048,0.23778708743138446,0.008989941240149494,5.070133002356837,0.00031610333040396756,0.0,0.0,0.0,0.0,0.0,0.008989941240149494,450,113,101.03075122833252,0.8940774444985179,0.2245127805074056,0.10478938162526322 +4,0.007256789765557793,0.1903947035916835,0.0005144578501850197,0.001130848757456988,0.0,0.0,0.0,0.0,0.01587584275592336,900,225,388.8551299571991,1.728245022031996,0.432061255507999,0.23386404902156857,0.0069510249215774264,2.9996348747551957,0.00017168078157728811,0.0,0.0,0.0,0.0,0.0,0.0069510249215774264,450,113,94.60210680961609,0.8371867859258061,0.2102269040213691,0.10559021631811487 +5,0.006384113563660776,0.29707657504114143,0.0002236975578378892,0.0011714805286222449,0.0,0.0,0.0,0.0,0.008635841717189629,900,225,390.0636169910431,1.733616075515747,0.43340401887893676,0.23355899076268544,0.010469536527711322,4.7843355605484685,0.00036496384345347286,0.0,0.0,0.0,0.0,0.0,0.010469536527711322,450,113,94.51667761802673,0.8364307753807676,0.21003706137339273,0.10331984527890432 +6,0.006239299076195392,0.22657988847652227,0.0001620542119322938,0.001147383804442749,0.0,0.0,0.0,0.0,0.009122197522083297,900,225,386.9022469520569,1.7195655420091418,0.42989138550228545,0.23747530813432402,0.007029949970184804,3.5702621474726217,0.00015399106053258602,0.0,0.0,0.0,0.0,0.0,0.007029949970184804,450,113,94.76029849052429,0.8385867123055247,0.21057844109005397,0.10414575381696321 diff --git a/treatment/tab_ddpm_concat/mlu-eval.ipynb b/treatment/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..654c3367e81ea697b536ab0ea543d3d65b4913dc --- /dev/null +++ b/treatment/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2347 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.273703Z", + "iopub.status.busy": "2024-03-23T09:20:33.272791Z", + "iopub.status.idle": "2024-03-23T09:20:33.308576Z", + "shell.execute_reply": "2024-03-23T09:20:33.307799Z" + }, + "papermill": { + "duration": 0.051185, + "end_time": "2024-03-23T09:20:33.310822", + "exception": false, + "start_time": "2024-03-23T09:20:33.259637", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.340215Z", + "iopub.status.busy": "2024-03-23T09:20:33.339740Z", + "iopub.status.idle": "2024-03-23T09:20:33.347271Z", + "shell.execute_reply": "2024-03-23T09:20:33.346295Z" + }, + "papermill": { + "duration": 0.024909, + "end_time": "2024-03-23T09:20:33.349556", + "exception": false, + "start_time": "2024-03-23T09:20:33.324647", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.377513Z", + "iopub.status.busy": "2024-03-23T09:20:33.377231Z", + "iopub.status.idle": "2024-03-23T09:20:33.381835Z", + "shell.execute_reply": "2024-03-23T09:20:33.380804Z" + }, + "papermill": { + "duration": 0.021176, + "end_time": "2024-03-23T09:20:33.384009", + "exception": false, + "start_time": "2024-03-23T09:20:33.362833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.411337Z", + "iopub.status.busy": "2024-03-23T09:20:33.411009Z", + "iopub.status.idle": "2024-03-23T09:20:33.415391Z", + "shell.execute_reply": "2024-03-23T09:20:33.414433Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.020654, + "end_time": "2024-03-23T09:20:33.417596", + "exception": false, + "start_time": "2024-03-23T09:20:33.396942", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.442400Z", + "iopub.status.busy": "2024-03-23T09:20:33.442132Z", + "iopub.status.idle": "2024-03-23T09:20:33.447753Z", + "shell.execute_reply": "2024-03-23T09:20:33.446840Z" + }, + "papermill": { + "duration": 0.01998, + "end_time": "2024-03-23T09:20:33.449630", + "exception": false, + "start_time": "2024-03-23T09:20:33.429650", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3611abb0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.472811Z", + "iopub.status.busy": "2024-03-23T09:20:33.472567Z", + "iopub.status.idle": "2024-03-23T09:20:33.477160Z", + "shell.execute_reply": "2024-03-23T09:20:33.476326Z" + }, + "papermill": { + "duration": 0.018538, + "end_time": "2024-03-23T09:20:33.479036", + "exception": false, + "start_time": "2024-03-23T09:20:33.460498", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tab_ddpm_concat/2\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010897, + "end_time": "2024-03-23T09:20:33.500938", + "exception": false, + "start_time": "2024-03-23T09:20:33.490041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.524436Z", + "iopub.status.busy": "2024-03-23T09:20:33.523632Z", + "iopub.status.idle": "2024-03-23T09:20:33.532665Z", + "shell.execute_reply": "2024-03-23T09:20:33.531895Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022772, + "end_time": "2024-03-23T09:20:33.534571", + "exception": false, + "start_time": "2024-03-23T09:20:33.511799", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tab_ddpm_concat/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:33.559479Z", + "iopub.status.busy": "2024-03-23T09:20:33.559222Z", + "iopub.status.idle": "2024-03-23T09:20:35.620506Z", + "shell.execute_reply": "2024-03-23T09:20:35.619544Z" + }, + "papermill": { + "duration": 2.07631, + "end_time": "2024-03-23T09:20:35.622702", + "exception": false, + "start_time": "2024-03-23T09:20:33.546392", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:35.649196Z", + "iopub.status.busy": "2024-03-23T09:20:35.648410Z", + "iopub.status.idle": "2024-03-23T09:20:35.665502Z", + "shell.execute_reply": "2024-03-23T09:20:35.664624Z" + }, + "papermill": { + "duration": 0.033322, + "end_time": "2024-03-23T09:20:35.667543", + "exception": false, + "start_time": "2024-03-23T09:20:35.634221", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:35.691803Z", + "iopub.status.busy": "2024-03-23T09:20:35.691229Z", + "iopub.status.idle": "2024-03-23T09:20:35.698842Z", + "shell.execute_reply": "2024-03-23T09:20:35.698003Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021831, + "end_time": "2024-03-23T09:20:35.700733", + "exception": false, + "start_time": "2024-03-23T09:20:35.678902", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:35.726607Z", + "iopub.status.busy": "2024-03-23T09:20:35.726324Z", + "iopub.status.idle": "2024-03-23T09:20:35.853846Z", + "shell.execute_reply": "2024-03-23T09:20:35.853112Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.143283, + "end_time": "2024-03-23T09:20:35.856063", + "exception": false, + "start_time": "2024-03-23T09:20:35.712780", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:35.880915Z", + "iopub.status.busy": "2024-03-23T09:20:35.880635Z", + "iopub.status.idle": "2024-03-23T09:20:40.913376Z", + "shell.execute_reply": "2024-03-23T09:20:40.912483Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 5.047872, + "end_time": "2024-03-23T09:20:40.915761", + "exception": false, + "start_time": "2024-03-23T09:20:35.867889", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-23 09:20:38.182216: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-23 09:20:38.182272: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-23 09:20:38.183859: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:40.943725Z", + "iopub.status.busy": "2024-03-23T09:20:40.942702Z", + "iopub.status.idle": "2024-03-23T09:20:40.950102Z", + "shell.execute_reply": "2024-03-23T09:20:40.949229Z" + }, + "papermill": { + "duration": 0.023338, + "end_time": "2024-03-23T09:20:40.952164", + "exception": false, + "start_time": "2024-03-23T09:20:40.928826", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:20:40.976220Z", + "iopub.status.busy": "2024-03-23T09:20:40.975935Z", + "iopub.status.idle": "2024-03-23T09:21:02.489500Z", + "shell.execute_reply": "2024-03-23T09:21:02.488466Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 21.528288, + "end_time": "2024-03-23T09:21:02.491947", + "exception": false, + "start_time": "2024-03-23T09:20:40.963659", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-23T09:21:02.520373Z", + "iopub.status.busy": "2024-03-23T09:21:02.520010Z", + "iopub.status.idle": "2024-03-23T09:21:02.526560Z", + "shell.execute_reply": "2024-03-23T09:21:02.525718Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023294, + "end_time": "2024-03-23T09:21:02.528438", + "exception": false, + "start_time": "2024-03-23T09:21:02.505144", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:02.554208Z", + "iopub.status.busy": "2024-03-23T09:21:02.553492Z", + "iopub.status.idle": "2024-03-23T09:21:02.558135Z", + "shell.execute_reply": "2024-03-23T09:21:02.557271Z" + }, + "papermill": { + "duration": 0.019428, + "end_time": "2024-03-23T09:21:02.559972", + "exception": false, + "start_time": "2024-03-23T09:21:02.540544", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:02.585563Z", + "iopub.status.busy": "2024-03-23T09:21:02.585297Z", + "iopub.status.idle": "2024-03-23T09:21:04.576662Z", + "shell.execute_reply": "2024-03-23T09:21:04.575789Z" + }, + "papermill": { + "duration": 2.006609, + "end_time": "2024-03-23T09:21:04.578856", + "exception": false, + "start_time": "2024-03-23T09:21:02.572247", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:04.607855Z", + "iopub.status.busy": "2024-03-23T09:21:04.607563Z", + "iopub.status.idle": "2024-03-23T09:21:04.926797Z", + "shell.execute_reply": "2024-03-23T09:21:04.925904Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.335597, + "end_time": "2024-03-23T09:21:04.928730", + "exception": false, + "start_time": "2024-03-23T09:21:04.593133", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:04.956945Z", + "iopub.status.busy": "2024-03-23T09:21:04.956673Z", + "iopub.status.idle": "2024-03-23T09:21:05.060877Z", + "shell.execute_reply": "2024-03-23T09:21:05.059814Z" + }, + "papermill": { + "duration": 0.120838, + "end_time": "2024-03-23T09:21:05.063125", + "exception": false, + "start_time": "2024-03-23T09:21:04.942287", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-23T09:21:05.092115Z", + "iopub.status.busy": "2024-03-23T09:21:05.091800Z", + "iopub.status.idle": "2024-03-23T09:21:05.602327Z", + "shell.execute_reply": "2024-03-23T09:21:05.601413Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.527293, + "end_time": "2024-03-23T09:21:05.604244", + "exception": false, + "start_time": "2024-03-23T09:21:05.076951", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:05.634189Z", + "iopub.status.busy": "2024-03-23T09:21:05.633862Z", + "iopub.status.idle": "2024-03-23T09:21:05.637975Z", + "shell.execute_reply": "2024-03-23T09:21:05.637113Z" + }, + "papermill": { + "duration": 0.021684, + "end_time": "2024-03-23T09:21:05.639981", + "exception": false, + "start_time": "2024-03-23T09:21:05.618297", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:05.666937Z", + "iopub.status.busy": "2024-03-23T09:21:05.666671Z", + "iopub.status.idle": "2024-03-23T09:21:05.673592Z", + "shell.execute_reply": "2024-03-23T09:21:05.672720Z" + }, + "papermill": { + "duration": 0.022666, + "end_time": "2024-03-23T09:21:05.675478", + "exception": false, + "start_time": "2024-03-23T09:21:05.652812", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18616321" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:05.702787Z", + "iopub.status.busy": "2024-03-23T09:21:05.702532Z", + "iopub.status.idle": "2024-03-23T09:21:05.795881Z", + "shell.execute_reply": "2024-03-23T09:21:05.795082Z" + }, + "papermill": { + "duration": 0.109166, + "end_time": "2024-03-23T09:21:05.797796", + "exception": false, + "start_time": "2024-03-23T09:21:05.688630", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 12] --\n", + "├─Adapter: 1-1 [2, 2648, 12] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 13,312\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,616,321\n", + "Trainable params: 18,616,321\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.71\n", + "========================================================================================================================\n", + "Input size (MB): 0.32\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.47\n", + "Estimated Total Size (MB): 1154.27\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T09:21:05.828508Z", + "iopub.status.busy": "2024-03-23T09:21:05.827985Z", + "iopub.status.idle": "2024-03-23T10:29:23.404054Z", + "shell.execute_reply": "2024-03-23T10:29:23.403245Z" + }, + "papermill": { + "duration": 4097.593681, + "end_time": "2024-03-23T10:29:23.406177", + "exception": false, + "start_time": "2024-03-23T09:21:05.812496", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0695428019142936, 'avg_role_model_std_loss': 28.1492625863294, 'avg_role_model_mean_pred_loss': 0.012808751637893953, 'avg_role_model_g_mag_loss': 0.003821606215917402, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.3170081154054949, 'n_size': 900, 'n_batch': 225, 'duration': 389.3622844219208, 'duration_batch': 1.7304990418752035, 'duration_size': 0.4326247604688009, 'avg_pred_std': 0.12492922414161714}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009141866957506769, 'avg_role_model_std_loss': 3.069006191681122, 'avg_role_model_mean_pred_loss': 0.00027733774792486793, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009141866957506769, 'n_size': 450, 'n_batch': 113, 'duration': 97.50238251686096, 'duration_batch': 0.8628529426270881, 'duration_size': 0.21667196114857992, 'avg_pred_std': 0.09880975214535895}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010684975302428939, 'avg_role_model_std_loss': 0.2685297108324734, 'avg_role_model_mean_pred_loss': 0.0004185057601977565, 'avg_role_model_g_mag_loss': 0.004228238970455196, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013845855410165515, 'n_size': 900, 'n_batch': 225, 'duration': 393.82890224456787, 'duration_batch': 1.7503506766425239, 'duration_size': 0.43758766916063097, 'avg_pred_std': 0.2299761895918184}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010794387987136922, 'avg_role_model_std_loss': 3.2538266676528425, 'avg_role_model_mean_pred_loss': 0.0005808094720025207, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010794387987136922, 'n_size': 450, 'n_batch': 113, 'duration': 102.51186180114746, 'duration_batch': 0.9071846177092696, 'duration_size': 0.22780413733588326, 'avg_pred_std': 0.09256724538113505}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008766504123186173, 'avg_role_model_std_loss': 0.18838597702945958, 'avg_role_model_mean_pred_loss': 0.0010075531042212967, 'avg_role_model_g_mag_loss': 0.004310787094161949, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.018823233489600473, 'n_size': 900, 'n_batch': 225, 'duration': 399.25857162475586, 'duration_batch': 1.7744825405544704, 'duration_size': 0.4436206351386176, 'avg_pred_std': 0.23340816134845632}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01156254135871633, 'avg_role_model_std_loss': 4.513857759436349, 'avg_role_model_mean_pred_loss': 0.0010884478876808565, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01156254135871633, 'n_size': 450, 'n_batch': 113, 'duration': 101.14439296722412, 'duration_batch': 0.8950831236037533, 'duration_size': 0.22476531770494249, 'avg_pred_std': 0.10850579791203765}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00745334779791051, 'avg_role_model_std_loss': 0.18830440280404173, 'avg_role_model_mean_pred_loss': 0.00034807503856150413, 'avg_role_model_g_mag_loss': 0.0012704849093238915, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012097826507647874, 'n_size': 900, 'n_batch': 225, 'duration': 397.8769977092743, 'duration_batch': 1.768342212041219, 'duration_size': 0.4420855530103048, 'avg_pred_std': 0.23778708743138446}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008989941240149494, 'avg_role_model_std_loss': 5.070133002356837, 'avg_role_model_mean_pred_loss': 0.00031610333040396756, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008989941240149494, 'n_size': 450, 'n_batch': 113, 'duration': 101.03075122833252, 'duration_batch': 0.8940774444985179, 'duration_size': 0.2245127805074056, 'avg_pred_std': 0.10478938162526322}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007256789765557793, 'avg_role_model_std_loss': 0.1903947035916835, 'avg_role_model_mean_pred_loss': 0.0005144578501850197, 'avg_role_model_g_mag_loss': 0.001130848757456988, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01587584275592336, 'n_size': 900, 'n_batch': 225, 'duration': 388.8551299571991, 'duration_batch': 1.728245022031996, 'duration_size': 0.432061255507999, 'avg_pred_std': 0.23386404902156857}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0069510249215774264, 'avg_role_model_std_loss': 2.9996348747551957, 'avg_role_model_mean_pred_loss': 0.00017168078157728811, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0069510249215774264, 'n_size': 450, 'n_batch': 113, 'duration': 94.60210680961609, 'duration_batch': 0.8371867859258061, 'duration_size': 0.2102269040213691, 'avg_pred_std': 0.10559021631811487}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006384113563660776, 'avg_role_model_std_loss': 0.29707657504114143, 'avg_role_model_mean_pred_loss': 0.0002236975578378892, 'avg_role_model_g_mag_loss': 0.0011714805286222449, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008635841717189629, 'n_size': 900, 'n_batch': 225, 'duration': 390.0636169910431, 'duration_batch': 1.733616075515747, 'duration_size': 0.43340401887893676, 'avg_pred_std': 0.23355899076268544}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010469536527711322, 'avg_role_model_std_loss': 4.7843355605484685, 'avg_role_model_mean_pred_loss': 0.00036496384345347286, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010469536527711322, 'n_size': 450, 'n_batch': 113, 'duration': 94.51667761802673, 'duration_batch': 0.8364307753807676, 'duration_size': 0.21003706137339273, 'avg_pred_std': 0.10331984527890432}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006239299076195392, 'avg_role_model_std_loss': 0.22657988847652227, 'avg_role_model_mean_pred_loss': 0.0001620542119322938, 'avg_role_model_g_mag_loss': 0.001147383804442749, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009122197522083297, 'n_size': 900, 'n_batch': 225, 'duration': 386.9022469520569, 'duration_batch': 1.7195655420091418, 'duration_size': 0.42989138550228545, 'avg_pred_std': 0.23747530813432402}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007029949970184804, 'avg_role_model_std_loss': 3.5702621474726217, 'avg_role_model_mean_pred_loss': 0.00015399106053258602, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007029949970184804, 'n_size': 450, 'n_batch': 113, 'duration': 94.76029849052429, 'duration_batch': 0.8385867123055247, 'duration_size': 0.21057844109005397, 'avg_pred_std': 0.10414575381696321}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005995685436751551, 'avg_role_model_std_loss': 0.14638709697454766, 'avg_role_model_mean_pred_loss': 0.0001228724541547233, 'avg_role_model_g_mag_loss': 0.0011365799427342912, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01081854196954939, 'n_size': 900, 'n_batch': 225, 'duration': 387.2872395515442, 'duration_batch': 1.7212766202290852, 'duration_size': 0.4303191550572713, 'avg_pred_std': 0.23766291744179197}\n", + "Time out: 3827.227990627289/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0043481008274275585, 'avg_g_mag_loss': 0.010324255653263465, 'avg_g_cos_loss': 0.0007838808940280051, 'pred_duration': 6.3304314613342285, 'grad_duration': 11.364734411239624, 'total_duration': 17.695165872573853, 'pred_std': 0.23463046550750732, 'std_loss': 0.0012389702023938298, 'mean_pred_loss': 5.348016202333383e-05, 'pred_rmse': 0.06594012677669525, 'pred_mae': 0.041026778519153595, 'pred_mape': 7420750.0, 'grad_rmse': 0.05075574293732643, 'grad_mae': 0.032667260617017746, 'grad_mape': 0.8455857038497925}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0043481008274275585, 'avg_g_mag_loss': 0.010324255653263465, 'avg_g_cos_loss': 0.0007838808940280051, 'avg_pred_duration': 6.3304314613342285, 'avg_grad_duration': 11.364734411239624, 'avg_total_duration': 17.695165872573853, 'avg_pred_std': 0.23463046550750732, 'avg_std_loss': 0.0012389702023938298, 'avg_mean_pred_loss': 5.348016202333383e-05}, 'min_metrics': {'avg_loss': 0.0043481008274275585, 'avg_g_mag_loss': 0.010324255653263465, 'avg_g_cos_loss': 0.0007838808940280051, 'pred_duration': 6.3304314613342285, 'grad_duration': 11.364734411239624, 'total_duration': 17.695165872573853, 'pred_std': 0.23463046550750732, 'std_loss': 0.0012389702023938298, 'mean_pred_loss': 5.348016202333383e-05, 'pred_rmse': 0.06594012677669525, 'pred_mae': 0.041026778519153595, 'pred_mape': 7420750.0, 'grad_rmse': 0.05075574293732643, 'grad_mae': 0.032667260617017746, 'grad_mape': 0.8455857038497925}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.0043481008274275585, 'avg_g_mag_loss': 0.010324255653263465, 'avg_g_cos_loss': 0.0007838808940280051, 'pred_duration': 6.3304314613342285, 'grad_duration': 11.364734411239624, 'total_duration': 17.695165872573853, 'pred_std': 0.23463046550750732, 'std_loss': 0.0012389702023938298, 'mean_pred_loss': 5.348016202333383e-05, 'pred_rmse': 0.06594012677669525, 'pred_mae': 0.041026778519153595, 'pred_mape': 7420750.0, 'grad_rmse': 0.05075574293732643, 'grad_mae': 0.032667260617017746, 'grad_mape': 0.8455857038497925}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:29:23.439974Z", + "iopub.status.busy": "2024-03-23T10:29:23.439667Z", + "iopub.status.idle": "2024-03-23T10:29:23.443969Z", + "shell.execute_reply": "2024-03-23T10:29:23.443136Z" + }, + "papermill": { + "duration": 0.023404, + "end_time": "2024-03-23T10:29:23.445933", + "exception": false, + "start_time": "2024-03-23T10:29:23.422529", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:29:23.477583Z", + "iopub.status.busy": "2024-03-23T10:29:23.477315Z", + "iopub.status.idle": "2024-03-23T10:29:23.603296Z", + "shell.execute_reply": "2024-03-23T10:29:23.602253Z" + }, + "papermill": { + "duration": 0.144635, + "end_time": "2024-03-23T10:29:23.605742", + "exception": false, + "start_time": "2024-03-23T10:29:23.461107", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:29:23.640888Z", + "iopub.status.busy": "2024-03-23T10:29:23.640575Z", + "iopub.status.idle": "2024-03-23T10:29:23.912180Z", + "shell.execute_reply": "2024-03-23T10:29:23.911267Z" + }, + "papermill": { + "duration": 0.291799, + "end_time": "2024-03-23T10:29:23.914044", + "exception": false, + "start_time": "2024-03-23T10:29:23.622245", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzyUlEQVR4nO3de3hTVb438O9Omnvb9J60WFsubblTBOkLgo6SoTB4KaIiD2Oxj8LIwBHsqExVCjM4FhEdRHhkrC+KjgieMzqvBz1VrJQ5DoUiFwXBcrHQcklv0KRN2iRN1vtHmk0DvSRpbm1+n+fZT5KdlZWV0HzZe6+11+YYYwyEEBIEBIFuACGEOFAgEUKCBgUSISRoUCARQoIGBRIhJGhQIBFCggYFEiEkaIQFugHeYLPZcPnyZURERIDjuEA3hxByA8YYmpubkZSUBIGg++2gARFIly9fRnJycqCbQQjpRU1NDW655ZZunx8QgRQREQHA/mEjIyMD3BpCyI30ej2Sk5P532p3BkQgOXbTIiMjKZAICWK9HVKhg9qEkKBBgUQICRoUSISQoDEgjiGRvrFarbBYLIFuBunHRCIRhEJhn+uhQAphjDFotVo0NTUFuilkAIiKioJare7TWEAKpBDmCKOEhATI5XIaVEo8whiD0WhEXV0dACAxMdHjukIqkNqtNvysbUb1VSN+M8bzL20gsFqtfBjFxsYGujmkn5PJZACAuro6JCQkeLz7FlIHtVtM7bj3re/w+4+OoLkttI+ZOI4ZyeXyALeEDBSOv6W+HI8MqUCKkouhipQAAM7UtQS4NcGBdtOIt3jjbymkAgkA0lX2oeuntc0Bbgkh5EYhG0iVtRRIhASbkAukDMcWEgUS8RDHcfjnP/8Z6GZ41Zo1a5CZmRnoZoReIKWrO7aQtHQMifRf77//PqKiorxW37PPPovS0lKv1eepkAuktIRwAEBDiwmNLaYAt4YQ3zKbzS6VCw8PD4rhHyEXSApJGJJj7GMmTtfSVlJnjDEYze1+X9y9eHJJSQmmTp2KqKgoxMbG4t5778W5c+cAAFOmTMHKlSudytfX10MkEuFf//oXAODKlSuYPXs2ZDIZBg8ejB07diA1NRUbN2706Hs7fvw47rnnHshkMsTGxmLx4sVoabn+t1VWVoZJkyZBoVAgKioKd9xxBy5cuAAA+OGHH3D33XcjIiICkZGRmDBhAr7//vse36+srAx5eXnQ6XTgOA4cx2HNmjUAgNTUVKxduxa5ubmIjIzE4sWLAQArV65Eeno65HI5hgwZglWrVjl1z9+4y/b4448jJycHGzZsQGJiImJjY7F06VKfn2IUUgMjHTJUEai52orTtc2YPDTw/ysEi1aLFSMLv/L7+578czbkYtf/FA0GA/Lz8zF27Fi0tLSgsLAQc+bMwbFjx7BgwQKsX78e69at47uhd+3ahaSkJEybNg0AkJubi4aGBpSVlUEkEiE/P58fZewug8GA7OxsTJ48GYcOHUJdXR2efPJJLFu2DO+//z7a29uRk5ODRYsW4eOPP4bZbEZFRQXftgULFmD8+PF4++23IRQKcezYMYhEoh7fc8qUKdi4cSMKCwtRWVkJwL6F47BhwwYUFhZi9erV/LqIiAi8//77SEpKwvHjx7Fo0SJERETg+eef7/Z99u7di8TEROzduxdnz57FvHnzkJmZiUWLFnn0XbnCoy2kLVu2IDU1FVKpFFlZWaioqOi27KeffoqJEyciKioKCoUCmZmZ+PDDD53KMMZQWFiIxMREyGQyaDQanDlzxpOmuYR62vq3uXPn4sEHH8SwYcOQmZmJbdu24fjx4zh58iQeeeQRXL58Gd999x1ffseOHZg/fz44jsPPP/+Mb775BsXFxcjKysJtt92Gd999F62trR61ZceOHWhra8MHH3yA0aNH45577sHmzZvx4Ycfora2Fnq9HjqdDvfeey+GDh2KESNGYOHChbj11lsBANXV1dBoNBg+fDjS0tLw8MMPY9y4cT2+p1gshlKpBMdxUKvVUKvVToF0zz334A9/+AOGDh2KoUOHAgBeeuklTJkyBampqbjvvvvw7LPP4pNPPunxfaKjo7F582YMHz4c9957L2bPnu3z40xubyHt2rUL+fn52Lp1K7KysrBx40ZkZ2ejsrISCQkJN5WPiYnBiy++iOHDh0MsFmP37t3Iy8tDQkICsrOzAQDr16/Hpk2bsH37dgwePBirVq1CdnY2Tp48CalU2vdPeYOMjgPbZyiQnMhEQpz8c3ZA3tcdZ86cQWFhIQ4ePIiGhgbYbDYA9h/36NGjMWPGDHz00UeYNm0aqqqqUF5ejr/97W8AgMrKSoSFheG2227j6xs2bBiio6M9avupU6cwbtw4KBQKft0dd9wBm82GyspK3HnnnXj88ceRnZ2NX//619BoNHjkkUf4873y8/Px5JNP4sMPP4RGo8HDDz/Mh4inJk6ceNO6Xbt2YdOmTTh37hxaWlrQ3t7e6+yqo0aNcjoFJDExEcePH+9T23rj9hbSG2+8gUWLFiEvLw8jR47E1q1bIZfLsW3bti7L/+pXv8KcOXMwYsQIDB06FMuXL8fYsWP5/8EYY9i4cSNeeuklPPDAAxg7diw++OADXL58uduuVZPJBL1e77S4g99C0ja7ffxiIOM4DnJxmN8Xd0f43nfffbh69SqKi4tx8OBBHDx4EMD1A7gLFizAf/3Xf8FisWDHjh0YM2YMxowZ4/Xvy1XvvfceysvLMWXKFOzatQvp6ek4cOAAAPuxm59++gmzZ8/Gt99+i5EjR+Kzzz7r0/t1DkcAKC8vx4IFC/Cb3/wGu3fvxtGjR/Hiiy/2esD7xl1HjuP48PcVtwLJbDbj8OHD0Gg01ysQCKDRaFBeXt7r6xljKC0t5f/nAICqqipotVqnOpVKJbKysrqts6ioCEqlkl/cveLIkHgFhAIO+rZ21Oqpp60/aWxsRGVlJV566SVMnz4dI0aMwLVr15zKPPDAA2hra0NJSQl27NiBBQsW8M9lZGSgvb0dR48e5dedPXv2pjpcNWLECPzwww8wGAz8un//+98QCATIyMjg140fPx4FBQXYv38/Ro8ejR07dvDPpaen45lnnsHXX3+NBx98EO+9916v7ysWi2G1Wl1q4/79+5GSkoIXX3wREydORFpaGn9QPdi4FUgNDQ2wWq1QqVRO61UqFbRabbev0+l0CA8Ph1gsxuzZs/HWW2/h17/+NQDwr3OnzoKCAuh0On6pqalx52NAEibE4Dj7/yJ0HKl/iY6ORmxsLN555x2cPXsW3377LfLz853KKBQK5OTkYNWqVTh16hTmz5/PPzd8+HBoNBosXrwYFRUVOHr0KBYvXgyZTObRuVgLFiyAVCrFwoULceLECezduxf/8R//gcceewwqlQpVVVUoKChAeXk5Lly4gK+//hpnzpzBiBEj0NraimXLlqGsrAwXLlzAv//9bxw6dAgjRozo9X1TU1PR0tKC0tJSNDQ0wGg0dls2LS0N1dXV2LlzJ86dO4dNmzb1eSvMV/zS7R8REYFjx47h0KFD+Mtf/oL8/HyUlZV5XJ9EIuGvMOLplUYy6Jy2fkkgEGDnzp04fPgwRo8ejWeeeQavvfbaTeUWLFiAH374AdOmTeMPIDt88MEHUKlUuPPOOzFnzhy+x8mT45VyuRxfffUVrl69ittvvx0PPfQQpk+fjs2bN/PP//zzz5g7dy7S09OxePFiLF26FL/73e8gFArR2NiI3NxcpKen45FHHsGsWbPwpz/9qdf3nTJlCp566inMmzcP8fHxWL9+fbdl77//fjzzzDNYtmwZMjMzsX//fqxatcrtz+oXzA0mk4kJhUL22WefOa3Pzc1l999/v8v1PPHEE2zGjBmMMcbOnTvHALCjR486lbnzzjvZ008/7VJ9Op2OAWA6nc7lNmzcc5qlrNzN/vDJMZdfM5C0traykydPstbW1kA3JeBqamoYAPbNN98Euin9Wk9/U67+Rt3aQhKLxZgwYYJT15/NZkNpaSkmT57scj02mw0mk/3YzeDBg6FWq53q1Ov1OHjwoFt1uitdZe8mpXPaQs+3336Lzz//HFVVVdi/fz8effRRpKam8sc1SeC4vcuWn5+P4uJibN++HadOncKSJUtgMBiQl5cHwD7orKCggC9fVFSEPXv24JdffsGpU6fw+uuv48MPP8Rvf/tbAPYj9ytWrMDLL7+Mzz//HMePH0dubi6SkpKQk5PjnU/ZBcc5badrm2GzUU9bKLFYLHjhhRcwatQozJkzB/Hx8fwgyY8++gjh4eFdLqNGjfJbG2fNmtVtO1555RW/tcPf3B6HNG/ePNTX16OwsBBarRaZmZkoKSnhD0pXV1dDILiecwaDAb///e9x8eJFyGQyDB8+HH//+98xb948vszzzz8Pg8GAxYsXo6mpCVOnTkVJSYlPxiA5pMTIIQ4ToM1iQ801I1JiFb2/iAwI2dnZ/Bi4G91///3Iysrq8rneRlB7U0+DNWNiYvzWDn/jGOv/A3H0ej2USiV0Op1bB7h/8+b/4uQVPd55bAJmjFL7sIXBp62tDVVVVRg8eLBPg5+Ejp7+plz9jYbcybWdZahpbiRCgklIB9L1c9rorH9CgkFIB1KGuqOnjcYiERIUQjqQHFtIvzS0wGL17Tk6hJDehXQgDYqSQSEWwmJlON9g6P0FhBCfCulA4jju+hzbdGCbuGggTvIfLEI6kAA6p430T96e5B+wT43LcRyampq8Wq87Qj6QaPZIQoJHyAfS9bFI1PUPxgCzwf8LTfLvtUn+TSYTnn32WQwaNAgKhQJZWVlOM2tcuHAB9913H6Kjo6FQKDBq1Ch8+eWXOH/+PO6++24A9ileOI7D448/7tH30RchOcl/Z2kdJ9mebzSgzWKF1M3pVAcUixF4Jcn/7/vCZUDs+qk7NMl/95P8L1u2DCdPnsTOnTuRlJSEzz77DDNnzsTx48eRlpaGpUuXwmw241//+hcUCgVOnjyJ8PBwJCcn4x//+Afmzp2LyspKREZGQiaTefSd9EXIB1J8uATRchGuGS04W9eC0YOUgW4S6cXcuXOdHm/btg3x8fH8JP8rVqzAd999xwdQV5P8Hzp0iJ97+t1330VaWppHbek8yb9j6tjNmzfjvvvuw6uvvgqRSOQ0yT8ApwnYqqur8dxzz2H48OEA4FI7bpzkv3Nd7733Hqqrq5GUZP+P5dlnn0VJSQnee+89vPLKK6iursbcuXP5KX2HDBnCv95xjlxCQoLXj0+5KuQDieM4pKsicLDqKiq1zaEdSCK5fWslEO/rBprkv2vHjx+H1WpFenq603qTycRfBPLpp5/GkiVL8PXXX0Oj0WDu3LkYO3asR+/nCyF/DAmgc9p4HGffdfL3QpP8e2WS/5aWFgiFQhw+fBjHjh3jl1OnTuHNN98EADz55JP45Zdf8Nhjj+H48eOYOHEi3nrrLa991r6iQAL1tPUnNMm/XVeT/I8fPx5WqxV1dXUYNmyY09J51y45ORlPPfUUPv30U/zhD39AcXExXycAly8e4AsUSOi0hURjkYIeTfJv19Uk/+np6ViwYAFyc3Px6aefoqqqChUVFSgqKsIXX3wBAFixYgW++uorVFVV4ciRI9i7dy//fikpKeA4Drt370Z9fb1TT6Hf+GZ2Xf/yZE7tzpoMZpaycjdLWbmb6VvNXm5dcOrPc2rv2bOHjRgxgkkkEjZ27FhWVlbGADjN9f7ll18yAOzOO++86fWXL19ms2bNYhKJhKWkpLAdO3awhIQEtnXrVpfe/8b3+vHHH9ndd9/NpFIpi4mJYYsWLWLNzc2MMca0Wi3LyclhiYmJTCwWs5SUFFZYWMisViszmUzs0UcfZcnJyUwsFrOkpCS2bNkyl/9NnnrqKRYbG8sAsNWrVzPGGDObzaywsJClpqYykUjEEhMT2Zw5c9iPP/7IGGNs2bJlbOjQoUwikbD4+Hj22GOPsYaGBr7OP//5z0ytVjOO49jChQtdaoeDN+bUDukJ2jr7P6+UQqtvwz+WTMGEFM8OcPYnNEHbdRcvXkRycjK++eYbTJ8+PdDN6be8MUFbyPeyOaSrI6DVt+F0bXNIBFIo+/bbb9HS0oIxY8bgypUreP7552mS/yBBx5A6ZHQMkKyk40gDHk3yH7xoC6mDo6ct5Lv+QwBN8h+8KJA60FgkAtivshwRERHoZmDQoEGBbkJA0C5bh2EJ4eA4oKHFjIYWU6Cb4zcDoE+DBAlv/C15FEhbtmxBamoqpFIpsrKyUFFR0W3Z4uJiTJs2DdHR0YiOjoZGo7mp/OOPP86ftexYZs6c6UnTPCYXhyE52n4KQyhsJTl2P4xGY4BbQgYKx99SX3Zt3d5l27VrF/Lz87F161ZkZWVh48aNyM7ORmVlJRISEm4qX1ZWhvnz52PKlCmQSqV49dVXMWPGDPz0009Om6UzZ850GqEqkUg8/EieS1dFoPqqEae1zZgyNM7v7+9PQqEQUVFR/Fnucrnco4GBhDDGYDQaUVdXh6ioKAiFns+Y4XYgvfHGG1i0aBF/6eytW7fiiy++wLZt2/DHP/7xpvIfffSR0+N3330X//jHP1BaWorc3Fx+vUQicRreHggZ6nB8c6o2ZC6L5Pi+PZ16g5DOoqKi+vwbdiuQzGYzDh8+jIKCAn6dQCCARqNBeXm5S3UYjUZYLJabegrKysqQkJCA6Oho3HPPPXj55Zf5M5RvZDKZYDJdP86j1+vd+RjdCrWeNo7jkJiYiISEBFgslkA3h/RjIpGoT1tGDm4FUkNDA6xWK1QqldN6lUqFn3/+2aU6Vq5ciaSkJGg0Gn7dzJkz8eCDD2Lw4ME4d+4cXnjhBcyaNQvl5eVdfsiioiL86U9/cqfpLul8ThtjLGR2YYRCoVf+mAjpK792+69btw47d+5EWVmZ09DyRx99lL8/ZswYjB07FkOHDkVZWVmXQ/kLCgqcTqjU6/VITk7uc/uGxIUjTMCh2dSOK7o2JEX5f8Y8QkKZW71scXFxEAqFqK2tdVpfW1vb677jhg0bsG7dOnz99de9Tgg1ZMgQxMXF4ezZs10+L5FIEBkZ6bR4gzhMgMFx9om2QmW3jZBg4lYgicViTJgwAaWlpfw6m82G0tJSTJ48udvXrV+/HmvXrkVJSQk/bWhPLl68iMbGRn5WPX9KpwGShASM2+OQ8vPzUVxcjO3bt+PUqVNYsmQJDAYD3+uWm5vrdND71VdfxapVq7Bt2zakpqZCq9VCq9Xyc620tLTgueeew4EDB3D+/HmUlpbigQcewLBhw7od3u9Ljuu0VWpDo6eNkGDi9jGkefPmob6+HoWFhdBqtcjMzERJSQl/oLu6uhoCwfWce/vtt2E2m/HQQw851bN69WqsWbMGQqEQP/74I7Zv346mpiYkJSVhxowZWLt2bcDGIgG0hURIINB8SDeoajDg7g1lkIoE+OlPMyEUhEZPGyG+5OpvlM5lu8GtMXJIwgRos9hQc5VOqyDEnyiQbiAUcPzFI2nSf0L8iwKpC+kJNOk/IYFAgdQFR9c/bSER4l8USF3IoJ42QgKCAqkLji2kX+oNMLfbAtwaQkIHBVIXkpRShEvC0G5jqGow9P4CQohXUCB1geM4pFNPGyF+R4HUDcdUJGcokAjxGwqkbqTz57RRIBHiLxRI3aCeNkL8jwKpG46etgtXjWg1WwPcGkJCAwVSN+LCJYhViMEYcLaOpiIhxB8okHrAH0ei3TZC/IICqQd0eW1C/IsCqQf8Wf/U00aIX1Ag9YB62gjxLwqkHqR1BNIVXRt0rXQhRUJ8jQKpB0qZCIlK+/XjaMQ2Ib5HgdQL6mkjxH8okHrR+fLahBDfokDqxfXLItHgSEJ8jQKpF9TTRoj/eBRIW7ZsQWpqKqRSKbKyslBRUdFt2eLiYkybNg3R0dGIjo6GRqO5qTxjDIWFhUhMTIRMJoNGo8GZM2c8aZrXDUsIB8cBjQYzGlpMgW4OIQOa24G0a9cu5OfnY/Xq1Thy5AjGjRuH7Oxs1NXVdVm+rKwM8+fPx969e1FeXo7k5GTMmDEDly5d4susX78emzZtwtatW3Hw4EEoFApkZ2ejra3N80/mJTKxECkxcgB0HIkQn2NumjRpElu6dCn/2Gq1sqSkJFZUVOTS69vb21lERATbvn07Y4wxm83G1Go1e+211/gyTU1NTCKRsI8//tilOnU6HQPAdDqdG5/EdYu2H2IpK3ezbd/94pP6CRnoXP2NurWFZDabcfjwYWg0Gn6dQCCARqNBeXm5S3UYjUZYLBbExMQAAKqqqqDVap3qVCqVyMrK6rZOk8kEvV7vtPgSndNGiH+4FUgNDQ2wWq1QqVRO61UqFbRarUt1rFy5EklJSXwAOV7nTp1FRUVQKpX8kpyc7M7HcBvNHkmIf/i1l23dunXYuXMnPvvsM0ilUo/rKSgogE6n45eamhovtvJmnbv+GWM+fS9CQplbgRQXFwehUIja2lqn9bW1tVCr1T2+dsOGDVi3bh2+/vprjB07ll/veJ07dUokEkRGRjotvjQ4ToEwAYcWUzsu6wJ/oJ2QgcqtQBKLxZgwYQJKS0v5dTabDaWlpZg8eXK3r1u/fj3Wrl2LkpISTJw40em5wYMHQ61WO9Wp1+tx8ODBHuv0J3GYAEPiFQCop40QX3J7ly0/Px/FxcXYvn07Tp06hSVLlsBgMCAvLw8AkJubi4KCAr78q6++ilWrVmHbtm1ITU2FVquFVqtFS4t95DPHcVixYgVefvllfP755zh+/Dhyc3ORlJSEnJwc73xKL6Bz2gjxvTB3XzBv3jzU19ejsLAQWq0WmZmZKCkp4Q9KV1dXQyC4nnNvv/02zGYzHnroIad6Vq9ejTVr1gAAnn/+eRgMBixevBhNTU2YOnUqSkpK+nScydsyVBHYjSu0hUSID3FsAByl1ev1UCqV0Ol0Pjue9NVPWvzuw8MYlRSJL56e5pP3IGSgcvU3SueyuchxTtvZuhZYbf0+wwkJShRILkqOkUMqEsDUbkP1VWOgm0PIgESB5CKhgENaAg2QJMSXKJDckE5TkRDiUxRIbshQd1wWiQKJEJ+gQHIDv4VEu2yE+AQFkhscZ/1XNRhgarcGuDWEDDwUSG5QR0oRIQlDu42hqsEQ6OYQMuBQILmB4zikq6mnjRBfoUByE/W0EeI7FEhuylB19LRp6bJIhHgbBZKb0mk6W0J8hgLJTY5z2qqvGmE0twe4NYQMLBRIbooNlyAuXAzAfqItIcR7KJA8QJP+E+IbFEgeoJ42QnyDAskDjhHblbW0y0aIN1EgeYDOaSPENyiQPJDeMRZJq2+DzmgJcGsIGTgokDwQIRVhUJQMAHC6jraSCPEWCiQPpfEjtimQCPEWCiQPZVBPGyFeR4HkIRqLRIj3eRRIW7ZsQWpqKqRSKbKyslBRUdFt2Z9++glz585FamoqOI7Dxo0bbyqzZs0acBzntAwfPtyTpvlNRqdz2gbApe0ICQpuB9KuXbuQn5+P1atX48iRIxg3bhyys7NRV1fXZXmj0YghQ4Zg3bp1UKvV3dY7atQoXLlyhV++++47d5vmV8MSwsFxwDWjBfUtpkA3h5ABwe1AeuONN7Bo0SLk5eVh5MiR2Lp1K+RyObZt29Zl+dtvvx2vvfYaHn30UUgkkm7rDQsLg1qt5pe4uDh3m+ZXUpEQqbEKAMBpmoqEEK9wK5DMZjMOHz4MjUZzvQKBABqNBuXl5X1qyJkzZ5CUlIQhQ4ZgwYIFqK6u7rasyWSCXq93WgLBMR6JDmwT4h1uBVJDQwOsVitUKpXTepVKBa1W63EjsrKy8P7776OkpARvv/02qqqqMG3aNDQ3d/1DLyoqglKp5Jfk5GSP37svqKeNEO8Kil62WbNm4eGHH8bYsWORnZ2NL7/8Ek1NTfjkk0+6LF9QUACdTscvNTU1fm6xHT+/NgUSIV4R5k7huLg4CIVC1NbWOq2vra3t8YC1u6KiopCeno6zZ892+bxEIunxeJS/ZHQ6p40xBo7jAtwiQvo3t7aQxGIxJkyYgNLSUn6dzWZDaWkpJk+e7LVGtbS04Ny5c0hMTPRanb6QGqeASMjBYLbiUlNroJtDSL/n9i5bfn4+iouLsX37dpw6dQpLliyBwWBAXl4eACA3NxcFBQV8ebPZjGPHjuHYsWMwm824dOkSjh075rT18+yzz2Lfvn04f/489u/fjzlz5kAoFGL+/Ple+Ii+IxIKMDSeDmwT4i1u7bIBwLx581BfX4/CwkJotVpkZmaipKSEP9BdXV0NgeB6zl2+fBnjx4/nH2/YsAEbNmzAXXfdhbKyMgDAxYsXMX/+fDQ2NiI+Ph5Tp07FgQMHEB8f38eP53vpqgj8rG1GpbYF9wxX9f4CQki3ODYAhhnr9XoolUrodDpERkb69b03f3sGG74+jTnjB+Gv8zL9+t6E9Beu/kaDopetP6Nz2gjxHgqkPnKc03a2vgXtVluAW0NI/0aB1EfJ0XJIRQKY2224cNUY6OYQ0q9RIPWRQMDRHNuEeAkFkhfwx5Go65+QPqFA8gLHiO0zdFkkQvqEAskL6Jw2QryDAskLHFtIVQ0GmNqtAW4NIf0XBZIXqCIliJSGwWpj+KXeEOjmENJvUSB5AcdxTnNsE0I8Q4HkJTRim5C+o0DyEtpCIqTvKJC8JC2BetoI6SsKJC9xTPhfc7UVBlN7gFtDSP9EgeQlseESxIXbp9U9U0cDJAnxBAWSF2WoO2aPpAPbhHiEAsmL6Jw2QvqGAsmL6DpthPQNBZIXpVPXPyF9QoHkRWkJ9mNItXoTmozmALeGkP6HAsmLIqQiDIqSAQBO01QkhLiNAsnLMmgqEkI8RoHkZTSdLSGe8yiQtmzZgtTUVEilUmRlZaGioqLbsj/99BPmzp2L1NRUcByHjRs39rnOYOYYi0RbSIS4z+1A2rVrF/Lz87F69WocOXIE48aNQ3Z2Nurq6rosbzQaMWTIEKxbtw5qtdordQaz9E5d/wPgGpyE+Bdz06RJk9jSpUv5x1arlSUlJbGioqJeX5uSksL++te/erVOxhjT6XQMANPpdC6V96VWczsb/MfdLGXlblaraw10cwgJCq7+Rt3aQjKbzTh8+DA0Gg2/TiAQQKPRoLy83KNA9KROk8kEvV7vtAQLqUiI1FgFANptI8RdbgVSQ0MDrFYrVCqV03qVSgWtVutRAzyps6ioCEqlkl+Sk5M9em9focnaCPFMv+xlKygogE6n45eamppAN8kJjdgmxDNh7hSOi4uDUChEbW2t0/ra2tpuD1j7ok6JRAKJROLR+/lDBn+SLQ2OJMQdbm0hicViTJgwAaWlpfw6m82G0tJSTJ482aMG+KLOQHN0/Z+pbYbNRj1thLjKrS0kAMjPz8fChQsxceJETJo0CRs3boTBYEBeXh4AIDc3F4MGDUJRUREA+0HrkydP8vcvXbqEY8eOITw8HMOGDXOpzv4mJVYBsVAAo9mKS02tSI6RB7pJhPQLbgfSvHnzUF9fj8LCQmi1WmRmZqKkpIQ/KF1dXQ2B4PqG1+XLlzF+/Hj+8YYNG7BhwwbcddddKCsrc6nO/kYkFGBIvAI/a5txuraZAokQF3GM9f/Re3q9HkqlEjqdDpGRkYFuDgBg+c6j+H/HLuP5mRn4/a+GBbo5hASUq7/RftnL1h/QOW2EuI8CyUeop40Q91Eg+YhjGpJzdS1ot9oC3BpC+gcKJB8ZFCWDXCyE2WrD+UZjoJtDSL9AgeQjAgHHT2lLI7YJcQ0Fkg/ROW2EuIcCyYcy6Jw2QtxCgeRDdOFIQtxDgeRDji2k8w0GtFmsAW4NIcGPAsmHEiIkUMpEsDHgXD2NRyKkNxRIPsRxHD9A8gwNkCSkVxRIPpZOVyEhxGUUSD6WQee0EeIyCiQfo542QlxHgeRjjkC6eK0VLab2ALeGkOBGgeRj0QoxEiLs83+foa0kQnpEgeQHNGKbENdQIPnB9XPaqOufkJ5QIPlBuorO+ifEFRRIfkA9bYS4hgLJD9I6Aqm+2YSrBnOAW0NI8KJA8oNwSRhuiZYBoN02QnpCgeQn/IhtCiRCuuVRIG3ZsgWpqamQSqXIyspCRUVFj+X/8z//E8OHD4dUKsWYMWPw5ZdfOj3/+OOPg+M4p2XmzJmeNC1opVPXPyG9cjuQdu3ahfz8fKxevRpHjhzBuHHjkJ2djbq6ui7L79+/H/Pnz8cTTzyBo0ePIicnBzk5OThx4oRTuZkzZ+LKlSv88vHHH3v2iYLU9XPaqOufkO64HUhvvPEGFi1ahLy8PIwcORJbt26FXC7Htm3buiz/5ptvYubMmXjuuecwYsQIrF27Frfddhs2b97sVE4ikUCtVvNLdHS0Z58oSHXuaRsAFwsmxCfcCiSz2YzDhw9Do9Fcr0AggEajQXl5eZevKS8vdyoPANnZ2TeVLysrQ0JCAjIyMrBkyRI0NjZ22w6TyQS9Xu+0BLsh8QoIBRx0rRbUNZsC3RxCgpJbgdTQ0ACr1QqVSuW0XqVSQavVdvkarVbba/mZM2figw8+QGlpKV599VXs27cPs2bNgtXa9bSvRUVFUCqV/JKcnOzOxwgIqUiI1Fg5ALoKCSHdCYpetkcffRT3338/xowZg5ycHOzevRuHDh1CWVlZl+ULCgqg0+n4paamxr8N9hCd00ZIz9wKpLi4OAiFQtTW1jqtr62thVqt7vI1arXarfIAMGTIEMTFxeHs2bNdPi+RSBAZGem09Ad0nTZCeuZWIInFYkyYMAGlpaX8OpvNhtLSUkyePLnL10yePNmpPADs2bOn2/IAcPHiRTQ2NiIxMdGd5gU9GotESM/c3mXLz89HcXExtm/fjlOnTmHJkiUwGAzIy8sDAOTm5qKgoIAvv3z5cpSUlOD111/Hzz//jDVr1uD777/HsmXLAAAtLS147rnncODAAZw/fx6lpaV44IEHMGzYMGRnZ3vpYwaHND6QWmCzUU8bITcKc/cF8+bNQ319PQoLC6HVapGZmYmSkhL+wHV1dTUEgus5N2XKFOzYsQMvvfQSXnjhBaSlpeGf//wnRo8eDQAQCoX48ccfsX37djQ1NSEpKQkzZszA2rVrIZFIvPQxg0NqrBxioQCtFisuXmvFrR0HuQkhdhwbAINi9Ho9lEoldDpd0B9PmvXm/+LUFT2Kcyfi1yNVvb+AkAHA1d9oUPSyhZIMmhuJkG5RIPmZ45w26mkj5GYUSH5GPW2EdI8Cyc8cY5F+qTfAYrUFuDWEBBcKJD8bFCWDQiyE2WrDhUZDoJtDSFChQPIzgYDjxyPRVUgIcUaBFAAZNOk/IV2iQAoAfvZI6mkjxAkFUgBQTxshXaNACoB0tX1w5PlGA9osXc/5REgookAKgPhwCaLkItgYcLaODmwT4kCBFAAcx/HjkWi3jZDr3D7bn3hHhioCFVVXA97T1maxoqHFhLpmE3StFiRESJAcI0ekVBTQdpHQRIEUIL7saWOMQddqQX2zPWjst203PLbf6lotXdahlImQHCNDcrQct8bIcUuMHMnRMiTHyDEoSgapSOj1dhNCgRQgGZ0ma3OVud2GhhZTj0HjWMxunJYiFgoQHyFBhDQMdc0mXDWYoWu1QHfJghOXur6iiypSguRoOZI7BZVjUUdKIRRwLr9/sLHZGJpN7WgymtFktKCp1QJpmAApsQokREgg6MefLdhRIAVIesc0JJeaWnFF1wqj2Yo6vQn1LSbU6dtQ32JCPf/YfnvVYHbrPZQyEeIjJEiIkNxwK3V6rJSJwHHXf2QtpnZcvGZEzdVW1Fw1ouaG+0azFbV6E2r1Jnx/4dpN7ysSckiKknUElgy3dGxlOcIrRiF2ej9fYYzBaLaiqdWCawZHuJhxzWiBzmi/vWY0Q9dx29RqQZPRAl2rBdZuZvSUigRIiVHg1lg5UmPlSIlVICVWjtRYBRKVUoQJ6bBsX9AEbQGU9co3qNW7d422MAGH+BsCJv6GgEmIkCAuXOKT3SrGGK4azKi55hxW9gAz4lJTKyzWnv+k5GKhU1g5gurWWDmSo+VQSG7+f7LNYoWutSM4jBY0dQSK436TU6g4Qsfi1pbijWQiIaLlIijlYhhM7bjU1NptUAH2f5vkGDlSYuVIibGHVWqcHLfGKJAcI4MkLHR3c139jVIgBdAf//Ejdh6yX8IpQhrWKVSkzls14SLEK8KQoAhDlISDAFbAZgNs7dcXZgVsVud1vZaxOt8yGyCWA+Jw+yIJd74vkgO9bNlYbQy1+jZUXzV2BFYrLnYKrtrmNvT2FxejEOOWaBksVsZvybT2YbyWWChAlFzUsYgRJRMhWi5GlEKEKJkY0Z2ei5aLESUXQSkT3RToFqsNl6614sJVIy40GnC+wYjqqwacbzSi+qoR5vbuw4/jgCSlzB5W/FbV9fty8cDeWaFA8hRj9h9nexvQbgIsrfZbx+P2ti7ut3VRpuPW0v3zrL0VNosJHLNCwKzdBwmC5Z+I6xRUii6CSwFIIjrdDwfEEZ3uh8MslOFKaxhqjEJc0APV10zXdwmvGdFk7PogOwAIONgDRS66Hiodjx1bMtFy+3qlTIRohT185GKhz3cRbTYGrb4N5xsNqG404nyjPbQudNwazD0HanyEBKnRUmREA2kRFqSEW5AsNUEtboXC1gK0XgNam4C2JsBqAaRKQBoFyKK6vxXJfPqZ3eHqb3Rgx/KNrBbg73O7CZhOj5l/5iniAPRpI54TAIIwgBPabwWOW8f9To9dKQMOsBgBcwtganG+BbMv5mb74iExgJSOZSpg3+pyBFhCONrDFGjlZDBAChYmh0iqgEimgEQWDrE0HAKx3P5DE8nsr3W65QBRGCASACIJECbpdYvOWwQC+3GzJKUUU26RAK0c0GoG2hhYqwkt1xqgu1aHlqZGmJobYDU2gWtrgsSig8LWAqXZgMhaIwR13vvPp50Toy0swr4I7Utrp8UoDIdREAGDIAKtwnC0cOEwCCJgFISjFRLYwMHGGGw2BhtjsNrsu+w2xmBl1+8vyErBb8Z455JloRVIgjCgap97rxFKgDCp/Y/bcSuSdty/YX1YT+s7bkU3PC8UdwoIN4KEEwICPx1AZcweVI5w4oPKYA8n/n4LYGrudL+H8qxji8FitC8dU0OFAYjoWPqO6wi8G4Orp1Drap0csJqvb6F03lq58X6brmOr1qkVPX+mG/4ZzZwUei4c12xyXLXJ0cTCoWMK6KCAjilgQRgiOQOUMEDZcdv5cSSMCONsCGNmhFsaEW5pdPubMzMhdFBAzxTQd7yvrotbPVNAn3wPQIHkAY4D5v5fewiIZN2HRpgECJN1hAX1moDjOnbRFAC8cKUUxuxbo70FmMVo32Xmb1t7Wddx3+bY7WOAxWBf/E0oBmTR13efXL6vhDhMgjgAcQCM5nZUXzXifIMRTVcNuNpoxNUWMwQCQMBxEHAchAIOHGd/LOQ4CDgGGWuF3NYChVUPOWuB3NoCua0ZUmszZNYWyNqbIbPqIbU2Q9reDEm7HpL2ZkjamyFk7RBzVsRDj3iu62EfnTVADuBOr3xtoRVIADDmoUC3gHCcfUtRJAUUcd6v32pxMcxcDDqzARCK3AsYkcwru4tycRiGqyMxXO2nzhrG7J+3ranTll/nW91N6+JuSfPm+7tv8+bNLCUlhUkkEjZp0iR28ODBHst/8sknLCMjg0kkEjZ69Gj2xRdfOD1vs9nYqlWrmFqtZlKplE2fPp2dPn3a5fbodDoGgOl0Ok8+DiHEx1z9jbq9P7Jr1y7k5+dj9erVOHLkCMaNG4fs7GzU1dV1WX7//v2YP38+nnjiCRw9ehQ5OTnIycnBiRMn+DLr16/Hpk2bsHXrVhw8eBAKhQLZ2dloa2vzNGcJIf2Ru0k3adIktnTpUv6x1WplSUlJrKioqMvyjzzyCJs9e7bTuqysLPa73/2OMWbfOlKr1ey1117jn29qamISiYR9/PHHXdbZ1tbGdDodv9TU1NAWEiFBzCdbSGazGYcPH4ZGo+HXCQQCaDQalJeXd/ma8vJyp/IAkJ2dzZevqqqCVqt1KqNUKpGVldVtnUVFRVAqlfySnJzszscghAQptwKpoaEBVqsVKpVzT4tKpYJWq+3yNVqttsfyjlt36iwoKIBOp+OXmpoadz4GISRI9cteNolEAolEEuhmEEK8zK0tpLi4OAiFQtTW1jqtr62thVqt7vI1arW6x/KOW3fqJIQMTG4FklgsxoQJE1BaWsqvs9lsKC0txeTJk7t8zeTJk53KA8CePXv48oMHD4ZarXYqo9frcfDgwW7rJIQMTG7vsuXn52PhwoWYOHEiJk2ahI0bN8JgMCAvLw8AkJubi0GDBqGoqAgAsHz5ctx11114/fXXMXv2bOzcuRPff/893nnnHQD2+aVXrFiBl19+GWlpaRg8eDBWrVqFpKQk5OTkuNQm1nF+sF7f+6hSQoj/OX6brLdz+T3pwnvrrbfYrbfeysRiMZs0aRI7cOAA/9xdd93FFi5c6FT+k08+Yenp6UwsFrNRo0Z1OzBSpVIxiUTCpk+fziorK11uj6PbnxZaaAnupaampsff8oCYfsRms+Hy5cuIiIjodZoJvV6P5ORk1NTU9Ku5k3yNvpfu0XfTNXe+F8YYmpubkZSUBEEP54f2y162GwkEAtxyyy1uvSYyMpL+uLpA30v36Lvpmqvfi1Kp7LUMncpOCAkaFEiEkKARcoEkkUiwevVqGlh5A/peukffTdd88b0MiIPahJCBIeS2kAghwYsCiRASNCiQCCFBgwKJEBI0KJAIIUEj5AJpy5YtSE1NhVQqRVZWFioqKgLdpIAqKirC7bffjoiICCQkJCAnJweVlZWBblbQWbduHX8iOAEuXbqE3/72t4iNjYVMJsOYMWPw/fff97nekAokdy9QEAr27duHpUuX4sCBA9izZw8sFgtmzJgBgyEA1zILUocOHcLf/vY3jB07NtBNCQrXrl3DHXfcAZFIhP/5n//ByZMn8frrryM6Orrvlbt8Sv0A4O4FCkJRXV0dA8D27dsX6KYEhebmZpaWlsb27NnD7rrrLrZ8+fJANyngVq5cyaZOneqTukNmC8mTCxSEIp1OBwCIiYkJcEuCw9KlSzF79uybLlQRyj7//HNMnDgRDz/8MBISEjB+/HgUFxd7pe6QCSRPLlAQamw2G1asWIE77rgDo0ePDnRzAm7nzp04cuQIP9kgsfvll1/w9ttvIy0tDV999RWWLFmCp59+Gtu3b+9z3QNi+hHiHUuXLsWJEyfw3XffBbopAVdTU4Ply5djz549kEqlgW5OULHZbJg4cSJeeeUVAMD48eNx4sQJbN26FQsXLuxT3SGzheTJBQpCybJly7B7927s3bvX7bmlBqLDhw+jrq4Ot912G8LCwhAWFoZ9+/Zh06ZNCAsLg9VqDXQTAyYxMREjR450WjdixAhUV1f3ue6QCSRPLlAQChhjWLZsGT777DN8++23GDx4cKCbFBSmT5+O48eP49ixY/wyceJELFiwAMeOHYNQKAx0EwPmjjvuuGloyOnTp5GSktL3yn1yqDxI7dy5k0kkEvb++++zkydPssWLF7OoqCim1WoD3bSAWbJkCVMqlaysrIxduXKFX4xGY6CbFnSol82uoqKChYWFsb/85S/szJkz7KOPPmJyuZz9/e9/73PdIRVIjPV8gYJQhG4mY3/vvfcC3bSgQ4F03X//93+z0aNHM4lEwoYPH87eeecdr9RL8yERQoJGyBxDIoQEPwokQkjQoEAihAQNCiRCSNCgQCKEBA0KJEJI0KBAIoQEDQokQkjQoEAihAQNCiRCSNCgQCKEBI3/D7+0H9pOjco2AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:29:23.948334Z", + "iopub.status.busy": "2024-03-23T10:29:23.948027Z", + "iopub.status.idle": "2024-03-23T10:33:56.993952Z", + "shell.execute_reply": "2024-03-23T10:33:56.993126Z" + }, + "papermill": { + "duration": 273.066041, + "end_time": "2024-03-23T10:33:56.996563", + "exception": false, + "start_time": "2024-03-23T10:29:23.930522", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:33:57.032326Z", + "iopub.status.busy": "2024-03-23T10:33:57.031861Z", + "iopub.status.idle": "2024-03-23T10:33:57.052436Z", + "shell.execute_reply": "2024-03-23T10:33:57.051560Z" + }, + "papermill": { + "duration": 0.040618, + "end_time": "2024-03-23T10:33:57.054506", + "exception": false, + "start_time": "2024-03-23T10:33:57.013888", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0004140.0073550.00434811.3877810.0326670.8455860.0507560.0000536.2931360.0410277420750.50.065940.234630.00123917.680917
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.000414 0.007355 0.004348 11.387781 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.032667 0.845586 0.050756 0.000053 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 6.293136 0.041027 7420750.5 0.06594 0.23463 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.001239 17.680917 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:33:57.087735Z", + "iopub.status.busy": "2024-03-23T10:33:57.087458Z", + "iopub.status.idle": "2024-03-23T10:33:57.600802Z", + "shell.execute_reply": "2024-03-23T10:33:57.599950Z" + }, + "papermill": { + "duration": 0.532195, + "end_time": "2024-03-23T10:33:57.602800", + "exception": false, + "start_time": "2024-03-23T10:33:57.070605", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:33:57.638179Z", + "iopub.status.busy": "2024-03-23T10:33:57.637852Z", + "iopub.status.idle": "2024-03-23T10:39:02.515566Z", + "shell.execute_reply": "2024-03-23T10:39:02.514652Z" + }, + "papermill": { + "duration": 304.898124, + "end_time": "2024-03-23T10:39:02.518059", + "exception": false, + "start_time": "2024-03-23T10:33:57.619935", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:39:02.555053Z", + "iopub.status.busy": "2024-03-23T10:39:02.554131Z", + "iopub.status.idle": "2024-03-23T10:39:02.581051Z", + "shell.execute_reply": "2024-03-23T10:39:02.580329Z" + }, + "papermill": { + "duration": 0.047842, + "end_time": "2024-03-23T10:39:02.583506", + "exception": false, + "start_time": "2024-03-23T10:39:02.535664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:39:02.617659Z", + "iopub.status.busy": "2024-03-23T10:39:02.616835Z", + "iopub.status.idle": "2024-03-23T10:39:02.622550Z", + "shell.execute_reply": "2024-03-23T10:39:02.621651Z" + }, + "papermill": { + "duration": 0.024429, + "end_time": "2024-03-23T10:39:02.624394", + "exception": false, + "start_time": "2024-03-23T10:39:02.599965", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.4323994534809771}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:39:02.660650Z", + "iopub.status.busy": "2024-03-23T10:39:02.660376Z", + "iopub.status.idle": "2024-03-23T10:39:03.023974Z", + "shell.execute_reply": "2024-03-23T10:39:03.023111Z" + }, + "papermill": { + "duration": 0.38491, + "end_time": "2024-03-23T10:39:03.025952", + "exception": false, + "start_time": "2024-03-23T10:39:02.641042", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIlElEQVR4nO3deXiU9b3w//c9+5KZyb6RDQg7sohgsVVEragcFfXpsVoVVKo9xePC8TyV81zHtb2oz6n7wumvT4Xaatvjc7TaWkUfq1J3ARFUBNlCCAlJSGbfZ76/P+7JwJAAyWSSmWS+r+vKNZl77rnnk5nkk+/+VYQQAkmSpCzSZDsASZIkmYgkSco6mYgkSco6mYgkSco6mYgkSco6mYgkSco6mYgkSco6mYgkSco6mYgkSco6mYhGmIaGBv7hH/5hSF9DURTuvffek5537733oijKkMYi5QeZiIbZBx98wL333ovT6cx2KNII8PTTT7Nu3bpshzHkZCIaZh988AH33XefTERSv8hEJEmSNExkIhpG9957L//6r/8KwNixY1EUBUVR2LdvH2vXruWcc86hvLwco9HI1KlTWbNmzXGv9cYbbzBr1ixMJhNTp07lxRdfHHA8oVCIO+64g7KyMmw2G5dccgkHDhzo89z33nuPuXPnYjKZGD9+PL/85S/7PE9RFG655Raee+45Jk2ahMlkYs6cOWzYsKHXe6EoCjt37uSaa67B4XBQVlbGv//7vyOEoLm5mUsvvRS73U5lZSUPPfTQgH8+gNdee40FCxZgs9mw2+3MnTuX559/PuWcF154gTlz5mA2myktLeWaa66hpaUl5Zxly5ZRUFBAS0sLS5YsoaCggLKyMu68805isVjKufF4nMcee4xTTjkFk8lEWVkZF1xwARs3bkye05/Pu6GhgS+//JJ33303+bty9tlnp/U+5DwhDZvPP/9cXHXVVQIQjzzyiPjtb38rfvvb3wqv1yvmzp0rli1bJh555BHxxBNPiPPPP18A4sknn0y5Rn19vZg4caIoLCwUd911l3j44YfFKaecIjQajXjjjTcGFM8111wjAHH11VeLJ598Ulx++eVixowZAhD33HNP8rytW7cKs9ks6urqxOrVq8UDDzwgKioqkuceDRDTp08XpaWl4v777xcPPvigqK+vF2azWWzbti153j333CMAMWvWLHHVVVeJp59+WixevFgA4uGHHxaTJk0S//RP/ySefvpp8e1vf1sA4t133x3Qz7d27VqhKIqYPn26+NnPfiaeeuopsXz5cnHttdemnAOIuXPnikceeUTcddddwmw2i4aGBtHd3Z08b+nSpcJkMolp06aJG264QaxZs0ZcccUVAhBPP/10yusuW7ZMAOLCCy8Ujz76qPjFL34hLr30UvHEE08kz+nP5/3SSy+JmpoaMXny5OTvykA/45FCJqJh9h//8R8CEHv37k057vf7e527aNEiMW7cuJRj9fX1AhD//d//nTzmcrlEVVWVmD17dr/j2LJliwDEj3/845TjV199da9EtGTJEmEymURTU1Py2FdffSW0Wm2fiQgQGzduTB5ramoSJpNJXHbZZcljPYnopptuSh6LRqOipqZGKIoifv7znyePd3d3C7PZLJYuXdrvn8/pdAqbzSZOP/10EQgEUh6Lx+NCCCHC4bAoLy8X06dPTznnL3/5iwDE3XffnTy2dOlSAYj7778/5VqzZ88Wc+bMSd7/29/+JgBx66239oqp53WF6P/nPW3aNLFgwYJ+/MQjm6ya5Qiz2Zz83uVy0dnZyYIFC9izZw8ulyvl3Orqai677LLkfbvdznXXXcdnn31GW1tbv17vr3/9KwC33npryvHbb7895X4sFmP9+vUsWbKEurq65PEpU6awaNGiPq89f/585syZk7xfV1fHpZdeyvr163tVY5YvX578XqvVctpppyGE4MYbb0weLywsZNKkSezZs6dfPxvAm2++icfj4a677sJkMqU81jPkYOPGjbS3t/PjH/845ZzFixczefJkXn311V7X/dGPfpRy/8wzz0yJ67//+79RFIV77rmn13OPHuowkM87H8hElCPef/99zjvvPKxWK4WFhZSVlfFv//ZvAL1+MRsbG3uN35k4cSIA+/bt69frNTU1odFoGD9+fMrxSZMmpdzv6OggEAgwYcKEXtc49twefZ07ceJE/H4/HR0dKcePTm4ADocDk8lEaWlpr+Pd3d3H/4GOsXv3bgCmT59+3HOampqAvn+OyZMnJx/v0dPec7SioqKUuHbv3k11dTXFxcUnjG8gn3c+0GU7AEn95T333HOZPHkyDz/8MLW1tRgMBv7617/yyCOPEI/Hsx3ikNFqtf06BiCyvKrx8eIaqHz+vI9HJqJh1tdI5D//+c+EQiFeeeWVlBLC22+/3ec1du3ahRAi5Vo7d+4E1J6W/qivrycej7N79+6UEsGOHTtSzisrK8NsNvPNN9/0usax5/bo69ydO3disVh6lSiGSk9J74svvqCxsbHPc+rr6wH15zjnnHNSHtuxY0fy8YG+7vr16+nq6jpuqWggn3e+jFyXVbNhZrVaAVIGNPb8pz36P77L5WLt2rV9XuPgwYO89NJLyftut5tnn32WWbNmUVlZ2a84LrzwQgAef/zxlOOPPvpoyn2tVsuiRYv405/+xP79+5PHt2/fzvr16/u89ocffsjmzZuT95ubm3n55Zc5//zzM1aqOJnzzz8fm83G6tWrCQaDKY/1vM+nnXYa5eXl/Od//iehUCj5+Guvvcb27dtZvHjxgF/3iiuuQAjBfffd1+uxntcdyOdttVrzYvCrLBENs55G3P/1v/4X3//+99Hr9Zx11lkYDAYuvvhibr75ZrxeL7/61a8oLy+ntbW11zUmTpzIjTfeyKeffkpFRQXPPPMMhw4dOm7i6susWbO46qqrePrpp3G5XJxxxhm89dZb7Nq1q9e59913H6+//jpnnnkmP/7xj4lGozzxxBNMmzaNrVu39jp/+vTpLFq0iFtvvRWj0cjTTz+dvM5wsdvtPPLIIyxfvpy5c+dy9dVXU1RUxOeff47f7+c3v/kNer2eBx98kOuvv54FCxZw1VVXcejQIR577DEaGhq44447Bvy6Cxcu5Nprr+Xxxx/nm2++4YILLiAej/P3v/+dhQsXcsstt3D++ef3+/OeM2cOa9as4ac//SmNjY2Ul5f3Kr2NCtnrsMtfDzzwgBgzZozQaDTJrvxXXnlFzJgxQ5hMJtHQ0CAefPBB8cwzz/Tq6q+vrxeLFy8W69evFzNmzBBGo1FMnjxZvPDCCwOOIxAIiFtvvVWUlJQIq9UqLr74YtHc3Nyr+14IId59910xZ84cYTAYxLhx48R//ud/JrvgjwaIFStWiN/97ndiwoQJwmg0itmzZ4u333475bye53Z0dKQcX7p0qbBarb1iXbBggZg2bdqAf8ZXXnlFnHHGGcJsNgu73S7mzZsnfv/736ec88c//lHMnj1bGI1GUVxcLH7wgx+IAwcO9Cuuvt6DaDQq/uM//kNMnjxZGAwGUVZWJi688EKxadOmlLj683m3tbWJxYsXC5vNJoBR25WvCCH3NZMyR1EUVqxYwZNPPpntUKQRRLYRSZKUdbKNaBQ62aBGs9mMw+EYpmgyr6Ojo9fAyKMZDIaTjuORcotMRKNQVVXVCR9funTpiF5aYu7cub0GGx5twYIFvPPOO8MXkDRoMhGNQm+++eYJH6+urh6y1x6OJsfnnnuOQCBw3MeLioqGPAYps2RjtSRJWScbqyVJyroRXTWLx+McPHgQm82WN0PhJWkkEULg8Xiorq5Gozl+uWdEJ6KDBw9SW1ub7TAkSTqJ5uZmampqjvv4iE5ENpsNUH9Iu92e5WgkSTqW2+2mtrY2+bd6PCM6EfVUx+x2u0xEkpTDTtZ0IhurJUnKOpmIJEnKOpmIJEnKuhHdRtQfQgii0egJ5yZJfdNqteh0Ojk0QhpyOZOIfv7zn7Nq1Spuu+22XqsEpiscDtPa2orf78/I9fKRxWKhqqoKg8GQ7VCkUSwnEtGnn37KL3/5S2bMmJGxa8bjcfbu3YtWq6W6uhqDwSD/sw+AEIJwOExHRwd79+5lwoQJJxyQJkmDkfVE5PV6+cEPfsCvfvUrfvrTn2bsuuFwmHg8Tm1tLRaLJWPXzSdmsxm9Xk9TUxPhcLjX/mD5wuWP0NztZ/qYkbt0Sq7L+r+4FStWsHjxYs4777yTnhsKhXC73SlfJyP/iw9Ovr9/Qgj+vPUgH+zuJB6X88OHSlZLRH/4wx/YvHkzn376ab/OX7169bAuwC5J+w776fCEMOg0hKJxzIbh2YUk32Tt311zczO33XYbzz33XL+L/KtWrcLlciW/mpubhzhKKd/taFNL3dOq7ZgNWlz+CC3O46+FJKUna4lo06ZNtLe3c+qpp6LT6dDpdLz77rs8/vjj6HS6PrvbjUZjcjqHnNaRGQ0NDRnrpRxt4nHB3k61x7WxvIDdHV6eeX8vf/u6PcuRjT5Zq5qde+65bNu2LeXY9ddfz+TJk/nJT34ybBvxSdLxdPpCBCMxDDoN1Q4zgYj6z/GwVz1u0svf0UzJWiKy2WxMnz495ZjVaqWkpKTXcenEwuGwHOczBNpc6g6xVQ4TGo2C1aij0KLH6Y9w0BlgXFlBliMcPfKySyQcjR/3KxqL9/vcSD/OTcfZZ5/NLbfcwi233ILD4aC0tJR///d/T64H3dDQwAMPPMB1112H3W7npptuAuC9997jzDPPxGw2U1tby6233orP50tet729nYsvvhiz2czYsWN57rnn0oovX7QmElGl40gbZlXi+3ZPqM/nSOnJ+jiiow3XzgtPvd17W+UeY0utLJk9Jnn//9uwm0is727bmiIz3zvtyMJsz7y/l0A4tW3rju9OTCvG3/zmN9x444188sknbNy4kZtuuom6ujp++MMfAvCLX/yCu+++m3vuuQeA3bt3c8EFF/DTn/6UZ555ho6OjmQy69mKetmyZRw8eJC3334bvV7PrbfeSnu7bO84nrkNxVQ7zFQ4jMljZTYj21s9dHplIsqknEpE0hG1tbU88sgjKIrCpEmT2LZtG4888kgyEZ1zzjn8y7/8S/L85cuX84Mf/IDbb78dgAkTJvD444+zYMEC1qxZw/79+3nttdf45JNPmDt3LgC//vWvmTJlyrD/bCNFsdVAsTW1yltaoCalTlkiyqi8TEQrFjYe9zHNMbNAbjpr/HHPPXbGyA3fHjuYsFJ861vfSpmSMn/+fB566KFkb+Jpp52Wcv7nn3/O1q1bU6pbQojkVJedO3ei0+mYM2dO8vHJkydTWFiYsZjzQU8icgYiRGJx9Nq8bN3IuLxMRAZd/395hurcwbJarSn3vV4vN998M7feemuvc+vq6ti5c+dwhTYqtLuDtLmDVDpMlNuOtBFZjTrOGF9CkVV2DmRSXiaikeDjjz9Ouf/RRx8xYcKE4w5rOPXUU/nqq69obOy7tDd58mSi0SibNm1KVs127NiB0+nMaNyjxe4OHx/tOcz0MQ6+OzV1wO3p40qyFNXoJcuVOWr//v2sXLmSHTt28Pvf/54nnniC22677bjn/+QnP+GDDz7glltuYcuWLXzzzTe8/PLL3HLLLQBMmjSJCy64gJtvvpmPP/6YTZs2sXz5csxm83D9SCPKYZ/aBnRsG5E0NGQiylHXXXcdgUCAefPmsWLFCm677bZkN31fZsyYwbvvvsvOnTs588wzmT17NnfffXfK9tJr166lurqaBQsWcPnll3PTTTdRXl4+HD/OiNPtCwN9JyJ/OMreTh/7D8t1rjJFVs1ylF6v59FHH2XNmjW9Htu3b1+fz5k7dy5vvPHGca9ZWVnJX/7yl5Rj11577aDiHI2EEDj9EQCKLPpej+/v8vPatjbGFJmpK5FLzGSCLBFJ0jF84RjRuEBRwGbqnYgKzWopyZVIVtLgyUQkScdwB9QEU2DUoT12PAdQmCgleUPRXqPrpfTIqlkOGq4R5lLfXIlE5DD3Lg0BmPRajHoNoUgcVyCSHFskpU8mIkk6xthSK/9jTg2aPkpDPewmPR2REJ5gVCaiDJCJSJKOYdJrqS0+cSO0zaSjwxPCG4wOU1Sjm2wjkqQ0FBjV/+GekGywzgRZIpKkY3y6rwuzXktjecFxFz+bUmVnTJE5ZfqHlD6ZiCTpKLG44P1dnQgBDaXW455XXWimGjkqPVNk1UySjuILRxECtBoFq9yxY9jIRCRJR+lpfC4w6k64M3AsLtjT4WXrAWdy5UwpfbJqJklH8YaOJKIUsSggQKuOLYoLwctbDgIwscImF9IfJJmIJOkonp4SkemoP432r2HHqxCPwdgFUHc6eq0Gs0FLIBzDE4zKRDRI+VU1EwKi4eH/GkDR/dlnn6WkpIRQKHUp0iVLlsgJqsOgV4nI3wVf/1n9HOMx2P036G4C1LFEAJ6g7MIfrPwqEcUi8PeHhv91z/wX0PVvXZvvfe973Hrrrbzyyit873vfA9TdN1599dUTzqyXMsN7bIlo/4dqtayoHkwOaN0K+/4ORfUUGHW0E8IX6r0ZqDQw+VUiGgHMZjNXX311cucNgN/97nfU1dVx9tlnZy+wPHHmxFL+x5waxpcVQCQA7V+pDzScqX4pGnA2g78Lq0FNVr6wHF09WPlVItLq1dJJNl53AH74wx8yd+5cWlpaGDNmDOvWrWPZsmUn7MWRMsNu0mPvWfrj4BdqachaCo4adbeE4rFweDcc+hKLUd0BxS8T0aDlVyJSlH5XkbJp9uzZzJw5k2effZbzzz+fL7/8kldffTXbYeWfw7vV2/KpR7ZsKZukHu/ag7X8FABZNcuA/EpEI8jy5ct59NFHaWlp4bzzzqO2tvbkT5IGJRiJsbmpG5tJzylVVujeqz5QctSWUkUN6q2njbpGDRedUtXnKo7SwMg2ohx19dVXc+DAAX71q19xww03ZDucvOAKRPh4bxcf7TkM7gNqtcxYAAUVR04yOcBSDCJOUeQQkyptlNvlfLPBkokoRzkcDq644goKCgpYsmRJtsPJCyljiFwt6kFHbe+dNAvr1FtX8zBGN7rJRJTDWlpa+MEPfoDRKBfeGg4pY4hcB9SDjpreJ9qqABDuVna1q9M85JKxgyPbiHJQd3c377zzDu+88w5PP/10tsPJG/6eRGTQQmeiRGQf0/tEu7pFk+JtY/0XrYRjgtoii9z9dRBkIspBs2fPpru7mwcffJBJkyZlO5y84QurvV/2uBOiIXXYxdHtQz0spaDVQTRMseKhjQK8oahMRIMgE1EOOt6+ZdLQ6hkP5IgdVg/YKkHTR+uFRqMmKFcLRcJJGwX4w7ILfzBkG5EkJfSMB7KEE4mor9JQD2sZAIVxl/pcOahxUEZ9iUiuFTM4+fT+LT6lCk8oQunuT9QDBSfYjttSCoAt3g0K+OWgxkEZtSUivV4dZOb3y/3JB6Pn/et5P0czh0VPTaEZQ7BDPXDCEpGaiAqiTkCWiAZr1JaItFothYWFtLe3A2CxWORcrQEQQuD3+2lvb6ewsBCtNk/W2wm5IRJUJ7cmSj19SiQiU9SNoovK+WaDNGoTEUBlZSVAMhlJA1dYWJh8H0czbyjK1gNOSoPNTAR19LT2BH8ehgLQm3CYY/zDeCMFJSdIWtJJjepEpCgKVVVVlJeXE4nIxasGSq/X501JqNsX5uM9XUwJ7WGilRNXy0AdbW0pxRQJ0mgNgkNO8xiMUZ2Iemi12rz5g5LS09PGYxdqL1hPr9gJWYrVEdiB7iGMLD+M2sZqSRqInnFAtrhbPWApOfmTzEUIBK1tB+U0j0GSiUiSSHS/C4E1ligR9TMRKSh8tbuJt7a34wvJBut0yUQkSahVM0PMh1GJqT1m5sKTP8lcBIBNeAEIRORYonRlNRGtWbOGGTNmYLfbsdvtzJ8/n9deey2bIUl5yh+OYo460WsVNcFo+tGmaCoEwEwIbTwsp3kMQlYTUU1NDT//+c/ZtGkTGzdu5JxzzuHSSy/lyy+/zGZYUh7yhWKYIi70Wo3aCN0fehMYLOi1Csaom4BMRGnLaq/ZxRdfnHL/Zz/7GWvWrOGjjz5i2rRpWYpKykcXz6gmYtiKrVvXv/ahHuYidJp2TFG3rJoNQs5038diMV544QV8Ph/z58/v85xQKJSy8aDb7R6u8KRRzmHRg8arzqwfSCIyFaLXKmoikiWitGW9sXrbtm0UFBRgNBr50Y9+xEsvvcTUqVP7PHf16tU4HI7kl1xQXsoof2LW/QBLRHqtBlPEJduIBiHriWjSpEls2bKFjz/+mH/6p39i6dKlfPXVV32eu2rVKlwuV/KruVmuGSwNnjsY4cMdLbR3Jia79reNCMDkoNCiZ161jlPrC4ckvnyQ9aqZwWCgsbERgDlz5vDpp5/y2GOP8ctf/rLXuUajUa7fLGVcty/Mtm/2YXEFKS8pBr25/082ObAYdFiMYbDJaR7pynqJ6FjxeDylHUiShpovFMMUTfSYmQdQGgIw2dXboBvyaO2mTMtqiWjVqlVceOGF1NXV4fF4eP7553nnnXdYv359NsOS8kwgEsUUdWPQKv0byHg0o52YAKfHR+u+NqY1VMrlZtKQ1UTU3t7OddddR2trKw6HgxkzZrB+/Xq++93vZjMsKc/4QjGMUQ96nSY5WrrfNFqEwcrOvYfZxm4m1lZg0MlENFBZTUS//vWvs/nykgSoo6pNUTd6kyY5WnogtGYHGgWMMS+BcAyDLudaPHKefMekvKe2EbkTbUQDLBEBismBXqvBGPXKQY1pkolIynuBYBBDzJ+YZ1Y48AuY7Oi0CsaYRy4Zm6asd99LUrZdMqkA3DbMZjPoLQO/QKJEZIj6ZIkoTTIRSXnPjhdMerAWq0vADpTRgV6jwRj2ymkeaZJVM0kKOtXbdKplcKRqFvXIElGaZIlIymuuQISWPU2UeoKU1w28oRoAk4OyAiM2UwzK5ejqdMhEJOW1Ll+YfQdaiMaClKfRdQ+AzojVasEaDYMunNH48oWsmkl5zRdKjCHS9XN52OMxJqZ6hOTSNOmQiUjKa/5QVB1VneYYoh4RnZXDvhB7WtoyGF3+kIlIymshvxONiKHX6cDoSP86Ggs7D3nZsutABqPLHzIRSXkt4lU3R9SY7erqjGkyWNUkpgl7iMr9zQZMJiIpr8V8XQDorANc/uMYeosDBTDEfASjMhENlExEUl6L+9US0WATkWK0odcqGGI+Oc0jDbL7Xspr36nRE9PZMRX3Y6/7EzHa0Gk1aokoLEtEAyUTkZTXrHG3Or2jYAAL5vfFaEOnUdCHgwRCYSCNOWt5TFbNpPwWcKq36Q5m7KG3oNPpAEEoIMcSDZQsEUl5y+X24D50GJNeS9lgBjMCKAoVZaWUWp3orbJqNlAyEUl5q7urnVZnAJPVRplu8LvDFBaWgBIATSAD0eUXWTWT8lbYo3bdY0p/RHUKY4F6G/Jm5np5RCYiKW+FE2OINJbCjFwvqFGnebQcas/I9fKJTERS3or51DFEWktmSkTOmJGdh7zsaj6YkevlE5mIpLwVD6iJSF8wuMGMPfRWdQa+CHkycr18klYi2rNnT6bjkKTh53cCYMhQIjImqnhKyIuQu74OSFqJqLGxkYULF/K73/2OYDCY6ZgkaejF44igCwCjbZCDGRNMiYmv+piXkFwydkDSSkSbN29mxowZrFy5ksrKSm6++WY++eSTTMcmSUMn5GJqVQHTaoopKc5MiUhntqPVKCgiTsAve84GIq1ENGvWLB577DEOHjzIM888Q2trK9/5zneYPn06Dz/8MB0dHZmOU5IyK+DEpNNiLyzFqM/QcDqtDkVvBiDok6OrB2JQjdU6nY7LL7+cF154gQcffJBdu3Zx5513Ultbm9zTXpJy0mB37jgeow2AsM+Z2euOcoNKRBs3buTHP/4xVVVVPPzww9x5553s3r2bN998k4MHD3LppZdmKk5JyihPdyf7u/zs9xsyet366komVhRQZoxk9LqjXVpl0ocffpi1a9eyY8cOLrroIp599lkuuugiNIkV7saOHcu6detoaGjIZKySlDFeVyctzgA+m4a6DF63vKQEwgdAyGkeA5FWIlqzZg033HADy5Yto6qqqs9zysvL+fWvfz2o4CRpqESTgxkz01CdlJzmIccSDURaiejNN9+krq4uWQLqIYSgubmZuro6DAYDS5cuzUiQkpRpyZUZCzI0zyzBg5mwL4Q4fJjSjF55dEurjWj8+PF0dnb2Ot7V1cXYsWMHHZQkDalIkFjID4BhkEvEHqvVr2PnIS8H5HyzAUkrER1v1KjX68VkklvuSjku6CQcixPRmDGbzRm9tCE5zUOOIxqIAVXNVq5cCYCiKNx9991YLEeWw4zFYnz88cfMmjUrowFKUsYFnERigpDOhsWQ2SW5jInR1SLsg3gMNNqMXn+0GtCn8NlnnwFqiWjbtm0YDEe6Pg0GAzNnzuTOO+/MbISSlGlBJ5FYnKDBjtWY2URhMtsQikbd2yzsBVP6mzbmkwElorfffhuA66+/nsceewy73T4kQUnSkAo4mVptp75yPDbr4FdmPJrZqCOssaDEvEQDbnQyEfVLWm1Ea9eulUlIGrmC6vSOktJyDLrMroRj1GmI6qzqy/hcGb32aNbvEtHll1/OunXrsNvtXH755Sc898UXXxx0YJI0ZDK1c0cfFEVRxxKFDhGWiajf+p2IHA6H+iYnvpekESkex+/potMdIODWMSmzw4gAmFxfjaX9EAWKXCKnv/qdiNauXdvn95I0ooTc+INhDrhCtLTHmFSf+ZeoqSgHnxFi/sxffJRKq4IcCATw+4+8yU1NTTz66KO88cYbGQtMkoZET4+Zzo7VpB+a10jMwJfTPPovrUR06aWX8uyzzwLgdDqZN28eDz30EJdeeilr1qzJaICSlFGBbnUMkdaG2TA0Y3xcMSOHfSFcrq4huf5olPYKjWeeeSYA//f//l8qKytpamri2Wef5fHHH+/3dVavXs3cuXOx2WyUl5ezZMkSduzYkU5IktQ/AXVUdVBnx5rhwYw9djph5yEvHZ2Hh+T6o1Faicjv92OzqcXPN954g8svvxyNRsO3vvUtmpqa+n2dd999lxUrVvDRRx/x5ptvEolEOP/88/H5fOmEJUknF+gmEo0T0tkyPpixR880j3gkBNHQkLzGaJPWv4TGxkb+9Kc/cdlll7F+/XruuOMOANrb2wc0vuj1119Pub9u3TrKy8vZtGkTZ511Vq/zQ6EQodCRD9btlstxSgMU6CYcixPQOygwDk2JyGSy4NHoicTi6q6vGdjOerRLq0R09913c+edd9LQ0MDpp5/O/PnzAbV0NHv27LSDcbnUcRfFx1nMfPXq1TgcjuRXbW1t2q8l5SEh1EQUjRPUDV0iMuu1hLVWonEBYdlg3R+KSHMDpra2NlpbW5k5c2ZyXaJPPvkEu93O5MmTB3y9eDzOJZdcgtPp5L333uvznL5KRLW1tbhcLjnSWzq5kAc+eJJgNE7XnNsod1jQaTO/x2iHJ8TmV56iJNLKaRcshcrpGX+NkcLtduNwOE76N5r2v4TKykoqKytTjs2bNy/dy7FixQq++OKL4yYhAKPRiNEoi7lSmhI7u5psxVQXFwzZy5gNWkJaK9GgQITcKEP2SqNHWonI5/Px85//nLfeeov29nbi8XjK4wPdCfaWW27hL3/5Cxs2bKCmpiadkCTp5PyJ7nTzEAynPopZryWitSCAsN+N/Nd5cmklouXLl/Puu+9y7bXXUlVVlZz6MVBCCP75n/+Zl156iXfeeUeu7igNrUA3nlCE/d1aDIc8TKiwDcnLaDUKsxprKWrZjTYie4D7I61E9Nprr/Hqq6/y7W9/e1AvvmLFCp5//nlefvllbDYbbW1tgDqXLdMr50kSgS48wShb/Roc7d4hS0QA42uqwGmEiFypsT/SaqkrKio6bs/WQKxZswaXy8XZZ59NVVVV8uuPf/zjoK8tSb0kxhAFdQ6sQ9RjlmRItEGFZSLqj7Q+jQceeIC7776b3/zmNynLxQ5Umh12kjRwPV33iVHVQ9V136MzYkDxhTBHnFiEgDSbL/JFWp/GQw89xO7du6moqKChoQG9PnXy4ObNmzMSnCRlTMgDsSihGIR0tiFPRJvbwtgO+agrNmOJ+MFgHdLXG+nS+jSWLFmS4TAkaYgluu69ihWhaIdsekcPs9FARGtKjK72yER0EmklonvuuSfTcUjS0Ap0IRC4UdtuhrpEZDFo6dZaicbc6jSPoWsXHxXSHlbqdDr5P//n/7Bq1Sq6utTxGZs3b6alpSVjwUlSxgS6icUFPq06uneoG6tNei1hrYWInObRL2l9Glu3buW8887D4XCwb98+fvjDH1JcXMyLL77I/v37k2sVSVLO8Heh1Sicf9oUPCV16IdgasfRkvPNIkIukNYPaX0aK1euZNmyZXzzzTcpO7tedNFFbNiwIWPBSVLG+LtQULAWllPpGPrdiM2GRImoZwa+dEJpJaJPP/2Um2++udfxMWPGJAclSlLOiMeSjdVYSoblJS16HRGt9chGi9IJpZWIjEZjn2sB7dy5k7KyskEHJUkZFegGEedwUPD3/SGau4Z+UXuLUctpE+tpLC9AyKrZSaWViC655BLuv/9+IpEIoO7ltH//fn7yk59wxRVXZDRASRo0v7pka3vUysYmJwe6A0P+knqthkn1VRRbjSgyEZ1UWonooYcewuv1UlZWRiAQYMGCBTQ2NmKz2fjZz36W6RglaXASicilUffjs5mGeHpHj55pHpEAxKLD85ojVFqfiMPh4M033+T999/n888/x+v1cuqpp3LeeedlOj5JGjxfJwBORe26tw/VNkLHaPMraP0xbAYwhb1gLhyW1x2JBpyI4vE469at48UXX2Tfvn0oisLYsWOprKxECJH2kiCSNGT8hxEIuuLqqMLhKhG9v/swJZ0xphULTCGPTEQnMKCqmRCCSy65hOXLl9PS0sIpp5zCtGnTaGpqYtmyZVx22WVDFackpUcI8B8mFhfJqlnBMCUitQtf9pz1x4A+kXXr1rFhwwbeeustFi5cmPLY3/72N5YsWcKzzz7Lddddl9EgJSltITfEIgQTk10tBu2QD2bsYdZr8WktRGMuOZboJAb0ifz+97/n3/7t33olIYBzzjmHu+66i+eeey5jwUnSoCUaqgNaO0LRYhum9iE4UiKKxOJymsdJDCgRbd26lQsuuOC4j1944YV8/vnngw5KkjLGpyaikvJqlp85lgumV57kCZljPnq+mezCP6EBVc26urqoqKg47uMVFRV0d3cPOihJyphEiUhjLRnW0hAc1UYUlNM8TmZAJaJYLIZOd/zcpdVqiUbleAkph/ja1Vvr8I/475n4GokJ2Vh9EgMqEQkhWLZs2XH3Fjt680NJyjohwNcBwKcdWoLuDk4Z46DQYhiWly8pMHDGlAaqdljURnO5ZOxxDSgRLV269KTnyB4zKWcEnRANg0bL1i4t7lA3E8qHb4Uyi0HH1LHV0GxQR1ZHQ6Af+pn/I9GAEtHatWuHKg5poPxd0LFDXYK0fCpoh2nawkiSGFEdN5fgcasbNQzb9I4eWr2afCJBtXomE1Gf5G/vSHR4N3zxIsQT7XGtW2DG90E3PFWOEcOrtg/59cUIAQadBothaNeqPlaLM4A2qKOEOPqQB6ylw/r6I8XwjOySMifkha9eVpOQrRJ0RnC1wO63sh1Z7kk0VLt16hbTdrN+2Kcg/W37IbYciuEPx2QX/gnIRDTSNL2vtjXYKuDU62D65erx1s+TJQApwas2VDs1aiJymIe3+x7UdqKw1kI4Kqd5nIhMRCNJ2KcmHIDx54JGC0UNUDZJ7ZFp+iCr4eWUWAQC6qYOh4XaQJ2NRGQ1HjW6Wo4lOi6ZiEaStm3qsqf2KiiqP3K8/gz1tnOn/GXv4etUk7PeTHdUHW5SmM0SUSyuduFLfZKN1SOFEEdKQ1WzUh+zVYK9GtwH4dCXUHf6sIeXc3oGMhaU8w8zqvGGoxiGabLr0axGnVoiisiq2YnIEtFI4W1Xu+y1OrW7/liV09Xbju3DG1eu6mkvs5aj0SjYTXpM+uHtMYMjVbNwVFbNTkQmopHi8DfqbdHYvrvpSyepo3bdrRBwDmtoOcnTqt7ahm+Sa1+siaqZOs3DB/F4VuPJVTIRjRSdiURUOqHvx40F4KhNnLtzeGLKVfE4eA8B0KEU8erWVjY1dWUllJICAwumN1BfWgAiDhFfVuLIdTIRjQRBN3ja1BJP8XiEELz9dTtfHTym8bMnSXXtGf4Yc4m/U51SoTPQHrWy85CHvZ1Dv4VQXywGHafUFlFUqA4hkNWzvslENBJ071VvbVVgLGBjUzdbmp18tOcw8bg6dcEfjuIy16jnOZvV7ut81VMtK6ik26+OPi+2Dn+PWQpjYkcP2WDdJ5mIRgLnfvW2qB6XP8KHu9U1duY2FKPRKHR6Qzz/8X7e3BdDGAvUUdeu5iwGnGUetVqGrZIufxiAomGacd+Xg84AB/w6QtGYWrqVepGJKNcJcSQRFdbz6b4uYnFBXbGF6WPU7XH0Wg2BcIzm7gDtumr13K69WQo4ByQbqqvo8qpL05RY+166Zjhs2NnBJ60xfKGouiKA1ItMRLku6FT/i2q0eE0VfNWq/kf91viS5Lwph1nPKTXqDhXb/MXq87r3ZSHYHBCPJbvuYwWVuAJq1awoi1Uzq1FHSGcjHBMQdGUtjlwmE1Gu6ykN2ar4uj1ILC6ocpgYU2hOOW12XRGKAl8Hi/CHo+qCYJFgFgLOMl+HWjXVGXHGLcSFwKDTUGDM3thdq1FLSFtAJBqXieg4ZCLKdYlEJApr2Z4oDU2rdvQ6zWHWM66sgIjWQmvEolbpXAeGNdSc4GpRb+1j8IZj6DQKxVZDVjf+tBh6SkQyER2PTES5LpFMhL2O2XVF1BZbmFBR0OepU6vUyZ27w0UIIfKzwdqdSL6OMdSXWFmxsJFLZ1VnNSSrQUdIV6BOfI0E1FUjpRRyrlkuC/uSo6Q1jmqm601MH9O7NNSjocSKUa+hXSnDEzqAPR8TUU8p0KEOZdBoFCyG7P6aF5h0xDRGAiLRThVyg04ukHY0mYhymTvR+2Mp6dcSozqthvOmVFCkFGD7cps6CDIWUZcrzQdBt/qlaMCW3VLQ0Xrap9zCCiSqZ3KlxhSyapbLPAcBCFoq2HrAiSd48kGKEytslJVVoBjtag+Su2Woo8wdPaWhgjJiGj3PfdzE61+0qhNOs8hh1nPulHKmjK1BIGQXfh+ymog2bNjAxRdfTHV1NYqi8Kc//Smb4eSeRInoQLSIt7a385etrf17nqIkqyY486h61pN0HbV0+cK0u0Ps7vCh12Z3Cx+DTsOMmkLKyypQUGSDdR+ymoh8Ph8zZ87kqaeeymYYuUmIZImoKaIOXGwosfbrqa2uAB92WTjoCuRnicg+hg6POpCxzGbMao9ZClOifU8mol6y2kZ04YUXcuGFF2YzhNwV6IZIEKHRsjdgBQQ1ReaTPg2g2xdhq7sAqydEteuAOhtdM8pr4ZFgcsY9jho69h9JRLmg3RPE7dZQFY5ilYmolxH12xkKhXC73Slfo5ZbLQ0FjKV4wgKtRqHS0b89scaWWgkYinFHNYRCwSOrFY5mzv1qKdJSAib7kRJRQW4koi37nfxtX4huX1iWiPowohLR6tWrcTgcya/a2tpshzR0EomoQykBoMJuRN/PpU7NBi2VDgseQwWuQOTIIL/RzNmk3hbVI4RIJqLyHCkRFZh0BLUFid08/HIs0TFGVCJatWoVLpcr+dXcPIobYhPtQy0xdR2bMYWWAT29rsSCx1iJ0x/Jj4GNPXPrCutx+iMEI+qo6pIcKRHZTXpiWhMBEqsAyJ6zFCNqHJHRaMRozI1frCEViyYnbu4Lqw2c1YUD26q4ocTKdmMFLk+EuLMZjRBqb9poFPKqu3YoChTWEfDHKC0wYNRp0Wpy42dOjiVSCoCwuv54QXl2g8ohIyoR5Q1fuzoGSG/msrnTaHEFGdPPhuoelXYTEWslkXbwebqxBV1gLhyaeLOtp1pWUA4GC9UGuHZ+A7HEonG5oMCk/qm5sAGH1c4IKSmricjr9bJr167k/b1797JlyxaKi4upq6vLYmRZlmgfwj4Gs1FHY3nfc8tORKNRqCl1oHRWEot71W780ZqIetZeKqxPOZwrpSEAWyIReRQbsXgnWpmIUmQ1EW3cuJGFCxcm769cuRKApUuXsm7duixFlQOSiahqUJe5cHoVWstMaP5UHWNTMS0DweUYIaBrt/p9yXjicYEgt5IQgFGnxaDTENTZCcfimAPZWcw/V2U1EZ199tnqLHEpVWKFwfcPGVBincysKcSaxno6Wo0C9hrg09HbYO0+qPZC6QzgqKXFGeDlLS2MLS1g8YzBJfJMO29KBeYg6HdvkVWzY8g2olwTCYC/i2g8zmaniai7i5k1helfz1FDTAhi7nYMkWC/Js+OKIcTVfvicaDRsr/LTyQmyMKmric1qdIGER3s1agN7NFw33vU5aEc/LjyXKJa5sRGVGOi0KJPqzTU4/P2KO8diNF82Dc6p3v0JKKSRgCau9Rtg2qLBzbcYdjozUf+GchSUZJMRLkmUS3rQB3IeOySsANlN+txJQY2itFWPQu61WEOigLF4whGYrS51eVxczEReYIRtre6ORRJxCYTUZJMRLkmUSI6EFPHDw202/5YNUVm/KZKQtE4vvamQYeXU3p2tLVXg8FKizOAEFBo0WM35d4aTIfcIV7/oo0d7kRsssE6SSaiXCIEuA8SE4L9UXVEdc0AR1QfS6/VYC1vAMDdvl8dnzRatG9Xb8umALCnQ93Oub+rFAw3h1lNQF0iMRxDloiSZCLKJYFuiATwhgUeXTEFRh128+D7E6qrxxDVGHF5/eqqjaNB0KUOSVAUKJuEEIK9neouquPKcjsRuSkgGo+ro6slQCai3JKolvmNpRgMesYUmTOyls7YMhseYyXuYIRw9/5BXy8ndOxQbx01YLITiwvmNhQzrsxKTVHutQ+BukCaxaAloC8kFImrWx/J4SuA7L7PLYlEVFUznh+NH69uP5MBRRY9wj6GeKCJrpY9VDbMz8h1s+qYaplOq2F2XRGz64qyGNTJOcx62kKFBCMCazQEYS8YbdkOK+tkiSiXeHpGVFej0SiY9NqMXFZRFMY1TqK+xEJh9NDI/y/sO6wmbUUDZZOyHc2AOMx6hKLDq0kkH19HdgPKETIR5YrEjHuBGPTUjr5MmTCJ6qICTPHQyG8kbd2i3paMB2MBra4Anzc7CUZyvyE+2U6kSSwb6+vMYjS5QyaiXOE9BPEYLT4Nz2zsZvP+DCcLrQ5siQQ3kneAjcfg0Bfq91UzgcTqh1+388Hu3P+jnlRpY8nsMYxraFAPyEQEyESUOxLtQ4coxhWMEh+CJSxC1iraPUEONO3M+LWHzeFd6twygxWKx+MNRfmmXe0t62sr7lxTUmBkbKkVa1Hin4KsmgEyEeUOdwsCQXO0EBj8QMa+HIiXsLvDx4G934zcycYtm9XbylNAo2Frs5NYXDCm0EyFfQTNo7MkNlj0d478NrsMkIkoV7gP4g/H6NKWYtBpKLdl/o+qpmECGkVB+A/T2TkCqwTednVJWEWB6tmEojG2tqgL0c+uK8xqaAOxp8PLh62CQBR14mtoFG8C0U8yEeWCoBuCLtzBKF5DOVUO05Csp2M0F2AuUTdebN7zRcavP+SaP1FvyyaBuZBNTd0EwjGKrQbGlw188bhs2bzfyUf7nHQriZ4zbx7ssnISMhHlgsRk1A6KiGkMQzogr6RuMgDdzTtGVvUs5IH2r9Tva0/HH47y2X4nAGeML0GTYwuhnUiJVV36w6kpVg94+rmD7ygmE1EucDar7UOxxIz7IWgf6lHRMBWtRsHoaeLAYe+QvU7G7f9I7TFz1IC9GgWFsaVWKuymtJbSzaaiRCLqUBKDLz2HshhNbpAjq3OBq5l4HEprGnEJE5VD2Oiqc4yhyOGgs9vJrl1fU1s6d8heK2OCLjj4mfp9w3cAde+2i06pIhSN5c6W0v1UbFET0aF4oVoU8LSqDdYj7OfIJFkiyrawD3ydaDUKp8+aydWn1w3tessaDeWJ6pnRvW9kVM/2va+WhgrrCBbUpsRs1GVm9PlwKrWpiag16iAqUH8HQp7sBpVlMhFlW8/gQmspGIZnsqajdgqn1hVyhqOLnP8f7DkEbdsACNedyUtbDvLnra0jYhT18VgMOmwmHXGNDq+2UD3oze/qmayaZZuzGSEEndpyimJxdMOx2HJJI0aDUV2GwnsIbJUZuWwsLjjoDLDvsI8uX5hAOIZGo+Aw6zl/asXAq1BCwDfrQcSJlkzklX0a2lx+THotoWg8Y3PxsqHMZsQTjNKlLaEw7lSX8S2dkO2wskYmomzr3os3HOW1AwbC/n3c+J2xQ9/moTNCyTjo2In3wBccLrdRP4jFxOJxweb93WxpduIJRns97g1GU36mA91+SguMJ08krVvA1UJIaHnZPYWWgB+DTsOS2dXJOVsj1bcbSzlrQhmFLh/s2D2yp91kgExE2RR0ga8TZyCK0zyGsXbT8DW8lk/F0/wlmz95ny/r6ll6xti0F+nXaBRanAE8wShmg5axpVaqHCYsBl2v3VYjsTivfH6QWEwwocLGlCobtUWW3t3v/i5i3/w/OtxB3mcmLRYdZoOWS2ZWU+UYul7F4VJa0LN1emIjUXerOvFZm59/kvn5U+eKxA6lbfEiYlrT8C5xWtKI1WKhWNuJwXuQDTsLuPCU/s/67/KFsRi0yVLNgolljC8rYHKl7YTVS28wis2kp9MTYnurm+2tbqxGLXXFFspsRuqKrZRZdbD9Ffa3u9gZKmR/+VQq7CYWz6ga8SWhXizFattg2K/2nhXWZjuirJCJKJu69xKNx9kv1Daa+tJhXFlQq0dTMZWxgU00ObfzdVsVNUUWTqk58cTRWFywcV8Xn+ztYlKljfOnqbEXWgwUWk6+R1eR1cA1p9fR5g6yvdXNzkNefKEY21s9bG/1cO5khbID74O7lbIiOxu032VhYyUzxjhG1KDF/viixUXTYT/fNlRQGN6rVs9kIpKGVTwO3ftwBSJ0mcZQbDUM/84T1bMpaP2cWaY29sX8vL2jHZNew4SK3isGCiHY3+Vnw84OOr1hAPzhGLG4GPBwA0VRqHKYqXKYWTCxnJbuAC3OAN3+MFXOTdC5DRSFgtlXcG3RuFGXgHrs6fSxu93LOEcZhewF536oHwWrZ6ZBJqJscTVDJEhHUIPXXs7cbMyVsleBvYoxHGRufC8fxKfxl62tzG0I8Z0J6uzwYCTG3k4f21pctHQHAHUw4dmTyphUYRt0m5ZWo1BXYqGu2Az7/g77PlUfaPwulIwf1eNLxhSa2d3upSleyhQA136IRUA7yqqf/TCaP+fc1rmTuBDsEVWgaLI3TaH2WygozNXv4dTq3iO6mw77ef2LNlq6A2g1CqfWF7F0fgOTK+2Za1iPRWHHX9WBiwDjF0LNnMxcO4f1bJ65N2BFGAvU98E5SjY3GCBZIsoGIaBjB4oC35p3BuWilAq78eTPGwplk6CgDI23gwWm3YybczoOy5H/yJUOE6U2I41lBUwfY8eW6eqj5xDseFW9VRRoPA9qTsvsa+SoMpsRvVYhGI3jKa7HHvoSuvaoS+DmGVkiygb3QQh5ULQGimsmcVpDcfbmSykKNJylfn/gU2oNvpS2KodZz7Xfqmf++JLMJqGQF775f7BpnZqE9CaY8Y95k4RArZZWJ0pF+0n0WB7elZcLpclElA2HvlRvSyfkxriR0gnqVzymlk5ivQclZoznEOxcDx+vgQOfgoirpbK5y6F43NC9bo5qKFWHbOwIl4BGBwFnXk73yIG/gjwTi8KhL+jwhNhmrGBitz/7GwIqCkw4X22fcLfCztdg8j9kZja4EOofVudO6PwmdREwezWMPTMvE1CPcaVW/r6zEzQGRMl4lI4d6j+qDE27GSlkIhpunTshGuJAQMfn3kJsrmD2ExGAyQ7TLoOt/wVtidUbJ14w8B4cIcB/WE1qrmb1NnTUukcaLZROhOpZUFif10tfgDr+6uYF49SBoR1+dQfb9q9g3ELQ5E+FRSai4XZwM75wlN2acSgaDVOq7NmO6IjisTD5Ivj6r2oycrWoJZbSSX1XIYVQl7DwtIG3Tb11t6ijhI+m1avXLpkAJY3DtsrASJGcc1c8Tp0HGPJC9968arSWiWg4uVrA2cxBV5g221QmVtgoSHN+15CpPEXdqufrv6obMX71itp2UVCuHtdo1bEuIQ8Eneri78fS6MAxBhy16khhe01utIXlOHdEYCqdiqHtMziwUSYiaYg0f0QwGuNrUUdEZ2VOfY7u0148Tm08btkIB7eoSSex71ovigKWErVNo6BSvbVVycQzQO/saGdLs5Oz6yYzS9miduN7O6CgLNuhDQv52zJcXAegYyfN3QFabDOoK7bk9j5cepO6LGv9t9WSkbcdIn61OqbVgd4K5iIwOWTSyYAiiwEhYHO7YEbJBDSdO6HpfZi2JNuhDQv5GzQchIBd/w9/OMpX8QYC+iLOTEyhyHmKos4QtxRnO5JRbUqVnQ/3HMYViLCraiYTlW+gfbs6rspRk+3whlz+NMtn04FPwd2K2Wxm3NwLmVHjoDyXS0PSsDPoNJxap1bV/96qJVo+XX1g5/qhHdeVI2QiGmrug7DnHQCU8edwyvgazp1Skd2YpJw0q7YQm0mHOxDhY6aD3qxWiRO/P6OZTERDydcJW/+LDrefcOF4qJ6d7YikHGbQaTh7UjkAn7SE2V+xUH3gwKdHdrkdpXIiET311FM0NDRgMpk4/fTT+eSTUfCmd+0htulZ9rV18rnTxB+8MwlG49mOSspxjeUFzKotxKTXYqqYpI7jAtj1Fuz+m7qO1SiU9UT0xz/+kZUrV3LPPfewefNmZs6cyaJFi2hvH6H7gfs6EV+9TNcHv+WLpkN8E7CzvewiJteUYdRl/e2WRoCzJpZx5dxatR2x/tuI+jMIx+Kw/2PY9Ax07Bx1CUkRWd5h7/TTT2fu3Lk8+eSTAMTjcWpra/nnf/5n7rrrrhM+1+1243A4cLlc2O3DPEI5HoNoEBHyogTUbXl8bbs41LIXpz9CKCo4VDCF9vJvc860ahrLe696KEn9savdw0cffcDs4EcUG2JYDDqMVjv60vFoi2rAXKwOo9Bb1FHsOTRtpr9/o1ntvg+Hw2zatIlVq1Ylj2k0Gs477zw+/PDDXueHQiFCoVDyvtvt7t8Lde1Ri7WJnOsORPi61QWIo3YNFerjQlBbbKXaYQQEnmCELw+6UYQA4urOwMTRxMIIoKbQnFzKIR6KcMgdpstcT2flqYwfN5FF9UWYDSN3/y0p+/Z1+ukwj+UtQxVjPJ9T3r0DXfwA7DqAVqMws8aR3PH2gDvEIW8cFA1xRQNoEIr6BTCjphBzYkpJc5efg65A4lXU5FVXbKHK0c8e3RlXqnMUMyCriaizs5NYLEZFRWovUkVFBV9//XWv81evXs1999038BeKhtVRqgkiECHmO0ESC0bBpCYXJRxFhLwcXWxUU5IqpBjUWeTWUkwFNURrHUytKKO2yDw8myVKo965U8qZVVfI7nYvbe4Sdrm/heI6gC14AEu4C8w6iPkgHiMSiRIOBo9/MV8UEolI+PzEvYHUx40W0PVzuyaRuerhiBrQuGrVKlauXJm873a7qa3tx64HhbUw8/vJIqspKqj3JeZIKYq6KJmiQQEUjUKBUQ8GHSgKxrig0R8BBRRFg6IoKIoGoTOhNZixGA2QaPvRA2eO/rFn0jBTFIXSAuNRe6GBEI2EonGCkRh6k14t0MQiFHncGHx+1BJ+HCUeA+LJ2oDOZoTEZgSFgTC6UCxlHbYCiw6Onf94vNYbQ+a2v8pqIiotLUWr1XLoUOpCUIcOHaKysvd6LEajEaMxjSVVDVZ19nfPdYDqfk7h0QNlJ95hR5KGnaIoas/a0bvl6gwUFpVS2M8pjPYiyJW1H7JadzAYDMyZM4e33noreSwej/PWW28xf35+bqsiSfko61WzlStXsnTpUk477TTmzZvHo48+is/n4/rrr892aJIkDZOsJ6Irr7ySjo4O7r77btra2pg1axavv/56rwZsSZJGr6yPIxqMrI4jkiTppPr7Nyr7lyVJyjqZiCRJyjqZiCRJyrqsN1YPRk/zVr+nekiSNKx6/jZP1hQ9ohORx+MB6N/oakmSssbj8eBwHH9k8IjuNYvH4xw8eBCbzZa9veOzqGeKS3Nzs+w17Cf5nqUn3fdNCIHH46G6uhrNCTaMHNElIo1GQ02NnNxlt9vlH9UAyfcsPem8bycqCfWQjdWSJGWdTESSJGWdTEQjmNFo5J577klvRYI8Jd+z9Az1+zaiG6slSRodZIlIkqSsk4lIkqSsk4lIkqSsk4lIkqSsk4kohw10B9wXXniByZMnYzKZOOWUU/jrX/86TJHmloG8b+vWrUtsiHDky2Tq53Y6o8SGDRu4+OKLqa6uRlEU/vSnP530Oe+88w6nnnoqRqORxsZG1q1bN6gYZCLKUQPdAfeDDz7gqquu4sYbb+Szzz5jyZIlLFmyhC+++GKYI8+udHYOttvttLa2Jr+ampqGMeLs8/l8zJw5k6eeeqpf5+/du5fFixezcOFCtmzZwu23387y5ctZv359+kEIKSfNmzdPrFixInk/FouJ6upqsXr16j7P/8d//EexePHilGOnn366uPnmm4c0zlwz0Pdt7dq1wuFwDFN0uQ8QL7300gnP+Z//83+KadOmpRy78sorxaJFi9J+XVkiykE9O+Ced955yWMn2gEX4MMPP0w5H2DRokXHPX80Sud9A/B6vdTX11NbW8ull17Kl19+ORzhjlhD8bsmE1EOOtEOuG1tbX0+p62tbUDnj0bpvG+TJk3imWee4eWXX+Z3v/sd8XicM844gwMHDgxHyCPS8X7X3G43gUDgOM86sRE9+16SBmv+/Pkpe+idccYZTJkyhV/+8pc88MADWYwsv8gSUQ4a6A64AJWVlQM6fzRK5307ll6vZ/bs2ezatWsoQhwVjve7ZrfbMZvNaV1TJqIclM4OuPPnz085H+DNN9/Mqx1zM7FzcCwWY9u2bVRVVQ1VmCPekPyupd3MLQ2pP/zhD8JoNIp169aJr776Stx0002isLBQtLW1CSGEuPbaa8Vdd92VPP/9998XOp1O/OIXvxDbt28X99xzj9Dr9WLbtm3Z+hGyYqDv23333SfWr18vdu/eLTZt2iS+//3vC5PJJL788sts/QjDzuPxiM8++0x89tlnAhAPP/yw+Oyzz0RTU5MQQoi77rpLXHvttcnz9+zZIywWi/jXf/1XsX37dvHUU08JrVYrXn/99bRjkIkohz3xxBOirq5OGAwGMW/ePPHRRx8lH1uwYIFYunRpyvn/9V//JSZOnCgMBoOYNm2aePXVV4c54twwkPft9ttvT55bUVEhLrroIrF58+YsRJ09b7/9tgB6ffW8T0uXLhULFizo9ZxZs2YJg8Egxo0bJ9auXTuoGOQyIJIkZZ1sI5IkKetkIpIkKetkIpIkKetkIpIkKetkIpIkKetkIpIkKetkIpIkKetkIpIkKetkIpJGlHXr1lFYWJi8f++99zJr1qzk/WXLlrFkyZJhj0saHJmIpD4tW7YMRVH40Y9+1OuxFStWoCgKy5YtSzk/0wmgoaGBRx99NOXYlVdeyc6dO4/7nMceeyxl/eSzzz6b22+/PaNxSZknE5F0XLW1tfzhD39IWewqGAzy/PPPU1dXl5WYzGYz5eXlx33c4XCklJikkUEmIum4Tj31VGpra3nxxReTx1588UXq6uqYPXv2oK7dV0llyZIlyVLW2WefTVNTE3fccUdydw3oXTU71tEls2XLlvHuu+/y2GOPJa+xd+9eGhsb+cUvfpHyvC1btqAoilyHKEtkIpJO6IYbbmDt2rXJ+8888wzXX3/9kL/uiy++SE1NDffff39yd42Beuyxx5g/fz4//OEPk9eoq6vr9TMBrF27lrPOOovGxsZM/QjSAMhEJJ3QNddcw3vvvUdTUxNNTU28//77XHPNNUP+usXFxWi1Wmw2G5WVlWmtNOlwODAYDFgsluQ1tFoty5YtY8eOHcn9ziKRCM8//zw33HBDpn8MqZ/kmtXSCZWVlbF48WLWrVuHEILFixdTWlqa7bAGpbq6msWLF/PMM88wb948/vznPxMKhfje976X7dDyliwRSSd1ww03sG7dOn7zm99krNSg0Wg4dimsSCSSkWv3x/Lly5MN8WvXruXKK6/EYrEM2+tLqWQikk7qggsuIBwOE4lEWLRoUUauWVZWltLuE4vFeu1KazAYiMVig3qd413joosuwmq1smbNGl5//XVZLcsyWTWTTkqr1bJ9+/bk98fjcrnYsmVLyrGSkhJqa2t7nXvOOeewcuVKXn31VcaPH8/DDz+M0+lMOaehoYENGzbw/e9/H6PRmFaVsKGhgY8//ph9+/ZRUFBAcXExGo0m2Va0atUqJkyYkFebDOQiWSKS+sVut2O32094zjvvvMPs2bNTvu67774+z73hhhtYunQp1113HQsWLGDcuHEsXLgw5Zz777+fffv2MX78eMrKytKK+84770Sr1TJ16lTKysrYv39/8rEbb7yRcDg8LL2A0onJNaulvPX3v/+dc889l+bm5l47l0rDSyYiKe+EQiE6OjpYunQplZWVPPfcc9kOKe/JqpmUd37/+99TX1+P0+nkf//v/53tcCRkiUiSpBwgS0SSJGWdTESSJGWdTESSJGWdTESSJGWdTESSJGWdTESSJGWdTESSJGWdTESSJGXd/w8ma4R8SteAgQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:39:03.061966Z", + "iopub.status.busy": "2024-03-23T10:39:03.061685Z", + "iopub.status.idle": "2024-03-23T10:39:03.413850Z", + "shell.execute_reply": "2024-03-23T10:39:03.413014Z" + }, + "papermill": { + "duration": 0.372618, + "end_time": "2024-03-23T10:39:03.415911", + "exception": false, + "start_time": "2024-03-23T10:39:03.043293", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFPUlEQVR4nO2deXxU1d3/3/fOPpkl+0YSAoRdNhUpuIBKtbjXPpVuCorWtb+qtVq7iNrHx25a6VPL08dWeKy7rUtbq2hpFUVREFEUBAnZIAlJyDIzyexzfn/cyUBIgGSy3Ely3q/XvCZz77nnfjOZ+eR7zvme71cRQggkEolER1S9DZBIJBIpRBKJRHekEEkkEt2RQiSRSHRHCpFEItEdKUQSiUR3pBBJJBLdkUIkkUh0x6i3Af0hFotRW1uL0+lEURS9zZFIJEcghMDr9VJYWIiqHt3vGdZCVFtbS3Fxsd5mSCSS41BTU0NRUdFRzw9rIXI6nYD2S7pcLp2tkUgkR+LxeCguLk58V4/GsBaizuGYy+WSQiSRpDDHmzqRk9USiUR3pBBJJBLdkUIkkUh0J2WE6Gc/+xmKonDzzTfrbYpEIhliUkKINm/ezO9//3tmzpyptykSiUQHdBcin8/HN7/5TR555BEyMjL0NkcikeiA7kJ04403cv7557N48WK9TZFIeqStI8wn+9v0NmNEo2sc0dNPP83WrVvZvHlzr9oHg0GCwWDitcfjGSzTJBJA26Lwt49r6QhFmFbgQlXlVqLBQDePqKamhu9+97s88cQTWK3WXl1z//3343a7Ew+5vUMy2FQe7KDRGyQcFQQjMb3NGbEoelXxePHFF/nyl7+MwWBIHItGoyiKgqqqBIPBLuegZ4+ouLiYtrY2GVktGRRe/aSOnXVe5pSks2hyLm0dYXyhCGPSbXqbNizweDy43e7jfkd1G5qdffbZbN++vcuxK6+8kilTpnDHHXd0EyEAi8WCxWIZKhMlo5xYTFDR1AFAWa6D8kYff91WS7bTwuVfGKuzdSML3YTI6XRywgkndDmWlpZGVlZWt+MSiR40tQcJhKOYjSqFbhv+cBSAgz7tuNXU/Z+lJDl0XzWTSFKV+rYAAAVuK6qqkGYxkm43IQTUtvp1tm5kkVK779944w29TZBIEtTFhSjffWgxpcBtpbUjTIM3yPgch16mjThSSogkklRibmkmhW4bee5D85I5Tgs767w0+YLHuFLSV6QQSSRHITPNTGaaucuxbIcmSk1eKUQDiZwjkkj6QKcQtfrDhKMyrmigkB6RRNIDDZ4A9Z4A+W4ruc5Dc0RpFiMLJmSRcYSnJOkf0iOSSHqgvLGd9Tsb+Kim+x6zeeOzmJTnxGSQX5+BQr6TEkkPHGzX5oCOnCOSDA5yaCaR9EBLewjoWYg6QhEOeIIYFIWSLPtQmzYikUIkkRyBEILWjjAAGXZTt/PVzR28sr2eMRk2KUQDhByaSSRH0B6KEokJFAWc1u5ClG7TvKS2uFhJ+o8UIonkCDx+TWAcFiOGHvIPpce9JF8wIpfwBwgpRBLJEbTFhcht6+4NAVhNBiwmtUtbSf+Qc0QSyRGMy07jP04qOmY2RpfVRGM4iDcQSQQ5SpJHCpFEcgRWk4HizGNPQjutRhq9QXyByBBZNbKRQzOJJAkcFu1/uDcoh2YDgfSIJJIj2FzZjM1koCzXcdTkZ1MLXIzJsHXZ/iFJHilEEslhRGOCjXuaEAJKs9OO2q4w3UYhMm/1QCGHZhLJYbSHIggBBlUhzSxTwQ4VUogkksPonHx2WIwoytFXzaIxwd5GHx/va0WnQjgjCjk0k0gOwxc8JERdiEYAAQYttigmBC9tqwVgUp5TJtLvJ1KIJJLD8HZ6RNbDvhoNn8GulyEWhXELoWQeJoOKzWzAH4riDUSkEPUTOTSTSA6jm0fU0Qyf/Q0iIU2Iyv8FLVWAFksE4A3IJfz+IoVIIjkM35EeUfW72rAsYywUzNSOVb6ltYmLVXswOuR2jjTk0EwiOYzTJ2Uzs8iNy2aCsB8admgnSk8HqxvqP4HWGuhoJs0cF6KQjK7uL9IjkkgOw2U1UZxp1za8Nn6meUNp2eAuAqsLMsdpDQ98it2izQt1SCHqN1KIJJKjcbBce86dBp1L+TmTtefmvYc8Ijk06zdyaCaRxAmEo2ytasFpNTGjIA1aKrQTWRMONcoo1Z699ZSUqZw3o6DHLI6SviGFSCKJ0+YP815FMw6LkRlpLdqwzOIAR96hRlY32DOho5mM8AEy8ifqZ/AIQg7NJJI4XWKI2vZrB93Fh4ZlnaSXaM9tNUNo3chGCpFEEqdLDFHbPu2gu6h7Q2cBAMJTx54GbZuHTBnbP+TQTCKJ09EpRGYDNMU9IteY7g1dhQAovnrWfVJHKCoozrDL6q/9QHpEEkmc9pC2+uWKtUIkqO0rO3x+qBN7NhiMEAmRqXiBQ96UJDmkEEkkcTrjgdzRg9oBZz6oPXxFVDUhUBmiNX6tXMLvD1KIJJI4nfFA9lBciHryhjpJywEgPdamXSuDGvuFnCOSSOKcP6MAbzBMdvn72gFH7tEb27MBcMZaQIEOGdTYL6QQSSRx3HYTbpsRAo3agWN6RJoQOSKtYJIeUX+RQzOJ5HCCHggHQFETXk+PxIXIGvGgiIjcb9ZPpEckkaCten28r5XsQA2TQIueNhzj62F2gMmK2xblggkWHFnHEC3JcZEekUQCtLSHeG9vM3sr9moHjjUsAy3a2p6N1WigLC1AvluWFeoPUogkEg7N8biEtgrWuSp2TOyZ2rO/ZZCsGj1IIZJIOBQH5Ix5tAP2rONfZMtAIKirr5XbPPqJFCKJhPjyuxCkReMeUS+FSEFhR3kV63c20C6jq5NGCpFEgjY0M0fbsShRbcXMln78i2wZADiFDwB/WMYSJYuuQrR69WpmzpyJy+XC5XIxf/58XnnlFT1NkoxSOkIRbJFWTAZFExi1F+WBrOkA2AhiiIXkNo9+oKsQFRUV8bOf/YwPPviALVu2cNZZZ3HxxRfz6aef6mmWZBTSHoxiDbdhMqiHJqGPh8kKZjsmg4Il4sEvhShpdI0juvDCC7u8vu+++1i9ejWbNm1i+vTpOlklGY1cOLOQsPljnC3G3s0PdWLLwKg2YI145NCsH6RMQGM0GuW5556jvb2d+fPn99gmGAwSDAYTrz0ez1CZJxnhuO0mUH3azvq+CJE1HZNB0YRIekRJo/tk9fbt23E4HFgsFq677jpeeOEFpk2b1mPb+++/H7fbnXgUFxcPsbWSEU1HfNd9Hz0ik0HFGm6Tc0T9QHchmjx5Mtu2beO9997j+uuvZ9myZezYsaPHtnfeeSdtbW2JR02NzBks6T+eQJh3d+2noSm+2bW3c0QAVjfpdhOnFBo5cWz6oNg3GtB9aGY2mykrKwPgpJNOYvPmzaxatYrf//733dpaLBYsFstQmygZ4bS0h9j+eSX2tgC5WZlgsvX+Yqsbu9mI3RICp9zmkSy6e0RHEovFuswDSSSDTXswijUSXzGz9cEbAq36K0DAA0IMvHGjBF09ojvvvJMlS5ZQUlKC1+vlySef5I033mDdunV6miUZZfjDEawRD2aD0rtAxsOxuIgKaPW2U1dZz/TSfJQjyw9JjouuQtTQ0MAVV1xBXV0dbrebmTNnsm7dOr74xS/qaZZklNEejGKJeDEZ1US0dK9RDQhzGrsrDrKdciYV52E2SiHqK7oK0R//+Ec9by+RAFpUtTXiwWRVE9HSfcFgc6MqYIn68IeimI0pN+OR8sh3TDLq0eaIPPE5oj56RIBidWMyqFgiPhnUmCRSiCSjHn8ggDnaEd9nlt73DqwujAYFS9QrU8Ymie7L9xKJ3lw02QEeJzabDUz2vncQ94jMkXbpESWJFCLJqMeFD6wmSMvUUsD2FYsbk6piCfnkNo8kkUMziSTQqj0nMyyDQ0OziFd6REkiPSLJqKbNH2b/3iqyvQFyS/o+UQ2A1U2Ow4LTGoVcGV2dDFKIJKOa5vYQlfv2E4kGyE1i6R4Ao4W0NDtpkRAYQwNq32hBDs0ko5r2YDyGyNjL9LBHwxLf6hGUqWmSQQqRZFTTEYxoUdVJxhB1EjamcbA9yN799QNo3ehBCpFkVBPsaEUVUUxGI1jcyfej2tl9wMe2PfsG0LrRgxQiyagm7NOKI6o2l5adMUnMaZqIqSEvEVnfrM9IIZKMaqLtzQAY0/qY/uMITHY3CmCOthOISCHqK1KIJKOaWIfmEfVXiBSLE5NBwRxtl9s8kkAu30tGNacVmYgaXVgze1Hr/lhYnBgNquYRhaRH1FekEElGNWkxj7a9w9GHhPk9YXFiVBVMoQD+YAhIYs/aKEYOzSSjG3+r9pxsMGMnJjtGoxEQBP0ylqivSI9IMmpp83jxHDiI1WQgpz/BjACKQl5ONtlprZjS5NCsr0ghkoxaWpobqGv1Y01zkmPsf3WY9PQsUPyg+gfAutGFHJpJRi0hr7Z0jzX5iOouWBzac9A3MP2NIqQQSUYtoXgMkWpPH5D+Aqq2zWP/gYYB6W80IYVIMmqJtmsxRAb7wHhErVELuw/42FNTOyD9jSbkHFGKEo1GCYfDepsxoomG/GBxYkjLIhAI9Ls/YXWCxYkSiwxIf8MBk8mEwWDodz9JCdHevXsZP358v28u6Y4Qgvr6elpbW/U2ZcRjyJmKLXsyEYODioqKfvcXi5qwTf4iVpQB6W+4kJ6eTn5+/wpLJiVEZWVlLFy4kBUrVvAf//EfWK0yK91A0SlCubm52O12WTV0sBCCjtYDCCEwObMxm0z97zIaoaOtERQFmzsPVR3ZfzshBB0dHTQ0aHNiBQUFSfeVlBBt3bqVNWvWcOutt3LTTTexdOlSVqxYwSmnnJK0IRJtONYpQllZ/Yz0lRybWARzmhkhFJQ0x8CIhhDE/C0IAWazKR7gOLKx2WyAVrU5Nzc36WFaUpPVs2fPZtWqVdTW1vLoo49SV1fHaaedxgknnMCDDz5IY2NjUsaMdjrnhOx2uT1g0IlFUVEwGIwD57koCijaVyoWGz1J9Ds/r/2Z0+zXqpnRaOTSSy/lueee4+c//zl79uzhtttuo7i4OFHTXtJ35HBsCOgUCrX/E61dULT+xCgSooH4vPZLiLZs2cINN9xAQUEBDz74ILfddhvl5eW8/vrr1NbWcvHFF/fbQIlkMIhGIwSjMYKxgRV9s9mE1WTAqIgB7Xekk5QQPfjgg8yYMYMFCxZQW1vLY489RlVVFf/5n//JuHHjOP3001m7di1bt24daHslI4jly5dzySWXDGifixYt4uabbz5mm9LSUn79m/8mHIkRjA6sEJkMRoyqggG536wvJCVEq1ev5hvf+AZVVVW8+OKLXHDBBahHpNnMzc3lj3/844AYKUl9eiMAKYWIC8VAD806vwdi+AvR3XffzezZs4fkXklN67/++uuUlJR0Ex8hBDU1NZSUlGA2m1m2bNmAGCmRDDhCGzophoFd2YqiImICEYnQ/4CA0UNSHtGECRNoamrqdry5uZlx48b12yjJ8GL58uW8+eabrFq1CkVRUBSF8vJyVqxYwbhx47DZbEyePJlVq1b1eP0999xDTk4OLpeL6667jlCod0UK29vbueKKK3A4HBQUFPDAAw90a9PQ0MCFF16IzWZj3LhxPPHEE9qJTiFSDSiKwurVq1myZAk2m43x48fz5z//OdFHZWUliqLw7LPPcvrpp2Oz2Zg7dy67d+9m8+bNnHzyyTgcDpYsWULtgSYC4d5FxT/66KNMnz4di8VCQUEBN910U+JcdXU1F198MQ6HA5fLxWWXXcaBAwcS5zu9lT/96U+Ulpbidrv52te+htfrTbSJxWL84he/oKysDIvFQklJCffdd1/i/B133MGkSZOw2+2MHz+en/zkJwm7165dyz333MNHH32U+JuuXbu2V3+XpBBJoCiKOHDgQLfjlZWVwm63J9NlUrS1tQlAtLW1Ddk9BxO/3y927Ngh/H5/t3PBcPSoj3Ak2uu2oV607Sutra1i/vz54pprrhF1dXWirq5OBAIBcdddd4nNmzeLvXv3iscff1zY7XbxzDPPJK5btmyZcDgcYunSpeKTTz4Rf//730VOTo744Q9/2Kv7Xn/99aKkpET885//FB9//LG44IILhNPpFN/97ncTbZYsWSJmzZol3n33XbFlyxaxYMECYbPZxC/+c6XwNO4T7cGwAERWVpZ45JFHxK5du8SPf/xjYTAYxI4dO4QQQlRUVAhATJkyRbz66qtix44d4gtf+II46aSTxKJFi8Tbb78ttm7dKsrKysQ111wjvA3Vov3g/mPa/rvf/U5YrVbx0EMPiV27don3339f/PrXvxZCCBGNRsXs2bPFaaedJrZs2SI2bdokTjrpJLFw4cLE9StXrhQOh0NceumlYvv27WLDhg0iPz+/y3t3++23i4yMDLF27VqxZ88e8dZbb4lHHnkkcf6nP/2p2Lhxo6ioqBB//etfRV5envj5z38uhBCio6NDfO973xPTp09P/E07Ojp6/F2O9bnt7Xe0T0J0yy23iFtuuUWoqiquvfbaxOtbbrlF/L//9//EvHnzxIIFC/rSZb8YTUL04Gu7jvp4Yeu+Lm3/e/3uo7Z9dnN1l7ar39jTrU0yLFy4sIsA9MSNN94ovvKVryReL1u2TGRmZor29vZD9qxeLRwOh4hGjy2IXq9XmM1m8eyzzyaOHTx4UNhstoQdu3btEoB4//33E2127twpAPHz/1wp2prqhD8UEYC47rrruvQ/b948cf311wshDgnRH/7wh8T5p556SgBi/fr1iWP333+/mDRpkvA2VAtfY40QsdhR7S8sLBQ/+tGPejz32muvCYPBIKqrD/2tPv300y6/y8qVK4XdbhcejyfR5vvf/76YN2+eEEIIj8cjLBZLF+E5Hr/85S/FSSedlHi9cuVKMWvWrONeNxBC1KcB8ocfftjpRbF9+3bMZnPinNlsZtasWdx22239d9MkI4KHH36YRx99lOrqavx+P6FQqNvk56xZs7oEcM6fPx+fz0dNTQ1jx449at/l5eWEQiHmzZuXOJaZmcnkyZMTr3fu3InRaOSkk05KHJsyZQrp6ekgIIaKGo+BmT9/fpf+58+fz7Zt27ocmzlzZuLnvLw8AGbMmNHlWGOjtsUDBIgoKN2/Yg0NDdTW1nL22Wf3+Lvt3LmT4uJiiouLE8emTZtGeno6O3fuZO7cuYC2+ud0OhNtCgoKEtstdu7cSTAYPOo9AJ555hl+85vfUF5ejs/nIxKJ4HK5jtp+MOmTEP373/8G4Morr2TVqlW6GT0aufHMsqOeOzIw+NtnTDhq2yNjz646dXDm9J5++mluu+02HnjgAebPn4/T6eSXv/wl77333qDcr28ITEYFh82Kauj98r3psP1onUF8Rx6LxWIIVBBRRCzWGWjdhc5tEf3FdMT+uM779+Ye7777Lt/85je55557OPfcc3G73Tz99NM9zrMNBUlNVq9Zs0aK0BBjNqpHfRgNaq/bmnrRNin7zGai0UPRxBs3bmTBggXccMMNzJkzh7KyMsrLy7td99FHH+H3H0qtumnTJhwORxdvoCcmTJiAyWTqImwtLS3s3r078XrKlClEIhE++OCDxLFdu3bR2tqGgoLRaEx4RJs2berS/6ZNm5g6dWovf/uuiONs83A6nZSWlrJ+/foez0+dOpWamhpqamoSx3bs2EFrayvTpk3rlQ0TJ07EZrMd9R7vvPMOY8eO5Uc/+hEnn3wyEydOpKqqqkubI/+mg0mvPaJLL72UtWvX4nK5uPTSS4/Z9vnnn++3YZLhRWlpKe+99x6VlZU4HA4mTpzIY489xrp16xg3bhx/+tOf2Lx5c7dV1VAoxIoVK/jxj39MZWUlK1eu5KabbuoWGnIkDoeDFStW8P3vf5+srCxyc3P50Y9+1OW6yZMn86UvfYlrr72W1atXYzQaufnmmw95C4fFED333HOcfPLJnHbaaTzxxBO8//77ycfBxYXoWNs87r77bq677jpyc3NZsmQJXq+XjRs38p3vfIfFixczY8YMvvnNb/LQQw8RiUS44YYbWLhwISeffHKvTLBardxxxx3cfvvtmM1mTj31VBobG/n0009ZsWIFEydOpLq6mqeffpq5c+fy8ssv88ILL3Tpo7S0lIqKCrZt20ZRURFOpxOLpf+5vXui1//+3G53wh11u93HfEhGH7fddhsGg4Fp06aRk5PDueeey6WXXsrSpUuZN28eBw8e5IYbbuh23dlnn83EiRM544wzWLp0KRdddBF33313r+75y1/+ktNPP50LL7yQxYsXc9ppp3WZDwLNey8sLGThwoVceumlfPuaa8jNySISEwQOi6q+5557ePrpp5k5cyaPPfYYTz31VK+9jyOxWsxYTQbUY0RXL1u2jIceeojf/e53TJ8+nQsuuIDPP/8c0IZYL730EhkZGZxxxhksXryY8ePH88wzz/TJjp/85Cd873vf46677mLq1KksXbo0MYd00UUXccstt3DTTTcxe/Zs3nnnHX7yk590uf4rX/kKX/rSlzjzzDPJycnhqaee6uM70XsUIcSw3RTj8Xhwu920tbWNiKFiIBCgoqKCcePGyRxPg0UsQtjTQDASI2TLITPNgqIovPDCCwO33STog6AXTDbob5miYcCxPre9/Y4mNSHg9/vp6OhIvK6qquKhhx7itddeS6Y7iWToiEURQhBDxTBYWQ46h3xi9OzA7y9JCdHFF1/MY489BkBrayunnHIKDzzwABdffDGrV6/udT/3338/c+fOxel0kpubyyWXXMKuXbuSMUkywqiursbhcBz1UV1dnVzHsSgCEIoBZZAyKEaEgqOwDEduSY+2v/XWW4Ny3+FM0hkaf/3rXwPw5z//mfz8fD788EP+8pe/cNddd3H99df3qp8333yTG2+8kblz5xKJRPjhD3/IOeecw44dO0hLS0vGNMkIobCwsFscz5HnkyIWQQiIKQbMcY9ooGcnglHYuP5VjEYVizuv2/kxY8YM6P1GAkkJUUdHRyKQ6rXXXuPSSy9FVVW+8IUvdFsCPBavvvpql9dr164lNzeXDz74gDPOOCMZ0yQjBKPRSFnZ0WOnkkZoQzOhqt3irwYKRTUwYXwpRoOCNaOQHoOJJF1I6h0qKyvjxRdfpKamhnXr1nHOOecAWsRofyaN29raAC1CtieCwSAej6fLQyLpE7EIAogpA5gi9ghUVUUoiravNjb804EMBUkJ0V133cVtt91GaWkp8+bNS4THv/baa8yZMycpQ2KxGDfffDOnnnoqJ5xwQo9t7r///i5hAscLepNIuiDEoclqxTBok9WKAgIVAXLCupckvXxfX19PXV0ds2bNSgSRvf/++7hcLqZMmdLn/q6//npeeeUV3n77bYqKinpsEwwGCQaDidcej4fi4mK5fC/pHbEo+BqIARF7LiaDOij5wcPRGB2tDRhFmDR3traMP4IZiOX7pLNC5efnk5+f3+VYsuWEbrrpJv7+97+zYcOGo4oQgMViGbTITskoIBYBQFUNmI0DnJnxMFRFQSgGRCyMiEWRpRCOT1JC1N7ezs9+9jPWr19PQ0NDYqNdJ3v37u1VP0IIvvOd7/DCCy/wxhtvyKRqksElUbljcOuNqfGhGSCFqJck9Re5+uqrefPNN7n88sspKChI2r298cYbefLJJ3nppZdwOp3U19cD2haSgdqhLJEkiEWICkEoCko4itU0OF6RoijYrGaMoSDKCMhdPRQkJUSvvPIKL7/8Mqeeemq/bt4Z/Lho0aIux9esWcPy5cv71bdkaFm0aBGzZ8/moYce0tsU4Cj2iCjRmKAjpmAcRCECsJpMEFHkZHUvSUqIMjIyjrrE3heG8TY3ySAQCoW6JNsbcOLBjEI1DH5demXkVPMYCpJavv/pT3/KXXfd1WW/mWT0kmzy/M66Zvfddx+FhYWJ7IrvvPMOs2fPxmq1cvLJJ/Piiy+iKEqXSOtPPvmEJUuW4HA4yMvL4/LLL08UdOjJnsqKivj2Dm3pXh2sfWZxwkIhEhNaPh/5D/e4JOURPfDAA5SXl5OXl0dpaWm3THGysOIAIgREk68pnjQGU/d0jkdh1apV7N69mxNOOIF7770X0LzmoqIinnvuObKysnjnnXf49re/TUFBAZdddlni2vXr1+NyuXj99dcBbbn3wgsv5LzzzuPJJ5+kqqqqW7201tZWzjrrLK6++mp+/etf4/f7ueOOO7jsssv417/+1aM9OVmZ4D+oeUSoGAbZI+qICAzhKGajikHEEqWoJT2TlBANdHVOyTGIhuEtHdJ3nv49MPZumOR2uzGbzdjt9i4hHffcc0/i53HjxvHuu+/y7LPPdhGitLQ0/vCHPySGZP/zP/+Doig88sgjWK1Wpk2bxv79+7nmmmsS1/z2t79lzpw5/Nd//Vfi2KOPPkpxcTG7d+9m0qRJ3e2JaPFnUQygKIO2vaMTbQlf1ZwhEQOkEB2LpIRo5cqVA22HZATSm+T5M2bM6DIvtGvXLmbOnNklMO7I+LSPPvqIf//73zgcjm73LC8vZ9KkSd2Nie+6j8ZnIwZ7aKYqClFFRRDVwgYMstzisUg6oKK1tZU///nPlJeX8/3vf5/MzEy2bt1KXl6e3F08kBhMmneix337QW+T5yeTZcHn83HhhRfy85//vNu5goKCni+KBzPG4kOkwZ6sVhWIoCJEVE5Y94KkhOjjjz9m8eLFuN1uKisrueaaa8jMzOT555+nuro6katIMgAoSq+HSHpyrOT5nfSUPP9IJk+ezOOPP04wGExE0W/evLlLmxNPPJG//OUvlJaWYjT2/BHulvg9LkSuNCsxg3lIPKJYYmgml/CPR1KrZrfeeivLly/n888/7+JCn3feeWzYsGHAjJMMHw5Pnt/U1MTEiRPZsmUL69atY/fu3fzkJz/pJig98Y1vfINYLMa3v/1tdu7cybp16/jVr34FHCrhc+ONN9Lc3MzXv/51Nm/eTHl5OevWrePKK69MiM+R9sQiYRTAYDBhSrJSSV9QFEXb+Cp34PeKpP4imzdv5tprr+12fMyYMYnoaMnoItnk+Uficrn429/+xrZt25g9ezY/+tGPuOuuuwAS//QKCwvZuHEj0WiUc845hxkzZnDzzTeTnp6e2IB9pD3V1fE8WerQTBqripYFMlFoUXJseleMtis5OTli69atQgghHA6HKC8vF0JopXKLioqS6TIpRlPJ6dHM448/Lkwm01Frrx+XSEiItloRbq0VHn9IBMORgTWwB6KxmPB3tItwy34R8zUM+v30ZCBKTiflEV100UXce++9hMNafIuiKFRXV3PHHXfwla98ZQBlUjIaeeyxx3j77bepqKjgxRdfTMQIJb3/ML7ZNSxUOkJRtHL3g4uqKFjNJoyqgiKHZsclKSF64IEH8Pl85OTk4Pf7WbhwIWVlZTidTu67776BtlEyyqivr+db3/oWU6dO5ZZbbuGrX/0q//u//5t8h/GJ6mg8lscwVJlbD9/mIaOrj0lSq2Zut5vXX3+djRs38tFHH+Hz+TjxxBNZvHjxQNsnGYXcfvvt3H777QPXYacQDdHSfSfhmAIxgUEFVURBGdz0I8OZPr8zsViMtWvX8vzzz1NZWYmiKIwbN478/HyEEIOS8U4i6RfxPNURYQCFwatndgS+YARjBKxGUGOxJMcfo4M+vTVCCC666CKuvvpq9u/fz4wZM5g+fTpVVVUsX76cL3/5y4Nlp0SSHELo5hGpCvFYIrlydjz65BGtXbuWDRs2sH79es4888wu5/71r39xySWX8Nhjj3HFFVcMqJGjjSMzXkr6gdB2v8fim11VZfC3d3SiKAoxVGBkR1cPxOe1T0L01FNP8cMf/rCbCAGcddZZ/OAHP+CJJ56QQpQkZrMZVVWpra0lJycHs9ksh7r9JRKCUJgwKmFCGBSFQGBobh0OR4iEo6hEiKl+iI2sja9CCEKhEI2Njaiq2q9cUn0Soo8//phf/OIXRz2/ZMkSfvOb3yRtzGhHVVXGjRtHXV0dtbW1epszMogEIRJAqCaEyQMCWodoaBaKxAiH/JhFCJPZAib7kNx3qLHb7ZSUlCSCSZOhT0LU3NxMXl73Erqd5OXl0dLSkrQxEs0rKikpIRKJdN0rJUmOiregeQcUzoHi6UN766Z2dm7bwnT/FsZNmASTLhzS+w8FBoMBo9HYb8+9T0IUjUaPusmw06hIJNIvgyTa3ILJZOqWcE6SBP46iHWAKwuGuFacwx7DJ6xE/R6s4eYhv/9wok9CJIRg+fLlR60tdnjxQ4lEd4SA9kYANjcaCHgamTHGTbp9aLIZZDnMLJhaSsEuOwQ9mj1yzq9H+iREy5YtO24bOVEtSRkCrdpktWrg42YDnmALE3OdQ3Z7u9nItHGFUGOGaESbrzJJr6gn+iREa9asGSw7JH2loxkad4E5DXKngUFG7XajXUumH7Nl4fVoWyyc1iF+nwwmTXzCAQj5pBAdBfnpHY4cLIdPnk8E6lG3DWZ+bVgkUBtSfA0AdJgyEQLMRhW7eWiX0Pe3+jEEjGQRwxT0Qlr2kN5/uCCDzocbQR/seEkTIWc+GC3Qth/K1+ttWerRrgmRx5gBgMtmGvK4rH/tPMC2A1E6QlEIeof03sMJKUTDjaqN2lyDMw9OvAJOuFQ7XvdRwgOQxPFpE9WtqiZEbtvQr0LazUZCBjuhSEwbmkl6RArRcCLUrgkOwISztWyDGaWQM1lbkal6R1fzUopoGPzNABwU2gS1HkKUZjEQMqQRjsY0b1bSI1KIhhP127UkX64CyBh76PjYBdpz0275Ye+kvUkTZ5ONlogWbpKup0cUjWlL+JIekZPVwwUhDnlDBbO7nnPmg6sQPLVw4FMomTfk5qUc8fkhHLlcMLMQXyiCecgyoh0izWLUPKKwHJodC+kRDRd8DdqSvcGoLdcfSf4J2nPjzqG1K1XpnC9Ly0VVFVxWE1bT0G867RyahSJyaHYspBANFw5+rj1njOt5mT57sha166kDf+uQmpaSeOu0Z2f+sdsNMmnxoVk4KrQ5PpnipUekEA0XmuJClD2x5/MWB7iL4213D41NqUosBr4DADQqGbz8cR0fVDXrYkqWw8zCE0oZm+3QchKF23WxI9WRQjQcCHjAW695PJkTEELw788a2FF7xORnp0g17x16G1OJjiZtS4XRTEMkjd0HvFQ0dehiit1sZEZxBhnpWgiBHJ71jBSi4UBLhfbsLACLgy1VLWyraWXT3oPEYtrWhY5QhDZbkdautUZbvh6tdA7LHPm0dGjR55lpOmcysDi0Zzlh3SNSiIYDrdXac8ZY2jrCvFt+EIC5pZmoqkKTL8iT71XzemUUYXFoUddtNToarDNebViGM5/mjhAAGUO0474nalv97OswEoxENe9W0g0pRKmOEIeEKH0smyubicYEJZl2ThjjAsBkUPGHotS0+GkwFmptmyt0MjgFSExUF9Ds01LTZKX1nLpmKNiwu5H366K0ByNaRgBJN6QQpTqBVu2/qGrAZ81jR532H/ULE7IS+6bcNhMzitwAbO/I1K5rqdTB2BQgFk0s3Ucd+bT5taFZho5DszSLkaDRSSgqINCmmx2pjBSiVKfTG3IW8FlDgGhMUOC2Mia9a/nlOSUZKAp8FsigIxTREoKFhyhLfCrR3qgNTY0WWmN2YkJgNqo4LPrF7qZZDAQNDsKRmBSioyCFKNWJC5FIL2Zn3BuaXuju1sxtMzE+x0HYYKcubNeGdG37htTUlKBtv/bsGoMvFMWoKmSm6VsNxW7u9IikEB0NKUSpTlxMhKuEOSUZFGfamZjn6LHptAJtc2d5KEMr6jcaJ6w9cfF1j2FsVho3nlnGxbMLdTUpzWwkaHRoG1/Dfi1rpKQLcq9ZKhNqT0RJq+5CTjBZOWFMd2+ok9KsNCwmlQYlB29wH67RKESdXqBbC2VQVQW7Wd+PucNqJKpa8Iv4PFXQA0aZIO1wpBClMp746o89q1cpRo0GlcVT88hQHDg/3a4FQUbDWrrS0UDAoz0UFZz6ekGH0zk/5RFpQHx4JjM1dkEOzVIZr1ZkMWDP4+N9rXgDxw9SnJTnJCcnD8Xi0laQPPsH28rUodMbcuQQVU088V4Vr35Sp2041RG3zcTZU3OZOq4IgZBL+D2gqxBt2LCBCy+8kMLCQhRF4cUXX9TTnNQj7hHti2SwfmcDf/+4rnfXKUpiaELrKBqedYquu5jm9hANniDlje2YDPqW8DEbVWYWpZObk4eCIiese0BXIWpvb2fWrFk8/PDDepqRmgiR8IiqwlrgYmlWWq8urWvz826zndo2/+j0iFxjaPRqgYw5TouuK2ZdsMbn96QQdUPXOaIlS5awZMkSPU1IXfwtEA4gVAMV/jRAUJRhO+5lAC3tYT72OEjzBils26ftRu9HXfJhQTiQ2HGPu4jG6kNClAo0eAN4PCoFoQhpUoi6Maw+ncFgEI/H0+UxYvFo3pDfko03JDCoCvnu3tXEGpedht+ciSeiEgwGDmUrHMm0VmtepD0LrK5DHpEjNYRoW3Ur/6oM0tIekh5RDwwrIbr//vtxu92JR3Fxsd4mDR5xIWpUsgDIc1kw9TLVqc1sIN9tx2vOo80fPhTkN5JprdKeM8YihEgIUW6KeEQOq5GAwRGv5tEhY4mOYFgJ0Z133klbW1viUVMzgidi4/ND+6NaHpsx6fY+XV6SZcdryae1Izw6Ahs799alj6W1I0wgrEVVZ6WIR+SymogarPiJZwGQK2ddGFZxRBaLBYslNT5Yg0o0kti4WRnSJjgL0/tWqrg0K42dljzavGFirTWoQmiraSORoE+r2qEokF6CvyNKtsOMxWjAoKbG75yIJVIcQEjLP+7I1deoFGJYCdGoob1BiwEy2fjy3OnsbwswppcT1Z3ku6yE0/IJN0C7twVnoA1s6YNjr950DsscuWC2U2iGy+eXEo0njUsFHFbtq9aGEzioLUZIEugqRD6fjz179iReV1RUsG3bNjIzMykpKdHRMp2Jzw/hGoPNYqQst+e9ZcdCVRWKst0oTflEYz5tGX+kClFn7qX0sV0Op4o3BOCMC5FXcRKNNWGQQtQFXYVoy5YtnHnmmYnXt956KwDLli1j7dq1OlmVAiSEqKBf3Sw5oQCDfRbUbNZibPKmD4BxKYYQ0Fyu/Zw1gVhMIEgtEQKwGA2YjSoBo4tQNIbNr08y/1RFVyFatGiRtktc0pV4hsGNB8wo0SZmFaWTlkQ+HYOqgKsI2DxyJ6w9tdoqlNEM7mL2t/p5adt+xmU7OH9m/4R8oFk8NQ9bAEzl2+TQ7AjkHFGqEfZDRzORWIytrVYinmZmFaUn35+7iKgQRD0NmMOBXm2eHVYcjA/tM8eDaqC6uYNwVKBDUdfjMjnfCWEjVKjaBHsk1HONulFICv65RjnxYVkrTiKqlXS7KSlvqJOPGiK8vS9KzcH2kbndo1OIssoAqGnWygYVZ/Yt3GHIMNkO/TOQXlECKUSpRnxY1ogWyHhkSti+4rKZaIsHNoqRNjwLeLQwB0WBzPEEwlHqPVp63FQUIm8gzM46DwfCcdukECWQQpRqxD2ifVEtfqivy/ZHUpRho8OaTzASo72hqt/mpRSdFW1dhWBOY3+rHyEg3W7CZU29HEwHPEFe/aSeXZ64bXLCOoEUolRCCPDUEhWC6ogWUV3Ux4jqIzEZVNJySwHwNFRr8UkjhYad2nPOVAD2NmrlnHubpWCocds0AWoW8XAM6RElkEKUSvhbIOzHFxJ4jZk4LEZctv6vJxQWjiGiWmjzdWhZG0cCgTYtJEFRIGcyQggqmrQqquNzUluIPDiIxGJadLUEkEKUWsSHZR2WbMxmE2MybAOSS2dcjhOvJR9PIEyopbrf/aUEjbu0Z3cRWF1EY4K5pZmMz0mjKCP15odAS5BmNxvwm9IJhmNa6SMZvgLI5fvUIi5EBUUTuG7CBK38zACQYTchXGOI+ato3r+X/NL5A9KvrhwxLDMaVOaUZDCnJENHo46P22aiPphOICxIiwQh5AOLU2+zdEd6RKmEtzOiuhBVVbCaDAPSraIojC+bzNgsO+mRA8P/v3D7QU20FRVyJuttTZ9w20wIxYhPjYtPe6O+BqUIUohShfiOe4Ho99aOnpg6cTKFGQ6sseDwnySt26Y9Z00Ai4O6Nj8f1bQSCKf+RHxinkiNp41tb9LRmtRBClGq4DsAsSj721Ue3dLC1uoBFguDEZxxgRvOFWBjUTjwifZzwSwgnv3wswbeKU/9L/XkfCeXzBnD+NJS7YAUIkAKUeoQnx86QCZtgQixQUhhEUwroMEbYF/V7gHve8g4uEfbW2ZOg8wJ+IIRPm/QVst6KsWdamQ5LIzLTiMtI/5PQQ7NAClEqYNnPwJBTSQd6H8gY0/si2VR3tjOvorPh+9m4/1btef8GaCqfFzTSjQmGJNuI881jPbR2eMFFjuahv+c3QAghShV8NTSEYrSbMjGbFTJdQ78l6qodCKqoiA6DtLUNAyHBL4GLSWsokDhHIKRKB/v1xLRzylJ19W0vrC30ce7dQJ/BG3ja3AEF4HoJVKIUoGABwJteAIRfOZcCtzWQcmnY7E5sGVphRdr9n4y4P0POjXva885k8GWzgdVLfhDUTLTzEzI6XvyOL3YWt3KpspWWpT4yplvFFRZOQ5SiFKB+GbURjKIquZBDcjLKpkCQEvNruE1PAt6oWGH9nPxPDpCET6sbgVgwYQs1BRLhHYsstK01B+taqZ2wNvLCr4jGClEqUBrjTY/FI3vuB+E+aFO8kqnYVAVLN4q9h30Ddp9BpzqTdqKmbsIXIUoKIzLTiPPZU0qla6eZMSFqFGJB196D+hoTWogI6tTgbYaYjHILiqjTVjJH8RJV6N7DBluN00trezZ8xnF2XMH7V4DRqANaj/Ufi49DdBqt503o4BgJJo6JaV7SaZdE6IDsXTNFfDWaRPWw+z3GEikR6Q3oXZob8KgKsybPYtvzCsZ3HzLqkpufHhm8VQOj+FZ5UbNG0ovIeAo7mKzxTgw0edDSbZTE6K6iJuIQPsMBL36GqUzUoj0pjO4MC0bzEOzWdNdPJUTS9JZ4G4m5f8Hew9A/XYAQiWn88K2Wv72cd2wiKI+GnazEafVSEw14jOkawd9o3t4JodmetNagxCCJkMuGdEYxqFItpxVhsVs0dJQ+A6AM39Auo3GBLWtfioPttPcHsIfiqKqCm6biXOm5fV9CCUEfL4ORIxI1iT+WqlS39aB1WQgGIkN2F48PchxWvAGIjQbskiPtWppfLMn6m2Wbkgh0puWCnyhCK/sMxPqqGTFaeMGf87DaIGs8dC4G9++TziY62RsP5KJxWKCrdUtbKtpxRuIdDvvC0S6/E77WjrIdliOLyR126BtP0Fh4CXPVPb7OzAbVS6ZU5jYszVcObUsmzMm5pDe1g67yof3tpsBQAqRngTaoL2JVn+EVtsYxrmsQzfxmjsNb82nbH1/I5+WjGXZgnFJJ+lXVYX9rX68gQg2s4Fx2WkUuK3YzcZu1VbD0Rh//aiWaFQwMc/J1AInxRn27svvHc1EP/8njZ4AG5nFfrsRm9nARbMKKXAP3qriUJHt6CydHi8k6qnTNj4bRudXcnT+1qlCvEJpfSyDqME6tClOs8pIs9vJNDRh9tWyYbeDJTN6v+u/uT2E3WxIeDULJ+UwIcfBlHznMYeXvkAEp9VEkzfIzjoPO+s8pFkMlGTayXFaKMlMIyfNCDv/SnVDG7uD6VTnTiPPZeX8mQXD3hPqhj1TmxsMdWirZ+nFelukC1KI9KSlgkgsRrXQ5mjGZg9hZkGDCTVvGuP8H1DVupPP6gsoyrAzo+jYG0ejMcGWymber2hmcr6Tc6ZrtqfbzaTbj1+jKyPNzLfmlVDvCbCzzsPuAz7ag1F21nnZWefl7CkKOfs2gqeOnAwXGwxf5MyyfGaOcQ+roMXe8Mn+NqoOdnCqOY/0UIU2PJNCJBlSYjFoqaTNH6bZOobMNPPQV54onIOj7iNmW+upjHbw710NWE0qE/O6ZwwUQlDd3MGG3Y00+UIAdISiRGOiz+EGiqJQ4LZR4LaxcFIu+1v87G/109IRoqD1A2jaDoqCY85XuDxj/IgToE72NrVT3uBjvDuHdCqgtRrGjoDsmUkghUgv2mogHKAxoOJz5TJXj71SrgJwFTCGWubGKngnNp2/f1zH3NIgp03UdocHwlEqmtrZvr+N/S1+QAsmXDQ5h8l5zn7PaRlUhZIsOyWZNqh8Cyo3ayfKvghZE0Z0fMmYdBvlDT6qYtlMBWirhmgYDCNs+NkLRvLfObVp2k1MCPaKAlBU/bYpFH8BBYW5pr2cWNg9orvqYAevflLP/hY/BlXhxLEZLJtfypR818BNrEcjsOsfWuAiwIQzoeikgek7heksnlnhT0NYHNr70DpCihv0EekR6YEQ0LgLRYEvnLKAXJFNnsty/OsGg5zJ4MhB9TWy0FrO+JPm4bYf+o+c77aS7bRQluPghDEunAM9fPQegF0va8+KAmWLoejkgb1HipLjtGAyKAQiMbyZY3EFP4XmvVoK3FGG9Ij0wFMLQS+KwUxm0WROLs3Ub7+UokDpGdrP+zZTbG7vMlfltpm4/AtjmT8ha2BFKOiDz/8JH6zVRMhkhZmXjRoRAm1YWhj3iqqJr1ge3DMqE6VJIdKDA59qz9kTUyNuJHui9ohFNe8k2j0occDwHoDd6+C91bBvM4iY5pXNvRoyxw/efVOU0mwtZGNXKAtUI/hbR+V2jxT4FowyohE48AmN3iDbLXlMaunQvyCgosDEc7T5CU8d7H4FplwwMLvBhdC+WE27oenzrknAXIUw7vRRKUCdjM9O463dTaCaEVkTUBp3af+oBmjbzXBBCtFQ07QbIkH2+Y185EvH2RbQX4gArC6Y/mX4+Fmoj2dvnPSlvq/gCAEdBzVRa6vRnoOH5T1SDZA9CQpnQ/rYUZ36ArT4q2sXjtcCQxs7tAq2DTtg/Jmgjp4BixSioaZ2K+2hCOXqeBRVZWqBS2+LDpE5DqacB5/9QxOjtv2ax5I9uechpBBaCgtvPfjqtWfPfi1K+HAMJq3vrImQVTZkWQaGC4k9d5njtX2AQR+0VIyqSWspRENJ235oraG2LUS9cxqT8pw4ktzfNWjkz9BK9Xz2D60Q446/anMXjlztuGrQYl2CXgi0asnfj0Q1gnsMuIu1SGFXUWrMhaU4nrDAmj0Nc/2HsG+LFCLJIFGziUAkymeihLAxjZPGpmid9szx2uTx/i1Qu00TnXjdtW4oCtiztDkNR7727CyQwtNH3tjVwLaaVhaVTGG2sk1bxvc1giNHb9OGBPlpGSra9kHjbmpa/Ox3zqQk057adbhMVi0t69hTNc/I1wDhDm04ZjCCKQ1sGWB1S9EZADLsZoSArQ2CmVkTUZt2Q9VGmH6J3qYNCfITNBQIAXv+SUcowo5YKX5TBqfHt1CkPIqi7RC3Z+ptyYhmaoGLd/cepM0fZk/BLCYpn0PDTi2uyl2kt3mDzuiZlteTfZvBU4fNZmP83CXMLHKTm8rekGTIMRtVTizRhupv1RmI5J6gndi9bnDjulIEKUSDjacW9r4BgDLhLGZMKOLsqXn62iRJSWYXp+O0GvH4w7zHCWCyaUPi+OdnJCOFaDBpb4KPn6XR00EofQIUztHbIkkKYzaqLJqcC8D7+0NU552pndi3+VCV2xFKSgjRww8/TGlpKVarlXnz5vH++yPgTW/eS/SDx6isb+KjVitP+2YRiMT0tkqS4pTlOphdnI7VZMCaN1mL4wLYsx7K/6XlsRqB6C5EzzzzDLfeeisrV65k69atzJo1i3PPPZeGhmFaD7y9CbHjJZrf+ROfVB3gc7+LnTnnMaUoB4tR97dbMgw4Y1IOS+cWa/OIY09FjF1AKBqD6vfgg0ehcfeIEyRF6Fxhb968ecydO5ff/va3AMRiMYqLi/nOd77DD37wg2Ne6/F4cLvdtLW14XINcYRyLAqRACLoQ/FrZXna6/dwYH8FrR1hghHBAcdUGnJP5azphZTlds96KJH0hj0NXjZteoc5gU1kmqPYzUYsaS5M2RMwZBSBLVMLozDZtSj2FNo209vvqK7L96FQiA8++IA777wzcUxVVRYvXsy7777brX0wGCQYDCZeezye3t2oea/m1sY11+MP81ldGyAOqxoqtPNCUJyZRqHbAgi8gTCf1npQhABiWmVgYqjREAIoSrclUjnEgmEOeEI028bSlH8iE8ZP4tyxGdjMw7f+lkR/Kps6aLSNY725gDHej8ht2YUxtg/27MOgKswqcicq3u7zBDngi4GiElNUQEUo2gNgZlE6tviWkprmDmrb/PG7aOJVkmmnwN3LFd2ZS7U9igOArkLU1NRENBolL6/rKlJeXh6fffZZt/b3338/99xzT99vFAlpUapxhD9MtP0YIhaIgFUTFyUUQQR9HO42apKkEVTM2i7ytGysjiIixW6m5eVQnGEbmmKJkhHP2VNzmV2STnmDj3pPFns8X0Bp24czsA97qBlsRoi2QyxKOBwhFAgcvbP2CMSFSLR3EPP5u5632MHYy3JNYuCGh8MqoPHOO+/k1ltvTbz2eDwUF/ei6kF6Mcz6WsJltUYEY9vje6QURUtKpqgogKIqOCwmMBtBUbDEBGUdYVBAUVQURUFRVITRisFsw24xQ3zuxwScPvJjzyRDjKIoZDssh9VCAyHKCEZiBMJRTFaT5tBEw2R4PZjbO9A8/BhKLArEEqMBo9MC8WIE6f4QxmC0Sx42h90IR+5/PNrsjXngyl/pKkTZ2dkYDAYOHOiaCOrAgQPk53fPx2KxWLBYkkipak7Tdn939gMU9nILjwnIOXaFHYlkyFEURVtZO7xartFMekY26b3cwujKgFTJ/aDr2MFsNnPSSSexfv36xLFYLMb69euZP390llWRSEYjug/Nbr31VpYtW8bJJ5/MKaecwkMPPUR7eztXXnml3qZJJJIhQnchWrp0KY2Njdx1113U19cze/ZsXn311W4T2BKJZOSiexxRf9A1jkgikRyX3n5H5fqyRCLRHSlEEolEd6QQSSQS3dF9sro/dE5v9Xqrh0QiGVI6v5vHm4oe1kLk9XoBehddLZFIdMPr9eJ2Hz0yeFivmsViMWpra3E6nfrVjteRzi0uNTU1ctWwl8j3LDmSfd+EEHi9XgoLC1GPUTByWHtEqqpSVCQ3d7lcLvml6iPyPUuOZN63Y3lCncjJaolEojtSiCQSie5IIRrGWCwWVq5cmVxGglGKfM+SY7Dft2E9WS2RSEYG0iOSSCS6I4VIIpHojhQiiUSiO1KIJBKJ7kghSmH6WgH3ueeeY8qUKVitVmbMmME//vGPIbI0tejL+7Z27dp4QYRDD6u1l+V0RggbNmzgwgsvpLCwEEVRePHFF497zRtvvMGJJ56IxWKhrKyMtWvX9ssGKUQpSl8r4L7zzjt8/etfZ8WKFXz44YdccsklXHLJJXzyySdDbLm+JFM52OVyUVdXl3hUVVUNocX6097ezqxZs3j44Yd71b6iooLzzz+fM888k23btnHzzTdz9dVXs27duuSNEJKU5JRTThE33nhj4nU0GhWFhYXi/vvv77H9ZZddJs4///wux+bNmyeuvfbaQbUz1ejr+7ZmzRrhdruHyLrUBxAvvPDCMdvcfvvtYvr06V2OLV26VJx77rlJ31d6RClIZwXcxYsXJ44dqwIuwLvvvtulPcC555571PYjkWTeNwCfz8fYsWMpLi7m4osv5tNPPx0Kc4ctg/FZk0KUghyrAm59fX2P19TX1/ep/Ugkmfdt8uTJPProo7z00ks8/vjjxGIxFixYwL59+4bC5GHJ0T5rHo8Hv99/lKuOzbDefS+R9Jf58+d3qaG3YMECpk6dyu9//3t++tOf6mjZ6EJ6RClIXyvgAuTn5/ep/UgkmfftSEwmE3PmzGHPnj2DYeKI4GifNZfLhc1mS6pPKUQpSDIVcOfPn9+lPcDrr78+qirmDkTl4Gg0yvbt2ykoKBgsM4c9g/JZS3qaWzKoPP3008JisYi1a9eKHTt2iG9/+9siPT1d1NfXCyGEuPzyy8UPfvCDRPuNGzcKo9EofvWrX4mdO3eKlStXCpPJJLZv367Xr6ALfX3f7rnnHrFu3TpRXl4uPvjgA/G1r31NWK1W8emnn+r1Kww5Xq9XfPjhh+LDDz8UgHjwwQfFhx9+KKqqqoQQQvzgBz8Ql19+eaL93r17hd1uF9///vfFzp07xcMPPywMBoN49dVXk7ZBClEK89///d+ipKREmM1mccopp4hNmzYlzi1cuFAsW7asS/tnn31WTJo0SZjNZjF9+nTx8ssvD7HFqUFf3rebb7450TYvL0+cd955YuvWrTpYrR///ve/BdDt0fk+LVu2TCxcuLDbNbNnzxZms1mMHz9erFmzpl82yDQgEolEd+QckUQi0R0pRBKJRHekEEkkEt2RQiSRSHRHCpFEItEdKUQSiUR3pBBJJBLdkUIkGVasXbuW9PT0xOu7776b2bNnJ14vX76cSy65ZMjtkvQPKUSSHlm+fDmKonDdddd1O3fjjTeiKArLly/v0n6gBaC0tJSHHnqoy7GlS5eye/fuo16zatWqLmlLFy1axM033zygdkkGHilEkqNSXFzM008/3SXHTCAQ4Mknn6SkpEQXm2w2G7m5uUc973a7u3hMkuGBFCLJUTnxxBMpLi7m+eefTxx7/vnnKSkpYc6cOf3quydP5ZJLLkl4WYsWLaKqqopbbrklkdQeug/NjuRwz2z58uW8+eabrFq1KtFHRUUFZWVl/OpXv+py3bZt21AURab/0AkpRJJjctVVV7FmzZrE60cffZQrr7xy0O/7/PPPU1RUxL333ptIat9XVq1axfz587nmmmsSfZSUlHT7nQDWrFnDGWecQVlZ2UD9CpI+IIVIcky+9a1v8fbbb1NVVUVVVRUbN27kW9/61qDfNzMzE4PBgNPpJD8/P6kEb263G7PZjN1uT/RhMBhYvnw5u3btSpQZCofDPPnkk1x11VUD/WtIeolMFSs5Jjk5OZx//vmsXbsWIQTnn38+2dnZepvVLwoLCzn//PN59NFHOeWUU/jb3/5GMBjkq1/9qt6mjVqkRyQ5LldddRVr167l//7v/wbMa1BVlSMz0ITD4QHpuzdcffXViYn4NWvWsHTpUux2+5DdX9IVKUSS4/KlL32JUChEOBzm3HPPHZA+c3Jyusz7RKPRbsUgzWYz0Wi0X/c5Wh/nnXceaWlprF69mldffVUOy3RGDs0kx8VgMLBz587Ez0ejra2Nbdu2dTmWlZVFcXFxt7ZnnXUWt956Ky+//DITJkzgwQcfpLW1tUub0tJSNmzYwNe+9jUsFktSQ8LS0lLee+89KisrcTgcZGZmoqpqYq7ozjvvZOLEiaMqt3cqIj0iSa9wuVy4XK5jtnnjjTeYM2dOl8c999zTY9urrrqKZcuWccUVV7Bw4ULGjx/PmWee2aXNvffeS2VlJRMmTCAnJycpu2+77TYMBgPTpk0jJyeH6urqxLkVK1YQCoWGZBVQcmxkqljJqOWtt97i7LPPpqamplvBQMnQIoVIMuoIBoM0NjaybNky8vPzeeKJJ/Q2adQjh2aSUcdTTz3F2LFjaW1t5Re/+IXe5kiQHpFEIkkBpEckkUh0RwqRRCLRHSlEEolEd6QQSSQS3ZFCJJFIdEcKkUQi0R0pRBKJRHekEEkkEt2RQiSRSHTn/wMestpm/GrBVwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:39:03.453868Z", + "iopub.status.busy": "2024-03-23T10:39:03.453506Z", + "iopub.status.idle": "2024-03-23T10:39:03.681986Z", + "shell.execute_reply": "2024-03-23T10:39:03.681094Z" + }, + "papermill": { + "duration": 0.250142, + "end_time": "2024-03-23T10:39:03.683918", + "exception": false, + "start_time": "2024-03-23T10:39:03.433776", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAviklEQVR4nO3dfVwU5d4/8A8Ly8K6gA8oICGroqCpoBCI6W2dw0NaHj12iswENw/3SeXc1qYZp4LIjlQqokVyZyJpmWS3PbzKEELtpIIopCECHvABUh59WgFdlt3r94c/prPuLiwKDLP7fb9evHCuuWbmO1yzX2eunbnGhjHGQAghAiPiOwBCCLkXlLwIIYJEyYsQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkh2fAfQ13Q6HS5fvgwnJyfY2NjwHQ4h5C6MMdy8eRPDhw+HSNTJ+RXrBz744APm7e3NJBIJCw4OZseOHTNZd+bMmQyAwc/s2bPN2lZNTY3R5emHfuinf/3U1NR0+lnm/cwrKysLSqUS6enpCAkJQWpqKiIjI1FRUYFhw4YZ1N+7dy/a2tq46StXrsDf3x9PPfWUWdtzcnICANTU1MDZ2blndqIf0mg0yMnJQUREBMRiMd/hkB5gLW2qUqng5eXFfVZN4T15paSkIDY2FgqFAgCQnp6O77//HhkZGXj11VcN6g8ePFhvevfu3ZBKpWYnr45LRWdnZ4tPXlKpFM7OzhZ9oFsTa2vTrrp1eE1ebW1tKCoqQnx8PFcmEokQFhaG/Px8s9axbds2PPPMMxgwYIDR+Wq1Gmq1mptWqVQA7hwIGo3mPqLv3zr2zZL30dpYS5uau3+8Jq+mpiZotVq4ubnplbu5uaG8vLzL5QsLC3H69Gls27bNZJ3k5GQkJSUZlOfk5EAqlXY/aIHJzc3lOwTSwyy9TVtbW82qx/tl4/3Ytm0bJk6ciODgYJN14uPjoVQquemO6+mIiAiLv2zMzc1FeHi4VVxiWANradOOq6Ou8Jq8XF1dYWtri/r6er3y+vp6uLu7d7psS0sLdu/ejbfeeqvTehKJBBKJxKBcLBZb9AHQwVr205pYepuau2+83qRqb2+PwMBA5OXlcWU6nQ55eXkIDQ3tdNk9e/ZArVbjueee6+0wCSH9EO+XjUqlEjExMQgKCkJwcDBSU1PR0tLCffsYHR0NT09PJCcn6y23bds2zJs3D0OGDOEjbEIIz3hPXlFRUWhsbERCQgLq6uoQEBCA7OxsrhO/urra4C7biooKHD58GDk5OXyETEiva21tNfjSqvmWGkdLqjDI9QRkjr93hfj5+VnFl093s2HMul7AoVKp4OLighs3blh8h/2+ffswe/Zsi+4fsVTFxcUIDAw0q25RURGmTJnSyxH1HXM/o7yfeRFCDPn5+aGoqEivrKL2OpR7SpDy1ET4egzUq2uNKHkR0g9JpVKDsynRxSuQ/HwL4yb4I8Cb+nppSBxCiCBR8iKECBIlL0KIIFHyIoQIEiUvQoggUfIihAgSJS9CiCBR8iKECBIlL0KIINEd9hbi7gd5TT3EC1jvg7zEslDyshDl5eVGH+R9z0hdS3uQl1gnSl4W4u4HeU09xNtRlxCho+RlIe5+kJce4iWWjjrsCSGCRMmLECJIlLwIIYJEyYsQIkiUvAghgsR78kpLS4NcLoeDgwNCQkJQWFjYaf3r169j+fLl8PDwgEQiwdixY7Fv374+ipYQ0l/weqtEVlYWlEol0tPTERISgtTUVERGRqKiogLDhg0zqN/W1obw8HAMGzYMX375JTw9PXHx4kUMHDiw74MnhPCK1+SVkpKC2NhY7gWz6enp+P7775GRkYFXX33VoH5GRgauXr2Ko0ePcq/zksvlfRkyIb3mfFMLWtTtJudXNbZwv+3sjH90B0jsMNJ1QK/E19/wlrza2tpQVFSE+Ph4rkwkEiEsLAz5+flGl/n2228RGhqK5cuX45tvvsHQoUPx7LPPYvXq1bC1tTW6jFqthlqt5qZVKhWAO+811Gg0PbhH/Ut7ezv325L301JcuNKC8NQjZtV9+cuSTufnvvgw5EOEm8DMPV55S15NTU3QarXcm7E7uLm5GbwpuMO5c+dw4MABLFy4EPv27UNlZSWWLVsGjUaDxMREo8skJycjKSnJoDwnJ0fwDyc33ALUWuPz6m8BgB2+OViAAkfT65DYAsM6mU/6Rk0zANhhkY8Wbo7G3wOt0QFX1cBgCSA20ltdf8sGOyttsT/vJ3jJejXcXtXa2mpWPUE9HqTT6TBs2DB89NFHsLW1RWBgIC5duoR169aZTF7x8fFQKpXctEqlgpeXFyIiIgT9xuwLV1qwwoz/qXdWdt3EQv+f2hKUXlZhfUkBngx/GA8ON35cajQa5ObmIjw83Ohb0Esvq7CzsgDTp083uQ4h6Lg66gpvycvV1RW2traor6/XK6+vr4e7u7vRZTw8PCAWi/UuEceNG4e6ujq0tbXB3t7eYBmJRAKJRGJQLhaLjR4AQqHW2gAAUqMC4DPM8L/ZlltqfHcoH088EooBjob7DwCVDc14Mesk1FobQf8tLEFHH5adnV2XbWHq2O3OOvozc2PnLXnZ29sjMDAQeXl5mDdvHoA7Z1Z5eXmIi4szuszDDz+MXbt2QafTQSS6c9589uxZeHh4GE1c1sBnmAwTPF0MyjUaDeqGAlO8Bwn6QCbEFF7v81Iqldi6dSs++eQTlJWVYenSpWhpaeG+fYyOjtbr0F+6dCmuXr2KFStW4OzZs/j++++xdu1aLF++nK9dIITwhNc+r6ioKDQ2NiIhIQF1dXUICAhAdnY214lfXV3NnWEBgJeXF/bv34+XXnoJkyZNgqenJ1asWIHVq1fztQuEEJ7w3mEfFxdn8jLx0KFDBmWhoaEoKCjo5agIIf0d748HEULIvaDkRQgRJN4vG8m9UWtvQ+RwCedVFRA5GN4q0d7ejsvtl1F2tczkoyTnVc0QOVyCWnsbgOE3loT0Z5S8BOpyy0UMGPk+/tH5IBz4MPvDTucPGAlcbglAINw6rUdIf0PJS6CGD/BGy/m/Y1NUAEYbuUm1vb0dRw4fwcPTHzZ55lXV0IwVWScx/FHv3g6XkB5HyUugJLYO0N32xEhnX4wfYvwm1fN25zFu8DiTN6nqbt+A7nYjJLYOvR0u6UJX3QBA110B1tYNQMmLkH7A3G4AoPOuAGvqBqDkRUg/0FU3ANB1V4C1dQNQ8iKkH+iqGwDouivA2roB6D4vQoggUfIihAgSJS9CiCBR8iKECBIlL0KIIFHyIoQIEiUvQoggUfIihAgSJS9CiCBR8iKECBIlL0KIIPWL5JWWlga5XA4HBweEhISgsND0o/WZmZmwsbHR+3FwsI5nuQghv+M9eWVlZUGpVCIxMRHFxcXw9/dHZGQkGhoaTC7j7OyM2tpa7ufixYt9GDEhpD/gPXmlpKQgNjYWCoUC48ePR3p6OqRSKTIyMkwuY2NjA3d3d+6n4z2PhBDrweuQOG1tbSgqKtJ7K7ZIJEJYWBjy8/NNLtfc3Axvb2/odDpMmTIFa9euxYMPPmi0rlqthlqt5qZVKhWAO8OLaDSaHtqTvtfe3s79NrYfHWWd7WNX6yB9x5y26KpNLaU9zY2d1+TV1NQErVZrcObk5uaG8vJyo8v4+voiIyMDkyZNwo0bN7B+/XpMmzYNpaWleOCBBwzqJycnIykpyaA8JycHUqm0Z3aEBzXNAGCHw4cP46LxsesAALm5ufe9DtL7utMWptrUUtqztbXVrHqCG4wwNDQUoaGh3PS0adMwbtw4/O///i/WrFljUD8+Ph5KpZKbVqlU8PLyQkREBJydnfsk5t5QelmF9SUFmD59Oh4cbrgfGo0Gubm5CA8PNzmGfVfrIH3HnLboqk0tpT07ro66wmvycnV1ha2tLerr6/XK6+vr4e7ubtY6xGIxJk+ejMrKSqPzJRIJJBKJ0eVMfaiFoGMYYDs7u073o7P9NHcdpPd1py1MtamltKe5sfPaYW9vb4/AwEDk5eVxZTqdDnl5eXpnV53RarUoKSmBh4dHb4VJCOmHeL9sVCqViImJQVBQEIKDg5GamoqWlhYoFAoAQHR0NDw9PZGcnAwAeOuttzB16lT4+Pjg+vXrWLduHS5evIi//vWvfO4GIaSP8Z68oqKi0NjYiISEBNTV1SEgIADZ2dlcJ351dTVEot9PEK9du4bY2FjU1dVh0KBBCAwMxNGjRzF+/Hi+doEQwgPekxcAxMXFIS4uzui8Q4cO6U1v3LgRGzdu7IOoCCH9Ge83qRJCyL2g5EUIESRKXoQQQaLkRQgRJEpehBBBouRFCBEkSl6EEEGi5EUIESRKXoQQQaLkRQgRJEpehBBB6nbyOnfuXG/EQQgh3dLt5OXj44NHH30Un376KW7fvt0bMRFCSJe6nbyKi4sxadIkKJVKuLu7429/+1un71kkhJDe0O3kFRAQgE2bNuHy5cvIyMhAbW0tpk+fjgkTJiAlJQWNjY29ESchhOi55w57Ozs7zJ8/H3v27MG7776LyspKrFy5El5eXoiOjkZtbW1PxkkIIXruOXmdOHECy5Ytg4eHB1JSUrBy5UpUVVUhNzcXly9fxty5c3syTkII0dPtkVRTUlKwfft2VFRUYPbs2dixYwdmz57NDdU8cuRIZGZmQi6X93SshBDC6Xby2rJlC55//nksXrzY5Bt7hg0bhm3btt13cIQQYkq3Lxtzc3OxevVqg8TFGEN1dTWAO680i4mJMXudaWlpkMvlcHBwQEhIiNnfXu7evRs2NjaYN2+e2dsihFiGbiev0aNHo6mpyaD86tWrGDlyZLcDyMrKglKpRGJiIoqLi+Hv74/IyEg0NDR0utyFCxewcuVKzJgxo9vbJIQIX7eTF2PMaHlzczMcHBy6HUBKSgpiY2OhUCgwfvx4pKenQyqVIiMjw+QyWq0WCxcuRFJSEkaNGtXtbRJChM/sPi+lUgkAsLGxQUJCAqRSKTdPq9Xi2LFjCAgI6NbG29raUFRUhPj4eK5MJBIhLCwM+fn5Jpd76623MGzYMCxZsgQ///xzt7ZJCLEMZievX375BcCdM6+SkhLY29tz8+zt7eHv74+VK1d2a+NNTU3QarXcC2Y7uLm5oby83Ogyhw8fxrZt23Dy5EmztqFWq6FWq7lplUoFANBoNNBoNN2Ktz9pb2/nfhvbj46yzvaxq3WQvmNOW3TVppbSnubGbnbyOnjwIABAoVBg06ZNcHZ2vrfI7sPNmzexaNEibN26Fa6urmYtk5ycjKSkJIPynJwcvbNHoalpBgA7HD58GBdlpuvl5ube9zpI7+tOW5hqU0tpz9bWVrPqdftWie3bt3c7GFNcXV1ha2uL+vp6vfL6+nq4u7sb1K+qqsKFCxcwZ84crkyn0wG4c8d/RUUFRo8erbdMfHw8d8kL3Dnz8vLyQkREBC8JuKeUXlZhfUkBpk+fjgeHG+6HRqNBbm4uwsPDIRaL72kdpO+Y0xZdtamltGfH1VFXzEpe8+fPR2ZmJpydnTF//vxO6+7du9esDQN3LjcDAwORl5fH3e6g0+mQl5eHuLg4g/p+fn4oKSnRK3v99ddx8+ZNbNq0CV5eXgbLSCQSSCQSg3KxWGzyQy0EdnZ23O/O9qOz/TR3HaT3dactTLWppbSnubGblbxcXFxgY2PD/bsnKZVKxMTEICgoCMHBwUhNTUVLSwsUCgUAIDo6Gp6enkhOToaDgwMmTJigt/zAgQMBwKCcEGLZzEpe/3mp2JOXjQAQFRWFxsZGJCQkoK6uDgEBAcjOzuY68aurq7lHjwghpEO3+7x6Q1xcnNHLRAA4dOhQp8tmZmb2fECEkH7PrOQ1efJk7rKxK8XFxfcVECGEmMOs5EXPDhJC+huzkldiYmJvx0EIId1CPeGEEEEy68xr8ODBOHv2LFxdXTFo0KBO+7+uXr3aY8ERQogpZiWvjRs3wsnJifu3uZ33hBDSW8xKXv85sODixYt7KxZCCDFbt/u8bG1tjQ4UeOXKFdja2vZIUIQQ0pUeG4xQrVbrDZNDCCG9yew77Ddv3gzgzmCEH3/8MWSy38fc0Gq1+Ne//gU/P7+ej5AQQowwO3lt3LgRwJ0zr/T0dL1LRHt7e8jlcqSnp/d8hIQQYoTZyev8+fMAgEcffRR79+7FoEGDei0oQgjpSrcfzO4YUZUQQvjU7eT1/PPPdzq/s7f+EEJIT+l28rp27ZretEajwenTp3H9+nX84Q9/6LHACCGkM91OXl999ZVBmU6nw9KlSw3GjyeEkN7SIw9mi0QiKJVK7htJQgjpbT02qkRVVRX33jhCCOlt3b5s/M/XiAF37vuqra3F999/r/cMJCGE9KZuJ6+ON2d3EIlEGDp0KDZs2NDlN5GEENJT+sV9XmlpaVi3bh3q6urg7++P999/H8HBwUbr7t27F2vXrkVlZSU0Gg3GjBmDl19+GYsWLerxuAgh/RfvI6lmZWVBqVQiMTERxcXF8Pf3R2RkpNGRK4A7AyO+9tpryM/Px6+//gqFQgGFQoH9+/f3ceSEED7xnrxSUlIQGxsLhUKB8ePHIz09HVKp1OTNro888gj+/Oc/Y9y4cRg9ejRWrFiBSZMm4fDhw30cOSGET7y+t7GtrQ1FRUWIj4/nykQiEcLCwpCfn9/l8owxHDhwABUVFXj33XeN1lGr1VCr1dy0SqUCcOfmWo1Gc597wJ+Ob3bb29uN7kdHWWf72NU6SN8xpy26alNLaU9zY+c1eTU1NUGr1XJvx+7g5uaG8vJyk8vduHEDnp6eUKvVsLW1xYcffojw8HCjdZOTk5GUlGRQnpOTA6lUen87wKOaZgCww+HDh3FRZrpebm7ufa+D9L7utIWpNrWU9mxtbTWrXo8lr99++w1vvfUWPvroo55apUlOTk44efIkmpubkZeXB6VSiVGjRuGRRx4xqBsfH693e4dKpYKXlxciIiLg7Ozc67H2ltLLKqwvKcD06dPx4HDD/dBoNMjNzUV4eDjEYvE9rYP0HXPaoqs2tZT27Lg66kqPJa8rV65g27Zt3Uperq6usLW1RX19vV55fX093N3dTS4nEong4+MDAAgICEBZWRmSk5ONJi+JRAKJRGJQLhaLTX6ohcDOzo773dl+dLaf5q6D9L7utIWpNrWU9jQ3dl477O3t7REYGIi8vDyuTKfTIS8vD6GhoWavR6fT6fVrEUIsH699XsCdO/ZjYmIQFBSE4OBgpKamoqWlBQqFAgAQHR0NT09PJCcnA7jThxUUFITRo0dDrVZj37592LlzJ7Zs2cLnbhBC+hjvySsqKgqNjY1ISEhAXV0dAgICkJ2dzXXiV1dXQyT6/QSxpaUFy5Ytw2+//QZHR0f4+fnh008/RVRUFF+7QAjhgdnJa/78+Z3Ov379+j0HERcXh7i4OKPzDh06pDf99ttv4+23377nbRFCLIPZycvFxaXL+dHR0fcdECGEmMPs5LV9+/bejIMQQrqF98eDCCHkXph95mXucDf0Ag5Cuu+WRgsAOH3phsk6LbfUONEIuF+8hgGOhvcuVjY091p8/ZHZySszMxPe3t6YPHkyGGO9GRMxQ1cHe1cHOmB9B3t/VvX/2+LVvSVd1LTDzsrjndYYIOH9JoI+YfZeLl26FJ9//jnOnz8PhUKB5557DoMHD+7N2EgnzDvYuz7QAes52PuziAfvPFEyepgMjmJbo3Uqam/g5S9LsOEvE+HrYfwLtAESO4x0HdBrcfYnNqwbp1FqtRp79+5FRkYGjh49iscffxxLlixBREQEbGxsejPOHqNSqeDi4oIbN24I+tnGqy1tyCmtM3mwm3OgA9Z1sAvdyYtXMG9LAb5eOhUB3kP4DqfXmPsZ7dZ/uRKJBAsWLMCCBQtw8eJFZGZmYtmyZWhvb0dpaSlkMgE/yi4wgwfY45ngESbndwyPMnroAEzw7Pw2F0KE6J6/bRSJRLCxsQFjDFqttidjIoSQLnUreanVanz++ecIDw/H2LFjUVJSgg8++ADV1dV01kUI6VNmXzYuW7YMu3fvhpeXF55//nl8/vnncHV17c3YCCHEJLOTV3p6OkaMGIFRo0bhp59+wk8//WS03t69e3ssOEIIMcXs5BUdHS2YbxQJIZavWzepEkJIf0HPNhJCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJD6RfJKS0uDXC6Hg4MDQkJCUFhYaLLu1q1bMWPGDAwaNAiDBg1CWFhYp/UJIZaJ9+SVlZUFpVKJxMREFBcXw9/fH5GRkWhoaDBa/9ChQ1iwYAEOHjyI/Px8eHl5ISIiApcuXerjyAkhfOI9eaWkpCA2NhYKhQLjx49Heno6pFKpyeGkP/vsMyxbtgwBAQHw8/PDxx9/zL1lmxBiPXgdQrOtrQ1FRUWIj4/nykQiEcLCwpCfn2/WOlpbW6HRaEyO6qpWq6FWq7lplUoFANBoNNBoNPcRff/WMZ5Xe3u7Re+nNbGWNjV333hNXk1NTdBqtdzbsTu4ubmhvLzcrHWsXr0aw4cPR1hYmNH5ycnJSEpKMijPycmBVCrtftACUdMMAHYoKCjApdN8R0N6grW0aWtrq1n1BD14+TvvvIPdu3fj0KFDcHBwMFonPj4eSqWSm1apVFw/mZCHge7KqeqrQMkJTJ06Ff4j6F0DlsBa2rTj6qgrvCYvV1dX2Nraor6+Xq+8vr4e7u7unS67fv16vPPOO/jxxx8xadIkk/UkEgkkEsO354jFYojF4nsLXADs7Oy435a8n9bEWtrU3H3jtcPe3t4egYGBep3tHZ3voaGhJpd77733sGbNGmRnZyMoKKgvQiWE9DO8XzYqlUrExMQgKCgIwcHBSE1NRUtLCxQKBYA744h5enoiOTkZAPDuu+8iISEBu3btglwuR11dHQBAJpPRUNSEWBHek1dUVBQaGxuRkJCAuro6BAQEIDs7m+vEr66uhkj0+wnili1b0NbWhr/85S9660lMTMSbb77Zl6ETQnjEe/ICgLi4OMTFxRmdd+jQIb3pCxcu9H5AhJB+j/ebVAkh5F5Q8iKECBIlL0KIIFHyIoQIEiUvQoggUfIihAgSJS9CiCBR8iKECBIlL0KIIFHyIoQIEiUvQoggUfIihAgSJS9CiCBR8iKECBIlL0KIIFHyIoQIEiUvQoggUfIihAgSJS9CiCBR8iKECBLvySstLQ1yuRwODg4ICQlBYWGhybqlpaV48sknIZfLYWNjg9TU1L4LlBDSr/CavLKysqBUKpGYmIji4mL4+/sjMjISDQ0NRuu3trZi1KhReOedd7p8ozYhxLLxmrxSUlIQGxsLhUKB8ePHIz09HVKpFBkZGUbrP/TQQ1i3bh2eeeYZSCSSPo6WENKf8Pbexra2NhQVFSE+Pp4rE4lECAsLQ35+fo9tR61WQ61Wc9MqlQoAoNFooNFoemw7/U17ezv325L305pYS5uau2+8Ja+mpiZotVruzdgd3NzcUF5e3mPbSU5ORlJSkkF5Tk4OpFJpj22nv6lpBgA7FBQU4NJpvqMhPcFa2rS1tdWsev3ijdm9KT4+HkqlkptWqVTw8vJCREQEnJ2deYysd52qvgqUnMDUqVPhP2Iw3+GQHmAtbdpxddQV3pKXq6srbG1tUV9fr1deX1/fo53xEonEaP+YWCyGWCzuse30N3Z2dtxvS95Pa2ItbWruvvHWYW9vb4/AwEDk5eVxZTqdDnl5eQgNDeUrLEKIQPB62ahUKhETE4OgoCAEBwcjNTUVLS0tUCgUAIDo6Gh4enoiOTkZwJ1O/jNnznD/vnTpEk6ePAmZTAYfHx/e9oMQ0vd4TV5RUVFobGxEQkIC6urqEBAQgOzsbK4Tv7q6GiLR7yeHly9fxuTJk7np9evXY/369Zg5cyYOHTrU1+ETQnjEe4d9XFwc4uLijM67OyHJ5XIwxvogKkJIf8f740GEEHIvKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSpXySvtLQ0yOVyODg4ICQkBIWFhZ3W37NnD/z8/ODg4ICJEydi3759fRQpIaS/4D15ZWVlQalUIjExEcXFxfD390dkZCQaGhqM1j969CgWLFiAJUuW4JdffsG8efMwb948nD59uo8jJ4TwiffklZKSgtjYWCgUCowfPx7p6emQSqXIyMgwWn/Tpk147LHHsGrVKowbNw5r1qzBlClT8MEHH/Rx5IQQPvH63sa2tjYUFRUhPj6eKxOJRAgLC0N+fr7RZfLz86FUKvXKIiMj8fXXXxutr1aroVaruWmVSgUA0Gg00Gg097kH/UdraysqKiq46bO1N6Cuq8Tpk/Zoq3fRq+vr6wupVNrXIZJuuLs9AdNtamntae7nktfk1dTUBK1Wy70hu4ObmxvKy8uNLlNXV2e0fl1dndH6ycnJSEpKMijPycmxqAavqqrCyy+/bFC+6BPDuhs2bMDo0aP7ICpyr0y1J2DYppbWnq2trWbV4/2N2b0tPj5e70xNpVLBy8sLERERcHZ25jGyntXa2orp06dz08231Nj/83FEzngIMkeJXl1L+5/aEt3dnoDpNrW09uy4OuoKr8nL1dUVtra2qK+v1yuvr6+Hu7u70WXc3d27VV8ikUAikRiUi8ViiMXie4y8/3FxcUFwcDA3rdFocPP6VcyYNtWi9tNa3N2egPW0qbn7xmuHvb29PQIDA5GXl8eV6XQ65OXlITQ01OgyoaGhevUBIDc312R9Qohl4v2yUalUIiYmBkFBQQgODkZqaipaWlqgUCgAANHR0fD09ERycjIAYMWKFZg5cyY2bNiAxx9/HLt378aJEyfw0Ucf8bkbhJA+xnvyioqKQmNjIxISElBXV4eAgABkZ2dznfLV1dUQiX4/QZw2bRp27dqF119/Hf/4xz8wZswYfP3115gwYQJfu0AI4YENY4zxHURfUqlUcHFxwY0bNyyqw/5uGo0G+/btw+zZsy26f8SaWEubmvsZ5f0mVUIIuReUvAghgkTJixAiSLx32Pe1ji4+c2+EEyqNRoPW1laoVCqL7h+xJtbSph2fza66460ued28eRMA4OXlxXMkhJDO3Lx5Ey4uLibnW923jTqdDpcvX4aTkxNsbGz4DqfXdDwGVVNTY9HfqloTa2lTxhhu3ryJ4cOH690mdTerO/MSiUR44IEH+A6jzzg7O1v0gW6NrKFNOzvj6kAd9oQQQaLkRQgRJEpeFkoikSAxMdHoiBpEmKhN9Vldhz0hxDLQmRchRJAoeRFCBImSFyFEkCh5dWHx4sWYN29ej67zkUcewYsvvthpHblcjtTU1B7dLiGWxKqSlzlJgwjLm2++iYCAAL7DMKm/HXP9LZ77YVXJixAhamtr4zuEfslqktfixYvx008/YdOmTbCxsYGNjQ2qqqqwZMkSjBw5Eo6OjvD19cWmTZuMLp+UlIShQ4fC2dkZL7zwgtkHVEtLC6KjoyGTyeDh4YENGzYY1GloaMCcOXPg6OiIkSNH4rPPPjOoY2Njgy1btmDWrFlwdHTEqFGj8OWXX3LzL1y4ABsbG3zxxReYMWMGHB0d8dBDD+Hs2bM4fvw4goKCIJPJMGvWLDQ2Npr5VwMyMjLw4IMPQiKRwMPDA3Fxcdy86upqzJ07FzKZDM7Oznj66af13uzUcVa0c+dOyOVyuLi44JlnnuEejgfuPGv63nvvwcfHBxKJBCNGjMA///lPbv7q1asxduxYSKVSjBo1Cm+88Qb3UtLMzEwkJSXh1KlTXJtmZmaavW+97V6PuY6uin/+858YPnw4fH19AQBHjx5FQEAAHBwcEBQUhK+//ho2NjY4efIkt+zp06cxa9YsyGQyuLm5YdGiRWhqajIZz4ULF/rqz9HzmJW4fv06Cw0NZbGxsay2tpbV1tay27dvs4SEBHb8+HF27tw59umnnzKpVMqysrK45WJiYphMJmNRUVHs9OnT7LvvvmNDhw5l//jHP8za7tKlS9mIESPYjz/+yH799Vf2xBNPMCcnJ7ZixQquzqxZs5i/vz/Lz89nJ06cYNOmTWOOjo5s48aNXB0AbMiQIWzr1q2soqKCvf7668zW1padOXOGMcbY+fPnGQDm5+fHsrOz2ZkzZ9jUqVNZYGAge+SRR9jhw4dZcXEx8/HxYS+88IJZsX/44YfMwcGBpaamsoqKClZYWMjFpNVqWUBAAJs+fTo7ceIEKygoYIGBgWzmzJnc8omJiUwmk7H58+ezkpIS9q9//Yu5u7vr/e1eeeUVNmjQIJaZmckqKyvZzz//zLZu3crNX7NmDTty5Ag7f/48+/bbb5mbmxt79913GWOMtba2spdffpk9+OCDXJu2traatW994X6PuUWLFrHTp0+z06dPsxs3brDBgwez5557jpWWlrJ9+/axsWPHMgDsl19+YYwxdu3aNTZ06FAWHx/PysrKWHFxMQsPD2ePPvqoyXja29v5+NP0CKtJXowxNnPmTL2kYczy5cvZk08+yU3HxMSwwYMHs5aWFq5sy5YtTCaTMa1W2+m6bt68yezt7dkXX3zBlV25coU5OjpycVRUVDAArLCwkKtTVlbGABgkr7uTTkhICFu6dClj7Pfk9fHHH3PzP//8cwaA5eXlcWXJycnM19e307g7DB8+nL322mtG5+Xk5DBbW1tWXV3NlZWWlurtS2JiIpNKpUylUnF1Vq1axUJCQhhjjKlUKiaRSPSSVVfWrVvHAgMDuenExETm7+9v9vJ97V6POTc3N6ZWq7myLVu2sCFDhrBbt25xZVu3btVLXmvWrGERERF6666pqWEAWEVFhdnxCIXVjSpxt7S0NGRkZKC6uhq3bt1CW1ubQQewv7+/3huJQ0ND0dzcjJqaGnh7e5tcd1VVFdra2hASEsKVDR48mLsMAICysjLY2dkhMDCQK/Pz88PAgQMN1nf3uylDQ0P1LhkAYNKkSdy/O97ANHHiRL2yhoYGkzF3aGhowOXLl/HHP/7R6PyysjJ4eXnpjYs2fvx4DBw4EGVlZXjooYcA3PnW1MnJiavj4eHBbb+srAxqtdrkNgAgKysLmzdvRlVVFZqbm9He3i74ERXMOeYmTpwIe3t7brqiogKTJk2Cg4MDV3b3S2lPnTqFgwcPQiaTGWyzqqoKY8eO7dkd4ZnV9HkZs3v3bqxcuRJLlixBTk4OTp48CYVCIegO0v8cYbNjvLK7y3Q6XZfrcXR07PF47t5+V9vIz8/HwoULMXv2bHz33Xf45Zdf8Nprrwm6fcw95gYMGNDtdTc3N2POnDk4efKk3s+///1v/Nd//VdP7UK/YVXJy97eHlqtlps+cuQIpk2bhmXLlmHy5Mnw8fFBVVWVwXKnTp3CrVu3uOmCggLIZLIuR2MdPXo0xGIxjh07xpVdu3YNZ8+e5ab9/PzQ3t6OoqIirqyiogLXr183WF9BQYHB9Lhx4zqN4V45OTlBLpcbvJ28w7hx41BTU4Oamhqu7MyZM7h+/TrGjx9v1jbGjBkDR0dHk9s4evQovL298dprryEoKAhjxozBxYsX9erc3ab9zb0ec3fz9fVFSUkJ1Go1V3b8+HG9OlOmTEFpaSnkcjl8fHz0fjqSYX//e3WHVSUvuVyOY8eO4cKFC2hqasKYMWNw4sQJ7N+/H2fPnsUbb7xhcEAAd76qXrJkCc6cOYN9+/YhMTERcXFxnY7yCAAymQxLlizBqlWrcODAAZw+fRqLFy/WW87X1xePPfYY/va3v+HYsWMoKirCX//6V6NnJXv27EFGRgbOnj2LxMREFBYW6n3719PefPNNbNiwAZs3b8a///1vFBcX4/333wcAhIWFYeLEiVi4cCGKi4tRWFiI6OhozJw5E0FBQWat38HBAatXr8Yrr7yCHTt2oKqqCgUFBdi2bRuAO8mturoau3fvRlVVFTZv3oyvvvpKbx1yuRznz5/HyZMn0dTUpPfh7g/u9Zi727PPPgudTof//u//RllZGfbv34/169cD+P0Me/ny5bh69SoWLFiA48ePo6qqCvv374dCoeAS1t3xmHMW3m/x3enWlyoqKtjUqVOZo6MjA8DKy8vZ4sWLmYuLCxs4cCBbunQpe/XVV/U6gGNiYtjcuXNZQkICGzJkCJPJZCw2Npbdvn3brG3evHmTPffcc0wqlTI3Nzf23nvvGXSa1tbWsscff5xJJBI2YsQItmPHDubt7W3QYZ+WlsbCw8OZRCJhcrlc7xuqjg77js5bxhg7ePAgA8CuXbvGlW3fvp25uLiY/TdLT09nvr6+TCwWMw8PD/b3v/+dm3fx4kX2pz/9iQ0YMIA5OTmxp556itXV1XHzjXWmb9y4kXl7e3PTWq2Wvf3228zb25uJxWI2YsQItnbtWm7+qlWruL97VFQU27hxo178t2/fZk8++SQbOHAgA8C2b99u9r71hfs55u525MgRNmnSJGZvb88CAwPZrl27uHV2OHv2LPvzn//MBg4cyBwdHZmfnx978cUXmU6nMxrP+fPne/kv0HtoSByBsLGxwVdffdXjjyoR4frss8+gUChw48aNHuujFBKr/7aREKHYsWMHRo0aBU9PT5w6dQqrV6/G008/bZWJC6DkdV+qq6s77Zw+c+YMRowY0YcRdY+xr9Q7/PDDD5gxY0YfRkO6UldXh4SEBNTV1cHDwwNPPfWU3tMI1oYuG+9De3t7p49XyOVy2Nn13/8fKisrTc7z9PS02v/RiTBQ8iKECJJV3SpBCLEclLwIIYJEyYsQIkiUvAghgkTJi/SqxYsXcwPficViuLm5ITw8HBkZGd16NCUzM9PoSBu9rTfeYUB6BiUv0usee+wx1NbW4sKFC/jhhx/w6KOPYsWKFXjiiSfQ3t7Od3hEqPh8NolYPlPP6eXl5TEA3ECEGzZsYBMmTGBSqZQ98MADbOnSpezmzZuMsd+f0fzPn8TERMYYYzt27GCBgYFMJpMxNzc3tmDBAlZfX89t5+rVq+zZZ59lrq6uzMHBgfn4+LCMjAxufnV1NXvqqaeYi4sLGzRoEPvTn/7EPe+XmJhosN2DBw/2yt+JdB+deRFe/OEPf4C/vz/27t0LABCJRNi8eTNKS0vxySef4MCBA3jllVcAANOmTUNqaiqcnZ1RW1uL2tparFy5EgCg0WiwZs0anDp1Cl9//TUuXLiAxYsXc9t54403cObMGfzwww8oKyvDli1b4Orqyi0bGRkJJycn/Pzzzzhy5AhkMhkee+wxtLW1YeXKlXj66ae5M8fa2lpMmzatb/9QxDS+syexbKbOvBhjLCoqio0bN87ovD179rAhQ4Zw0+aOhnH8+HEGgDtrmzNnDlMoFEbr7ty5k/n6+nIjLjDGmFqtZo6Ojmz//v1dxk/4RWdehDeMMW4sqh9//BF//OMf4enpCScnJyxatAhXrlxBa2trp+soKirCnDlzMGLECDg5OWHmzJkA7jx3CgBLly7F7t27ERAQgFdeeQVHjx7llj116hQqKyvh5OQEmUwGmUyGwYMH4/bt22YNEEj4RcmL8KasrAwjR47EhQsX8MQTT2DSpEn4v//7PxQVFSEtLQ1A5+8sbGlpQWRkJJydnfHZZ5/h+PHj3GCFHcvNmjULFy9exEsvvcSNyd9xydnc3IzAwECDYZPPnj2LZ599tpf3ntyv/vvUMLFoBw4cQElJCV566SUUFRVBp9Nhw4YN3CizX3zxhV59Y8MXl5eX48qVK3jnnXe4IblPnDhhsK2hQ4ciJiYGMTExmDFjBlatWoX169djypQpyMrKwrBhw0y+1MOShk22NHTmRXqdWq1GXV0dLl26hOLiYqxduxZz587FE088gejoaPj4+ECj0eD999/HuXPnsHPnTqSnp+utQy6Xo7m5GXl5eWhqakJraytGjBgBe3t7brlvv/0Wa9as0VsuISEB33zzDSorK1FaWorvvvuOG/d/4cKFcHV1xdy5c/Hzzz/j/PnzOHToEP7nf/4Hv/32G7fdX3/9FRUVFWhqauJeeEv6Ab473Yhli4mJ4W4zsLOzY0OHDmVhYWEsIyND772XKSkpzMPDgzk6OrLIyEi2Y8cOgyGsX3jhBTZkyBC9WyV27drF5HI5k0gkLDQ0lH377bcG7zIcN24cc3R0ZIMHD2Zz585l586d49ZZW1vLoqOjmaurK5NIJGzUqFEsNjaW3bhxgzHGWENDAwsPD2cymYxulehnaEgcQogg0WUjIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIE6f8B4mOdoOEeKq8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:39:03.721755Z", + "iopub.status.busy": "2024-03-23T10:39:03.721395Z", + "iopub.status.idle": "2024-03-23T10:39:03.941121Z", + "shell.execute_reply": "2024-03-23T10:39:03.940199Z" + }, + "papermill": { + "duration": 0.240931, + "end_time": "2024-03-23T10:39:03.943047", + "exception": false, + "start_time": "2024-03-23T10:39:03.702116", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATMAAAEmCAYAAAAOQ3d2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSpElEQVR4nO2dd1hUx/rHv2d32V16kSqCgCKKBQxNNMaGojFqyv2FGKNojClqTCQmtkRjTIKJ5XIT243GmGKNUeNN0Shq7A3FAoihCUoHYalb5/fHsgeWuotbYJnP85xHd86cc97ZZb8788477zCEEAIKhULp5HCMbQCFQqHoAipmFArFJKBiRqFQTAIqZhQKxSSgYkahUEwCKmYUCsUkoGJGoVBMAipmFArFJOAZ2wBDo1AokJubC2trazAMY2xzKBRKIwghqKioQPfu3cHhaN7f6nJilpubCw8PD2ObQaFQ2iAnJwc9evTQuH6XEzNra2sAyjfKxsbGyNZQKJTGiEQieHh4sN9VTelyYqYaWtrY2FAxo1A6MNq6gegEAIVCMQmomFEoFJOgQ4jZpk2b4OXlBaFQiLCwMFy5cqXFujt37gTDMGqHUCg0oLUUCqUjYnQx27dvH2JiYrBy5Upcv34dAQEBiIyMRGFhYYvX2NjYIC8vjz3u379vQIspFEpHxOhitmHDBsyZMwezZs2Cv78/tm7dCgsLC+zYsaPFaxiGgaurK3u4uLgY0GIKhdIRMaqYSSQSJCQkICIigi3jcDiIiIjAxYsXW7yusrISPXv2hIeHB6ZMmYKkpKQW64rFYohEIrWDQqGYHkYVs+LiYsjl8iY9KxcXF+Tn5zd7jZ+fH3bs2IFff/0VP/30ExQKBYYOHYoHDx40Wz82Nha2trbsQQNmKaZKjUSOebuu45eE5r8Lpo7Rh5naEh4ejhkzZiAwMBAjRozAwYMH4eTkhP/+97/N1l+6dCnKy8vZIycnx8AWUyiGIf5uAX6/nYfVvydDJlcY2xyDY9SgWUdHR3C5XBQUFKiVFxQUwNXVVaN7mJmZYfDgwUhLS2v2vEAggEAgeGxbKZSOTkZRFQCgrFqKK1mlGNrL0cgWGRaj9sz4fD6CgoIQHx/PlikUCsTHxyM8PFyje8jlcty+fRtubm76MpNC6RRkFlex//8rqaCVmqaJ0YeZMTEx2LZtG77//nukpKTgrbfeQlVVFWbNmgUAmDFjBpYuXcrW/+STT/DXX38hIyMD169fxyuvvIL79+/jtddeM1YTKJQOQUYDMTueXICutouk0ddmRkVFoaioCCtWrEB+fj4CAwNx9OhRdlIgOztbLQ3Io0ePMGfOHOTn58Pe3h5BQUG4cOEC/P39jdUECsXoEEKQUVTJvn5YVoOkXBEGuNsa0SrDwnS1TYBFIhFsbW1RXl5OF5pTTIbiSjGCPz0BhgFG9nHCqdQivD26N94b52ds07Smvd9Row8zKRTK46Pyl3W3NceUQHcAXc9vRsWMQjEBVENMHydLjPJzBo/DILWgAlkN/GimDhUzCsUEUDn/vR0tYWthhvBe3QAAx5KaDz43RaiYUSgmQGZdjJmPoyUAYJy/cgLtr+SuM9SkYkahmABsz8zJCgAw1l8ZdH49+xEKK2qNZpchoWJGoXRy5AqC+yXqPTNXWyECPOxACHAiueV0WqYEFTMKpZPz8FENpHICPo+D7nbmbHlkf+VQs6v4zaiYUSidnPRi5UymVzcLcDn1m4CMqxtqXkgvRkWt1Ci2GRIqZhRKJ6fe+W+lVt7b2Qq9nCwhlROcSi0yhmkGhYoZhdLJyWSd/5ZNzo3rr+yd/dUFhppUzCiUTk5G3TDT27GpmEXWidnp1CKIZXKD2mVoqJhRKJ0c1TCzVzM9s0HutnCxEaBSLMOF9BJDm2ZQqJhRKJ2YGokcueXKODLvRj4zAOBwGHYiwNSHmlTMKJROjMpfZmtuBnsLs2brjKsL0TieXAC5wnST5FAxo1A6MSox83GyBMMwzdYZ4tMN1kIeiisluJH9yJDmGRQqZhRKJyazFee/CjMuB2P6OgMw7bWaVMwolE5MRqMF5i2hmtU8lpRvsum0qZhRKJ2YDHaY2dT535Cn+jiBz+Pgfkk17hVUtlq3s0LFjELppDTM+9/aMBMALAU8POWr3HrOVNdqUjGjUDoppVUSiGplAACvbq2LGVC/GoCKGYVC6VCoZjLd7cxhzue2WX9MX2dwGCApV4QHj6r1bZ7BoWJGoXRSGqbK1oRuVgKEeDkAMM3NTqiYUSidlEwtxQxosPA82fSGmlTMKJROSsMdmTRFtTfAlcxSlFZJ9GKXsaBiRqF0UtrTM/NwsIC/mw0UBIhPMa2hJhUzCqUTIlcQZJUonfiNkzK2RX0ALRUzCoViZHLLaiCRKcDncuBub972BQ1QLTw/+08RqiUyfZhnFKiYUSidENVMZs9Gef81oa+rNTwdLCCWKXDmnumk06ZiRqF0QjI1jPxvDoZh6jcJNqGhJhUzCqUToumazJaIHKD0m51IKYBUrtCZXcaEihmF0glh85i1o2cGAE942qObJR+iWhmuZJbq0jSj0SHEbNOmTfDy8oJQKERYWBiuXLmi0XV79+4FwzB49tln9WsghdLBUKX+aW5HJk3gchiM9TetTYKNLmb79u1DTEwMVq5cievXryMgIACRkZEoLGx9S/msrCwsWrQIw4cPN5ClFErHoFYqR255DYD298yA+lnNv5IKTCLHmdHFbMOGDZgzZw5mzZoFf39/bN26FRYWFtixY0eL18jlckybNg2rVq2Cj4+PAa2lUIxPVkkVCAFshDw4WPLbfZ+hvRxhyeciX1SLWw/KdWihcTCqmEkkEiQkJCAiIoIt43A4iIiIwMWLF1u87pNPPoGzszNmz57d5jPEYjFEIpHaQaF0ZjLZIaZVi3n/NUFoxsVIP2U6bVMYahpVzIqLiyGXy+Hi4qJW7uLigvz85t/cc+fO4dtvv8W2bds0ekZsbCxsbW3Zw8PD47HtplCMiWoms9djDDFVsENNE9gbwOjDTG2oqKjA9OnTsW3bNjg6Omp0zdKlS1FeXs4eOTk5eraSQtEvrPNfB2I2qq8zzLgM0gorkV7UudNp84z5cEdHR3C5XBQUqP8qFBQUwNXVtUn99PR0ZGVlYdKkSWyZQqGMkeHxeEhNTUWvXr3UrhEIBBAIBHqwnkIxDuyOTO2cyWyIjdAM4b0cceZeEf5KKsBbI9sXt9YRMGrPjM/nIygoCPHx8WyZQqFAfHw8wsPDm9Tv27cvbt++jcTERPaYPHkyRo0ahcTERDqEpHQJ2IBZLReYt8Q4EwnRMGrPDABiYmIQHR2N4OBghIaGIi4uDlVVVZg1axYAYMaMGXB3d0dsbCyEQiEGDBigdr2dnR0ANCmnUEyRR1USlFVLAQBejhY6uec4fxd89OsdJOaUoUBUCxcboU7ua2iMLmZRUVEoKirCihUrkJ+fj8DAQBw9epSdFMjOzgaH06lcexSK3lD1ytxshbDg6+br62wjxGAPO1zPLsNfyQWYPqSnTu5raIwuZgAwf/58zJ8/v9lzp0+fbvXanTt36t4gCqWD0p7sspowrr+rUsyS8jutmNEuD4XSiWhPdllNUCVsvJhegvIaqU7vbSiomFEonYh6MdPtrKO3oyV8na0gUxCcTm19KWFHhYoZhdKJUMWY6XqYCTRMp905ZzWpmFEonQSFgiCz5PFS/7SGajXA6dQi1ErlOr+/vqFiRqF0EnLLlXn/zbgM3O20y/uvCQPdbeFmK0S1RI7zacU6v7++oWJGoXQSVEPMnt0swePq/qvb2dNpUzGjUDoJ+prJbIjKb3YipQByRefKcUbFjELpJDxuqmxNCPF2gK25GUqqJLiW1bnSaT+WmFVWVtJcYRSKgUjXU8BsQ8y4HIzpp8xx1tnSAmktZpmZmZg4cSIsLS1ha2sLe3t72Nvbw87ODvb29vqwkUKhQH8xZo0Z518fotGZ0mlrvZzplVdeASEEO3bsgIuLy2NluqRQKJpRK5XjYZky778+fWYAMKKPE4RmHDx4VIOUvAr4d7fR6/N0hdZidvPmTSQkJMDPz08f9lAolGa4X1INQgBrIQ+OVu3P+68J5nwuhvs64XhyAY4l5XcaMdN6mBkSEkKztVIoBkaVkNHH0dIgoyHVrGZn8ptp3TPbvn073nzzTTx8+BADBgyAmZmZ2vlBgwbpzDgKhaIkwwBhGQ0Z09cZXA6DlDwRckqr4eGgm9xp+kRrMSsqKkJ6ejqbPBFQBtsRQsAwDOTyzrcMgkLp6NSvyTRMWmt7Sz5CvRxwMaMEx5Ly8drwjr+lo9Zi9uqrr2Lw4MHYs2cPnQCgUAyEIQJmGzOuvwsuZpTgr6QC0xSz+/fv48iRI+jdu7c+7KFQKM1gHDFzxar/JePa/VIUV4rhaNWxNwbSegJg9OjRuHnzpj5soVAozVBWLUFplQSAYcXM3c4cA9xtoCBAfErHnwjQumc2adIkLFy4ELdv38bAgQObTABMnjxZZ8ZRKJR657+rjRCWAsNmuo/0d8WdhyL8lVSAqBBPgz5bW7R+Z958800AwCeffNLkHJ0AoFB0T6YON/3VlsgBrlh//B7OphWjUiyDlYHFVBu0HmYqFIoWDypkFIruySjW/5rMlvB1toJXNwtIZAqcuVdk8Odrg1ZiJpVKwePxcOfOHX3ZQ6FQGmEM578KhmE6TTptrcTMzMwMnp6etAdGoRgQfeb91wRVOu2TdwshkSmMYoMmaD3MXL58OZYtW4bS0s6V64hC6YwoFARZbN5/wwTMNmawhz0crQSoqJXhUkaJUWzQBK29eRs3bkRaWhq6d++Onj17wtJS/dfi+vXrOjOOQunq5IlqUStVgMdh0MNe93n/NYHDYTDW3wV7rmTjWFI+nurjZBQ72kJrMXv22Wf1YAaFQmkO1UymZzcLveT915TI/koxO55cgNVTBoDD6Xgrf7QWs5UrV+rDDgqF0gzsTKaRhpgqwnt1g5WAh8IKMRIflOEJz46XiLXdQSMJCQlISUkBAPTv3x+DBw/WmVEUCkWJsZ3/KgQ8Lkb1dcb/bubir6SCDilmWvdbCwsLMXr0aISEhGDBggVYsGABgoKCMGbMGBQVdew4FAqls2HMsIzG1G9D1zHTaWstZm+//TYqKiqQlJSE0tJSlJaW4s6dOxCJRFiwYIE+bKRQuiwZDZIyGpuRfk7gcznIKK5iN1fpSGgtZkePHsXmzZvRr18/tszf3x+bNm3Cn3/+qVPjKJSujFgmx4NHdXn/jTzMBABroRmG9u4GADjWATcJbtdypsaLywFlQK1C0b6Auk2bNsHLywtCoRBhYWG4cuVKi3UPHjyI4OBg2NnZwdLSEoGBgfjxxx/b9VwKpSOTXZf330rAg1MHSb/DptPugKsB2pUC6J133kFubi5b9vDhQyxcuBBjxozR2oB9+/YhJiYGK1euxPXr1xEQEIDIyEgUFhY2W9/BwQHLly/HxYsXcevWLcyaNQuzZs3CsWPHtH42hdKRSW/g/O8oSVAj+rmAYYCbD8qRV15jbHPUYIiWnrycnBxMnjwZSUlJ8PDwYMsGDBiAI0eOoEePHloZEBYWhpCQEGzcuBGAsufn4eGBt99+G0uWLNHoHk888QQmTpyI1atXt1lXJBLB1tYW5eXlsLFpedcZuVwOqVSqWSMoFD2w50o2tp/NwOi+zlg+0d/Y5rAs2HMDSbnleHu0L54d7N5iPTMzM3C5XK3vr+l3tDFaixkAEEJw4sQJ3L17FwDQr18/REREaHsbSCQSWFhY4MCBA2rBuNHR0SgrK8Ovv/7aph0nT57E5MmTcfjwYYwdO7ZJHbFYDLFYzL4WiUTw8PBo8Y0ihCA/Px9lZWVat4dC0SWPqiWoEsthY86DjbCpa8dYVNRKUV4jg5DHgaN168NfOzs7uLq6atWzbK+YtSvOjGEYjB07tlnx0Ibi4mLI5XK4uLiolbu4uLBC2Rzl5eVwd3eHWCwGl8vF5s2bW7QlNjYWq1at0tgmlZA5OzvDwsKiw3TvKV0PbkkVLKVyuNkKYWOu370ytUEilSOzpAoMGHg4WTa7MoEQgurqatZd5Obmpne72iVm8fHxiI+PR2FhYROn/44dO3RiWGtYW1sjMTERlZWViI+PR0xMDHx8fDBy5MgmdZcuXYqYmBj2tapn1hxyuZwVsm7duunLfApFI2SMBAyPC2tLCwj5HScpolAImFcrUCuVQwoerITNC625uXItaWFhIZydnds15NQGrd+hVatW4ZNPPkFwcDDc3Nweq+fi6OgILpeLggL1ad6CggK4urq2eB2Hw2E3VAkMDERKSgpiY2ObFTOBQACBQLOZIJWPzMKi4+8RSDFtZHIFZHUdBT5PvyLQHmyEZqiVyiGqlcLesuVeo+q7JJVKO56Ybd26FTt37sT06dMf++F8Ph9BQUGIj49nfWYKhQLx8fGYP3++xvdRKBRqfrHHhQ4tKcZGlTfMjMsBtwMu6rY156GwAqiolUGhIC0uPDfkd0lrMZNIJBg6dKjODIiJiUF0dDSCg4MRGhqKuLg4VFVVsZsMz5gxA+7u7oiNjQWg9IEFBwejV69eEIvF+OOPP/Djjz9iy5YtOrOJQjE2YpmqV2a8TBmtITTjgs/lQCJXoEIsg6258ScotH6nXnvtNezevVtnBkRFRWHdunVYsWIFAgMDkZiYiKNHj7KTAtnZ2cjLy2PrV1VVYe7cuejfvz+GDRuGX375BT/99BNee+01ndnUVZg5c6bOUzqNHDkS7777bqt1vLy8EBcXp9PnmhoqMRN0UDFjGAY25maY/X/PIGbhu8Y2B0A7ema1tbX45ptvcOLECQwaNKjJaoANGzZobcT8+fNbHFaePn1a7fWnn36KTz/9VOtnmDojR45EYGAgFQkTQSyTY8uGNTh74k/cudUx96lVhYtIZAoQQozuntFazG7duoXAwEAAaLKxibEbQ6GYCqqemSH6ZVKptNklim1hKeCCYQACoEosh5XQuDOuWr9Xp06davE4efKkPmyktMHMmTPx999/4z//+Q8YhgHDMEhPT8fs2bPh7e0Nc3Nz+Pn54T//+U+z169atQpOTk6wsbHBm2++CYlEotFzq6qqMGPGDFhZWcHNzQ3r169vUqewsBCTJk2Cubk5vL29sWvXriZ1GIbBli1bMGHCBJibm8PHxwcHDhxgz2dlZYFhGOzfvx/Dhw+Hubk5QkJCcO/ePVy9ehXBwcGwsrLChAkTtEpDtWPHDvTv3x8CgQBubm5qo4Ps7GxMmTIFVlZWsLGxwYsvvqg26/7xxx+z64K9vLxga2uLl156CRUVFWwdhUKBL7/8Er1794ZAIICnpyc+++wz9vzixYvRp08fWFhYwMfHBx999BGkUikIIdi/+0ds/fcXuH37FvuZ7ty5s8023b17F08++SSEQiH8/f1x4sQJMAyDw4cPq72X+/btw4gRIyAUCrFr1y6UlJRg6tSpcHd3h4WFBQYOHIg9e/ao3bvx571hwwZwOUoJEdV2gNUypItRXl5OAJDy8vIm52pqakhycjKpqalhyxQKBakSS41yKBQKjdpUVlZGwsPDyZw5c0heXh7Jy8sjtbW1ZMWKFeTq1askIyOD/PTTT8TCwoLs27ePvS46OppYWVmRqKgocufOHfLbb78RJycnsmzZMo2e+9ZbbxFPT09y4sQJcuvWLfLMM88Qa2tr8s4777B1JkyYQAICAsjFixfJtWvXyNChQ4m5uTn597//zdYBQLp160a2bdtGUlNTyYcffki4XC5JTk4mhBCSmZlJAJC+ffuSo0ePkuTkZDJkyBASFBRERo4cSc6dO0euX79OevfuTd58802NbN+8eTMRCoUkLi6OpKamkitXrrA2yeVyEhgYSJ588kly7do1cunSJRIUFERGjBjBXr9y5UpiZWVFnn/+eXL79m1y5swZ4urqqvbeffDBB8Te3p7s3LmTpKWlkbNnz5Jt27ax51evXk3Onz9PMjMzyZEjR4iLiwv54osviFgqJ5fv5ZIZr88n/fv3Zz/T6urqVtskk8mIn58fGTt2LElMTCRnz54loaGhBAA5dOiQ2nvp5eVFfvnlF5KRkUFyc3PJgwcPyNq1a8mNGzdIeno6+eqrrwiXyyWXL19u8/OeNvtNkpJb3uzfa3PfqbZo7TvaGlTMGtDcG18llpKei38zylEllmrcrhEjRqiJSHPMmzePvPDCC+zr6Oho4uDgQKqqqtiyLVu2ECsrKyKXy1u9V0VFBeHz+WT//v1sWUlJCTE3N2ftSE1NJQDIlStX2DopKSkEQBMxayxCYWFh5K233iKE1H8Bt2/fzp7fs2cPAUDi4+PZstjYWOLn59eq3Sq6d+9Oli9f3uy5v/76i3C5XJKdnc2WJSUlqbVl5cqVxMLCgohEIrbO+++/T8LCwgghhIhEIiIQCNTEqy3Wrl1LgoKCSEWNhNzMeUTmvbeEBAQEaHz9n3/+SXg8HsnLy2PLjh8/3qyYxcXFtXm/iRMnkvfee48Q0vrn/crsN8nNnEekupm/V0OKWccJK6bonE2bNmHHjh3Izs5GTU0NJBIJ6+9UERAQoBYkHB4ejsrKSuTk5KBnz54t3js9PR0SiQRhYWFsmYODA/z8/NjXKSkp4PF4CAoKYsv69u0LOzu7JvcLDw9v8joxMVGtbNCgQez/VbPdAwcOVCtrKdtKQwoLC5Gbm9tilpeUlBR4eHiorRTx9/eHnZ0dUlJSEBISAkA5K2ttbc3WcXNzY5+fkpICsVjcaiaZffv24auvvkJ6ejoqKyshk8lgY2PD+st4WsaXpaamwsPDQy3gPDQ0tNm6wcHBaq/lcjk+//xz7N+/Hw8fPoREIoFYLGb/Nlr7vM3qZlzLa2QwN+JKBSpmbWBuxkXyJ5FGe3Z72bt3LxYtWoT169cjPDwc1tbWWLt2LS5fvqxDCw1LQye1arKpcZkmOfVUy2x0aU/j57f1jIsXL2LatGlYtWoVIiMjYWtri71792L9+vWsmOkzWLbxFpFr167Ff/7zH8TFxWHgwIGwtLTEu+++q5H/lM+t95u52gr1Yq8maD0BcObMGchksiblMpkMZ86c0YlRHQmGYWDB5xnl0GZ2mM/nq+00f/78eQwdOhRz587F4MGD0bt3b6Snpze57ubNm6ipqc9LdenSJVhZWbW4flVFr169YGZmpiaOjx49wr1799jXffv2hUwmQ0JCAluWmprabEaSS5cuNXndMJuxLrG2toaXlxfi4+ObPd+vXz/k5OQgJyeHLUtOTkZZWRn8/TVLxePr6wtzc/MWn3HhwgX07NkTy5cvR3BwMHx9fXH//n0A9TOZ5gKB2mfaFn5+fsjJyVGbqLh69apG154/fx5TpkzBK6+8goCAAPj4+Kh9lq193nweBwwY1ErlEEs1t1fXaN0zGzVqFPLy8uDs7KxWXl5ejlGjRmn15lN0h5eXFy5fvoysrCxYWVnB19cXP/zwA44dOwZvb2/8+OOPuHr1Kry9vdWuk0gkmD17Nj788ENkZWVh5cqVmD9/Pjic1n/nrKysMHv2bLz//vvo1q0bnJ2dsXz5crXr/Pz8MH78eLzxxhvYsmULeDwe3n333WZ7LT///DOCg4Px5JNPYteuXbhy5Qq+/fZb3bw5zfDxxx/jzTffhLOzMyZMmICKigqcP38eb7/9NiIiIjBw4EBMmzYNcXFxkMlkmDt3LkaMGNFkeNYSQqEQixcvxgcffAA+n49hw4ahqKgISUlJmD17Nnx9fZGdnY29e/ciJCQEv//+Ow4dOgRAGWMGAD7e3sjMzERiYiJ69OgBa2vrVtcZjx07Fr169UJ0dDS+/PJLVFRU4MMPPwTQdtiUr68vDhw4gAsXLsDe3h4bNmxAQUEBK96tfd4choGlgItKsQyiWimcHmNE8Tho3TMjLQTHlZSUNOm6UgzHokWLwOVy4e/vDycnJ0RGRuL5559HVFQUwsLCUFJSgrlz5za5bsyYMfD19cVTTz2FqKgoTJ48GR9//LFGz1y7di2GDx+OSZMmISIiAk8++aSafwwAvvvuO3Tv3h0jRozA888/j9dff73JDyGgDA/Zu3cvBg0ahB9++AF79uzRuBfUHqKjoxEXF4fNmzejf//+eOaZZ/DPP/8AUH7xf/31V9jb2+Opp55CREQEfHx8sG/fPq2e8dFHH+G9997DihUr0K9fP0RFRbE+tcmTJ2PhwoWYP38+AgMDceHCBXz00UcAAGldz+zFF/+F8ePHY9SoUXBycmoSKtEYLpeLw4cPo7KyEiEhIXjttdewfPlyAEpxbY0PP/wQTzzxBCIjIzFy5Ei4uro2WR3S2udtU7ecSVTTdNRmKDROzvj8888DAH799VeMHz9e7RdCLpfj1q1b8PPzw9GjR/VjqY5oLfFbbW0tMjMz4e3t3eaHT9EdDMPg0KFDOl9a1Rmplcpxr6ACHIZB/+42jx2Ifv78eTz55JNIS0tDr169dGRlUyQyBe7miwAA/dxsYFbnR2vPd0rvyRltbW0BKHtm1tbWakMFPp+PIUOGYM6cORo/mEKhNKXhmsz2CNmhQ4dYN0NaWhreeecdDBs2TK9CBigXxFvwuaiWKNMCdbM0/AYsGovZd999B0Dpm1m0aBEdUpo42dnZrQ7zkpOT4enpaUCLtMPKyqrFc3/++SeGDx9uQGs0R+UvEzSTw2zXrl144403mr2uZ8+eSEpKQkVFBRYvXozs7Gw4OjoiIiKi2ZUZ+sBGaKYUsxqZUcSsXXsAdGboMFMzZDIZsrKyWjzv5eUFHq/jRvakpaW1eM7d3V1n4Rm65kFpNUqrJXCxEcLFRv1vsKKiokkiUxVmZmatxgUaAtUQmWEY+LtZg8vhdMxhpoqCggIsWrSITZvdWAvpbKZpwOPx2Gy+nZHOantrqX+sra3VgnQ7GgIeBwIeF2KZHBW1MthZGHbfAq3FbObMmcjOzsZHH3302GmzKRSKOh09KWNrKHOc8VBUIYeoRtrxxezcuXM4e/Zsk2UxFArl8ZAr6vP+d9SkjG1hIzRDUYVYmU7bwB4srd8xDw+PJkNLCoXy+LBrMrkcNrVOZ8OCz4UZlwM5IagUGzbmTOt3LC4uDkuWLGnVOUyhULSno6fK1gSGYWBTl6RRVGPYHGdaDzOjoqJQXV2NXr16wcLCosli29LSUp0ZR6F0JcTSzi9mgHI1QEmVRBmiYcCd2LUWM5pj3vTw8vLCu+++y25EQiPyjYOEjTHr3GJmKeCBy2EgUyhQY8CF51qLWXR0tD7soHQg8vLyYG9vb2wzuhz1w8yOt+mvNnAYBtZCM5RVSwzqN2vXT0B6ejo+/PBDTJ06lV04++effyIpKUmnxlGMg6urq8a7wOsLQkizqaZMFUJIpw7LaIxtnd+ssrYDi9nff/+NgQMH4vLlyzh48CAqKysBKPNirVy5UucGUjSjoqIC06ZNg6WlJdzc3PDvf/9boz0sm6O5DTAOHjyIUaNGwcLCAgEBAbh48aLaNefOnWM3G/Hw8MCCBQtQVVXFnv/xxx8RHBwMa2truLq64uWXX1bLCnv69GkwDIM///wTQUFBEAgEOHfuXLvei86ITE6gIAQMGJMQMyuhGRiGgVSugFTedsJMXaD1u7ZkyRJ8+umnOH78OPj8+qC40aNHN0mwZxIQAkiqjHNoEQITExOD8+fP48iRIzh+/DjOnj2L69ev6+xtWL58ORYtWoTExET06dMHU6dOZXtO6enpGD9+PF544QXcunUL+/btw7lz59R2O5JKpVi9ejVu3ryJw4cPIysrCzNnzmzynCVLlmDNmjVISUlRS5Nt6qjWZPJ5yvxgnR0uh4G1QNk7M5TfTGuf2e3bt5vd0dzZ2RnFxcU6MapDIa0GPu9unGcvywX4bS/or6iowPfff4/du3ezOedVecR0xaJFizBx4kQAytxj/fv3R1paGvr27YvY2FhMmzaN7QX6+vriq6++wogRI7BlyxYIhUK8+uqr7L18fHzw1VdfISQkBJWVlWqLwj/55BOMHTtWZ3Z3FkwhLKMxNuY8lFcCtRLDiJnW75ydnR3y8vKalN+4cQPu7u46MYqiHRkZGZBKpWqbV9ja2qptLvK4NOwlubm5AQA7TLx58yZ27twJKysr9oiMjIRCoUBmZiYAICEhAZMmTYKnpyesra0xYsQIAMrsHA3RNJOrqWFK/jIVNkIzMAAkcoICUa3en6d1z+yll17C4sWL8fPPP7MbOJw/fx6LFi3CjBkz9GGjcTGzUPaQjPXsDkJzm4moNu+orKzEG2+8gQULFjS5ztPTE1VVVYiMjERkZCR27doFJycnZGdnIzIyssmGGV01tZTEBHtmPC4HwroU2ufTitHT2U6/z9P2gs8//xzz5s2Dh4cH5HI5/P39IZfL8fLLL7P5xk0KhtFoqGdMfHx8YGZmhqtXr7I5xsrLy3Hv3j089dRTen/+E088geTk5BYzVdy+fRslJSVYs2YNu1HKtWvX9G5XZ0JsIjFmjbGqC5o9/08xXh6q30wmWosZn8/Htm3b8NFHH+HOnTuorKzE4MGD4evrqw/7KBpgbW2N6OhovP/++3BwcICzszNWrlwJDqd92Uq1ZfHixRgyZAjmz5+P1157DZaWlkhOTsbx48exceNGeHp6gs/n4+uvv8abb76JO3fuYPXq1Xq3q7OgIAQSmXKyp7PHmDXGSqBsz63ccjyqksDeUn+ZNNr9M+Dp6Ymnn34aL774IhWyDsCGDRsQHh6OZ555BhERERg2bBj69etnkCSTgwYNwt9//4179+5h+PDhGDx4MFasWMFOQDg5OWHnzp34+eef4e/vjzVr1mDdunV6t6uzIJEpQEDAYRjwuJ1/JrMhfB4XfC4DhYLgRErziSV1hUaZZmNiYrB69WpYWloiJiam1bobNmzQmXH6oKtkmq2qqoK7uzvWr1+P2bNnG9scSiuIaqTIKqmCuRkXvi4dN/lie6itrcXN5HuI+TMX/h6O2Daj7QkevWaavXHjBqRSKfv/lqCJGo3HjRs3cPfuXYSGhqK8vByffPIJAGDKlClGtozSFqbqL1NhzlcONc/cK0K1RAYLvn7SrWt011OnTjX7f12xadMmrF27Fvn5+QgICMDXX3+tFmbQkG3btuGHH37AnTt3AABBQUH4/PPPW6zflVi3bh1SU1PB5/MRFBSEs2fPIiUlBRMmTGjxGtUKDorxYMMyjLR5rr4x43LgYiPEw4oqnLlXjPEDXPXyHKPvSLFv3z7ExMRg69atCAsLQ1xcHCIjI5GamtrsZrGnT5/G1KlTMXToUAiFQnzxxRcYN24ckpKSunSc2+DBg5GQkNCkvKamBomJiYY3iKIxphgw25ioEA88Hcgg0MNOb8/QyGem2gBYEw4ePKiVAWFhYQgJCcHGjRsBKGOXPDw88Pbbb2PJkiVtXi+Xy2Fvb4+NGzdqFOfWVXxmlM5Dcp4IMrkCvZ2t9DYEMxaG3J1Jo58CW1tb9rCxsUF8fLxanFBCQgLi4+PZjYI1RSKRICEhAREREfUGcTiIiIhospC5JaqrqyGVSuHg4NDsebFYDJFIpHZQKB0FuUIBmdz0e2aGQKOfAdUGwIAypujFF1/E1q1bweUqx/hyuRxz587VSkUBoLi4GHK5HC4uLmrlLi4uuHv3rkb3WLx4Mbp3764miA2JjY3FqlWrtLJLFdlOoegbNu8/p/Pm/W8NQ36XtO7T7tixA+fOnWOFDAC4XC5iYmIwdOhQrF27VqcGtsaaNWuwd+9enD59usUu7NKlS9XCSUQiERuF3hg+nw8Oh4Pc3Fw4OTmBz+fTGVqKXqmskYDIJOCZcVFbq//1i4aCEAKJRIKioiJwOBy1DDv6Qmsxk8lkuHv3bpNFzHfv3tVahR0dHcHlcpvs0lxQUABX19ZnPNatW4c1a9bgxIkTraaKEQgEGica5HA48Pb2Rl5eHnJzjbQek9KlENVIIaqVwVLAhVxk2H0mDYGFhQU8PT3BMUCvU2sxmzVrFmbPno309HQ2HOLy5ctYs2YNZs2apdW9VCEE8fHxbL55hUKB+Ph4tVxYjfnyyy/x2Wef4dixYzrPssDn8+Hp6QmZTEZ3Z6fonU9/S8ap1EK8PtwHUf09jW2OTuFyueDxeAYb3WgtZuvWrYOrqyvWr1/PpgJyc3PD+++/j/fee09rA2JiYhAdHY3g4GCEhoYiLi4OVVVVrDDOmDED7u7uiI2NBQB88cUXWLFiBXbv3g0vLy/k5+cDAJt6RhcwDAMzM7MmO09RKLomMa8KDyvkcOtmQ2fQHxOtxYzD4eCDDz7ABx98wM4Mauv4b0hUVBSKioqwYsUK5OfnIzAwEEePHmUnBbKzs9W6qFu2bIFEIsG//vUvtfusXLkSH3/8cbvtoFAMDSEEmUXK1OI+Trr5Ie7KaBRnZkq0N4aFQtE1BaJahH0eDw4D3F09waQSMz4Oel2b2ZgDBw5g//79yM7ObpJcT5d55ykUUyajrlfm4WBBhUwHaP0OfvXVV5g1axZcXFxw48YNhIaGolu3bsjIyGh1DSCFQlEns7huiOnYsZN/dha0FrPNmzfjm2++wddffw0+n48PPvgAx48fx4IFC1BeXq4PGykUkySjSLnI39uR+st0gdZilp2djaFDhwIAzM3NUVFRAQCYPn069uzZo1vrKBQTRtUz83aiPTNdoLWYubq6orS0FIAy26xqr8zMzEx0sbkECuWxUIlZLzrM1Alai9no0aNx5MgRAMoA2oULF2Ls2LGIiorCc889p3MDKRRTRCpXILu0GgDtmekKrWczv/nmG3bZ0rx589CtWzdcuHABkydPxhtvvKFzAykUUySntBoyBYG5GRcu1jRYVhdoJWYymQyff/45Xn31VfTo0QOAch/Nl156SS/GUSimCusvc7QEh0OTGegCrYaZPB4PX375JWQymb7soVC6BKoYMzrE1B1a+8zGjBmDv//+Wx+2UChdhgwaY6ZztPaZTZgwAUuWLMHt27cRFBQES0v1D2Py5Mk6M45CMVUyi5UxZj60Z6YztBazuXPnAmh+f0yGYWjaHApFA9hhZlcKmJVUA3wLvd1e62GmQqFo8aBCRqG0TaVYhsIKMQDlBIBJQwiQcwX4ZQ6w3g+oKtHbo0xrKxgKpROQVecvc7Tiw9bcRHPmSaqB2z8DV7cD+bfqy1P/AJ6YrpdHaixmNTU1iI+PxzPPPANAmVtfLBaz57lcLlavXk0TzFEobZDOrsk0wV5ZcRpw7VsgcRdQW7dWmysABv4LCJkNuAfp7dEai9n333+P33//nRWzjRs3on///jA3Nweg3AOge/fuWLhwoX4spVBMhIYxZiaBQg7cOwpc2QZknKovt/cCgmcDg18BLJrfClKXaCxmu3btwgcffKBWtnv3bvj4+AAAfvrpJ2zatImKGYXSBmzqn86eXbayCLj+PZCwEyjPqStkAN9xQMhrQO8IwIDb52ksZmlpaRg4cCD7WigUqqWzDg0Nxbx583RrHYVigtTPZHbCnpnKoX91O5B8GJDXJWc1twcGTweCXwUcvI1imsZiVlZWpuYjKyoqUjuvUCjUzlMolKYQQjpnUkZJVQOH/u36cvcgZS+s/3OAmbnx7IMWYtajRw/cuXOnyX6ZKm7dusWu16RQKM1TVClGpVgGDgN4dtNfzJXOKE5TCljibkBc59DnCYEBKof+E8a1rwEai9nTTz+NFStWYOLEiU1mLGtqarBq1SpMnDhR5wZSKKaEaojZw94CAh7XyNa0gFymdOhf3W5Uh762aCxmy5Ytw/79++Hn54f58+ejT58+AIDU1FRs3LgRMpkMy5Yt05uhFIop0KFnMisLges/ANe+A0QP6grrHPqhc4BeYwzq0NcWjcXMxcUFFy5cwFtvvYUlS5awWWUZhsHYsWOxefNmdq9LCoXSPPUzmR1EzAgBci4re2FJhwGFVFlu7qAMbg1+Vdkj6wRotQLA29sbR48eRWlpKdLS0gAAvXv3hoNDx+tyUigdEdUmJkZ3/kuqgFv7gavfAgUNHfrBDRz6nSsAvl3LmRwcHBAaGqprWygUkyej2MgLzIv/UQpYcw790NeA7oONY5cOoGszKRQDIZMrkF2izPtv0GGmXAbc+7POoX+6vtzeWzkjGTitQzr0tYWKGYViIHIe1UCmIBCaceBqY4AhXGWhMkL/2k51h36fSCBkDtBrdId26GsLFTMKxUCoEjJ6ddNj3n+VQ//KNiD510YO/Rl1Dv2e+nm2kaFiRqEYCFWMWS99rMkUV9ZH6BfcqS93D1aGVfg/2+kc+tpCxYxCMRAZ+ogxK7pXl3JnNyAWKct4wrqUO53boa8tVMwoFAORqasF5iqH/pVtQGaDzYUcfJQR+oEvm4RDX1uM7v3btGkTvLy8IBQKERYWhitXrrRYNykpCS+88AK8vLzAMAzi4uIMZyiF8pg8dsBsZSHw91rgP4OAfa/UCRkD9JkAvPILMD8BGDq/SwoZYOSe2b59+xATE4OtW7ciLCwMcXFxiIyMRGpqKpydnZvUr66uho+PD/7v//6P5k2jdCqqxDLki2oBaNkzIwTIvgRc3QYkH6l36Ft0Uzr0g2aZrENfW4wqZhs2bMCcOXMwa9YsAMDWrVvx+++/Y8eOHViyZEmT+iEhIQgJCQGAZs/rmtyyGhRXijGoh53en0UxbVS9MgdLPuws+G1fIK4Ebqsi9Bs49HuEKMMq/KeYvENfW4wmZhKJBAkJCVi6dClbxuFwEBERgYsXL+rsOWKxWC3Pmkgk0ui6k3cL8OrOa+jjYoW/Fo7QmT2UronGOcyK7ilnJG/uaeDQN2/g0A/Ur6GdGKOJWXFxMeRyeZPF6S4uLrh7967OnhMbG4tVq1ZpfZ2qN3avoBJl1RLNfk0pJsW2Mxk4nlyAb2cGw1r4eLsotZpdVi5T7lp0dRuQeaa+3MFHKWCBLyszuVJaxeRnM5cuXYqYmBj2tUgkgoeHR5vXOVoJ4ONoiYziKlzPfoTRfWlGkK7Gd+czkVtei4vpJRjX3/Wx7qUKmPVu6PyvKKiL0P8OqMhVljEcoM94pYj5jDKpCH19YzQxc3R0BJfLRUFBgVp5QUEBXF0f7w+nIQKBAAKBoF3XBnvZI6O4ClezqJh1NaRyBeuwf/Co5rHvxw4zu1kC9y8owypSjgAKmbKChWNdhP4swM7zsZ/XFTGa7PP5fAQFBSE+Pp4tUygUiI+PR3h4uLHMUiO4p3KK+1pW6WPfS5X/jdI5yC+vhaLuI8t5VP1Y9yKEIK+oBNO4JzDq1LPAdxOApINKIesRCjy/DYhJBiJWUiF7DIw6zIyJiUF0dDSCg4MRGhqKuLg4VFVVsbObM2bMgLu7O2JjYwEoJw2Sk5PZ/z98+BCJiYmwsrJC7969dW5fsJfST3Ezpxy1UjmEZu1Lc7ztTAb+eyYd+94I189SForOadgbyyl9jJ5ZUSpqz/8X8dgNa7MaoBRKh/6g/1MOJd0CHt9YCgAji1lUVBSKioqwYsUK5OfnIzAwEEePHmUnBbKzs9W2s8vNzcXgwfXLM9atW4d169ZhxIgROH36tM7t83a0RDdLPkqqJLjzsBzBXu0LRjx9rxDFlRKcTi2iYtZJeFhWL2APtO2ZyWVA6u/KWcnMMzAHAAbIYdzgMW4BEDiVOvT1gNEnAObPn4/58+c3e66xQHl5eRl0uMYwDIK97HEsqQDX7j9qt5jJ68YrqiyjlI7Pw0cNxawGhBAwTBuZLirygYS6TXEbOPRTbZ/E6sJhsPQbjf+G06Sm+sLoYtbRCfFyUIpZVikwole77qESM5UTmNLxadgbqxTLUFYthb1lM+E5hCgd+le3ASn/U3foB0WjauAM/GtTKioUMnw3xMswxndRqJi1gao3du3+IygUpF15qOp7ZlTMOgsNh5mAsnemJmbiCuDWPmWEfmFyfblHmNIX5j8F4Anw8/lMVIhl8HGyxAhfJwNZ3zWhYtYG/bvbQGjGQVm1FOlFlfB1sdb6HvK6kXG+qBZVYhksBfRt7+ioxMyMy0AqJ8h5VI2BPWyBwrt1Efp7AUmFsrKZBTBQ5dAfxN5DoSD47kIWAGDWMG/9JWSkAKBi1iZmXA4CPexwKaMUV7MetUvMFIp6P19mcRUGuNvq0kSKjlEoCHLrxCzQww43sorAu3sESPgfkHW2vmK33koBC5gKmNs1uc/Ju4W4X1INGyEPLzzhbiDruy5UzDQgxMsBlzJKce1+KV4O0z4OSE7FTO9IZAqcSCnAsN6OsDV/vKVHhRViSOUErpwyLOCegq/gAFyTHilPMhzA72mliHmPaDVCf8f5TADA1DBPWPDpV03f0HdYA1i/WdajJufkCoLKWhlsLcyQX16L7NJqhHqrz3oqGszANuc3u5pViu525nC3M9ex5V2HfVez8dGvSXh1mDdWTPJv/40IQfndk9ho9hXGc6+C91AOMEA5xw62w14DgmYCdm0vh0vOFeFCegm4HAYzwr3abw9FY6iYacATnnbgMEB2aTUKRLVwabCzztxdCTiVWoSj7wxH9HdXkFNag6PvDkdfVxu2jnrPTD084/aDcvzfVmWWkKw1E/XcEtPlQnoJAKCsWtK+G4grlH6wq9/CrygFfnXx0eVOQfjo4RDccxiFo2PGany77+p6ZeMHuNIfKQNBV7FqgLXQDH514tSwd0YIwbGkAkhkCjy3+QIbKZ6Sp55mSN6wZ9YoPON6tnpvjxCCk3cLkFf++OsBuxI3sssAAFpHIRbeBX5fBKzvB/yxCChKgZQjxG7ZaKzz2Y6yl/6HI4phyCyTaRzjWFwpxq+JyjizV4d5a2sRpZ3QnpmGhHjZIyVPhKtZpZg4yA0AUFpV3wsor5Gy/3/QaPmL2gRAUZVaAGbDXptcQXAsKR9zd10HQHtqmpJXXsMuCtdIcORS4G5dhH4zDv3YBwHYce0RFrj5ws3WHBwGEMsUKKoQw1mD/S53XcqGRK5AgIcdnvC0a2erKNpCxaw1ZBKAp4wtCvZywA8X7+Pa/fpF51klzceNNV6Y3LBnViGWoahSDGdr5ZeioT8tYNVfqBTLdGZ+V+H6/TLNKorylCl3EnYCFXnKsoYOfZ+RAMMgbYdyH4oedubg85Qb9uaW1yLnUU2bYiaWyfHjpfsAgFeHebW9aoCiM6iYtYSkCvhqMNA7AgiZjRAvpVM5OVeESrEMVgIeMovVRSvU2wFXMkubpIxRKNRvnVlUxYpZw44EFbL2caPBUL1Jv4wQIOucshd297f6CH1LJ+CJaGXKHdseapc8rPsxcrdX+rp6OFggt7wWDx5VI6hn62sqf7uZh+JKMVxsBHh6oNtjtYuiHdRn1hL3jgGVBUDiLmDbaLjtm4A5VhdgRiRIrPPPFNQNbQBggLsN3h3jC6A+4wIhBHIFYYeSjlbKXl5DvxnR3stDacSNnLKmhbUiZc6wzUOA758Bkg8rhcwzHHjhW2BhMjDmoyZCRghhA2Z71ImZh70FACCntPUF54QQNhxjRrgXzLj062VIaM+sJfo/p8wtdXU7cOcgkHsDy3ED8wSWSD81BXBYBLFM2eV6KcQDq6b0Z31ouWU1kMkVmP7tFRRXilFUqdyDoLezFYorS9XWaCo00LIL6cXoZimAn6v2AbumjkSmwO2H5exrl9pM4Pef6iL062aOzSyAQS8qh5KuA1u935GbuaiVKsAwgJttnZg5KP9tLUljoagWB64/QFKuCAIeBy+H0rxkhoaKWUswDNAjWHmM+wy48SMqzn8Du5qHCMrdDXy9G8/aDkEyZxhsBJ4Q8Lhwthayy1/O/FOEixklarfs7WyFSxmlyCiqZNd5yttQs+ySary87TIAYMHo3ogZ58eeu5RRAjsLM7UwkK5Gcp4ICpkET3OuYQbvOIZkpQBZdSe7+dbl0J8KCNsOVD55twDv7E1UXmopAJ+n7Fn1UPXMGvlCc0qrcSwpH3/eycf17Eesy+DFYI/mF6VT9AoVM02w7AY8+S4e+kRjzcZNmGkWjxHMDfiUX8J2/iWU39oF2MwB94kZ6G5njvsl1Xh157Umt/F1VvasTqQU4olPj2POcJ82Z98aTjJ8dTINzz3RA96OlnjwqBovfXNJWaerznqKckFOxeG8YB9cmDIAgBwccPtNrI/Q19ABn19eiw8O3GZfl1TV7+jlUTfcvF9SjbTCChy9k4+jSfm481A9BGewpx0mDHDFzKE0HMMYUDHTgj6udkjgh+B07WAcjfZEwcktGFj4Kxwk+cDJ1cDpNYgVPIkNzHBcI34A1L9IvZ3rEzOWVUux9lgq3hvbp9Vn8rjq95DJlUPb+yWPl8q5I1BeLYWthZZLj1iH/jYg5TcMJsoI/WLYYZdsFIp8p+LTqMhWbyGTK5BXXoucR9V4UFqDB4+q8cedfBRX1guYW4NZS28nS3A5DB48qkHEhvrdkziMctJnwgA3jOvvwg5LKcaBipkWcDgMgnva41RqES6UWCHJ6Q28njMWmwfnYEzF/4AHVzC05hSGCk4hReGBn+RjcVg+DFXKXKNwsxVCaMZBrbR+erMtn1ljJ7KA177U3R2NdcdSsfFUGv7zUiCmBCoXYVdLZOAwTPPpyWtFdSl3tgNF9VsR3uT0w/baMZD0eQbH7pbiGbPm0+woFASLf7mFC+klyBfVNju8d7Ti49NnB+CbMxn48Jn6JVHO1kJ8/twAfHwkGTKFAsN6O2J8f1eM9XdBN6v2bZZD0T1UzLQk2MsBp1KLcO1+KTgMAzH4yO4xCRi2AMi7iaQj/4Z37h/ox8nBZ5wdWMLbg4PyJ/GTfCx4HA68Ha3UVggo2hhmchuljVH5cTr7/igbT6UBAFYeScKUQHdUimUIXPUXvB0tcTymwabLBclKAbu1r4FD3xIY9CJK+k/HlG/ywTDAe56OOHa3tMncsESmwMHrD/D1yTS1HGV8Hgc97M3Rw94CPezN4WFvgUkBbuhhb4HxA5qGVESFeOLZwe6QyQlN4dRBoZ+KlgTXxRldzXqEQA87AA16S24BSAv7DFP3Po0XuGfwCvcEenHyEM07jmjecdQePIR/CcZiDXpDWvfWt+UzM2uUlaE5F5BGKZ07KCqrL6WXQKYg+KewUhmhn/I/pYjdP19f2bFPXcqdlwChLa4l5QPIh5+LNazqBOafggr8kvAAdhZmqKiVIe7EPWQ1GpJfWTYGjlYCrfOLCXhcUB3ruNCPRksCPOxgxmVQVCHGvQJlcj5VbwlQxiaJYInv5BPwnXw8hnKSMJ17HGM5CRA+vIjZuIhJAjvskY/CHtloKEjru0o17pk1p30KAnA7sJYRQnA9uwx+rvWio0IlwhnFlXBBKV7mnQT+vVAZ4wcADBfo+zQQMgcP7IIRf7cIY2v56C6sX4852NOOvc+9gkq89/NNtWc4WglYf1iYt4NGS5IonQ8qZloiNONioLstrmeXsU54QQMxUwVYKmFwQTEAFxQD4IoSHB+RCbPEH+AsLsY7vEOYx/0VGXefwnXOUFxQ9EfjCQNAmem0ORoG28oVpInodSR+TniADw7cQj83G/z5znC1cwwhQOYZhCesw6uCs+AxCqASgKWzMt1O0ExcKBJg57ksnEg5DQUBbuaUYeHYPjh1txAAMNjDXm3JmLudORws+aiRyjG+vyveHNkLv9/KxfazmfjyX4NAMU2omLWD9yP74oujd3HzQRkIAfo0yD7raKWMT5LI1Ncw5aMbqoe9iDz/ufjmv19hOvcEwrnJ6PPoNHbzTyNd4Yaf5BH4Rf4URLBkr2tu+EgIwewGoR9t+d2MzYFrDwCoZxOxQjWe557FTEU88P0DDAQABris6IuQ//sAnH6TAB4fO89n4uP/Jard7+CNh/j1Zi7kCgIBj4PwXt1gJeDhXkEFwrwdENnftcn7FhXiiagQGshqylAxawfhvbrh8LxheFQlQa1MrjYlz+Ew8LA3R3ozSRgZAN7OdvhDMQR/KIbAV/YAK90uIaDkT/Ti5GEl50d8wNuHw/Jh+Enecu6sSrEMEnm9WMo0WUZgRNTEtiAZuLoNlwW7YMnUhUKYWeKAbBi21Y5GKvHEnT6RSMh4hAtpxfjvmQwAwPOD3THY0w4f/ZoEQNkbHe7riAVjfOHhoOwNr5zU36DtonQsqJg9Bi1Feb8b0Qe/3cpFtUSOIT7dsPZYKgBlL6thSud/SA/8YDcP53MnYgr3AqZz/0I/Tg6m8k5hKu8UsP0QrP1egQC2EEP5LALSJKygrVUEuoQQgte+vwYOh8E304NanHiQyRXYfDodQ3y6gVFI8AznIqbzjgNblGEVlgzwj8Idh3jjEbNwBRZ9Uu/oH7DymNq9bM3N8MW/BkFBCP64nQ+BGQeLxvnR9OMUNaiY6YFJAd0xKaA7AKCwopYVs+bcWmU1UlTBHLvlY7BbPhrBTCqm805gAucy+A+uwuXBVVwUWGG/fBR+ko8B0LQnpjCgmBVVihFf56s6fa8IHvbm8Ha0wv5rOQjuac9u+PK/W7nYffwiCO8k/ss/DQe+Koc+F6TvRLx8cyAuKvwBCYPNDYRMhSWfiyqJHIDSB6aKt9vz+hADtJLSGaFipmecrAR4wtMOXA4Dh7qe3Ji+zqwgVNY2TPvD4Brpi2vSvliN6Tj6VBZsk3+EQ2Uu3uT9D69zf8O5f38P34nvggMuFHVJT767kAUbIQ+vDffRf4Ma6Oas764CAL781yAsPahcCvT3ohHoKUpA4IU4nBOcVjr0FUAhscMe+WgseP9T+H5xs82h8YSBbvB3s8EnvyXj3QhfvTWHYjowRNNcwCaCSCSCra0tysvLYWNjmAXaqrdYNSQ7n1aMadsvw9vREl7dLHAqtajFa7mQYxQnEdO5xzGCe4stz1E4YZd8DPbLR6IUynYEetjhl7eGtjizeT37EezMzeDjZNXseU0oFNUi9PP4JuXWdQ796dzj6M3JZcsvK/riB9k4HFMEQ6bFb+fnzw3Ey2GeqJbI6M5GXYz2fkfpX4kBaOxX6utqDYZRbjvHrxs+/TsqAAv33WxyrRxcnFAE4YQiCF6yPLzMPYlXBGfggSIs4ezFQt4B/KYYgp9kY3EjpzfO/lOEkX7OTe6TU1qN5zdfAABkxj6tVZAtIQQyBVEO9Rpd5sdkYzr3OJ7jnmMd+oRvhVsOkXj/fijukbZ3MmqOAA+lP4wKGUVT6F+KEehmJcAgd1vcfFCO1LrAWwGPC69uFk2i1RuSRdzwuWwa1sv+D5O4FzGdexwBnAy8wD2HF7jncEfhBdx7DfB+FeDXh3fM3ZWAP27ns6//tfUifn4jXOMI+Pl7biA+pQDnFo8GAJhBhvGcK3iFdwJhnPp1kv8o3PGDfCwO1T6JSpFFS7drwsyhXhjW2xHFlWIsO3Qbzwzqjn5dOK0RpX3QYaaR2HD8Hr6K/4d9vX1GMPq4WOPnhByM9HNCaZUUHg7mGB93tpW7AIOYdEznHsck7kUImbpNVQS2QODLqA6IxltHK/D3vabD2BMxT6G3s2bJHr2W/A4A+Gy0PZ5V/IXqizvgxCgTIsoIB8cUwfhRPg6XFP3QXOBvQwQ8DgZ72uGrqYPhbC2EVK6gGVkparT3O0rFzEgk3H+EF7ZcYF//8GoonurTNONDtUSGWw/KEXfiHi5llDY5r8IOFfhX3XpQL04BW35O3h8/ysfihCIIctRno2i8t2dz1ErlSCuowOeb/4sZ3OOI4CQoHfoACuoc+ntko1EAh1bvc3jeMHg7Wj72TuOUrkF7v6Md4idx06ZN8PLyglAoRFhYGK5cudJq/Z9//hl9+/aFUCjEwIED8ccffxjIUt0R6GEHuwa5vBouiWqIBZ+HIT7d8NEzre/SXQZrbJdPxCjJesyQLMZxeRDkhMGT3CT8lx+Hc4J3sIB7EE5QhkgUVYhxIb0Y0gbBt4QQrD12F15Lfse8HadwYPOHMP9mCHbzP1fu7s0ocEnRD3MlCzBM/BXiZP9qUcguLR2DrDUTkbVmIgI97KiQUfSO0X1m+/btQ0xMDLZu3YqwsDDExcUhMjISqampcHZu6si+cOECpk6ditjYWDzzzDPYvXs3nn32WVy/fh0DBgwwQgvaB5fDYLivE/53Uznzx29BzFRI5fUd6IYxWPHvjcCY9X+z5wg4OKMIwBlFANxRhKm8k3iJewpuTClizA7gbd4hHFMEY+OOcbhM+iLcxxG754SBYRgcSHiA+NOn8BnvOJ69X+fQ5wCVRIiD8uH4SR7RrEPf3c4cHz3jj+TcckSFetIdvClGwejDzLCwMISEhGDjxo0AAIVCAQ8PD7z99ttYsmRJk/pRUVGoqqrCb7/9xpYNGTIEgYGB2Lp1a5vP6yjDTAD4JeEBm+HhjwXD4d+9ZXuuZpXi/7ZeBABY8LmorhOzrDUTIZMrcCdXBLFUjte+v4aKRlvW8SHFeM5VvMI7jlBOKlt+T+GOH+Vj4TXiFdw8fbjZ8z/Ix+GwfBgqoe7QP/vBKJRUSeDpYMHGz1EouqBThmZIJBIkJCRg6dKlbBmHw0FERAQuXrzY7DUXL15ETEyMWllkZCQOHz7cbH2xWAyxuD4dskgkaraeMWjoI2urZ9Zw4Xrjnx8el8PmVru9KrJBPYKNJ9Ow/vg9HFEMxRHJUPRj7uMV7gk8yz2HPpyHWM3ZCVzYibrVUpASrtKhL1P23FQO/ebEVrUmkkLpCBhVzIqLiyGXy+Hi4qJW7uLigrt37zZ7TX5+frP18/Pzm60fGxuLVatW6cZgHeNkLcCrw7yRVlQJr26tC0M/t3oh2TYjGG/9lIBPn2t9WM0wDN4e44u3x/jiQnoxXt52GSmkJ5bLZmONbCqe457FdO4J+HIeooDYYbdsDPbIR6MQ9lj/fwHY+4R7p036SOl6GN1npm+WLl2q1pMTiUTw8GhfIKc+WDGpdce+CgdLPq4sGwNzPhfWQjPcXDlOq0ypQ3s54vSikXCzE0LA40Isk+PU3eHguXyOQ0l3MDY0AAstzLGwvQ2hUIyMUcXM0dERXC4XBQUFauUFBQVwdXVt9hpXV1et6gsEAggEprHpRMMMqdqmfAYAL8f6QFoBj4vxA5TvmfdIunib0vkxamgGn89HUFAQ4uPr1/opFArEx8cjPDy82WvCw8PV6gPA8ePHW6xPoVC6BkYfZsbExCA6OhrBwcEIDQ1FXFwcqqqqMGvWLADAjBkz4O7ujtjYWADAO++8gxEjRmD9+vWYOHEi9u7di2vXruGbb74xZjMoFIqRMbqYRUVFoaioCCtWrEB+fj4CAwNx9OhR1smfnZ0NToMdioYOHYrdu3fjww8/xLJly+Dr64vDhw93qhgzCoWie4weZ2ZoOlKcGYVCaUqnXs5EoVAojwsVMwqFYhJQMaNQKCaB0ScADI3KRdiRljVRKJR6VN9Nbd35XU7MKiqUmV070ioACoXSlIqKCtjaar6dYJebzVQoFMjNzYW1tXW71h2qlkPl5OSYxGyoqbUHML02mVp7gNbbRAhBRUUFunfvrhaW1RZdrmfG4XDQo0ePx76PjY2NyfxhAabXHsD02mRq7QFabpM2PTIVdAKAQqGYBFTMKBSKSUDFTEsEAgFWrlxpMpk4TK09gOm1ydTaA+inTV1uAoBCoZgmtGdGoVBMAipmFArFJKBiRqFQTAIqZhQKxSSgYqYBpaWlmDZtGmxsbGBnZ4fZs2ejsrKy1fpvv/02/Pz8YG5uDk9PTyxYsADl5eUGtLoeU9wxXps2bdu2DcOHD4e9vT3s7e0RERHR5ntgaLT9jFTs3bsXDMPg2Wef1a+B7UDbNpWVlWHevHlwc3ODQCBAnz59tPvbI5Q2GT9+PAkICCCXLl0iZ8+eJb179yZTp05tsf7t27fJ888/T44cOULS0tJIfHw88fX1JS+88IIBrVayd+9ewufzyY4dO0hSUhKZM2cOsbOzIwUFBc3WP3/+POFyueTLL78kycnJ5MMPPyRmZmbk9u3bBra8ZbRt08svv0w2bdpEbty4QVJSUsjMmTOJra0tefDggYEtbx5t26MiMzOTuLu7k+HDh5MpU6YYxlgN0bZNYrGYBAcHk6effpqcO3eOZGZmktOnT5PExESNn0nFrA2Sk5MJAHL16lW27M8//yQMw5CHDx9qfJ/9+/cTPp9PpFKpPsxskdDQUDJv3jz2tVwuJ927dyexsbHN1n/xxRfJxIkT1crCwsLIG2+8oVc7tUHbNjVGJpMRa2tr8v333+vLRK1oT3tkMhkZOnQo2b59O4mOju5wYqZtm7Zs2UJ8fHyIRCJp9zPpMLMNLl68CDs7OwQHB7NlERER4HA4uHz5ssb3UaUA5vEMtxxWtWN8REQEW6bJjvEN6wPKHeNbqm9o2tOmxlRXV0MqlcLBwUFfZmpMe9vzySefwNnZGbNnzzaEmVrRnjYdOXIE4eHhmDdvHlxcXDBgwAB8/vnnkMvlGj+3yy0015b8/Hw4OzurlfF4PDg4OLS4i3pjiouLsXr1arz++uv6MLHV5+p7x3hD0542NWbx4sXo3r17E9E2Bu1pz7lz5/Dtt98iMTHRABZqT3valJGRgZMnT2LatGn4448/kJaWhrlz50IqlWLlypUaPbfL9syWLFkChmFaPTT9crSGSCTCxIkT4e/vj48//vjxDac8FmvWrMHevXtx6NAhCIXCti/oYFRUVGD69OnYtm0bHB0djW2OzlAoFHB2dsY333yDoKAgREVFYfny5di6davG9+iyPbP33nsPM2fObLWOj48PXF1dUVhYqFYuk8lQWlra4i7qKioqKjB+/HhYW1vj0KFDMDMze1yztcIQO8Ybmva0ScW6deuwZs0anDhxAoMGDdKnmRqjbXvS09ORlZWFSZMmsWUKhQKAcsSQmpqKXr166dfoNmjPZ+Tm5gYzMzNwuVy2rF+/fsjPz4dEIgGfz2/zuV22Z+bk5IS+ffu2evD5fISHh6OsrAwJCQnstSdPnoRCoUBYWFiL9xeJRBg3bhz4fD6OHDlilF6AKe4Y3542AcCXX36J1atX4+jRo2r+T2OjbXv69u2L27dvIzExkT0mT56MUaNGITExsUNkUG7PZzRs2DCkpaWxwgwA9+7dg5ubm0ZCBoCGZmjC+PHjyeDBg8nly5fJuXPniK+vr1poxoMHD4ifnx+5fPkyIYSQ8vJyEhYWRgYOHEjS0tJIXl4ee8hkMoPavnfvXiIQCMjOnTtJcnIyef3114mdnR3Jz88nhBAyffp0smTJErb++fPnCY/HI+vWrSMpKSlk5cqVHTI0Q5s2rVmzhvD5fHLgwAG1z6KiosJYTVBD2/Y0piPOZmrbpuzsbGJtbU3mz59PUlNTyW+//UacnZ3Jp59+qvEzqZhpQElJCZk6dSqxsrIiNjY2ZNasWWpfhMzMTAKAnDp1ihBCyKlTpwiAZo/MzEyD2//1118TT09PwufzSWhoKLl06RJ7bsSIESQ6Olqt/v79+0mfPn0In88n/fv3J7///ruBLW4bbdrUs2fPZj+LlStXGt7wFtD2M2pIRxQzQrRv04ULF0hYWBgRCATEx8eHfPbZZ1r9+NMUQBQKxSTosj4zCoViWlAxo1AoJgEVMwqFYhJQMaNQKCYBFTMKhWISUDGjUCgmARUzCoViElAxo1AoJgEVM0qHYObMmc1mLhk/fryxTaN0Erps1gxKx2P8+PH47rvv1Mpa2vFaKpU2yUKiaXaFxrT3OkrHgvbMKB0GgUAAV1dXtcPe3h4AwDAMtmzZgsmTJ8PS0hKfffYZPv74YwQGBmL79u3w9vZmM5NkZ2djypQpsLKygo2NDV588UW1dDQtXUfp3FAxo3QaPv74Yzz33HO4ffs2Xn31VQBAWloafvnlFxw8eBCJiYlQKBSYMmUKSktL8ffff+P48ePIyMhAVFSU2r0aX0fp/NBhJqXD8Ntvv8HKykqtbNmyZVi2bBkA4OWXX8asWbPUzkskEvzwww9wcnICoMy9dvv2bWRmZrK5vX744Qf0798fV69eRUhISLPXUTo/VMwoHYZRo0Zhy5YtamUNNx1pLqliz5491QQpJSUFHh4eakkK/f39YWdnh5SUFFbMGl9H6fxQMaN0GCwtLdG7d+9Wz2tSpumzKKYF9ZlRTIp+/fohJycHOTk5bFlycjLKysrg7+9vRMso+ob2zCgdBrFY3GRLOx6Pp9UuRBERERg4cCCmTZuGuLg4yGQyzJ07FyNGjOhQuf8puof2zCgdhqNHj8LNzU3tePLJJ7W6B8Mw+PXXX2Fvb4+nnnoKERER8PHxwb59+/RkNaWjQNNmUygUk4D2zCgUiklAxYxCoZgEVMwoFIpJQMWMQqGYBFTMKBSKSUDFjEKhmARUzCgUiklAxYxCoZgEVMwoFIpJQMWMQqGYBFTMKBSKSUDFjEKhmAT/D5MODL9GqCrHAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018353, + "end_time": "2024-03-23T10:39:03.980131", + "exception": false, + "start_time": "2024-03-23T10:39:03.961778", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4714.87906, + "end_time": "2024-03-23T10:39:06.721011", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tab_ddpm_concat/2/mlu-eval.ipynb", + "output_path": "eval/treatment/tab_ddpm_concat/2/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tab_ddpm_concat/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-23T09:20:31.841951", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/model.pt b/treatment/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..72ab8dbb76f735ec9e881224b9e44b34e5be23f0 --- /dev/null +++ b/treatment/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47ec99d611d421725d9edf587e668e18b56165b247b4961d5a067ae47fb552eb +size 74520513 diff --git a/treatment/tab_ddpm_concat/params.json b/treatment/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..58c4d564a346f44dd192fc21139f794e7508e0ac --- /dev/null +++ b/treatment/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "tab_ddpm_concat", "mse_mag": true, "mse_mag_target": 0.2, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tvae/eval.csv b/treatment/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..2b3b13da6304fb1b5f597098d112a1e4fa119577 --- /dev/null +++ b/treatment/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.0,0.013790099427079105,0.005082991473580131,11.677511215209961,0.07804063707590103,1.4367523193359375,0.129799485206604,6.618953921133652e-05,6.3949174880981445,0.04794245585799217,3503210.25,0.0712951049208641,0.24372297525405884,1.4872253814246505e-05,18.072428703308105 diff --git a/treatment/tvae/history.csv b/treatment/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..4b7a2e2838f8db4c4d1a8673b1b2ed0accaeb1d6 --- /dev/null +++ b/treatment/tvae/history.csv @@ -0,0 +1,8 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.09367916755792167,22.42829442501068,0.017235794447822684,0.000614058285072032,0.0,0.0,0.0,0.0,0.6507627824445565,900,225,402.8643465042114,1.7905082066853841,0.44762705167134603,0.024596532245632262,0.036389211933645936,5.391759709776824,0.0017475698325281996,0.0,0.0,0.0,0.0,0.0,0.036389211933645936,450,113,104.99674201011658,0.929174708054129,0.2333260933558146,0.04841550381079448 +1,0.014980487561493241,0.3094676796827943,0.0015274671318595311,0.12126932171639054,0.0,0.0,0.0,0.0,0.01596135459627476,900,225,404.5609152317047,1.7980485121409098,0.44951212803522744,0.22056979837516943,0.00846542533677687,2.915443131132165,0.0002598197966862772,0.0,0.0,0.0,0.0,0.0,0.00846542533677687,450,113,105.198903799057,0.9309637504341328,0.23377534177568224,0.11661221853713531 +2,0.008436545321653992,0.35172939089605787,0.0012636855114381994,0.11215860046653284,0.0,0.0,0.0,0.0,0.00857960265895397,900,225,404.56075525283813,1.798047801123725,0.44951195028093127,0.23612414244251947,0.00871398364906055,2.157759373814455,0.00026264469392604274,0.0,0.0,0.0,0.0,0.0,0.00871398364906055,450,113,104.34595584869385,0.9234155384840164,0.23187990188598634,0.11704388770882779 +3,0.0067718393057612045,0.07337658431866786,0.0009447876299363208,0.11110870275827539,0.0,0.0,0.0,0.0,0.006862115198129383,900,225,403.8835325241089,1.7950379223293729,0.4487594805823432,0.23448577124677183,0.008347708936015611,1.7172627388828352,0.0002486144137671114,0.0,0.0,0.0,0.0,0.0,0.008347708936015611,450,113,104.71852970123291,0.9267126522233001,0.23270784378051756,0.11048408394689214 +4,0.005389175902948611,0.16187325817722736,0.00014434781082973955,0.09723918036661214,0.0,0.0,0.0,0.0,0.005458275656003227,900,225,404.1273407936096,1.796121514638265,0.44903037865956624,0.23944718190365366,0.007817784738700482,1.7797075561753128,0.0002088686810555831,0.0,0.0,0.0,0.0,0.0,0.007817784738700482,450,113,104.49570226669312,0.9247407280238329,0.23221267170376247,0.11337851887234073 +5,0.004446125861679522,0.05840130616491845,9.408990984527262e-05,0.09586656246696496,0.0,0.0,0.0,0.0,0.004507343026266931,900,225,403.9254059791565,1.795224026574029,0.4488060066435072,0.24089082530803152,0.00899352906204879,1.7173922716789354,0.00027806640467408996,0.0,0.0,0.0,0.0,0.0,0.00899352906204879,450,113,103.82342529296875,0.9187913742740598,0.2307187228732639,0.12656746556639462 +6,0.003227566026788635,0.04552710757783138,2.7508186136415947e-05,0.10205126650217507,0.0,0.0,0.0,0.0,0.003272674533940921,900,225,403.4123239517212,1.7929436620076498,0.44823591550191244,0.2458179177592198,0.010771220862358304,2.951861702030632,0.00042820363848689307,0.0,0.0,0.0,0.0,0.0,0.010771220862358304,450,113,104.2169623374939,0.9222740029866716,0.23159324963887531,0.12846026598753776 diff --git a/treatment/tvae/mlu-eval.ipynb b/treatment/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3d34c6cb7bb0d45d667761868ac0c932a60d01eb --- /dev/null +++ b/treatment/tvae/mlu-eval.ipynb @@ -0,0 +1,2380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:16.905915Z", + "iopub.status.busy": "2024-03-22T19:11:16.905542Z", + "iopub.status.idle": "2024-03-22T19:11:16.943219Z", + "shell.execute_reply": "2024-03-22T19:11:16.942251Z" + }, + "papermill": { + "duration": 0.056174, + "end_time": "2024-03-22T19:11:16.945575", + "exception": false, + "start_time": "2024-03-22T19:11:16.889401", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:16.974878Z", + "iopub.status.busy": "2024-03-22T19:11:16.974495Z", + "iopub.status.idle": "2024-03-22T19:11:16.982327Z", + "shell.execute_reply": "2024-03-22T19:11:16.981363Z" + }, + "papermill": { + "duration": 0.025471, + "end_time": "2024-03-22T19:11:16.984646", + "exception": false, + "start_time": "2024-03-22T19:11:16.959175", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:17.013496Z", + "iopub.status.busy": "2024-03-22T19:11:17.013133Z", + "iopub.status.idle": "2024-03-22T19:11:17.017986Z", + "shell.execute_reply": "2024-03-22T19:11:17.016999Z" + }, + "papermill": { + "duration": 0.021833, + "end_time": "2024-03-22T19:11:17.020302", + "exception": false, + "start_time": "2024-03-22T19:11:16.998469", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:17.048828Z", + "iopub.status.busy": "2024-03-22T19:11:17.047900Z", + "iopub.status.idle": "2024-03-22T19:11:17.052708Z", + "shell.execute_reply": "2024-03-22T19:11:17.051773Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.021226, + "end_time": "2024-03-22T19:11:17.054892", + "exception": false, + "start_time": "2024-03-22T19:11:17.033666", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:17.084676Z", + "iopub.status.busy": "2024-03-22T19:11:17.084303Z", + "iopub.status.idle": "2024-03-22T19:11:17.091649Z", + "shell.execute_reply": "2024-03-22T19:11:17.090522Z" + }, + "papermill": { + "duration": 0.025032, + "end_time": "2024-03-22T19:11:17.093923", + "exception": false, + "start_time": "2024-03-22T19:11:17.068891", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0975b811", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:17.120337Z", + "iopub.status.busy": "2024-03-22T19:11:17.119604Z", + "iopub.status.idle": "2024-03-22T19:11:17.125853Z", + "shell.execute_reply": "2024-03-22T19:11:17.124975Z" + }, + "papermill": { + "duration": 0.021671, + "end_time": "2024-03-22T19:11:17.127997", + "exception": false, + "start_time": "2024-03-22T19:11:17.106326", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tvae/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011409, + "end_time": "2024-03-22T19:11:17.151375", + "exception": false, + "start_time": "2024-03-22T19:11:17.139966", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:17.175210Z", + "iopub.status.busy": "2024-03-22T19:11:17.174900Z", + "iopub.status.idle": "2024-03-22T19:11:17.184210Z", + "shell.execute_reply": "2024-03-22T19:11:17.183335Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023566, + "end_time": "2024-03-22T19:11:17.186239", + "exception": false, + "start_time": "2024-03-22T19:11:17.162673", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tvae/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:17.211923Z", + "iopub.status.busy": "2024-03-22T19:11:17.211645Z", + "iopub.status.idle": "2024-03-22T19:11:19.348163Z", + "shell.execute_reply": "2024-03-22T19:11:19.347183Z" + }, + "papermill": { + "duration": 2.151444, + "end_time": "2024-03-22T19:11:19.350433", + "exception": false, + "start_time": "2024-03-22T19:11:17.198989", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:19.375740Z", + "iopub.status.busy": "2024-03-22T19:11:19.374782Z", + "iopub.status.idle": "2024-03-22T19:11:19.390301Z", + "shell.execute_reply": "2024-03-22T19:11:19.389535Z" + }, + "papermill": { + "duration": 0.029907, + "end_time": "2024-03-22T19:11:19.392191", + "exception": false, + "start_time": "2024-03-22T19:11:19.362284", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:19.415872Z", + "iopub.status.busy": "2024-03-22T19:11:19.415581Z", + "iopub.status.idle": "2024-03-22T19:11:19.423336Z", + "shell.execute_reply": "2024-03-22T19:11:19.422624Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021841, + "end_time": "2024-03-22T19:11:19.425242", + "exception": false, + "start_time": "2024-03-22T19:11:19.403401", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:19.451213Z", + "iopub.status.busy": "2024-03-22T19:11:19.450477Z", + "iopub.status.idle": "2024-03-22T19:11:19.546271Z", + "shell.execute_reply": "2024-03-22T19:11:19.545454Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.111156, + "end_time": "2024-03-22T19:11:19.548491", + "exception": false, + "start_time": "2024-03-22T19:11:19.437335", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:19.575722Z", + "iopub.status.busy": "2024-03-22T19:11:19.575427Z", + "iopub.status.idle": "2024-03-22T19:11:24.307455Z", + "shell.execute_reply": "2024-03-22T19:11:24.306618Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.747493, + "end_time": "2024-03-22T19:11:24.310220", + "exception": false, + "start_time": "2024-03-22T19:11:19.562727", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 19:11:21.854880: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 19:11:21.854960: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 19:11:21.856789: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:24.337212Z", + "iopub.status.busy": "2024-03-22T19:11:24.336069Z", + "iopub.status.idle": "2024-03-22T19:11:24.342983Z", + "shell.execute_reply": "2024-03-22T19:11:24.342248Z" + }, + "papermill": { + "duration": 0.022101, + "end_time": "2024-03-22T19:11:24.344842", + "exception": false, + "start_time": "2024-03-22T19:11:24.322741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:24.368995Z", + "iopub.status.busy": "2024-03-22T19:11:24.368701Z", + "iopub.status.idle": "2024-03-22T19:11:47.073211Z", + "shell.execute_reply": "2024-03-22T19:11:47.072047Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.719957, + "end_time": "2024-03-22T19:11:47.076299", + "exception": false, + "start_time": "2024-03-22T19:11:24.356342", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:11:47.121403Z", + "iopub.status.busy": "2024-03-22T19:11:47.120277Z", + "iopub.status.idle": "2024-03-22T19:11:47.127583Z", + "shell.execute_reply": "2024-03-22T19:11:47.126693Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.033251, + "end_time": "2024-03-22T19:11:47.130100", + "exception": false, + "start_time": "2024-03-22T19:11:47.096849", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:47.162389Z", + "iopub.status.busy": "2024-03-22T19:11:47.161717Z", + "iopub.status.idle": "2024-03-22T19:11:47.166738Z", + "shell.execute_reply": "2024-03-22T19:11:47.165828Z" + }, + "papermill": { + "duration": 0.021659, + "end_time": "2024-03-22T19:11:47.168750", + "exception": false, + "start_time": "2024-03-22T19:11:47.147091", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:11:47.195798Z", + "iopub.status.busy": "2024-03-22T19:11:47.195496Z", + "iopub.status.idle": "2024-03-22T19:30:19.744397Z", + "shell.execute_reply": "2024-03-22T19:30:19.743454Z" + }, + "papermill": { + "duration": 1112.578585, + "end_time": "2024-03-22T19:30:19.759717", + "exception": false, + "start_time": "2024-03-22T19:11:47.181132", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:30:19.788850Z", + "iopub.status.busy": "2024-03-22T19:30:19.788533Z", + "iopub.status.idle": "2024-03-22T19:30:20.113978Z", + "shell.execute_reply": "2024-03-22T19:30:20.113001Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.343325, + "end_time": "2024-03-22T19:30:20.116605", + "exception": false, + "start_time": "2024-03-22T19:30:19.773280", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:30:20.151145Z", + "iopub.status.busy": "2024-03-22T19:30:20.150808Z", + "iopub.status.idle": "2024-03-22T19:54:03.730648Z", + "shell.execute_reply": "2024-03-22T19:54:03.729585Z" + }, + "papermill": { + "duration": 1423.613592, + "end_time": "2024-03-22T19:54:03.746480", + "exception": false, + "start_time": "2024-03-22T19:30:20.132888", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:54:03.777333Z", + "iopub.status.busy": "2024-03-22T19:54:03.776930Z", + "iopub.status.idle": "2024-03-22T19:54:04.288234Z", + "shell.execute_reply": "2024-03-22T19:54:04.287309Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.529147, + "end_time": "2024-03-22T19:54:04.290337", + "exception": false, + "start_time": "2024-03-22T19:54:03.761190", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:54:04.324252Z", + "iopub.status.busy": "2024-03-22T19:54:04.323326Z", + "iopub.status.idle": "2024-03-22T19:54:04.328280Z", + "shell.execute_reply": "2024-03-22T19:54:04.327335Z" + }, + "papermill": { + "duration": 0.024551, + "end_time": "2024-03-22T19:54:04.330477", + "exception": false, + "start_time": "2024-03-22T19:54:04.305926", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:54:04.361033Z", + "iopub.status.busy": "2024-03-22T19:54:04.360744Z", + "iopub.status.idle": "2024-03-22T19:54:04.367832Z", + "shell.execute_reply": "2024-03-22T19:54:04.366994Z" + }, + "papermill": { + "duration": 0.024706, + "end_time": "2024-03-22T19:54:04.369884", + "exception": false, + "start_time": "2024-03-22T19:54:04.345178", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18701313" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:54:04.398497Z", + "iopub.status.busy": "2024-03-22T19:54:04.398219Z", + "iopub.status.idle": "2024-03-22T19:54:04.495979Z", + "shell.execute_reply": "2024-03-22T19:54:04.495196Z" + }, + "papermill": { + "duration": 0.114539, + "end_time": "2024-03-22T19:54:04.498143", + "exception": false, + "start_time": "2024-03-22T19:54:04.383604", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 95] --\n", + "├─Adapter: 1-1 [2, 2648, 95] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 98,304\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 95] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,701,313\n", + "Trainable params: 18,701,313\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 74.05\n", + "========================================================================================================================\n", + "Input size (MB): 2.51\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.81\n", + "Estimated Total Size (MB): 1156.80\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:54:04.532186Z", + "iopub.status.busy": "2024-03-22T19:54:04.531904Z", + "iopub.status.idle": "2024-03-22T21:05:06.564871Z", + "shell.execute_reply": "2024-03-22T21:05:06.563849Z" + }, + "papermill": { + "duration": 4262.053695, + "end_time": "2024-03-22T21:05:06.567346", + "exception": false, + "start_time": "2024-03-22T19:54:04.513651", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.09367916755792167, 'avg_role_model_std_loss': 22.42829442501068, 'avg_role_model_mean_pred_loss': 0.017235794447822684, 'avg_role_model_g_mag_loss': 0.000614058285072032, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.6507627824445565, 'n_size': 900, 'n_batch': 225, 'duration': 402.8643465042114, 'duration_batch': 1.7905082066853841, 'duration_size': 0.44762705167134603, 'avg_pred_std': 0.024596532245632262}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.036389211933645936, 'avg_role_model_std_loss': 5.391759709776824, 'avg_role_model_mean_pred_loss': 0.0017475698325281996, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.036389211933645936, 'n_size': 450, 'n_batch': 113, 'duration': 104.99674201011658, 'duration_batch': 0.929174708054129, 'duration_size': 0.2333260933558146, 'avg_pred_std': 0.04841550381079448}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014980487561493241, 'avg_role_model_std_loss': 0.3094676796827943, 'avg_role_model_mean_pred_loss': 0.0015274671318595311, 'avg_role_model_g_mag_loss': 0.12126932171639054, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01596135459627476, 'n_size': 900, 'n_batch': 225, 'duration': 404.5609152317047, 'duration_batch': 1.7980485121409098, 'duration_size': 0.44951212803522744, 'avg_pred_std': 0.22056979837516943}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00846542533677687, 'avg_role_model_std_loss': 2.915443131132165, 'avg_role_model_mean_pred_loss': 0.0002598197966862772, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00846542533677687, 'n_size': 450, 'n_batch': 113, 'duration': 105.198903799057, 'duration_batch': 0.9309637504341328, 'duration_size': 0.23377534177568224, 'avg_pred_std': 0.11661221853713531}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008436545321653992, 'avg_role_model_std_loss': 0.35172939089605787, 'avg_role_model_mean_pred_loss': 0.0012636855114381994, 'avg_role_model_g_mag_loss': 0.11215860046653284, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00857960265895397, 'n_size': 900, 'n_batch': 225, 'duration': 404.56075525283813, 'duration_batch': 1.798047801123725, 'duration_size': 0.44951195028093127, 'avg_pred_std': 0.23612414244251947}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00871398364906055, 'avg_role_model_std_loss': 2.157759373814455, 'avg_role_model_mean_pred_loss': 0.00026264469392604274, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00871398364906055, 'n_size': 450, 'n_batch': 113, 'duration': 104.34595584869385, 'duration_batch': 0.9234155384840164, 'duration_size': 0.23187990188598634, 'avg_pred_std': 0.11704388770882779}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0067718393057612045, 'avg_role_model_std_loss': 0.07337658431866786, 'avg_role_model_mean_pred_loss': 0.0009447876299363208, 'avg_role_model_g_mag_loss': 0.11110870275827539, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006862115198129383, 'n_size': 900, 'n_batch': 225, 'duration': 403.8835325241089, 'duration_batch': 1.7950379223293729, 'duration_size': 0.4487594805823432, 'avg_pred_std': 0.23448577124677183}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008347708936015611, 'avg_role_model_std_loss': 1.7172627388828352, 'avg_role_model_mean_pred_loss': 0.0002486144137671114, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008347708936015611, 'n_size': 450, 'n_batch': 113, 'duration': 104.71852970123291, 'duration_batch': 0.9267126522233001, 'duration_size': 0.23270784378051756, 'avg_pred_std': 0.11048408394689214}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005389175902948611, 'avg_role_model_std_loss': 0.16187325817722736, 'avg_role_model_mean_pred_loss': 0.00014434781082973955, 'avg_role_model_g_mag_loss': 0.09723918036661214, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005458275656003227, 'n_size': 900, 'n_batch': 225, 'duration': 404.1273407936096, 'duration_batch': 1.796121514638265, 'duration_size': 0.44903037865956624, 'avg_pred_std': 0.23944718190365366}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007817784738700482, 'avg_role_model_std_loss': 1.7797075561753128, 'avg_role_model_mean_pred_loss': 0.0002088686810555831, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007817784738700482, 'n_size': 450, 'n_batch': 113, 'duration': 104.49570226669312, 'duration_batch': 0.9247407280238329, 'duration_size': 0.23221267170376247, 'avg_pred_std': 0.11337851887234073}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004446125861679522, 'avg_role_model_std_loss': 0.05840130616491845, 'avg_role_model_mean_pred_loss': 9.408990984527262e-05, 'avg_role_model_g_mag_loss': 0.09586656246696496, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004507343026266931, 'n_size': 900, 'n_batch': 225, 'duration': 403.9254059791565, 'duration_batch': 1.795224026574029, 'duration_size': 0.4488060066435072, 'avg_pred_std': 0.24089082530803152}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00899352906204879, 'avg_role_model_std_loss': 1.7173922716789354, 'avg_role_model_mean_pred_loss': 0.00027806640467408996, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00899352906204879, 'n_size': 450, 'n_batch': 113, 'duration': 103.82342529296875, 'duration_batch': 0.9187913742740598, 'duration_size': 0.2307187228732639, 'avg_pred_std': 0.12656746556639462}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003227566026788635, 'avg_role_model_std_loss': 0.04552710757783138, 'avg_role_model_mean_pred_loss': 2.7508186136415947e-05, 'avg_role_model_g_mag_loss': 0.10205126650217507, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003272674533940921, 'n_size': 900, 'n_batch': 225, 'duration': 403.4123239517212, 'duration_batch': 1.7929436620076498, 'duration_size': 0.44823591550191244, 'avg_pred_std': 0.2458179177592198}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010771220862358304, 'avg_role_model_std_loss': 2.951861702030632, 'avg_role_model_mean_pred_loss': 0.00042820363848689307, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010771220862358304, 'n_size': 450, 'n_batch': 113, 'duration': 104.2169623374939, 'duration_batch': 0.9222740029866716, 'duration_size': 0.23159324963887531, 'avg_pred_std': 0.12846026598753776}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0027504112272628035, 'avg_role_model_std_loss': 0.05725725870194452, 'avg_role_model_mean_pred_loss': 0.00010518741055586947, 'avg_role_model_g_mag_loss': 0.08298002037892326, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002791692248569638, 'n_size': 900, 'n_batch': 225, 'duration': 403.34395694732666, 'duration_batch': 1.7926398086547852, 'duration_size': 0.4481599521636963, 'avg_pred_std': 0.24101652820077207}\n", + "Time out: 3970.833259820938/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.005082991521107293, 'avg_g_mag_loss': 0.006355337803827987, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.387144327163696, 'grad_duration': 11.690799713134766, 'total_duration': 18.077944040298462, 'pred_std': 0.24372297525405884, 'std_loss': 1.4872253814246505e-05, 'mean_pred_loss': 6.618953921133652e-05, 'pred_rmse': 0.07129509747028351, 'pred_mae': 0.04794245585799217, 'pred_mape': 3503210.5, 'grad_rmse': 0.1297994703054428, 'grad_mae': 0.07804063707590103, 'grad_mape': 1.436752438545227}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.005082991521107293, 'avg_g_mag_loss': 0.006355337803827987, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 6.387144327163696, 'avg_grad_duration': 11.690799713134766, 'avg_total_duration': 18.077944040298462, 'avg_pred_std': 0.24372297525405884, 'avg_std_loss': 1.4872253814246505e-05, 'avg_mean_pred_loss': 6.618953921133652e-05}, 'min_metrics': {'avg_loss': 0.005082991521107293, 'avg_g_mag_loss': 0.006355337803827987, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.387144327163696, 'grad_duration': 11.690799713134766, 'total_duration': 18.077944040298462, 'pred_std': 0.24372297525405884, 'std_loss': 1.4872253814246505e-05, 'mean_pred_loss': 6.618953921133652e-05, 'pred_rmse': 0.07129509747028351, 'pred_mae': 0.04794245585799217, 'pred_mape': 3503210.5, 'grad_rmse': 0.1297994703054428, 'grad_mae': 0.07804063707590103, 'grad_mape': 1.436752438545227}, 'model_metrics': {'tvae': {'avg_loss': 0.005082991521107293, 'avg_g_mag_loss': 0.006355337803827987, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.387144327163696, 'grad_duration': 11.690799713134766, 'total_duration': 18.077944040298462, 'pred_std': 0.24372297525405884, 'std_loss': 1.4872253814246505e-05, 'mean_pred_loss': 6.618953921133652e-05, 'pred_rmse': 0.07129509747028351, 'pred_mae': 0.04794245585799217, 'pred_mape': 3503210.5, 'grad_rmse': 0.1297994703054428, 'grad_mae': 0.07804063707590103, 'grad_mape': 1.436752438545227}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:05:06.603446Z", + "iopub.status.busy": "2024-03-22T21:05:06.603128Z", + "iopub.status.idle": "2024-03-22T21:05:06.607378Z", + "shell.execute_reply": "2024-03-22T21:05:06.606505Z" + }, + "papermill": { + "duration": 0.024613, + "end_time": "2024-03-22T21:05:06.609254", + "exception": false, + "start_time": "2024-03-22T21:05:06.584641", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:05:06.642072Z", + "iopub.status.busy": "2024-03-22T21:05:06.641781Z", + "iopub.status.idle": "2024-03-22T21:05:06.772599Z", + "shell.execute_reply": "2024-03-22T21:05:06.771754Z" + }, + "papermill": { + "duration": 0.149744, + "end_time": "2024-03-22T21:05:06.774884", + "exception": false, + "start_time": "2024-03-22T21:05:06.625140", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:05:06.811014Z", + "iopub.status.busy": "2024-03-22T21:05:06.810639Z", + "iopub.status.idle": "2024-03-22T21:05:07.094104Z", + "shell.execute_reply": "2024-03-22T21:05:07.093114Z" + }, + "papermill": { + "duration": 0.304127, + "end_time": "2024-03-22T21:05:07.096204", + "exception": false, + "start_time": "2024-03-22T21:05:06.792077", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAESCAYAAAAv/mqQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAt30lEQVR4nO3de1gTZ74H8O8kIeF+EeTaKF7AS63ggrDoqmublu3adnXdlvZhi+U51a0HurZZdy1thbZ7Vmy1fThVH926q/bGkWd7ak/X9WAtSnustFgoLa0Wbyh4CRcvAQISmHnPHyGBaIAMhEwIv8/zzGMymcsvKF9n3nfmHY4xxkAIIU4gk7oAQsjYQYFDCHEaChxCiNNQ4BBCnIYChxDiNBQ4hBCnocAhhDiNQuoC7CEIAi5fvgw/Pz9wHCd1OYSQPhhjaG1tRWRkJGSygY9hRkXgXL58GWq1WuoyCCEDqK+vxx133DHgMqMicPz8/ACYvpC/v7/E1RBC+mppaYFarbb8ng5kVASO+TTK39+fAocQF2VPcwc1GhNCnIYChxDiNBQ4hBCnGRVtOGToeJ5HV1eX1GWQUU6pVA7a5W0PChw3xRiDTqfDjRs3pC6FuAGZTIZJkyZBqVQOazsUOG7KHDahoaHw9vamCybJkJkvvL1y5QomTJgwrH9LbhU455sN+FHXiuRJ4xDkM7wkHs14nreETXBwsNTlEDcwfvx4XL58Gd3d3fDw8Bjydtyq0Xj1+5V46r0KVNZdl7oUSZnbbLy9vSWuhLgL86kUz/PD2o5bBU5smC8AoKahVeJKXAOdRhFHcdS/JTcLHNOl1ad0FDiEuCK3CpxpPYFT09AmcSWEEFvcK3DCTYFztqkN3bwgcTVktOI4Dh999JHUZTjUSy+9hPj4eKnLcK/AiQr0grdSDmO3gAvX2qUuh5Ah27NnDwIDAx22vbVr16KkpMRh2xsqtwocmYxDTKip4ZjacchYYDQa7VrO19fXJS6RcKvAAXobjqmnyhpjDO3GbkkmsQ93LS4uxs9+9jMEBgYiODgYDzzwAM6ePQsAmDdvHtatW2e1fFNTEzw8PPD5558DAK5cuYIlS5bAy8sLkyZNQmFhIaKjo1FQUDCkn111dTXuvvtueHl5ITg4GKtWrUJbW287YWlpKZKSkuDj44PAwEDMnz8fFy5cAAB8++23WLx4Mfz8/ODv74+EhAR8/fXXA+6vtLQUmZmZ0Ov14DgOHMfhpZdeAgBER0fjz3/+MzIyMuDv749Vq1YBANatW4fY2Fh4e3tj8uTJWL9+vdUtLbeeUj3xxBNYunQpNm/ejIiICAQHByMrK2vEb4Nxqwv/gN52nFMUOFY6unjMzD0oyb5PvJIKb6X9/9QMBgO0Wi1mz56NtrY25ObmYtmyZaiqqkJ6ejpee+01bNy40dJVW1RUhMjISCxYsAAAkJGRgebmZpSWlsLDwwNarRaNjY1Dqt1gMCA1NRUpKSk4fvw4Ghsb8eSTTyI7Oxt79uxBd3c3li5dipUrV+K//uu/YDQaUV5ebqktPT0dc+bMwfbt2yGXy1FVVTXohXPz5s1DQUEBcnNzUVNTA8B0hGK2efNm5ObmIi8vzzLPz88Pe/bsQWRkJKqrq7Fy5Ur4+fnhT3/6U7/7OXLkCCIiInDkyBGcOXMGaWlpiI+Px8qVK4f0s7KH2wWOpWuceqpGreXLl1u937VrF8aPH48TJ07gkUcewTPPPIOjR49aAqawsBCPPfYYOI7Djz/+iE8//RTHjx9HYmIiAOBvf/sbYmJihlRLYWEhbt68iXfeeQc+Pj4AgK1bt+LBBx/Eq6++Cg8PD+j1ejzwwAOYMmUKAGDGjBmW9evq6vDHP/4R06dPBwC76lAqlQgICADHcQgPD7/t87vvvht/+MMfrOa9+OKLltfR0dFYu3Yt9u7dO2DgBAUFYevWrZDL5Zg+fTqWLFmCkpISChwxzEc4tc0GdHbzUCnkElfkGrw85DjxSqpk+xbj9OnTyM3NxVdffYXm5mYIgqnHsa6uDrNmzcJ9992H999/HwsWLEBtbS3Kysrw17/+FQBQU1MDhUKBn/zkJ5btTZ06FUFBQUOq/eTJk4iLi7OEDQDMnz8fgiCgpqYGCxcuxBNPPIHU1FTce++90Gg0eOSRRxAREQEA0Gq1ePLJJ/Huu+9Co9Hg4YcftgTTUJmDtK+ioiK8+eabOHv2LNra2tDd3T3o6Jh33nkn5PLev5uIiAhUV1cPq7bBuF0bTqifCv6eCvACw7kmg9TluAyO4+CtVEgyib1K9cEHH8S1a9ewc+dOfPXVV/jqq68A9DaQpqen44MPPkBXVxcKCwtx11134a677nL4z8xeu3fvRllZGebNm4eioiLExsbiyy+/BGBqO/nhhx+wZMkSHD58GDNnzsS+ffuGtb++4QcAZWVlSE9Pxy9/+Uvs378f33zzDV544YVBG5RvPbXjOM4S7iPF7QKH4zhqxxnFrl69ipqaGrz44ou45557MGPGDFy/bn1v3K9+9SvcvHkTxcXFKCwsRHp6uuWzadOmobu7G998841l3pkzZ27bhr1mzJiBb7/9FgZD739eX3zxBWQyGaZNm2aZN2fOHOTk5ODYsWOYNWsWCgsLLZ/Fxsbi2WefxSeffIJf//rX2L1796D7VSqVdt+3dOzYMUycOBEvvPACEhMTERMTY2m0djVDCpxt27YhOjoanp6eSE5ORnl5+YDL37hxA1lZWYiIiIBKpUJsbCwOHDgwpILtYempoq7xUScoKAjBwcF46623cObMGRw+fBhardZqGR8fHyxduhTr16/HyZMn8dhjj1k+mz59OjQaDVatWoXy8nJ88803WLVqFby8vIZ0P1B6ejo8PT2xYsUKfP/99zhy5AiefvppPP744wgLC0NtbS1ycnJQVlaGCxcu4JNPPsHp06cxY8YMdHR0IDs7G6Wlpbhw4QK++OILHD9+3KqNpz/R0dFoa2tDSUkJmpub0d7e/3VlMTExqKurw969e3H27Fm8+eabwz6KGimiA6eoqAharRZ5eXmorKxEXFwcUlNT++0FMBqNuPfee3H+/Hl88MEHqKmpwc6dOxEVFTXs4vvTe4RDDcejjUwmw969e1FRUYFZs2bh2WefxaZNm25bLj09Hd9++y0WLFiACRMmWH32zjvvICwsDAsXLsSyZcssPTaenp6i6/H29sbBgwdx7do1zJ07F7/5zW9wzz33YOvWrZbPf/zxRyxfvhyxsbFYtWoVsrKy8Lvf/Q5yuRxXr15FRkYGYmNj8cgjj+D+++/Hyy+/POh+582bh6eeegppaWkYP348XnvttX6Xfeihh/Dss88iOzsb8fHxOHbsGNavXy/6uzoFEykpKYllZWVZ3vM8zyIjI1l+fr7N5bdv384mT57MjEaj2F1Z6PV6BoDp9Xq7li8728wmrtvPFrx6eMj7HM06OjrYiRMnWEdHh9SluIT6+noGgH366adSlzJqDfRvSszvp6gjHKPRiIqKCmg0Gss8mUwGjUaDsrIym+t8/PHHSElJQVZWFsLCwjBr1ixs2LBhwPPTzs5OtLS0WE1imE+p6q61o93YLWpdMvodPnwYH3/8MWpra3Hs2DE8+uijiI6OxsKFC6UubcwTFTjNzc3geR5hYWFW88PCwqDT6Wyuc+7cOXzwwQfgeR4HDhzA+vXr8frrr+M//uM/+t1Pfn4+AgICLJPYx/yO81EixFcFADhNp1VjTldXF55//nnceeedWLZsGcaPH2+5CPD999+Hr6+vzenOO+90Wo33339/v3Vs2LDBaXU424hfhyMIAkJDQ/HWW29BLpcjISEBly5dwqZNm6yulOwrJyfHqqHQ/ChRMaaF+6L5TCdqGloRpw4czlcgo0xqaipSU21fc/TQQw8hOTnZ5mfDGTpTrL/97W/o6Oiw+dm4ceOcVoeziQqckJAQyOVyNDQ0WM1vaGiweUUkYLqYyMPDw+oCoxkzZkCn08FoNNocBV6lUkGlUokp7TaxYX744sxVnKaucdKHn5+fXc/AHmkj2WniykSdUimVSiQkJFjd5i4IAkpKSpCSkmJznfnz5+PMmTNWFxSdOnUKERERw37kxEBoMC5CXI/obnGtVoudO3fi7bffxsmTJ7F69WoYDAZkZmYCMN04l5OTY1l+9erVuHbtGtasWYNTp07hX//6FzZs2ICsrCzHfQsbYmi4UUJcjug2nLS0NDQ1NSE3Nxc6nQ7x8fEoLi62NCTX1dVZPaFPrVbj4MGDePbZZzF79mxERUVhzZo1tw0x4GjmAdV1LTehb+9CgLfzzs8JIbYNqdE4Ozsb2dnZNj8rLS29bV5KSorl3hJn8fP0QFSgFy7d6MCpxlbMjXbfhjhCRgu3u5eqL/NRDt1TRYhrcO/ACad2HCKeOw6i7ircO3BCabhRMjo5ehB1wNTcwXEcbty44dDtiuHWgWO+ibNG1yp6XF1CiOO5deBMDfUFxwHX27vQ3Gbf6PZuizHAaJBmokHUHTaIemdnJ9auXYuoqCj4+PggOTnZqqPmwoULePDBBxEUFAQfHx/ceeedOHDgAM6fP4/FixcDMA0BwnEcnnjiiSH9PIbD7YYY7cvTQ47oYB/UNhtwuqEV4/2Gd/XyqNbVDmyIlGbfz18GlD6DL9eDBlHvfxD17OxsnDhxAnv37kVkZCT27duHX/ziF6iurkZMTAyysrJgNBrx+eefw8fHBydOnICvry/UajX++7//G8uXL0dNTQ38/f3h5eU1pJ/JcLh14ACmnqraZgNqGloxb2qI1OUQO9Ag6rYHUa+rq8Pu3btRV1eHyEjTfx5r165FcXExdu/ejQ0bNqCurg7Lly+3DLk6efJky/rme7RCQ0Md3j5krzEQOH44+EMDdY17eJuONKTatwg0iLpt1dXV4HkesbGxVvM7OzstD7n7/e9/j9WrV+OTTz6BRqPB8uXLMXv27CHtbyS4dRsOQMONWnCc6bRGiokGUXfIIOptbW2Qy+WoqKhAVVWVZTp58iT+8z//EwDw5JNP4ty5c3j88cdRXV2NxMREbNmyxWHfdbjcPnDMPVWnG9qop2oUoEHUTWwNoj5nzhzwPI/GxkZMnTrVaup76qVWq/HUU0/hww8/xB/+8Afs3LnTsk0Adg/OPhLcPnCig33gIefQ2tmNK/qbUpdDBkGDqJvYGkQ9NjYW6enpyMjIwIcffoja2lqUl5cjPz8f//rXvwAAzzzzDA4ePIja2lpUVlbiyJEjlv1NnDgRHMdh//79aGpqsuppcxrHj37qeGLHNL7VfW98xiau288O/9jg4Mpc02gf0/jQoUNsxowZTKVSsdmzZ7PS0lIGgO3bt8+yzIEDBxgAtnDhwtvWv3z5Mrv//vuZSqViEydOZIWFhSw0NJTt2LHDrv3fuq/vvvuOLV68mHl6erJx48axlStXstbWVsYYYzqdji1dupRFREQwpVLJJk6cyHJzcxnP86yzs5M9+uijTK1WM6VSySIjI1l2drbdfy9PPfUUCw4OZgBYXl4eY4wxo9HIcnNzWXR0NPPw8GARERFs2bJl7LvvvmOMMZadnc2mTJnCVCoVGz9+PHv88cdZc3OzZZuvvPIKCw8PZxzHsRUrVthVB2OOG9OYY8z1zzNaWloQEBAAvV4/6NMEbckurMT+764g5/7p+N2i4T31cDS4efMmamtrMWnSpCE9qcDdXLx4EWq1Gp9++inuueceqcsZlQb6NyXm99Pte6kA02Bc+3GFbnEYIw4fPoy2tjbcdddduHLlCv70pz/RIOouwu3bcIA+N3FS4IwJNIi66xozRzgAcKaxDbzAIJeJbzwkowcNou66xkTgqMd5w9NDhptdAuqvtSM6xP7L7Il7oUHUpTUmTqnkMg5TQ033ooyldpxR0B9ARglH/VsaE4ED9F5xPBYG4zKfGrS3t0tcCXEX5qu8+z7uaSjGxCkV0PexMe4fOHK5HIGBgZY7pL29vYd00RshgOlRUE1NTfD29oZCMbzIGDOBE9vnFoexwHyp+1CHZSCkL5lMhgkTJgz7P64xEzjmI5yzTW0wdgtQKtz7bJLjOERERCA0NBRdXV1Sl0NGOaVSafX4p6EaUuBs27YNmzZtgk6nQ1xcHLZs2YKkpCSby+7Zs8fykDwzlUqFmzede19TRIAn/FQKtHZ24/xVg6VNx93J5fJhn3cT4iiiI6uoqAharRZ5eXmorKxEXFwcUlNTBzx09/f3x5UrVyyTefhFZ+I4DjE9j40Z80NVECIR0YHzxhtvYOXKlcjMzMTMmTOxY8cOeHt7Y9euXf2uYx65zDyZn9LpbNPoimNCJCUqcIxGIyoqKqDRaHo3IJNBo9GgrKys3/Xa2towceJEqNVq/OpXv8IPP/ww4H46OzvR0tJiNTmCpWucAocQSYgKnObmZvA8f9sRSlhYGHQ6nc11pk2bhl27duF//ud/8N5770EQBMybNw8XL17sdz/5+fkICAiwTGq1WkyZ/ZpmCZyx0VNFiKsZ8a6alJQUZGRkID4+HosWLcKHH36I8ePHW8agtSUnJwd6vd4y1dfXO6SWmJ7AOX/VgJtd0o16RshYJaqXKiQkBHK5HA0NDVbzGxoarIY4HIiHhwfmzJmDM2fO9LuMSqWCSuX4R7qE+CoxzkeJawYjzjS2YVZUgMP3QQjpn6gjHKVSiYSEBJSUlFjmCYKAkpISpKSk2LUNnudRXV1tGdXemTiOQyz1VBEiGdGnVFqtFjt37sTbb7+NkydPYvXq1TAYDJZrbTIyMpCTk2NZ/pVXXsEnn3yCc+fOobKyEr/97W9x4cIFPPnkk477FiJMo4ZjQiQj+sK/tLQ0NDU1ITc3FzqdDvHx8SguLrY0JNfV1VldkXj9+nWsXLkSOp0OQUFBSEhIwLFjxzBz5kzHfQsRaDAuQqQzJsY07uv4+Wt4eEcZogK98MVzdzuoQkLGLjG/n+59Q5ENsaGmI5xLNzrQepPuMSLEmcZc4AR4eyDc3zTqPF2PQ4hzjbnAAagdhxCpjMnAmdbTNU6BQ4hzjcnAiaGucUIkMSYDxzLcqI7acAhxpjEZOOZxcZrbOnG1rVPiaggZO8Zk4HgrFZgwzhsA9VQR4kxjMnCA3rFxTjdSOw4hzjKGA4du4iTE2cZs4NBwo4Q435gNnFhLT1UrPRKXECcZs4EzebwP5DIOLTe70dBCPVWEOMOYDRyVQo5JIT4A6LSKEGcZs4ED9DYcU+AQ4hxjPHB623EIISNvTAcODTdKiHON6cDpHaaiDYJAPVWEjLQxHTgTx3lDqZCho4vHpRsdUpdDiNsb04GjkMswZTxdcUyIs4zpwAF6B+OqoXYcQkbckAJn27ZtiI6OhqenJ5KTk1FeXm7Xenv37gXHcVi6dOlQdjsiaLhRQpxHdOAUFRVBq9UiLy8PlZWViIuLQ2pqKhobGwdc7/z581i7di0WLFgw5GJHwjTqGifEaUQHzhtvvIGVK1ciMzMTM2fOxI4dO+Dt7Y1du3b1uw7P80hPT8fLL7+MyZMnD6tgRzNfi3OuyYBuXpC4GkLcm6jAMRqNqKiogEaj6d2ATAaNRoOysrJ+13vllVcQGhqKf/u3f7NrP52dnWhpabGaRkpUoBe8lXIYeQHnr7aP2H4IISIDp7m5GTzPWx7raxYWFgadTmdznaNHj+Lvf/87du7cafd+8vPzERAQYJnUarWYMkWRyTgaVJ0QJxnRXqrW1lY8/vjj2LlzJ0JCQuxeLycnB3q93jLV19ePYJV9eqqoHYeQEaUQs3BISAjkcjkaGhqs5jc0NCA8PPy25c+ePYvz58/jwQcftMwTBFM7iUKhQE1NDaZMmXLbeiqVCiqVSkxpwxJLRziEOIWoIxylUomEhASUlJRY5gmCgJKSEqSkpNy2/PTp01FdXY2qqirL9NBDD2Hx4sWoqqoa0VMlMWj0P0KcQ9QRDgBotVqsWLECiYmJSEpKQkFBAQwGAzIzMwEAGRkZiIqKQn5+Pjw9PTFr1iyr9QMDAwHgtvlSMh/hnL/ajptdPDw95BJXRIh7Eh04aWlpaGpqQm5uLnQ6HeLj41FcXGxpSK6rq4NMNrouYA71UyHAywP6ji6cazJgZqS/1CUR4pY4NgoG9G1paUFAQAD0ej38/UcmDB7ZUYby89dQkBaPpXOiRmQfhLgjMb+fo+tQZATFhtM9VYSMNAqcHuZbHE5T4BAyYihwepgv/qMjHEJGDgVOD3NPVf21Dhg6uyWuhhD3RIHTY5yPEuP9TBcbnm5sk7gaQtwTBU4flkHV6RYHQkYEBU4fdIsDISOLAqePWBpulJARRYHTBw03SsjIosDpIybUdITT0NKJG+1GiashxP1Q4PTh5+mBqEAvAKaH4xFCHIsC5xY0VAUhI4cC5xYxPQ3HFDiEOB4Fzi3osTGEjBwKnFv0vRZnFIzcQcioQoFzi6mhvpBxwPX2LjS1dUpdDiFuhQLnFp4eckQH+wAATlNPFSEORYFjQww9NoaQEUGBY8M0uqeKkBFBgWOD+RYHuqeKEMeiwLGhd7jRNuqpIsSBKHBsiA7xgYecQ1tnNy7rb0pdDiFuY0iBs23bNkRHR8PT0xPJyckoLy/vd9kPP/wQiYmJCAwMhI+PD+Lj4/Huu+8OuWBn8JDLMDmk54pjajgmxGFEB05RURG0Wi3y8vJQWVmJuLg4pKamorGx0eby48aNwwsvvICysjJ89913yMzMRGZmJg4ePDjs4kcSteMQ4niiA+eNN97AypUrkZmZiZkzZ2LHjh3w9vbGrl27bC7/85//HMuWLcOMGTMwZcoUrFmzBrNnz8bRo0eHXfxImhZGRziEOJqowDEajaioqIBGo+ndgEwGjUaDsrKyQddnjKGkpAQ1NTVYuHBhv8t1dnaipaXFanK2WHpsDCEOJypwmpubwfO85TniZmFhYdDpdP2up9fr4evrC6VSiSVLlmDLli249957+10+Pz8fAQEBlkmtVosp0yHMw1ScaWwDL1BPFSGO4JReKj8/P1RVVeH48eP4y1/+Aq1Wi9LS0n6Xz8nJgV6vt0z19fXOKNOKOsgbnh4ydHYLqLvW7vT9E+KOFGIWDgkJgVwuR0NDg9X8hoYGhIeH97ueTCbD1KlTAQDx8fE4efIk8vPz8fOf/9zm8iqVCiqVSkxpDieTcYgJ9UP1JT1qdK2YFOIjaT2EuANRRzhKpRIJCQkoKSmxzBMEASUlJUhJSbF7O4IgoLPT9e/EpsfGEOJYoo5wAECr1WLFihVITExEUlISCgoKYDAYkJmZCQDIyMhAVFQU8vPzAZjaYxITEzFlyhR0dnbiwIEDePfdd7F9+3bHfpMRMC2cHhtDiCOJDpy0tDQ0NTUhNzcXOp0O8fHxKC4utjQk19XVQSbrPXAyGAz493//d1y8eBFeXl6YPn063nvvPaSlpTnuW4yQWMstDhQ4hDgCx0bBzUItLS0ICAiAXq+Hv7+/0/Z7+UYH5m08DIWMw4lXfgGlgu4EIeRWYn4/6TdoABEBnvBTKdAtMNQ2G6Quh5BRjwJnABzH0S0OhDgQBc4gLD1VdIsDIcNGgTMI8z1VdIRDyPBR4AyCeqoIcRwKnEGY23AuXGtHh5GXuBpCRjcKnEGE+KoQ7KMEY6YbOQkhQ0eBYwcaqoIQx6DAsYN5qAq6p4qQ4aHAsYP5wXgUOIQMDwWOHabRtTiEOAQFjh1iegLnsv4mWm52SVwNIaMXBY4dArw8EBHgCYCuxyFkOChw7GTpqdJR1zghQ0WBY6dYajgmZNgocOxEw40SMnwUOHaia3EIGT4KHDtNDfUFxwHNbUY0t7n+APCEuCIKHDt5KxWYMM4bAB3lEDJUFDgixISah6qgnipChoICRwR6bAwhwzOkwNm2bRuio6Ph6emJ5ORklJeX97vszp07sWDBAgQFBSEoKAgajWbA5V0ZDTdKyPCIDpyioiJotVrk5eWhsrIScXFxSE1NRWNjo83lS0tL8dhjj+HIkSMoKyuDWq3Gfffdh0uXLg27eGeb1mdA9VHwdB1CXI7o51IlJydj7ty52Lp1KwDTY3vVajWefvppPPfcc4Ouz/M8goKCsHXrVmRkZNi1T6meS3UrY7eAmbnF6BYYynLuRkSAl2S1EOIqRuy5VEajERUVFdBoNL0bkMmg0WhQVlZm1zba29vR1dWFcePG9btMZ2cnWlparCZXoFTIEB3iAwA4RQ3HhIgmKnCam5vB87zlsb5mYWFh0Ol0dm1j3bp1iIyMtAqtW+Xn5yMgIMAyqdVqMWWOKBqqgpChc2ov1caNG7F3717s27cPnp6e/S6Xk5MDvV5vmerr651Y5cBouFFChk4hZuGQkBDI5XI0NDRYzW9oaEB4ePiA627evBkbN27Ep59+itmzZw+4rEqlgkqlElOa05i7xuniP0LEE3WEo1QqkZCQgJKSEss8QRBQUlKClJSUftd77bXX8Oc//xnFxcVITEwcerUuoPc5VW0QBOqpIkQMUUc4AKDVarFixQokJiYiKSkJBQUFMBgMyMzMBABkZGQgKioK+fn5AIBXX30Vubm5KCwsRHR0tKWtx9fXF76+vg78Ks4xMdgHSoUMHV08Ll7vwIRgb6lLImTUEB04aWlpaGpqQm5uLnQ6HeLj41FcXGxpSK6rq4NM1nvgtH37dhiNRvzmN7+x2k5eXh5eeuml4VUvAbmMw9TxvjhxpQU1Da0UOISIIPo6HCm4ynU4Zs8WVWHfN5fwx9RpyFo8VepyCJHUiF2HQ0x6hxulhmNCxKDAGQLqqSJkaChwhsA8TMW5JgO6eEHiaggZPShwhiAq0As+SjmMvIALVw1Sl0PIqEGBMwQyGWd5OB49NoYQ+1HgDNE0usWBENEocIYoNpxu4iRELAqcIbI8GK+RAocQe1HgDJH5lOp8swE3u3iJqyFkdKDAGaLxfioEentAYMDZJmo4JsQeFDhDxHEcPf6XEJEocIZhGnWNEyIKBc4wmBuOT9MRDiF2ocAZBhpulBBxKHCGwRw4F693oK2zW+JqCHF9FDjDEOSjRKifaexlOq0iZHAUOMNkfhon9VQRMjgKnGEyD1VBD8YjZHAUOMNEg3ERYj8KnGGi4UYJsR8FzjCZx8VpbO3EdYNR4moIcW0UOMPkq1LgjiAvAHRaRchghhQ427ZtQ3R0NDw9PZGcnIzy8vJ+l/3hhx+wfPlyREdHg+M4FBQUDLVWl2W5p6qRGo4JGYjowCkqKoJWq0VeXh4qKysRFxeH1NRUNDY22ly+vb0dkydPxsaNGwd9/vhoZQkcaschZECiA+eNN97AypUrkZmZiZkzZ2LHjh3w9vbGrl27bC4/d+5cbNq0CY8++ihUKpVd++js7ERLS4vV5MrMPVV0iwMhAxMVOEajERUVFdBoNL0bkMmg0WhQVlbmsKLy8/MREBBgmdRqtcO2PRL6DlMxCh5kSohkRAVOc3MzeJ63PEfcLCwsDDqdzmFF5eTkQK/XW6b6+nqHbXskTBnvCxkH3GjvQlNrp9TlEOKyFFIXYItKpbL79MsVeHrIER3sg3PNBpxqaEOov6fUJRHikkQd4YSEhEAul6OhocFqfkNDg9s2CNuLhqogZHCiAkepVCIhIQElJSWWeYIgoKSkBCkpKQ4vbjShx8YQMjjRp1RarRYrVqxAYmIikpKSUFBQAIPBgMzMTABARkYGoqKikJ+fD8DU0HzixAnL60uXLqGqqgq+vr6YOnWqA7+KtOjBeIQMTnTgpKWloampCbm5udDpdIiPj0dxcbGlIbmurg4yWe+B0+XLlzFnzhzL+82bN2Pz5s1YtGgRSktLh/8NXIS5a/x0T08Vx3ESV0SI6+HYKOjHbWlpQUBAAPR6Pfz9/aUux6YuXsDM3GJ08QxH1y3GHUHeUpdEiFOI+f2ke6kcxEMuw5TxNFQFIQOhwHGgWHpsDCEDosBxIBpulJCBUeA4EA3GRcjAKHAcyNw1fqapDbzg8m3xhDgdBY4D3RHkBS8POYzdAi5cNUhdDiEuhwLHgWQyDjFh1FNFSH8ocByMeqoI6R8FjoNNC6OeKkL6Q4HjYOabOOmeKkJuR4HjYLE9bTjnmw3o7OYlroYQ1+JegdPWCHRLO+JeuL8n/DwV6BYYapupp4qQvtwrcIqfAzZOBN5ZCvzf68DFrwG+26klcBzXO1QFXQBIiBWXHGJ0yJpOAd0dwLkjpgkAlH5A9Hxg0kIgegEQNguQjWzOxob74esL16nhmJBbuFfgPPV/QFMNUPs5UPsZcP4ocPMGcKrYNAGA1zgg+memAJq0CAiJARw8ds006honxCb3ChyOA0Knm6bkVYDAAw3f9wTQ58CFY0DHNeDkx6YJAHzDesKnZwqKHnYZ5ov/TjfSEQ4hfblX4NxKJgci4kzTvKcBvgu4XGU6+qn9HKj/CmhrAKr/YZoAIHACEG0OoAWAf6To3ZqPcOqutaPd2A1vpXv/mAmx19j6TZB7AOq5pmnhWqDrJnDxuCl8zv+f6fWNOqDqPdMEAMExveETvQDwCRl0N8G+KoT4KtHcZsSZxjbMviNwZL8XIaPE2AqcW3l4moJk0gLT+842oP7L3lOwK98CV0+bpq//blombFbv6dfEeYBngM1Nx4b5obntKv757WUYOnkEensgwMs0eSvlNOYxGZNoTOOBdNwwtfuYA6jxB+vPORkQEd8bQBN+Cih9AAAv//MH7P7ivM3Nesg5S/iYp0BvpY15vX/698xTKeQj+pUJEUvM7ycFjhhtTaZTL/Mp2NUz1p/LPIA7EoFJC3FlXBI2fOeDS60CbnR0oaWjC/qOLnTxw/txe3nITYHkqUCAtwcCvRQI9FL0hJQCgV4e8PdSmALMU2F67WkKLLmMA9BzZMX1vO77Jx11jQxBAJgAMN7UkcF403uh7599Xgvdff7suuV9z8R3W78Xvbx5mS4b2+g2tXcKPBCVACz644BfT8zv55BOqbZt24ZNmzZBp9MhLi4OW7ZsQVJSUr/L/+Mf/8D69etx/vx5xMTE4NVXX8Uvf/nLoexaWr7jgVm/Nk0AoL/UG0C1nwP6eqCuDKgrQwSALTIFIFcBYAAHMC9mes0ABgYw03sGWF73vLHaLQcGWd95xp6pZeS+qgAOrCeczH/CPI8zz+tdxhxW5vngTJ9xPZ+xAYKOQdbnPUxHjlbLmD7nLMHYuzzX5z3XE5qc1fKy3vdc7/JWywK9v/y3hcJg8255bWs7xEJ04BQVFUGr1WLHjh1ITk5GQUEBUlNTUVNTg9DQ0NuWP3bsGB577DHk5+fjgQceQGFhIZYuXYrKykrMmjXLIV9CMgFRQNyjpokx4Pr53vCp/RwwNJr+t+jR9/jB1Y8lTAHXz9GYrdkuf5zs2njIIEAOxnHgOQUEyCFw5klhec0s7xU9r+VgMtNr0599X/f8KVMAPe/R8xqW+aZ5nEwBJvMAJzfN42QKQK6AKngSJjrwe4o+pUpOTsbcuXOxdetWAKZH/arVajz99NN47rnnbls+LS0NBoMB+/fvt8z76U9/ivj4eOzYscOufbrMKZUYjAH6i72BY/7f2vIa6P1fHLd8Zu9yt36GPp/1LtfFC2i9ycPI8+jmGQRBQDcvgBf6TLwAnjF0dwsQBAZe4C2fdfMMvCCY1hPMnwsQeB7dgunfAN9nPi/wPa9Zn/k8eJ5BYKZlmWCqQ2CmPxkzTULPfLA+nwkMjPFgjEEQmGVZJpi2xwQGQOjZDsD6rM+YAA6ADILV8Zr5PdeTlHzPHAGyntem9+bXPLP+nIcMrM/rW9e1tfxt2+x5z1z4DqN7pofi70/MHXCZETulMhqNqKioQE5OjmWeTCaDRqNBWVmZzXXKysqg1Wqt5qWmpuKjjz7qdz+dnZ3o7Oy9CbOlZQTPHUYKxwGBaqmrAAB4ABjnI3UV0mCMQWAALzAIzBSC3YIpuHhmDkXWE04whVzP8uZ1zev1ft47XxBMrxkzbc88nzEGXuh93bcGZnObMK3fp06hzzb61skL1nXxt61zy3pW24WNZfu8ZrD8ZyEwBvU4xz7QUVTgNDc3g+d5y2N9zcLCwvDjjz/aXEen09lcXqfT9buf/Px8vPzyy2JKI8QmjuMg59DTYE6k5pLHcjk5OdDr9Zapvr5e6pIIIQ4g6ggnJCQEcrkcDQ0NVvMbGhoQHh5uc53w8HBRywOASqWCSqUSUxohZBQQdYSjVCqRkJCAkpISyzxBEFBSUoKUlBSb66SkpFgtDwCHDh3qd3lCiPsS3S2u1WqxYsUKJCYmIikpCQUFBTAYDMjMzAQAZGRkICoqCvn5+QCANWvWYNGiRXj99dexZMkS7N27F19//TXeeustx34TQojLEx04aWlpaGpqQm5uLnQ6HeLj41FcXGxpGK6rq4OszwBX8+bNQ2FhIV588UU8//zziImJwUcffTT6r8EhhIhGtzYQQoZFzO+nS/ZSEULc06gYnsJ8EDYqLwAkxM2Zfy/tOVkaFYHT2moaqlOtdo0rdwkht2ttbUVAgO3xocxGRRuOIAi4fPky/Pz8Bhy4qqWlBWq1GvX19dTWcwv62dhGP5f+2fuzYYyhtbUVkZGRVh1GtoyKIxyZTIY77rjD7uX9/f3pH08/6GdjG/1c+mfPz2awIxszajQmhDgNBQ4hxGncKnBUKhXy8vLoPiwb6GdjG/1c+jcSP5tR0WhMCHEPbnWEQwhxbRQ4hBCnocAhhDgNBQ4hxGkocAghTuNWgbNt2zZER0fD09MTycnJKC8vl7okSeXn52Pu3Lnw8/NDaGgoli5dipqaGqnLckkbN24Ex3F45plnpC5FcpcuXcJvf/tbBAcHw8vLC3fddRe+/vprh2zbbQLH/IC+vLw8VFZWIi4uDqmpqWhsbJS6NMl89tlnyMrKwpdffolDhw6hq6sL9913HwwGg9SluZTjx4/jr3/9K2bPni11KZK7fv065s+fDw8PD/zv//4vTpw4gddffx1BQUGO2QFzE0lJSSwrK8vynud5FhkZyfLz8yWsyrU0NjYyAOyzzz6TuhSX0draymJiYtihQ4fYokWL2Jo1a6QuSVLr1q1jP/vZz0Zs+25xhGN+QJ9Go7HMG+wBfWORXq8HAIwbN07iSlxHVlYWlixZYvVvZyz7+OOPkZiYiIcffhihoaGYM2cOdu7c6bDtu0XgDPSAvoEeuDeWCIKAZ555BvPnz6fxpHvs3bsXlZWVlgH/CXDu3Dls374dMTExOHjwIFavXo3f//73ePvttx2y/VExPAUZvqysLHz//fc4evSo1KW4hPr6eqxZswaHDh2Cp6en1OW4DEEQkJiYiA0bNgAA5syZg++//x47duzAihUrhr19tzjCGcoD+saS7Oxs7N+/H0eOHBE1rpA7q6ioQGNjI37yk59AoVBAoVDgs88+w5tvvgmFQgGe56UuURIRERGYOXOm1bwZM2agrq7OIdt3i8AZygP6xgLGGLKzs7Fv3z4cPnwYkyZNkrokl3HPPfeguroaVVVVlikxMRHp6emoqqqCXC6XukRJzJ8//7ZLJ06dOoWJEyc6Zgcj1hztZHv37mUqlYrt2bOHnThxgq1atYoFBgYynU4ndWmSWb16NQsICGClpaXsypUrlqm9vV3q0lwS9VIxVl5ezhQKBfvLX/7CTp8+zd5//33m7e3N3nvvPYds320ChzHGtmzZwiZMmMCUSiVLSkpiX375pdQlSQqAzWn37t1Sl+aSKHBM/vnPf7JZs2YxlUrFpk+fzt566y2HbZvGwyGEOI1btOEQQkYHChxCiNNQ4BBCnIYChxDiNBQ4hBCnocAhhDgNBQ4hxGkocAghTkOBQwhxGgocQojTUOAQQpzm/wGR7UeNMuRGagAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:05:07.134463Z", + "iopub.status.busy": "2024-03-22T21:05:07.134150Z", + "iopub.status.idle": "2024-03-22T21:10:01.335745Z", + "shell.execute_reply": "2024-03-22T21:10:01.334892Z" + }, + "papermill": { + "duration": 294.223202, + "end_time": "2024-03-22T21:10:01.338394", + "exception": false, + "start_time": "2024-03-22T21:05:07.115192", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:01.375882Z", + "iopub.status.busy": "2024-03-22T21:10:01.375558Z", + "iopub.status.idle": "2024-03-22T21:10:01.396142Z", + "shell.execute_reply": "2024-03-22T21:10:01.395270Z" + }, + "papermill": { + "duration": 0.042261, + "end_time": "2024-03-22T21:10:01.398302", + "exception": false, + "start_time": "2024-03-22T21:10:01.356041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.00.013790.00508311.6775110.0780411.4367520.1297990.0000666.3949170.0479423503210.250.0712950.2437230.00001518.072429
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.0 0.01379 0.005083 11.677511 0.078041 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 1.436752 0.129799 0.000066 6.394917 0.047942 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 3503210.25 0.071295 0.243723 0.000015 18.072429 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:01.433398Z", + "iopub.status.busy": "2024-03-22T21:10:01.433107Z", + "iopub.status.idle": "2024-03-22T21:10:01.943094Z", + "shell.execute_reply": "2024-03-22T21:10:01.942082Z" + }, + "papermill": { + "duration": 0.529995, + "end_time": "2024-03-22T21:10:01.945257", + "exception": false, + "start_time": "2024-03-22T21:10:01.415262", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:10:01.982376Z", + "iopub.status.busy": "2024-03-22T21:10:01.982041Z", + "iopub.status.idle": "2024-03-22T21:15:29.306320Z", + "shell.execute_reply": "2024-03-22T21:15:29.305467Z" + }, + "papermill": { + "duration": 327.346112, + "end_time": "2024-03-22T21:15:29.308842", + "exception": false, + "start_time": "2024-03-22T21:10:01.962730", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:15:29.348351Z", + "iopub.status.busy": "2024-03-22T21:15:29.348017Z", + "iopub.status.idle": "2024-03-22T21:15:29.374002Z", + "shell.execute_reply": "2024-03-22T21:15:29.373187Z" + }, + "papermill": { + "duration": 0.047293, + "end_time": "2024-03-22T21:15:29.376162", + "exception": false, + "start_time": "2024-03-22T21:15:29.328869", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:15:29.413991Z", + "iopub.status.busy": "2024-03-22T21:15:29.413152Z", + "iopub.status.idle": "2024-03-22T21:15:29.419579Z", + "shell.execute_reply": "2024-03-22T21:15:29.418593Z" + }, + "papermill": { + "duration": 0.028986, + "end_time": "2024-03-22T21:15:29.422024", + "exception": false, + "start_time": "2024-03-22T21:15:29.393038", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.41921933552430435}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:15:29.464081Z", + "iopub.status.busy": "2024-03-22T21:15:29.463136Z", + "iopub.status.idle": "2024-03-22T21:15:29.885474Z", + "shell.execute_reply": "2024-03-22T21:15:29.884472Z" + }, + "papermill": { + "duration": 0.445301, + "end_time": "2024-03-22T21:15:29.887736", + "exception": false, + "start_time": "2024-03-22T21:15:29.442435", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAE8CAYAAACRnDftAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMUElEQVR4nO3deXhU5d34//eZfctM9pVAQJBVFilosAooFZWfSm0tLi2goLYPPpVS24pXv1q1/WJb16ePxW5Afay11SJ+H6wLRYEquLApooBgSAJkJclMZt/O74+TDASSkGVmzkxyv65rriRn7jnzOVk+ue/73Isky7KMIAhCmtGoHYAgCEJfiOQlCEJaEslLEIS0JJKXIAhpSSQvQRDSkkhegiCkJZG8BEFISyJ5CYKQlkTyEgQhLYnkJQhCWhLJS1Dd9u3b+dnPfkZLS4vaoQhpRCQvQXXbt2/noYceEslL6BWRvARBSEsieQmq+tnPfsaPfvQjAIYPH44kSUiShM1mY/bs2WeVj0ajlJSU8M1vfjN27LHHHmPGjBnk5ORgNpuZOnUqL7/8cqfv9/zzzzN16lTMZjPZ2dncdNNNVFdXJ+bihISSxJI4gpo++eQTHn30Uf7617/y5JNPkpubC8CRI0d4+OGHOX78OIWFhbHy27ZtY+bMmbz00kuxBFZaWsp1113HuHHjCAaDvPjii3z44Yds3LiRefPmxV77i1/8gv/zf/4P3/rWt5g5cyYNDQ385je/wWazsWfPHjIzM5N67UI/yYKgsl//+tcyIFdUVMSOHTx4UAbk3/zmNx3K/sd//Idss9lkr9cbO3b657Isy8FgUJ4wYYJ8+eWXx44dPXpU1mq18i9+8YsOZfft2yfrdLqzjgupTzQbhZR0/vnnM3nyZP72t7/FjkUiEV5++WWuvfZazGZz7Pjpnzc3N+N0Orn00kvZvXt37Pj69euJRqN861vforGxMfYoLCxk1KhRvPPOO8m5MCFudGoHIAhdWbBgAffffz/Hjx+npKSELVu2UF9fz4IFCzqU27hxIz//+c/Zu3cvgUAgdlySpNjnX3zxBbIsM2rUqE7fS6/XJ+YihIQRyUtIWQsWLGDlypW89NJLLF++nL///e84HA6uuuqqWJl///vfXHfddVx22WX89re/paioCL1ez9q1a3nhhRdi5aLRKJIk8frrr6PVas96L5vNlpRrEuJHJC9BdafXkE43fPhwpk+fzt/+9jfuvvtu1q9fz/z58zEajbEy//jHPzCZTLz55psdjq9du7bDuc477zxkWWb48OGcf/75ibkQIalEn5egOqvVCtDpINUFCxbw/vvvs2bNGhobG89qMmq1WiRJIhKJxI4dPXqUDRs2dCh3ww03oNVqeeihh5DPuMEuyzInT56Mz8UISSOGSgiq++ijj5g+fTrXXHMNN910E3q9nmuvvRar1cqxY8cYOnQoNpsNvV5PbW1th/6pt99+myuuuIJLL72UW265hfr6ep555hkKCwv55JNPOiSqRx99lJUrVzJjxgzmz59PRkYGFRUVvPLKK9x5553ce++9aly+0Fcq3ukUhJhHHnlELikpkTUazVnDJi655BIZkJcuXdrpa//0pz/Jo0aNko1GozxmzBh57dq18oMPPih39uv9j3/8Q/7qV78qW61W2Wq1ymPGjJGXLVsmHzx4MFGXJiSIqHkJgpCWRJ+XIAhpSSQvQRDSkkhegiCkJZG8BEFISyJ5CYKQlkTyEgQhLQ266UHRaJQTJ06QkZHR5bQUQRDUI8syra2tFBcXo9F0Xb8adMnrxIkTlJaWqh2GIAjnUF1dzZAhQ7p8ftAlr4yMDED5xtjtdpWjEQThTC6Xi9LS0tjfalcGXfJqbyra7XaRvAQhhZ2rW0d02AuCkJZE8hIEIS2J5CUIQlpStc9r9erVrF69mqNHjwIwfvx4HnjgAa6++upOy69bt47bbrutwzGj0Yjf749rXLIsEw6HOyxwJ/SMVqtFp9OJYShCwqmavIYMGcKjjz7KqFGjkGWZP//5z1x//fXs2bOH8ePHd/oau93OwYMHY1/H+48kGAxSU1OD1+uN63kHE4vFQlFREQaDQe1QhAFM1eR17bXXdvj6F7/4BatXr+b999/vMnlJktRhE9J4ikajVFRUoNVqKS4uxmAwiBpEL8iyTDAYpKGhgYqKCkaNGtXtIENB6I+UGSoRiUR46aWX8Hg8lJeXd1nO7XYzbNgwotEoF154If/3//7fLhMdQCAQ6LAdlsvl6rJsMBgkGo1SWlqKxWLp24UMcmazGb1eT2VlJcFgEJPJpHZIvedrhsObQWuAUVeCPg2vYRBQ/d/ivn37sNlsGI1Gvvvd7/LKK68wbty4TsuOHj2aNWvW8Oqrr/L8888TjUaZMWMGx44d6/L8q1atwuFwxB49GV0vagv9k9bfP1mG/Rug8Quo2w9HNqsdkdAF1ZeBDgaDVFVV4XQ6efnll/njH//I1q1bu0xgpwuFQowdO5abb76ZRx55pNMyndW8SktLcTqdZw1S9fv9VFRUMHz48PSsMaSItP4+tlTDnudPfS1poHwZGMW+jsnicrlwOByd/o2eTvVmo8FgYOTIkQBMnTqVjz76iKeffprf/e5353ytXq9nypQpHD58uMsyRqOxw35+gtCtuv3Kx8ILwNMArbVw8gsonqJuXMJZUq5+H41GO9SUuhOJRNi3bx9FRUUJjko4XVlZGU899ZTaYSRGc4XyMX8s5I5SPm+qUC8eoUuq1rxWrlzJ1VdfzdChQ2ltbeWFF15gy5YtvPnmmwAsXLiQkpISVq1aBcDDDz/MxRdfzMiRI2lpaeHXv/41lZWVLF26VM3LEAYKvwt8LSBJ4BgCmrY/D9cJVcMSOqdq8qqvr2fhwoXU1NTgcDiYOHEib775Jl/72tcAqKqq6tD529zczB133EFtbS1ZWVlMnTqV7du396h/TOgoGAyKcVhncrbd+LEVgM4IGUVKIgu0KonNJCbypxJVk9ef/vSnbp/fsmVLh6+ffPJJnnzyyQRG1LVgONrlcxoJdFpNj8pKEuh7UNag612LftasWUyYMAGA//mf/0Gv1/O9732Phx9+GEmSKCsrY8mSJXzxxRds2LCBG264gXXr1vHuu++ycuVKdu7cSW5uLl//+tdZtWoVVqsVUP7BLFmyhH/9618UFhby85//vFdxpRV3nfLRXqx81BnAmgvutr4vkbxSiuod9unimXe6vikwPNfK/Cklsa9/v+0IoUjnN3GHZJm58Sunhmusea8CX/DsaUg/+Nr5vY7xz3/+M0uWLOHDDz9k586d3HnnnQwdOpQ77rgDgMcee4wHHniABx98EIAjR45w1VVX8fOf/5w1a9bQ0NDA3Xffzd13383atWsBWLx4MSdOnOCdd95Br9fz/e9/n/r6+l7HlhY8DcpHa96pY9Z8JXl5G4He/0yExBHJawApLS3lySefRJIkRo8ezb59+3jyySdjyevyyy/nhz/8Yaz80qVLufXWW1m+fDkAo0aN4r/+67+YOXMmq1evpqqqitdff50PP/yQadOmAUpteezYsUm/tqRwtyVlW/6pY9Zc5aOnMfnxCN0SyauHls0e2eVzmjNmEN152Xldlj1zttHtlwzvT1gdXHzxxR2mM5WXl/P444/HJph/5Stf6VD+448/5pNPPuEvf/lL7Jgsy7FpUocOHUKn0zF16tTY82PGjCEzMzNuMaeMkE/p24KONS9LW/LyiuSVakTy6qHe9EElqmx/tfdjtXO73dx11118//vfP6vs0KFDOXToULJCU197k9HkUDrr27XXvLwnldH3Yq5ryhDJawD54IMPOnz9/vvvM2rUKLRabaflL7zwQj777LPYIOEzjRkzhnA4zK5du2LNxoMHD9LS0hLXuFOCt0n5aMnpeNzkUBJWJAxBNxi7X1ddSJ6UG6Qq9F1VVRUrVqzg4MGD/PWvf+U3v/kN99xzT5flf/KTn7B9+3buvvtu9u7dyxdffMGrr77K3XffDShzSa+66iruuusuPvjgA3bt2sXSpUsxm83JuqTk8bcoH82ZHY9rtKcSlt+ZzIiEcxDJawBZuHAhPp+P6dOns2zZMu655x7uvPPOLstPnDiRrVu3cujQIS699FKmTJnCAw88QHFxcazM2rVrKS4uZubMmdxwww3ceeed5Ofnd3nOtOVrVj6as85+zpTZVqYlWdEIPSCajQOIXq/nqaeeYvXq1Wc9175a7ZmmTZvGW2+91eU5CwsL2bhxY4dj3/nOd/oVZ0rqNnk5lI+i5pVSRM1LEGT5VK2qvZZ1uvamZHvTUkgJInkJQsgH4bbFAM7s84JTCU3UvFKKaDYOEGdOpRJ6ob1GZcwArf7s50WzMSWJmpcgxPq7Mjt/vn0hwoBbaWIKKUEkL0Horr8LwNA2VCIaVpqYQkoQyUsQ2qcFdbVqhFYH+raxbUF3cmISzkkkL0EItO0oZexmyZv2gartiU5QneiwF4RY8sogGI7y9oF6GtwBppVlMabQHnsOd71IXilEJC9BaE9IRjvbDjXweY2SzN74tJYsi4ECuwkMbZ32otmYMkSzURjcwkEI+ZXPTXZGF2YwMt9GhkmHLMOuyrY7kaLZmHJEzUsY3NqTkc4AOiOl2VCabaG+1c9f3q/icL0bXzCC+fThEkJKULXmtXr1aiZOnIjdbsdut1NeXs7rr7/e7WteeuklxowZg8lk4oILLuCf//xnkqIVBqQuOuvzM0zk2gxEojJVTd5Tz7eXF1SnavIaMmQIjz76KLt27WLnzp1cfvnlXH/99ezfv7/T8tu3b+fmm29myZIl7Nmzh/nz5zN//nw+/fTTxAUpy0rTQo1HDwdEPvfcc+Tk5Jy13+X8+fMH5iTqeGpLRgGdlR1HTlLjPDWOqyxXWbzx6EkPGNoWcgx6kh6i0DlJllNryHB2dja//vWvWbJkyVnPLViwAI/H02GVg4svvpjJkyfz7LPP9uj83W0l3uk29eEg/Pvxvl9Qf1z6Q6U5cw4+n4+ioiL+8Ic/cOONNwLKrj8lJSW89dZbzJ49O9GRdtDp9zFVHX0XKv5NtWkUL7snkm83cutFwwBwekO4g2HybEYMEQ9s/2+QNDDzx2JF1QTq7m/0dCnTYR+JRHjxxRfxeDyUl5d3WmbHjh3MmTOnw7G5c+eyY8eOLs8bCARwuVwdHgON2Wzmlltuie34A/D8888zdOhQZs2apV5g6cCv/D40hJSln4dkWWJPOSx6SjLNylLd+rbjchRC3qSHKZxN9Q77ffv2UV5ejt/vx2az8corr3S5iWxtbS0FBQUdjhUUFFBbW9vl+VetWsVDDz3U9wC1eqUGpIbOJgl34Y477mDatGkcP36ckpIS1q1bx+LFiztsyCF0oq3Dvs5vAAmKHF3UFDVa0JuUO5NB76lmpKAa1ZPX6NGj2bt3L06nk5dffplFixaxdevWuO2CvXLlSlasWBH72uVyUVpa2s0rziBJPWq6qW3KlClMmjSJ5557jiuvvJL9+/fz2muvqR1W6gu4iMoy9UEDGFHGdJ3meIuPQ3Wt5NmMTNBbleQV8gB5nZ9PSBrVk5fBYIhtADF16lQ++ugjnn76aX73u9+dVbawsJC6uroOx+rq6igsLOzy/EajEaPR2OXzA8nSpUt56qmnOH78OHPmzOldkh6sgm48wTA+yYzFoMVu6vgnUefys7eqhZH5NiYYrMouQkHRbEwFKdPn1S4ajZ5116xdeXk5mzdv7nBs06ZNXfaRDTa33HILx44d4w9/+AO333672uGkvkgYQn68gQghrYV8u/GsZnaOVal1N3mCYGjr9xJ3HFOCqslr5cqVbNu2jaNHj7Jv3z5WrlzJli1buPXWWwFlQ4mVK1fGyt9zzz288cYbPP744xw4cICf/exn7Ny5M7bbzWDncDj4xje+gc1mY/78+WqHk/pCShLyhGXCGiPZ1rNr6NltyavFGyKiNXd4naAuVZuN9fX1LFy4kJqaGhwOBxMnTuTNN9/ka1/7GqBs5aXRnMqvM2bM4IUXXuCnP/0p999/P6NGjWLDhg1MmDBBrUtIOcePH+fWW28dNE3lfmmrQQ0rzOfWC4Z1ugGwzajDqNcQCEVplU1kgmg2pghVk9ef/vSnbp/vbGnjG2+8MTaWSTilubmZLVu2sGXLFn7729+qHU56aJvqozXZyLd3fpdRkiRyrAZOtPhpCevbkpeoeaUC1TvshfiYMmUKzc3N/PKXv2T06NFqh5Me2leIaF8xogtZllPJCxDNxhQhktcA0dW+jEI3gh48gTBf1IUx5LgYV9z5aG6HWUlarZG2ITOi2ZgSRPISBq+gG3cgzBduGX1da5fJa1JpJhOHZGIKu+ADlJqXLIspQioTyasTKTbdM+2kzfcv6MEfjhDUWsi1dD2bwaTXKp9o2kbVR8IQCYJO3BRRU8qN81KTXq/8Anu9olnQH+3fv/bvZ8oKuvGHooQ0ZjLNPYhVqwdN2/97Mb9RdaLmdRqtVktmZib19fUAWCwWMTewF2RZxuv1Ul9fT2ZmJlqtVu2Quhdw4w9FCNosZFq6nwK2/XAjDe4AX5MNWFAGt2JOUpxCp0TyOkP7VKP2BCb0XmZmZrdTtlKCLCMH3QRCyuj6c9W8Kk56qHcF8Oj0WEDUvFKASF5nkCSJoqIi8vPzCYVCaoeTdvR6ferXuABCPkLhCBEZIjoL9nMkL4dZT70rgDtqIE+D2Hw2BYjk1QWtVpsef4RC3wSVJmNYY8JmNqLVdN89YDe19YdG9UpPsUheqhPJSxicgm7sJj0Xjynkgkkl5yye0bbahCfaPlBVNBvVJu42CoNT2xQfvcV+zs56OJW8WqNtZcP+hIUm9IxIXsLg1L6FWQ9XRM1oaza2hkXNK1WI5CUMTkEPFY0e9taFcHrPfWOmveblw6AMwhV9XqoTfV7C4BR0c9IT4AutTEkkes7iZr2W7806D6NLh/TJXlHzSgEieQmDUtjfSigiE9KYY7Wq7kiSpEwT0rcvSChqXmoTzUZhUAp623fKtmHsZBHCLsWSl7/HmwILiSGSlzAoBb3KlmdGi73HU8A+O+Hi/33eTH2rH6JhiIhBzGoSyUsYfMJBggGl2Weydr0j85mavUGONAZxt+cs0e+lKpG8hMEn6CYQjhKVdFjNlnOXb5Nh0oEk4ZXbxnqJfi9VqZq8Vq1axbRp08jIyCA/P5/58+dz8ODBbl+zbt06JEnq8DCZutjlWBA6E/QQikQJai1k9GQpnDY2o9Kx7xWj7FOCqncbt27dyrJly5g2bRrhcJj777+fK6+8ks8++wyrtevBg3a7vUOSE8vWCL0SdDMi10aprZhIaWaPXxZLXrIB8Imal8pUTV5vvPFGh6/XrVtHfn4+u3bt4rLLLuvydZIkpf6SK0Lqap8aZM5Ar+/55HtLW/LyyAZk2YskpgipKqX6vJxOJwDZ2dndlnO73QwbNozS0lKuv/569u/f32XZQCCAy+Xq8BAGufZdg4wZvXqZRa9FkiAkmQhFZdFsVFnKJK9oNMry5cu55JJLut1EdvTo0axZs4ZXX32V559/nmg0yowZMzh27Fin5VetWoXD4Yg9SktLE3UJQpoIels5UOtid02gV+vtazQSFoMWWW8mHImKZqPKJDlFdkv43ve+x+uvv867777LkCFDevy6UCjE2LFjufnmm3nkkUfOej4QCBAIBGJfu1wuSktLcTqd2O09v00uDBzOD57ns0/3cqJwNt+YN69Xrw2Go+hrdiId3gz5Y2D81xMU5eDlcrlwOBzn/BtNielBd999Nxs3bmTbtm29SlygrNw5ZcoUDh8+3OnzRqMRo1Hs8iKcEmgboGqw9P6fl0GnAX3b8ApR81KVqs1GWZa5++67eeWVV3j77bcZPnx4r88RiUTYt28fRUVFCYhQGIhCbVODTJbe9XnFiPmNKUHVmteyZct44YUXePXVV8nIyKC2thYAh8OB2az8gixcuJCSkhJWrVoFwMMPP8zFF1/MyJEjaWlp4de//jWVlZUsXbpUtesQ0kg0SsivdNibbY5ev/xIg5vDh51c0OKj2CiSl5pUTV6rV68GYNasWR2Or127lsWLFwNQVVWFRnOqgtjc3Mwdd9xBbW0tWVlZTJ06le3btzNu3LhkhS2ks5CXYCgCSFhsvW82uv1hjrREKQuEISySl5pUTV49uVewZcuWDl8/+eSTPPnkkwmKSBjwgh6C4SghjYk887mXfz6T1agjrDESDEfbds4OKZvRCkmXMkMlBCEpgm7C0SghrTm2tHNvWI1aIpKBYKTtH6/o91JNStxtFISkCXqYXJpF0DEMra33d6GtRmVytg8DMjJSyAcmMeRGDaLmJQwubVODDOaMc+7V2BmrQfl/H5SMhCOy6PdSkUhewuDSlrx6umvQmbQaCbNBS1hjIhiJKiuqCqoQzUZhUKk7eZLmWhdkyIzp4zmsBi2yzkgk6hXzG1UkkpcwqHhanTR7Q0SDfb9DeNP0oegyhyPVusXmsyoSyUsYVEK+ttH1vVj++Ux6rea0Ufai5qUW0eclDCphX9voemvvR9d3EJvfKGpeahHJSxg8IiHCQeXuoLUPo+vbVZ70sKWilepmrxjnpSKRvITBo210fVTSYutmmfFz8QYjVDplWv1iipCaRPISBg2/10U4Kiuj6/swNaidxaAlrDESEgsSqkokL2HQCHhalYGpepuyLlcfWQw6QhqTSF4qE3cbhUHDoQ0wvSybcPawfp2nveYVjshEQz40sgxiB6ukEzUvYfBo23hDZ+rjIoRtzHotEZ0RGZS17MVYL1WI5CUMHv2cGtROo5EwGYxEJL1oOqqoT8nryy+/jHccgpBwh6pqOFDrosbX870au2KJTRGSRfJSSZ+S18iRI5k9ezbPP/88fr+oMgvpweVSpgb5JXO/z3Xz9KFcOm4odpNeNBtV0qfktXv3biZOnMiKFSsoLCzkrrvu4sMPP4x3bIIQV2G/smuQqR8DVNvptBrQiSlCaupT8po8eTJPP/00J06cYM2aNdTU1PDVr36VCRMm8MQTT9DQ0BDvOAWhX6KRKHJbn5c1o59Tg9rF5jeKmpca+tVhr9PpuOGGG3jppZf45S9/yeHDh7n33nspLS1l4cKF1NTUdPv6VatWMW3aNDIyMsjPz2f+/PkcPHjwnO/70ksvMWbMGEwmExdccAH//Oc/+3MZwiDg9XkgGkECbH3YNehMFY0e3q/2cqxZLIujln4lr507d/If//EfFBUV8cQTT3Dvvfdy5MgRNm3axIkTJ7j++uu7ff3WrVtZtmwZ77//Pps2bSIUCnHllVfi8Xi6fM327du5+eabWbJkCXv27GH+/PnMnz+fTz/9tD+XIgxwntYWADQGExpd/zfM8ATCVLdCayAs+rxUIsk92cLnDE888QRr167l4MGDXHPNNSxdupRrrrmmwxZlx44do6ysjHA43OPzNjQ0kJ+fz9atW7nssss6LbNgwQI8Hg8bN26MHbv44ouZPHkyzz777Dnfo6dbiQsDy5df7Kdu21qM9jwuvPEn/T5fRaOH97e9wXjvh0ycUg7j5/c/SAHo+d9on0bYr169mttvv53Fixd3uVN1fn4+f/rTn3p1XqfTCUB2dnaXZXbs2MGKFSs6HJs7dy4bNmzotHwgECAQCMS+drlcvYpJGBgiPmVqkNbcvwGq7WLzG8NiqIRa+pS8Nm3axNChQzvUtEDZh7G6upqhQ4diMBhYtGhRj88ZjUZZvnw5l1xyCRMmTOiyXG1tLQUFBR2OFRQUxHbbPtOqVat46KGHehyHMDCNygTKsonkl8TlfJa2dexDkShyyIeYHJR8ferzOu+882hsbDzreFNTE8OHD+9TIMuWLePTTz/lxRdf7NPru7Jy5UqcTmfsUV1dHdfzC2kioEwN0vZzalA7i0HZfFYGQoGu+2iFxOlTzaurbjK3243JZOr1+e6++242btzItm3bGDJkSLdlCwsLqaur63Csrq6OwsLCTssbjUaMxt7vzycMMEFljBeG+CQvrUZCY1RWUw37vfR9gR2hr3qVvNr7miRJ4oEHHsBiscSei0QifPDBB0yePLnH55Nlmf/8z//klVdeYcuWLT2qtZWXl7N582aWL18eO7Zp0ybKy8t7/L7C4PPRwSoyAi5KzzfTv5mNpxhNVrQaiUgoAJEwaMUiLcnUq+/2nj17ACXp7Nu3D4Ph1P8bg8HApEmTuPfee3t8vmXLlvHCCy/w6quvkpGREeu3cjgcmM3KAMCFCxdSUlLCqlWrALjnnnuYOXMmjz/+OPPmzePFF19k586d/P73v+/NpQiDSDAcxdPaTDgcoixOzUaAm8pHoYvkgNy2+aw2fucWzq1Xyeudd94B4LbbbuPpp5/u91CD1atXAzBr1qwOx9euXcvixYsBqKqq6nBjYMaMGbzwwgv89Kc/5f7772fUqFFs2LCh205+YXBz+0MYIl60GgmTJU6j6wGdTgs6k3K3MeQDo0heydSneu7atWvj8uY9GWK2ZcuWs47deOON3HjjjXGJQRj4WltdSHIUo14LBlt8T663nEpeQlL1OHndcMMNrFu3Drvdzg033NBt2fXr1/c7MEGIF2+rMn5QZ7LGtV/qSIMbV42fvKiXIWKUfdL1+CfpcDiQ2pa6dTjiV/UWhETzeVqA/q+geiZvIEKNF8xSWMxvVEGPk9fpTcV4NRsFIRl8bhcaQBfH/i4Ac/tA1bAsVpZQQZ8Gqfp8PrzeU/9pKisreeqpp3jrrbfiFpggxIs+4karkTBa4zuX1Wo8fQs0UfNKtj4lr+uvv57nnnsOgJaWFqZPn87jjz/O9ddfH7uDKAipYnqRgWllWZTk58X1vBa9rsMUISG5+ryS6qWXXgrAyy+/TGFhIZWVlTz33HP813/9V1wDFIR+C7qRkJDiPJTB3DY5OypDOCBqXsnWp+Tl9XrJyFB+Ed566y1uuOEGNBoNF198MZWVlXENUBD6LdA2NSjOycug04BBmWUS8ov5jcnW5w04NmzYQHV1NW+++SZXXnklAPX19WKNLCGl1Ln87PyimiMN7oQMItUZLWg1kqh5qaBPyeuBBx7g3nvvpaysjIsuuig2r/Ctt95iypQpcQ1QEPqjxRMg5G3FH4rEf4AqcMP0kUwvy8au6/mim0J89GnE3je/+U2++tWvUlNTw6RJk2LHr7jiCr7+9a/HLThB6C9l+WcZg16njIaPM13byhKE/MocR0ms7JUsfR5uXFhYeNYyNNOnT+93QIIQTz63UxnjZcoATQI2iG/fQUiOQjgA+t4vCSX0TZ+Sl8fj4dFHH2Xz5s3U19cTjUY7PC921BZSRcDdjBnQWRMzK+TwyQDBOi8Oo0RJ2CeSVxL1KXktXbqUrVu38p3vfIeioqLYtCFBSDUBjxMzYLBmJeT8nkCYFr8GLSFlcrY5Me8jnK1Pyev111/ntdde45JLLol3PIIQN7IsE/a2AGDOSExSsRq1NGpMhMIBsbJEkvWpEyArK6vbHX4EIRWEIjIOjQ+tRsKSkZmQ97AYdIS1RoKRqEheSdan5PXII4/wwAMPdJjfKAipxqDTcNlQI9PKstCZMxPyHlaDjpDGRFhMEUq6PjUbH3/8cY4cOUJBQQFlZWXo9R13IN69e3dcghOEfvO7kJDAlJjB0+1ThCKysouQ2IgjefqUvObPnx/nMAQhASJhCLZN2zEmJnkZdJrYcImATySvZOpT8nrwwQfjHYcgxN27+78k41gLxdkZ5LWPx0qA2BQhv+hGSaY+j9praWnhj3/8IytXrqSpqQlQmovHjx/v8Tm2bdvGtddeS3FxMZIksWHDhm7Lb9myBUmSznp0tVu2MLi5nE14gxHC+oyEjny/espwppdlk2WIJOw9hLP1qeb1ySefMGfOHBwOB0ePHuWOO+4gOzub9evXU1VVFVvr61w8Hg+TJk3i9ttvP+e6+Kc7ePBghwng+fn5vb4GYeALupsBMNoyE/o+WkPbTpCiwz6p+pS8VqxYweLFi/nVr34VWxoH4JprruGWW27p8Xmuvvpqrr766l6/f35+PpmZmb1+nTB4RKMyIa8LAHOCk1dsVL1IXknVp2bjRx99xF133XXW8ZKSkqQ04SZPnkxRURFf+9rXeO+997otGwgEcLlcHR7CwNfqD2MIt6KRwJSgAartKlxRDtS6qK4/mdD3ETrqU/IyGo2dJoFDhw6RlxffpXZPV1RUxLPPPss//vEP/vGPf1BaWsqsWbO6HZqxatUqHA5H7FFaWpqw+ITU4fSFMITdGHVaJFNid7vyRAw0e0N4vF6Iin6vZOlT8rruuut4+OGHCYVCAEiSRFVVFT/5yU/4xje+EdcATzd69Gjuuusupk6dyowZM1izZg0zZszgySef7PI1K1euxOl0xh7V1dUJi09IHU5fCGPEjUmvSfhO1maLBZAIRWTRdEyiPiWvxx9/HLfbTV5eHj6fj5kzZzJy5EgyMjL4xS9+Ee8YuzV9+nQOHz7c5fNGoxG73d7hMWhFI3B8Nxx9b8D/kWklyNL4MOu1YMpM6HtZjHrCGkPbLkID+/uaSvrUYe9wONi0aRPvvfceH3/8MW63mwsvvJA5c+bEO75z2rt3L0VFRUl/37Qjy3BgI9R9pnzd8DlcuAi0+u5fl6bG5WqhxKYMkUjQANV2FkP7LkIB5JAXscZKcvQ6eUWjUdatW8f69es5evQokiQxfPhwCgsLkWW5V8vjuN3uDrWmiooK9u7dS3Z2NkOHDmXlypUcP348NvTiqaeeYvjw4YwfPx6/388f//hH3n77bbFfZE+0VCmJS5JA0oC7AY7thGHlakeWGP4W5aMxA7R9XnOzR6ztuwiFIej3YEzouwntevVTlWWZ6667jn/+859MmjSJCy64AFmW+fzzz1m8eDHr168/50DT0+3cuZPZs2fHvl6xYgUAixYtYt26ddTU1FBVVRV7PhgM8sMf/pDjx49jsViYOHEi//rXvzqcQ+hC9YcARAonEbAUYjnyBpzYDaUXJWaFURXJsgy+FqUGlOAmI4BOqwGDCYLg94nklSy9Sl7r1q1j27ZtbN68+ayE8fbbbzN//nyee+45Fi5c2KPzzZo1S/lF6+b9TvfjH/+YH//4x70JWQBlffXmCrzBMOuP55Gbm8tVOiP4XdBSCdnD1Y4wrjzBCG/t+IxRHicTCu1JacZpDVZ0GomQmCKUNL36l/vXv/6V+++/v9OazuWXX859993HX/7yl7gFJ8TJycNEI2H2tRhoiNopzsmA/LGx5waaZk8QbcBJOBpFStLKpjPHD2VaWTa5RjFUIll6lbw++eQTrrrqqi6fv/rqq/n444/7HZQQZycP0+wNUq0pwaTXcl6eDbJHKM81Dbz9Bpq9QUxhV1LuNLbTGk7bRUhIil4lr6amJgoKCrp8vqCggObm5n4HJcSRLIPzGHWuAE5TCZOGOLAadXitJVS3+DlRcxx8A+tn1uwNYQy3YtJrIUGLEJ4lNkVINBuTpVfJKxKJoNN13U2m1WoJh8XmmynF7yToddLiC+M25DNhiDLavNol87nHRp3Lj9wysAbutnh8GCOepNa8qlrh81oXR2oak/J+Qh/uNi5evBijsfP7KYFAIC5BCXHkPEaLN4jHkEteZgZ2kzKua3iulc9NhfiddTjrK8ksmqhyoPHjcTYBMkaTEdpXfEgwn6ynxRvCYmhNyvsJvUxeixYtOmeZnt5pFJLEeYxmb4hWYyHDc0/9IRt0GuwFw8D5Mc01lWRO6uYcaSQSlQm2Ks1gU0ZO0nawNpltyvsHxQj7ZOlV8lq7dm2i4hASxV1LkcNEoOB8SgpsHZ7KLR6O5xB4m09AOAi69F/EOBiOMtwWxNqixWhL3h6KZqvyvZWDHqWfUexlmnADa3Si0FE0Cp4G7CY9E8eMJsfWsblfUpBPUGvF7QsSdp5QKcj4Mhu0zCzVMnFIJpIledvzmSxKrTYciSCHRfdJMojkNZD5mpVNKLS6Tjuus60GwtYCojKcrDuW/PgSxdt299ScvORlMZqISlqiMvh94o5jMojkNZC562h0B6iL2PFHzp7JIEkS1uwitBqJQMvA2AfAH4oge9vu+CWx5qXTaZHaNvnweUWnfTKI5DWQues43uJjR72OEy2ddyRPGXs+08qyGDpA7pKt31XJRwcqcPpCYMlJ6ntrDRZ0GomgmCKUFImdbi+oKtJahy8YwWvNJt9u6rSMMbMIkMDTqPSRpfEk7WhUxtNykkgkisFoBIPt3C+Ko2mjitE4I2COJvV9B6v0/U0VzsnXXIMMhC15WA3azguZMpU+sWgY2deUzPDizukLoQ80o5XAaM9L+h0/TWyKkBgukQwieQ1UIR9+txMAc2Zh1+usaTQc9lrYU91MZdXR5MWXACc9AcxhJ2aDDk0S+7ti2gfEtu/SLSSUSF4DlfckvlCEoNZKpr37Ndw9+mz8oSiuhvQeLtHoDmIKtWA2aJPe3wVQ79fyea2LTyrS+/uYLkTyGqi8TfiCEXz6TLKt3Q8+zcgpVl7SXJOMyBLmpDuIKezEYtAm9U5ju4DWQos3RKvLmfT3HoxE8hqo2mpefp2DnHMkL3uOslJIqLWh28UhU12jO4A55MSi1yZ1jFc7i1Wp4Ub8A+PObaoTdxsHKl8T5xdkkF10HnmOzu80tsvKLUYjgS7gxOX147CakxRk/MiyzOhsLQ5jGIvRpEqz0WxVNvqQgx4iURmtRkwRSiRR8xqovE2Y9VpKi4co61p1Q2vKwGA0AzJNDXXJiS/OJEni4oIoY4vsGGzZqszTNNuUJad1ER/eQCjp7z/YqJq8tm3bxrXXXktxcTGSJPVo844tW7Zw4YUXYjQaGTly5Fnr3Aso47XaFxjsSQ1EktDb8wFwnkzjfi9P28h6a+J2be+OZLBh0GnQyBHcXnHHMdFUTV4ej4dJkybxzDPP9Kh8RUUF8+bNY/bs2ezdu5fly5ezdOlS3nzzzQRHmmb8LTS5fRxzBakP9mwvG2tmPlajFmu4JbGxJUijO0DA2TbFyZqrThBaPdq2FVV9bpc6MQwiqvZ5XX311Vx99dU9Lv/ss88yfPhwHn/8cQDGjh3Lu+++y5NPPsncuXMTFWb68TVz0h2gKmDD3uwj33HuPqzhw4ZB+EuwpeeKCK/vq6HgyGdMzwqSac1XLQ6t2YYu4CMSEDWvREurDvsdO3actSv33LlzWb58eZevCQQCHVZ4dbkGwX9E70l8wQh+vYOyc9xpjGlvXnpPJi6uBAmGo5x0BxgeasJitKnWbAQYN7QQTWYEMkVnfaKlVYd9bW3tWRuAFBQU4HK58Pk6n5KxatUqHA5H7FFaWpqMUFUV9SjDJHw9GCYR05a8op6ThNJsH4JGdwB92INZCmPQ6VQZ49VOY2ybTylG2SdcWiWvvli5ciVOpzP2qK4eWJtNdMbvrCcqQ8iQFVuz/pxMmVQ0B/joy3o+q0ivtb1qnH4soSasxrbEpen+7mpCGUTySpa0ajYWFhZSV9fxVn5dXR12ux2zufN+HaPR2OWGIQOV19kAgNGRi6anY400GmRTFlHZjetkDYwqS1yAcXaixYcl1ESGSa9qkxHAFdVzvMaFy1/FRSNUDWXAS6uaV3l5OZs3b+5wbNOmTZSXl6sUUQoKBwh4lH49a2bXe2x2xpypdHR7muvjHlaiyLJMjdOHNdhIhkkHtt5dc9zj0dto8YVwOltUjWMwUDV5ud1u9u7dy969ewFlKMTevXupqqoClCbf6bsRffe73+XLL7/kxz/+MQcOHOC3v/0tf//73/nBD36gRvipqW1OY0hjPueE7DNl5BQCEHLVE46kx5pUTl8ITyBCRqgRm1EHGYWqxmO2KaPsNSEPgXBE1VgGOlWbjTt37mT27Nmxr1esWAEoW6ytW7eOmpqaWCIDGD58OK+99ho/+MEPePrppxkyZAh//OMfxTCJ03lPMiLPSr65GIrtvXqpNTMfnUbCGGqhyRPscgHDVGLSa5k72oHDHUYj6VVPXgaTDZ1GQhfx0+oPY7Sp2P82wKmavGbNmtXtRODORs/PmjWLPXv2JDCqNOdrQiNJ2LPyoaed9W0kax5Wo45AoJl6lz9tktc4mxcyzWDOBL3K8zINVow6DaGQF5c3SK5tcPW3JlNa9XkJPdA+TqsvE5PN2ViMOrTRIE3NabSqamvbyHqVa10AGDMw6rVIchSPW6wukUgieQ0wzpN1HGlwc8TTh4nJWh22zDxyrAaKDal/q7/VH2JXZROuhrbhLxlF6gYEoNGiNSnDJTzuZpWDGdjSaqiEcA6yjLulnvrWAK1uI+f14RS5+cXkajxgTf1pQkcbvWw71Iim+TBT8qTUqHkBeosDXXMzUsitdigDmkheA0mgFb8/gIxERmYfJydbc6Hxi1MrNKSwyiYPuoifPJ0PsIAtNZLX0MJ8huldUNi7Pkehd0SzcSBpm9MY0NnJsfex49qah4yMt6WOVn/qrkkVjcpUNXnJCNTisOiVpKtPjRsMkqntLm9A9HklkkheA4jsbVLmNOod51y3vkuWXI42etn12SE+rmqJa3zxVOvyEwhFyYnUYTPowDFE7ZBOMbaNrxPJK6FE8hpA/K5GwlGZgD6LTHMfmyyWbMxGHbpogCZn6nY4H2lQ+pOG6VqUbd1SLHkdqmvl3/sraPEG1Y5mwBLJawDxtCjTenQZOei0ffzRavWY7cowC/fJ2pTckEOWZQ7Xu9FEwxRrW5SDKZa8fKEIIa8Tly+9VuhIJyJ5DSChVmVCtsnev8nJ1qwCJADvSTzB1Jvi0hoI4w1GcIQbyTJpwWhTdv5OFYYMDDoNhogHVwr3G6Y7cbdxoIiEKTIGyS/LxjdhZL9OpbPlYTZosYSUkfa2PFucgowPu0nPXZeNwHngGNp6Cewl0NWO4GowZmDUadBGA7S63YBD7YgGJFHzGij8LSDLaA1GbBmZ/TuXNRerQYc51Eytyx+P6OJOp9WQE2rbLCRzqLrBnElnwGC0AOBtbVE3lgFMJK+Bwts2ncec3f9aiCUXm0mHOdRCrTO1klcgHFH64SIhcLYtmpg1XN2gOmGwKbUtj6tF3UAGMNFsHCD8rnqOnHAi55cwXpaVO3B9ZcnBYdYzzO5naFFqTSx+50ADtU4fVxR4KI2GlWEJKi773BWzLRMAv6cJub8/D6FTInkNEK1N9bj8YZp9Jib09w9FZ8Cckc0wvRNM3vgEGAeBcITD9a2EIjIWT9t8xuzhqdXf1cZsz0GnkcjR+glFZAy61Isx3YnkNUD4XW13GjPjtO2XLR/8TnDXQ9aw+Jyzn/afcBGKyOTYDGQHjisHU7DJCKCzZPGVsiykAgPoRO9MIojv6gARcClzEa1xS14FhKNR6msqYwNC1RSNyuyuVAbNXligQ/I0KDWurDJ1A+uKKRMJSfkHICSESF4DQdBLwKtMRbFnxy95uXxh9h04xHuH1Z+k/UW9m1Z/GItByxh9+/pdRWCwqBtYV0xtwyN8LaqGMZCJ5DUAyJ4GvMEIAZ2NHHucxmRlFJBhUoZLNLV68QTUGykuyzIfHlXupk4qzUTXdFh5Ivd81WI6J3MmTZ4AH395nLf2Dfzt9tQgktcA4G+pIxyV8emzyerrhOwzGe3oTRZsBg2WYDPHmjvf1DcZqpt8NLYGMOo1TC40QnOl8kTeaNViOiedCVlrxBuM0NqcfruQp4OUSF7PPPMMZWVlmEwmLrroIj788MMuy65btw5Jkjo8TKbUWApFLQFnHQatBo0tF31f5zSeSZLAVojdrMcaOkl1k3p3HYfmWPjm1CHMHp2PyXUU5KiyBE4KDpGIkSSMGVkA+N1NKTlHNN2pnrz+9re/sWLFCh588EF2797NpEmTmDt3LvX1Xe8daLfbqampiT0qKyuTGHHqyZRdTB2WxeVTx8f3xLZ8HGY91mAjVSomL4DSbAtji+xQ/5lyIJVrXW3MGTlIgOR3puQc0XSnevJ64oknuOOOO7jtttsYN24czz77LBaLhTVr1nT5GkmSKCwsjD0KCtTdaFRVsgweZZiE3h7n70NGIRkmHbbQSZy+UNKXd2n1h3D6TpvYHHBDU4XyecGEpMbSF1prFia9BlPYRZNbLI0Tb6omr2AwyK5du5gzZ07smEajYc6cOezYsaPL17ndboYNG0ZpaSnXX389+/fv77JsIBDA5XJ1eAwoQQ+EfEozry87BnUnowidRkORtgVJDie19iXLMm8fqOd/dhzlUF3bon71nylNRntxajcZ25mzMRuUaVaNntTfEyDdqJq8GhsbiUQiZ9WcCgoKqK2t7fQ1o0ePZs2aNbz66qs8//zzRKNRZsyYwbFjxzotv2rVKhwOR+xRWloa9+tQk+xpYG91Cx83gjcS51Hc5iwwWBiWZeI7441cUJK81REO1LbyZYOHqAw5VoNSw6z9RHmyMPVrXQBYcrAYtJjDLZwUNa+4U73Z2Fvl5eUsXLiQyZMnM3PmTNavX09eXh6/+93vOi2/cuVKnE5n7FFdPbBuW3ua6vCFIpwI2TDq4rw7c9sKpTajjpxwfdLm5zm9Id4+oPR5TivLJsdmhJYqcDeAVgf545ISR79ZcrAadGRpfdjEXhxxp+r0oNzcXLRaLXV1dR2O19XVUVjYs51g9Ho9U6ZM4fDhw50+bzQaMRpTa3JxPLkalWkyBkcBWk0Ckot9CDQcApfyPomeZByJyrz+aQ3BcJSSTDMXDW9rHh7fqXwsuED9XbF7ymAl25Gh7CdQJOY2xpuqNS+DwcDUqVPZvHlz7Fg0GmXz5s2Ul5f36ByRSIR9+/ZRVJQCG46qwNekJBVzdnFi3qBteWXfyUre2HeCl3Z23jyPl39/0UCN049Rr2HuhEI0Ggl8zcp2bABDvpLQ94+r0/shvWKsV7yp3mxcsWIFf/jDH/jzn//M559/zve+9z08Hg+33XYbAAsXLmTlypWx8g8//DBvvfUWX375Jbt37+bb3/42lZWVLF26VK1LUE8kTMCl9A3a8xK0hntGIWh06MJ+qo9Vc7zFR32CFig82uhhT9uORVeOK8DRvolI5Xalzyt7hDK+K520Ja+Iu5FAWAyXiCfVV5VYsGABDQ0NPPDAA9TW1jJ58mTeeOONWCd+VVUVGs2pHNvc3Mwdd9xBbW0tWVlZTJ06le3btzNuXJr0g8SR7KnH6w8R1hjJzYnTnMYzabSQWYq+qYLx2kY+CGfxyTEnc8bFf2DwsBwL08qy0WklRua3bR/mbYLaT5XPy74a9/dMOEsOx5q9fFK/n8zoWGaMTLPkm8IkeZAN/XW5XDgcDpxOJ3a7Xe1w+sVd8SH73v47LvMQrljw/b7vGHQu1R/B4X/RZCziz54ZaDUSt11SRoYpMb3QHfrV9r2sNBlzRsLEGxPyfgnVeJja9/6H/U4DzgmLuOHCFNrlKEX19G9U9Waj0HdRVy12kw5jVnHiEhcozTUgO1RPqUNHJCrzYUVTXE590h3grf21hCPR2LFY4mr8QnlIGhgxKy7vl3S2fGxGZaxXQ4tbTBOKI5G80pg9dJLxxQ5mTb0gsW9kyQZzJkQjfDVHGTC677iTGmf/JmvXOH28tOsY+0+4eO/IGR3aQS8celP5vHQ62Pq3nZtqjBlYbHa0kozkbew4Y0DoF5G80lUkpKxyCkqneiJJEuSOAqDQX8HYIjuyTKxzvS8O17fyj13H8AUjFDpMTC87bcS8LMOBjRBoVRJnOvZ1tZMkNBmFWA06bMHGlN2NKR2J5JWmIi3HCEfCydtwNb9t0nfjF8wcYWfK0Ey+Nq73cylDkShbDtbzvx/XEIrIlOVa+MaFQzAb2gbYyjIc/hecPAIaHYz/OmjTfIRnRgFWkw5rqJGaFNuNKZ2pfrdR6Jummi85fLQZOa+QGckY+Z5RqNz2957E3HKIWaMnx54KR6K0+ELk2rofDNzoDrBhz3Fa/crChlOHZXHJyNxTg2tlGY68DcfaBqSOvlpZSz/d2Qqxm3RYmxs5pvLqHAOJSF5pqrVOWV0hklGSnDeUJCia1JZcPlI+b0ua24+cZHdVMyPybJyXZyXPZsSo1xKKRAlFohQ5lBHxNqOOQDhKhknH5WPyGXH6TtwhHxz8pzKaH2DUlekzh/FcMgqwm/SU6N1kFVrEVmhxIpJXOopG8TVWAWAvGpG89y2aBJXvgqcRTh6G3FHIsow7EFYqTfVujtR33KzDYdZz+1eVHX5Mei3/38QiijPNpxZNlGVoOAiHNylL3mi0MPqagZO4AEyZ6C0OxhZoINObklu1pSORvNJQqOUYHo+XiMZASUkSV8nQm6D4Qqh6X6mBZQ1H0uq45oIiLhqezYHaVo41e3H6QgRCUbRaCZNeSzQqK9N8gGE5VuVc0aiSACvfhda2ua2WHBgzDxxJqk0miyRBZinUfQYt1am741GaEckrDTVVfU5EhoB9GDm2JC+BPWwG1O5TRr5XbIWRVwCQYzNyycgeTID3NSuvr/lEuZsISod86XQYWp7+nfNdcSjJK9B4lC8MExlfbBdNx34SySsNOU8o/UKWovOT/wegM8L5c+HT9VD9IRgzYMi07ptC3ialadhwAFpPW6dNb1aaoqUXpe4WZvGSOZSoLLNn/34+aJpKXoaRAvvg3nuhv0TySjNyoBVPg7ImWd6wseoEkTcayi6Bo+/B4c3QfFRpTlpzldHwQbfSL+Y8Bi2VHfculCTIHKYkrdzzlfW5BgNLDhpzJtkmNw7/MQ7X54vk1U+D5Ddn4JDrDjA0x8KJaBbDilQcRlB2KWiN8OUWZUzWySNdl5U0kDVMSXq554PBmrQwU4YkQc5IshvryfJWcahuFDPOyxFNx34QySvNaBo/J8dqJGfkDEjkfMZzkSQYepEy8v7YTmiuOLW1vc6k1MJs+ZA1XFkTTDdwF4TssdyRZFZ/RG5zFV96Ahxr9lGaPcCbywkkklc68TaB87iSOPLHqB2NwpIN51+pfC7LYhhAdzKHoTNaKTJ5yfQf49PjDpG8+kFMD0ojJz7fTnWTF5+tVOkoTzUicXVPo4WCCeTbTeS7D/BFvVtM1O4HkbzShBzy03DwQ461+DigTf0NV4UuFE3EZtRRJp3AFGlN2Kq0g4FoNqaJE59uxevzEjRkMnHsJLXDEfrKlg/ZwxkROsx5xScwFVyodkRpS9S80kDI00zdp1sBsI+9AmuCVjAVkmTYJZj1WkyN+0/NLhB6TSSvVBeNcnjrXwkEgwStxYydOE3tiIT+yixVho3IUTiwkaN1LfzrszqiUbHKam+kRPJ65plnKCsrw2QycdFFF/Hhhx92W/6ll15izJgxmEwmLrjgAv75z38mKdIki0Y5/O+/01LzJRFJz4hLF2AyiJb+gDDqStCbCDprOfD283x67CQb99XgD4kdhnpK9eT1t7/9jRUrVvDggw+ye/duJk2axNy5c6mvr++0/Pbt27n55ptZsmQJe/bsYf78+cyfP59PP/00yZEnmLeJ6Mcv4qvei4yEY+oNlJYMsAnLg5nRBhO+gUGvZ1pGIxc0vMaJY1U8t+Mou6ua8QVFEjsX1XcPuuiii5g2bRr//d//DSibzpaWlvKf//mf3HfffWeVX7BgAR6Ph40bN8aOXXzxxUyePJlnn332nO+XirsHydEowYAXj6sJZ301uYFjZLiPghzFL2v4Mmc2YydOE6OxB6KmL2H/K7R6PBxp8FKjKaTZPAyPMY/MrFyunDwCh8UAgNMbIhSNotdq0GsldBrl40D7vejp36iqbZBgMMiuXbs6bCqr0WiYM2cOO3bs6PQ1O3bsYMWKFR2OzZ07lw0bNnRaPhAIEAgEYl+7XK4ex+f0hvh/Hx+nPbu3p/n2fD+u2MH04dnwyUv43U3srmw6raCMhIzyYpmCDCMjci2ATCgSYefRZqWMLEM4gCyf2j1HthvJyLVB9ghMI69gXLpttCr0XPYI+MoSMo5sZqLhIEWtzdS5avA0R6AWrP4c0BtAa6C23scJpx9QkpXclrQ0koSkkSgfkYtBpzSmjjR4qHX5kdpKtye49jQ3/qql2BzKvgG7Kpv47ETXfxfXTSrBYVFuEu2tbmHfsZYuy159QVFsRd1PjzvZU32q7FXjC8nLiN9MC1WTV2NjI5FIJLbBbLuCggIOHDjQ6Wtqa2s7LV9bW9tp+VWrVvHQQw/1Kb6ILNPoDnb5vK+9f8LXhORpQPK2dH0yvRECbeUjUQh6ADi92ivrTGgyCvDll8HkiwbGEsjCuZkzYcI30HibKKj/nAJnNf6WWjytTnQSEA5COIgh5CFDDhCJynTWty955NiUMcntRuM59U9bPuNjNBqOPecJRLr9PQ9HT/1j9QbD3ZaNnBaYLxShsTXQ6XPxMOB7f1euXNmhpuZyuSgt7dkCfjajjm9cOOSsgePtX2cY24YsjJmHNhTivNFBJCRkqa0qL0lIbf/7jAYtmJTqv1aG8y8IAUoZnd6MxZaBXi+GQAxqlmxltQ7ABJiiEeWfXDQMkRDDoyGGt9X6ZTlKKCITiUYJR6NEIjJaky5WtcoNhMkIRoiiJKwz72RarI7Y5xeUOBie2/Vk+dM3Fx5XZGdIZtdTmjItp8qeX5BBQYap0+fiQdXklZubi1arpa6u41iXuro6Cgs7386rsLCwV+WNRiNGY9+qqgadhqE5PZh75hiCDuhpPUkD5KRGd5uQyjRaMHX+iyIBhm5e2ptfryyrgSxrd2c7JdNiINPSs7IOsx6HOXH/kFW922gwGJg6dSqbN2+OHYtGo2zevJny8vJOX1NeXt6hPMCmTZu6LC8IwsCkerNxxYoVLFq0iK985StMnz6dp556Co/Hw2233QbAwoULKSkpYdWqVQDcc889zJw5k8cff5x58+bx4osvsnPnTn7/+9+reRmCICSZ6slrwYIFNDQ08MADD1BbW8vkyZN54403Yp3yVVVVaDSnKogzZszghRde4Kc//Sn3338/o0aNYsOGDUyYMIB2mxEE4ZxUH+eVbKk4zksQhFN6+jeq+gh7QRCEvhDJSxCEtCSSlyAIaUn1Dvtka+/i6800IUEQkqf9b/Nc3fGDLnm1tiq7NPd0lL0gCOpobW3F4XB0+fygu9sYjUY5ePAg48aNo7q6etDdcWyfHiWuffBce7pdtyzLtLa2Ulxc3GGY1JkGXc1Lo9FQ0rYult1uT4sfZiKIax98155O191djaud6LAXBCEtieQlCEJaGpTJy2g08uCDD/Z5tYl0Jq598F37QL3uQddhLwjCwDAoa16CIKQ/kbwEQUhLInkJgpCWRPISBCEtDZrk1dTUxK233ordbiczM5MlS5bgdru7fc2sWbOQ2jbTaH9897vfTVLEfTeYdyDvzbWvW7furJ+vyWTqsnyq2rZtG9deey3FxcVIktTlNoCn27JlCxdeeCFGo5GRI0eybt26hMcZb4Mmed16663s37+fTZs2sXHjRrZt28add955ztfdcccd1NTUxB6/+tWvkhBt3w3mHch7e+2gjDo//edbWVmZxIjjw+PxMGnSJJ555pkela+oqGDevHnMnj2bvXv3snz5cpYuXcqbb76Z4EjjTB4EPvvsMxmQP/roo9ix119/XZYkST5+/HiXr5s5c6Z8zz33JCHC+Jk+fbq8bNmy2NeRSEQuLi6WV61a1Wn5b33rW/K8efM6HLvooovku+66K6FxJkJvr33t2rWyw+FIUnTJAcivvPJKt2V+/OMfy+PHj+9wbMGCBfLcuXMTGFn8DYqa144dO8jMzOQrX/lK7NicOXPQaDR88MEH3b72L3/5C7m5uUyYMIGVK1fi9XoTHW6fte9APmfOnNixnuxAfnp5UHYg76p8qurLtQO43W6GDRtGaWkp119/Pfv3709GuKoaKD/zQTExu7a2lvz8jrsq6nQ6srOzu9xpG+CWW25h2LBhFBcX88knn/CTn/yEgwcPsn79+kSH3CfJ2IE8VfXl2kePHs2aNWuYOHEiTqeTxx57jBkzZrB//36GDBmSjLBV0dXP3OVy4fP5MJvNKkXWO2mdvO677z5++ctfdlvm888/7/P5T+8Tu+CCCygqKuKKK67gyJEjnHfeeX0+r5AaysvLO+z3OWPGDMaOHcvvfvc7HnnkERUjE3oirZPXD3/4QxYvXtxtmREjRlBYWHhWp204HKapqanLnbY7c9FFFwFw+PDhlExeydiBPFX15drPpNfrmTJlCocPH05EiCmjq5+53W5Pm1oXpPndxry8PMaMGdPtw2AwUF5eTktLC7t27Yq99u233yYajcYSUk/s3bsXgKKionhfSlwM5h3I+3LtZ4pEIuzbty9lf77xMlB+5oPibqMsy/JVV10lT5kyRf7ggw/kd999Vx41apR88803x54/duyYPHr0aPmDDz6QZVmWDx8+LD/88MPyzp075YqKCvnVV1+VR4wYIV922WVqXUKPvPjii7LRaJTXrVsnf/bZZ/Kdd94pZ2ZmyrW1tbIsy/J3vvMd+b777ouVf++992SdTic/9thj8ueffy4/+OCDsl6vl/ft26fWJfRZb6/9oYcekt988035yJEj8q5du+SbbrpJNplM8v79+9W6hD5pbW2V9+zZI+/Zs0cG5CeeeELes2ePXFlZKcuyLN93333yd77znVj5L7/8UrZYLPKPfvQj+fPPP5efeeYZWavVym+88YZal9AngyZ5nTx5Ur755ptlm80m2+12+bbbbpNbW1tjz1dUVMiA/M4778iyLMtVVVXyZZddJmdnZ8tGo1EeOXKk/KMf/Uh2Op0qXUHP/eY3v5GHDh0qGwwGefr06fL7778fe27mzJnyokWLOpT/+9//Lp9//vmywWCQx48fL7/22mtJjjh+enPty5cvj5UtKCiQr7nmGnn37t0qRN0/77zzjgyc9Wi/1kWLFskzZ8486zWTJ0+WDQaDPGLECHnt2rVJj7u/xJI4giCkpbTu8xIEYfASyUsQhLQkkpcgCGlJJC9BENKSSF6CIKQlkbwEQUhLInkJgpCWRPISBCEtieQlDHjr1q0jMzMz9vXPfvYzJk+eHPt68eLFzJ8/P+lxCf0jkpcQN4sXL+5ynf9ly5YhSVKHVUASkTTKysp46qmnOhxbsGABhw4d6vI1Tz/9dIc13GfNmsXy5cvjGpcQfyJ5CXFVWlrKiy++iM/nix3z+/288MILDB06VJWYzGbzWYtRns7hcHSomQnpQSQvIa4uvPBCSktLO6w2u379eoYOHcqUKVP6de7OakTz58+P1eZmzZpFZWUlP/jBD2K7AcHZzcYznV4DXLx4MVu3buXpp5+OnaOiooKRI0fy2GOPdXjd3r17kSRpwK//lapE8hLi7vbbb2ft2rWxr9esWcNtt92W8Pddv349Q4YM4eGHH47tBtRbTz/9NOXl5R12jRo6dOhZ1wSwdu1aLrvsMkaOHBmvSxB6QSQvIe6+/e1v8+6771JZWUllZSXvvfce3/72txP+vtnZ2Wi1WjIyMigsLOzTarAOhwODwYDFYomdQ6vVsnjxYg4ePBjbBzIUCvHCCy9w++23x/syhB5K62WghdSUl5fHvHnzWLduHbIsM2/ePHJzc9UOq1+Ki4uZN28ea9asYfr06fzv//4vgUCAG2+8Ue3QBi1R8xIS4vbbb2fdunX8+c9/jlvtRKPRcObyc6FQKC7n7omlS5fGbkasXbuWBQsWYLFYkvb+QkcieQkJcdVVVxEMBgmFQsydOzcu58zLy+vQjxWJRM7a2dtgMBCJRPr1Pl2d45prrsFqtbJ69WreeOMN0WRUmWg2Cgmh1Wpj285ptdouyzmdztjGJu1ycnIoLS09q+zll1/OihUreO211zjvvPN44oknaGlp6VCmrKyMbdu2cdNNN2E0GvvUXC0rK+ODDz7g6NGj2Gw2srOz0Wg0sb6vlStXMmrUqPTbsGKAETUvIWHsdjt2u73bMlu2bGHKlCkdHg899FCnZW+//XYWLVrEwoULmTlzJiNGjGD27Nkdyjz88MMcPXqU8847j7y8vD7Ffe+996LVahk3bhx5eXlUVVXFnluyZAnBYDApd0+F7ok17AWhF/79739zxRVXUF1dfdau00JyieQlCD0QCARoaGhg0aJFFBYW8pe//EXtkAY90WwUhB7461//yrBhw2hpaeFXv/qV2uEIiJqXIAhpStS8BEFISyJ5CYKQlkTyEgQhLYnkJQhCWhLJSxCEtCSSlyAIaUkkL0EQ0pJIXoIgpKX/Hw3xqWrI/OTXAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:15:29.926636Z", + "iopub.status.busy": "2024-03-22T21:15:29.925854Z", + "iopub.status.idle": "2024-03-22T21:15:30.341699Z", + "shell.execute_reply": "2024-03-22T21:15:30.340762Z" + }, + "papermill": { + "duration": 0.43778, + "end_time": "2024-03-22T21:15:30.343969", + "exception": false, + "start_time": "2024-03-22T21:15:29.906189", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABK9klEQVR4nO29eXxU9b3//zyzz2Qyk30jgYAgsm8VDLaCS0XkWqm91lor4Fbban+l1Cr47cWvem26iJXeWmzrBepV6nVB/FarSFGgAi6sIqtsCUtWksxk9u38/jjJQCAJWWbmzCSf5+NxHsmc8znnvM8k85rP8l4kWZZlBAKBIMXQqG2AQCAQ9AQhXgKBICUR4iUQCFISIV4CgSAlEeIlEAhSEiFeAoEgJRHiJRAIUhIhXgKBICXRqW1AoolEIpw+fZr09HQkSVLbHIFAcB6yLNPc3ExRUREaTcf9q34nXqdPn6akpERtMwQCwUU4ceIExcXFHR7vd+KVnp4OKG+MzWZT2RqBQHA+TqeTkpKS6Ge1I/qdeLUOFW02mxAvgSCJudi0jpiwFwgEKYkQL4FAkJII8RIIBCmJqnNey5YtY9myZRw/fhyAUaNGsXjxYmbOnNlu+5UrV3LXXXe12Wc0GvH5fDG1S5ZlQqEQ4XA4ptcVtEWr1aLT6YTLiqBHqCpexcXF/OpXv2LYsGHIssxf//pXbr75Znbu3MmoUaPaPcdms3Hw4MHo61j/4wcCAaqqqvB4PDG9rqB9LBYLhYWFGAwGtU0RpBiqitdNN93U5vVTTz3FsmXL+PjjjzsUL0mSKCgoiIs9kUiEY8eOodVqKSoqwmAwiF5BnJBlmUAgQF1dHceOHWPYsGGdOiQKBOeTNK4S4XCY1157DbfbTVlZWYftXC4XgwYNIhKJMHHiRH75y192KHQAfr8fv98ffe10OjtsGwgEiEQilJSUYLFYevYggi5jNpvR6/VUVFQQCAQwmUxqm6TgbYTD60FrgGHXgz5J7BK0QfWvuj179mC1WjEajfzgBz/gzTffZOTIke22HT58OMuXL+ett97ipZdeIhKJMHXqVE6ePNnh9cvLy7Hb7dGtK971ogeQOJLuvZZl2LsG6r+Emr1wZL3aFgk6QFK7AEcgEKCyshKHw8Hrr7/OCy+8wMaNGzsUsHMJBoOMGDGC22+/nSeffLLdNu31vEpKSnA4HBc4qfp8Po4dO8bgwYOTpxfQx0m697zpBOx86exrSQNlD4DRqp5N/Qyn04ndbm/3M3ouqg8bDQYDQ4cOBWDSpEl89tlnLF26lD/96U8XPVev1zNhwgQOHz7cYRuj0YjRaIyZvYI+Ts1e5WfBGHDXQXM1nPkSiiaoa5fgApKsz65Mmp/bU+qMcDjMnj17KCwsjLNVgn5D4zHlZ94IyBmm/N5wTD17BB2ias9r0aJFzJw5k4EDB9Lc3MyqVavYsGEDa9euBWDOnDkMGDCA8vJyAJ544gmuuOIKhg4dSlNTE7/97W+pqKjg3nvvVfMxkoLp06czfvx4nn32WbVNSV18TvA2gSSBvRg0LR8P52lVzRK0j6riVVtby5w5c6iqqsJutzN27FjWrl3L17/+dQAqKyvbTOg2NjZy3333UV1dTWZmJpMmTWLLli1dmh8TCC6Ko2Xhx5oPOiOkFypC5m9WhM0kAvmTCrmf4XA4ZEB2OBwXHPN6vfK+fftkr9d7wTF/MNzhFgyFu9w20MW23WHu3Lky0GYbMGCA/Mc//rFNux07dsiSJMnHjx+XZVmWlyxZIo8ePVq2WCxycXGx/MMf/lBubm5uc86//vUv+atf/apsMpnk4uJi+cc//rHscrm6ZV9ndPaeJ5zDH8jyB7+U5YPvnd336V+UfbUH1bOrn9HZZ/RcVJ+wTxWe+7DjRYHBOWnMnjAg+vrPm44QDLe/iFucaebWr5x111i++RjewIVhSD/9+qVdtm3p0qUcOnSI0aNH88QTTwCwZMkSVq1axQ9/+MNou5dffpkrr7ySQYMGAYqbwu9//3sGDx7M0aNH+dGPfsTDDz/MH//4RwCOHDnCDTfcwH/+53+yfPly6urqePDBB3nwwQdZsWJFl+1LGdx1ys+03LP70vLAVQeeeqDrfxNB/Em6CXtB97Hb7RgMBiwWCwUFBRQUFHDHHXewefNmKisrAWUh5JVXXuGOO+6Injd//nyuvvpqSktLueaaa/jP//xPXn311ejx8vJy7rjjDubPn8+wYcOYOnUqv//973nxxRdjHk+aFLhqlZ/WvLP70nKUn+76xNsj6BTR8+oiD1w9tMNjmvMiiL5/1SUdtj0/2ujuKwf3xqwOGT9+PCNGjGDVqlUsXLiQjRs3Ultby6233hpt889//pPy8nIOHDiA0+kkFArh8/nweDxYLBZ2797N559/zssvvxw9R5blaBjViBEj4mK7KgS9ytwWtO15WVrEyyPEK9kQPa8uYtBpOtx0Wk2X2+q72DYW3HHHHaxatQqAVatWccMNN5CdnQ3A8ePH+bd/+zfGjh3LG2+8wfbt23nuuecAxXEYlFCs+++/n127dkW33bt38+WXX3LJJR0LdErSOmQ02ZXJ+lZae16eM4r3vSBpED2vPoLBYLgghc93v/tdfvGLX7B9+3Zef/11nn/++eix7du3E4lEWLJkSXRF99whI8DEiRPZt29f1Im4T+NpUH5astvuN9mV7nI4BAEXGDvPqy5IHKLn1UcoLS3lk08+4fjx49TX1xOJRCgtLWXq1Kncc889hMNhvvGNb0TbDx06lGAwyH/9139x9OhR/ud//qeNuAE88sgjbNmyhQcffJBdu3bx5Zdf8tZbb/Hggw8m+vHij69J+WnOaLtfoz0rWD5HIi0SXAQhXn2Ehx56CK1Wy8iRI8nNzY1O1N9xxx3s3r2bb37zm5jN5mj7cePG8cwzz/DrX/+a0aNH8/LLL0edgVsZO3YsGzdu5NChQ3zta19jwoQJLF68mKKiooQ+W0LwNio/zZkXHjNltLRpSpQ1gi6gemB2ouks6DPpgoT7AUnznm9bDs01MObfz4YFtbL/bajeA4OvgtIr1bGvH9HVwGzR8xIIZPlsr6q1l3UurUPJ1qGlICkQ4iUQBL0QakkGcP6cF5wVNDHnlVQI8RIIWntUxnTQ6i88brK3tBPilUwI8RIIopP1Ge0fb01E6HcJX68kQoiXQNDZfBeAocVVIhJShpiCpECIl0DQGhbUUcobrQ70LW4mAVdibBJcFCFeAoG/paKUsZN8Xa2Oqq1CJ1AdER4kEETFK51AKMIHB2qpc/m5vDSTywps0WO4aoV4JRFCvASCVkEy2th0qI79VYqYvfdFNZkWA/k2ExhaJu3FsDFpEMPGPsL06dOZP3++2mZESTZ7OiQUgGBLbjKTjeEF6QzNs5Ju0iHLsL2iZSVSDBuTDtHzEkQJBAIYDAa1zUgsrWKkM4DOSEkWlGRZqG328fLHlRyudeENhDGf6y4hSApU7XktW7aMsWPHYrPZsNlslJWV8e6773Z6zmuvvcZll12GyWRizJgx/OMf/0iQtcnLvHnz2LhxI0uXLkWSJCRJ4siRI9xzzz0MHjwYs9nM8OHDWbp06QXnzZ49m6eeeoqioiKGDx8OwJYtWxg/fjwmk4mvfOUrrFmzBkmS2LVrV/TcL774gpkzZ2K1WsnPz+fOO++kvr6+Q3uOHz+eqLeje3QwWZ+XbiLHaiAckals8Jw93tpeoDqqildxcTG/+tWv2L59O9u2beOaa67h5ptvZu/eve2237JlC7fffjv33HMPO3fuZPbs2cyePZsvvvgifkbKsjK0UGProkPk0qVLKSsr47777qOqqoqqqiqKi4spLi7mtddeY9++fSxevJhHH330gpxd69ev5+DBg6xbt463334bp9PJTTfdxJgxY9ixYwdPPvkkjzzySJtzmpqauOaaa5gwYQLbtm3jvffeo6amhm9/+9sd2lNSUkJS0iJGfl0aW4+cocpx1o+rNCcNgONn3GBQfifgTriJgvZRddh40003tXn91FNPsWzZMj7++GNGjRp1QfulS5dyww038POf/xyAJ598knXr1vGHP/zhglxUMSMchH8tic+1L8bXfqYMZy7C+TnsW3n88cejvw8ePJitW7fy6quvRkUGIC0tjRdeeCE6XHz++eeRJIm//OUvmEwmRo4cyalTp7jvvvui5/zhD39gwoQJ/PKXv4zuW758OSUlJRw6dIhLL720XXuSkpZhY63fwMdHz3C03sUdU5QCJWMHZDAk10qu1QjhFtEKepUvlfPzeQsSTtLMeYXDYV577TXcbjdlZWXtttm6dSsLFixos2/GjBmsWbOmw+v6/f42Fbidzv7T7X/uuedYvnw5lZWVeL1eAoEA48ePb9NmzJgxbea5Dh48yNixY9ukp5k8eXKbc3bv3s2HH36I1Wq94J5Hjhzh0ktTqMqOT/l/qAsqqZ+LMy3RQ3aLHrulJdZR07JfjkDQc7YnJlAN1cVrz549lJWV4fP5sFqtvPnmmx0Wka2uriY/P7/Nvvz8fKqrqzu8fnl5eZseSLfR6pUekBq0FyTcRV555RUeeughlixZQllZGenp6fz2t7/lk08+adMuLa37H0KXy8VNN93Er3/96wuOFRYW9thmVWjpedX4DCBBob2DnGIaLehNyspkQIhXMqC6eA0fPpxdu3bhcDh4/fXXmTt3Lhs3boxZFexFixa16a05nc7uzb9IUpeGbmpzfg77zZs3M3XqVH70ox9F9x05cuSi1xk+fDgvvfQSfr8fo1HpjXz22Wdt2kycOJE33niD0tJSdLr2/4Xay6mflPidRGSZ2oABjCg+XedwqsnLoZpmcq1GRuvTFPEKuoHc9q8nSBiq+3kZDAaGDh3KpEmTKC8vZ9y4cResirVSUFBATU1Nm301NTWdzqsYjcboambr1hc5P4f9sGHD2LZtG2vXruXQoUP8x3/8xwUi1B7f/e53iUQifP/732f//v2sXbuWp59+GgCpZZ7ngQceoKGhgdtvv53PPvuMI0eOsHbtWu66666oYLWXUz8pCbhwB0J4JTMWgxabqa0Y1zh97Kps4lj9uZP2HhUMFZyP6uJ1PpFIpM0c1bmUlZWxfv36NvvWrVvX4RxZf+L8HPYzZszglltu4bbbbmPKlCmcOXOmTS+sI2w2G3//+9/ZtWsX48eP5//8n//D4sWLAaLzYEVFRWzevJlwOMz111/PmDFjmD9/PhkZGdFKRB3l1E8qwiEI+vD4wwS1FvJsxqhAt5KdpvS6G9wBMLTMe4kVx+RAVpGFCxfKGzdulI8dOyZ//vnn8sKFC2VJkuT3339flmVZvvPOO+WFCxdG22/evFnW6XTy008/Le/fv19+7LHHZL1eL+/Zs6fL93Q4HDIgOxyOC455vV553759stfr7f3D9SFeeuklWa/Xyx6PJ+bXVvU99zbJ8ge/lI+8/h/yM2sPyBsO1l7QxOkNyM+8f1B+dt0hObT/XVn+4JeyfOTDxNvaj+jsM3ouqs551dbWMmfOHKqqqrDb7YwdO5a1a9fy9a9/HYDKysroNznA1KlTWbVqFb/4xS949NFHGTZsGGvWrGH06NFqPUKf5MUXX2TIkCEMGDCA3bt388gjj/Dtb3+7TfWhPkFLD2pQQR53jBnUbrFfq1GHUa/BH4zQLJvIADFsTBJUFa///u//7vT4hg0bLth36623tilZL4g91dXVLF68mOrqagoLC7n11lt56qmn1DYr9rSE+mhNVvJs7a8ySpJEdpqB000+mkL6FvESw8ZkQPXVRkHy8fDDD/Pwww+rbUb8ac0QYbjQX+1cMi1nxQtoWW0UqE3STdgLBAkj4MbtD7GrJsS+0x07L9vNimg1h1tcZsSwMSkQPS9B/yXgwuUP8aVLRl/TzMii9t1oxpVkMLY4A1PICZ+g9LxEiJDqCPFqB1lUiEkYqr7XATe+UJiA1kKOpeNoBpNeq/yiafHzCocgHACdMQFGCjpCDBvPQa9X/oE9HjEsSBSt73Xre59QAi58wQhBjZkMcxfur9WDpuX7Pij+R9RG9LzOQavVkpGRQW1tLQAWi+UCp0VBbJBlGY/HQ21tLRkZGWi12sQb4XfhC4YJWC1kWDoPAdtyuJ46l5+vywYsKM6t9DHPkVRDiNd5tIYatQqYIL5kZGSokzZHlpEDLvxBxbv+Yj2vY2fc1Dr9uHV6LCB6XkmAEK/zkCSJwsJC8vLyCAaDapvTp9Hr9er0uACCXoKhMGEZwjoLtouIl92sp9bpxxUxkKtBFJ9NAoR4dYBWq1XvgyWIPwFlyBjSmLCajWg1nU8P2Ewt86ERvTJTLMRLdYR4CfonARc2k54rLitgzLgBF22e3pJtwh1pdVQVw0a1EauNgv5JS4iP3mK76GQ9nBWv5khL25AvbqYJuoYQL0H/pLWEWRczoqa3DBubQ6LnlSwI8RL0TwJujtW72VUTxOG5+MJMa8/Li0FxrBVzXqoj5rwE/ZOAizNuP19qZQaEL57l1azX8sPpl2B06pA+3yV6XkmAEC9BvyTkayYYlglqzNFeVWdIkqSECelbPFNFz0t1xLBR0C8JeForZVsxtpOEsEOi4uXrclFgQXwQ4iXolwQ8Sskzo8XW5RCwfaed/L/9jdQ2+yASUgoSC1RDiJeg/xEKEPArwz5TWterSTV6AhypD+Bq1Swx76UqQrwE/Y+AC38oQkTSkWa2XLx9C+kmHUgSHrnF10vMe6mKquJVXl7O5ZdfTnp6Onl5ecyePZuDBw92es7KlSuRJKnNdm5peoHgogTcBMMRAloL6V1JhdOC1ahM7HuEl31SoOpq48aNG3nggQe4/PLLCYVCPProo1x//fXs27ev0zL0NputjciJtDWCbhFwMSTHSom1iHBJRpdPi4qXbAC8ouelMqqK13vvvdfm9cqVK8nLy2P79u1cddVVHZ4nSZI6aVQEfYPW0CBzOnp914PvLS3i5ZYNyLIHSYQIqUpSzXk5HA4AsrKyOm3ncrkYNGgQJSUl3Hzzzezdu7fDtn6/H6fT2WYT9HNaqwYZ07t1mkWvRZIgKJkIRmQxbFSZpBGvSCTC/PnzufLKKzstIjt8+HCWL1/OW2+9xUsvvUQkEmHq1KmcPHmy3fbl5eXY7fboVlJSEq9HEKQIAU8zB6qd7KjydyuHvkYjYTFokfVmQuGIGDaqjCQnSbWJH/7wh7z77rt89NFHFBcXd/m8YDDIiBEjuP3223nyyScvOO73+/H7/dHXTqeTkpISHA4HNlvXl8kFfQfHJy+x74tdnC64mm/NmtWtcwOhCPqqbUiH10PeZTDqm3Gysv/idDqx2+0X/YwmRXjQgw8+yNtvv82mTZu6JVygZOOcMGEChw8fbve40WjEaBRVXgRn8bc4qBos3f/yMug0oG9xrxA9L1VRddgoyzIPPvggb775Jh988AGDBw/u9jXC4TB79uyhsLAwDhYK+iLBltAgk6V7c15RRHxjUqBqz+uBBx5g1apVvPXWW6Snp1NdXQ2A3W7HbFb+QebMmcOAAQMoLy8H4IknnuCKK65g6NChNDU18dvf/paKigruvfde1Z5DkEJEIgR9yoS92Wrv9ulH6lwcPuxgTJOXIqMQLzVRVbyWLVsGwPTp09vsX7FiBfPmzQOgsrISjeZsB7GxsZH77ruP6upqMjMzmTRpElu2bGHkyJGJMluQygQ9BIJhQMJi7f6w0eULcaQpQqk/BCEhXmqiqnh1Za1gw4YNbV7/7ne/43e/+12cLBL0eQJuAqEIQY2JXPPF0z+fT5pRR0hjJBCKtFTODirFaAUJJ2lcJQSChBBwEYpECGrN0dTO3SHNqCUsGQiEW754xbyXaiTFaqNAkDACbsaXZBKwD0Jr7f4qdJpRCc72YkBGRgp6wSRcbtRA9LwE/YuW0CCDOf2itRrbI82gfN8HJCOhsCzmvVREiJegf9EiXl2tGnQ+Wo2E2aAlpDERCEeUjKoCVRDDRkG/oubMGRqrnZAuc1kPr5Fm0CLrjIQjHhHfqCJCvAT9Cnezg0ZPkEig5yuE35k8EF3GYKRqlyg+qyJCvAT9iqC3xbu+G+mfz0ev1ZzjZS96Xmoh5rwE/YqQt8W7Pq373vVtiMY3ip6XWgjxEvQfwkFCAWV1MK0H3vWtVJxxs+FYMycaPcLPS0WEeAn6Dy3e9RFJi7WTNOMXwxMIU+GQafaJECE1EeIl6Df4PE5CEVnxru9BaFArFoOWkMZIUCQkVBUhXoJ+g9/drDim6q1KXq4eYjHoCGpMQrxURqw2CvoNdq2fyaVZhLIG9eo6rT2vUFgmEvSikWUQFawSjuh5CfoPLYU3dKYeJiFswazXEtYZkUHJZS98vVRBiJeg/9DL0KBWNBoJk8FIWNKLoaOK9Ei8jh49Gms7BIK4c6iyigPVTqq8Xa/V2BGWaIiQLMRLJXokXkOHDuXqq6/mpZdewucTXWZBauB0KqFBPsnc62vdPnkgXxs5EJtJL4aNKtEj8dqxYwdjx45lwYIFFBQUcP/99/Ppp5/G2jaBIKaEfErVIFMvHFRb0Wk1oBMhQmrSI/EaP348S5cu5fTp0yxfvpyqqiq++tWvMnr0aJ555hnq6upibadA0Csi4Qhyy5xXWnovQ4NaicY3ip6XGvRqwl6n03HLLbfw2muv8etf/5rDhw/z0EMPUVJSwpw5c6iqqur0/PLyci6//HLS09PJy8tj9uzZHDx48KL3fe2117jsssswmUyMGTOGf/zjH715DEE/wON1QySMBFh7UDXofI7Vu/n4hIeTjSItjlr0Sry2bdvGj370IwoLC3nmmWd46KGHOHLkCOvWreP06dPcfPPNnZ6/ceNGHnjgAT7++GPWrVtHMBjk+uuvx+12d3jOli1buP3227nnnnvYuXMns2fPZvbs2XzxxRe9eRRBH8fd3ASAxmBCo+t9wQy3P8SJZmj2h8Scl0pIcldK+JzHM888w4oVKzh48CA33ngj9957LzfeeGObEmUnT56ktLSUUCjU5evW1dWRl5fHxo0bueqqq9ptc9ttt+F2u3n77bej+6644grGjx/P888/f9F7dLWUuKBvcfTLvdRsWoHRlsvEWx/p9fWO1bv5eNN7jPJ8ytgJZTBqdu+NFABd/4z2yMN+2bJl3H333cybN6/DStV5eXn893//d7eu63A4AMjKyuqwzdatW1mwYEGbfTNmzGDNmjXttvf7/fj9/uhrp9PZLZsEfYOwVwkN0pp756DaSjS+MSRcJdSiR+K1bt06Bg4c2KanBUodxhMnTjBw4EAMBgNz587t8jUjkQjz58/nyiuvZPTo0R22q66uJj8/v82+/Pz8aLXt8ykvL+fxxx/vsh2CvsmwDKA0i3DegJhcz9KSxz4YjiAHvYjgoMTTozmvSy65hPr6+gv2NzQ0MHjw4B4Z8sADD/DFF1/wyiuv9Oj8jli0aBEOhyO6nThxIqbXF6QIfiU0SNvL0KBWLAal+KwMBP0dz9EK4kePel4dTZO5XC5MJlO3r/fggw/y9ttvs2nTJoqLizttW1BQQE1NTZt9NTU1FBQUtNveaDRiNHa/Pp+gjxFQfLwwxEa8tBoJjVHJphryeeh5gh1BT+mWeLXONUmSxOLFi7FYLNFj4XCYTz75hPHjx3f5erIs8+Mf/5g333yTDRs2dKnXVlZWxvr165k/f35037p16ygrK+vyfQX9j88OVpLud1JyqZneRTaexWhKQ6uRCAf9EA6BViRpSSTderd37twJKKKzZ88eDIaz3zcGg4Fx48bx0EMPdfl6DzzwAKtWreKtt94iPT09Om9lt9sxmxUHwDlz5jBgwADKy8sB+MlPfsK0adNYsmQJs2bN4pVXXmHbtm38+c9/7s6jCPoRgVAEd3MjoVCQ0hgNGwG+UzYMXTgb5Jbis9rYXVtwcbolXh9++CEAd911F0uXLu21q8GyZcsAmD59epv9K1asYN68eQBUVla2WRiYOnUqq1at4he/+AWPPvoow4YNY82aNZ1O8gv6Ny5fEEPYg1YjYbLEyLse0Om0oDMpq41BLxiFeCWSHvVzV6xYEZObd8XFbMOGDRfsu/XWW7n11ltjYoOg79Pc7ESSIxj1WjBYY3txveWseAkSSpfF65ZbbmHlypXYbDZuueWWTtuuXr2614YJBLHC06z4D+pMaTGdlzpS58JZ5SM34qFYeNknnC7/Je12O1JLqlu7PXZdb4Eg3njdTUDvM6iej8cfpsoDZikk4htVoMvide5QMVbDRoEgEXhdTjSALobzXQDmVkfVkCwyS6hAj5xUvV4vHs/Zb5qKigqeffZZ3n///ZgZJhDECn3YhVYjYUyLbSxrmvHcEmii55VoeiReN998My+++CIATU1NTJ48mSVLlnDzzTdHVxAFgmRhcqGBy0szGZCXG9PrWvS6NiFCgsTS40yqX/va1wB4/fXXKSgooKKighdffJHf//73MTVQIOg1ARcSElKMXRnMLcHZERlCftHzSjQ9Ei+Px0N6uvKP8P7773PLLbeg0Wi44oorqKioiKmBAkGv8beEBsVYvAw6DRiUKJOgT8Q3JpoeF+BYs2YNJ06cYO3atVx//fUA1NbWihxZgqSixulj25cnOFLniosTqc5oQauRRM9LBXokXosXL+ahhx6itLSUKVOmROMK33//fSZMmBBTAwWC3tDk9hP0NOMLhmPvoArcMnkok0uzsOm6nnRTEBt65LH37//+73z1q1+lqqqKcePGRfdfe+21fPOb34yZcQJBb1HSP8sY9DrFGz7G6FoySxD0KTGOksjslSh67G5cUFBwQRqayZMn99oggSCWeF0OxcfLlA6aOBSIb60gJEcg5Ad991NCCXpGj8TL7Xbzq1/9ivXr11NbW0skEmlzXFTUFiQLflcjZkCXFp+okMNn/ARqPNiNEgNCXiFeCaRH4nXvvfeyceNG7rzzTgoLC6NhQwJBsuF3OzADhrTMuFzf7Q/R5NOgJagEZ5vjcx/BhfRIvN59913eeecdrrzyyljbIxDEDFmWCXmaADCnx0dU0oxa6jUmgiG/yCyRYHo0CZCZmdlphR+BIBkIhmXsGi9ajYQlPSMu97AYdIS0RgLhiBCvBNMj8XryySdZvHhxm/hGgSDZMOg0XDXQyOWlmejMGXG5R5pBR1BjIiRChBJOj4aNS5Ys4ciRI+Tn51NaWope37YC8Y4dO2JinEDQa3xOJCQwxcd5ujVEKCwrVYREIY7E0SPxmj17dozNEAjiQDgEgZawHWN8xMug00TdJfxeIV6JpEfi9dhjj8XaDoEg5ny09yjpJ5soykont9UfKw5EQ4R8YholkfTYa6+pqYkXXniBRYsW0dDQACjDxVOnTnX5Gps2beKmm26iqKgISZJYs2ZNp+03bNiAJEkXbB1Vyxb0b5yOBjyBMCF9elw932dOGMzk0iwyDeG43UNwIT3qeX3++edcd9112O12jh8/zn333UdWVharV6+msrIymuvrYrjdbsaNG8fdd9990bz453Lw4ME2AeB5eXndfgZB3yfgagTAaM2I6320hpZKkGLCPqH0SLwWLFjAvHnz+M1vfhNNjQNw44038t3vfrfL15k5cyYzZ87s9v3z8vLIyMjo9nmC/kMkIhP0OAEwx1m8ol71QrwSSo+GjZ999hn333//BfsHDBiQkCHc+PHjKSws5Otf/zqbN2/utK3f78fpdLbZBH2fZl8IQ6gZjQSmODmotnLMGeFAtZMTtWfieh9BW3okXkajsV0ROHToELm5sU21ey6FhYU8//zzvPHGG7zxxhuUlJQwffr0Tl0zysvLsdvt0a2kpCRu9gmSB4c3iCHkwqjTIpniW+3KHTbQ6Ani9nggIua9EkWPxOsb3/gGTzzxBMFgEABJkqisrOSRRx7hW9/6VkwNPJfhw4dz//33M2nSJKZOncry5cuZOnUqv/vd7zo8Z9GiRTgcjuh24sSJuNknSB4c3iDGsAuTXhP3StZmiwWQCIZlMXRMID0SryVLluByucjNzcXr9TJt2jSGDh1Keno6Tz31VKxt7JTJkydz+PDhDo8bjUZsNlubrd8SCcOpHXB8c5//kGklyNR4Meu1YMqI670sRj0hjaGlilDffl+TiR5N2NvtdtatW8fmzZvZvXs3LpeLiRMnct1118Xavouya9cuCgsLE37flEOW4cDbULNPeV23HybOBa2+8/NSlJE5WhhgVVwk4uSg2orF0FpFyI8c9CByrCSGbotXJBJh5cqVrF69muPHjyNJEoMHD6agoABZlruVHsflcrXpNR07doxdu3aRlZXFwIEDWbRoEadOnYq6Xjz77LMMHjyYUaNG4fP5eOGFF/jggw9Evciu0FSpCJckgaQBVx2c3AaDytS2LD74mpSfxnTQ9jjnZpdIa60iFIKAz40xrncTtNKtv6osy3zjG9/gH//4B+PGjWPMmDHIssz+/fuZN28eq1evvqij6bls27aNq6++Ovp6wYIFAMydO5eVK1dSVVVFZWVl9HggEOBnP/sZp06dwmKxMHbsWP75z3+2uYagA058CkC4YBx+SwGWI+/B6R1QMiU+GUZVRJZl8DYpPaA4DxkBdFoNGEwQAJ9XiFei6JZ4rVy5kk2bNrF+/foLBOODDz5g9uzZvPjii8yZM6dL15s+fbryj9bJ/c7l4Ycf5uGHH+6OyQJQ8qs3HsMTCLH6VC45OTncoDOCzwlNFZA1WG0LY4o7EOb9rfsY5nYwusCWkGGc1pCGTiMRFCFCCaNbX7l/+9vfePTRR9vt6VxzzTUsXLiQl19+OWbGCWLEmcNEwiH2NBmoi9goyk6HvBHRY32NRncArd9BKBJBSlBm02mjBnJ5aRY5RuEqkSi6JV6ff/45N9xwQ4fHZ86cye7du3ttlCDGnDlMoyfACc0ATHotl+RaIWuIcqyh79UbaPQEMIWcCVlpbEVrOKeKkCAhdEu8GhoayM/P7/B4fn4+jY2NvTZKEENkGRwnqXH6cZgGMK7YTppRhydtACeafJyuOgXevvU3a/QEMYaaMem1EKckhBcQDRESw8ZE0S3xCofD6HQdT5NptVpCIVF8M6nwOQh4HDR5Q7gMeYwuVrzNTzhl9rut1Dh9yE19y3G3ye3FGHYntOdV2Qz7q50cqapPyP0EPVhtnDdvHkZj++spfr8/JkYJYojjJE2eAG5DDrkZ6dhMil/X4Jw09psK8DlqcNRWkFE4VmVDY4fb0QDIGE1GaM34EGe8sp4mTxCLoTkh9xN0U7zmzp170TZdXWkUJAjHSRo9QZqNBQzOOftBNug02PIHgWM3jVUVZIzr5BopRDgiE2hWhsGm9OyEVbA2ma3K/QPCwz5RdEu8VqxYES87BPHCVU2h3YQ//1IG5FvbHMopGoz7EHgaT0MoALrUT2IcCEUYbA2Q1qTFaE1cDUVzmvLeygG3Ms8oapnGnb7lnShoSyQC7jpsJj1jLxtOtrXtcH9Afh4BbRoub4CQ47RKRsYWs0HLtBItY4szkCyJK89nsii92lA4jBwS0yeJQIhXX8bbqBSh0OranbjOSjMQSssnIsOZmpOJty9eeFpWT82JEy+L0URE0hKRwecVK46JQIhXX8ZVQ73LT03Yhi98YSSDJEmkZRWi1Uj4m/pGHQBfMIzsaVnxS2DPS6fTIrUU+fB6xKR9IhDi1Zdx1XCqycvWWh2nm9qfSJ4w4lIuL81kYB9ZJVu9vYLPDhzD4Q2CJTuh99YaLOg0EgERIpQQ4htuL1CVcHMN3kAYT1oWeTZTu22MGYWABO56ZY4shYO0IxEZd9MZwuEIBqMRDNaLnxRDLh9WhMYRBnMkofftr6Tuf6rgongbq5CBkCWXNIO2/UamDGVOLBJC9jYk0ryY4/AG0fsb0UpgtOUmfMVPEw0REu4SiUCIV18l6MXncgBgzijoOM+aRsNhj4WdJxqpqDyeOPviwBm3H3PIgdmgQ5PA+a4orQ6xrVW6BXFFiFdfxXMGbzBMQJtGhq3zHO5ufRa+YARnXWq7S9S7ApiCTZgN2oTPdwHU+rTsr3by+bHUfh9TBSFefRVPA95AGK8+g6y0zp1P07OLlFMaqxJhWdw44wpgCjmwGLQJXWlsxa+10OQJ0ux0JPze/REhXn2Vlp6XT2cn+yLiZctWMoUEm+s6TQ6Z7NS7/JiDDix6bUJ9vFqxpCk93LCvb6zcJjtitbGv4m3g0vx0sgovIdfe/kpjK5k5RWgk0PkdOD0+7GnmBBkZO2RZZniWFrsxhMVoUmXYaE5TCn3IATfhiIxWI0KE4onoefVVPA2Y9VpKioqVvFadoDWlYzCaAZmGuprE2BdjJEniivwIIwptGKxZqsRpmq1Kymld2IvHH0z4/fsbqorXpk2buOmmmygqKkKSpC4V79iwYQMTJ07EaDQydOjQC/LcC1D8tVoTDHalByJJ6G15ADjOpPC8l7vFsz4tflXbO0MyWDHoNGjkMC6PWHGMN6qKl9vtZty4cTz33HNdan/s2DFmzZrF1Vdfza5du5g/fz733nsva9eujbOlKYaviQaXl5POALWBrtWyScvII82oJS3UFF/b4kS9y4/f0RLilJajjhFaPdqWjKpel1MdG/oRqs55zZw5k5kzZ3a5/fPPP8/gwYNZsmQJACNGjOCjjz7id7/7HTNmzIiXmamHt5EzLj+Vfiu2Ri959ovPYQ0eNAhCR8GamhkR3t1TRf6RfUzODJCRlqeaHVqzFZ3fS9gvel7xJqUm7Ldu3XpBVe4ZM2Ywf/78Ds/x+/1tMrw6nf3gG9FzBm8gjE9vp/QiK41RWoeXnjPxsytOBEIRzrj8DA42YDFaVRs2AowcWIAmIwwZYrI+3qTUhH11dfUFBUDy8/NxOp14ve2HZJSXl2O326NbSUlJIkxVlYhbcZPwdsFNIkqLeEXcZwimWB2CepcffciNWQph0OlU8fFqRWNsiacUXvZxJ6XEqycsWrQIh8MR3U6c6FvFJtrD56glIkPQkBnNWX9RTBkca/Tz2dFa9h1LrdxeVQ4flmADacYW4dJ0vroaVwxCvBJFSg0bCwoKqKlpu5RfU1ODzWbDbG5/XsdoNHZYMKSv4nHUAWC056Dpqq+RRoNsyiQiu3CeqYJhpfEzMMacbvJiCTaQbtKrOmQEcEb0nKpy4vRVMmWIqqb0eVKq51VWVsb69evb7Fu3bh1lZWUqWZSEhPz43cq8XlpGxzU228OcoUx0uxtrY25WvJBlmSqHl7RAPekmHVi798wxt0dvpckbxOFoUtWO/oCq4uVyudi1axe7du0CFFeIXbt2UVlZCShDvnOrEf3gBz/g6NGjPPzwwxw4cIA//vGPvPrqq/z0pz9Vw/zkpCWmMagxXzQg+3zSswsACDprCYVTIyeVwxvE7Q+THqzHatRBeoGq9pitipe9JujGHwqraktfR9Vh47Zt27j66qujrxcsWAAoJdZWrlxJVVVVVMgABg8ezDvvvMNPf/pTli5dSnFxMS+88IJwkzgXzxmG5KaRZy6CIlu3Tk3LyEOnkTAGm2hwBzpMYJhMmPRaZgy3Y3eF0Eh61cXLYLKi00jowj6afSGMVhXn3/o4qorX9OnTOw0Ebs97fvr06ezcuTOOVqU43gY0koQtMw+6OlnfgpSWS5pRh9/fSK3TlzLiNdLqgQwzmDNAr3JcpiENo05DMOjB6QmQY+1f862JJKXmvARdoNVPqyeByeYsLEYd2kiAhsYUyqra3OJZr3KvCwBjOka9FkmO4HaJ7BLxRIhXH8NxpoYjdS6OuHsQmKzVYc3IJTvNQJEh+Zf6m31Btlc04KxrcX9JL1TXIACNFq1JcZdwuxpVNqZvk1KuEoKLIMu4mmqpbfbT7DJySQ8ukZNXRI7GDWnJHyZ0vN7DpkP1aBoPMyFXSo6eF6C32NE1NiIFXWqb0qcR4tWX8Dfj8/mRkUjP6GFwcloO1H95NkNDElPR4EYX9pGr8wIWsCaHeA0syGOQ3gkF3ZtzFHQPMWzsS7TENPp1NrJtPZy4TstFRsbTVEOzL3lzUkUiMpUNHtL91dgtekV09cmxwCCZWlZ5/WLOK54I8epDyJ4GJaZRb79o3voOseRwvN7D9n2H2F3ZFFP7Ykm104c/GCE7XIPVoAN7sdomncXY4l8nxCuuCPHqQ/ic9YQiMn59JhnmHg5ZLFmYjTp0ET8NjuSdcD5Sp8wnDdI1KWXdkky8DtU086+9x2jyBNS2ps8ixKsP4W5Swnp06dnotD3802r1mG2Km4XrTHVSFuSQZZnDtS40kRBF2iZlZ5KJlzcYJuhx4PSmVoaOVEKIVx8i2KwEZJtsvQtOTsvMRwLwnMEdSL4Ql2Z/CE8gjD1UT6ZJC0arUvk7WTCkY9BpMITdOJN43jDVEauNfYVwiEJjgLzSLLyjh/bqUjprLmaDFktQ8bS35lpjZGRssJn03H/VEBwHTqKtlcA2ADqqCK4GxnSMOg3aiJ9mlwuwq21Rn0T0vPoKviaQZbQGI9b0jN5dKy2HNIMOc7CRaqcvFtbFHJ1WQ3awpVhIxkB1jTkfnQGD0QKAp7lJXVv6MEK8+gqelnAec1bveyGWHKwmHeZgE9WO5BIvfyiszMOFg+BoSZqYOVhdo9rBYFV6W25nk7qG9GHEsLGP4HPWcuS0AzlvAKNkWVmB6ymWbOxmPYNsPgYWJldg8YcH6qh2eLk2301JJKS4JaiY9rkjzNYMAHzuBuTe/j0E7SLEq4/Q3FCL0xei0WtidG8/KDoD5vQsBukdYPLExsAY4A+FOVzbTDAsY3G3xDNmDU6u+a4WzLZsdBqJbK2PYFjGoEs+G1MdIV59BJ+zZaUxI0Zlv6x54HOAqxYyB8Xmmr1k72knwbBMttVAlv+UsjMJh4wAOksmXynNRMo3gE7MzsQD8a72EfxOJRYxLWbilU8oEqG2qiLqEKomkYjMjgrFaXZivg7JXaf0uDJL1TWsI0wZSEjKF4AgLgjx6gsEPPg9SiiKLSt24uX0hthz4BCbD6sfpP1lrYtmXwiLQctl+tb8XYVgsKhrWEeYWtwjvE2qmtGXEeLVB5DddXgCYfw6K9m2GPlkpeeTblLcJRqaPbj96nmKy7LMp8eV1dRxJRnoGg4rB3IuVc2mi2LOoMHtZ/fRU7y/p++X21MDIV59AF9TDaGIjFefRWZPA7LPx2hDb7JgNWiwBBo52dh+Ud9EcKLBS32zH6New/gCIzRWKAdyh6tm00XRmZC1RjyBMM2NqVeFPBVICvF67rnnKC0txWQyMWXKFD799NMO265cuRJJktpsJlNypEJRC7+jBoNWg8aag76nMY3nI0lgLcBm1pMWPMOJBvVWHQdmW/j3ScVcPTwPk/M4yBElBU4SukhEkSSM6ZkA+FwNSRkjmuqoLl7/+7//y4IFC3jsscfYsWMH48aNY8aMGdTWdlw70GazUVVVFd0qKioSaHHykSE7mTQok2smjYrtha152M160gL1VKooXgAlWRZGFNqgdp+yI5l7XS2Y07ORAMnnSMoY0VRHdfF65plnuO+++7jrrrsYOXIkzz//PBaLheXLl3d4jiRJFBQURLf8fHULjaqKLINbcZPQ22L8PqQXkG7SYQ2eweENJjy9S7MviMN7TmCz3wUNx5Tf80cn1JaeoE3LxKTXYAo5aXCJ1DixRlXxCgQCbN++neuuuy66T6PRcN1117F169YOz3O5XAwaNIiSkhJuvvlm9u7d22Fbv9+P0+lss/UpAm4IepVhXk8qBnVGeiE6jYZCbROSHEpo70uWZT44UMv/bD3OoZqWpH61+5Qho60ouYeMrZizMBuUMKt6d/LXBEg1VBWv+vp6wuHwBT2n/Px8qqur2z1n+PDhLF++nLfeeouXXnqJSCTC1KlTOXnyZLvty8vLsdvt0a2kpCTmz6EmsruOXSea2F0PnnCMvbjNmWCwMCjTxJ2jjIwZkLjsCAeqmzla5yYiQ3aaQelhVn+uHCxI/l4XAJZsLAYt5lATZ0TPK+aoPmzsLmVlZcyZM4fx48czbdo0Vq9eTW5uLn/605/abb9o0SIcDkd0O3Giby1buxtq8AbDnA5aMepiXJ25JUOp1agjO1SbsPg8hyfIBweUOc/LS7PIthqhqRJcdaDVQd7IhNjRayzZpBl0ZGq9WEUtjpijanhQTk4OWq2WmpqaNvtramooKOhaJRi9Xs+ECRM4fPhwu8eNRiNGY3IFF8cSZ70SJmOw56PVxEFcbMVQdwicyn3iHWQcjsi8+0UVgVCEARlmpgxuGR6e2qb8zB+jflXsrmJII8uertQTKBSxjbFG1Z6XwWBg0qRJrF+/ProvEomwfv16ysrKunSNcDjMnj17KCxMgoKjKuBtUETFnFUUnxu0pFf2nqngvT2neW1b+8PzWPGvL+uocvgw6jXMGF2ARiOBt1EpxwZQ/JW43j+mnDsP6RG+XrFG9WHjggUL+Mtf/sJf//pX9u/fzw9/+EPcbjd33XUXAHPmzGHRokXR9k888QTvv/8+R48eZceOHXzve9+joqKCe++9V61HUI9wCL9TmRu05cYph3t6AWh06EI+Tpw8wakmL7VxSlB4vN7NzpaKRdePzMfeWkSkYosy55U1RPHvSiVaxCvsqscfEu4SsUT1rBK33XYbdXV1LF68mOrqasaPH897770XncSvrKxEozmrsY2Njdx3331UV1eTmZnJpEmT2LJlCyNHpsg8SAyR3bV4fEFCGiM52TGKaTwfjRYyStA3HGOUtp5PQpl8ftLBdSNj7xg8KNvC5aVZ6LQSQ/Nayod5GqD6C+X30q/G/J5xx5LNyUYPn9fuJSMygqlDU0x8kxhJ7meuv06nE7vdjsPhwGazqW1Or3Ad+5Q9H7yK01zMtbf9fz2vGHQxTnwGh/9Jg7GQv7qnotVI3HVlKemm+MxCt5lX2/O6MmTMHgpjb43L/eJK/WGqN/8Pex0GHKPncsvEJKpylKR09TOq+rBR0HMizmpsJh3GzKL4CRcowzUgK1hLiV1HOCLz6bGGmFz6jMvP+3urCYUj0X1R4ar/UtkkDQyZHpP7JRxrHlaj4utV1+QSYUIxRIhXCmMLnmFUkZ3pk8bE90aWLDBnQCTMV7MVh9E9pxxUOXoXrF3l8PLa9pPsPe1k85HzJrQDHji0Vvm9ZDJYe1fOTTWM6VisNrSSjOSpbxsxIOgVQrxSlXBQyXIKyqR6PJEkyBkGQIHvGCMKbcgy0cn1nnC4tpk3tp/EGwhTYDcxufQcj3lZhgNvg79ZEc5UnOtqRZLQpBeQZtBhDdQnbTWmVESIV4oSbjpJKBxKXMHVvJag7/ovmTbExoSBGXx9ZPdjKYPhCBsO1vL33VUEwzKlORa+NbEYs6HFwVaW4fA/4cwR0Ohg1DdBm+Ienun5pJl0pAXrqUqyakypjOqrjYKe0VB1lMPHG5FzC5iaCM/39AJl2d9zBnPTIaYPHx89FApHaPIGybF27gxc7/KzZucpmn1KYsNJgzK5cmjOWedaWYYjH8DJFofU4TOVXPqpjrUAm0lHWmM9J1XOztGXEOKVojTXKNkVwukDEnNDSYLCcS3i8pnye4tobjlyhh2VjQzJtXJJbhq5ViNGvZZgOEIwHKHQrnjEW406/KEI6SYd11yWx5BzK3EHvXDwH4o3P8Cw61MnhvFipOdjM+kZoHeRWWARpdBihBCvVCQSwVtfCYCtcEji7ls4Dio+Anc9nDkMOcOQZRmXP6R0mmpdHKltW6zDbtZz91eVCj8mvZZ/G1tIUYb5bNJEWYa6g3B4nZLyRqOF4Tf2HeECMGWgt9gZka+BDE9SlmpLRYR4pSDBppO43R7CGgMDBiQwS4beBEUTofJjpQeWORhJq+PGMYVMGZzFgepmTjZ6cHiD+IMRtFoJk15LJCIrYT7AoOw05VqRiCKAFR9Bc0tsqyUbLpsF9gT1JhOFJEFGCdTsg6YTyVvxKMUQ4pWCNFTuJyyD3zaIbGuCU2APmgrVexTP92MbYei1AGRbjVw5tAsB8N5G5fyqz5XVRFAm5Esmw8Cy1J+c7wi7Il7++uN8aRjLqCKbGDr2EiFeKYjjtDIvZCm8NPEfAJ0RLp0BX6yGE5+CMR2KL+98KORpUIaGdQeg+Zw8bXqzMhQtmZK8JcxiRcZAIrLMzr17+aRhErnpRvJt/bv2Qm8R4pViyP5m3HVKTrLcQSPUMSJ3OJReCcc3w+H10HhcGU6m5Sje8AGXMi/mOAlNFW1rF0oSZAxSRCvnUiU/V3/Ako3GnEGWyYXdd5LDtXlCvHpJP/nP6TvINQcYmG3hdCSTQYUquhGUfg20Rji6QfHJOnOk47aSBjIHKaKXcykY0hJmZtIgSZA9lKz6WjI9lRyqGcbUS7LF0LEXCPFKMTT1+8lOM5I9dCrEM57xYkgSDJyieN6f3AaNx86WtteZlF6YNQ8yBys5wXR9NyFkl8kZSsaJz8hprOSo28/JRi8lWX18uBxHhHilEp4GcJxShCPvMrWtUbBkwaXXK7/LsnAD6IyMQeiMaRSaPGT4TvLFKbsQr14gwoNSiNP7t3CiwYPXWqJMlCcbQrg6R6OF/NHk2UzkuQ7wZa1LBGr3AiFeKYIc9FF38FNONnk5oE3+gquCDigci9Woo1Q6jSncHLestP0BMWxMEU5/sRGP10PAkMHYEePUNkfQU6x5kDWYIcHDXFJ0GlP+RLUtSllEzysFCLobqfliIwC2EdeSFqcMpoIEMehKzHotpvq9Z6MLBN1GiFeyE4lweOPf8AcCBNKKGDH2crUtEvSWjBLFbUSOwIG3OV7TxD/31RCJiCyr3SEpxOu5556jtLQUk8nElClT+PTTTztt/9prr3HZZZdhMpkYM2YM//jHPxJkaYKJRDj8r1dpqjpKWNIz5Gu3YTKIkX6fYNj1oDcRcFRz4IOX+OLkGd7eU4UvKCoMdRXVxet///d/WbBgAY899hg7duxg3LhxzJgxg9ra2nbbb9myhdtvv5177rmHnTt3Mnv2bGbPns0XX3yRYMvjjKeByO5X8J7YhYyEfdItlAzoYwHL/RmjFUZ/C4Nez+Xp9Yype4fTJyt5cetxdlQ24g0IEbsYqlcPmjJlCpdffjl/+MMfAKXobElJCT/+8Y9ZuHDhBe1vu+023G43b7/9dnTfFVdcwfjx43n++ecver9krB4kRyIE/B7czgYctSfI8Z8k3XUc5Ag+WcPR7KsZMfZy4Y3dF2k4CnvfpNnt5kidhypNAY3mQbiNuWRk5nD9+CHYLQYAHJ4gwUgEvVaDXiuh0yg/+9r/RVc/o6qOQQKBANu3b29TVFaj0XDdddexdevWds/ZunUrCxYsaLNvxowZrFmzpt32fr8fv98ffe10Ortsn8MT5P/tPkWrurfKfKvejyyyM3lwFnz+Gj5XAzsqGs5pKCMho5wsk59uZEiOBZAJhsNsO96otJFlCPmR5bPVc2SbkfQcK2QNwTT0WkamWqFVQdfJGgJfuYf0I+sZazhIYXMjNc4q3I1hqIY0XzboDaA1UF3r5bTDByhiJbeIlkaSkDQSZUNyMOiUwdSROjfVTh9SS+tWgWuVuVE33IvVrtQN2F7RwL7THX8uvjFuAHaLski060QTe042ddh25pjCaEbdL0452HnibNsbRhWQmx67SAtVxau+vp5wOBwtMNtKfn4+Bw4caPec6urqdttXV1e32768vJzHH3+8R/aFZZl6V6DD497W+QlvA5K7DsnT1PHF9Ebwt7QPRyDgBuDcbq+sM6FJz8ebVwrjp/SNFMiCi2POgNHfQuNpIL92P/mOE/iaqnE3O9BJQCgAoQCGoJt02U84ItPe3L7klqMhY5LLhcZ99ktbPu9nJBKKHnP7w53+n4ciZ79YPYFQp23D5xjmDYapb/a3eywW9PnZ30WLFrXpqTmdTkpKupbAz2rU8a2JxRc4jre+Tje2uCxcNgttMMglwwNISMhSS1dekpBavvuMBi2YlO6/VoZLxwQBpY1Ob8ZiTUevFy4Q/RpLlpKtAzABpkhY+ZKLhCAcZHAkyOCWXr8sRwiGZcKRCKFIhHBYRmvSRbtWOf4Q6YEwERTBOn8l05Jmj/4+ZoCdwTkdB8ufW1x4ZKGN4oyOQ5oyLGfbXpqfTn66qd1jsUBV8crJyUGr1VJT09bXpaamhoKC9st5FRQUdKu90WjEaOxZV9Wg0zAwuwuxZ/ZidEBX+0kaIDs5ptsEyYxGC6b2/1EkwNDJqd3598pMM5CZ1tnVzpJhMZBh6Vpbu1mP3Ry/L2RVVxsNBgOTJk1i/fr10X2RSIT169dTVlbW7jllZWVt2gOsW7euw/YCgaBvovqwccGCBcydO5evfOUrTJ48mWeffRa3281dd90FwJw5cxgwYADl5eUA/OQnP2HatGksWbKEWbNm8corr7Bt2zb+/Oc/q/kYAoEgwaguXrfddht1dXUsXryY6upqxo8fz3vvvRedlK+srESjOdtBnDp1KqtWreIXv/gFjz76KMOGDWPNmjWMHt2Hqs0IBIKLorqfV6JJRj8vgUBwlq5+RlX3sBcIBIKeIMRLIBCkJEK8BAJBSqL6hH2iaZ3i606YkEAgSBytn82LTcf3O/FqblaqNHfVy14gEKhDc3Mzdru9w+P9brUxEolw8OBBRo4cyYkTJ/rdimNreJR49v7z7Kn23LIs09zcTFFRURs3qfPpdz0vjUbDgJa8WDabLSX+mPFAPHv/e/ZUeu7OelytiAl7gUCQkgjxEggEKUm/FC+j0chjjz3W42wTqYx49v737H31ufvdhL1AIOgb9Muel0AgSH2EeAkEgpREiJdAIEhJhHgJBIKUpN+IV0NDA3fccQc2m42MjAzuueceXC5Xp+dMnz4dqaWYRuv2gx/8IEEW95z+XIG8O8++cuXKC/6+JpOpw/bJyqZNm7jpppsoKipCkqQOywCey4YNG5g4cSJGo5GhQ4eycuXKuNsZa/qNeN1xxx3s3buXdevW8fbbb7Np0ya+//3vX/S8++67j6qqquj2m9/8JgHW9pz+XIG8u88Oitf5uX/fioqKBFocG9xuN+PGjeO5557rUvtjx44xa9Ysrr76anbt2sX8+fO59957Wbt2bZwtjTFyP2Dfvn0yIH/22WfRfe+++64sSZJ86tSpDs+bNm2a/JOf/CQBFsaOyZMnyw888ED0dTgclouKiuTy8vJ223/729+WZ82a1WbflClT5Pvvvz+udsaD7j77ihUrZLvdniDrEgMgv/nmm522efjhh+VRo0a12XfbbbfJM2bMiKNlsadf9Ly2bt1KRkYGX/nKV6L7rrvuOjQaDZ988kmn57788svk5OQwevRoFi1ahMfjibe5Paa1Avl1110X3deVCuTntgelAnlH7ZOVnjw7gMvlYtCgQZSUlHDzzTezd+/eRJirKn3lb94vArOrq6vJy2tbVVGn05GVldVhpW2A7373uwwaNIiioiI+//xzHnnkEQ4ePMjq1avjbXKPSEQF8mSlJ88+fPhwli9fztixY3E4HDz99NNMnTqVvXv3UlxcnAizVaGjv7nT6cTr9WI2m1WyrHuktHgtXLiQX//615222b9/f4+vf+6c2JgxYygsLOTaa6/lyJEjXHLJJT2+riA5KCsra1Pvc+rUqYwYMYI//elPPPnkkypaJugKKS1eP/vZz5g3b16nbYYMGUJBQcEFk7ahUIiGhoYOK223x5QpUwA4fPhwUopXIiqQJys9efbz0ev1TJgwgcOHD8fDxKSho7+5zWZLmV4XpPhqY25uLpdddlmnm8FgoKysjKamJrZv3x4994MPPiASiUQFqSvs2rULgMLCwlg/SkzozxXIe/Ls5xMOh9mzZ0/S/n1jRV/5m/eL1UZZluUbbrhBnjBhgvzJJ5/IH330kTxs2DD59ttvjx4/efKkPHz4cPmTTz6RZVmWDx8+LD/xxBPytm3b5GPHjslvvfWWPGTIEPmqq65S6xG6xCuvvCIbjUZ55cqV8r59++Tvf//7ckZGhlxdXS3Lsizfeeed8sKFC6PtN2/eLOt0Ovnpp5+W9+/fLz/22GOyXq+X9+zZo9Yj9JjuPvvjjz8ur127Vj5y5Ii8fft2+Tvf+Y5sMpnkvXv3qvUIPaK5uVneuXOnvHPnThmQn3nmGXnnzp1yRUWFLMuyvHDhQvnOO++Mtj969KhssVjkn//85/L+/fvl5557TtZqtfJ7772n1iP0iH4jXmfOnJFvv/122Wq1yjabTb7rrrvk5ubm6PFjx47JgPzhhx/KsizLlZWV8lVXXSVnZWXJRqNRHjp0qPzzn/9cdjgcKj1B1/mv//oveeDAgbLBYJAnT54sf/zxx9Fj06ZNk+fOndum/auvvipfeumlssFgkEeNGiW/8847CbY4dnTn2efPnx9tm5+fL994443yjh07VLC6d3z44YcycMHW+qxz586Vp02bdsE548ePlw0GgzxkyBB5xYoVCbe7t4iUOAKBICVJ6TkvgUDQfxHiJRAIUhIhXgKBICUR4iUQCFISIV4CgSAlEeIlEAhSEiFeAoEgJRHiJejzrFy5koyMjOjr//t//y/jx4+Pvp43bx6zZ89OuF2C3iHESxAz5s2b12Gq7AceeABJktoE0sdDNEpLS3n22Wfb7Lvttts4dOhQh+csXbq0TRrk6dOnM3/+/JjaJYg9QrwEMaWkpIRXXnkFr9cb3efz+Vi1ahUDBw5UxSaz2XxBPrdzsdvtbXpmgtRAiJcgpkycOJGSkpI2CRtXr17NwIEDmTBhQq+u3V6PaPbs2dHe3PTp06moqOCnP/1ptKAGXDhsPJ9ze4Dz5s1j48aNLF26NHqNY8eOMXToUJ5++uk25+3atQtJkvp8Cp1kRYiXIObcfffdrFixIvp6+fLl3HXXXXG/7+rVqykuLuaJJ56IFtToLkuXLqWsrKxN4ZWBAwde8EwAK1as4KqrrmLo0KGxegRBNxDiJYg53/ve9/joo4+oqKigoqKCzZs3873vfS/u983KykKr1ZKenk5BQUGPEira7XYMBgMWiyV6Da1Wy7x58zh48GC0lFowGGTVqlXcfffdsX4MQRdJ6UyqguQkNzeXWbNmsXLlSmRZZtasWeTk5KhtVq8oKipi1qxZLF++nMmTJ/P3v/8dv9/PrbfeqrZp/RbR8xLEhbvvvpuVK1fy17/+NWa9E41Gw/kZnILBYEyu3RXuvffe6GLEihUruO2227BYLAm7v6AtQrwEceGGG24gEAgQDAaZMWNGTK6Zm5vbZh4rHA5fUBzXYDAQDod7dZ+OrnHjjTeSlpbGsmXLeO+998SQUWXEsFEQF7RabbRyk1ar7bCdw+GI1gZoJTs7m5KSkgvaXnPNNSxYsIB33nmHSy65hGeeeYampqY2bUpLS9m0aRPf+c53MBqNPRqulpaW8sknn3D8+HGsVitZWVloNJro3NeiRYsYNmxY6uV872OInpcgbthsNmw2W6dtNmzYwIQJE9psjz/+eLtt7777bubOncucOXOYNm0aQ4YM4eqrr27T5oknnuD48eNccskl5Obm9sjuhx56CK1Wy8iRI8nNzaWysjJ67J577iEQCCRk9VTQOSINtEDQDf71r39x7bXXcuLEiQsKtwoSixAvgaAL+P1+6urqmDt3LgUFBbz88stqm9TvEcNGgaAL/O1vf2PQoEE0NTXxm9/8Rm1zBIiel0AgSFFEz0sgEKQkQrwEAkFKIsRLIBCkJEK8BAJBSiLESyAQpCRCvAQCQUoixEsgEKQkQrwEAkFKIsRLIBCkJP8/wqZFauCUJqMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:15:30.383600Z", + "iopub.status.busy": "2024-03-22T21:15:30.383265Z", + "iopub.status.idle": "2024-03-22T21:15:30.546874Z", + "shell.execute_reply": "2024-03-22T21:15:30.545909Z" + }, + "papermill": { + "duration": 0.185637, + "end_time": "2024-03-22T21:15:30.549081", + "exception": false, + "start_time": "2024-03-22T21:15:30.363444", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkJ0lEQVR4nO3de1xUdf4/8BfDZbjjhYuI2JimWCooLorZ2kXAtUzXTf1pCaKrG8quOZFKq7Cum9SmiBXKIxPNymQrM3c1lVBb85IK6qIJBIKYAoqkCOgwMJ/vH/2YGhlgMOBwZl7Px6MHzWc+Z+Z9+MDLcz6c+RwrIYQAEZHMKKQugIjofjC8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIlmykbqAjqbT6XD16lW4uLjAyspK6nKI6B5CCNy+fRs9e/aEQtH08ZXFhdfVq1fh6+srdRlE1ILLly+jV69eTT5vceHl4uIC4KdvjKurq8TVtB+tVov9+/cjNDQUtra2UpdDbcBSxrSyshK+vr7639WmWFx4NZwqurq6mn14OTo6wtXV1ax/0C2JpY1pS9M6nLAnIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEcmSxV0qYa5qamqQk5Ojf1x1R4Oj2QXo6n4Kzg5Kg75+fn5wdHTs6BKJ2hTDy0zk5OQgMDCwUfs/jfTNzMzEsGHD2r8oonbE8DITfn5+yMzM1D/OLbkJ9SfZSJwyGAO8uzTqSyR3DC8z4ejoaHA0pbh0A8rDdzBwkD8CHuguYWVE7YMT9kQkSwwvIpIlhhcRyZLk4ZWcnAyVSgV7e3uMGDECJ06caLZ/UlISBgwYAAcHB/j6+mLRokW4e/duB1VLRJ2FpOGVlpYGtVqN+Ph4ZGVlwd/fH2FhYbh27ZrR/tu2bcPSpUsRHx+PCxcuYNOmTUhLS8Orr77awZUTkdQkDa/ExETMnTsXkZGRePjhh5GSkgJHR0ekpqYa7X/06FE8+uijmDFjBlQqFUJDQzF9+vQWj9aIyPxIdqlEbW0tMjMzERsbq29TKBQYO3Ysjh07ZnSbUaNG4cMPP8SJEycQFBSEixcvYs+ePZg5c2aT76PRaKDRaPSPKysrAfy0sJtWq22jvel86urq9F/NeT8tScM4mvt4mrp/koVXeXk56uvr4eXlZdDu5eVl8DGXX5oxYwbKy8sxevRoCCFQV1eHF198sdnTxoSEBKxYsaJR+/79+836IzKXqwDABsePH8eVc1JXQ20pPT1d6hLaVU1NjUn9ZHWR6qFDh7Bq1SqsX78eI0aMQH5+PhYuXIiVK1di+fLlRreJjY2FWq3WP25YHzs0NNSsl4E+W1wBZJ/CyJEj4d+7m9TlUBvQarVIT09HSEiIWS8D3XB21BLJwsvd3R3W1tYoKyszaC8rK0OPHj2MbrN8+XLMnDkTf/zjHwEAgwcPRnV1NebNm4e//vWvRm+TpFQqoVQqG7Xb2tqa9Q+AjY2N/qs576clMvefXVP3TbIJezs7OwQGBiIjI0PfptPpkJGRgeDgYKPb1NTUNAooa2trAD/d642ILIekp41qtRoREREYPnw4goKCkJSUhOrqakRGRgIAwsPD4ePjg4SEBADAhAkTkJiYiKFDh+pPG5cvX44JEyboQ4yILIOk4TVt2jRcv34dcXFxKC0tRUBAAPbu3aufxC8uLjY40lq2bBmsrKywbNkyXLlyBR4eHpgwYQJee+01qXaBiCRiJSzsfKuyshJubm64deuWWU/Yn7l0A5M2HMfOqJFcVcJMaLVa7NmzB+PHjzfrOS9Tf0cl/3gQEdH9YHgRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJkqT3bSQi42pqapCTk2PQVnVHg6PZBejqfgrODkp9u5+fHxwdHTu6RMkxvIg6oZycHAQGBhp97p/3PM7MzMSwYcPav6hOhuFF1An5+fkhMzPToC235CbUn2QjccpgDPDuYtDXEjG8iDohR0fHRkdTiks3oDx8BwMH+fMu6OCEPRHJFMOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSxJHl7JyclQqVSwt7fHiBEjcOLEiWb737x5EwsWLIC3tzeUSiX69++PPXv2dFC1RNRZSLokTlpaGtRqNVJSUjBixAgkJSUhLCwMubm58PT0bNS/trYWISEh8PT0xKeffgofHx9cunQJXbp06fjiiUhSkoZXYmIi5s6di8jISABASkoKdu/ejdTUVCxdurRR/9TUVFRUVODo0aOwtbUFAKhUqo4smajdFJZXo1pT1+TzBder9V9tbIz/6jopbdDH3ald6utsJAuv2tpaZGZmIjY2Vt+mUCgwduxYHDt2zOg2u3btQnBwMBYsWIAvvvgCHh4emDFjBpYsWQJra2uj22g0Gmg0Gv3jyspKAIBWq4VWq23DPep4RTeqUa2pN/pcXuktg69NcVJaQ9XdMn7YO7OiG9UISTpiUt+XP81u9vn0lx6V9Zia+nspWXiVl5ejvr4eXl5eBu1eXl6NbjzQ4OLFizhw4ACef/557NmzB/n5+Zg/fz60Wi3i4+ONbpOQkIAVK1Y0at+/f7+sb1pw7Q7w2pmWh2/x5xda7PPXgDp4OrRFVXS/LlcBgA1m9quHl4Mw2kerAyo0QDclYGtktrrsjhU+yLfGvoyv4evcruW2q5qaGpP6yWoZaJ1OB09PT7z77ruwtrZGYGAgrly5gjfffLPJ8IqNjYVardY/rqyshK+vL0JDQ+Hq6tpRpbe581crgTPHsfq5wejn0fhf2eq7Guw9fBLjHvsNnOyVRl4ByL9ejZhPs/Gb4NF4pKd8vxfm4PzVSqzOPo4/hDza5FhotVqkp6cjJCREP21y72t8kH8co0fLezwbzo5aIll4ubu7w9raGmVlZQbtZWVl6NGjh9FtvL29YWtra3CKOHDgQJSWlqK2thZ2dnaNtlEqlVAqG//y2traGv0BkIuGOQ8/bzcM8nFr9LxWq0V5DhD0oEeT+9nwGjY2NrL+XpiD1oxFUz+75jKeptYu2aUSdnZ2CAwMREZGhr5Np9MhIyMDwcHBRrd59NFHkZ+fD51Op2/Ly8uDt7e30eAiIvMl6XVearUaGzduxPvvv48LFy4gKioK1dXV+r8+hoeHG0zoR0VFoaKiAgsXLkReXh52796NVatWYcGCBVLtAhFJRNI5r2nTpuH69euIi4tDaWkpAgICsHfvXv0kfnFxMRSKn/PV19cX+/btw6JFizBkyBD4+Phg4cKFWLJkiVS7QEQSkXzCPjo6GtHR0UafO3ToUKO24OBgHD9+vJ2rIqLOTvKPBxER3Q+GFxHJEsOLiGRJ8jkvIgI09XehsL+CwspcKOyNXx5fV1eHq3VXcaHigtHPNhZWVkFhfwWa+rsAGl/7Z24YXkSdwNXqS3Dq8zZebX5FKADA+r3rm3zOqQ9wtToAgfBqso+5YHgRdQI9nR5AdeGfsW5aAPp6Nn3kdeSbI3h09KNGj7wKrlVhYdoZ9HzigfYut1NgeBF1Akpre+ju+qCP6wA83N34KZ9Wq0WhTSEGdhto9CM0uru3oLt7HUpr+/Yut1No9YT9xYsX26MOIqJWafWRV79+/TBmzBjMmTMHzz33HOztLSPlO5uWJnhbmtwFLG+Cl8xLq8MrKysLmzdvhlqtRnR0NKZNm4Y5c+YgKCioPeqjJpg6wdvc5C5gWRO8ZF5aHV4BAQFYt24d1qxZg127dmHLli0YPXo0+vfvj9mzZ2PmzJnw8PBoj1rpF1qa4G1pchewvAleMi/3PWFvY2ODyZMn4+mnn8b69esRGxuLmJgYvPrqq5g6dSreeOMNeHt7t2Wt9AstTfC2NLkLWN4EL5mX+77C/tSpU5g/fz68vb2RmJiImJgYFBQUID09HVevXsXEiRPbsk4iIgOtPvJKTEzE5s2bkZubi/Hjx2Pr1q0YP368fumaPn36YMuWLbyrDxG1q1aH14YNGzB79mzMmjWrydNCT09PbNq06VcXR0TUlFaHV3p6Onr37m2wSCAACCFw+fJl9O7dG3Z2doiIiGizIomI7tXqOa++ffuivLy8UXtFRQX69OnTJkUREbWk1eElhPF7ylVVVfGCVSLqMCafNjbc+9DKygpxcXEGN2ytr6/Ht99+i4CAgDYvkIjIGJPD6/Tp0wB+OvLKzs42uNWYnZ0d/P39ERMT0/YVEhEZYXJ4HTx4EAAQGRmJdevWyfpu00Qkf63+a+PmzZvbow4iolYxKbwmT56MLVu2wNXVFZMnT262744dO9qkMCKi5pgUXm5ubrCystL/PxGR1EwKr1+eKvK0kYg6A976jIhkyaQjr6FDh+pPG1uSlZX1qwoiIjKFSeE1adKkdi6DiKh1TAqv+Pj49q6DiKhVOOdFRLJk0pFXt27dkJeXB3d3d3Tt2rXZ+a+Kioo2K46IqCkmhdfatWvh4uKi/39TJ++JiNqLSeH1y4UFZ82a1V61EBGZrNVzXtbW1rh27Vqj9hs3bsDa2rpNiiIiakmbLUao0WgMlskhImpPJq8q8dZbbwH4aTHC9957D87OP9/otL6+Hv/973/h5+fX9hUSERlhcnitXbsWwE9HXikpKQaniHZ2dlCpVEhJSWn7ComIjDA5vAoLCwEATzzxBHbs2IGuXbu2W1FERC1p9WKEDSuqEhFJqdXhNXv27GafT01NbXURycnJePPNN1FaWgp/f3+8/fbbCAoKanG77du3Y/r06Zg4cSJ27tzZ6vclIvlqdXj9+OOPBo+1Wi3OnTuHmzdv4sknn2x1AWlpaVCr1UhJScGIESOQlJSEsLAw5ObmwtPTs8ntioqKEBMTg8cee6zV70lE8tfq8Pr8888btel0OkRFRaFv376tLiAxMRFz585FZGQkACAlJQW7d+9Gamoqli5danSb+vp6PP/881ixYgUOHz6Mmzdvtvp9iUjeWh1exigUCqjVajz++ONYvHixydvV1tYiMzMTsbGxBq81duxYHDt2rMnt/v73v8PT0xNz5szB4cOHm30PjUYDjUajf1xZWQngpyNGrVZrcq2dTV1dnf6rsf1oaGtuH1t6Deo4poxFS2NqLuNpau1tEl4AUFBQoP/mmaq8vBz19fXw8vIyaPfy8kJOTo7Rbb755hts2rQJZ86cMek9EhISsGLFikbt+/fvN7hxrtxcrgIAG3zzzTe45Nx0v/T09F/9GtT+WjMWTY2puYxnTU2NSf1aHV4Nd85uIIRASUkJdu/ebfAZyPZw+/ZtzJw5Exs3boS7u7tJ28TGxhrUXFlZCV9fX4SGhsr63pPnr1ZidfZxjB49Go/0bLwfWq0W6enpCAkJga2t7X29BnUcU8aipTE1l/FsODtqSavDq+HO2Q0UCgU8PDywZs2aFv8SeS93d3dYW1ujrKzMoL2srAw9evRo1L+goABFRUWYMGGCvk2n0wEAbGxskJub22jeTalUQqlUNnotW1vbJn+p5cDGxkb/tbn9aG4/TX0Nan+tGYumxtRcxtPU2iW9zsvOzg6BgYHIyMjQLzWt0+mQkZGB6OjoRv39/PyQnZ1t0LZs2TLcvn0b69atg6+vb5vVRkSdW5vNed0vtVqNiIgIDB8+HEFBQUhKSkJ1dbX+r4/h4eHw8fFBQkIC7O3tMWjQIIPtu3TpAgCN2onIvEkeXtOmTcP169cRFxeH0tJSBAQEYO/evfpJ/OLiYigUXK2aiAxJHl4AEB0dbfQ0EQAOHTrU7LZbtmxp+4KIqNPjIQ0RyVKbhdcPP/yAefPmtdXLERE1q83C68aNG9i0aVNbvRwRUbN42khEssTwIiJZYngRkSyZfKnE5MmTm32ey9IQUUcyObzc3NxafD48PPxXF0REZAqTw2vz5s3tWQcRUatwzouIZMnkIy9Tl7u5nxtwEBG1lsnhtWXLFjzwwAMYOnQohBDtWRMRUYtMDq+oqCh8/PHHKCwsRGRkJF544QV069atPWsjImqSyXNeycnJKCkpweLFi/Hvf/8bvr6+mDp1Kvbt28cjMSLqcK2asFcqlZg+fTrS09Px3Xff4ZFHHsH8+fOhUqlQVVXVXjUSETVy339tVCgUsLKyghAC9fX1bVkTEVGLWhVeGo0GH3/8MUJCQtC/f39kZ2fjnXfeQXFxMZydZXyvJSKSHZMn7OfPn4/t27fD19cXs2fPxscff2zy7ceIiNqayeGVkpKC3r1748EHH8TXX3+Nr7/+2mi/HTt2tFlxRERNMTm8wsPDYWVl1Z61EBGZrFUXqRIRdRb8bCMRyRLDi4hkieFFRLLE8CIiWWJ4EZEsmfzXRiJqP3e0P33E7tyVW032qb6jwanrQI9LP8LJQdno+fxrlvX5YoYXUSdQ8P+DZ+mO7BZ62uCD/JPN9nBSWsavtWXsJVEnF/pIDwBAX09nONhaG+2TW3ILL3+ajTXPDcYAb+M3xHFS2qCPu1O71dmZMLyIOoFuTnb4f0G9m+1TV1cHAOjr4YRBPs3fzcsScMKeiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLvFRCplq6Irulq7EBy7sim8wLw0umTLsiu+WrsQHLuSKbzAt/amWqpSuyTbkaG7CsK7LJvHSK8EpOTsabb76J0tJS+Pv74+2330ZQUJDRvhs3bsTWrVtx7tw5AEBgYCBWrVrVZH9z1dIV2bwam8yd5BP2aWlpUKvViI+PR1ZWFvz9/REWFoZr164Z7X/o0CFMnz4dBw8exLFjx+Dr64vQ0FBcuXKlgysnIilJHl6JiYmYO3cuIiMj8fDDDyMlJQWOjo5ITU012v+jjz7C/PnzERAQAD8/P7z33nvQ6XTIyMjo4MqJSEqSnjbW1tYiMzMTsbGx+jaFQoGxY8fi2LFjJr1GTU0NtFotunXrZvR5jUYDjUajf1xZWQkA0Gq10Gq1v6L6zq3htLGurs6s99OSWMqYmrpvkoZXeXk56uvr4eXlZdDu5eWFnJwck15jyZIl6NmzJ8aOHWv0+YSEBKxYsaJR+/79++Ho6Nj6omXichUA2OD48eO4ck7qaqgtWMqY1tTUmNSvU0zY36/XX38d27dvx6FDh2Bvb2+0T2xsLNRqtf5xZWWlfp7M1dW1o0rtcGeLK4DsUxg5ciT8exs/KiV5sZQxbTg7aomk4eXu7g5ra2uUlZUZtJeVlaFHjx7Nbrt69Wq8/vrr+OqrrzBkyJAm+ymVSiiVjS/StLW1ha2t7f0VLgM2Njb6r+a8n5bEUsbU1H2TdMLezs4OgYGBBpPtDZPvwcHBTW73z3/+EytXrsTevXsxfPjwjiiViDoZyU8b1Wo1IiIiMHz4cAQFBSEpKQnV1dWIjIwEAISHh8PHxwcJCQkAgDfeeANxcXHYtm0bVCoVSktLAQDOzs5wdnaWbD+IqGNJHl7Tpk3D9evXERcXh9LSUgQEBGDv3r36Sfzi4mIoFD8fIG7YsAG1tbV47rnnDF4nPj4ef/vb3zqydCKSkOThBQDR0dGIjo42+tyhQ4cMHhcVFbV/QUTU6Ul+kSoR0f1geBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZKlThFdycjJUKhXs7e0xYsQInDhxotn+n3zyCfz8/GBvb4/Bgwdjz549HVQpEXUWkodXWloa1Go14uPjkZWVBX9/f4SFheHatWtG+x89ehTTp0/HnDlzcPr0aUyaNAmTJk3CuXPnOrhyIpKS5OGVmJiIuXPnIjIyEg8//DBSUlLg6OiI1NRUo/3XrVuHcePG4ZVXXsHAgQOxcuVKDBs2DO+8804HV05EUrKR8s1ra2uRmZmJ2NhYfZtCocDYsWNx7Ngxo9scO3YMarXaoC0sLAw7d+402l+j0UCj0egfV1ZWAgC0Wi20Wu2v3IPOo6amBrm5ufrHeSW3oCnNx7kzdqgtczPoO2DAADg6OnZ0idQK944n0PSYmtt4mvp7KWl4lZeXo76+Hl5eXgbtXl5eyMnJMbpNaWmp0f6lpaVG+yckJGDFihWN2vfv329WA15QUICXX365UfvM9xv3XbNmDfr27dsBVdH9amo8gcZjam7jWVNTY1I/ScOrI8TGxhocqVVWVsLX1xehoaFwdXWVsLK2VVNTg9GjR+sfV93RYN/hkwh77DdwdlAa9DW3f6nN0b3jCTQ9puY2ng1nRy2RNLzc3d1hbW2NsrIyg/aysjL06NHD6DY9evRoVX+lUgmlUtmo3dbWFra2tvdZeefj5uaGoKAg/WOtVovbNyvw2KiRZrWfluLe8QQsZ0xN3TdJJ+zt7OwQGBiIjIwMfZtOp0NGRgaCg4ONbhMcHGzQHwDS09Ob7E9E5kny00a1Wo2IiAgMHz4cQUFBSEpKQnV1NSIjIwEA4eHh8PHxQUJCAgBg4cKFGDNmDNasWYOnn34a27dvx6lTp/Duu+9KuRtE1MEkD69p06bh+vXriIuLQ2lpKQICArB37179pHxxcTEUip8PEEeNGoVt27Zh2bJlePXVV/HQQw9h586dGDRokFS7QEQSsBJCCKmL6EiVlZVwc3PDrVu3zGrC/l5arRZ79uzB+PHjzXp+xJJYypia+jsq+UWqRET3g+FFRLLE8CIiWZJ8wr6jNUzxmXohnFxptVrU1NSgsrLSrOdHLImljGnD72ZL0/EWF163b98GAPj6+kpcCRE15/bt23Bzc2vyeYv7a6NOp8PVq1fh4uICKysrqctpNw0fg7p8+bJZ/1XVkljKmAohcPv2bfTs2dPgMql7WdyRl0KhQK9evaQuo8O4urqa9Q+6JbKEMW3uiKsBJ+yJSJYYXkQkSwwvM6VUKhEfH290RQ2SJ46pIYubsCci88AjLyKSJYYXEckSw4uIZInhRUSyxPCSgccffxwvvfSS1GXQfehsY9fZ6vk1GF5EnVxtba3UJXROgjq1iIgIAcDgPx8fH7F+/XqDfllZWcLKykoUFRUJIYRYs2aNGDRokHB0dBS9evUSUVFR4vbt2wbbHD58WIwePVrY29uLXr16iT//+c+iqqqqw/bN3Bkbu/z8fDF79myhUqmEvb296N+/v0hKSmq03cSJE8U//vEP4e3tLVQqlRBCiCNHjgh/f3+hVCpFYGCg+PzzzwUAcfr0af222dnZYty4ccLJyUl4enqKF154QVy/fr3JegoLCzvq29HmGF6d3M2bN0VwcLCYO3euKCkpESUlJSImJkaMHj3aoN/LL79s0LZ27Vpx4MABUVhYKDIyMsSAAQNEVFSU/vn8/Hzh5OQk1q5dK/Ly8sSRI0fE0KFDxaxZszps38ydsbG7e/euiIuLEydPnhQXL14UH374oXB0dBRpaWn67SIiIoSzs7OYOXOmOHfunDh37py4deuW6Natm3jhhRfE+fPnxZ49e0T//v0NwuvHH38UHh4eIjY2Vly4cEFkZWWJkJAQ8cQTTzRZT11dnRTfmjbB8JKBMWPGiIULF+ofnz59WlhZWYlLly4JIYSor68XPj4+YsOGDU2+xieffCK6d++ufzxnzhwxb948gz6HDx8WCoVC3Llzp213wILdO3bGLFiwQPzhD3/QP46IiBBeXl5Co9Ho2zZs2CC6d+9uMDYbN240CK+VK1eK0NBQg9e+fPmyACByc3NNrkcuOOclQwEBARg4cCC2bdsGAPj6669x7do1TJkyRd/nq6++wlNPPQUfHx+4uLhg5syZuHHjhv5W6mfPnsWWLVvg7Oys/y8sLAw6nQ6FhYWS7JelSE5ORmBgIDw8PODs7Ix3330XxcXFBn0GDx4MOzs7/ePc3FwMGTIE9vb2+rZ7b0p79uxZHDx40GBM/fz8AAAFBQXtuEfSYHjJ1PPPP68Pr23btmHcuHHo3r07AKCoqAjPPPMMhgwZgs8++wyZmZlITk4G8PPkb1VVFf70pz/hzJkz+v/Onj2L77//Hn379pVmpyzA9u3bERMTgzlz5mD//v04c+YMIiMjG03KOzk5tfq1q6qqMGHCBIMxPXPmDL7//nv89re/batd6DQsbj0vObKzs0N9fb1B24wZM7Bs2TJkZmbi008/RUpKiv65zMxM6HQ6rFmzRr+Y27/+9S+D7YcNG4bvvvsO/fr1a/8dsGD3jt2RI0cwatQozJ8/X99mylHRgAED8OGHH0Kj0eg/mH3y5EmDPsOGDcNnn30GlUoFGxvjv9rGfpbkikdeMqBSqfDtt9+iqKgI5eXl0Ol0UKlUGDVqFObMmYP6+no8++yz+v79+vWDVqvF22+/jYsXL+KDDz4wCDcAWLJkCY4ePYro6Gj9v85ffPEFoqOjO3r3zNq9Y/fQQw/h1KlT2LdvH/Ly8rB8+fJGIWTMjBkzoNPpMG/ePFy4cAH79u3D6tWrAUC/IvCCBQtQUVGB6dOn4+TJkygoKMC+ffsQGRmpDyxjP0uyJfWkG7UsNzdXjBw5Ujg4OBj8eXv9+vUCgAgPD2+0TWJiovD29hYODg4iLCxMbN26VQAQP/74o77PiRMnREhIiHB2dhZOTk5iyJAh4rXXXuugvbIM945dTk6OmDVrlnBzcxNdunQRUVFRYunSpcLf31+/TcOlEvc6cuSIGDJkiLCzsxOBgYFi27Zt+tdskJeXJ37/+9+LLl26CAcHB+Hn5ydeeuklodPpjNYj50sluCQOkUx99NFHiIyMxK1bt+Dg4CB1OR2Oc15EMrF161Y8+OCD8PHxwdmzZ7FkyRJMnTrVIoMLYHgRyUZpaSni4uJQWloKb29vTJkyBa+99prUZUmGp41EJEv8ayMRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyoXc2aNQtWVlawsrKCra0tvLy8EBISgtTU1FZ9NGXLli3o0qVL+xXahFmzZmHSpEkd/r7UMoYXtbtx48ahpKQERUVF+PLLL/HEE09g4cKFeOaZZ1BXVyd1eSRX0n46icxdU5/Ty8jIEADExo0bhRDNL1t98ODBRssXx8fHCyGE2Lp1qwgMDBTOzs7Cy8tLTJ8+XZSVlenfp6KiQsyYMUO4u7sLe3t70a9fP5Gamqp/vri4WEyZMkW4ubmJrl27imeffVb/eb/4+PhG73vw4MF2+T5R6/HIiyTx5JNPwt/fHzt27AAAKBQKvPXWWzh//jzef/99HDhwAIsXLwYAjBo1CklJSXB1dUVJSQlKSkoQExMDANBqtVi5ciXOnj2LnTt3oqioCLNmzdK/z/Lly/Hdd9/hyy+/xIULF7Bhwwa4u7vrtw0LC4OLiwsOHz6MI0eOwNnZGePGjUNtbS1iYmIwdepU/ZFjSUkJRo0a1bHfKGqa1OlJ5q2pIy8hhJg2bZoYOHCg0efuXbZ68+bNws3NrcX3O3nypACgP2qbMGGCiIyMNNr3gw8+EAMGDNCvuCCEEBqNRjg4OIh9+/a1WD9Ji0deJBkhhH4tqpaWrW5KZmYmJkyYgN69e8PFxQVjxowBAP2yylFRUdi+fTsCAgKwePFiHD16VL/t2bNnkZ+fDxcXF/2yyd26dcPdu3fNctlkc8PwIslcuHABffr0MWnZamOqq6sRFhYGV1dXfPTRRzh58iQ+//xzg+1+97vf4dKlS1i0aBGuXr2Kp556Sn/KWVVVhcDAwEbLJufl5WHGjBntvPf0a3FVCZLEgQMHkJ2djUWLFpm0bLWx5YtzcnJw48YNvP766/D19QUAnDp1qtF7eXh4ICIiAhEREXjsscfwyiuvYPXq1Rg2bBjS0tLg6ekJV1dXo3Wa07LJ5oZHXtTuNBoNSktLceXKFWRlZWHVqlWYOHEinnnmGYSHh5u0bLVKpUJVVRUyMjJQXl6Ompoa9O7dG3Z2dvrtdu3ahZUrVxpsFxcXhy+++AL5+fk4f/48/vOf/2DgwIEAfrqJibu7OyZOnIjDhw+jsLAQhw4dwl/+8hf88MMP+vf93//+h9zcXJSXl0Or1XbMN41aJvWkG5m3X96l2cbGRnh4eIixY8eK1NRUUV9fr+9nyrLVL774oujevbvBpRLbtm0TKpVKKJVKERwcLHbt2tXoXoYDBw4UDg4Oolu3bmLixIni4sWL+tcsKSkR4eHhwt3dXSiVSvHggw+KuXPnilu3bgkhhLh27Zp+qWzwUolOhet5EZEs8bSRiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSz9H4w5sijv0b6oAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:15:30.591580Z", + "iopub.status.busy": "2024-03-22T21:15:30.591189Z", + "iopub.status.idle": "2024-03-22T21:15:30.829339Z", + "shell.execute_reply": "2024-03-22T21:15:30.828392Z" + }, + "papermill": { + "duration": 0.26235, + "end_time": "2024-03-22T21:15:30.831419", + "exception": false, + "start_time": "2024-03-22T21:15:30.569069", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAEmCAYAAAD8/yLTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABYzElEQVR4nO3dd1yV1R/A8c8dXJZMB0NBcOEeOUjL1MTQzJGVM1emldkic6epGWZmppk2nLmycvRrWEpuURN3igNRUBkisoW7nt8fV65cAeGyLuO8X6/nJfe553nu9wH5cp5zznOOTJIkCUEQhCpKbukABEEQLEkkQUEQqjSRBAVBqNJEEhQEoUoTSVAQhCpNJEFBEKo0kQQFQajSRBIUBKFKU1o6gPJIr9dz69YtHBwckMlklg5HEISHSJJEamoqnp6eyOXFq8uJJJiHW7du4eXlZekwBEEoQHR0NHXq1CnWOUQSzIODgwNg+AY7OjpaOBpBEB6WkpKCl5eX8Xe1OEQSzEP2LbCjo6NIgoJQjpVEc5XoGBEEoUoTSVAQhCpNJEFBEKo00SZYRJIkodVq0el0lg5FKCEKhQKlUimGRVUxIgkWgVqtJiYmhoyMDEuHIpQwOzs7PDw8UKlUlg5FKCMiCZpJr9cTGRmJQqHA09MTlUolag6VgCRJqNVqbt++TWRkJA0bNiz2IFzBPJIkWeR3SSRBM6nVavR6PV5eXtjZ2Vk6HKEE2draYmVlxfXr11Gr1djY2Fg6pCpDr5fovGAP9Wra88Wg1tSoZl1mn23RP3X79++nT58+eHp6IpPJ2L59+yPLjxo1CplMlmtr1qyZscxHH32U6/3GjRuXeOyillA5iZ+rZVyOT+Nm0j3Crt/F2daqTD/boj/x9PR0WrVqxbJlywpV/ssvvyQmJsa4RUdH4+rqyksvvWRSrlmzZiblDh48WBrhC4JQQk5E3QWgVR1nlIqyTUsWvR3u1asXvXr1KnR5JycnnJycjK+3b9/O3bt3GT16tEk5pVKJu7t7icUpCELpOnHdkAQfq+tc5p9doev+K1euJCAggLp165rsv3z5Mp6entSrV49hw4YRFRX1yPNkZWWRkpJisgmV06hRo+jfv7+lwxAekl0TbFvXpcw/u8ImwVu3bvHnn3/y6quvmuz39/dnzZo17Ny5k+XLlxMZGUnnzp1JTU3N91zBwcHGWqaTk1OlnUGma9euvPvuu5YOQxBMJGWoibidDkAbL5EEC23t2rU4Ozvn+qveq1cvXnrpJVq2bElgYCB//PEHSUlJbNmyJd9zTZ06leTkZOMWHR1dytELxaHRaCwdglCCTkYlAVCvhj0u9mU/PrNCJkFJkli1ahXDhw8vcFCrs7MzjRo14sqVK/mWsba2Ns4YU5SZYyRJIkOtLfNNkqRCxzhq1Cj27dvHl19+aew1r1OnDsuXLzcpd/LkSeRyOdevXwdg0aJFtGjRAnt7e7y8vBg/fjxpaWkmxxw8eJDOnTtja2uLl5cXb7/9Nunp6YWKKyYmht69e2Nra4uvry8bN27Ex8eHxYsXG8vIZDKWL19O3759sbe3Z968eeh0OsaMGYOvry+2trb4+fnx5Zdfmpxbp9MRFBSEs7Mz1atXZ9KkSWZ9z4SykX0r3Ma77GuBUEHHCe7bt48rV64wZsyYAsumpaURERHB8OHDSy2eexodTWf+VWrnz8/5OYHYqQr3I/zyyy+5dOkSzZs3Z86cOQB8/vnnbNy4kTfeeMNYbsOGDTzxxBPGdla5XM6SJUvw9fXl6tWrjB8/nkmTJvH1118DEBERQc+ePfn4449ZtWoVt2/fZsKECUyYMIHVq1cXGNeIESNISEhg7969WFlZERQURHx8fK5yH330EfPnz2fx4sUolUr0ej116tThp59+onr16hw+fJhx48bh4eHBwIEDjde3Zs0aVq1aRZMmTfj888/Ztm0bTz/9dKG+Z0LZyE6ClugUAQsnwbS0NJMaWmRkJKdOncLV1RVvb2+mTp3KzZs3WbdunclxK1euxN/fn+bNm+c658SJE+nTpw9169bl1q1bzJo1C4VCwZAhQ0r9esozJycnVCoVdnZ2xp7zYcOG8fnnnxMVFYW3tzd6vZ7NmzczY8YM43E52xB9fHz4+OOPef31141JMDg4mGHDhhnLNWzYkCVLltClSxeWL1/+yAHH4eHh7N69m3///Zd27doB8P3339OwYcNcZYcOHZprFMDs2bONX/v6+hIaGsqWLVuMSXDx4sVMnTqVAQMGALBixQr++qvs/1gJ+dPpJU7dvx1+rCrWBI8fP063bt2Mr4OCggAYOXIka9asISYmJlfPbnJyMr/88kuuW59sN27cYMiQIdy5c4eaNWvy5JNPcuTIEWrWrFlq12FrpeD8nMBSO/+jPrc4WrduTZMmTdi4cSNTpkxh3759xMfHm4y73L17N8HBwYSHh5OSkoJWqyUzM5OMjAzs7Ow4ffo0Z86cYcOGDcZjJEkyPl7YpEmTfD//4sWLKJVKHnvsMeO+Bg0a4OKS+5chO0nmtGzZMlatWkVUVBT37t1DrVbTunVrwPD/JCYmBn9/f2N5pVJJu3btxC1xOXIpLpV0tY5q1koauRV/luiisGgS7Nq16yP/Q65ZsybXPicnp0dOXLB58+aSCM0sMpms0Lel5c2wYcOMSXDjxo307NmT6tWrA3Dt2jWee+453njjDebNm4erqysHDx5kzJgxqNVq7OzsSEtL47XXXuPtt9/OdW5vb+8Si9Pe3t7k9ebNm5k4cSKff/45HTt2xMHBgc8++4yjR4+W2GcKpS/s/vjA1l7OKOSWeQa/Yv7mCkWiUqlyTf01dOhQZsyYQVhYGD///DMrVqwwvhcWFoZer+fzzz83Pk72cC/7Y489xvnz52nQoIHZ8fj5+aHVajl58iRt27YF4MqVK9y9e7fAYw8dOkSnTp0YP368cV9ERITxaycnJzw8PDh69ChPPfUUAFqtlrCwMJOap2BZxvZAb2eLxVAhe4eFovHx8eHo0aNcu3aNhIQE9Ho9Pj4+dOrUiTFjxqDT6ejbt6+xfIMGDdBoNCxdupSrV6/yww8/mCRJgMmTJ3P48GEmTJjAqVOnuHz5Mjt27GDChAkFxtO4cWMCAgIYN24cx44d4+TJk4wbNw5bW9sCZxNp2LAhx48f56+//uLSpUt8+OGH/PvvvyZl3nnnHebPn8/27dsJDw9n/PjxJCUlFf4bJpS67OExbSwwSDqbSIJVyMSJE1EoFDRt2pSaNWsa21uHDRvG6dOnef7557G1tTWWb9WqFYsWLeLTTz+lefPmbNiwgeDgYJNztmzZkn379nHp0iU6d+5MmzZtmDlzJp6enoWKad26dbi5ufHUU0/x/PPPM3bsWBwcHAqcweW1115jwIABDBo0CH9/f+7cuWNSKwR4//33GT58OCNHjjTeMj///POFiksofYnpaiITDEOpHrPAIOlsMkm0EueSkpKCk5MTycnJucYMZmZmEhkZia+vr5hqqRTcuHEDLy8vdu/eTffu3cv888XPt+yEXIhjzNrjNKhVjd1BXcw69lG/o+YSbYKCRf3zzz+kpaXRokULYmJimDRpEj4+PsZ2PKHyKg/tgSCSoFCKDhw48MhZgtLS0tBoNEybNo2rV6/i4OBAp06d2LBhA1ZWZTunnFD2snuGLTU+MJtIgkKpadeuHadOnXpkmcDAQAIDy36MpWBZWp2e09HJADxmwU4REElQKEW2trZFGjojVH7hsanc0+hwsFHSoGY1i8YieocFQShzJ6MeDJKWW2iQdDaRBAVBKHMn7o8PtMQkqg8TSVAQhDL3oGdYJEFBEKqYhLQsrt/JQCaD1hYeHgMiCQqCUMayF1VqWKsajjaWHwolkqBQKHnN9lzQOtGCkJcTFp4/8GFiiIxQJDExMXnO+ycIBSlP7YEgkqBQROVhXWdJktDpdCiV4r9xRaHR6TlzIwmw/CDpbOJ2uCRIEqjTy34zc+6L1NRUhg0bhr29PR4eHnzxxRdFXoYz5+3wtWvXkMlkbN26lW7dumFnZ0erVq0IDQ01OaagBZl++OEH2rVrh4ODA+7u7gwdOtRkvZG9e/cik8n4888/adu2LdbW1hw8eNDs2AXLuRCTQqZGj5OtFfVq2Bd8QBkQf0JLgiYDPinc1FElatotUBX+P1JQUBCHDh3i119/xc3NjZkzZ3LixAnjlPTFNX36dBYuXEjDhg2ZPn06Q4YM4cqVKyiVykItyKTRaJg7dy5+fn7Ex8cTFBTEqFGj+OOPP0w+Z8qUKSxcuJB69eqJW/IKJrtTpI235QdJZ7NoTXD//v306dMHT0/PQjW0Z9cEHt5iY2NNyi1btgwfHx9sbGzw9/fn2LFjpXgVFUNqaipr165l4cKFdO/enebNm7N69epcM00Xx8SJE+nduzeNGjVi9uzZXL9+3biQVs4FmRo2bEinTp1YsmQJ69atIzMzE4BXXnmFXr16Ua9ePR5//HGWLFnCn3/+mWuJzzlz5tCjRw/q16+Pq6tricUvlL7y1ikCFq4Jpqen06pVK1555RXjimCFcfHiRZM5xGrVqmX8+scffyQoKIgVK1bg7+/P4sWLCQwM5OLFiyblSpSVnaFWVtas7Apd9OrVq2g0Gjp06GDc5+TkhJ+fX4mF07JlS+PXHh4eAMTHx9O4ceNCLcgUFhbGRx99xOnTp7l79y56vR6AqKgomjZtajwur0WXhIqhvHWKgIWTYK9evR451VJ+atWqhbOzc57vLVq0iLFjxxqXZ1yxYgW///47q1atYsqUKcUJN38ymVm3pZVVzumvsqfHz05kBS3IlJ6ebpxRZsOGDcaZrwMDA1Gr1SblH150SagY4lMzuXH3HjIZtPJysnQ4RhWyY6R169Z4eHjQo0cPDh06ZNyvVqsJCwsjICDAuE8ulxMQEJCrkT6nrKwsUlJSTLbKpl69elhZWZmsw5GcnMylS5fK5PNzLsj08KZSqQgPD+fOnTvMnz+fzp0707hx4zwXYRcqrhPXkwDwc3PAoRwMks5WoZKgh4cHK1as4JdffuGXX37By8uLrl27cuLECQASEhLQ6XS4ubmZHOfm5par3TCn4OBgnJycjJuXl1epXoclODg4MHLkSD744AP27NnDf//9x5gxY5DL5QUualQSClqQydvbG5VKZVzU6ddff2Xu3LmlHpdQdoy3wuVkaEy2CpUE/fz8eO2112jbti2dOnVi1apVdOrUiS+++KJY5506dSrJycnGLTo6uoQiLl8WLVpEx44dee655wgICOCJJ56gSZMmZbKWRkELMtWsWZM1a9bw008/0bRpU+bPn8/ChQtLPS6h7JwoJzNJP6zCD5Hp0KGDcaxYjRo1UCgUxMXFmZSJi4t75OBea2trrK2tSzXO8sDBwcGkYyI9PZ3Zs2czbty4Ao+9du2ayeuc63P5+Pjw8Hpdzs7Oufa1b9+ev//+O9/PGDJkCEOGDMn3c7p27ZrrnELFoNbqOXPz/kzS5WDShJwqVE0wL6dOnTL2RKpUKtq2bUtISIjxfb1eT0hICB07drRUiOXGyZMn2bRpExEREZw4cYJhw4YB0K9fPwtHJlR252NSUGv1uNhZ4VtOBklns2hNMC0tzTiODCAyMpJTp07h6uqKt7c3U6dO5ebNm6xbtw6AxYsX4+vrS7NmzcjMzOT777/nn3/+MaldBAUFMXLkSNq1a0eHDh1YvHgx6enpxt7iqm7hwoVcvHjR+AfjwIEDXLhwocAFkQShOHLeCpdFG7Q5LJoEjx8/Trdu3Yyvg4KCABg5ciRr1qwhJibGuEA4GHp/33//fW7evImdnR0tW7Zk9+7dJucYNGgQt2/fZubMmcTGxtK6dWt27tyZq7OkKmrTpg1hYWG59t+7d6/ABZEEoTjKa6cIiMXX8yQWX6+6xM+3dHQKDuFWciYbx/rTqX6NYp+vJBdfr/BtgpYi/nZUTuLnWvJikzO5lZyJXAat6jhbOpxcRBI0U/ZTERkZGRaORCgN2T9Xsfh7ycm+FW7s7oi9dfkbkFL+IirnFAoFzs7OxqcZ7Ozsyl1Dr2A+SZLIyMggPj4eZ2dnFAqFpUOqNIydInWdLRtIPkQSLILsMYfisa7Kx9nZuVxMGFuZZNcEy8PymnkRSbAIZDIZHh4e1KpVC41GY+lwhBJiZWUlaoAlLEur49xNw7P45e1JkWwiCRaDQqEQvzSC8Ajnbqag1umpbq/C27XwU7+VJdExIghCqTkZlT2TdPkbJJ1NJEFBEErNg0HSzpYN5BFEEhQEodRkzyFYXtsDQSRBQRBKya2ke8SmZKKQy8rlIOlsIgkKglAqwu6PD2zq4Yitqvx2IIokKAhCqXiwqJKzZQMpgEiCgiCUCuPymuV0kHQ2kQQFQShxmRod529lzyQtkqAgCFXMuZvJaHQSNR2sqeNia+lwHkkkQUEQSlzO9sDyOkg6W7Eem0tLSzMurp2tuBMcCoJQ8YWV05Xl8mJ2TTAyMpLevXtjb2+Pk5MTLi4uuLi44OzsjItL+b9gQRBKlyRJFaZTBIpQE3z55ZeRJIlVq1bh5uZW7qu6giCUrRt373E7NQulXEaL2k6WDqdAZtcET58+zerVqxk0aBBdu3alS5cuJps59u/fT58+ffD09EQmk7F9+/ZHlt+6dSs9evSgZs2aODo60rFjR/766y+TMh999BEymcxka9y4sbmXKQhCEWW3BzbzdMTGqvwOks5mdhJs37490dHRJfLh6enptGrVimXLlhWq/P79++nRowd//PEHYWFhdOvWjT59+nDy5EmTcs2aNSMmJsa4ZS/OLghC6TtZgW6FoQi3w99//z2vv/46N2/epHnz5rnWYmjZsmWhz9WrV69Hrnf7sMWLF5u8/uSTT9ixYwf/+9//aNOmjXG/UqkUswMLgoU86BmupEnw9u3bREREmCxmLpPJkCQJmUyGTqcr0QAfRa/Xk5qaiqurq8n+y5cv4+npiY2NDR07diQ4OBhvb+98z5OVlUVWVpbxdUpKSqnFLAiV2T21jvO37s8kXVlrgq+88gpt2rRh06ZNFu8YWbhwIWlpaQwcONC4z9/fnzVr1uDn50dMTAyzZ8+mc+fOnDt3DgcHhzzPExwczOzZs8sqbEGotM7cSEKrl3BztMbTqYKs2yyZyc7OTrp8+bK5hxUIkLZt21bo8hs2bJDs7OykXbt2PbLc3bt3JUdHR+n777/Pt0xmZqaUnJxs3KKjoyVASk5OLnQ8giBI0td7rkh1J/8mvf7D8VL9nOTk5BL7HTW7Jvj0009z+vRpGjRoUOIJubA2b97Mq6++yk8//URAQMAjyzo7O9OoUSOuXLmSbxlra2usra1LOkxBqHIqWnsgFOF2uE+fPrz33nucPXuWFi1a5OoY6du3b4kFl5dNmzbxyiuvsHnzZnr37l1g+bS0NCIiIhg+fHipxiUIVZ0kScY1RSpKeyAUIQm+/vrrAMyZMyfXe+Z2jKSlpZnU0CIjIzl16hSurq54e3szdepUbt68ybp16wDYuHEjI0eO5Msvv8Tf35/Y2FgAbG1tcXIyDMqcOHEiffr0oW7duty6dYtZs2ahUCgYMmSIuZcqCIIZohIzSEhTo1LIaV674jw+a/Y4Qb1en+9mbs/w8ePHadOmjXF4S1BQEG3atGHmzJkAxMTEEBUVZSz/7bffotVqefPNN/Hw8DBu77zzjrHMjRs3GDJkCH5+fgwcOJDq1atz5MgRatasae6lCoJgBuMg6dqOWCvL/yDpbGbVBDUaDba2tpw6dYrmzZsX+8O7du2KJEn5vr9mzRqT13v37i3wnJs3by5mVIIgFEVFWFQpL2bVBK2srPD29i7TsYCCIFQMFbFTBIpwOzx9+nSmTZtGYmJiacQjCEIFlKHWEh6bCpTvNYbzYnbHyFdffcWVK1fw9PSkbt262Nvbm7x/4sSJEgtOEISK4XR0Mjq9hKeTDR5O5Xsm6YeZnQT79+9fCmEIglCRZd8Kt6lAQ2OymZ0EZ82aVRpxCIJQgZ2oQDNJP6zI0+uHhYVx4cIFwDB1Vc5ZXARBqDokSeJkdBJQ/tcYzovZSTA+Pp7Bgwezd+9enJ2dAUhKSqJbt25s3rxZjMcThCrm2p0MEtPVqJRymnmW/5mkH2Z27/Bbb71Famoq//33H4mJiSQmJnLu3DlSUlJ4++23SyNGQRDKsexb4Za1nVApK94ClmbXBHfu3Mnu3btp0qSJcV/Tpk1ZtmwZzzzzTIkGJwhC+XeiAj4vnFORHpt7eNIEMAykfnj5TUEQKr8Hy2s6WzaQIjI7CT799NO888473Lp1y7jv5s2bvPfee3Tv3r1EgxMEoXxLy9JyKe7+IOkK2DMMRUiCX331FSkpKfj4+FC/fn3q16+Pr68vKSkpLF26tDRiFAShnDodnYRegtrOttRyrCAzST/E7DZBLy8vTpw4we7duwkPDwegSZMmBU5uKghC5WMcH1hB2wOhiOMEZTIZPXr0oEePHiUdjyAIFUh2p0jbCtoeCEVMgiEhIYSEhBAfH5+rM2TVqlUlEpggCOWbXp9jkHRVqgnOnj2bOXPm0K5dOzw8PCy62pwgCJZzNSGdpAwNNlZymnhUnJmkH2Z2ElyxYgVr1qwRa3YIQhWXfSvcsrYzVoqKN0g6m9mRq9VqOnXqVBqxCIJQgZw0zhzjbNlAisnsJPjqq6+ycePGEvnw/fv306dPHzw9PZHJZGzfvr3AY/bu3ctjjz2GtbU1DRo0yDUFP8CyZcvw8fHBxsYGf39/jh07ViLxCoLwQEWdTv9hZt8OZ2Zm8u2337J7925atmyZ6+mRRYsWFfpc6enptGrVildeeYUBAwYUWD4yMpLevXvz+uuvs2HDBkJCQnj11Vfx8PAgMDAQgB9//JGgoCBWrFiBv78/ixcvJjAwkIsXL1KrVi3zLlYQhDylZGq4FF/Cg6SPrwJbV2jWv2TOV0gy6VErHeWhW7du+Z9MJuOff/4pWiAyGdu2bXvkpK2TJ0/m999/59y5c8Z9gwcPJikpiZ07dwLg7+9P+/bt+eqrrwDDY35eXl689dZbTJkypVCxpKSk4OTkRHJyMo6OFbfBVxBKy/5Ltxmx6hjernbsn5R/Tii0E+vg17dApoBxe8Gj5SOLl+TvqNk1wT179hTrA4sjNDQ016DswMBA3n33XcDQXhkWFsbUqVON78vlcgICAggNDc33vFlZWWRlZRlfp6SklGzgglDJPFhUybn4Jzu1EX69PwOV/2vg3qL45zRDherSiY2Nxc3NzWSfm5sbKSkp3Lt3j4SEBHQ6XZ5lshdqz0twcDBOTk7GzcvLq1TiF4TK4kRUElAC4wPP/ATbxwMStB8LgZ9AGQ+7q1BJsLRMnTqV5ORk4xYdHW3pkASh3NLrJWPPcLHaA//bBtvGARK0HQW9FpR5AoRiTK9vCe7u7sTFxZnsi4uLw9HREVtbWxQKBQqFIs8y7u7u+Z7X2toaa2vrUolZECqbiNtppGZqsbVS0NjdoWgnufA/+HkMSHpo/TL0/gLklqmTVaiaYMeOHQkJCTHZt2vXLjp27AiASqWibdu2JmX0ej0hISHGMoIgFE92e2ArLyeURRkkfXEn/DQaJB20HAx9l1gsAUIRkuD+/fvRarW59mu1Wvbv32/WudLS0jh16hSnTp0CDENgTp06RVRUFGC4TR0xYoSx/Ouvv87Vq1eZNGkS4eHhfP3112zZsoX33nvPWCYoKIjvvvuOtWvXcuHCBd544w3S09MZPXq0uZcqCEIewoqzstzl3bBlOOg10PwF6P81yBUlHKGZJDPJ5XIpLi4u1/6EhARJLpebda49e/ZIQK5t5MiRkiRJ0siRI6UuXbrkOqZ169aSSqWS6tWrJ61evTrXeZcuXSp5e3tLKpVK6tChg3TkyBGz4kpOTpYAKTk52azjBKEq6P75Xqnu5N+kXf/FmnfglX8kaU5NSZrlKEmbX5YkrabIMZTk76jZ4wTlcjlxcXG5VpW7dOkS7dq1qxTDS8Q4QaEqSEjLYu3hawxs54WXq12hjknO0NBqzt8AhM0IoHq1QralRx6ADS+B9h749YaBa0GRe5mOwrLIOMHsJzpkMhmjRo0y6UjQ6XScOXNGPFMsCBXIluPRLP3nCtGJGSweXLh1w09GG26FfarbFT4BXg+FjYMMCbDhM/DS6mIlwJJW6CTo5GRYT1SSJBwcHLC1tTW+p1KpePzxxxk7dmzJRygIQqm4m64G4OCVBPR6Cbm84OEpZo8PjP4XNrwImnSo/zQM/AGU5WskRqGT4OrVqwHw8fFh4sSJ2Nvbl1pQFUnIhTger1cde+sKNdpIEEhX6wBISFNzMS61UHMCmjU+8OYJWD8A1Gng+xQM3ghW5W8dErN7h2fNmiUS4H3z/wxnzNrjzPvjgqVDEQSzpWc9GOVx8HJCgeV1eomT2TXBgpJgzGn4oT9kpUDdJ2DIZrCyffQxFmJ2EoyLi2P48OF4enqiVCqNA5Szt6rkqUY1ANh4NIq9F+MtHI0gmCc9S2f8+uCVgpPg5fhU0rK02KsU+D1qkHTsOVjXDzKTwcsfhv4IqvJbcTL7Hm7UqFFERUXx4YcfVvnp9TvVr8GoTj6sOXyNyb+c4e93u+BkV34afAXhUXLWBI9G3iFLq8NamX9FJnv+wFZezijyaz+MDzckwHt3oXZbGPYzWBfxqZIyYnYSPHjwIAcOHKB169alEE7FM7lnY/Zfus3VhHRm/nqOLwvZyyYIlpaufpAEMzV6TlxPomP96vmWP1FQe2DCZVjbBzISwKMVvLwVbMr/EDOzb4e9vLwwc2hhpWarUvD5wFbIZbDj1C3+OBtj6ZAEoVCya4KeTobOioNXbj+yvHF5zbx6hu9EGBJgejy4tYDh28HWuSTDLTVmJ8HFixczZcoUrl27VgrhVExtvF0Y37UBANO3nSU+NdPCEQlCwbLbBJ9pZphc5OCVO/mWvZuu5urtdADaPDyH4N1rhgSYGgO1msKIHWDnWhohlwqzk+CgQYPYu3cv9evXx8HBAVdXV5Otqnq7e0OaejhyN0PDtK1nRW1ZKPeyb4efaWaYf/PsjSSSMzR5ls0eJF2vpj3OdqoHbyRFw5o+kHITajQyJED7/G+pyyOz2wQXL15cCmFUfCqlnEWDWtF36SF2X4jnp+M3GNheTM4qlE+SJBlvh+vXrEbDWtW4HJ9G6NUEejb3yFU+z0WVkm/C2ucgOQpc68PI/0G1ireOj9lJcOTIkaURR6XQ2N2RoGcaMf/PcOb8dp6O9atT08Ga9UeuE5mQzqSejXGyFb3HguVlafXo79+s2KkUPNGgBpfj0zhwOZ8k+HCnSGqs4Rb47jVw8TEkQIf85+wsz4r0mENERASrV68mIiKCL7/8klq1avHnn3/i7e1Ns2bNSjrGCmVs53rsPh/H8et3GfztETQ6PfGphvVL7ml0LBrY2rIBCgKQlmN4jJ1KSeeGNVhz+BqH8hgvqNNLnI5OAu53iqTFGxJgYgQ4eRsSoFPtsgq9xJndJrhv3z5atGjB0aNH2bp1K2lpaQCcPn2aWbNmlXiA5ZpeD6FfQ1aacZdCLmPRwNbUcbHlZtI94lOz8HSyQSaDrSduskcMqhbKgYz7nSK2VgoUchn+9aqjkMu4dieD6MQMk7IXY1NJV+twsFbS0D4T1vaFhEvgWBtG/Q+cvS1xCSXG7CQ4ZcoUPv74Y3bt2oVK9aCB9Omnn+bIkSMlGly59/d0+GsqbBoM6gf/cbyr2/HHO52Z0K0BH/dvzp4PujLmCV8A5v523lLRCoJRdk0w+5n3atZK2ng5A+SqDYbdvxV+orYc+frn4fYFcPAw1ABdfMos5tJidhI8e/Yszz//fK79tWrVIiGh4EdvKpXmL4DKAa4dgM1DQHPP+JajjRUTA/14+fG6WCsVvPV0QwCu3k7PtwdOEMpKds+wvfWDJ0SebGh4DPTAQ0nw5PW7OJLO7OQZEHcW7GsZEmD1+mUXcCkyOwk6OzsTE5N7QPDJkyepXbvitgsUSZ128PLPYGUPV/fCjy+DNivPok52VtR2NjxA/vr6MPZfevTAVEEoTdk9w/aqB90CTzYwJMHD96fWynbx+g3Wqebjlh4OdjUMCbBGw7INuBSZnQQHDx7M5MmTiY2NRSaTodfrOXToEBMnTjRZD6TK8H4chm0BpS1c2Q1bRoJWnWfRJh6GZyhDr95hxKpjJo3TglCWsgdKV8sxBVwrL2eqWSu5m6HhfIxhhvg7iXeYnTab1vII9LauMPJXqNXYIjGXFrOT4CeffELjxo3x8vIiLS2Npk2b8tRTT9GpUydmzJhRGjGWfz5PwtDNoLSBS3/Cz6NBl/uWt7G76XOUPx0X6xsLlpF9O2yX43bYSiHn8XqGBx4OXkkAdTryjQNpJ79EKvbIR2wHt8o3+sPsJKhSqfjuu++IiIjgt99+Y/369YSHh/PDDz8UeSqtZcuW4ePjg42NDf7+/hw7dizfsl27dkUmk+XaevfubSwzatSoXO/37NmzSLEVWr2uMHgDKFQQ/htsHQs605peYw/T2TRWH7pGYrqaxHQ1d9KySEjLMrkNEYTSkv5Qx0i27FviYxdvwMZBuCQcJ0WyZXX9xYZJESqhIk+H7O3tjbd38bvGf/zxR4KCglixYgX+/v4sXryYwMBALl68SK1auUefb926FbX6we3mnTt3aNWqFS+99JJJuZ49expnwwbKZnH1BgEwaD1sHgb/bQO5Ep7/xrik4MM1wajEDB6bu8tkX6s6Tmx/84kqPUWZUPoy7s8qba8yrbg82bAG1qh55cZ0kJ8lQ2bHyKzJDG78uCXCLBOFSoJBQUHMnTsXe3t7goKCHll20aJFZgWwaNEixo4da1wXeMWKFfz++++sWrWKKVOm5Cr/8PPJmzdvxs7OLlcStLa2xt3dAiPYGwUaVtLaMgLO/gRyK+i3DORyfKqbruilkMvQPVTzO30jmasJ6dSvWa0soxaqmIeHyGSr72LFKtslPCGdRae0Y2zWZE5KDVlQlDWGK4hCJcGTJ0+i0WiMX+fH3NqLWq0mLCyMqVOnGvfJ5XICAgIIDQ0t1DlWrlzJ4MGDc035v3fvXmrVqoWLiwtPP/00H3/8MdWr5/1gd1ZWFllZD3p1i71saOPe8MJK+PkVOL0RFEp47kuUigetD880dWP5y22RJMlwyw4M+e4IRyMTORaZKJKgUKry6h1Gq0b282iekE5wT1KxyGU2h6Jr42ijrNT/HwuVBPfs2ZPn18WVkJCATqfDzc3NZL+bmxvh4eEFHn/s2DHOnTvHypUrTfb37NmTAQMG4OvrS0REBNOmTaNXr16Ehobm2W4ZHBzM7Nmzi3cxD2vWH/RaQ9vgiXWGGmHvz5nRuwnL9lzh3YBG92fnffCHw9/X1ZgEh3So2KPwhfItu3fYWBPUaeCXMXDxD3Rya17NDOJQtGHIW2tvl0KtRFdRVegl0lauXEmLFi3o0KGDyf7Bgwcbv27RogUtW7akfv367N27l+7du+c6z9SpU01u81NSUvDyKoEZYFq8aEiE216H4ytBYcWrPecz5knfPGvNHXyrA1c4FplY/M8WhEdISDPc+TjYKA0deNtegwu/gkJFar81HNr4oJmmbSW+FYZCJsHshdcLY+vWrYUuW6NGDRQKBXFxcSb74+LiCmzPS09PZ/PmzcyZM6fAz6lXrx41atTgypUreSZBa2vr0us4aTXY8Ff21wlwdAXIlcie+TjPoo/VdUYpl3Ez6R437mZQx8Uuz3KCUBw6vWScFaalZzXYMR7O/WK4Wxn4A85+PWn8z37CY1MBw//LyqxQQ2ScnJyMm6OjIyEhIRw/ftz4flhYGCEhIcYF2gtLpVLRtm1bQkJCjPv0ej0hISF07Njxkcf+9NNPZGVl8fLLLxf4OTdu3ODOnTt4eOSeIqhMPDYcnvvC8HXoVxAyB/KYdNVOpaR5bcP3UNQGhdJyMTaV1Ewt1VQymod9CGd+NIxkeGkN+BmGkj1xf6iMTAat7z9TXFkVqiaYc6jJ5MmTGThwICtWrDC2r+l0OsaPH4+jo/mLqgQFBTFy5EjatWtHhw4dWLx4Menp6cbe4hEjRlC7dm2Cg4NNjlu5ciX9+/fP1dmRlpbG7NmzeeGFF3B3dyciIoJJkybRoEEDAgMDzY6vxLR7xXDb8ecHcHCRYTxht6m5ivn7unIqOoljkYkMeKyOBQIVKrtjkXeQoWeJw0bkp/8AmQJe+B6aPGcs071xLVYejKRFbSccbCr5HJiSmWrUqCGFh4fn2h8eHi65urqaezpJkiRp6dKlkre3t6RSqaQOHTpIR44cMb7XpUsXaeTIkbk+C5D+/vvvXOfKyMiQnnnmGalmzZqSlZWVVLduXWns2LFSbGxsoeNJTk6WACk5OblI1/NIh7+SpFmOhm3fglxv7z4fK9Wd/JvU7bM9Jf/ZgiBJ0vgfjktrpr9g+D/4kbMknd6SZ7m//4uVIm+nlXF0hVOSv6Nmd4xotVrCw8Px8/Mz2R8eHo5ery9SIp4wYQITJkzI8729e/fm2ufn55fvGh62trb89ddfRYqjTHR809BGuHsW/POxoUb4xDvGt9vVdUUmg6sJ6cSnZlLLwcaCwQqVjaTX80TEIoYqdyEhQ9bva2j5Up5lezR1y3N/ZWN2Ehw9ejRjxowhIiLC2Ct79OhR5s+fb7yFFQrw5LuGRLjnY9g109Ag3XE8YJhtprG7IxdiUvg38i69W1qoHVOofCSJ5P9NY6j0GwDa3ouxaj3EwkFZntlJcOHChbi7u/P5558bp9Ty8PDggw8+4P333y/xACutLh+AXgP7PjVMzKqwgg5jAUO74IWYFI5F3hFJUCgZkgT/fIzzyeUAfOf4FmPbj7JsTOWE2RMoyOVyJk2axM2bN0lKSiIpKYmbN28yadKkIk+gUGV1nQpPvmf4+o+JcNzQAdXB1/Bo4KGIO2RqdJaKTqhM9i2AAwsBmKUZSXKz4RYOqPwwOwnm5OjoWKQeYeE+mQy6z4KO99tDf3sXTq6nvY8rchlciU+j0/x/WPjXRWKTxYLuQhEd+Bz2fgLAUuUo1uoCjX9oBZBJ+fUwPMLPP//Mli1biIqKMpnRBeDEiRMlFpylpKSk4OTkRHJyctkkeUmCPyfDsW8AGTz/Ddv1T/LpznBi7ic/pVxGYHN3RnfyoW1dFzHLjFA4h5fC34Z5PlOemE7LkGYo5DJOz3rGZELViqYkf0fNrgkuWbKE0aNH4+bmxsmTJ+nQoQPVq1fn6tWr9OrVq1jBVFkyGfT61DCWEAm2v05/ZSgHJnVj+bDH6ODrilYv8fuZGF5cEUqfrw5y4LKYnr+qK7D+cmSFMQHSbTp7ag4DoJmnY4VOgCXN7CT49ddf8+2337J06VJUKhWTJk1i165dvP322yQnJ5dGjFWDTAbPfg5thoOkh63jUF78H71aeLDltY788XZnBrXzwlop59zNFF5de5zke2LBpqoq4nYa/p+EMPnnM7mmYwPg3+9h52TD1099AF0mGZ9C6uAjboVzMjsJRkVF0alTJ8AwJi811fB84fDhw9m0aVPJRlfVyOXQZwm0GgKSzjAVV/jvADT1dOTTF1tyZGp36tWwJ0ur55/wuAJOKFRW/0YmEp+axY/Ho9n/8F1B2Fr4/f5IjSfehW7TgQePYor2QFNmJ0F3d3cSEw3fTG9vb+Naw5GRkQVXz4WCyeWGSVhbvGSYgWbLSLj0t/FtF3uVcdjMn2djLRWlYGE5F+mKScrRaXZqI/zv/uD7x9+EgI9AJiMxXc3l+DQA2ouaoAmzk+DTTz/Nr7/+ChgGTr/33nv06NGDQYMG5bkesVAEcgX0XwFN+xvGEv74Mlx5MMlEz+aGGXb2XbpNhlqsWFcVpWY++LmnZN5vFjmzBbaPByToMA4C54FMRpZWx7StZwHwc3PAxV5lgYjLL7NbR7/99lvj43Fvvvkm1atX5/Dhw/Tt25fXXnutxAOsshRKw0Pteq1h4abNQ2HoFqjXhaYejni52hKdeI99F2/Tq4UYUF3V5EyCqZkaOLfVMCcgErQdDb0WgExGepaW19eHceByAiqFnCnPVq7lMkuCWTVBrVbLxx9/TGzsg9uwwYMHs2TJEt566y1UKvEXpkQprODF1dCoJ2gzYdNguHYImUxGr+aGxPfN/quMW3ecbgv3Ep8ixhJWFWlZDzrFaseEwC+vGjrU2rwMvReBTEZyhoaXVx7lwOUE7FQKVo9uTze/3IuXVXVmJUGlUsmCBQvQasUtWJlRqmDgOsNKdpoM2PASRB0lsJnhlvhUdBJ/n48jMiGdE1FJlo1VKDPZbYLd5WEMvDbT0JHWcrChY00uJz41k0HfhnIyKgknWys2vOpvnCNQMGV2m2D37t3Zt29facQi5EdpbVjKs15X0KTD+hdoI4/IVeyexvSPk1qrF4/dVVKpmVq6yk/xtdWXKNFC8xeh/9cgVxCdmMFLK0IJj02lloM1W17rSJtKPkV+cZjdJtirVy+mTJnC2bNnadu2ba5V3vr27VtiwQk5WNnC4E2wcSBcO4B8/QBWdP+Gry86kKnRcSkujau3043FJUmi0/x/yFBrOTXzGVTKYj0hKVjQ3ovxpGVpea6lp3FfvdR/mWb1BdYyLYetn6TT/fWtL8el8vLKo8SlZOHlasv6Mf7UrW7/iLMLZj82J5fn/8skk8nQ6Sp+zaPMH5szR1YabHgRokLBxhlG/UbQfh1bT9wE4ODkbtRxsSNTo6PxhzsB2DOxK741xC9CRXMnLYvriRkM+PowAIemPE1tZ1uIPEDm2gHYoOZvXVsWOU9n5/vdOR2dxKjVx7iboaFhrWqsf9UfN8fKOR+lRR+b0+v1+W6VIQGWe9bVYNhPUKc9ZCbBun746qKMb/8Qeh3A5CmCPJ8oEMq9NzeeMCZAgFtJ9+D6Ydg4EBvUhOjaMEHzNnezJEIj7jD0uyPczdDQqo4TW17rWGkTYEkT90gVkbUDvPwLeLaBjDuMjnib+jJDTXD9kevEp2SizZH4xCD2iic9S8u/1+6a7JPf+NfQMabJ4JDUivGad1BjRUKampGrj5Gu1tGxXnU2jH1cjAU0Q6GT4L179/jtt9+Mr7PX6s3ePvjgAzIzizZEY9myZfj4+GBjY4O/vz/Hjh3Lt+yaNWuQyWQmm42N6V88SZKYOXMmHh4e2NraEhAQwOXLl4sUW7ll4wTDt4F7C6pp77JJNQ9fWQzpah0vrgglMuFB+6BOJMEKI0truJs6EXXXpAbfUhZBi72vgDoNyecpXsl6jywMiU6nl1Br9fRo6sbq0e3F5AhmKnQSXLt2Ld98843x9VdffcXhw4c5efIkJ0+eZP369SxfvtzsAH788UeCgoKYNWsWJ06coFWrVgQGBhIfH5/vMY6OjsTExBi369evm7y/YMEClixZwooVKzh69Cj29vYEBgYWOUmXW7YuMHwHcTb1qSVLYqNqHo87JxOVmMGIlUeNxTRakQQrgh9Cr+E3Yyc7z8Xwb44lV5vJIvlBFYxKmwZ1nyBtwA/GBJhtQJvaLB/2GDZWYmJjcxU6CW7YsIFx48aZ7Nu4cSN79uxhz549fPbZZ2zZssXsABYtWsTYsWMZPXo0TZs2ZcWKFdjZ2bFq1ap8j5HJZLi7uxs3N7cHC8JIksTixYuZMWMG/fr1o2XLlqxbt45bt26xfft2s+Mr9+yrs7bBl1zS18ZDlsgGq3k8VTODlBxPFCz4K5xnvthHTPI9CwYq5CctS8vWEzf4cMd/ALyx4QTHrhmSYGNZFOtVwTjJMoiybwlDt5AmWQOgUshZOqQNs/s2Y+FLrVAqROtWURT6u3blyhVatGhhfG1jY2PSU9yhQwfOnz9v1oer1WrCwsIICAh4EJBcTkBAAKGhofkel5aWRt26dfHy8qJfv378999/xvciIyOJjY01OaeTkxP+/v75njMrK4uUlBSTrSJJs3JhmHo6EXoPFKk3WC2fS0DtB08UHLicwKW4NOPzo0L50u+rgwRtOW18LUlwMiqJhrIbbFDNw0WWxkl9A5a4fwLW1YyPzFWzUdKnlScjO/kgl4tJdouq0EkwKSmJrKws4+vbt2/j4+NjfK3X603eL4yEhAR0Op1JTQ7Azc3N5NG8nPz8/Fi1ahU7duxg/fr16PV6OnXqxI0bNwCMx5lzzuDgYJycnIybl5eXWddhaQ1qVeM2zgxVTwcXXxTJ1/lW9xFuJJqU23PxtrHNSSgf9HqJiBzjO7PV0UWzUTWP6rJUzuh9GamezI17hrY+YxIUbX8lotBJsE6dOpw7dy7f98+cOUOdOnVKJKhH6dixIyNGjKB169Z06dKFrVu3UrNmTZP2SnNNnTqV5ORk4xYdHV2CEZe+IR28ebt7Q5a93htG/QbOdZEnRbLVPpiaJJmU/es/MQdheaDXS/wSdoP/buW+6/CRxbBRNY+asmT+09dluHoqKdhzO9VQych+ZE4kwZJR6CT47LPPMnPmzDw7F+7du8fs2bPp3bu3WR9eo0YNFAoFcXGmv5hxcXG4u7sX6hxWVla0adOGK1euABiPM+ec1tbWxkWjKuLiUVYKOUE9GtHOxxWc6sDI/4GTF7V1N9mgmkd1Hsz4veGIoRNJo9Oj1ektFXKVt+FYFO//dJo+Xx002e8li2Ojah5usiTC9V68rJ5KMtUAHiTBHLfDQvEVOglOmzaNxMRE/Pz8+Oyzz9ixYwc7duxgwYIF+Pn5cffuXaZNm2bWh6tUKtq2bUtIyIO58vR6PSEhIXTs2LFQ59DpdJw9exYPD8OsKr6+vri7u5ucMyUlhaNHjxb6nBWeS10Y+SuJiho0kt9kveoTXEhBLoOjkYlcjE0l8Iv9PLf0IHoxkNoiTl6/m2tfbW6zSTUPT1kil/W1ed92Dnd58Ac5JVPLraR7xhlkHERNsEQU+rvo5ubG4cOHeeONN5gyZYpxAK5MJqNHjx58/fXXudrhCiMoKIiRI0fSrl07OnTowOLFi0lPT2f06NEAjBgxgtq1axMcHAzAnDlzePzxx2nQoAFJSUl89tlnXL9+nVdffdUYz7vvvsvHH39Mw4YN8fX15cMPP8TT05P+/fubHV+F5VqPhe4LeefGuzSRR7PZ5lO+9l7Ejkv3+HRnOFfvjyO8m6GmejVrCwdb9SSkm67S6M4dNqk+po4sgQi9B2OkD2nuXZf/Hpo9/ETUXWOboIOoCZYIs76Lvr6+7Ny5k8TEROPtZ4MGDXB1Lfp03YMGDeL27dvMnDmT2NhYWrduzc6dO40JNSoqyqQX+u7du4wdO5bY2FhcXFxo27Ythw8fpmnTpsYykyZNIj09nXHjxpGUlMSTTz7Jzp07cw2qruzu2nozVD2dzaq5+MkimZs2kz28zT/hD8ZgJqSJJFgW9HqJ7w5cxc/dga5+tUhIfdCJWIu7bFTNw1t+m2t6N4aqp9OggS82ytxj/sKu38XRxgoQt8MlpUjrDld25XoCBTO8s/kkO07dopEsmh+t5+FCCif0DRihnkIadgCsHtWebo3FRJul7bv9V5n3xwUA3uxWn5/DbhCXkkUNktmsmksD+S2i9TUZpP6QW9Tg3YCG3Lx7j5/CDKMeOvi6ciwykVZezrSr68LKg5G83qU+U3pVzZmiLTqBglBxqO4Pnr0kefGmchbYOPOY/AqrVQuww9DB9damk2w8GvWo0wj5SM7QMPGn0xyOSCiw7KpDkcavl+2J4E6aGldS2KCaRwP5LW5K1Rmimc4tDBOfdvBxRal4MPavS6OaAJy/lUxCmqEWKW6HS4ZIgpWYtdWDH+9lmQ+M2E66zJ728kusUn2GDVmkZWmZtu0sZ24kWSzOimrBX+H8HHaDod8dfWS55HsaYpJNR1VU06ewQfUJfvIbxEouDFVP54b0oEbextsFRY4B0I3dHahRzRqNTuLI1TuGc4iOkRIhkmAlplI81Kbk2Yawp1aRKtnyuPwC31l9jjWGBvq+Xx3i3c0n2XD0eh5nEvJy427hHkP865xp54YjaaxXBdNEHkW8ZBjkfl0yHb5lq1KgzNEWXs1aSdu6zgDEpWQZ9wnFJ5JgJZbXbNKPd36GkerJpEvWdFac4xurL4yJcPupW0zfdk4MmykkZY6a2s2ke/k+jbPj9E3j1w5ksE41n+bya9zFkaHqaVyVPE3Ku9+fBzBnTbCajZLHHpoiX9wOlwyRBCuxvJKgSikn6JXhbG60CI3chq6K0yyz+hIrHky4IFJg4eR8XveJ+f/Q76tDucqkZWk5HGG4fbXnHmtUn9JafpVEqRofV/+UK1Lup6ym9W4CmCZZB2sr2tY1TYKid7hkiCRYiVnns67Ikw1rMGbYy/zZYjGZkhUBipMstVpqWLAH0IsBA4WifGjSgvDY1Fxl4lMykSSwJZNVqs9oK79MkmTPy+ppZLj45XleTydDTVAme3B+e2sFzWs7YaUwTYxC8YkkWImpCphaKdPrScZq3idLsqKn4l8WWy1DgWFS1gOXb5dRlBWXIo+ZWyRJ4tzNZNRawyOJielqbMhilWoh/vJwUiQ7XlZP5bzkQ418xmdmzwqd8/a6mo0SGysFTT2dTPYJxSeSYCVW0ApzHk42HNC35DXNu6glBc8pjvK51XLORicyfOUxVuyLYOy645yKTiqbgCuYh2uCAK3n7OK5pQd5a9MJABKTU/nWahEd5edJlWwZoZ7COakeANWrqfJMhNXvJ8Gcy6Va3x843TZHu6DoGCkZIglWYgUlwfY+rni52rJX34bxmnfRSAr6Kw7zmdU3yNAz/89wdp2Po/+y3G1dAnnO4Zd8z/Bc71//xYE2i+YHx/OU4iz3sGGUehKnpAbGsjWqWbPulQ7Yq0x78bOfCMnU5J7g4rH7PcQgOkZKikiClVhBt8M2Vgr2TuwGwG59W97SvIVWkvOC4gDByu+RIWaZeZS8aoLZrNDClpF43j7IPUnFMo9PCJNM2wDrVrejqacjZz8KNNmfnVzvqXP3Nrf3cUUhl+FsZ5Vvm69gHvFdrMRy1gTz+3VVyGU43q9R7NR34F3Nm+gkGYOVe5mrXI3oK86fIp81uJVoWWK1FC79iUamYoxmIkluHXKVa3P/1ja/WaHvaXInQTdHG9a90oFVo9qbdJwIRSeSYCVW0O1wtt/f7mz8+jd9R97XvIFekvGyMoRZynWARNSdDDYejTI2+AuQV0VbgY4vrL6ml+JftDIrJimncFjfPM+2v4La9DLzSIIATzSokWvMoFB0olGhEivs7ZKXqx3fjWjHP+HxbDoWxXb9k1hptXxm9S2jlX+hRUHXhaCXZCSkZfF294alHHnFoHyoJihHz0KrFfRRHEEtKXhN/S577hkmOGji4UjPZu7s/C/vJR6y5WzCECOVyoaoCVZiha0JAvRo6kbwgBbGB/V/0nVlqmYMAGOVfzBR8SMgsWjXJVbsixBrlQDyHLejMvR8qvyW5xWH0EgK3tS8wx59GwA+faEF3RvXYsXwtgWe09H2wdi/2f2aUaOairn9mpV88IKRSIKVWFEazr8f2Y7eLQ2zdG/SdedDzSgAxit/5T3lLwDM/zOcub+Zt7JgZaPR6Y2zvMjQM0+5kpeU+9FKct7WTGCXvh0ALz/uzaD23oVeDtPJ9sHNWRMPR/6dHsDwjj4lHr/wgEiClViuCRQKwUohZ9nQx4yvf9A9wxzNcADeUW5lgmIbAOuPRLH1xA2e/fIA0YkZJRNwBbHpWBTNZv1FaMQdQGK2ci1DlXvQSTKCNOP5U+9vLOvn5mDWuZ1sTZ8CEZ0fpU8kwUrMnNvhR1ml68UnmiEATLT6idcU/wMgaMtpzsekMPmXMyXyORXF1K1nUWv1nL2ZxEzlD4xQ7kIvyZioeZ1f9Z2M5RRyGc+19HzEmR7IHiD9bAuPUolZyJ/oGKnESioJAnyr64MSHZOstjDVahNaFKzUPQvA4Yg7fLj9HLP6NC30bV/FJzFVuZFXlDsBmKwdyzb9g172Sx/3Muv7/9vbT3IsMrHQSVMoOVXlf2yVVFJJcOIzjQD4WtefxdoBAHxotZ4Rir+MZX44cp0tx2+UyOeVR+duJjPom1BORN3FwUbBROUWXlP+DsA0zRh+0nU1KW/u997DyZZ+rWvn+TyyULrKRRJctmwZPj4+2NjY4O/vz7Fjx/It+91339G5c2dcXFxwcXEhICAgV/lRo0Yhk8lMtp49e5b2ZZQ7JsMtinGeYf51jV8v1r7AMm1fAOZYrWWo4sHSptfvpBfjU8q3tzed5GhkIgO+Psxkmx1MUO4AYKZmJBt13U3KijxWsVg8Cf74448EBQUxa9YsTpw4QatWrQgMDCQ+Pj7P8nv37mXIkCHs2bOH0NBQvLy8eOaZZ7h586ZJuZ49exITE2PcNm3aVBaXU67krI1oirHQevasJgYyPtMO4httbwA+sVrJS4q9APx2JqbIn1HepWQangker9jOy5kbAZireZl1ugePvPm5OTB/QAv+fq9LvudZ+FIrAKZW0QWSyiOLtwkuWrSIsWPHGtcZXrFiBb///jurVq1iypQpucpv2LDB5PX333/PL7/8QkhICCNGjDDut7a2xt3d/eHDq5ScQ2SK+6SHbw17IhOya3oygrVDsULHK8qdfKr8Dq2kYFtSZ8KuJ9K2btGXYC2vajvb8vy9rUyy2gJAsGaIsU00244JT2Bj9ege+Rfb1qFHEzec7MRcgOWFRWuCarWasLAwAgICjPvkcjkBAQGEhoYW6hwZGRloNJpcax/v3buXWrVq4efnxxtvvMGdO3fyPUdWVhYpKSkmW2WQ83a4uEkw5x1eI7dqgIw52uH8oA1ALpNYaLWCvvLDDPzmSKUaQ6jV6ZEkiWH8wXQrQw1woeYlvtH1MSnXza9mgQkwm0iA5YtFk2BCQgI6nc640Ho2Nzc3YmMf/XhRtsmTJ+Pp6WmSSHv27Mm6desICQnh008/Zd++ffTq1QudLu+nHIKDg3FycjJuXl5eRb+ociTng/naYq4bknO42s53nsrey0ztKDZpu6GQSSyy+ppnOMLKg5GEx6ZQ0Ze0Ts/S8uSne9i0bCYDE5YB8KX2eb7SPW9SzkohY9qzTSwRolACLH47XBzz589n8+bN7N27FxsbG+P+wYMHG79u0aIFLVu2pH79+uzdu5fu3bvnOs/UqVMJCgoyvk5JSak0ibCk5By0mzO5SsiZph2DEh0vKfezxOorxmsU9FxsWGT8g8CK1fYlSRL/3UphwV8XOXr1Ds9Luxmq/h6Ar7V9+UL7orFsYDM3pj/bFCc7q1yDnIWKw6I1wRo1aqBQKIiLizPZHxcXV2B73sKFC5k/fz5///03LVu2fGTZevXqUaNGDa5cuZLn+9bW1jg6OppsgqmHezy/GtoGb1c7wJAIJ2vHsU33BFYyHcusvqSb/CTL9kRYINLi+fX0LZ5bepD9l27TR9rDJ8qVAHynfZYF2kFkNww80aA63wxvh3d1O5EAKziLJkGVSkXbtm0JCXkwzEKv1xMSEkLHjh3zPW7BggXMnTuXnTt30q5duwI/58aNG9y5cwcPDzEav6hkD81I+FxLT/ZP6kbEJ4bOAT1yJmpe5zfd46hkOlZYfcFT8tMcvXqnwtwWh12/y0e//gdAf/lBFii/RS6TWK0NZJ52GDlbRte94p/PWYSKxuJDZIKCgvjuu+9Yu3YtFy5c4I033iA9Pd3YWzxixAimTp1qLP/pp5/y4YcfsmrVKnx8fIiNjSU2Npa0tDQA0tLS+OCDDzhy5AjXrl0jJCSEfv360aBBAwIDA/OMQShYfo+wKuQyto03PCqmQ8G7mvH8qWuPtUzLt1aL+PL77/k5rPwPog6NuMMLyw9zN0PDc/JQPrdajlwmsV7bndnaEWQnwDoutvw7PUAMaq5ELJ4EBw0axMKFC5k5cyatW7fm1KlT7Ny509hZEhUVRUzMg/Fny5cvR61W8+KLL+Lh4WHcFi5cCIBCoeDMmTP07duXRo0aMWbMGNq2bcuBAwewts57dS+hYI96kL+NtwuRwc9iYyVHi5K3NW+xS/cYNjINK60WcuLAb2UYqfnO3EhiyHdHAAiUHzOsuieT2Kztyofa0eSsAW4b/wQ1HcT/o8qkXHSMTJgwgQkTJuT53t69e01eX7t27ZHnsrW15a+//npkGcF8BVV8ZDIZSwa3YdwPYWhQ8qbmHb5hEd0Up5mRNAui2rAsogb/hMfzw5gO2KnKxX89APreXzQ9QB7GV1ZLUcr0/KLrzFTtq0j36wmz+jRlmH/dEn0eWygfxE+0kitosaXCKsyMTjlb/tRY8brmPfbrWmAvy0Ja/yK7/v6dsOt32Xg0qkRiKi6NTs87m08C0FV+kq+tFmMl07Fd14kPNK8ZEyDAqE4+IgFWUuKnWsmV1C/uwx0jhZGFinGaIEJ1TZGpU1mnmk8L2VXSs8rHrNT/O32LHadu0Vl+hm+sFqOS6fhN529YYwU51ko5Qzp4ET63p5jXrxITSbCSK6kkaG4/QI+mhjbdTKwZo5nIMb0fjrIMflAF45oaXiIxFceiXZcI2nKajvL/+M7qc6xlGv7StTOstoeCatZKwuf2JHhAy0I/CSJUTCIJVnIldTtcqPvhHBQ5ymdgw2j1JML0DXGWpdP71OtM/noT6VnakomtCJaEXKa9LJyVVguxkWnYrWvDBM3baFESPKAFZ2Y9I2p/VYRIgpWctZVlaoI580eL2k6kY8so9WRO6evhKkvjg7hJjPt8Q77LSpa2x2SXWK1agJ0si726VozXvIvmfj9hn1ae+a4FLFQ+IglWciXWMWJm+ZwrsW0b34k3utYnFTtGqKdwTu9DDVkKX2TNZO2vu0skvsJIylAz8JtQdu36nTWqT6kmy+Sgrhmvad5DjRXvdG/I1U+eLXA9YKFyEUmwkiu5NsGC06DJgyE5iisVcib3bMz2N58ghWq8rJ7KBb03tWRJ9D3zOiReLZEYHyUxXU3rObtIvxZGh4Ov4ii7xxF9E17VTCQLFatGteO9Ho1EDbAKEkmwkiux3mFzb4fz2Nfay5nZfZuRhAPD1NO4pK+NhyyRtG96wd3rJRJnfh6bu4smsuusVwXjJMvgX30jXlF/QCbWXPq4F083div4JEKlJJJgJWdVYuMEzcuC+ZUf2cmHI1O7k4gjw9TTidB7UC0rFv2a5yApuiRCzVMjWTTrVZ/gIkvjhL4Bo9WTyMCGkx/2EOP/qjjx06/kirIAe17MbxPM/z13JxuOTuvObZwZop5BpN4NeXIUmSt7Q8qtYsWZp9uX2KCaR3VZKqf19Rilnkwadmx/84mHlg4QqiKRBCu5kuoYKUybYE4FlXZztCEy+FnicWGoegZR+prYpF4nYuHTzFofUsDRZrgTQdq3vagpS+E/fV1GqKeQgj27g7rQ2su55D5HqLBEEqzkLNYmWIgDssvEUJ0h6hnckGpQXx7Dy5fegrTbRQnTSKPTo0u4imbVs1TTJHBB78XL6qkkU41f3uhIg1rVinV+ofIQSbCSK8ve4ZxPD5t7+3yTmgxRT+eW5EpD+U3CF3TjdtzNgg/Mg0an58XgzcQs7YFVeiyX9LV5WT2NuzhybX7vSrkQlFB0IglWckW9HX4455VGTTCnuf2b4+nThKHq6cRJzjSWRxO/rBdkJJr3wUD0tcssVc+kjiyBCL0Hw9TTsXZy4+onzxZ8sFDliCRYyRX1iZGHU5j5vcPmf96qUe1RO/oyVD2d25ITzeTXOTP/aS5fL/ykrHfjonDaMgBv+W2u6d0Yqp7O92/25vDU7mIMoJAnkQQrOZWiaA//P5z0zE0fRUk39tZKDk/tztxXBzBUPZ07kgMt5ZGkrexH8t2Ca4Qbdh8jcdkzVM+6QZS+JkPUM6hZ24dWogNEeASRBCu5orYJPlxpMv92uEgfC0Cn+jVo2eZxQzueVI028itc+iIQstLyPyg9gfb7R1FfHsNNqTpDNTOoXtuX/014suiBCFWCSIKVXFGT4MPzB5o7RMbc8g/r3qQWF6S6hh5dyY728kscmRfApgN5LOyekciFT7vRSH6TGMmVIeoZ/D5zGL+91VnMBCMUqFwkwWXLluHj44ONjQ3+/v4cO3bskeV/+uknGjdujI2NDS1atOCPP/4weV+SJGbOnImHhwe2trYEBARw+fLl0ryEcqvIg6UfrgkW4pCczw4XN/dkn+s/yZfh6qmkSLY8Lr+A19+vcvlmvLHc7fhYzs3vRhN5FPGSM0PV0wn5eJRYBlMoNIsnwR9//JGgoCBmzZrFiRMnaNWqFYGBgcTHx+dZ/vDhwwwZMoQxY8Zw8uRJ+vfvT//+/Tl37pyxzIIFC1iyZAkrVqzg6NGj2NvbExgYSGZmZlldVrlR5N7hh1+X0GNzhaXPkVG3z3vL8JSHZMOTiv+4teIFUtJSuRUXx82vetFcfo3bkiOvMpN/PhlTYo8KClWDxf+3LFq0iLFjxzJ69GiaNm3KihUrsLOzY9WqVXmW//LLL+nZsycffPABTZo0Ye7cuTz22GN89dVXgKEWuHjxYmbMmEG/fv1o2bIl69at49atW2zfvr0Mr6x8KPLtcHHbBIv0qQ/knJBGLpfx87x3Dc/7StZ0UZzhwoIA7ix7htbyqyRK1fjWZzG/zh4jbn8Fs1k0CarVasLCwggICDDuk8vlBAQEEBoamucxoaGhJuUBAgMDjeUjIyOJjY01KePk5IS/v3++58zKyiIlJcVkqyxqVCva8pAPt+mZO8decTtSHl6wXS6X8a/UmDGaidyTVPjLw2khv0aSZM/n7p8xffQL5n2gINxn0SSYkJCATqczrjGczc3NjdjY2DyPiY2NfWT57H/NOWdwcDBOTk7GzcvLq0jXUx71bO7OwHZ1+OT5FmYdN7tvMwDGd60PwJRejWnm6UjwgPzP07F+dQDq1bRnUDtvANp4Oz/yc5rXdgSgRxPTn5e/r+Fc9irTIT6h+ma8qnmfTMmKFMmOCz3WMe+NoYW8KkHITUyhC0ydOpWgoCDj65SUlEqTCBVyGQtebGX2cS+18+LpxrVwvT/LipujDb+/3fmRxzjbqfhvdiDWSjlKhZxj07obj8/Pjjef5J5Gl6um6e5kw7Fp3alm82B/+NyeHItMpFP9Xqzc+RTPt/Olo5uH2dcmCDlZNAnWqFEDhUJBXFycyf64uDjc3d3zPMbd3f2R5bP/jYuLw8PDw6RM69at8zyntbU11tZFu22szKoX4VbaPkcyq+VoU2B5hVyW7632w8fbWCl4qlFNAF7r3cns2AQhLxa9HVapVLRt25aQkAdTJ+n1ekJCQujYsWOex3Ts2NGkPMCuXbuM5X19fXF3dzcpk5KSwtGjR/M9pyAIVZhkYZs3b5asra2lNWvWSOfPn5fGjRsnOTs7S7GxsZIkSdLw4cOlKVOmGMsfOnRIUiqV0sKFC6ULFy5Is2bNkqysrKSzZ88ay8yfP19ydnaWduzYIZ05c0bq16+f5OvrK927d69QMSUnJ0uAlJycXLIXKwhCiSjJ31GLtwkOGjSI27dvM3PmTGJjY2ndujU7d+40dmxERUUhlz+osHbq1ImNGzcyY8YMpk2bRsOGDdm+fTvNmzc3lpk0aRLp6emMGzeOpKQknnzySXbu3ImNTcG3Z4IgVC0ySXpoLIJASkoKTk5OJCcn4+joaOlwBEF4SEn+jlp8sLQgCIIliSQoCEKVJpKgIAhVmsU7Rsqj7GbSyvT4nCBUJtm/myXRpSGSYB5SU1MBKs1TI4JQWaWmpuLk5FSsc4je4Tzo9Xpu3bqFg4NDuZ6VJPvxvujo6CrXiy2uvWpfe1RUFDKZDE9PT5MhdEUhaoJ5kMvl1KlTx9JhFJqjo2OV+2XIJq69al67k5NTiV276BgRBKFKE0lQEIQqTSTBCsza2ppZs2ZVyRlwxLWLay8pomNEEIQqTdQEBUGo0kQSFAShShNJUBCEKk0kQUEQqjSRBCuYxMREhg0bhqOjI87OzowZM4a0tLRHln/rrbfw8/PD1tYWb29v3n77bZKTk8sw6qJZtmwZPj4+2NjY4O/vz7Fjxx5Z/qeffqJx48bY2NjQokUL/vjjjzKKtOSZc+3fffcdnTt3xsXFBRcXFwICAgr8XpVn5v7cs23evBmZTEb//v3N+8Biz00tlKmePXtKrVq1ko4cOSIdOHBAatCggTRkyJB8y589e1YaMGCA9Ouvv0pXrlyRQkJCpIYNG0ovvPBCGUZtvs2bN0sqlUpatWqV9N9//0ljx46VnJ2dpbi4uDzLHzp0SFIoFNKCBQuk8+fPSzNmzMi17EJFYe61Dx06VFq2bJl08uRJ6cKFC9KoUaMkJycn6caNG2UcefGZe+3ZIiMjpdq1a0udO3eW+vXrZ9ZniiRYgZw/f14CpH///de4788//5RkMpl08+bNQp9ny5YtkkqlkjQaTWmEWSI6dOggvfnmm8bXOp1O8vT0lIKDg/MsP3DgQKl3794m+/z9/aXXXnutVOMsDeZe+8O0Wq3k4OAgrV27trRCLDVFuXatVit16tRJ+v7776WRI0eanQTF7XAFEhoairOzM+3atTPuCwgIQC6Xc/To0UKfJ3tKcqWyfD46rlarCQsLIyAgwLhPLpcTEBBAaGhonseEhoaalAcIDAzMt3x5VZRrf1hGRgYajQZXV9fSCrNUFPXa58yZQ61atRgzZkyRPrd8/hYIeYqNjaVWrVom+5RKJa6ursTGxhbqHAkJCcydO5dx48aVRoglIiEhAZ1OZ1xsK5ubmxvh4eF5HhMbG5tn+cJ+X8qLolz7wyZPnoynp2euPwrlXVGu/eDBg6xcuZJTp04V+XNFTbAcmDJlCjKZ7JFbYX8BHiUlJYXevXvTtGlTPvroo+IHLpQ78+fPZ/PmzWzbtq3Sr66YmprK8OHD+e6776hRo0aRzyNqguXA+++/z6hRox5Zpl69eri7uxMfH2+yX6vVkpiYiLu7+yOPT01NpWfPnjg4OLBt2zasrKyKG3apqVGjBgqFgri4OJP9cXFx+V6nu7u7WeXLq6Jce7aFCxcyf/58du/eTcuWLUszzFJh7rVHRERw7do1+vTpY9yn1+sBwx3SxYsXqV+/fsEfXNQGTKHsZXeMHD9+3Ljvr7/+KrBjJDk5WXr88celLl26SOnp6WURarF16NBBmjBhgvG1TqeTateu/ciOkeeee85kX8eOHStsx4g51y5JkvTpp59Kjo6OUmhoaFmEWGrMufZ79+5JZ8+eNdn69esnPf3009LZs2elrKysQn2mSIIVTM+ePaU2bdpIR48elQ4ePCg1bNjQZIjMjRs3JD8/P+no0aOSJBkSoL+/v9SiRQvpypUrUkxMjHHTarWWuowCbd68WbK2tpbWrFkjnT9/Xho3bpzk7OwsxcbGSpIkScOHD5emTJliLH/o0CFJqVRKCxculC5cuCDNmjWrQg+RMefa58+fL6lUKunnn382+fmmpqZa6hKKzNxrf1hReodFEqxg7ty5Iw0ZMkSqVq2a5OjoKI0ePdrkP3tkZKQESHv27JEkSZL27NkjAXlukZGRlrmIQlq6dKnk7e0tqVQqqUOHDtKRI0eM73Xp0kUaOXKkSfktW7ZIjRo1klQqldSsWTPp999/L+OIS4451163bt08f76zZs0q+8BLgLk/95yKkgTFVFqCIFRpondYEIQqTSRBQRCqNJEEBUGo0kQSFAShShNJUBCEKk0kQUEQqjSRBAVBqNJEEhQEoUoTSVCoMEaNGpXnDDs9e/a0dGhCBSZmkREqlJ49e7J69WqTfdbW1nmW1Wg0uWbLUavVqFQqsz+3qMcJ5Z+oCQoVirW1Ne7u7iabi4sLADKZjOXLl9O3b1/s7e2ZN28eH330Ea1bt+b777/H19fXOMdeVFQU/fr1o1q1ajg6OjJw4ECTKZzyO06ofEQSFCqVjz76iOeff56zZ8/yyiuvAHDlyhV++eUXtm7dyqlTp9Dr9fTr14/ExET27dvHrl27uHr1KoMGDTI518PHCZWTuB0WKpTffvuNatWqmeybNm0a06ZNA2Do0KGMHj3a5H21Ws26deuoWbMmALt27eLs2bNERkbi5eUFwLp162jWrBn//vsv7du3z/M4oXISSVCoULp168by5ctN9uVcUCjnIlTZ6tata5LILly4gJeXlzEBAjRt2hRnZ2cuXLhgTIIPHydUTiIJChWKvb09DRo0eOT7hdlX2M8SKj/RJihUOU2aNCE6Opro6GjjvvPnz5OUlETTpk0tGJlgCaImKFQoWVlZuZbRVCqVZq02FhAQQIsWLRg2bBiLFy9Gq9Uyfvx4unTpkufttFC5iZqgUKHs3LkTDw8Pk+3JJ5806xwymYwdO3bg4uLCU089RUBAAPXq1ePHH38spaiF8kxMry8IQpUmaoKCIFRpIgkKglCliSQoCEKVJpKgIAhVmkiCgiBUaSIJCoJQpYkkKAhClSaSoCAIVZpIgoIgVGkiCQqCUKWJJCgIQpUmkqAgCFXa/wGr3l37Plz/TwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019032, + "end_time": "2024-03-22T21:15:30.870163", + "exception": false, + "start_time": "2024-03-22T21:15:30.851131", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 7458.159698, + "end_time": "2024-03-22T21:15:33.617760", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tvae/42/mlu-eval.ipynb", + "output_path": "eval/treatment/tvae/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tvae/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "tvae" + }, + "start_time": "2024-03-22T19:11:15.458062", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tvae/model.pt b/treatment/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..87c21bad88e012c204f0033f8e7f15452405dc90 --- /dev/null +++ b/treatment/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1d74dc64fb3cc9d690fe2dcff53509bba6bbb5c61b5c692d8abb13933e9d2a9 +size 74860097 diff --git a/treatment/tvae/params.json b/treatment/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..94245ab853a4399ba21b46e04507344a04c0d274 --- /dev/null +++ b/treatment/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 0.2, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file