diff --git "a/insurance/realtabformer/mlu-eval.ipynb" "b/insurance/realtabformer/mlu-eval.ipynb" new file mode 100644--- /dev/null +++ "b/insurance/realtabformer/mlu-eval.ipynb" @@ -0,0 +1,2459 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.565090Z", + "iopub.status.busy": "2024-03-22T17:48:32.564211Z", + "iopub.status.idle": "2024-03-22T17:48:32.602921Z", + "shell.execute_reply": "2024-03-22T17:48:32.602187Z" + }, + "papermill": { + "duration": 0.054165, + "end_time": "2024-03-22T17:48:32.605109", + "exception": false, + "start_time": "2024-03-22T17:48:32.550944", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.632778Z", + "iopub.status.busy": "2024-03-22T17:48:32.632342Z", + "iopub.status.idle": "2024-03-22T17:48:32.639689Z", + "shell.execute_reply": "2024-03-22T17:48:32.638818Z" + }, + "papermill": { + "duration": 0.023919, + "end_time": "2024-03-22T17:48:32.641677", + "exception": false, + "start_time": "2024-03-22T17:48:32.617758", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.666219Z", + "iopub.status.busy": "2024-03-22T17:48:32.665872Z", + "iopub.status.idle": "2024-03-22T17:48:32.670288Z", + "shell.execute_reply": "2024-03-22T17:48:32.669459Z" + }, + "papermill": { + "duration": 0.019208, + "end_time": "2024-03-22T17:48:32.672274", + "exception": false, + "start_time": "2024-03-22T17:48:32.653066", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.698128Z", + "iopub.status.busy": "2024-03-22T17:48:32.697757Z", + "iopub.status.idle": "2024-03-22T17:48:32.702899Z", + "shell.execute_reply": "2024-03-22T17:48:32.701664Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.021273, + "end_time": "2024-03-22T17:48:32.705320", + "exception": false, + "start_time": "2024-03-22T17:48:32.684047", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.734913Z", + "iopub.status.busy": "2024-03-22T17:48:32.734569Z", + "iopub.status.idle": "2024-03-22T17:48:32.740963Z", + "shell.execute_reply": "2024-03-22T17:48:32.740031Z" + }, + "papermill": { + "duration": 0.023874, + "end_time": "2024-03-22T17:48:32.743198", + "exception": false, + "start_time": "2024-03-22T17:48:32.719324", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d343101a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.770956Z", + "iopub.status.busy": "2024-03-22T17:48:32.770581Z", + "iopub.status.idle": "2024-03-22T17:48:32.776641Z", + "shell.execute_reply": "2024-03-22T17:48:32.775636Z" + }, + "papermill": { + "duration": 0.022705, + "end_time": "2024-03-22T17:48:32.778873", + "exception": false, + "start_time": "2024-03-22T17:48:32.756168", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/realtabformer/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.012826, + "end_time": "2024-03-22T17:48:32.804618", + "exception": false, + "start_time": "2024-03-22T17:48:32.791792", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.831861Z", + "iopub.status.busy": "2024-03-22T17:48:32.831454Z", + "iopub.status.idle": "2024-03-22T17:48:32.842155Z", + "shell.execute_reply": "2024-03-22T17:48:32.841141Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.026984, + "end_time": "2024-03-22T17:48:32.844341", + "exception": false, + "start_time": "2024-03-22T17:48:32.817357", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/realtabformer/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:32.873217Z", + "iopub.status.busy": "2024-03-22T17:48:32.872829Z", + "iopub.status.idle": "2024-03-22T17:48:34.907078Z", + "shell.execute_reply": "2024-03-22T17:48:34.905937Z" + }, + "papermill": { + "duration": 2.050961, + "end_time": "2024-03-22T17:48:34.909240", + "exception": false, + "start_time": "2024-03-22T17:48:32.858279", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:34.934797Z", + "iopub.status.busy": "2024-03-22T17:48:34.934314Z", + "iopub.status.idle": "2024-03-22T17:48:34.952680Z", + "shell.execute_reply": "2024-03-22T17:48:34.951911Z" + }, + "papermill": { + "duration": 0.033772, + "end_time": "2024-03-22T17:48:34.955024", + "exception": false, + "start_time": "2024-03-22T17:48:34.921252", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:34.980303Z", + "iopub.status.busy": "2024-03-22T17:48:34.979475Z", + "iopub.status.idle": "2024-03-22T17:48:34.989126Z", + "shell.execute_reply": "2024-03-22T17:48:34.988220Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024507, + "end_time": "2024-03-22T17:48:34.991219", + "exception": false, + "start_time": "2024-03-22T17:48:34.966712", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:35.018864Z", + "iopub.status.busy": "2024-03-22T17:48:35.018499Z", + "iopub.status.idle": "2024-03-22T17:48:35.498861Z", + "shell.execute_reply": "2024-03-22T17:48:35.497941Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.497969, + "end_time": "2024-03-22T17:48:35.501411", + "exception": false, + "start_time": "2024-03-22T17:48:35.003442", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:35.528095Z", + "iopub.status.busy": "2024-03-22T17:48:35.527397Z", + "iopub.status.idle": "2024-03-22T17:48:48.370968Z", + "shell.execute_reply": "2024-03-22T17:48:48.370163Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 12.85894, + "end_time": "2024-03-22T17:48:48.373403", + "exception": false, + "start_time": "2024-03-22T17:48:35.514463", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 17:48:39.992016: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 17:48:39.992132: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 17:48:40.117715: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:48.400981Z", + "iopub.status.busy": "2024-03-22T17:48:48.400390Z", + "iopub.status.idle": "2024-03-22T17:48:48.406841Z", + "shell.execute_reply": "2024-03-22T17:48:48.406076Z" + }, + "papermill": { + "duration": 0.02259, + "end_time": "2024-03-22T17:48:48.408784", + "exception": false, + "start_time": "2024-03-22T17:48:48.386194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:48.433480Z", + "iopub.status.busy": "2024-03-22T17:48:48.432733Z", + "iopub.status.idle": "2024-03-22T17:48:56.990631Z", + "shell.execute_reply": "2024-03-22T17:48:56.989549Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.57279, + "end_time": "2024-03-22T17:48:56.993041", + "exception": false, + "start_time": "2024-03-22T17:48:48.420251", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T17:48:57.020225Z", + "iopub.status.busy": "2024-03-22T17:48:57.019880Z", + "iopub.status.idle": "2024-03-22T17:48:57.026502Z", + "shell.execute_reply": "2024-03-22T17:48:57.025618Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.02276, + "end_time": "2024-03-22T17:48:57.028481", + "exception": false, + "start_time": "2024-03-22T17:48:57.005721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:57.054648Z", + "iopub.status.busy": "2024-03-22T17:48:57.053907Z", + "iopub.status.idle": "2024-03-22T17:48:57.059601Z", + "shell.execute_reply": "2024-03-22T17:48:57.058618Z" + }, + "papermill": { + "duration": 0.021435, + "end_time": "2024-03-22T17:48:57.061565", + "exception": false, + "start_time": "2024-03-22T17:48:57.040130", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:48:57.087847Z", + "iopub.status.busy": "2024-03-22T17:48:57.087014Z", + "iopub.status.idle": "2024-03-22T17:54:39.984658Z", + "shell.execute_reply": "2024-03-22T17:54:39.983743Z" + }, + "papermill": { + "duration": 342.926299, + "end_time": "2024-03-22T17:54:39.999916", + "exception": false, + "start_time": "2024-03-22T17:48:57.073617", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:54:40.027722Z", + "iopub.status.busy": "2024-03-22T17:54:40.027401Z", + "iopub.status.idle": "2024-03-22T17:54:40.643997Z", + "shell.execute_reply": "2024-03-22T17:54:40.643048Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.633398, + "end_time": "2024-03-22T17:54:40.646155", + "exception": false, + "start_time": "2024-03-22T17:54:40.012757", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:54:40.674361Z", + "iopub.status.busy": "2024-03-22T17:54:40.673611Z", + "iopub.status.idle": "2024-03-22T18:02:02.384439Z", + "shell.execute_reply": "2024-03-22T18:02:02.383434Z" + }, + "papermill": { + "duration": 441.742571, + "end_time": "2024-03-22T18:02:02.401879", + "exception": false, + "start_time": "2024-03-22T17:54:40.659308", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.431631Z", + "iopub.status.busy": "2024-03-22T18:02:02.431317Z", + "iopub.status.idle": "2024-03-22T18:02:02.840156Z", + "shell.execute_reply": "2024-03-22T18:02:02.839186Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.426339, + "end_time": "2024-03-22T18:02:02.842464", + "exception": false, + "start_time": "2024-03-22T18:02:02.416125", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.872250Z", + "iopub.status.busy": "2024-03-22T18:02:02.871890Z", + "iopub.status.idle": "2024-03-22T18:02:02.875980Z", + "shell.execute_reply": "2024-03-22T18:02:02.875132Z" + }, + "papermill": { + "duration": 0.02141, + "end_time": "2024-03-22T18:02:02.877923", + "exception": false, + "start_time": "2024-03-22T18:02:02.856513", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.906802Z", + "iopub.status.busy": "2024-03-22T18:02:02.906111Z", + "iopub.status.idle": "2024-03-22T18:02:02.913535Z", + "shell.execute_reply": "2024-03-22T18:02:02.912635Z" + }, + "papermill": { + "duration": 0.024448, + "end_time": "2024-03-22T18:02:02.915615", + "exception": false, + "start_time": "2024-03-22T18:02:02.891167", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10420892" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:02.943512Z", + "iopub.status.busy": "2024-03-22T18:02:02.942817Z", + "iopub.status.idle": "2024-03-22T18:02:03.040150Z", + "shell.execute_reply": "2024-03-22T18:02:03.039190Z" + }, + "papermill": { + "duration": 0.114006, + "end_time": "2024-03-22T18:02:03.042749", + "exception": false, + "start_time": "2024-03-22T18:02:02.928743", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 15200] --\n", + "├─Adapter: 1-1 [2, 1071, 15200] --\n", + "│ └─Embedding: 2-1 [2, 1071, 19, 800] (440,800)\n", + "│ └─TensorInductionPoint: 2-2 [19, 1] 19\n", + "│ └─Sequential: 2-3 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 820,224\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 15200] (recursive)\n", + "│ └─Embedding: 2-4 [2, 267, 19, 800] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [19, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,861,692\n", + "Trainable params: 10,420,892\n", + "Non-trainable params: 440,800\n", + "Total mult-adds (M): 43.07\n", + "========================================================================================================================\n", + "Input size (MB): 0.20\n", + "Forward/backward pass size (MB): 632.89\n", + "Params size (MB): 43.45\n", + "Estimated Total Size (MB): 676.54\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:02:03.074601Z", + "iopub.status.busy": "2024-03-22T18:02:03.074272Z", + "iopub.status.idle": "2024-03-22T19:01:15.975389Z", + "shell.execute_reply": "2024-03-22T19:01:15.974397Z" + }, + "papermill": { + "duration": 3552.935987, + "end_time": "2024-03-22T19:01:15.993965", + "exception": false, + "start_time": "2024-03-22T18:02:03.057978", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.029822943683765413, 'avg_role_model_std_loss': 0.7290258940829034, 'avg_role_model_mean_pred_loss': 0.00741166268384139, 'avg_role_model_g_mag_loss': 4.97963041305542, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.030589072092229292, 'n_size': 900, 'n_batch': 113, 'duration': 167.66863656044006, 'duration_batch': 1.4837932438976997, 'duration_size': 0.18629848506715563, 'avg_pred_std': 0.11875602419990881}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011320239888348927, 'avg_role_model_std_loss': 0.8666478302036187, 'avg_role_model_mean_pred_loss': 0.0016214420898055298, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011320239888348927, 'n_size': 450, 'n_batch': 57, 'duration': 53.24812388420105, 'duration_batch': 0.9341776120035272, 'duration_size': 0.11832916418711345, 'avg_pred_std': 0.05855883448793177}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008769770529793783, 'avg_role_model_std_loss': 0.7716960565807296, 'avg_role_model_mean_pred_loss': 0.00043396698297856246, 'avg_role_model_g_mag_loss': 1.4175501622094049, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008993633385075049, 'n_size': 900, 'n_batch': 113, 'duration': 166.8349289894104, 'duration_batch': 1.4764153007912424, 'duration_size': 0.18537214332156712, 'avg_pred_std': 0.09298099682921857}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006308909958720001, 'avg_role_model_std_loss': 0.1587160864532443, 'avg_role_model_mean_pred_loss': 0.0010557503438670954, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006308909958720001, 'n_size': 450, 'n_batch': 57, 'duration': 54.493409872055054, 'duration_batch': 0.9560247345974571, 'duration_size': 0.12109646638234456, 'avg_pred_std': 0.07172276902322967}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0052478324769375225, 'avg_role_model_std_loss': 0.4535665847330644, 'avg_role_model_mean_pred_loss': 0.00015836110432902994, 'avg_role_model_g_mag_loss': 0.8015708598825667, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005376618540095579, 'n_size': 900, 'n_batch': 113, 'duration': 167.3072214126587, 'duration_batch': 1.4805948797580415, 'duration_size': 0.18589691268073189, 'avg_pred_std': 0.09632732554347115}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005088021330302581, 'avg_role_model_std_loss': 0.6213396075144173, 'avg_role_model_mean_pred_loss': 0.00011936115010066849, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005088021330302581, 'n_size': 450, 'n_batch': 57, 'duration': 52.64925193786621, 'duration_batch': 0.9236710866292318, 'duration_size': 0.11699833763970269, 'avg_pred_std': 0.052176826527309525}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0030236450636746464, 'avg_role_model_std_loss': 0.41981709804675077, 'avg_role_model_mean_pred_loss': 4.2389841936938424e-05, 'avg_role_model_g_mag_loss': 0.5216422769096163, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031053333073052473, 'n_size': 900, 'n_batch': 113, 'duration': 167.31541466712952, 'duration_batch': 1.4806673864347746, 'duration_size': 0.18590601629681058, 'avg_pred_std': 0.09325551584494852}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019911888805735443, 'avg_role_model_std_loss': 0.2196139024221849, 'avg_role_model_mean_pred_loss': 1.918494357696417e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019911888805735443, 'n_size': 450, 'n_batch': 57, 'duration': 53.6285560131073, 'duration_batch': 0.9408518598790754, 'duration_size': 0.11917456891801623, 'avg_pred_std': 0.07260068974523037}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016811677904075219, 'avg_role_model_std_loss': 0.24856906805416656, 'avg_role_model_mean_pred_loss': 1.2625722286744727e-05, 'avg_role_model_g_mag_loss': 0.3810585221648216, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001737628386148976, 'n_size': 900, 'n_batch': 113, 'duration': 167.82943487167358, 'duration_batch': 1.4852162378024212, 'duration_size': 0.1864771498574151, 'avg_pred_std': 0.09434814767631809}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027509150341696416, 'avg_role_model_std_loss': 0.07755351969834473, 'avg_role_model_mean_pred_loss': 2.1412663001758838e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027509150341696416, 'n_size': 450, 'n_batch': 57, 'duration': 53.66714072227478, 'duration_batch': 0.941528784601312, 'duration_size': 0.11926031271616618, 'avg_pred_std': 0.08398696716482702}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001128127839474473, 'avg_role_model_std_loss': 0.21778014314879, 'avg_role_model_mean_pred_loss': 6.985574985683568e-06, 'avg_role_model_g_mag_loss': 0.28518845240275065, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001170292465992841, 'n_size': 900, 'n_batch': 113, 'duration': 160.84526014328003, 'duration_batch': 1.4234093817989384, 'duration_size': 0.1787169557147556, 'avg_pred_std': 0.09777768919310342}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021597746671694847, 'avg_role_model_std_loss': 0.3930031767821128, 'avg_role_model_mean_pred_loss': 3.3223664668092175e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021597746671694847, 'n_size': 450, 'n_batch': 57, 'duration': 49.775447607040405, 'duration_batch': 0.8732534667901826, 'duration_size': 0.11061210579342312, 'avg_pred_std': 0.07324986261511712}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010482495646445184, 'avg_role_model_std_loss': 0.11189875107239877, 'avg_role_model_mean_pred_loss': 8.224311742242863e-06, 'avg_role_model_g_mag_loss': 0.2797618282172415, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010887739290612647, 'n_size': 900, 'n_batch': 113, 'duration': 162.833313703537, 'duration_batch': 1.4410027761374955, 'duration_size': 0.1809259041150411, 'avg_pred_std': 0.09814084033621887}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019042760821240436, 'avg_role_model_std_loss': 0.35190855250582836, 'avg_role_model_mean_pred_loss': 1.6172142232353588e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019042760821240436, 'n_size': 450, 'n_batch': 57, 'duration': 52.383286476135254, 'duration_batch': 0.9190050258971098, 'duration_size': 0.11640730328030056, 'avg_pred_std': 0.06503926100732203}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001563700584617133, 'avg_role_model_std_loss': 0.21518477240972333, 'avg_role_model_mean_pred_loss': 4.5778025960834125e-06, 'avg_role_model_g_mag_loss': 0.30026551425457, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016092181016897989, 'n_size': 900, 'n_batch': 113, 'duration': 165.6556372642517, 'duration_batch': 1.465979090834086, 'duration_size': 0.1840618191825019, 'avg_pred_std': 0.09587243578470914}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003619586681533191, 'avg_role_model_std_loss': 0.5286780524212229, 'avg_role_model_mean_pred_loss': 0.00025007503121890473, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003619586681533191, 'n_size': 450, 'n_batch': 57, 'duration': 52.2312707901001, 'duration_batch': 0.9163380840368438, 'duration_size': 0.11606949064466689, 'avg_pred_std': 0.07745434307460591}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011870296497040222, 'avg_role_model_std_loss': 0.15783826198738787, 'avg_role_model_mean_pred_loss': 2.913096641859988e-06, 'avg_role_model_g_mag_loss': 0.26780749612384375, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012269385414159235, 'n_size': 900, 'n_batch': 113, 'duration': 164.9515302181244, 'duration_batch': 1.4597480550276494, 'duration_size': 0.18327947802013822, 'avg_pred_std': 0.0955763464315539}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.000885065957877992, 'avg_role_model_std_loss': 0.12204710721440885, 'avg_role_model_mean_pred_loss': 1.0928920943189783e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000885065957877992, 'n_size': 450, 'n_batch': 57, 'duration': 50.95037126541138, 'duration_batch': 0.8938661625510768, 'duration_size': 0.11322304725646973, 'avg_pred_std': 0.07468080061912667}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007502347028801321, 'avg_role_model_std_loss': 0.07230353024762727, 'avg_role_model_mean_pred_loss': 2.765409718966213e-06, 'avg_role_model_g_mag_loss': 0.2302845541636149, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007828545368586977, 'n_size': 900, 'n_batch': 113, 'duration': 159.93213367462158, 'duration_batch': 1.4153286165895715, 'duration_size': 0.17770237074957954, 'avg_pred_std': 0.09794058360620937}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002954557936366958, 'avg_role_model_std_loss': 0.5175472071741417, 'avg_role_model_mean_pred_loss': 0.0001666847069735606, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002954557936366958, 'n_size': 450, 'n_batch': 57, 'duration': 48.75002574920654, 'duration_batch': 0.8552636096352025, 'duration_size': 0.10833339055379232, 'avg_pred_std': 0.06763517065790661}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008960025814141975, 'avg_role_model_std_loss': 0.1276295194669632, 'avg_role_model_mean_pred_loss': 9.211538751067532e-06, 'avg_role_model_g_mag_loss': 0.24726980176236896, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009317982033179659, 'n_size': 900, 'n_batch': 113, 'duration': 158.89508271217346, 'duration_batch': 1.406151174444013, 'duration_size': 0.17655009190241497, 'avg_pred_std': 0.09495367640546992}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002206301508348487, 'avg_role_model_std_loss': 0.6127840376083674, 'avg_role_model_mean_pred_loss': 3.638367311774459e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002206301508348487, 'n_size': 450, 'n_batch': 57, 'duration': 48.69274377822876, 'duration_batch': 0.8542586627759432, 'duration_size': 0.1082060972849528, 'avg_pred_std': 0.07679086615609233}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012414162013576263, 'avg_role_model_std_loss': 0.15656802731034372, 'avg_role_model_mean_pred_loss': 7.65507348889812e-06, 'avg_role_model_g_mag_loss': 0.2784161967039108, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012826652981392625, 'n_size': 900, 'n_batch': 113, 'duration': 159.0150740146637, 'duration_batch': 1.4072130443775548, 'duration_size': 0.17668341557184855, 'avg_pred_std': 0.09698942330031268}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0015209214665810578, 'avg_role_model_std_loss': 1.1925622708846475, 'avg_role_model_mean_pred_loss': 1.541826602484448e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015209214665810578, 'n_size': 450, 'n_batch': 57, 'duration': 48.32934379577637, 'duration_batch': 0.8478832244873047, 'duration_size': 0.10739854176839193, 'avg_pred_std': 0.060592792895540856}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000802709650840067, 'avg_role_model_std_loss': 0.1468484333481462, 'avg_role_model_mean_pred_loss': 6.78528227740518e-07, 'avg_role_model_g_mag_loss': 0.21540525201294158, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000833754398206818, 'n_size': 900, 'n_batch': 113, 'duration': 158.784077167511, 'duration_batch': 1.4051688244912477, 'duration_size': 0.17642675240834554, 'avg_pred_std': 0.09867627354981624}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001989033992609216, 'avg_role_model_std_loss': 0.6257210954253898, 'avg_role_model_mean_pred_loss': 2.2637783627021217e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001989033992609216, 'n_size': 450, 'n_batch': 57, 'duration': 48.381258964538574, 'duration_batch': 0.8487940169217294, 'duration_size': 0.10751390881008573, 'avg_pred_std': 0.07426875439807445}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006656886564370426, 'avg_role_model_std_loss': 0.08703389815338669, 'avg_role_model_mean_pred_loss': 6.284277396025041e-07, 'avg_role_model_g_mag_loss': 0.20659642385111915, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006952917674950893, 'n_size': 900, 'n_batch': 113, 'duration': 157.6981496810913, 'duration_batch': 1.3955588467353213, 'duration_size': 0.17522016631232368, 'avg_pred_std': 0.09772866989065589}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010757716005254123, 'avg_role_model_std_loss': 0.41428317912765056, 'avg_role_model_mean_pred_loss': 2.2639515015956634e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010757716005254123, 'n_size': 450, 'n_batch': 57, 'duration': 47.70015549659729, 'duration_batch': 0.8368448332736367, 'duration_size': 0.10600034554799398, 'avg_pred_std': 0.07492752365049041}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0003420950226265834, 'avg_role_model_std_loss': 0.03358766082648436, 'avg_role_model_mean_pred_loss': 3.4995474424948957e-07, 'avg_role_model_g_mag_loss': 0.13486777688066165, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003607326176521989, 'n_size': 900, 'n_batch': 113, 'duration': 157.660076379776, 'duration_batch': 1.3952219148652742, 'duration_size': 0.17517786264419555, 'avg_pred_std': 0.0989597013500412}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008682416723038639, 'avg_role_model_std_loss': 1.067326461550605, 'avg_role_model_mean_pred_loss': 1.39864987777777e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008682416723038639, 'n_size': 450, 'n_batch': 57, 'duration': 47.50433969497681, 'duration_batch': 0.8334094683329264, 'duration_size': 0.10556519932217068, 'avg_pred_std': 0.06998455429269948}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00032250956939404, 'avg_role_model_std_loss': 0.03796051059134845, 'avg_role_model_mean_pred_loss': 2.0489448113398992e-07, 'avg_role_model_g_mag_loss': 0.13379798481861752, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00034089527511645834, 'n_size': 900, 'n_batch': 113, 'duration': 158.42128372192383, 'duration_batch': 1.4019582630258747, 'duration_size': 0.17602364857991537, 'avg_pred_std': 0.09779412188954585}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008600625935489208, 'avg_role_model_std_loss': 1.0569037955788663, 'avg_role_model_mean_pred_loss': 1.91862969400389e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008600625935489208, 'n_size': 450, 'n_batch': 57, 'duration': 48.475810289382935, 'duration_batch': 0.8504528120944375, 'duration_size': 0.10772402286529541, 'avg_pred_std': 0.06928659294192728}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'pred_duration': 5.626660346984863, 'grad_duration': 4.452868223190308, 'total_duration': 10.079528570175171, 'pred_std': 0.1404879093170166, 'std_loss': 0.001446777256205678, 'mean_pred_loss': 4.327647218360653e-07, 'pred_rmse': 0.02408498339354992, 'pred_mae': 0.015106265433132648, 'pred_mape': 1.252137541770935, 'grad_rmse': 0.47066444158554077, 'grad_mae': 0.20095407962799072, 'grad_mape': 9.563034057617188}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'avg_pred_duration': 5.626660346984863, 'avg_grad_duration': 4.452868223190308, 'avg_total_duration': 10.079528570175171, 'avg_pred_std': 0.1404879093170166, 'avg_std_loss': 0.001446777256205678, 'avg_mean_pred_loss': 4.327647218360653e-07}, 'min_metrics': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'pred_duration': 5.626660346984863, 'grad_duration': 4.452868223190308, 'total_duration': 10.079528570175171, 'pred_std': 0.1404879093170166, 'std_loss': 0.001446777256205678, 'mean_pred_loss': 4.327647218360653e-07, 'pred_rmse': 0.02408498339354992, 'pred_mae': 0.015106265433132648, 'pred_mape': 1.252137541770935, 'grad_rmse': 0.47066444158554077, 'grad_mae': 0.20095407962799072, 'grad_mape': 9.563034057617188}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0005800863997310877, 'avg_g_mag_loss': 0.005636774159637363, 'avg_g_cos_loss': 0.007875667856340961, 'pred_duration': 5.626660346984863, 'grad_duration': 4.452868223190308, 'total_duration': 10.079528570175171, 'pred_std': 0.1404879093170166, 'std_loss': 0.001446777256205678, 'mean_pred_loss': 4.327647218360653e-07, 'pred_rmse': 0.02408498339354992, 'pred_mae': 0.015106265433132648, 'pred_mape': 1.252137541770935, 'grad_rmse': 0.47066444158554077, 'grad_mae': 0.20095407962799072, 'grad_mape': 9.563034057617188}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.030571Z", + "iopub.status.busy": "2024-03-22T19:01:16.030259Z", + "iopub.status.idle": "2024-03-22T19:01:16.034407Z", + "shell.execute_reply": "2024-03-22T19:01:16.033634Z" + }, + "papermill": { + "duration": 0.025063, + "end_time": "2024-03-22T19:01:16.036311", + "exception": false, + "start_time": "2024-03-22T19:01:16.011248", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.070909Z", + "iopub.status.busy": "2024-03-22T19:01:16.070227Z", + "iopub.status.idle": "2024-03-22T19:01:16.159934Z", + "shell.execute_reply": "2024-03-22T19:01:16.158930Z" + }, + "papermill": { + "duration": 0.110121, + "end_time": "2024-03-22T19:01:16.162741", + "exception": false, + "start_time": "2024-03-22T19:01:16.052620", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.199073Z", + "iopub.status.busy": "2024-03-22T19:01:16.198774Z", + "iopub.status.idle": "2024-03-22T19:01:16.494447Z", + "shell.execute_reply": "2024-03-22T19:01:16.493514Z" + }, + "papermill": { + "duration": 0.315967, + "end_time": "2024-03-22T19:01:16.496397", + "exception": false, + "start_time": "2024-03-22T19:01:16.180430", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCTUlEQVR4nO3de1xUdf4/8NfMwAz3Qa7DKBcvIAqIhjFipbWyYpGGWSixSn5dtV01k9yMfinWbqFZ38zLN7NttbYIdNdq81aImqsgKJfEUEJDQWFAQIY7AzOf3x8HRgcGnBmQAXk/H4/zgDnnc875zMi8/JxzPud8eIwxBkIIGST4pq4AIYQYgkKLEDKoUGgRQgYVCi1CyKBCoUUIGVQotAghgwqFFiFkUDEzdQX6i1qtRmlpKWxtbcHj8UxdHUJIJ4wx1NXVQSqVgs/vvj01ZEKrtLQU7u7upq4GIeQeSkpKMGLEiG6XD5nQsrW1BcB9IHZ2diauDSGks9raWri7u2u+q90ZMqHVcUhoZ2dHoUXIAHav0zd0Ip4QMqhQaBFCBhUKLULIoDJkzmmR3lGpVGhtbTV1NcggZm5uDoFA0OvtUGiRHjHGIJfLUVNTY+qqkAeAvb09JBJJr/pKUmiRHnUElouLC6ysrKhjLjEKYwyNjY2oqKgAALi5uRm9LQqtThRNrbhcVgsAkI1yNHFtTEulUmkCy9FxaH8WpPcsLS0BABUVFXBxcTH6UJFOxHeSXXwb83efxVvf55u6KibXcQ7LysrKxDUhD4qOv6XenB81KrR27twJLy8vWFhYQCaTITMzs8fy+/fvh6+vLywsLBAQEIDDhw9rLd+4cSN8fX1hbW2NYcOGITQ0FBkZGVplqqurER0dDTs7O9jb22PJkiWor683pvo9crYRAQBu1bf0+bYHKzokJH2lL/6WDA6t5ORkxMbGIj4+HtnZ2QgMDERYWJjmWLWztLQ0REVFYcmSJcjJyUFERAQiIiJw8eJFTRkfHx/s2LEDeXl5OH36NLy8vDBz5kzcunVLUyY6Ohq//PILUlJScPDgQZw6dQrLli0z4i33zMWWC62q+hao1DTmByEDDjNQcHAwW7Fihea1SqViUqmUJSQk6CwfGRnJwsPDtebJZDK2fPnybvehUCgYAHbs2DHGGGP5+fkMADt37pymzJEjRxiPx2M3b97Uq94d21QoFD2Wa1Op2cjXDzLPdQdZRW2zXtt+UDU1NbH8/HzW1NRk6qqQB0RPf1P6fkcNamkplUpkZWUhNDRUM4/P5yM0NBTp6ek610lPT9cqDwBhYWHdllcqldi9ezfEYjECAwM127C3t8fkyZM15UJDQ8Hn87scRnZoaWlBbW2t1qQPAZ8HB+v2Q8Q6OkQkxuHxePj2229NXY0+tXHjRkycONHU1TDs8LCyshIqlQqurq5a811dXSGXy3WuI5fL9Sp/8OBB2NjYwMLCAh9++CFSUlLg5OSk2YaLi4tWeTMzMzg4OHS734SEBIjFYs1kyGNpnG3pvBYZ/Pbu3Qt7e/s+297atWuRmpraZ9sz1oC5evjEE08gNzcXaWlpmDVrFiIjI7s9T6aPuLg4KBQKzVRSUqL3uprQopYWGQKUSqVe5WxsbAZE1xeDQsvJyQkCgQDl5eVa88vLyyGRSHSuI5FI9CpvbW2NMWPGYMqUKfjss89gZmaGzz77TLONzgHW1taG6urqbvcrEok0j6Ex9HE0HVcQK+qa9V5nKGCMoVHZZpKJGTgQ+tGjR/Hoo4/C3t4ejo6OePrpp3H16lUAwNSpU7Fu3Tqt8rdu3YK5uTlOnToFACgrK0N4eDgsLS0xcuRIJCYmwsvLC1u3bjXqs8vLy8Pvfvc7WFpawtHREcuWLdO6+n3y5EkEBwfD2toa9vb2eOSRR3D9+nUAwM8//4wnnngCtra2sLOzQ1BQEM6fP9/j/k6ePInFixdDoVCAx+OBx+Nh48aNAAAvLy/89a9/xaJFi2BnZ6e5oLVu3Tr4+PjAysoKo0aNwvr167W6JnQ+PHzxxRcRERGB999/H25ubnB0dMSKFSvu++1eBnUuFQqFCAoKQmpqKiIiIgBwjzFOTU3FypUrda4TEhKC1NRUvPLKK5p5KSkpCAkJ6XFfarUaLS0tmm3U1NQgKysLQUFBAIDjx49DrVZDJpMZ8hb0Qi0t3ZpaVRi/4QeT7Dv/7TBYCfX/c21oaEBsbCwmTJiA+vp6bNiwAXPnzkVubi6io6Px3nvvYdOmTZpL8MnJyZBKpXjssccAAIsWLUJlZSVOnjwJc3NzxMbGGt3yb2hoQFhYGEJCQnDu3DlUVFTgj3/8I1auXIm9e/eira0NERERWLp0Kb7++msolUpkZmZq6hYdHY1Jkybh448/hkAgQG5uLszNzXvc59SpU7F161Zs2LABBQUFALiWUof3338fGzZsQHx8vGaera0t9u7dC6lUiry8PCxduhS2trZ47bXXut3PiRMn4ObmhhMnTuDKlSuYP38+Jk6ciKVLlxr1WenD4B7xsbGxiImJweTJkxEcHIytW7eioaEBixcvBsD9Yw8fPhwJCQkAgNWrV2P69On44IMPEB4ejqSkJJw/fx67d+8GwP2DvvPOO5gzZw7c3NxQWVmJnTt34ubNm3j++ecBAOPGjcOsWbOwdOlS7Nq1C62trVi5ciUWLFgAqVTaV5+FBoXW4Ddv3jyt1//4xz/g7OyM/Px8REZG4pVXXsHp06c1IZWYmIioqCjweDxcvnwZx44dw7lz5zQXf/7+97/D29vbqLokJiaiubkZX3zxBaytrQEAO3bswOzZs7F582aYm5tDoVDg6aefxujRowFwf/MdiouL8Ze//AW+vr4AoFc9hEIhxGIxeDyezqOR3/3ud3j11Ve15r355pua3728vLB27VokJSX1GFrDhg3Djh07IBAI4Ovri/DwcKSmpg6s0Jo/fz5u3bqFDRs2QC6XY+LEiTh69KjmZHtxcbHWQ+mnTp2KxMREvPnmm3jjjTfg7e2Nb7/9Fv7+/gAAgUCAy5cv4/PPP0dlZSUcHR3x8MMP47///S/8/Pw02/nqq6+wcuVKzJgxA3w+H/PmzcO2bdt6+/51otDSzdJcgPy3w0y2b0MUFhZiw4YNyMjIQGVlJdRqNQDu79Pf3x8zZ87EV199hcceewxFRUVIT0/HJ598AgAoKCiAmZkZHnroIc32xowZg2HDhhlV90uXLiEwMFATWADwyCOPQK1Wo6CgANOmTcOLL76IsLAw/P73v0doaCgiIyM19+fFxsbij3/8I/75z38iNDQUzz//vCbcjHX3lfgOycnJ2LZtG65evYr6+nq0tbXd87SKn5+f1u04bm5uyMvL61Xd7sWoew9XrlzZ7eHgyZMnu8x7/vnnNa2mziwsLHDgwIF77tPBwQGJiYkG1dNY1CteNx6PZ9AhminNnj0bnp6e+PTTTyGVSqFWq+Hv76856RwdHY2XX34Z27dvR2JiIgICAhAQEGCy+u7Zswcvv/wyjh49iuTkZLz55ptISUnBlClTsHHjRrzwwgs4dOgQjhw5gvj4eCQlJWHu3LlG7+/uAAW4bkXR0dF46623EBYWBrFYjKSkJHzwwQc9bqfzYSqPx9P8B3G/DJirhwMJtbQGt6qqKhQUFODNN9/EjBkzMG7cONy+fVurzDPPPIPm5mYcPXoUiYmJiI6O1iwbO3Ys2trakJOTo5l35cqVLtvQ17hx4/Dzzz+joaFBM+/MmTPg8/kYO3asZt6kSZMQFxeHtLQ0+Pv7a/0n7ePjgzVr1uDHH3/Es88+iz179txzv0KhECqVSq86pqWlwdPTE//v//0/TJ48Gd7e3poLAQMNhZYOHaFV19yG5lb9/tHJwDFs2DA4Ojpi9+7duHLlCo4fP47Y2FitMtbW1oiIiMD69etx6dIlREVFaZb5+voiNDQUy5YtQ2ZmJnJycrBs2TJYWloade9cdHQ0LCwsEBMTg4sXL+LEiRNYtWoVFi5cCFdXVxQVFSEuLg7p6em4fv06fvzxRxQWFmLcuHFoamrCypUrcfLkSVy/fh1nzpzBuXPntM55dcfLywv19fVITU1FZWUlGhsbuy3r7e2N4uJiJCUl4erVq9i2bRu++eYbg99rf6DQ0sHOwgxCM+6jodbW4MPn85GUlISsrCz4+/tjzZo12LJlS5dy0dHR+Pnnn/HYY4/Bw8NDa9kXX3wBV1dXTJs2DXPnztVcSbOwsDC4PlZWVvjhhx9QXV2Nhx9+GM899xxmzJiBHTt2aJZfvnwZ8+bNg4+PD5YtW4YVK1Zg+fLlEAgEqKqqwqJFi+Dj44PIyEg8+eSTeOutt+6536lTp+Kll17C/Pnz4ezsjPfee6/bsnPmzMGaNWuwcuVKTJw4EWlpaVi/fr3B77U/8JihHWAGqdraWojFYigUCr36bD2y6Thu1jThwJ+n4iEP407ADnbNzc0oKirCyJEjjfqyPkhu3LgBd3d3HDt2DDNmzDB1dQatnv6m9P2ODo6zqibgbCvCzZomamkNUcePH0d9fT0CAgJQVlaG1157DV5eXpg2bZqpqzbk0eFhN+hk/NDW2tqKN954A35+fpg7dy6cnZ01HU2/+uor2NjY6Jzu7qZzvz355JPd1uPdd9/tt3r0N2ppdYNCa2gLCwtDWJjuPmlz5szp9k6Me/VU70t///vf0dTUpHOZg4NDv9Wjv1FodYP6apHu2NrawtbW1tTVwPDhw01dBZOgw8NuuNhRS4uQgYhCqxualhaFFiEDCoVWN+icFiEDE4VWN+4OrSHSlY2QQYFCqxtO7YeHSpUatU1tJq4NIaQDhVY3LMwFsLPgLq7eqqcnmBLDPIgDWwwUFFo96DhErKDzWmQQ6uuBLQDu0VM8Hg81NTV9ul1DUGj1gE7GEzLwUGj1wNmWu6GTQqsdY4CywTQTDWzRZwNbtLS0YO3atRg+fDisra0hk8m0Ht55/fp1zJ49G8OGDYO1tTX8/Pxw+PBhXLt2DU888QQA7vE/PB4PL774olGfR29Qj/geUK/4TlobgXf7/pn8enmjFBBa37tcOxrYovuBLVauXIn8/HwkJSVBKpXim2++waxZs5CXlwdvb2+sWLECSqUSp06dgrW1NfLz82FjYwN3d3f8+9//xrx581BQUAA7OztYWloa9Zn0BoVWD+jwcPCigS10D2xRXFyMPXv2oLi4WDMozNq1a3H06FHs2bMH7777LoqLizFv3jzN46dHjRqlWb/jnkYXF5c+P1+mLwqtHlBodWJuxbV4TLVvA9DAFrrl5eVBpVLBx8dHa35LS4tmINaXX34Zf/rTn/Djjz8iNDQU8+bNw4QJE4za3/1A57R6QKHVCY/HHaKZYjLwMcezZ89GdXU1Pv30U2RkZCAjIwMAtAa2+Ne//oXW1tYBM7BFeno6pk6diuTkZPj4+ODs2bMAuEFSf/nlF4SHh+P48eMYP3680Y9Crq+vh0AgQFZWFnJzczXTpUuX8NFHHwEA/vjHP+K3337DwoULkZeXh8mTJ2P79u199l57i0KrBx3ntCrpnNagQgNbcHQNbDFp0iSoVCpUVFRgzJgxWtPdh5Hu7u546aWXcODAAbz66qv49NNPNdsEoPeAGfcDhVYPOlpaVQ1KtKnu77BIpO/QwBYcXQNb+Pj4IDo6GosWLcKBAwdQVFSEzMxMJCQk4NChQwCAV155BT/88AOKioqQnZ2NEydOaPbn6ekJHo+HgwcP4tatW1pXQPsNGyIUCgUDwBQKhd7rtKnUbOTrB5nnuoOsXNF0H2s3MDU1NbH8/HzW1DT43ntKSgobN24cE4lEbMKECezkyZMMAPvmm280ZQ4fPswAsGnTpnVZv7S0lD355JNMJBIxT09PlpiYyFxcXNiuXbv02n/nfV24cIE98cQTzMLCgjk4OLClS5eyuro6xhhjcrmcRUREMDc3NyYUCpmnpyfbsGEDU6lUrKWlhS1YsIC5u7szoVDIpFIpW7lypd7/Ji+99BJzdHRkAFh8fDxjjDGlUsk2bNjAvLy8mLm5OXNzc2Nz585lFy5cYIwxtnLlSjZ69GgmEomYs7MzW7hwIausrNRs8+2332YSiYTxeDwWExOjVz069PQ3pe931KjQ2rFjB/P09GQikYgFBwezjIyMHsvv27ePjR07lolEIubv788OHTqkWaZUKtlrr73G/P39mZWVFXNzc2MLFy5kN2/e1NqGp6cnA6A1JSQk6F1nY0KLMcYm/y2Fea47yPJu1Bi03oNgMIdWXyspKWEA2LFjx0xdlUGtL0LL4MPD5ORkxMbGIj4+HtnZ2QgMDERYWFi3fVjS0tIQFRWFJUuWICcnBxEREYiIiMDFixcBAI2NjcjOzsb69euRnZ2NAwcOoKCgAHPmzOmyrbfffhtlZWWaadWqVYZW32AuttRXayg6fvw4/vOf/6CoqAhpaWlYsGABDWwxUBialMHBwWzFihWa1yqVikml0m5bPZGRkSw8PFxrnkwmY8uXL+92H5mZmQwAu379umaep6cn+/DDDw2troaxLa2Yf2Qwz3UHWfK5YqP3PVgN5ZbW0aNHmZ+fH7O0tGQuLi4sIiKCXbt2jTHG2Jdffsmsra11TuPHj++3Os6aNavberzzzjv9Vg9D9EVLy6B+WkqlEllZWYiLi9PM4/P5CA0NRXp6us510tPTu5wEDQsL6/EO+I5bDzp3Xtu0aRP++te/wsPDAy+88ALWrFkDMzPdb6GlpQUtLXdaR7W1tfd4d7rRE0yHJhrYYuAyKLQqKyuhUqng6uqqNd/V1RWXL1/WuY5cLtdZXi6X6yzf3NyMdevWISoqSmvAxpdffhkPPfQQHBwckJaWhri4OJSVleF///d/dW4nISFBr1F474X6apHOaGAL0xpQPeJbW1sRGRkJxhg+/vhjrWV3t9YmTJgAoVCI5cuXIyEhASKRqMu24uLitNapra2Fu7u7wXWi0AI9uZX0mb74WzIotJycnCAQCFBeXq41v7y8XKtj2t0kEole5TsC6/r16zh+/Pg9h66XyWRoa2vDtWvXtDrodRCJRDrDzFBDObQ6DnUaGxtNcmMsefA0NjYC6N1htEGhJRQKERQUhNTUVERERAAA1Go1UlNTsXLlSp3rhISEIDU1Fa+88opmXkpKCkJCQjSvOwKrsLAQJ06c0NwD1ZPc3Fzw+Xy4uLgY8hYMNpSf9CAQCGBvb6+5MmxlZWVU50pCGGNobGxERUUF7O3tIRAIjN6WwYeHsbGxiImJweTJkxEcHIytW7eioaEBixcvBsA90mP48OFISEgAAKxevRrTp0/HBx98gPDwcCQlJeH8+fPYvXs3AC6wnnvuOWRnZ+PgwYNQqVSa810ODg4QCoVIT09HRkaG5plC6enpWLNmDf7whz8YfROrvoZySwuApkVs7GNZCLmbvb19t0dl+jI4tObPn49bt25hw4YNkMvlmDhxIo4ePao52V5cXAw+/073r6lTpyIxMRFvvvkm3njjDXh7e+Pbb7+Fv78/AODmzZv4z3/+AwCYOHGi1r5OnDiBxx9/HCKRCElJSdi4cSNaWlowcuRIrFmzpstVyfuhI7TqW9rQqGyDlXBAnQa873g8Htzc3ODi4oLW1lZTV4cMYubm5r1qYXXgsSFylrW2thZisRgKheKe58vuxhjDuA1H0dyqxqm/PAEPR8MekUII0Y++31G6YfoeeDzenUNEGpWHEJOj0NIDdTAlZOCg0NLDUD8ZT8hAQqGlBwotQgYOCi09ONu0DyU2BPtqETLQUGjpgVpahAwcFFp6oNAiZOCg0NKDC4UWIQMGhZYenO96eukQ6YtLyIBFoaUHRxtu2KRWFUNNI93KQogpUWjpQWQmgL0V9ygNuoJIiGlRaOmJesUTMjBQaOmJriASMjBQaOmJQouQgYFCS09D+QmmhAwkFFp6opYWIQMDhZaeKLQIGRgotPREoUXIwEChpae7e8UTQkyHQktPHSfiqxuUaFWpTVwbQoYuCi09DbMSQsDnxvyrqleauDaEDF0UWnri83lwar8Hkc5rEWI6FFoGoFF5CDE9Ci0D0P2HhJieUaG1c+dOeHl5wcLCAjKZDJmZmT2W379/P3x9fWFhYYGAgAAcPnxYs6y1tRXr1q1DQEAArK2tIZVKsWjRIpSWlmpto7q6GtHR0bCzs4O9vT2WLFmC+vp6Y6pvNBfb9mfFU2gRYjIGh1ZycjJiY2MRHx+P7OxsBAYGIiwsDBUVFTrLp6WlISoqCkuWLEFOTg4iIiIQERGBixcvAgAaGxuRnZ2N9evXIzs7GwcOHEBBQQHmzJmjtZ3o6Gj88ssvSElJwcGDB3Hq1CksW7bMiLdsvI7DwwoKLUJMhxkoODiYrVixQvNapVIxqVTKEhISdJaPjIxk4eHhWvNkMhlbvnx5t/vIzMxkANj169cZY4zl5+czAOzcuXOaMkeOHGE8Ho/dvHlT5zaam5uZQqHQTCUlJQwAUygUer/XzvaeKWKe6w6yl/553uhtEEJ0UygUen1HDWppKZVKZGVlITQ0VDOPz+cjNDQU6enpOtdJT0/XKg8AYWFh3ZYHAIVCAR6PB3t7e8027O3tMXnyZE2Z0NBQ8Pl8ZGRk6NxGQkICxGKxZnJ3d9f3bXaLesUTYnoGhVZlZSVUKhVcXV215ru6ukIul+tcRy6XG1S+ubkZ69atQ1RUFOzs7DTbcHFx0SpnZmYGBweHbrcTFxcHhUKhmUpKSvR6jz2hXvGEmJ6ZqStwt9bWVkRGRoIxho8//rhX2xKJRBCJRH1UMw5dPSTE9AxqaTk5OUEgEKC8vFxrfnl5OSQSic51JBKJXuU7Auv69etISUnRtLI6ttH5RH9bWxuqq6u73e/90NHSalSq0NDS1m/7JYTcYVBoCYVCBAUFITU1VTNPrVYjNTUVISEhOtcJCQnRKg8AKSkpWuU7AquwsBDHjh2Do6Njl23U1NQgKytLM+/48eNQq9WQyWSGvIVesRaZwUooAECtLUJMxtAz/ElJSUwkErG9e/ey/Px8tmzZMmZvb8/kcjljjLGFCxey119/XVP+zJkzzMzMjL3//vvs0qVLLD4+npmbm7O8vDzGGGNKpZLNmTOHjRgxguXm5rKysjLN1NLSotnOrFmz2KRJk1hGRgY7ffo08/b2ZlFRUXrXW98rE/cy7b3jzHPdQZZZVNWr7RBCtOn7HTU4tBhjbPv27czDw4MJhUIWHBzMzp49q1k2ffp0FhMTo1V+3759zMfHhwmFQubn58cOHTqkWVZUVMQA6JxOnDihKVdVVcWioqKYjY0Ns7OzY4sXL2Z1dXV617mvQmve/51hnusOskMXSnu1HUKINn2/ozzGhsaQybW1tRCLxVAoFFrnywz1py+zcOSiHG/N8UPMVK++qyAhQ5y+31G699BA1FeLENOi0DIQdXsgxLQotAxEHUwJMS0KLQPR4SEhpkWhZSAKLUJMi0LLQB2hVVnfArV6SFx4JWRAodAykFP7ifg2NUNNU6uJa0PI0EOhZSBzAR8O1twAFxV19Kx4QvobhZYRqNsDIaZDoWUEOhlPiOlQaBmBQosQ06HQMgKFFiGmQ6FlBM05LeoVT0i/o9AyArW0CDEdCi0jUGgRYjoUWkagm6YJMR0KLSN0nNOqaWxFS5vKxLUhZGih0DKC2NIc5gIeAKCqXmni2hAytFBoGYHP52nuQaTzWoT0LwotI9HJeEJMg0LLSNRXixDToNAyErW0CDENCi0jUWgRYhpGhdbOnTvh5eUFCwsLyGQyZGZm9lh+//798PX1hYWFBQICAnD48GGt5QcOHMDMmTPh6OgIHo+H3NzcLtt4/PHHwePxtKaXXnrJmOr3CZf20KJnahHSvwwOreTkZMTGxiI+Ph7Z2dkIDAxEWFgYKioqdJZPS0tDVFQUlixZgpycHERERCAiIgIXL17UlGloaMCjjz6KzZs397jvpUuXoqysTDO99957hla/z1BLixATMXTo6uDgYLZixQrNa5VKxaRSKUtISNBZPjIykoWHh2vNk8lkbPny5V3KFhUVMQAsJyeny7Lp06ez1atXG1pdDX2H3NbX+WtVzHPdQfbo5tQ+2R4hQ52+31GDWlpKpRJZWVkIDQ3VzOPz+QgNDUV6errOddLT07XKA0BYWFi35Xvy1VdfwcnJCf7+/oiLi0NjY2O3ZVtaWlBbW6s19SVnGwsAXEuLMRrggpD+YmZI4crKSqhUKri6umrNd3V1xeXLl3WuI5fLdZaXy+UGVfSFF16Ap6cnpFIpLly4gHXr1qGgoAAHDhzQWT4hIQFvvfWWQfswhJMt95z45lY16lvaYGthft/2RQi5w6DQMqVly5Zpfg8ICICbmxtmzJiBq1evYvTo0V3Kx8XFITY2VvO6trYW7u7ufVYfK6EZbERmqG9pw626FgotQvqJQYeHTk5OEAgEKC8v15pfXl4OiUSicx2JRGJQeX3JZDIAwJUrV3QuF4lEsLOz05r6Gp2MJ6T/GRRaQqEQQUFBSE1N1cxTq9VITU1FSEiIznVCQkK0ygNASkpKt+X11dEtws3NrVfb6Q3qFU9I/zP48DA2NhYxMTGYPHkygoODsXXrVjQ0NGDx4sUAgEWLFmH48OFISEgAAKxevRrTp0/HBx98gPDwcCQlJeH8+fPYvXu3ZpvV1dUoLi5GaWkpAKCgoAAA10qTSCS4evUqEhMT8dRTT8HR0REXLlzAmjVrMG3aNEyYMKHXH4KxqKVFiAkYc2ly+/btzMPDgwmFQhYcHMzOnj2rWTZ9+nQWExOjVX7fvn3Mx8eHCYVC5ufnxw4dOqS1fM+ePQxAlyk+Pp4xxlhxcTGbNm0ac3BwYCKRiI0ZM4b95S9/Maj7Ql93eWCMsfjvLjLPdQfZ5iOX+mybhAxV+n5HeYwNjev1tbW1EIvFUCgUfXZ+a+eJK9jyQwGeDxqBLc8H9sk2CRmq9P2O0r2HvUDntAjpfxRavUDntAjpfxRavUChRUj/o9DqhY7QqmpQQqUeEqcGCTE5Cq1ecLAWgscDVGqG2400wAUh/YFCqxfMBXw4WHH3INIhIiH9g0Krl5w1DwOk0CKkP1Bo9RKdjCekf1Fo9RKFFiH9i0Krlyi0COlfFFq9RL3iCelfFFq9dKelRaPyENIfKLR6iQ4PCelfFFq95EKhRUi/otDqpY5ReWqb29DcqjJxbQh58FFo9ZKdpRmEAu5jrKST8YTcdxRavcTj8ei8FiH9iEKrs4ZK4MxHwPk9eq/iRKFFSL+h0Ors6nEgZQNw6n1Ard85KuqrRUj/odDqbNwcwNIBqL0BFKbotQodHhLSfyi0OjO3ACa+wP2epd8hIoUWIf2HQkuXIG4MR/z6A1BTfM/iFFqE9B8KLV2cxgAjpwFgQPYX9yzecU6LnqlFyP1HodWdyf/D/cz+J6Bq7bEotbQI6T9GhdbOnTvh5eUFCwsLyGQyZGZm9lh+//798PX1hYWFBQICAnD48GGt5QcOHMDMmTPh6OgIHo+H3NzcLttobm7GihUr4OjoCBsbG8ybNw/l5eXGVF8/Y8MBaxegXg4UHOmxqOZWnvoWDJGxbwkxGYNDKzk5GbGxsYiPj0d2djYCAwMRFhaGiooKneXT0tIQFRWFJUuWICcnBxEREYiIiMDFixc1ZRoaGvDoo49i8+bN3e53zZo1+P7777F//3789NNPKC0txbPPPmto9fVnJgQm/YH7/fw/eiza0dJStqlR29x2/+pECAGYgYKDg9mKFSs0r1UqFZNKpSwhIUFn+cjISBYeHq41TyaTseXLl3cpW1RUxACwnJwcrfk1NTXM3Nyc7d+/XzPv0qVLDABLT0/Xq94KhYIBYAqFQq/yjDHGqosYixczFm/HWNXVHov6xx9lnusOssLyOv23TwjR0Pc7alBLS6lUIisrC6GhoZp5fD4foaGhSE9P17lOenq6VnkACAsL67a8LllZWWhtbdXajq+vLzw8PLrdTktLC2pra7Umgw3zAsa07zNrb49F6bwWIf3DoNCqrKyESqWCq6ur1nxXV1fI5XKd68jlcoPKd7cNoVAIe3t7vbeTkJAAsVismdzd3fXen5bJ7d0fcr4E2roPJOoVT0j/eGCvHsbFxUGhUGimkpIS4zbkHQbYSoHGKuDS990Wo5YWIf3DoNBycnKCQCDoctWuvLwcEolE5zoSicSg8t1tQ6lUoqamRu/tiEQi2NnZaU1GEZgBQTHc7z3cRE2hRUj/MCi0hEIhgoKCkJqaqpmnVquRmpqKkJAQneuEhIRolQeAlJSUbsvrEhQUBHNzc63tFBQUoLi42KDtGG3SQoDHB66fBm4V6CxCoUVI/zAzdIXY2FjExMRg8uTJCA4OxtatW9HQ0IDFi7lzP4sWLcLw4cORkJAAAFi9ejWmT5+ODz74AOHh4UhKSsL58+exe/duzTarq6tRXFyM0tJSAFwgAVwLSyKRQCwWY8mSJYiNjYWDgwPs7OywatUqhISEYMqUKb3+EO5JPBzweRIoOMSdkJ+V0KUIndMipJ8Yc2ly+/btzMPDgwmFQhYcHMzOnj2rWTZ9+nQWExOjVX7fvn3Mx8eHCYVC5ufnxw4dOqS1fM+ePQxAlyk+Pl5Tpqmpif35z39mw4YNY1ZWVmzu3LmsrKxM7zob1eXhbr+mcF0fEtwZUzZ2WXzicjnzXHeQzdp6yrjtEzLE6fsd5TE2NLpw19bWQiwWQ6FQGHd+S60GtgVyN1BHfHznSRDt8ktr8dS2/8JGZIb0uN/B1sK8j2pOyNCg73f0gb162Of4fCDoRe53HSfkfVxtMNLJGvUtbfj45NX+rRshQwiFliEmLQT4ZsCNTECep7XITMBH3JO+AIC/ny7CjduNpqghIQ88Ci1D2LgAvk9zv+tobf1+vCumjHKAsk2NLT/ovspICOkdCi1DdTyy5sI+oKVeaxGPx8Ob4ePB4wHf5ZYit6Sm/+tHyAOOQstQI6cBjmMAZR1w8V9dFvsPF+PZSSMAAH87mE+PqiGkj1FoGYrHu+uEvO5H1vwlbCwszPk4f/02jlzU/x5LQsi9UWgZI/AFQCACyn4GbmZ3WSwRW2DZtNEAgE1HLqOlTb+hyAgh90ahZQxrR8Avgvu9m9bW8mmj4GIrQnF1I75Iu95/dSPkAUehZayOEXsu/htoVnRZbC0yw9qZYwEA244XorpB2Z+1I+SBRaFlLI8pgPM4oLWRu5Kow7ygERjnZoe65jZsSy3s5woS8mCi0DIWj3en+8P5fwA6rhIK+Dy8GT4OAPDPs9dxpaK+SxlCiGEotHpjQiRgZglU5AMlGTqLPDLGCTN8XaBSM2w6cqmfK0jIg4dCqzcs7YGAedzvPYzYE/fUOAj4PBy7VIG0K5X9UzdCHlAUWr3VcYj4y7dAY7XOImNcbBAt8wAA/O3QJajU1OGUEGNRaPWW9CFAMgFQtQC5id0WWz3DG7YWZsgvq8WB7Bv9WEFCHiwUWr119wn5rD06T8gDgKONCKt+NwYAsOWHAjQqaVBXQoxBodUXAp4DhLZA1RXg2n+7LRYz1QvuDpaoqGvBJz/91o8VJOTBQaHVF0S2wITnud/PfAS0NukuZibA67O4LhC7T/0GuaK5v2pIyAODQquvTF4CgAdcOQb83xTg1x90FnsqQIIgz2FoalXh/R/pmVuEGIpCq69I/IH5X3IDu96+BiRGAl9Hcb/fhXvmFtfa+nf2DVy82fUWIEJI9yi0+tK4p4GV54BHVnOPZS44DOyUASc3A613DgUneQzDnEApGAPeOXSJnrlFiAEotPqayAb4/dvAn9K4Bwa2NQMn3+UOGQtTNMVemzUWQjM+0n+rwrFLFSasMCGDC4XW/eI8Flj0H+C5fwC2bsDtIuCr54CkaOD2dYwYZoUlj44EACQcvoRWldrEFSZkcKDQup94PMB/HnfIOHUVd8h4+SB3yHhqC/786Ag4WgvxW2UDvjpLz9wiRB9GhdbOnTvh5eUFCwsLyGQyZGZm9lh+//798PX1hYWFBQICAnD48GGt5YwxbNiwAW5ubrC0tERoaCgKC7Uf5eLl5QUej6c1bdq0yZjq9z+RLTDzb8BLpwGvx4C2JuD432D7j8fw3kTu0PCDlF9xpaLOxBUlZOAzOLSSk5MRGxuL+Ph4ZGdnIzAwEGFhYaio0H1eJi0tDVFRUViyZAlycnIQERGBiIgIXLx4UVPmvffew7Zt27Br1y5kZGTA2toaYWFhaG7W7sf09ttvo6ysTDOtWrXK0Oqblss4IOZ7YN5ngI0EqP4NM7L+jCTxDtg2yxHzj3Mor6W+W4T0iBkoODiYrVixQvNapVIxqVTKEhISdJaPjIxk4eHhWvNkMhlbvnw5Y4wxtVrNJBIJ27Jli2Z5TU0NE4lE7Ouvv9bM8/T0ZB9++KHe9WxubmYKhUIzlZSUMABMoVDovY37qknB2NE3GNs4jLF4O1a10Z09tO4rFvbhT0zRpDR17QjpdwqFQq/vqEEtLaVSiaysLISGhmrm8fl8hIaGIj09Xec66enpWuUBICwsTFO+qKgIcrlcq4xYLIZMJuuyzU2bNsHR0RGTJk3Cli1b0NbW/f17CQkJEIvFmsnd3d2Qt3r/WdgBYe9wh4xOY+HAFNhk+U9cltdh+RdZNBgGId0wKLQqKyuhUqng6uqqNd/V1RVyue6hsuRyeY/lO37ea5svv/wykpKScOLECSxfvhzvvvsuXnvttW7rGhcXB4VCoZlKSkr0f6P9yXU88OxugCfA71kanhGeR/pvVXh1389Q0yNsCOnCzNQV0FdsbKzm9wkTJkAoFGL58uVISEiASCTqUl4kEumcPyBJJwKPvgL89wNssfoCp9vG4eCFMrjaWWD90+NNXTtCBhSDWlpOTk4QCAQoLy/Xml9eXg6JRKJzHYlE0mP5jp+GbBMAZDIZ2tracO3aNUPewsA17TXAaSyEzZX4ZvRBAMBnp4vw6Sl6GgQhdzMotIRCIYKCgpCamqqZp1arkZqaipCQEJ3rhISEaJUHgJSUFE35kSNHQiKRaJWpra1FRkZGt9sEgNzcXPD5fLi4uBjyFgYucwvgmR0AePAo+Q7/F1wFAHjn8CV8l3vTtHUjZCAx9Ax/UlISE4lEbO/evSw/P58tW7aM2dvbM7lczhhjbOHChez111/XlD9z5gwzMzNj77//Prt06RKLj49n5ubmLC8vT1Nm06ZNzN7enn333XfswoUL7JlnnmEjR45kTU1NjDHG0tLS2Icffshyc3PZ1atX2ZdffsmcnZ3ZokWL9K63vlcmTO5IHGPxdkz9wTj27jcZzHPdQTbmjUPsdOEtU9eMkPtK3++owaHFGGPbt29nHh4eTCgUsuDgYHb27FnNsunTp7OYmBit8vv27WM+Pj5MKBQyPz8/dujQIa3larWarV+/nrm6ujKRSMRmzJjBCgoKNMuzsrKYTCZjYrGYWVhYsHHjxrF3332XNTc3613nQRNaLQ2MbQ3kgus/q9mfv8xinusOMr8NR9nFmzWmrh0h942+31EeY0PjEQO1tbUQi8VQKBSws7MzdXV6VvRf4POnAQDK6O+w8LgIGUXVcLYV4cCfpsLdwcrEFSSk7+n7HaV7DweikY+1P1QQEB5ejd0LxsFXYotbdS2I2ZOJ2w1KE1ewHw2N/1OJASi0BqrQjYDdCOD2NYjTN2Pv4mBIxRb47VYDlnx+Dk3KB7zzqaoNOPoGsGU0cOl7U9eGDCAUWgOVhR0w+yPu97MfQ6L4GXv/Jxh2FmbILq7Bqq9z0PagPs6muRb4egFwdifQWAV88yeg6qqpa0UGCAqtgcw7FJgYDYAB362Aj4M5PnvxYQjN+Dh2qRzrv/vlwXvq6e3rwGczgSspgJkl4DwOUNYB/1oMtLWYunZkAKDQGujC3gFsXIGqQuCnzXjYywHbFkwEjwd8nVmMD48VPjjBVZIJfPo74NYl7ikY/3ME+MO/AUsHoOxnIGWDqWtIBgAKrYHOchjw9Ifc72c+AkpzMMvfDW/N8QMAbEstxJLPzw/+R9pc2A/sfRporAQkAcDS44B0EiAeDszdxZXJ2AVcOnj/6tDWAvzyDVBZeO+yxGQotAYD33DA71mAqYDvVgJtSiwK8cKGp8dDKODj+OUK/P5/f8I3OTcGX6uLMeBEAnDgj4CqBRj7FLD4KBdWHXzCgJCV3O/f/RmoKe77erS1AMkLgf0vAjsmA5/OADI/BRqr+35fpFeon9Zg0VAJ7AzmTkw//gbw+DoAwK/ldXh138/Iax+KbOZ4V7wzNwDOtjpuFq8rB/L2AT8nAfUV3MAb3jOBMaGAtWN/vhtOaxPw3Qrg4r+511Nf5q6a8gVdy7YpgT2zgJtZwIiHgcVHAIF539RD1QrsiwEKDgECIaBWcf9BAADfHBg7CwiMAsb8HjAT9s0+SRf6fkcptAaTvH8B/17CfZGWn+IeawOgVaXGrpNXse14IVpVDMOszPHXCH88PUHKtSAKjgA/f82NBsR0dZXgASMmA95hgM9MQDKBe779/VRfASS9ANw4xz07/+kPgYcW9bzO7WvArmlAi4Ibpu33b/e+HqpW4F//A1z6DyAQAS8kA65+QN5+7jOT590pa+UI+D8HBC7gDl3v92c0xFBodfJAhBZj3Gg+BYcA6UPAkhRAcOfpQvmltXh1/8+4VKZAAK8Ia13O47GWk+A319zZxoiHgYkvAE4+wJVUoPBHoPyi9n5sJID377lW2OgnuGfc96XyX4DE+YCiBLCwB+b/k2v16SP/O2Bfe7hF/4urp7FUbcCBpcAvB7gW1oKvuSu2d5Nf5MIrbz9Qf9eTSJx9ufAKiNQ+lCVGo9Dq5IEILQCoLeNG82lRcC2NR1bfWVYnR1tuMmrS9sKp6c4jbZotXWERFN0eVt5dt6m4yYVXYQrw20mgteHOMr454BnCtcK8Z3Lr96aF8euPXPcFZT3gMBp4YR/gNMawbRxaC5z7lGv5vHQasJMaXg+1CvjmJe5wmW/OjQ4+dlb35VVtwG8nuAC7fIgbzxIAwANGPc4dPvqGc+NeEqNQaHXywIQWAOR8yZ0LMrPgrrJV/grkJnItp/bDP7VAhJN8GfY0TMUZtT/mTByBjXP8YG91j3MybS3A9TNcuBT+CFR36tRpKwUcRwP2HoC9Z/vP9slOqvt8FMC1EjM+AX6IA5iaG5Uo8gvAysHw99/aDHwWyh26eT7CjS8pMOB5lmo18J+VQO5X3KHp859zo4Prq1nBtfhyvwaK0+7MN7PkDq/9nuUCXkj3iBqCQquTByq0GAO+fBa4erzrMncZ16Lym4sWMxtsPVaIT366CjUDnG1F2PRsAGaMc+26XneqrnLh9esPXJiperjvkW8GiEfcFWSed4Lt4r+Ac3/nyk36AxD+Ye9OaldeAXZP51ps09cBT7yh33pqNXBwNZD9BcATcIPp+kUYX4/qIuDCPuBCElB91wMbza0An1mA/7PchQ5zS+P30UGt4g6tSzK4CxLiEcDk/zGupTkAUWh18kCFFsBd9v+/qVxvcbvh3PmVwCidh3/Zxbexdv/P+O0Wd9j3XNAIbJg9HnYWBl59a6nnWjeKEq7nes11rh41xdw8dfcDjXB43CHt1FV9cxL7wn6uqwR4wKLvgFHTey7PGHDoVeD8ZwCPDzz7KRDwXO/r0bHtsp+582O/fKPdLUNow3Xl8JsLjJkBmOn5GPCWOuDGeS6kis9yvys7jY3JN+cGBJ66kuvfNohRaHXywIUWwLWC6su51lV3h2XtmltVeP+HAnx2pgiMAcOszPGErwum+zjjMW9nOFj38lK+WgXUld0JsZpiLtRutwcbjwfMfMeww7D2eluY9/DevlsJ5PyTu2vgpdOATTdPsmUMOLIOyPwEAA+Y+wkQON+guuiNMeBmdnuAfQvU3rizTGR3p9/dqMe1W5uKG1w4dYRU+UXuUPpuQlvA/WFgeBBw7Yz24emox4GQVVwwDsIrmxRanTyQoWWEc9eq8Zf9P+NaVaNmHo8HTBguxjQfZ0z3ccZEd3uYCUzT77i8thnpV6uQfrUKab9VoqS6CW5iC4x3s8N4qR3Gu9nBTyqGu4MleDweoGwEPn0CuHUZGPUE8IcDAL9T3RkDfnwTSN/BvX5mJ3eI2h/UauDmeeDiASD/Wy7YO1jYcwHW1gwUZ2iHWwexB+Ah4/5j8pgCuIzX/g/qZhaQtoM7x9bRncV5HBCyApgQqX+rbgCg0OqEQusOZZsa569V46dfb+GnX2/hslz7kMPWwgyPjnHCdB9nTPNxhtS+D87HdKO6QYmzv1Uh7Wol0q9W4eqthnuvBMBWZIZx7UE2xbYCM08vAF/VDPxuPTBt7Z2CjAHHNgJntnKvZ38EBL3Y129DP2o1UHKWO3z85VugodOo7DwBIPEH3KdwAeUu0787xe3r3G1O2V9w5/kArvUZvJR7NpsxFzz6GYVWJxRa3ZMrmnGq8BZO/XoL/y2shKKpVWu5t4uNJsCCRzr0fLh2D4qmVmQWVXMtqauVXQKTxwP8pWJMHe2IKaMd4Se1w/WqRuSX1iK/tBa/lCnwq7weyk6P5XlecBJbzHdDBT7+z3MrrLynYaK7GIG/7oDZmQ+4Qk+9z32JTai6QYkCeR0K5TVo+e00JPKfAJEteB4ySP0ew3gvaa8+XzTVANmfA2d3AXWl3DwzS2BSNDDlz9yV3wGKQqsTCi39qNQMF27U4KdfuRDLLalB5zFjhQI+LMz5sBQKYGkugKXQDJaa12btP/l3LROgQdmGjN+qkHdT0WV7vhJbTBnliKmjHSEb6QixVc8XCFpValy9VX8nyEprkV+qwEbVR5grOIMy5oCnWt7FQsExxJr/CwBwZMQrUE5ehknuw+4cWt5HDS1t+LW8Dr+W16FAXo9fy+twWV6HyvqeH69jLuBhvJsdJnkMwyQPezzkMQwjhhlRX1Ur16JL2w7IL7TP5HGHowHPcQ+YtJVwU1/dDtVLFFqdUGgZp6ZRidNXKnGq/VCyvLb3z7Qa5WSNkNGOCBntiCmjHOFk0/vzLowxlN2qhPiLUFjXX8MtMzc4t3Hnj/7WGo2/q8I1ZR2thZjkYc8Fg7s9Jrjbw0akfz8vtZqhrqUNNY1K3G5sxe1GJarrlbh6iwungvI6lFQ3dbu+h4MVfFxtMVZigzEuNpArWpBTfBvZxTU6Q83ZVoRJ7vZ4yLO9viPsYSnUszXGGFB0ijufV/ij7jLWzu0B5nbXTzft19ZO97zY01sUWp1QaPUeYwyKplY0tarQpFShUalCc6tK8/run5pl7a95PCDIcxhCRjlBIra4f5UsuwD8PZR7YgSAmqlv4JTrQuQU30ZOcQ1+KVWgVaX9J8/nAT6utpjkYY+A4fZgYKhpbMXtBi6UFE13wqmmsRWKplaoOjcXdXC2FcFXYssFlKstfCS28HaxgXU3AckYw43bTchur2tO8W38UlqLtk77EvB5GOdmi5FONrC3NIfY0hz2VuawszS/67VQM19zuFlxGcjczXVbqZNzFwXUrTpqogNPwN3OxeNx3UXAa79Cqc9PAFHJmntlu0Oh1QmF1hCSmwj88Ab31IjHYrUWNbeqkF9WqwmFnOIa3KzpvlXUE0tzAYZZcQExzNocno7WGOtqi7HtQdXrbiTt9c27qeBaYtdrkF18GxV1hrV2hWZ8rXCTiC3h7WIDb2crjBW3wt1MAfPGCi7Easu4nx2hVifnLhh07nphqGU/AdKJPRah0OqEQmuIYUzvvkoVtc3IKalBTnEN8stqIRTwMczKHMOshbC3MscwKyHsLe+E07D2VkyvTpgbiTGGUkUzcopvQ65oRm1TK2qauNZfRyvw7kmfFqG5gIeRTtbwdrHFGBcbeLvawNvFFl5OVhCZCbj7LhsquM6ujAFgBvwE99NlHCC07rEe9zW0du7ciS1btkAulyMwMBDbt29HcHBwt+X379+P9evX49q1a/D29sbmzZvx1FNPaZYzxhAfH49PP/0UNTU1eOSRR/Dxxx/D2/tO7+7q6mqsWrUK33//Pfh8PubNm4ePPvoINjb63aBKoUWGGsYY6lvaNIFW29SK242tuHG7EYUV9SisqMeV8jo0dDOyk4DPg5ejFbxdbOHtagOJ2AICHg98Hg98Pg98HleGx+O1z0f7fB4EfNw1n4dAdzFs73EHxn0LreTkZCxatAi7du2CTCbD1q1bsX//fhQUFMDFpWtv5LS0NEybNg0JCQl4+umnkZiYiM2bNyM7Oxv+/v4AgM2bNyMhIQGff/45Ro4cifXr1yMvLw/5+fmwsODOfzz55JMoKyvDJ598gtbWVixevBgPP/wwEhMT9ao3hRYhXXW03ArL63ClgruQwIVZPepa7nVblv6+X/koAkaIeyxz30JLJpPh4Ycfxo4dXO9itVoNd3d3rFq1Cq+//nqX8vPnz0dDQwMOHrzzbO8pU6Zg4sSJ2LVrFxhjkEqlePXVV7F2LdcpUKFQwNXVFXv37sWCBQtw6dIljB8/HufOncPkyZMBAEePHsVTTz2FGzduQCq99w2jFFqE6I8xhvLaFk2IFZbXoapBCcYYVGoGNQPUjHGTGlAxpnOZSs1ta8cLD2GMS89HRfp+Rw14ngegVCqRlZWFuLg4zTw+n4/Q0FCkp6frXCc9PR2xsdonQ8PCwvDtt98CAIqKiiCXyxEaeufha2KxGDKZDOnp6ViwYAHS09Nhb2+vCSwACA0NBZ/PR0ZGBubOndtlvy0tLWhpuXPCsra21pC3SsiQxuPxIBFbQCK2wDQfZ1NXR4tBN5hVVlZCpVLB1VX70Saurq6Qy+U615HL5T2W7/h5rzKdDz3NzMzg4ODQ7X4TEhIgFos1k7u7u57vkhAykD2wo/HExcVBoVBoppKSElNXiRDSBwwKLScnJwgEApSXl2vNLy8vh0Qi0bmORCLpsXzHz3uVqajQvrm0ra0N1dXV3e5XJBLBzs5OayKEDH4GhZZQKERQUBBSU1M189RqNVJTUxESEqJznZCQEK3yAJCSkqIpP3LkSEgkEq0ytbW1yMjI0JQJCQlBTU0NsrKyNGWOHz8OtVoNmUxmyFsghAx2zEBJSUlMJBKxvXv3svz8fLZs2TJmb2/P5HI5Y4yxhQsXstdff11T/syZM8zMzIy9//777NKlSyw+Pp6Zm5uzvLw8TZlNmzYxe3t79t1337ELFy6wZ555ho0cOZI1NTVpysyaNYtNmjSJZWRksNOnTzNvb28WFRWld70VCgUDwBQKhaFvmRDSD/T9jhocWowxtn37dubh4cGEQiELDg5mZ8+e1SybPn06i4mJ0Sq/b98+5uPjw4RCIfPz82OHDh3SWq5Wq9n69euZq6srE4lEbMaMGaygoECrTFVVFYuKimI2NjbMzs6OLV68mNXV1eldZwotQgY2fb+jdBsPIWRAuC/9tAazjmym/lqEDEwd3817taOGTGjV1XFPyKT+WoQMbHV1dRCLu7/lZ8gcHqrVapSWlsLW1vaeT4Gsra2Fu7s7SkpK6FDyLvS5dI8+G90M+VwYY6irq4NUKgW/8+AkdxkyLS0+n48RI0YYtA7179KNPpfu0Wejm76fS08trA4PbI94QsiDiUKLEDKoUGjpIBKJEB8fD5Fo8Ax02R/oc+kefTa63Y/PZciciCeEPBiopUUIGVQotAghgwqFFiFkUKHQIoQMKhRahJBBhUKrk507d8LLywsWFhaQyWTIzMw0dZVMbuPGjeDxeFqTr6+vqavV706dOoXZs2dDKpWCx+NpBmfpwBjDhg0b4ObmBktLS4SGhqKwsNA0le1n9/psXnzxxS5/Q7NmzTJqXxRad0lOTkZsbCzi4+ORnZ2NwMBAhIWFdXnU81Dk5+eHsrIyzXT69GlTV6nfNTQ0IDAwEDt37tS5/L333sO2bduwa9cuZGRkwNraGmFhYWhubu7nmva/e302ADBr1iytv6Gvv/7auJ3d16d6DTLBwcFsxYoVmtcqlYpJpVKWkJBgwlqZXnx8PAsMDDR1NQYUAOybb77RvFar1UwikbAtW7Zo5tXU1DCRSMS+/vprE9TQdDp/NowxFhMTw5555pk+2T61tNp1jOl49/iL9xrTcSgpLCyEVCrFqFGjEB0djeLiYlNXaUC51/idBDh58iRcXFwwduxY/OlPf0JVVZVR26HQamfMmI5DhUwmw969e3H06FF8/PHHKCoqwmOPPaZ5RhnRb/zOoWzWrFn44osvkJqais2bN+Onn37Ck08+CZVKZfC2hsyjaYjxnnzySc3vEyZMgEwmg6enJ/bt24clS5aYsGZksFiwYIHm94CAAEyYMAGjR4/GyZMnMWPGDIO2RS2tdsaM6ThU2dvbw8fHB1euXDF1VQYMfcbvJHeMGjUKTk5ORv0NUWi1M2ZMx6Gqvr4eV69ehZubm6mrMmDoM34nuePGjRuoqqoy6m+IDg/vEhsbi5iYGEyePBnBwcHYunUrGhoasHjxYlNXzaTWrl2L2bNnw9PTE6WlpYiPj4dAIEBUVJSpq9av6uvrtVoGRUVFyM3NhYODAzw8PPDKK6/gb3/7G7y9vTFy5EisX78eUqkUERERpqt0P+nps3FwcMBbb72FefPmQSKR4OrVq3jttdcwZswYhIWFGb6zPrkG+QDpaUzHoWr+/PnMzc2NCYVCNnz4cDZ//nx25coVU1er3504cYIB6DJ1jPOpz/idD6qePpvGxkY2c+ZM5uzszMzNzZmnpydbunSpZoBnQ9HztAghgwqd0yKEDCoUWoSQQYVCixAyqFBoEUIGFQotQsigQqFFCBlUKLQIIYMKhRYhZFCh0CKEDCoUWoSQQYVCixAyqPx/ThKEiYyLfQoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:01:16.535119Z", + "iopub.status.busy": "2024-03-22T19:01:16.534818Z", + "iopub.status.idle": "2024-03-22T19:03:19.526880Z", + "shell.execute_reply": "2024-03-22T19:03:19.526064Z" + }, + "papermill": { + "duration": 123.014918, + "end_time": "2024-03-22T19:03:19.529471", + "exception": false, + "start_time": "2024-03-22T19:01:16.514553", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:03:19.568996Z", + "iopub.status.busy": "2024-03-22T19:03:19.568673Z", + "iopub.status.idle": "2024-03-22T19:03:19.589191Z", + "shell.execute_reply": "2024-03-22T19:03:19.588310Z" + }, + "papermill": { + "duration": 0.042336, + "end_time": "2024-03-22T19:03:19.591200", + "exception": false, + "start_time": "2024-03-22T19:03:19.548864", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0109920.0049930.000584.4443540.2009549.5630330.4706644.327647e-075.6215140.0151061.2521370.0240850.1404880.00144710.065868
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.010992 0.004993 0.00058 4.444354 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.200954 9.563033 0.470664 4.327647e-07 5.621514 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.015106 1.252137 0.024085 0.140488 0.001447 \n", + "\n", + " total_duration \n", + "realtabformer 10.065868 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:03:19.626727Z", + "iopub.status.busy": "2024-03-22T19:03:19.626449Z", + "iopub.status.idle": "2024-03-22T19:03:20.096934Z", + "shell.execute_reply": "2024-03-22T19:03:20.096067Z" + }, + "papermill": { + "duration": 0.490882, + "end_time": "2024-03-22T19:03:20.099222", + "exception": false, + "start_time": "2024-03-22T19:03:19.608340", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:03:20.137539Z", + "iopub.status.busy": "2024-03-22T19:03:20.136699Z", + "iopub.status.idle": "2024-03-22T19:05:40.481644Z", + "shell.execute_reply": "2024-03-22T19:05:40.480813Z" + }, + "papermill": { + "duration": 140.366735, + "end_time": "2024-03-22T19:05:40.484194", + "exception": false, + "start_time": "2024-03-22T19:03:20.117459", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:40.523302Z", + "iopub.status.busy": "2024-03-22T19:05:40.522476Z", + "iopub.status.idle": "2024-03-22T19:05:40.548513Z", + "shell.execute_reply": "2024-03-22T19:05:40.547780Z" + }, + "papermill": { + "duration": 0.047483, + "end_time": "2024-03-22T19:05:40.550447", + "exception": false, + "start_time": "2024-03-22T19:05:40.502964", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:40.585958Z", + "iopub.status.busy": "2024-03-22T19:05:40.585710Z", + "iopub.status.idle": "2024-03-22T19:05:40.591049Z", + "shell.execute_reply": "2024-03-22T19:05:40.590252Z" + }, + "papermill": { + "duration": 0.025293, + "end_time": "2024-03-22T19:05:40.592955", + "exception": false, + "start_time": "2024-03-22T19:05:40.567662", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.04078416973075253}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:40.630543Z", + "iopub.status.busy": "2024-03-22T19:05:40.630098Z", + "iopub.status.idle": "2024-03-22T19:05:41.096810Z", + "shell.execute_reply": "2024-03-22T19:05:41.095666Z" + }, + "papermill": { + "duration": 0.490429, + "end_time": "2024-03-22T19:05:41.101208", + "exception": false, + "start_time": "2024-03-22T19:05:40.610779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAh0lEQVR4nO3deXhU1f348fedO0sm22QhkAQSEgiCiiCiIG5gpSqu1IoVbQEVsC3WWqQ/xadF0SrWKoVqS7XVRFqXVr+obS3u4lYBBVQUZRNCWMOSPZPZ7vn9McmQIdtkstxJ+LyeZ55kbs7c+7mTzCfnnnsWTSmlEEIIE1nMDkAIISQRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOkkEYkW3XPPPWia1qn7LCoqQtM0Pv3004jK/+53v2PQoEHous6pp57aqbGI2CGJSLTLAw88wMsvv9wtx3rjjTf4f//v/3H22WdTWFjIAw880C3HFd3PanYAomd54IEHuPrqq5k8eXKXH+udd97BYrHw5JNPYrfbu/x4wjxSI+olampqzA6h05WWluJ0OjstCSmlcLvdnbKvjqitrTU7hJgjiagHami72bRpE9dddx2pqamcc845APz9739n9OjROJ1O0tLSuPbaaykpKQl7/QcffMCUKVPIzc3F4XCQk5PDL37xizY/pJqmUVNTw9NPP42maWiaxowZMwAoLi7mpz/9KUOHDsXpdJKens6UKVPYuXNns/uqra3l5ptvJj09neTkZKZNm0ZZWVnYsQoLC6mpqQkdq6ioCAC/3899993H4MGDcTgc5OXlcdddd+HxeMKOkZeXx2WXXcbrr7/O6aefjtPp5PHHH2fVqlVomsY///lPFi5cSP/+/UlKSuLqq6+moqICj8fDbbfdRt++fUlMTOSGG25osu9I3+sJEyYwfPhw1q1bx3nnnUd8fDx33XVXq+/z8UguzXqwKVOmMGTIEB544AGUUtx///38+te/5pprrmHmzJkcPHiQRx99lPPOO48NGzaQkpICwAsvvEBtbS0/+clPSE9PZ+3atTz66KPs3r2bF154ocXj/e1vf2PmzJmMGTOG2bNnAzB48GAAPvnkE/73v/9x7bXXMmDAAHbu3MmyZcuYMGECmzZtIj4+Pmxft9xyCykpKdxzzz1s3ryZZcuWUVxcHEoSf/vb33jiiSdYu3Ytf/3rXwE466yzAJg5cyZPP/00V199Nbfffjtr1qxh0aJFfP3117z00kthx9m8eTNTp07l5ptvZtasWQwdOjT0s0WLFuF0OrnzzjvZtm0bjz76KDabDYvFQllZGffccw+rV6+mqKiI/Px8FixYEHptpO81wOHDh5k0aRLXXnstP/zhD+nXr187f9PHASV6nLvvvlsBaurUqaFtO3fuVLquq/vvvz+s7MaNG5XVag3bXltb22SfixYtUpqmqeLi4ibHaSwhIUFNnz69yeub2+fHH3+sALV8+fLQtsLCQgWo0aNHK6/XG9r+0EMPKUC98soroW3Tp09XCQkJYfv87LPPFKBmzpwZtn3evHkKUO+8805o28CBAxWgXnvttbCy7777rgLU8OHDw2KYOnWq0jRNTZo0Kaz8uHHj1MCBA0PP2/Nejx8/XgHqz3/+c5P3Rxwll2Y92I9//OPQ9ytWrMAwDK655hoOHToUemRmZjJkyBDefffdUFmn0xn6vqamhkOHDnHWWWehlGLDhg1RxdJ4nz6fj8OHD1NQUEBKSgrr169vUn727NnYbLbQ85/85CdYrVb++9//tnqchp/PnTs3bPvtt98OwKuvvhq2PT8/n4suuqjZfU2bNi0shrFjx6KU4sYbbwwrN3bsWEpKSvD7/UD73msAh8PBDTfc0Op5He/k0qwHy8/PD32/detWlFIMGTKk2bKNP3C7du1iwYIF/Otf/wprlwGoqKiIKha3282iRYsoLCxkz549qEYTfza3z2PjTExMJCsrq8U2pQbFxcVYLBYKCgrCtmdmZpKSkkJxcXHY9sbv0bFyc3PDnrtcLgBycnKabDcMg4qKCtLT09v1XgP0799f7vq1QRJRD9a4FmIYBpqmsXLlSnRdb1I2MTERgEAgwHe/+12OHDnCHXfcwbBhw0hISGDPnj3MmDEDwzCiiuVnP/sZhYWF3HbbbYwbNw6Xy4WmaVx77bVR77M1kXa0bPweHau596m17Q3JNdL3OpIYRJAkol5i8ODBKKXIz8/nhBNOaLHcxo0b2bJlC08//TTTpk0LbX/zzTcjOk5LCeDFF19k+vTpPPLII6FtdXV1lJeXN1t+69atnH/++aHn1dXV7Nu3j0suuaTV4w8cOBDDMNi6dSsnnnhiaPuBAwcoLy9n4MCBEZ1HR0T6XovISRtRL3HVVVeh6zoLFy4MuyyC4H/yw4cPA0f/2zcuo5Ri6dKlER0nISGh2eSi63qT4z766KMEAoFm9/PEE0/g8/lCz5ctW4bf72fSpEmtHr8hUS1ZsiRs++LFiwG49NJL2zqFDov0vRaRkxpRLzF48GB+85vfMH/+fHbu3MnkyZNJSkpix44dvPTSS8yePZt58+YxbNgwBg8ezLx589izZw/Jycn83//9X5O2opaMHj2at956i8WLF5OdnU1+fj5jx47lsssu429/+xsul4uTTjqJjz/+mLfeeov09PRm9+P1erngggu45ppr2Lx5M3/6058455xzuOKKK1o9/siRI5k+fTpPPPEE5eXljB8/nrVr1/L0008zefLksFpWV4n0vRaRk0TUi9x5552ccMIJ/P73v2fhwoVAsOH1wgsvDH3AbTYb//73v7n11ltZtGgRcXFxfO973+OWW25h5MiRbR5j8eLFzJ49m1/96le43W6mT5/O2LFjWbp0Kbqu88wzz1BXV8fZZ5/NW2+91eIdq8cee4xnnnmGBQsW4PP5mDp1Kn/4wx8iavv561//yqBBgygqKuKll14iMzOT+fPnc/fdd7fj3eqYSN5rETlNHVu3FEKIbiZtREII00kiEkKYThKREMJ0koiEEKaTRCSEMJ0kIiGE6Xp0PyLDMNi7dy9JSUmdPsm7EKLjlFJUVVWRnZ2NxdJyvadHJ6K9e/c2GSkthIg9JSUlDBgwoMWf9+hElJSUBARPMjk52eRohBDHqqysJCcnJ/RZbUmPTkQNl2PJycmSiISIYW01nUhjtRDCdJKIhBCmk0QkhDBdj24jioRSCr/f3+IEXaJluq5jtVqla4Tocr06EXm9Xvbt2ycra3ZAfHw8WVlZMvm76FK9NhEZhsGOHTvQdZ3s7Gzsdrv8Z28HpRRer5eDBw+yY8cOhgwZ0mqHNCE6otcmIq/Xi2EY5OTkNFllVETG6XRis9koLi7G6/USFxdndkiil+r1/+Lkv3jHyPsXud1ltWzeX0WdT9oj20v+yoToJBt3V/DfjftYvyuyhQjEUZKIhOgkZbXB5ZGsFgulVXUmR9OzSCI6zuXl5TVZI0xEp7IumIg+2naI97ccMjmankUSkRCdwOs3cHuPtg1Vun2tlBbHkkTUC3i9XrNDOO5V1YUnnlqv36RIeqbjMhF5/UaLD3/AiLisL4Ky0ZgwYQK33HILt9xyCy6Xiz59+vDrX/86tLxxXl4e9913H9OmTSM5OZnZs2cD8OGHH3LuuefidDrJycnh1ltvpaamJrTf0tJSLr/8cpxOJ/n5+TzzzDNRxSeaqqwLJh6X0waAL6Ci/v0fj3ptP6LW/PHdbS3+LL9PApNH9Q89f+L97fgCza9BOSDVyZTTj07M9tRHO8Kq5wC/+O4JUcX49NNPc9NNN7F27Vo+/fRTZs+eTW5uLrNmzQLg4YcfZsGCBaHVTbdv387FF1/Mb37zG5566ikOHjwYSmaFhYUAzJgxg7179/Luu+9is9m49dZbKS0tjSo+Ea6hBpQSb6PW68cXULi9AezW4/J/fbsdl4moJ8jJyeH3v/89mqYxdOhQNm7cyO9///tQIvrOd77D7bffHio/c+ZMrr/+em677TYAhgwZwh/+8AfGjx/PsmXL2LVrFytXrmTt2rWcccYZADz55JOceOKJ3X5uvVFuWjyXj8zCYdV5Y9MBfG4fNV4/rnib2aH1CMdlIppzfkGLP7McMwpk9nmDWyx77IiRG8/O70hYYc4888ywISnjxo3jkUceCQ3ePf3008PKf/7553zxxRdhl1tKqdBQly1btmC1Whk9enTo58OGDSMlJaXTYj6eJcXZSIoLJh2nTafS7cMjl2YRMz0R7dmzhzvuuIOVK1dSW1tLQUEBhYWFTT5onak91eWuKttRCQkJYc+rq6u5+eabufXWW5uUzc3NZcuWLd0V2nFveP9kBmckkOKU2lCkTE1EZWVlnH322Zx//vmsXLmSjIwMtm7dSmpqqplhxYQ1a9aEPV+9ejVDhgxB1/Vmy5922mls2rSJgoLma3vDhg3D7/ezbt260KXZ5s2bKS8v79S4j1c7D9Xg8RtkJemM0IshOw/iZMaCSJmaiH7729+Sk5MTakwFyM/vvMubnmzXrl3MnTuXm2++mfXr1/Poo4/yyCOPtFj+jjvu4Mwzz+SWW25h5syZJCQksGnTJt58800ee+wxhg4dysUXX8zNN9/MsmXLsFqt3HbbbTidzm48q97r0+IySo7UcrXra5JrNkFCHzhjZtPrd9EsU5v0//Wvf3H66aczZcoU+vbty6hRo/jLX/7SYnmPx0NlZWXYo7eaNm0abrebMWPGMGfOHH7+85+HbtM3Z8SIEbz33nts2bKFc889l1GjRrFgwQKys7NDZQoLC8nOzmb8+PFcddVVzJ49m759+3bH6fR67vqBrgmV3+ILGNSU7afyyEGTo+o5TK0Rffvttyxbtoy5c+dy11138cknn3Drrbdit9uZPn16k/KLFi1i4cKFJkTa/Ww2G0uWLGHZsmVNfrZz585mX3PGGWfwxhtvtLjPzMxM/vOf/4Rt+9GPftShOEWQxxfAGnBjp5oDtV5Kytxorq2cmS6JPhKm1ogMw+C0007jgQceYNSoUcyePZtZs2bx5z//udny8+fPp6KiIvQoKSnp5oiFaJ7bGyDOX4lV19Abbr3WSo0oUqYmoqysLE466aSwbSeeeCK7du1qtrzD4QitYSZrmYlY4QsY+A2Fw1+F1aJhrZ/DSdX13qaDzmbqpdnZZ5/N5s2bw7Zt2bKFgQMHmhRRbFi1apXZIYh2aGgfcho16JajNSLNU2VmWD2KqTWiX/ziF6xevZoHHniAbdu28eyzz/LEE08wZ84cM8MSol0axpQlqGo0NHDVDxHySI0oUqYmojPOOIOXXnqJ5557juHDh3PfffexZMkSrr/+ejPDEqJdEh1WLh+ZxWl9gx8nS30isvhk9ZhImd6z+rLLLuOyyy4zOwwhohZn0ynomwS7AlAFlsSM4A/8HjAMkHm/22R6IhKi1/C5AbAnZ5CdEoeuaeCvA7usItMWSURCdNCRGi8HK+vIrqogyaYRl+BiYN9U8HuDyUkSUZukzihEB+08XMPrX+xif1l1cIMtHqz1Q2f8bvMC60GkRiREB3n9BlajLngpplvBaqcWO3j92D218iGLgNSIhOggr9/AFnAH+w/ZgpdhHxfX8PnuCqqr5RZ+JCQRCdFBvkDTRKRswUuzgEcuzSJxfCUipYINiN39UM3Ped2c5cuXk56ejsfjCds+efJkGaAao4KXZp6wRKQ1JCKv9CWKxPF1+RrwwQctz+nTZc69HayRTZI1ZcoUbr31Vv71r38xZcoUILj6xquvvtrqyHphHm/AQFdedF0DqyO40RoHQMArK75G4viqEfUATqeT6667LmyyuL///e/k5uYyYcIE8wITLfL4DXTDG6wR1Scirf4fj+H3tPZSUe/4qhHptmDtxIzjtsOsWbM444wz2LNnD/3796eoqIgZM2aETaYvYse4QekEAgkkVFpDiUi3BWtEhk8SUSSOr0SkaRFfIplp1KhRjBw5kuXLl3PhhRfy1Vdf8eqrr5odlmhBTlo8uHSo1UEPJiKLLfjV8MmlWSSOr0TUg8ycOZMlS5awZ88eJk6cSE5OTtsvEuZpuASrbxvKTk/B6ooj2SZLCkVC2ohi1HXXXcfu3bv5y1/+wo033mh2OKIVWw5UcbC8EkOp0KXZ4Kw0BqYnkOqQy+lISCKKUS6Xi+9///skJiYyefJks8MRLQgYile/2MdnOw6EJaKGSzQC0kYUCbk0i2F79uzh+uuvx+FwmB2KaEHDpGhWw4uu6aFE5MWG4Q+g4UZ+e22TRBSDysrKWLVqFatWreJPf/qT2eGIVngDwURkx4umxYfaiL46WEfdrnL6JNoZInMStUkSUQwaNWoUZWVl/Pa3v2Xo0KFmhyNa0VAjsuEPbtCDd2VtNgd1BC/dCHjBEmdShD2DJKIY1NK6ZSL2eAMGmvJj04IT6DfUiHSrDaVZMBTBdiKbJKLWSH1RiA4IjjPzHl3LrKFGpGsENHuwAdvvNTHCnqHXJyLVjgGnoil5/1rnC9QP72joLFvfFqRbLAQs1mAiMnwmRxn7em0istmCwypqa2X0c0c0vH8N76cI1y8pjvMLXGSlOEOXZQBWi4ahWTEMgm1EolW9to1I13VSUlIoLS0FID4+XsZqtYNSitraWkpLS0lJSUHXdbNDikmueBuuPnbYYw9dlgFYdY2AZsMwFAT8JkbYM5iaiO655x4WLlwYtm3o0KF88803nbL/zMxMgFAyEu2XkpISeh9FC44Z3gGQ4LDSPz2ZpLpqqRFFwPQa0cknn8xbb70Vem61dl5ImqaRlZVF37598fnkOr29bDab1ITaUFpVh+9QGen+AHGNElFynI2TBvSBw+XSRhQB0xOR1WqN+D+ux+MJm7mwsjKy+YB1XZcPlOgSX5RUcOjrPZyjeRjQ/5iZHfT6j1dAElFbTG+s3rp1K9nZ2QwaNIjrr7+eXbt2tVh20aJFuFyu0ENGpAuzeQONJ0U7WiNSSuExdDz+AEouzdpkaiIaO3YsRUVFvPbaayxbtowdO3Zw7rnnUlVV1Wz5+fPnU1FREXqUlJR0c8RChPP6Dayqfr7qRo3VfkPx302HWb+rHJ9PElFbTL00mzRpUuj7ESNGMHbsWAYOHMg///lPbrrppiblHQ6HDAAVMcXrN4gzfE1qRFaLhmEJfrwM6dDYJtMvzRpLSUnhhBNOYNu2bWaHIkREPIGm81VD8EYJlmDfq4DUiNoUU4mourqa7du3k5WVZXYoQkTE5zewqqaJCECzBhOR1IjaZmoimjdvHu+99x47d+7kf//7H9/73vfQdZ2pU6eaGZYQEfOGDfEIH9iq6Q0reUgiaoupbUS7d+9m6tSpHD58mIyMDM455xxWr15NRkaGmWEJEbFzh/QhudaG3RoIa6wG0HSpEUXK1ET0/PPPm3l4ITrs5GwXfKuDz9KkRmQJXZpJP6K2mN6hUYgeTalGQzzC24gG9HHhKnPgsARMCKxnkUQkRJS8foO9hyvoU+cl0WFtkohOy+8L5YlglalU2hJTd82E6EnK3V7+s34nm/dXBhfvPKaNqOH2vQzxaJvUiISIkvfYW/fHTDMTsNhQhoHm9yAjHVsniUiIKHn9DZ0ZLUfXMWvknS1HSNxZxsCMJLJNiK8nkUszIaLkC6hme1U30KzBSzUVCIAhDdatkUQkRJSCE+d7sGg0m4is9YnIUEraidogiUiIKHkDAXTlw2pp2qsawKJbUWjB6WJlcrRWSSISIkqexksJNVcj0i0YFmv92maSiFojjdVCRCm/TwJ9sp2klTmabay26hbqNCuGMiQRtUESkRBRynI5yUq3Q52t2RqRbqlfyUPVyQT6bZBEJERH+OuCX5tJROkJdhyuRBLxgyFLCrVGEpEQUSqtrMNWWU2SYWBtJhHl9UmAnHSo9EgiaoM0VgsRpVWbD7J2214q3L5m75oBoMswj0hIIhIiSp5Ao7tmx44zq2doVgJKbt+3RRKREFHy+g101XQpoQYlR2r576ZDfLmnQpadboMkIiGi5G3oR9TMNLEQvGtmaFbp0BgBSURCREEphc8fQA8tJdRMP6KGRCQdGtskiUiIKAQMBQEPoFq8NLPqFgKaNTjWTGpErZJEJEQUvPUN1QC61XZ0nftGQpdmSkkbURukH5EQUdAtGmfnJZLudqK1cOvepmuhsWYq4EVrtpSAGKoRPfjgg2iaxm233WZ2KEK0yWHVGZkZx4DU+Bb7EDXUiAACspJHq2KiRvTJJ5/w+OOPM2LECLNDESJyLaze0cBmsZCVlkS61y6N1W0wvUZUXV3N9ddfz1/+8hdSU1PNDkeIiNR6/Rwsr6DOF2ixRmSxaEw4MZsT+iVhRWZobI3piWjOnDlceumlTJw4sc2yHo+HysrKsIcQZth1pJa3Nu7i20M1LdaIgKMrechds1aZvtLr+vXr+eSTTyIqv2jRIhYuXNjFUQnRNm/YpGgtjDMD0K0oglPFSmN1y0yrEZWUlPDzn/+cZ555hri4Vn6RjcyfP5+KiorQo6SkpIujFKJ5wRU8PPW9qluuEb2wfj+rvz1CVY27G6PreUyrEa1bt47S0lJOO+200LZAIMD777/PY489hsfjQdfDV4NyOBw4HK1Ug4XoJqFpYq2t14gMS/AjZsjEaK2KKhF9++23DBo0qEMHvuCCC9i4cWPYthtuuIFhw4Zxxx13NElCQsQSjz+AbnjrJ85v+Z+jJbSkkLQRtSaqRFRQUMD48eO56aabuPrqqyO+tGosKSmJ4cOHh21LSEggPT29yXYhYo3HF1xKqKUVPBpY6ucjMqQfUauiaiNav349I0aMYO7cuWRmZnLzzTezdu3azo5NiJhV5w+0uoJHA80aTEQq4AfD6K7wepyoEtGpp57K0qVL2bt3L0899RT79u3jnHPOYfjw4SxevJiDBw9GFcyqVatYsmRJVK8VojudlOViUJqVBIe11RqR3niRRZkutkUdumtmtVq56qqreOGFF/jtb3/Ltm3bmDdvHjk5OUybNo19+/Z1VpxCxJShmUnku3QS7K0notClmUL6ErWiQ4no008/5ac//SlZWVksXryYefPmsX37dt5880327t3LlVde2VlxChFblDq6RFArl2Z9k+NITnBit1pkmEcromqsXrx4MYWFhWzevJlLLrmE5cuXc8kll2CxBPNafn4+RUVF5OXldWasQsSMfUfKSfH6ibNaWhx9DzB2UDrsTQefWy7NWhFVIlq2bBk33ngjM2bMICsrq9kyffv25cknn+xQcELEIn/A4KW13zJqbzmnD8rA1sxcRGF0WzARSY2oRVElojfffJPc3NxQDaiBUoqSkhJyc3Ox2+1Mnz69U4IUIpYEOzN60ACr3dn2C2S8WZuiaiMaPHgwhw4darL9yJEj5OfndzgoIWKZx29gDdShWzQ0W+uJ6JOdR1i1rYwdh2ukRtSKqBKRUqrZ7dXV1VF1bhSiJ/H4A1iNumBnRlt8q2WVAq/S61fykDailrTr0mzu3LkAaJrGggULiI8/+ksIBAKsWbOGU089tVMDFCLW1NX3qtZ1DdqoEVn1xvNWS42oJe1KRBs2bACCNaKNGzditx9d3dJutzNy5EjmzZvXuREKEWM8/gA2ow6rxdJmjchq0epX8kDaiFrRrkT07rvvAsHBqUuXLiU5OblLghIilnl8R9uI2qwRWSyykkcEorprVlhY2NlxCNFj9EuOIy7DSlqNve0aka5haDpGQNY2a03Eieiqq66iqKiI5ORkrrrqqlbLrlixosOBCRGrMl1xZKZZQXO0WSPSLfVLCvmRNqJWRJyIXC4XmqaFvhfiuOarDX5tIxEl2K2kJiWQUKVLjagVESeixpdjcmkmjmeHqz04a6pxKIXexqVZpiuOzBOzYcd2aSNqRVT9iNxuN7W1taHnxcXFLFmyhDfeeKPTAhMiVr3+5X42fLuXSrevzRoRID2rIxBVIrryyitZvnw5AOXl5YwZM4ZHHnmEK6+8kmXLlnVqgELEGo/Xg0UFsEbQjwiAhrFo0kbUoqhnaDz33HMBePHFF8nMzKS4uJjly5fzhz/8oVMDFCLW+D01AFitNtDtrZatrPPxf58dYF1xmfSsbkVUiai2tpakpCQA3njjDa666iosFgtnnnkmxcXFnRqgELHEFzBQnmCzhNURD1rrq5VZNI0avxZ8nazk0aKoElFBQQEvv/wyJSUlvP7661x44YUAlJaWSidH0au5fQFsRi3BOfMT2yzf0LNaIRPotyaqRLRgwQLmzZtHXl4eY8eOZdy4cUCwdjRq1KhODVCIWFLnDWAPuLFaLGiOpDbLWy3BsWYgiag1UfWsvvrqqznnnHPYt28fI0eODG2/4IIL+N73vtdpwQkRa9y+ALZATbCh2p7QZnndoqFCiyxKImpJ1Cu9ZmZmkpmZGbZtzJgxHQ5IiFiWHGfjlAwrKWUOsLd9aaZpGlpobTNpI2pJVJdmNTU1/PrXv+ass86ioKCAQYMGhT0itWzZMkaMGEFycjLJycmMGzeOlStXRhOSEN0iNcHOiWkaWS4nONpORABa/Z01qRG1LKoa0cyZM3nvvff40Y9+RFZWVmjoR3sNGDCABx98kCFDhqCU4umnn+bKK69kw4YNnHzyyVHtU4gu5w3evo+kRgTQx5VAksOKxfAHZ0qL8vPSm0WViFauXMmrr77K2Wef3aGDX3755WHP77//fpYtW8bq1aslEYmYdKTGi6O6gjil0CNMRFeMGgg19eMzlQGa3oUR9kxRJaLU1FTS0tI6NZBAIMALL7xATU1N6C7csTweDx6PJ/S8srKyU2MQoi3vbT5Av+27KejjpG8EjdVAcBWPBgEfWCQRHSuqNqL77ruPBQsWhI03i9bGjRtJTEzE4XDw4x//mJdeeomTTjqp2bKLFi3C5XKFHjk5OR0+vhDt4XbXoCkDu26J+NIMzRJ8gIw3a0FUNaJHHnmE7du3069fP/Ly8rDZbGE/X79+fcT7Gjp0KJ999hkVFRW8+OKLTJ8+nffee6/ZZDR//vzQvNkQrBFJMhLdKVBTDoDVmXR0DFkbVn65nz4llQxOs5MmDdbNiioRTZ48udMCsNvtFBQUADB69Gg++eQTli5dyuOPP96krMPhwOFoeXlfIbpSwFAY7goA7AkpEb+uzh/AHbAQCMhKHi2JKhHdfffdnR1HiGEYYe1AQsSKGq8fe6Aaiwa2diQiPTRvtSEj8FsQdYfG8vJyXnzxRbZv384vf/lL0tLSWL9+Pf369aN///4R7WP+/PlMmjSJ3NxcqqqqePbZZ1m1ahWvv/56tGEJ0WVqPH4c/ipsugUtLvJZSo+u5OGRNqIWRJWIvvjiCyZOnIjL5WLnzp3MmjWLtLQ0VqxYwa5du0JzFbWltLSUadOmsW/fPlwuFyNGjOD111/nu9/9bjRhCdGlgomoOthQ3c5EZDQkIpmlsVlRJaK5c+cyY8YMHnroodB0IACXXHIJ1113XcT7efLJJ6M5vBCmSI23MyxVkeh1tC8R6Rq+hiWFpEbUrKgS0SeffNJsY3L//v3Zv39/h4MSIhalJ9hJT/KDLw4ckU93o1sseGS111ZFlYgcDkeznQm3bNlCRkZGh4MSIib5asFXFxyiER95h95EhxUtPg4bFqkRtSCqDo1XXHEF9957Lz5f8E3VNI1du3Zxxx138P3vf79TAxQiVpTu343bF8BwJIf3lm7D6IGpnDcsm+wUp7QRtSCqRPTII49QXV1NRkYGbreb8ePHU1BQQFJSEvfff39nxyhETPjoiy18VlJOlSWKdf1kJY9WRXVp5nK5ePPNN/noo4/4/PPPqa6u5rTTTmPixImdHZ8QMcEfMKDmEABxKf3avwNZyaNV7U5EhmFQVFTEihUr2LlzJ5qmkZ+fT2ZmJkqpqKcEESKWVdb5ifeXoVs07MntawfdfrCabV8fpsBdzeABcmnWnHZdmimluOKKK5g5cyZ79uzhlFNO4eSTT6a4uJgZM2bINLGi16p0+3D6ynBYLWjxfdr1Wn9AUeEBj196VrekXTWioqIi3n//fd5++23OP//8sJ+98847TJ48meXLlzNt2rRODVIIs1VVV2ELuHE4bJDQvkSkWzQCFiuGIf2IWtKuGtFzzz3HXXfd1SQJAXznO9/hzjvv5Jlnnum04ISIFe7yUgCs8S6wtm/g9dGe1dKPqCXtSkRffPEFF198cYs/nzRpEp9//nmHgxIi1ngqgolIT2x/Pzmr3igRyej7ZrXr0uzIkSP069fyHYN+/fpRVlbW4aCEiDVDEj34Up0k9slq92utodH3SI2oBe1KRIFAAKu15Zfouo7fLxlf9D6Z1mpIjYeMKBKR3jD6XtqIWtKuRKSUYsaMGS1OTibzCIleqzbYh4h23jEDsOkWEpxxOKy69KxuQbsS0fTp09ssI3fMRG/jdrvxHDlEnE0nLj693a93OW38YOwgWOeSGlEL2pWICgsLuyoOIWJW6YE97N1fhSM+kdPs8dHtpGFsmrQRNSuqsWZCHE8abt1rCR2YWcJS/z9fakTNkkQkRBu8lfV9iJKiS0RKKf65YT+f7y7H5/UGV3sVYSQRCdEGf9VBgHaPMWugaRoHawxqvQEC0peoWZKIhGiDUd2BUff1tPpuL0r6EjVLEpEQrQn4Ue5gJ934DiQim27F0HTpS9QCSURCtMJTdRB/wCBgsZOcHMWEaPUahnkEDCV9iZoR9bpmQhwPLO7DDM5IoDouE4ct+o+LVbc0Gm8mNaJjmVojWrRoEWeccQZJSUn07duXyZMns3nzZjNDEiKMzVNO36Q4BuXmdmg/9oaBr4aMwG+OqYnovffeY86cOaxevZo333wTn8/HhRdeSE1NjZlhCXFU7eHgV2fkq3Y0J95uxWa3B1cAkRpRE6Zemr322mthz4uKiujbty/r1q3jvPPOMykqIY4qP3wA3F4ctmScHdjP5SOzIZAFFXukjagZMdVGVFFRAUBaWvP/fTweT9jA2ubWVhOi0yjFvn17KK+qIakgjuEd3Z+s5NGimLlrZhgGt912G2effTbDhzf/K1+0aBEulyv0yMnJ6eYoxXHFW43XUwdoJKS0f9R9EzLerEUxk4jmzJnDl19+yfPPP99imfnz51NRURF6lJSUdGOE4nijag/j8RvUWZNIjo/r0L4+Lynnw2/L2Vfhlp7VzYiJS7NbbrmF//znP7z//vsMGDCgxXIOh6PFuZCE6GzeykMEDEWd3UWyM/KVXZtT4/Fz2K3or8tKHs0xNREppfjZz37GSy+9xKpVq8jPzzczHCHCuCuCY8xUfBo2vWMXDzZrsB9RQPoRNcvURDRnzhyeffZZXnnlFZKSkti/fz8QXEnW6ezIPQohOq6uftS93s7lg5pjtWgYFitGQPoRNcfUNqJly5ZRUVHBhAkTyMrKCj3+8Y9/mBmWEAB4q4KDXe3JHU9EtrCe1dJGdCzTL82EiEkBPxl6HXEZCXgGDuzw7hoSUUB6VjcrJhqrhYg5deU4bRaccUmQ2YGZGevZQit5IG1EzYiZ2/dCxJTaI8Gv8enBYRkdZNMtWG02LBrSs7oZUiMSohmq9hCllXVY4hNINxQWS8eSUU5aPDljBsGmjVIjaobUiIRohrviIN8equH93Ubn7VR6VrdIEpEQzair70OkJ6Z3uDYU0rCSR8DbOfvrRSQRCdGMzrx1D+DxB1j59WG+3FuB4ZdEdCxJREIcy1uL1x2cE8vZSYnIomkUVwSoqvNj+GVp9mNJIhLiWO4j1PkCePREkhMTO2WXVotGwGIHIODzyNpmx5BEJMSxGkbd21y4OjjYtYGmaVhswRH8KhCQButjSCIS4li1h6nzBXBbU0iJ75xEBKBbbYAWHPgakMuzxqQfkRDHULVHOKFfEmlZgzqtRgRgs+oELLbgMA+/F2RGmxBJREIcQ6s9gstpwzUwBzo4/UdjdqsFv2avX8lDakSNyaWZEI0ZAahf2ZX49E7dtcOqo1ntGABy5yyM1IiEaKyugiPVdXiVTlwgjpRO3PX3T+uPpuVAxW5JRMeQGpEQjdUe5kBVHV+V6ZSU1XXqrjVNA2t9w5BcmoWRRCREY7WHcXsDuG0ppCZ0XkN1iB7sS4T0rg4jl2ZCNOKrLMXjN6hNSCE9oXNva32zv5KyXVXkedxkSY0ojNSIhGik+sheAFRCBk673qn7rnT72VulqPUGpI3oGJKIhGhgBHCXHwAgKT2703dvt1qO9iOSEfhhJBEJ0aD2MDVuDwGLjbT0vp2+e7tuIWCx13do7NyG8J5OEpEQDapLqfUGqLWlk+nq/OWs7FYLAc1+tGe1CJHGaiEa1JRycnYyfZMLSHV1bInp5jis9TUiGWvWhKk1ovfff5/LL7+c7OxsNE3j5ZdfNjMccbyr3IvVYiFrQD5xts5tqIaGRNQw1kwSUWOmJqKamhpGjhzJH//4RzPDEAKMAKoyeMeM5AFdcoiGsWZKIYnoGKZemk2aNIlJkyZFXN7j8eDxHP0FVlZWdkVYopczDMXuMjduX4DslDiS4myUl+5mZ/Eh0lNSGOBMpZNmqQ7jctqYef5JWNZ+AH53Fxyh5+pRbUSLFi1i4cKFZochergPth1ifXFwYKumwcD0eJz7P8PlMzhAGjmWrrlQ0DQNzR4ffBLwBx96j/oIdpkeddds/vz5VFRUhB4lJSVmhyR6oDPyUklPtNM32YFSsPNQLdqR7VgtGiecOKJrD251gFb/sfPVdu2xepAelY4dDgcOh8wmJTom3m7lR2cORNM0ymq8bN5ziKyqCvpmuHD2H9alx353y0H6lHoY7NKI97khLrlLj9dT9KgakRDRUkpRWnm0E6FWv4x0aoKdM5MOMTA1DqerL8SndWkce8rcHKyz4PEb0k7UiCQicVwoOeLmmTW7eGnDblTjFTSUgr3rg99nDu/yOJw2Hb8lDr+hwCeJqIGpl2bV1dVs27Yt9HzHjh189tlnpKWlkZuba2Jkorf5ZOcRAFLi7aHaEAAHN0PlvuAqrJld3D4ExNl0fJY4/IFKSUSNmJqIPv30U84///zQ87lz5wIwffp0ioqKTIpK9DYHKuvYdaQWi6ZxWm5qcKNScHg7bP5v8HnOGHB0zhpmrXHaLVRZHFIjOoapiWjChAnh1WQhukBDbWhoZhKuOCsc2go7P4Sq/cECKTmQd063xBJn1SnT4/AHlLQRNdKj7poJ0V5lNV62lVYDcHq2HT57Fsp3BX+oWyF7FOSPB0vnD+lojsOm47c48PsNqRE1IolI9GrristQCoYl19Fn8/NQVxFMQP1PD16O2RO6NZ6GxmpDIYmoEUlEotcKGIo95W5c7hLOUp+CzQBnKpwyBRI6d6mgSA3NTGLY2cOwfPUVeGtMiSEWSSISvZauwY8GHKSi/CNcNluwLejkq6BhmIUZMVk0iEsKPvFWmxZHrJFEJHonnxu2vonlwFekOm2QeQoMndRtbUGtarg7560Bw4AuGtvWk0giEr2LYcD+z6nc9BaJFh8Wiw6Dz4cBZwRHuJpMKcXrW6rI2V/FCRkJ2Hw14EgyOyzTSSISvUflPtjyGnVH9vD17nL8cemMvOAaEvvlmx1ZiKZpFB9x4/TYyAsY2DzVkoiQRCR6AyMAO96DkrWgFN+W+/g2ZRy23NM5q2/s9dBPcFjx6vF4/XUkSDsRIIlI9HQ+N3y5ItQ3aI8jn3eShxGwJvDDYZnhwzliRKLDildPwBeoBY9M7geSiERP5vfAF/8IXpLpNiryLubl7XH4dIOx+amkJ8bmlDEJDis1ejxevwEeqRGBjL4XPVXABxtfDCYhm5Oa4dfxf7sS8PoN+qc4OTPfnH5CkUhw6Hj1BLwBI9jBUkgiEj2QEYCvXg5ejlntMOIHfLjPQoXbh8tp49IRWVgssXdJ1iDJYcNtcwVrRO4ys8OJCXJpJnoWw4Cv/wWHtwWHapwyBZKzOG9IgKo6PxNP7EuCI7b/rF1OG3VWV3BZIfcRs8OJCVIjEj2HUrDlNSj9Biw6h/Mug5TgXTGnXefq0QNIibebHGTb+qc6ufG7p3Fytgt8deCVuaslEYmeQSnY9hbs+xwD2JB4Lss362zc3fPaWHSLhs3uODpftdSKJBGJHkAp+PZd2P0p3oDBKu10Vh0JNkaX1fbgNeSd9fNj10oiiu2LaSGUgh3voXatpqzGy7tqNLviBmLTNS44sR8nZvXMVTA27a3k0B4Y4nOTVX3A7HBMJ4lIxC4jAJtXUlfyGd8equHzuDPYnzSUjCQHFw/PpE+M9hOKRK3Xz06fi8w6P1kNS10fxyQRiZhiGIoKt49AzWH6lLwOlftQmoUNzjM5nHwSYwamMjY/Daves1sV+iXHsdaeQXWFH6oPHPervh6/Zy5ixqFqD1v2V7HrcDV1h0tIrdpMgbGDPpmJYHXgPOUKhtWkUZCRhCveZna4naJvsgOfzUWViqPO4yWuYhekDTI7LNNIIhLdTymo3Ev14d18s30nBw/uJ85XSa6/Eg2FRQOL0xb8YA6dBHHJjO5jdtCdy2HVyU51Un4kl7LaXWQd2iaJyGx//OMf+d3vfsf+/fsZOXIkjz76KGPGjDE7rNhVVxHsVewuo8YbICGlH6QOBHsCtV4/DqsenAkwligFVfugdFOwH5CniuK9Ffjq/KQAqfE20lLjSUqIx5E5FEv2qOA59WKDMhL5fG8+Byu3knngS7RBE4I9xY9Dpieif/zjH8ydO5c///nPjB07liVLlnDRRRexefNm+vbta3Z4sSPgCy4GuP8LvId2cLjaw6FqL9UeP2fkpWLVdUjN4xt3Pz6t6UNuvwwK+iYyMD0e27HtKb46qKug7MhBaqorMTQLSneg4lIgLhXdZsemW8h0xYVesr+ijjibhaQ4W+RJzjCCyefQZji4GX9N8Da11WIBq53M/JM5UmHnpII80vv0C84n7UiOiQnMusNJWcn8b2sOh8sSKKusIq1kNeSfZ3ZYptCUyQuLjR07ljPOOIPHHnsMAMMwyMnJ4Wc/+xl33nlnq6+trKzE5XJRUVFBcnLPvI3booA/OEVE1T78h7ZTt3cTVTW1lLt9VNT6qHT0o9aWhgXFhP4Bkn3BD/mWA1UcqvFRY++D25YKVjvp8VaStDqSLXXkJfiCiYjgChfegHHMgTU81gSI78OEUSdCQh9wpvHi56UcrAlg1QzS4qCPQ5EWp0i1G6Q4FIkWf3A0vL8u+NVXi6o5hM/npdbjp6zWR2ltgJQBJzF0xNjgZchx3Djb4IOtB/Hv38RZdR/gsFkh79zgEkcmzqvdmSL9jJr6l+D1elm3bh3z588PbbNYLEycOJGPP/64SXmPx4PH4wk9r6yMcC6XIztg+9vBywNgf2UdxYePWUGhUT4+oV8iqfVDBUor3ew4VEPDT7X674LFFSf0TSItIVj2UHVdaA0tOLo/rX7fgzISyUhygFKU1XrZfKAq7LiNvx+UYqFvUrBGUlHjYcuBajzWJEoTTuRQ9gmkpWcwNDOJE/olBcdW1R6Bg99QkPgNmYd2c7imirKaw3hqDKiAGsBvtZDXsNKpPR49xUlAObBiYDU8WL3lWPwe4nFj9+2BkqpQPKccqKK81htcBqdedf2j1Go5uoIq8NXeCty+AAFD4dNslMflciQ+j7LkXPolJXFCn5yYnCfIDGcP7oOl4DzY6oE96zi48U1K3nsFQ4/D0B2gaSjNUl9L1BjaL4mU+gb70koPO479O25kSN8k0hKCZQ9Ve9l+sOUpRwZlJJBR3x3iSI2PraVVrZYB4JRrjvYO7yBTE9GhQ4cIBAL069cvbHu/fv345ptvmpRftGgRCxcubP+B/B6oPhh6qqrdGNUtj+/Rar1A/bW6uw6j9ugv+9jqo6oLgF7/y6nzoOqa/rJDr/EAdl9wm9eL8rT8R6RUQnA99oQ+aGmZbLUmk9Q3l7zUBC7sl9h0TFV8Ggw8C8vAs0h2l5NctY+82sOUV7sprwtQYzhQjkTU4IFocSlgtXNq04OCrxZqD0PNoeDX2sPgLmNoTgLK8FMX0KjxW6gO6FT6LFT4LOhx8TAgO9i+YXWy2X2ISp+O25aCx5ZMSryD7BQn5/ZLIi89XpJQI6FZAoZ8F5KzqP5sFR5/NfhrgWb+RhPdYNT/7mvqMKpaWZIovhZU/d9mtQejqpW5j5yJQH3ZGi9GVdNEFFYGQB1bm45ej6obz58/n7lz54aeV1ZWkpOT0/YLU3Jg5LXB7zWNRI+f3Fo/aFqzzRFxiQ6wWgCNRG+A/DpfM+0WwdfGx9nAHnwbE3wBBtf5Qz8Llav/LjHOCrb6soEAQ+oCR/emBcsGv9eIT0wO/uI1jVSlmDayHR9eZwo4U9CA1PpHRDQtuOCgPSE0mDT8jMFZ/2h8E0spFfb+nOOqC46nslpw2vSmbVSiKU2DzFPIuuAk4quqMOoqwe9FKQNlqNCHPi7BDrbg+xnvrf87boEzwQY2vb5sgNxaX4tl4+NtYA+WdfoC5NY0Ldu4DNCpi1Oamoj69OmDruscOBDexf3AgQNkZmY2Ke9wOHA4ouhNa0+AtKMTqCfVPyIRX/+IRMOHNBIOwv63tCrWaxDHxtc3Oa6FkqItcTaduLQUIKXNson1j0i09++4f4RlO4up/6rsdjujR4/m7bffDm0zDIO3336bcePGmRiZEKI7mX5pNnfuXKZPn87pp5/OmDFjWLJkCTU1Ndxwww1mhyaE6CamJ6If/OAHHDx4kAULFrB//35OPfVUXnvttSYN2EKI3sv0fkQd0av7EQnRC0T6GZXbGUII00kiEkKYThKREMJ0pjdWd0RD81bEQz2EEN2q4bPZVlN0j05EVfXd0CPqXS2EME1VVRUul6vFn/fou2aGYbB3716SkpJivvdxg4ZhKSUlJb3qTl9vPK/eeE7QveellKKqqors7GwslpZbgnp0jchisTBgwACzw4hKcnJyr/rjbtAbz6s3nhN033m1VhNqII3VQgjTSSISQphOElE3czgc3H333dHNIhDDeuN59cZzgtg8rx7dWC2E6B2kRiSEMJ0kIiGE6SQRCSFMJ4lICGE6SURd7P777+ess84iPj6elJSUiF6jlGLBggVkZWXhdDqZOHEiW7du7dpA2+nIkSNcf/31JCcnk5KSwk033UR1dSurRAATJkxA07Swx49//ONuirh5f/zjH8nLyyMuLo6xY8eydu3aVsu/8MILDBs2jLi4OE455RT++9//dlOk7dOe8yoqKmrye4mL6+Z5x5XoUgsWLFCLFy9Wc+fOVS6XK6LXPPjgg8rlcqmXX35Zff755+qKK65Q+fn5yu12d22w7XDxxRerkSNHqtWrV6sPPvhAFRQUqKlTp7b6mvHjx6tZs2apffv2hR4VFRXdFHFTzz//vLLb7eqpp55SX331lZo1a5ZKSUlRBw4caLb8Rx99pHRdVw899JDatGmT+tWvfqVsNpvauHFjN0feuvaeV2FhoUpOTg77vezfv79bY5ZE1E0KCwsjSkSGYajMzEz1u9/9LrStvLxcORwO9dxzz3VhhJHbtGmTAtQnn3wS2rZy5UqlaZras2dPi68bP368+vnPf94NEUZmzJgxas6cOaHngUBAZWdnq0WLFjVb/pprrlGXXnpp2LaxY8eqm2++uUvjbK/2nlekf5tdSS7NYsyOHTvYv38/EydODG1zuVyMHTu22dVvzfDxxx+TkpLC6aefHto2ceJELBYLa9asafW1zzzzDH369GH48OHMnz+f2tqWF7rsSg2rDDd+n1tbZRiC5924PMBFF10UM78XiO68AKqrqxk4cCA5OTlceeWVfPXVV90RbkiPHvTaG+3fvx+g2dVvG35mtv3799O3b9+wbVarlbS0tFZjvO666xg4cCDZ2dl88cUX3HHHHWzevJkVK1Z0dchNtHeVYQiedyz/XiC68xo6dChPPfUUI0aMoKKigocffpizzjqLr776qtsGlUuNKAp33nlnk8a9Yx8t/dJjWVef1+zZs7nooos45ZRTuP7661m+fDkvvfQS27dv78SzEO01btw4pk2bxqmnnsr48eNZsWIFGRkZPP74490Wg9SIonD77bczY8aMVssMGjQoqn03rHB74MABsrKyQtsPHDjAqaeeGtU+IxXpeWVmZlJaWhq23e/3c+TIkWZX6G3J2LFjAdi2bRuDBw9ud7wd0d5VhiH4u2lPeTNEc17HstlsjBo1im3btnVFiM2SRBSFjIwMMjIyumTf+fn5ZGZm8vbbb4cST2VlJWvWrOEnP/lJlxyzQaTnNW7cOMrLy1m3bh2jR48G4J133sEwjFByicRnn30GEJZwu0vjVYYnT54MHF1l+JZbbmn2NePGjePtt9/mtttuC2178803Y2pV4mjO61iBQICNGzdyySWXdGGkxzC1qfw4UFxcrDZs2KAWLlyoEhMT1YYNG9SGDRtUVVVVqMzQoUPVihUrQs8ffPBBlZKSol555RX1xRdfqCuvvDImb9+PGjVKrVmzRn344YdqyJAhYbfvd+/erYYOHarWrFmjlFJq27Zt6t5771Wffvqp2rFjh3rllVfUoEGD1HnnnWfWKajnn39eORwOVVRUpDZt2qRmz56tUlJSQreuf/SjH6k777wzVP6jjz5SVqtVPfzww+rrr79Wd999d8zevm/PeS1cuFC9/vrravv27WrdunXq2muvVXFxceqrr77qtpglEXWx6dOnK6DJ49133w2VAVRhYWHouWEY6te//rXq16+fcjgc6oILLlCbN2/u/uBbcfjwYTV16lSVmJiokpOT1Q033BCWXHfs2BF2nrt27VLnnXeeSktLUw6HQxUUFKhf/vKXpvYjUkqpRx99VOXm5iq73a7GjBmjVq9eHfrZ+PHj1fTp08PK//Of/1QnnHCCstvt6uSTT1avvvpqN0ccmfac12233RYq269fP3XJJZeo9evXd2u8Mg2IEMJ0ctdMCGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOkkEQkhTCeJSPQoRUVFYVPu3nPPPWGDgWfMmBEaYyV6DklEolkzZsxocU7pOXPmoGla2Ej9rkgAeXl5LFmyJGzbD37wA7Zs2dLia5YuXUpRUVHo+YQJE8IGqYrYJIlItCgnJ4fnn38et9sd2lZXV8ezzz5Lbm6uKTE5nc4mk7I15nK5Il6kQMQOSUSiRaeddho5OTlhMyiuWLGC3NxcRo0a1aF9N1dTmTx5cqiWNWHCBIqLi/nFL34RmpQNml6aHatxzWzGjBm89957LF26NLSPHTt2UFBQwMMPPxz2us8++wxN07p1Dh5xlCQi0aobb7yRwsLC0POnnnqKG264ocuPu2LFCgYMGMC9997Lvn372LdvX7v3sXTpUsaNG8esWbNC+8jNzW1yTgCFhYWcd955FBQUdNYpiHaQRCRa9cMf/pAPP/yQ4uJiiouL+eijj/jhD3/Y5cdNS0tD13WSkpLIzMyMahZEl8uF3W4nPj4+tA9d15kxYwabN28OrfXl8/l49tlnufHGGzv7NESEZIZG0aqMjAwuvfRSioqKUEpx6aWX0qdPH7PD6pDs7GwuvfRSnnrqKcaMGcO///1vPB4PU6ZMMTu045bUiESbbrzxRoqKinj66ac7rdZgsVg4diosn8/XKfuOxMyZM0MN8YWFhfzgBz8gPj6+244vwkkiEm26+OKL8Xq9+Hw+Lrrook7ZZ0ZGRli7TyAQ4MsvvwwrY7fbCQQCHTpOS/u45JJLSEhIYNmyZbz22mtyWWYyuTQTbdJ1na+//jr0fUsqKipCE+I3SE9PJycnp0nZ73znO8ydO5dXX32VwYMHs3jxYsrLy8PK5OXl8f7773PttdficDiiuiTMy8tjzZo17Ny5k8TERNLS0rBYLKG2ovnz5zNkyJCYmgD/eCQ1IhGR5ORkkpOTWy2zatUqRo0aFfZYuHBhs2VvvPFGpk+fzrRp0xg/fjyDBg3i/PPPDytz7733snPnTgYPHhz1qinz5s1D13VOOukkMjIy2LVrV+hnN910E16vt1vuAorWyZzV4rj1wQcfcMEFF1BSUtJkZVTRvSQRieOOx+Ph4MGDTJ8+nczMTJ555hmzQzruyaWZOO4899xzDBw4kPLych566CGzwxFIjUgIEQOkRiSEMJ0kIiGE6SQRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCm+/9mtOU6DexJtAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:41.147054Z", + "iopub.status.busy": "2024-03-22T19:05:41.146225Z", + "iopub.status.idle": "2024-03-22T19:05:41.582765Z", + "shell.execute_reply": "2024-03-22T19:05:41.581740Z" + }, + "papermill": { + "duration": 0.460683, + "end_time": "2024-03-22T19:05:41.584886", + "exception": false, + "start_time": "2024-03-22T19:05:41.124203", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9qUlEQVR4nO3deXxU5b348c+ZM1smyUxWskBCwiKKrIpQRAWXVsVa0Vat2gri0gXbKpcWub8riq3Ful256qXWW0BfdWlV1LaK1LqgoiCCIC4EiSxhCZB9m/Wc8/tjkiFD9slyJuH7fr3mRWbmmTPfk8l8eZ7nPItiGIaBEEKYyGJ2AEIIIYlICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOmsZgfQHbquc/DgQZKTk1EUxexwhBDHMQyD2tpacnNzsVjarvf060R08OBB8vLyzA5DCNGBkpIShgwZ0ubz/ToRJScnA+GTdLvdJkcjhDheTU0NeXl5ke9qW/p1ImpqjrndbklEQsSxjrpOpLNaCGE6SURCCNNJIhJCmK5f9xF1hmEYhEIhNE0zOxTRD9lsNlRVNTuMAW9AJ6JAIMChQ4doaGgwOxTRTymKwpAhQ0hKSjI7lAFtwCYiXdfZvXs3qqqSm5uL3W6XQY+iSwzD4OjRo+zfv5+RI0dKzagXDdhEFAgE0HWdvLw8XC6X2eGIfiozM5M9e/YQDAYlEfWiAd9Z3d6wciE60pVa9P7KBopKa/EFpT+yq+RbKkQP2b6/mte3H2LLvkqzQ+l3JBEJ0UMqG4IAWC0WjtT6TI6mf5FEdILYs2cPiqKwdevWPjnO+vXrGTt2LDabjVmzZnXrPfuLGl84Ea3fVcZ7O8tMjqZ/kUR0gnr33XdRFIWqqqpeOf78+fOZMGECu3fvZtWqVb3yHvEkENLxBo71DdV4gyZG0/9IIuoHAoGA2SF0WXFxMeeddx5DhgwhJSUlpmP05Xk3DXyNVa0vOvE0BGI/1onohExEgZDe5i2k6Z0uG+xE2VjMmDGDW2+9ldtuu42MjAwuvPBCPv/8cy6++GKSkpLIysrixz/+MWVlx6r/b7zxBmeddRYpKSmkp6fz3e9+l+Li4laPv2fPHs4991wAUlNTURSFOXPmdOk4O3bs4Mwzz8TpdDJmzBjWrVsXObaiKJSXlzN37lwURYnUiNatW8fkyZNxOBzk5ORwxx13RH35Wzvvpprb2rVrmThxIgkJCZx33nkcOXKENWvWcMopp+B2u7n22mujBq7qus7SpUspLCwkISGB8ePH8+KLL0aebzrumjVrOP3003E4HHzwwQcxfV4ANb7weXgSbAAENSPmz/9ENGDHEbXn8Xd2tflcYUYisyYOjtz/03vFBLXWd+UekprAlZOOLcy2Yv3uqOo5wO3fPimmGJ966il+9rOfsX79eqqqqjjvvPO46aab+O///m+8Xi8LFy7kqquu4u233wagvr6e+fPnM27cOOrq6li8eDGXX345W7dubTGEIS8vj5deeonvf//7FBUV4Xa7SUhI6NJxfv3rX/PII48wevRoHn74YS699FJ2795NXl4ehw4dYtSoUdxzzz1cffXVeDweDhw4wMyZM5kzZw5PP/00O3bs4Oabb8bpdHL33Xe3et4Ahw4dAuDuu+/msccew+VycdVVV3HVVVfhcDh49tlnqaur4/LLL+fRRx9l4cKFACxdupS//OUv/PGPf2TkyJG89957/OhHPyIzM5Pp06dH3u+OO+7gwQcfZNiwYaSmpsb0WcGxGlCKy0ZDIERQM/AGNOzWE/L/+i47IRNRfzBy5Ejuv/9+AH73u98xceJEfv/730eeX7FiBXl5eezcuZOTTjqJ73//+1GvX7FiBZmZmXz55ZeMGTMm6jlVVUlLSwNg0KBBUU2nzh7n1ltvjZRdvnw5b7zxBn/+85/5zW9+Q3Z2Noqi4PF4yM7OBuB///d/ycvL47HHHkNRFE4++WQOHjzIwoULWbx4cSTJNT9vOJaIfve73zFt2jQAbrzxRhYtWkRxcTHDhg0D4Ac/+AHvvPMOCxcuxO/38/vf/55///vfTJ06FYBhw4bxwQcf8MQTT0QlonvuuYdvf/vbHX8gHchPc3Hp+BwcVpV/fXmYoDdIfSCEx2Xr9rFPBKYnogMHDrBw4ULWrFlDQ0MDI0aMYOXKlUyaNKnX3nPeuSPafM5y3Pi1W84Z3mbZ48e6zZ1W2J2wopx++umRn7dt28Y777zT6nyn4uJiTjrpJL7++msWL17Mxo0bKSsrQ9fDzYJ9+/a1SETt6exxmr7gAFarlUmTJvHVV1+1edyvvvqKqVOnRg0QnDZtGnV1dezfv5/8/PwW593cuHHjIj9nZWXhcrkiSajpsY8//hiAXbt20dDQ0CLBBAIBJk6cGPVYT/2dJTttJDvDSSfBplLjDeKXplmnmZqIKisrmTZtGueeey5r1qwhMzOTr7/+ultV5M7oSnW5t8p2JDExMfJzXV0dl156KX/4wx9alMvJyQHg0ksvZejQoTz55JPk5uai6zpjxozpcodvTx0nVs3Puzmb7VjNQlGUqPtNjzUlzbq6OgBee+01Bg8eHFXO4XB06v26Y8xgN8MzE0lJkNpQZ5maiP7whz+Ql5fHypUrI48VFvZcrWKgOO2003jppZcoKCjAam35kZWXl1NUVMSTTz7J2WefDdBhx6vdbgeIWh6lK8fZsGED55xzDgChUIjNmzdz6623tvl+p5xyCi+99BKGYURqRevXryc5ObndRdVjMXr0aBwOB/v27YtqhvWmPWX1+EM6Ockq49S9kFsATnufvPdAYGpP2t///ncmTZrElVdeyaBBg5g4cSJPPvmkmSHFpXnz5lFRUcE111zDpk2bKC4uZu3atdxwww1omkZqairp6en86U9/YteuXbz99tvMnz+/3WMOHToURVH45z//ydGjR6mrq+vScR5//HFefvllduzYwbx586isrGTu3Lltvt/Pf/5zSkpK+MUvfsGOHTt49dVXueuuu5g/f36PzwdMTk5mwYIF3H777Tz11FMUFxezZcsWHn30UZ566qkefa8mn+yt5PXth6j+4k3Y8Tp89jcwWr/IIVoyNRF98803LF++nJEjR7J27Vp+9rOf8ctf/rLNPxa/309NTU3U7USQm5vL+vXr0TSN73znO4wdO5bbbruNlJQULBYLFouF559/ns2bNzNmzBhuv/12HnjggXaPOXjwYJYsWcIdd9xBVlYWt956a5eOc99993Hfffcxfvx4PvjgA/7+97+TkZHR7vu9/vrrfPzxx4wfP56f/vSn3HjjjfzXf/1Xt343bfntb3/LnXfeydKlSznllFO46KKLeO2113qtxu1tnOiaWPMNQU2nvrKUmoqjvfJeA5FiGOalbbvdzqRJk/jwww8jj/3yl79k06ZNfPTRRy3K33333SxZsqTF49XV1S128fD5fOzevZvCwkKcTmfPBy9OCJ39O/q/97/BW1/LDfydqoYAJZVelFMv51vfmtaH0cafmpoaPB5Pq9/R5kytEeXk5DB69Oiox0455RT27dvXavlFixZRXV0duZWUlPRFmEJ0yBvQcIZqsKoKatOl1wapEXWWqZ3V06ZNo6ioKOqxnTt3MnTo0FbLOxyOFlc9hDBbUNMJ6QaOUC3WBAVrY5+X4Tsxug56gqk1ottvv50NGzbw+9//nl27dvHss8/ypz/9iXnz5pkZlhBd0tQ/lKDXo1qO1YgUf62ZYfUrpiaiM844g5dffpnnnnuOMWPG8Nvf/pZHHnmE6667zsywhOiSpjlliUYdCgp4Gscu+aVG1Fmmj6z+7ne/y3e/+12zwxAiZkkOK5eOzyGpyAIhsHgGAzuwBGX3mM6SGXlCdJPTpjJiUDLZCeEmmiUpM/xEyA+6TPPoDNNrREIMGEEvAHZ3JrkpTlRFgZAP7LKLTEckEQnRTRX1AY7W+MitrSbZpuBM9DB0UCqEAuHkJImoQ9I0E6Kb9pTXs/azfZRWhifbYnOBNby+EyGveYH1I5KI4tCMGTO47bbbzA4jIt7iiTeBkI5V94WbYqoVrHYasNMQCBHyS4d1Z0giGqD64zrX/VUgpGPTvOHxQ7ZwM+yjvfVs219NXZ1cwu8MSURxZs6cOaxbt45ly5ahKAqKolBcXMyNN94YWX951KhRLFu2rMXrZs2axb333ktubi6jRo0C4MMPP2TChAk4nU4mTZrEK6+80mI7oPbWw24tnj179vTVr6NfCGotE5FhCzfNNL80zTrjxOqsNgzQTNjmRbW1XM6xDcuWLWPnzp2MGTOGe+65BwgvcD9kyBBeeOEF0tPT+fDDD7nlllvIycnhqquuirz2rbfewu128+abbwLhCYeXXnopM2fO5Nlnn2Xv3r0tmlgdrYfdWjyZmZk98EsZOMJNM39UIlKaElFAmmadcWIlIi0I7z/U9+979n+AtXOLZHk8Hux2Oy6XK7LeMxC16kBhYSEfffQRf/vb36ISUWJiIv/3f/8XWfTsj3/8I4qi8OSTT+J0Ohk9ejQHDhzg5ptvjrzmscce63A97NbiEccENB3VCKCqClgb50JawzP1tYDs+NoZJ1Yi6scef/xxVqxYwb59+/B6vQQCASZMmBBVZuzYsZEkBFBUVMS4ceOilq+YPHly1Gs6sx62aJ8/pKPqgXCNqDERKY3/8eghv5mh9RsnViJSbeHaiRnv2w3PP/88CxYs4KGHHmLq1KkkJyfzwAMPsHHjxqhysay/3Jn1sEX7pg5LR9MSSayxRhKRagsnfz0oiagzTqxEpCidbiKZyW63R60lvX79es4880x+/vOfRx5ra/PE5kaNGsVf/vIX/H5/ZPmUTZs2RZXpaD3s1uIR0fLSXOBRoUEFNfx7ttjC/+pBaZp1hlw1i0MFBQVs3LiRPXv2UFZWxsiRI/nkk09Yu3YtO3fu5M4772yRUFpz7bXXous6t9xyC1999RVr167lwQcfBIgsYN/RetitxaPL/KmWmppgjX1Duekp5HqcuG3yu+oMSURxaMGCBaiqyujRo8nMzOTCCy/kiiuu4Oqrr2bKlCmUl5dH1Y7a4na7+cc//sHWrVuZMGEC/+///T8WL14MEOk36mg97NbiaWsFzRPVzsO1HK2qQTeMSNNseE4aQ9MTSXV07mrpic7UNau7q731cGXN6tY988wz3HDDDVRXV0e2mRZt6+jvSNMN/uetrxlT+jLn5oawjr8KMkZC7WH4ZAU4kuDMX5gQeXzo7JrVJ1Yf0Qno6aefZtiwYQwePJht27ZFxghJEuoZTYuiWfUAqqJGakQBbOghDQUvsrhxxyQRDXClpaUsXryY0tJScnJyuPLKK7n33nvNDmvACGjhRGQngKK4In1EXxz14dtXRUaSnZG6Dj28d9tAI4logPvNb37Db37zG7PDGLCaakQ2QuEH1PBVWZvNgY9w0w0tABbpHmiPpGkhuiGg6ShGCJvSOLyhsUakWm0YigXdADQZS9QRSURCdEN4nlng2F5mTTUiVUFT7OEraSFZCaEjAz4R9eOLgiIOdPT3E9Qap3c0DZZt7AtSLRY0izWciHQTJlr3MwM2Edls4WkVDQ0y+1nErmldJ1VVW30+K9nJuSM85KQkRJplAFaLgq5Yw2vna1Ij6oipndWt7WU/atQoduzY0e1jq6pKSkoKR44cAcDlckVGEwvRGbquc/ToUVwuV5vTXzwuG54MOxywR5plAFZVQVNs6LoBWqivQu63TL9qduqpp/Lvf/87cr+tDzwWTctWNCUjIbrKYrGQn5/f/n9ix03vAEh0WBmc7ibZVyc1ok4wPRFZrdZeW+dGURRycnIYNGgQwaC000XX2e32yFSX1hyp9REsqyQ9pOFslojcThujh2RAeZX0EXWC6Yno66+/Jjc3F6fTydSpU1m6dCn5+fmtlvX7/fj9xy6F1tR0bj1gVVXbbOML0R2flVRT9tUBzlL8DBl83MoOauPXy4xVQfsZUzurp0yZwqpVq3jjjTdYvnw5u3fv5uyzz6a2trbV8kuXLsXj8URueXl5fRyxENECWvNF0Y7ViAzDwK+r+EMahjTNOmRqIrr44ou58sorGTduHBdeeCGvv/46VVVV/O1vf2u1/KJFi6iuro7cSkpK+jhiIaIFQjpWo3G96mad1SHd4PUvy9myr4pgUBJRR0xvmjWXkpLCSSedxK5du1p93uFwRBb4EiIeBEI6Tj3YokZktSjolvDXS5cBjR2Kq3FEdXV1FBcXyxKlot/way3Xq4bGhecs4bFsmtSIOmRqIlqwYAHr1q1jz549fPjhh1x++eWoqso111xjZlhCdFowpGM1WiYiAMUaTkRSI+qYqU2z/fv3c80111BeXk5mZiZnnXUWGzZskH2zRL8RiJriET3DXlGbdvKQRNQRUxPR888/b+bbC9FtZ4/MwN1gw27VojqrARRVakSdFVed1UL0N6fmeuAbFYKWFjUiS6RpJuOIOiKJSIjuMIxmUzyi+4iGZHjwVDpwWGQrpo5IIhIiRoGQzsHyajJ8AZIc1haJ6LTCQVCVBFZZiqYjcXX5Xoj+pMob4J9b9lBUWhPevPO4PqKmy/cyxaNjUiMSIkaB4y/dHzdDX7PYMHQdJeRHZjq2TxKREDEKhJoGM1oiW0039/bOCpL2VDI0M5lcE+LrT6RpJkSMgprR6qjqJoo13FQzNA106bBujyQiIWIUXjjfj0Wh1URkbUxEumFIP1EHJBEJEaOApqEaQayWlqOqASyqFQMlvFysLI7WLklEQsTI33wrodZqRKoF3WJt3NtMElF7pLNaiBgVZiSSkZtAWqWj1c5qq2rBp1jRDV0SUQckEQkRoxxPAjnpdvDZWq0RqZbGnTwMnyyg3wFJREJ0R8gX/reVRJSeaMfhSSKJEOiypVB7JBEJEaMjNT5sNXUk6zrWVhJRQUYi5KVDjV8SUQeks1qIGL1bdJSPdx2k2hts9aoZAKpM8+gMSURCxMivNbtqdvw8s0a6YkUz5PJ9RyQRCRGjQEhHNVpuJdSkpKKB178s4/MD1bLtdAckEQkRo0DTOKJWlomF8FUzXbHKgMZOkEQkRAwMwyAY0lAjWwm1Mo6oKRHJgMYOSSISIgaaboDmB4w2m2ZW1YKmWMNzzaRG1C5JRELEINDYUQ2gWm3H9rlvJtI0MwzpI+qAjCMSIgaqRWFaQRLp3gSUNi7d21QlMtfM0AIorZYSEEc1ovvuuw9FUbjtttvMDkWIDjmsKuOznQxJdbU5hqipRgSgyU4e7YqLGtGmTZt44oknGDdunNmhCNF5beze0cRmsZCTlkx6wC6d1R0wvUZUV1fHddddx5NPPklqaqrZ4QjRKQ2BEEerqvEFtTZrRBaLwoxTcjkpKxkrskJje0xPRPPmzeOSSy7hggsu6LCs3++npqYm6iaEGfZVNPDv7fv4pqy+zRoRcGwnD7lq1i7Tt5zesmULmzZt6lT5pUuXsmTJkl6OSoiOBaIWRWtjnhmAasUgvFSsdFa3zbQaUUlJCb/61a945plncDrb+SCbWbRoEdXV1ZFbSUlJL0cpROvCO3j4G0dVt10jemFLKRu+qaC23tuH0fU/ptWINm/ezJEjRzjttNMij2maxnvvvcdjjz2G3+9HVaN3g3I4HDgc7VSDhegjkWVire3XiHRL+Cumy8Jo7YopEX3zzTcMGzasW298/vnns3379qjHbrjhBk4++WQWLlzYIgkJEU/8IQ1VDzQunN/2f46WyJZC0kfUnpgS0YgRI5g+fTo33ngjP/jBDzrdtGouOTmZMWPGRD2WmJhIenp6i8eFiDf+YHgrobZ28GhiaVyPSJdxRO2KqY9oy5YtjBs3jvnz55Odnc1PfvITPv74456OTYi45Qtp7e7g0USxhhORoYVA1/sqvH4npkQ0YcIEli1bxsGDB1mxYgWHDh3irLPOYsyYMTz88MMcPXo0pmDeffddHnnkkZheK0RfGp3jYVialUSHtd0akdp8k0VZLrZN3bpqZrVaueKKK3jhhRf4wx/+wK5du1iwYAF5eXlcf/31HDp0qKfiFCKujMpOptCjkmhvPxFFmmYGMpaoHd1KRJ988gk///nPycnJ4eGHH2bBggUUFxfz5ptvcvDgQS677LKeilOI+GIYx7YIaqdpNsjtxJ2YgN1qkWke7Yips/rhhx9m5cqVFBUVMXPmTJ5++mlmzpyJxRLOa4WFhaxatYqCgoKejFWIuHGoooqUQAin1dLm7HuAKcPS4WA6BL3SNGtHTIlo+fLlzJ07lzlz5pCTk9NqmUGDBvHnP/+5W8EJEY9Cms7LH3/DxINVTBqWia2VtYiiqLZwIpIaUZtiSkRvvvkm+fn5kRpQE8MwKCkpIT8/H7vdzuzZs3skSCHiSXgwox8FsNoTOn6BzDfrUEx9RMOHD6esrKzF4xUVFRQWFnY7KCHimT+kY9V8qBYFxdZ+Itq0p4J3d1Wyu7xeakTtiCkRGYbR6uN1dXUxDW4Uoj/xhzSsui88mNHmaresYUDAUBt38pA+orZ0qWk2f/58ABRFYfHixbhcxz4ETdPYuHEjEyZM6NEAhYg3vsZR1aqqQAc1IqvafN1qqRG1pUuJ6NNPPwXCNaLt27djtx/b3dJutzN+/HgWLFjQsxEKEWf8IQ2b7sNqsXRYI7JalMadPJA+onZ0KRG98847QHhy6rJly3C73b0SlBDxzB881kfUYY3IYpGdPDohpqtmK1eu7Ok4hOg3stxOnJlW0urtHdeIVAVdUdE12dusPZ1ORFdccQWrVq3C7XZzxRVXtFt29erV3Q5MiHiV7XGSnWYFxdFhjUi1NG4pFEL6iNrR6UTk8XhQFCXysxAntGBD+N8OElGi3UpqciKJtarUiNrR6UTUvDkmTTNxIiuv85NQX4fDMFA7aJple5xkn5ILu4ulj6gdMY0j8nq9NDQ0RO7v3buXRx55hH/96189FpgQ8Wrt56V8+s1BarzBDmtEgIys7oSYEtFll13G008/DUBVVRWTJ0/moYce4rLLLmP58uU9GqAQ8cYf8GMxNKydGEcEQNNcNOkjalPMKzSeffbZALz44otkZ2ezd+9enn76af7nf/6nRwMUIt6E/PUAWK02UO3tlq3xBXlp62E2762UkdXtiCkRNTQ0kJycDMC//vUvrrjiCiwWC9/61rfYu3dvjwYoRDwJajqGP9wtYXW4QGl/tzKLolAfUsKvk5082hRTIhoxYgSvvPIKJSUlrF27lu985zsAHDlyRAY5igHNG9Sw6Q2E18xP6rB808hqA1lAvz0xJaLFixezYMECCgoKmDJlClOnTgXCtaOJEyf2aIBCxBNfQMOuebFaLCiO5A7LWy3huWYgiag9MY2s/sEPfsBZZ53FoUOHGD9+fOTx888/n8svv7zHghMi3niDGjatPtxRbU/ssLxqUTAimyxKImpLzDu9Zmdnk52dHfXY5MmTux2QEPHM7bQxNtNKSqUD7B03zRRFQYnsbSZ9RG2JqWlWX1/PnXfeyZlnnsmIESMYNmxY1K2zli9fzrhx43C73bjdbqZOncqaNWtiCUmIPpGaaOeUNIUcTwI4Ok5EAErjlTWpEbUtphrRTTfdxLp16/jxj39MTk5OZOpHVw0ZMoT77ruPkSNHYhgGTz31FJdddhmffvopp556akzHFKLXBcKX7ztTIwLI8CSS7LBi0UPhldJi/L4MZDElojVr1vDaa68xbdq0br35pZdeGnX/3nvvZfny5WzYsEESkYhLFfUBHHXVOA0DtZOJ6HsTh0J94/xMQwdF7cUI+6eYElFqaippaWk9GoimabzwwgvU19dHrsIdz+/34/f7I/dramp6NAYhOrKu6DBZxfsZkZHAoE50VgPhXTyaaEGwSCI6Xkx9RL/97W9ZvHhx1HyzWG3fvp2kpCQcDgc//elPefnllxk9enSrZZcuXYrH44nc8vLyuv3+QnSF11uPYujYVUunm2YolvANZL5ZG2KqET300EMUFxeTlZVFQUEBNpst6vktW7Z0+lijRo1i69atVFdX8+KLLzJ79mzWrVvXajJatGhRZN1sCNeIJBmJvqTVVwFgTUg+NoesA2s+LyWjpIbhaXbSpMO6VTElolmzZvVYAHa7nREjRgBw+umns2nTJpYtW8YTTzzRoqzD4cDhaHt7XyF6k6Yb6N5qAOyJKZ1+nS+k4dUsaJrs5NGWmBLRXXfd1dNxROi6HtUPJES8qA+EsGt1WBSwdSERqZF1q3WZgd+GmAc0VlVV8eKLL1JcXMyvf/1r0tLS2LJlC1lZWQwePLhTx1i0aBEXX3wx+fn51NbW8uyzz/Luu++ydu3aWMMSotfU+0M4QrXYVAuKs/OrlB7bycMvfURtiCkRffbZZ1xwwQV4PB727NnDzTffTFpaGqtXr2bfvn2RtYo6cuTIEa6//noOHTqEx+Nh3LhxrF27lm9/+9uxhCVErwonorpwR3UXE5HelIhklcZWxZSI5s+fz5w5c7j//vsjy4EAzJw5k2uvvbbTx/nzn/8cy9sLYYpUl52TUw2SAo6uJSJVIdi0pZDUiFoVUyLatGlTq53JgwcPprS0tNtBCRGP0hPtpCeHIOgER+eXu1EtFvyy22u7YkpEDoej1cGEO3fuJDMzs9tBCRGXgg0Q9IWnaLg6P6A3yWFFcTmxYZEaURtiGtD4ve99j3vuuYdgMPxLVRSFffv2sXDhQr7//e/3aIBCxIsjpfvxBjV0hzt6tHQHTh+ayjkn55KbkiB9RG2IKRE99NBD1NXVkZmZidfrZfr06YwYMYLk5GTuvffeno5RiLiw/rOdbC2potYSw75+spNHu2Jqmnk8Ht58803Wr1/Ptm3bqKur47TTTuOCCy7o6fiEiAshTYf6MgCcKVldP4Ds5NGuLiciXddZtWoVq1evZs+ePSiKQmFhIdnZ2RiGEfOSIELEsxpfCFeoEtWiYHd3rR+0+Ggdu74qZ4S3juFDpGnWmi41zQzD4Hvf+x433XQTBw4cYOzYsZx66qns3buXOXPmyDKxYsCq8QZJCFbisFpQXBldem1IM6j2gz8kI6vb0qUa0apVq3jvvfd46623OPfcc6Oee/vtt5k1axZPP/00119/fY8GKYTZautqsWleHA4bJHYtEakWBc1iRddlHFFbulQjeu655/jP//zPFkkI4LzzzuOOO+7gmWee6bHghIgX3qojAFhdHrB2beL1sZHVMo6oLV1KRJ999hkXXXRRm89ffPHFbNu2rdtBCRFv/NXhRKQmdX2cnFVtlohk9n2rutQ0q6ioICur7SsGWVlZVFZWdjsoIeLNyCQ/wdQEkjJyuvxaa2T2PVIjakOXEpGmaVitbb9EVVVCIcn4YuDJttZBqgsyY0hEatPse+kjakuXEpFhGMyZM6fNxclkHSExYDWExxDRxStmADbVQmKCE4dVlZHVbehSIpo9e3aHZeSKmRhovF4v/ooynDYVpyu9y6/3JNi4esow2OyRGlEbupSIVq5c2VtxCBG3jhw+wMHSWhyuJE6zu2I7SNPcNOkjalVMc82EOJE0XbpXEruxsoSl8f98qRG1ShKREB0I1DSOIUqOLREZhsHfPi1l2/4qgoFAeLdXEUUSkRAdCNUeBejyHLMmiqJwtF6nIaChyViiVkkiEqIDel03Zt03UhqHvRgylqhVkoiEaI8WwvCGB+m6upGIbKoVXVFlLFEbJBEJ0Q5/7VFCmo5mseN2x7AgWqOmaR6abshYolaYmoiWLl3KGWecQXJyMoMGDWLWrFkUFRWZGZIQUSzecoZnJpI7OA+HLeZtALGqlmbzzaRGdDxTE9G6deuYN28eGzZs4M033yQYDPKd73yH+vp6M8MSIsLmr2JQspNh+fndOo69aeKrLjPwWxN7iu8Bb7zxRtT9VatWMWjQIDZv3sw555xjUlRCNNNQHv43ofO7drTGZbdis9tB8UmNqBWmJqLjVVdXA5CW1r0PXYieUlV+GLwBHDY3Cd04zqXjc0HLgeoD0kfUirhJRLquc9tttzFt2jTGjBnTahm/3x81sba1vdWE6DGGwaFDB6iqrSd5hJPW/yq7QHbyaFPcXDWbN28en3/+Oc8//3ybZZYuXYrH44nc8vLy+jBCccIJ1BHw+wCFxJSuz7pvQeabtSkuEtGtt97KP//5T9555x2GDBnSZrlFixZRXV0duZWUlPRhlOJEYzSU4w/p+KzJuF3Obh1rW0kVH3xTxaFqr4ysboWpTTPDMPjFL37Byy+/zLvvvkthYWG75R0OR5trIQnR0wI1ZWi6gc/uwZ3Q+Z1dW1PvD1HuNRisyk4erTE1Ec2bN49nn32WV199leTkZEpLS4HwBo4JCd3pGhSi+7zV4TlmhisNm9q9xoPNGh5HpMk4olaZ2jRbvnw51dXVzJgxg5ycnMjtr3/9q5lhCQGAr3HWvdrF7YNaY7Uo6BYZR9QW05tmQsSrQG14sqvd3f1EZIsaWS19RMeLm8v3QsQVLUSm6sOZmYh/6NBuH64pEWlSI2qVJCIhWuOrIsFmIcGZDNndWJmxkS2ykwfSR9SKuLh8L0TcaagI/+tKB0Xp9uFsqgWrzYZFQUZWt0JqREK0wmgo40iND4srkXTdwGLpXjLKS3ORN3kYfLldakStkBqREK3wVh/lm7J63tuv99xBZWR1myQRCdEKX+MYIjUpvdu1oYimnTy0QM8cbwCRRCREK3ry0j2AP6Sx5qtyPj9YjR6SRHQ8SURCHC/QQMAbXpwvoYcSkUVR2FutUesLoYdka/bjSSIS4njeCnxBDb+ahDspqUcOabUoaBY7AFrQL3ubHUcSkRDHa5p1b/Pg6eZk1yaKomCxhWfwG5omHdbHkUQkxPEayvEFNbzWFFJcPZOIAFSrDVDCE181aZ41J+OIhDiO0VDBSVnJpOUM67EaEYDNqqJZbOFpHqEAyIo2EZKIhDiO0lCBJ8GGZ2gedHP5j+bsVgshxd44A19qRM1J00yI5nQNGnd2xZXeo4d2WFUUqx0dQK6cRZEakRDN+aqpqPMRMFScmpOUHjz0908bjKLkQfV+SUTHkRqREM01lHO41scXlSollb4ePbSiKGBt7BiSplkUSURCNNdQjjeg4bWlkJrYcx3VEWp4LBEyujqKNM2EaCZYcwR/SKchMYX0xJ69rLWjtIbKfbUU+L3kSI0oitSIhGimruIgAEZiJgl2tUePXeMNcbDWoCGgSR/RcSQRCdFE1/BWHQYgOT23xw9vt1qOjSOSGfhRJBEJ0aShnHqvH81iIy19UI8f3q5a0Cz2xgGNPdsR3t9JIhKiSd0RGgIaDbZ0sj09v6+e3WpBU+zHRlaLCFMT0Xvvvcell15Kbm4uiqLwyiuvmBmOONHVH+HUXDdjR40g29O9LaZb47A21ohkrlkLpiai+vp6xo8fz+OPP25mGEKE1RzEarGQM6QQp61nO6qhKRE1zTWTRNScqZfvL774Yi6++GIzQxAiTNcwag6iALiH9MpbNM01MwwkER2nX40j8vv9+P3HPsCamhoToxH9la4b7K/04g1q5KY4SXbaqDqynz17y0hPSWFIQio9tEp1FE+CjZvOHY3l4/ch5O2Fd+i/+lUiWrp0KUuWLDE7DNHPvb+rjC17wxNbFQWGprtIKN2KJ6hzmDTyLL3TY6EoCordFb6jhcI3tV99BXtNv7pqtmjRIqqrqyO3kpISs0MS/dAZBamkJ9kZ5HZgGLCnrAGlohirReGkU8b17ptbHaA0fu2CDb37Xv1Iv0rHDocDh0NWkxLd47Jb+fG3hqIoCpX1AYoOlJFTW82gTA8Jg0/u1fd+Z+dRMo74Ge5RcAW94HT36vv1F/2qRiRErAzD4EjNsUGESuM20qmJdr6VXMbQVCcJnkHgSuvVOA5Uejnqs+AP6dJP1IypNaK6ujp27doVub979262bt1KWloa+fn5JkYmBpqSCi8vbdlPQYaLWRMGRxIRhgEHt4R/zh7T63Ek2FRCFichvQ6CkoiamJqIPvnkE84999zI/fnz5wMwe/ZsVq1aZVJUYiDatKcCgBSX/VgSAjhaBDWHwruwZvdy/xDgtKkELU5CWo0komZMTUQzZszAkP2dRC87XONjX0UDFkXhtPzU8IOGAeXFUPR6+H7eZHD0zB5m7UmwW6i1OAjphiSiZvpVZ7UQsWiqDY3KTsbjtELZ17DnA6gtDRdIyYOCs/okFqdVpVJ1EtIM6SNqRhKRGNAq6wPsOlIHwKRcO2x9Fqr2hZ9UrZA7EQqng6Xnp3S0xmFTCVkchEK61IiakUQkBrTNeysxDDjZ7SOj6HnwVYcT0OBJ4eaYPbFP42nqrNYNJBE1I4lIDFiabnCgyovHW8KZxidg0yEhFcZeCYk9u1VQZ43KTubkaSdj+eILCNSbEkM8kkQkBixVgR8POUp11Xo8Nlu4L+jUK6BpmoUZMVkUcCaH7wTqTIsj3kgiEgNT0Atfv4nl8BekJtggeyyMurjP+oLa1XR1LlAPug69NLetP5FEJAYWXYfSbdR8+W+SLEEsFhWGnwtDzgjPcDWZYRis3VlLXmktJ2UmYgvWgyPZ7LBMJ4lIDBw1h2DnG/gqDvDV/ipCznTGn38VSVmFZkcWoSgKeyu8JPhtFGg6Nn+dJCIkEYmBQNdg9zoo+RgMg2+qgnyTMhVb/iTOHBR/U4USHVYCqotAyEei9BMBkohEfxf0wuerI2ODDjgKedt9Mpo1kR+dnB09nSNOJDmsBNREgloD+GVxP5BEJPqzkB8++2u4SabaqC64iFeKnQRVnSmFqaQnxeeSMYkOK/Wqi0BIB7/UiECWARH9lRaE7S+Gk5Atgfox1/LSvkQCIZ3BKQl8q9CccUKdkehQCaiJBDQ9PMBSSCIS/ZCuwRevhJtjVjuMu5oPDlmo9gbxJNi4ZFwOFkv8NcmaJDtseG2ecI3IW2l2OHFBmmaif9F1+OrvUL4rPFVj7JXgzuGckRq1vhAXnDKIREd8/1l7Emz4rJ7wtkLeCrPDiQtSIxL9h2HAzjfgyA6wqJQXfBdSwlfFEuwqPzh9CCkuu8lBdmxwagJzv30ap+Z6IOiDgKxdLYlI9A+GAbv+DYe2oQOfJp3N00Uq2/f3vz4W1aJgszuOrVcttSJJRKIfMAz45h3Y/wkBTeddZRLvVoQ7oysb+vEe8gmN62M3SCKK78a0EIYBu9dh7NtAZX2Ad4zT2eccik1VOP+ULE7J6Z+7YHx5sIayAzAy6CWn7rDZ4ZhOEpGIX7oGRWvwlWzlm7J6tjnPoDR5FJnJDi4ak01GnI4T6oyGQIg9QQ/ZvhA5NQfNDsd0kohEXNF1g2pvEK2+nIyStVBzCEOx8GnCtyh3j2by0FSmFKZhVft3r0KW28nH9kzqqkNQd/iE3/X1xD1zETfK6vzsLK1lX3kdvvISUmuLGKHvJiM7CawOEsZ+j5Pr0xiRmYzHZTM73B4xyO0gaPNQazjx+QM4q/dB2jCzwzKNJCLR9wwDag5SV76fHcV7OHq0FGewhvxQDQoGFgUsCbbwF3PUxeB0c3qG2UH3LIdVJTc1gaqKfCob9pFTtksSkdkef/xxHnjgAUpLSxk/fjyPPvookydPNjus+OWrDo8q9lZSH9BITMmC1KFgT6QhEMJhVcMrAcYTw4DaQ3Dky/A4IH8tew9WE/SFSAFSXTbSUl0kJ7pwZI/CkjsxfE4D2LDMJLYdLORozddkH/4cZdiM8EjxE5Dpieivf/0r8+fP549//CNTpkzhkUce4cILL6SoqIhBgwaZHV780ILhzQBLPyNQtpvyOj9ldQHq/CHOKEjFqqqQWsAObxaf1GeQn5XJiEFJDE13YTu+PyXoA181lRVHqa+rQVcsGKoDw5kCzlRUmx2baiHb44y8pLTah9NmIdlp63yS0/Vw8ikrgqNFhOrDl6mtFgtY7WQXnkpFtZ3RIwpIz8gKryftcMfFAmZ9YXSOmw+/zqO8MpHKmlrSSjZA4Tlmh2UKxTB5h8MpU6Zwxhln8NhjjwGg6zp5eXn84he/4I477mj3tTU1NXg8Hqqrq3G7++dl3DZpofASEbWHCJUV4zv4JbX1DVR5g1Q3BKlxZNFgS8OCwYzBGu5g+Eu+83AtZfVB6u0ZeG2pYLWT7rKSrPhwW3wUJAbDiYjwDhcBTT/ujRX81kRwZTBj4imQmAEJaby47QhH6zWsik6aEzIcBmlOg1S7TorDIMkSCs+GD/nC/wYbMOrLCAYDNPhDVDYEOdKgkTJkNKPGTQk3Q07gztkm7399lFDpl5zpex+HzQoFZ4e3ODJxXe2e1NnvqKl/CYFAgM2bN7No0aLIYxaLhQsuuICPPvqoRXm/34/f74/cr6np5FouFbuh+K1w8wAorfGxt/y4HRSa5eOTspJIbZwqcKTGy+6yepqeVRp/Chc3OGlQMmmJ4bJldb7IHlpw7HhK47GHZSaRmewAw6CyIUDR4dqo923+87AUC4OSwzWS6no/Ow/X4bcmcyTxFMpyTyItPZNR2cmclJUcnlvVUAFHdzAiaQfZZfspr6+lsr4cf70O1VAPhKwWCpp2OrW7UFMS0AwHVnSsuh9roApLyI8LL/bgASipjcQz9nAtVQ2B8DY4jeoab0eslmM7qAJfHKzGG9TQdIOgYqPKmU+Fq4BKdz5ZycmclJEXl+sEmWHa8AwsI86Br/1wYDNHt79JybpX0VUnuuoARcFQLI21RIVRWcmkNHbYH6nxs/v4v+NmRg5KJi0xXLasLkDx0baXHBmWmUhm43CIivogXx+pbbcMAGOvOjY6vJtMTURlZWVomkZWVlbU41lZWezYsaNF+aVLl7JkyZKuv1HID3VHI3eNOi96Xdvze5SGANDYVvf60BuOfdjHVx8NnwZq44fj82P4Wn7Ykdf4AXsw/FgggOFv+4/IMBLD+7EnZqCkZfO11U3yoHwKUhP5TlZSyzlVrjQYeiaWoWfi9lbhrj1EQUM5VXVeqnwa9boDw5GEMXwoijMFrHYmtHxTCDZAQznUl4X/bSgHbyWj8hIx9BA+TaE+ZKFOU6kJWqgOWlCdLhiSG+7fsCZQ5C2jJqjitaXgt7lJcTnITUng7KxkCtJdkoSaiawSMPLb4M6hbuu7+EN1EGoAWvkbTfKC3vjZ1/vQa9vZksjVAEbj32adH722nbWPEpKAxrL1AfTalokoqgyAcXxtOnb9qm68aNEi5s+fH7lfU1NDXl5exy9MyYPxPwz/rCgk+UPkN4RAUVrtjnAmOcBqARSSAhqFvmAr/Rbh17qcNrCHf42JQY3hvlDkuUi5xp+SnFawNZbVNEb6tGNHU8Jlwz8ruJLc4Q9eUUg1DK4f34Uvb0IKJKSgAKmNt05RlPCGg/bEyGTS6DOGhMZb84tYhmFE/X7O8vjC86msFhJsass+KtGSokD2WHLOH42rthbdVwOhAIahY+hG5EvvTLSDLfz7dAUa/47bkJBoA5vaWFYjvyHYZlmXywb2cNmEoEZ+fcuyzcsAPbo5pamJKCMjA1VVOXw4eoj74cOHyc7OblHe4XDgcMQwmtaeCGnHFlBPbrx1hqvx1hlNX9LOcBD1f0u74r0GcXx8g9zONkqKjjhtKs60FCClw7JJjbfO6Orf8eBOlu0ppv5XZbfbOf3003nrrbcij+m6zltvvcXUqVNNjEwI0ZdMb5rNnz+f2bNnM2nSJCZPnswjjzxCfX09N9xwg9mhCSH6iOmJ6Oqrr+bo0aMsXryY0tJSJkyYwBtvvNGiA1sIMXCZPo6oOwb0OCIhBoDOfkflcoYQwnSSiIQQppNEJIQwnemd1d3R1L3V6akeQog+1fTd7Kgrul8notrGYeidGl0thDBNbW0tHo+nzef79VUzXdc5ePAgycnJcT/6uEnTtJSSkpIBdaVvIJ7XQDwn6NvzMgyD2tpacnNzsVja7gnq1zUii8XCkCFDzA4jJm63e0D9cTcZiOc1EM8J+u682qsJNZHOaiGE6SQRCSFMJ4mojzkcDu66667YVhGIYwPxvAbiOUF8nle/7qwWQgwMUiMSQphOEpEQwnSSiIQQppNEJIQwnSSiXnbvvfdy5pln4nK5SElJ6dRrDMNg8eLF5OTkkJCQwAUXXMDXX3/du4F2UUVFBddddx1ut5uUlBRuvPFG6ura2SUCmDFjBoqiRN1++tOf9lHErXv88ccpKCjA6XQyZcoUPv7443bLv/DCC5x88sk4nU7Gjh3L66+/3keRdk1XzmvVqlUtPhens4/XHTdEr1q8eLHx8MMPG/Pnzzc8Hk+nXnPfffcZHo/HeOWVV4xt27YZ3/ve94zCwkLD6/X2brBdcNFFFxnjx483NmzYYLz//vvGiBEjjGuuuabd10yfPt24+eabjUOHDkVu1dXVfRRxS88//7xht9uNFStWGF988YVx8803GykpKcbhw4dbLb9+/XpDVVXj/vvvN7788kvjv/7rvwybzWZs3769jyNvX1fPa+XKlYbb7Y76XEpLS/s0ZklEfWTlypWdSkS6rhvZ2dnGAw88EHmsqqrKcDgcxnPPPdeLEXbel19+aQDGpk2bIo+tWbPGUBTFOHDgQJuvmz59uvGrX/2qDyLsnMmTJxvz5s2L3Nc0zcjNzTWWLl3aavmrrrrKuOSSS6IemzJlivGTn/ykV+Psqq6eV2f/NnuTNM3izO7duyktLeWCCy6IPObxeJgyZUqru9+a4aOPPiIlJYVJkyZFHrvggguwWCxs3Lix3dc+88wzZGRkMGbMGBYtWkRDQ9sbXfampl2Gm/+e29tlGMLn3bw8wIUXXhg3nwvEdl4AdXV1DB06lLy8PC677DK++OKLvgg3ol9Peh2ISktLAVrd/bbpObOVlpYyaNCgqMesVitpaWntxnjttdcydOhQcnNz+eyzz1i4cCFFRUWsXr26t0Nuoau7DEP4vOP5c4HYzmvUqFGsWLGCcePGUV1dzYMPPsiZZ57JF1980WeTyqVGFIM77rijRefe8be2PvR41tvndcstt3DhhRcyduxYrrvuOp5++mlefvlliouLe/AsRFdNnTqV66+/ngkTJjB9+nRWr15NZmYmTzzxRJ/FIDWiGPzHf/wHc+bMabfMsGHDYjp20w63hw8fJicnJ/L44cOHmTBhQkzH7KzOnld2djZHjhyJejwUClFRUdHqDr1tmTJlCgC7du1i+PDhXY63O7q6yzCEP5uulDdDLOd1PJvNxsSJE9m1a1dvhNgqSUQxyMzMJDMzs1eOXVhYSHZ2Nm+99VYk8dTU1LBx40Z+9rOf9cp7NunseU2dOpWqqio2b97M6aefDsDbb7+NruuR5NIZW7duBYhKuH2l+S7Ds2bNAo7tMnzrrbe2+pqpU6fy1ltvcdttt0Uee/PNN+NqV+JYzut4mqaxfft2Zs6c2YuRHsfUrvITwN69e41PP/3UWLJkiZGUlGR8+umnxqeffmrU1tZGyowaNcpYvXp15P59991npKSkGK+++qrx2WefGZdddllcXr6fOHGisXHjRuODDz4wRo4cGXX5fv/+/caoUaOMjRs3GoZhGLt27TLuuece45NPPjF2795tvPrqq8awYcOMc845x6xTMJ5//nnD4XAYq1atMr788kvjlltuMVJSUiKXrn/84x8bd9xxR6T8+vXrDavVajz44IPGV199Zdx1111xe/m+K+e1ZMkSY+3atUZxcbGxefNm44c//KHhdDqNL774os9ilkTUy2bPnm0ALW7vvPNOpAxgrFy5MnJf13XjzjvvNLKysgyHw2Gcf/75RlFRUd8H347y8nLjmmuuMZKSkgy3223ccMMNUcl19+7dUee5b98+45xzzjHS0tIMh8NhjBgxwvj1r39t6jgiwzCMRx991MjPzzfsdrsxefJkY8OGDZHnpk+fbsyePTuq/N/+9jfjpJNOMux2u3Hqqacar732Wh9H3DldOa/bbrstUjYrK8uYOXOmsWXLlj6NV5YBEUKYTq6aCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCLRr6xatSpqpcu77747ag7enDlzIlMbRP8hiUi0as6cOW0u5Tpv3jwURYmaINsbCaCgoIBHHnkk6rGrr76anTt3tvmaZcuWsWrVqsj9GTNmRM0NE/FJEpFoU15eHs8//zxerzfymM/n49lnnyU/P9+UmBISElqshdScx+Pp9NrgIn5IIhJtOu2008jLy4tauGz16tXk5+czceLEbh27tZrKrFmzIrWsGTNmsHfvXm6//fbIWkjQsml2vOY1szlz5rBu3TqWLVsWOcbu3bsZMWIEDz74YNTrtm7diqIofbr0hThGEpFo19y5c1m5cmXk/ooVK7jhhht6/X1Xr17NkCFDuOeeezh06BCHDh3q8jGWLVvG1KlTufnmmyPHyM/Pb3FOACtXruScc85hxIgRPXUKogskEYl2/ehHP+KDDz5g79697N27l/Xr1/OjH/2o1983LS0NVVVJTk4mOzs7psXHPB4Pdrsdl8sVOYaqqsyZM4eioqLIFjvBYJBnn32WuXPn9vRpiE6ShdFEuzIzM7nkkktYtWoVhmFwySWXkJGRYXZY3ZKbm8sll1zCihUrmDx5Mv/4xz/w+/1ceeWVZod2wpIakejQ3LlzWbVqFU899VSP1RosFgvHr0ATDAZ75NidcdNNN0U64leuXMnVV1+Ny+Xqs/cX0SQRiQ5ddNFFBAIBgsEgF154YY8cMzMzM6rfR9M0Pv/886gydrsdTdO69T5tHWPmzJkkJiayfPly3njjDWmWmUyaZqJDqqry1VdfRX5uS3V1dWQd6ibp6enk5eW1KHveeecxf/58XnvtNYYPH87DDz9MVVVVVJmCggLee+89fvjDH+JwOGJqEhYUFLBx40b27NlDUlISaWlpWCyWSF/RokWLGDlyZFytO30ikhqR6BS3243b7W63zLvvvsvEiROjbkuWLGm17Ny5c5k9ezbXX38906dPZ9iwYZx77rlRZe655x727NnD8OHDY96sYMGCBaiqyujRo8nMzGTfvn2R52688UYCgUCfXAUU7ZOlYsUJ6/333+f888+npKSkxYaEom9JIhInHL/fz9GjR5k9ezbZ2dk888wzZod0wpOmmTjhPPfccwwdOpSqqiruv/9+s8MRSI1ICBEHpEYkhDCdJCIhhOkkEQkhTCeJSAhhOklEQgjTSSISQphOEpEQwnSSiIQQppNEJIQw3f8H9WVoupmTblwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:41.625103Z", + "iopub.status.busy": "2024-03-22T19:05:41.624806Z", + "iopub.status.idle": "2024-03-22T19:05:41.843809Z", + "shell.execute_reply": "2024-03-22T19:05:41.842964Z" + }, + "papermill": { + "duration": 0.241397, + "end_time": "2024-03-22T19:05:41.845833", + "exception": false, + "start_time": "2024-03-22T19:05:41.604436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6PklEQVR4nO3deVwT19oH8F/CEkB2AUFBEXBHRRERrWIVUVEvFK0WtSLa9taWLuJS6Fu1vvYWqWi9Va++9V617m0R0VJccKFSF0SsrQsu4K4gm7IbQjjvH7yZl8iWKJBk5vl+Pn4kM2cmz5DMw5lzZs4RMcYYCCGE58SaDoAQQtoCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIOhcstuwYQOcnZ1hZGQEb29vnD9/vtGymzdvxvDhw2FlZQUrKyv4+fk1WZ4Qwl86lex+/PFHREREYNmyZbh48SL69++PsWPHIi8vr8HyKSkpCAkJwcmTJ3H27Fk4OTnB398fjx49auPICSGaJtKlgQC8vb3h5eWF9evXAwBqamrg5OSEjz76CJGRkc1uL5fLYWVlhfXr12PWrFkqvWdNTQ0eP34MMzMziESiV4qfENKyGGMoLS1Fx44dIRY3XXfTb6OYXllVVRUyMjIQFRXFLROLxfDz88PZs2dV2kdFRQVkMhmsra0bLSOVSiGVSrnXjx49Qu/evV8+cEJIq3vw4AEcHR2bLKMzya6goAByuRwdOnRQWt6hQwdcv35dpX189tln6NixI/z8/BotEx0djeXLl9db/u9//xsmJibqBU0IaVUVFRV45513YGZm1mxZnUl2r2rlypXYu3cvUlJSYGRk1Gi5qKgoREREcK9LSkrg5OSEoKAgmJubt0WoGiGTyZCcnIwxY8bAwMBA0+GQVySUz7OkpATvvPOOSk1MOpPsbGxsoKenhydPnigtf/LkCezt7ZvcNjY2FitXrsSxY8fQr1+/JstKJBJIJJJ6yw0MDHj9pVEQynEKBd8/T3WOTWd6Yw0NDeHp6Ynjx49zy2pqanD8+HH4+Pg0ut0333yDFStW4PDhwxg0aFBbhEoI0UI6U7MDgIiICISGhmLQoEEYPHgw1q5di/LycoSFhQEAZs2ahU6dOiE6OhoAEBMTg6VLl2L37t1wdnZGbm4uAMDU1BSmpqYaOw5CSNvTqWQ3bdo05OfnY+nSpcjNzYWHhwcOHz7MdVrcv39fqft548aNqKqqwpQpU5T2s2zZMnz55ZdtGTohRMN0KtkBQHh4OMLDwxtcl5KSovT67t27rR8QIUQn6EybHSGEvAqdq9kRQuqrqKhQut+0rFKKM5ezYWVzAabGyncX9OzZU5D3jFKyI4QHrl+/Dk9Pz3rLv2mgbEZGBgYOHNj6QWkZSnaE8EDPnj2RkZHBvb6R8wwRP1/Gmjf7ooeDZb2yQkTJTqBUvewR6iWPrjExMVGqrYnvFUKSWole7v3h0aW9BiPTHpTsBErVyx6hXvIQ/qFkJ1CqXvYI9ZJHF9wpKEe5tLrBddn55dz/+vqNn+btJProatOuVeLTNpTsBIoue3TbnYJyvB6b0my5BXGXmy1zcuFIQSQ8SnYC0VQtAKCagK5RfJZrp3nAza7+o4/llVIkppzFxJE+aGdcf2ALAMjKK8OnP15q8nvBJ5TsBEDVWgBANQFd42ZnCvdOFvWWy2Qy5NoCA7tY8XrUE3VQshOA5moBANUECP9RshOQxmoBANUECP9RsiNEB0nlzyE2eoQ7JTcgNqpfW6+ursbj6sfILMpstA32TkkZxEaPIJU/B9DwH0E+oWQnAM2dGACdHLrmcfk9tOu6Dp83Mw3yvw7/q8n17boCj8s94IkOTZbjA0p2AqDqiQHQyaErOrbrgvI7H+Gf0zzg2kA7bHV1NU7/fhrDXhvW6B+v7LwyfPLjJXR8vUtrh6sVKNkJQHMnBkAnh66R6Bmh5nkndDXvgd7tG+6NvaN/B72sezXaBlvzvBg1z/Mh0Wt8Aio+oWQnAM2dGACdHIT/aPBOQoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCDfEkAJUyOQDgyqPiRsuUV0pxIR+wv/e0yQl3CNFVOpfsNmzYgFWrViE3Nxf9+/fHunXrMHjw4EbL//zzz1iyZAnu3r2Lbt26ISYmBgEBAW0YseZl/1+SioxvbppEfezISm92f+0kOve14Z3m/oDRH6/6dOpb++OPPyIiIgKbNm2Ct7c31q5di7Fjx+LGjRuws7OrV/7MmTMICQlBdHQ0Jk6ciN27dyMoKAgXL16Eu7u7Bo5AM/z72AMAXO1MYWyg12CZGznFWBB3Gaun9EUPh8bnl6BJsrWDan/A6I9XXSLGGNN0EKry9vaGl5cX1q9fDwCoqamBk5MTPvroI0RGRtYrP23aNJSXlyMxMZFbNmTIEHh4eGDTpk0qvWdJSQksLCxQXFwMc3PzljkQLXTpXiGCNp5Dwrwh8OjSXtPhkGYUlVfh6NXcRv+ACeWPlzrnp86k9KqqKmRkZCAqKopbJhaL4efnh7Nnzza4zdmzZxEREaG0bOzYsUhISGj0faRSKaRSKfe6pKQEQO2w5TKZ7BWOQLtVV1dz//P5OPnCzFCEyQMcGl3//PlzAEAXKwl62Jk0uS9d/rzViV1nkl1BQQHkcjk6dFCe1apDhw64fv16g9vk5uY2WD43N7fR94mOjsby5cvrLT969ChMTJr+0uiyB2UAoI9z587h0RVNR0NelVA+z4qKCpXL6kyyaytRUVFKtcGSkhI4OTnB39+f15exf94vAi5fwJAhQ9C/s7WmwyGvSCifp+LKSxU6k+xsbGygp6eHJ0+eKC1/8uQJ7O3tG9zG3t5erfIAIJFIIJHU770yMDBodNYtPlBMn6ivr8/r4xQKoXye6hybztxUbGhoCE9PTxw/fpxbVlNTg+PHj8PHx6fBbXx8fJTKA0BycnKj5Qkh/KUzNTsAiIiIQGhoKAYNGoTBgwdj7dq1KC8vR1hYGABg1qxZ6NSpE6KjowEAn3zyCXx9fbF69WpMmDABe/fuxYULF/D9999r8jAIIRqgU8lu2rRpyM/Px9KlS5GbmwsPDw8cPnyY64S4f/8+xOL/r6wOHToUu3fvxhdffIHPP/8c3bp1Q0JCgqDusSOE1NKpZAcA4eHhCA8Pb3BdSkpKvWVvvvkm3nzzzVaOihCi7XSmzY4QQl4FJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiMoLi7G7MkBePiv2Zg9OQDFxcWaDomQFqdzs4uRluXm5obs7Gzu9Z8ZBbC0tISrqyuysrI0GBkhLYtqdgL2YqKrKzs7G25ubm0cESGth5KdQOXk5CglutdG+cF60iK8NsqPW5adnY2cnBxNhEdIixMxxpimg9BmJSUlsLCwQHFxMczNzTUdTovp27cvrly50mw5d3d3XL58uQ0iIi2lsrISs/8ejoOpf+Bvwwdg2/+sh7GxsabDahXqnJ9UsxMoRXtcu3btGlyvODmo3U63BAUFwcTEBD/t2ILnd//ATzu2wMTEBEFBQZoOTeMo2QmUSCQCAJSXlze4vrKyUqkc0X5BQUE4cOBAg+sOHDgg+IRHyU6gXnvtNe5nCwsLuLi4wMrKCi4uLrCwsGiwHNFelZWVjSY6hQMHDnB/xISIkp1A6enpcT8XFxfj9u3bePr0KW7fvq10n13dckR7ffzxxyqVmzVrFioqKlo5Gu1EyU6gLl261KLliGalpqaqVC4uLg7Xr19v5Wi0EyU7gTIwMGjRckSzSktLVSpna2uLnj17tnI02omSnUDV1NRwP0skEqV1dV/XLUe0l5GRkUrlzMzMYGJi0srRaCedSXZFRUWYMWMGzM3NYWlpiblz56KsrKzJ8h999BF69OgBY2NjdO7cGR9//DE99/l/6tYEpFKp0rq6r1WtMRDNev78eYuW4yOdSXYzZszA1atXkZycjMTERJw6dQrvvfdeo+UfP36Mx48fIzY2FleuXMG2bdtw+PBhzJ07tw2j1l6qdjxQB4VuoGTXPLWfoLh9+zZcXFxaK54GZWZmonfv3khPT8egQYMAAIcPH0ZAQAAePnyIjh07qrSfn3/+GTNnzkR5eTn09RseA0EqlSrVbEpKSuDk5ISCggJePUExevRolRq1hw8fjuPHj7dBRORVdOjQAU+fPm22nJWVFZ48edIGEbWNkpIS2NjYqPQEhdqjnri5ucHX1xdz587FlClTVG4reBVnz56FpaUll+gAwM/PD2KxGGlpaXjjjTdU2o/iF9JYogOA6OhoLF++vN7yo0eP8qqtQ5UTQ1EuKSmplaMhr0rV20kqKip49XmqcxuN2snu4sWL2Lp1KyIiIhAeHo5p06Zh7ty5GDx4sLq7Ullubi7s7OyUlunr68Pa2hq5ubkq7aOgoAArVqxo8tIXAKKiohAREcG9VtTs/P39eVWzW7lypUrlTE1NERAQ0MrRkFf1YrtrU+X49HmWlJSoXFbtZOfh4YF//vOfWL16NQ4ePIht27bhtddeQ/fu3TFnzhy8/fbbsLW1VWlfkZGRiImJabJMZmamuiHWU1JSggkTJqB379748ssvmywrkUjq9U4Ctbdg8Ok2jPz8fJXL8em4Cb9uJ1LnWF551BOpVIp//etfiIqKQlVVFQwNDTF16lTExMTAwcGhyW3z8/NRWFjYZBkXFxfs3LkTCxYsULr0qq6uhpGREX7++ecmL2NLS0sxduxYmJiYIDExUe3Lbr6OetK+fXsUFRU1W87a2rrZz4honlgshiqnskgk4tXtROqcny89UvGFCxewZcsW7N27F+3atcPChQsxd+5cPHz4EMuXL0dgYCDOnz/f5D5sbW1VqgX6+Pjg2bNnyMjIgKenJwDgxIkTqKmpgbe3d6PblZSUYOzYsZBIJDh48GCbtC/qCrFYtY54VcsRzerZs6dKV0FCvaEYAMDUtHr1aubu7s4MDAxYYGAg++WXX5hcLlcq8+DBA6anp6furps0btw4NmDAAJaWlsZ+//131q1bNxYSEsKtf/jwIevRowdLS0tjjDFWXFzMvL29Wd++fVlWVhbLycnh/lVXV6v8vsXFxQwAKy4ubtHj0TRXV1cGoNl/rq6umg6VqKBPnz4qfZ59+vTRdKgtSp3zU+2a3caNGzFnzhzMnj270ctUOzs7/Oc//1F3103atWsXwsPDMXr0aIjFYkyePBnfffcdt14mk+HGjRtc78zFixeRlpYGAPWGF79z5w6cnZ1bND5dY2tr2+iQ7C+WI9rP3t4eV69eVamcUKmd7JKTk9G5c+d6lzeMMTx48ACdO3eGoaEhQkNDWyxIoLbtaPfu3Y2ud3Z2VmqzGDlypEptGEKl6si1fB3hlm9cXV1Vuh/S1dW1DaLRTmo3yLi6uqKgoKDe8qKiInTt2rVFgiKtT9VBOWnwTt2g6h92IVcA1E52jf2yysrKqANAh9DjRfxy9+7dFi3HRypfxiputBWJRFi6dKnS0wRyuRxpaWnw8PBo8QBJ61D1DxP9AdMNNGRX81ROdn/88QeA2prd5cuXYWhoyK0zNDRE//79sXDhwpaPkLQKVTseqINCNzx8+JD72cjISKlGXvd13XJCo3KyO3nyJAAgLCwM//znP3l1g60QqXpjKZ9uQOWzO3fucD+/2PRQ93XdckKjdm/s1q1bWyMO0sZUnSKRplIkfKFSsgsODsa2bdtgbm6O4ODgJsvGx8e3SGCkdTU2heLLliOa1bVrV/z1118qlRMqlZKdhYUFdwtC3Wn2iO7q3Lkzbt68qVI5ov3at2/fouX4SKVkV/fSlS5j+SE8PBzHjh0DUNvBVFVVxa2r+zo8PFwj8RH1qDPEk1DRU94CVbctrm6ie/E1tdnpBnoipnkq1ewGDBig8p30Fy9efKWASNugm1D5pX///io9Lta/f/82iEY7qZTsgoKCWjkM0tbqDoRgYGAAmUzGva57GSv0ARN0xa1bt1q0HB+plOyWLVvW2nGQNta7d28AtbOHOTg44P79+9w6e3t7PHr0CHK5nCtHtFtlZWWLluMjarMTqDNnzgCofdSvbqIDgPv370MulyuVI9rN1NSU+/nFaQXqPvJXt5zQqJTsrK2tuZFOrKysYG1t3eg/ohvoCQp+UTQ16evr13vEz9bWlptRT8hNUipdxn777bcwMzPjfqZhf3Rf3b/wnTt3Vqrd1X0t5JqALunSpQuA2rlZXnz+9cGDB/XKCdErT7jDd3ydcMfPz4/rvbOzs8OMGTNQXl6Odu3aYdeuXcjLywNQO5m24n48or3kcjkcHByanDXOzs4Ojx8/hp6eXhtG1rpadcIdPT095OTk1JvHtbCwEHZ2dlxbD9Fuz549437Oy8vDt99+22w5ot0UNwxbW1tDIpGgpKQE5ubmkEqlKCoqEvzYhC02eKdUKlUa9oloN0dHxxYtRzQrJSUFJSUlMDU1RVFREXJyclBeXo6cnBwUFRXB1NQUJSUlSElJ0XSoGqNyzU4xuY1IJMK///1vpbYcuVyOU6dOCXuaNh0TGBiIAwcOAKhtwB4xYgSePn0KKysrnDp1irscCgwM1GSYREWKJFZWVgZDQ0MEBwfD2NgYlZWViI+PR1lZGVdu9OjRGoxUc1ROdorLHMYYNm3apHTdb2hoCGdnZ2zatKnlIyStou4E2fn5+di3b1+z5Yj2UtwEbmBggNLSUohEIiQlJSEgIAA//PADTE1NIZPJ6j0aKCQqJzvFoH+vv/464uPjYWVl1WpBkdanahKjZKcbFBNk29nZQV9fX6ntXHE7yuPHj1WaSJuv1G6zO3nyJCU6nhk/fjyGDRsGJycnDBs2DOPHj9d0SERNis6HR48eITAwEOfOnUNlZSXOnTuHwMBAPH78WKmcEKndGztnzpwm12/ZsuWlgyFtR3EDuK2tLY4ePcrVBB48eAA9PT3Y2toiPz+fbhTXEd27d0dycjIA4NixY0hMTOTW1R3ppHv37m0em7ZQu2b39OlTpX95eXk4ceIE4uPj6TYFHaKYGT4/P7/e7UJyuZzroBDyDPK6ZNWqVQBqbw17sV1OKpVybeyKckKkds1u//799ZbV1NRg3rx5gp5tXNeomsQo2ekGY2NjeHl5IT09HQAwcOBArjdWMeyal5cXjWf3qsRiMSIiIjBy5EgsXry4JXZJWhnNQcEvitq44mmCF8eVtLCwQEFBAeRyOa+eoFBHiyQ7AMjOzkZ1dXVL7Y60suXLl3M/N/W42PLly/G3v/1NU2ESFaWmpuLu3bsQiUQYP348SktLce/ePXTp0gVmZmY4fPgwiouLkZqaipEjR2o6XI1QO9lFREQovWaMIScnB7/++itCQ0NbLDDSuh49egQAcHBwgKGhodLjYs7OznBwcEBOTg5Xjmg3xefk4eGBy5cvc4MBPHjwAI6OjvDw8MAff/wh6M9T7WT3xx9/KL0Wi8WwtbXF6tWrm+2pJdrD3NwcT548gUwmw7179/Dbb7/h0KFDGD9+PHx9feHg4MCVI9pP0aH04vkJAA8fPuSSX1MDBfCd2snu5MmTrREHaWNBQUFYtWoVCgoKEBQUhMjISHh5eUEikSAoKAiFhYVcOaL9aCrF5rVYmx3RLWPHjuVuQ0hKSkJSUlKj5Yj2U9w0DNS2wU6fPh0VFRUwMTHB7t27uTbYuuUEh+mIwsJCNn36dGZmZsYsLCzYnDlzWGlpqUrb1tTUsHHjxjEAbP/+/Wq9b3FxMQPAiouLXyJq7VVdXc3s7OwYgEb/2dnZserqak2HSlQwatQoBoDp6+szsVis9Dnq6ekxfX19BoCNGjVK06G2KHXOT52Zg2LGjBm4evUqkpOTkZiYiFOnTuG9995Tadu1a9fS6Mov0NPTw8aNGyESiZTmKABq79kSiUTYuHGjYG9T0DWKNrnq6mrY2tpi/vz5eO+99zB//nzY2Nhwd0q8OIqxkOjEZWxmZiYOHz6M9PR0DBo0CACwbt06BAQEIDY2Fh07dmx020uXLmH16tW4cOEC1+jeFKlUqjRreklJCQBAJpMpTTfIB5MmTcLevXuxePFi3Lt3j1veoUMHxMTEYNKkSbw7Zr7q1KkTbt68CUNDQxgZGdXrXVdMj9mpUydefabqHEuLJbuHDx/iv//7v/H999+31C45Z8+ehaWlJZfogNphxcViMdLS0vDGG280uF1FRQWmT5+ODRs2qPwkQHR0tNI9aApHjx6FiYnJyx2AFpNIJFizZg2uXbvGjWfXu3dv6OnpNdqOR7SPYpDVqqoqWFtbw9/fHxKJBFKpFBcuXOAmO3d0dOTV51pRUaFy2RZLdoWFhfjPf/7TKskuNze33jDw+vr6sLa2Rm5ubqPbzZ8/H0OHDlVrAMqoqCilewlLSkrg5OQEf39/Xt+GMW7cOCQnJ2PMmDEwMDDQdDhETc+ePcOOHTsA1N5+0tAtKEDtnCIBAQFtGVqrUlx5qUKjl7GRkZGIiYlpsszLjr918OBBnDhxotEPvTESiaTevJtA7aCIQkgCQjlOvlF11rAuXbrw6vNV51g0muwWLFiA2bNnN1nGxcUF9vb2XNe5QnV1NYqKihq9PD1x4gSys7NhaWmptHzy5MkYPny4oMfiJ/wzfPhwODs7Q09PD3fv3lUayUZPTw/Ozs6oqanB8OHDNRilZmk02dna2tab0LchPj4+ePbsGTIyMuDp6QmgNpnV1NTA29u7wW0iIyPxzjvvKC3r27cvvv32W0yaNOnVgydEi+jp6eHNN9/EqlWrGnzWOTs7G4sWLRJ077rK88YGBwc3uf7Zs2f47bffWm0qxfHjx+PJkyfYtGkTZDIZwsLCMGjQIOzevRtA7bOBo0ePxvbt2zF48OAG9yESibB//361ngrg67yxL5LJZNycBXy6zBEKuVwONzc32NjYoKCggOuQAICuXbuiffv2KCwsxK1bt3iV8Fpl3lgLC4tm18+aNUvV3alt165dCA8Px+jRoyEWizF58mRuxjOg9mS9ceOGWr0zhPCFYtSTPXv2wMvLCydPnuSedX799ddx/vx5DB06lEY9UcXWrVtbM45mWVtbc7W4hjg7Ozc6p62CipVYQnROTk4OAMDd3R16enrw9fVFeXk5fH19oaenB3d3d6VyQqQzT1AQQhqnuGH+ypUrDa5XLFflxnq+Urlmp+rwTTThDiFtT9Eb+/XXXyMhIUFpXU1NDaKjo9G1a1fqjVXFtm3b0KVLFwwYMIAuBwnRMnp6eli9ejWmTJmCoKAgLFq0iJtKcdWqVUhMTERcXByvOifUpXKymzdvHvbs2YM7d+4gLCwMM2fOpGn2CNEiwcHBiIuLw4IFCzBixAhuedeuXREXF9fsHRV8p3Kb3YYNG5CTk4PFixfjl19+gZOTE6ZOnYojR45QTY8QLREcHIysrCwkJycjIiICycnJuHXrluATHaBmB4VEIkFISAiSk5Nx7do19OnTBx988AGcnZ1RVlbWWjESQtSg6I0dMWIE1xtLXqE3ViwWQyQSgTHWajcSE0JIS1Er2UmlUuzZswdjxoxB9+7dcfnyZaxfvx7379+Hqalpa8VICFGDXC7Hb7/9hlOnTrXqU026RuUOig8++AB79+6Fk5MT5syZgz179sDGxqY1YyOEqCk+Ph4LFizgHhdbs2YNnJ2dsXr1asG326mc7DZt2oTOnTvDxcUFv/32G3777bcGy8XHx7dYcIQQ1cXHx2PKlCmYOHEiduzYgYcPH8LR0RHffPMNpkyZIvgeWZWT3axZs2geB0K0lFwux4IFCzBx4kQkJCRALpejsLAQ3t7eSEhIQFBQEBYuXIjAwEDBdliodVMxIUQ71R0IQCwWK7XTicViREVFCX4gAHo2lhAeqDsQQEMdFDQQgI7MLkYIaZriAf/169fjf/7nf+p1UCimHRXyQABUsyOEB4YPHw5bW1tERUXB3d0dqamp2LNnD1JTU+Hu7o7PP/8cdnZ2gh4IgJIdITxRtwNR8QgnPcr5/yjZEcIDqampyMvLQ3R0NK5cuYIRI0YgJCQEI0aMwNWrV/H1118jLy8Pqampmg5VYyjZEcIDio6H8PDwBgcCCA8PVyonRJTsCOGBuiMVNzQQAI1UTMmOEF6oO1JxTU2N0joaqbgWJTtCeEAxUnFiYiKCgoJw7tw5bqTioKAgJCYmIjY2VrBPTwB0nx0hvEEjFTeNkh0hPBIcHIzAwMB688YKuUanQMmOEJ5paN5YQm12BDTYIxEGSnYCFx8fD1dXV4wZMwZr1qzBmDFj4OrqSuMSEt6hZCdg8fHxmDx5MvLy8pSW5+XlYfLkyZTwCK9QshMouVyO999/v8ky8+bNo0tawhuU7AQqJSUF+fn5AOo/LK54nZeXh5SUlLYOjZBWQclOoE6cOMH97OfnpzQkkJ+fX4PlCNFllOwE6t69ewBqR7Y9cOAAvL29YWxsDG9vbxw4cAB9+vRRKkeIrtOZZFdUVIQZM2bA3NwclpaWmDt3LsrKyprd7uzZsxg1ahTatWsHc3NzjBgxApWVlW0QsW6g8c6IUOhMspsxYwauXr2K5ORkJCYm4tSpU9xQ0405e/Ysxo0bB39/f5w/fx7p6ekIDw+HWKwzh91qunTpAgC4evUqAgMDlZ6lDAwMxNWrV5XKEaLzmA64du0aA8DS09O5ZYcOHWIikYg9evSo0e28vb3ZF1988UrvXVxczACw4uLiV9qPtjl27BgDwAAwY2Nj7ucXXx87dkzToZKXUFVVxRISElhVVZWmQ2lV6pyfOvG42NmzZ2FpaYlBgwZxy/z8/CAWi5GWloY33nij3jZ5eXlIS0vDjBkzMHToUGRnZ6Nnz574xz/+gddee63R95JKpZBKpdzrkpISAIBMJoNMJmvBo9KsYcOGwdbWluuRbYidnR2GDRvGq+MWCsVnxvfPTp3j04lkl5ubCzs7O6Vl+vr6sLa2Rm5uboPb3L59GwDw5ZdfIjY2Fh4eHti+fTtGjx6NK1euoFu3bg1uFx0djeXLl9dbfvToUZiYmLzikWiXOXPmICYmBtXV1UrLFa/DwsJw5MgRTYRGXoFcLse1a9fw9OlTXL58Gb179+bt87EVFRUql9VosouMjERMTEyTZTIzM19q34oBDP/+978jLCwMADBgwAAcP34cW7ZsQXR0dIPbRUVFISIigntdUlICJycn+Pv7w9zc/KVi0VYBAQEYOHAgFi9erNTr2qlTJ8TExDRYYybabf/+/fjss8+4qRQBwNnZmbefp+LKSxUaTXYLFizA7Nmzmyzj4uICe3v7eo80VVdXo6ioCPb29g1upxh+unfv3krLe/Xqhfv37zf6fhKJBBKJpN5yAwMDGBgYNBmrLpo6dSomT55MQwLxQHx8PN566y1MnDgRO3bswMOHD+Ho6IhvvvkGb731Fi/HtFPrnGyDNsRXpuiguHDhArfsyJEjTXZQ1NTUsI4dO9broPDw8GBRUVEqvzdfOyheJJQGbb6qrq5mzs7ObNKkSUwulyt9nnK5nE2aNIl17dqVVVdXazrUFqXO+akT92D06tUL48aNw7vvvovz58/j9OnTCA8Px1tvvYWOHTsCAB49eoSePXvi/PnzAGrn0Fy0aBG+++47xMXFISsrC0uWLMH169cxd+5cTR4OIS0uNTUVd+/exeeff17v1iqxWIyoqCjcuXNH0FMp6kQHBQDs2rUL4eHhGD16NMRiMSZPnozvvvuOWy+TyXDjxg2lBstPP/0Uz58/x/z581FUVIT+/fsjOTkZrq6umjgEQlqNYopEd3f3Btcrlgt5KkWdSXbW1tbYvXt3o+udnZ0bfBogMjISkZGRrRkaIRpXdyrFIUOG1FtPUynq0BMUhJDG1Z1KUSaTKY08LZPJaCpF6FDNjrSeusOyt2vXjnpjdZBiKsUpU6bA3Nwcz58/BwCsWbMGRkZGkEqliIuLE/TnSjU7gYuPj4eLi4vSsOwuLi40SrGOYowpPQEE1D4V1FATj9CIGP0WmlRSUgILCwsUFxfz7qZixbDsjdm3bx/v7sviK7lcDgcHB+Tn58PY2FhpZB/Fazs7Ozx+/JhXtTt1zk+q2QmUXC7HzJkzAQCGhoZYvHgxNm7ciMWLF8PQ0BAAMHPmTBqWXUfUHXl69OjRSoOxjh49GgCNPE3JTqCOHj2KyspKGBgYoLS0FF999RUcHBzw1VdfobS0FAYGBqisrMTRo0c1HSpRgWJE6SFDhjQ4GKuih1bII09TshOoNWvWAAA++ugjrianYGhoiA8//FCpHNFuDx48AFA77iNjTKk3ljGGkJAQpXJCRL2xAvXs2TMAgI+PT4Prvb29lcoR7ebk5AQAWLduHWJjY7mBHdasWYMuXbpwz3srygkR1ewESjE2YGRkJDdCjEJNTQ2++OILpXJEu40aNQoAcPPmTVRWVmLjxo3YsmULNm7ciMrKSty8eVOpnBBRb2wz+NobW1lZyY3PFxAQgMjISDx69AidOnXCypUrkZSUBKB2vDBjY2NNhkpUUFVVBWNjY9TU1DTaGysWi1FZWVmv2UKXqXN+0mWsQBkbGyMwMBAHDhxAUlISl9zqCgwMpESnI86cOVOvhq4gEokA1NbYz5w5g5EjR7ZhZNqDLmMFLCEhAYGBgQ2uCwwMREJCQtsGRF6a4gH/nTt31hvV287ODjt37lQqJ0SU7AQuISEBFRUVeP/99+Hh4YH3338fFRUVlOh0jOIBf1dXV2RnZyM5ORkRERFITk5GVlYWXFxclMoJEbXZNYOvbXYvkslkSEpKQkBAAC9HZOY7uVwONzc39O3bFwkJCZDL5dznqaenh6CgIFy5cgW3bt0S7BMU1GZHCA/UHQggMDAQY8aMwa1bt3Dv3j0kJyfj119/FfxAAJTsCOGJ4OBgLFy4EN9++y0SExO55fr6+li4cKHgn3OmZEcIT8THxyM2NhYTJkzganbdunVDcnIyYmNjMWTIEEEnPGqzawa12RFdQG12NOoJIYJAE+40j5IdITxAE+40j5IdITxQd8KdhtCEO5TsCOGFuhPuNDSwA024Q8mOEF5Q3GeXmJiIoKAgnDt3DpWVlTh37hyCgoKQmJiI2NhYXnVOqItuPSGEJ4KDgxEXF4cFCxZgxIgR3PKuXbsiLi5O0LedAJTsCOGV4OBgBAYG4uTJkzh06BDGjx9PU2P+H0p2hPCMnp4efH19UV5eDl9fX0p0/4fa7AghgkDJjqCyshIff/wxvvzyS3z88cdKo9wSwheU7AQuKCgIJiYm2LRpEy5duoRNmzbBxMQEQUFBmg6NkBZFyU7AgoKCcODAgQbXHThwgBIe4RVKdgJVWVnJJbqAgAClGeQDAgIA1CY8uqQlfEHJTqAiIiIA1A7j/csvvyjNIP/LL7/A1dVVqRwhuk5nkl1RURFmzJgBc3NzWFpaYu7cuSgrK2tym9zcXLz99tuwt7dHu3btMHDgQOzbt6+NItZuFy5cAACsXLmywRnkv/rqK6VyhOg6nbnPbsaMGcjJyUFycjJkMhnCwsLw3nvvYffu3Y1uM2vWLDx79gwHDx6EjY0Ndu/ejalTp+LChQsYMGBAG0avfSwtLQEA27dvx8KFC+vNIN+3b1+lcoToPKYDrl27xgCw9PR0btmhQ4eYSCRijx49anS7du3ase3btysts7a2Zps3b1b5vYuLixkAVlxcrH7gWiwpKYkBaPZfUlKSpkMlaqqurmbJycksIiKCJScns+rqak2H1GrUOT91omZ39uxZWFpaYtCgQdwyPz8/iMVipKWl4Y033mhwu6FDh+LHH3/EhAkTYGlpiZ9++gnPnz9vcpJgqVQKqVTKvS4pKQFQO5KvTCZrmQPSAnWfnQQALy8vBAQEICkpCenp6Url+HTcfLd//34sWrQI9+/fB1BbU+/cuTNWrVrV6Hmiy9T5bupEssvNza038a++vj6sra2Rm5vb6HY//fQTpk2bhvbt20NfXx8mJibYv38/3NzcGt0mOjoay5cvr7f86NGjMDExefmD0DIZGRlKr9PT05WSnEJMTAw8PT3bKizyCs6ePYuYmJh6y+/fv49p06bhs88+g4+PjwYiaz0VFRUql9VosouMjGzww6krMzPzpfe/ZMkSPHv2DMeOHYONjQ0SEhIwdepUpKamcm1SL4qKilLqgSwpKYGTkxP8/f15NQfFunXrAAATJ07En3/+iQcPHnDrOnfuDHd3dyQlJeHMmTNYsmSJpsIkKpLL5QgJCWmyzLp16/Dll1/y6llZxZWXKjSa7BYsWIDZs2c3WcbFxQX29vbIy8tTWl5dXY2ioiLY29s3uF12djbWr1+PK1euoE+fPgCA/v37IzU1FRs2bMCmTZsa3E4ikUAikdRbbmBgwKuJaIqLiwEAoaGheOONN+qNkhEXF4ekpCQUFxfz6rj56vjx40r3RM6cOROenp7IyMjAzp07AdTWglJSUjBu3DhNhdni1PputkEb4itTdFBcuHCBW3bkyJEmOyj++usvBoBdu3ZNabm/vz979913VX5vvnZQzJs3jwFgbm5uTC6Xs6qqKpaQkMCqqqqYXC5nrq6uDACbN2+epkMlKhg5ciTXqVRZWan0eVZWVnLrRo4cqelQW5Q656dOJDvGGBs3bhwbMGAAS0tLY7///jvr1q0bCwkJ4dY/fPiQ9ejRg6WlpTHGGKuqqmJubm5s+PDhLC0tjWVlZbHY2FgmEonYr7/+qvL78jXZVVRUcCfAhAkT2KlTp9iePXvYqVOn2IQJE7h1FRUVmg6VqMDc3JwBYCNGjGCMMaVkxxhjw4YNYwCYubm5JsNscbxMdoWFhSwkJISZmpoyc3NzFhYWxkpLS7n1d+7cYQDYyZMnuWU3b95kwcHBzM7OjpmYmLB+/frVuxWlOXxNdowxFhgY2ORtJ4GBgZoOkajIysqKAWBOTk4N1tQdHR0ZAGZlZaXpUFsUL5OdpvA52THWeMKjRKdbJk6cqFJNfeLEiZoOtUWpc36KGGNM7VZBAVFnxnFdVVlZiYiICJw7dw5DhgzBmjVrYGxsrOmwiBrKyspgZmbWbLnS0lKYmpq2QURtQ53zUyfusyOty9jYGN999x2SkpIQEBBAva86yNTUFF5eXg3eK6ng5eXFq0SnLp0ZCIAQ0rTz58/Dy8urwXVeXl44f/58G0ekXSjZEcIj58+fR2lpKSZNmoQuXbpg0qRJKC0tFXyiA+gylhDeMTU1xb59+6hZ4gVUsyOECAIlO0KIIFCyI4QIArXZNUNxG6I6oyvoIplMhoqKCpSUlFAbDw8I5fNUnJeq3C5Mya4ZpaWlAAAnJycNR0IIaUxpaSksLCyaLENPUDSjpqYGjx8/hpmZGUQikabDaTWKcfsePHjA2ydFhEQonydjDKWlpejYsSPE4qZb5ahm1wyxWAxHR0dNh9FmzM3NeX1yCI0QPs/manQK1EFBCBEESnaEEEGgZEcA1A5Hv2zZsgaHpCe6hz7P+qiDghAiCFSzI4QIAiU7QoggULIjhAgCJTsdcPfuXYhEIly6dKlN9nP69Gn07dsXBgYGCAoKeqX3JERbULLTQSkpKRCJRHj27Fmr7D8iIgIeHh64c+cOtm3b1irvQeobOXIkPv30U02HwdG2eF4VJbtWVlVVpekQ1JadnY1Ro0bB0dERlpaWL7WPtjxuxhiqq6vb7P20mS5+39pMq81xJlC+vr7sww8/ZJ988glr3749GzlyJLt8+TIbN24ca9euHbOzs2MzZ85k+fn53DaHDh1iw4YNYxYWFsza2ppNmDCBZWVlcesVc+L+8ccf3M91/4WGhqq1nz179jAfHx8mkUhYnz59WEpKitL6uv+2bt3KGGMsJSWFeXl5MUNDQ2Zvb88+++wzJpPJmjzukydPMgDs8OHDzMPDgxkZGbHXX3+dPXnyhCUlJbGePXsyMzMzFhISwsrLy7l9yeVy9vXXXzNnZ2dmZGTE+vXrx37++WduvWK/SUlJbODAgczAwEBpvmBdFBoaWu93n5WVxebMmcP9Hrp3787Wrl1bb7vAwED21VdfMQcHB+bs7MwYY+z06dOsf//+TCKRME9PT7Z//37uO6TQ1PeyoXju3LnTVr+OVkHJroX5+voyU1NTtmjRInb9+nV27tw5Zmtry6KiolhmZia7ePEiGzNmDHv99de5beLi4ti+ffvYrVu32B9//MEmTZrE+vbty+RyOWNMOdlVV1ezffv2MQDsxo0bLCcnhz179kyt/Tg6OrK4uDh27do19s477zAzMzNWUFDAqqurWU5ODjM3N2dr165lOTk5rKKigj18+JCZmJiwDz74gGVmZrL9+/czGxsbtmzZskaP+/r161xSGjJkCPv999/ZxYsXmZubG/P19WX+/v7s4sWL7NSpU6x9+/Zs5cqV3L6++uor1rNnT3b48GGWnZ3Ntm7dyiQSCZeUFfvt168fO3r0KMvKymKFhYWt/dG2qmfPnjEfHx/27rvvspycHJaTk8OeP3/Oli5dytLT09nt27fZzp07mYmJCfvxxx+57UJDQ5mpqSl7++232ZUrV9iVK1dYcXExs7a2ZjNnzmRXr15lSUlJrHv37krJ7unTp01+LxuKp7q6WhO/mhZDya6F+fr6sgEDBnCvV6xYwfz9/ZXKPHjwgEtWDcnPz2cA2OXLlxljysmOsf8/2Z8+fdpkLI3tp25ikclkzNHRkcXExHDLLCwsuBodY4x9/vnnrEePHqympoZbtmHDBmZqasol0hePu26cx44d45ZFR0czACw7O5tb9ve//52NHTuWMcbY8+fPmYmJCTtz5ozSvubOnctCQkKU9puQkNDk8esaX19f9sknnzRZ5sMPP2STJ0/mXoeGhrIOHTowqVTKLdu4cSNr3749q6ys5JZt3rxZ6TukyvdSlXh0CbXZtQJPT0/u5z///BMnT56Eqakp969nz54AatvGAODWrVsICQmBi4sLzM3N4ezsDAC4f/++Wu+r6n58fHy4n/X19TFo0CBkZmY2ut/MzEz4+PgoDXE1bNgwlJWV4eHDhw0ed139+vXjfu7QoQNMTEzg4uKitCwvLw8AkJWVhYqKCowZM0bpd7Z9+3bu96UwaNCgRmPmiw0bNsDT0xO2trYwNTXF999/X+/z7Nu3LwwNDbnXN27cQL9+/WBkZMQtGzx4sNI2qnwv+YaGeGoF7dq1434uKyvDpEmTEBMTU6+cg4MDAHDT3m3evBkdO3ZETU0N3N3d1W5sbqn9vKy6x11X3ZFyRSJRvZFzRSIRampqANT+vgDg119/RadOnZTKvficZ2Pvxxd79+7FwoULsXr1avj4+MDMzAyrVq1CWlqaUrmX+T2o8r3kG0p2rWzgwIHYt28fnJ2doa9f/9ddWFiIGzduYPPmzRg+fDgA4Pfff29yn4q/4nK5/KX2c+7cOYwYMQIAUF1djYyMDISHhzf6fr169cK+ffvAGONqd6dPn4aZmVmLj/XXu3dvSCQS3L9/H76+vi26b21naGio9JmePn0aQ4cOxQcffMAtU6XW1aNHD+zcuRNSqZT7A5Genq5UprnvZUPx6Dq6jG1lH374IYqKihASEoL09HRkZ2fjyJEjCAsLg1wuh5WVFdq3b4/vv/8eWVlZOHHiBCIiIprcZ5cuXSASiZCYmIj8/HyUlZWptZ8NGzZg//79uH79Oj788EM8ffoUc+bMafT9PvjgAzx48AAfffQRrl+/jgMHDmDZsmWIiIhodnRYdZmZmWHhwoWYP38+fvjhB2RnZ+PixYtYt24dfvjhhxZ9L23j7OyMtLQ03L17FwUFBejWrRsuXLiAI0eO4ObNm1iyZEm9pNWQ6dOno6amBu+99x4yMzNx5MgRxMbGAgD3x6q572VD8Shq37qKkl0r69ixI06fPg25XA5/f3/07dsXn376KSwtLSEWiyEWi7F3715kZGTA3d0d8+fPx6pVq5rcZ6dOnbB8+XJERkaiQ4cOCA8PV2s/K1euxMqVK9G/f3/8/vvvOHjwIGxsbJp8v6SkJJw/fx79+/fH+++/j7lz5+KLL754pd9NY1asWIElS5YgOjoavXr1wrhx4/Drr7+ia9eurfJ+2mLhwoXQ09ND7969YWtri7FjxyI4OBjTpk2Dt7c3CgsLlWp5jTE3N8cvv/yCS5cuwcPDA//1X/+FpUuXAgDXjtfc97KheNRtQ9Y2NMQTIQKwa9cuhIWFobi4GMbGxpoORyOozY4QHtq+fTtcXFzQqVMn/Pnnn/jss88wdepUwSY6gJIdIbyUm5uLpUuXIjc3Fw4ODnjzzTfxj3/8Q9NhaRRdxhJCBIE6KAghgkDJjhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7IjWmD17NkQiETcySocOHTBmzBhs2bJFrecyt23b9tLDyb+K2bNn0wRFWoySHdEq48aNQ05ODu7evYtDhw7h9ddfxyeffIKJEyfSPBPk1Wh06FBC6lDMp/Ci48ePMwBs8+bNjDHGVq9ezdzd3ZmJiQlzdHRk8+bNY6WlpYyx/x/FuO4/xfDx27dvZ56enszU1JR16NCBhYSEsCdPnnDvU1RUxKZPn85sbGyYkZERc3NzY1u2bOHW379/n7355pvMwsKCWVlZsb/97W/cvAzLli2r9766Pi8G31DNjmi9UaNGoX///oiPjwcAiMVifPfdd7h69Sp++OEHnDhxAosXLwYADB06FGvXroW5uTlycnKQk5ODhQsXAgBkMhlWrFiBP//8EwkJCbh79y5mz57Nvc+SJUtw7do1HDp0CJmZmdi4cSM3GoxMJsPYsWNhZmaG1NRUnD59Gqamphg3bhyqqqqwcOFCTJ06lauZ5uTkYOjQoW37iyJN03S2JUShsZodY4xNmzaN9erVq8F1P//8M2vfvj33euvWrczCwqLZ90tPT2cAuFrhpEmTWFhYWINld+zYUW8eDqlUyoyNjdmRI0eajZ9oHtXsiE5gdUZJPnbsGEaPHo1OnTrBzMwMb7/9NgoLC1FRUdHkPjIyMjBp0iR07twZZmZm3EjIinHa5s2bh71798LDwwOLFy/GmTNnuG3//PNPZGVlwczMjJuzwdraGs+fP+ftnA18Q8mO6ITMzEx07doVd+/excSJE9GvXz/s27cPGRkZ2LBhA4CmJ4guLy/H2LFjYW5ujl27diE9PR379+9X2m78+PG4d+8e5s+fj8ePH2P06NHcJXBZWRk8PT1x6dIlpX83b97E9OnTW/noSUugIZ6I1jtx4gQuX76M+fPnIyMjAzU1NVi9ejU3ou5PP/2kVL6huROuX7+OwsJCrFy5Ek5OTgCACxcu1HsvW1tbhIaGIjQ0FMOHD8eiRYsQGxuLgQMH4scff4SdnR3Mzc0bjJNvczbwDdXsiFaRSqXIzc3Fo0ePcPHiRXz99dcIDAzExIkTMWvWLLi5uUEmk2HdunW4ffs2duzYgU2bNintw9nZGWVlZTh+/DgKCgpQUVGBzp07w9DQkNvu4MGDWLFihdJ2S5cuxYEDB5CVlYWrV68iMTERvXr1AgDMmDEDNjY2CAwMRGpqKu7cuYOUlBR8/PHH3HSSzs7O+Ouvv3Djxg0UFBRAJpO1zS+NqEbTjYaEKISGhnK3bejr6zNbW1vm5+fHtmzZwk3GzRhja9asYQ4ODszY2JiNHTuWbd++vd6k4e+//z5r37690q0nu3fvZs7OzkwikTAfHx928ODBehNH9+rVixkbGzNra2sWGBjIbt++ze0zJyeHzZo1i9nY2DCJRMJcXFzYu+++y4qLixljjOXl5bExY8YwU1NTuvVEC9HgnYQQQaDLWEKIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCD8L6jSNtuByeREAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:05:41.885904Z", + "iopub.status.busy": "2024-03-22T19:05:41.885619Z", + "iopub.status.idle": "2024-03-22T19:05:42.161890Z", + "shell.execute_reply": "2024-03-22T19:05:42.160988Z" + }, + "papermill": { + "duration": 0.298816, + "end_time": "2024-03-22T19:05:42.164011", + "exception": false, + "start_time": "2024-03-22T19:05:41.865195", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXFUlEQVR4nO2dd3xT5f7H3yerezFKKZQpe2+BqwxBQAUc94LIVUHc4EJkqBfcoKLyExVwgIvhQIWriBeRIVO27FGKIKstpbvNPL8/TpMmadImadqk7fN+vfJqcnLGkzT55Pt81yPJsiwjEAgEAUQV6AEIBAKBECKBQBBwhBAJBIKAI4RIIBAEHCFEAoEg4AghEggEAUcIkUAgCDhCiAQCQcDRBHoA5cFisXDhwgWioqKQJCnQwxEIBE7IskxOTg6JiYmoVO7tniotRBcuXCApKSnQwxAIBGVw7tw5GjZs6Pb5Ki1EUVFRgPIio6OjAzwagUDgTHZ2NklJSbbvqjuqtBBZp2PR0dFCiASCIKYs14lwVgsEgoAjhEggEAQcIUQCgSDgVGkfUVVBlmVMJhNmsznQQxEI/IparUaj0ZQ7fUYIUQVjMBi4ePEi+fn5gR6KQFAhhIeHU79+fXQ6nc/nEEJUgVgsFlJSUlCr1SQmJqLT6UTipaDaIMsyBoOBtLQ0UlJSaNGiRalJi6VRI4Row/FU5v5ynDb1o5n7r06Vdl2DwYDFYiEpKYnw8PBKu65AUFmEhYWh1Wr566+/MBgMhIaG+nSeGiFEJrPM4QvZqFWBsUZ8/ZUQCKoC/vh814hvSHxUCACp2foAj0QgELiiZghRtCJE6bl6LBaxaIlAEGzUCCGqExmCJIHJInM13xDo4dRYzpw5gyRJ7N+/v1LOs3XrVjp06IBWq+XWW28t1zVrKhs3bkSSJDIzMyv0OjVCiLRqFbXCldBiao6YngULFf0hnzx5Mp07dyYlJYVPP/20Qq4h8A81QogA6lr9REKIfMJgqHqWZHJyMgMHDqRhw4bExsb6dI7KfN3WxFd/UxX+dzVPiLILAzoOWZbJN5gCcvNmUd/+/fszadIknnzySerUqcOQIUM4dOgQw4YNIzIyknr16nH33XeTnp5uO2bt2rX84x//IDY2ltq1a3PLLbeQnJzs8vxnzpxhwIABAMTFxSFJEuPGjfPqPMeOHaNPnz6EhobSvn17Nm3aZDu3JElcuXKF++67D0mSbBbRpk2b6NmzJyEhIdSvX5/p06c7fPldvW6r5fbLL7/QpUsXwsLCGDhwIKmpqfz888+0adOG6Oho7rrrLofEVYvFwuzZs2natClhYWF06tSJb7/91va89bw///wz3bp1IyQkhC1btpT5v3nllVeIj48nKiqK+++/n+nTp9O5c2fb8+PGjePWW2/l1VdfJTExkVatWgHwxRdf0L17d6KiokhISOCuu+4iNTXV4dxr1qyhZcuWhIWFMWDAAM6cOVPmePxBjQjfA8RHKfkNgbaICoxm2s78JSDXPvLSEMJ1nv/LP/vsMx555BG2bt1KZmYmAwcO5P777+edd96hoKCAadOmMWrUKH777TcA8vLymDx5Mh07diQ3N5eZM2dy2223sX///hIh3qSkJFauXMkdd9zB8ePHiY6OJiwszKvzPPPMM8ybN4+2bdvy9ttvM3z4cFJSUkhKSuLixYu0atWKl156idGjRxMTE8P58+e56aabGDduHJ9//jnHjh3jgQceIDQ0lBdeeMHl6wa4ePEiAC+88ALvvfce4eHhjBo1ilGjRhESEsKyZcvIzc3ltttuY/78+UybNg2A2bNn8+WXX7Jw4UJatGjB5s2b+fe//03dunXp16+f7XrTp09n7ty5NGvWjLi4uFL/J0uXLuXVV1/lgw8+oG/fvqxYsYK33nqLpk2bOuy3fv16oqOjWbdunW2b0Wjk5ZdfplWrVqSmpjJ58mTGjRvHmjVrAKWv1+23387EiRN58MEH2b17N08//XTZHxQ/UHOEqChyliamZh7TokUL3njjDUD5Fe7SpQuvvfaa7fnFixeTlJTEiRMnaNmyJXfccYfD8YsXL6Zu3bocOXKE9u3bOzynVqupVasWAPHx8Q5TJ0/PM2nSJNu+CxYsYO3atXzyySdMnTqVhIQEJEkiJiaGhIQEAD744AOSkpJ47733kCSJ1q1bc+HCBaZNm8bMmTNtImf/uqFYiF555RX69u0LwIQJE5gxYwbJyck0a9YMgH/+859s2LCBadOmodfree211/j111/p3bs3AM2aNWPLli0sWrTIQYheeuklBg8eXPY/BJg/fz4TJkxg/PjxAMycOZP//e9/5ObmOuwXERHBxx9/7FB2cd9999nuN2vWjHfffZcePXqQm5tLZGQkCxYsoHnz5rz11lsAtGrVioMHD/L66697NLbyUHOEKCo4hChMq+bIS0MCdm1v6Natm+3+gQMH2LBhA5GRkSX2S05OpmXLlpw8eZKZM2eyc+dO0tPTsVgsAJw9e7aEEJWGp+exfsEBNBoN3bt35+jRo27Pe/ToUXr37u1QZtO3b19yc3P5+++/adSoUYnXbU/Hjh1t9+vVq0d4eLhNhKzb/vjjDwBOnTpFfn5+CYExGAx06dLFYVv37t1t9y0WmcwCIzFhGtQuEgWPHz/Oo48+6rCtZ8+eNqvUSocOHUrUfu3Zs4cXXniBAwcOcPXqVYf3tW3bthw9epRevXo5HGP/HlckNUaIip3VgfURSZLk1fQokERERNju5+bmMnz4cJe/jvXr1wdg+PDhNG7cmI8++ojExEQsFgvt27f32lnqr/P4iv3rtker1druS5Lk8Ni6zfrltlooP/30Ew0aNHDYLyQkxO310vP0XMoqJE+vI6mW72VBzq8hLy+PIUOGMGTIEJYuXUrdunU5e/YsQ4YMCQpndtX4RviBYPERVVW6du3KypUradKkCRpNyY/NlStXOH78OB999BHXXXcdQJmOV+svtn17FG/Os2PHDq6//noATCYTe/bsYdKkSW6v16ZNG1auXIksyzaraOvWrURFRZXa2N0X2rZtS0hICGfPnnWYhpVFvl55L7IKjDSwyKicypJatWrFrl27uOeee2zbdu3aVeZ5jx07xpUrV5gzZ45twYndu3c77NOmTRtWr17tsG3Hjh0ej7081JiomX2ZhzfRI4HCxIkTycjIYMyYMezatYvk5GR++eUXxo8fj9lsJi4ujtq1a/Phhx9y6tQpfvvtNyZPnlzqORs3bowkSfz444+kpaWRm5vr1Xnef/99vv/+e44dO8bEiRO5evWqgx/EmUcffZRz587x2GOPcezYMVatWsWsWbOYPHmy3+sBo6KimDJlCk899RSfffYZycnJ7N27l/nz5/PZZ5+5Pa7AqAiRRZbJLjSWeP6xxx7jk08+4bPPPuPkyZO88sor/Pnnn2V2dWjUqBE6nY758+dz+vRpVq9ezcsvv+ywz8MPP8zJkyd55plnOH78OMuWLau0/KuaI0RFzuoCo5lcvf9zNao7iYmJbN26FbPZzI033kiHDh148skniY2NRaVSoVKpWLFiBXv27KF9+/Y89dRTvPnmm6Wes0GDBrz44otMnz6devXqMWnSJK/OM2fOHObMmUOnTp3YsmULq1evpk6dOqVeb82aNfzxxx906tSJhx9+mAkTJvD888+X671xx8svv8x//vMfZs+eTZs2bRg6dCg//fRTiQiXFZPZgtFssT3OzC8pRGPHjmXGjBlMmTKFrl27kpKSwrhx48qseq9bty6ffvop33zzDW3btmXOnDnMnTvXYZ9GjRqxcuVKfvjhBzp16sTChQsdghMViSRXYfMgOzubmJgYsrKyPFrFo/2sX8jVm/jt6X40q1vS6epvCgsLSUlJoWnTpj63RxDUHHIKjaSk56GWJMxF08c2CVFo1KXbC4MHDyYhIYEvvviikkbqSGmfc0+/ozXGRwTK9CxXbyI1R18pQiQQeIN1WhYVqqHQZKHQaCa70EitiGLndn5+PgsXLmTIkCGo1WqWL1/Or7/+6pAvVBWpUUJUJyqE0+l5wmEtCEoKDGZuu6E3l86fA0CWQQKs7p9FixZx++23s2bNGl599VUKCwtp1aoVK1euZNCgQYEbuB+oUUIUHyRlHgKBKwqNZt7/7CsSorTo1CpOp+UB0Dw+Eq1aRb169QgLC+PXX38N8Ej9Tw0TImX+GuikRoHAGbPFgt5kIbFhI9rWj0ajVhFSK5c8g4k6MWG2PLjqSo2JmoEo8xAELwVGJVqmU6tszunYcCVhMrMg8AmHFU3QCNGcOXOQJIknn3yywq4RL1qBCIKUAoPiqA61K8OJCdMiIVFgMKM3Vu818YJCiHbt2sWiRYscankqgmAp8xAInCksEpowXbEQadQqIkMV70lmQcmcoupEwIUoNzeXsWPH8tFHH5XZAqG8iDIPQbBiDd07Fybbpmf5xmpdERBwIZo4cSI333xzpYQfrVOzzHwjelP1NnUFVQeLRUZf5CNyFqLoUC0qSUJvMtvEqjoSUCFasWIFe/fuZfbs2R7tr9fryc7Odrh5Q2y4EhYFSM+t/g7AQNKkSRPmzZtneyxJEj/88EPAxhPMFJrMyMhoVCo0aseaMbVKIto6PXNR8lFdCJgQnTt3jieeeIKlS5d6XP4we/ZsYmJibDdrFbGnSJIUNC1jaxoXL15k2LBhgR5GUGJ1VIfp1C6LV2OLFn7IKqi+07OACdGePXtITU2la9euaDQaNBoNmzZt4t1330Wj0Ti0hrAyY8YMsrKybLdz5855fV3RRD8wJCQklOjDU9lUVHP68lLsH3L9dYwM1aBWSRjNFvKqacF2wITohhtu4ODBg+zfv9926969O2PHjmX//v2o1SW7CYaEhBAdHe1w85aAC5EsgyEvMDcvf01zcnIYO3YsERER1K9fn3feeYf+/fv7lGJhPzWzNrf/7rvvGDBgAOHh4XTq1Int27c7HLNlyxauu+46wsLCSEpK4vHHHycvL8/2fFnN4H1tTl/ZuHNUW1FJEjFh1pyi6jk9C1hmdVRUVIn2oREREdSuXdurtqLeYmsZG6ipmTEfXksMzLWfvQA6190HXTF58mS2bt3K6tWrqVevHjNnzmTv3r0OK0aUh+eee465c+fSokULnnvuOcaMGcOpU6fQaDQkJyczdOhQXnnlFRYvXkxaWhqTJk1i0qRJLFmyBCi7GbwVb5rTVzYWWaawyFEdqnPfyjc2TEtGnoGsAiOJsTKqMvoPVTVqVIkHiBC+p+Tk5PDZZ5+xbNkybrjhBgCWLFlCYqL/RHTKlCncfPPNALz44ou0a9eOU6dO0bp1a2bPns3YsWNt1leLFi1499136devHwsWLCA0NLTMZvBWvGlOX9nojRZkWUatkmyBFFdEhGjQqlUYzRZyCk02C6m6EFRCtHHjxgq/RsDLPLThimUSqGt7yOnTpzEajfTs2dO2LSYmxrZGlj+wT2C19r1OTU2ldevWHDhwgD///JOlS5fa9pFlGYvFQkpKCm3atCmzGbwV++b0wYZ1Whaqde2otiIVTc/Sc/Vk5huEEFV1Al7mIUleTY+qM87N6AGH5vMPPfQQjz/+eInjGjVq5FUzeHfN8IOBwjL8Q/bEhitClFNowmxRrKjqQo0TIlHm4RnNmjVDq9Wya9cu2zI7WVlZnDhxwtawviLp2rUrR44c4ZprrnH5/MGDB8tsBl8VsA/dl0WYVk2IRo3eZCa7wEhchK7MY6oKAc+srmysPqL0XANmS/XMyfAHUVFR3HvvvTzzzDNs2LCBw4cPM2HCBFQqVZmN2v3BtGnT2LZtG5MmTWL//v2cPHmSVatW2Vbp8KQZfLAjy3KZETN7JEmyq8ivXtGzGidEdSJ1SBKYLTIZeSK7ujTefvttevfuzS233MKgQYPo27cvbdq0qZT+2x07dmTTpk2cOHGC6667ji5dujBz5kybs9yTZvDBjt5kwSIrEbAQjWdfxdgi31Buocmh0X5Vp0Y1z7fS/ZV1pOcaWPP4dbRN9D4XyVOqW/P8vLw8GjRowFtvvcWECRMCPZwqT2a+gbMZ+YTrNFwT73kP9ZOpORQYzCTGhlEnMvAN0/zRPL/GWUQAdW0hfOEnKo19+/axfPly25pcY8eOBWDkyJEBHln1wJtpmT2xYYpvqDrVntVIIQp45KwKMXfuXDp16sSgQYPIy8vj999/5+jRo0RGRrq9CTyj2FHt3dfQ6ifKN5gwVJMuEjUuagbFkTPRMrZ0unTpwp49e0psLygoYP/+/ZU/oGqEt45qe7RqFZEhGnL1JjILjMRHeXd8MFIjhUis5lE+wsLC3IbVBZ5hNFswW5RFFEO8FCJQrKJcvYnMfKMtElyVqdFTs7TcyrGIqnA8QFBBWJvlh2pUPtWNRYdqkSSJQmPgG6b54/NdM4UoushZnV2xQmTNHM7Pz6/Q6wiqHjb/kA/WECj9rKNClAlNVn5g01Csn2/7THlvqdlTswr2EanVamJjY22tKcLDwyslGVBQuVhkGWRQeVFykZefj2wyoZZVFBb65iIIV5vJMhnIyDIRo6PSP1uyLJOfn09qaiqxsbEuW/d4So0UIvsyD1mWK/QfmJCQoFzLrk+OoHpxObsQk0UmMSbU48/SxaxCJbM/KoRMD5MZnZFlmfSsQiwyGLNC0Pl4nvISGxtr+5z7So0UIqtzr9BoIUdvIjq04iqZJUmifv36xMfHYzRWn7wPQTH3v7URgHl3dqZDg9gy97+Sp+f+77aDBD9O+gdhOt+/hl//fJR1Ry4zsnMij9/Q3Ofz+IpWqy2XJWSlRgpRmE5NVIiGHL2J1Gx9hQqRFbVa7Zd/mCC4kGWZ8zmKvydTL3mUQX/ir2zO55hpXjeCuOjy5V0NaNeQT3de4Ku9l3lqaHu0pfQ0Cmaq5qj9QN1A9yUSVAvs66aveug0PnJBWX2mfYOYcl+/b/Pa1I7QcSXPwNZT6eU+X6CosUIUL9qBCPyAfeg600MhOnQ+C4D2ieUXIo1axS0dlaZyq/cHqOGeH6jBQqSY0MIiEpQHe4vI09qvQxcUIWrnp4LrEZ0bAPDL4Uu2tICqRo0VooCv5iGoFljsLKKrHghRVr6RcxkFALTzg0UE0LVRLA3jwsgzmFl/7LJfzlnZ1FghEmUeAn9gn1ScVVD21OzwRcUaSqoVRky4f4IkkiQxsrPSp2lVFZ2e1Vwhiq7cMg9B9cTeIsryoGvi4fOKo7pdff9YQ1ZGFk3PNh5PJasKtgepuUIUVTllHoLqjddCVOQfat/Avw35WtaLonVCFEazzM+HLvr13JVBDRYi4SMSlB97Z3V2QdnLQR8qCt2380Po3pkRVXh6VmOFyOqsziow2pZ0EQi8RfbCIso3mDidlgv4L2Jmz/COihDtSLnCpayq5fussUIUE6a11eaIEL7AVxwsokIjllJWhjl6MQeLrFjjFdFDKKlWON0bxyHL8OOfVcsqKpcQ5ebmkp2d7XCrKkiSRN1I4bAWlA97H5EsQ06h++lZsX/I/9MyK1U1eua1EKWkpHDzzTcTERFBTEwMcXFxxMXFERsbS1xcXEWMscKwRs6Ew1rgKxanpmCllXnYImYVuHLMTR3qo1ZJHDyfRXLRNLAq4HXR67///W9kWWbx4sXUq1evSvfXsXVqFGUeAh9xbk54Nd9AE1wvcV2cUV1xFlHtyBCua1GHjcfTWL3/Ak8Nbllh1/InXgvRgQMH2LNnD61ataqI8VQqthC+8BEJfMTZInJX5mEwWThxOQfwf+jemZGdExUhOnCBJwe1qBLGgtdTsx49enDu3LmKGEulYyvzEFMzgY84W0SZbrKrT1zOwWiWiQnT0iA2rELHNLhtAqFaFSnpeRwsKrANdry2iD7++GMefvhhzp8/T/v27Uv0qe3YsaPfBlfRiAp8QXkp4SPKc20R2ScyVrSFEhmiYVCbevz450VW7b9Ax4axFXo9f+C1EKWlpZGcnMz48eNt2yRJsrVcNZurTk6OKPMQlJcSFpEbZ/WhIke1P1p/eMLIzg348c+L/PfABZ69qQ1qL/ppBwKvhei+++6jS5cuLF++vBo4q0WZh6B8lIyalW4Rta3AiJk9/VrWJSZMS2qOnp2nr9DnmjqVcl1f8VqI/vrrL1avXl0tFtizTs3Sc/WYLXLQ/2oIgg/n/EVX4XuzRebIRf91ZfQEnUbFTR0SWP7HOVbtvxD0QuS1s3rgwIEcOHCgIsZS6dSK0CFJyofpSp6wigTe40nU7HRaLoVGC+E6NU1ruw7tVwQjOikV+WsOXURvCm6XidcW0fDhw3nqqac4ePAgHTp0KOGsHjFihN8GV9Fo1CpqR4SQnqsnNVtfLZbuFVQM6bl6fvrzIrd1beCw2ILzKqeuLKLDRYWubetHe7X2WXnp1bQWkSEacgpNnMvI55r4qEq7trd4LUQPP/wwAC+99FKJ57x1Vi9YsIAFCxZw5swZANq1a8fMmTMZNmyYt8PymfgoRYiEw1pQGpO/PsDmE2lsOpHG4nE9bNudp2auLCJbj+pKmpZZUakkYsK05OpN5OmD2yLyempmsVjc3ryNmDVs2JA5c+awZ88edu/ezcCBAxk5ciSHDx/2dlg+Y4ucCYe1oBQ2n0gD4LdjjgtllpyalbSIDlWyo9qecJ2yhFWeoewWJYHEKyEyGo1oNBoOHTrkl4sPHz6cm266iRYtWtCyZUteffVVIiMj2bFjh1/O7wkil0jgCXUidS63WyzK35CiTg55BjMGk8X2vCzLtqlZZYXu7QkPUSY91coi0mq1NGrUqEJyhcxmMytWrCAvL4/evXu73Eev1/u92l+UeQg8oVGtcNt9e7+Q1SKKDtNidf/YW0XnMgrIKTShU6toUa98iyn6QkSRRZRfnSwigOeee45nn32WjIwMvwzg4MGDREZGEhISwsMPP8z3339P27ZtXe47e/ZsYmJibLekpKRyX1+UeQg8IdGuLMP+R8uqSRqVRGy4YjXZ5xJZ84daJUQFZBXWiCpiEXntrH7vvfc4deoUiYmJNG7cmIgIx3Dk3r17vTpfq1at2L9/P1lZWXz77bfce++9bNq0yaUYzZgxg8mTJ9seZ2dnl1uMbBX4wlktKAWNXbTrdFoe9aIVS1pGUSKVJBEbpiUjz+AQOTtUQT2qPaWqWEReC9Gtt97q1wHodDpbcmS3bt3YtWsX//d//8eiRYtK7BsSEkJISIhfr2/rSSR8RIJSMNv5pM9cyaN389pAcdRMkiC2aHkg+6nZIVsPosr3D0EF+YhMepBUoPbPckjggxDNmjXLbxd3hcViQa+vPOvEvszDWi8nEDhj3wI2JT2veLtcbBHFFU3NrCF8xVHt31VdvSXC31EzWYbVj0HORRj1BYTF+uW0XguRlT179nD06FFAyf/p0qWL1+eYMWMGw4YNo1GjRuTk5LBs2TI2btzIL7/84uuwvMbqI9KbLGQXmogJ85/KC6oPZjshOp1WLESyTYgo4SNKzdGTnmtArZJoUz9AQmSziPwkRJtehz+/AkkNlw5C0+v8clqvhSg1NZU777yTjRs3EhsbC0BmZiYDBgxgxYoV1K1b16tz3XPPPVy8eJGYmBg6duzIL7/8wuDBg70dls+EatVEhSrZp2k5hUKIBC4x20XK/rpibxEpfxWLyHFqZk1kvKZuJKFadSWN1JEInfIVzzf4YWp24CvYOFu5f8vbfhMh8EGIHnvsMXJycjh8+DBt2rQB4MiRI9x77708/vjjLF++3ONzffLJJ95evkKIjwohp9BEarY+qNPgBZVPVoGRrafSKbD7Iuvt8oSsUzZJgrgIq0WkCJE1fyhQ0zKA8JCiqVl5LaK/tsHqScr9vk9At3HlO58TXgvR2rVr+fXXX20iBNC2bVvef/99brzxRr8OrrKIjwolOS1PRM4EJViwMZmFm5Idtlkc8oiUvypJsjmrrVMzq0VUEYspeorVIiqXj+hKMqy4C8wGaDMCbnjBP4Ozw6cSD+dCV1CSHS0Wi4sjgh+xmofAHRcyC0pss6/qkGX78L3VWe1oEbUPoEVU7jyi/AxY+k8ouAoNusFti0Dl/3won9qAPPHEE1y4ULxu0vnz53nqqae44YYb/Dq4ykKUeQjcketiSuPKIpIkbD6iq/lGruYZOF8kYoGoMbNSrjwikx5WjIWM0xDTCMasAF142cf5gNdC9N5775GdnU2TJk1o3rw5zZs3p2nTpmRnZzN//vyKGGOFI8o8BO7IdbFgor1FZBUlSSrOrM7MN9isoSa1w4kKDVwAxOc8IlmGVZPg7DYIiYaxX0NkfAWMUMFrH1FSUhJ79+7l119/5dixYwC0adOGQYMG+X1wlYUo8xC4o2yLqDh8HxdhjZoZK2UNM0/wOY9o4xw4+DWoNDDqc4hvU/Yx5cCnPCJJkhg8eHClhtkrElHmIXCHayEqvi87hO8Vi8hkkdl5+goA7QJU2mHF6iPK98YiOvAVbJqj3L/5bWg+oAJG5ohPQrR+/XrWr19PampqCQf14sWL/TKwyqTYWS18RAJHXIW97avvi2vNlJy0UK2KQqOFnSlKUXggWn/YY42aGcwWDCYLOk0Z3pgzW2HVROV+3yeh270VO8AivPYRvfjii9x4442sX7+e9PR0rl696nCritQt8hFlF5ooNAZ3lbKgcskpa2pW9DtsLQ2yRs6sCYSBzCECCNMVJ1IWlJXUmH4KvhoLFiO0HQk3VGw5lz1eW0QLFy7k008/5e67766I8QSE6FANIRoVepOFtBw9SbUqJjIgqFoYTBaHJmdWLC6c1dbi/NhwLZeKLOv6MaHUjvRvkba36DQqdGoVBrOFPIOJmHA3jvO8K7DsX0Vh+u4VFqZ3h9dXMhgM9OnTpyLGEjAkSSp2WIsQvqAId9nIspuERsDmJ4LAO6qtRJSVXW3SK5ZQxmmIbQRjloO2YpfFdsZrIbr//vtZtmxZRYwloNgc1iKELyjClaMa3Cc0QnHkDAI/LbMSbsuudjE1k2XFJ3R2O4TEwF3fVGiY3h1eT80KCwv58MMP+fXXX+nYsWOJLOu3337bb4OrTEQukcAZd0LkLqERiivwofJX7XCH1SLKd/V6Ns6Gg98Uhek/g/jWlTw6Ba+F6M8//6Rz584AJZroV+VePqLMQ+CMeyGyv+9kEdn5YALVldEZtxbR/uVKWw+AW96plDC9O7wWog0bNlTEOAKOKPMQOOOZRVQkREVODmvUrFaEjoTo4FiwM9JVT6IzW5QGZwD/eAq63hOAkRVT+d28gxQxNRM446q8A8B+JTOrJkkoFpG1yX7npNigmSGUWNss/aRSQ2YxQttbYeDMwA2uCJ87NFY3RJmHwBn3zuqSFpFVcwa1jefV29pz3TWeNwisaByyq/OuwNJ/QWFmUZh+oVdh+iu5epb/cZY/zlzls/E9/Ca2QoiKqCvKPAROuAt3O/qIlL9WH1GIRs3YXo0remheYbWICgvzYMUjcDWlKEy/wuMw/ZEL2SzZmsKqAxdsuVXbk6/Q55o6fhmjEKIirM7qK7l6zBYZtSo4zGpB4MhxMzVzV/QarCg+IpmBx16CjB12YfrSrTazRWbdkUss2XrGVrIC0KlhDOP7NqV7k1p+G6PXQrR582b69OmDRuN4qMlkYtu2bVx//fV+G1xlUjsiBJWk/MJdydUTHySORkHgcJ/QSPGKL04WUTASrtPwlOZb2mX8TwnTj/681DB9Vr6Rr3af5bNtf9l6KqlVEsPaJzC+b1O6NvK//8trIRowYAAXL14kPt4x6SkrK4sBAwZUyHLUfmH3YrhmMMS6XpBRrZKoExlCao6e1BwhRDWZU6m5PP/DQS5muY+gyrLiF7LvRxSsdMpYQ3/N98qDW+ZBs/4u9zuVmsun21JYuec8BUU1l3HhWu7q1Yh/X9uY+jEVl23ttRC5W/vrypUrJVZ9DRq2vgvr/gPx7eC+tRDqOr+jbpRViAqB4EhGc0VWgRGT2RLwOqbqyv2f7eLMlfxS97FOzop9RBU7Jp9J+Z3rjr4EwJrYMdzU1bFG1GKR2XQijcVbU/j9ZLpte+uEKMb3bcLIzg0qZQUSj4Xo9ttvBxTlHzdunMOKq2azmT///DN4a9Da3Qbb34PUw/DNOLjra1CXfOnxUSEcJvjLPDq9+D8Ajrw0xJasJvAfZYkQKJaQGqlEQmNQkX4Svvo3atnEj+ZerIi4h5uKnsrVm1i5528+23aG00ULRkoSDG5Tj/F9m3Jts1qVauV5/CmOiVEsBFmWiYqKIiys2EzT6XRce+21PPDAA/4foT+ITYK7voIlN0Hyevj5GaXhk9Mbbb/qa7Biv+Lo+asFtKgnlj8KBFYBkp0SGoOGvHSl6X1hJpm1O/P0+UdobVDG+v6GUyzcmGxrcRIVqmF09yTu7dMkYJ0nPBaiJUuWANCkSROmTJkSvNMwdyR2gTs+VhK5di+GWs2hzySHXWxlHkFsEZnshEgVtPOB6o81cFZcaxZE/wtjobL8z9UzENuYkwMWof8ymXy9iat5Bt785TgAzepEMK5vE+7o2tCWaxQovNbxWbNmVT0RstL6ZhjyqnL/f8/D0R8dnq4KZR72Sx+rg+nDX8OwWkRBNzWzWGDVo3BupxKmH/sNupgEQGnWduSi0tQ/qVYYv07uxz29mwRchMAHIbp8+TJ33303iYmJaDQa1Gq1wy3oufZR6D4BkGHl/XB+r+2pulWgzMNk15pX5DoFDouTRRQ0/4qNr8GhlUVh+i+gbqvifkQGE0dsa63FBJVF7bUUjhs3jrNnz/Kf//yH+vXrB5dJ6gmSBMPegMy/4NSvsPxOuH89xCYVZ1cHsRCZxdQsKJCdfERB8Z/Yvww2v6ncH/5/0KwfYL/IoslmEbWtHxydAax4LURbtmzh999/t7UCqZKoNfDPJbBkGFw+BMtGwX2/2E3N9G7TFAKNvY8o+EZXcyi2iIJkapbyO6x+XLl/3dPQ5d+2p6yRVaNZZv+5TCDwq4s44/XULCkpyaHor8oSGq1E0iITIPUIfHMvdcOVt8NgspBdUI61wisQe4uoGvwXggJfPs+yzUekPA7oj1baieKm9+1uhwHPOzwdbtdAP6UoVN+2fnDlyXktRPPmzWP69OmcOXOmAoZTycQ0hLtWgDYckn8jdN10YkKVX49gdVjbW0TV4gchwJy8nEPP19bzxfYzAC6b5buipEVUEaPzgLx0pel9YRY07Am3flAil0CrVjksI1QrQke96OBKhvVaiEaPHs3GjRtp3rw5UVFR1KpVy+FW5bCG9ZFgzxIeDfkZCF6HtdlsL0QBHEg14dnvD5KWo+c/qw4DkFNo9Oi44jwi5XFApmb2Yfq4JqU2vY+0i4y1rR8ddG4Hr31E8+bNq4BhBJjWN8OQ1+CXGTyg/5S9qlhSczoFelQuMQv18Sv2Fia4r7h3JuAJjRYL/PCIEqYPLaqmj3DfkiNcpyZDmZUFTVN/e7wWonvvrZyVHyudax+BjGRUuz5mnvZ91p7vCl0aBnpUJTDbhe8tQpTKjfNb6KkQYZuaKX8r3cLY8Coc/q4oTP8l1G1Z6u4RdqVAbYNQiHzS8eTkZJ5//nnGjBlDamoqAD///DOHDx/26+AqFUmCoa9zKqY3YZKBwfufgMyzgR5VCRx9RAEcSBVlz18Z/HPBNg7+neXy+ewypmaaImdQQH1E+5bC73OV+8PfhaZlt94JDyl2WAdb6B58EKJNmzbRoUMHdu7cyXfffUdubi4ABw4cYNasyluitkJQa9ja+U2OWhoRacqApaMUJ2AQYTKLqFl5uGPBdnb/dZUxH+1w+XxpPiKVVJy7ZXGKmlWajyhlM/zXGqafAl3GenSY1UcUolHRtE7wVUZ4LUTTp0/nlVdeYd26deh0xWs4DRw4kB07XP9z3TF79mx69OhBVFQU8fHx3HrrrRw/ftzbIfmV2LhajDc8Q4aqFqQdha/vBbNnDszKwCyiZn7BXT/q7FKmZmqVZMvdKuEjqgwhSjsBX/0bLKaiMP1zHh9qDeG3rh+NRh1sFbo+CNHBgwe57bbbSmyPj48nPT3dxRHu2bRpExMnTmTHjh2sW7cOo9HIjTfeSF5enrfD8hvxUaFcojYzQp9XwvqnN8CaKUEzDzKJPKIKpTQfkUqSbIIjO03NKlyHbNX0WZDUC25d4JWH3OojCsZpGfggRLGxsVy8eLHE9n379tGgQQOvzrV27VrGjRtHu3bt6NSpE59++ilnz55lz5493g7Lb1jLPLblN4Q7PkEJ638K2+YHbEz2mIWPqEJxnppp7Jw/apVk8wVV6tTMWAjLxyhlSXFN4M5loPWug2ivZrXQqVXc2K5exYyxnHgtRHfeeSfTpk3j0qVLSJKExWJh69atTJkyhXvuKd8ibVlZij/GXT6SXq8nOzvb4eZvrK1AcgpNFDYfAkNnK0+smwlHVvv9et5iX/Qqpmb+xzmjPiq0ONpUqkVUUQOyhun//sOjML07RvdoxOGXhjCgVeWva+8JXgvRa6+9RuvWrUlKSiI3N5e2bdty/fXX06dPH55//vmyT+AGi8XCk08+Sd++fWnfvr3LfWbPnk1MTIztlpTkuv90eYgK0RCqVd6W1Gw99HoYejwAyPDdg/B34Kw1ECUe/sb5PSw0OfZcjwotXkJaJRVPwUokNFZU2GzDK0Vhei2MXlpmmL40tEHoG7Li9ch0Oh0fffQRycnJ/Pjjj3z55ZccO3aML774olxtQCZOnMihQ4dYsWKF231mzJhBVlaW7Xbu3Dmfr+cOSZLsVn0tLArrz4EWN4KpAJaPhqt/+f26niKmZhWLfeY6OFpEapVkFzVTtlk7ZlbIzGzfl/D7W8r9Ee9C0+sq4CLBgc8dkRo1akSjRo38MohJkybx448/snnzZho2dJ9EGBIS4tAru6KIjwrhbEZ+cZmHWgP/XAyLh8Hlg0q1/oT/KaZyJeNoEQkl8jfOmdbOQmR9/52LXv3uIzq9Cf77hHL/+meg813+PX+Q4ZEQTZ48mZdffpmIiAgmT55c6r5vv/22xxeXZZnHHnuM77//no0bN9K0aVOPj61IXPYlColSqvU/vgHSjilh/bHfgFrr5iwVg0horFjsM9cBIkPsp2YSsm1qZv1bAQmNacfhq7uVMH37O7wK01dVPBKiffv2YTQabffd4W2a+8SJE1m2bBmrVq0iKiqKS5cuAUqjfvvm/JWN25axMQ2UZXqX3KSE9X96WmlAVYnp/fYWkSjx8D9OMzOinSwi69vvbI36zSLKTVPWptcXhelHflCpn69A4ZEQbdiwweX98rJgwQIA+vfv77B9yZIljBs3zm/X8Rbr4oouV/NI7Az//ESpet77GdRuDn2fqLSxCYvId/6+WvYyQc4WkX1phBI1K5qSFe3m1wUWjQWwwhqmb+pTmL6qEtCu2cEafq4bVcZqHq2GwZDZsHaaEtaPawJtR1bK2Jy/KALPKDCY+cfrLn5EnT6DJieTyD7SpPiInEs8/DQ1s4Xpd0ForDLt9yFMX1XxSIisiyt6wnfffefzYIKF+LKECODahyEjGf74UAnrRzeAht0rfGwm0Y/IJzLyDR7t5zzddRYiq+A4LydU7qnZby/D4e+LwvRfQp0W5TtfFcOj8L197k50dDTr169n9+7dtuf37NnD+vXrbYswVnWs4fu0sro0DpkNLYaAqVBpwl8JYX0RNfMNt0svOW13jprZZ1YreUSOFpHsD4to7xewpSjIM2J+tQ7Tu8Mji8i6uCLAtGnTGDVqFAsXLrTlDZnNZh599FGio4OzjsVbrFOzK3kGTGaL+yJBa1h/yVC4dNDWhJ+w2Aobm/AR+YZboXB6E83OQuRkETknNFpnyj77iE5vhB+fVO5fPxU6j/HtPFUcrxMaFy9ezJQpUxySF9VqNZMnT2bx4sV+HVygqB2hQ62SkGVFjEolJBLGfAVR9ZWw/jfuq/VPXs7halnnKwMRNfMNT4WihI/IwSKyK/Eo2lauotfUY/DVPUVh+n/CgGd9OEn1wGshMplMHDt2rMT2Y8eOYakmjlSVSqJOpNLixGXkzJmYBkqOkTZC+YX7aXKJX9qvd59j8DubeeDz3a7P4SGixMM3PF2M0rkVr71FpAiRcr/cCY25qUrTe30WJF0LI9+vEWF6d3gdNRs/fjwTJkwgOTmZnj17ArBz507mzJnD+PHj/T7AQBEfFcrlbH1RLpEHvq/6nZRp2ooxsPdzqNUc/vEkAL8du8yM7w4CsPfsVQoMZsJ0vpXDiBIP78jTm1i59296Na3t0f7OUzOt2rn63rHEwycfkbGgqJr+bI0L07vDayGaO3cuCQkJvPXWW7Z2IPXr1+eZZ57h6aef9vsAA4VHkTNnWg1V6tJ+ngq/zoK4JuyN6sejS/faPuAWGY5dyqZLozifxuXoTBVKVBYzVx1m5d6/qR2hc/m88ztYqrPa3kdk+3962RjNYoHvH4bzu4vC9N9ChGciWZ3xWohUKhVTp05l6tSptjYc1cVJbY/Py0/3egiuJMMfi7B89xBvW2ZSaGzCgFZ1MZgtbD11hcMXfBcis0MbEJ9OUaP46eAFwANfXxHOeVpau/XA1A5RMxz+euys/u0lOPKDEqa/cynUucaz46o55eoLEB0dXS1FCEop8/CEobPRNxuMylzIO5Y5DE4s5P2xXenUMBaAwxd874NtcnBW+3yaGkOhsXS/pbN8mJ1216rc5RH5kNC493PY8o5yf8R8aPIPDw6qGfiUWf3tt9/y9ddfc/bsWQwGx1+avXv3+mVggaZuaWUeZZClt3DvlQd4xXKS9qozLJTmoDbfQPsGiq/p0HnfG7p527M6V28iLUcflA3TgwHnd9DZIrI3dFxFzWS750oleQP8+JRyv9+0Ghumd4fXFtG7777L+PHjqVevHvv27aNnz57Url2b06dPM2zYsIoYY0DwyUcEFBrNPPj5bvZfNjFV9yymiPqor5yAr++hfb1wAI5fysHo/NPrId72rL7+jQ0MmLuRoxf9382yOuLsI7Kfcil5RD4kNKYWdWuwmKDDv6D/DP8OuhrgtRB98MEHfPjhh8yfPx+dTsfUqVNZt24djz/+uK3Va3Ug3gcfkdkiM/nr/exMySAqRMPc+4ah+ffXSlg/ZRNJ258jKlSNwWzh5OVcn8blbdQso8g3svlEmk/Xq2lYnIXI7r5jz2rr/kX7ubOI7MP0jXrX+DC9O7wWorNnz9KnTx8AwsLCyMnJAeDuu+9m+fLl/h1dALFW4Kfl6D2aAsmyzIv/Pcyag5fQqVUsuqebsqJm/Y7wryUgqZD2fcEToWsAuOph7ZMzjuuaee4k0mmCt01oMFHSIiq+r5JcZFaXFjWzD9PXaqa0etVUfGO/qojXn86EhAQyMjIApUujdS2zlJSUoK2m9wVrQqPBbCGroOx1zT7YmMzn2/9CkuDt0Z3o09yucrrlEBj6OgD3F37GTaodHke8dp/J4N7Ff3A6TbGgHHwYXrzdwdyvOJhwziOy1xf7PKKSCY1OJ7JY4PuHlDB9WFxR03sRpneH15/OgQMHsnq1sprF+PHjeeqppxg8eDCjR492ud5ZVSVEoyY2XOnOV5af6Ovd53jzF2VhyFm3tOWWjokld+r1oNKIH3hbu4DIdM+c+v9cuJ1NJ9J4+Eulab995q83UbOabhGFe5hAWsIispucORS9Fv0euF1gcf2LcGRVcdN7EaYvFa+jZh9++KGtlGPixInUrl2bbdu2MWLECB566CG/DzCQxEeFkJlvJDVbT8t6US73WX+0OGv6kf7NGde3lHa3Q15jx969XGv8g7YbH4LWG5ReRh7w99UCwLvqe3sLNaTGC5GGfIO5xHZny7SEj6hE1KzoOOv+1hPY69Cez2DrPOX+yPehSV+fx11T8OrTaTKZeOWVV2wtXUFZ5+zdd9/lsccec1iCujpQr8hPdD7TdWe/vWevMnGZkjV9R9eGTB3SqvQTqtS8EfEMhyxN0OkzlJagBZkejcX6efemH5H9r7uuhk/NIkJ8s4jscSzxcFNrlvybXZh+OnQa7duAaxhefTo1Gg1vvPEGJpP7ZXmrE9a8n50pGSWeO5Way32f7qLQaKF/q7rMuaODR9m1BnUYEwxTKAyrB+kn4Ou7wVS249r6wfem6NVgKvYn1cSpmckuRSJc55nxX9JH5FjiUWpCY+pRJUwvm6HjaOg/vRyjr1l4/em84YYb2LRpU0WMJejoW+Rw3nbqisM053J2Ifcu/oPMfCOdGsbwwdiuHjuDm9WJ5DK1+LrlW6CLhJTN8NNTHtdrmLxIaNTbCVFNdFbn6YunYp74iGRZdlhJF5zC95Jk8xkVF70qf8P0V2DpKNBnQ6M+Sua0CNN7jNc+omHDhjF9+nQOHjxIt27diIhwzNgdMWKE3wYXaLo3iUOnUXEpu5DT6Xk0rxtJVoGRexf/wfnMAprWiWDxuB4e/9oCXNeiDqsPXGDlhVrc888lyoKN+75UqvWvc79Uk1VyvMkjsreIauJ3ItdQbLlrPKjBsMjFTmgrzlEzV+H7UPRc+8dEyDqr/B/vFGF6b/FaiB599FHA9fplkiRhNpd0CFZVQrVqujWKY/vpK2w7lU6D2DAe/Hw3xy7lUDcqhM/v60ntSO8+cNe1qAvAn39nktlwMLHD3oA1U5QoS1wTaF96f3D7X+yynNV6U/X5X/hCnr5YiDyxN80WVxaRm8ZoVovIYuZt7QLirh5UwvRjv4HwWuUee03Da3vdYrG4vVUnEbLyjxbK9GzzyXRb1nRkiIZPx/cgqVa41+dLiAmlZb1IZBm2nroCPR+AaxVx5/uH4dwfrg8s+uB7YxHZT82qUYqXx+TaCZEnSmSR5RIpEY4WEVhrYK0W0Z3ZS7hJ/QdmlVbpK1S7eTlHXTOpeY4DL+nTXElCW3fksi1r+sN7utEu0YNmaW6wWkW/n1TKLjY0epzf1T3ArFcycTNSShxjtX686VltPzWrgTrkZBG5fgfst7uq/1M5Tc0cLKI9nzIy7xsADnV7FRr38cOoayYeT80KCgpYv349t9xyCwAzZsxAry9O9FOr1bz88suEhlavTnMdGsQQFaIhR29ynTXtA9e1qMMnW1L4/WQ6siwz/vO9hPMIX4ek0z4/RWnCP+F/iqlfhOzKIirjOvZTs+qU9e4pDkLkwcu3F+5iHKdm1ihanctbYafi03vHeAdtGo+kU7lGW7Px2CL67LPPWLRoke3xe++9x7Zt29i3bx/79u3jyy+/tK3cWp3QqFXc2C4BgJnusqa9pFfT2ujUKs5nFnA6PQ+AfEK5Tz9FWR8tXanWtw/ru3ZWex41M5plUrOV3kqHzmex56+SKQnVjVy7qJknMmx0Xm8aF85qoKV0jl67nwLZzMaQAfyf+fbyL7BYw/FYiJYuXcqDDz7osG3ZsmVs2LCBDRs28Oabb/L111/7fYDBwGu3t+f3qQMYX1rWtBeE6dT0aKpYO1tOptu2pxKnNOG3hvV/fArnr5A3jdHsheiBz3fT87X1HLuUzS3zt3DHgu1k+lh4W1VwtIjKliJXUzN7fVFJErGWDBbr3kRryoXGfXkv6klA8r55vsABj4Xo1KlTdOjQwfY4NDQUlV33up49e3LkyBH/ji5ICNGofXJMl4azn8hGQgf4p1Ktz/4veVSt1PXJLhIay/qd17voTvjfAxds9zPKubRRsJPrZdTM4EqI7ARGJ+t5InUmDaV0ciIaw+gvMRZ5N1TC21ouPH77MjMzHXxCaWlpNGnSxPbYYrE4PC8oneuKonHbk6+UfLLljTDsDQCmar/iFtV2ZGDtoYvs+euqbbcyndUuna/FXyyfFwWsInjiI7LfXppFJGHh9jMv0sxwnKtyJJu6vw/htYp7VpdoOivwBo+FqGHDhhw6dMjt83/++ScNGzb0y6BqAm0SoqkTqSPPqRDTNoXo+QBcOxGAt7QL6cIJPv7dMZpWprPaWHo6RXX3a3iSR2RvYBpN7n1E0zQraH11I0a0PGiYTHZ446LjZYf9BL7hsRDddNNNzJw5k8LCks3kCwoKePHFF7n55pv9OrjqjEol8Y9rSkbfHCrEb3yZdeZuhEhGPtK9RepZx4UtrZp1/FIOU745wLkMx+JcvYsokLPPozpj76x2ZxLZV9u7nprBGPV6Htb8CMAX8c+wS27tvuhV4BMeC9Gzzz5LRkYGrVq14s0332TVqlWsWrWKN954g1atWnH16lWefbbmLpnrC1Y/kT0OfhuVmseNEzloaUJtKYcl2jeIprjFrEWWWXPwIkPmbebbPX/zyNI9DudyFY6u7tMxezyziErPI6p1cQsva5YAsDXpQXZFD1LOV6Jndc15XysCj4WoXr16bNu2jTZt2jB9+nRuu+02brvtNmbMmEHbtm3ZsmUL9erVq8ixVjusfiJ70nKL/WwWi0wBoUwwPMMFuRbNVRdZqJ2HFuULJgOr9xc7n0849cF2aRFJru9XR/IMZfuIzKUIUUvpHO23PoZGsrDS/A/+SLq/xEqvXi0nJHCLV77+pk2bsnbtWtLS0tixYwc7duwgLS2NtWvX0qxZs4oaY7UlPjqU1gmODdd2nC52XhuL6p5SieM+w1Ry5VD6qI/wmuZjQEaWZdLthCvEqcLepUVE1XdW601mXv7xSMmIoxO5nmRWu3FW1yWTxbo30Zjy2GlpzQzjA6jVKpt4O7eKrarvZbDgU9CxVq1a9OzZk549e1KrlijwKw/OVtGagxdt9+2boB2TGzHJ+DhmWeJfms08ql4F4CBEzj2HXBW9Vofvy+fb/uKTLSnc/YmburwiPIma2U/NDEXO6lD0fKSbS0MpnfyopjxomIwBrVNjNMfjhUVUPkT2Q4AZ2FqZzjavG4FaJXHofDZnryhOZ+epwu90YZZpHABTtV+TeG4NV3KLfUolhaj02qmqWvZx7qrrjpnOeNLN0j4vy2i2IGFhnvYDOqtOkyFHcqj/R2QRCTi2irUJmNVZLZSoXAghCjC9m9fmw7u7sWRcT65tpliXPxVZRc5RnGZ1IvjSPJiPTDcB0HXfs7QwFCeROjc/K8tZXUV1yOOMHXtrx91LdZ6aTdcsZ6h6F3pZw4OGyeijm9ieV6uK3z9ZWER+JaBCtHnzZoYPH05iYiKSJPHDDz8EcjgB48Z2CTSqHc5NHeoDxdMz59onq8Uz23QX/zN3Q20x8JHuLRpJlx2et1JWPyJnIcopNDJ60Xa+2H7G15dSKXjqj3HIQXejuvYWUaPTK3hI8xMAzxgfZrfcukQ/opKN0bwbk8A1ARWivLw8OnXqxPvvvx/IYQQNQ9oloJLg4PkszmXkY3SyaKxdBi2oeMI4kfNhrRzC+s4N8l1bRMX3nR24n2xJYWdKBv9ZddhPr6hi8DRUbvHA5LPuc73qAF0PvwbAW8Z/strSp+hajtd17yMSQlQeAipEw4YN45VXXqlW66GVhzqRIVzbTOl/tObgxRI+IrXdt6KAUN6t9wrn5do0V11kkXYe4WpHC8h1QqP7qVluYdVYFMHT77zDWpSlOKtbSWd5X/suKtnMSvN1zDfbfR5L9CMqOh/WPKISuwl8oEr5iPR6PdnZ2Q636ob99MzZRxTm1AD+dGEkEwzPkCOH0Vt9hEm58x2+cWU5q62/5huOpfL8DwddZhYHI7586d2F72tZrvKJbi5RUgEXYrsx3fiAwxUcpmbOjdEQFpG/qFJCNHv2bGJiYmy3pKSkQA/J71inZwf+zuJMenF0aFCbelzb1HHJ4tQcvS2sb5JV9C/8FX6fa3vedaOvYqxfzfGf7uLLHWf5fPtffnsdFYmnESoHZ7UrHTLk8bZlDg2ldJIt9fmpzRu2anorDv2I7BqjWUtDRK2Zf6hSQjRjxgyysrJst3PnzgV6SH6nblQIPZsq0bNV+88D0KR2OB/f273EFzCtaCnsTZZOvGC6V9n42ytw8FvAtbN6/7lM2/2aHDWLCVXBdw/SnmQy5EjGG6eSo4ou9VpK1Mx6bse/wiIqH16v4hFIQkJCCAmp/su03NyhPjtOZ7DxuJI57G5NMvsC2S/Ng+lXJ5fBWd/AD49ATEOXU7OfD12ye+ReiWRZDt5IUBnDevG/h4kL1zkIrXPU7DntCjj2IwZZwwOGpzkr13PdBsR+gUUXeUS2WrMq9ZMefIi3LwgZ0j4BSSrOI7IKUVkJiP+t9zC0vgXMBlg+htr6v0vdv7QOj498ude7QVcipfX+SUnPY8nWM7y97oTb/t5j1b8yyvgDADPkR9kjK0uFO0cpoWSr2GIfkai+9ycBFaLc3Fz279/P/v37AUhJSWH//v2cPXs2kMMKOPFRofRsUlw6o1U7hozdodVo4fYPIbELFGTwYu6LxJDrdv/SdG3t4UvunwwwpbmI7P1iDj6yotfaT3WAFzWfKg8GPM+PluKVN8pqFatWSUSHagH4O7MAEAmN/iKgQrR79266dOlCly5dAJg8eTJdunRh5syZgRxWUHBzx/q2+1aLqKy8GI1K4lCaie9avYUc3ZBG8gUW6d6xVes7U9YCjcGKyiE73PE12M9i9U7LKbWSzvKe9l00koWf1QPg+imOtWZlNM9XSRK9i5aX2nIynVy9icKi5nPCIiofARWi/v37I8tyidunn34ayGEFBUOLpmdgPzUr/RgZmVvmb2Hyz5fY1P19cgnjWtVR5mg/wpU/yHl55aqC/Xe+x6vrWbAx2fbYXhDs0xHizFdYrHuTKKmA7ea2vKF7BCTJwcr8apcrS7z4fGqVRLfGcYRqVaTm6Hl8+T4KjRaSaoX5vad5TUP4iIKU+KhQehRNz7Qaz3xE9k9vz63H0/JTmGQVd6h/Z5L6hxL7P7Z8r0OHwqqCve2Rnqvn9bXFnStdvZowCnlN/yoNpCskW+rzsPFJDLIyxTKXsSqKo0WkLEPeqyiN4rdjqQC8NKK924CCwDPEuxfEjOysrKFWN1KJFJalGfbPmy0yG8wdmWkaD8AU7TeMUG112D85LY/tp1007w92SpkGmZ3eJBUW/k/7Pq0tyVyRoxhvnGqrpvek+4Cr1rr2rVuGtktgQOt4LwYvcIUQoiBmTI9GvDumC88MUaI6ZfmI7L9YJouMwWRhmfkGFpmUXuJvahfRXXLse23fs8eZPX9ddenADTSlOYZNTn6eGZpl3Kjeg76o6f1ZWWm7YpHlEqLlCvvwvbXEpn+reCQJwrRqZg5v68MrEDgjhCiIUakkRnRKJCFGWca7rO+N0W6Hn+warM0xjWGtuQchkokPdW/TWCqOiJV2zjsWbOOF1cFXAFta+N5erP+tXscDmjUAvKp9zBamt+7nyay0YVwYOrUKlQTN6iqW1DXxkXw6viffPNybxNgwH1+FwB4hRFWIMvOI7BZPtGZdA8ioeNL4KPstzagl5bJY+6ZdWL/0cy7dGXypFKUFqKwr4fZX7beF6d80jmK9+jqH/WTZs+r8OpEh7Hj2BnY+O4imdSJs2/u1rEv7BjHeD17gEiFEVQhPvjjuKCSEBwxT+FuuQ3PVRT7UvY0Oo8uQdbBT2tTMbLHQuihMr5ZkvjFdz/vmkSX2s5QiRC3rRdIwLowZw1oDUCtCR92o6p/RH0iEEFUhylsblkYs9xmeIVsOo5fqGLO1H1GgN5Z53Kr95/nz78zyXdyPlFZ6osq9zGLdG0RKhWwzt+VZ0/2A5EJ0HKdmm57pb7sfptOwZdpAHurX3K/jFrhHCFEVwh+R9hNyEhONTxSF9bfQ4vjCMo95YsV+Rry3tcT2q3kG5v16wtZju7Jwq0OGPFqsn0CilGEL0ztX01uxyI4Rtvoxxb6e7IKyxVngX4QQVSHKMzWz53dLR/5TFNbvmvwBI1VbfDrP1JV/Mu/Xk9z6QUmRqkhcOqstZlh5P5EZh7kiRzHOOJXsojA9lLQmrcmzVuybzmXmGxBULkKIqhCunNW+VhYsN9/AQtMtALyh/ZAeTmF9T9hZlIPksDptJeDqNet/fhaOr8Gi0vGA4WnOyY6LfTqLuLNFZO93yhIWUaUjhKgK4Tw1iw7VlMtv9LrpTo7E9idEMrFI9zZNpItlHxQEODur/61eR8guZYp5uNfr7JVblnkOWZY5UzSllCRHv1MVTDav8gghqkI4/6o3qh3OyyPb+Xw+GRXLEp9zCOvHkuPF8YFh66nibPD+qn3F1fQD/8OFhje5PMZ5rNmFJu5YsA0ozph+qJ+yWvHEAcJJXdlUqcZoNR37X+qh7RJ49qY2NKodzs+HLrEt2bdSjSyzlgcMU/g+ZCbNVJdYpHuHuw0zMKD106jLx4nLOaz44xwxYVqyCoz0b1WXTSeUhnFtpL94TzsftSTztakfF/UjaOnGnCktB8tqYU0d0ppbOzegVb0ot/sKKgZhEVUh7L9MC+/uRqPaSsW3fcsQbykwmEqE9d1V66/af56/ruTZHldG44sh8zazeGsK7/x6gsVbU7hnsbLMdDxX+UT3pi1M/5xpAu+sP2lLaPQG6/pxapVEm/rRYtXWACCEqApRJ9J1Ut2o7knMH9PFoRjTU/L0Sj+dE3ISjxqfxCSruF29hcfV35fY94kV++n35kbb48qYmrkyZMIpZLHuTRKlDE5ZEh3C9O7qx3KKlkpSqyRiwrTEiwTFoEIIURXi4f7NublDfT4Y29Vhu1atYninRJrXjXRzpHvyjcV9r7dYOvC86T4AJmu/5VYfw/oViVJN/x7tVWdIl6MZb3zGIUxvXXDAGWuTtHCtmt+nDWDTMwOEGAURQoiqEJEhGt4f29W29pmr573lgN2qHgArzANZaBoOwOtuwvq7zmR4fR1fcOXXeU6zlMHqvehlpZreOUy/oWjBAXuub1m3+IEE0aFawnRq4sJ1fh+zwDeEEFUjIkP9E3t43TSaNeaetmp957D+vxZuJznNfS9sfzHv15MOj+9W/48Jmp8BmGx8xKMwfat6UdzZw/X6dzHhweGQFwghqlb4YhG5QkbFZOMj7Lc0J07KZYn2jRJh/eOXPA/z+4LJbOH/1hcLUX/VPl7QfAbAG8bR/GS51qPz3NKxPgNbxxNRtEquvRs6TghR0CCEqBoR5SeLCJRq/fuLqvWbqi6zSPcOOoozjiu6Wfx3+4p9PfZh+q9M/fnAPMLj89SLCSVUq+bGdgklnosNE1OzYEEIUTWia6M4v54vnRjGG6bawvqvaz/EGiu7kqe3RaLKgyzL7D17lZxCIyazhYnL9rJkawpTv/0TgHpksLgoTL/V3I7nTPfhTeJAbtEYR3RS2u7a15TFRQghChaEEFUjkmqF8393dvbrOU/KDW1h/dvUW3lSsxKA574/5Jfz/3TwIrd/sI0R721l7eFL/PTnRV787xFACdN/optLfSmDk5YGPGJ8EpOXObjWhQeua1GHsb0aMXHANbbnxNQseBBCVM2w7yLoL+zD+k9qvvNrWH/1fqWrZEp6nkP/bNdheu9fW7hW8Q1p1Cpeva0D91/XzPbcDW2UiFstYRkFHCFE1YxmdrlEnZJiHZ6zb/7lLSvMA1lQFNZ/Q7uIntJRn8/ljmkrD9ruP6/5ksHqvRTKWh4wPM3fsm8rZYQXOaldcU18JBum9GfDlP4+nVvgP4QQVTMiQzTs/c9gDr84hFUT+zo817h2BGfm3My4Pk1oVCucd8d04eVb29MuMdqjc79hGs1P5p7oJDOLdO/Q1C6sn1PoW+sMV3nQ96h/4T7NWkAJ0++TW/h0boCwUoQIFAsyJkxM0QKNEKJqSK0IHRGlhPJfGNGOzVMHMKJTIndf25jbujTw6LxKWP9R9lmuIU7KZbH2DeLIBqDDC//zy9JDA1T7mKX5HIDXjXeyxsMw/RM3uBarMG3pQiQIDoQQCRzquZY90KvUffUUNR6z1C0R1r9azgZpbaUztqb3K0z9WWAe7vGx7vw8HRqKlTaqAkKIqjm3Fq0WO7ZXI7f7mO2UqE/zsgtn04lRnMdyOD1Vx3lDuwhfS2D3/nUVKA7TR0h6tpjbFTnHPQ/TG0yurbFwneh0UxUQQlTNmXNHR5bd34tZw903UPOlF/YpuSGPGJ/AKKu5Vb2Np4rC+t6QmW/gSp7BVk2fIF3lpKWBki7gQZj+8YHFoXhDEK5IK/AcIUTVnFCtmj7X1EGncf+vdvajOEfb3LHVLqz/hOY73p77okfryVuZ8d1BVFh4Vzufdqq/SPMgTN+mvuJYlySYfGPxyq16FxZRaREzQXAhhEjAnT0acW2zWjx/cxsAvnrQMwcxwFfmAXxgUkouXpIW8fmKpR4fu/30Ff6j+YJB6n1FYfopZYbpVz7Sm2lDW7N+cj+H7QaThZFF09DnbmrD/566nm3TB3o8FkFgkWRvfsKCjOzsbGJiYsjKyiI62rMQtMAzDv6dxetrj5GjN/H5+J50eul/bveVsDBfO59b1DvJlCNY33cpwwf2K9UKe+m/RzDvWMiLWqWQ9RHDE/xsKd1RDnBmzs0Oj5tM/wmAhf/uSq+mtTmZmkvPprU8eYmCSsDT76gQIoFHHPw7i+Hvuc+oDsHACt0rdFGd4oylHuv6LOWBoT1c7qs3mXlk5mt8pH0LtSQzx3gnCz0sZHUWopT0PA6ez2J4x/qlrgArCAyefkfF1EzgER0axpRaPqJHx/1FYf0mqst03vYoP+87U2K/cxn5zP7kK+YXVdMvNw1goRdhemea1olgRKdEIUJVHCFEAo9pW790q/OKXVi/h+oEhu8eKdF0+vnPf+HhC88SIen53dy+aMVZz0Skd7Pavg5dEOQEhRC9//77NGnShNDQUHr16sUff/wR6CEJXDBreNsy9zklN1Sa2ctqRqq3cebb5ygs6ou9YssRpmbMIkG6yglLAyYan/AoTL/o7m78q1tD3h7dqdyvQRCcBNxH9NVXX3HPPfewcOFCevXqxbx58/jmm284fvw48fGlR1CEjygwpGYX0vO19aXu8y/1Rt7UfgjAZMPDtBx0Hy02PMQN6n2kydHcZniZv+W6bo+vFaGjVoSO+/o25a5SkjEFwU2VcVb36tWLHj168N577wFgsVhISkriscceY/r06aUeK4QocBQazSz/46ytd5ArntGsYKJmNQZZTbKcSBvVOQplLXca/sN++Rq3x6198jpaJ4j/Z3XA0+9oQPPfDQYDe/bsYcaMGbZtKpWKQYMGsX379hL76/V69Hq97XF2dnaljFNQklCtmvF9mzK+b1Oy8o1czTfw6pqjrDty2bbPXNMoGkuXuUW9kzbSOQCeNE4sIULThrbmoeublViDXlBzCKgQpaenYzabqVfPcUmYevXqcexYyWVsZs+ezYsvvlhZwxN4SEy4lphwLR/d0x2AfIOJ5384xHd7z/O08RESpKt0V53gNeMY1lp6AkpxrSd1bYKaQZWqCJwxYwaTJ0+2Pc7OziYpyfVSMYLAEa7T8Paozrw9qjMAeQXD2HvsENM6deNZsZyzwAUBFaI6deqgVqu5fPmyw/bLly+TkFBy1YWQkBBCQsTqnFWNiLBQunbpHuhhCIKYgIbvdTod3bp1Y/364giMxWJh/fr19O7dO4AjEwgElUnAp2aTJ0/m3nvvpXv37vTs2ZN58+aRl5fH+PHjAz00gUBQSQRciEaPHk1aWhozZ87k0qVLdO7cmbVr15ZwYAsEgupLwPOIyoPIIxIIghtR9CoQCKoMQogEAkHAEUIkEAgCTsCd1eXB6t4SpR4CQXBi/W6W5Yqu0kKUk5MDILKrBYIgJycnh5gY92vMVemomcVi4cKFC0RFRQVFsaS15OTcuXMiimeHeF/cU93fG1mWycnJITExEZXKvSeoSltEKpWKhg0bBnoYJYiOjq6WH6ryIt4X91Tn96Y0S8iKcFYLBIKAI4RIIBAEHCFEfiQkJIRZs2aJDgFOiPfFPeK9UajSzmqBQFA9EBaRQCAIOEKIBAJBwBFCJBAIAo4QIoFAEHCEEJWTjIwMxo4dS3R0NLGxsUyYMIHc3NxSj/nwww/p378/0dHRSJJEZmZm5Qy2AvF2td5vvvmG1q1bExoaSocOHVizZk0ljbTy8ea9OXz4MHfccQdNmjRBkiTmzZtXeQMNIEKIysnYsWM5fPgw69at48cff2Tz5s08+OCDpR6Tn5/P0KFDefbZZytplBXLV199xeTJk5k1axZ79+6lU6dODBkyhNTUVJf7b9u2jTFjxjBhwgT27dvHrbfeyq233sqhQ4cqeeQVj7fvTX5+Ps2aNWPOnDkuF5CotsgCnzly5IgMyLt27bJt+/nnn2VJkuTz58+XefyGDRtkQL569WoFjrLi6dmzpzxx4kTbY7PZLCcmJsqzZ892uf+oUaPkm2++2WFbr1695IceeqhCxxkIvH1v7GncuLH8zjvvVODoggdhEZWD7du3ExsbS/fuxUvlDBo0CJVKxc6dOwM4ssrDulrvoEGDbNtKW60XlPfNfn+AIUOGuN2/quLLe1NTEUJUDi5dukR8fLzDNo1GQ61atbh06VKARlW5lLZar7v34NKlS17tX1Xx5b2pqQghcsH06dORJKnUm6slsQUCgW9U6TYgFcXTTz/NuHHjSt2nWbNmJCQklHA6mkwmMjIyaoyj0dvVegESEhK82r+q4st7U1MRFpEL6tatS+vWrUu96XQ6evfuTWZmJnv27LEd+9tvv2GxWOjVq1cAX0Hl4ctqvb1793bYH2DdunXVbnVfsZKxFwTaW17VGTp0qNylSxd5586d8pYtW+QWLVrIY8aMsT3/999/y61atZJ37txp23bx4kV537598kcffSQD8ubNm+V9+/bJV65cCcRLKDcrVqyQQ0JC5E8//VQ+cuSI/OCDD8qxsbHypUuXZFmW5bvvvluePn26bf+tW7fKGo1Gnjt3rnz06FF51qxZslarlQ8ePBiol1BhePve6PV6ed++ffK+ffvk+vXry1OmTJH37dsnnzx5MlAvoVIQQlROrly5Io8ZM0aOjIyUo6Oj5fHjx8s5OTm251NSUmRA3rBhg23brFmzZKDEbcmSJZX/AvzE/Pnz5UaNGsk6nU7u2bOnvGPHDttz/fr1k++9916H/b/++mu5ZcuWsk6nk9u1ayf/9NNPlTziysOb98b6eXG+9evXr/IHXomINiACgSDgCB+RQCAIOEKIBAJBwBFCJBAIAo4QIoFAEHCEEAkEgoAjhEggEAQcIUQCgSDgCCESCAQBRwiRwO+MGzfOZceCoUOHBnpogiBFVN8LKoShQ4eyZMkSh23uVjM1Go1otVqHbQaDAZ1O5/V1fT1OEFiERSSoEEJCQkhISHC4xcXFASBJEgsWLGDEiBFERETw6quv8sILL9C5c2c+/vhjmjZtSmhoKABnz55l5MiRREZGEh0dzahRoxzaarg7TlC1EEIkCAgvvPACt912GwcPHuS+++4D4NSpU6xcuZLvvvuO/fv3Y7FYGDlyJBkZGWzatIl169Zx+vRpRo8e7XAu5+MEVQ8xNRNUCD/++CORkZEO25599lnbyiV33XUX48ePd3jeYDDw+eefU7duXUDpUXTw4EFSUlJISkoC4PPPP6ddu3bs2rWLHj16uDxOUPUQQiSoEAYMGMCCBQscttWqVct2337BASuNGzd2EJOjR4+SlJRkEyGAtm3bEhsby9GjR21C5HycoOohhEhQIURERHDNNdeU+rwn2zy9lqBqI3xEgqClTZs2nDt3jnPnztm2HTlyhMzMTNq2bRvAkQn8jbCIBBWCXq8vsWSORqOhTp06Hp9j0KBBdOjQgbFjxzJv3jxMJhOPPvoo/fr1czm1E1RdhEUkqBDWrl1L/fr1HW7/+Mc/vDqHJEmsWrWKuLg4rr/+egYNGkSzZs346quvKmjUgkAhWsUKBIKAIywigUAQcIQQCQSCgCOESCAQBBwhRAKBIOAIIRIIBAFHCJFAIAg4QogEAkHAEUIkEAgCjhAigUAQcIQQCQSCgCOESCAQBBwhRAKBIOD8P6Z6FThJrrjWAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019827, + "end_time": "2024-03-22T19:05:42.203827", + "exception": false, + "start_time": "2024-03-22T19:05:42.184000", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4633.875758, + "end_time": "2024-03-22T19:05:44.947278", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/realtabformer/42/mlu-eval.ipynb", + "output_path": "eval/insurance/realtabformer/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/realtabformer/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "realtabformer" + }, + "start_time": "2024-03-22T17:48:31.071520", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file