diff --git "a/insurance/lct_gan/mlu-eval.ipynb" "b/insurance/lct_gan/mlu-eval.ipynb" new file mode 100644--- /dev/null +++ "b/insurance/lct_gan/mlu-eval.ipynb" @@ -0,0 +1,2475 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.775742Z", + "iopub.status.busy": "2024-03-22T13:31:10.775388Z", + "iopub.status.idle": "2024-03-22T13:31:10.809260Z", + "shell.execute_reply": "2024-03-22T13:31:10.808445Z" + }, + "papermill": { + "duration": 0.049185, + "end_time": "2024-03-22T13:31:10.811626", + "exception": false, + "start_time": "2024-03-22T13:31:10.762441", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.837166Z", + "iopub.status.busy": "2024-03-22T13:31:10.836720Z", + "iopub.status.idle": "2024-03-22T13:31:10.843587Z", + "shell.execute_reply": "2024-03-22T13:31:10.842788Z" + }, + "papermill": { + "duration": 0.022047, + "end_time": "2024-03-22T13:31:10.845565", + "exception": false, + "start_time": "2024-03-22T13:31:10.823518", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.868871Z", + "iopub.status.busy": "2024-03-22T13:31:10.868586Z", + "iopub.status.idle": "2024-03-22T13:31:10.872665Z", + "shell.execute_reply": "2024-03-22T13:31:10.871831Z" + }, + "papermill": { + "duration": 0.017967, + "end_time": "2024-03-22T13:31:10.874536", + "exception": false, + "start_time": "2024-03-22T13:31:10.856569", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.898144Z", + "iopub.status.busy": "2024-03-22T13:31:10.897843Z", + "iopub.status.idle": "2024-03-22T13:31:10.902004Z", + "shell.execute_reply": "2024-03-22T13:31:10.901136Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018546, + "end_time": "2024-03-22T13:31:10.904116", + "exception": false, + "start_time": "2024-03-22T13:31:10.885570", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.928121Z", + "iopub.status.busy": "2024-03-22T13:31:10.927759Z", + "iopub.status.idle": "2024-03-22T13:31:10.933667Z", + "shell.execute_reply": "2024-03-22T13:31:10.932778Z" + }, + "papermill": { + "duration": 0.020413, + "end_time": "2024-03-22T13:31:10.935553", + "exception": false, + "start_time": "2024-03-22T13:31:10.915140", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fa29630e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:10.960407Z", + "iopub.status.busy": "2024-03-22T13:31:10.960101Z", + "iopub.status.idle": "2024-03-22T13:31:10.965227Z", + "shell.execute_reply": "2024-03-22T13:31:10.964361Z" + }, + "papermill": { + "duration": 0.019757, + "end_time": "2024-03-22T13:31:10.967135", + "exception": false, + "start_time": "2024-03-22T13:31:10.947378", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/lct_gan/0\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011013, + "end_time": "2024-03-22T13:31:10.989313", + "exception": false, + "start_time": "2024-03-22T13:31:10.978300", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:11.013132Z", + "iopub.status.busy": "2024-03-22T13:31:11.012736Z", + "iopub.status.idle": "2024-03-22T13:31:11.022355Z", + "shell.execute_reply": "2024-03-22T13:31:11.021514Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024011, + "end_time": "2024-03-22T13:31:11.024335", + "exception": false, + "start_time": "2024-03-22T13:31:11.000324", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/lct_gan/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:11.048821Z", + "iopub.status.busy": "2024-03-22T13:31:11.048470Z", + "iopub.status.idle": "2024-03-22T13:31:13.092182Z", + "shell.execute_reply": "2024-03-22T13:31:13.091205Z" + }, + "papermill": { + "duration": 2.058715, + "end_time": "2024-03-22T13:31:13.094289", + "exception": false, + "start_time": "2024-03-22T13:31:11.035574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.121361Z", + "iopub.status.busy": "2024-03-22T13:31:13.120262Z", + "iopub.status.idle": "2024-03-22T13:31:13.134109Z", + "shell.execute_reply": "2024-03-22T13:31:13.133344Z" + }, + "papermill": { + "duration": 0.029569, + "end_time": "2024-03-22T13:31:13.136200", + "exception": false, + "start_time": "2024-03-22T13:31:13.106631", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.161072Z", + "iopub.status.busy": "2024-03-22T13:31:13.160367Z", + "iopub.status.idle": "2024-03-22T13:31:13.167785Z", + "shell.execute_reply": "2024-03-22T13:31:13.167005Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021972, + "end_time": "2024-03-22T13:31:13.169788", + "exception": false, + "start_time": "2024-03-22T13:31:13.147816", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.194948Z", + "iopub.status.busy": "2024-03-22T13:31:13.194311Z", + "iopub.status.idle": "2024-03-22T13:31:13.291036Z", + "shell.execute_reply": "2024-03-22T13:31:13.289868Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.112234, + "end_time": "2024-03-22T13:31:13.293545", + "exception": false, + "start_time": "2024-03-22T13:31:13.181311", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:13.321198Z", + "iopub.status.busy": "2024-03-22T13:31:13.320221Z", + "iopub.status.idle": "2024-03-22T13:31:18.059075Z", + "shell.execute_reply": "2024-03-22T13:31:18.058235Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.755204, + "end_time": "2024-03-22T13:31:18.061484", + "exception": false, + "start_time": "2024-03-22T13:31:13.306280", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 13:31:15.593181: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 13:31:15.593242: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 13:31:15.594874: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:18.086449Z", + "iopub.status.busy": "2024-03-22T13:31:18.085796Z", + "iopub.status.idle": "2024-03-22T13:31:18.092975Z", + "shell.execute_reply": "2024-03-22T13:31:18.092258Z" + }, + "papermill": { + "duration": 0.02172, + "end_time": "2024-03-22T13:31:18.094921", + "exception": false, + "start_time": "2024-03-22T13:31:18.073201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:18.122003Z", + "iopub.status.busy": "2024-03-22T13:31:18.121152Z", + "iopub.status.idle": "2024-03-22T13:31:26.530810Z", + "shell.execute_reply": "2024-03-22T13:31:26.529807Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.426043, + "end_time": "2024-03-22T13:31:26.533331", + "exception": false, + "start_time": "2024-03-22T13:31:18.107288", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T13:31:26.562557Z", + "iopub.status.busy": "2024-03-22T13:31:26.562182Z", + "iopub.status.idle": "2024-03-22T13:31:26.568987Z", + "shell.execute_reply": "2024-03-22T13:31:26.568139Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023527, + "end_time": "2024-03-22T13:31:26.571019", + "exception": false, + "start_time": "2024-03-22T13:31:26.547492", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:26.596763Z", + "iopub.status.busy": "2024-03-22T13:31:26.596424Z", + "iopub.status.idle": "2024-03-22T13:31:26.601516Z", + "shell.execute_reply": "2024-03-22T13:31:26.600685Z" + }, + "papermill": { + "duration": 0.020363, + "end_time": "2024-03-22T13:31:26.603428", + "exception": false, + "start_time": "2024-03-22T13:31:26.583065", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:31:26.628735Z", + "iopub.status.busy": "2024-03-22T13:31:26.628397Z", + "iopub.status.idle": "2024-03-22T13:36:01.416808Z", + "shell.execute_reply": "2024-03-22T13:36:01.415837Z" + }, + "papermill": { + "duration": 274.815243, + "end_time": "2024-03-22T13:36:01.430583", + "exception": false, + "start_time": "2024-03-22T13:31:26.615340", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:36:01.459705Z", + "iopub.status.busy": "2024-03-22T13:36:01.458691Z", + "iopub.status.idle": "2024-03-22T13:36:01.798957Z", + "shell.execute_reply": "2024-03-22T13:36:01.798123Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.357702, + "end_time": "2024-03-22T13:36:01.801276", + "exception": false, + "start_time": "2024-03-22T13:36:01.443574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:36:01.832024Z", + "iopub.status.busy": "2024-03-22T13:36:01.831204Z", + "iopub.status.idle": "2024-03-22T13:41:58.932397Z", + "shell.execute_reply": "2024-03-22T13:41:58.931173Z" + }, + "papermill": { + "duration": 357.136543, + "end_time": "2024-03-22T13:41:58.951952", + "exception": false, + "start_time": "2024-03-22T13:36:01.815409", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T13:41:58.986118Z", + "iopub.status.busy": "2024-03-22T13:41:58.985668Z", + "iopub.status.idle": "2024-03-22T13:41:59.466837Z", + "shell.execute_reply": "2024-03-22T13:41:59.465865Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.5011, + "end_time": "2024-03-22T13:41:59.469128", + "exception": false, + "start_time": "2024-03-22T13:41:58.968028", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.500820Z", + "iopub.status.busy": "2024-03-22T13:41:59.500461Z", + "iopub.status.idle": "2024-03-22T13:41:59.504956Z", + "shell.execute_reply": "2024-03-22T13:41:59.504022Z" + }, + "papermill": { + "duration": 0.023179, + "end_time": "2024-03-22T13:41:59.506852", + "exception": false, + "start_time": "2024-03-22T13:41:59.483673", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.535267Z", + "iopub.status.busy": "2024-03-22T13:41:59.534881Z", + "iopub.status.idle": "2024-03-22T13:41:59.542637Z", + "shell.execute_reply": "2024-03-22T13:41:59.541738Z" + }, + "papermill": { + "duration": 0.024826, + "end_time": "2024-03-22T13:41:59.544773", + "exception": false, + "start_time": "2024-03-22T13:41:59.519947", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9631369" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.579995Z", + "iopub.status.busy": "2024-03-22T13:41:59.578894Z", + "iopub.status.idle": "2024-03-22T13:41:59.686595Z", + "shell.execute_reply": "2024-03-22T13:41:59.685598Z" + }, + "papermill": { + "duration": 0.129451, + "end_time": "2024-03-22T13:41:59.689262", + "exception": false, + "start_time": "2024-03-22T13:41:59.559811", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 29] --\n", + "├─Adapter: 1-1 [2, 1071, 29] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 30,720\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 29] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,631,369\n", + "Trainable params: 9,631,369\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.31\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.53\n", + "Estimated Total Size (MB): 346.32\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:41:59.723605Z", + "iopub.status.busy": "2024-03-22T13:41:59.723221Z", + "iopub.status.idle": "2024-03-22T14:46:40.208564Z", + "shell.execute_reply": "2024-03-22T14:46:40.207591Z" + }, + "papermill": { + "duration": 3880.522327, + "end_time": "2024-03-22T14:46:40.227825", + "exception": false, + "start_time": "2024-03-22T13:41:59.705498", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03242904957539092, 'avg_role_model_std_loss': 0.8712406825969596, 'avg_role_model_mean_pred_loss': 0.006855423091376893, 'avg_role_model_g_mag_loss': 0.38299333698219723, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03276979403259853, 'n_size': 900, 'n_batch': 113, 'duration': 157.391695022583, 'duration_batch': 1.3928468586069294, 'duration_size': 0.17487966113620335, 'avg_pred_std': 0.13700437690831918}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006329906290500528, 'avg_role_model_std_loss': 0.46090240039369457, 'avg_role_model_mean_pred_loss': 5.679718335689661e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006329906290500528, 'n_size': 450, 'n_batch': 57, 'duration': 52.734307527542114, 'duration_batch': 0.9251632899568792, 'duration_size': 0.1171873500612047, 'avg_pred_std': 0.06423525986049258}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007500169986807224, 'avg_role_model_std_loss': 1.1092925603606028, 'avg_role_model_mean_pred_loss': 0.00019880676637914756, 'avg_role_model_g_mag_loss': 0.05190713240040673, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007618296743732773, 'n_size': 900, 'n_batch': 113, 'duration': 157.97713208198547, 'duration_batch': 1.3980277175396945, 'duration_size': 0.17553014675776163, 'avg_pred_std': 0.0854555291609954}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005137449055619072, 'avg_role_model_std_loss': 0.6586513487276945, 'avg_role_model_mean_pred_loss': 2.703822330727585e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005137449055619072, 'n_size': 450, 'n_batch': 57, 'duration': 53.497430086135864, 'duration_batch': 0.9385514050199274, 'duration_size': 0.1188831779691908, 'avg_pred_std': 0.05167298022200141}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004720190377233343, 'avg_role_model_std_loss': 0.5550237497275922, 'avg_role_model_mean_pred_loss': 4.577079138076836e-05, 'avg_role_model_g_mag_loss': 0.02987108025078972, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004798027821105077, 'n_size': 900, 'n_batch': 113, 'duration': 156.7364845275879, 'duration_batch': 1.3870485356423707, 'duration_size': 0.17415164947509765, 'avg_pred_std': 0.09026794963046512}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004182245211753374, 'avg_role_model_std_loss': 0.24288320739538394, 'avg_role_model_mean_pred_loss': 0.00014376330885359007, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004182245211753374, 'n_size': 450, 'n_batch': 57, 'duration': 52.81124806404114, 'duration_batch': 0.9265131239305463, 'duration_size': 0.11735832903120252, 'avg_pred_std': 0.07229022658838515}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004049827164530547, 'avg_role_model_std_loss': 0.43805476618803796, 'avg_role_model_mean_pred_loss': 5.451335522204717e-05, 'avg_role_model_g_mag_loss': 0.03185774852211277, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004109540785429999, 'n_size': 900, 'n_batch': 113, 'duration': 155.9995777606964, 'duration_batch': 1.3805272368203223, 'duration_size': 0.17333286417855157, 'avg_pred_std': 0.08781432131288854}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005127925792023436, 'avg_role_model_std_loss': 0.8044920190759293, 'avg_role_model_mean_pred_loss': 3.6524204451706925e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005127925792023436, 'n_size': 450, 'n_batch': 57, 'duration': 52.598074197769165, 'duration_batch': 0.92277323153981, 'duration_size': 0.11688460932837592, 'avg_pred_std': 0.04705578919393909}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004900188291147869, 'avg_role_model_std_loss': 0.6228896175905209, 'avg_role_model_mean_pred_loss': 0.00012289669273024944, 'avg_role_model_g_mag_loss': 0.031160058855182596, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005004006718000811, 'n_size': 900, 'n_batch': 113, 'duration': 156.05951523780823, 'duration_batch': 1.3810576569717543, 'duration_size': 0.17339946137534248, 'avg_pred_std': 0.08326448575980895}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00360431135011216, 'avg_role_model_std_loss': 0.1391123805354558, 'avg_role_model_mean_pred_loss': 0.00017833255974409213, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00360431135011216, 'n_size': 450, 'n_batch': 57, 'duration': 52.58910870552063, 'duration_batch': 0.9226159422021163, 'duration_size': 0.11686468601226807, 'avg_pred_std': 0.07722438271402528}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00338898796432962, 'avg_role_model_std_loss': 0.3835711898057332, 'avg_role_model_mean_pred_loss': 5.93510390443841e-05, 'avg_role_model_g_mag_loss': 0.04150173789097203, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034270015977866325, 'n_size': 900, 'n_batch': 113, 'duration': 156.12185072898865, 'duration_batch': 1.3816092984866253, 'duration_size': 0.17346872303220962, 'avg_pred_std': 0.09039239540893947}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004042578568138803, 'avg_role_model_std_loss': 0.16148795401670143, 'avg_role_model_mean_pred_loss': 2.5237032979771928e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004042578568138803, 'n_size': 450, 'n_batch': 57, 'duration': 53.37223672866821, 'duration_batch': 0.9363550303275125, 'duration_size': 0.11860497050815158, 'avg_pred_std': 0.07075024978257716}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002903090084760657, 'avg_role_model_std_loss': 0.2928461281526556, 'avg_role_model_mean_pred_loss': 1.3530937127217902e-05, 'avg_role_model_g_mag_loss': 0.03275197486082713, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029446042016045087, 'n_size': 900, 'n_batch': 113, 'duration': 157.1868932247162, 'duration_batch': 1.3910344533160723, 'duration_size': 0.174652103583018, 'avg_pred_std': 0.09083371352305454}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004153869318348977, 'avg_role_model_std_loss': 0.08165856703591276, 'avg_role_model_mean_pred_loss': 0.0004118713491015787, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004153869318348977, 'n_size': 450, 'n_batch': 57, 'duration': 52.67383909225464, 'duration_batch': 0.9241024402149937, 'duration_size': 0.11705297576056586, 'avg_pred_std': 0.08548504119869649}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026706402006998866, 'avg_role_model_std_loss': 0.20781586027989052, 'avg_role_model_mean_pred_loss': 0.00013795001301429672, 'avg_role_model_g_mag_loss': 0.037672711697717506, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026999003414271607, 'n_size': 900, 'n_batch': 113, 'duration': 155.8716015815735, 'duration_batch': 1.3793947042617123, 'duration_size': 0.17319066842397055, 'avg_pred_std': 0.09302399396500756}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036898051684774043, 'avg_role_model_std_loss': 0.04890317060488071, 'avg_role_model_mean_pred_loss': 0.00016461873778845238, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036898051684774043, 'n_size': 450, 'n_batch': 57, 'duration': 52.411773443222046, 'duration_batch': 0.9195047972495096, 'duration_size': 0.11647060765160455, 'avg_pred_std': 0.08178644566878415}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002238100019361203, 'avg_role_model_std_loss': 0.3384687315130162, 'avg_role_model_mean_pred_loss': 1.1111032782711483e-05, 'avg_role_model_g_mag_loss': 0.03593144379142258, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002262775602414169, 'n_size': 900, 'n_batch': 113, 'duration': 155.77839064598083, 'duration_batch': 1.3785698287254942, 'duration_size': 0.1730871007177565, 'avg_pred_std': 0.09169465267157133}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00233326900155387, 'avg_role_model_std_loss': 0.18033007517983266, 'avg_role_model_mean_pred_loss': 6.032218973090211e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00233326900155387, 'n_size': 450, 'n_batch': 57, 'duration': 52.599425315856934, 'duration_batch': 0.9227969353659111, 'duration_size': 0.11688761181301541, 'avg_pred_std': 0.07049635971545062}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012590437038711064, 'avg_role_model_std_loss': 0.15078724619232048, 'avg_role_model_mean_pred_loss': 4.081550376790824e-06, 'avg_role_model_g_mag_loss': 0.0263645450067189, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012720353449630138, 'n_size': 900, 'n_batch': 113, 'duration': 156.98792576789856, 'duration_batch': 1.3892736793619342, 'duration_size': 0.1744310286309984, 'avg_pred_std': 0.09350108472317194}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001966165854424212, 'avg_role_model_std_loss': 0.3141539510364837, 'avg_role_model_mean_pred_loss': 9.730311759192344e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001966165854424212, 'n_size': 450, 'n_batch': 57, 'duration': 53.111929416656494, 'duration_batch': 0.9317882353799385, 'duration_size': 0.11802650981479221, 'avg_pred_std': 0.06570776830950197}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001532604441874557, 'avg_role_model_std_loss': 0.1929226224414785, 'avg_role_model_mean_pred_loss': 3.391608327091929e-06, 'avg_role_model_g_mag_loss': 0.03449563190340996, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00154851471255016, 'n_size': 900, 'n_batch': 113, 'duration': 156.23109221458435, 'duration_batch': 1.3825760372972067, 'duration_size': 0.17359010246064926, 'avg_pred_std': 0.0936134795996204}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002808738038454774, 'avg_role_model_std_loss': 0.16682867217481775, 'avg_role_model_mean_pred_loss': 0.0001628730561616128, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002808738038454774, 'n_size': 450, 'n_batch': 57, 'duration': 52.47261381149292, 'duration_batch': 0.9205721721314547, 'duration_size': 0.11660580846998427, 'avg_pred_std': 0.07371748221646014}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019407396270738294, 'avg_role_model_std_loss': 0.3653798322266867, 'avg_role_model_mean_pred_loss': 2.2870681320568346e-05, 'avg_role_model_g_mag_loss': 0.03755757513559527, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001960661863623601, 'n_size': 900, 'n_batch': 113, 'duration': 157.5604350566864, 'duration_batch': 1.3943401332450125, 'duration_size': 0.1750671500629849, 'avg_pred_std': 0.09302689506779466}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022611901594912828, 'avg_role_model_std_loss': 0.223774224152935, 'avg_role_model_mean_pred_loss': 9.727328464069852e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022611901594912828, 'n_size': 450, 'n_batch': 57, 'duration': 52.695061922073364, 'duration_batch': 0.9244747705626906, 'duration_size': 0.11710013760460748, 'avg_pred_std': 0.06702947569602545}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015795660438016057, 'avg_role_model_std_loss': 0.20200802482515098, 'avg_role_model_mean_pred_loss': 3.2623586478791824e-06, 'avg_role_model_g_mag_loss': 0.03782492588998543, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015964161236964476, 'n_size': 900, 'n_batch': 113, 'duration': 157.62508010864258, 'duration_batch': 1.3949122133508194, 'duration_size': 0.17513897789849175, 'avg_pred_std': 0.09343803705301433}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019024741732250226, 'avg_role_model_std_loss': 0.16194683409677263, 'avg_role_model_mean_pred_loss': 0.00010373163840122158, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019024741732250226, 'n_size': 450, 'n_batch': 57, 'duration': 53.39478373527527, 'duration_batch': 0.9367505918469345, 'duration_size': 0.11865507496727837, 'avg_pred_std': 0.07240477975523263}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012615600261617348, 'avg_role_model_std_loss': 0.17734704436294219, 'avg_role_model_mean_pred_loss': 6.4640958073491015e-06, 'avg_role_model_g_mag_loss': 0.032005395059370334, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012749444810389024, 'n_size': 900, 'n_batch': 113, 'duration': 157.78692436218262, 'duration_batch': 1.3963444633821471, 'duration_size': 0.17531880484686957, 'avg_pred_std': 0.09475085942025206}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003454031080505552, 'avg_role_model_std_loss': 0.19643008009194018, 'avg_role_model_mean_pred_loss': 0.00048754797153507685, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003454031080505552, 'n_size': 450, 'n_batch': 57, 'duration': 53.61415338516235, 'duration_batch': 0.9405991821958307, 'duration_size': 0.11914256307813856, 'avg_pred_std': 0.08090297263850899}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001146070581356374, 'avg_role_model_std_loss': 0.12761228089974574, 'avg_role_model_mean_pred_loss': 5.205212322840684e-06, 'avg_role_model_g_mag_loss': 0.029757227330572074, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001157807001274907, 'n_size': 900, 'n_batch': 113, 'duration': 158.21624660491943, 'duration_batch': 1.4001437752647738, 'duration_size': 0.1757958295610216, 'avg_pred_std': 0.09795315121918653}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023975990749345835, 'avg_role_model_std_loss': 0.05156347854368074, 'avg_role_model_mean_pred_loss': 0.00027527655832503, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023975990749345835, 'n_size': 450, 'n_batch': 57, 'duration': 53.09577012062073, 'duration_batch': 0.9315047389582584, 'duration_size': 0.11799060026804606, 'avg_pred_std': 0.07778614515177253}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008811901032135615, 'avg_role_model_std_loss': 0.11619655411389741, 'avg_role_model_mean_pred_loss': 1.1726831726480937e-06, 'avg_role_model_g_mag_loss': 0.026715771118178962, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008903484087042872, 'n_size': 900, 'n_batch': 113, 'duration': 157.73501706123352, 'duration_batch': 1.3958851067365798, 'duration_size': 0.17526113006803726, 'avg_pred_std': 0.09735740258036989}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022739232058585105, 'avg_role_model_std_loss': 0.07977884817566928, 'avg_role_model_mean_pred_loss': 0.0003206384845873516, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022739232058585105, 'n_size': 450, 'n_batch': 57, 'duration': 52.94484996795654, 'duration_batch': 0.9288570169816938, 'duration_size': 0.11765522215101454, 'avg_pred_std': 0.08302696749339239}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007510672794726109, 'avg_role_model_std_loss': 0.07160761223355694, 'avg_role_model_mean_pred_loss': 8.719960328579189e-07, 'avg_role_model_g_mag_loss': 0.025627725821816258, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007592219165558668, 'n_size': 900, 'n_batch': 113, 'duration': 155.887836933136, 'duration_batch': 1.3795383799392564, 'duration_size': 0.17320870770348443, 'avg_pred_std': 0.10122318941671236}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001969620921461481, 'avg_role_model_std_loss': 0.14806897564466326, 'avg_role_model_mean_pred_loss': 0.0002989729056185993, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001969620921461481, 'n_size': 450, 'n_batch': 57, 'duration': 52.3817937374115, 'duration_batch': 0.9189788374984473, 'duration_size': 0.11640398608313667, 'avg_pred_std': 0.07865735263514675}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006008681999972194, 'avg_role_model_std_loss': 0.07322366024851942, 'avg_role_model_mean_pred_loss': 2.489029027904118e-07, 'avg_role_model_g_mag_loss': 0.019117836964627107, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006073361239072659, 'n_size': 900, 'n_batch': 113, 'duration': 156.76917052268982, 'duration_batch': 1.3873377922361931, 'duration_size': 0.17418796724743313, 'avg_pred_std': 0.0980093978114624}\n", + "Time out: 3743.244676589966/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'pred_duration': 2.398756980895996, 'grad_duration': 6.545684576034546, 'total_duration': 8.944441556930542, 'pred_std': 0.14884799718856812, 'std_loss': 2.9791326596750878e-06, 'mean_pred_loss': 9.03553154785186e-06, 'pred_rmse': 0.030479786917567253, 'pred_mae': 0.018268786370754242, 'pred_mape': 0.8310969471931458, 'grad_rmse': 0.05661950632929802, 'grad_mae': 0.018045680597424507, 'grad_mape': 0.6029329299926758}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'avg_pred_duration': 2.398756980895996, 'avg_grad_duration': 6.545684576034546, 'avg_total_duration': 8.944441556930542, 'avg_pred_std': 0.14884799718856812, 'avg_std_loss': 2.9791326596750878e-06, 'avg_mean_pred_loss': 9.03553154785186e-06}, 'min_metrics': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'pred_duration': 2.398756980895996, 'grad_duration': 6.545684576034546, 'total_duration': 8.944441556930542, 'pred_std': 0.14884799718856812, 'std_loss': 2.9791326596750878e-06, 'mean_pred_loss': 9.03553154785186e-06, 'pred_rmse': 0.030479786917567253, 'pred_mae': 0.018268786370754242, 'pred_mape': 0.8310969471931458, 'grad_rmse': 0.05661950632929802, 'grad_mae': 0.018045680597424507, 'grad_mape': 0.6029329299926758}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0009290174014001553, 'avg_g_mag_loss': 0.0073665124891392425, 'avg_g_cos_loss': 0.005373399430023883, 'pred_duration': 2.398756980895996, 'grad_duration': 6.545684576034546, 'total_duration': 8.944441556930542, 'pred_std': 0.14884799718856812, 'std_loss': 2.9791326596750878e-06, 'mean_pred_loss': 9.03553154785186e-06, 'pred_rmse': 0.030479786917567253, 'pred_mae': 0.018268786370754242, 'pred_mape': 0.8310969471931458, 'grad_rmse': 0.05661950632929802, 'grad_mae': 0.018045680597424507, 'grad_mape': 0.6029329299926758}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.265195Z", + "iopub.status.busy": "2024-03-22T14:46:40.264825Z", + "iopub.status.idle": "2024-03-22T14:46:40.269352Z", + "shell.execute_reply": "2024-03-22T14:46:40.268411Z" + }, + "papermill": { + "duration": 0.025753, + "end_time": "2024-03-22T14:46:40.271239", + "exception": false, + "start_time": "2024-03-22T14:46:40.245486", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.307605Z", + "iopub.status.busy": "2024-03-22T14:46:40.307245Z", + "iopub.status.idle": "2024-03-22T14:46:40.388209Z", + "shell.execute_reply": "2024-03-22T14:46:40.387226Z" + }, + "papermill": { + "duration": 0.101979, + "end_time": "2024-03-22T14:46:40.390711", + "exception": false, + "start_time": "2024-03-22T14:46:40.288732", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.428666Z", + "iopub.status.busy": "2024-03-22T14:46:40.428323Z", + "iopub.status.idle": "2024-03-22T14:46:40.724550Z", + "shell.execute_reply": "2024-03-22T14:46:40.723557Z" + }, + "papermill": { + "duration": 0.317504, + "end_time": "2024-03-22T14:46:40.726627", + "exception": false, + "start_time": "2024-03-22T14:46:40.409123", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+qElEQVR4nO3de1xUZf4H8M/MwMxwHW7CAHLxhhoimsaElWSyoZGGuaHEKrmmtmleWDeXNqXa34Zpba6XzbVard0MdFfbTc0i1MwYQbl4l1VDQGFAQBjul5nn98dxjg4MODNcBuT7fr3OCznnOec8MzGfnjnnOc8jYIwxEEJIPyG0dAUIIcQUFFqEkH6FQosQ0q9QaBFC+hUKLUJIv0KhRQjpVyi0CCH9ipWlK9BbtFotiouL4eDgAIFAYOnqEELaYIyhpqYGXl5eEAo7bk8NmNAqLi6Gj4+PpatBCLmPoqIiDB48uMPtAya0HBwcAHBviKOjo4VrQwhpS61Ww8fHh/+sdmTAhJbuK6GjoyOFFiF92P0u39CFeEJIv0KhRQjpVyi0CCH9yoC5pkW6RqPRoKWlxdLVIP2YtbU1RCJRl49DoUU6xRiDSqVCVVWVpatCHgBOTk6Qy+Vd6itJoUU6pQssd3d32NraUsdcYhbGGOrr61FWVgYA8PT0NPtYFFptVDe04FKJGgIAiqGulq6ORWk0Gj6wXF0H9ntBus7GxgYAUFZWBnd3d7O/KtKF+DZyCm9j7o6TeOvri5auisXprmHZ2tpauCbkQaH7W+rK9VEKrTZc7SQAgMq6JgvXpO+gr4Sku3TH3xKFVhvOdtYAgNt1LaA5Pwjpeyi02tC1tJo1WtQ2tVq4NoSQtii02rARiyC15t6W23XUL4mYRyAQ4KuvvrJ0NbrVW2+9hXHjxlm6GhRahuhaWxV0XYv0Y7t27YKTk1O3HW/16tVIS0vrtuOZi0LLAP66Vn2zhWtCSM9rbjbu79ze3r5PdH2h0DLAhb+DSF8P78UYQ31zq0UWU2+KHD58GI8//jicnJzg6uqKZ599FteuXQMATJo0CWvWrNErf+vWLVhbW+P48eMAgJKSEkRGRsLGxgZDhgzB7t274e/vj02bNpn13p07dw5PPfUUbGxs4OrqisWLF6O2tpbffuzYMYSEhMDOzg5OTk547LHHUFBQAAA4c+YMpkyZAgcHBzg6OmLChAk4ffp0p+c7duwYFixYgOrqaggEAggEArz11lsAAH9/f/zxj3/E/Pnz4ejoiMWLFwMA1qxZg4CAANja2mLo0KFYu3atXteEtl8PX3rpJURFReH999+Hp6cnXF1dsXTp0h5/3Is6lxrgYsu1tKjbg76GFg0eWvetRc598Z0I2IqN/3Otq6tDfHw8xo4di9raWqxbtw6zZs1Cbm4uYmNjsWHDBqxfv56/BZ+SkgIvLy888cQTAID58+ejvLwcx44dg7W1NeLj4/ne3Kaqq6tDREQEQkNDcerUKZSVleHll1/GsmXLsGvXLrS2tiIqKgqLFi3Cl19+iebmZmRmZvJ1i42Nxfjx4/HRRx9BJBIhNzcX1tbWnZ5z0qRJ2LRpE9atW4e8vDwAXEtJ5/3338e6deuQmJjIr3NwcMCuXbvg5eWFc+fOYdGiRXBwcMDrr7/e4XmOHj0KT09PHD16FFevXsWcOXMwbtw4LFq0yKz3yhgUWgZQS6v/mz17tt7vf//73zFo0CBcvHgR0dHRWLlyJU6cOMGH1O7duxETEwOBQIDLly/j+++/x6lTpzBx4kQAwCeffIIRI0aYVZfdu3ejsbERn3/+Oezs7AAAW7duxYwZM/Dee+/B2toa1dXVePbZZzFs2DAAwOjRo/n9CwsL8bvf/Q6jRo0CAKPqIRaLIZPJIBAIIJfL221/6qmn8Nvf/lZv3Ztvvsn/29/fH6tXr0ZycnKnoeXs7IytW7dCJBJh1KhRiIyMRFpaGoVWb3Oxo5aWITbWIlx8J8Ji5zbFlStXsG7dOmRkZKC8vBxarRYAFwBjxozB008/jS+++AJPPPEE8vPzoVQq8be//Q0AkJeXBysrKzz88MP88YYPHw5nZ2ez6n7p0iUEBwfzgQUAjz32GLRaLfLy8jB58mS89NJLiIiIwC9+8QuEh4cjOjqafz4vPj4eL7/8Mv7xj38gPDwcL7zwAh9u5tKF8b1SUlKwefNmXLt2DbW1tWhtbb3vKL+BgYF6j+N4enri3LlzXarb/dA1LQOopWWYQCCArdjKIoupPalnzJiByspKfPzxx8jIyEBGRgaAuxedY2Nj8a9//QstLS3YvXs3goKCEBQU1O3vmbF27twJpVKJSZMmISUlBQEBATh58iQA7lrShQsXEBkZiSNHjuChhx7C/v37u3S+ewMUAJRKJWJjY/HMM8/gwIEDyMnJwR/+8If7XqRv+zVVIBDw/4PoKRRaBlBLq3+rqKhAXl4e3nzzTUydOhWjR4/G7du39co899xzaGxsxOHDh7F7927Exsby20aOHInW1lbk5OTw665evdruGMYaPXo0zpw5g7q6On7dTz/9BKFQiJEjR/Lrxo8fj4SEBKSnp2PMmDHYvXs3vy0gIACrVq3Cd999h+effx47d+6873nFYjE0Go1RdUxPT4efnx/+8Ic/YOLEiRgxYgR/I6CvodAyQNfSul1PLa3+yNnZGa6urtixYweuXr2KI0eOID4+Xq+MnZ0doqKisHbtWly6dAkxMTH8tlGjRiE8PByLFy9GZmYmcnJysHjxYtjY2Jj17FxsbCykUini4uJw/vx5HD16FK+99hrmzZsHDw8P5OfnIyEhAUqlEgUFBfjuu+9w5coVjB49Gg0NDVi2bBmOHTuGgoIC/PTTTzh16pTeNa+O+Pv7o7a2FmlpaSgvL0d9fX2HZUeMGIHCwkIkJyfj2rVr2Lx5c5dbcz2FQssAXUuropZaWv2RUChEcnIysrKyMGbMGKxatQobN25sVy42NhZnzpzBE088AV9fX71tn3/+OTw8PDB58mTMmjWLv5MmlUpNro+trS2+/fZbVFZW4pFHHsEvf/lLTJ06FVu3buW3X758GbNnz0ZAQAAWL16MpUuXYsmSJRCJRKioqMD8+fMREBCA6OhoTJ8+HW+//fZ9zztp0iS88sormDNnDgYNGoQNGzZ0WHbmzJlYtWoVli1bhnHjxiE9PR1r1641+bX2CmaGrVu3Mj8/PyaRSFhISAjLyMjotPyePXvYyJEjmUQiYWPGjGEHDx7U256YmMhGjhzJbG1tmZOTE5s6dSo7efKkXpmKigr24osvMgcHByaTydivf/1rVlNTY3Sdq6urGQBWXV1937IVtU3Mb80B5rfmAGtu1Rh9jgdNQ0MDu3jxImtoaLB0VSyuqKiIAWDff/+9pavSr3X2N2XsZ9TkllZKSgri4+ORmJiI7OxsBAcHIyIiosM+LOnp6YiJicHChQuRk5ODqKgoREVF4fz583yZgIAAbN26FefOncOJEyfg7++Pp59+Grdu3eLLxMbG4sKFC0hNTcWBAwdw/PhxvlNcd5PZWEP3LYB6xQ9MR44cwX//+1/k5+cjPT0dc+fOhb+/PyZPnmzpqhFTkzIkJIQtXbqU/12j0TAvLy+WlJRksHx0dDSLjIzUW6dQKNiSJUs6PIcucXX/V7t48SIDwE6dOsWX+eabb5hAIGA3b940qt6mtLQYY2z8O98xvzUH2OUStVHlH0QDuaV1+PBhFhgYyGxsbJi7uzuLiopi169fZ4wx9s9//pPZ2dkZXB566KFeq+O0adM6rMef/vSnXquHKbqjpWVSP63m5mZkZWUhISGBXycUChEeHg6lUmlwH6VS2e4iaERERIdPwDc3N2PHjh2QyWQIDg7mj+Hk5KTXtyQ8PBxCoRAZGRmYNWtWu+M0NTWhqenuNSm1Wm306wQAZ1trVNY133louvNpusmDJyIiAhERhvukzZw5EwqFwuC2+/VU706ffPIJGhoaDG5zcXHptXr0NpNCq7y8HBqNBh4eHnrrPTw8cPnyZYP7qFQqg+VVKpXeugMHDmDu3Lmor6+Hp6cnUlNT4ebmxh/D3d1dv+JWVnBxcWl3HJ2kpCSjLlZ2xNVOgmu36mh4GtKOg4MDHBws/z8yb29vS1fBIvrM3cMpU6YgNzcX6enpmDZtGqKjo81+1gsAEhISUF1dzS9FRUUm7e9MfbUI6ZNMCi03NzeIRCKUlpbqrS8tLTX4fBMAyOVyo8rb2dlh+PDhePTRR/Hpp5/CysoKn376KX+MtgHW2tqKysrKDs8rkUjg6Oiot5iCesUT0jeZFFpisRgTJkzQGwhMq9UiLS0NoaGhBvcJDQ1tN3BYampqh+XvPa7umlRoaCiqqqqQlZXFbz9y5Ai0Wm2H1xa6inrFE9I3mfzAdHx8POLi4jBx4kSEhIRg06ZNqKurw4IFCwBwQ3p4e3sjKSkJALBixQqEhYXhgw8+QGRkJJKTk3H69Gns2LEDADdsx5/+9CfMnDkTnp6eKC8vx7Zt23Dz5k288MILALjHIKZNm4ZFixZh+/btaGlpwbJlyzB37lx4eXl113uhx9lWDACopF7xhPQpJofWnDlzcOvWLaxbtw4qlQrjxo3D4cOH+YvthYWFEArvNuAmTZqE3bt3480338Qbb7yBESNG4KuvvsKYMWMAACKRCJcvX8Znn32G8vJyuLq64pFHHsGPP/6IwMBA/jhffPEFli1bhqlTp0IoFGL27NnYvHlzV19/h1zt74QWtbQI6VMEjA2MebLUajVkMhmqq6uNur51LK8ML+08hdGejvhmxRO9UMO+p7GxEfn5+RgyZIhZj68MZAKBAPv370dUVJSlq9KndPY3ZexntM/cPexraNJW0t9198QWADeMs0AgQFVVVbce1xQUWh2gSVsJ6ZsotDpAk7YawBjQXGeZhSa26LaJLZqamrB69Wp4e3vDzs4OCoUCx44d4/ctKCjAjBkz4OzsDDs7OwQGBuLQoUO4fv06pkyZAoAb/kcgEOCll14y6/3oChpuuQO6SVsbW7S4XdcCB2nvPZ7RZ7XUA+/2zN3a+3qjGBDb3b/cHTSxRccTWyxbtgwXL15EcnIyvLy8sH//fkybNg3nzp3DiBEjsHTpUjQ3N+P48eOws7PDxYsXYW9vDx8fH/z73//G7NmzkZeXB0dHR9jY2Jj1nnQFhVYnXO0kuFnVgIq6Jvi62lq6OsQENLGF4YktCgsLsXPnThQWFvLdhVavXo3Dhw9j586dePfdd1FYWIjZs2fzw08PHTqU31/3TKO7u3u3Xy8zFoVWJ5ztrHGzqoGGp9GxtuVaPJY6twloYgvDzp07B41Gg4CAAL31TU1N/ESsy5cvx29+8xt89913CA8Px+zZszF27FizztcT6JpWJ3SP8lTUUmgBAAQC7iuaJRaa2KJbJraora2FSCRCVlYWcnNz+eXSpUv4y1/+AgB4+eWX8fPPP2PevHk4d+4cJk6ciC1btnTba+0qCq1O6CZtpZZW/0ITW3AMTWwxfvx4aDQalJWVYfjw4XrLvV8jfXx88Morr2Dfvn347W9/i48//pg/JgCjJ8zoCRRaneBbWnUUWv0JTWzBMTSxRUBAAGJjYzF//nzs27cP+fn5yMzMRFJSEg4ePAgAWLlyJb799lvk5+cjOzsbR48e5c/n5+cHgUCAAwcO4NatW3p3QHtNjwxP2AeZOnIpY4xtSfsf81tzgP1ub24P1qzv6s8jl6amprLRo0cziUTCxo4dy44dO8YAsP379/NlDh06xACwyZMnt9u/uLiYTZ8+nUkkEubn58d2797N3N3d2fbt2406f9tznT17lk2ZMoVJpVLm4uLCFi1axM9xoFKpWFRUFPP09GRisZj5+fmxdevWMY1Gw5qamtjcuXOZj48PE4vFzMvLiy1btszo/yavvPIKc3V1ZQBYYmIiY4yx5uZmtm7dOubv78+sra2Zp6cnmzVrFjt79ixjjLFly5axYcOGMYlEwgYNGsTmzZvHysvL+WO+8847TC6XM4FAwOLi4oyqh053jFxKj/F0YndGId7Yfw7ho93xSdwjPVzDvoce47nrxo0b8PHxwffff4+pU6daujr9Vnc8xkN3Dztxd3ga+no40Bw5cgS1tbUICgpCSUkJXn/9dZrYoo+ga1qduDsQIIXWQNPS0oI33ngDgYGBmDVrFgYNGsR3NP3iiy9gb29vcLl3ZJKeNn369A7r8e677/ZaPXobtbQ6QS2tgYsmtui7KLQ6oWtpqRtb0aLRwlpEDVNCE1tYGn0KO0GTtnIGyL0a0gu642+JQqsTIqHg7rDLA/Arou6rTn19vYVrQh4Uur+lrnyNpq+H96GbtHUghpZIJIKTkxM/uoGtra1ZnSsJYYyhvr4eZWVlcHJygkgkMvtYFFr3oZu0dSCGFgD+0Y6uzEFJiI6Tk1OH0/4Zi0LrPu6OYDowQ0sgEMDT0xPu7u5oaaGZiYj5rK2tu9TC0qHQug96/pAjEom65Q+OkK6iC/H34TLAW1qE9DVmhda2bdvg7+8PqVQKhUKBzMzMTsvv3bsXo0aNglQqRVBQEA4dOsRva2lpwZo1axAUFAQ7Ozt4eXlh/vz5KC7WH2zO39+fH+tat6xfv96c6puEWlqE9C0mh1ZKSgri4+ORmJiI7OxsBAcHIyIiosMLtenp6YiJicHChQuRk5ODqKgoREVF4fz58wC4W6DZ2dlYu3YtsrOzsW/fPuTl5WHmzJntjvXOO++gpKSEX1577TVTq28yvqU1gPtpEdKnmDSuBGMsJCSELV26lP9do9EwLy8vlpSUZLB8dHQ0i4yM1FunUCjYkiVLOjxHZmYmA8AKCgr4dX5+fuzDDz80tbo8c4amYYyxY3llzG/NARbx4Q9mn5sQcn/GfkZNamk1NzcjKysL4eHh/DqhUIjw8HAolUqD+yiVSr3yAPdcV0flAfDTHrUdOH/9+vVwdXXF+PHjsXHjRrS2djy1V1NTE9Rqtd5iDpc7nUuppUVI32DS3cPy8nJoNBp4eHjorffw8MDly5cN7qNSqQyWV6lUBss3NjZizZo1iImJ0RtTZ/ny5Xj44Yfh4uKC9PR0JCQkoKSkBH/+858NHicpKQlvv/22KS/PIBf7uz3iGWPUuZIQC+tTXR5aWloQHR0Nxhg++ugjvW33Dpc7duxYiMViLFmyBElJSZBIJO2OlZCQoLePWq2Gj4+PyXXStbRaNAy1Ta00/yEhFmbS10M3NzeIRCKUlpbqrS8tLe2wl6tcLjeqvC6wCgoKkJqaet/RRRUKBVpbW3H9+nWD2yUSCRwdHfUWc9iIRbCx5vonDdRe8YT0JSaFllgsxoQJE5CWlsav02q1SEtLQ2hoqMF9QkND9coDQGpqql55XWBduXIF33//PT//Wmdyc3MhFArh7u5uykswi4vdwH1ompC+xuSvh/Hx8YiLi8PEiRMREhKCTZs2oa6uDgsWLADATSfu7e2NpKQkAMCKFSsQFhaGDz74AJGRkUhOTsbp06exY8cOAFxg/fKXv0R2djYOHDgAjUbDX+9ycXGBWCyGUqlERkYGpkyZAgcHByiVSqxatQq/+tWvzJ5A0xQudmLcrGqg0CKkLzDn1uSWLVuYr68vE4vFLCQkhJ08eZLfFhYW1m6Gjj179rCAgAAmFotZYGAgO3jwIL8tPz+fATC4HD16lDHGWFZWFlMoFEwmkzGpVMpGjx7N3n33XdbY2Gh0nc3t8sAYY/M+zWB+aw6wPacKTd6XEGIcmo2nDXNm49FZlZKL/Tk3kTB9FJaEmTcdOSGkc8Z+RunZQyPwAwFSXy1CLI5Cywiuur5atRRahFgahZYRnKlXPCF9BoWWEXRdHmikB0Isj0LLCLrQojG1CLE8Ci0jUEuLkL6DQssIutCquTNpKyHEcii0jCCzsYZQN2krtbYIsSgKLSOIhAI4UV8tQvoECi0j8Q9NU18tQiyKQstILtTSIqRPoNAyEg1PQ0jfQKFlJGcKLUL6BAotI7lSaBHSJ1BoGYlaWoT0DRRaRqKWFiF9A4WWkailRUjfQKFlJGppEdI3UGgZSdfSul3PTdpKCLEMCi0j3Ttpa01Tq4VrQ8jARaFlpHsnbaWHpgmxHAotE9C4WoRYnlmhtW3bNvj7+0MqlUKhUCAzM7PT8nv37sWoUaMglUoRFBSEQ4cO8dtaWlqwZs0aBAUFwc7ODl5eXpg/fz6Ki4v1jlFZWYnY2Fg4OjrCyckJCxcuRG1trTnVNxuNYEqI5ZkcWikpKYiPj0diYiKys7MRHByMiIgIlJWVGSyfnp6OmJgYLFy4EDk5OYiKikJUVBTOnz8PAKivr0d2djbWrl2L7Oxs7Nu3D3l5eZg5c6becWJjY3HhwgWkpqbiwIEDOH78OBYvXmzGSzYftbQI6QNMnQU2JCSELV26lP9do9EwLy8vlpSUZLB8dHQ0i4yM1FunUCjYkiVLOjxHZmYmA8AKCgoYY4xdvHiRAWCnTp3iy3zzzTdMIBCwmzdvGlXvrswwrbMyOYf5rTnAth+7avYxCCGGGfsZNaml1dzcjKysLISHh/PrhEIhwsPDoVQqDe6jVCr1ygNAREREh+UBoLq6GgKBAE5OTvwxnJycMHHiRL5MeHg4hEIhMjIyDB6jqakJarVab+kqGumBEMszKbTKy8uh0Wjg4eGht97DwwMqlcrgPiqVyqTyjY2NWLNmDWJiYvipsVUqFdzd3fXKWVlZwcXFpcPjJCUlQSaT8YuPj49Rr7EzFFqEWF6funvY0tKC6OhoMMbw0UcfdelYCQkJqK6u5peioqIu149CixDLszKlsJubG0QiEUpLS/XWl5aWQi6XG9xHLpcbVV4XWAUFBThy5AjfytIdo+2F/tbWVlRWVnZ4XolEAolEYvRrM4YzjV5KiMWZ1NISi8WYMGEC0tLS+HVarRZpaWkIDQ01uE9oaKheeQBITU3VK68LrCtXruD777+Hq6tru2NUVVUhKyuLX3fkyBFotVooFApTXkKXuNpTS4sQizP1Cn9ycjKTSCRs165d7OLFi2zx4sXMycmJqVQqxhhj8+bNY7///e/58j/99BOzsrJi77//Prt06RJLTExk1tbW7Ny5c4wxxpqbm9nMmTPZ4MGDWW5uLispKeGXpqYm/jjTpk1j48ePZxkZGezEiRNsxIgRLCYmxuh6d8fdwyulNcxvzQE2JvGw2ccghBhm7GfU5NBijLEtW7YwX19fJhaLWUhICDt58iS/LSwsjMXFxemV37NnDwsICGBisZgFBgaygwcP8tvy8/MZAIPL0aNH+XIVFRUsJiaG2dvbM0dHR7ZgwQJWU1NjdJ27I7Qqa5uY35oDzG/NAdbUojH7OISQ9oz9jAoYGxhDFqjVashkMlRXV+tdLzOFVssw/A+HoGVA5htT4e4o7eZaEjJwGfsZ7VN3D/s6oVDAX4ynXvGEWAaFlomc6flDQiyKQstEfF8t6vZAiEVQaJmIn2maWlqEWASFlolcqK8WIRZFoWUiamkRYlkUWiai5w8JsSwKLRNRaBFiWRRaJqLQIsSyKLRMRKFFiGVRaJnIhSZtJcSiKLRMpAstmrSVEMug0DKR1FoEWzFN2kqIpVBomYEemibEcii0zKAbwZRaWoT0PgotM1BLixDLodAygysNT0OIxVBomcGZ+moRYjEUWmagDqaEWA6FlhkotAixHAotM9DopYRYDoWWGailRYjlmBVa27Ztg7+/P6RSKRQKBTIzMzstv3fvXowaNQpSqRRBQUE4dOiQ3vZ9+/bh6aefhqurKwQCAXJzc9sd48knn4RAINBbXnnlFXOq32UUWoRYjsmhlZKSgvj4eCQmJiI7OxvBwcGIiIhAWVmZwfLp6emIiYnBwoULkZOTg6ioKERFReH8+fN8mbq6Ojz++ON47733Oj33okWLUFJSwi8bNmwwtfrdQjd6aU1jK5pbtRapAyEDlqmzwIaEhLClS5fyv2s0Gubl5cWSkpIMlo+OjmaRkZF66xQKBVuyZEm7srrZpnNyctptCwsLYytWrDC6no2Njay6uppfioqKujzDtI5Go2VDfs/NNF1a3dDl4xFCjJ9h2qSWVnNzM7KyshAeHs6vEwqFCA8Ph1KpNLiPUqnUKw8AERERHZbvzBdffAE3NzeMGTMGCQkJqK+v77BsUlISZDIZv/j4+Jh8vo7QpK2EWI6VKYXLy8uh0Wjg4eGht97DwwOXL182uI9KpTJYXqVSmVTRF198EX5+fvDy8sLZs2exZs0a5OXlYd++fQbLJyQkID4+nv9drVZ3a3C52IlRUddMveIJ6WUmhZYlLV68mP93UFAQPD09MXXqVFy7dg3Dhg1rV14ikUAikfRYfXS94qmlRUjvMunroZubG0QiEUpLS/XWl5aWQi6XG9xHLpebVN5YCoUCAHD16tUuHcdcrveMYEoI6T0mhZZYLMaECROQlpbGr9NqtUhLS0NoaKjBfUJDQ/XKA0BqamqH5Y2l6xbh6enZpeOYi29p1VJoEdKbTP56GB8fj7i4OEycOBEhISHYtGkT6urqsGDBAgDA/Pnz4e3tjaSkJADAihUrEBYWhg8++ACRkZFITk7G6dOnsWPHDv6YlZWVKCwsRHFxMQAgLy8PANdKk8vluHbtGnbv3o1nnnkGrq6uOHv2LFatWoXJkydj7NixXX4TzEEtLUIsxJxbk1u2bGG+vr5MLBazkJAQdvLkSX5bWFgYi4uL0yu/Z88eFhAQwMRiMQsMDGQHDx7U275z504GoN2SmJjIGGOssLCQTZ48mbm4uDCJRMKGDx/Ofve735nUfcHY26nG+vTHn5nfmgPs1S+yuuV4hAx0xn5GBYwNjCll1Go1ZDIZqqur4ejo2OXj/Sf3JlYk52LSMFfsXvRoN9SQkIHN2M8oPXtoJl0/LXqUh5DeRaFlJnr+kBDLoNAyE03aSohlUGiZiSZtJcQyKLTMdO+krZXUV4uQXkOh1QX8xXjqq0VIr6HQ6gLdpK3U0iKk91BodQG1tAjpfRRaXeBK3R4I6XUUWl3gTDNNE9LrKLS6wIXG1CKk11FodYELtbQI6XUUWl1ALS1Ceh+FVhe40JhahPQ6Cq0u4B+apn5ahPQaCq0u4CdtbaJJWwnpLRRaXSCzsYZQwP2bviIS0jsotLrg3klbqYMpIb2DQquLaDBAQnoXhVYXOVNoEdKrKLS6iJ4/JKR3mRVa27Ztg7+/P6RSKRQKBTIzMzstv3fvXowaNQpSqRRBQUE4dOiQ3vZ9+/bh6aefhqurKwQCAT8R670aGxuxdOlSuLq6wt7eHrNnz243c7UlUEuLkN5lcmilpKQgPj4eiYmJyM7ORnBwMCIiIlBWVmawfHp6OmJiYrBw4ULk5OQgKioKUVFROH/+PF+mrq4Ojz/+ON57770Oz7tq1Sp8/fXX2Lt3L3744QcUFxfj+eefN7X63Y5aWoT0MlMnVAwJCWFLly7lf9doNMzLy4slJSUZLB8dHc0iIyP11ikUCrZkyZJ2ZfPz8xkAlpOTo7e+qqqKWVtbs7179/LrLl26xAAwpVJpVL27e7JWHZq0lZDuYexn1KSWVnNzM7KyshAeHs6vEwqFCA8Ph1KpNLiPUqnUKw8AERERHZY3JCsrCy0tLXrHGTVqFHx9fTs8TlNTE9Rqtd7SE2j0UkJ6l0mhVV5eDo1GAw8PD731Hh4eUKlUBvdRqVQmle/oGGKxGE5OTkYfJykpCTKZjF98fHyMPp8pdP20qHMpIb3jgb17mJCQgOrqan4pKirqkfPQSA+E9C4rUwq7ublBJBK1u2tXWloKuVxucB+5XG5S+Y6O0dzcjKqqKr3WVmfHkUgkkEgkRp/DXPeOqcUYg0Ag6PFzEjKQmdTSEovFmDBhAtLS0vh1Wq0WaWlpCA0NNbhPaGioXnkASE1N7bC8IRMmTIC1tbXecfLy8lBYWGjScXqCLrRatQzqRpq0lZCeZlJLCwDi4+MRFxeHiRMnIiQkBJs2bUJdXR0WLFgAAJg/fz68vb2RlJQEAFixYgXCwsLwwQcfIDIyEsnJyTh9+jR27NjBH7OyshKFhYUoLi4GwAUSwLWw5HI5ZDIZFi5ciPj4eLi4uMDR0RGvvfYaQkND8eijj3b5TegK3aSt9c0a3K5rhszG2qL1IeSBZ86tyS1btjBfX18mFotZSEgIO3nyJL8tLCyMxcXF6ZXfs2cPCwgIYGKxmAUGBrKDBw/qbd+5cycD0G5JTEzkyzQ0NLBXX32VOTs7M1tbWzZr1ixWUlJidJ17qssDY4w9tj6N+a05wE5fr+z2YxMyUBj7GRUwxpgFM7PXqNVqyGQyVFdXw9HRsVuPPXPrCZy9UY1P5k9E+EMe99+BENKOsZ/RB/buYW+ikR4I6T0UWt3AhWaaJqTXUGh1A2ppEdJ7KLS6AY30QEjvodDqBjTSAyG9h0KrG1BLi5DeQ6HVDailRUjvodDqBs73PH9ICOlZFFrdwM2OezC7pqkVBRV1Fq4NIQ82Cq1uILO1RuhQVwDAypRctGhotmlCegqFVjfZ+MJYOEitkFNYhS1pVyxdHUIeWBRa3WSwsy3+NCsIALD16FVk5ldauEaEPJgotLrRzGAvPP+wN7QMWJWSi+qGFktXiZAHDoVWN3vnuTHwdbHFzaoGrP3qPAbIIBqE9BoKrW5mL7HCX+aOg0gowH/PFGN/zk1LV4mQBwqFliFNtV3afbyvM1ZOHQEAWPefCyisqO+OWhFCQKHVXn0lsOVh4HAC0Fht9mFenTIcj/g7o7apFStSctBK3SAI6RYUWm1d/A9QWwqc/CuwZQKQ809Aa3rgiIQCfDhnHN8NYvORqz1QWUIGHgqttiYuAH61D3AdAdTdAv6zFPj0F8DNbJMPpdcN4sgVnLpO3SAI6SoKLUOGTwV+kw784o+A2B64eRr4+Cngv8uBugqTDnVvN4iVydQNgpCuotDqiJUYeGw5sOw0EBQNgAHZn3HXuzI/BjTGz3F4bzeIdf8533N1JmQAoNC6H0dPYPbHwIJvAI8xQGMVcGg1sONJoEBp1CHsJVbYdKcbxH9yi7E/50aPVpmQB5lZobVt2zb4+/tDKpVCoVAgMzOz0/J79+7FqFGjIJVKERQUhEOHDultZ4xh3bp18PT0hI2NDcLDw3Hliv7ze/7+/hAIBHrL+vXrzam+efwmAYt/AJ55H5DKgNJzwM5pwL8XAeqS++7+sK8zVtzpBrH2K+oGQYi5TA6tlJQUxMfHIzExEdnZ2QgODkZERATKysoMlk9PT0dMTAwWLlyInJwcREVFISoqCufP3/2atGHDBmzevBnbt29HRkYG7OzsEBERgcbGRr1jvfPOOygpKeGX1157zdTqd43ICghZBLyWDTwcB0AAnNsDbJ0I/PQXoLXz8bSW3tMNYiV1gyDEPKbOAhsSEsKWLl3K/67RaJiXlxdLSkoyWD46OppFRkbqrVMoFGzJkiWMMca0Wi2Ty+Vs48aN/PaqqiomkUjYl19+ya/z8/NjH374oanV5fXIDNM3TjO24ynGEh25ZVMwY6d3MdbS2OEuRZV1bEziYea35gD783d5pp2v/Cpj/3mNsb+MYyz7nybtWtvYwrRarWnnI6QXGfsZNaml1dzcjKysLISHh/PrhEIhwsPDoVQavr6jVCr1ygNAREQEXz4/Px8qlUqvjEwmg0KhaHfM9evXw9XVFePHj8fGjRvR2trxxfCmpiao1Wq9pdt5TwAWpgLPbQPsBgG384GvlwN/GQcotxnsWX9vN4gtR67gtDHdIIpzgD1xXL+x7M+Ayp+B/7wKfPUq0Nzx10zGGE5cKceCnZkITPwWCz87jSqam5H0cyaFVnl5OTQaDTw89Kd+9/DwgEqlMriPSqXqtLzu5/2OuXz5ciQnJ+Po0aNYsmQJ3n33Xbz++usd1jUpKQkymYxffHx8jH+hphAKgfG/ApbnAk//CXDwBGqKgW/fADaNAY69x/Wyv8e93SCWf5mDPaeL2neFYAzIPw58HsVd9L/4FQAGBEwDJi0HBEIg9wvgk6nArf/p7drUqsGe00WY/pcf8atPM3A07xYA4MjlMszYegIXis3v6U+IpVlZugLGio+P5/89duxYiMViLFmyBElJSZBIJO3KJyQk6O2jVqt7LrgAQGIPTFrGXfM68yVwYhPX8jr2LpC+GZjwEhC6jLsbCeDtmYHIKriNgop6vP6vs3hz/3lMDnDDs0FyRFhlwSZjM3Azizu2QAQE/RJ4bAXgEcitGx4O/PtloOwiF2oz/oKKoTPxRUYhPlcWoLy2CQBgKxYheqIPJge4IfG/F1BU2YDn/5qOpOeD8PzDg3vu/SCkh5gUWm5ubhCJRCgtLdVbX1paCrlcbnAfuVzeaXndz9LSUnh6euqVGTduXId1USgUaG1txfXr1zFy5Mh22yUSicEw63FWEi6gxv2Kax2d+BAoPQ8otwKZO4BxLwKPrYCDy1D8+zeTkHKqCF+fKcY11W04/28Pgq59DRshdzdSI5SAjZ8Hq8eXA85++ucZGga8cgL490Lg+o/AvpeRqg3HtuZfoQlieMqkeGmSP+Y+4guZrTUA7g7mypRcHMu7hfg9Z5BTWIW1zz4EsRX1fCH9h0l/rWKxGBMmTEBaWhq/TqvVIi0tDaGhoQb3CQ0N1SsPAKmpqXz5IUOGQC6X65VRq9XIyMjo8JgAkJubC6FQCHd3d1NeQu8RWXGto1dOAC/uBXweBTTNQNYu7trUvxbCrfYKlk6S47DiPC66rsFG6x0YJiyBmtliS2sUQuo3YeypX2DFt5VIvViKplYNf3jGGH5UCfGS5g/Y3BoFLRNgrvB7fGP3Dj551hnHX5+CJWHD+MACACdbMf4e9wiW3+l68Y+TBZizQwlVdWPb2hPSZwkYM22UupSUFMTFxeFvf/sbQkJCsGnTJuzZsweXL1+Gh4cH5s+fD29vbyQlJQHgujyEhYVh/fr1iIyMRHJyMt59911kZ2djzJgxAID33nsP69evx2effYYhQ4Zg7dq1OHv2LC5evAipVAqlUomMjAxMmTIFDg4OUCqVWLVqFaZPn47PPvvMqHqr1WrIZDJUV1fD0dHRxLepmxSkAz9+AFz9/u46sQPQXMP9294D7NGluOQ9G/+9XIuvzxTjZlUDX9RBaoWIQDke8nREyqki5JVy+wkEwEr/QrxauQHWTZWAxBF4bivw0HMdViXtUilWpeRC3dgKN3sxtsQ8jNBhrp3XX6sBmmruLGrD/25UAxIHwCcE8BwHWEvNfbfIAGPsZ9Tk0AKArVu3YuPGjVCpVBg3bhw2b94MhUIBAHjyySfh7++PXbt28eX37t2LN998E9evX8eIESOwYcMGPPPMM/x2xhgSExOxY8cOVFVV4fHHH8df//pXBAQEAACys7Px6quv4vLly2hqasKQIUMwb948xMfHG/0VsE+Elk7JGeDHP3MjSoABLkO561Vj5+p9yBljyC2qwtdnSnDwXDFK1U16h9Fdr1rwmD/8XO2A6pvAv34NFJ3kCihe4Z6ftBIbrEZBRR2W/CMLl1U1EAkF+P20UXg5WALBzSzgxinuIfHa0ruB1GLi9GgiMeA1HvBRAL6Pcj/t3Ew7BhkwejS0+qM+FVo6lT8DNSruwywUdVpUq2U4db0SB86W4LJKjfDRHpgb4guZjbV+QU0LcOSPXGdXgOuW8cud7a+JAUBzPZqKcnD4269hVZKF8cKr8BIY0QXDSsq1pvjF8c7iwN2QqFEBRRncKBltuQ7nvir7KrifbiO4piIZ8Ci02uiTodWT8r4B9r/CPSsplQFR24FBI7kWlG4pvQBo9fu6aZgA+SJ/DBr9GGTDQwHnIW3CyaHDlpsexrhQLsoACk9yP29dbl/OxoUL7aFh3A0Ma5tuefmk/6HQamPAhRYA3C4A/rXgbtcJQ+w9gMGPAIMnIs96JJZ8r8H1GiHsJVZ4/4WxmDbGs+N9TVVfyYWlLsRuZgGt99wEcAsAoj4CBk/svnOSfoNCq40BGVoA9zxk6jog4yNAJAE8g/mQwuBHANlgva9nZTWNWLY7h5+3MTLIE48Nd0PIEGcMG2QPQXd+lWttBlRnuRsUyq3c9TOBEHhsJfDk77nuI6RjjAEZfwNObgMCpgOTVwP2ffRuuhEotNoYsKGl01gNWNkY9dWuRaPFe99cxicn8vXWu9iJ8Yi/Mx7xd0HIEBc85OkIK1E39fGqrwS+WcM9gA4A7g9xrS6vcd1z/AdNUw3w39eAC/vvrrO2BR79DffEhI2TxapmLgqtNgZ8aJkhu/A2juXdQmZ+BXIKq9DUqj8qhZ1YhIf9nBHi74JHhrhgnI8TpNad31C4r0tfA1+vBOrLuScBJq8Gnlht3HW0gaLsEpAyD6i4AgituJD6+RhQfGdIcKmMa60qlgBiO0vW1CQUWm1QaHVNc6sW525WIzO/Eqeuc0tNo/5FfLFIiLGDZQgaLMNgZ1t4O9lgsDO3yGysjf9qWVcOHPztnectAciDuFaXPKh7X1R/dHYv91B+Sz3g4AVEf8b1iWMMuHyQu3Osu+Fh7wFM/h03jFI/CH0KrTYotLqXRsuQp6rBqeuVyLxeiVP5lSiraeqwvJ1YBG9nG3g72dz5acv/7uNsAzd7CYTCNqF2/t9ceDXcBoTWQNga4PGVgMja4Dm6HWPAjdPAmd1cq2/ok8CQJ7iWTG9rbeIewj/1Cff70CeB2Z+27/em1QDn9gJH3wWqCrh1Tr7AkwnA2Dn37VpjSRRabVBo9SzGGAoq6pF5vRJXSmtws6oBN2834GZVA8pr7z8cjo21CEGDZRjv44Txvk4Y7+sMD0cpUFMKHFgF5B3kCnqOA2ZtB9xH99yLaWkAzv0LOPUx1xH4XgIRdwNj2BRg2FOA18PcI1s9qaoI2Bt39y7w5N9xIdRZALU2c8MYHd/I3eAAALeRwFNvAqNn9Mm+cRRabVBoWU5ji0YvxHQ/b9yux83bDVCpG6E18FfoKZNivK8Txg2WIbz1Bww59TYEjdVcT/spb3DXcrqz5VCZD5z+lJvrsuE2t04kAcbM5jrNXjsCVLSZv1Ii41pfuhBzGdp99QG4R77+vQhoqASkTsDzO4CACOP3b67nHtQ/8SHXZw/gnlKYug4YOqVPhReFVhsUWn1Xi0aL6+V1yCmqQk5hFXKLqpCnUrcLMk/hbWy224lHWk4DABo9Hob04bnchCMegebdMdNquWA49TFwJRXAnZM6+QITFwLj5wF29zyTWVUIXDvKBVj+D3fDTcfJjwuvYVMA/ycAWxfT66Sr1/ENwLH1XJ08xwHRnxt+ssEYDVVctxLlX+8+jiWScF91pY53fsq4DsS6f7ddJI7c0Eoynx75mkmh1QaFVv9S19SKszeqkVtUhZzC28gpqsKtmiYADC+IfsBaq3/AUdCgv5PMlwsv+Zg7QTaGa/kIDXTLqK/kWlSnPwVuX7+7fng48MgiYMQv7v/B1GqAklwuwK4d4575bPOEARy9ue4bHoF36vQQNxFwZxfG6yqAfYuAa3dGPpmwAJi2vnsePq+9BZz4M3DqU0DT8TXITokk3PvqNpx7LMt1BPfTbYT5IQ0KrXYotPo3xhiKqxuRU3gbuYVVKLx+BUGlX2E08jFKWITBgnLDO1rbcqGhCzInP+DSf7hrVrre+FIZ16Ka+GvAdZj5lWyqAa7/BPx8FJqraRBVXDFcTmjNPVLFh9mdxcGTe0h9bxxQXcT1q3v2Q2BcjPl16khzPdetpFHN9eHTLU33/l7Vfrv6JjfEUkdsXO4GmOvwu/92GXbfO5gUWm1QaD14blY1YPuxa0g5VQSpRo1RgiI87XoLke4VkDdeg6DsEtDa0PEB5EFcqyroBUBsa3Y9Gpo1uFhSjTNF1Th3sxpnblTh51t1cEA9QuxUeNyhFEHWN+Hfmg/n2qsQtdQYPpCNMzevgLaF+5DP+cfdkWr7Cq2GC9Tyq1w/sYqrQPmdn+qbHe83/7/c86WdoNBqg0LrwaWqbsT2H67hy8xCvgNs8GAZlk8ZiqfcayAovcCNHlt6gftweY3nwsonxOQL0U2tGuSpanDmRjXO3ajC2RvVuFJWC42BOwkCAddrQh+Dj7ACU5zK8IitCqMEhfBqugbbmusQsDuDPI6ewU2WYomuFV3RXAdUXLsTZtfuhNkVLuCWZQKOXp3uTqHVBoXWg69M3Ygdx3/GPzMK0NjChVeglyOWTx2BX4z2aN8PrBMaLUNxVQMKK+txvaIOF4vVOHezGpdLatBsYL7KQQ4SBA+WIcjbCWN9ZAjylsFObIX/ldbgskqNSyU1uFSixmVVTftJTABI0IzxNmUY5iqBwPthDHd3uLPYw8NR0r3PfPY2XcTc5zVQaLVBoTVwlNc24ZMf8/G58jrqm7nWyyi5A5ZPHYFpgXI+vBpbNCiqrEdBRT0KKutRUFGHgop6FFbW48bterRoDH80nGytMXawE8Z6yzB2sAxjBzsZHSyMMajUjbhUwgXZZRUXZj/fqjXY7QMAHCRWGOpuj+GD7DHc/e7i42zTfc9+9gEUWm1QaA08lXXN+PuJfOxKv47aJu6u3nB3e7jYiVFYUQ+VuvOx8cUiIQa72MDPxRYjPBwwdrAMwYOdMNjZpttbPo0tGlwtq8VlVQ2ultXi2q1aXCurRUFlvcGvnrr6DXGzw9BBdnC2E8NBagVHqTXsJVZwkFrBQWp95ye33kFqBXuJVZ8NOgqtNii0Bq7q+hb8/ad8/P2n/HbPSzpIrODrags/V1v4udrBz8X2zu92kDtKITLhK2VPaGrVoKCiHlfLavkw0/3UfQU2lY21CI42VpA7SuHjYgvfexYfF1t4yqQWCTYKrTYotIi6sQWHz6kgsRbC14ULJmdbEx7k7kO0WoabVQ24eqsW18vroG5oRU1jC2oaW1HTxP1UN95dV9vYioYWzf0PDMBKKIC3sw0fYveGmrujBI5S666P5mEAhVYbFFpkoGvRaFHb2IqaxlZUN7SguLoBRZXcNTzdcqOyweCNhrYkVkLIbKzhaGMN2Z3FUWp199/3bHOUWmOMtyMcpJ0/6G7sZ7TfzDBNCOkaa5EQznZiONtxnTyDBrfvUqHVMpTWNKLwzg0J/VBrQEVdExgDmlq1KKtp6nRkj3vtf3USxvs6d8vroNAihPCEQgE8ZTbwlNlAMbT9PJhaLUNNUyvUDS2obmi5+7OR+8mta+X/rdvmYteN43kxM2zdupX5+fkxiUTCQkJCWEZGRqfl9+zZw0aOHMkkEgkbM2YMO3jwoN52rVbL1q5dy+RyOZNKpWzq1Knsf//7n16ZiooK9uKLLzIHBwcmk8nYr3/9a1ZTU2N0naurqxkAVl1dbfwLJYT0GmM/oybfIkhJSUF8fDwSExORnZ2N4OBgREREoKyszGD59PR0xMTEYOHChcjJyUFUVBSioqJw/vx5vsyGDRuwefNmbN++HRkZGbCzs0NERAQaG+/eko6NjcWFCxeQmpqKAwcO4Pjx41i8eLHJIU0I6edMTcOQkBC2dOlS/neNRsO8vLxYUlKSwfLR0dEsMjJSb51CoWBLlixhjHGtLLlczjZu3Mhvr6qqYhKJhH355ZeMMcYuXrzIALBTp07xZb755hsmEAjYzZs3jao3tbQI6dt6pKXV3NyMrKwshIeH8+uEQiHCw8OhVCoN7qNUKvXKA0BERARfPj8/HyqVSq+MTCaDQqHgyyiVSjg5OWHixLvz4YWHh0MoFCIjI8PgeZuamqBWq/UWQkj/Z1JolZeXQ6PRwMPDQ2+9h4cHVCqVwX1UKlWn5XU/71fG3V1/PjcrKyu4uLh0eN6kpCTIZDJ+8fHxMfJVEkL6sr7Zn78bJCQkoLq6ml+KioosXSVCSDcwKbTc3NwgEolQWlqqt760tBRyudzgPnK5vNPyup/3K9P2Qn9raysqKys7PK9EIoGjo6PeQgjp/0wKLbFYjAkTJiAtLY1fp9VqkZaWhtDQUIP7hIaG6pUHgNTUVL78kCFDIJfL9cqo1WpkZGTwZUJDQ1FVVYWsrCy+zJEjR6DVaqFQKEx5CYSQfs7kzqXx8fGIi4vDxIkTERISgk2bNqGurg4LFiwAAMyfPx/e3t5ISkoCAKxYsQJhYWH44IMPEBkZieTkZJw+fRo7duwAAAgEAqxcuRL/93//hxEjRmDIkCFYu3YtvLy8EBUVBQAYPXo0pk2bhkWLFmH79u1oaWnBsmXLMHfuXHh5dT6wmA6787QSXZAnpG/SfTbZ/Z4sNOfW5JYtW5ivry8Ti8UsJCSEnTx5kt8WFhbG4uLi9Mrv2bOHBQQEMLFYzAIDAzvsXOrh4cEkEgmbOnUqy8vL0ytTUVHBYmJimL29PXN0dGQLFiwwqXNpUVERAzfVCi200NKHl6Kiok4/ywPmgWmtVovi4mI4ODjc96l+tVoNHx8fFBUVDehrYfQ+cOh94PT0+8AYQ01NDby8vCA0NIPSHQPm2UOhUIjBgwebtA9dwOfQ+8Ch94HTk++DTCa7b5kHtssDIeTBRKFFCOlXKLQMkEgkSExMhEQisXRVLIreBw69D5y+8j4MmAvxhJAHA7W0CCH9CoUWIaRfodAihPQrFFqEkH6FQosQ0q9QaLWxbds2+Pv7QyqVQqFQIDMz09JV6nVvvfUWBAKB3jJq1ChLV6vHHT9+HDNmzICXlxcEAgG++uorve2MMaxbtw6enp6wsbFBeHg4rly5YpnK9qD7vQ8vvfRSu7+PadOm9Vr9KLTuYeqkHQ+ywMBAlJSU8MuJEycsXaUeV1dXh+DgYGzbts3gdmMmYHkQ3O99AIBp06bp/X18+eWXvVdBo4dJGABMnbTjQZWYmMiCg4MtXQ2LAsD279/P/27MBCwPorbvA2OMxcXFseeee84i9WHMjCnEHlTmTNrxILty5Qq8vLwwdOhQxMbGorCw0NJVsihjJmAZSI4dOwZ3d3eMHDkSv/nNb1BRUdFr56bQusOcSTseVAqFArt27cLhw4fx0UcfIT8/H0888QRqamosXTWLMWYCloFi2rRp+Pzzz5GWlob33nsPP/zwA6ZPnw6NRtMr5x8wQ9MQ402fPp3/99ixY6FQKODn54c9e/Zg4cKFFqwZ6Qvmzp3L/zsoKAhjx47FsGHDcOzYMUydOrXHz08trTvMmbRjoHByckJAQACuXr1q6apYjDETsAxUQ4cOhZubW6/9fVBo3WHOpB0DRW1tLa5duwZPT09LV8VijJmAZaC6ceMGKioqeu3vg74e3uN+k3YMFKtXr8aMGTPg5+eH4uJiJCYmQiQSISYmxtJV61G1tbV6rYX8/Hzk5ubCxcUFvr6+952A5UHR2fvg4uKCt99+G7Nnz4ZcLse1a9fw+uuvY/jw4YiIiOidClrsvmUf1dmkHQPFnDlzmKenJxOLxczb25vNmTOHXb161dLV6nFHjx41ONGCbqIWYyZgeRB09j7U19ezp59+mg0aNIhZW1szPz8/tmjRIqZSqXqtfjSeFiGkX6FrWoSQfoVCixDSr1BoEUL6FQotQki/QqFFCOlXKLQIIf0KhRYhpF+h0CKE9CsUWoSQfoVCixDSr1BoEUL6lf8HXCPIJJjQZeQAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:46:40.769360Z", + "iopub.status.busy": "2024-03-22T14:46:40.768952Z", + "iopub.status.idle": "2024-03-22T14:48:58.416842Z", + "shell.execute_reply": "2024-03-22T14:48:58.416011Z" + }, + "papermill": { + "duration": 137.673684, + "end_time": "2024-03-22T14:48:58.419405", + "exception": false, + "start_time": "2024-03-22T14:46:40.745721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:48:58.458680Z", + "iopub.status.busy": "2024-03-22T14:48:58.458035Z", + "iopub.status.idle": "2024-03-22T14:48:58.478208Z", + "shell.execute_reply": "2024-03-22T14:48:58.477275Z" + }, + "papermill": { + "duration": 0.042235, + "end_time": "2024-03-22T14:48:58.480335", + "exception": false, + "start_time": "2024-03-22T14:48:58.438100", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0011460.0139830.0009296.5465270.0180460.6029330.056620.0000092.3627930.0182690.8310970.030480.1488480.0000038.90932
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.001146 0.013983 0.000929 6.546527 0.018046 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.602933 0.05662 0.000009 2.362793 0.018269 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.831097 0.03048 0.148848 0.000003 8.90932 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:48:58.517267Z", + "iopub.status.busy": "2024-03-22T14:48:58.516680Z", + "iopub.status.idle": "2024-03-22T14:48:58.914736Z", + "shell.execute_reply": "2024-03-22T14:48:58.913883Z" + }, + "papermill": { + "duration": 0.418878, + "end_time": "2024-03-22T14:48:58.917032", + "exception": false, + "start_time": "2024-03-22T14:48:58.498154", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:48:58.956992Z", + "iopub.status.busy": "2024-03-22T14:48:58.955979Z", + "iopub.status.idle": "2024-03-22T14:51:25.628364Z", + "shell.execute_reply": "2024-03-22T14:51:25.627328Z" + }, + "papermill": { + "duration": 146.695763, + "end_time": "2024-03-22T14:51:25.631495", + "exception": false, + "start_time": "2024-03-22T14:48:58.935732", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:25.673508Z", + "iopub.status.busy": "2024-03-22T14:51:25.673079Z", + "iopub.status.idle": "2024-03-22T14:51:25.702166Z", + "shell.execute_reply": "2024-03-22T14:51:25.701069Z" + }, + "papermill": { + "duration": 0.053561, + "end_time": "2024-03-22T14:51:25.705052", + "exception": false, + "start_time": "2024-03-22T14:51:25.651491", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:25.743874Z", + "iopub.status.busy": "2024-03-22T14:51:25.743504Z", + "iopub.status.idle": "2024-03-22T14:51:25.749616Z", + "shell.execute_reply": "2024-03-22T14:51:25.748693Z" + }, + "papermill": { + "duration": 0.027677, + "end_time": "2024-03-22T14:51:25.751678", + "exception": false, + "start_time": "2024-03-22T14:51:25.724001", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.03444043153561541}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:25.792667Z", + "iopub.status.busy": "2024-03-22T14:51:25.792298Z", + "iopub.status.idle": "2024-03-22T14:51:26.187177Z", + "shell.execute_reply": "2024-03-22T14:51:26.186182Z" + }, + "papermill": { + "duration": 0.417606, + "end_time": "2024-03-22T14:51:26.189398", + "exception": false, + "start_time": "2024-03-22T14:51:25.771792", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA910lEQVR4nO3deXhU9b0/8PdZZs0sSchOEhIIi7JGBETUgFBRrEptvVapggjaFqo2ch/N7S0qrcVaF2jrQ729JZGnImp/uNxaVIqCSgUF2QQLBEMIISEhyySzz5zz/f1xJgNDZpLJZJIzCZ+XzzwyM98585ksn3z3L8cYYyCEEBXxagdACCGUiAghqqNERAhRHSUiQojqKBERQlRHiYgQojpKRIQQ1VEiIoSojhIRIUR1lIhIlyoqKsBxHE6ePKl2KGQQo0RE4m7jxo1Ys2aN2mGQAYQSEYk7SkSkpygREUJUR4mI9NiWLVtQUlICs9kMi8WCKVOmYOPGjQCAmTNn4r333kN1dTU4jgPHcSgoKIj62rIs48knn0ROTg6MRiNmzZqFI0eOoKCgAIsWLQqWa25uxooVKzB+/HiYTCZYLBbcdNNNOHDgQMj1tm/fDo7j8MYbb+Dpp59Gbm4u9Ho9Zs+ejcrKynh8OUgciGoHQAaWiooKLF68GGPHjkVZWRmSk5Oxb98+vP/++7j77rvxi1/8AjabDadPn8aLL74IADCZTFFfv6ysDM8++yxuueUWzJ07FwcOHMDcuXPhdrtDyn377bd4++23cccdd6CwsBBnz57Fyy+/jJKSEhw5cgQ5OTkh5Z955hnwPI8VK1bAZrPh2WefxYIFC7B79+7ef1FI7zFCulBeXs4AsKqqKtba2srMZjObNm0ac7lcIeVkWQ7+++abb2bDhg3r8XvV19czURTZ/PnzQx5/8sknGQC2cOHC4GNut5tJkhRSrqqqiul0OrZq1argYx9//DEDwC677DLm8XiCj69du5YBYIcOHepxnCT+qGlGorZ161a0t7fj8ccfh16vD3mO47heX3/btm3w+/346U9/GvL4z372s05ldTodeF758ZUkCU1NTTCZTBg9ejS++uqrTuXvu+8+aLXa4P1rr70WgFKzIuqjRESiduLECQDAuHHj+uT61dXVAICioqKQx1NTU5GSkhLymCzLePHFFzFy5EjodDqkpaUhPT0dBw8ehM1m63Tt/Pz8kPsd12tpaYnnRyAxokREBqTf/OY3KC0txXXXXYe//vWv+OCDD7B161aMHTsWsix3Ki8IQtjrMNopOSFQZzWJ2ogRIwAAX3/9daday4VibaYNGzYMAFBZWYnCwsLg401NTZ1qLn/7298wa9Ys/OUvfwl5vLW1FWlpaTG9P1EP1YhI1G644QaYzWasXr260yjWhTWLpKSksM2j7syePRuiKGLdunUhj//xj3/sVFYQhE61mTfffBO1tbU9fl+iPqoRkahZLBa8+OKLWLJkCaZMmYK7774bKSkpOHDgAJxOJ1555RUAwOTJk/H666+jtLQUU6ZMgclkwi233NLt9TMzM/Hwww/j+eefx6233oobb7wRBw4cwJYtW5CWlhZS0/rud7+LVatW4b777sPVV1+NQ4cO4dVXX8Xw4cP77POTPqTyqB1JcBcO33d499132dVXX80MBgOzWCxs6tSp7LXXXgs+b7fb2d13382Sk5MZgB4N5fv9fvbLX/6SZWVlMYPBwK6//nr2zTffsCFDhrAf//jHwXJut5s9+uijLDs7mxkMBjZjxgz2+eefs5KSElZSUhIs1zF8/+abb4a8T1VVFQPAysvLe/olIX2AY4x660hia21tRUpKCn7961/jF7/4hdrhkD5AfUQkobhcrk6PdSygnTlzZv8GQ/oN9RGRftHY2AhJkiI+r9VqkZqaitdffx0VFRWYN28eTCYTPvvsM7z22mu44YYbMGPGjH6MmPQnSkSkX0yZMiU4YTGckpISbN++HRMmTIAoinj22WfR1tYW7MD+9a9/3Y/Rkv5GfUSkX+zcuTNss6tDSkoKJk+e3I8RkURCiYgQojrqrCaEqG5A9xHJsowzZ87AbDbHZfU3ISS+GGNob29HTk5OcLeEcAZ0Ijpz5gzy8vLUDoMQ0o2amhrk5uZGfH5AJyKz2QxA+ZAWi0XlaAghF2tra0NeXl7wdzWSAZ2IOppjFouFEhEhCay7rhPqrCaEqI4SESFEdZSICCGqG9B9RNFgjMHv93e5zomEJwgCRFGkqRGkzw3qROT1elFXVwen06l2KAOW0WhEdnZ2yAkYhMSb6omotrYWjz32GLZs2QKn04mioiKUl5fjyiuv7NV1ZVlGVVUVBEFATk4OtFot/WXvAcYYvF4vGhsbUVVVhZEjR3Y5IY2Q3lA1EbW0tGDGjBmYNWsWtmzZgvT0dBw/frzT0TGx8Hq9kGUZeXl5MBqNcYj20mMwGKDRaFBdXQ2v19vpLDMShr0ROLENyJoAZF6udjQDhqqJ6Le//S3y8vJQXl4efOzC0xsu5vF44PF4gvfb2tq6fQ/6K9479PXroaodQHOVcksbBQiqNzoGBFV/yt59911ceeWVuOOOO5CRkYHi4mL8+c9/jlh+9erVsFqtwRst7yCJxOuT8M2/D+NEo105YcTZpHZIA4aqiejbb7/FunXrMHLkSHzwwQf4yU9+goceeih4GsTFysrKYLPZgreampp+jpiQyI6eqkVrWzsa2j2wuXyAo0HtkAYMVRORLMu44oor8Jvf/AbFxcV44IEHsHTpUvzpT38KW16n0wWXc9CyjvgoKCgI7glNeqehsREAoBU46LUC4GxWOaKBQ9VElJ2djcsvD+3Qu+yyy3Dq1CmVIiIkdu2tSlOsKMMMvSgAnnaVIxo4VE1EM2bMwNGjR0MeO3bsWPDoYRIdr9erdgiXPElm8DmU0231msCvldeuYkQDi6qJ6Oc//zl27dqF3/zmN6isrMTGjRvxP//zP1i2bFmfvq/XL0e8+SU56rK+KMrGYubMmVi+fDmWL18Oq9WKtLQ0/PKXvwwesVxQUIBf/epXuPfee2GxWPDAAw8AAD777DNce+21MBgMyMvLw0MPPQSHwxG8bkNDA2655RYYDAYUFhbi1VdfjSk+0lm72weNvx08B3iMWWh2eMCoRhQ1VccWp0yZgrfeegtlZWVYtWoVCgsLsWbNGixYsKBP3/eljysjPleYloT5xUOD9//nkxPwSeG39c5NMeCOK8+P3K3fWQWXN3Qpyc+/MyqmGF955RXcf//9+OKLL7Bnzx488MADyM/Px9KlSwEAzz33HFauXIknnngCAHDixAnceOON+PWvf43169ejsbExmMw6pkcsWrQIZ86cwccffwyNRoOHHnoIDQ3UoRoP7W4/tJIdWpHHR7UCMtvtmGxqA81Hj47qkxy++93v4rvf/a7aYSScvLw8vPjii+A4DqNHj8ahQ4fw4osvBhPR9ddfj0cffTRYfsmSJViwYAEeeeQRAMDIkSPx+9//HiUlJVi3bh1OnTqFLVu24IsvvsCUKVMAAH/5y19w2WWX9ftnG4zSTDpMydFBdCThWzkNaAd8Hhe0jAE0o79bqiciNSybVRTxOf6in5kHrhsRsezFP1+LZ0SejNlTV111VciSlOnTp+P5558PLt69eAnMgQMHcPDgwZDmFmMsuNTl2LFjEEUx5MieMWPGIDk5OW4xX8oMWgEGvQwwLThZWRngl2TA7wE0NCO9O5dkItKK0XeN9VXZ3kpKSgq5b7fb8eCDD+Khhx7qVDY/Px/Hjh3rr9AuXT5lcTVvsELmRKUP0e+iRBSFSzIRDQS7d+8Oub9r1y6MHDkSgiCELX/FFVfgyJEjKCoKX9sbM2YM/H4/9u7dG2yaHT16FK2trXGN+1L17VkbzC3tMBtEaJOT4Od18MsM8LkBg9rRJT5aSJSgTp06hdLSUhw9ehSvvfYa/vCHP+Dhhx+OWP6xxx7Dv/71Lyxfvhz79+/H8ePH8c4772D58uUAgNGjR+PGG2/Egw8+iN27d2Pv3r1YsmQJDAb6LYmHgyfrUdloR7tHhk5nhJ/Xnq8RkW5RIkpQ9957L1wuF6ZOnYply5bh4YcfDg7ThzNhwgTs2LEDx44dw7XXXovi4mKsXLkSOTk5wTLl5eXIyclBSUkJbr/9djzwwAPIyMjoj48z6PlcypwhUWeEQScqNaKOPiLSLWqaJSiNRoM1a9Zg3bp1nZ47efJk2NdMmTIFH374YcRrZmVl4e9//3vIY/fcc0+v4iQKyav0Dwk6IwqGJME9NB1D3E7ARzWiaFAiIiQOWCARiTojcpINQHYaUHcG8LtVjmxgoKYZIb0kywyyT0k4ojawCZ+oU/5PNaKoUI0oAW3fvl3tEEgPePwyBFlZ7ydodfD6ZbS5AK3bB4vkUzm6gYFqRIT0ktcvQ5B9EHgOgsYAm8uHfx5rxbGz7YBEndXRoBoRIb1k0Aq4ptAMbX0SIGqhFXlIvAaSzACqEUWFEhEhvaQVeWQncUCSDhB00Ik8JE4DJQ+5EX4KKrkQNc0IiYeOJpiohVZQakQA4PdR0ywaVCMipJea7B64z9lg8fhgFnTgeQ5CYH2Z5KXh+2hQIiKkl6qbnWiqaUSB1g1zYNhe0Cg7EUk+2j0zGtQ0I6SXvH4ZAlNGzTrmD4lUI+oRqhER0ksd84gEngMEJRGNG5aBlBYj9IIE0OZo3aJEREgveXwSkmQvRF4LiEqTbGxeGlAV2NlA8p6faU3CurSaZowBfm//31j4Pa/D2bBhA4YMGRJytDYAzJ8/nxaoJiiPzw+e+UNqROBFgAv8eknUT9SdS6tGJPmAT5/v//e99tHgX8ru3HHHHXjooYfw7rvv4o477gCgnL7x3nvvdbmynqjHH+gHEi/oI2rz+AEfBy0k6P1egCpEXbq0akQDgMFgwN133x08eQMA/vrXvyI/Px8zZ85ULzASUUci4kUR4JXpi3tONmNfrQON7R5AptnV3bm0akSCRqmdqPG+PbB06VJMmTIFtbW1GDp0KCoqKrBo0aKQzfRJ4ri6wAxtiwlJSabgYxqBh58TIcseWuYRhUsrEXFc1E0kNRUXF2PixInYsGEDbrjhBhw+fBjvvfee2mGRCIaaOMCkA3Tnt93VCDy8nAiJualGFAVVm2ZPPvkkOI4LuY0ZM0bNkBLGkiVLUFFRgfLycsyZMwd5eXndv4iowx/ojBbO/5HTCBwkXoQsM0DyqxTYwKF6H9HYsWNRV1cXvH322Wdqh5QQ7r77bpw+fRp//vOfsXjxYrXDIRFIMsOJ+iY0O72QQxIRD5kTITHQqFkUVE9EoigiKysreEtLS1M7pIRgtVrx/e9/HyaTCfPnz1c7HBKByydh97E6HKtvB3fBXCGNoKzAl2RGTbMoqJ6Ijh8/jpycHAwfPhwLFizAqVOnIpb1eDxoa2sLuQ1mtbW1WLBgAXQ6GvtNVB6fFFzecXEikjlqmkVL1UQ0bdo0VFRU4P3338e6detQVVWFa6+9Fu3t7WHLr169GlarNXgbrP0mLS0teOutt7B9+3YsW7ZM7XBIF0KWd4jnT3QdkqTFyOwUZFh0VCOKgqqjZjfddFPw3xMmTMC0adMwbNgwvPHGG7j//vs7lS8rK0NpaWnwfltb26BMRsXFxWhpacFvf/tbjB49Wu1wSBe8gUQk8lxIZ3VKkhYpQ1MBpqfh+ygk1PB9cnIyRo0ahcrKyrDP63S6S6KZEuncMpJ4PH4ZAvOGrLwP4gO/XlQj6pbqfUQXstvtOHHiBLKzs9UOhZCoePxScOP8C2tEsszQ6gXsHj/1EUVB1US0YsUK7NixAydPnsS//vUvfO9734MgCLjrrrvi9h6sBwtOSWf09eua1y9DZF6IPB9SI3L7Jbx/5BwO1dog0/B9t1Rtmp0+fRp33XUXmpqakJ6ejmuuuQa7du1Cenp6r6+t0SjLKpxOJwwGQzelSSROp3KCacfXk4QqTEvC0CwDkry68yvvcX7UDAAkvzexmh4JSNVEtGnTpj67tiAISE5ORkNDAwDAaDTSWq0eYIzB6XSioaEBycnJEAQ6iyKcISYdYOIBpyakRiTyHFigj0jyeUFpvGsJ1Vkdb1lZWQAQTEak55KTk4NfRxJBx/n2FyQijuPAB9Y1yn5qmnVnUCcijuOQnZ2NjIwM+Hw0ctFTGo2GakLdqGl2wtjWDrOGg1YIXVDNBXZdkPz0s9edQZ2IOgiCQL9QpE/sqmxA7plWjMo0YchFw/cc1YiiRn1ohPSCz+sCAGXUTAhNRLyo1IhkqhF165KoERHSV/xeZW9xQaMB+NC/66NzUpHSYoCel9UIbUChRERIL0iBGhGv6TxFZEJ+GnDGCAiUiLpDTTNCYiTLDHLgbHtRG2bpER8YtJf9PTrJ5VJEiYiQGHklZcErAIjazjUip8TB6fXD55cAWerv8AYUSkSExMjTcdQ0B/BhDlDcUdmKA6dtaLR7aJfGblAfESEx0mt4zCgww8QnhT2UQRRFMI5XNkejFfhdokRESIx0ooDhKRqgRd9p6B4ARIFTdmlkoBX43aCmGSG9EVzeoe/0lIbnIXEiZEY1ou5QIiIkRjanD3XNNji8/vBNs2CNiNEujd2gRERIjL49Z8eeE/WobXGFb5rxgUQkg2pE3aBEREiMlP2qfcp+1WFrRPz5GhEN33eJOqsJiVHICR5hakQZZh306RakeJ3UNOsGJSJCYhScRxRu43wAOckG5OSkAk0t1DTrBjXNCImRxy9BlD0QBT5sIgJw/iQPGr7vEiUiQmLU0UcUqWkmyQwuiYfbLynrzUhElIgIiZHHJ0GQPYGmWefO6rNtbnz473P4pq6NmmbdoD4iQmI0dZgFSfVGGLVC2AmNIs8pExplUGd1NygRERKjESkiYNEDXOjhih1o+D561DQjJFYde1ELWiUZXUSZWS3QEo8oUCIiJAaSzHCyoQltbh9YmNoQoKw161j0Shvody1hEtEzzzwDjuPwyCOPqB0KId1y+yR8dOg0jpxpC9s/BJxfawbQBvrdSYhE9OWXX+Lll1/GhAkT1A6FkKh4/DJEpoyYcZoIiYg/n4gk6qzukuqJyG63Y8GCBfjzn/+MlJQUtcMhJCoev9TlHCJAOeCzKCsZWVY9eJpH1CXVE9GyZctw8803Y86cOd2W9Xg8aGtrC7kRogZlMqMn4vKODleNzELhkCRoOBo164qqw/ebNm3CV199hS+//DKq8qtXr8ZTTz3Vx1ER0j1PyMr7yIkouMSDakRdUq1GVFNTg4cffhivvvoq9PrwbeyLlZWVwWazBW81NTV9HCUh4Xl85/uIukpEHsbDK8mQqLO6S6rViPbu3YuGhgZcccUVwcckScInn3yCP/7xj/B4PJ3Oq9fpdNDpuvjrQ0g/cQf6iESh81HTF/r7oUbkVLdgdL4eqf0Y30CjWiKaPXs2Dh06FPLYfffdhzFjxuCxxx7rlIQISSTDUo1Iy9IhxaHtskbEa5RDFmVafd8l1RKR2WzGuHHjQh5LSkrCkCFDOj1OSKLJsOiBZA3Auk5EgtCRiGhCY1dUHzUjZMDq4gSPDrwYOHZaomOnu5JQi163b9+udgiERKXO5oKh3Q6TLEOMsMQDAPjA9iCSHDjJI8x2IYRqRITEZMfRRhw62YA2l7/LGpEQqBHJDDSE3wVKRITEwOOTIDAvRCH8pmgdNKKgHDtNK/C7lFBNM0IGCq/XA47JgXlEkWtE2VYDBKsJSVpG+1Z3gRIRIT3EGIPfq3RUCwIfdlO0DpdlW4DsFMBjp6ZZF2Jqmn377bfxjoOQAcMvMyAwHC9q9GE3RQsRXOZBTbNIYkpERUVFmDVrFv7617/C7XbHOyZCEprHL0OU3eAACFpDl2VlmcEHAX5Zpn2ruxBTIvrqq68wYcIElJaWIisrCw8++CC++OKLeMdGSELy+DrOM+PAabpORF+fsWH78RacaHTQvtVdiCkRTZo0CWvXrsWZM2ewfv161NXV4ZprrsG4cePwwgsvoLGxMd5xEpIwDFoBV+cbMTTZAGiMXZYVeV7Zt1qmUbOu9Gr4XhRF3H777XjzzTfx29/+FpWVlVixYgXy8vJw7733oq6uLl5xEpIwjFoRo4dokG01ABF2Z+ygCWwXKzNGTbMu9CoR7dmzBz/96U+RnZ2NF154AStWrMCJEyewdetWnDlzBrfddlu84iQksfhcyv/Frptm548UAtWIuhDT8P0LL7yA8vJyHD16FPPmzcOGDRswb9488LyS1woLC1FRUYGCgoJ4xkpIQrC5fPC3tsLkl6Drpo9I5DnIvAhZorPNuhJTIlq3bh0WL16MRYsWITs7O2yZjIwM/OUvf+lVcIQkon/XtaHxaC0uE1wY0V0iEjjIEM6vNSNhxZSItm7divz8/GANqANjDDU1NcjPz4dWq8XChQvjEiQhicTlk6CR3dBo+S5nVQOBzmpepCUe3Yipj2jEiBE4d+5cp8ebm5tRWFjY66AISWRunwxR8ij7VXdTIzJoBWSnmpFq1NLM6i7EVCNiEfZVsdvtUe8/TchA5fZJ0AfmEXWXiEw6EVeNyAQEE60160KPElFpaSkA5bymlStXwmg8P4dCkiTs3r0bkyZNimuAhCQal09CuuyGKBi6bZoBAAK7NFLTLLIeJaJ9+/YBUGpEhw4dglZ7frGfVqvFxIkTsWLFivhGSEiCcbs94JkfGp7rdkIjALBAHxEv+dDNqrRLVo8S0ccffwxA2eR+7dq1sFgsfRIUIYnM53ECgHLAQ1dnmkH5o/3anjoUNjWjONkL6rgIL6Y+ovLy8njHQciAwBjD1cOMsLYboNUbu115z3EcOEH5NZPpbLOIok5Et99+OyoqKmCxWHD77bd3WXbz5s29DoyQRMRxHMala4AUI6BLiuo1fKCPSPLTSR6RRJ2IrFYruED2t1qtfRYQIQnP61D+r40uEXV0Vst+GjWLJOpEdGFzjJpm5FLl8kpob2qGxSdBH2UiCm6gT2ebRRTThEaXywWn0xm8X11djTVr1uDDDz+MW2CEJKLaVic+PXwSlY32qGtEHWebMeojiiimRHTbbbdhw4YNAIDW1lZMnToVzz//PG677TasW7curgESkkhcXhkayaXMqo62RhTY01qmeUQRxbxD47XXXgsA+Nvf/oasrCxUV1djw4YN+P3vfx/1ddatW4cJEybAYrHAYrFg+vTp2LJlSywhEdIvXD4JGskFjcBHnYgyU0xITdJCw8l9HN3AFdPwvdPphNlsBgB8+OGHuP3228HzPK666ipUV1dHfZ3c3Fw888wzGDlyJBhjeOWVV3Dbbbdh3759GDt2bCyhEdKnlAWvHTUiU1SvuaooE2hQfl/AWPeb7V+CYt48/+2330ZNTQ0++OAD3HDDDQCAhoaGHk1yvOWWWzBv3jyMHDkSo0aNwtNPPw2TyYRdu3bFEhYhfc7llaCRnBB7UCMKLvEAaOFrBDElopUrV2LFihUoKCjAtGnTMH36dABK7ai4uDimQCRJwqZNm+BwOILXu5jH40FbW1vIjZD+5PH5A02z6JZ3AAB4ESzwHyWi8GJqmv3gBz/ANddcg7q6OkycODH4+OzZs/G9732vR9c6dOgQpk+fDrfbDZPJhLfeeguXX3552LKrV6/GU089FUvIhMSF22kHBwZREKKuEe2obIa2qgW5yXrkSb5uV+xfimLeszorKwvFxcUhm6NNnToVY8aM6dF1Ro8ejf3792P37t34yU9+goULF+LIkSNhy5aVlcFmswVvNTU1sYZPSEwmZ/HITTHAkGQGeCGq13AAJE4MnORBNaJwYqoRORwOPPPMM9i2bRsaGhogy6GjAT05CVar1aKoqAgAMHnyZHz55ZdYu3YtXn755U5ldToddLquFxkS0peKLFCWd5hTon6NKHDwcyJkJlMiiiCmRLRkyRLs2LED99xzD7Kzs4NLP+JBlmV4PJ64XY+QuPIE+iV15qhfohF4eDkBEqPTXiOJKRFt2bIF7733HmbMmNGrNy8rK8NNN92E/Px8tLe3Y+PGjdi+fTs++OCDXl2XkL7g9kloO9cIs0+CQRf9ekuRV842Y8xLm6NFEFMiSklJQWpqaq/fvKGhIXgQo9VqxYQJE/DBBx/gO9/5Tq+vTUi8NbZ7cOCbk8j1tWPSZdFPU9EEzjaTZDpSKJKYEtGvfvUrrFy5Eq+88krIdrE9RccNkYHE6ZWglRzK0H0PmmYCz50/ZJGaZmHFlIief/55nDhxApmZmSgoKIBGowl5/quvvopLcIQkEofXD63fDo2eB3TR14jMehEpZiNMfhs1zSKIKRHNnz8/zmEQkvjcbjd0kgMaUQcYkqN+XW6KEbnDM4BzNqoRRRBTInriiSfiHQchCc/b3gQdGASNPvpZ1R2CJ3lQH1E4MU9obG1txf/+7/+irKwMzc3NAJQmWW1tbdyCIySRSHblUFHBlNbzhat84G8+Nc3CiqlGdPDgQcyZMwdWqxUnT57E0qVLkZqais2bN+PUqVPBvYoIGUxkp/IHVzAN6dHrmh1e/OtwI7LbWzB5GCWicGKqEZWWlmLRokU4fvx4yMmu8+bNwyeffBK34AhJJJdZfchLMcCUktmj1wkcB48sQJJpZnUkMdWIvvzyy7BLMIYOHYr6+vpeB0VIIsrVOpXlHT1MRKLAQYYAmQGMDlkMK6YakU6nC7sFx7Fjx5Cent7roAhJOIwBLqVpBkPPJvOKAgeZF8FAZ5tFElMiuvXWW7Fq1Sr4fMoXleM4nDp1Co899hi+//3vxzVAQhKBy26Drd0Bt18GDNEveAUAkVdmVgOARCd5hBVTInr++edht9uRnp4Ol8uFkpISFBUVwWw24+mnn453jISo7uzZMzhS14bDLTwg9KxHQ+C54KiZRDWisGLqI7Jardi6dSt27tyJAwcOwG6344orrsCcOXPiHR8hCcHTpgzdc8bY1lhywUMWKRGF0+NEJMsyKioqsHnzZpw8eRIcx6GwsBBZWVlgjMV1SxBCEoWvvRFAYA5RDNKsSbC2a8DRqFlYPWqaMcZw6623YsmSJaitrcX48eMxduxYVFdXY9GiRT3eJpaQgUJyNAEAND2cQ9RhztihuDzbAqPI4hnWoNGjGlFFRQU++eQTbNu2DbNmzQp57qOPPsL8+fOxYcMG3HvvvXENkhC1sUAi0lpiHBXumFktUY0onB7ViF577TX813/9V6ckBADXX389Hn/8cbz66qtxC46QhCD5wVytAAC9JSO2awTXmlEfUTg9SkQHDx7EjTfeGPH5m266CQcOHOh1UIQkEuZqhs8vQeI1MJmj3/7jQh/+uwlfnmzGuTZHnKMbHHrUNGtubkZmZuRZpZmZmWhpael1UIQkEuZoQkFaEtq1GUjSa7p/QRhexsEvMxo1i6BHiUiSJIhi5JcIggC/n9rAZHDh3S3IMOuRkZkHCLFtWMEJyukzsuSjY6fD6FEiYoxh0aJFEY/0odM3yKDkVDqqYYxtxAwARDEwj0iWlT2JejgpcrDr0Vdj4cKF3ZahETMy2DhbG+Bz+SCIVphivAYvagEolSHIfkpEF+nRV6O8vLyv4iAkMTGGcw11qD3XBm8Wh5Lc2C4jigIALnDaqw+AvruXXFJi3qGRkEuC1wG/xwWAg84ce9NMIwiQOBESY7RvdRiUiAjpiqsZXkmGR0yCyWiI+TIWg4gkox5akad9q8OghiohXXE2w+OX4RaTYdLF/usyITcZGJYGuNtoUmMYqtaIVq9ejSlTpsBsNiMjIwPz58/H0aNH1QyJkFCuFnj9MlyiFWZ9L/9u84E5SNQ060TVRLRjxw4sW7YMu3btwtatW+Hz+XDDDTfA4aDZpyQxeO3n4JcZ3BoLzDFOZgzqGCmjFfidqNo0e//990PuV1RUICMjA3v37sV1112nUlSEnOdtV+YQMX2K0r8To28b7Th9ogWZrB2jKRF1klB9RDabDQCQmhp+8ymPxxMyaTLcvtmExA1j0PnaMCI9CeaRhb26lMwAp5+Hj8nUNAsjYUbNZFnGI488ghkzZmDcuHFhy6xevRpWqzV4y8vL6+coySXFa4cGEjIsRowbkd+rS2kFZd9qiTHqrA4jYRLRsmXL8PXXX2PTpk0Ry5SVlcFmswVvNTU1/RghueQEDlSE3gLwQq8upRE5yJwYmNBIw/cXS4im2fLly/H3v/8dn3zyCXJzI09d1el0Ede5ERJ3rhbYXD74tElI8kowaGNPRhqBh8wJNKExAlVrRIwxLF++HG+99RY++ugjFBb2rh1OSFy5WnC6xYlPT0s41ezs1aU0gabZ+SUe5EKq1oiWLVuGjRs34p133oHZbA6eEmu1WmEwxD6LlZC4cAUmMxosvZ5DpBX4wBIPQPb7E6dPJEGo+vVYt24dbDYbZs6ciezs7ODt9ddfVzMsQgAAslOZzOiOw2RGjcDBZNDBpBPpkMUwVK0RMUYnGpAExRi89iYwAD6ttVfLOwBAFHjMnZAHfPstAOqsvhjVEAkJx2uH1+MBAwdNUkp8zuvrWOJBExo7oURESDjOZnj9MryiCSZDnEZqO6YA0KhZJwkxfE9IwnG1wOOX4RJTe7/GLGD7iVZYT7WgQO9ESlyuOHhQIiIkHFcLUowaXJY1DHx2bEcIXczu5aDzy/D7qLP6YpSICAnH1QyjVkRBbi4wxBiXSwqBDfQlOlKoE+ojIiQcV+B8PmP4BdixEDTKBvqyn2pEF6NERMjFGANcLWhs9+C0SwdJjs80E0pEkVEiIuRinnb4fV4cb3Tgb0fa45iIlJM7mM8dl+sNJpSICLmYqwUenwyPaIZBp+3VhmgXEjSB0179gdNeSRAlIkIu5mqG2y/BLVpgNcRn6B4AjAYDDBoeIg+aS3QRGjUj5GKBGpFbY41rIpo8PBOoSVVqQ5IHCJz+SqhGREhnzo4aUXwTETgOEALJhzqsQ1AiIuRiwVnVVljimYgAQAwsF5E8XZe7xFDTjJALMQa4WuHxSXFvmtU0O3HmlB2paMdI2gokBCUiQi7ktgGyH4UZFojjRiDNFL+tiWXG0ObloIdETbOLUCIi5EKBGdXWlAxYc+O7NFUr8pB4LSQ/o6bZRaiPiJALuQIndxjivz5eJwqQeI0yQZJqRCEoERFyIWcLHB4/Tjp1aGiP7wxonchD4pRExKhGFIISESEXcjSiyeHFZ3Ucvq61xfXSStNMo2w/66FlHheiRETIhRyNcPkkuDTJSDHGd8KhyHNggtL57af1ZiEoERHSwesEvA64vBJcmhSkJsU3EXEcB0NgmYfspabZhWjUjJAOznOQGYONGSHxWqTEOREBwE0T84Fj/wY0ctyvPZBRjYiQDo5GePwyHJoUaEUe5l4eIRRWoGlGo2ahKBER0sHeGGyWJRs18TlC6GK0xCMsVRPRJ598gltuuQU5OTngOA5vv/22muGQS137Gbh8EhzaNKTGuaO6w95aBw6ebkVdU1ufXH+gUjURORwOTJw4ES+99JKaYRCi7A9kb0SmRYdZU4pRnN83B/7Y/QIcXglej6tPrj9QqdpZfdNNN+Gmm26KurzH44HHc75K29ZGf1VInNjPAkyGqDcjLztT2bKjD2h0BgCATMP3IQZUH9Hq1athtVqDt7y8PLVDIoOFrVb5vyWnz5IQAIg65Wgi2e+lXRovMKASUVlZGWw2W/BWU1OjdkhksGj+Fg6vH/vtFnzbaO+zt9Fo9WDgAuvNqFbUYUDNI9LpdNDp4rctAyEAlKF0Ww1aHF58LiVjWH07hqeb+uStdBoRdl4HSZYAnxvQmfvkfQaaAVUjIqRPtJwEZAlNfj3cohVDkw199lY6DQ8/r4NfZoCfOqw7UCIipP4gZMZwks8HOA65KX2XiAwaAZzWAJHnlBoRAaBy08xut6OysjJ4v6qqCvv370dqairy8/NVjIwMRowxOL0SDBoBPB/okLY3Ak2VaHX6UGccBbNejPsaswvlJBuQMzYfaPJRjegCqiaiPXv2YNasWcH7paWlAICFCxeioqJCpajIYFTZYMf2ow1od/uh0/C4clgqJucmQTj6D4AxnORz4dKkoDjD1Dczqi8kKie+Uo3oPFUT0cyZM8HoxEvSxw6fseHDw2eD9z1eP47t3wntnsOYmCbDKWuwm58EABibY+37gDSBph/ViIIG1KgZIT1V0+zEP480AAAm5llx7RA7mva/j5rWGuSmJYHTpsJVcDP0J3nkJGmRbu77UdnPTzlgPNOKwtR2WPr83QYGSkRkUDPrRaSatEhL0mKWsQrc1x8jSwSGjMiBOOwqYOhkpIla3J0pKXN7+kGrlwe8EjwuZ7+830BAiYgMaslGLX54ZS64bz8Gd+JL5cGcYmiGzwQ0+mA5nSj0W0y8LgkA4He399t7JjpKRGTQcXj8qLO5UZRhAmQJmuP/AOq/Vp4ccT2QP03V+DQGZRKj3+1QNY5EQomIDGj1Nje+rrWh2emFT5IhyQw2pw9+meGGUVaMbf0YaK4COB4YMw/IGq92yMFEJHuoRtSBEhEZsPadasGOY40IN/A6XNuK/Op/ApINEETg8u8BaUX9H2QYWoMFMgDJ61IWvgrxO9Z6oKJERAas4Wkm7K1uwdBkA4anm6AVeYiyF9b6f8HcdBAcAOhMwLgfAJZstcMNMhiNaOcEeP0y4HUAhmS1Q1IdJSIyYFmNGtwzfdj5juamE0DlFqCjyZM1XukT0hrVCzIMk16Ddk0SBN4NeO2UiECJiAwwjDG0ufywGpXmjE4UAMkPVP4TOLNPKWRIAUbNBVILVYw0ssK0JAy/fBjQdkapERFKRGRgOXbWjve/rsfkYSm4ZmSaskzi6/8HtJ5SCuROAYaXJHS/C8dxSpMRUGpEhBIRGTj8koydlcrZYxqBUzp6D72h7K4oaoHL5wNDRqgdZnS0gUTkoUQEUCIiA8jBWhtsLh9MOhHFeVbgyNuBJKQDJi0AzJlqhxi1L+t80J5uRa65CUOGqx2N+mg/IjIguH0SvqhqBgBcVZgKbdVHwLnjAC8A438woJIQALTISXB6JXjaz6kdSkKgREQGhD0nW+DyShhi0mKs9A1Qu1fZ5P6yW4Dkgbd3ldacCgDw2VtUjiQxUCIiCc/m8mHfKeUXdlZKI/iq7coTI2YDGZepF1gvmJLTAQA+p42OnwYlIjIANNk94HkOY3RNyK3/SHkwb4pyG6CSLRb4eR2cPglwt6odjuooEZGENzzdhPsm6jHT/yk4JgMZY5Ta0ACWmqSFW7TA7ZXAnM1qh6M6SkQk8TmbYfzm/8HASUDKMGDMLX16CGJ/sBo08GqtkBjgtDWoHY7qaPieJCRZZvjwSD1GW3woPP0u4HUCpgxg7O3KItYBTuA5CJZsmH0nIdvq1Q5HdQP/O0oGHZ8kY+uRs6itPg5z0z8xdKgOWksGMOHOkM3MBrobpk4Af/AI4KMhfEpEJGE4PH4cb7Djq5PnkNTwFcbZ9mJUhgFaaxYw8YeANkntEOOKt2QrTUy3TZlhreub02UHAkpERHWtDjd2HDiO1nN1MHqbMNxxFBY4MSLLBOuw8cDoecrs6cFGowdMmfC2ngF37gQ0QyeqHZFqKBGRfscc5+BpqITedRawN8DkaEZWVSMyGGDWiRhi1SJ9yFCIRbOUrTwGeMd0V/a0J8NffRj5+sPIoUSkrpdeegm/+93vUF9fj4kTJ+IPf/gDpk6dqnZYiYkxoLVa2f7U1QxwPCTDEJzmstAqpGNifsoFRVnUhwW6vBLOtrnh8Poh8BxMOhGZFj00QpwGVr1OeM4cxNlje9BytgYcgPG5VnDgIAIoykqBMTkdBmsGkDpcmaiYwCvo40VOGwP27U601BxGziXcPFM9Eb3++usoLS3Fn/70J0ybNg1r1qzB3LlzcfToUWRkZKgdXuLw2IGzXwNn9gOuFjAwOD0SGuweNNk98EkMXo0V8ozrwGePB4yp+OjfDbB7/BiVacaIwA6GkGVl6wl3q9I34W7D2XYPtlW2wS1a4NSkwscbAI4Dz3HIsuowITcZl2X38AQuxgC/B1JLNc5VHURr9SG02p2QGcDAwWEciuFDi2FKHQokDcEQffKgrvlEMrqoCF8eyAScZ3H24D+ROWW+2iGpgmMqH7U6bdo0TJkyBX/84x8BALIsIy8vDz/72c/w+OOPd/natrY2WK1W2Gw2WCyD7Kg6yackCttpoKlS2X2QyXB4/TjnAqowFHVyMjgmw+RtRKavBql6DsOGGCHyPGRDKraekuCWNeCZBD1zYYjoRhJzQJYkWI0a5FiVE0clmWFPdTN0ogCdhoeP16MZVjRzyXBqhmBiUS7GD8sCeBFOtxtHTjcjSfBCDw80kgui5IHgdwE+J8yiD3rmAbxO1LfacarZGTwvzKkZAk/mBAwddQVG52ZAr+m/I3wS2ed7vwL2b4TAc8gaV4KssddBaxwcP8/R/o6qWiPyer3Yu3cvysrKgo/xPI85c+bg888/71Te4/HA4/EE77e1tUX3Rs1VwIltCLfL+ukWJ860KAfddTzLgQX/PTbHApNW+TKdaXWiJlD2wmsxABxjuCzHAoteAzCG+jY3qpucAJPDhjQ6y4xkg3LdhnYPqhrbg5flwCDI5z/n8HQTUo1awJKNU9wIfNiYCpnXQOQ5DE83YWyOBflWEXzTcaDhCNBcBd7VjGutfjQ5vDhn98Dtk+EB4EGgRiKYkJMyDNBbIQAoTnVA620BXC0AY2BwwuO3o81VBUvdAeCckjQc7R74Gu1ojfClHplhgt6kdCxrBR4O3gyXtQDWwkkYP3wk0i36vj9bfoC5ctIk7Gysgq72c9Qe3I4zB7djQmEmjIYkgBdwstmFs20eABwQ5ks3IdcKQyCp1zQ7ccbmjvhe43KsSNIpZWtb3DjdGvmQx8uzLTDrlZ/ROpsbJp0YvA8AGP8fgD4+CVPVRHTu3DlIkoTMzNAtHDIzM/Hvf/+7U/nVq1fjqaee6vkb+T2AvTHsU8zhhOTo4gxypwRIgS+TywXZGf4bxwAwFwAE+jU8bshdnFvFPBwgaJU7Xjdk7/kYGKCc8sBp4NQOQVb2OKSOmgyY0pFu92Ckphl5KUaMzDSF1iqyxik3rxNor4PR2Qyj5EUuL6BN0uGcTwcnnwROb0GKSQ8kG4Iv1Xb8Q/IBjnPgHA3QOxqhtzcCPoeyE6Lsh2DUYUi6CU6mhYfTwcvp4OUN8HE6SKIBzsJsIDMN0Bhg4g0Y6wXSTTpKPl3QCDyu+c73cPhALtqOfQrRUQ8t8wJuPwCAORyQ7ZGTC2f3AoGfA2Z3Qm7v4ufZ4QF8YnRlzW7ArwmUdSmv81/Qbxfhj2wsVO8j6omysjKUlpYG77e1tSEvL6/7FybnKfNQgIv6IThY3X7wgW946FPKHY1JBwQ2Z7d4/CjwSBcU4UKK641aQAiU9UkY4VbKcnznX0KDXhP44eFg8UsY6fGj488dx3FgGiMgKrUHk14MxpBm0mHe+G5OpNAalZ0KA7sVcgCsgVu3BI1y4kWEUy9SA7doaAFkDMJR974gCjwmXjENuGIanE4HBNmh/FFgMlKdHug8vrDHJgGAYNEBgZ+xZLcPglsKXxCAaNYCgQEIi9uP/MDPfjhakxYQlbImjx9agQ/eVwrEb16XqokoLS0NgiDg7NmzIY+fPXsWWVlZncrrdDrodDH8ZGuTIm6kbgncomEK3KJhDNyioQ/cCAEAozEJwPlfcmtylH9E0LOf556W7UuqLnrVarWYPHkytm3bFnxMlmVs27YN06dPVzEyQkh/Ur1pVlpaioULF+LKK6/E1KlTsWbNGjgcDtx3331qh0YI6SeqJ6I777wTjY2NWLlyJerr6zFp0iS8//77nTqwCSGDl+rziHpjUM8jImQQiPZ3lDZGI4SojhIRIUR1lIgIIapTvbO6Nzq6t6Je6kEI6Vcdv5vddUUP6ETU3q6sz4pqdjUhRDXt7e2wWiNPyxzQo2ayLOPMmTMwm80DZi1Tx7KUmpqaQTXSNxg/12D8TED/fi7GGNrb25GTkwOej9wTNKBrRDzPIzc3V+0wYmKxWAbVD3eHwfi5BuNnAvrvc3VVE+pAndWEENVRIiKEqI4SUT/T6XR44oknYttFIIENxs81GD8TkJifa0B3VhNCBgeqERFCVEeJiBCiOkpEhBDVUSIihKiOElEfe/rpp3H11VfDaDQiOTk5qtcwxrBy5UpkZ2fDYDBgzpw5OH78eN8G2kPNzc1YsGABLBYLkpOTcf/998Nut3f5mpkzZ4LjuJDbj3/8436KOLyXXnoJBQUF0Ov1mDZtGr744osuy7/55psYM2YM9Ho9xo8fj3/84x/9FGnP9ORzVVRUdPq+6PX9vIs6I31q5cqV7IUXXmClpaXMarVG9ZpnnnmGWa1W9vbbb7MDBw6wW2+9lRUWFjKXy9W3wfbAjTfeyCZOnMh27drFPv30U1ZUVMTuuuuuLl9TUlLCli5dyurq6oI3m83WTxF3tmnTJqbVatn69evZ4cOH2dKlS1lycjI7e/Zs2PI7d+5kgiCwZ599lh05coT993//N9NoNOzQoUP9HHnXevq5ysvLmcViCfm+1NfX92vMlIj6SXl5eVSJSJZllpWVxX73u98FH2ttbWU6nY699tprfRhh9I4cOcIAsC+//DL42JYtWxjHcay2tjbi60pKStjDDz/cDxFGZ+rUqWzZsmXB+5IksZycHLZ69eqw5f/jP/6D3XzzzSGPTZs2jT344IN9GmdP9fRzRfuz2ZeoaZZgqqqqUF9fjzlz5gQfs1qtmDZtWtjTb9Xw+eefIzk5GVdeeWXwsTlz5oDneezevbvL17766qtIS0vDuHHjUFZWBmeEAyv7Wscpwxd+nbs6ZRhQPveF5QFg7ty5CfN9AWL7XABgt9sxbNgw5OXl4bbbbsPhw4f7I9ygAb3odTCqr68HgLCn33Y8p7b6+npkZGSEPCaKIlJTU7uM8e6778awYcOQk5ODgwcP4rHHHsPRo0exefPmvg65k56eMgwonzuRvy9AbJ9r9OjRWL9+PSZMmACbzYbnnnsOV199NQ4fPtxvi8qpRhSDxx9/vFPn3sW3SN/0RNbXn+uBBx7A3LlzMX78eCxYsAAbNmzAW2+9hRMnTsTxU5Cemj59Ou69915MmjQJJSUl2Lx5M9LT0/Hyyy/3WwxUI4rBo48+ikWLFnVZZvjw4TFdu+OE27NnzyI7+/yxz2fPnsWkSZNiuma0ov1cWVlZaGhoCHnc7/ejubk57Am9kUybNg0AUFlZiREjRvQ43t7o6SnDgPK96Ul5NcTyuS6m0WhQXFyMysrKvggxLEpEMUhPT0d6enqfXLuwsBBZWVnYtm1bMPG0tbVh9+7d+MlPftIn79kh2s81ffp0tLa2Yu/evZg8eTIA4KOPPoIsy8HkEo39+/cDQEjC7S8XnjI8f/58AOdPGV6+fHnY10yfPh3btm3DI488Enxs69atCXUqcSyf62KSJOHQoUOYN29eH0Z6EVW7yi8B1dXVbN++feypp55iJpOJ7du3j+3bt4+1t7cHy4wePZpt3rw5eP+ZZ55hycnJ7J133mEHDx5kt912W0IO3xcXF7Pdu3ezzz77jI0cOTJk+P706dNs9OjRbPfu3YwxxiorK9mqVavYnj17WFVVFXvnnXfY8OHD2XXXXafWR2CbNm1iOp2OVVRUsCNHjrAHHniAJScnB4eu77nnHvb4448Hy+/cuZOJosiee+459s0337AnnngiYYfve/K5nnrqKfbBBx+wEydOsL1797If/vCHTK/Xs8OHD/dbzJSI+tjChQsZgE63jz/+OFgGACsvLw/el2WZ/fKXv2SZmZlMp9Ox2bNns6NHj/Z/8F1oampid911FzOZTMxisbD77rsvJLlWVVWFfM5Tp06x6667jqWmpjKdTseKiorYf/7nf6o6j4gxxv7whz+w/Px8ptVq2dSpU9muXbuCz5WUlLCFCxeGlH/jjTfYqFGjmFarZWPHjmXvvfdeP0ccnZ58rkceeSRYNjMzk82bN4999dVX/RovbQNCCFEdjZoRQlRHiYgQojpKRIQQ1VEiIoSojhIRIUR1lIgIIaqjREQIUR0lIkKI6igRkQGloqIiZMvdJ598MmQx8KJFi4JrrMjAQYmIhLVo0aKIe0ovW7YMHMeFrNTviwRQUFCANWvWhDx255134tixYxFfs3btWlRUVATvz5w5M2SRKklMlIhIRHl5edi0aRNcLlfwMbfbjY0bNyI/P1+VmAwGQ6dN2S5ktVqjPqSAJA5KRCSiK664Anl5eSE7KG7evBn5+fkoLi7u1bXD1VTmz58frGXNnDkT1dXV+PnPfx7clA3o3DS72IU1s0WLFmHHjh1Yu3Zt8BpVVVUoKirCc889F/K6/fv3g+O4ft2Dh5xHiYh0afHixSgvLw/eX79+Pe67774+f9/NmzcjNzcXq1atQl1dHerq6np8jbVr12L69OlYunRp8Br5+fmdPhMAlJeX47rrrkNRUVG8PgLpAUpEpEs/+tGP8Nlnn6G6uhrV1dXYuXMnfvSjH/X5+6ampkIQBJjNZmRlZcW0C6LVaoVWq4XRaAxeQxAELFq0CEePHg2e9eXz+bBx40YsXrw43h+DRIl2aCRdSk9Px80334yKigowxnDzzTcjLS1N7bB6JScnBzfffDPWr1+PqVOn4v/+7//g8Xhwxx13qB3aJYtqRKRbixcvRkVFBV555ZW41Rp4nsfFW2H5fL64XDsaS5YsCXbEl5eX484774TRaOy39yehKBGRbt14443wer3w+XyYO3duXK6Znp4e0u8jSRK+/vrrkDJarRaSJPXqfSJdY968eUhKSsK6devw/vvvU7NMZdQ0I90SBAHffPNN8N+R2Gy24Ib4HYYMGYK8vLxOZa+//nqUlpbivffew4gRI/DCCy+gtbU1pExBQQE++eQT/PCHP4ROp4upSVhQUIDdu3fj5MmTMJlMSE1NBc/zwb6isrIyjBw5MqE2wL8UUY2IRMViscBisXRZZvv27SguLg65PfXUU2HLLl68GAsXLsS9996LkpISDB8+HLNmzQops2rVKpw8eRIjRoyI+dSUFStWQBAEXH755UhPT8epU6eCz91///3wer39MgpIukZ7VpNL1qefforZs2ejpqam08mopH9RIiKXHI/Hg8bGRixcuBBZWVl49dVX1Q7pkkdNM3LJee211zBs2DC0trbi2WefVTscAqoREUISANWICCGqo0RECFEdJSJCiOooERFCVEeJiBCiOkpEhBDVUSIihKiOEhEhRHX/H07YC6mtqHbFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:26.231009Z", + "iopub.status.busy": "2024-03-22T14:51:26.230151Z", + "iopub.status.idle": "2024-03-22T14:51:26.577757Z", + "shell.execute_reply": "2024-03-22T14:51:26.576763Z" + }, + "papermill": { + "duration": 0.370764, + "end_time": "2024-03-22T14:51:26.579890", + "exception": false, + "start_time": "2024-03-22T14:51:26.209126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8ZElEQVR4nO3deXhU9b348fc5Z7ZsM0kgK0nYwiI7ilDckMqVirVau1hrK4hLW2lvLZdWuf2VFn0stlUrbX1ob1uJ3qpU70W7KnJRUFEpoiyCgmCAsIQEsmf2c87vj5kMBCbbZDmT8Hn5nEdm5syZz8nyyXf/KqZpmgghhIVUqwMQQghJREIIy0kiEkJYThKREMJykoiEEJaTRCSEsJwkIiGE5SQRCSEsZ7M6gO4wDINjx46RkZGBoihWhyOEOItpmjQ2NlJYWIiqtl3u6deJ6NixYxQXF1sdhhCiAxUVFRQVFbX5er9ORBkZGUDkJt1ut8XRCCHO1tDQQHFxcex3tS39OhG1VMfcbrckIiGSWEdNJ9JYLYSwnCQiIYTlJBEJISzXr9uIOsM0TcLhMLquWx3Kec1ut6NpmtVhiCRleSI6evQo9957Ly+99BJer5fS0lJWr17NtGnTun3tYDDI8ePH8Xq9PRCp6A5FUSgqKiI9Pd3qUEQSsjQR1dbWcumllzJ79mxeeuklcnJy+Pjjj8nKyur2tQ3DoLy8HE3TKCwsxOFwyKBHi5imSXV1NUeOHGHUqFFSMhLnsDQR/exnP6O4uJjVq1fHnhs+fHiPXDsYDGIYBsXFxaSmpvbINUXicnJyOHjwIKFQaGAnoqZqOLAB8idB3jiro+k3LG2s/utf/8q0adP40pe+RG5uLlOnTuX3v/99m+cHAgEaGhpaHR1pb1i56DvnTWm0fBPUlMOev4AetjqafsPS39JPPvmEVatWMWrUKNatW8e3vvUt/v3f/50nn3wy7vkrVqzA4/HEDpneIZJJMKTz4Ue7OVDdhGma4D1ldUj9hqWJyDAMLrzwQn76058ydepU7rrrLu68805++9vfxj1/6dKl1NfXx46Kioo+jliItu09fJS6hkaqGgPU+0LQXGV1SP2GpYmooKCAceNa16MvuOACDh8+HPd8p9MZm84xkKd1XHnlldxzzz1WhyG6qKq6GgCHpuByaOCtsTii/sPSRHTppZeyd+/eVs/t27ePoUOHWhRR/7Jx40YURaGurs7qUATQWBepipXmZuCyaRBotDii/sPSRPS9732Pd955h5/+9Kfs37+fZ555hv/6r/9i0aJFVoYlRJfphkmouR4Alz36axVssjCi/sXSRHTxxRfzwgsv8OyzzzJhwgQeeOABHnvsMW655ZZe/dxg2GjzCOtGp88NdeLc7goEAtx7770UFxfjdDopLS3lj3/8IwcPHmT27NkAZGVloSgKCxYs6PB6jY2N3HLLLaSlpVFQUMAvf/nLc6qC//3f/820adPIyMggPz+fr371q1RVnW7vaCmJbdiwgWnTppGamsoll1xyTun2fNLoD2EPN6IqEEjNp6Y5gCklok6zfGT1Zz/7WT772c/26Wc+/tr+Nl8bPjiNG6YOiT3+r9cPENLj78pdlJXCl6ad7rl7YnM5vmDrqSTf+7fR3Yr11ltv5e233+ZXv/oVkydPpry8nJMnT1JcXMz//u//8oUvfIG9e/fidrtJSUnp8HqLFy9m8+bN/PWvfyUvL49ly5bx3nvvMWXKlNg5oVCIBx54gDFjxlBVVcXixYtZsGAB//znP1td64c//CGPPPIIOTk5fPOb32ThwoVs3ry5W/fbXzX6wzj0Jhw2lVePauQ1NnFRegMOqwPrJyxPRKJt+/bt47nnnmP9+vXMmTMHgBEjRsRez87OBiA3N5fMzMwOr9fY2MiTTz7JM888w1VXXQXA6tWrKSwsbHXewoULY/8eMWIEv/rVr7j44otpampqNUXjwQcfZNasWQDcd999XHvttfj9flwuV2I33I8NTndycaETW3ManxiDoRFCAR8O04TzZQxVN5yXiWjR7NI2X1PP+pm564qRbZ579s/Xwkt7ZlR4i+3bt6NpWuyXvbs++eQTQqEQ06dPjz3n8XgYM2ZMq/O2bdvGT37yE3bs2EFtbS2GEaliHj58uFUv56RJk2L/LigoAKCqqoqSkpIeibc/SXFopLgMMB0oRmSKUlg3IBwA+/mXmLvqvExEDlvnm8Z669zO6ExVq6c1Nzczd+5c5s6dy9NPP01OTg6HDx9m7ty5BIPBVufa7fbYv1tGTrckrfNSKDK5Wk3xYCi2SBti2CeJqBNk/kMSmzhxIoZhsGnTprivOxyRFojOLnEyYsQI7HY7W7dujT1XX1/Pvn37Yo8/+ugjTp06xUMPPcTll1/O2LFjWzVUi/g+OVFPdW0j/rCOIyWNsOokbJgQ8lsdWr8giSiJDRs2jPnz57Nw4UJefPFFysvL2bhxI8899xwAQ4cORVEU/v73v1NdXU1TU/vdxRkZGcyfP5/vf//7vPbaa+zevZvbb78dVVVjJZqSkhIcDge//vWv+eSTT/jrX//KAw880Ov32t/tPFjJ/uomGgMGTmcqYdVxukQkOiSJKMmtWrWKL37xi9x9992MHTuWO++8k+bmZgCGDBnC8uXLue+++8jLy+Pb3/52h9d79NFHmTlzJp/97GeZM2cOl156KRdccEGsgTknJ4eysjKef/55xo0bx0MPPcTDDz/cq/c4EIR8kT8CNmcqKU5bpETU0kYkOqSYphm/b7ofaGhowOPxUF9ff850D7/fT3l5OcOHDz8ve3E6q7m5mSFDhvDII49w++2399rnDPTvx5pX3mBoxV8YP2oETZNuw//+cwz2H8I96bMw5EKrw7NMe7+jZzovG6vPZ++//z4fffQR06dPp76+nvvvvx+A66+/3uLI+jczGGmotjlTKcxMgYLBcPwYhKWNqDMkEQ0gZ3evn23Pnj0APPzww+zduxeHw8FFF13EG2+8weDBg/sqzAHHMEyMaKO0zRFdhM/mjPw/JG1EnSGJaAApLCxk+/bt7b5eUlLCtm3b+i6o80AgbKAZkaENmsNJMGzQ4AOHP4RbD1kcXf8giWgAsdlslJa2PVhT9I5g2EAzQmiqgmZPocYXYsO+OkobG5k2VBqrO0MSkRDdlOLQuGx4Bo7KNLA5cNhUdNWObpggJaJOkUQkRDc5bCoFaQqkOUFz4rSp6IqdSB7yM4C3CugxMo5IiJ6gR6tgNgcOLVIiAgiHpGrWGVIiEqKbTjUF8J+sxx0IkaE5UVUFLTq/TA9K931nSCISopsO1Xg5VVHNMIefjGi3vWaPzgMMBdt7q4iSqpkQ3RQMG2hmpNesZfyQTUpEXSKJKAkl2y4eyRZPsmkZR6SpCmiRRDRhaC5DB6Xi0nTov7Oo+owkogHq7LWDRO8JhHRsRhCbqoItUiUbXzyYQk9KZDcPXb4XHTm/EpFpQjjY90cX/iIuWLCATZs2sXLlShRFQVEUDhw4wO23387w4cNJSUlhzJgxrFy58pz33XDDDTz44IMUFhbGVl186623mDJlCi6Xi2nTpvHiiy+iKEqrEdgffPAB11xzDenp6eTl5fH1r3+dkydPthnPwYMHu/2tGEgCoTCqGW5VIkK1gRL99ZJE1KHzq7FaD8Ebj/T9517+H7G/lB1ZuXIl+/btY8KECbEJqVlZWRQVFfH8888zaNAg3nrrLe666y4KCgr48pe/HHvvhg0bcLvdrF+/HojMfL7uuuuYN28ezzzzDIcOHTqnilVXV8enP/1p7rjjDn75y1/i8/m49957+fKXv8yrr74aN56cnJwe+KIMHOFoO5DtjDaihkAYQgoOdFzhIDitjDD5nV+JqB/weDw4HA5SU1PJz8+PPb98+fLYv4cPH87bb7/Nc8891yoRpaWl8Yc//CG2cuNvf/tbFEXh97//PS6Xi3HjxnH06FHuvPPO2Ht+85vfMHXqVH7605/GnnviiScoLi5m3759jB49Om484rSWRKTabKBGhi++e7AG7WgzI9wGxYaMru7I+ZWINHukdGLF53bT448/zhNPPMHhw4fx+XwEg8FWWwBBZGnZliQEsHfvXiZNmtRq/Z8zF84H2LFjB6+99lqr3TlaHDhwgNGju7cd0vngkmEZOGrTSUs7/TW0ayphxYZhBGSaRyecX4lIUTpdRUoma9asYcmSJTzyyCPMnDmTjIwMfvGLX7Bly5ZW56WlpXX52k1NTVx33XX87Gc/O+e1lp05RPuGpCuQ7gTn6c0O7JpKULGhm36QElGHLG2s/slPfhJrAG05xo4da2VIScHhcLRaEH/z5s1ccskl3H333UydOpXS0lIOHDjQ4XXGjBnDrl27CAROTzM4c+F8gAsvvJDdu3czbNgwSktLWx0tie3seMRZwtHGaO30Hzm7pqCrNgzDBD1sUWD9h+W9ZuPHj+f48eOx480337Q6JMsNGzaMLVu2cPDgQU6ePMmoUaN49913WbduHfv27eNHP/rROQklnq9+9asYhsFdd93Fhx9+yLp162LrT7cslr9o0SJqamq4+eab2bp1KwcOHGDdunXcdtttseRzdjzn9ZZBZ9ENkwOVp6jxBjFaJSIVQ7Ghm0ivWSdYnohsNhv5+fmxQ1YKhCVLlqBpGuPGjSMnJ4e5c+dy4403ctNNNzFjxgxOnTrF3Xff3eF13G43f/vb39i+fTtTpkzhhz/8IcuWLQOItRsVFhayefNmdF3n6quvZuLEidxzzz1kZmaiqmrceA4fPtx7N9/P+EI6W/YdZ19lI4rtdNeYXYvMwNcNU6pmnWB5G9HHH39MYWEhLpeLmTNnsmLFijZ3Cg0EAq2qGQ0NDX0VZp8aPXo0b7/9dqvnVq9ezerVq1s9t2LFiti/y8rK4l7rkksuYceOHbHHTz/9NHa7vdXXeNSoUaxdu7ZL8YiIQEiPTe84OxEZig1Dl6pZZ1haIpoxYwZlZWW8/PLLrFq1ivLyci6//HIaGxvjnr9ixQo8Hk/sKC4u7uOI+5+nnnqKN998k/Lycl588cXYGCErdpEdiFpN77Cd7p0clOZgVEEWuW6nlIg6wdIS0TXXXBP796RJk5gxYwZDhw7lueeei7u1zdKlS1m8eHHscUNDgySjDlRWVrJs2TIqKyspKCjgS1/6Eg8++KDVYQ0YwWgisqlKq8bqrDQHWUOywXRJ930nWF41O1NmZiajR49m//79cV93Op04nTJEtSt+8IMf8IMf/MDqMAasQNhAM4OtZt7HqNFfLykRdcjyxuozNTU1ceDAARm/IvqNQFiPLZx/ZonIMEzqgtAUCEsbUSdYmoiWLFnCpk2bOHjwIG+99Raf//zn0TSNm2++ucc+ox9vZDugDNTvQzBsYDNbZt6fLhH5wzov7znJrqP1GNJ93yFLq2ZHjhzh5ptv5tSpU+Tk5HDZZZfxzjvv9MikSrs9Mq3C6/VKw2wSaFmWRNMG1lLywwenMSQ/hbSg8/TMe073mgHo4WByVT2SkKWJaM2aNb12bU3TyMzMpKqqCoDU1NTYID7RtwzDoLq6mtTUVGy2pGqW7LZB6U5IV8Frb1UisqkKZrSNSA8F6f5sw4FtYP1UnKVltnhLMhLWUVWVkpKSgfnHoGV/+zMSkaIoqNF5jUZYqmYdGdCJSFEUCgoKyM3NJRSSngsrORyO2EjtgaSixktqQyMZdgWH1npCtRJddUEPy89eRwZ0ImqhadqAa5sQyeGd/VUUHatjdF46g87qvlekRNRpA+9PlBB9KBT0AUR6zbTWiUi1RUpEhpSIOnRelIiE6C3hYGTuo2a3w1lVzzGF2WTVpuBSZbWCjkgiEqIb9GiJSLWfO0RkUslgOJYKmiSijkjVTIgEGYaJEd3b3uaIM/VIjXbaG2HZ26wDkoiESFBQj0x4BbA5zi0ReXUFbzBMKKyDIStctkcSkRAJCrRsNa2AevaEV2DT/jp2HKmnuikgqzR2QNqIhEiQy65y6bAM0tW0uJsy2Gw2TEWNrFstM/DbJYlIiAQ5bRojsuxQ6zqn6x7ApimRVRpNZAZ+B6RqJkR3xKZ3uM55ya6q6IoNw5QSUUckEQmRoHpviOM19TQHw/GrZrESkSmrNHZAEpEQCfrkZBPvHqjkaK0vftVMjSYiAykRdUASkRAJiqxXHYqsVx23RKSeLhFJ9327pLFaiAS12sEjTokoN8OJK8dNVtArVbMOSCISIkGxcUTxFs4HCjNTKCzMhlO1UjXrgFTNhEhQIKxjMwLYNDVuIgJO7+Qh3fftkkQkRIJa2ojaqprpholPV/GH9ch8M9EmSURCJCgQ0tGMQLRqdm5j9YkGP698dJIPjzdI1awD0kYkRIKmD3WTVplKqkOLO6DRpiqRAY0G0ljdAUlEQiRoZJYN3C5QWm+u2EK67ztPqmZCJKplLWrNEUlGZ4mMrNZkikcnSCISIgG6YXKw6hQN/hBmnNIQROaatUx6lQX025c0ieihhx5CURTuueceq0MRokP+kM6ru46w51hD3PYhOD3XDGQB/Y4kRSLaunUrv/vd75g0aZLVoQjRKYGwgc2M9Jgp9jYSkXo6EenSWN0uyxNRU1MTt9xyC7///e/JysqyOhwhOiUQ1tsdQwSRDT5L8zPJ97hQZRxRuyxPRIsWLeLaa69lzpw5HZ4bCARoaGhodQhhhchgxkCb0ztafGpUPsMHpWFXpNesPZZ2369Zs4b33nuPrVu3dur8FStWsHz58l6OSoiOBVrNvG87EcWmeEiJqF2WlYgqKir47ne/y9NPP43LFb+OfbalS5dSX18fOyoqKno5SiHiC4ROtxG1l4gCpkpQN9ClsbpdlpWItm3bRlVVFRdeeGHsOV3Xef311/nNb35DIBA4Z796p9OJ09nOXx8h+og/2kZk087davpMf99VTeGhWsaUuMjuw/j6G8sS0VVXXcWuXbtaPXfbbbcxduxY7r333nOSkBDJZGh2KoPznWQ1O9otEan2yCaLhsy+b5dliSgjI4MJEya0ei4tLY1Bgwad87wQySbX7YJMO5jtJyJNa0lEMqCxPZb3mgnRb7Wzg0cL1RbddlqXbafbk1STXjdu3Gh1CEJ0yvF6HymNTaQbBrY2pngAqNHlQXQjupNHnOVChJSIhEjIpr3V7DpYRYMv3G6JSIuWiAwT6cJvhyQiIRIQCOloZhCbFn9RtBZ2mxbZdlpm4LcrqapmQvQXwWAAxTSi44jaLhEVeFLQPOmkOUxZt7odkoiE6CLTNAkHIw3VmqbGXRStxQUFbijIgkCTVM3akVDV7JNPPunpOIToN8KGCdHueJvdFXdRtFZi0zykataWhBJRaWkps2fP5k9/+hN+v7+nYxIiqQXCBjbDjwJojpR2zzUMkxAaYcOQdavbkVAieu+995g0aRKLFy8mPz+fb3zjG/zrX//q6diESEqBUMt+ZgqKvf1E9MGxejZ+XMuB6mZZt7odCSWiKVOmsHLlSo4dO8YTTzzB8ePHueyyy5gwYQKPPvoo1dXVPR2nEEkjxaFxSUkqQzJTwJ7a7rk2VY2sW21Ir1l7utV9b7PZuPHGG3n++ef52c9+xv79+1myZAnFxcXceuutHD9+vKfiFCJppDpsjBlkp8CTAm2sztjCHl0u1jBNqZq1o1uJ6N133+Xuu++moKCARx99lCVLlnDgwAHWr1/PsWPHuP7663sqTiGSS8gX+b+t/arZ6S2FkBJROxLqvn/00UdZvXo1e/fuZd68eTz11FPMmzcPVY3kteHDh1NWVsawYcN6MlYhkkK9L0S4ro70sI6zgzYim6pgqDYMXfY2a09CiWjVqlUsXLiQBQsWUFBQEPec3Nxc/vjHP3YrOCGS0UfHG6jee5QLNB8jO0pEmoKBdnqumYgroUS0fv16SkpKYiWgFqZpUlFRQUlJCQ6Hg/nz5/dIkEIkE19Ix274sTvUdkdVQ7SxWrXJFI8OJNRGNHLkSE6ePHnO8zU1NQwfPrzbQQmRzPwhA5seiKxX3UGJKMWhUZCdQXaqQ0ZWtyOhEpHZxroqTU1NnV5/Woj+yh/ScUXHEXWUiNKdNj41Mg+0dJlr1o4uJaLFixcDkf2ali1bRmrq6TEUuq6zZcsWpkyZ0qMBCpFsfCGdHMOPTUvpsGoGQHSVRqmata1Liej9998HIiWiXbt24XCcnuzncDiYPHkyS5Ys6dkIhUgyfn8A1QxjV5UOBzQCmNE2IlUP0cGstPNWlxLRa6+9BkQWuV+5ciVut7tXghIimYUCXoDIBg/t7WlG5I/2s+8eZ/ipGqZmBpGGi/gSaiNavXp1T8chRL9gmiaXDE3F05iCw5Xa4cx7RVFQtMivmSF7m7Wp04noxhtvpKysDLfbzY033tjuuWvXru12YEIkI0VRmJBjh6xUcKZ16j1qtI1ID8tOHm3pdCLyeDwo0ezv8Xh6LSAhkl6wOfJ/R+cSUUtjtRGWXrO2dDoRnVkdk6qZOF/5gjqNp2pwh3RcnUxEsQX0ZW+zNiU0oNHn8+H1emOPDx06xGOPPcYrr7zSY4EJkYyO1nl5Y/dB9lc3dbpE1LK3mSltRG1KKBFdf/31PPXUUwDU1dUxffp0HnnkEa6//npWrVrVowEKkUx8QQO77ouMqu5siSi6prUh44jalPAKjZdffjkA//M//0N+fj6HDh3iqaee4le/+lWnr7Nq1SomTZqE2+3G7XYzc+ZMXnrppURCEqJP+EI6dt2HXVM7nYjystLJTnNgV4xejq7/Sqj73uv1kpGRAcArr7zCjTfeiKqqfOpTn+LQoUOdvk5RUREPPfQQo0aNwjRNnnzySa6//nref/99xo8fn0hoQvSqyITXlhJReqfe86nSPKiK/L5gmh0vtn8eSnjx/BdffJGKigrWrVvH1VdfDUBVVVWXBjled911zJs3j1GjRjF69GgefPBB0tPTeeeddxIJS4he5wvq2HUvti6UiGJTPEAmvrYhoUS0bNkylixZwrBhw5gxYwYzZ84EIqWjqVOnJhSIruusWbOG5ubm2PXOFggEaGhoaHUI0ZcCoXC0ata56R0AqDbM6H+SiOJLqGr2xS9+kcsuu4zjx48zefLk2PNXXXUVn//857t0rV27djFz5kz8fj/p6em88MILjBs3Lu65K1asYPny5YmELESP8HubUDCxaVqnS0Sb9tfgKK+lKNNFsR7qcMb++SjhNavz8/OZOnVqq8XRpk+fztixY7t0nTFjxrB9+3a2bNnCt771LebPn8+ePXvinrt06VLq6+tjR0VFRaLhC5GQi/JVirJSSEnLAFXr1HsUQFds0Z08pEQUT0IloubmZh566CE2bNhAVVUVhtG6N6ArO8E6HA5KS0sBuOiii9i6dSsrV67kd7/73TnnOp1OnM72JxkK0ZtK3USmd2Rkdfo9Nk0hrNgwTEMSURsSSkR33HEHmzZt4utf/zoFBQWxqR89wTAMAoFAj11PiB4ViLZLOjM6/Ra7phJUNHRTdnttS0KJ6KWXXuIf//gHl156abc+fOnSpVxzzTWUlJTQ2NjIM888w8aNG1m3bl23ritEb/CHdBpOVpMR0klxdn6+pU2N7G1mmkFZHK0NCSWirKwssrOzu/3hVVVVsY0YPR4PkyZNYt26dfzbv/1bt68tRE+rbgyw48ODFIUamXJB54ep2KN7m+mGbCnUloQS0QMPPMCyZct48sknWy0X21Wy3ZDoT7xBHYfeHOm670LVTFOV05ssStUsroQS0SOPPMKBAwfIy8tj2LBh2O32Vq+/9957PRKcEMmkORjGEW7C7lLB2fkSUYbLRlZGKunheqmatSGhRHTDDTf0cBhCJD+/349Tb8Zuc0JKZqffV5SVStGIXDhZLyWiNiSUiH784x/3dBxCJL1g4ymcmGh2V+dHVbeI7eQhbUTxJDygsa6ujj/84Q8sXbqUmpoaIFIlO3r0aI8FJ0Qy0Zsim4pq6YO7PnFVjf7Nl6pZXAmViHbu3MmcOXPweDwcPHiQO++8k+zsbNauXcvhw4djaxUJMZAY3sgfXC19UJfeV9Mc5K3d1RQ01nLRUElE8SRUIlq8eDELFizg448/brWz67x583j99dd7LDghkskFnhDFWSmkZ+V16X2aohAwNHRDRla3JaES0datW+NOwRgyZAiVlZXdDkqIZFTk8Eamd3QxEdk0BQMNwwRTNlmMK6ESkdPpjLsEx759+8jJyel2UEIkHdMEX6RqRkrXBvPaNAVDtWEie5u1JaFE9LnPfY7777+fUCjyRVUUhcOHD3PvvffyhS98oUcDFCIZ+JrqqW9sxh82IKXzE14BbGpkZDWALjt5xJVQInrkkUdoamoiJycHn8/HrFmzKC0tJSMjgwcffLCnYxTCcidOHGPP8QZ216qgda1FQ1OVWK+ZLiWiuBJqI/J4PKxfv57NmzezY8cOmpqauPDCC5kzZ05PxydEUgg0RLruldTE5lgqsU0WJRHF0+VEZBgGZWVlrF27loMHD6IoCsOHDyc/Px/TNHt0SRAhkkWosRqIjiFKwGBPGp5GO4r0msXVpaqZaZp87nOf44477uDo0aNMnDiR8ePHc+jQIRYsWNDlZWKF6C/05lMA2Ls4hqjFnPFDGFfgJtVm9mRYA0aXSkRlZWW8/vrrbNiwgdmzZ7d67dVXX+WGG27gqaee4tZbb+3RIIWwmhlNRA53gr3CLSOrdSkRxdOlEtGzzz7Lf/7nf56ThAA+/elPc9999/H000/3WHBCJAU9jOmrA8Dlzk3sGrG5ZtJGFE+XEtHOnTv5zGc+0+br11xzDTt27Oh2UEIkE9NXQyiso6t20jM6v/zHmV756BRbD9ZwsqG5h6MbGLpUNaupqSEvr+1RpXl5edTW1nY7KCGSidl8imGD02h05JLmsnf8hjiCpkLYMKXXrA1dSkS6rmOztf0WTdMIh6UOLAYW1V9LboaL3Lxi0BJbsELRIrvPGHpItp2Oo0uJyDRNFixY0OaWPrL7hhiQvJGGalIT6zEDsNmi44gMI7ImURcHRQ50XfpqzJ8/v8NzpMdMDDTeuipCvhCazUN6gtdQbQ4gUhjCCEsiOkuXvhqrV6/urTiESE6mycmq4xw92UAwX2FWUWKXsdk0QInu9hoCXB295byS8AqNQpwXgs2EAz5AwZmReNXMrmnoig3dNGXd6jgkEQnRHl8NQd0gYEsjPTUl4cu4U2ykpbpw2FRZtzoOSxPRihUruPjii8nIyCA3N5cbbriBvXv3WhmSEK15awiEDfy2TNKdibfrTCrKZOrQwRR6UmRQYxyWJqJNmzaxaNEi3nnnHdavX08oFOLqq6+muVkGfYkk4aslGDbw2TxkuLrZwKxGxyBJ1ewcljbdv/zyy60el5WVkZuby7Zt27jiiissikqI04JNJwkbJn67m4wEBzPGtPSUyQz8cyRVH2J9fT0A2dmJrfkiRE8LNkbGEJmurEj7ToI+qW7iyIFa8sxGxkgiOkfSJCLDMLjnnnu49NJLmTBhQtxzAoFAq0GT8dbNFqLHmCbOUAMjc9LIGDW8W5cyTPCGVUKmIVWzOJKm12zRokV88MEHrFmzps1zVqxYgcfjiR3FxcV9GKE47wSbsKOT605lwsiSbl3KoUXWrdZNUxqr40iKRPTtb3+bv//977z22msUFbU9Ymzp0qXU19fHjoqKij6MUpx3ohsq4nKDqnXrUnabgqHYogMapfv+bJZWzUzT5Dvf+Q4vvPACGzduZPjw9ou/TqezzXluQvQ4Xy31vhAhRxppQZ0UR+LJyK6pGIomAxrbYGmJaNGiRfzpT3/imWeeISMjg8rKSiorK/H5fFaGJUSEr5YjtV7eOKJzuMbbrUvZo1Wz01M8xJksTUSrVq2ivr6eK6+8koKCgtjx5z//2cqwhIjwRQcz2t3dHkPk0NToFA8wZKmcc1heNRMiWRneyGBGfw8MZrRrCukpTtKDNnQ9mByNs0kkabrvhUgqpkmw6RQmEHJ4ujW9A8CmqcydVAyffAJIY/XZJDELEU+wiWAggImCPS2rZ/bra5niIQMazyGJSIh4vDUEwwZBWzrpKT3UU9syBEB6zc4hVTMh4vHVEggb+GzZ3Z9jFrXxQB2ew7UMc3nJ6pErDhySiISIx1dLVqqdC/KHohYktoXQ2ZqCCs6wQTgU7JHrDSSSiISIx1dDqsPGsKIiGJTaI5fUogvo67Kl0DmkjUiIeHzR/flSe24lCM0eWUDfCEuJ6GySiIQ4m2mCr5bqxgBHfE50o2fGu0kiapskIiHOFmgkHArycXUz/7OnsQcTUWTnDjPk75HrDSSSiIQ4m6+WQMggYMsgxeno1oJoZ9Ls0d1ew9HdXkWMJCIhzuarwR/W8dvceFJ6puseIDUlhRS7ik1FxhKdRXrNhDhbtETkt3t6NBFdNCIPKrIjpSE9ANHdX4WUiIQ4l7elRNSziQhFAS2afKTBuhVJREKcLTaq2oO7JxMRgC06XUQPtH/eeUaqZkKcyTTBV0cgpPd41ayixsuxw01k08goXUpEZ5JEJMSZ/PVghBme68Y2YSSD03tuaWLDNGkIKrjQpWp2FklEQpwpOqLak5WLp6hnp6Y6bCq66kAPm1I1O4u0EQlxJl90546Unp8f77Rp6Ko9MkBSSkStSCIS4kzeWpoDYQ56nVQ19uwIaKdNRVciiciUElErkoiEOFNzNaeag7x5XOGDo/U9eulI1cweWX42INM8ziSJSIgzNVfjC+n47JlkpfbsgEObqmBqkcbvsMw3a0USkRAtgl4INuML6vjsWWSn9WwiUhSFlOg0DyMoVbMzSa+ZEC28JzFMk3ozFV11kNXDiQjgmsklsO8jsBs9fu3+TEpEQrRoriYQNmi2Z+GwqWR0cwuhuKJVM+k1a83SRPT6669z3XXXUVhYiKIovPjii1aGI853TdWxallmqr1nthA6m0zxiMvSRNTc3MzkyZN5/PHHrQxDiIjGY/hCOs2OwWT3cEN1i21Hm9l5pI7jpxp65fr9laVtRNdccw3XXHONlSEIEaGHoKmaPLeT2RdMxZ6W2Ssf0xTWcAR1PAFfr1y/v+pXjdWBQIBA4HSRtqFB/qqIHtJ0AkwDmyuD4oK8yJIdvcDuTAHAkO77VvpVY/WKFSvweDyxo7i42OqQxEBRfzTyf3dhryUhAJszsjWREQ7KKo1n6FeJaOnSpdTX18eOiooKq0MSA0XNJzQHw2xvcvNJdVOvfYzd4cJEic43k1JRi35VNXM6nTidPbcsgxBApCu9voLa5iBv65kMrWxkRE56r3yU026jSXWiGzqE/ODM6JXP6W/6VYlIiF5RexAMnVNhF36bhyGZKb32UU67Slh1EjZMCEuDdQtLS0RNTU3s378/9ri8vJzt27eTnZ1NSUmJhZGJ80rlTgzT5KBaAopCUVbvJaIUu4biSMFmeiMlIgFYnIjeffddZs+eHXu8ePFiAObPn09ZWZlFUYmByjRNvEGdFLuGqkYbpJuq4dR+6rwhjqeOJsNl6/E5ZmcqzEyhcHwJnApJiegMliaiK6+8ElM2mhN9YH9VExv3VtHoD+O0q0wbms1FRWloe/8JpslBtQifPYupuem9M6L6TLbIjq9SIjqtXzVWC5GI3cfqeWX3idjjQDDMvu2bcby7m8mDDbyGnS3qFADGF3p6PyB7tOonJaIYSURiQKuo8fJ/e6oAmFzs4fJBTZza/jIVdRUUDU5DcWTjG3YtroMqhWkOcjJ6v1f27cPNpB6rY3h2I+5e/7T+QRKRGNAyXDay0x0MTnMwO7Uc5YPXyLfBoJGF2IZ+CoZcxGCbg6/m6ZGxPX2gLqhCUCfg8/bJ5/UHkojEgJaZ6uAr04pQPnkN5cDWyJOFU7GPuBLsrth5TpvWZzGpzjQAwv7GPvvMZCeJSAw4zYEwx+v9lOamg6Fj//ifUPlB5MWRn4aSGZbGZ0+JDGIM+5stjSOZSCIS/VplvZ8PjtZT4w0S0g10w6TeGyJsmFw92sP4utegphwUFcbOg/yJVoccS0RGQEpELSQRiX7r/cO1bNpXTbwRICMcdZQc+j/Q60GzwbjPw+DSvg8yDkeKGwPQg77IxFet57a17q8kEYl+a8TgdLYdqmVIZgojctJx2FRsRhBP5VtknNqJAuBMhwlfBHeB1eHGpKSm0qhoBMMGBJshJdPqkCwniUj0W55UO1+fOfR0Q/OpA7D/JWip8uRPjLQJOVKtCzKOdJedRnsamuqHYJMkIiQRiX7GNE0afGE8qZHqjNOmgR6G/f8Hx96PnJSSBaPnQvZwCyNt2/DBaYwYNxQajkVKREISkehf9p1o4uUPKrloaBaXjRocmSbxwf9C3eHICUUXw4hZSd3uoihKpMoIkRKRkEQk+o+wbrB5f2TvMbumRBp6dz0XWV3R5oBxN8CgkVaH2TmOaCIKSCICSUSiH9l5tJ56X4h0p42pxR7Y82I0CTlhyi2QkWd1iJ229XgIx5E6ijJOMWiE1dFYTxZGE/2CP6Tzr/IaAD41PBtH+atw8mNQNZj4xX6VhABqjTS8QZ1A40mrQ0kKkohEv/DuwVp8QZ1B6Q7G6x/C0W2RRe4vuA4y+98ieo6MbABCTbUWR5IcJBGJpFfvC/H+4cgv7OysatTyjZEXRl4FuRdYF1g3pGfmABDy1sv200giEv3AqaYAqqow1nmKospXI08WXxw5+qlMt5uw6sQb0sFfZ3U4lpNEJJLeiJx0bpvs4srwGyimAbljI6Whfiw7zYHf5sYf1DG9NVaHYzlJRCL5eWtI/fB/SVF0yBoKY6/r1U0Q+4InxU7Q4UE3wVtfZXU4lpPue5GUDMPklT2VjHGHGH7krxD0QnoujL8xMom1n9NUBc1dQEboIEZ9pdXhWK7/f0fFgBPSDdbvOcHRQx+Tcer/GDLEicOdC5NuarWYWX939fRJqDv3QEi68CURiaTRHAjzcVUT7x08SVrVe0yo38bo3BQcnnyY/BVwpFkdYo9S3QWRKqa/PjLC2tk7u8v2B5KIhOXqmv1s2vExdSePkxo8xYjmvbjxMjI/Hc/QiTBmXmT09EBjd0F6HsG6YygnD2AfMtnqiCwjiUj0ObP5JIGq/bh8J6CpivTmGvLLq8k1IcNpY5DHQc6gIdhKZ0eW8ujnDdPtebcxk/Ch3ZS4dlMoichajz/+OL/4xS+orKxk8uTJ/PrXv2b69OlWh5WcTBPqDkWWP/XVgKKipwziiJJPnZbD5JKsM041O71ZoC+oc6LBT3MwjKYqpDtt5Lld2LUe6lgNegkc28mJfe9Se6ICBZhY5EFBwQaU5meRmplDiicXskdEBiom8Qz6nmIMHov5yWZqK3ZTeB5XzyxPRH/+859ZvHgxv/3tb5kxYwaPPfYYc+fOZe/eveTm5lodXvIINMGJD+DYdvDVYmLiDehUNQU41RQgpJsE7R6MS69ALZgIqdm8+lEVTYEwo/MyGBldwRDDiCw94a+LtE34GzjRGGDD/gb8NjdeezYhNQUUBVVRyPc4mVSUyQUFXdyByzQhHECvPcTJ8p3UHdpFXZMXwwQThebUIYwYMpX07CGQNohBrswBXfJpy5jSUrbuyAPvCU7s/D/yLr7B6pAsoZgW7/k8Y8YMLr74Yn7zm98AYBgGxcXFfOc73+G+++5r970NDQ14PB7q6+txuwfYVnV6KJIo6o/Aqf2R1QdNg+ZgmJM+KGcIx41MFNMgPVhNXqiCbJfC0EGp2FQVIyWb9Yd1/IYd1dRxmT4G2fykmc0Yuo4n1U6hJ7LjqG6YvHuoBqdNw2lXCakuavBQo2TitQ9icmkRE4fmg2rD6/ez50gNaVoQFwHsug+bHkAL+yDkJcMWwmUGIOilsq6JwzXe2H5hXvsgAnmTGDL6QsYU5eKy990WPsns7W3vwfZn0FSF/AmzyB9/BY7UgfHz3NnfUUtLRMFgkG3btrF06dLYc6qqMmfOHN5+++1zzg8EAgQCgdjjhoaGzn1QTTkc2EC8VdaP1Ho5VhvZ6K7lVQUz9u/xhW7SHZEv07E6LxXRc8+8lgkopskFhW7cLjuYJpUNfg6d8oJpxA1pTH4GmSmR61Y1BiivboxdVsFEM07f54icdLJTHeAu4LAykleqszFUOzZVYUROOuML3ZR4bKinPoaqPVBTjuqr4XJPmFPNQU42BfCHDAJAgGiJREunMGsouDxowNTsZhzBWvDVgmli4iUQbqLBV477+A44GUkazY0BQtVN1LXxpR6Vm44rPdKw7NBUmtUMfJ5heIZPYeKIUeS4Xb2/t3w/M23KFDZXl+M8+jZHd27k2M6NTBqeR2pKGqgaB2t8nGgIAArE+dJNKvKQEk3qFTVejtX72/ysCYUe0pyRc4/W+jlS1/Ymj+MK3GS4Ij+jx+v9pDttsccATPwyuHomYVqaiE6ePImu6+TltV7CIS8vj48++uic81esWMHy5cu7/kHhADRVx33JbPaiN7ezB7lXBz36ZfL5MLzxv3EmYPoAou0aAT9GO/tWmQEFNEfkQdCPETwdgwmRXR4UO17HIPILJpA9+iJIzyGnKcAoew3FWamMyktvXarInxA5gl5oPE6qt4ZUPUiRqtGgOzkZcuJV01BcbrLSXZCZEnuro+UfegiaT6I0V+FqrsbVVA2h5shKiEYYLdXJoJx0vKaDgOIkqDgJqimEFCe6LQXv8ALIGwz2FNLVFMYHISfdKcmnHXZN5bJ/+zy7dxTRsO8NbM2VOMwg+MMAmM3NGE1tJxelKQjRnwOzyYvR2M7Pc3MAQrbOnZvhh7A9eq4v8r7wGe12bfyRTYTlbURdsXTpUhYvXhx73NDQQHFxccdvzCyOjEOBs9ohFDz+MGr0G976pcgDe7oToouzuwNhhgX0M05RWp3uSnWAFj03pDPSHzlXUc/9JUxx2aM/PArusM6oQJiWP3eKomDaU8EWKT2ku2yxGAanO5k3sYMdKRypkZUKo6sVKoAnenRIs0d2vGhj14vs6NEZDiB3APa69wabpjL5whlw4Qy83mY0oznyR8E0yPYGcAZCcbdNAtDcToj+jGX6Q2h+Pf6JgC3DAdEOCLc/TEn0Zz8eR7oDbJFz0wNhHJoaexw5oefGdVmaiAYPHoymaZw4caLV8ydOnCA/P/+c851OJ05nAj/ZjrQ2F1J3R4/OSI8enZEaPTrDFT2EAEhNTQNO/5J7Mjv5R4Su/Tx39dzeZOmkV4fDwUUXXcSGDRtizxmGwYYNG5g5c6aFkQkh+pLlVbPFixczf/58pk2bxvTp03nsscdobm7mtttuszo0IUQfsTwR3XTTTVRXV7Ns2TIqKyuZMmUKL7/88jkN2EKIgcvycUTdMaDHEQkxAHT2d1QWRhNCWE4SkRDCcpKIhBCWs7yxujtamrc6PdVDCNGnWn43O2qK7teJqLExMj+rU6OrhRCWaWxsxONpe1hmv+41MwyDY8eOkZGR0W/mMrVMS6moqBhQPX0D8b4G4j1B396XaZo0NjZSWFiIqrbdEtSvS0SqqlJUVGR1GAlxu90D6oe7xUC8r4F4T9B399VeSaiFNFYLISwniUgIYTlJRH3M6XTy4x//OLFVBJLYQLyvgXhPkJz31a8bq4UQA4OUiIQQlpNEJISwnCQiIYTlJBEJISwniaiXPfjgg1xyySWkpqaSmZnZqfeYpsmyZcsoKCggJSWFOXPm8PHHH/duoF1UU1PDLbfcgtvtJjMzk9tvv52mpqZ233PllVeiKEqr45vf/GYfRRzf448/zrBhw3C5XMyYMYN//etf7Z7//PPPM3bsWFwuFxMnTuSf//xnH0XaNV25r7KysnO+Ly5XH6+ibopetWzZMvPRRx81Fy9ebHo8nk6956GHHjI9Ho/54osvmjt27DA/97nPmcOHDzd9Pl/vBtsFn/nMZ8zJkyeb77zzjvnGG2+YpaWl5s0339zue2bNmmXeeeed5vHjx2NHfX19H0V8rjVr1pgOh8N84oknzN27d5t33nmnmZmZaZ44cSLu+Zs3bzY1TTN//vOfm3v27DH/3//7f6bdbjd37drVx5G3r6v3tXr1atPtdrf6vlRWVvZpzJKI+sjq1as7lYgMwzDz8/PNX/ziF7Hn6urqTKfTaT777LO9GGHn7dmzxwTMrVu3xp576aWXTEVRzKNHj7b5vlmzZpnf/e53+yDCzpk+fbq5aNGi2GNd183CwkJzxYoVcc//8pe/bF577bWtnpsxY4b5jW98o1fj7Kqu3ldnfzZ7k1TNkkx5eTmVlZXMmTMn9pzH42HGjBlxd7+1wttvv01mZibTpk2LPTdnzhxUVWXLli3tvvfpp59m8ODBTJgwgaVLl+JtY8PK3tayy/CZX+f2dhmGyH2feT7A3Llzk+b7AondF0BTUxNDhw6luLiY66+/nt27d/dFuDH9etLrQFRZWQkQd/fbltesVllZSW5ubqvnbDYb2dnZ7cb41a9+laFDh1JYWMjOnTu599572bt3L2vXru3tkM/R1V2GIXLfyfx9gcTua8yYMTzxxBNMmjSJ+vp6Hn74YS655BJ2797dZ5PKpUSUgPvuu++cxr2zj7a+6cmst+/rrrvuYu7cuUycOJFbbrmFp556ihdeeIEDBw704F2Irpo5cya33norU6ZMYdasWaxdu5acnBx+97vf9VkMUiJKwH/8x3+wYMGCds8ZMWJEQtdu2eH2xIkTFBSc3vb5xIkTTJkyJaFrdlZn7ys/P5+qqqpWz4fDYWpqauLu0NuWGTNmALB//35GjhzZ5Xi7o6u7DEPke9OV862QyH2dzW63M3XqVPbv398bIcYliSgBOTk55OTk9Mq1hw8fTn5+Phs2bIglnoaGBrZs2cK3vvWtXvnMFp29r5kzZ1JXV8e2bdu46KKLAHj11VcxDCOWXDpj+/btAK0Sbl85c5fhG264ATi9y/C3v/3tuO+ZOXMmGzZs4J577ok9t379+qTalTiR+zqbruvs2rWLefPm9WKkZ7G0qfw8cOjQIfP99983ly9fbqanp5vvv/+++f7775uNjY2xc8aMGWOuXbs29vihhx4yMzMzzb/85S/mzp07zeuvvz4pu++nTp1qbtmyxXzzzTfNUaNGteq+P3LkiDlmzBhzy5Ytpmma5v79+83777/ffPfdd83y8nLzL3/5izlixAjziiuusOoWzDVr1phOp9MsKysz9+zZY951111mZmZmrOv661//unnffffFzt+8ebNps9nMhx9+2Pzwww/NH//4x0nbfd+V+1q+fLm5bt0688CBA+a2bdvMr3zlK6bL5TJ3797dZzFLIupl8+fPN4Fzjtdeey12DmCuXr069tgwDPNHP/qRmZeXZzqdTvOqq64y9+7d2/fBt+PUqVPmzTffbKanp5tut9u87bbbWiXX8vLyVvd5+PBh84orrjCzs7NNp9NplpaWmt///vctHUdkmqb561//2iwpKTEdDoc5ffp085133om9NmvWLHP+/Pmtzn/uuefM0aNHmw6Hwxw/frz5j3/8o48j7pyu3Nc999wTOzcvL8+cN2+e+d577/VpvLIMiBDCctJrJoSwnCQiIYTlJBEJISwniUgIYTlJREIIy0kiEkJYThKREMJykohEv1JWVtZqpcuf/OQnrebgLViwIDa1QfQfkohEXAsWLGhzKddFixahKEqrCbK9kQCGDRvGY4891uq5m266iX379rX5npUrV1JWVhZ7fOWVV7aaGyaSkyQi0abi4mLWrFmDz+eLPef3+3nmmWcoKSmxJKaUlJRz1kI6k8fj6fTa4CJ5SCISbbrwwgspLi5utXDZ2rVrKSkpYerUqd26drySyg033BArZV155ZUcOnSI733ve7G1kODcqtnZziyZLViwgE2bNrFy5crYNcrLyyktLeXhhx9u9b7t27ejKEqfLn0hTpNEJNq1cOFCVq9eHXv8xBNPcNttt/X6565du5aioiLuv/9+jh8/zvHjx7t8jZUrVzJz5kzuvPPO2DVKSkrOuSeA1atXc8UVV1BaWtpTtyC6QBKRaNfXvvY13nzzTQ4dOsShQ4fYvHkzX/va13r9c7Ozs9E0jYyMDPLz8xNafMzj8eBwOEhNTY1dQ9M0FixYwN69e2Nb7IRCIZ555hkWLlzY07chOkkWRhPtysnJ4dprr6WsrAzTNLn22msZPHiw1WF1S2FhIddeey1PPPEE06dP529/+xuBQIAvfelLVod23pISkejQwoULKSsr48knn+yxUoOqqpy9Ak0oFOqRa3fGHXfcEWuIX716NTfddBOpqal99vmiNUlEokOf+cxnCAaDhEIh5s6d2yPXzMnJadXuo+s6H3zwQatzHA4Huq5363Pausa8efNIS0tj1apVvPzyy1Its5hUzUSHNE3jww8/jP27LfX19bF1qFsMGjSI4uLic8799Kc/zeLFi/nHP/7ByJEjefTRR6mrq2t1zrBhw3j99df5yle+gtPpTKhKOGzYMLZs2cLBgwdJT08nOzsbVVVjbUVLly5l1KhRSbXu9PlISkSiU9xuN263u91zNm7cyNSpU1sdy5cvj3vuwoULmT9/PrfeeiuzZs1ixIgRzJ49u9U5999/PwcPHmTkyJEJb1awZMkSNE1j3Lhx5OTkcPjw4dhrt99+O8FgsE96AUX7ZKlYcd564403uOqqq6ioqDhnQ0LRtyQRifNOIBCgurqa+fPnk5+fz9NPP211SOc9qZqJ886zzz7L0KFDqaur4+c//7nV4QikRCSESAJSIhJCWE4SkRDCcpKIhBCWk0QkhLCcJCIhhOUkEQkhLCeJSAhhOUlEQgjLSSISQlju/wM8DZSjgrZJdgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:26.621078Z", + "iopub.status.busy": "2024-03-22T14:51:26.620698Z", + "iopub.status.idle": "2024-03-22T14:51:26.847753Z", + "shell.execute_reply": "2024-03-22T14:51:26.846754Z" + }, + "papermill": { + "duration": 0.249855, + "end_time": "2024-03-22T14:51:26.849809", + "exception": false, + "start_time": "2024-03-22T14:51:26.599954", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA43ElEQVR4nO3deVgT194H8C9hCSC7bGLRYFHRCxUXRGkVFcQNr1Sp1mJFa/VWL91wKfa2Wm/flqrgVr362FatVdSqlFpqVVxQXqWKW4uKqAhiWURECYskITnvH97MS2QLCiST+X2ex0cycyb5DcP8cubMmXOMGGMMhBBi4ES6DoAQQtoDJTtCiCBQsiOECAIlO0KIIFCyI4QIAiU7QoggULIjhAgCJTtCiCBQsiOECAIlO0KIIPAu2W3YsAESiQTm5ubw9/fHuXPnGi37zTffYMiQIbC3t4e9vT2Cg4ObLE8IMVy8SnZ79uxBdHQ0li5diosXL6JPnz4YNWoUSkpKGiyfmpqKqVOn4sSJE0hPT4e7uztCQkJQUFDQzpETQnTNiE8DAfj7+8PPzw/r168HAKhUKri7u+Pdd99FTExMs9srlUrY29tj/fr1mD59ulafqVKpUFhYCGtraxgZGT1X/ISQ1sUYQ0VFBdzc3CASNV13M2mnmJ6bXC7HhQsXsHjxYm6ZSCRCcHAw0tPTtXqP6upqKBQKODg4NFpGJpNBJpNxrwsKCtC7d+9nD5wQ0ubu3r2LF154ockyvEl2paWlUCqVcHFx0Vju4uKC69eva/UeH330Edzc3BAcHNxomdjYWCxbtqze8m+//RaWlpYtC5oQ0qaqq6vx9ttvw9rautmyvEl2z+urr77C7t27kZqaCnNz80bLLV68GNHR0dxrqVQKd3d3hIWFwcbGpj1C1QmFQoGUlBSMHDkSpqamug6HPCehHE+pVIq3335bqyYm3iQ7R0dHGBsb4969exrL7927B1dX1ya3jYuLw1dffYWjR4/ipZdearKsWCyGWCyut9zU1NSg/2jUhLKfQmHox7Ml+8abu7FmZmbo378/jh07xi1TqVQ4duwYBg8e3Oh2K1aswOeff45Dhw5hwIAB7REqIUQP8aZmBwDR0dGIjIzEgAEDMHDgQKxZswZVVVWYOXMmAGD69Ono3LkzYmNjAQDLly/HkiVLkJCQAIlEguLiYgCAlZUVrKysdLYfhJD2x6tkN2XKFNy/fx9LlixBcXExfH19cejQIe6mRX5+vsbt540bN0IulyM8PFzjfZYuXYrPPvusPUMnhOgYr5IdAERFRSEqKqrBdampqRqv8/Ly2j4gQggv8KbNjhBCngfvanaEkPqqq6s1+ptWPpbhTGYO7B3Pw8pCs3eBl5eXIPuMUrITKG1PDqGeGHxz/fp19O/fv97yFQ2UvXDhAvr169f2QekZSnYCpe3JIdQTg2+8vLxw4cIF7nV20SNE783Eqtd80LOTXb2yQkTJTqC0PTmEemLwjaWlpcaXkujOA4jTHqOXdx/4du2ow8j0ByU7gaKTgwgNJTtCeCq3tApVstoG1+Xcr+L+NzFp/DTvIDaBh2OHNolP31CyI4SHckurMDwutdly8/dlNlvmxIJhgkh4lOwEoqlaAEA1Ab5RH8s1U3zh6Vz/0ceqxzIkp6YjdNhgdLCoP7AFANwqqcQHey43+XdhSCjZCYC2tQCAagJ84+lsBe/OtvWWKxQKFDsB/braG/SoJy1ByU4AmqsFAFQTIIaPkp2ANFYLAKgmQAwfJTtCeEimrIHIvAC50myIzOvX1mtra1FYW4issqxG22BzpZUQmRdApqwB0PCXoCGhZEcIDxVW3UEHj6/xcTPTIP/n0H+aXN/BAyis8kV/uDRZzhBQshOA5moBANUE+MatQ1dU5b6LtVN88WID7bC1tbU4/b+n8fIrLzd6PHNKKvH+nstwG961rcPVC5TsBEDbWgBANQG+EBubQ1XTGR42PdG7Y8N3Y3NNctHLoVejbbCqmnKoau5DbNz4BFSGhJKdADRXCwCoJkAMHyU7AWiuFgBQTYAYPhqpmBAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7AghgkDJjhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7AghgkDJjhAiCJTsCCGCwLshnjZs2ICVK1eiuLgYffr0wddff42BAwc2Wn7v3r349NNPkZeXh+7du2P58uUYO3ZsO0ase48VSgDAlYLyRstUPZbh/H3A9c7DJmcXI/qhuWNKx7M+XiW7PXv2IDo6Gps2bYK/vz/WrFmDUaNGITs7G87OzvXKnzlzBlOnTkVsbCxCQ0ORkJCAsLAwXLx4Ed7e3jrYA93I+e8fdUxic3PCmuCHWxnNvl8HMa/+bAySdseUjmddRowxpusgtOXv7w8/Pz+sX78eAKBSqeDu7o53330XMTEx9cpPmTIFVVVVSE5O5pYNGjQIvr6+2LRpU4OfIZPJIJPJuNdSqRTu7u4oLS2FjY1NK+9R+yirkuNoVgm6OXWAhalxg2VuFJdj0U9ZWPFqL/RwbXx+iQ5iY0g60gTZutbcMRXK8ZRKpXB0dER5eXmz5ydvUrpcLseFCxewePFibplIJEJwcDDS09Mb3CY9PR3R0dEay0aNGoWkpKRGPyc2NhbLli2rt/zIkSOwtLR8tuD1gBWAkpLG1z+pKJig5FYmxMVNv9e1VoyLPLumjqlQjmd1dbXWZXmT7EpLS6FUKuHiojnRi4uLC65fv97gNsXFxQ2WLy5u/OgvXrxYI0Gqa3YhISG8rdlp44/8MiDzPAYNGoQ+XRx0HQ55TkI5nlKpVOuyvEl27UUsFkMsrt+ga2pq2ujcDIZAPcmOiYmJQe+nUAjleLZk33jT9cTR0RHGxsa4d++exvJ79+7B1dW1wW1cXV1bVJ4QYrh4k+zMzMzQv39/HDt2jFumUqlw7NgxDB48uMFtBg8erFEeAFJSUhotTwgxXLy6jI2OjkZkZCQGDBiAgQMHYs2aNaiqqsLMmTMBANOnT0fnzp0RGxsLAHj//fcRGBiI+Ph4jBs3Drt378b58+exefNmXe4GIUQHeJXspkyZgvv372PJkiUoLi6Gr68vDh06xN2EyM/Ph0j0/5XVgIAAJCQk4JNPPsHHH3+M7t27IykpSVB97AghT/Aq2QFAVFQUoqKiGlyXmppab9lrr72G1157rY2jIoToO9602RFCyPOgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkRwgRBEp2hBBBoGRHCBEESnaEEEGgZEcIEQRKdoQQQaBkR6BUKnE+/X9Rde0kzqf/L5RKpa5DIqTVUbITuMTERHh6emL21DCU/rISs6eGwdPTE4mJiboOjZBWRclOwBITExEeHg5vb2/E/Hs5HMa8j5h/L4e3tzfCw8Mp4RGDYsQYY7oOQp9JpVLY2tqivLwcNjY2ug6n1VRUVMDLywt2dnZ4+PAhioqKuHWdOnWCvb09ysvLkZWVBWtrax1GSlqqsrIS4ydOxulL1/By3974JfFHWFlZ6TqsNtGS85NqdgK1a9cuFBYW4tq1axqJDgCKiopw7do1FBQUYNeuXTqKkDyLgQMHwtraGqkpv0FRegepKb/B2toaAwcO1HVoOkfJTqBMTExatRzRvYEDByIjI6PBdRkZGYJPeJTsBOrRo0etWo7oVklJSaOJTi0jIwOnTp1CdXV1O0WlXyjZCVTdS9fg4GB4e3vDwcEB3t7eCA4ObrAc0V+vvfaaVuUCAwNx/fr1No5GP9ENimYY6g2KHj164ObNm82W6969O27cuNEOEZHn0bVrV+Tn5zdbrlOnTrh16xYsLS3bIaq2RzcoSLOkUmmrliO6pe2lqUKhMJhE11KU7ATK2dm5VcsR3XJzc2vVcoaIN8murKwMERERsLGxgZ2dHWbNmoXKysomy7/77rvo2bMnLCws0KVLF7z33nsoLy9vx6j1V01NTauWI7oll8tbtZwh4k2yi4iIwNWrV5GSkoLk5GScOnUKc+bMabR8YWEhCgsLERcXhytXrmDbtm04dOgQZs2a1Y5R66/c3NxWLUd0i5olmtfiGxS3b99Gt27d2iqeBmVlZaF3797IyMjAgAEDAACHDh3C2LFj8ddff2ldNd+7dy+mTZuGqqqqRvuPyWQyyGQy7rVUKoW7uztKS0sN6gaFmZmZ1mWFXBvgC2dnZ626CdnZ2aGkpKTtA2onUqkUjo6OWt2gaHGPUU9PTwQGBmLWrFkIDw+Hubn5MweqrfT0dNjZ2XGJDnjSXUIkEuHs2bN49dVXtXof9S+kqY6ysbGxWLZsWb3lR44cEWzD7sGDB3UdAmlGU006T5czpOPZkj6DLa7ZXb58GVu3bsWuXbsgl8sxZcoUzJo1q017Z3/55Zf4/vvvkZ2drbHc2dkZy5Ytw9y5c5t9j9LSUvTv3x/Tpk3DF1980Wg5qtnVRzU7/ff08bSxsYFCoYCpqWm9S1dDOp5tWrPz9fXF2rVrER8fjwMHDmDbtm145ZVX0KNHD7z11lt488034eTkpNV7xcTEYPny5U2WycrKammI9UilUowbNw69e/fGZ5991mRZsVgMsVhcb7mpqSlMTU2fOxY+Eup+85k6wT1+/LjeOkM6ni3Zl+fuVCyTyfCf//wHixcvhlwuh5mZGSZPnozly5ejU6dOTW57//59PHjwoMky3bp1w44dOzB//nw8fPiQW15bWwtzc3Ps3bu3ycvYiooKjBo1CpaWlkhOTm7xZbehdioWiUTQ5tAbGRlBpVK1Q0TkeRgZGWld1pCeI2jR+cmeUUZGBps7dy6zt7dnL7zwAvvXv/7Fbt++zU6dOsWCgoKYn5/fs751PdeuXWMA2Pnz57llhw8fZkZGRqygoKDR7crLy9mgQYNYYGAgq6qqeqbPLi8vZwBYeXn5M22vr6ysrBiAZv9ZWVnpOlSihdDQUK2OZ2hoqK5DbVUtOT9b3PVk1apV8PHxQUBAAAoLC7F9+3bcuXMH//M//wMPDw8MGTIE27Ztw8WLF1v61o3q1asXRo8ejdmzZ+PcuXM4ffo0oqKi8Prrr3N3YgsKCuDl5YVz584BeJLxQ0JCUFVVhe+++w5SqRTFxcUoLi6mYceh/be7tuUI0XctbrPbuHEj3nrrLcyYMaPRy1RnZ2d89913zx1cXTt37kRUVBSCgoIgEokwadIkrFu3jluvUCiQnZ3N3Z25ePEizp49C+DJHeS6cnNzIZFIWjU+vqFOqIaFOok3r8XJLiUlBV26dIFIpFkpZIzh7t276NKlC8zMzBAZGdlqQQKAg4MDEhISGl0vkUg0aiHDhg2jWkkTFApFq5YjumVhYdGq5QxRiy9jX3zxRZSWltZbXlZWBg8Pj1YJihDSMpTsmtfiZNdYbamysrJdOhiT1vF0zfx5yxHdaqiLyfOUM0RaX8ZGR0cDeHKLe8mSJRpPEyiVSpw9exa+vr6tHiBpG6amphqdp5sqR/Tf/fv3W7WcIdI62V26dAnAk5pdZmamRo9tMzMz9OnTBwsWLGj9CEmbsLCw0CrZCfmyh0/qJjEjIyONK7C6rynZaeHEiRMAgJkzZ2Lt2rUG1cFWiCQSCS5fvqxVOaL/ysrKuJ+fbmqq+7puOaFpcYPM1q1bKdEZgH79+rVqOaJbDT3i+DzlDJFWNbuJEydi27ZtsLGxwcSJE5ssS7PI84O2zxy3xrPJpO25uLiguLhYq3JCpVWys7W15Z69s7W1bdOASPvQdsRmGtmZH7R9NrYlz9AaGq2S3datWxv8mfBX3busxsbGGo/Q1X1Nd2P5gboSNY+mexeouqPaGhsbY8iQIVCpVBCJRDhz5gyX7GiSbH5wd3fX6nl0d3f3dohGP2mV7Pr27at19bc1BwAgbadu1yG5XI7U1NRmyxH9NW7cOPz8889alRMqrZJdWFhYG4dB2puXl5dWk2R7eXm1QzTkeWk71PrBgwcxe/bsNo5GP2mV7JYuXdrWcZB29uqrr+KXX37RqhzRf+o5KExNTRscvEG9XNu5KgwRtdkJFN2NNSxWVlYAnoxSM3bsWJiZmSEnJwcvvvgi5HI5V/NTlxMirZKdg4MDbty4AUdHR9jb2zfZfifkHtp8op4nRD2k9dPUy7WdT4ToVlhYGJKSkmBiYoLMzEzcvXsXAJCZmYkuXbrAxMQEtbW1gm6S0irZrV69GtbW1tzPQu6rYyg6d+4M4EnNzdnZGS4uLigrK4ODgwPu3bvHzS2qLkf0W9euXQE8mZtFnejU8vPz65UToueecMfQGeqEO3K5HB06dICZmRlkMlm9fnZisRhyuRxVVVV0R5YHlEolOnXq1OSD/s7OzigsLISxsXE7Rta2WnJ+triHobGxcYMzij948MCgfomG7syZM6itrUV1dTUcHBwQHh6OESNGIDw8HA4ODqiurkZtbS3OnDmj61CJltSj2Dg4OKBTp07o0KEDOnXqBAcHBwDCHpIdeIYbFI1VBGUyGdUAeKSgoAAA4OHhgTt37mDfvn3cOpFIBA8PD+Tm5nLliH5LTU2FVCqFlZWVRrt5VVUVgCc3JqRSKVJTUxEUFKSrMHVK62SnntzGyMgI3377rcZdHaVSiVOnTlGfLB5RX+7k5ubWW6dSqbjlQh7/jE/UncIrKythZmaGiRMnwsLCAo8fP0ZiYiLX5YSSnRZWr14N4EnNbtOmTRqXrGZmZpBIJNi0aVPrR0jaRMeOHVu1HNEt9SxwpqamqKiogJGREQ4ePIixY8fi+++/h5WVFRQKhaBni9M62am/6YcPH47ExETY29u3WVCk7RUWFrZqOaJb6qG4nJ2dYWJionHDycTEBE5OTigsLBT0kF0tvkFx4sQJSnQG4MiRI9zPT7e11h3gsW45or/UNx8KCgowYcIE/P7773j8+DF+//13TJgwgfvSEvJNihbfoHjrrbeaXL9ly5ZnDoa0n7/++ov7+enHi+pe6tQtR/RXjx49kJKSAgA4evQokpOTuXV15xHp0aNHu8emL1pcs3v48KHGv5KSEhw/fhyJiYk0HBCP1L3B9HQn8bqvhfx4EZ+sXLkSwJOuYU+3y8lkMq6NXV1OiFpcs/vpp5/qLVOpVJg7dy5efPHFVgmKtL3hw4dzw3GNHDkS3bt3x40bN9CjRw/cvHkThw8f5soR/WdhYQE/Pz9kZGQAeDJ3iPpurPo4+/n5CXq2uFYZCEAkEiE6OhrDhg3DokWLWuMtSRurrq7mfj58+DCX3J5uo6tbjugvpVKJ+/fvc08TPD2upK2tLUpLS6FUKgXb+b/VxmjOyclBbW1ta70daWM0Z4FhSUtLQ15eHqRSKUaPHg1vb2907NgR3t7eGD16NKRSKXJzc5GWlqbrUHWmxTW76OhojdeMMRQVFeHXX39FZGRkqwVG2la3bt1atRzRLfWTLhKJBEeOHIFKpQLw5DHOa9euQSKRCP6JmBYnu0uXLmm8FolEcHJyQnx8fLN3aon+8PHx4X42NzfX6JJQ93XdckR/0RMxzWtxsjtx4kRbxEHaWd0/ehsbG7zzzjuorq6GpaUlEhISuGQn5JODT+r2fTUyMtJ4hr3uayH3kaWRigVKncRGjRqFo0ePYs2aNdw6ExMThISE4MiRI5TseCI9PZ37ualkl56eLtjmJt5MIllWVoaIiAjY2NjAzs4Os2bN0no8fcYYxowZAyMjIyQlJbVtoDyhHoG4pKSk3gCdbm5uXJKjkYr54c8//+R+buqJmLrlhIY3NbuIiAgUFRUhJSUFCoUCM2fOxJw5c5CQkNDstmvWrKG7ik9RJ7hLly7BxcUFH3zwAXcZu2vXLq5tlkYq5oeKigru56CgIISEhODmzZvo3r07jhw5gl9//bVeOaHhRbLLysrCoUOHkJGRgQEDBgAAvv76a4wdOxZxcXFwc3NrdNvLly8jPj4e58+fR6dOnZr9LJlMxg2CCDwZCRV48khVQ7M28ZWfnx9MTExgZmaG0tJSjctYY2NjWFpaQi6Xw8/Pz6D221Cpa+BmZma4cuUKl9yAJ3dozczMIJfL4eTkZFDHsyX70mrJ7q+//sK///1vbN68ubXekpOeng47Ozsu0QFAcHAwRCIRzp492+h0f9XV1XjjjTewYcMGuLq6avVZsbGxWLZsWb3lR44cgaWl5bPtgB7KzMxEbW0tamtrYWNjAx8fH4jFYshkMmRmZnJJfvXq1XRHlgfUNx7kcjkePnyIv//973B1dUVxcTFSU1O5R8js7e21nmOWD1rS6b3Vkt2DBw/w3XfftUmyKy4uhrOzs8YyExMTODg4oLi4uNHtPvzwQwQEBGDChAlaf9bixYs1+hJKpVK4u7sjJCTEoOagUD/HLJFIcPfuXZw+fZpbZ2JiAolEgry8PLi7u2Ps2LE6ipJoy9zcHImJiQCe/M0eOHCgwXJz5szBiBEj2jO0NqX+UtaGTi9jY2JisHz58ibLPOv4WwcOHMDx48fr9Qtsjlgs1mjQVTM1NYWpqekzxaKPHj58CAC4c+cOxo0bh5EjR3JtPCkpKdxl0MOHDw1qvw1VcHAwnJycmp1wJzg42KAeF2vJ36ZOk938+fMxY8aMJst069YNrq6u9Sb5qa2tRVlZWaOXp8ePH0dOTg7s7Ow0lk+aNAlDhgzhhrEWKvUIxE5OTvjpp5/AGONGtp03bx46d+6MkpISGqmYJ4yNjbFp0yZMmjSp0TIbN240qETXUjpNdk5OTlp1bRg8eDAePXqECxcuoH///gCeJDOVSgV/f/8Gt4mJicHbb7+tsczHxwerV6/G+PHjnz94nnvw4AGAJ11PJk6ciIULF3KDPa5cuZL7clGXI/zx9BMx6tFPhE7rZDdx4sQm17flWHa9evXC6NGjMXv2bGzatAkKhQJRUVF4/fXXuTuxBQUFCAoKwvbt2zFw4EC4uro2WOvr0qULPDw82ixWvlB/yfTt2xeZmZkYOnQot87DwwN9+/bFpUuXqJ8dTyiVSsyfPx/jx4/H/v37cfLkSfz2228YM2YMAgMDMWnSJCxYsAATJkwQbO1O62Rna2vb7Prp06c/d0CN2blzJ6KiohAUFASRSIRJkyZxM54BT25BZ2dn05BEWqrbzy40NBQffvihRpudeqRb6mfHD+pRT3bt2gVTU1MEBgaiqqoKgYGBMDU1xeLFixEQEIC0tDQMGzZM1+HqhNbJbuvWrW0ZR7McHBya7EAskUgandNWrbn1QjJkyBBIJBI4OjoiMzNTYxhviUSCAQMG4MGDBxgyZIgOoyTaKioqAgB4e3s3uF69XF1OiHjRqZi0PmNjY8THxyM8PBzjxo1DdHR0vbux+/btE+wlD9+oO8xfuXIFgwYNqrf+ypUrGuWEyIhpWd3RdvgmQ5twRyqVcqO/GlI/O7XExETMnz8feXl53DIPDw/ExcU1205L9IdSqYSnpyd8fHyQlJQEpVLJ3V03NjZGWFgYrly5gps3bxrUF1hLzk+tk51IJELXrl3Rt2/fJi8HG5qjgs8MPdkBT06UEydOcA3aw4cPN6gTQigSExMRHh6O0NBQLFy4EAUFBejcuTNWrlyJ5ORk7Nu3z+C+wFp0fjItzZs3j9nb2zNfX1+2du1a9uDBA2035bXy8nIGgJWXl+s6lDYll8tZUlISk8vlug6FPIf9+/cziUTCAHD/PDw82P79+3UdWptoyfmp9RBPGzZsQFFRERYtWoRffvkF7u7umDx5Mg4fPkwN/4ToiYkTJ+LWrVtISUlBdHQ0UlJScPPmTYOr0T2LFo1nJxaLMXXqVKSkpODatWv429/+hnnz5kEikWg9thwhpG0ZGxsjMDAQQ4cORWBgIDVJ/NczD94pEom4EVCVSmVrxkQIIa2uRclOJpNh165dGDlyJHr06IHMzEysX78e+fn5NHM8IXpCqVTi5MmTOHXqFE6ePEmVkf/Sup/dvHnzsHv3bri7u+Ott97Crl274Ojo2JaxEUJa6OmuRKtWrYJEIkF8fLzg2+20TnabNm1Cly5d0K1bN5w8eRInT55ssJx6TC1CSPuq2/Xkhx9+wF9//YUXXngBK1asQHh4uEF2PWkJrZPd9OnTaR4HQvSUeiCA0NBQrlPxgwcP4O/vj6SkJISFhdFAANoW3LZtWxuGQQh5HnUHAhCJRBrtdCKRiAYCAI+mUiRthxq0+a/uQAANHU8aCICSneAlJibC09MTI0eOxKpVqzBy5Eh4enpS2yvPqB/wX79+fYPHc/369RrlhIiSnYCpG7R9fHyQlpaGXbt2IS0tDT4+PggPD6eExyNDhgyBk5MTFi9eDG9vb43j6e3tjY8//hjOzs6CHrKLkp1APd2g7e/vDwsLC65BOzQ0FAsWLKBLWh6pewNR/QgnPcr5/yjZCZS6Qfvjjz+GSKT5Z6Bu0M7NzUVaWpqOIiQtkZaWhpKSEsTGxuLKlSsYOnQopk6diqFDh+Lq1av48ssvUVJSIujjSclOoGhkW8OiPk5RUVENDgQQFRWlUU6IKNkJVN2RbRtCI9vyS93j2dBAAHQ8KdkJlnoOii+//BIqlUpjnUqlQmxsLDw8PATdoM0ndDybR8lOoNRzUCQnJyMsLAy///47N29sWFgYkpOTERcXJ9je9nxDx1MLbTyQKO8Z+kjFQhvZ1tAJ7Xi25PzUeg4KoaI5KAjfCOl4tuT8pKkUCdegrZ5U2VBPDKGg49kwarMjhAgCJTtCiCBQsiOECAIlO0KIIFCyIzSeHREESnYCR+PZEaGgZCdgNJ4dERLeJLuysjJERETAxsYGdnZ2mDVrFiorK5vdLj09HSNGjECHDh1gY2ODoUOH4vHjx+0QsX6j8eyI0PAm2UVERODq1atISUlBcnIyTp06hTlz5jS5TXp6OkaPHo2QkBCcO3cOGRkZiIqKqjd+mxDVHc+OMabRZscYo/HsiOFp40fXWsW1a9cYAJaRkcEt++2335iRkRErKChodDt/f3/2ySefPNdnG+qzsQkJCQwA27FjR71nKSUSCduxYwcDwBISEnQdKnkGcrmcJSUlMblcrutQ2lRLzk9ePC6Wnp4OOzs7DBgwgFsWHBwMkUiEs2fP4tVXX623TUlJCc6ePYuIiAgEBAQgJycHXl5e+OKLL/DKK680+lkymQwymYx7LZVKAQAKhQIKhaIV90q3nJycAADTpk3D2LFj8f777yMnJwcvvvgiUlJSMG3aNK6cIe23ECiVSqSmpuLUqVMQi8UYNmyYwT4y1pK/TV4ku+LiYjg7O2ssMzExgYODA4qLixvc5vbt2wCAzz77DHFxcfD19cX27dsRFBSEK1euoHv37g1uFxsbi2XLltVbfuTIEVhaWj7nnugPuVwOkUgEc3NzZGRk4ODBg9w6JycnWFpaQiaT4eHDhxrriH5LT0/H1q1bUVJSAgBYtWoVnJ2dMXPmTAwePFjH0bW+6upqrcvqNNnFxMRg+fLlTZbJysp6pvdWD2D4j3/8AzNnzgQA9O3bF8eOHcOWLVsQGxvb4HaLFy9GdHQ091oqlcLd3R0hISEGNerJyZMnoVKpUF1dDSsrK7z33nuoqamBubk5du/ezf0R2dvbIzAwUMfREm389NNPWLFiBcaOHYsFCxaguLgYrq6uiIuLw4oVK7B79+4Gr4L4TH3lpQ2dJrv58+djxowZTZbp1q0bXF1duW8qtdraWpSVlcHV1bXB7dTDT/fu3Vtjea9evZCfn9/o54nFYojF4nrLTU1NYWpq2mSsfHLv3j0AgIeHB/Lz87Fu3TpunbGxMTw8PJCbm4t79+4Z1H4bKqVSiY8++oi7u65UKnHw4EG8/PLLGDp0KMLCwhATE4NJkyYZ1CVtS/42dZrsnJycuLajpgwePBiPHj3ChQsX0L9/fwDA8ePHoVKp4O/v3+A2EokEbm5uyM7O1lh+48YNjBkz5vmD57n79+8DAHJzcyEWizW6mJiYmCA3N1ejHNFv6rvru3btgkgk0jie6tniAgICkJaWhmHDhukuUB3iRR+MXr16YfTo0Zg9ezbOnTuH06dPIyoqCq+//jrc3NwAAAUFBfDy8sK5c+cAPJlDc+HChVi3bh327duHW7du4dNPP8X169cxa9YsXe6OXujYsWOrliO6RbPFNY8XNygAYOfOnYiKikJQUBBEIhEmTZqkcemlUCiQnZ2t0WD5wQcfoKamBh9++CHKysrQp08fpKSk4MUXX9TFLuiVus0CcrlcY13d1083HxD9VHd2sUGDBtVbT7OL8SjZOTg4ICEhodH1EomkwdnPY2JiEBMT05ah8VJpaWmrliO6VXd2saSkJI11NLvYE7xJdqR11b1J4+TkhIiICFRVVaFDhw7YuXMnV6Nr6mYO0R/q2cXCw8MxYcIEjBw5Ejdv3sSdO3eQkpKCX3/9Ffv27TOomxMtRclOoNS1YHNzc1hYWGD16tXcOolEAnNzc9TU1DRYWyb6aeLEiViwYAFWrVqF5ORkbrmxsTEWLFiAiRMn6jA63aNkJ1Dq54Nramrg7e2N6Oho3Lx5E927d8eRI0eQl5enUY7ov8TERKxcuRIWFhYag12YmZlh5cqVGDRokKATHiU7geratSv387Fjx/Drr79yry0sLBosR/SXUqnEO++802SZuXPnYsKECYK9lKWvbYEaMWJEq5YjupWamsr1iQwKCtIYnzAoKAjAkzvrqampOoxStyjZCdSwYcPqPW/8NGdnZ8F2QOWb48ePAwAGDRqExMRE1NTUICMjAzU1NUhMTOS6o6jLCRElO4EyNjbGxo0bYWRkBCMjI4116mUbN24U7CUP39y9exfAk87DPXr00Bhmv0ePHtxjk+pyQkTJTsAmTpyIffv2wcXFRWO5i4sL9u3bJ+jGbL5xd3cHAHz77bfw9vbWuIz19vbGli1bNMoJESU7gZs4cSJu3bqFlJQUREdHIyUlBTdv3qRExzN1R6ZhjHFdhur+/HQ5oaG7sQTGxsYIDAxEVVUVAgMD6dKVh+oes+PHjzd6d13Ix5ZqdoQYgKaeYa7bJivkZ50p2RFiANQP+MfGxta7y+7s7Iwvv/xSo5wQUbIjxACoBwI4c+YMbt68qdEGe+PGDaSnpwt+IABKdoQYAPVAAMnJyZg0aRLEYjH8/PwgFosxadIkJCcnIy4uTtBtdnSDghADoe5KFB0djaFDh3LLJRIJdSUC1ewIMThPdxInT1CyI8RAJCYmIjw8HD4+Phqdin18fBAeHo7ExERdh6hTlOwIMQBKpRLz58/nZhfz9/eHhYUF/P39kZSUhNDQUCxYsEBjIh6hoWRHiAFQzy728ccf1xuDUD27WG5uLtLS0nQUoe5RsiPEANDsYs2jZEeIAag7u1hDaHYxSnaEGIS6s4upVCqNdTS72BOU7AgxAHU7FYeFheH333/H48eP8fvvvyMsLIw6FYM6FRNiMNSdiufPn6/RqdjDw4M6FQMwYjRXXpOkUilsbW1RXl4OGxsbXYfTJuRyOb7++mscP34cI0aMwLvvvgszMzNdh0WekVKpxIkTJ/Dbb79hzJgxGD58uMHW6FpyflKya4ahJ7tFixZh1apVGv2vjI2NER0djRUrVugwMvI8FAoFDh48iLFjx8LU1FTX4bSZlpyfdBkrYIsWLcLKlSvrLVcqldxySnjEUNANCoGSy+WIi4sDUP9ZSvXruLg4yOXydo+NkLZAyU6g1q5dy81N8PRljvo1Ywxr165t99gIaQuU7ATq559/5n5+uvG67uu65QjhM0p2AlVeXt6q5QjRd5TsBMrJyYn7efjw4RpDAg0fPrzBcoTwGW+SXVlZGSIiImBjYwM7OzvMmjULlZWVTW5TXFyMN998E66urujQoQP69euH/fv3t1PE+q3upCznz59HZmYmHj9+jMzMTJw/f77BcoTwGW+6nkRERKCoqAgpKSlQKBSYOXMm5syZg4SEhEa3mT59Oh49eoQDBw7A0dERCQkJmDx5Ms6fP4++ffu2Y/T6x8Tk/w99SUkJ5s2b12w5QviMFzW7rKwsHDp0CN9++y38/f3xyiuv4Ouvv8bu3btRWFjY6HZnzpzBu+++i4EDB6Jbt2745JNPYGdnhwsXLrRj9Pqpa9eurVqO6A+5XI5169Zh8+bNWLduHXUf+i9efG2np6fDzs4OAwYM4JYFBwdDJBLh7NmzePXVVxvcLiAgAHv27MG4ceNgZ2eHH3/8ETU1NRg2bFijnyWTySCTybjXUqkUwJMe6QqFonV2SA8MHTqUm0vUzMxM44QQi8Xc72Do0KEGtd+GLiYmBmvWrOFGPjl48CAWLVqEDz74AF999ZWOo2t9Lfnb5EWyKy4urtd2ZGJiAgcHBxQXFze63Y8//ogpU6agY8eOMDExgaWlJX766Sd4eno2uk1sbCyWLVtWb/mRI0dgaWn57DuhZ5RKJWxsbLhkXpe6/52trS2qqqpw8ODB9g6PPINt27YhKSmp3nKVSoVVq1bh9u3bmDFjRrvH1Zaqq6u1LqvTZBcTE4Ply5c3WSYrK+uZ3//TTz/Fo0ePcPToUTg6OiIpKQmTJ0/mJiFpyOLFixEdHc29lkqlcHd3R0hIiME9G/vNN99gypQpjfaz27x5M8aPH6+L0EgLyeXyRq9w1H7++Wfs2LHDoAZ5aOjLujE6TXbz589v9pumW7ducHV1RUlJicby2tpalJWVwdXVtcHtcnJysH79ely5cgV/+9vfAAB9+vRBWloaNmzYgE2bNjW4nVgshlgsrrfc1NTU4B6onjx5MkxMTBAdHY07d+5wy11cXBAfHy/4IYH4ZM2aNag7pse0adPQv39/XLhwATt27ADwpMb+n//8BwsXLtRVmK2uReck44Fr164xAOz8+fPcssOHDzMjIyNWUFDQ4DZ//vknA8CuXbumsTwkJITNnj1b688uLy9nAFh5efmzBc8DtbW1LCUlhUVHR7OUlBRWW1ur65BICw0aNIgBYADY48ePmVwuZ0lJSUwul7PHjx9z6wYNGqTrUFtVS85PXtyN7dWrF0aPHo3Zs2fj3LlzOH36NKKiovD666/Dzc0NAFBQUAAvLy+cO3cOAODl5QVPT0/84x//wLlz55CTk4P4+HikpKQgLCxMh3ujf4yNjREYGIihQ4ciMDDQYMc+M2TZ2dkAgH79+sHc3Fxjnbm5OXx9fTXKCREvkh0A7Ny5E15eXggKCsLYsWPxyiuvYPPmzdx6hUKB7OxsrsHS1NQUBw8ehJOTE8aPH4+XXnoJ27dvx/fff4+xY8fqajcIaRPqdrjr16+jtrZWY11tbS1u3LihUU6IeHE3FgAcHBya7EAskUg02iwAoHv37vTEBBEEPz8/JCcno7q6Gp07d8Znn30Gc3NzfPvtt/jss8+4SoCfn5+OI9Ud3iQ7Qkjjdu3aBWtrawBNPxGza9eu9gxLr/DmMpYQ0jgrK6tma21+fn6wsrJqp4j0DyU7QgzEuXPnGk14fn5+3M07oaJkR4gBOXfuHCoqKjB+/Hh07doV48ePR0VFheATHUBtdoQYHCsrK+zfv18Qs4u1BNXsCCGCQMmOECIIlOwIIYJAbXbNUHdUbsnoCnykUChQXV0NqVRKbTwGQCjHU31ePv1AQUMo2TWjoqICAODu7q7jSAghjamoqICtrW2TZYyYNilRwFQqFQoLC2FtbQ0jIyNdh9Nm1OP23b171+DG7RMioRxPxhgqKirg5uYGkajpVjmq2TVDJBLhhRde0HUY7cbGxsagTw6hEcLxbK5Gp0Y3KAghgkDJjhAiCJTsCIAnw9EvXbq0wSHpCf/Q8ayPblAQQgSBanaEEEGgZEcIEQRKdoQQQaBkxyPDhg3DBx98oOswCOElSnYGKDU1FUZGRnj06JGuQyEtoG9fZvoWz/OiZEeIAZHL5boOQW9RsuMpmUyGjz76CO7u7hCLxfD09MR3332HvLw8DB8+HABgb28PIyMjzJgxo9n3q6ioQEREBDp06IBOnTph9erV9b7Zf/jhBwwYMADW1tZwdXXFG2+8gZKSEm69ukZ57NgxDBgwAJaWlggICBD0xMzamjFjBk6ePIm1a9fCyMgIRkZGyMnJwaxZs+Dh4QELCwv07NkTa9eurbddWFgYvvjiC7i5uaFnz54AgDNnzsDX1xfm5uYYMGAAkpKSYGRkhMuXL3PbXrlyBWPGjIGVlRVcXFzw5ptvorS0tNF48vLy2uvX0TYY4Y3AwED2/vvvM8YYmzx5MnN3d2eJiYksJyeHHT16lO3evZvV1tay/fv3MwAsOzubFRUVsUePHjX73m+//Tbr2rUrO3r0KMvMzGSvvvoqs7a25j6PMca+++47dvDgQZaTk8PS09PZ4MGD2ZgxY7j1J06cYACYv78/S01NZVevXmVDhgxhAQEBrf2rMDiPHj1igwcPZrNnz2ZFRUWsqKiI1dTUsCVLlrCMjAx2+/ZttmPHDmZpacn27NnDbRcZGcmsrKzYm2++ya5cucKuXLnCysvLmYODA5s2bRq7evUqO3jwIOvRowcDwC5dusQYY+zhw4fMycmJLV68mGVlZbGLFy+ykSNHsuHDhzcaT21trS5+Na2Gkh2PqJNddnY2A8BSUlIaLKdOOg8fPtTqfaVSKTM1NWV79+7llj169IhZWlpqJLunZWRkMACsoqJC43OPHj3Klfn1118ZAPb48WOtYhGyul9mjfnnP//JJk2axL2OjIxkLi4uTCaTccs2btzIOnbsqPE7/+abbzSS3eeff85CQkI03vvu3bvcl6S28fAJXcby0OXLl2FsbIzAwMBWeb/bt29DoVBg4MCB3DJbW1vukkjtwoULGD9+PLp06QJra2vu8/Pz8zXKvfTSS9zPnTp1AgCNy12ivQ0bNqB///5wcnKClZUVNm/eXO/37ePjAzMzM+51dnY2XnrpJZibm3PL6h5bAPjjjz9w4sQJWFlZcf+8vLwAADk5OW24R7pDQzzxkIWFRbt/ZlVVFUaNGoVRo0Zh586dcHJyQn5+PkaNGlWvUbzuyLjqMQBVKlW7xmsIdu/ejQULFiA+Ph6DBw+GtbU1Vq5cibNnz2qU69ChQ4vfu7KyEuPHj8fy5cvrrVN/QRkaSnY85OPjA5VKhZMnTyI4OLjeevW3vFKp1Or9unXrBlNTU2RkZKBLly4AgPLycty4cQNDhw4FAFy/fh0PHjzAV199xY3afP78+dbYHfJfZmZmGsfs9OnTCAgIwLx587hl2tS6evbsiR07dkAmk3EDAWRkZGiU6devH/bv3w+JRAITk4bTwNPx8B1dxvKQRCJBZGQk3nrrLSQlJSE3Nxepqan48ccfAQBdu3aFkZERkpOTcf/+fVRWVjb5ftbW1oiMjMTChQtx4sQJXL16FbNmzYJIJOJqZl26dIGZmRm+/vpr3L59GwcOHMDnn3/e5vsqJBKJBGfPnkVeXh5KS0vRvXt3nD9/HocPH8aNGzfw6aef1ktaDXnjjTegUqkwZ84cZGVl4fDhw4iLiwPw/zXtf/7znygrK8PUqVORkZGBnJwcHD58GDNnzuQS3NPx8L12TsmOpzZu3Ijw8HDMmzcPXl5emD17NqqqqgAAnTt3xrJlyxATEwMXFxdERUU1+36rVq3C4MGDERoaiuDgYLz88svo1asX1+7j5OSEbdu2Ye/evejduze++uor7gQirWPBggUwNjZG79694eTkhFGjRmHixImYMmUK/P398eDBA41aXmNsbGzwyy+/4PLly/D19cW//vUvLFmyBAC44+nm5obTp09DqVQiJCQEPj4++OCDD2BnZ8cNb/50PE+3FfINDfFEGlRVVYXOnTsjPj4es2bN0nU45Dnt3LkTM2fORHl5uU7afPUBtdkRAMClS5dw/fp1DBw4EOXl5fj3v/8NAJgwYYKOIyPPYvv27ejWrRs6d+6MP/74Ax999BEmT54s2EQHULIThPz8fPTu3bvR9deuXQMAxMXFITs7G2ZmZujfvz/S0tLg6OjYXmGSVlRcXIwlS5aguLgYnTp1wmuvvYYvvvhC12HpFF3GCkBtbW2Tj/o0dUeOEENByY4QIgh0N5YQIgiU7AghgkDJjhAiCJTsCCGCQMmOECIIlOyI3pgxYwY3Kq6pqSlcXFwwcuRIbNmypUXPZW7btg12dnZtF2gj1KMGE/1EyY7oldGjR6OoqAh5eXn47bffMHz4cLz//vsIDQ1FbW2trsMjfKbLkUMJqSsyMpJNmDCh3vJjx44xAOybb75hjDEWHx/PvL29maWlJXvhhRfY3Llz642WXPff0qVLGWOMbd++nfXv359ZWVkxFxcXNnXqVHbv3j3uc8rKytgbb7zBHB0dmbm5OfP09GRbtmzh1ufn57PXXnuN2draMnt7e/b3v/+d5ebmMsYYW7p0ab3PPXHiRJv8nsizoZod0XsjRoxAnz59kJiYCAAQiURYt24drl69iu+//x7Hjx/HokWLAAABAQFYs2YNbGxsUFRUhKKiIixYsAAAoFAo8Pnnn+OPP/5AUlIS8vLyNCYj+vTTT3Ht2jX89ttvyMrKwsaNG7nH5RQKBUaNGgVra2ukpaXh9OnTsLKywujRoyGXy7FgwQJMnjyZq5kWFRUhICCgfX9RpGm6zraEqDVWs2OMsSlTprBevXo1uG7v3r2sY8eO3OutW7cyW1vbZj/v6Tk0xo8fz2bOnNlg2R9++IH17NmTqVQqbplMJmMWFhbs8OHDzcZPdI9qdoQXGGPcwJNHjx5FUFAQOnfuDGtra7z55pt48OABqqurm3yP5ubQmDt3Lnbv3g1fX18sWrQIZ86c4bb9448/cOvWLVhbW3NzNjg4OKCmpsZg52wwNJTsCC9kZWXBw8MDeXl5CA0NxUsvvYT9+/fjwoUL2LBhA4CmJ4hWz6FhY2ODnTt3IiMjAz/99JPGdmPGjMGdO3fw4YcforCwEEFBQdwlcGVlJfr374/Lly9r/Ltx4wbeeOONNt570hpoqAui944fP47MzEx8+OGHuHDhAlQqFeLj47kRddXD0as1NHeCtnNoODk5ITIyEpGRkRgyZAgWLlyIuLg49OvXD3v27IGzszNsbGwajNPQ5mwwNFSzI3pFJpOhuLgYBQUFuHjxIr788ktMmDABoaGhmD59Ojw9PaFQKLi5MH744Qds2rRJ4z0kEgkqKytx7NgxlJaWorq6Wqs5NJYsWYKff/4Zt27dwtWrV5GcnIxevXoBACIiIuDo6IgJEyYgLS2Nm/fjvffew19//cV97p9//ons7GyUlpZCoVC0zy+NaEfXjYaEqEVGRnLdNkxMTJiTkxMLDg5mW7ZsYUqlkiu3atUq1qlTJ2ZhYcFGjRrFtm/fXm9S8HfeeYd17NhRo+tJQkICk0gkTCwWs8GDB7MDBw7Umzi6V69ezMLCgjk4OLAJEyaw27dvc+9ZVFTEpk+fzhwdHZlYLGbdunVjs2fPZuXl5YwxxkpKStjIkSOZlZUVdT3RQzSeHSFEEOgylhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgiU7AghgkDJjhAiCJTsCCGCQMmOECIIlOwIIYJAyY4QIgj/B6O+PK7MkOS0AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:51:26.891264Z", + "iopub.status.busy": "2024-03-22T14:51:26.890884Z", + "iopub.status.idle": "2024-03-22T14:51:27.172219Z", + "shell.execute_reply": "2024-03-22T14:51:27.171220Z" + }, + "papermill": { + "duration": 0.304536, + "end_time": "2024-03-22T14:51:27.174494", + "exception": false, + "start_time": "2024-03-22T14:51:26.869958", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWUElEQVR4nO2dd3iTZffHP0madNFBGR3QsilDlixBEZEiKOJ+ReSVraIsrYOhgoiKvgLyAxRkCSoIuHkF8cUKKLL3LqvQMroo3SNt8vz+CEmTNEmTNm2Scn+uK5fNM8+DyTfnPve5z5FJkiQhEAgEHobc1QYIBAJBRRDiJRAIPBIhXgKBwCMR4iUQCDwSIV4CgcAjEeIlEAg8EiFeAoHAIxHiJRAIPBIhXgKBwCPxcrUB1Y1Wq+XatWsEBAQgk8lcbY5AIDBDkiRycnKIiIhALrfhX0kuZMeOHdLDDz8shYeHS4D0008/lXvOtm3bpE6dOkkqlUpq1qyZ9OWXXzp0z6SkJAkQL/ESLzd/JSUl2fwuu9TzysvLo0OHDowaNYonnnii3OMTEhIYOHAgY8eOZc2aNcTFxTFmzBjCw8Pp37+/XfcMCAgAICkpicDAwErZLxAInE92djaRkZGG76o1ZJLkHguzZTIZP/30E4899pjVYyZPnsymTZs4ceKEYdszzzxDZmYmW7Zsses+2dnZBAUFkZWVJcRLIHBD7P2OelTAfvfu3cTExJhs69+/P7t377Z6TlFREdnZ2SYvgUDg+XiUeCUnJxMaGmqyLTQ0lOzsbAoKCiyeM3v2bIKCggyvyMjI6jBVIBBUMR4lXhVh6tSpZGVlGV5JSUmuNkkgEDgBj0qVCAsLIyUlxWRbSkoKgYGB+Pr6WjzH29sbb2/v6jBPAEiSRElJCRqNxtWmCNwUhUKBl5dXpVOVPEq8evTowebNm022bd26lR49erjIIoExarWa69evk5+f72pTBG6On58f4eHhqFSqCl/DpeKVm5vL+fPnDe8TEhI4cuQIISEhREVFMXXqVK5evcpXX30FwNixY1m0aBFvvvkmo0aN4s8//2TDhg1s2rTJVY8guIVWqyUhIQGFQkFERAQqlUokAQvKIEkSarWatLQ0EhISaNGihe1EVBu4VLwOHDhAnz59DO9jY2MBGD58OKtWreL69eskJiYa9jdp0oRNmzbx6quv8n//9380bNiQ5cuX253jVRFm/XoKf28vYvu1rLJ71ATUajVarZbIyEj8/PxcbY7AjfH19UWpVHL58mXUajU+Pj4Vuo5Lxeu+++7DVprZqlWrLJ5z+PDhKrSqlLScIlbsTADgoXZhtAoTeWHlUdFfUcHthTM+J+KTZoNijdbw97p9YpZSIHAnhHjZQKMt9Qp/OnyVwmIxgyYQuAtCvGxgPKLNKijm95PJrjNGUGXcd999vPLKK642o8awfft2ZDIZmZmZVXofIV420JjF48TQ8famur6UAvsQ4mUD/bBRqZAhk8Huize4lJ7nYqsEgqpHrVa72oRyEeJlA+0tzyvQR0nvlvUAWH9AeF/2IkkS+eoSl7wqWiylqKiIyZMnExkZibe3N82bN2fFihVcunTJkNZTu3ZtZDIZI0aMKPd6OTk5DB06FH9/f8LDw/n000/LDFO//vprunTpQkBAAGFhYTz77LOkpqYa9us9vri4OLp06YKfnx89e/YkPj7e7ud6//33qV+/PgEBAYwZM4YpU6bQsWNHw/4RI0bw2GOP8cEHHxAREUF0dLRdtgFs3ryZli1b4uvrS58+fbh06ZLddlUGj8qwr270npdcLuOZrpFsj0/juwNXiO3XEqVC6H55FBRraDP9d5fc+9R7/fFTOf7xHjZsGLt372bBggV06NCBhIQE0tPTiYyM5IcffuDJJ58kPj7e5pI0Y2JjY/nnn3/YuHEjoaGhTJ8+nUOHDpkIR3FxMbNmzSI6OprU1FRiY2MZMWJEmdUkb731FnPnzqVevXqMHTuWUaNG8c8//5Rrw5o1a/jggw/4/PPPufvuu1m3bh1z586lSZMmJsfFxcURGBjI1q1b7bYtKSmJJ554gnHjxvHCCy9w4MABXnvttXJtcgZCvGxgEC8Z9G0dSt1a3qTnFvHnmVT6tw1zsXUCZ3P27Fk2bNjA1q1bDaWXmjZtatgfEhICQP369QkODi73ejk5OaxevZq1a9fSt29fAL788ksiIiJMjhs1apTh76ZNm7JgwQK6du1Kbm4utWrVMuz74IMP6N27NwBTpkxh4MCBFBYWlpvkuXDhQkaPHs3IkSMBmD59Ov/73//Izc01Oc7f35/ly5ebLNkpz7bFixfTrFkz5s6dC0B0dDTHjx/n448/Lvffp7II8bKBftiokMlQKuQ81bkhS3ZcYN2+RCFeduCrVHDqvapb/VDevR3lyJEjKBQKg0BUlosXL1JcXEy3bt0M24KCggxDMj0HDx7k3Xff5ejRo9y8eROtVpdfmJiYSJs2bQzHtW/f3vB3eHg4AKmpqURFRdm0Iz4+npdfftlkW7du3fjzzz9NtrVr167MWsPybDt9+jTdu3c3Oae61hoL8bKB8bARYHDXSJbsuMCOs2lcyywgIrj8YcPtjEwmq9DQzVXYMwx0Nnl5efTv35/+/fuzZs0a6tWrR2JiIv379y8TNFcqlYa/9etG9WLiDPz9/StsmysQgRsb6HNUFbfEq0ldf+5qGoJWgu8OXHGhZYKqoF27dmi1Wnbs2GFxv94rsbfcT9OmTVEqlezfv9+wLSsri7Nnzxrenzlzhhs3bvDRRx/Rq1cvWrVqVSYgXlmio6NNbADKvLeEPba1bt2affv2mWzbs2dP5Y22AyFeNjAeNuoZ0k3nom84kGSSgS/wfBo3bszw4cMZNWoUP//8MwkJCWzfvp0NGzYA0KhRI2QyGb/++itpaWllYkbmBAQEMHz4cN544w22bdvGyZMnGT16NHK53OA5RUVFoVKpWLhwIRcvXmTjxo3MmjXLqc81YcIEVqxYwerVqzl37hzvv/8+x44dK7fqhz22jR07lnPnzvHGG28QHx/P2rVrLa5JrgqEeNnAfNgI0L9tGMF+Sq5mFrDx6FVXmSaoIhYvXsxTTz3Fyy+/TKtWrXj++efJy9Pl9jVo0ICZM2cyZcoUQkNDGT9+fLnXmzdvHj169ODhhx8mJiaGu+++m9atWxuC7PXq1WPVqlV89913tGnTho8++og5c+Y49ZmGDh3K1KlTef3117nzzjtJSEhgxIgR5Qb67bEtKiqKH374gZ9//pkOHTqwZMkSPvzwQ6fabw236R5UXTjSPWjX+XSeXb6X6NAAfn/1XsP2z7ef5z9b4okM8SUu9j5UXuI3oLCwkISEBJo0aVLhEie3A3l5eTRo0IC5c+cyevRol9nRr18/wsLC+Prrr11yf1ufF3u/o54TTXUB+uVBxp4XwMieTVj1zyWSMgpYtz+RYT0au8A6gSdw+PBhzpw5Q7du3cjKyuK9994D4NFHH602G/Lz81myZAn9+/dHoVDw7bff8scff5jkc3kiwmWwgXGelzG+KgUT+rYAYEHcefLVJdVtmsANSExMpFatWlZf+kKac+bMoUOHDsTExJCXl8fff/9N3bp1nWZH27ZtrdqwZs0aZDIZmzdv5t5776Vz587897//5YcffijTRtDTEJ6XDQwBe3P1Ap7pGsnyvy9y+UY+X/5ziXF9mle3eQIXExERwZEjR2zuj4qK4uDBg1Vqx+bNmykuLra4LzQ0FF9fX/74448qtcEVCPGygb4WodzCrIxSISe2X0smrTvCkh0XGNo9imC/ijcTEHgeXl5eNG/u+h+tRo0audoElyCGjTbQDxsteV4Ag9pH0CosgJzCEhbvuFCdpgkEtz1CvGwgWcjzMkYul/HmAN1Sj1X/XCI5q7DabBMIbneEeNmgdLbR+jF9ouvTpVFtikq0LPjzXDVZJhAIhHjZoLxhI+jWmE1+sBUA6/cnkSCKFQoE1YIQLxvoZxstBeyN6do4hD7R9dBoJeZtPWvzWIFA4ByEeNnA1myjOa/318W+/nv0GievZVWlWQI3o3HjxsyfP9/wXiaT8fPPP7vMntsFIV420NoxbNTTNiKIRzroiszN+d3+8ryCmsf169d58MEHXW1GjUeIlw00dg4b9cT2a4mXXMa2+DT2JWRUpWkCNyYsLAxvb2+X2iBJEiUlNXvlhxAvG5QG7O07vnFdf57uGgnAf7acqXATiBqDJIE6zzUvB//t7WmUYS/Gw8ZLly4hk8n48ccf6dOnD35+fnTo0IHdu3ebnLNz50569eqFr68vkZGRTJw40VDNAuxv0vHbb7/RuXNnvL292blzp8O2exIiw94Gko3lQdaY1LcFPxy8woHLN/nzTCp9W4dWlXnuT3E+fBhR/nFVwbRroPIv/7hb2NMoozK89dZbzJkzhxYtWvDWW28xZMgQzp8/j5eXFxcuXGDAgAG8//77rFy5krS0NMaPH8/48eP58ssvAfubdEyZMoU5c+bQtGlTateu7RTb3RUhXjYoXZhtv3iFBvow4u7GfLHjIp/8Hk+f6PplqlII3At7G2VUhtdff52BAwcCMHPmTNq2bcv58+dp1aoVs2fPZujQoQYvr0WLFixYsIDevXuzePFifHx87G7S8d5779GvXz+n2e3OCPGygcasDLS9vNS7GWv3JnImOYf/HrvGox0bVIF1HoDST+cBueredmJvo4zKYK15RqtWrTh69CjHjh1jzZo1hmMkSUKr1ZKQkEDr1q3tbtLRpUsXp9ns7gjxsoFhttEBzwsg2E/F2N7N+OT3eOb+7ywP3hF+exYslMkcGrrVZGw1z8jNzeXFF19k4sSJZc6LiopyqBGGeRONmsxt+I2yH/1sY3m1vi0x8u7G1K3lTWJGvuiy7ebY0yijKrnzzjs5deoUzZs3L/NSqVTV0qTDE3G5eH322Wc0btwYHx8funfvXqYTiTnz588nOjraMCvz6quvUlhYNQuiHZ1tNMZP5cXEvrpyKQvizomChW6MPY0yqpLJkyeza9cuxo8fz5EjRzh37hy//PKLoUZ+dTTp8ERcKl7r168nNjaWGTNmcOjQITp06ED//v2t/qqsXbuWKVOmMGPGDE6fPs2KFStYv34906ZNqxL7HElStcQzXaOIDPElLaeIVbsuOdEygbMpr1FGVdK+fXt27NjB2bNn6dWrF506dWL69OmGCYPqaNLhkUgupFu3btK4ceMM7zUajRQRESHNnj3b4vHjxo2T7r//fpNtsbGx0t133233PbOysiRAysrKKvfYT7fGS40m/ypN+/GY3dc358dDSVKjyb9K7WZskTLz1BW+jrtTUFAgnTp1SiooKHC1KU4hNzdXCgoKkpYvX+5qU2oktj4v9n5HXeZ5qdVqDh48aFJHWy6XExMTUyaBT0/Pnj05ePCgYWh58eJFNm/ezEMPPWT1PkVFRWRnZ5u87KWynhfAIx0aEB0aQHZhCUv+EgUL3ZXDhw/z7bffcuHCBQ4dOsTQoUOB6m2UIXAMl4lXeno6Go2G0FDTJM7Q0FCSk5MtnvPss8/y3nvvcc8996BUKmnWrBn33XefzWHj7NmzCQoKMrwiIyPttlHfU9aRPC9zFHKZYdH2l/8kkJotCha6K5YaZZw+fdpmkw2B6/CoVInt27fz4Ycf8vnnn9O9e3fOnz/PpEmTmDVrFu+8847Fc6ZOnUpsbKzhfXZ2tt0CpqlAhr0lYlrX586oYA4lZrLgz3O8/1i7Sl1P4Hw6depksVFGQUGBzSYbAtfhMvGqW7cuCoWClJQUk+0pKSmEhYVZPOedd97hueeeY8yYMQC0a9eOvLw8XnjhBd566y3kFkqeent7V3iRrDOGjaBLtXhzQCueWbqHdfuSeL5XUxrVuX3ycTwZX19ft2iyISiLy4aNKpWKzp07ExcXZ9im1WqJi4ujR48eFs/Jz88vI1AKhQKgShZB61MlnDFbflfTOvRuWY+SGl6wsCr+PwhqHs74nLg0VSI2NpZly5axevVqTp8+zUsvvUReXh4jR44EYNiwYUydOtVw/KBBg1i8eDHr1q0jISGBrVu38s477zBo0CCDiDkTTTkNOBzljVuxr41Hr3H6uv0TB56APoM8Pz/fxZYIPAH958R45YGjuDTmNXjwYNLS0pg+fTrJycl07NiRLVu2GIL4iYmJJp7W22+/jUwm4+233+bq1avUq1ePQYMG8cEHH1SJfc4aNuq5o0EQA9uHs+nYdeb8Hs+KEV2dcl13QKFQEBwcbMjR8/Pzq5YET4FnIUkS+fn5pKamEhwcXCmnQybdZn5+dnY2QUFBZGVlERgYaPPYt38+zjd7EpnUtwWv9mvplPtfTMul36d/odFKfD+2B10ahzjluu6AJEkkJyeTmZnpalMEbk5wcDBhYWEWf+Ds/Y561GxjdaOvYe8szwugab1aPN2lId/uS+LjLWfY8GKPGuOhyGQywsPDqV+/vtX28wKBUql0SphHiJcNJCelSpgzsW8Lfjx0lf2XbrI9Po0+reo79fquRqFQVEkMUiAwxuULs92ZihQjtIfwIF+G92wMwH9+jzfE1gQCgf0I8bJBaZKq86/9Uu9mBHh7cfp6Nv895qKCfQKBByPEywbaKvK8AGr7q3jh3qYAzNt6lmJ9gE0gENiFEC8baJywttEWo+5pQt1aKi7fyGeDKFgoEDiEEC8bODvPyxx/by/G99EtPfm/P85RoNZUyX0EgpqIEC8bGAL2Vdj9Z0j3KBoE+5KaU8Tq3Zeq7D4CQU1DiJcNnL08yBLeXgpDAuzi7RfIKhD5UQKBPQjxsoFUhbONxjzeqQEt6tciq6CYpaJgoUBgF0K8bFBVeV7mGBcsXLnzEqk5omChQFAeQrxsUNGmsxXhgTahdIwMpqBYw6I/z1f5/QQCT6dS4pWbm1vh+vCeQFXmeZmjK1io876+3ZdI4g1RWkYgsIXD4pWQkMDAgQPx9/cnKCiI2rVrU7t2bYKDg6ldu3ZV2OgyqmO20ZiezerSq0VdijUSn/5RcwsWCgTOwOGF2f/+97+RJImVK1cSGhpaYyoiWKI6ZhvNeaN/NH+fS+fnI1d5sXdTWoXZLtsjENyuOCxeR48e5eDBg0RHR1eFPW6FthIdsytK+4bBPNQujM3Hk5nzezzLh9ecgoUCgTNx+GvZtWtXkpJuj6Uses+rOmJexrz2QDQKuYw/Tqdy8HJGtd5bIPAUHPa8li9fztixY7l69Sp33HFHmRrU7du3d5pxrqaqlwdZo1m9Wjx1Z0PWH0ji4y3xrH/hrho9PBcIKoLD4pWWlsaFCxcMTTJAN1MmSRIymQyNpuaszzM0na1m8QKYFNOCn45cZV9CBjvOpnFfdM0qWCgQVBaHxWvUqFF06tSJb7/9tuYH7LXVH7DXExHsy7C7GrF8ZwKf/B7PvS3quUREBQJ3xWHxunz5Mhs3brwtGnFqXRTz0vNyn+as25/EyWvZbDp+nUEdIlxih0DgjjgcsL///vs5evRoVdjidpTmebnm/iH+Kp7vJQoWCgSWcNjzGjRoEK+++irHjx+nXbt2ZQL2jzzyiNOMczWuyPMyZ3SvJny1+xIJ6Xl8d+AKz3aPcpktAoE74bB4jR07FoD33nuvzL4aF7B30WyjMbW8vXi5T3Nm/XqK/4s7yxN3NsBHKTrzCAQOD4i0Wq3VV00SLjDK83JxoHzorYKFKdlFfCUKFgoEgIPiVVxcjJeXFydOnKgqe9wKrb7prItnVH2UCibFtADg8+0XyC4UBQsFAofES6lUEhUVVeM8LGtoq6jpbEV4olMDmtevRWZ+Mcv+uuhqcwQCl+PwsPGtt95i2rRpZGTU/GUr1VWM0B68FHJef0BXLnrFzgTScopcbJFA4FocDtgvWrSI8+fPExERQaNGjfD39zfZf+jQIacZ52oMeV5uUrKxf9swOjQM4uiVLD7bdp53H2nrapMEApfhsHg99thjVWCGe+LKDHtL6AoWtmLo8r2s2XuZ0fc0ITLEz9VmCQQuwWHxmjFjRlXY4ZZUdzFCe7i7eV3ubl6Hf87f4Js9l5n6UGtXmyQQuASHxUvPwYMHOX36NABt27alU6dOTjPKXdAvzHYXz0vPoPYR/HP+BqeuV77sdm5RCS98dYCY1qGMuqeJE6wTCKoHh6M5qamp3H///XTt2pWJEycyceJEOnfuTN++fUlLS3PYgM8++4zGjRvj4+ND9+7d2bdvn83jMzMzGTduHOHh4Xh7e9OyZUs2b97s8H3tQeMGSaqWiA4LAOBMck6lr7UjPo1dF27wzd7Llb6WQFCdOCxeEyZMICcnh5MnT5KRkUFGRgYnTpwgOzubiRMnOnSt9evXExsby4wZMzh06BAdOnSgf//+pKamWjxerVbTr18/Ll26xPfff098fDzLli2jQYMGjj6GXbhLkqo5LUJ14pWWU0RGnrpS1zp1PQuAAvXtkf4iqDk4PGzcsmULf/zxB61bl8Za2rRpw2effcYDDzzg0LXmzZvH888/b6gNtmTJEjZt2sTKlSuZMmVKmeNXrlxJRkYGu3btMqypbNy4saOPYDdaNwvY66nl7UXD2r5cuVnA2ZQc7mpap8LXOn1d570VFAvxEngWFVoeZL4YG3QJrFqt/VUP1Go1Bw8eJCYmptQYuZyYmBh2795t8ZyNGzfSo0cPxo0bR2hoKHfccQcffvhhlSXNuluqhDHRt7yvsymVGzqeuqaLm+ULz0vgYVSoJM6kSZO4du2aYdvVq1d59dVX6du3r93XSU9PR6PREBoaarI9NDSU5ORki+dcvHiR77//Ho1Gw+bNm3nnnXeYO3cu77//vtX7FBUVVai3pCRJpZVU3czzAmh5K+4VX4m4V0aemuRsXXdudYnWEOMTCDwBh8Vr0aJFZGdn07hxY5o1a0azZs1o0qQJ2dnZLFy4sCpsNKDVaqlfvz5Lly6lc+fODB48mLfeeoslS5ZYPWf27NkEBQUZXpGRkfbdy+h77G7DRnCO53XabLZSDB0FnoTDMa/IyEgOHTrEH3/8wZkzZwBo3bq1yfDPHurWrYtCoSAlJcVke0pKCmFhYRbPCQ8PR6lUolCUloRp3bo1ycnJqNVqVCpVmXOmTp1KbGys4X12drZdAmbshbhbwB5KZxzjk3MM/QMcxVy88tUl1PKucPaMQFCtVOiTKpPJ6NevH/369avwjVUqFZ07dyYuLs6Qta/VaomLi2P8+PEWz7n77rtZu3YtWq0W+a1A1NmzZwkPD7coXADe3t54e3s7bJ8+3gXulyoB0LSePwq5jOzCEpKzCwkP8nX4GuZ5YmLGUeBJVEi84uLiiIuLIzU1tUyQfuXKlXZfJzY2luHDh9OlSxe6devG/PnzycvLM8w+Dhs2jAYNGjB79mwAXnrpJRYtWsSkSZOYMGEC586d48MPP3Q4RcMejD0vdxw2enspaFLXn/OpucQn51RMvK6JYaPAc3FYvGbOnMl7771Hly5dCA8Pr1T3oMGDB5OWlsb06dNJTk6mY8eObNmyxRDET0xMNHhYoBuy/v7777z66qu0b9+eBg0aMGnSJCZPnlxhG6yhkYyHjU6/vFOIDg3gfGouZ1NyHG6Npi7RciEtFwBvLzlFJVox4yjwKBwWryVLlrBq1Sqee+45pxgwfvx4q8PE7du3l9nWo0cP9uzZ45R720Lr5p4XQMvQADYdv058cq7D555LzaFYIxHo40VYkA9nU3LFsFHgUTgsXmq1mp49e1aFLW6FyWyjG8a8AKLDagEVm3HUJ6e2Dg+ksEQ39BfiJfAkHB4QjRkzhrVr11aFLW6FRisRLUvEG7XbNtZteStd4lxqjsM5Wvp4V5uIQPxuNfTIFzEvgQfhsOdVWFjI0qVL+eOPP2jfvn2ZbPt58+Y5zThXIkuPZ71qFuekhpDfG/xCXG1SGRrV8cfbS05hsZbEjHya1PUv/6Rb6NMkWocHkngjH4ACdUmV2CkQVAUOi9exY8fo2LEjQJlGHO7qoVSI/BvIkegqj4eV/WHo91C7kautMkEhl9EitBYnrmYTn5xjt3hJksTp5FueV3ggO87qqoGIYaPAk3BYvLZt21YVdrgdBeHdeVY9g9WqjwlPPwsr+sGzGyCio6tNM6FlaAAnrmZzNiWHAXdYTu4153pWIZn5xXjdEj8xbBR4Im6aBOB6tJLEWSmSZ6UPIPQOyE2BLx+Cc3+42jQT9MuE4h0I2uuHjM3r18LbS4GfSidewvMSeBJCvKygD4Cny+vAyM3QpDcU58Hap+HQ1y62rhT9Au2zDizQ1gfrW4cHAuAjxEvggQjxsoJJz0afIF3Mq/1gkDSwcTxsmw2S66sw6D2vhPQ8ikrsEx/jeBeAn1IXPRDDRoEnIcTLChrzbtleKnj8C+j1mu79jo/gl/GgcW336vAgHwJ8vCjRSiSk59l1jnGOFyCGjQKPxGHx+uuvvygpKTulXlJSwl9//eUUo9wBvedlMoMqk0Hf6fDwpyCTw5FvYO1gKKp8LfmKIpPJSuNedg4dM/N1paNDA3UL1sWwUeCJOCxeffr0sdgtOysriz59+jjFKHegtPmGhZ1dRsEz34LSDy7E6QL5OZYLKFYHFS1MqNdlMdso8EQcFi9rtaNu3LhRpnu2J2OIeVnLXYseAMN/Bb+6kHwMlveDtPhqtNDIlEoWJiwdNookVYHnYHee1xNPPAHohikjRowwqZGl0Wg4duxYjVrzGB0WwN9vluNJNuwMY7bCN09CxkVY8QAMWQeNelSPkbdoWYF0CWMMw0bheQk8CLvFKygoCNB5XgEBAfj6ltaPUqlU3HXXXTz//PPOt9BFeHspiAzxK//AkKYweit8+wxc2Q9fPQpPLIW2j1W5jXr0VVWTMgrIKyrB38FqqIZho4h5CTwIuz/lX375JaBrNfb666/XqCFipfGvC8M2wg9jIH4TfDcCsj+AHuOq5fYh/irqBXiTllPEudRcOkYG2zzePMHDT6X7GIiAvcCTcDjmNWPGDCFcllD5weCvoesYQILfp8GWqeBAO7jKYIh7ORS018XzfFW6j4EYNgo8CYfFKyUlheeee46IiAi8vLxQKBQmr9sauQIemgMxM3Xv93wO34+A4sIqv7U+7nWmAq3QfG95XmLYKPAkHF6YPWLECBITE3nnnXcqXQa6RiKTwT2vQFBD+GksnPoFclPhmbVVWlanMoUJ9TEvfe9Gdy2+KBAY47B47dy5k7///ttQFkdghXZPQa36sO7fkLi7ysvqVGbG0VdV6jEXFGtE+zOBR+DwsDEyMhLJDdb0eQRN7oVRWyCwAejL6lw7UiW3anFLvNJyisjIU9s81vx/n7eX3JCwmi9yvQQegsPiNX/+fKZMmcKlS5eqwJwaSGgbGPNHlZfVqeXtRWSILn3F0aGjTCYzDB3FjKPAU3BYvAYPHsz27dtp1qwZAQEBhISEmLwEFgiMqJayOo5m2huHK31FoqrAw3A4uDF//vwqMOM2QF9WZ+N4OLZe99+sK3DfFFMVqQQtQwP443Sqw2scoVS8xIyjwFNwWLyGDx9eFXbcHujL6gQ1hL/n6srqZF2BQfNBoSz39PKIruACbSit6SWGjQJPoUL1vC5cuMDbb7/NkCFDSE1NBeC3337j5MmTTjWuRlKFZXWMZxxtTapY2ifK4gg8DYfFa8eOHbRr1469e/fy448/kpur69Z89OhRZsyY4XQDayxVUFanaT1/FHIZOYUlJGc7lhgryuIIPA2HxWvKlCm8//77bN26FZVKZdh+//33s2fPHqcaV+Nxclkdby8FTW+1P3N06CjK4gg8DYfF6/jx4zz++ONlttevX5/09HSnGHVboS+rE9IUshJ1ZXUu767w5QwNOeyYcTSeJhDDRoGn4bB4BQcHc/369TLbDx8+TIMGDZxi1G2HvqxOw65QmKkrq3Py5wpdqrQkdK5D51kbNpZoqmdhuUDgKA6L1zPPPMPkyZNJTk5GJpOh1Wr5559/eP311xk2bFhV2Hh7oC+rEz0QNEW6sjq7P3P4Mi3tyPWyFMq31IRj9ubTdHxvK4k38h22QyCoahwWrw8//JBWrVoRGRlJbm4ubdq04d5776Vnz568/fbbVWHj7YOhrM7zVLSsTrTRsFFfh98eLA0bv/jrIrlFJcyPO2v3dQSC6sLhPC+VSsWyZct45513OHHiBLm5uXTq1IkWLVpUhX23H3IFPPQJBEfC1um6sjrZV+HxpaD0Kff0qBA/fJRyCou1JGbk06SufbXXbPVuLCoWQ0eB+1Hh8gFRUVFERUU50xaBHpkM7p6kW9DtYFkdhVxGi/oBHL+aRXxyjv3iZSNgXyjSJwRuiF3iFRsby6xZs/D39yc2NtbmsfPmzXPYiM8++4xPPvmE5ORkOnTowMKFC+nWrVu5561bt44hQ4bw6KOP8vPPPzt8X7engmV1WobqxOtsSg4D7gizepxxLTYfw/KgsqkSRSXC8xK4H3aJ1+HDhykuLjb8bY2KFCZcv349sbGxLFmyhO7duzN//nz69+9PfHw89evXt3repUuXeP311+nVq5fD9/Qo9GV11jxVWlbn2Q0Q0dHqKfrChFZre1kIhRmqSlgYIgrPS+CO2CVe27Zts/i3M5g3bx7PP/88I0eOBGDJkiVs2rSJlStXMmXKFIvnaDQahg4dysyZM/n777/JzMx0qk1uh76szpp/QcoJXTb+019BixiLh7esQD17W0mqhSVCvATuR4XWNjoLtVrNwYMHiYkp/RLK5XJiYmLYvdt6ouZ7771H/fr1GT16dLn3KCoqIjs72+TlkThQVkc/43gxPY8iO4XHx0ZVCRGwF7gjdnle+oaz9vDjjz/afWx6ejoajYbQ0FCT7aGhoZw5c8biOTt37mTFihUcOXLErnvMnj2bmTNn2m2TW2MoqzMBjq2zWlYnLNCHAB8vcgpLuJiWR+vwwHIvXTpstCBeIuYlcEPs8ryCgoIMr8DAQOLi4jhw4IBh/8GDB4mLizM0pq0qcnJyeO6551i2bBl169a165ypU6eSlZVleCUlJVWpjVWOlwoeXwK9Xte93/ER/DIeNMWGQ2QyGa0cWCYEtns3ipiXwB2xy/PSN5wFmDx5Mk8//TRLliwxtDrTaDS8/PLLBAaW/wtvTN26dVEoFKSkpJhsT0lJISys7CzZhQsXuHTpEoMGDTJs095K4PTy8iI+Pp5mzZqZnOPt7Y23t7dDdrk9Mhn0fQeCGsCm13RldXKuw9OrwVsnWi1DA9h/6abFBdr6eL3x9Iq+d6OlYaMQL4E74nDMa+XKlbz++usmPRoVCgWxsbGsXLnSoWupVCo6d+5MXFycYZtWqyUuLo4ePXqUOb5Vq1YcP36cI0eOGF6PPPIIffr04ciRI0RGRjr6OJ6NjbI60Q56XvrejWLYKPAUHBavkpISi/GoM2fOGLwgR4iNjWXZsmWsXr2a06dP89JLL5GXl2eYfRw2bBhTp04FwMfHhzvuuMPkFRwcTEBAAHfccYdJiZ7bBitldRxthWbeu9EYZ4nX/ksZfL3nsug+JXAKDmfYjxw5ktGjR3PhwgVDIunevXv56KOPDILjCIMHDyYtLY3p06eTnJxMx44d2bJliyGIn5iYiFzu0klR90dfVuebJyHjIqx4gDaP62YikzIKyCsqwb+cXozGvRvz1SUE+FS+LLU5/1qim0FuUsefe1rYF7MUCKzhsHjNmTOHsLAw5s6dayiNEx4ezhtvvMFrr71WISPGjx/P+PHjLe7bvn27zXNXrVpVoXvWOPRldb59Bq7sJ3DDUwz2n8D6vDs5m5JDp6jaNk/X926UJN3QsSrES8/ljDzuQYiXJ3H5Rh5vfH+MNuGBTHmwFT5KRfknVTEOuzRyuZw333yTq1evkpmZSWZmJlevXuXNN980iYMJXIBZWZ3ZmrmMUvxWJu6lvTVsM14QUZ29GxVGN37ju6NM/v5Yld5PUDmuZhbw7LK97EvIYNWuSzz9xW6uZha42qzKJakGBgY6PMMoqGKMyurIkZiu/JpG+983lNUpUGsMM4q1/U1jhLbanxU7sSihXK4Tr8x8Nd8dvML6A0ncyC1y2vUFziMlu5Bnl+3hamYBjer4UdtPybErWQxauJNdF1xbOblCVSW+//57NmzYQGJiImq1aWv5Q4cOOcUwQSW4VVbnaE4AHc7M467U9fB9ITy+lNQcnTD5KhUEmMXBbDWezVdrCPJ1TuxR73kp5KUeWL5aQx2nXF3gLNJzixi6fC+Xb+QTGeLLuhfuokQjMfabg5y8ls1zK/Yx9cFWjL6nSYXWNVcWhz+NCxYsYOTIkYSGhnL48GG6detGnTp1uHjxIg8++GBV2CioCDIZ0t2TmKgeTzFeurI6Xz9Gepoupy400LvMB868d6PxbkvVJiqKfv7FeM5RNLt1LzLz1Ty3Yh/nU3MJD/Jh7Zi7CA/yJTLEjx9e6skTdzZAo5V4f9NpJq474tTPh704LF6ff/45S5cuZeHChahUKt588022bt3KxIkTycrKqgobBRWkRf1abNT2ZJh6MlrvQEjcTctfn6ShLI36gWULGxqvb9RqJYwzGiorLlqj9Av5LVU0vr4lb0/gGrILixm+ch+nr2dTL8CbNWO6ExniZ9jvo1Qw918dmPlIW7zkMv579BpPfL6LyzfyqtVOh8UrMTGRnj17AuDr60tOji4Y/Nxzz/Htt9861zpBpfD39iIyxJfd2rYcf2A9BDYgIPciP6pm0FmVWOZ44/WNGrNcrPyiyolLsVEOoGG4aCKOouWaO5CvLmHUl/s5eiWL2n5K1ozpTtN6tcocJ5PJGN6zMWufv4u6tbw5k5zDoIU72RafWm22OixeYWFhZGRkALpqqvpejQkJCSL50A2JDtVNqBwpioAxf5Di25z6skxeTZoE5/4wOda4LI55ompeJcWlWFN6vXMpuTz/1QFOJ5dW+BAt11xPYbGGMasPcODyTQJ8vPh6dHdDsrM1ujUJYdPEe7gzKpjswhJGrdrPgrhzJp52VeGweN1///1s3LgR0CWsvvrqq/Tr14/Bgwdb7OcocC0mhQkDI5jXcD47NW1RaQvKlNUxGTaae16VFC/jFmrfH7zC1lMprN9fukheDBtdS1GJhrHfHGTXhRv4qxSsHtWNOxrYV2ghNNCHdS/0YGj3KCQJ5m09y4vfHCS7sLj8kyuBw7ONS5cuNSwDGjduHHXq1GHXrl088sgjvPjii043UFA5zAsTXs7zYmTxZP5s/D2RSRtNyuqYDBu15uJVyWGjkeeVmlNo8l9nXF9QcYo1WiZ+e5jt8Wn4KOWsHNGVO8tJajZH5SXng8fb0aFhMG//coKtp1J4bNE/LB3Wmeb1bXtvFcUhz6ukpIT333+f5ORkw7ZnnnmGBQsWMGHChNtzbaGbo1+gHZ+SgyRJpGYXUYwXV+/7tExZnVpKncAUqDVlPKFKx7yMPC+9kKXllOZ22Vu5QpIkh1q6CWyj0Uq8tuEov59MQeUlZ9mwLnRvWvGklae7RvLdiz2ICPLhYnoejy76hy0nyjapdgYOiZeXlxf/+c9/KCkRwVVPoWndWnjJZeQUlpCcXUhKts7bCQ3y1ZXVefhTkMnhyDc8lzAZfwrIV2vKxKAqG/Mq0ZQVHGPxstfzmvDtYbp/GEdOFQ9Jbge0WokpPxxj49FreMllLB56J71a1Kv0dTtEBrNxwj3c1TSEPLWGsd8c4j9bzjj9R8fhmFffvn3ZsWOHU40QVB0qL7mh/dmhy5nk3RKJ+gG3apwZldVpmrWH9apZeOWnlBGTyg7r1BYy9G/mlwqQvdf/9dh10nOL+O1EcvkHC6wiSRIzNp7ku4NXkMtgwZBO9G0dWv6JdlK3ljffjO7OmHuaALB5x042LHiTzHx1OWfaj8MxrwcffJApU6Zw/PhxOnfujL+/aV/ARx55xGnGCZxDdFgA51Jz+ftcGgAB3l6mVSZuldUpWP0kdxRfIuLcWK5Hf2NyjUoH7Mspl2Sp8cfByzfZdiaV8fc3L7MQ2Ete/RndNQVJkvhw82m+3nMZmQzmPt2Bh9qFO/0+Xgo5bz/chnv9LtFux7vUzsxhz28NuevJSc65vqMnvPzyy4Dl/owymQyNRgRe3Y3o0AB+5Tp/ndWJV/1AC5VlG3ZmS49v6Lh9FE2KkwnY9CRdZJM4ILUCIK+yMa8S20OGPAue15OLdwFQy8eLsb1NK+QqhHhVmI1Hr7Hs7wQAPny8HY93alh1NzuziXt3jQJZIdf9W9P9gWecdmmHh41ardbqSwiXe9LyVtD+WtateJeF7HoAbVBjnlTP5LyqNUp1FmtUs3lQvheovOdVXK7nZf2zcym9bOa2EK+Ks+fiDQCG92jEkG5V2PV+3zJY/28oKYQW/QmfFIcswHlDU1Hl7zYg2izR0Jp4+akUZBDIjNofcj28L96yYj5TLmCU4rdKx7wsBeyNySuyLo4qr7IfU4ULFgLXFBIz8gFo1zC4am6g1cLWGbD5dZC0cOdweGYtqPzLP9cB7B42FhQUEBcXx8MPPwzouvIUFZXOFikUCmbNmoWPj+UvhsB1RIb44aOUU3ir/6LFYSOlSaqZxUr+6j6XwqQ3GO61lenKr/lfch5oV1AswaHLN+kYFYy3l/3128orqWNLHFUKC+IlPK8KoxevKKP1ik6jpAh+GQfHv9O9v/9tXUpOFfzY2O15rV69mi+++MLwftGiRezatYvDhw9z+PBhvvnmGxYvXux0AwWVRyGX0cIoUTA0wIrnZZSkmlcMM0pGMLtkCAAPZP8I349g/e5zDF66h8XbLzhkQ/niVb7nZbzk5HYXr6ISDVtOJJNV4FjKSIlGy7VMXfggMsTXuUYVZOpKkR//DuRe8NhiuPeNKhEucEC81qxZwwsvvGCybe3atWzbto1t27bxySefsGHDBqcbKHAO+mRVsDVsLC2Jo0tSlfGj71MmZXV6/DOaIHJJynCskmax2bDRfLbQpud1S7wsLu6+Tfn4t3jGfnOQUav2O3Te9axCNFoJlUJu9UesQmRdgS8fhEt/g6oWPLsBOj7rvOtbwG7xOn/+PO3atTO89/HxMWmM0a1bN06dOuVc6wROwzjuFWpl2Gjcu1EfQK/jr2KjtidTfGeAdxDNCo7zg+pd/PKvOnT/EjPPq5lZpQJz8TL21PTipRGel4HvD+rWhR68fNOh85JuDRkbhvgaKtpWmpSTuq5VqaegVhiM/A2a93XOtW1gt3hlZmaaxLjS0tJo3Lix4b1WqzXZL3AvWtrheRn3btSLSd1aOqHbpW2LduRvXJdCaC6/xmtJ4+DaEbvvb56k2ibCtHy4+bDReHmSPuZl7L3d7uJV0Vx1fbwrsraT4l0Xd8DKAZBzDepG67pYhbd3zrXLwW7xatiwISdOnLC6/9ixYzRsWIX5IoJK0eqWeHnJZdQLsOx5GfduzC3SxVLq1tKtV80rKuGqqgmPFb3HaW0UwZoMXZNbs7I61jCfbWwdbjoDau55FRq91wuVieclMx92ljDh28NsOlY16+hqCk4N1h/7ThfjKsqGRnfD6N8huApTL8ywW7weeughpk+fTmFhYZl9BQUFzJw5k4EDBzrVOIHzCA30YdajbfnwiXZW21YZ927MyNMt49B7XvlqDedTc0khhH+pp3NE2RGK88qU1bGGeYZ98/plh43GAXljMdOLlvHQ07yE9dK/LvLfo9cYt/Y26aFQQdcr6aYuVlmpYL0kwc5P4ccxoC2Gto/Dv38EX8cqUVQWu1Mlpk2bxoYNG4iOjmb8+PG0bNkSgPj4eBYtWkRJSQnTpk2rMkMFlee5Ho1t7jfu3XjjlnjVuSVeJVqJU9d1xQNz8WOa7ztsbv0dHFtnUlbH2syS2szzsvTLX1iiKZ00MBo26muLlRiJm3nhy2tu0IrLE6i056XVwG9vwv7luvc9xkO/WaWNCaoRu8UrNDSUXbt28dJLLzFlyhTDh0cmk9GvXz8+//xzQ5drgWei792Yp9YYPK86tUrLHB1JyjT8nV0sR/PoYvak+3D3tVW6sjpZV2DQfFCUbVhrHrAPDyr7y59XZFm89KcaDz3NHQ/z2UyBZa7oY14VES91PvwwBuI3ATLo/yH0eNm5BjqAQ2sbmzRpwpYtW8jIyOD8+fMANG/enJCQkCoxTlD9+Kp04nUjVydegT5eqBRy1Botx65kGo4rLNZwOCmToRcf4AU/b6ZKy5Ad+QZyrsPTq8HbNKZlnuflpyo7dNUF7XWeXmp2aXhCc2vIaTz0NK84bqlqhcCUvKISg0ftsHjl3YBvB8OV/aDwhieWQtvHnG+kA1TI1wsJCaFbt25069ZNCFcNQx/3yr21XMdX5YWft25bSrZp/S29d7Y0vzcpD30JSj+4EKcL5OeYlqwx94xkMhn+ZgJmHOcyTgEweF426kGZe3aCsiTd1HldwX5KAn3KesdWybgIK/rphMsnGIb94nLhArG2UWCGvnej4b1Kgb+qrINeUKwhp7A0veEvOsPwX8GvLiQf0+X9pMUb9lsqRKePp+kxTpc4YCJetzwvk2Gj6fVut2FjRZ428UYF4l1XD+r+X2ZcgKAoGP0/aNSjAnd3PkK8BCb4mHlDvkqFySykPsFVF9Qv9cT2X8qAhp11eT4hzSArEVY8AJd3G443xzieBqWeV2GxhhNXS3uAagwBeyPvyux65S0/qmlUpFOXwzle8Vtg1cOQnw5h7XX/b+tFO3zfqkKIl8AEP7M0Cl+VwmR4165BsOFv42Hk/ksZHLiUQZfPL/Jb96+gYVcozISvHoWTP5fxlADq+Jt6XvqaYceuZJl4UpaGjWUD9reXeJmz81w6zyzdzcW0XKvHXDGkSdghXge+hHVDoDgfmvWFkZshIMxZ5joFIV4CE8wD6X4qhWEGEHTJrvrlOqlGNegv3cjnqSW7Sc8tYuLGRBi2EVo9DJoi+G4Ena6uLXOvumaeV0Gxbth44HKGyXaLw8YyntftNWw0598r9rLnYgaT1h2xeozB87KV4yVJ8Of78OsrunI2Hf8Nz64vMwHjDgjxEphgPmz0U3rh7126rUVoLXyV+gB+2YRlgNp+KlD5wdNfkXnHcECid8KnvOP1NTJKPSTzYaPe8zp4SRfvKl3TqNtvMttYJuZ1e3teevSTKJZIKi/Hq0QNP78Ef32ie997Cjy6yGLqizvgFuL12Wef0bhxY3x8fOjevTv79u2zeuyyZcvo1asXtWvXpnbt2sTExNg8XuAYloaNvkaeV/P6tQzemb77j3mxwIa1db/saq2Mjgce4MNiXVmd0V6/sUi5AG90X7BgX/OYl87zOpSoE6/uTXQz2fZ4XuqS20u8rPmZXgrLScKSJNlOUC3M1q2WOPotyBTwyELoM7XKytk4A5eL1/r164mNjWXGjBkcOnSIDh060L9/f1JTUy0ev337doYMGcK2bdvYvXs3kZGRPPDAA1y96liVA4FljIeNXnIZKi+5IeYlk+mqQeg9L30u1hOdGphcQz8L+eeZFEDGUs0gJjMRtaRgoGIfX6tmQ35Gme9FvlqDukRr6CqkrzyhD9hrbMS8bKVR3E5YW7CellNEUYkWuQwigs2GjdnXdektF7eB0l83TLxzWDVYWzlcLl7z5s3j+eefZ+TIkbRp04YlS5bg5+fHypUrLR6/Zs0aXn75ZTp27EirVq1Yvnw5Wq2WuLi4ara8ZmI8bNSLlD7mFRXih4/R7KO+acZ90fX4enQ32oTrKkXoxWvDgSuGa60vvIvhxVPIlvzoJo+Hlf3xz79mcu98tcakH2Ogr264oh8RGg8NzWfbbO2riVh7RGtdlfQ5XuFBviiNK9OmnoHlMZByHPzrw8hN0KKfs82tElwqXmq1moMHDxITE2PYJpfLiYmJYffu3XZdIz8/n+LiYqvJskVFRWRnZ5u8BNYxzvPSi5TeG2txazG1r9nQMsBHSa8W9fh86J0ABgHan2AaeN+tbctT6hkU+IZD+lkePTictrIEw/58dYlB+PxUCrwNMS/trf/amG00Gjaae2Gbj183Sb2oySisrDG0OGS89A+sfACyr0Cd5rpUiIhO1WGmU3CpeKWnp6PRaMqsiQwNDSU52b6mopMnTyYiIsJEAI2ZPXs2QUFBhldkZGSl7a7JGA8b9X93blwblUJOzK2mpL4qc/HyMvlvnlpDiUZrsct2j7vuwfelPyH0DvzU6WxQvUdv+VEA8otKE18DfLyQy/SlcHTnFhuJknmJHeN9xiJ3KPEmL685xMMLd9r7T+ARWEo9ARueV4ZZNYkTP8LXj0FhFkR2h9FboXbjKrC06nD5sLEyfPTRR6xbt46ffvrJauOPqVOnkpWVZXglJSVVs5Wehcmw8dZwsU90fU7M7M8zt9pkWfK8QDfM08dcrmYWoNeQBkYxFplMBoERMPI3rtTujr+siBXKT/iXYvutrP1iwzW9DHW8tCb/BfjPljMmNhgPG43/Ppuc4+C/gOtIyS6stIdoLeZl8Lxq+8KuRfD9SNCodeksw34BP89b5udS8apbty4KhYKUlBST7SkpKYSF2U6ImzNnDh999BH/+9//aN/eeuVGb29vAgMDTV4C6xjPNhp7YcYziua5YHqPS6mQ06yerr3V4cRMw/67m9cx/G0I0vsE8lv7BfyguQcvmZZPlEsZkL6K7IJiwzX1ZYr1TpZxLte5VNNkTONho7HnpfGg+Ff3D+N4eOFOLthINC0Pa+KVlJGPHC0PXl0A/3tLt7Hbi/D0V6B0ciOOasKl4qVSqejcubNJsF0ffO/Rw/r6qf/85z/MmjWLLVu20KVLl+ow9bbBz0LA3hxrw0aAVmG6H4fDt9IdFHIZ3ZvUwRIauZLXil9iYcljADya+RXR+6biRYmJ56XVShy7ksmb3x+zardxnOuej7dxLiXHcK6ncfxKxb0va+KVcuMmi5QLaHbxVuHIB96HBz8Guf3t69wNlw8bY2NjWbZsGatXr+b06dO89NJL5OXlMXLkSACGDRvG1KlTDcd//PHHvPPOO6xcuZLGjRuTnJxMcnIyubkV/7USlGI6bLT8wTauxKrykpv0b2x9a8bx8K3aX75KBV0al1bYlFH65dI5RTLmljzNtOLRaJHTJOknVijnUFepNnheJVptuV1yjB2s3KISXvtOF0eztCDc3alMapXSQp5XUXYanxTO4CHFPiSFCp5cAT0nuHUOlz04VM+rKhg8eDBpaWlMnz6d5ORkOnbsyJYtWwxB/MTERJMuRYsXL0atVvPUU0+ZXGfGjBm8++671Wl6jcTasNHkGKPtgT6mH6FWt2rTn7qmm9X1UcqJCvGjbi0V6blqk++LcdB5raYvAfUieT37I3pzjKZXXmF/lK5PqEZLuf0JzYeHx65kcfDyTTxQu8qUuLaE8ePO+19p9Y4ys403LyFb/QRd5RfIlvwI+Pd6aHKvs0x1KS4XL4Dx48czfvx4i/u2b99u8v7SpUtVb9BtjPE6RmviZTycrOVt+hHS53rph3E+SgUymYxOUbXZeioF46+leThqj1dXNnVezj37Xiay6Bwhe/5NM9mraKX6t2YerSuRJQ/rycW7mPpgK6vnuCv2NEYyftoFf543/G0y23jtCKz5F6q8VK5KdZgZMJOlNUS4wA2GjQL3Qt+7EbDRqKNUsALMitrVD/Cmtl/pNr3QPdIhAqVCRseoYKv3VmskUgPa8IR6JqmqhvgXXOMH1bs0zT9uNQUAbMe1ijxw2ZC8EsM5Q8zr3B+6rPm8VG4GtOSJoplo63mekNtCiJfABF8HPa8As2GjTCYzxL1019MdO6hDBKfeG8DD7SOs3rtYo0WSIFEKZWGjz8io3YFgWR5vpr7JAPleq+fZmlE0XvNYmVm86qQykSgvuUzXzWnt07ruTk3vY2XLxaQQUrG69W6MEC+BCaYxL8tRBWNRMxcvKJ1xBFPvzWRZCpaX+Oi35CmD2X3Pl/yu6YKKYj7hU0YpfrNoj62gvHFt+75zd1BYrLF6rCsx9h7tiXlZRuLhm1/pujlJGmj/DDz7HadvLXRwSq9GN0KIl8AEXztSJXxMPK+y5VKMG8pauwZYqMlVojW0OZPLZKD046XiV9jiNwg5EtOVX5cpqwOlrdEsYV5tIrucwL+rME71sKsZuNkje1HCx17LGHhjlW5Dr9fg8SWUyLzYm3ADgE5R1dtXsaoR4iUwQd+7EeybbbTkeZkMG22Jl9l7tUYyCJoMXfxGi5xltV6yWlYHbHte5jEvd60+oTERL8c8Lz8KWa6cy2Cv7WiRw8B50Hc6yGQcvZJFTmEJQb5K2jUIcrbZLkWIl8AEfe9GsJ7n5auy7Xk1r1/LEDi2dg1LpOcW8cnvuml/idLgc4kESzWDmKgeX6asDpj2eDTn232JJu/dNe/LuNCiPdqlTzOpRybrVe9xn+IoBZKK1VEfQNfRhuN2nksHdKscrCWweipukSohcC/0vRutZtgrred5gW5Y2ayeP2dTcvFRVuz30UsuQx8iK7olThu1PUktDmapch7d5PFIK/vztt+7rDlr/3Xd0fNKSM/jv0dLywPZ63k1lV1jtfJjIuVp3JACGKV+gzbBvUyO+ftcGgC9WtRznsFugvC8BGUoLYVj+bfNt5xhI5QG7a2lW4D1mlQAD7YLNyRc3jAqbbxH24Yn1e9yVaqDLP0sky6/bFJWpzzc0fPqM2c787Y6oMBAJ+L5QfUukfI0LmlDeUI9k6NSc5OUkpzCYsNKh3ua13WmyW6BEC9BGe5pXo8Qf5UhW94c45hXLW/L9c0HdYgg2E/J3c2sf2mslXVZ9Gwneresh+KWB5KeW2Sy/5zUkCeKZlIQ0pr6skyTsjrl4Y7iZY61fxcDpzbytdcH1JblckTbjCfUM7ks6QoZGJeB3n3hBhqtRJO6/jUuTQKEeAksMPuJduyb1pe6Zk1h9djK89LTr00oh9/pR0ybUIv7wbrnpb+vPkZj6bgUQjg1YD1/a+4wKatTHiVaLZIkMXvzaX454p6lw7W28mr3fgEbhuEtK2ar5k6eUb9NBqUTJMae187zunhXTfS6QIiXwApeCusfDXuGjVB+vpI1/6JegKl4WWPc9+cZVfymSVmdSYofbFxZJwzb4lP54q+LNtuEVRX2NAqxmHSr1cL/3obf3gQkvi6JYWzxqxRi+gNjvLbx71vB+l4thHgJBACoFHJDLpKl2cbKUup52T4uObuQYrxMyuq8qvyBj72W4UXZKq6g87zGfn3Imebazfu/nqLl278ZFq1bQ5+8W1is4YsdFzh/LR1+GA27FuoO6DuDH8JeRUPZeKLe80rKyCchPQ+FXMZdzSyXJPJ0hHgJHEYmk9GzWV2iQvwMbc4qhJVxo34G01o9dgsWGcrqaCQZg722s0I5B38Kyhyp0UomWffVyfKduomFT/+wHZyf+uNx4pNzWLz9Ap/9doD0JQ/DyR9BroTHl0KvWEO5oJjW9U3O1W/XDxk7RgYTWAU/MO6ASJUQVIivR3dDo5VsDi/Lw9rgTj/cVDiYrLlW05dkqTaLlAvprTjGetksRqrfII3SzHJ3SJUo77lu5hcz5qv93BmYx/eqmbSUXwXvQBj8NTS9DyitLlvbz7T3pd7z2lnDh4wgPC9BBZHJZJUSLrDseBnH0CqSVPmnVhfETpcCuUN+iZ+8Z9BMVhqYd4fKqvY8V62bZ5iROomW8qtcl0Jg5G8G4YLSoWWIv6l4KeQyNFrJ4HkJ8RIIqgBLKQFBvqVDHHvFyzwR9pikSx+4qA2joSydH1Tv0kWma9hx4PLNSljsHLbHW26orOdu+XE2qN4jRHuDeK0uLYSwO0yO0VoRL7lMxomrWWQVFBPg7UWHhsFOtd2dEOIlcBn+3mWjFsbxGXsdO0vDsEQplCfV73JI25xgWR5rVLN5UL63TDLogUsZZc6tavTNei3xuPxvVin/Q4CsgJOq9vxLPYPrlA2469MpapuJl0xWmlXfo1mdSnvH7kzNfTKB2zOiZ2Publ6HWY+2NWwz9bxKP571AyznnOmOs+yh3SSQZ9Vv8bumC96yYj5TLihTVuepJbtJziqs6CM4EYmXFT/zqWoxSpmGjZoeTPF9l2z8LR5t8LzMYl5glCLRsuYtCTJGiJfAZfipvFgz5i6e69HYsM1EvIw8qr5ms2rG2BpeFuLNS8WvsLqkH3KZ5bI617PKzkpWJwo0vO+1kjeVGwBYUjKIScXjSMop9dDM+zmeudWP0kshM3QWB8gvKuHQrc5NvWpocqoeIV4CtyLYqIS0caZE31bWM/XLW8isRc6MkhFWy+oYdz8y5sTVLDYfv05JVaZWqPNYovyUf3vFoZVkTC8ezkclQ5CQk5lfWnvMuOO3cTOSyzfyTf7N9iRkUKyRiAzxpVGdmrckyBghXgK3YGD7cABG39PEsC2vqNTzuLt5Xfq20nlfb/SPNjnXvsC+zFBWp0jyMpTVCSLXpKGunjPJ2Ty8cCcvrznEZ9suVOCJ7CA3DVYPop/iEIWSkpeKX+ErTf9yTzubUtoF3FepINi3dOi4L0EXw7uneb1KVGT1DIR4CdyChc904uiMB2gRWroYXF+22Ecpx1elYNmwLhx/9wEesLFesjw2ansyvHgK2ZIf3eS6ygzvrN5UZlgWd7p0RrBK1kDeuAAr+sHVg9yUaulic9qudp2qHzLqCfIrm4R6bw1OkdAjxEvgFsjlMpN4F+jWUB579wGOTH/AcEyAj7JMhQRbZaAtYVxWp7n8Gv+X+yaTF31tSOwEXePaqqKj7DyaZTFwMwGCG+lmRaWW5Z6nH76eNRYvGQSb/bvJZdDTRjWPmoIQL4FbE+ijLFMTzEepMInnVCRrXl9W57Q2ylBWZ9mXS1m3L5Evdlxg8fbKDRUlSSI+OadMw48Y+UG+Vb2PojADwjvCmD+4KFnvqGSMvqR1vJnnZZ5l375hsEVvrKYhxEvgkfz+Smnz1BJNxbLmUwjhX+rpJmV1Dv6ykNm/nTE57mJ6XplOR+Xx24lk+s//i2eW7iGroJhL6Xn8W7GVL5Tz8JWp+VPTkS1dV0At67Oo5ny7LxFJkjiTbLqwO9hMqGpyVr0xQrwEHomxN1aZxhK5+NlVVkcfCLcXfe38I0mZdJy5hc3zx/K+8ksUMolvS/rwfPFrjN0Q79A13990muTsQrILTYe05l5WTSz5bAmxMFvg8QzqEMHjnRpwZ6NgYub95fD5+rI616S6TPD6mVeVPxAhu8FbJaMoufUVMS5F7QhKSviP8gseV/wDwNzip1ioeZyKtpY9fsV0YqFBsG8Zz7OTja7kNQnheQk8Ho1Wy9NdI2leP4CPnmhHeJBPBa6iK6sz1UpZHUfjahl5agLI50vlxzyu+IdiScHrxS+yUPMExsI19cfjDl131qZTADSq48fbA1vTs1mdMsNG8+a+NZXb4ykFNZpiI8/jmW5R7J7at8LX+lbTl+eLXyNf8taV1VHNoh432XvxBtmFthvWFpWUBudvXEtgg2om9yhOkiv5MLr4db7X9C57P7PWbOWRlFFAVIgfG8fdw5heTZHJZLeNp2WOEC+Bx2OpUUhYoM77stX01hqWyurs2bebQUZZ7ua8+f1Rot/ewsW0XKSUk/zoPYPW8iRSpWAGq6fzl7aDw3ZYwttLzpJ/dzaJc4UH+dKkrm4N5F1NQ5xyH09AJjk6jeLhZGdnExQURFZWFoGBgeWfIHBbzqbkcD41l4fahZfZdyEtl2V/XeSl+5rx0+GrzP/jnMPXj5KlsEr5MU3lyWRK/oxRv8b6D2ItZvQ3nrIJgB7ykyz3no+/lMd5bQQjiidzRXIsgH5nVDBP3NmQ+X+co0ldP0q0Egue6cTGo9fo3Kg2dzUtW2UiIT2P9fuTeL5XE+pYaZziKdj7HRXiJajxSJJEk6mbK3RubbJZoZrDnfLzFElKXil+mddeeYPm9U29vcZTNvGIfBefKJfgLSthnzaa59WvkUUtu+7z8n3N+PxWbtneaX0JDaxI3K5mYO931C2GjZ999hmNGzfGx8eH7t27s2/fPpvHf/fdd7Rq1QofHx/atWvH5s0V+2AKbg9kMhnP99KtmezWJIT3jErwlIelsjpr/28qGUazj4XqEl5Q/JcFqkV4y0rYpOnGc+qpdgsXwGsPRPP92B6cmTXgthYuR3C5eK1fv57Y2FhmzJjBoUOH6NChA/379yc11XK1yV27djFkyBBGjx7N4cOHeeyxx3jsscc4ceJENVsu8CTeHNCKr0Z1Y9XIrgzr0ZipD7bilZgWdp1rqazOT7OHsSguHk1JCRs/+jfTlN8CsLzkQcYXT6SIsnW2zNHff0DbMBRyGV0ah9jsMC4wxeXDxu7du9O1a1cWLVoEgFarJTIykgkTJjBlypQyxw8ePJi8vDx+/fVXw7a77rqLjh07smTJknLvJ4aNAnP2JWTw9Be77ThS4gXFrwah2qTphgKJAYr9aCUZH5QMZYXmIbvu+euEe2gbEciJq9m0DKtltSzP7YhHDBvVajUHDx4kJibGsE0ulxMTE8Pu3ZY/TLt37zY5HqB///5Wjy8qKiI7O9vkJRAY061JCIuH3mnHkbqyOhOMyuoMUOynSFIyvniCTeFSKeREhvjy95t9SJj9EHc0CEImk9GuYZAQrgri0gz79PR0NBoNoaGmJU5CQ0M5c+aMxXOSk5MtHp+cnGzx+NmzZzNz5kznGCyosTzYLpxLHw002SZJEsnZhTz22T+kZBcZtv9X25O04mC+UM5DQsbz6tfYL7UyObdXi7p8+Hg7dp5Pp0SjNakWK3AONX550NSpU4mNjTW8z87OJjIy0oUWCTwFmUxGeJAve6eZevqSJJFd8ACf/683YYF+LLurJcEWaskDDOkWVR2m3pa4VLzq1q2LQqEgJSXFZHtKSgphYWEWzwkLC3PoeG9vb7y9PTvvReBeyGQygvyUTH2sm6tNua1xacxLpVLRuXNn4uLiDNu0Wi1xcXH06NHD4jk9evQwOR5g69atVo8XCAQ1E5cPG2NjYxk+fDhdunShW7duzJ8/n7y8PEaOHAnAsGHDaNCgAbNnzwZg0qRJ9O7dm7lz5zJw4EDWrVvHgQMHWLp0qSsfQyAQVDMuF6/BgweTlpbG9OnTSU5OpmPHjmzZssUQlE9MTERu1EamZ8+erF27lrfffptp06bRokULfv75Z+644w5rtxAIBDUQl+d5VTciz0sgcG88Is9LIBAIKooQL4FA4JEI8RIIBB6JywP21Y0+xCeWCQkE7on+u1leOP62E6+cHF3PO5FlLxC4Nzk5OQQFWe8MddvNNmq1Wq5du0ZAQAAyWcU6uFQE/bKkpKSkGj3Lebs8J4hnrSokSSInJ4eIiAiTNClzbjvPSy6X07BhQ5fdPzAwsMZ/0OH2eU4Qz1oV2PK49IiAvUAg8EiEeAkEAo9EiFc14e3tzYwZM2p8hYvb5TlBPKurue0C9gKBoGYgPC+BQOCRCPESCAQeiRAvgUDgkQjxEggEHokQryoiIyODoUOHEhgYSHBwMKNHjyY3N9fm8RMmTCA6OhpfX1+ioqKYOHEiWVlZ1Wh1xXD0WQGWLl3KfffdR2BgIDKZjMzMzOox1kFul27ujjznyZMnefLJJ2ncuDEymYz58+dXn6FGCPGqIoYOHcrJkyfZunUrv/76K3/99RcvvPCC1eOvXbvGtWvXmDNnDidOnGDVqlVs2bKF0aNHV6PVFcPRZwXIz89nwIABTJs2rZqsdJzbpZu7o8+Zn59P06ZN+eijj6w2vqkWJIHTOXXqlARI+/fvN2z77bffJJlMJl29etXu62zYsEFSqVRScXFxVZjpFCr7rNu2bZMA6ebNm1VoZcXo1q2bNG7cOMN7jUYjRURESLNnz7Z4/NNPPy0NHDjQZFv37t2lF198sUrtrCyOPqcxjRo1kj799NMqtM46wvOqAnbv3k1wcDBdunQxbIuJiUEul7N37167r6Mvg+vl5b5LUJ31rO5GdXRzdwcq8pzughCvKiA5OZn69eubbPPy8iIkJMRqZ29z0tPTmTVrVrnDL1fjjGd1R2x1c7f2XI52c3cHKvKc7oIQLweYMmUKMpnM5uvMmTOVvk92djYDBw6kTZs2vPvuu5U3vAJU17MKBBXFfccjbshrr73GiBEjbB7TtGlTwsLCygQ7S0pKyMjIKDfAmZOTw4ABAwgICOCnn35CqVRW1uwKUR3P6s5URzd3d6Aiz+kuCPFygHr16lGvXr1yj+vRoweZmZkcPHiQzp07A/Dnn3+i1Wrp3r271fOys7Pp378/3t7ebNy4ER8fH6fZ7ihV/azujnE398ceewwo7eY+fvx4i+fou7m/8sorhm3u3s29Is/pNrhkmuA2YMCAAVKnTp2kvXv3Sjt37pRatGghDRkyxLD/ypUrUnR0tLR3715JkiQpKytL6t69u9SuXTvp/Pnz0vXr1w2vkpISVz2GXTj6rJIkSdevX5cOHz4sLVu2TAKkv/76Szp8+LB048YNVzyCRdatWyd5e3tLq1atkk6dOiW98MILUnBwsJScnCxJkiQ999xz0pQpUwzH//PPP5KXl5c0Z84c6fTp09KMGTMkpVIpHT9+3FWPYBeOPmdRUZF0+PBh6fDhw1J4eLj0+uuvS4cPH5bOnTtXrXYL8aoibty4IQ0ZMkSqVauWFBgYKI0cOVLKyckx7E9ISJAAadu2bZIklaYMWHolJCS45iHsxNFnlSRJmjFjhsVn/fLLL6v/AWywcOFCKSoqSlKpVFK3bt2kPXv2GPb17t1bGj58uMnxGzZskFq2bCmpVCqpbdu20qZNm6rZ4orhyHPq/3+av3r37l2tNouSOAKBwCMRs40CgcAjEeIlEAg8EiFeAoHAIxHiJRAIPBIhXgKBwCMR4iUQCDwSIV4CgcAjEeIlEAg8EiFeArdgxIgRFitXDBgwwNWmCdwUsTBb4DYMGDCAL7/80mSbtQ7NxcXFZSpuqNVqVCqVw/et6HkC1yI8L4Hb4O3tTVhYmMmrdu3aAMhkMhYvXswjjzyCv78/H3zwAe+++y4dO3Zk+fLlNGnSxFCFIzExkUcffZRatWoRGBjI008/bVLyxdp5As9CiJfAY3j33Xd5/PHHOX78OKNGjQLg/Pnz/PDDD/z4448cOXIErVbLo48+SkZGBjt27GDr1q1cvHiRwYMHm1zL/DyB5yGGjQK34ddff6VWrVom26ZNm2boMPTss88ycuRIk/1qtZqvvvrKUHts69atHD9+nISEBCIjIwH46quvaNu2Lfv376dr164WzxN4HkK8BG5Dnz59WLx4scm2kJAQw9/GTT70NGrUyESATp8+TWRkpEG4ANq0aUNwcDCnT582iJf5eQLPQ4iXwG3w9/enefPmNvfbs83eewk8GxHzEtQoWrduTVJSEklJSYZtp06dIjMzkzZt2rjQMoGzEZ6XwG0oKioq027Ly8uLunXr2n2NmJgY2rVrx9ChQ5k/fz4lJSW8/PLL9O7d2+KwU+C5CM9L4DZs2bKF8PBwk9c999zj0DVkMhm//PILtWvX5t577yUmJoamTZuyfv36KrJa4CpEGWiBQOCRCM9LIBB4JEK8BAKBRyLESyAQeCRCvAQCgUcixEsgEHgkQrwEAoFHIsRLIBB4JEK8BAKBRyLESyAQeCRCvAQCgUcixEsgEHgkQrwEAoFH8v+BgjL1SdrB8gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020824, + "end_time": "2024-03-22T14:51:27.215878", + "exception": false, + "start_time": "2024-03-22T14:51:27.195054", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4820.639414, + "end_time": "2024-03-22T14:51:29.960241", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/lct_gan/0/mlu-eval.ipynb", + "output_path": "eval/insurance/lct_gan/0/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/lct_gan/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T13:31:09.320827", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file