diff --git "a/models/spabert/notebooks/GAN-SpaBERT_pytorch.ipynb" "b/models/spabert/notebooks/GAN-SpaBERT_pytorch.ipynb" deleted file mode 100644--- "a/models/spabert/notebooks/GAN-SpaBERT_pytorch.ipynb" +++ /dev/null @@ -1 +0,0 @@ -{"cells":[{"cell_type":"markdown","metadata":{"id":"view-in-github"},"source":["\"Open"]},{"cell_type":"code","source":["!sudo update-alternatives --config python3"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dGLAJQpUJNW1","executionInfo":{"status":"ok","timestamp":1724035718040,"user_tz":420,"elapsed":247,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"5dc39b9c-4f73-4fa9-a0ca-1133a0c6cfd4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["update-alternatives: error: no alternatives for python3\n"]}]},{"cell_type":"code","source":["# install pip\n","!curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py\n","!python3 get-pip.py --force-reinstall\n","\n","#install colab's dependencies\n","!python3 -m pip install ipython ipython_genutils ipykernel jupyter_console prompt_toolkit httplib2 astor\n","\n","# link to the old google package\n","!ln -s /usr/local/lib/python3.9/dist-packages/google \\\n"," /usr/local/lib/python3.8/dist-packages/google\n","\n","# There has got to be a better way to do this...but there's a bad import in some of the colab files\n","# IPython no longer exposes traitlets like this, it's a separate package now\n","!sed -i \"s/from IPython.utils import traitlets as _traitlets/import traitlets as _traitlets/\" /usr/local/lib/python3.8/dist-packages/google/colab/*.py\n","!sed -i \"s/from IPython.utils import traitlets/import traitlets/\" /usr/local/lib/python3.8/dist-packages/google/colab/*.py"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2K_XqKbyJPCE","executionInfo":{"status":"ok","timestamp":1721346937085,"user_tz":420,"elapsed":10689,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"65c6e909-4638-4b55-8eaa-b8df8cec2871"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" % Total % Received % Xferd Average Speed Time Time Time Current\n"," Dload Upload Total Spent Left Speed\n","100 2224k 100 2224k 0 0 17.4M 0 --:--:-- --:--:-- --:--:-- 17.5M\n","Collecting pip\n"," Using cached pip-24.1.2-py3-none-any.whl.metadata (3.6 kB)\n","Using cached pip-24.1.2-py3-none-any.whl (1.8 MB)\n","Installing collected packages: pip\n"," Attempting uninstall: pip\n"," Found existing installation: pip 23.1.2\n"," Uninstalling pip-23.1.2:\n"," Successfully uninstalled pip-23.1.2\n","Successfully installed pip-24.1.2\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n","\u001b[0mRequirement already satisfied: ipython in /usr/local/lib/python3.10/dist-packages (7.34.0)\n","Requirement already satisfied: ipython_genutils in /usr/local/lib/python3.10/dist-packages (0.2.0)\n","Requirement already satisfied: ipykernel in /usr/local/lib/python3.10/dist-packages (5.5.6)\n","Requirement already satisfied: jupyter_console in /usr/local/lib/python3.10/dist-packages (6.1.0)\n","Requirement already satisfied: prompt_toolkit in /usr/local/lib/python3.10/dist-packages (3.0.47)\n","Requirement already satisfied: httplib2 in /usr/local/lib/python3.10/dist-packages (0.22.0)\n","Collecting astor\n"," Downloading astor-0.8.1-py2.py3-none-any.whl.metadata (4.2 kB)\n","Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.10/dist-packages (from ipython) (67.7.2)\n","Collecting jedi>=0.16 (from ipython)\n"," Downloading jedi-0.19.1-py2.py3-none-any.whl.metadata (22 kB)\n","Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython) (4.4.2)\n","Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython) (0.7.5)\n","Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipython) (5.7.1)\n","Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from ipython) (2.16.1)\n","Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython) (0.2.0)\n","Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython) (0.1.7)\n","Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython) (4.9.0)\n","Requirement already satisfied: jupyter-client in /usr/local/lib/python3.10/dist-packages (from ipykernel) (6.1.12)\n","Requirement already satisfied: tornado>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipykernel) (6.3.3)\n","Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt_toolkit) (0.2.13)\n","Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2) (3.1.2)\n","Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython) (0.8.4)\n","Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython) (0.7.0)\n","Requirement already satisfied: jupyter-core>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from jupyter-client->ipykernel) (5.7.2)\n","Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.10/dist-packages (from jupyter-client->ipykernel) (24.0.1)\n","Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.10/dist-packages (from jupyter-client->ipykernel) (2.8.2)\n","Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core>=4.6.0->jupyter-client->ipykernel) (4.2.2)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.1->jupyter-client->ipykernel) (1.16.0)\n","Downloading astor-0.8.1-py2.py3-none-any.whl (27 kB)\n","Downloading jedi-0.19.1-py2.py3-none-any.whl (1.6 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m70.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: jedi, astor\n","Successfully installed astor-0.8.1 jedi-0.19.1\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n","\u001b[0mln: failed to create symbolic link '/usr/local/lib/python3.8/dist-packages/google': No such file or directory\n","sed: can't read /usr/local/lib/python3.8/dist-packages/google/colab/*.py: No such file or directory\n","sed: can't read /usr/local/lib/python3.8/dist-packages/google/colab/*.py: No such file or directory\n"]}]},{"cell_type":"code","source":["#check python version\n","import sys\n","print(sys.version)\n","!python3 --version\n","!python --version"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GhL6RHY0JRCg","executionInfo":{"status":"ok","timestamp":1721408603412,"user_tz":420,"elapsed":445,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"cb270ecd-abef-47d6-feb6-662610b2022c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["3.10.12 (main, Mar 22 2024, 16:50:05) [GCC 11.4.0]\n","Python 3.10.12\n","Python 3.10.12\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NV6AqCjjJVMC","executionInfo":{"status":"ok","timestamp":1721421519160,"user_tz":420,"elapsed":17174,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"38047a2c-7d5b-4a1b-a8cb-466eef2c9869"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"markdown","metadata":{"id":"fUpqAwtN8rTA"},"source":["# GAN-BERT (in Pytorch and compatible with HuggingFace)\n","\n","This is a Pytorch (+ **Huggingface** transformers) implementation of the GAN-BERT model from https://github.com/crux82/ganbert. While the original GAN-BERT was an extension of BERT, this implementation can be adapted to several architectures, ranging from Roberta to Albert!\n","\n","**NOTE**: given that this implementation is different from the original one in Tensorflow, some results can be slighty different.\n"]},{"cell_type":"markdown","metadata":{"id":"Q0m5KR34gmRH"},"source":["Let's GO!\n","\n","Required Imports."]},{"cell_type":"code","source":["!pip list | grep packaging"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"I26sjPpPDa47","executionInfo":{"status":"ok","timestamp":1721408610190,"user_tz":420,"elapsed":2169,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"aee49b21-c551-4f98-be2b-d5952193e748"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["packaging 24.1\n"]}]},{"cell_type":"code","source":["pip install packaging==21.3"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":287},"id":"JvCgPn-ADfOe","executionInfo":{"status":"ok","timestamp":1721408618079,"user_tz":420,"elapsed":5330,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"c365806f-cdf0-4574-87e9-8175a8f6babd"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting packaging==21.3\n"," Downloading packaging-21.3-py3-none-any.whl (40 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/40.8 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from packaging==21.3) (3.1.2)\n","Installing collected packages: packaging\n"," Attempting uninstall: packaging\n"," Found existing installation: packaging 24.1\n"," Uninstalling packaging-24.1:\n"," Successfully uninstalled packaging-24.1\n","Successfully installed packaging-21.3\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["packaging"]},"id":"96e342c5396c4c3c9ce457aa4b22a4bf"}},"metadata":{}}]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UIqpm34x2rms","outputId":"57e03d2f-0543-457c-f6a9-00bcd5da3a33","executionInfo":{"status":"ok","timestamp":1721408671643,"user_tz":420,"elapsed":41237,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting transformers==4.3.2\n"," Downloading transformers-4.3.2-py3-none-any.whl (1.8 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.3.2) (3.15.4)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.3.2) (1.25.2)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from transformers==4.3.2) (21.3)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.3.2) (2024.5.15)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.3.2) (2.31.0)\n","Collecting sacremoses (from transformers==4.3.2)\n"," Downloading sacremoses-0.1.1-py3-none-any.whl (897 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m897.5/897.5 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting tokenizers<0.11,>=0.10.1 (from transformers==4.3.2)\n"," Downloading tokenizers-0.10.3.tar.gz (212 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.7/212.7 kB\u001b[0m \u001b[31m16.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n"," Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n"," Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.3.2) (4.66.4)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from packaging->transformers==4.3.2) (3.1.2)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.3.2) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.3.2) (3.7)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.3.2) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.3.2) (2024.7.4)\n","Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from sacremoses->transformers==4.3.2) (8.1.7)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from sacremoses->transformers==4.3.2) (1.4.2)\n","Building wheels for collected packages: tokenizers\n"," \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n"," \n"," \u001b[31m×\u001b[0m \u001b[32mBuilding wheel for tokenizers \u001b[0m\u001b[1;32m(\u001b[0m\u001b[32mpyproject.toml\u001b[0m\u001b[1;32m)\u001b[0m did not run successfully.\n"," \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n"," \u001b[31m╰─>\u001b[0m See above for output.\n"," \n"," \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n"," Building wheel for tokenizers (pyproject.toml) ... \u001b[?25l\u001b[?25herror\n","\u001b[31m ERROR: Failed building wheel for tokenizers\u001b[0m\u001b[31m\n","\u001b[0mFailed to build tokenizers\n","\u001b[31mERROR: Could not build wheels for tokenizers, which is required to install pyproject.toml-based projects\u001b[0m\u001b[31m\n","\u001b[0mLooking in links: https://download.pytorch.org/whl/torch_stable.html\n","\u001b[31mERROR: Could not find a version that satisfies the requirement torch==1.7.1+cu101 (from versions: 1.11.0, 1.11.0+cpu, 1.11.0+cu102, 1.11.0+cu113, 1.11.0+cu115, 1.11.0+rocm4.3.1, 1.11.0+rocm4.5.2, 1.12.0, 1.12.0+cpu, 1.12.0+cu102, 1.12.0+cu113, 1.12.0+cu116, 1.12.0+rocm5.0, 1.12.0+rocm5.1.1, 1.12.1, 1.12.1+cpu, 1.12.1+cu102, 1.12.1+cu113, 1.12.1+cu116, 1.12.1+rocm5.0, 1.12.1+rocm5.1.1, 1.13.0, 1.13.0+cpu, 1.13.0+cu116, 1.13.0+cu117, 1.13.0+cu117.with.pypi.cudnn, 1.13.0+rocm5.1.1, 1.13.0+rocm5.2, 1.13.1, 1.13.1+cpu, 1.13.1+cu116, 1.13.1+cu117, 1.13.1+cu117.with.pypi.cudnn, 1.13.1+rocm5.1.1, 1.13.1+rocm5.2, 2.0.0, 2.0.0+cpu, 2.0.0+cpu.cxx11.abi, 2.0.0+cu117, 2.0.0+cu117.with.pypi.cudnn, 2.0.0+cu118, 2.0.0+rocm5.3, 2.0.0+rocm5.4.2, 2.0.1, 2.0.1+cpu, 2.0.1+cpu.cxx11.abi, 2.0.1+cu117, 2.0.1+cu117.with.pypi.cudnn, 2.0.1+cu118, 2.0.1+rocm5.3, 2.0.1+rocm5.4.2, 2.1.0, 2.1.0+cpu, 2.1.0+cpu.cxx11.abi, 2.1.0+cu118, 2.1.0+cu121, 2.1.0+cu121.with.pypi.cudnn, 2.1.0+rocm5.5, 2.1.0+rocm5.6, 2.1.1, 2.1.1+cpu, 2.1.1+cpu.cxx11.abi, 2.1.1+cu118, 2.1.1+cu121, 2.1.1+cu121.with.pypi.cudnn, 2.1.1+rocm5.5, 2.1.1+rocm5.6, 2.1.2, 2.1.2+cpu, 2.1.2+cpu.cxx11.abi, 2.1.2+cu118, 2.1.2+cu121, 2.1.2+cu121.with.pypi.cudnn, 2.1.2+rocm5.5, 2.1.2+rocm5.6, 2.2.0, 2.2.0+cpu, 2.2.0+cpu.cxx11.abi, 2.2.0+cu118, 2.2.0+cu121, 2.2.0+rocm5.6, 2.2.0+rocm5.7, 2.2.1, 2.2.1+cpu, 2.2.1+cpu.cxx11.abi, 2.2.1+cu118, 2.2.1+cu121, 2.2.1+rocm5.6, 2.2.1+rocm5.7, 2.2.2, 2.2.2+cpu, 2.2.2+cpu.cxx11.abi, 2.2.2+cu118, 2.2.2+cu121, 2.2.2+rocm5.6, 2.2.2+rocm5.7, 2.3.0, 2.3.0+cpu, 2.3.0+cpu.cxx11.abi, 2.3.0+cu118, 2.3.0+cu121, 2.3.0+rocm5.7, 2.3.0+rocm6.0, 2.3.1, 2.3.1+cpu, 2.3.1+cpu.cxx11.abi, 2.3.1+cu118, 2.3.1+cu121, 2.3.1+rocm5.7, 2.3.1+rocm6.0)\u001b[0m\u001b[31m\n","\u001b[0m\u001b[31mERROR: No matching distribution found for torch==1.7.1+cu101\u001b[0m\u001b[31m\n","\u001b[0mRequirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.1.99)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.0.3)\n","Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.4)\n","Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.1)\n","Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas) (1.25.2)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/transformers/deepspeed.py:24: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations\n"," warnings.warn(\n","/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n","The secret `HF_TOKEN` does not exist in your Colab secrets.\n","To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n","You will be able to reuse this secret in all of your notebooks.\n","Please note that authentication is recommended but still optional to access public models or datasets.\n"," warnings.warn(\n"]}],"source":["!pip install transformers==4.3.2\n","!pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html\n","!pip install sentencepiece\n","!pip install pandas\n","\n","import torch\n","import io\n","import torch.nn.functional as F\n","import random\n","import numpy as np\n","import time\n","import math\n","import datetime\n","import torch.nn as nn\n","from transformers import *\n","from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n","import random\n","import pandas as pd\n","import pickle\n","import csv\n","\n","\n","##Set random values\n","seed_val = 42\n","random.seed(seed_val)\n","np.random.seed(seed_val)\n","torch.manual_seed(seed_val)\n","if torch.cuda.is_available():\n"," torch.cuda.manual_seed_all(seed_val)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"LeZgRup520II","executionInfo":{"status":"ok","timestamp":1721408695804,"user_tz":420,"elapsed":249,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"c571191e-2c8c-4ed3-a4c5-55deb6f889f7"},"outputs":[{"output_type":"stream","name":"stdout","text":["There are 1 GPU(s) available.\n","We will use the GPU: Tesla T4\n"]}],"source":["# If there's a GPU available...\n","if torch.cuda.is_available():\n"," # Tell PyTorch to use the GPU.\n"," device = torch.device(\"cuda\")\n"," print('There are %d GPU(s) available.' % torch.cuda.device_count())\n"," print('We will use the GPU:', torch.cuda.get_device_name(0))\n","# If not...\n","else:\n"," print('No GPU available, using the CPU instead.')\n"," device = torch.device(\"cpu\")"]},{"cell_type":"markdown","metadata":{"id":"AU3ns8Ic7I-h"},"source":["### Input Parameters\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jw0HC_hU3FUy"},"outputs":[],"source":["#--------------------------------\n","# Transformer parameters\n","#--------------------------------\n","max_seq_length = 64\n","batch_size = 64\n","\n","#--------------------------------\n","# GAN-BERT specific parameters\n","#--------------------------------\n","# number of hidden layers in the generator,\n","# each of the size of the output space\n","num_hidden_layers_g = 1;\n","# number of hidden layers in the discriminator,\n","# each of the size of the input space\n","num_hidden_layers_d = 1;\n","# size of the generator's input noisy vectors\n","noise_size = 100\n","# dropout to be applied to discriminator's input vectors\n","out_dropout_rate = 0.2\n","\n","# Replicate labeled data to balance poorly represented datasets,\n","# e.g., less than 1% of labeled material\n","apply_balance = True\n","\n","#--------------------------------\n","# Optimization parameters\n","#--------------------------------\n","learning_rate_discriminator = 5e-5\n","learning_rate_generator = 5e-5\n","epsilon = 1e-8\n","num_train_epochs = 10\n","multi_gpu = True\n","# Scheduler\n","apply_scheduler = False\n","warmup_proportion = 0.1\n","# Print\n","print_each_n_step = 10\n","\n","#--------------------------------\n","# Adopted Tranformer model\n","#--------------------------------\n","# Since this version is compatible with Huggingface transformers, you can uncomment\n","# (or add) transformer models compatible with GAN\n","\n","#model_name = \"bert-base-cased\"\n","model_name = \"bert-base-uncased\"\n","#model_name = \"roberta-base\"\n","#model_name = \"albert-base-v2\"\n","#model_name = \"xlm-roberta-base\"\n","#model_name = \"amazon/bort\"\n","\n","#--------------------------------\n","# Retrieve the TREC QC Dataset\n","#--------------------------------\n","#! git clone https://github.com/crux82/ganbert\n","\n","# NOTE: in this setting 50 classes are involved\n","labeled_file = \"/content/ganbert/data/labeled.tsv\"\n","unlabeled_file = \"/content/ganbert/data/unlabeled.tsv\"\n","test_filename = \"/content/ganbert/data/test.tsv\"\n","opspam = \"./data/opspam.txt\" #Don't have\n","chicago_unlab = \"./data/chicago_unlab.txt\" #Don't Have\n","opspam_test = \"./data/opspam_test.txt\" #Don't have\n","review_content = \"/content/drive/MyDrive/Master_Project_2024_JP/yelpZip/YelpZip/reviewContent\"\n","metadata = \"/content/drive/MyDrive/Master_Project_2024_JP/yelpZip/YelpZip/metadata\"\n","outlabsp = \"/content/drive/MyDrive/Master_Project_2024_JP/CSC502 Thomas Project/Yelp_dataset_Sirish/outlabsp.txt\"\n","outrevsp = \"/content/drive/MyDrive/Master_Project_2024_JP/CSC502 Thomas Project/Yelp_dataset_Sirish/outrevsp.txt\"\n","#review_content = \"./Yelp/YelpNYC/reviewContent\"\n","#metadata = \"./Yelp/YelpNYC/metadata\"\n","\n","\n","\n"]},{"cell_type":"markdown","metadata":{"id":"R6Q5jzVioTHb"},"source":["Load the Tranformer Model"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"gxghkkZq3Gbn","executionInfo":{"status":"ok","timestamp":1721408720743,"user_tz":420,"elapsed":3492,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["c869a537fc794e27918eaac6e9a960fe","6e38efa9cbc947fe834dcf54dca2229a","7904dce7c1b74a04b94f5b98a211981d","136118b664ab4424b2d1ff7c86c7d0f7","b0c3e480d7f54023a25f073400fc1519","8f89cfc39e7f4df0aa57a403d4509f74","002428f871114b97a48aa237dff7b6d9","0c19a6c54a5d4d0d84cc308830e970a2","60f87b21653a476c8ae0e260fc7df97d","ebafff9ee1c04b64ab166c3aac13a6f0","46b3280d400843039617466ae7285130","a007bd30def44eb789acb386f8dcc0f4","26e35ebcee904fcc8e59d8ef2cc4efd3","52114ba15a6b42aa97b1f0522ef4cc0b","a43c1a714f3d4f738a5415294c409215","c2781eb2f03d476fb58a330b0a0c3b1b","27566833a689489ba7fd4b68dd2d13f6","ab7f04c391d54ae4ba9ae31188b9099d","9da4fd83f6cd461b9692597a46d0ccc1","366f2fc4fb3c480aaba32d3216bd3d24","e25b2951c77f44e7aa722bfe608a80a4","a1d08faa838646da8b97b377cd6a7913","62df311454c44e489e4130900dfec014","2d156fa7e87a4e60b7081d321f1118ac","1077abf659c64aaa8cbd5f84257cf3d3","2464d9ad1361412b99e55d7a7a21bef5","22638fcba8d441de910886946c159366","0081cf36c860483ca6dea6dbbd3c57fa","4fce33104c5d49d78a0b627c1f48def1","fcdf50a400ef43b1b4fb0e88be7f4f23","48ce848d10f44d029b8f91d6a4a41a69","ec11c0dda2a341318977a5ba706dcd2e","1e816e9b63a24e80b277f7b873171532","80708c1bab0e441db238db1f6de14aa1","d0fb79a264b04957895de907c288f747","c515dfd4e23843debcd530c1e29ce63b","1a7864a6a429493888b053ea305263b9","5eba8eaef376428691f503e9a1b09921","a81ebe0b66d342a9a7ef4495cbc12430","9dbb1bb20cea4de49910683ebed17bbf","5d033b9c8bd445db8e9fb41e5d21d969","7023abd045f9488fba6ad0af5b7b374e","e04a3cc02da345e382d44406fdf67f7f","118a9e15bdb847b590a7db22d0f2ca00","bb9a56fe72f341a18309f3fa9e94e736","87d0a7104d7e41acad70fd849a5ee931","3529ec18d150499aa74cc565a56e0e8b","a42c4c42aee546d680ebcbbef9aae803","2ebf4f8809654720b33597d4e4a0cb53","b379c4b7bf0c4ff7b7ac7bdd44f5bb90","41fc4a9f66d74e8eb626bc005c43809c","e1e43783014c4ae1a627400dfb6a6fe4","51665cdd88aa4862ad4f360f8853fe0b","a5110700268243a081a925e4f0728aab","97bd3ba8bc354929a382a399d5653bcd"]},"outputId":"50a74de3-2e00-4403-c61e-61ec5b8dcbf3"},"outputs":[{"output_type":"display_data","data":{"text/plain":["config.json: 0%| | 0.00/570 [00:00 0) for token_id in sent]\n"," input_mask_array.append(att_mask)\n"," # Convertion to Tensor\n"," input_ids = torch.tensor(input_ids)\n"," input_mask_array = torch.tensor(input_mask_array)\n"," label_id_array = torch.tensor(label_id_array, dtype=torch.long)\n"," label_mask_array = torch.tensor(label_mask_array)\n","\n"," # Building the TensorDataset\n"," dataset = TensorDataset(input_ids, input_mask_array, label_id_array, label_mask_array)\n","\n"," if do_shuffle:\n"," sampler = RandomSampler\n"," else:\n"," sampler = SequentialSampler\n","\n"," # Building the DataLoader\n"," return DataLoader(\n"," dataset, # The training samples.\n"," sampler = sampler(dataset),\n"," batch_size = batch_size) # Trains with this batch size.\n","\n","def format_time(elapsed):\n"," '''\n"," Takes a time in seconds and returns a string hh:mm:ss\n"," '''\n"," # Round to the nearest second.\n"," elapsed_rounded = int(round((elapsed)))\n"," # Format as hh:mm:ss\n"," return str(datetime.timedelta(seconds=elapsed_rounded))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4c-nsMXlKX-D","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721358688756,"user_tz":420,"elapsed":1399,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"53efc151-c0bd-4b04-e463-153ad7632378"},"outputs":[{"output_type":"stream","name":"stderr","text":[":54: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index\n"," label_mask_array = torch.tensor(label_mask_array)\n"]}],"source":["label_map = {}\n","for (i, label) in enumerate(label_list):\n"," label_map[label] = i\n","#------------------------------\n","# Load the train dataset\n","#------------------------------\n","train_examples = labeled_examples\n","#The labeled (train) dataset is assigned with a mask set to True\n","train_label_masks = np.ones(len(labeled_examples), dtype=bool)\n","#If unlabel examples are available\n","if unlabeled_examples:\n"," train_examples = train_examples + unlabeled_examples\n"," #The unlabeled (train) dataset is assigned with a mask set to False\n"," tmp_masks = np.zeros(len(unlabeled_examples), dtype=bool)\n"," train_label_masks = np.concatenate([train_label_masks,tmp_masks])\n","\n","train_dataloader = generate_data_loader(train_examples, train_label_masks, label_map, do_shuffle = True, balance_label_examples = apply_balance)\n","\n","#------------------------------\n","# Load the test dataset\n","#------------------------------\n","#The labeled (test) dataset is assigned with a mask set to True\n","test_label_masks = np.ones(len(test_examples), dtype=bool)\n","\n","test_dataloader = generate_data_loader(test_examples, test_label_masks, label_map, do_shuffle = False, balance_label_examples = False)"]},{"cell_type":"markdown","metadata":{"id":"6Ihcw3vquaQm"},"source":["We define the Generator and Discriminator as discussed in https://www.aclweb.org/anthology/2020.acl-main.191/"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"18kY64-n3I6y"},"outputs":[],"source":["#------------------------------\n","# The Generator as in\n","# https://www.aclweb.org/anthology/2020.acl-main.191/\n","# https://github.com/crux82/ganbert\n","#------------------------------\n","class Generator(nn.Module):\n"," def __init__(self, noise_size=100, output_size=512, hidden_sizes=[512], dropout_rate=0.1):\n"," super(Generator, self).__init__()\n"," layers = []\n"," hidden_sizes = [noise_size] + hidden_sizes\n"," for i in range(len(hidden_sizes)-1):\n"," layers.extend([nn.Linear(hidden_sizes[i], hidden_sizes[i+1]), nn.LeakyReLU(0.2, inplace=True), nn.Dropout(dropout_rate)])\n","\n"," layers.append(nn.Linear(hidden_sizes[-1],output_size))\n"," self.layers = nn.Sequential(*layers)\n","\n"," def forward(self, noise):\n"," output_rep = self.layers(noise)\n"," return output_rep\n","\n","#------------------------------\n","# The Discriminator\n","# https://www.aclweb.org/anthology/2020.acl-main.191/\n","# https://github.com/crux82/ganbert\n","#------------------------------\n","class Discriminator(nn.Module):\n"," def __init__(self, input_size=512, hidden_sizes=[512], num_labels=2, dropout_rate=0.1):\n"," super(Discriminator, self).__init__()\n"," self.input_dropout = nn.Dropout(p=dropout_rate)\n"," layers = []\n"," hidden_sizes = [input_size] + hidden_sizes\n"," for i in range(len(hidden_sizes)-1):\n"," layers.extend([nn.Linear(hidden_sizes[i], hidden_sizes[i+1]), nn.LeakyReLU(0.2, inplace=True), nn.Dropout(dropout_rate)])\n","\n"," self.layers = nn.Sequential(*layers) #per il flatten\n"," self.logit = nn.Linear(hidden_sizes[-1],num_labels+1) # +1 for the probability of this sample being fake/real.\n"," self.softmax = nn.Softmax(dim=-1)\n","\n"," def forward(self, input_rep):\n"," input_rep = self.input_dropout(input_rep)\n"," last_rep = self.layers(input_rep)\n"," logits = self.logit(last_rep)\n"," probs = self.softmax(logits)\n"," return last_rep, logits, probs"]},{"cell_type":"markdown","metadata":{"id":"Uje9s2zQunFc"},"source":["We instantiate the Discriminator and Generator"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Ylz5rvqE3U2S","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721358705173,"user_tz":420,"elapsed":654,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"af260e3b-e521-4fae-f765-0e7896185296"},"outputs":[{"output_type":"stream","name":"stderr","text":["loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--bert-base-cased/snapshots/cd5ef92a9fb2f889e972770a36d4ed042daf221e/config.json\n","Model config BertConfig {\n"," \"_name_or_path\": \"bert-base-cased\",\n"," \"architectures\": [\n"," \"BertForMaskedLM\"\n"," ],\n"," \"attention_probs_dropout_prob\": 0.1,\n"," \"classifier_dropout\": null,\n"," \"gradient_checkpointing\": false,\n"," \"hidden_act\": \"gelu\",\n"," \"hidden_dropout_prob\": 0.1,\n"," \"hidden_size\": 768,\n"," \"initializer_range\": 0.02,\n"," \"intermediate_size\": 3072,\n"," \"layer_norm_eps\": 1e-12,\n"," \"max_position_embeddings\": 512,\n"," \"model_type\": \"bert\",\n"," \"num_attention_heads\": 12,\n"," \"num_hidden_layers\": 12,\n"," \"pad_token_id\": 0,\n"," \"position_embedding_type\": \"absolute\",\n"," \"transformers_version\": \"4.42.4\",\n"," \"type_vocab_size\": 2,\n"," \"use_cache\": true,\n"," \"vocab_size\": 28996\n","}\n","\n"]}],"source":["# The config file is required to get the dimension of the vector produced by\n","# the underlying transformer\n","config = AutoConfig.from_pretrained(model_name)\n","hidden_size = int(config.hidden_size)\n","# Define the number and width of hidden layers\n","hidden_levels_g = [hidden_size for i in range(0, num_hidden_layers_g)]\n","hidden_levels_d = [hidden_size for i in range(0, num_hidden_layers_d)]\n","\n","#-------------------------------------------------\n","# Instantiate the Generator and Discriminator\n","#-------------------------------------------------\n","generator = Generator(noise_size=noise_size, output_size=hidden_size, hidden_sizes=hidden_levels_g, dropout_rate=out_dropout_rate)\n","discriminator = Discriminator(input_size=hidden_size, hidden_sizes=hidden_levels_d, num_labels=len(label_list), dropout_rate=out_dropout_rate)\n","\n","# Put everything in the GPU if available\n","if torch.cuda.is_available():\n"," generator.cuda()\n"," discriminator.cuda()\n"," transformer.cuda()\n"," if multi_gpu:\n"," transformer = torch.nn.DataParallel(transformer)\n","\n","# print(config)"]},{"cell_type":"markdown","metadata":{"id":"VG3qzp2-usZE"},"source":["Let's go with the training procedure"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"NhqylHGK3Va4","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721359345781,"user_tz":420,"elapsed":216427,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"9ba1b4e5-1895-4a3e-a81e-8fb1b1fc1ac1"},"outputs":[{"output_type":"stream","name":"stderr","text":[":54: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index\n"," label_mask_array = torch.tensor(label_mask_array)\n","loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--bert-base-cased/snapshots/cd5ef92a9fb2f889e972770a36d4ed042daf221e/config.json\n","Model config BertConfig {\n"," \"_name_or_path\": \"bert-base-cased\",\n"," \"architectures\": [\n"," \"BertForMaskedLM\"\n"," ],\n"," \"attention_probs_dropout_prob\": 0.1,\n"," \"classifier_dropout\": null,\n"," \"gradient_checkpointing\": false,\n"," \"hidden_act\": \"gelu\",\n"," \"hidden_dropout_prob\": 0.1,\n"," \"hidden_size\": 768,\n"," \"initializer_range\": 0.02,\n"," \"intermediate_size\": 3072,\n"," \"layer_norm_eps\": 1e-12,\n"," \"max_position_embeddings\": 512,\n"," \"model_type\": \"bert\",\n"," \"num_attention_heads\": 12,\n"," \"num_hidden_layers\": 12,\n"," \"pad_token_id\": 0,\n"," \"position_embedding_type\": \"absolute\",\n"," \"transformers_version\": \"4.42.4\",\n"," \"type_vocab_size\": 2,\n"," \"use_cache\": true,\n"," \"vocab_size\": 28996\n","}\n","\n"]},{"output_type":"stream","name":"stdout","text":["\n","======== Epoch 1 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.371\n"," Average training loss discriminator: 1.887\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.681\n"," Test Loss: 1.133\n"," Test took: 0:00:00\n","\n","======== Epoch 2 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.548\n"," Average training loss discriminator: 1.065\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.550\n"," Test Loss: 1.849\n"," Test took: 0:00:00\n","\n","======== Epoch 3 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.776\n"," Average training loss discriminator: 0.724\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.512\n"," Test Loss: 2.449\n"," Test took: 0:00:01\n","\n","======== Epoch 4 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.837\n"," Average training loss discriminator: 0.656\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.562\n"," Test Loss: 2.314\n"," Test took: 0:00:01\n","\n","======== Epoch 5 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.747\n"," Average training loss discriminator: 0.731\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.675\n"," Test Loss: 1.710\n"," Test took: 0:00:01\n","\n","======== Epoch 6 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.724\n"," Average training loss discriminator: 0.804\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.625\n"," Test Loss: 1.833\n"," Test took: 0:00:01\n","\n","======== Epoch 7 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.745\n"," Average training loss discriminator: 0.760\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.569\n"," Test Loss: 2.122\n"," Test took: 0:00:01\n","\n","======== Epoch 8 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.730\n"," Average training loss discriminator: 0.724\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.588\n"," Test Loss: 2.160\n"," Test took: 0:00:01\n","\n","======== Epoch 9 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.712\n"," Average training loss discriminator: 0.741\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.625\n"," Test Loss: 1.866\n"," Test took: 0:00:01\n","\n","======== Epoch 10 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.729\n"," Average training loss discriminator: 0.770\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.675\n"," Test Loss: 2.143\n"," Test took: 0:00:01\n","\n","Training complete!\n","Total training took 0:01:47 (h:mm:ss)\n"]},{"output_type":"stream","name":"stderr","text":["loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--bert-base-cased/snapshots/cd5ef92a9fb2f889e972770a36d4ed042daf221e/config.json\n","Model config BertConfig {\n"," \"_name_or_path\": \"bert-base-cased\",\n"," \"architectures\": [\n"," \"BertForMaskedLM\"\n"," ],\n"," \"attention_probs_dropout_prob\": 0.1,\n"," \"classifier_dropout\": null,\n"," \"gradient_checkpointing\": false,\n"," \"hidden_act\": \"gelu\",\n"," \"hidden_dropout_prob\": 0.1,\n"," \"hidden_size\": 768,\n"," \"initializer_range\": 0.02,\n"," \"intermediate_size\": 3072,\n"," \"layer_norm_eps\": 1e-12,\n"," \"max_position_embeddings\": 512,\n"," \"model_type\": \"bert\",\n"," \"num_attention_heads\": 12,\n"," \"num_hidden_layers\": 12,\n"," \"pad_token_id\": 0,\n"," \"position_embedding_type\": \"absolute\",\n"," \"transformers_version\": \"4.42.4\",\n"," \"type_vocab_size\": 2,\n"," \"use_cache\": true,\n"," \"vocab_size\": 28996\n","}\n","\n"]},{"output_type":"stream","name":"stdout","text":["\n","======== Epoch 1 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.361\n"," Average training loss discriminator: 1.992\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.531\n"," Test Loss: 1.476\n"," Test took: 0:00:01\n","\n","======== Epoch 2 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.535\n"," Average training loss discriminator: 1.100\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.569\n"," Test Loss: 1.879\n"," Test took: 0:00:01\n","\n","======== Epoch 3 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.758\n"," Average training loss discriminator: 0.746\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.550\n"," Test Loss: 2.150\n"," Test took: 0:00:01\n","\n","======== Epoch 4 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.845\n"," Average training loss discriminator: 0.648\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.581\n"," Test Loss: 2.390\n"," Test took: 0:00:01\n","\n","======== Epoch 5 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.750\n"," Average training loss discriminator: 0.724\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.613\n"," Test Loss: 2.081\n"," Test took: 0:00:01\n","\n","======== Epoch 6 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.709\n"," Average training loss discriminator: 0.757\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.625\n"," Test Loss: 1.980\n"," Test took: 0:00:01\n","\n","======== Epoch 7 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.731\n"," Average training loss discriminator: 0.727\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.606\n"," Test Loss: 2.000\n"," Test took: 0:00:01\n","\n","======== Epoch 8 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.734\n"," Average training loss discriminator: 0.718\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.588\n"," Test Loss: 1.982\n"," Test took: 0:00:01\n","\n","======== Epoch 9 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.717\n"," Average training loss discriminator: 0.730\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.575\n"," Test Loss: 2.250\n"," Test took: 0:00:01\n","\n","======== Epoch 10 / 10 ========\n","Training...\n","\n"," Average training loss generetor: 0.716\n"," Average training loss discriminator: 0.726\n"," Training epcoh took: 0:00:10\n","\n","Running Test...\n"," Accuracy: 0.575\n"," Test Loss: 2.248\n"," Test took: 0:00:01\n","\n","Training complete!\n","Total training took 0:01:47 (h:mm:ss)\n"]}],"source":["for z in range(1, 3):\n"," label_map = {}\n"," for (i, label) in enumerate(label_list):\n"," label_map[label] = i\n"," #------------------------------\n"," # Load the train dataset\n"," #------------------------------\n"," train_examples = labeled_examples\n"," #The labeled (train) dataset is assigned with a mask set to True\n"," train_label_masks = np.ones(len(labeled_examples), dtype=bool)\n"," #If unlabel examples are available\n"," if unlabeled_examples:\n"," train_examples = train_examples + unlabeled_examples\n"," #The unlabeled (train) dataset is assigned with a mask set to False\n"," tmp_masks = np.zeros(len(unlabeled_examples), dtype=bool)\n"," train_label_masks = np.concatenate([train_label_masks,tmp_masks])\n","\n"," train_dataloader = generate_data_loader(train_examples, train_label_masks, label_map, do_shuffle = True, balance_label_examples = apply_balance)\n","\n"," #------------------------------\n"," # Load the test dataset\n"," #------------------------------\n"," #The labeled (test) dataset is assigned with a mask set to True\n"," test_label_masks = np.ones(len(test_examples), dtype=bool)\n","\n"," test_dataloader = generate_data_loader(test_examples, test_label_masks, label_map, do_shuffle = False, balance_label_examples = False)\n","\n"," ###########################################################\n","\n","\n"," # The config file is required to get the dimension of the vector produced by\n"," # the underlying transformer\n"," config = AutoConfig.from_pretrained(model_name)\n"," hidden_size = int(config.hidden_size)\n"," # Define the number and width of hidden layers\n"," hidden_levels_g = [hidden_size for i in range(0, num_hidden_layers_g)]\n"," hidden_levels_d = [hidden_size for i in range(0, num_hidden_layers_d)]\n","\n"," #-------------------------------------------------\n"," # Instantiate the Generator and Discriminator\n"," #-------------------------------------------------\n"," generator = Generator(noise_size=noise_size, output_size=hidden_size, hidden_sizes=hidden_levels_g, dropout_rate=out_dropout_rate)\n"," discriminator = Discriminator(input_size=hidden_size, hidden_sizes=hidden_levels_d, num_labels=len(label_list), dropout_rate=out_dropout_rate)\n","\n"," # Put everything in the GPU if available\n"," if torch.cuda.is_available():\n"," generator.cuda()\n"," discriminator.cuda()\n"," transformer.cuda()\n"," if multi_gpu:\n"," transformer = torch.nn.DataParallel(transformer)\n","\n"," # print(config)\n","\n","\n"," ###########################################################\n","\n","\n","\n"," training_stats = []\n","\n"," # Measure the total training time for the whole run.\n"," total_t0 = time.time()\n","\n"," #models parameters\n"," transformer_vars = [i for i in transformer.parameters()]\n"," d_vars = transformer_vars + [v for v in discriminator.parameters()]\n"," g_vars = [v for v in generator.parameters()]\n","\n"," #optimizer\n"," dis_optimizer = torch.optim.AdamW(d_vars, lr=learning_rate_discriminator)\n"," gen_optimizer = torch.optim.AdamW(g_vars, lr=learning_rate_generator)\n","\n"," #scheduler\n"," if apply_scheduler:\n"," num_train_examples = len(train_examples)\n"," num_train_steps = int(num_train_examples / batch_size * num_train_epochs)\n"," num_warmup_steps = int(num_train_steps * warmup_proportion)\n","\n"," scheduler_d = get_constant_schedule_with_warmup(dis_optimizer,\n"," num_warmup_steps = num_warmup_steps)\n"," scheduler_g = get_constant_schedule_with_warmup(gen_optimizer,\n"," num_warmup_steps = num_warmup_steps)\n","\n"," # For each epoch...\n"," for epoch_i in range(0, num_train_epochs):\n"," # ========================================\n"," # Training\n"," # ========================================\n"," # Perform one full pass over the training set.\n"," print(\"\")\n"," print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, num_train_epochs))\n"," print('Training...')\n","\n"," # Measure how long the training epoch takes.\n"," t0 = time.time()\n","\n"," # Reset the total loss for this epoch.\n"," tr_g_loss = 0\n"," tr_d_loss = 0\n","\n"," # Put the model into training mode.\n"," transformer.train()\n"," generator.train()\n"," discriminator.train()\n","\n"," # For each batch of training data...\n"," for step, batch in enumerate(train_dataloader):\n","\n"," # Progress update every print_each_n_step batches.\n"," if step % print_each_n_step == 0 and not step == 0:\n"," # Calculate elapsed time in minutes.\n"," elapsed = format_time(time.time() - t0)\n","\n"," # Report progress.\n"," print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n","\n"," # Unpack this training batch from our dataloader.\n"," b_input_ids = batch[0].to(device)\n"," b_input_mask = batch[1].to(device)\n"," b_labels = batch[2].to(device)\n"," b_label_mask = batch[3].to(device)\n","\n"," real_batch_size = b_input_ids.shape[0]\n","\n"," # Encode real data in the Transformer\n"," model_outputs = transformer(b_input_ids, attention_mask=b_input_mask)\n"," #model_outputs = transformer(input_ids = None, inputs_embeds = inputs_embeds, attention_mask=b_input_mask) #Verify this with Dr.Chen\n"," hidden_states = model_outputs[-1]\n","\n"," # Generate fake data that should have the same distribution of the ones\n"," # encoded by the transformer.\n"," # First noisy input are used in input to the Generator\n"," noise = torch.zeros(real_batch_size, noise_size, device=device).uniform_(0, 1)\n"," # Gnerate Fake data\n"," gen_rep = generator(noise)\n","\n"," # Generate the output of the Discriminator for real and fake data.\n"," # First, we put together the output of the tranformer and the generator\n"," disciminator_input = torch.cat([hidden_states, gen_rep], dim=0)\n"," # Then, we select the output of the disciminator\n"," features, logits, probs = discriminator(disciminator_input)\n","\n"," # Finally, we separate the discriminator's output for the real and fake\n"," # data\n"," features_list = torch.split(features, real_batch_size)\n"," D_real_features = features_list[0]\n"," D_fake_features = features_list[1]\n","\n"," logits_list = torch.split(logits, real_batch_size)\n"," D_real_logits = logits_list[0]\n"," D_fake_logits = logits_list[1]\n","\n"," probs_list = torch.split(probs, real_batch_size)\n"," D_real_probs = probs_list[0]\n"," D_fake_probs = probs_list[1]\n","\n"," #---------------------------------\n"," # LOSS evaluation\n"," #---------------------------------\n"," # Generator's LOSS estimation\n"," g_loss_d = -1 * torch.mean(torch.log(1 - D_fake_probs[:,-1] + epsilon))\n"," g_feat_reg = torch.mean(torch.pow(torch.mean(D_real_features, dim=0) - torch.mean(D_fake_features, dim=0), 2))\n"," g_loss = g_loss_d + g_feat_reg\n","\n"," # Disciminator's LOSS estimation\n"," logits = D_real_logits[:,0:-1]\n"," log_probs = F.log_softmax(logits, dim=-1)\n"," # The discriminator provides an output for labeled and unlabeled real data\n"," # so the loss evaluated for unlabeled data is ignored (masked)\n"," label2one_hot = torch.nn.functional.one_hot(b_labels, len(label_list))\n"," per_example_loss = -torch.sum(label2one_hot * log_probs, dim=-1)\n"," per_example_loss = torch.masked_select(per_example_loss, b_label_mask.to(device))\n"," labeled_example_count = per_example_loss.type(torch.float32).numel()\n","\n"," # It may be the case that a batch does not contain labeled examples,\n"," # so the \"supervised loss\" in this case is not evaluated\n"," if labeled_example_count == 0:\n"," D_L_Supervised = 0\n"," else:\n"," D_L_Supervised = torch.div(torch.sum(per_example_loss.to(device)), labeled_example_count)\n","\n"," D_L_unsupervised1U = -1 * torch.mean(torch.log(1 - D_real_probs[:, -1] + epsilon))\n"," D_L_unsupervised2U = -1 * torch.mean(torch.log(D_fake_probs[:, -1] + epsilon))\n"," d_loss = D_L_Supervised + D_L_unsupervised1U + D_L_unsupervised2U\n","\n"," #---------------------------------\n"," # OPTIMIZATION\n"," #---------------------------------\n"," # Avoid gradient accumulation\n"," gen_optimizer.zero_grad()\n"," dis_optimizer.zero_grad()\n","\n"," # Calculate weigth updates\n"," # retain_graph=True is required since the underlying graph will be deleted after backward\n"," g_loss.backward(retain_graph=True)\n"," d_loss.backward()\n","\n"," # Apply modifications\n"," gen_optimizer.step()\n"," dis_optimizer.step()\n","\n"," # A detail log of the individual losses\n"," #print(\"{0:.4f}\\t{1:.4f}\\t{2:.4f}\\t{3:.4f}\\t{4:.4f}\".\n"," # format(D_L_Supervised, D_L_unsupervised1U, D_L_unsupervised2U,\n"," # g_loss_d, g_feat_reg))\n","\n"," # Save the losses to print them later\n"," tr_g_loss += g_loss.item()\n"," tr_d_loss += d_loss.item()\n","\n"," # Update the learning rate with the scheduler\n"," if apply_scheduler:\n"," scheduler_d.step()\n"," scheduler_g.step()\n","\n"," # Calculate the average loss over all of the batches.\n"," avg_train_loss_g = tr_g_loss / len(train_dataloader)\n"," avg_train_loss_d = tr_d_loss / len(train_dataloader)\n","\n"," # Measure how long this epoch took.\n"," training_time = format_time(time.time() - t0)\n","\n"," print(\"\")\n"," print(\" Average training loss generetor: {0:.3f}\".format(avg_train_loss_g))\n"," print(\" Average training loss discriminator: {0:.3f}\".format(avg_train_loss_d))\n"," print(\" Training epcoh took: {:}\".format(training_time))\n","\n"," # ========================================\n"," # TEST ON THE EVALUATION DATASET\n"," # ========================================\n"," # After the completion of each training epoch, measure our performance on\n"," # our test set.\n"," print(\"\")\n"," print(\"Running Test...\")\n","\n"," t0 = time.time()\n","\n"," # Put the model in evaluation mode--the dropout layers behave differently\n"," # during evaluation.\n"," transformer.eval() #maybe redundant\n"," discriminator.eval()\n"," generator.eval()\n","\n"," # Tracking variables\n"," total_test_accuracy = 0\n","\n"," total_test_loss = 0\n"," nb_test_steps = 0\n","\n"," all_preds = []\n"," all_labels_ids = []\n","\n"," #loss\n"," nll_loss = torch.nn.CrossEntropyLoss(ignore_index=-1)\n","\n"," # Evaluate data for one epoch\n"," for batch in test_dataloader:\n","\n"," # Unpack this training batch from our dataloader.\n"," b_input_ids = batch[0].to(device)\n"," b_input_mask = batch[1].to(device)\n"," b_labels = batch[2].to(device)\n","\n"," # Tell pytorch not to bother with constructing the compute graph during\n"," # the forward pass, since this is only needed for backprop (training).\n"," with torch.no_grad():\n"," model_outputs = transformer(b_input_ids, attention_mask=b_input_mask)\n"," hidden_states = model_outputs[-1]\n"," _, logits, probs = discriminator(hidden_states)\n"," ###log_probs = F.log_softmax(probs[:,1:], dim=-1)\n"," filtered_logits = logits[:,0:-1]\n"," # Accumulate the test loss.\n"," total_test_loss += nll_loss(filtered_logits, b_labels)\n","\n"," # Accumulate the predictions and the input labels\n"," _, preds = torch.max(filtered_logits, 1)\n"," all_preds += preds.detach().cpu()\n"," all_labels_ids += b_labels.detach().cpu()\n","\n"," # Report the final accuracy for this validation run.\n"," all_preds = torch.stack(all_preds).numpy()\n"," all_labels_ids = torch.stack(all_labels_ids).numpy()\n"," test_accuracy = np.sum(all_preds == all_labels_ids) / len(all_preds)\n"," print(\" Accuracy: {0:.3f}\".format(test_accuracy))\n","\n"," # Calculate the average loss over all of the batches.\n"," avg_test_loss = total_test_loss / len(test_dataloader)\n"," avg_test_loss = avg_test_loss.item()\n","\n"," # Measure how long the validation run took.\n"," test_time = format_time(time.time() - t0)\n","\n"," print(\" Test Loss: {0:.3f}\".format(avg_test_loss))\n"," print(\" Test took: {:}\".format(test_time))\n","\n"," # Record all statistics from this epoch.\n"," training_stats.append(\n"," {\n"," 'epoch': epoch_i + 1,\n"," 'Training Loss generator': avg_train_loss_g,\n"," 'Training Loss discriminator': avg_train_loss_d,\n"," 'Valid. Loss': avg_test_loss,\n"," 'Valid. Accur.': test_accuracy,\n"," 'Training Time': training_time,\n"," 'Test Time': test_time,\n"," 'Preds': all_preds,\n"," 'Labels': all_labels_ids\n"," }\n"," )\n","\n"," last_pred = []\n"," last_label = []\n"," for stat in training_stats:\n"," last_pred = stat['Preds']\n"," last_label = stat['Labels']\n","\n","\n"," print(\"\\nTraining complete!\")\n","\n"," print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n","\n"," with open('/content/drive/MyDrive/Master_Project_2024_JP/CSC502 Thomas Project/models/GANBERT/runs/'+ str(z) +'/training_stats_lab_yelp_100L_ns_400R.pkl', 'wb') as f:\n"," pickle.dump(training_stats, f)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"dDm9NProRB4c","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1679283224281,"user_tz":420,"elapsed":173,"user":{"displayName":"Tom Hoang","userId":"06929039740252442952"}},"outputId":"ac51e265-4e9c-4436-97ef-38c72f33ca64"},"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Training complete!\n","Total training took 0:03:46 (h:mm:ss)\n"]}],"source":["last_pred = []\n","last_label = []\n","for stat in training_stats:\n"," last_pred = stat['Preds']\n"," last_label = stat['Labels']\n","\n","\n","print(\"\\nTraining complete!\")\n","\n","print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"]},{"cell_type":"code","source":["with open('/content/drive/MyDrive/Master_Project_2024_JP/CSC502 Thomas Project/models/GANBERT/runs/1/training_stats_lab_ns_100.pkl', 'wb') as f:\n"," pickle.dump(training_stats, f)"],"metadata":{"id":"LJ-ki1aykw58"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"id":"T_IaS--gcYyl","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721359427900,"user_tz":420,"elapsed":409,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"b337ee04-8db6-4bfc-fd14-1564cd1a6666"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n"," 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1,\n"," 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0,\n"," 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0,\n"," 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1,\n"," 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0,\n"," 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1,\n"," 1, 1, 1, 1, 1, 1])"]},"metadata":{},"execution_count":29}],"source":["last_pred"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Qq76zKxmck5T","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721359431548,"user_tz":420,"elapsed":1074,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"88fceb85-8b0f-45f6-d376-0358d419e630"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n"," 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n"," 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n"," 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n"," 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n"," 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n"," 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n"," 1, 1, 1, 1, 1, 1])"]},"metadata":{},"execution_count":30}],"source":["last_label"]},{"cell_type":"code","source":["from sklearn.metrics import classification_report"],"metadata":{"id":"EKnqunrU10RW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0d6xC-nG-ej3","executionInfo":{"status":"ok","timestamp":1679684179863,"user_tz":420,"elapsed":23006,"user":{"displayName":"Thong Hoang","userId":"01246065757324293441"}},"outputId":"c48badd7-1869-4e2b-b7e9-596844469493"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["import pickle"],"metadata":{"id":"csIUAyIg-ruH"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for z in range(1, 2):\n"," with open('/content/drive/MyDrive/CSC 502/GAN-BERT/runs/'+ str(z) +'/training_stats_lab_op_spam_100_s.pkl', 'rb') as f:\n"," print(z)\n"," training_stats = pickle.load(f)\n","\n"," last_pred = []\n"," last_label = []\n"," for stat in training_stats:\n"," last_pred = stat['Preds']\n"," last_label = stat['Labels']\n","\n"," target_names = ['real', 'fake']\n"," print(classification_report(last_label, last_pred, target_names=target_names))"],"metadata":{"id":"SUn_VlD815Yk"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"id":"tWlGOpascAaa","colab":{"base_uri":"https://localhost:8080/","height":507},"executionInfo":{"status":"ok","timestamp":1721359527836,"user_tz":420,"elapsed":1062,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"b62b7cf9-f3f0-44e7-b2c9-33fd3cbd8d64"},"outputs":[{"output_type":"display_data","data":{"text/plain":["
"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAcUAAAHqCAYAAAB1O1VnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8dUlEQVR4nO3deVxV1f7/8fdhRplRcUKcMofKOVMrh8yhwUwrb1nO1zJtsuxbWU71y8q+t7L8lpk5VE6ZerUsU9NrjjmkaWGlASoomSggIAis3x9uzpWYDxyOyOv5ePDwsM9ae33OBs+bvc/ea9uMMUYAAEBuri4AAIDLBaEIAICFUAQAwEIoAgBgIRQBALAQigAAWAhFAAAshCIAABZCEQAAC6EI/M2BAwd03333qVatWvLw8JDNZlOrVq1cVs+mTZtks9lks9lcVgPyFx0dbf/ZREdHu7oclAFCEU6RlZWlpUuXavDgwWrSpImCgoLk5eWlGjVq6MYbb9Tzzz+vgwcPurrMPKKiotS5c2d9/vnnOnnypAIDAxUWFqZq1aq5urQKKScwbDabmjVrVmT7Xbt25eozdOjQMq1n3759mjx5st5+++0yXS+uHB6uLgBXnh07dmjIkCH67bff7Ms8PT3l7++v06dPa+vWrdq6datee+019e/fX4sWLZKXl5cLK/6vWbNmKTk5WY0bN9amTZtUp04dV5ekKlWq6Oqrr3Z1GaV26NAhbd++XR07diywzccff+zUGvbt26cpU6YoIiJCTz75ZKnX5+npaf/ZeHp6lnp9cD32FFGmVq9era5du+q3335TaGiopk2bpt9++00ZGRk6ffq0MjIytGvXLj333HMKCAjQ8uXLlZqa6uqy7Q4cOCBJuuuuuy6LQJSk66+/XocOHdKhQ4dcXYrD6tevL0maO3dugW3Onz+vxYsXy2azKSIiopwqK506derYfzaXy+8LSodQRJn5/fff9eCDDyo9PV3NmzfXvn379Nxzz+mqq66yt3F3d1e7du00bdo0RUVF6a677nJhxXnlBLSfn5+LK7myDB48WDabTUuWLCnwj6Dly5fr7Nmz6tKliz1EgfJGKKLMvPjii0pKSpKPj49WrFihunXrFto+JCREK1euVGBgYJ7nTp48qfHjx6tFixaqWrWqqlatqhYtWujZZ59VfHx8vuv7+0kP8fHxeuKJJ9SgQQP5+PgoLCxM//jHP/Ld46pfv75sNps2bdokSZoyZUquz7Zylk+ePFk2m01du3Yt8HUVdWLMzp07NWjQIHtdVatWVUREhLp06aKXX35Zx48fL9H6XLG9SqpBgwbq0qWLkpKS9MUXX+TbJufQ6bBhwwpdV2pqqhYtWqTBgwerVatWql69ury9vVW7dm3169dPX3/9db79bDabfd0xMTG5fr42m02TJ0+2tx06dKj9M01jjD766CPdeOONCg0Nlc1m07x58yQVfKLN6dOnVbduXdlsNvXr1y/fejIzM9W5c2fZbDZdd911On/+fKGvG+XEAGXg5MmTxs3NzUgyI0aMKNW6Nm3aZIKCgowkI8lUrVrVVK1a1f59cHCw+f777/P0i4qKsrf58ssvTY0aNYwkU6VKFePt7W1/LiAgwOzbty9X33bt2pmwsDDj6elpHzMsLMz+tXXrVmOMMZMmTTKSTJcuXQqsf+PGjfax/m7evHnGZrPZn/f29jYBAQH27yWZuXPnFnt9rtpexXXpa5o/f76RZLp165anXXR0tLHZbMbf39+kpKSYLl26GElmyJAhedrOnTvXvl6bzWYCAwNNlSpVcm3Dp59+Ok+/sLAw+7Z2c3PL9fMNCwsz06dPt7cdMmSIkWQGDx5sBgwYYO8THBxs3Nzc7D+jS7dhVFRUrvE2bdpk/z/x3nvv5alnwoQJRpLx9fU1P//8c8k2LJyGUESZWLRoUa43WEcdPXrU/gbfvHlzs2XLFvtzmzdvNldffbWRZEJCQszx48dz9b30DSo4ONh07tzZ7Nq1yxhjzIULF8y6detMrVq1jCRz00035Tt+zpvxpEmT8n2+NKGYkpJi/P39jSTz4IMPmsOHD9ufO3funNm9e7cZP368+eqrr4q1vsthexXl0lDMef02m8388ccfudpNnjzZSDIjR440xphCQ3HlypXmmWeeMVu2bDEpKSn25XFxcWbKlCn2P2z+/e9/5+mbE6gRERGF1p0Tin5+fsbDw8O8+eabJjEx0RhjTHJysomLizPGFB6Kxhjz0ksvGUnGx8fH/PTTT/blGzdutAfmBx98UGgtKF+EIsrEiy++aH9ziI2NdXg9jzzyiP1N+sSJE3meP3bsmP2v/TFjxuR67tI3qKZNm5rU1NQ8/VetWmVvc+zYsTzPOzMUd+7cad+Tu3DhQoH9i7s+Y1y/vYry973fkSNHGklm4sSJ9jbZ2dmmfv36RpJ9j7ywUCzK9OnTjSRzyy235HmupKEoycyYMaPAdkWFYmZmpuncubP9j5bU1FTz119/mTp16hhJpn///iV9eXAyPlNEmTh9+rT9cUhIiEPrMMZo6dKlkqRHHnlENWvWzNOmbt26euSRRyRJixcvLnBdTz/9tHx9ffMs79Onj/3yj5wzTctLUFCQJNnPxC2tiri9hg8fLkmaP3++jDGSpI0bNyo6OlpXX321OnXqVOoxbr/9dknS9u3blZWVVap1BQcH6+GHH3a4v7u7uxYuXKjg4GD98ssveuKJJzR8+HDFxsYqPDxcH330UanqQ9kjFHHZiIqKUkJCgiSpR48eBba79dZbJV0M4qioqHzbdOjQId/lHh4eql69uiTZxyovjRo1UtOmTXXhwgV16NBBr7/+uvbt2+fwG3dF3F4dO3ZU06ZNFRMTow0bNkgq/gk2l4qPj9ekSZPUsWNHhYaG2mcestlsat68uaSLJ+ScOXOmVPW2b9++1NfQ1qtXT7Nnz5YkzZ49W6tWrZK7u7s+/fRTBQcHl2rdKHuEIspEaGio/bGjb55//vmn/XFh13xdelbrpX0u5e/vX2B/D4+Lc1ZcuHChpCWWiru7uxYvXqwGDRooJiZGzz33nFq3bq2AgADdeuutev/990t0zWZF3V454Td37lwlJSVp+fLlcnd31+DBg4vVf/v27WratKmmTp2qHTt2KCEhQb6+vqpRo0ae2YdSUlJKVWuNGjVK1T/HgAEDNGDAAPv3zzzzjG6++eYyWTfKFqGIMtGiRQv74x9//NGFlVzeWrZsqUOHDumLL77QqFGjdM011ygtLU3r16/Xo48+qqZNm5b7Yd3y9tBDD8nd3V0rVqzQBx98oLS0NPXu3Vu1atUqsm9mZqbuv/9+nT17Vq1atdKaNWuUlJSk5ORkxcfH6+TJk9qxY4e9fc4hWke5u7uXqn+O6OhorV+/3v791q1bS31oF85BKKJMdOvWTW5uF3+dVqxY4dA6Lv2r/O/X6l3q0ufK6i/54srZayrsmrLExMRC1+Hl5aX+/ftr1qxZOnDggE6dOqUPPvhAISEhOnbsmIYMGVKsWirC9spPrVq11Lt3b6Wlpemll16SVPxDp9u3b1dMTIzc3d315Zdfqk+fPnn2ck+ePFnmNZdGTpAnJiaqSZMm8vb21pYtW/Tyyy+7ujTkg1BEmQgLC7MfHlq4cGGueU+LkvPXfIMGDewn6eR83pSfnL+4Q0ND1aBBA0dLdkjOZ0DHjh0rsM3OnTtLtM7Q0FA9/PDDev311yVd3NMuzok4FWF7FSTnhJuMjAxVq1ZNffv2LVa/nO1evXr1Ag8ZX7pH9nc5f7iVdg+yJCZNmqQdO3aoSpUqWrlypf3n/Morr2jLli3lVgeKh1BEmXnllVfk5+entLQ09e/fX7GxsYW2P3PmjAYMGGDfs7LZbBo4cKCkixNz5/cXf1xcnGbNmiVJuv/++8v4FRStZcuW9jryC78///zTflLF36Wnpxe67kvP/sx58y5MRdheBbnzzjs1fvx4Pf3003r77beLPZl2zuxH8fHx+c7Uc/z4cc2YMaPA/gEBAZKks2fPlrxoB2zcuFGvvfaaJOmtt95Ss2bN9MQTT+j2229XVlaWBg0aVOqTgVC2CEWUmSZNmuiTTz6Rl5eXfv75Z7Vq1Uqvv/66Dh8+bG+TlZWlH3/8URMnTlTDhg21fPnyXOt44YUXFBQUpISEBPXo0UPbtm2zP7d161b16NFDZ8+eVUhIiJ577rlye205OnXqZJ+sesiQIdq9e7eMMcrOztamTZvUtWtXZWdn59t38eLF6ty5s2bNmqU//vjDvjwrK0tr1661v56OHTsW+6zEy317FcTT01NvvPGG3nzzTQ0aNKjY/W688UZVrVpVxhjdd9999iMSOduwa9euhU6Hd80110iSkpKS7JezOMvp06f10EMPKTs7W/3799eoUaPsz82dO1e1atXS0aNH9c9//tOpdaCEXHeJJK5UW7ZsMY0bN8417ZaXl5cJCQmxz+Iha4qu+++/32RkZOTqv2nTJhMYGFjgtGVBQUFm8+bNecYt6kLqHBEREflOp2ZM0RfvG2PMN998Y581Rda0aD4+PkaSueqqq3LN7nOpS6cnkzXFW2hoaK5tUrt2bRMZGZmrX3GmeXPV9ipKzvpL2rewi/fff//9XNvRz8/Pvv2rVauWa8KB/F7XLbfcYn/e39/fREREmIiICPPWW2/Z2+RcvF/U5AGFbcO+ffsaSSY8PNwkJCTk6btu3Tr7lH8ffvhhMbYKygN7iihznTt31qFDh7Ro0SINGjRIjRs3lo+Pj5KTkxUSEqIbb7xREyZMUGRkpBYuXJjn0FmXLl0UGRmpp59+Ws2aNVN2draMMWrWrJmeeeYZRUZG6qabbnLRq5N69eql77//XnfccYeCg4OVlZWl8PBwPffcc9qzZ0++F9FLUt++fbVgwQINGzZMLVu2VGBgoBITE+Xv76/rr79eL7/8sn7++Wc1bdq0RPVc7turrD3yyCP66quv1LVrV/n5+SkzM1N16tTRY489pv379+vaa68ttP+yZcv01FNPqUmTJrpw4YJiYmIUExNTpodUZ86cqVWrVsnNza3A6xF79Oih8ePHS5KefPJJRUZGltn4cJzNmHL8xBkAgMsYe4oAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIpwupkzZ6p+/fry8fFRhw4d9MMPP7i6JKBcbN68WXfeeadq164tm82mlStXurokFIFQhFMtWbJE48aN06RJk7R37161bNlSvXr1KvAO8MCVJCUlRS1bttTMmTNdXQqKiWne4FQdOnRQ+/bt9d5770mSsrOzFR4erscee+yyumsD4Gw2m00rVqxQv379XF0KCsGeIpwmIyNDe/bsUY8ePezL3Nzc1KNHD23fvt2FlQFA/ghFOM1ff/2lrKwshYWF5VoeFhaW7w1xAcDVCEUAACyEIpymWrVqcnd3V3x8fK7l8fHxBd5zEABciVCE03h5ealt27basGGDfVl2drY2bNigjh07urAyAMifh6sLwJVt3LhxGjJkiNq1a6frr79eb7/9tlJSUjRs2DBXlwY43blz53T48GH791FRUdq3b59CQkJUr149F1aGgnBJBpzuvffe0/Tp03Xy5Em1atVKM2bMUIcOHVxdFuB0mzZtUrdu3fIsHzJkiObNm1f+BaFIhCIAABY+UwQAwEIoAgBgIRQBALAQigAAWAhFAAAshCIAABZCEQAAC6EIp0tPT9fkyZOVnp7u6lIAl+D/QMXBxftwuqSkJAUGBioxMVEBAQGuLgcod/wfqDjYUwQAwEIoAgBgqRR3ycjOzlZcXJz8/f1ls9lcXU6lk5SUlOtfoLLh/4BrGWOUnJys2rVry82t8H3BSvGZ4vHjxxUeHu7qMgAALnTs2DHVrVu30DaVYk/R399fknR07wYF+Pu5uBqg/F2IP+bqEgCXSU5JVcNeg+1ZUJhKEYo5h0wD/P0IRVRKF1KquroEwOWK8/EZJ9oAAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsHi4ugBUTLv3HdS/136n3fsO6nDUUZ06fUbn09NVLSRY7Vq20NB/3K1+fW7J0y/6aKwaXt+zyPX/8M0StWt1jTNKB5ziXGqaWt49Ssfj/5IkzZ4yToPvujVXm+jYeF19+9Ai17Xts3fUtkUTZ5SJIhCKcMhHC7/QhwuW2r/3q1pFbm5uijv5p1ad/FOr1m7UgNtv1cIPpsvT0zPfdYRVDy1w/Z6e/GqiYpn03nx7IBZHWGhwgc95evD77ypseTikY9uWatq4gW6+oZ2aNIqQX9WqkqRjsSf07pzP9Ob/zdUXX63Ta+9+pJfGjc53HScObC7PkgGn+THysN5fslrXX3u1fjjwa7H6HN2w0MlVwRF8pgiHDBnYT0+OGqw21zW3B6IkhdeppTcmPqMH77lTkjR/yb9dVSJQLrKzszXm5RmSpHcnjHVxNSgtQhFOkfN5YFz8ny6uBHCumYtWac8vv2vUvberVdPGri4HpUQowim279onSWoQXse1hQBOFBv/l6b83wKFhQZrypghri4HZYDPFFFmzqWk6I+Y45q1YKmW/PtrSdKY4Q8U2L7T7Q/o518P60JmpmpWr6bO17fWw4Pv040d2pZXyUCpPPX6+0pOSdOMF8Yq0L9q0R0ucfPgp/TLkRhdyMxSWGiwOrVqrn/ee5s6t+asa1eqUHuKM2fOVP369eXj46MOHTrohx9+cHVJld7xuJNyq9lCbjVbKKDR9WrVvb/en7dYPj7emvo/j+nRYfcX2HfHnv1yc7v4Kxh9LFafffGlbr5rsJ58aZqMMeX1EgCHfPmfHfr3d9vUpd11euD27iXuv/OnQ3KzXfz9j4mL16I1G9V92Hg9/cYH/P67UIUJxSVLlmjcuHGaNGmS9u7dq5YtW6pXr176808+s3Ild3d3hVUPVVj1UHl5Xbz0wsPDQ889NlJj8glEHx8vjR76D/1n5QIlHflBZ37boZSoPdr97ee6s2dXSdKM2Z9q2ozZ5fkygBJJSTuvp157X54eHnrn+UeL3c/H21MP33eHNnw8Xae3LdefW5bp7I6V2rHoXd3epYMk6b2F/9Ybc5Y4q3QUwWYqyJ8kHTp0UPv27fXee+9JunjGV3h4uB577DE999xzhfZNSkpSYGCgzv6+UwH+fuVRbqWUnZ2tw1FH9fp7czR30XLVq1NLX332gVqU4OSDgf8cp89Xr1XVKr469uN3CgoMcGLFlceFk0ddXcIV5dn/na13Plmup4feo1efHJHrOe9WfSTlf/F+UR4Y/6q+WPe9qvr66I+1nygogPerspB0LkXVb7xHiYmJCggo/D2lQuwpZmRkaM+ePerRo4d9mZubm3r06KHt27fnaZ+enq6kpKRcX3A+Nzc3NWlUX3PeellPPTxER2NPaPBjzyk7O7vY63jtxXGSpJTUNG34foezSgUctv/QEb23cKXCa1bXhIcHlem6/9+TwyVd3BPd+MO+Ml03iqdChOJff/2lrKwshYWF5VoeFhamkydP5mk/bdo0BQYG2r/Cw8PLq1RYHhtx8c3ixwOR+vFAZLH7NYioq+qhIZKkP2KOO6U2oDSenv6BsrKyNWXsEBljdC41LddXjvQLF3QuNU2paeeLve4GdWqqenCgJOmP43nf2+B8V+TZp88//7zGjRtn/z4pKYlgLGd1atWwPz4SfUxtW7ZwYTVA2Tkad/E8huEvvllou7GvvKuxr7yriFo19NvX88ujNJSBChGK1apVk7u7u+Lj43Mtj4+PV82aNfO09/b2lre3d3mVh3xEHY21P/arWqX4/WKO69TpBElSg3pc44jKJSr2pE6dSZQkNagTVkRrOEOFCEUvLy+1bdtWGzZsUL9+/SRdPKljw4YNGjuWaZXKW1ZWltzc3GSz2QpsM/3/PpZ08UzUju1a2pcbYwrtN2Ha25IkX18fdb+xQ9kUDJShovb6CjvRpqjf/4nvzpMk+fp4q+v1rUpVJxxTIT5TlKRx48Zp9uzZmj9/viIjIzV69GilpKRo2LBhri6t0jkWe1Lte92njxcu1/G4/37ukZ2drX0HI/Xgo89qzmdfSJLGjnhAwUGB9jbd7h6qaTNm62Dk78rKypJ08Y3ixwOR6j/scS1eefGi/2fHjFBIcFD5vSigHNw68lm9MWeJfj4cnev3f9+hw7r3qala+s1/JEnPDL1XIYH+riy10qoQe4qSNHDgQJ06dUoTJ07UyZMn1apVK33zzTd5Tr5B+dj70y8aOe4lSZKPj7f8qlRRckqK0tMz7G2GDuynN156Ole/mONxmvDq25rw6tvy9PRQgL+fUtPOK+2SkxHGjhikiU/nf2cNoCI7GvenXnp3nl56d548PTwU4FdFqefTlXY+3d7m0fv7asLDBc8EBeeqMKEoSWPHjuVw6WWgds3qWjzrf7Vhyw7t+vGATsSf0ukzifLx9lKjiHDd0K6Vhv2jnzpf3yZP3zcmPqN1m7dp148HdPLPv5RwNlFenp66unEDdWrfWqMeulcd2lznglcFON+0p0Zq/Y692n3wN8WfPqOExGR5eXqoSf266tSquUYM6KPrr23q6jIrtQpz8X5pcPE+Kjsu3kdldsVdvA8AQHkgFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFqeG4pkzZ5SYmOjMIQAAKDMOh2JcXJwWLFigb775Js9zP//8s9q1a6dq1aopJCREN910k3777bdSFQoAgLM5HIoff/yxhg0bpk2bNuVanpaWpttuu00//vijjDEyxmjr1q3q0aOHkpKSSlsvAABO43Aorl+/XpI0cODAXMvnz5+vY8eOKSQkRLNnz9ann36qunXrKjY2VjNnzixdtQAAOJHDoRgdHS1Jatq0aa7ly5cvl81m06uvvqoRI0bogQce0OzZs2WM0apVq0pVLAAAzuRwKP71118KCAiQr6+vfVl2dra2bdsmm82me+65x7781ltvlZubm3799dfSVQsAgBM5HIpZWVlKT0/PtezAgQNKTU1VixYtFBwc/N9B3NwUHByslJQUxysFAMDJHA7FWrVqKT09XVFRUfZla9eulSR16tQpT/tz584pJCTE0eEAAHA6h0OxY8eOkqQpU6YoOztbp06d0vvvvy+bzaZevXrlahsVFaX09HTVqlWrdNUCAOBEDofiE088IUn65JNPFBQUpPDwcMXExKhBgwa64447crVdt26dJKlNmzalKBUAAOdyOBSvv/56ffzxx/Lz89O5c+eUkZGhpk2bavny5fLw8MjVdsGCBZKkbt26la5aAACcyGaMMaVZQVpamg4ePKigoCA1atRIbm65czYjI0OLFy+WMUZ33XWXgoKCSjOcQ5KSkhQYGKizv+9UgL9fuY8PuNqFk0ddXQLgMknnUlT9xnuUmJiogICAQtt6FPpsMfj6+qp9+/YFPu/l5aXBgweXdhgAAJyOu2QAAGAhFAEAsBTr8GnDhg3LZDCbzaYjR46UyboAAChrxQrFnHlOS8tms5XJegAAcIZiheLcuXOdXQcAAC5XrFAcMmSIs+sAAMDlONEGAAALoQgAgIVQBADAUupQ3L9/v0aNGqXmzZsrICBA7u7uBX79fU5UAAAuJ6VKqffee0/jxo1TVlaWSjmFKgAALufwnuLOnTv1xBNPKCsrS48++qjWrFkjSQoJCdH69ev16aefaujQofLy8lK1atW0cOFCfffdd2VWOAAAZc3hu2QMGjRIixYt0pNPPql//etfkiQ3NzfVrFlTcXFx9nb79u1Tr169FBAQoL1798rf379sKi8B7pKByo67ZKAyK8ldMhzeU9y6datsNpv9ZsM5/p6xrVq10rvvvqsjR45o+vTpjg4HAIDTORyK8fHx8vb2VkRExH9X5uam8+fP52l79913y9PTU8uXL3d0OAAAnM7hE22qVKmSZy5Tf39/JSUlKT09Xd7e3vblnp6eqlKlimJiYhyvFAAAJ3N4T7FOnTpKSkpSZmamfVmjRo0kSbt27crVNi4uTomJiZyhCgC4rDkcis2aNVNWVpYOHDhgX9a1a1cZYzR16lT7YdSMjAw9/vjjkqRrr722lOUCAOA8Dodiz549ZYzR6tWr7cvGjBkjb29vbdiwQXXr1lXnzp1Vp04drVixQjabTWPHji2TogEAcAaHP1McMGCAjh8/rtq1a9uXNWjQQAsXLtSwYcOUkJCg7du3S7p4As748eM1aNCg0lcMAICTOHydYmESEhK0Zs0aHTt2TIGBgerZs6caN25c1sMUG9cporLjOkVUZiW5TtEpk5GGhITowQcfdMaqAQBwGu6SAQCAhVAEAMDi8OHT7t27l7iPzWbThg0bHB0SAACncjgUN23aVKx2ObPeGGPyzIADAMDlxOFQnDRpUqHPJyYmaufOndq+fbtCQ0M1evRoubu7OzocAABO57RQzPHdd9+pf//++uWXX7Rs2TJHhwMAwOmcfqJN9+7d9c4772jFihX66KOPnD0cAAAOK5ezTwcOHCh3d3dCEQBwWXPKxft/5+Pjo6pVqyoyMrI8hiuQLbShbEXMZgBciaJ+iXJ1CYDLnEvJLnbbctlTjI2N5dZRAIDLntNDMS0tTY8++qgkbh0FALi8OXz4dOrUqYU+f/78eR07dkxr167V6dOnZbPZNGbMGEeHAwDA6RwOxcmTJxfrYnxjjNzc3PTiiy/qgQcecHQ4AACczuFQvPnmmwsNRQ8PDwUHB6tly5a67777dNVVVzk6FAAA5cLp07wBAFBRcJcMAAAsDofi1KlT9a9//avY7WfMmFHkyTkAALiSzTh48aCbm5tq1qypuLi4YrVv0KCBjh49qqysLEeGK5WkpCQFBgYqMeGUArh4H5XQr5vXuboEwGXOpaSq3Z33KTExscgM4PApAACWcgvFhIQE+fj4lNdwAACUWLmE4ueff67k5GTVq1evPIYDAMAhxb4k45133tE777yTa9mpU6fUsGHDAvsYY3T27FklJSXJZrPp9ttvd7xSAACcrNihePbsWUVHR+dalpWVlWdZQW655RZNnDixJLUBAFCuih2K/fr1U/369SVd3AMcPny4AgMD9fbbbxfYx83NTQEBAbrmmmvUqFGj0tYKAIBTldslGa7EJRmo7LgkA5VZSS7JcHiat+zs4t+0EQCAioDrFAEAsDgcijt27FCbNm2KdY/EkSNHqk2bNtq9e7ejwwEA4HQOh+LChQu1f/9+3XTTTUW2veGGG7Rv3z4tXLjQ0eEAAHA6h0PxP//5jySpZ8+eRba9++67JUkbN250dDgAAJzO4VA8fvy4AgMDFRISUmTb0NBQBQYGKjY21tHhAABwOodDMS0trURnoBpjlJyc7OhwAAA4ncOhWKNGDSUnJxfrOsXY2FglJSWpWrVqjg4HAIDTORyKN9xwgyRp5syZRbbNadOhQwdHhwMAwOkcDsURI0bIGKM33nhDH374YYHtZs2apTfeeEM2m00jRoxwdDgAAJzO4Rltbr31Vt1zzz1atmyZRo8erZkzZ+qOO+5QRESEJCkmJkarV6/Wzz//LGOMBgwYoD59+pRZ4QAAlDWHQ1GS5s+fL5vNps8//1wHDhzQwYMHcz2fM63qP/7xD82ZM6c0QwEA4HSlmubN19dXS5Ys0fr16/XAAw8oIiJC3t7e8vHxUf369TVo0CB99913WrhwoXx9fcuqZgAAnKJUe4o5unfvru7duxf4fHZ2tr766ivNmTNHK1euLIshAQAoc2USigX5/fffNWfOHC1YsEDx8fHOHAoAgFIr81BMTU3V0qVLNWfOHG3btk3Sfz9bbNasWVkPBwBAmSmzUNyxY4fmzJmjpUuX6ty5c5IuhmHTpk1177336t5779U111xTVsMBAFDmShWKp06d0oIFC/Txxx/r0KFDkv67V2iz2bRr1y61bdu29FUCAFAOShyKxhitWbNGH3/8sb788ktlZmbKGCNfX1/169dPQ4YMUe/evSVxuBQAULEUOxSPHDmijz/+WPPnz9eJEydkjJHNZtONN96owYMH67777pO/v78zawUAwKmKHYpXXXWVbDabjDFq0KCBBg8erMGDB6tBgwbOrA8AgHJT4sOnjz/+uN544w15eXk5ox4AAFym2DPaeHt7yxijd999V7Vr19aYMWO0Y8cOZ9YGAEC5KnYonjhxQjNmzNB1112nhIQEvf/+++rcubOuvvpqvfrqqzp69Kgz6wQAwOmKHYpBQUEaO3asfvzxR+3Zs0ejR49WYGCgfv/9d7300ktq2LChunfvrrlz5zqzXgAAnMahCcFbt26tmTNn6sSJE/rkk0/UpUsXGWO0adMmjRw50t7u22+/VWZmZpkVCwCAM5XqLhne3t72O2EcPnxYEyZMUJ06dSTJfg/FGjVqaNiwYVqzZg0BCQC4rNlMzhQ0ZcQYo7Vr1+qjjz7S6tWrdeHCBdlsNkkXD8GePn26LIcrlqSkJAUGBiox4ZQCAgLKfXzA1X7dvM7VJQAucy4lVe3uvE+JiYlFZkCp9hTzY7PZ1Lt3by1btkyxsbF688031axZMxljdPbs2bIeDgCAMlPmoXipatWqady4cTp48KC2bdumESNGOHM4AABKxan3U7zUDTfcoBtuuKG8hgMAoMScuqcIAEBFQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFg9XF4CK6ejRo1q+YqU2fLdR+386oPj4eHl5ealhwwbq06uXnnh8rGrVqpWnX2xsrBYuWqwfdu3Wz7/8oj//PKXExEQFBgaqRfPmumfA3Rr1z5Hy9vZ2wasCimf5N+v1whtvF9qmio+P9q5ZlmvZc6+/pZVrNxRrjLt79dC0/3nSwQrhKEIRJXbs2DHVb9RExhj7soCAAKWkpOinnw7op58O6MOP5uiLpYvVrVvXXH2/37JVzz73gv17b29vValSRadPn9bm77/X5u+/1/uzPtS6b9aoTp065fOCAAd5engo0N8v3+d8fXzyLPOvWkXVgoMKXN+FrCwlJiVLkppf1ahMakTJEIoosaysLEnS7bf10dAhg3VL924KDg5WRkaGNmz4TmMef0JRUdHqN+Be/frLAdWsWdPet154uCa99KK63HyTWra8TiEhIZKkpKQkLVn6ucY986wiIw9p8NDh2rBurUteH1BcrVo01SdvvVbs9hPGPqwJYx8u8Pl5n6/Ua+9/JE9PD91xS5eyKBElRCiixIKDg/Xj7h/UsuV1uZZ7eXmpT5/eWrP632rdroOSkpI068OPNGnii/Y2nTp1VKdOHfOsMyAgQP8cOUJubm4aOeoRfbdxk44dO6bw8HCnvx7gcrHy24uHVrt2aK/gwAAXV1M5caINSiwwMDBPIF6qadOmuqFDB0nSnr17S7Tu9u3a2R/HxZ1wrECgAjp05A8dOhIl6eLniXANQhFOERp68bBozqHW4tq2fbv9cf36EWVaE3A5W2GdgBMSFKibOrR1cTWVV4U4fLp582ZNnz5de/bs0YkTJ7RixQr169fP1WWhAJmZmdq67WK4XdOiRZHtMzIyFBsbqxUr/62XJk2RJN17zwCFhYU5tU6gtA5HH9Udwx7VsRMn5e7urjph1dWpbWs91P9O1a1Vs+gVWDKzsvTlhv9Iku7o3kWeHhXirfmKVCG2fEpKilq2bKnhw4erf//+ri4HRZj5f+/r5MmTcnNz05DBDxXYrvHVzXTkyB+5ltlsNt17zwB9/NGHzi4TKLUziUk6m5SsAH8/paSk6vfoo/o9+qiWfPmNXn7mMd15S9dirWfzzt06feasJOnu3hw6daUKEYp9+vRRnz59XF0GiuGnnw7o+QkvSZLGjhmt5s2bFdi2erXqOncuRSkpKTp37pwk6b5779ErUyfLzy//09yBy0GN0BA9NnSQet7USRF1a8vL01MZGRe0fe8+TZ81V4djjur5195SzWrV1L7lNUWuL+cEm6sb1lezxg2dXT4KcUV+ppienq6kpKRcX3C+EydOqN+Ae5SWlqa2bdvo9WmvFtp++9bNOhl7VMlnT+tk7FH9v5en6suv1ujaVm31+bIvyqlqoORubN9GYwbfr6saRMjL01OS5OXlqS43tNeid6crok4tZWZl6X9nzytyXWeTkrVx+w+SpH69bnFm2SiGKzIUp02bpsDAQPsXp/U7X0JCgnr2uV1RUdG66qrG+mrVSvnkc/FyQcLCwvTC8/+jRZ8u0Pnz5zV0+EjFxsY6sWLAOfz9qmrUA/dJkvZH/qoziYmFtl+zcbMuXMiUh7u7+vboVh4lohBXZCg+//zzSkxMtH8dO3bM1SVd0RITE9Xrtjt08ODPqlevntav/drhk2TuvPMORUREKDU1VYuXLC3jSoHy0bLZ1ZIkY4yOn4gvtG3OtG83tm+j0EJmu0H5qBCfKZaUt7c3c2eWk5SUFN12x13avXuPatasqfVr16hevXqlWmed2rUVExOjI3/8UXRjoAI7EnNMPx36TRKHTi8XV+SeIspHWlqa7ryrv7Zt367Q0FCtX/u1rrrqqlKvNzomRpLkV5WTbVAx7Y/81f64Ts2Cj5rk7CUGBvire6cOTq8LRasQe4rnzp3T4cOH7d9HRUVp3759CgkJKfVeCRyTkZGh/vfcp42bNikoKEjffv2VWrRoXmS/zMxMeRRyDdZnCxcpLi5OknTTjZ3LrF6grBhjZLPZCnz+XEqqZi+6eHeM65o2UUhQYL7tsrOztWr9RknS7d1utp+wA9eqEHuKu3fvVuvWrdW6dWtJ0rhx49S6dWtNnDjRxZVVTllZWXrgwcH6Zu238vf319dfrlKbNq2L1ffmrrfotden65dfInPNdnP06FFNmfqKho8cJUlq27aNbr/9NqfUD5RGbPyfGjjmaS1b863i4v+0L8+4cEHf/7BH9z8+XtHHY+Xm5qZxI4cUuJ5te/Yp/q/TkqR+Pbs7vW4UT4XYU+zatWuu2xTBtbZu3aYvlq+QJF24cEH9BtxbYNvw8LratWOb/fu4Eyf0/IQX9fyEF+Xp6amAgACdP39eKSkp9jbt27XTqpVfyM2tQvzNhkpof+Sv9kOk3l5e8vXxUUpqqi5kZkqSfH28NfnJMbqhTcsC15FzbWKjiHBdZ52YA9erEKGIy0t2drb98fnz53X+/PkC2/79sox5c2brm7XfavP3W3Ts+HGdOnVKbm5uql8/Qm1at9a99wzQvfcMkLu7u9PqB0qjWnCQJox9WHsP/qJDR6KUkJiocykp8vXx0dWNGqhj65b6R9/bVKdmjQLXcS4lVeu37JAk9evJCTaXE5upBLtgSUlJCgwMVGLCKQUEcDsWVD6/bl7n6hIAlzmXkqp2d96nxMTEIjOA41MAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYCEUAQCwEIoAAFgIRQAALIQiAAAWQhEAAAuhCACAhVAEAMBCKAIAYPFwdQHlwRgjSUpKSnZxJYBrnEtJdXUJgMucS734+5+TBYWpFKGYnHwxDMPrN3RxJQAAV0lOTlZgYGChbWymONFZwWVnZysuLk7+/v6y2WyuLqfSSUpKUnh4uI4dO6aAgABXlwOUO/4PuJYxRsnJyapdu7bc3Ar/1LBS7Cm6ubmpbt26ri6j0gsICOANAZUa/wdcp6g9xBycaAMAgIVQBADAQijC6by9vTVp0iR5e3u7uhTAJfg/UHFUihNtAAAoDvYUAQCwEIoAAFgIRQAALIQicBnr2rWrbDabJk+enOe5+vXry2azad68eeVa07x582Sz2VS/fv1yHRcoD4QirmiTJ0+WzWbL8+Xj46O6deuqb9++Wrp0abHmRLzSRUdHa/LkyfkGMFBZVIoZbQBJCgsLsz9OTExUbGysYmNjtXr1as2bN08rVqyoUKfMN2rUSD4+PsWeqaMo0dHRmjJliiQVGoyBgYG6+uqrVadOnTIZF7icsKeISuPkyZP2r5SUFB08eFC33nqrJOnrr7/Wiy++6OIKS2bDhg06dOiQ7r777nId9+6779ahQ4e0YcOGch0XKA+EIiolNzc3tWjRQqtWrVLjxo0lSbNmzVJmZqaLKwPgSoQiKjUfHx/de++9ki7eVubQoUOKjo62f/YYHR2tI0eOaNSoUWrQoIG8vb3znGCSnZ2tzz77TLfddpvCwsLk5eWl6tWrq2fPnlq0aFGhn1dmZWXp3XffVZs2bVS1alWFhISoa9euWrZsWZG1F+dEm507d2rYsGFq3LixqlSpooCAADVv3lzDhw/X2rVrc62rW7du9u///hns0KFD7c8V50SbI0eOaPTo0brqqqvk6+urgIAAtWnTRlOnTlVSUlK+fTZt2mQfT5IOHz6s4cOHKzw8XN7e3qpbt67++c9/KjY2tsBxDx06pFGjRqlJkyaqUqWKfHx8FB4erhtuuEEvvPCCDh06VGBfQJJkgCvYpEmTjCRT2K/6zJkz7W22bt1qoqKi7N9/9tlnxs/Pz0gyVapUMVWrVjURERH2vqdPnzY333yzvb0kExgYmOv7vn37mvT09Dzjnj9/3vTq1cvezs3NzQQFBRmbzWYkmf/5n/8xXbp0MZLMpEmT8vSPiIgwkszcuXPzPJeZmWkef/zxXHVUrVrVBAcH29cfGBhob9+uXTsTHBxsbxsWFpbr6/HHH7e3nTt3rpGUaztcasmSJcbb29u+Ln9//1zfh4eHm19++SVPv40bN9rbfPfdd/bt7u/vbzw8POzP1a5d2xw/fjxP/2+//TbXOJ6eniYoKCjXNshvOwKXIhRxRStOKI4fP97eJjIyMlco+vn5mQ4dOphdu3bZ2//666/GmIvBkxNarVq1MqtXrzYpKSnGGGPOnTtn5s+fb2rUqGEkmSeffDLPuE899ZSRZGw2m3nllVdMYmKiMcaY+Ph4M3r06FwBW9JQfPbZZ+2vYfjw4faajTHm7NmzZuXKlWbgwIG5+lwaSoUpLBT37NljPD09jSTTuXNn89NPPxljjMnKyjKrVq0ytWrVMpJMo0aNTHJycoHjBwcHm759+5rIyEhjjDHp6elmyZIlxt/f30gyDz30UJ6xGzVqZCSZnj17mgMHDtiXp6WlmYMHD5opU6bku62ASxGKuKIVFYqJiYmmdu3aRpIJCQkxWVlZuUIxIiIiz5t3jgULFhhJpmnTpubs2bP5ttm9e7ex2WzGy8vLxMfH25fHxsba935eeumlfPvef//9he7hFBSKv/76q3FzczOSzLPPPpvvuvNTFqHYu3dvI8k0btzY/gfCpfbu3Wt/3dOnTy9w/G7dupmsrKw8/WfMmGEkGV9fX3PhwgX78vj4eHvfuLi4Yr5iIC8+U0SldPbsWW3YsEHdu3dXXFycJOmJJ57Ic1fusWPHys/PL991zJkzR5I0evToAi+LaNu2rVq0aKGMjAxt3LjRvnzZsmXKzMyUr6+vnnnmmXz7Onq94Pz585Wdna3Q0FD7JRbl4ezZs/bPKcePH68qVarkadO6dWv1799fkrRo0aIC1/XCCy/ke4f0u+66S5KUlpam33//3b7c39/f3v7EiROOvwhUeoQiKo1LTxwJDg5Wjx49tGfPHknSgw8+qAkTJuTp07lz53zXlZWVpR07dki6GF41a9Ys8OvXX3+VJMXExNj77969W5LUrl27Au/E3qRJE4euBdy2bZsk6dZbb5WPj0+J+ztq79699pOKevToUWC7nMtgfvrpJ124cCHfNh06dMh3ee3ate2PExIS7I99fX11yy23SJJ69+6tiRMnaufOncrIyCjZi0Clx8X7qDQuvXjf29tb1apVU+vWrTVo0KBcZ15eqkaNGvkuT0hIUHp6uiTpzJkzxRo/NTXV/vjPP/+UpCJDr27duoWebZmfkydPSpIiIiJK1K+0cl6TVPjrqlu3riQpMzNTCQkJuX4uOfz9/fPt6+Hx37esvwfqRx99pL59+2r//v16+eWX9fLLL8vLy0vt27fXXXfdpREjRigkJKRErwmVD6GISiMnLErC3d093+VZWVn2x19//bV69+7tcF1lLeeShsqmXr162rt3r9atW6c1a9Zo69at2r9/v7Zu3aqtW7dq2rRpWrZsmbp37+7qUnEZ4/Ap4IDQ0FD7Xsulh0WLK2cPtKi9wJLuJUpSzZo1Ha6rNC7dqz5+/HiB7XKe8/DwKPM9Nzc3N/Xq1UvvvPOOdu/erYSEBH322WeqV6+ezpw5owceeIBDqigUoQg4wNPTU9dff70kafXq1SXu365dO0kXP1s8d+5cvm1+//33QsOlIJ06dZIkrVu3TufPny92v0tPbDEOTJDepk0b+zoKmwJu/fr1kqSWLVvK09OzxOOUhL+/vx544AH7SVHx8fE6cOCAU8dExUYoAg4aNWqUJGnNmjVas2ZNoW0vPSlEkgYMGCB3d3elpaXpzTffzLfP1KlTHapr6NChcnd31+nTpzVp0qRi97v0hJ+zZ8+WeNygoCD16tVLkjR9+vRcn6Hm2L9/v7744gtJ0v3331/iMQpS1N6fr6+v/XF+Z7UCOfjtABz04IMPqkePHjLG6O6779Yrr7xiv7xDklJSUrRx40aNGTNGDRs2zNW3Tp06GjNmjCTp5Zdf1rRp05ScnCxJOnXqlMaOHatPP/3UoTtgNG7cWOPHj5ckvfHGGxo5cmSuyxeSkpK0ZMmSPBOJN2nSRF5eXpIunrTiyN7iK6+8Ik9PTx0+fFi9evWy75VlZ2drzZo1uu2225SZmalGjRrp4YcfLvH6C7Jt2zZdd911euuttxQZGans7GxJF/d4t23bptGjR0u6eJLPddddV2bj4grk0qskAScrzow2f3fpxftRUVGFtk1MTDR33HFHrqnEAgICck3XJsl4eHjk6ZuWlmZ69Ohhb+Pu7p5rGrbSTvM2ZsyYXHX5+fkVOM1bjhEjRtjbV6lSxdSrV89ERESYp59+2t6mqGneFi9ebLy8vHJtDx8fnxJN81aYnDYbN27Mt6+sKd5CQ0NzTQ8XEBBgNm/eXOi6AfYUgVIICAjQ6tWrtWbNGg0cOFD16tVTenq6UlNTVadOHfXs2VPTpk2zX6t4KR8fH3399dd655131KpVK3l5eckYo5tuuklLly7Va6+95nBd7u7ueu+997RlyxYNGjRI9erV04ULF2SMUfPmzTVixAj7YcxLzZw5U5MnT9a1114rSTp69KhiYmL0119/FXvsgQMH6ueff9bDDz+sRo0aKT09XR4eHmrVqpWmTJmigwcPqlmzZg6/tvy0b99eS5cu1ejRo9W2bVtVq1ZNSUlJ8vHxUatWrfTss88qMjJSN910U5mOiyuPzRhuOQ4AgMRnigAA2BGKAABYCEUAACyEIgAAFkIRAAALoQgAgIVQBADAQigCAGAhFAEAsBCKAABYCEUAACyEIgAAFkIRAAALoQgAgOX/A2v4OhhcyZ3SAAAAAElFTkSuQmCC\n"},"metadata":{}}],"source":["from sklearn.preprocessing import StandardScaler\n","from sklearn.svm import SVC\n","from sklearn.metrics import confusion_matrix\n","from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score\n","import matplotlib.pyplot as plt\n","#\n","# Standardize the data set\n","#\n","\n","#\n","# Fit the SVC model\n","#\n","\n","#\n","# Get the predictions\n","#\n","\n","#\n","# Calculate the confusion matrix\n","#\n","conf_matrix = confusion_matrix(y_true=last_label, y_pred=last_pred)\n","#\n","# Print the confusion matrix using Matplotlib\n","#\n","fig, ax = plt.subplots(figsize=(5, 5))\n","ax.matshow(conf_matrix, cmap=plt.cm.Oranges, alpha=0.3)\n","for i in range(conf_matrix.shape[0]):\n"," for j in range(conf_matrix.shape[1]):\n"," ax.text(x=j, y=i,s=conf_matrix[i, j], va='center', ha='center', size='xx-large')\n","\n","plt.xlabel('Predictions', fontsize=18)\n","plt.ylabel('Actuals', fontsize=18)\n","plt.title('Confusion Matrix', fontsize=18)\n","plt.show()"]},{"cell_type":"code","source":["from sklearn.metrics import classification_report\n","\n","target_names = ['non-spam', 'spam']\n","print(classification_report(last_label, last_pred, target_names=target_names))"],"metadata":{"id":"f1ze-Zf6EzAR","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1721359533938,"user_tz":420,"elapsed":404,"user":{"displayName":"Jason Phillips","userId":"10136472498761089328"}},"outputId":"148e3ee8-6d6c-4092-89d2-76f1123fe61a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" precision recall f1-score support\n","\n"," non-spam 0.60 0.44 0.51 80\n"," spam 0.56 0.71 0.63 80\n","\n"," accuracy 0.57 160\n"," macro avg 0.58 0.57 0.57 160\n","weighted avg 0.58 0.57 0.57 160\n","\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"id":"951PfCSbH08S"},"outputs":[],"source":["rN"]}],"metadata":{"accelerator":"GPU","colab":{"provenance":[]},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.7"},"gpuClass":"standard","widgets":{"application/vnd.jupyter.widget-state+json":{"c869a537fc794e27918eaac6e9a960fe":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_6e38efa9cbc947fe834dcf54dca2229a","IPY_MODEL_7904dce7c1b74a04b94f5b98a211981d","IPY_MODEL_136118b664ab4424b2d1ff7c86c7d0f7"],"layout":"IPY_MODEL_b0c3e480d7f54023a25f073400fc1519"}},"6e38efa9cbc947fe834dcf54dca2229a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8f89cfc39e7f4df0aa57a403d4509f74","placeholder":"​","style":"IPY_MODEL_002428f871114b97a48aa237dff7b6d9","value":"config.json: 100%"}},"7904dce7c1b74a04b94f5b98a211981d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_0c19a6c54a5d4d0d84cc308830e970a2","max":570,"min":0,"orientation":"horizontal","style":"IPY_MODEL_60f87b21653a476c8ae0e260fc7df97d","value":570}},"136118b664ab4424b2d1ff7c86c7d0f7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ebafff9ee1c04b64ab166c3aac13a6f0","placeholder":"​","style":"IPY_MODEL_46b3280d400843039617466ae7285130","value":" 570/570 [00:00<00:00, 36.3kB/s]"}},"b0c3e480d7f54023a25f073400fc1519":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8f89cfc39e7f4df0aa57a403d4509f74":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"002428f871114b97a48aa237dff7b6d9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0c19a6c54a5d4d0d84cc308830e970a2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"60f87b21653a476c8ae0e260fc7df97d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ebafff9ee1c04b64ab166c3aac13a6f0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"46b3280d400843039617466ae7285130":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a007bd30def44eb789acb386f8dcc0f4":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_26e35ebcee904fcc8e59d8ef2cc4efd3","IPY_MODEL_52114ba15a6b42aa97b1f0522ef4cc0b","IPY_MODEL_a43c1a714f3d4f738a5415294c409215"],"layout":"IPY_MODEL_c2781eb2f03d476fb58a330b0a0c3b1b"}},"26e35ebcee904fcc8e59d8ef2cc4efd3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_27566833a689489ba7fd4b68dd2d13f6","placeholder":"​","style":"IPY_MODEL_ab7f04c391d54ae4ba9ae31188b9099d","value":"model.safetensors: 100%"}},"52114ba15a6b42aa97b1f0522ef4cc0b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9da4fd83f6cd461b9692597a46d0ccc1","max":435755784,"min":0,"orientation":"horizontal","style":"IPY_MODEL_366f2fc4fb3c480aaba32d3216bd3d24","value":435755784}},"a43c1a714f3d4f738a5415294c409215":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e25b2951c77f44e7aa722bfe608a80a4","placeholder":"​","style":"IPY_MODEL_a1d08faa838646da8b97b377cd6a7913","value":" 436M/436M [00:01<00:00, 290MB/s]"}},"c2781eb2f03d476fb58a330b0a0c3b1b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"27566833a689489ba7fd4b68dd2d13f6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ab7f04c391d54ae4ba9ae31188b9099d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9da4fd83f6cd461b9692597a46d0ccc1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"366f2fc4fb3c480aaba32d3216bd3d24":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e25b2951c77f44e7aa722bfe608a80a4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a1d08faa838646da8b97b377cd6a7913":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"62df311454c44e489e4130900dfec014":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_2d156fa7e87a4e60b7081d321f1118ac","IPY_MODEL_1077abf659c64aaa8cbd5f84257cf3d3","IPY_MODEL_2464d9ad1361412b99e55d7a7a21bef5"],"layout":"IPY_MODEL_22638fcba8d441de910886946c159366"}},"2d156fa7e87a4e60b7081d321f1118ac":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0081cf36c860483ca6dea6dbbd3c57fa","placeholder":"​","style":"IPY_MODEL_4fce33104c5d49d78a0b627c1f48def1","value":"tokenizer_config.json: 100%"}},"1077abf659c64aaa8cbd5f84257cf3d3":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_fcdf50a400ef43b1b4fb0e88be7f4f23","max":49,"min":0,"orientation":"horizontal","style":"IPY_MODEL_48ce848d10f44d029b8f91d6a4a41a69","value":49}},"2464d9ad1361412b99e55d7a7a21bef5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ec11c0dda2a341318977a5ba706dcd2e","placeholder":"​","style":"IPY_MODEL_1e816e9b63a24e80b277f7b873171532","value":" 49.0/49.0 [00:00<00:00, 2.87kB/s]"}},"22638fcba8d441de910886946c159366":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0081cf36c860483ca6dea6dbbd3c57fa":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4fce33104c5d49d78a0b627c1f48def1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"fcdf50a400ef43b1b4fb0e88be7f4f23":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"48ce848d10f44d029b8f91d6a4a41a69":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ec11c0dda2a341318977a5ba706dcd2e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1e816e9b63a24e80b277f7b873171532":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"80708c1bab0e441db238db1f6de14aa1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d0fb79a264b04957895de907c288f747","IPY_MODEL_c515dfd4e23843debcd530c1e29ce63b","IPY_MODEL_1a7864a6a429493888b053ea305263b9"],"layout":"IPY_MODEL_5eba8eaef376428691f503e9a1b09921"}},"d0fb79a264b04957895de907c288f747":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a81ebe0b66d342a9a7ef4495cbc12430","placeholder":"​","style":"IPY_MODEL_9dbb1bb20cea4de49910683ebed17bbf","value":"vocab.txt: 100%"}},"c515dfd4e23843debcd530c1e29ce63b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_5d033b9c8bd445db8e9fb41e5d21d969","max":213450,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7023abd045f9488fba6ad0af5b7b374e","value":213450}},"1a7864a6a429493888b053ea305263b9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e04a3cc02da345e382d44406fdf67f7f","placeholder":"​","style":"IPY_MODEL_118a9e15bdb847b590a7db22d0f2ca00","value":" 213k/213k [00:00<00:00, 6.38MB/s]"}},"5eba8eaef376428691f503e9a1b09921":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a81ebe0b66d342a9a7ef4495cbc12430":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9dbb1bb20cea4de49910683ebed17bbf":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5d033b9c8bd445db8e9fb41e5d21d969":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7023abd045f9488fba6ad0af5b7b374e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e04a3cc02da345e382d44406fdf67f7f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"118a9e15bdb847b590a7db22d0f2ca00":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"bb9a56fe72f341a18309f3fa9e94e736":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_87d0a7104d7e41acad70fd849a5ee931","IPY_MODEL_3529ec18d150499aa74cc565a56e0e8b","IPY_MODEL_a42c4c42aee546d680ebcbbef9aae803"],"layout":"IPY_MODEL_2ebf4f8809654720b33597d4e4a0cb53"}},"87d0a7104d7e41acad70fd849a5ee931":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b379c4b7bf0c4ff7b7ac7bdd44f5bb90","placeholder":"​","style":"IPY_MODEL_41fc4a9f66d74e8eb626bc005c43809c","value":"tokenizer.json: 100%"}},"3529ec18d150499aa74cc565a56e0e8b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e1e43783014c4ae1a627400dfb6a6fe4","max":435797,"min":0,"orientation":"horizontal","style":"IPY_MODEL_51665cdd88aa4862ad4f360f8853fe0b","value":435797}},"a42c4c42aee546d680ebcbbef9aae803":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a5110700268243a081a925e4f0728aab","placeholder":"​","style":"IPY_MODEL_97bd3ba8bc354929a382a399d5653bcd","value":" 436k/436k [00:00<00:00, 13.1MB/s]"}},"2ebf4f8809654720b33597d4e4a0cb53":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b379c4b7bf0c4ff7b7ac7bdd44f5bb90":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"41fc4a9f66d74e8eb626bc005c43809c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e1e43783014c4ae1a627400dfb6a6fe4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"51665cdd88aa4862ad4f360f8853fe0b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a5110700268243a081a925e4f0728aab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"97bd3ba8bc354929a382a399d5653bcd":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0} \ No newline at end of file