{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "%pip install onnx"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yE8Z_9M87Mth",
        "outputId": "ca9c9cdd-54c2-4527-dad9-5947dc8b7345"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: onnx in /usr/local/lib/python3.10/dist-packages (1.15.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from onnx) (1.23.5)\n",
            "Requirement already satisfied: protobuf>=3.20.2 in /usr/local/lib/python3.10/dist-packages (from onnx) (3.20.3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V2O5yNWuifv3"
      },
      "outputs": [],
      "source": [
        "import ast\n",
        "import torch\n",
        "import pandas as pd\n",
        "import torch.nn as nn\n",
        "from tqdm import tqdm\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.metrics import accuracy_score\n",
        "from transformers import BertTokenizer, AdamW, BertForSequenceClassification"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G4PfTgErIXIj"
      },
      "outputs": [],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TSKX7sHE6Bnr"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('dataset.csv')\n",
        "df['classes'] = df['classes'].apply(ast.literal_eval)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QTwsZltwhPTt"
      },
      "outputs": [],
      "source": [
        "classes_count = [0 for i in range(630)]\n",
        "\n",
        "for classes in df['classes']:\n",
        "  for c in classes:\n",
        "    classes_count[c] +=1\n",
        "\n",
        "classes_min = min(classes_count)\n",
        "classes_max = max(classes_count)\n",
        "\n",
        "pos_weights = torch.tensor([0.3 + 0.7 * (1 - (c - classes_min)  / (classes_max - classes_min)) for c in classes_count]).to(device)  # Adjust weights for each class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZRao8_5mitBa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201,
          "referenced_widgets": [
            "e3ebca197a0a426babfb75caf8f81799",
            "548e251c68d044368ecb68898037bb01",
            "7be5e1bf58f34d359bc1726c88c2006f",
            "ab4a6233ff044573958ef204775509c5",
            "887871dfa5414422b63285477cda1930",
            "2af63a8b833a4582babaec7bfc219a36",
            "5b046c94863140dd858051e1b30cf02a",
            "7ae4e4874f704c7d95479a435d3f5a18",
            "78b5111866184eb096cdfb432a4d2c49",
            "2e91d31351fe4c06b0d7827e12686607",
            "44dfbae45a8848a1bc84e12ebd9e93b6",
            "d0afdae8789b42d4985b0c4fda3c2564",
            "7c960dc2e1264d6d813ea2b9c878ffa2",
            "bd09a7a58ff14190b3a683c0e2e4ff79",
            "4b832750eab0451997a0f892d38db423",
            "1bbf5fae9a2c4e32b3d3d162db1c9b5f",
            "7ec362f6554f416384b1a03a3ea41eb9",
            "594493b81e6f45cc94d9d44c2804d042",
            "141ba72718e043edb793f7869b2fedb5",
            "494f55304a184a518e4e35edf39dd7a2",
            "48d903cf22d84ecb8cd5af5874567536",
            "c97fb6b54c474f11ab1fee9aff91cf47",
            "03f2530b9b444d48931daa2ca900bb9e",
            "ded0e4189e594d31b190dbca2efd9cd4",
            "1bed16416cb945828b250748028e4be9",
            "75b649dfd7304bebad995e992d379fa5",
            "a20d444fee7b4919ab47e5bcd69e853a",
            "c330eadc3acc4ddc95f061bf8e841dad",
            "777ec9ccae5240048cdd3212a40ce632",
            "1a61aa0c0b6047f58eff4feb7aa32959",
            "17a3d48a545b4876849fe4ffc764752c",
            "681827839e92433e8dd25faa888c0680",
            "514ab933abbb44eda2cbfd7b6d5ff47a",
            "338ca79c5d294a2e967166437813b668",
            "841711bda59a4625b9a8bae26caed181",
            "3f04bf02bc2149c189bbf00c36ac7107",
            "21e9b97580f440a38a484e136ddadfb2",
            "d1e66db731e949cb8fb16c0bb35729ae",
            "725592c335604e9b95eaaf9852fd588e",
            "8c25c42889db4936af2626284c8c2bfc",
            "d0c60871f88d4405881aabac6e732de9",
            "c7f502323c1e45b696d1c79db8b93ba7",
            "ef643c62650f419ca01d4266049f6c30",
            "e3c9de4c383e489cb1d4e39ab9bd3683"
          ]
        },
        "outputId": "0e9bba12-bd74-4ece-e7d6-aeaae0fa0da9"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e3ebca197a0a426babfb75caf8f81799"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d0afdae8789b42d4985b0c4fda3c2564"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "03f2530b9b444d48931daa2ca900bb9e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "338ca79c5d294a2e967166437813b668"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "mlb = MultiLabelBinarizer()\n",
        "mlb.fit([range(len(classes_count))])\n",
        "train_df, val_df = train_test_split(df, test_size=0.2, random_state=42)\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "def tokenize_data(data, max_length=16):\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "    labels = []\n",
        "\n",
        "    for _, row in data.iterrows():\n",
        "        text = row['domain'].replace('.', ' ')\n",
        "        classes = row['classes']\n",
        "\n",
        "        encoding = tokenizer.encode_plus(\n",
        "            text,\n",
        "            max_length=max_length,\n",
        "            padding='max_length',\n",
        "            truncation=True,\n",
        "            return_tensors='pt'\n",
        "        )\n",
        "\n",
        "        input_ids.append(encoding['input_ids'])\n",
        "        attention_masks.append(encoding['attention_mask'])\n",
        "        labels.append(torch.tensor(mlb.transform([classes])))\n",
        "\n",
        "    input_ids = torch.cat(input_ids, dim=0)\n",
        "    attention_masks = torch.cat(attention_masks, dim=0)\n",
        "    labels = torch.cat(labels, dim=0)\n",
        "\n",
        "    return TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "\n",
        "train_dataset = tokenize_data(train_df)\n",
        "val_dataset = tokenize_data(val_df)\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
        "val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ScZSPDtsFST9"
      },
      "outputs": [],
      "source": [
        "class SmallBERT(nn.Module):\n",
        "    def __init__(self, hidden_size=256, num_layers=2, num_attention_heads=4, num_classes=2, vocab_size=30522):\n",
        "        super(SmallBERT, self).__init__()\n",
        "\n",
        "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
        "\n",
        "        self.transformer = nn.TransformerEncoder(\n",
        "            encoder_layer=nn.TransformerEncoderLayer(\n",
        "                d_model=hidden_size,\n",
        "                nhead=num_attention_heads,\n",
        "                dim_feedforward=hidden_size * 4\n",
        "            ),\n",
        "            num_layers=num_layers\n",
        "        )\n",
        "\n",
        "        self.dropout = nn.Dropout(0.1)\n",
        "        self.classifier = nn.Linear(hidden_size, num_classes)\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        embedded = self.embedding(input_ids)\n",
        "        transformer_output = self.transformer(embedded)\n",
        "        pooled_output = transformer_output.mean(dim=1)\n",
        "        pooled_output = self.dropout(pooled_output)\n",
        "        logits = self.classifier(pooled_output)\n",
        "\n",
        "        return logits"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uRGMQeqxygB-",
        "outputId": "2ff0b6bf-bd9a-4edd-dd3c-eaf4af9ed972"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
            "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SmallBERT(\n",
              "  (embedding): Embedding(30522, 64)\n",
              "  (transformer): TransformerEncoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): TransformerEncoderLayer(\n",
              "        (self_attn): MultiheadAttention(\n",
              "          (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
              "        )\n",
              "        (linear1): Linear(in_features=64, out_features=256, bias=True)\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "        (linear2): Linear(in_features=256, out_features=64, bias=True)\n",
              "        (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "        (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "        (dropout1): Dropout(p=0.1, inplace=False)\n",
              "        (dropout2): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=64, out_features=630, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "model = SmallBERT(hidden_size=64, num_layers=1, num_attention_heads=2, num_classes=len(mlb.classes_), vocab_size=30522)\n",
        "model.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I6SmEEwfyu02",
        "outputId": "ef09ac44-2346-4940-9cb4-a5cbf955f8d9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "Epoch 1: 100%|██████████| 17333/17333 [01:34<00:00, 183.05it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.01455639958662022\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 2: 100%|██████████| 17333/17333 [01:32<00:00, 187.41it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.009372596806357445\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 3: 100%|██████████| 17333/17333 [01:33<00:00, 185.43it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.00895946430422158\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 4: 100%|██████████| 17333/17333 [01:33<00:00, 184.48it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.008654748138614369\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 5: 100%|██████████| 17333/17333 [01:37<00:00, 178.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.008430107505195222\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 6: 100%|██████████| 17333/17333 [01:35<00:00, 181.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.00825132777584388\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 7: 100%|██████████| 17333/17333 [01:35<00:00, 181.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.008098585383865228\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 8: 100%|██████████| 17333/17333 [01:36<00:00, 178.89it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007973489129822018\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 9: 100%|██████████| 17333/17333 [01:34<00:00, 182.66it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007862447003308942\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 10: 100%|██████████| 17333/17333 [01:34<00:00, 184.37it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007767102444360781\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 11: 100%|██████████| 17333/17333 [01:32<00:00, 187.08it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007681683442265059\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 12: 100%|██████████| 17333/17333 [01:34<00:00, 183.56it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.0076040456217698555\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 13: 100%|██████████| 17333/17333 [01:33<00:00, 185.58it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007536577820711911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 14: 100%|██████████| 17333/17333 [01:33<00:00, 185.95it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007473697705334162\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 15: 100%|██████████| 17333/17333 [01:35<00:00, 182.32it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.0074216005095400365\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 16: 100%|██████████| 17333/17333 [01:34<00:00, 183.65it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007366839836092643\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 17: 100%|██████████| 17333/17333 [01:35<00:00, 182.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007323595992545943\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 18: 100%|██████████| 17333/17333 [01:35<00:00, 182.39it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007281322365534597\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 19: 100%|██████████| 17333/17333 [01:34<00:00, 184.08it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007237244195990475\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 20: 100%|██████████| 17333/17333 [01:32<00:00, 186.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007204567027232412\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 21: 100%|██████████| 17333/17333 [01:34<00:00, 182.81it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007169707122033225\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 22: 100%|██████████| 17333/17333 [01:34<00:00, 183.90it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.0071396396113856736\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 23: 100%|██████████| 17333/17333 [01:34<00:00, 184.27it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.0071061778510986525\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 24: 100%|██████████| 17333/17333 [01:32<00:00, 187.58it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007079116968562793\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 25: 100%|██████████| 17333/17333 [01:35<00:00, 182.40it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007053613749052303\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 26: 100%|██████████| 17333/17333 [01:33<00:00, 184.62it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007027668537672933\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 27: 100%|██████████| 17333/17333 [01:32<00:00, 187.06it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.007002266663781792\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 28: 100%|██████████| 17333/17333 [01:33<00:00, 185.74it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006983264061160072\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 29: 100%|██████████| 17333/17333 [01:32<00:00, 186.62it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006960326002187834\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 30: 100%|██████████| 17333/17333 [01:33<00:00, 185.09it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006937417333392953\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 31: 100%|██████████| 17333/17333 [01:34<00:00, 183.75it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006922033323461952\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 32: 100%|██████████| 17333/17333 [01:33<00:00, 184.46it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006898388598138539\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 33: 100%|██████████| 17333/17333 [01:33<00:00, 185.13it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006879616245666734\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 34: 100%|██████████| 17333/17333 [01:33<00:00, 184.60it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006862940901992164\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 35: 100%|██████████| 17333/17333 [01:42<00:00, 169.91it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006847698244776712\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 36: 100%|██████████| 17333/17333 [01:34<00:00, 183.32it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006830427775150129\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 37: 100%|██████████| 17333/17333 [01:34<00:00, 183.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006812137309707921\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 38: 100%|██████████| 17333/17333 [01:33<00:00, 184.78it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.0067945668246664325\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 39: 100%|██████████| 17333/17333 [01:35<00:00, 181.25it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.0067820949417340725\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 40: 100%|██████████| 17333/17333 [01:35<00:00, 182.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006767237709435089\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 41: 100%|██████████| 17333/17333 [01:33<00:00, 186.12it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006754826359595461\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 42: 100%|██████████| 17333/17333 [01:34<00:00, 184.04it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006737771247406829\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 43: 100%|██████████| 17333/17333 [01:33<00:00, 185.51it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006729339890418221\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 44: 100%|██████████| 17333/17333 [01:34<00:00, 183.86it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006713867076464513\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 45: 100%|██████████| 17333/17333 [01:36<00:00, 180.05it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006700394152881958\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 46: 100%|██████████| 17333/17333 [01:34<00:00, 183.63it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006688476914265044\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 47: 100%|██████████| 17333/17333 [01:33<00:00, 185.67it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006676017345109116\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 48: 100%|██████████| 17333/17333 [01:33<00:00, 184.72it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006663939434730227\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 49: 100%|██████████| 17333/17333 [01:33<00:00, 185.32it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006649965298220924\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch 50: 100%|██████████| 17333/17333 [01:33<00:00, 185.48it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Loss: 0.006640829532034061\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "model.train()\n",
        "\n",
        "optimizer = AdamW(model.parameters(), lr=2e-4)\n",
        "criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weights, reduction='mean')\n",
        "\n",
        "epochs = 50\n",
        "for epoch in range(epochs):\n",
        "  total_loss = 0\n",
        "\n",
        "  for batch in tqdm(train_dataloader, desc=f'Epoch {epoch + 1}'):\n",
        "    input_ids = batch[0].to(device)\n",
        "    attention_mask = batch[1].to(device)\n",
        "    labels = batch[2].to(device, dtype=torch.float)\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    logits = model(input_ids=input_ids, attention_mask=attention_mask)\n",
        "\n",
        "    loss = criterion(logits, labels)\n",
        "    total_loss += loss.item()\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "  average_loss = total_loss / len(train_dataloader)\n",
        "  print(f'Training Loss: {average_loss}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GO0fi0qQ7oKb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2863272e-3956-4e9a-b7ba-fd8b850ec406"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Validation: 100%|██████████| 4334/4334 [00:07<00:00, 563.03it/s]\n",
            "<ipython-input-11-49aedd21c825>:16: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:261.)\n",
            "  predicted_labels = mlb.inverse_transform(torch.tensor(predicted_logits))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predicted labels [(), (), (332,), (183, 194, 215), (215,), (23,), (534,), (), (140,), (183,), (229,), (239,), (), (227, 542), (536,), (140,), (1,), (140, 299), (), (250,), (173,), (227, 542), (239,), (363, 365), (), (215,), (243,), (1, 183, 363), (332, 340), (215,), (289,), (), (96, 289, 572), (), (), (), (), (160,), (215,), (1, 104, 183), (140, 227, 439), (), (289,), (408, 412, 601), (), (140, 215, 250, 528), (332,), (), (183,), (289,), (183, 194), (183,), (), (48, 363), (), (215,), (104,), (), (239, 423), (215, 250), (), (183,), (1,), (444,), (215,), (243, 245), (299, 325, 363, 364, 365), (104, 533), (239,), (534,), (183, 533), (400,), (332,), (), (250,), (215,), (103, 215, 439), (), (), (1, 183, 186, 215), (), (), (103, 126, 289), (363,), (172,), (215,), (140, 215, 439), (183,), (), (23,), (351,), (227, 229, 542), (1, 23), (), (), (), (183,), (215,), (183, 351), (325,)]\n",
            "True inputs ['usw1.green.ops.kargo.com', '6002359.global.siteimproveanalytics.io', 'didiglobal.com.dob.sibl.support-intelligence.net', 'ns500248.ns500202.ns500248.ns500242.ns500197.ns500219.sweetchicksclub.com', 'ms.email.nextdoor.com', 'd.hello.plowandhearth.com', 'p45-contentws.icloud.com', 'pakasak.com', 'vm3-proxy-hisgeneral-scus-sat02p-6.connector.his.msappproxy.net', 'gahu.hit.gemius.pl', 'ep.wakehealth.edu', 'fi-prj-poc-slim.frontsrv.com', 'v3.pebblepad.co.uk', 'osceolak12.schoology.com', 'mydnspt.net', 'default-download.splashtop.com', 'flow.performgroup.io', 'instagram.feoh11-1.fna.fbcdn.net', 'arin.authdns.ripe.net', 'lee.ns.cloudflare.com', 'goodhoodstore.com', 'gw01mem01-eastus.classroom.cloud', 'www.sos.state.mn.us', 'security.netflix.net', 'prod.lb.mlsmatrix.com', 'apps.paris.fr', 'rc.samsungweather.com', 'pull-flv-f6.douyincdn.com.hdlvcloud.ks-cdn.com', 'visitutah.com', 'ogs.google.co.kr', 'daraz-sg.alibaba.com.gds.alibabadns.com', 'stor.g2.ph.dell.com', 'zalando.es', 'perfumist.net', 'lifesparking.com', 'signon.hoovers.dnb.com', 'appdl2drcndbankcdn.cache.qcloudcdn.qq.com', 'api.my.healthequity.com', 'db3pcor006-meta.fe.1drv.com', 'cdn.pubble.io', 'jminsure-my.sharepoint.com', 'aws.upstart.com', 'shop.rewe-static.de', 'anonymous.print.avery.com', 'mygts.dhl.com', 'app.chat.global.xiaomi.net', 'citifxvelocity.com', 'core.gss-service.com', 'kdev.msap.io', 'i-subscriptions.thx.lazparking.com', 'ns500245.ns500249.ns500197.hostmaster.sweetxladies.com', 's7.tengu0.xyz', 'link.entrata.com', 'r5.sn-q4flrnsl.c.2mdn.net', 'static.one.network', 'expurgate.de', 'a2467.casalemedia.com', 'squadup.com', 'licensemanager.kaspersky-labs.com', 'doc-0k-7c-docstext.googleusercontent.com', 'perficient.com', 'r4---sn-vgqsrnzz.gvt1.com', 'blockworks-com.gallerycdn.vsassets.io', 'prod-s6-piv.nubank.com.br', 'mailchi.mp.dob.sibl.support-intelligence.net', 'www.steelexpress.co.uk', 'arlostreaming19261-z2-prod.ar.arlo.com', 'ic3-calling-enterpriseproxy.brazilsouth-prod.cosmic.office.net', 'eservices.dor.nc.gov', 'nevacloud.io', 'fceb4-1.fna.fbcdn.net', 'api.cosmopolitan.com.hk', 'map3.viamichelin.com', 'api.foreflight.com', 'o1276079.ingest.sentry.io', 'up.cm.ksmobile.com', 'triconenergy.sharepoint.com', 'mb.cdn.srv-hub.org', 'serv.ad-adapex.io', 'cname.mail.cname.cname.mail.cname.cname.cname.cname.cname.cdn.wan02.com', 'ssb-eu-secure-6.smartadserver.com', 'cap.co.uk', 'ppc-oauth.wac.trafficmanager.net.wac-0003.wac-dc-msedge.net.wac-0003.wac-msedge.net', 'nvidia.tt.omtrdc.net', 'euc.vision.meraki.com', 'ginkgobioworks.zoom.us', 'capanoinc-my.sharepoint.com', 'r2---sn-vgqskned.gvt1.com', 'beplb01.portal.hewitt.com', 'external-media.grailed.com', 'api-senso-cloud-anbaathcgfgmhqhw.z01.azurefd.net', 'classroom.emeritus.org', 'assets.omny.fm', 'r5---sn-nx5s7nel.c.2mdn.net', 'api.kfdealeraccess.com', 'puntown.com', 'st-v3-univ-srs-linux-3040.api.splashtop.com', 'planetoftheapps.com', 'ekbhgdvznq.dbysdbuylr.net', 'fcs.myforcura.com']\n",
            "True labels [[238], [183, 164], [332], [215, 183], [215], [351], [1], [1], [140], [578], [229], [363, 351, 365], [250, 439], [227, 542], [250], [1, 363], [103], [299], [140], [137, 534], [298], [227], [215, 140], [423], [570], [215], [243], [1, 183, 186], [332], [215], [1, 289], [103], [289, 572, 96], [94], [250], [183, 194], [103], [158, 160], [215], [1], [140, 227, 439], [103, 533], [289], [601], [103], [215], [310], [224, 439], [215, 140], [100], [1, 183, 194], [243], [1], [1, 363], [533], [243, 245], [245], [1, 183], [423], [250], [436, 439], [183], [263], [444], [1], [103, 164], [363, 325, 364], [104, 533], [239], [1, 183, 250], [183, 533], [254, 400], [332], [1], [1], [126, 215, 243, 183], [439], [250], [140, 227], [215, 183, 186], [183], [1], [215], [140], [172, 414], [215, 140], [439], [183], [151], [23], [243], [227, 542], [23], [1], [436], [1], [140, 363], [215, 250], [243], [243, 299, 325]]\n",
            "Validation Accuracy: 0.18336410315582993\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "\n",
        "predicted_logits = []\n",
        "with torch.no_grad():\n",
        "    for batch in tqdm(val_dataloader, desc='Validation'):\n",
        "      input_ids = batch[0].to(device)\n",
        "      attention_mask = batch[1].to(device)\n",
        "      labels = batch[2].to(device, dtype=torch.float)\n",
        "\n",
        "      logits = model(input_ids=input_ids, attention_mask=attention_mask)\n",
        "      probabilities = torch.sigmoid(logits)\n",
        "\n",
        "      preds = (probabilities > 0.2).cpu().numpy().astype(int)\n",
        "      predicted_logits.extend(preds)\n",
        "\n",
        "predicted_labels = mlb.inverse_transform(torch.tensor(predicted_logits))\n",
        "print('Predicted labels', predicted_labels[1000:1100])\n",
        "\n",
        "true_inputs = val_df['domain'].tolist()\n",
        "print('True inputs', true_inputs[1000:1100])\n",
        "\n",
        "true_labels = val_df['classes'].tolist()\n",
        "print('True labels', true_labels[1000:1100])\n",
        "\n",
        "true_logits = mlb.transform(true_labels)\n",
        "accuracy = accuracy_score(true_logits, predicted_logits)\n",
        "print(f'Validation Accuracy: {accuracy}')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model, \"bert_domain_classifier\")\n",
        "torch.save(model.state_dict(), \"bert_domain_classifier.pth\")"
      ],
      "metadata": {
        "id": "jSo-SvyK8Nan"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dummy_input_ids = torch.zeros((1, 16), dtype=torch.long).to(device)\n",
        "dummy_attention_mask = torch.zeros((1, 16), dtype=torch.long).to(device)\n",
        "input_names = ['input_ids', 'attention_mask']\n",
        "output_names = ['logits']\n",
        "dynamic_axes = {'input_ids': {0: 'batch_size'}, 'attention_mask': {0: 'batch_size'},\n",
        "                'logits': {0: 'batch_size'}}\n",
        "\n",
        "torch.onnx.export(model, (dummy_input_ids, dummy_attention_mask),\n",
        "                  \"bert_domain_classifier.onnx\", opset_version=14,\n",
        "                  input_names=input_names,\n",
        "                  output_names=output_names,\n",
        "                  dynamic_axes=dynamic_axes)"
      ],
      "metadata": {
        "id": "pqanQz3i8DBx"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "gpuType": "V100"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e3ebca197a0a426babfb75caf8f81799": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_548e251c68d044368ecb68898037bb01",
              "IPY_MODEL_7be5e1bf58f34d359bc1726c88c2006f",
              "IPY_MODEL_ab4a6233ff044573958ef204775509c5"
            ],
            "layout": "IPY_MODEL_887871dfa5414422b63285477cda1930"
          }
        },
        "548e251c68d044368ecb68898037bb01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2af63a8b833a4582babaec7bfc219a36",
            "placeholder": "​",
            "style": "IPY_MODEL_5b046c94863140dd858051e1b30cf02a",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "7be5e1bf58f34d359bc1726c88c2006f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ae4e4874f704c7d95479a435d3f5a18",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_78b5111866184eb096cdfb432a4d2c49",
            "value": 28
          }
        },
        "ab4a6233ff044573958ef204775509c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e91d31351fe4c06b0d7827e12686607",
            "placeholder": "​",
            "style": "IPY_MODEL_44dfbae45a8848a1bc84e12ebd9e93b6",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.54kB/s]"
          }
        },
        "887871dfa5414422b63285477cda1930": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2af63a8b833a4582babaec7bfc219a36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b046c94863140dd858051e1b30cf02a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7ae4e4874f704c7d95479a435d3f5a18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78b5111866184eb096cdfb432a4d2c49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e91d31351fe4c06b0d7827e12686607": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44dfbae45a8848a1bc84e12ebd9e93b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0afdae8789b42d4985b0c4fda3c2564": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7c960dc2e1264d6d813ea2b9c878ffa2",
              "IPY_MODEL_bd09a7a58ff14190b3a683c0e2e4ff79",
              "IPY_MODEL_4b832750eab0451997a0f892d38db423"
            ],
            "layout": "IPY_MODEL_1bbf5fae9a2c4e32b3d3d162db1c9b5f"
          }
        },
        "7c960dc2e1264d6d813ea2b9c878ffa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ec362f6554f416384b1a03a3ea41eb9",
            "placeholder": "​",
            "style": "IPY_MODEL_594493b81e6f45cc94d9d44c2804d042",
            "value": "vocab.txt: 100%"
          }
        },
        "bd09a7a58ff14190b3a683c0e2e4ff79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_141ba72718e043edb793f7869b2fedb5",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_494f55304a184a518e4e35edf39dd7a2",
            "value": 231508
          }
        },
        "4b832750eab0451997a0f892d38db423": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_48d903cf22d84ecb8cd5af5874567536",
            "placeholder": "​",
            "style": "IPY_MODEL_c97fb6b54c474f11ab1fee9aff91cf47",
            "value": " 232k/232k [00:00&lt;00:00, 2.28MB/s]"
          }
        },
        "1bbf5fae9a2c4e32b3d3d162db1c9b5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ec362f6554f416384b1a03a3ea41eb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "594493b81e6f45cc94d9d44c2804d042": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "141ba72718e043edb793f7869b2fedb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "494f55304a184a518e4e35edf39dd7a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "48d903cf22d84ecb8cd5af5874567536": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c97fb6b54c474f11ab1fee9aff91cf47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "03f2530b9b444d48931daa2ca900bb9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ded0e4189e594d31b190dbca2efd9cd4",
              "IPY_MODEL_1bed16416cb945828b250748028e4be9",
              "IPY_MODEL_75b649dfd7304bebad995e992d379fa5"
            ],
            "layout": "IPY_MODEL_a20d444fee7b4919ab47e5bcd69e853a"
          }
        },
        "ded0e4189e594d31b190dbca2efd9cd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c330eadc3acc4ddc95f061bf8e841dad",
            "placeholder": "​",
            "style": "IPY_MODEL_777ec9ccae5240048cdd3212a40ce632",
            "value": "tokenizer.json: 100%"
          }
        },
        "1bed16416cb945828b250748028e4be9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1a61aa0c0b6047f58eff4feb7aa32959",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_17a3d48a545b4876849fe4ffc764752c",
            "value": 466062
          }
        },
        "75b649dfd7304bebad995e992d379fa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_681827839e92433e8dd25faa888c0680",
            "placeholder": "​",
            "style": "IPY_MODEL_514ab933abbb44eda2cbfd7b6d5ff47a",
            "value": " 466k/466k [00:00&lt;00:00, 9.05MB/s]"
          }
        },
        "a20d444fee7b4919ab47e5bcd69e853a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c330eadc3acc4ddc95f061bf8e841dad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "777ec9ccae5240048cdd3212a40ce632": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1a61aa0c0b6047f58eff4feb7aa32959": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "17a3d48a545b4876849fe4ffc764752c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "681827839e92433e8dd25faa888c0680": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "514ab933abbb44eda2cbfd7b6d5ff47a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "338ca79c5d294a2e967166437813b668": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_841711bda59a4625b9a8bae26caed181",
              "IPY_MODEL_3f04bf02bc2149c189bbf00c36ac7107",
              "IPY_MODEL_21e9b97580f440a38a484e136ddadfb2"
            ],
            "layout": "IPY_MODEL_d1e66db731e949cb8fb16c0bb35729ae"
          }
        },
        "841711bda59a4625b9a8bae26caed181": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_725592c335604e9b95eaaf9852fd588e",
            "placeholder": "​",
            "style": "IPY_MODEL_8c25c42889db4936af2626284c8c2bfc",
            "value": "config.json: 100%"
          }
        },
        "3f04bf02bc2149c189bbf00c36ac7107": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0c60871f88d4405881aabac6e732de9",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c7f502323c1e45b696d1c79db8b93ba7",
            "value": 570
          }
        },
        "21e9b97580f440a38a484e136ddadfb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef643c62650f419ca01d4266049f6c30",
            "placeholder": "​",
            "style": "IPY_MODEL_e3c9de4c383e489cb1d4e39ab9bd3683",
            "value": " 570/570 [00:00&lt;00:00, 17.8kB/s]"
          }
        },
        "d1e66db731e949cb8fb16c0bb35729ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "725592c335604e9b95eaaf9852fd588e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c25c42889db4936af2626284c8c2bfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0c60871f88d4405881aabac6e732de9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7f502323c1e45b696d1c79db8b93ba7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ef643c62650f419ca01d4266049f6c30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3c9de4c383e489cb1d4e39ab9bd3683": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}