{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e7940e4-1ba3-4192-a82c-5008dc5ed698",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradio as gr\n",
    "from transformers import pipeline\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "54c98c8e-a9d2-453c-933a-6a945143361f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4be240ca-2fe4-4d02-9955-c54d414fc795",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e93873a-e119-4fb5-aa38-b8814098f758",
   "metadata": {},
   "source": [
    "### first we need to import our libraries dataset and convert them into .csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ad1e3444-0197-4a5f-945d-bee10e977ffe",
   "metadata": {},
   "outputs": [],
   "source": [
    "spam_folder = \"enron2/spam\"\n",
    "ham_folder = \"enron2/ham\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "262cfd88-9c03-4131-8e62-94dc9e5cce7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_emails(folder, label):\n",
    "    emails = []\n",
    "    for filename in os.listdir(folder):\n",
    "        if filename.endswith(\".txt\"):\n",
    "            with open(os.path.join(folder, filename), \"r\", encoding=\"utf-8\", errors=\"ignore\") as file:\n",
    "                text = file.read()\n",
    "                emails.append((text, label))\n",
    "    return emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f54a8a8a-9e31-46b4-82c1-6111e158594b",
   "metadata": {},
   "outputs": [],
   "source": [
    "spam_emails = load_emails(spam_folder, label=1)  \n",
    "ham_emails = load_emails(ham_folder, label=0)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8229102b-ec10-47f3-9cc8-dc42f7c4d6e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSV file created successfully!\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(all_emails, columns=[\"text\", \"label\"])\n",
    "df.to_csv(\"dataset/emails.csv\", index=False, escapechar='\\\\', quotechar='\"')\n",
    "print(\"CSV file created successfully!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a5484a4b-6a93-40ea-993a-36118c5a31b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"dataset/emails.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "643bbc72-bfaa-4e82-85f8-3f216bc9e843",
   "metadata": {},
   "source": [
    "### now we need to clean our messy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d778066a-5526-428c-a118-4e1178b206c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\raofb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\raofb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\raofb\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "24c1784f-8262-487a-9868-44762dc976ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     C:\\Users\\raofb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt_tab.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt_tab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e043a0fb-f7e0-4727-8aee-b8ecce88b835",
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "def clean_text(text):\n",
    "    # Lowercase\n",
    "    text = text.lower()\n",
    "    \n",
    "    # Remove HTML\n",
    "    text = BeautifulSoup(text, \"html.parser\").get_text()\n",
    "    \n",
    "    # Replace URLs and emails\n",
    "    text = re.sub(r'http\\S+', '<URL>', text)\n",
    "    text = re.sub(r'\\S+@\\S+', '<EMAIL>', text)\n",
    "    \n",
    "    # Remove special characters (retain $ for monetary values)\n",
    "    text = re.sub(r'[^\\w\\s$]', '', text)\n",
    "    \n",
    "    # Replace numbers (except after $)\n",
    "    text = re.sub(r'\\b\\d+\\b', '<NUM>', text)\n",
    "    \n",
    "    # Normalize whitespace\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "\n",
    "    tokens = word_tokenize(text)\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stopwords.words('english')]\n",
    "\n",
    "    return ' '.join(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f60b360e-c425-47f2-ad9c-6caf1d04f8ba",
   "metadata": {},
   "source": [
    "### now we apply our nltk function to dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "90b9a80e-0937-46c0-8430-226fa0f4d0cf",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('Dataset/emails.csv')\n",
    "df['cleaned_text'] = df['text'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a425c6e7-5628-4af2-9f0f-c2fe8b7d5eb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('cleaned_emails.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "54941e16-8791-426e-ba66-d207b90d7876",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>cleaned_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Subject: fw : this is the solution i mentioned...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject fw solution mentioned lsc oo thank ema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Subject: advs\\ngreetings ,\\ni am benedicta lin...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject advs greeting benedicta lindiwe hendri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Subject: whats new in summer ? bawled\\ncarolyn...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject whats new summer bawled carolyn regret...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Subject: \\nh $ ello\\ndea 54 r home owner ,\\nwe...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject h $ ello dea &lt; NUM &gt; r home owner beet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Subject: : ) ) you can not save the world by q...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject save world quitting smoking save self ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Subject: need software ? click here .\\ntop qua...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject need software click top quality softwa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Subject: spend too much on your phone bill ? 2...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject spend much phone bill &lt; NUM &gt; crystal ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Subject: slotting order confirmation may 18 , ...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject slotting order confirmation may &lt; NUM ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Subject: we shiip to ur country for mircosoft ...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject shiip ur country mircosoft adobe norto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Subject: urgent business proposal ,\\nmrs . reg...</td>\n",
       "      <td>1</td>\n",
       "      <td>subject urgent business proposal mr regina ros...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label  \\\n",
       "0  Subject: fw : this is the solution i mentioned...      1   \n",
       "1  Subject: advs\\ngreetings ,\\ni am benedicta lin...      1   \n",
       "2  Subject: whats new in summer ? bawled\\ncarolyn...      1   \n",
       "3  Subject: \\nh $ ello\\ndea 54 r home owner ,\\nwe...      1   \n",
       "4  Subject: : ) ) you can not save the world by q...      1   \n",
       "5  Subject: need software ? click here .\\ntop qua...      1   \n",
       "6  Subject: spend too much on your phone bill ? 2...      1   \n",
       "7  Subject: slotting order confirmation may 18 , ...      1   \n",
       "8  Subject: we shiip to ur country for mircosoft ...      1   \n",
       "9  Subject: urgent business proposal ,\\nmrs . reg...      1   \n",
       "\n",
       "                                        cleaned_text  \n",
       "0  subject fw solution mentioned lsc oo thank ema...  \n",
       "1  subject advs greeting benedicta lindiwe hendri...  \n",
       "2  subject whats new summer bawled carolyn regret...  \n",
       "3  subject h $ ello dea < NUM > r home owner beet...  \n",
       "4  subject save world quitting smoking save self ...  \n",
       "5  subject need software click top quality softwa...  \n",
       "6  subject spend much phone bill < NUM > crystal ...  \n",
       "7  subject slotting order confirmation may < NUM ...  \n",
       "8  subject shiip ur country mircosoft adobe norto...  \n",
       "9  subject urgent business proposal mr regina ros...  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "read = pd.read_csv(\"cleaned_emails.csv\")\n",
    "read.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f4b111d3-d794-4aaa-ac45-ef86a398cddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('cleaned_emails.csv')\n",
    "\n",
    "# Drop the 'text' column\n",
    "df = df.drop(columns=['text'])\n",
    "\n",
    "# Rename 'cleaned_text' to 'text'\n",
    "df = df.rename(columns={'cleaned_text': 'text'})\n",
    "\n",
    "# Save the updated DataFrame to a new CSV file (or overwrite the existing one)\n",
    "df.to_csv('Cleaned_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04a92aca-fa9c-468b-84dd-b6bc404b2851",
   "metadata": {},
   "source": [
    "### 2nd cleaning ---------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "977a3274-d470-4bf1-9467-dd63a0774ee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langdetect import detect, LangDetectException\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e690b5d1-aa74-4146-9bda-9437790ae571",
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def clean_text(text):\n",
    "    try:\n",
    "        # Detect and remove non-English text\n",
    "        if detect(text) != 'en':\n",
    "            return ''\n",
    "    except LangDetectException:\n",
    "        return ''\n",
    "    \n",
    "    # Lowercase\n",
    "    text = text.lower()\n",
    "    \n",
    "    # Remove HTML\n",
    "    text = BeautifulSoup(text, \"html.parser\").get_text()\n",
    "    \n",
    "    # Enhanced URL/email replacement\n",
    "    text = re.sub(r'\\b(?:https?://|www\\.)\\S+', '<URL>', text)\n",
    "    text = re.sub(r'\\b[\\w\\.-]+@[\\w\\.-]+\\.\\w{2,}\\b', '<EMAIL>', text)\n",
    "    \n",
    "    # Date pattern replacement (MM/DD/YYYY, DD-MM-YYYY, Month names, etc.)\n",
    "    text = re.sub(r'\\b(\\d{1,2}[/-]\\d{1,2}[/-]\\d{2,4}\\b|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\\s+\\d{1,2},?\\s+\\d{4})', '<DATE>', text)\n",
    "    \n",
    "    # Time pattern replacement\n",
    "    text = re.sub(r'\\b\\d{1,2}:\\d{2}\\s*(?:am|pm)?\\b', '<TIME>', text)\n",
    "    \n",
    "    # Handle monetary values before general number replacement\n",
    "    money_values = re.findall(r'\\$\\d+[\\d,.]*', text)\n",
    "    text = re.sub(r'\\$\\d+[\\d,.]*', '<MONEY>', text)\n",
    "    \n",
    "    # Replace other numbers\n",
    "    text = re.sub(r'\\b\\d+\\b', '<NUM>', text)\n",
    "    \n",
    "    # Split and recombine long sentences\n",
    "    sentences = sent_tokenize(text)\n",
    "    processed_sentences = []\n",
    "    for sent in sentences:\n",
    "        # Limit sentence length to 15 words\n",
    "        tokens = word_tokenize(sent)[:15]\n",
    "        processed_sentences.append(' '.join(tokens))\n",
    "    text = ' '.join(processed_sentences)\n",
    "    \n",
    "    # Remove special characters (keep $ for money handling)\n",
    "    text = re.sub(r'[^\\w\\s<>]', '', text)\n",
    "    \n",
    "    # Tokenization and lemmatization\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]\n",
    "    \n",
    "    return ' '.join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "64a46265-6d3e-4f80-bca9-5c6b418e0b55",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('Cleaned_data.csv')\n",
    "df['cleaned_text'] = df['text'].apply(clean_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "038cbb13-af40-4376-8721-d8a337aaf59e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('cleaned_data1.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecbb27b3-0a56-40c2-a3a9-79ccb0752c78",
   "metadata": {},
   "source": [
    "#### we save our final data into our directories "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0a079539-bbb3-47b2-9488-71c429a13dd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('Final_data.csv')\n",
    "\n",
    "# Drop the 'text' column\n",
    "df = df.drop(columns=['text'])\n",
    "\n",
    "# Rename 'cleaned_text' to 'text'\n",
    "df = df.rename(columns={'cleaned_text': 'text'})\n",
    "\n",
    "# Save the updated DataFrame to a new CSV file (or overwrite the existing one)\n",
    "df.to_csv('test.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60e531d9-0caa-4fc7-a691-e30e91a6d515",
   "metadata": {},
   "source": [
    "### now we split our data into training and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "128b90d9-b454-4f88-970a-8f5257b17671",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "95b7cbf5-127d-4af1-bf25-1a7c9c9c859d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  category                                               text\n",
      "0     spam  subject fw solution mentioned lsc oo thank ema...\n",
      "1     spam  subject advs greeting benedicta lindiwe hendri...\n",
      "2     spam  subject whats new summer bawled carolyn regret...\n",
      "3     spam  subject h $ ello dea < NUM > r home owner beet...\n",
      "4     spam  subject save world quitting smoking save self ...\n",
      "category\n",
      "spam    5996\n",
      "ham     5861\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"updated_dataset.csv\")\n",
    "print(df.head())  # we check the first row\n",
    "print(df[\"category\"].value_counts())  # Check classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9aa78718-154e-498e-b792-b63c2410cc71",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[\"text\"]  # Input text\n",
    "y = df[\"category\"]  # Labels (0 or 1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X,\n",
    "    y,\n",
    "    test_size=0.2,           ## 20% of data for testing\n",
    "    random_state=42,         ## Seed for reproducibility\n",
    "    stratify=y               ## Preserve spam/ham ratio in both splits\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bbc78d97-0357-464b-b26b-b4e39b185f94",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f417943-a5f7-4db6-bef5-b046c5d874fd",
   "metadata": {},
   "source": [
    "### SAVE OUR TRAIN AND TEST DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6bd3a326-0e59-4b53-94c1-7e54dcab6834",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training samples: 9485\n",
      "Testing samples: 2372\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.DataFrame({\"text\": X_train, \"category\": y_train})\n",
    "test_df = pd.DataFrame({\"text\": X_test, \"category\": y_test})\n",
    "\n",
    "# Save to CSV files\n",
    "train_df.to_csv(\"Train and Test/train.csv\", index=False)\n",
    "test_df.to_csv(\"Train and Test/test.csv\", index=False)\n",
    "\n",
    "print(f\"Training samples: {len(train_df)}\")\n",
    "print(f\"Testing samples: {len(test_df)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e7dffbbc-765e-4093-9ec7-baac9f63f873",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  category                                               text\n",
      "0     spam  subject fw solution mentioned lsc oo thank ema...\n",
      "1     spam  subject advs greeting benedicta lindiwe hendri...\n",
      "2     spam  subject whats new summer bawled carolyn regret...\n",
      "3     spam  subject h $ ello dea < NUM > r home owner beet...\n",
      "4     spam  subject save world quitting smoking save self ...\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Load your CSV dataset\n",
    "df = pd.read_csv('Final_data.csv')\n",
    "\n",
    "# Rename the 'label' column to 'category'\n",
    "df.rename(columns={'label': 'category'}, inplace=True)\n",
    "\n",
    "# Replace 0 with 'ham' and 1 with 'spam' in the 'category' column\n",
    "df['category'] = df['category'].replace({0: 'ham', 1: 'spam'})\n",
    "\n",
    "# Save the updated DataFrame back to a CSV (optional)\n",
    "df.to_csv('updated_dataset.csv', index=False)\n",
    "\n",
    "# Display the updated DataFrame\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97a86b3e-f6e2-4aba-a924-8af72ff049a1",
   "metadata": {},
   "source": [
    "### Check our split data values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0efe481d-e9f2-44b4-91f8-e42aa7c9709a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train class distribution:\n",
      " category\n",
      "spam    4796\n",
      "ham     4689\n",
      "Name: count, dtype: int64\n",
      "Test class distribution:\n",
      " category\n",
      "spam    1200\n",
      "ham     1172\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(\"Train class distribution:\\n\", train_df[\"category\"].value_counts())\n",
    "print(\"Test class distribution:\\n\", test_df[\"category\"].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4272c7ef-3e31-4779-92b3-9385c1b9f3d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>subject need outstanding logo working company ...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>subject weijun decided interview guess mean ba...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>subject investment partnership proposal dear s...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>subject tage resume submittal need time extern...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>subject trading ag prod vince wanted let know ...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>subject utf &lt; NUM &gt; q best dvd collect utf &lt; N...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>subject portcalc methodology keith power emrs ...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>subject ibuyit approver ah wonderful new syste...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>subject osman sezgen steve kromer requesting o...</td>\n",
       "      <td>ham</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>subject read email change life dear homeowner ...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text category\n",
       "0  subject need outstanding logo working company ...     spam\n",
       "1  subject weijun decided interview guess mean ba...      ham\n",
       "2  subject investment partnership proposal dear s...     spam\n",
       "3  subject tage resume submittal need time extern...      ham\n",
       "4  subject trading ag prod vince wanted let know ...      ham\n",
       "5  subject utf < NUM > q best dvd collect utf < N...     spam\n",
       "6  subject portcalc methodology keith power emrs ...      ham\n",
       "7  subject ibuyit approver ah wonderful new syste...      ham\n",
       "8  subject osman sezgen steve kromer requesting o...      ham\n",
       "9  subject read email change life dear homeowner ...     spam"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\"Train and Test/train.csv\")\n",
    "df_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a75e71c9-468d-49e0-811d-2b416d1cbe5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4689, 2)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ham = df_train[df_train['category']=='ham']\n",
    "df_ham.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1bc2aece-1656-4019-94dd-0b7fb4710b78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4796, 2)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_spam = df_train[df_train['category']=='spam']\n",
    "df_spam.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fc422e3e-df9d-4e06-b722-daee1659179a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9485, 2)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_balanced = pd.concat([df_ham, df_spam])\n",
    "df_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "81ce7ac6-619b-4d4e-ad41-769e2159a6d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9456</th>\n",
       "      <td>subject get thru july newsletter get thru news...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9458</th>\n",
       "      <td>subject cheap med nowrpmsqp every ph armaceuti...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9459</th>\n",
       "      <td>subject trusted source prescription medication...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9460</th>\n",
       "      <td>subject would hate reject chance reduce expend...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9469</th>\n",
       "      <td>subject backplanes &lt; NUM &gt; vicodin whim cocked...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9473</th>\n",
       "      <td>subject love girl idn wan na see girl naked pc...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9476</th>\n",
       "      <td>subject clalis $ &lt; NUM &gt; &lt; NUM &gt; discount drug...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9479</th>\n",
       "      <td>subject shoot spe rm drink increase cum volume...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9483</th>\n",
       "      <td>subject george williams george williams abidja...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9484</th>\n",
       "      <td>subject utf &lt; NUM &gt; q election result r utf &lt; ...</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text category\n",
       "9456  subject get thru july newsletter get thru news...     spam\n",
       "9458  subject cheap med nowrpmsqp every ph armaceuti...     spam\n",
       "9459  subject trusted source prescription medication...     spam\n",
       "9460  subject would hate reject chance reduce expend...     spam\n",
       "9469  subject backplanes < NUM > vicodin whim cocked...     spam\n",
       "9473  subject love girl idn wan na see girl naked pc...     spam\n",
       "9476  subject clalis $ < NUM > < NUM > discount drug...     spam\n",
       "9479  subject shoot spe rm drink increase cum volume...     spam\n",
       "9483  subject george williams george williams abidja...     spam\n",
       "9484  subject utf < NUM > q election result r utf < ...     spam"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_balanced.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "65f1fe2c-7a36-414c-8315-3647bd222686",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>spam</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2123</th>\n",
       "      <td>subject reminder cera executive roundtable hou...</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3480</th>\n",
       "      <td>subject fw tw docket rp &lt; NUM &gt; &lt; NUM &gt; &lt; NUM ...</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>subject &lt; NUM &gt; sm unique logo business lack v...</td>\n",
       "      <td>spam</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>subject satellite dish click info stop getting...</td>\n",
       "      <td>spam</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1920</th>\n",
       "      <td>subject truly amazed hi new product offer c _ ...</td>\n",
       "      <td>spam</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2371</th>\n",
       "      <td>subject chess board widow living &lt; NUM &gt; hi</td>\n",
       "      <td>spam</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8423</th>\n",
       "      <td>subject enron stanford program stinson monday ...</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7240</th>\n",
       "      <td>subject change lifestyle &lt; NUM &gt; k month bait ...</td>\n",
       "      <td>spam</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7885</th>\n",
       "      <td>subject cera monthly summary july &lt; NUM &gt; cera...</td>\n",
       "      <td>ham</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7869</th>\n",
       "      <td>subject save money buying ftware enough money ...</td>\n",
       "      <td>spam</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text category  spam\n",
       "2123  subject reminder cera executive roundtable hou...      ham     0\n",
       "3480  subject fw tw docket rp < NUM > < NUM > < NUM ...      ham     0\n",
       "1113  subject < NUM > sm unique logo business lack v...     spam     1\n",
       "365   subject satellite dish click info stop getting...     spam     1\n",
       "1920  subject truly amazed hi new product offer c _ ...     spam     1\n",
       "2371        subject chess board widow living < NUM > hi     spam     1\n",
       "8423  subject enron stanford program stinson monday ...      ham     0\n",
       "7240  subject change lifestyle < NUM > k month bait ...     spam     1\n",
       "7885  subject cera monthly summary july < NUM > cera...      ham     0\n",
       "7869  subject save money buying ftware enough money ...     spam     1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_balanced['spam'] = df_balanced['category'].apply(lambda x: 1 if x==\"spam\" else 0)\n",
    "df_balanced.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "022e9214-90da-476c-b3ec-12bee25e80a4",
   "metadata": {},
   "source": [
    "### -----------------------------------------------Part-2------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "047a7701-f782-4f80-aaf5-6bafc5301fbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text\n",
    "import keras_nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "645881bc-ecfc-461b-9a14-61c64c5f3764",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_path = \"/Users/techgarage/Projects/Data Engineering/Final Project/bert-tensorflow2-en-uncased-preprocess-v3\"\n",
    "encoder_path = \"/Users/techgarage/Projects/Data Engineering/Final Project/bert-tensorflow2-en-uncased-l-12-h-768-a-12-v4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d5e2d09b-c72f-4693-89fb-7d86134b7acd",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_preprocess = hub.KerasLayer(preprocess_path)\n",
    "bert_encoder = hub.KerasLayer(encoder_path, trainable=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d7023825-23df-4ab1-85c3-ea261b3a1761",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'default': <tf.Tensor: shape=(2, 768), dtype=float32, numpy=\n",
      "array([[-0.9430887 , -0.57238555, -0.8907942 , ..., -0.8635234 ,\n",
      "        -0.7363553 ,  0.93383116],\n",
      "       [-0.8458787 , -0.3137438 , -0.5593949 , ..., -0.3633124 ,\n",
      "        -0.6121973 ,  0.8722634 ]], dtype=float32)>, 'pooled_output': <tf.Tensor: shape=(2, 768), dtype=float32, numpy=\n",
      "array([[-0.9430887 , -0.57238555, -0.8907942 , ..., -0.8635234 ,\n",
      "        -0.7363553 ,  0.93383116],\n",
      "       [-0.8458787 , -0.3137438 , -0.5593949 , ..., -0.3633124 ,\n",
      "        -0.6121973 ,  0.8722634 ]], dtype=float32)>, 'sequence_output': <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[-0.19839856, -0.20800582, -0.1962688 , ..., -0.47272685,\n",
      "          0.0864562 ,  0.7098722 ],\n",
      "        [-0.5383126 , -0.7158387 , -0.28930873, ..., -0.72038037,\n",
      "          0.5794679 ,  0.39614454],\n",
      "        [-0.14156267, -0.7373614 ,  0.37026387, ..., -0.37185436,\n",
      "          0.07691361,  0.9704544 ],\n",
      "        ...,\n",
      "        [ 0.06476687, -0.4518682 ,  0.21092355, ...,  0.10411158,\n",
      "         -0.15344658,  0.15741925],\n",
      "        [ 0.00561799, -0.4243467 ,  0.17027403, ...,  0.12554014,\n",
      "         -0.1677104 ,  0.14828748],\n",
      "        [-0.06908819, -0.43356168,  0.25077093, ...,  0.12376285,\n",
      "         -0.21341398,  0.12534308]],\n",
      "\n",
      "       [[-0.05573511,  0.17838944, -0.12837237, ..., -0.16575927,\n",
      "          0.15549946,  0.21796423],\n",
      "        [ 0.27656567, -0.06027734,  0.24292594, ...,  0.05262224,\n",
      "          0.5221071 , -0.07443378],\n",
      "        [ 0.05904233,  0.11985369, -0.04732013, ...,  0.05382551,\n",
      "          0.24277204,  0.04877569],\n",
      "        ...,\n",
      "        [-0.06302132,  0.17733227,  0.40315932, ...,  0.11782701,\n",
      "         -0.20818709, -0.0962089 ],\n",
      "        [-0.0760133 ,  0.21092619,  0.45218655, ...,  0.14849319,\n",
      "         -0.26165274, -0.06360148],\n",
      "        [ 0.1626576 ,  0.12422754,  0.5222455 , ...,  0.24048476,\n",
      "         -0.24223803, -0.14883336]]], dtype=float32)>, 'encoder_outputs': [<tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 0.04222755, -0.00565784, -0.15892257, ...,  0.24197707,\n",
      "         -0.0975021 , -0.04534519],\n",
      "        [-0.57877016,  0.4591414 ,  0.17042653, ...,  0.17379816,\n",
      "          0.3257624 , -0.1035484 ],\n",
      "        [-1.0044422 , -0.39177877, -0.36441994, ...,  0.4770085 ,\n",
      "          0.24922882,  0.28879505],\n",
      "        ...,\n",
      "        [-0.02801086, -0.17419498,  0.6732487 , ...,  0.30962443,\n",
      "         -0.48121578,  0.01905968],\n",
      "        [-0.11255493, -0.16464029,  0.5454035 , ...,  0.33624443,\n",
      "         -0.38363996, -0.0586778 ],\n",
      "        [ 0.01631033, -0.09114645,  0.5857597 , ...,  0.65954405,\n",
      "         -0.7021383 , -0.12894379]],\n",
      "\n",
      "       [[ 0.20487462,  0.04901898,  0.01649245, ...,  0.04353925,\n",
      "          0.04925472,  0.02829387],\n",
      "        [ 1.4605433 ,  0.41298807,  0.85494065, ...,  0.6134594 ,\n",
      "          0.80718094, -0.2633693 ],\n",
      "        [-0.05972779, -0.18533123, -0.5527816 , ..., -0.10191233,\n",
      "          0.65774244,  0.43938908],\n",
      "        ...,\n",
      "        [ 0.01185893, -0.2301354 ,  0.6187198 , ...,  0.19465885,\n",
      "         -0.42088404,  0.1006694 ],\n",
      "        [-0.07991942, -0.24889088,  0.48138082, ...,  0.21367387,\n",
      "         -0.30915743,  0.01614393],\n",
      "        [ 0.04677403, -0.17776218,  0.5053538 , ...,  0.582287  ,\n",
      "         -0.6344621 , -0.06023622]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[-0.08198933, -0.24693474, -0.39003628, ...,  0.439112  ,\n",
      "          0.05109447, -0.0044647 ],\n",
      "        [-0.9807272 ,  0.6689387 ,  0.48308071, ...,  0.1564306 ,\n",
      "          0.09792089, -0.02700958],\n",
      "        [-1.1450135 , -0.21649039,  0.16978282, ...,  0.54555625,\n",
      "         -0.08833696,  0.4793641 ],\n",
      "        ...,\n",
      "        [-0.07667708, -0.46201542,  0.5964664 , ...,  0.88089323,\n",
      "         -0.61731744, -0.07153333],\n",
      "        [-0.22768137, -0.37603164,  0.567706  , ...,  0.7776769 ,\n",
      "         -0.6923214 , -0.16456038],\n",
      "        [-0.07292495, -0.47088498,  0.42633674, ...,  1.1841507 ,\n",
      "         -0.885921  , -0.15032959]],\n",
      "\n",
      "       [[ 0.06639995, -0.14721206, -0.10672337, ...,  0.06945898,\n",
      "          0.08428381,  0.07299946],\n",
      "        [ 1.2143568 ,  0.48814198,  1.1617565 , ...,  0.427473  ,\n",
      "          1.390525  , -0.6787748 ],\n",
      "        [-0.0177076 , -0.1816837 , -0.19067988, ..., -0.30094966,\n",
      "          0.28298748,  0.4206936 ],\n",
      "        ...,\n",
      "        [-0.04438182, -0.12137772,  0.4709124 , ...,  0.74642366,\n",
      "         -0.4703179 , -0.0582533 ],\n",
      "        [-0.06758315, -0.02387481,  0.50011903, ...,  0.79208016,\n",
      "         -0.4944227 , -0.11274293],\n",
      "        [ 0.13245018, -0.14612037,  0.41312754, ...,  1.0047185 ,\n",
      "         -0.68071204, -0.25335985]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 1.88205242e-02, -4.19564575e-01, -9.72785205e-02, ...,\n",
      "          4.34135646e-01,  2.40221262e-01,  2.04072416e-01],\n",
      "        [-4.48446095e-01,  1.81707352e-01,  4.33682859e-01, ...,\n",
      "         -5.54058142e-02, -4.61606085e-02,  1.36725694e-01],\n",
      "        [-1.06392920e+00, -2.69372582e-01,  6.61358237e-01, ...,\n",
      "          3.99683654e-01, -2.06476793e-01,  2.66633153e-01],\n",
      "        ...,\n",
      "        [-1.29950047e-03, -5.26915431e-01,  8.76984775e-01, ...,\n",
      "          1.12286353e+00, -8.32752362e-02, -8.66764039e-02],\n",
      "        [-5.30661419e-02, -4.99284506e-01,  7.59432733e-01, ...,\n",
      "          1.04773223e+00, -1.10542774e-01, -2.57611960e-01],\n",
      "        [ 4.16158885e-03, -5.81846416e-01,  6.97214842e-01, ...,\n",
      "          1.34556234e+00, -2.37027630e-01, -1.53350607e-01]],\n",
      "\n",
      "       [[ 1.98504999e-02, -2.34437555e-01,  1.12557501e-01, ...,\n",
      "          2.38051996e-01,  1.49298698e-01,  2.20182493e-01],\n",
      "        [ 1.29915905e+00,  3.28751534e-01,  1.23260486e+00, ...,\n",
      "          3.10731322e-01,  7.98960507e-01, -5.44931173e-01],\n",
      "        [-5.44543229e-02, -6.17580451e-02, -6.97334185e-02, ...,\n",
      "         -2.71830112e-01,  2.33011786e-03,  4.77785051e-01],\n",
      "        ...,\n",
      "        [-8.16972479e-02, -1.25493705e-01,  7.47347534e-01, ...,\n",
      "          9.28182423e-01, -6.34126663e-02, -1.33340880e-01],\n",
      "        [-1.30778015e-01, -3.14075388e-02,  6.66660905e-01, ...,\n",
      "          9.91367698e-01, -1.05210334e-01, -1.73617572e-01],\n",
      "        [ 6.76381961e-02, -1.79536566e-01,  7.31914282e-01, ...,\n",
      "          1.12215555e+00, -2.49684200e-01, -3.13854456e-01]]],\n",
      "      dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 0.23253876, -0.5878328 , -0.5117078 , ...,  0.54637855,\n",
      "          0.3729315 ,  0.8276504 ],\n",
      "        [-0.4173653 ,  0.10359846,  0.07069035, ...,  0.13642198,\n",
      "         -0.09496864, -0.17008804],\n",
      "        [-1.3769395 , -0.33059418,  0.7084929 , ...,  0.23613824,\n",
      "         -0.13318482,  0.6639668 ],\n",
      "        ...,\n",
      "        [-0.14409971, -0.95972544,  0.8196844 , ...,  1.029249  ,\n",
      "         -0.505237  , -0.0820753 ],\n",
      "        [-0.24914083, -0.8525534 ,  0.71119857, ...,  0.9299053 ,\n",
      "         -0.5289181 , -0.19867681],\n",
      "        [-0.17688558, -1.0652891 ,  0.8114191 , ...,  1.232535  ,\n",
      "         -0.5275141 , -0.18064196]],\n",
      "\n",
      "       [[ 0.07835185, -0.16930445, -0.25908536, ...,  0.11454112,\n",
      "          0.27769732,  0.53462124],\n",
      "        [ 1.1889614 , -0.39503306,  0.7059822 , ...,  0.3740296 ,\n",
      "          0.11370531, -0.6435863 ],\n",
      "        [ 0.31502503, -0.49227   , -0.01700327, ..., -0.00761845,\n",
      "          0.27753487,  0.73732984],\n",
      "        ...,\n",
      "        [-0.27396932, -0.601428  ,  0.8877718 , ...,  0.7409698 ,\n",
      "         -0.56265104,  0.03451739],\n",
      "        [-0.3649508 , -0.45706168,  0.78661674, ...,  0.76761526,\n",
      "         -0.5909752 , -0.01154372],\n",
      "        [-0.01357281, -0.62979025,  0.93073106, ...,  0.9332208 ,\n",
      "         -0.68241715, -0.3531196 ]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 0.11884356, -0.5148154 , -0.32688135, ...,  0.101791  ,\n",
      "          0.23662724,  0.88309544],\n",
      "        [-0.2888226 ,  0.30233493, -0.06162377, ..., -0.19382872,\n",
      "         -0.0729757 , -0.23562919],\n",
      "        [-1.2452599 , -0.5015673 ,  0.32685408, ...,  0.53150743,\n",
      "         -0.3750096 ,  0.73725295],\n",
      "        ...,\n",
      "        [ 0.03977381, -0.8479642 ,  0.7485787 , ...,  0.6571404 ,\n",
      "         -0.6224219 , -0.21472959],\n",
      "        [-0.0742109 , -0.7208384 ,  0.60503834, ...,  0.6880955 ,\n",
      "         -0.6776467 , -0.3155314 ],\n",
      "        [-0.06136909, -0.9506144 ,  0.76140445, ...,  0.8818397 ,\n",
      "         -0.8236465 , -0.3216293 ]],\n",
      "\n",
      "       [[-0.04276581,  0.07925801, -0.28103375, ..., -0.33148646,\n",
      "          0.41494718,  0.64737356],\n",
      "        [ 0.7152701 , -0.49672917,  1.1861763 , ..., -0.18042506,\n",
      "          0.06279897, -0.78743297],\n",
      "        [ 0.49038363, -0.30374768, -0.25442252, ..., -0.3617459 ,\n",
      "          0.42115545,  0.861674  ],\n",
      "        ...,\n",
      "        [-0.17577647, -0.24708512,  1.2082454 , ..., -0.04786705,\n",
      "         -0.13006926, -0.3941232 ],\n",
      "        [-0.17560306, -0.11561126,  1.0242093 , ...,  0.03608295,\n",
      "         -0.18673371, -0.4092454 ],\n",
      "        [ 0.18188742, -0.33772412,  1.145873  , ...,  0.23452786,\n",
      "         -0.31088895, -0.6601856 ]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 0.06071234, -0.7956739 , -0.4278758 , ...,  0.02831682,\n",
      "          0.33086035,  0.78214604],\n",
      "        [-0.48937097,  0.10409439, -0.5440423 , ..., -0.6400165 ,\n",
      "          0.06334624, -0.07265621],\n",
      "        [-0.8370529 , -0.43852547,  0.02520663, ...,  0.22630142,\n",
      "         -0.22112286,  0.7913871 ],\n",
      "        ...,\n",
      "        [-0.10923568, -0.8002732 ,  1.0630294 , ...,  0.44094047,\n",
      "         -0.6448848 , -0.13665396],\n",
      "        [-0.2489643 , -0.7156884 ,  0.97479945, ...,  0.57076055,\n",
      "         -0.72722065, -0.20035143],\n",
      "        [-0.17111725, -0.9605686 ,  1.0823051 , ...,  0.782978  ,\n",
      "         -0.85319215, -0.20169927]],\n",
      "\n",
      "       [[-0.03043595, -0.3351068 , -0.13165352, ..., -0.32476133,\n",
      "          0.35392705,  0.56300366],\n",
      "        [ 0.58081585, -0.35410577,  1.2072893 , ..., -0.2792115 ,\n",
      "          0.03580071, -0.7630123 ],\n",
      "        [ 0.5630262 , -0.4679435 , -0.38500452, ..., -0.45557278,\n",
      "         -0.00384355,  0.8763245 ],\n",
      "        ...,\n",
      "        [-0.23482245, -0.10766596,  1.4201454 , ...,  0.07200695,\n",
      "         -0.3376395 , -0.47948518],\n",
      "        [-0.25911236,  0.01417249,  1.3014742 , ...,  0.17332402,\n",
      "         -0.39001083, -0.49856895],\n",
      "        [ 0.21144739, -0.26251334,  1.3444276 , ...,  0.28540578,\n",
      "         -0.39566213, -0.74265194]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[-0.02161034, -0.9120089 , -0.2800715 , ..., -0.10675792,\n",
      "          0.6008877 ,  0.961232  ],\n",
      "        [-0.76047474, -0.03780157, -0.7370079 , ..., -0.39759997,\n",
      "         -0.07949114,  0.102171  ],\n",
      "        [-1.0510209 , -0.33599663,  0.21717477, ..., -0.04515497,\n",
      "         -0.0876311 ,  1.1958765 ],\n",
      "        ...,\n",
      "        [ 0.24660665, -0.8857314 ,  1.0813076 , ...,  0.4609538 ,\n",
      "         -0.3729293 ,  0.05634779],\n",
      "        [ 0.11600631, -0.79443914,  0.9784327 , ...,  0.57947654,\n",
      "         -0.50094205, -0.02501383],\n",
      "        [ 0.06238839, -1.0905796 ,  1.1894618 , ...,  0.7936153 ,\n",
      "         -0.64365435, -0.00459876]],\n",
      "\n",
      "       [[-0.02165605, -0.00627999, -0.09550503, ..., -0.46217042,\n",
      "          0.49604496,  0.5488441 ],\n",
      "        [ 0.82693785, -0.37880576,  1.3755609 , ...,  0.05833477,\n",
      "          0.0601463 , -0.9805254 ],\n",
      "        [ 0.6558955 , -0.07985266, -0.06040744, ..., -0.0520392 ,\n",
      "         -0.07544921,  0.7840497 ],\n",
      "        ...,\n",
      "        [-0.12956248, -0.13533002,  1.4172117 , ...,  0.5473515 ,\n",
      "         -0.3337415 , -0.3622912 ],\n",
      "        [-0.12117871, -0.05489191,  1.328096  , ...,  0.51856905,\n",
      "         -0.3706308 , -0.34924036],\n",
      "        [ 0.34187743, -0.27316713,  1.5144311 , ...,  0.68457603,\n",
      "         -0.24899381, -0.59764874]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 0.267469  , -0.62187463, -0.20508286, ..., -0.06582171,\n",
      "          0.3819914 ,  0.92700386],\n",
      "        [-0.49007717, -0.19729286, -0.63019806, ..., -0.4946556 ,\n",
      "          0.5848053 ,  0.48207486],\n",
      "        [-0.7382609 , -0.42379066,  0.04920355, ...,  0.22922839,\n",
      "          0.4823346 ,  1.3403559 ],\n",
      "        ...,\n",
      "        [ 0.36012125, -0.72662765,  1.2360562 , ...,  0.4134717 ,\n",
      "         -0.7964586 , -0.0168159 ],\n",
      "        [ 0.19020146, -0.6944481 ,  1.0574071 , ...,  0.40931013,\n",
      "         -0.8179945 , -0.10516895],\n",
      "        [ 0.08327669, -0.9118013 ,  1.1854435 , ...,  0.6869288 ,\n",
      "         -0.96097374, -0.18211707]],\n",
      "\n",
      "       [[-0.15515378,  0.24348043, -0.5878669 , ..., -0.8551526 ,\n",
      "          0.37197673,  0.77416694],\n",
      "        [ 0.82778156, -0.5299998 ,  1.1944404 , ..., -0.18086453,\n",
      "          0.22024067, -0.62813413],\n",
      "        [ 0.743023  , -0.2938047 , -0.23586163, ..., -0.41389942,\n",
      "         -0.06697719,  0.97216934],\n",
      "        ...,\n",
      "        [-0.06196451, -0.00945033,  1.3805895 , ...,  0.41180745,\n",
      "         -0.6445146 , -0.17519078],\n",
      "        [-0.08102008,  0.17403589,  1.4231976 , ...,  0.429969  ,\n",
      "         -0.68531966, -0.1757994 ],\n",
      "        [ 0.65523195, -0.11694056,  1.641292  , ...,  0.7779677 ,\n",
      "         -0.36524218, -0.4030809 ]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[ 0.262105  , -0.49547008, -0.21239214, ..., -0.00404145,\n",
      "          0.11047935,  1.0197922 ],\n",
      "        [-0.48511976, -0.2839242 , -0.02731339, ..., -0.695119  ,\n",
      "          0.44140273,  0.6733612 ],\n",
      "        [-0.4911195 , -0.29700607,  0.18301496, ..., -0.37084612,\n",
      "          0.44985855,  1.3507713 ],\n",
      "        ...,\n",
      "        [ 0.3708187 , -0.71213204,  0.8711745 , ...,  0.37050912,\n",
      "         -0.65640986,  0.17500949],\n",
      "        [ 0.24236405, -0.6819058 ,  0.8009118 , ...,  0.3233849 ,\n",
      "         -0.6954117 ,  0.0448739 ],\n",
      "        [ 0.06622784, -0.8053169 ,  0.8632049 , ...,  0.46156222,\n",
      "         -0.81855065, -0.05946006]],\n",
      "\n",
      "       [[-0.04873446,  0.4529825 , -0.92919904, ..., -0.8306872 ,\n",
      "          0.24304585,  0.5967375 ],\n",
      "        [ 0.63002735, -0.47839594,  0.7894074 , ..., -0.39560497,\n",
      "          0.16381933, -0.5277528 ],\n",
      "        [ 0.4487518 , -0.16525362, -0.53505677, ..., -0.24357045,\n",
      "          0.38862032,  0.5095053 ],\n",
      "        ...,\n",
      "        [ 0.04551447,  0.29526532,  0.99386215, ...,  0.27326238,\n",
      "         -0.3456278 , -0.23659004],\n",
      "        [-0.01135593,  0.43595943,  1.0248154 , ...,  0.3244376 ,\n",
      "         -0.46688926, -0.21477409],\n",
      "        [ 0.83194435,  0.27556145,  1.482031  , ...,  0.6464286 ,\n",
      "         -0.01236844, -0.57039464]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[-0.287879  , -0.80240715, -0.20276153, ..., -0.19355206,\n",
      "         -0.6109808 ,  1.0038904 ],\n",
      "        [-0.8657336 , -0.31282604, -0.22360814, ..., -0.6522808 ,\n",
      "          0.48126665,  0.6094352 ],\n",
      "        [-0.76809317, -0.24397063,  0.23806915, ..., -0.6669281 ,\n",
      "          0.12250681,  1.3389902 ],\n",
      "        ...,\n",
      "        [-0.24871868, -0.92885894,  0.6644963 , ...,  0.44990155,\n",
      "         -0.61868006,  0.21143207],\n",
      "        [-0.33172315, -0.86779344,  0.68440217, ...,  0.48378584,\n",
      "         -0.6701277 ,  0.1242202 ],\n",
      "        [-0.47390074, -0.971211  ,  0.7892387 , ...,  0.5549451 ,\n",
      "         -0.82587767, -0.04555673]],\n",
      "\n",
      "       [[-0.1323342 ,  0.56171477, -0.44893602, ..., -0.71494114,\n",
      "         -0.39547947,  0.38463563],\n",
      "        [ 0.5482401 , -0.60376394,  0.7847973 , ..., -0.3984929 ,\n",
      "          0.06032612, -0.7970418 ],\n",
      "        [ 0.25018427, -0.15513963, -0.18558574, ..., -0.08695153,\n",
      "          0.0445721 , -0.04240235],\n",
      "        ...,\n",
      "        [-0.24507912,  0.46590155,  0.5433186 , ...,  0.13668773,\n",
      "         -0.598839  , -0.45822692],\n",
      "        [-0.27610523,  0.70140356,  0.6461801 , ...,  0.17265077,\n",
      "         -0.7190671 , -0.4098673 ],\n",
      "        [ 0.6226735 ,  0.5686493 ,  1.1515373 , ...,  0.5658265 ,\n",
      "         -0.32691854, -0.5872827 ]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[-0.26158482, -0.6000199 , -0.14985457, ..., -0.3541886 ,\n",
      "         -0.0744052 ,  0.8018481 ],\n",
      "        [-0.5006118 , -0.8921    , -0.40147567, ..., -1.1037706 ,\n",
      "          0.4858664 ,  0.48425287],\n",
      "        [-0.53629315, -0.6906576 ,  0.29962227, ..., -0.62254155,\n",
      "          0.23217814,  1.3942658 ],\n",
      "        ...,\n",
      "        [ 0.16818719, -1.0856829 ,  0.5047239 , ...,  0.0128454 ,\n",
      "         -0.3836278 , -0.03140625],\n",
      "        [ 0.0023242 , -0.9831716 ,  0.5245629 , ...,  0.05957055,\n",
      "         -0.4703085 , -0.05248631],\n",
      "        [-0.21012276, -1.0518107 ,  0.6366271 , ...,  0.0917865 ,\n",
      "         -0.6165241 , -0.13285404]],\n",
      "\n",
      "       [[-0.11200634,  0.32884985, -0.1780588 , ..., -0.31254804,\n",
      "          0.03413404, -0.0067618 ],\n",
      "        [ 0.730604  , -0.37129667,  0.45334762, ..., -0.27244237,\n",
      "         -0.08073802, -0.4595476 ],\n",
      "        [ 0.26112485, -0.16279833, -0.12435186, ...,  0.01330798,\n",
      "          0.23121914,  0.2716457 ],\n",
      "        ...,\n",
      "        [-0.17112523,  0.3180147 ,  0.5765437 , ...,  0.03357287,\n",
      "         -0.97571343, -0.28733844],\n",
      "        [-0.22259393,  0.5266339 ,  0.7034917 , ...,  0.11205668,\n",
      "         -1.0813433 , -0.23581836],\n",
      "        [ 0.43861124,  0.44458354,  1.0901142 , ...,  0.34537375,\n",
      "         -0.9213069 , -0.32855487]]], dtype=float32)>, <tf.Tensor: shape=(2, 128, 768), dtype=float32, numpy=\n",
      "array([[[-0.19839856, -0.20800582, -0.1962688 , ..., -0.47272685,\n",
      "          0.0864562 ,  0.7098722 ],\n",
      "        [-0.5383126 , -0.7158387 , -0.28930873, ..., -0.72038037,\n",
      "          0.5794679 ,  0.39614454],\n",
      "        [-0.14156267, -0.7373614 ,  0.37026387, ..., -0.37185436,\n",
      "          0.07691361,  0.9704544 ],\n",
      "        ...,\n",
      "        [ 0.06476687, -0.4518682 ,  0.21092355, ...,  0.10411158,\n",
      "         -0.15344658,  0.15741925],\n",
      "        [ 0.00561799, -0.4243467 ,  0.17027403, ...,  0.12554014,\n",
      "         -0.1677104 ,  0.14828748],\n",
      "        [-0.06908819, -0.43356168,  0.25077093, ...,  0.12376285,\n",
      "         -0.21341398,  0.12534308]],\n",
      "\n",
      "       [[-0.05573511,  0.17838944, -0.12837237, ..., -0.16575927,\n",
      "          0.15549946,  0.21796423],\n",
      "        [ 0.27656567, -0.06027734,  0.24292594, ...,  0.05262224,\n",
      "          0.5221071 , -0.07443378],\n",
      "        [ 0.05904233,  0.11985369, -0.04732013, ...,  0.05382551,\n",
      "          0.24277204,  0.04877569],\n",
      "        ...,\n",
      "        [-0.06302132,  0.17733227,  0.40315932, ...,  0.11782701,\n",
      "         -0.20818709, -0.0962089 ],\n",
      "        [-0.0760133 ,  0.21092619,  0.45218655, ...,  0.14849319,\n",
      "         -0.26165274, -0.06360148],\n",
      "        [ 0.1626576 ,  0.12422754,  0.5222455 , ...,  0.24048476,\n",
      "         -0.24223803, -0.14883336]]], dtype=float32)>]}\n"
     ]
    }
   ],
   "source": [
    "# Example input text\n",
    "text = [\"This is a sample sentence.\", \"BERT is powerful!\"]\n",
    "\n",
    "# Preprocess the text\n",
    "preprocessed_text = bert_preprocess(text)\n",
    "\n",
    "# Encode the text\n",
    "bert_output = bert_encoder(preprocessed_text)\n",
    "\n",
    "# bert_output contains the contextualized embeddings\n",
    "print(bert_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cd44a41e-b6cc-4f8c-8452-c9294a50b4ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 768), dtype=float32, numpy=\n",
       "array([[-0.8435169 , -0.51327276, -0.8884573 , ..., -0.7474886 ,\n",
       "        -0.75314736,  0.91964495],\n",
       "       [-0.87208366, -0.5054398 , -0.9444669 , ..., -0.85847515,\n",
       "        -0.71745366,  0.88082975]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_sentence_embeding(sentences):\n",
    "    preprocessed_text = bert_preprocess(sentences)\n",
    "    return bert_encoder(preprocessed_text)['pooled_output']\n",
    "\n",
    "get_sentence_embeding([\n",
    "    \"500$ discount. hurry up\", \n",
    "    \"Bhavin, are you up for a volleybal game tomorrow?\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "335a2120-1cbb-41b0-824b-e70a3193160c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8d7b58eb-e09f-4043-bce7-003ea7e7f22b",
   "metadata": {},
   "outputs": [],
   "source": [
    "sl=get_sentence_embeding([\n",
    "    \"apple\",\n",
    "    \"mango\",\n",
    "    \"orange\",\n",
    "    \"Bill gates\",\n",
    "    \"tim cook\",\n",
    "    \"jeff bezos\"\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd167304-6e8d-4300-bc6a-7265dfe941bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9614038]], dtype=float32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity([sl[0]],[sl[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e882510f-afad-400d-bfb8-714a27503ac4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9564287]], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity([sl[2]],[sl[3]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a002b3e9-85e9-4a38-9a71-a184664cd29c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.987975]], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity([sl[0]],[sl[4]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1574cef7-1543-4393-b3be-da7eae69f029",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_1\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"functional_1\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ text (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)                 │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bert_preprocess_encode          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BertPreprocessEncode</span>)          │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ output (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │           <span style=\"color: #00af00; text-decoration-color: #00af00\">769</span> │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ text (\u001b[38;5;33mInputLayer\u001b[0m)               │ (\u001b[38;5;45mNone\u001b[0m)                 │             \u001b[38;5;34m0\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ bert_preprocess_encode          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n",
       "│ (\u001b[38;5;33mBertPreprocessEncode\u001b[0m)          │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ dropout (\u001b[38;5;33mDropout\u001b[0m)               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ output (\u001b[38;5;33mDense\u001b[0m)                  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │           \u001b[38;5;34m769\u001b[0m │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">769</span> (3.00 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m769\u001b[0m (3.00 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">769</span> (3.00 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m769\u001b[0m (3.00 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class BertPreprocessEncode(tf.keras.layers.Layer):\n",
    "    def __init__(self, preprocess_path, encoder_path, **kwargs):\n",
    "        super(BertPreprocessEncode, self).__init__(**kwargs)\n",
    "        self.preprocess = hub.KerasLayer(preprocess_path)\n",
    "        self.encoder = hub.KerasLayer(encoder_path)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        preprocessed = self.preprocess(inputs)\n",
    "        encoded = self.encoder(preprocessed)\n",
    "        return encoded['pooled_output']\n",
    "\n",
    "# Define the input layer\n",
    "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "\n",
    "# Use the custom preprocessing and encoding layer\n",
    "bert_output = BertPreprocessEncode(preprocess_path, encoder_path)(text_input)\n",
    "\n",
    "# Add a dropout layer and a dense layer for classification\n",
    "dropout = tf.keras.layers.Dropout(0.1, name=\"dropout\")(bert_output)\n",
    "output = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(dropout)\n",
    "\n",
    "# Build the model\n",
    "model = tf.keras.Model(inputs=[text_input], outputs=[output])\n",
    "\n",
    "# Compile the model\n",
    "model.compile(\n",
    "    optimizer='adam',\n",
    "    loss='binary_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")\n",
    "\n",
    "# Print the model summary\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "36f5f101-c490-487e-981e-2599ffa25d42",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "ConnectionError",
     "evalue": "('Connection aborted.', ConnectionResetError(54, 'Connection reset by peer'))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mConnectionResetError\u001b[0m                      Traceback (most recent call last)",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:715\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    714\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 715\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    716\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    717\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    718\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    719\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    720\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    721\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    722\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    723\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    725\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    726\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    727\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    728\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:404\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    403\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 404\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    405\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    406\u001b[0m     \u001b[38;5;66;03m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:1060\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m   1059\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(conn, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msock\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m):  \u001b[38;5;66;03m# AppEngine might not have  `.sock`\u001b[39;00m\n\u001b[0;32m-> 1060\u001b[0m     \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1062\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn\u001b[38;5;241m.\u001b[39mis_verified:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connection.py:419\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    417\u001b[0m     context\u001b[38;5;241m.\u001b[39mload_default_certs()\n\u001b[0;32m--> 419\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[43mssl_wrap_socket\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    420\u001b[0m \u001b[43m    \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    421\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkeyfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    422\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcertfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcert_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    423\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkey_password\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey_password\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    424\u001b[0m \u001b[43m    \u001b[49m\u001b[43mca_certs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_certs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    425\u001b[0m \u001b[43m    \u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    426\u001b[0m \u001b[43m    \u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_cert_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    427\u001b[0m \u001b[43m    \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    428\u001b[0m \u001b[43m    \u001b[49m\u001b[43mssl_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    429\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    430\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    432\u001b[0m \u001b[38;5;66;03m# If we're using all defaults and the connection\u001b[39;00m\n\u001b[1;32m    433\u001b[0m \u001b[38;5;66;03m# is TLSv1 or TLSv1.1 we throw a DeprecationWarning\u001b[39;00m\n\u001b[1;32m    434\u001b[0m \u001b[38;5;66;03m# for the host.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/util/ssl_.py:449\u001b[0m, in \u001b[0;36mssl_wrap_socket\u001b[0;34m(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)\u001b[0m\n\u001b[1;32m    448\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m send_sni:\n\u001b[0;32m--> 449\u001b[0m     ssl_sock \u001b[38;5;241m=\u001b[39m \u001b[43m_ssl_wrap_socket_impl\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[43m        \u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\n\u001b[1;32m    451\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    452\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/util/ssl_.py:493\u001b[0m, in \u001b[0;36m_ssl_wrap_socket_impl\u001b[0;34m(sock, ssl_context, tls_in_tls, server_hostname)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m server_hostname:\n\u001b[0;32m--> 493\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mssl_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/ssl.py:500\u001b[0m, in \u001b[0;36mSSLContext.wrap_socket\u001b[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[0m\n\u001b[1;32m    494\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    495\u001b[0m                 do_handshake_on_connect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    496\u001b[0m                 suppress_ragged_eofs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    497\u001b[0m                 server_hostname\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, session\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m    498\u001b[0m     \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[1;32m    499\u001b[0m     \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[0;32m--> 500\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msslsocket_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    503\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    504\u001b[0m \u001b[43m        \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    505\u001b[0m \u001b[43m        \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    506\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    507\u001b[0m \u001b[43m        \u001b[49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msession\u001b[49m\n\u001b[1;32m    508\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/ssl.py:1040\u001b[0m, in \u001b[0;36mSSLSocket._create\u001b[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[0m\n\u001b[1;32m   1039\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1040\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1041\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m):\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/ssl.py:1309\u001b[0m, in \u001b[0;36mSSLSocket.do_handshake\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m   1308\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1309\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1310\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "\u001b[0;31mConnectionResetError\u001b[0m: [Errno 54] Connection reset by peer",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mProtocolError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/requests/adapters.py:667\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    666\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 667\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    668\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    669\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    671\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    672\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    673\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    674\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    675\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    676\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    677\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    678\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    679\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:801\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    799\u001b[0m     e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[0;32m--> 801\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    802\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m    803\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    804\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/util/retry.py:552\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    551\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_method_retryable(method):\n\u001b[0;32m--> 552\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43merror\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    553\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/packages/six.py:769\u001b[0m, in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    768\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m value\u001b[38;5;241m.\u001b[39m__traceback__ \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m tb:\n\u001b[0;32m--> 769\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m value\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m    770\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m value\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:715\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    714\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 715\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    716\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    717\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    718\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    719\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    720\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    721\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    722\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    723\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    725\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    726\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    727\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    728\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:404\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    403\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 404\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    405\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    406\u001b[0m     \u001b[38;5;66;03m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connectionpool.py:1060\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m   1059\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(conn, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msock\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m):  \u001b[38;5;66;03m# AppEngine might not have  `.sock`\u001b[39;00m\n\u001b[0;32m-> 1060\u001b[0m     \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1062\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn\u001b[38;5;241m.\u001b[39mis_verified:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/connection.py:419\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    417\u001b[0m     context\u001b[38;5;241m.\u001b[39mload_default_certs()\n\u001b[0;32m--> 419\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[43mssl_wrap_socket\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    420\u001b[0m \u001b[43m    \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    421\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkeyfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    422\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcertfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcert_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    423\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkey_password\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey_password\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    424\u001b[0m \u001b[43m    \u001b[49m\u001b[43mca_certs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_certs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    425\u001b[0m \u001b[43m    \u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    426\u001b[0m \u001b[43m    \u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_cert_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    427\u001b[0m \u001b[43m    \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    428\u001b[0m \u001b[43m    \u001b[49m\u001b[43mssl_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    429\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    430\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    432\u001b[0m \u001b[38;5;66;03m# If we're using all defaults and the connection\u001b[39;00m\n\u001b[1;32m    433\u001b[0m \u001b[38;5;66;03m# is TLSv1 or TLSv1.1 we throw a DeprecationWarning\u001b[39;00m\n\u001b[1;32m    434\u001b[0m \u001b[38;5;66;03m# for the host.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/util/ssl_.py:449\u001b[0m, in \u001b[0;36mssl_wrap_socket\u001b[0;34m(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)\u001b[0m\n\u001b[1;32m    448\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m send_sni:\n\u001b[0;32m--> 449\u001b[0m     ssl_sock \u001b[38;5;241m=\u001b[39m \u001b[43m_ssl_wrap_socket_impl\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[43m        \u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\n\u001b[1;32m    451\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    452\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/urllib3/util/ssl_.py:493\u001b[0m, in \u001b[0;36m_ssl_wrap_socket_impl\u001b[0;34m(sock, ssl_context, tls_in_tls, server_hostname)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m server_hostname:\n\u001b[0;32m--> 493\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mssl_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/ssl.py:500\u001b[0m, in \u001b[0;36mSSLContext.wrap_socket\u001b[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[0m\n\u001b[1;32m    494\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    495\u001b[0m                 do_handshake_on_connect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    496\u001b[0m                 suppress_ragged_eofs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    497\u001b[0m                 server_hostname\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, session\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m    498\u001b[0m     \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[1;32m    499\u001b[0m     \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[0;32m--> 500\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msslsocket_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    503\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    504\u001b[0m \u001b[43m        \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    505\u001b[0m \u001b[43m        \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    506\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    507\u001b[0m \u001b[43m        \u001b[49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msession\u001b[49m\n\u001b[1;32m    508\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/ssl.py:1040\u001b[0m, in \u001b[0;36mSSLSocket._create\u001b[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[0m\n\u001b[1;32m   1039\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1040\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1041\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m):\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/ssl.py:1309\u001b[0m, in \u001b[0;36mSSLSocket.do_handshake\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m   1308\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1309\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1310\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "\u001b[0;31mProtocolError\u001b[0m: ('Connection aborted.', ConnectionResetError(54, 'Connection reset by peer'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m text_input \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mlayers\u001b[38;5;241m.\u001b[39mInput(shape\u001b[38;5;241m=\u001b[39m(), dtype\u001b[38;5;241m=\u001b[39mtf\u001b[38;5;241m.\u001b[39mstring)\n\u001b[0;32m----> 2\u001b[0m preprocessor \u001b[38;5;241m=\u001b[39m \u001b[43mkeras_nlp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mBertPreprocessor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_preset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mbert_base_en_uncased\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43mtrainable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m      3\u001b[0m encoder_inputs \u001b[38;5;241m=\u001b[39m preprocessor(text_input)\n\u001b[1;32m      4\u001b[0m encoder \u001b[38;5;241m=\u001b[39m keras_nlp\u001b[38;5;241m.\u001b[39mmodels\u001b[38;5;241m.\u001b[39mBertBackbone\u001b[38;5;241m.\u001b[39mfrom_preset(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbert_base_en_uncased\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/models/preprocessor.py:186\u001b[0m, in \u001b[0;36mPreprocessor.from_preset\u001b[0;34m(cls, preset, config_file, **kwargs)\u001b[0m\n\u001b[1;32m    184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mbackbone_cls \u001b[38;5;241m!=\u001b[39m backbone_cls:\n\u001b[1;32m    185\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m find_subclass(preset, \u001b[38;5;28mcls\u001b[39m, backbone_cls)\n\u001b[0;32m--> 186\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_preprocessor\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig_file\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/utils/preset_utils.py:690\u001b[0m, in \u001b[0;36mKerasPresetLoader.load_preprocessor\u001b[0;34m(self, cls, config_file, **kwargs)\u001b[0m\n\u001b[1;32m    684\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_preprocessor\u001b[39m(\n\u001b[1;32m    685\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mcls\u001b[39m, config_file\u001b[38;5;241m=\u001b[39mPREPROCESSOR_CONFIG_FILE, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m    686\u001b[0m ):\n\u001b[1;32m    687\u001b[0m     \u001b[38;5;66;03m# If there is no `preprocessing.json` or it's for the wrong class,\u001b[39;00m\n\u001b[1;32m    688\u001b[0m     \u001b[38;5;66;03m# delegate to the super class loader.\u001b[39;00m\n\u001b[1;32m    689\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m check_file_exists(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpreset, config_file):\n\u001b[0;32m--> 690\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_preprocessor\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    691\u001b[0m     preprocessor_json \u001b[38;5;241m=\u001b[39m load_json(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpreset, config_file)\n\u001b[1;32m    692\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(check_config_class(preprocessor_json), \u001b[38;5;28mcls\u001b[39m):\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/utils/preset_utils.py:618\u001b[0m, in \u001b[0;36mPresetLoader.load_preprocessor\u001b[0;34m(self, cls, config_file, **kwargs)\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_preprocessor\u001b[39m(\n\u001b[1;32m    610\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mcls\u001b[39m, config_file\u001b[38;5;241m=\u001b[39mPREPROCESSOR_CONFIG_FILE, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m    611\u001b[0m ):\n\u001b[1;32m    612\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Load a prepocessor layer from the preset.\u001b[39;00m\n\u001b[1;32m    613\u001b[0m \n\u001b[1;32m    614\u001b[0m \u001b[38;5;124;03m    By default, we create a preprocessor from a tokenizer with default\u001b[39;00m\n\u001b[1;32m    615\u001b[0m \u001b[38;5;124;03m    arguments. This allow us to support transformers checkpoints by\u001b[39;00m\n\u001b[1;32m    616\u001b[0m \u001b[38;5;124;03m    only converting the backbone and tokenizer.\u001b[39;00m\n\u001b[1;32m    617\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 618\u001b[0m     kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_add_missing_kwargs\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    619\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mcls\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/models/preprocessor.py:201\u001b[0m, in \u001b[0;36mPreprocessor._add_missing_kwargs\u001b[0;34m(cls, loader, kwargs)\u001b[0m\n\u001b[1;32m    190\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Fill in required kwargs when loading from preset.\u001b[39;00m\n\u001b[1;32m    191\u001b[0m \n\u001b[1;32m    192\u001b[0m \u001b[38;5;124;03mThis is a private method hit when loading a preprocessing layer that\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;124;03mencoders.\u001b[39;00m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    200\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m kwargs \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mtokenizer_cls:\n\u001b[0;32m--> 201\u001b[0m     kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mloader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_tokenizer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenizer_cls\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    202\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maudio_converter\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m kwargs \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39maudio_converter_cls:\n\u001b[1;32m    203\u001b[0m     kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maudio_converter\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m loader\u001b[38;5;241m.\u001b[39mload_audio_converter(\n\u001b[1;32m    204\u001b[0m         \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39maudio_converter_cls\n\u001b[1;32m    205\u001b[0m     )\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/utils/preset_utils.py:634\u001b[0m, in \u001b[0;36mKerasPresetLoader.load_tokenizer\u001b[0;34m(self, cls, config_file, **kwargs)\u001b[0m\n\u001b[1;32m    633\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_tokenizer\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mcls\u001b[39m, config_file\u001b[38;5;241m=\u001b[39mTOKENIZER_CONFIG_FILE, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 634\u001b[0m     tokenizer_config \u001b[38;5;241m=\u001b[39m \u001b[43mload_json\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpreset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig_file\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    635\u001b[0m     tokenizer \u001b[38;5;241m=\u001b[39m load_serialized_object(tokenizer_config, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    636\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(tokenizer, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mload_preset_assets\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/utils/preset_utils.py:444\u001b[0m, in \u001b[0;36mload_json\u001b[0;34m(preset, config_file)\u001b[0m\n\u001b[1;32m    443\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_json\u001b[39m(preset, config_file\u001b[38;5;241m=\u001b[39mCONFIG_FILE):\n\u001b[0;32m--> 444\u001b[0m     config_path \u001b[38;5;241m=\u001b[39m \u001b[43mget_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpreset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig_file\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    445\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(config_path, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m config_file:\n\u001b[1;32m    446\u001b[0m         config \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mload(config_file)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras_hub/src/utils/preset_utils.py:158\u001b[0m, in \u001b[0;36mget_file\u001b[0;34m(preset, path)\u001b[0m\n\u001b[1;32m    150\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    151\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnexpected Kaggle preset. Kaggle model handles should have \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    152\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthe form kaggle://\u001b[39m\u001b[38;5;132;01m{org}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{model}\u001b[39;00m\u001b[38;5;124m/keras/\u001b[39m\u001b[38;5;132;01m{variant}\u001b[39;00m\u001b[38;5;124m[/\u001b[39m\u001b[38;5;132;01m{version}\u001b[39;00m\u001b[38;5;124m]. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    155\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion). Received: preset=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpreset\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    156\u001b[0m     )\n\u001b[1;32m    157\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 158\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mkagglehub\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_download\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkaggle_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    159\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m KaggleApiHTTPError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    160\u001b[0m     message \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(e)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/kagglehub/models.py:35\u001b[0m, in \u001b[0;36mmodel_download\u001b[0;34m(handle, path, force_download)\u001b[0m\n\u001b[1;32m     33\u001b[0m h \u001b[38;5;241m=\u001b[39m parse_model_handle(handle)\n\u001b[1;32m     34\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDownloading Model: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mh\u001b[38;5;241m.\u001b[39mto_url()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m ...\u001b[39m\u001b[38;5;124m\"\u001b[39m, extra\u001b[38;5;241m=\u001b[39m{\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mEXTRA_CONSOLE_BLOCK})\n\u001b[0;32m---> 35\u001b[0m path, _ \u001b[38;5;241m=\u001b[39m \u001b[43mregistry\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_resolver\u001b[49m\u001b[43m(\u001b[49m\u001b[43mh\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m path\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/kagglehub/registry.py:28\u001b[0m, in \u001b[0;36mMultiImplRegistry.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m impl \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_impls):\n\u001b[1;32m     27\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m impl\u001b[38;5;241m.\u001b[39mis_supported(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 28\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mimpl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     29\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     30\u001b[0m         fails\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28mtype\u001b[39m(impl)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/kagglehub/resolver.py:29\u001b[0m, in \u001b[0;36mResolver.__call__\u001b[0;34m(self, handle, path, force_download)\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;28mself\u001b[39m, handle: T, path: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m, force_download: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m     17\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Optional[\u001b[38;5;28mint\u001b[39m]]:\n\u001b[1;32m     18\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Resolves a handle into a path with the requested file(s) and the resource's version number.\u001b[39;00m\n\u001b[1;32m     19\u001b[0m \n\u001b[1;32m     20\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;124;03m        Some cases where version number might be missing: Competition datasource, API-based models.\u001b[39;00m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 29\u001b[0m     path, version \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_resolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;66;03m# Note handles are immutable, so _resolve() could not have altered our reference\u001b[39;00m\n\u001b[1;32m     32\u001b[0m     register_datasource_access(handle, version)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/kagglehub/http_resolver.py:167\u001b[0m, in \u001b[0;36mModelHttpResolver._resolve\u001b[0;34m(self, h, path, force_download)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m path:\n\u001b[1;32m    165\u001b[0m     \u001b[38;5;66;03m# Downloading a single file.\u001b[39;00m\n\u001b[1;32m    166\u001b[0m     os\u001b[38;5;241m.\u001b[39mmakedirs(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(out_path), exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 167\u001b[0m     \u001b[43mapi_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mh\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextract_auto_compressed_file\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    168\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    169\u001b[0m     \u001b[38;5;66;03m# List the files and decide how to download them:\u001b[39;00m\n\u001b[1;32m    170\u001b[0m     \u001b[38;5;66;03m# - <= 25 files: Download files in parallel\u001b[39;00m\n\u001b[1;32m    171\u001b[0m     \u001b[38;5;66;03m# > 25 files: Download the archive and uncompress\u001b[39;00m\n\u001b[1;32m    172\u001b[0m     (files, has_more) \u001b[38;5;241m=\u001b[39m _list_files(api_client, h)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/kagglehub/clients.py:175\u001b[0m, in \u001b[0;36mKaggleApiV1Client.download_file\u001b[0;34m(self, path, out_file, resource_handle, cached_path, extract_auto_compressed_file)\u001b[0m\n\u001b[1;32m    167\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    168\u001b[0m \u001b[38;5;124;03mIssues a call to kaggle api and downloads files. For competition downloads,\u001b[39;00m\n\u001b[1;32m    169\u001b[0m \u001b[38;5;124;03mcall may return early if local cache is newer than the last time the file was modified.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    172\u001b[0m \u001b[38;5;124;03mbool:  If downloading remote was necessary\u001b[39;00m\n\u001b[1;32m    173\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    174\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_url(path)\n\u001b[0;32m--> 175\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    176\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    177\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mUser-Agent\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mget_user_agent\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    178\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    179\u001b[0m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_auth\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    180\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mDEFAULT_CONNECT_TIMEOUT\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mDEFAULT_READ_TIMEOUT\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    181\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m response:\n\u001b[1;32m    182\u001b[0m     kaggle_api_raise_for_status(response, resource_handle)\n\u001b[1;32m    184\u001b[0m     total_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(response\u001b[38;5;241m.\u001b[39mheaders[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Length\u001b[39m\u001b[38;5;124m\"\u001b[39m]) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Length\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m response\u001b[38;5;241m.\u001b[39mheaders \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/requests/api.py:73\u001b[0m, in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(url, params\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     63\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \n\u001b[1;32m     65\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mget\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    585\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m    586\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m    587\u001b[0m }\n\u001b[1;32m    588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/requests/adapters.py:682\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    667\u001b[0m     resp \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39murlopen(\n\u001b[1;32m    668\u001b[0m         method\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[1;32m    669\u001b[0m         url\u001b[38;5;241m=\u001b[39murl,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    678\u001b[0m         chunked\u001b[38;5;241m=\u001b[39mchunked,\n\u001b[1;32m    679\u001b[0m     )\n\u001b[1;32m    681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 682\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m    684\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m MaxRetryError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    685\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, ConnectTimeoutError):\n\u001b[1;32m    686\u001b[0m         \u001b[38;5;66;03m# TODO: Remove this in 3.0.0: see #2811\u001b[39;00m\n",
      "\u001b[0;31mConnectionError\u001b[0m: ('Connection aborted.', ConnectionResetError(54, 'Connection reset by peer'))"
     ]
    }
   ],
   "source": [
    "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string)\n",
    "preprocessor = keras_nlp.models.BertPreprocessor.from_preset(\"bert_base_en_uncased\",trainable=True)\n",
    "encoder_inputs = preprocessor(text_input)\n",
    "encoder = keras_nlp.models.BertBackbone.from_preset(\"bert_base_en_uncased\")\n",
    "outputs = encoder(encoder_inputs)\n",
    "pooled_output = outputs[\"pooled_output\"]      # [batch_size, 768].\n",
    "sequence_output = outputs[\"sequence_output\"]  # [batch_size, seq_length, 768]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ca5f1364-47d2-4dde-afa2-51f75ce9850c",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Exception encountered when calling layer 'keras_layer' (type KerasLayer).\n\nA KerasTensor is symbolic: it's a placeholder for a shape an a dtype. It doesn't have any actual numerical value. You cannot convert it to a NumPy array.\n\nCall arguments received by layer 'keras_layer' (type KerasLayer):\n  • inputs=<KerasTensor shape=(None,), dtype=string, sparse=None, name=test>\n  • training=None",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Bert layers\u001b[39;00m\n\u001b[1;32m      2\u001b[0m text_input \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mlayers\u001b[38;5;241m.\u001b[39mInput(shape\u001b[38;5;241m=\u001b[39m(), dtype\u001b[38;5;241m=\u001b[39mtf\u001b[38;5;241m.\u001b[39mstring, name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtest\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m preprocessed_text \u001b[38;5;241m=\u001b[39m \u001b[43mbert_preprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext_input\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m outputs \u001b[38;5;241m=\u001b[39m bert_encoder(preprocessed_text)\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# Neural network layers\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tf_keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow_hub/keras_layer.py:250\u001b[0m, in \u001b[0;36mKerasLayer.call\u001b[0;34m(self, inputs, training)\u001b[0m\n\u001b[1;32m    247\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;66;03m# Behave like BatchNormalization. (Dropout is different, b/181839368.)\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 250\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[43msmart_cond\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msmart_cond\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    251\u001b[0m \u001b[43m                                 \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    252\u001b[0m \u001b[43m                                 \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    254\u001b[0m \u001b[38;5;66;03m# Unwrap dicts returned by signatures.\u001b[39;00m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_key:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow_hub/keras_layer.py:252\u001b[0m, in \u001b[0;36mKerasLayer.call.<locals>.<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    247\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;66;03m# Behave like BatchNormalization. (Dropout is different, b/181839368.)\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m    250\u001b[0m   result \u001b[38;5;241m=\u001b[39m smart_cond\u001b[38;5;241m.\u001b[39msmart_cond(training,\n\u001b[1;32m    251\u001b[0m                                  \u001b[38;5;28;01mlambda\u001b[39;00m: f(training\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m),\n\u001b[0;32m--> 252\u001b[0m                                  \u001b[38;5;28;01mlambda\u001b[39;00m: \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m)\n\u001b[1;32m    254\u001b[0m \u001b[38;5;66;03m# Unwrap dicts returned by signatures.\u001b[39;00m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_key:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow/core/function/polymorphism/function_type.py:583\u001b[0m, in \u001b[0;36mcanonicalize_to_monomorphic\u001b[0;34m(args, kwargs, default_values, capture_types, polymorphic_type)\u001b[0m\n\u001b[1;32m    577\u001b[0m       parameters\u001b[38;5;241m.\u001b[39mappend(\n\u001b[1;32m    578\u001b[0m           _make_validated_mono_param(kwarg_name, arg[kwarg_name],\n\u001b[1;32m    579\u001b[0m                                      Parameter\u001b[38;5;241m.\u001b[39mKEYWORD_ONLY, type_context,\n\u001b[1;32m    580\u001b[0m                                      poly_parameter\u001b[38;5;241m.\u001b[39mtype_constraint))\n\u001b[1;32m    581\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    582\u001b[0m     parameters\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m--> 583\u001b[0m         \u001b[43m_make_validated_mono_param\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpoly_parameter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkind\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    584\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mtype_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    585\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mpoly_parameter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtype_constraint\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    587\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m FunctionType(parameters, capture_types), type_context\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow/core/function/polymorphism/function_type.py:522\u001b[0m, in \u001b[0;36m_make_validated_mono_param\u001b[0;34m(name, value, kind, type_context, poly_type)\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_make_validated_mono_param\u001b[39m(\n\u001b[1;32m    519\u001b[0m     name, value, kind, type_context, poly_type\n\u001b[1;32m    520\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Parameter:\n\u001b[1;32m    521\u001b[0m \u001b[38;5;250m  \u001b[39m\u001b[38;5;124;03m\"\"\"Generates and validates a parameter for Monomorphic FunctionType.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 522\u001b[0m   mono_type \u001b[38;5;241m=\u001b[39m \u001b[43mtrace_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_value\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtype_context\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    524\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m poly_type \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m mono_type\u001b[38;5;241m.\u001b[39mis_subtype_of(poly_type):\n\u001b[1;32m    525\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mParameter `\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` was expected to be of type \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    526\u001b[0m                     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpoly_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m but is \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmono_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow/core/function/trace_type/trace_type_builder.py:185\u001b[0m, in \u001b[0;36mfrom_value\u001b[0;34m(value, context)\u001b[0m\n\u001b[1;32m    178\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m default_types\u001b[38;5;241m.\u001b[39mAttrs\u001b[38;5;241m.\u001b[39mfrom_type_and_attributes(\n\u001b[1;32m    179\u001b[0m       \u001b[38;5;28mtype\u001b[39m(value),\n\u001b[1;32m    180\u001b[0m       \u001b[38;5;28mtuple\u001b[39m(\n\u001b[1;32m    181\u001b[0m           from_value(\u001b[38;5;28mgetattr\u001b[39m(value, a\u001b[38;5;241m.\u001b[39mname), context)\n\u001b[1;32m    182\u001b[0m           \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m value\u001b[38;5;241m.\u001b[39m__attrs_attrs__))\n\u001b[1;32m    184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m util\u001b[38;5;241m.\u001b[39mis_np_ndarray(value):\n\u001b[0;32m--> 185\u001b[0m   ndarray \u001b[38;5;241m=\u001b[39m \u001b[43mvalue\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__array__\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    186\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m default_types\u001b[38;5;241m.\u001b[39mTENSOR(ndarray\u001b[38;5;241m.\u001b[39mshape, ndarray\u001b[38;5;241m.\u001b[39mdtype)\n\u001b[1;32m    188\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value, custom_nest_protocol\u001b[38;5;241m.\u001b[39mCustomNestProtocol):\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras/src/backend/common/keras_tensor.py:61\u001b[0m, in \u001b[0;36mKerasTensor.__array__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__array__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 61\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m     62\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mA KerasTensor is symbolic: it\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms a placeholder for a shape \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     63\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124man a dtype. It doesn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt have any actual numerical value. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     64\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou cannot convert it to a NumPy array.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     65\u001b[0m     )\n",
      "\u001b[0;31mValueError\u001b[0m: Exception encountered when calling layer 'keras_layer' (type KerasLayer).\n\nA KerasTensor is symbolic: it's a placeholder for a shape an a dtype. It doesn't have any actual numerical value. You cannot convert it to a NumPy array.\n\nCall arguments received by layer 'keras_layer' (type KerasLayer):\n  • inputs=<KerasTensor shape=(None,), dtype=string, sparse=None, name=test>\n  • training=None"
     ]
    }
   ],
   "source": [
    "# Bert layers\n",
    "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='test')\n",
    "preprocessed_text = bert_preprocess(text_input)\n",
    "outputs = bert_encoder(preprocessed_text)\n",
    "\n",
    "# Neural network layers\n",
    "l = tf.keras.layers.Dropout(0.1, name=\"dropout\")(outputs['pooled_output'])\n",
    "l = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(l)\n",
    "\n",
    "# Use inputs and outputs to construct a final model\n",
    "model = tf.keras.Model(inputs=[text_input], outputs = [l])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "04f8b330-e5a6-4509-8bd0-5cfa2f50c541",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/techgarage/anaconda3/envs/DE/lib/python3.9/site-packages/keras/src/models/functional.py:237: UserWarning: The structure of `inputs` doesn't match the expected structure.\n",
      "Expected: ['text']\n",
      "Received: inputs=Tensor(shape=(None,))\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 3s/step - accuracy: 0.5000 - loss: 1.0744\n",
      "Epoch 2/3\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 209ms/step - accuracy: 0.5000 - loss: 0.7670\n",
      "Epoch 3/3\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 207ms/step - accuracy: 0.5000 - loss: 0.7240\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.history.History at 0x2d69dca00>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Example training data (spam detection)\n",
    "train_texts = [\n",
    "    \"Win a free iPhone now!!!\",  # Spam (1)\n",
    "    \"Meeting at 3 PM tomorrow\"   # Ham (0)\n",
    "]\n",
    "train_labels = np.array([1, 0])  # Use numpy arrays for labels\n",
    "\n",
    "# Convert to a TensorFlow Dataset\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_texts, train_labels))\n",
    "train_dataset = train_dataset.batch(32)  # Adjust batch size as needed\n",
    "\n",
    "# Train the model\n",
    "model.fit(train_dataset, epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "05be4db7-be33-49be-ac6e-441650cece64",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Exception encountered when calling layer 'keras_layer_4' (type KerasLayer).\n\nA KerasTensor is symbolic: it's a placeholder for a shape an a dtype. It doesn't have any actual numerical value. You cannot convert it to a NumPy array.\n\nCall arguments received by layer 'keras_layer_4' (type KerasLayer):\n  • inputs=<KerasTensor shape=(None,), dtype=string, sparse=False, name=text>\n  • training=None",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[39], line 14\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;66;03m# Define the model architecture\u001b[39;00m\n\u001b[1;32m     13\u001b[0m text_input \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mlayers\u001b[38;5;241m.\u001b[39mInput(shape\u001b[38;5;241m=\u001b[39m(), dtype\u001b[38;5;241m=\u001b[39mtf\u001b[38;5;241m.\u001b[39mstring, name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m preprocessed_text \u001b[38;5;241m=\u001b[39m \u001b[43mbert_preprocess\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext_input\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     15\u001b[0m encoded_output \u001b[38;5;241m=\u001b[39m bert_encoder(preprocessed_text)\n\u001b[1;32m     16\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m encoded_output[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpooled_output\u001b[39m\u001b[38;5;124m'\u001b[39m]\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tf_keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow_hub/keras_layer.py:250\u001b[0m, in \u001b[0;36mKerasLayer.call\u001b[0;34m(self, inputs, training)\u001b[0m\n\u001b[1;32m    247\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;66;03m# Behave like BatchNormalization. (Dropout is different, b/181839368.)\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 250\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[43msmart_cond\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msmart_cond\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    251\u001b[0m \u001b[43m                                 \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    252\u001b[0m \u001b[43m                                 \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    254\u001b[0m \u001b[38;5;66;03m# Unwrap dicts returned by signatures.\u001b[39;00m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_key:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow_hub/keras_layer.py:252\u001b[0m, in \u001b[0;36mKerasLayer.call.<locals>.<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    247\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;66;03m# Behave like BatchNormalization. (Dropout is different, b/181839368.)\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m    250\u001b[0m   result \u001b[38;5;241m=\u001b[39m smart_cond\u001b[38;5;241m.\u001b[39msmart_cond(training,\n\u001b[1;32m    251\u001b[0m                                  \u001b[38;5;28;01mlambda\u001b[39;00m: f(training\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m),\n\u001b[0;32m--> 252\u001b[0m                                  \u001b[38;5;28;01mlambda\u001b[39;00m: \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m)\n\u001b[1;32m    254\u001b[0m \u001b[38;5;66;03m# Unwrap dicts returned by signatures.\u001b[39;00m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_key:\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow/core/function/polymorphism/function_type.py:583\u001b[0m, in \u001b[0;36mcanonicalize_to_monomorphic\u001b[0;34m(args, kwargs, default_values, capture_types, polymorphic_type)\u001b[0m\n\u001b[1;32m    577\u001b[0m       parameters\u001b[38;5;241m.\u001b[39mappend(\n\u001b[1;32m    578\u001b[0m           _make_validated_mono_param(kwarg_name, arg[kwarg_name],\n\u001b[1;32m    579\u001b[0m                                      Parameter\u001b[38;5;241m.\u001b[39mKEYWORD_ONLY, type_context,\n\u001b[1;32m    580\u001b[0m                                      poly_parameter\u001b[38;5;241m.\u001b[39mtype_constraint))\n\u001b[1;32m    581\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    582\u001b[0m     parameters\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m--> 583\u001b[0m         \u001b[43m_make_validated_mono_param\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpoly_parameter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkind\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    584\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mtype_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    585\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43mpoly_parameter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtype_constraint\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    587\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m FunctionType(parameters, capture_types), type_context\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow/core/function/polymorphism/function_type.py:522\u001b[0m, in \u001b[0;36m_make_validated_mono_param\u001b[0;34m(name, value, kind, type_context, poly_type)\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_make_validated_mono_param\u001b[39m(\n\u001b[1;32m    519\u001b[0m     name, value, kind, type_context, poly_type\n\u001b[1;32m    520\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Parameter:\n\u001b[1;32m    521\u001b[0m \u001b[38;5;250m  \u001b[39m\u001b[38;5;124;03m\"\"\"Generates and validates a parameter for Monomorphic FunctionType.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 522\u001b[0m   mono_type \u001b[38;5;241m=\u001b[39m \u001b[43mtrace_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_value\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtype_context\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    524\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m poly_type \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m mono_type\u001b[38;5;241m.\u001b[39mis_subtype_of(poly_type):\n\u001b[1;32m    525\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mParameter `\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m` was expected to be of type \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    526\u001b[0m                     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpoly_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m but is \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmono_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/tensorflow/core/function/trace_type/trace_type_builder.py:185\u001b[0m, in \u001b[0;36mfrom_value\u001b[0;34m(value, context)\u001b[0m\n\u001b[1;32m    178\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m default_types\u001b[38;5;241m.\u001b[39mAttrs\u001b[38;5;241m.\u001b[39mfrom_type_and_attributes(\n\u001b[1;32m    179\u001b[0m       \u001b[38;5;28mtype\u001b[39m(value),\n\u001b[1;32m    180\u001b[0m       \u001b[38;5;28mtuple\u001b[39m(\n\u001b[1;32m    181\u001b[0m           from_value(\u001b[38;5;28mgetattr\u001b[39m(value, a\u001b[38;5;241m.\u001b[39mname), context)\n\u001b[1;32m    182\u001b[0m           \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m value\u001b[38;5;241m.\u001b[39m__attrs_attrs__))\n\u001b[1;32m    184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m util\u001b[38;5;241m.\u001b[39mis_np_ndarray(value):\n\u001b[0;32m--> 185\u001b[0m   ndarray \u001b[38;5;241m=\u001b[39m \u001b[43mvalue\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__array__\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    186\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m default_types\u001b[38;5;241m.\u001b[39mTENSOR(ndarray\u001b[38;5;241m.\u001b[39mshape, ndarray\u001b[38;5;241m.\u001b[39mdtype)\n\u001b[1;32m    188\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value, custom_nest_protocol\u001b[38;5;241m.\u001b[39mCustomNestProtocol):\n",
      "File \u001b[0;32m~/anaconda3/envs/DE/lib/python3.9/site-packages/keras/src/backend/common/keras_tensor.py:108\u001b[0m, in \u001b[0;36mKerasTensor.__array__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__array__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 108\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    109\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mA KerasTensor is symbolic: it\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms a placeholder for a shape \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    110\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124man a dtype. It doesn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt have any actual numerical value. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    111\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou cannot convert it to a NumPy array.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    112\u001b[0m     )\n",
      "\u001b[0;31mValueError\u001b[0m: Exception encountered when calling layer 'keras_layer_4' (type KerasLayer).\n\nA KerasTensor is symbolic: it's a placeholder for a shape an a dtype. It doesn't have any actual numerical value. You cannot convert it to a NumPy array.\n\nCall arguments received by layer 'keras_layer_4' (type KerasLayer):\n  • inputs=<KerasTensor shape=(None,), dtype=string, sparse=False, name=text>\n  • training=None"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "\n",
    "# Load the preprocessing and encoder layers from local directories\n",
    "preprocess_path = \"/Users/techgarage/Projects/Data Engineering/Final Project/bert-tensorflow2-en-uncased-preprocess-v3\"\n",
    "encoder_path = \"/Users/techgarage/Projects/Data Engineering/Final Project/bert-tensorflow2-en-uncased-l-12-h-768-a-12-v4\"\n",
    "\n",
    "# Load the BERT layers\n",
    "bert_preprocess = hub.KerasLayer(preprocess_path)\n",
    "bert_encoder = hub.KerasLayer(encoder_path)\n",
    "\n",
    "# Define the model architecture\n",
    "text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "preprocessed_text = bert_preprocess(text_input)\n",
    "encoded_output = bert_encoder(preprocessed_text)\n",
    "pooled_output = encoded_output['pooled_output']\n",
    "\n",
    "# Add classification head\n",
    "dropout = tf.keras.layers.Dropout(0.1, name=\"dropout\")(pooled_output)\n",
    "output = tf.keras.layers.Dense(1, activation='sigmoid', name=\"output\")(dropout)\n",
    "\n",
    "# Build the model\n",
    "model = tf.keras.Model(inputs=[text_input], outputs=[output])\n",
    "\n",
    "# Compile the model\n",
    "model.compile(\n",
    "    optimizer='adam',\n",
    "    loss='binary_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7233ee9f-f86e-46cb-91d6-c248bfa81ede",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_USE_LEGACY_KERAS']='1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57d3f18e-5cc2-463c-a17b-012712d63f32",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}