# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.10 64-bit (''venv'': conda)'
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="ddAEv4mympEq" outputId="bd2a44da-50d1-44ae-be49-c7218e516bea"
import torch
from transformers import BertTokenizer
from transformers import BertModel
#pip install transformers
## Load pretrained model/tokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased',output_hidden_states=True)


# + id="g1_yw6Erm4LZ"
def GetBertEmbeddings(text):
  marked_text =  text
  tokenized_text = tokenizer.tokenize(marked_text)
  indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
  segments_ids = [1] * len(tokenized_text)
  # Convert inputs to PyTorch tensors
  tokens_tensor = torch.tensor([indexed_tokens])
# Load pre-trained model (weights)
  model = BertModel.from_pretrained('bert-base-uncased',output_hidden_states=True)
 
# Put the model in "evaluation" mode, meaning feed-forward operation.
  model.eval()
 #Run the text through BERT, get the output and collect all of the hidden states produced from all 12 layers.
 #Run the text through BERT, get the output and collect all of the hidden states produced from all 12 layers.
  with torch.no_grad():
    outputs = model(tokens_tensor)
# can use last hidden state as word embeddings
    last_hidden_state = outputs[0]
    word_embed_1 = last_hidden_state
# Evaluating the model will return a different number of objects               based on how it's  configured in the `from_pretrained` call earlier. In this case, becase we set `output_hidden_states = True`, the third item will be the hidden states from all layers. See the documentation for more details:https://huggingface.co/transformers/model_doc/bert.html#bertmodel
    hidden_states = outputs[2]
# initial embeddings can be taken from 0th layer of hidden states
    word_embed_2 = hidden_states[0]
# sum of all hidden states
    word_embed_3 = torch.stack(hidden_states).sum(0)
# sum of second to last layer
    word_embed_4 = torch.stack(hidden_states[2:]).sum(0)
# sum of last four layer
    word_embed_5 = torch.stack(hidden_states[-4:]).sum(0)
# concatenate last four layers
    word_embed_6 = torch.cat([hidden_states[i] for i in [-1,-2,-3,-4]], dim=-1)
  return word_embed_5
  
 

  


# + colab={"base_uri": "https://localhost:8080/"} id="KjEb75AlnD5v" outputId="27a68f16-ac54-481a-e91b-08ab06561c76"
GetBertEmbeddings("It is time to play valorant").shape

# + id="XBCLIi9onJpP"


StarcoderdataJupyterScriptsDedupFiltered