code: def extract_bert_embeddings(
      model_name,
      tokenizer,
      model,
      data,
      device,
      keep_endpoints,
      num_layers=None,
      detach=True,
  ):
      """
      Extract transformer embeddings using a generic roberta extraction
  
      data: list of list of string (the text tokens)
      num_layers: how many to return.  If None, the average of -2, -3, -4 is returned
      """
      if model_name.startswith("vinai/phobert"):
          return extract_phobert_embeddings(
              model_name,
              tokenizer,
              model,
              data,
              device,
              keep_endpoints,
              num_layers,
              detach,
          )
  
      if "bart" in model_name:
          # this should work with "vinai/bartpho-word"
          # not sure this works with any other Bart
          return extract_bart_word_embeddings(
              model_name,
              tokenizer,
              model,
              data,
              device,
              keep_endpoints,
              num_layers,
              detach,
          )
  
      if isinstance(data, tuple):
          data = list(data)
  
      if "xlnet" in model_name:
          return extract_xlnet_embeddings(
              model_name,
              tokenizer,
              model,
              data,
              device,
              keep_endpoints,
              num_layers,
              detach,
          )
  
      data = fix_blank_tokens(tokenizer, data)
  
      # add add_prefix_space = True for RoBerTa-- error if not
      # using attention masks makes contextual embeddings much more useful for downstream tasks
      tokenized = tokenizer(
          data,
          padding="longest",
          is_split_into_words=True,
          return_offsets_mapping=False,
          return_attention_mask=True,
      )
      list_offsets = [[None] * (len(sentence) + 2) for sentence in data]
      for idx in range(len(data)):
          offsets = tokenized.word_ids(batch_index=idx)
          for pos, offset in enumerate(offsets):
              if offset is None:
                  continue
              # this uses the last token piece for any offset by overwriting the previous value
              list_offsets[idx][offset + 1] = pos
          list_offsets[idx][0] = 0
          list_offsets[idx][-1] = list_offsets[idx][-2] + 1
          # print(list_offsets[idx])
          if any(x is None for x in list_offsets[idx]):
              raise ValueError(
                  "OOPS, hit None when preparing to use Bert\ndata[idx]: {}\noffsets: {}\nlist_offsets[idx]: {}".format(
                      data[idx], offsets, list_offsets[idx], tokenized
                  )
              )
  
          if len(offsets) > tokenizer.model_max_length - 2:
              logger.error(
                  "Invalid size, max size: %d, got %d.\nTokens: %s\nTokenized: %s",
                  tokenizer.model_max_length,
                  len(offsets),
                  data[idx],
                  offsets,
              )
              raise TextTooLongError(
                  len(offsets),
                  tokenizer.model_max_length,
                  idx,
                  " ".join(data[idx]),
              )
  
      features = []
      for i in range(int(math.ceil(len(data) / 128))):
          if detach:
              with torch.no_grad():
                  attention_mask = torch.tensor(
                      tokenized["attention_mask"][
                          128 * i : 128 * i + 128
                      ],
                      device=device,
                  )
                  id_tensor = torch.tensor(
                      tokenized["input_ids"][128 * i : 128 * i + 128],
                      device=device,
                  )
                  feature = model(
                      id_tensor,
                      attention_mask=attention_mask,
                      output_hidden_states=True,
                  )
                  features += cloned_feature(
                      feature.hidden_states, num_layers, detach
                  )
          else:
              attention_mask = torch.tensor(
                  tokenized["attention_mask"][128 * i : 128 * i + 128],
                  device=device,
              )
              id_tensor = torch.tensor(
                  tokenized["input_ids"][128 * i : 128 * i + 128],
                  device=device,
              )
              feature = model(
                  id_tensor,
                  attention_mask=attention_mask,
                  output_hidden_states=True,
              )
              features += cloned_feature(
                  feature.hidden_states, num_layers, detach
              )
  
      processed = []
      # process the output
      if not keep_endpoints:
          # remove the bos and eos tokens
          list_offsets = [sent[1:-1] for sent in list_offsets]
      for feature, offsets in zip(features, list_offsets):
          new_sent = feature[offsets]
          processed.append(new_sent)
  
      return processed
  
imports: import math
  import logging
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence, pack_sequence, PackedSequence
label_desc: Use image node_id 11 for calling a global function with example usage: extract_bert_embeddings(model_name, tokenizer, model, data, device, keep_endpoints, num_layers, detach) and returns: processed, extract_phobert_embeddings, extract_bart_word_embeddings, extract_xlnet_embeddings