code: def extract_phobert_embeddings(
      model_name,
      tokenizer,
      model,
      data,
      device,
      keep_endpoints,
      num_layers,
      detach=True,
  ):
      """
      Extract transformer embeddings using a method specifically for phobert
  
      Since phobert doesn't have the is_split_into_words / tokenized.word_ids(batch_index=0)
      capability, we instead look for @@ to denote a continued token.
      data: list of list of string (the text tokens)
      """
      processed = (
          []
      )  # final product, returns the list of list of word representation
      tokenized_sents = (
          []
      )  # list of sentences, each is a torch tensor with start and end token
      list_tokenized = []  # list of tokenized sentences from phobert
      for idx, sent in enumerate(data):
          tokenized, tokenized_sent = tokenize_manual(
              model_name, sent, tokenizer
          )
  
          # add tokenized to list_tokenzied for later checking
          list_tokenized.append(tokenized)
  
          if len(tokenized_sent) > tokenizer.model_max_length:
              logger.error(
                  "Invalid size, max size: %d, got %d %s",
                  tokenizer.model_max_length,
                  len(tokenized_sent),
                  data[idx],
              )
              raise TextTooLongError(
                  len(tokenized_sent),
                  tokenizer.model_max_length,
                  idx,
                  " ".join(data[idx]),
              )
  
          # add to tokenized_sents
          tokenized_sents.append(torch.tensor(tokenized_sent).detach())
  
          processed_sent = []
          processed.append(processed_sent)
  
          # done loading bert emb
  
      size = len(tokenized_sents)
  
      # padding the inputs
      tokenized_sents_padded = torch.nn.utils.rnn.pad_sequence(
          tokenized_sents,
          batch_first=True,
          padding_value=tokenizer.pad_token_id,
      )
  
      features = []
  
      # Feed into PhoBERT 128 at a time in a batch fashion. In testing, the loop was
      # run only 1 time as the batch size for the outer model was less than that
      # (30 for conparser, for example)
      for i in range(int(math.ceil(size / 128))):
          padded_input = tokenized_sents_padded[128 * i : 128 * i + 128]
          start_sentence = i * 128
          end_sentence = start_sentence + padded_input.shape[0]
          attention_mask = torch.zeros(
              end_sentence - start_sentence,
              padded_input.shape[1],
              device=device,
          )
          for sent_idx, sent in enumerate(
              tokenized_sents[start_sentence:end_sentence]
          ):
              attention_mask[sent_idx, : len(sent)] = 1
          if detach:
              with torch.no_grad():
                  # TODO: is the clone().detach() necessary?
                  feature = model(
                      padded_input.clone().detach().to(device),
                      attention_mask=attention_mask,
                      output_hidden_states=True,
                  )
                  features += cloned_feature(
                      feature.hidden_states, num_layers, detach
                  )
          else:
              feature = model(
                  padded_input.to(device),
                  attention_mask=attention_mask,
                  output_hidden_states=True,
              )
              features += cloned_feature(
                  feature.hidden_states, num_layers, detach
              )
  
      assert len(features) == size
      assert len(features) == len(processed)
  
      # process the output
      # only take the vector of the last word piece of a word/ you can do other methods such as first word piece or averaging.
      # idx2+1 compensates for the start token at the start of a sentence
      offsets = [
          [
              idx2 + 1
              for idx2, _ in enumerate(list_tokenized[idx])
              if (
                  idx2 > 0
                  and not list_tokenized[idx][idx2 - 1].endswith("@@")
              )
              or (idx2 == 0)
          ]
          for idx, sent in enumerate(processed)
      ]
      if keep_endpoints:
          # [0] and [-1] grab the start and end representations as well
          offsets = [[0] + off + [-1] for off in offsets]
      processed = [
          feature[offset] for feature, offset in zip(features, offsets)
      ]
  
      # This is a list of tensors
      # Each tensor holds the representation of a sentence extracted from phobert
      return processed
  
imports: import math
  import logging
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence, pack_sequence, PackedSequence
label_desc: Use image node_id 8 for calling a global function with example usage: extract_phobert_embeddings(model_name, tokenizer, model, data, device, keep_endpoints, num_layers, detach) and returns: processed