class_docstr: A modified wrapper for a public python BPE tokenizer. The modifications allow encoding directly into the formats
  required for finetune. Particularly with respect to formatting with multiple inputs.
class_docstr_tok: ["A","modified","wrapper","for","a","public","python","BPE","tokenizer",".","The","modifications","allow","encoding","directly","into","the","formats","required","for","finetune",".","Particularly","with","respect","to","formatting","with","multiple","inputs","."]
docstr: Convert a sample of raw text to a list of byte-pair encoded token indices.
docstr_tok: ["Convert","a","sample","of","raw","text","to","a","list","of","byte-pair","encoded","token","indices","."]
code: def _encode(self, texts):
      """
      Convert a sample of raw text to a list of byte-pair encoded token indices.
      """
      batch_tokens = []
      batch_token_idxs = []
      batch_char_ends = []
      # to account for the fact that some BPEs have different lengths than their original tokens
      # (e.g. special characters such as bullets)
      batch_char_starts = []
  
      for i, text in enumerate(texts):  # text = one label span
          subtokens = []
          subtoken_idxs = []
          char_ends = []
          char_starts = []
          token_start = 0
  
          tokens = re.findall(self.pat, text)
          for j, token in enumerate(tokens):
              encoded_token = "".join(
                  self.byte_encoder[b] for b in token.encode("utf-8")
              )
              bpe_toks = self.bpe(encoded_token).split(" ")
              decoded_bpe_toks = self._decode_token(bpe_toks)
              try:
                  if token.strip():
                      token_start = text.index(
                          token.strip(), token_start
                      )
              except ValueError:
                  # text_standardization oddity
                  traceback.print_exc()
                  continue
  
              subtokens.extend(decoded_bpe_toks)
              subtoken_idxs.extend(
                  [self.encoder.get(t, self.UNK_IDX) for t in bpe_toks]
              )
              lens = [None for _ in bpe_toks]
  
              for i, tok in enumerate(decoded_bpe_toks):
                  lens[i] = len(tok.strip())
  
              token_char_ends = np.cumsum(lens) + token_start
              token_char_starts = [token_start] + token_char_ends[
                  :-1
              ].tolist()
              token_start += len(token.strip())
              char_ends.extend(token_char_ends)
              char_starts.extend(token_char_starts)
  
          batch_tokens.append(subtokens)
  
          for k in range(len(subtoken_idxs)):
              subtoken_idxs[k] = self._convert_to_embed_idx(
                  subtoken_idxs[k]
              )
  
          batch_token_idxs.append(subtoken_idxs)
          batch_char_ends.append(char_ends)
          batch_char_starts.append(char_starts)
      return EncodedOutput(
          token_ids=batch_token_idxs,
          tokens=batch_tokens,
          token_ends=batch_char_ends,
          token_starts=batch_char_starts,
      )
  
file_path: finetune/finetune/base_models/gpt2/encoder.py
imports: import os
  import json
  import traceback
  import regex
  from functools import lru_cache
  import numpy
  import finetune
  from finetune.encoding.input_encoder import BaseEncoder, EncodedOutput, get_pairs
label_desc: Use image node_id 7 for calling the GPT2Encoder obj's underlying member method code with example usage: obj._encode(texts) and returns: EncodedOutput