class_docstr: A modified wrapper for a public python BPE tokenizer. The modifications allow encoding directly into the formats
  required for finetune. Particularly with respect to formatting with multiple inputs.
class_docstr_tok: ["A","modified","wrapper","for","a","public","python","BPE","tokenizer",".","The","modifications","allow","encoding","directly","into","the","formats","required","for","finetune",".","Particularly","with","respect","to","formatting","with","multiple","inputs","."]
docstr: Convert a batch of raw text to a batch of byte-pair encoded token indices.
docstr_tok: ["Convert","a","batch","of","raw","text","to","a","batch","of","byte-pair","encoded","token","indices","."]
code: def _encode(self, texts):
      """
      Convert a batch of raw text to a batch of byte-pair encoded token indices.
      """
      nlp = get_spacy()
  
      batch_tokens = []
      batch_token_idxs = []
      batch_char_ends = []
      # to account for the fact that some BPEs have different lengths than their original tokens
      # (e.g. special characters such as bullets)
  
      batch_char_starts = []
      for i, text in enumerate(texts):
          raw_text = text.lower()
  
          # Only fine to apply this fix because it preserves character locations
          ftfy_text = uncurl_quotes(raw_text)
          tokens = nlp(_text_standardize(text))
          subtokens = []
          subtoken_idxs = []
          char_starts = []
          char_ends = []
          token_start = 0
  
          for j, token in enumerate(tokens):
              bpe_toks = self.bpe(token.text).split(" ")
  
              try:
                  stripped_text = token.text.strip()
                  if stripped_text:
                      token_start = ftfy_text.index(
                          stripped_text, token_start
                      )
              except ValueError:
                  warnings.warn(
                      "Failed to find token `{}` in text.".format(
                          token.text
                      )
                  )
                  continue
  
              subtokens.extend(bpe_toks)
              subtoken_idxs.extend(
                  [
                      self.encoder.get(SUBS.get(t, t), self.UNK_IDX)
                      for t in bpe_toks
                  ]
              )
  
              assert len("".join(bpe_toks).replace("</w>", "")) == len(
                  token.text.replace(" ", "")
              )
  
              token_char_ends = (
                  np.cumsum(
                      [
                          len(tok.strip().replace("</w>", ""))
                          for tok in bpe_toks
                      ]
                  )
                  + token_start
              )
              token_char_starts = [token_start] + token_char_ends[
                  :-1
              ].tolist()
              token_start += len(token.text.strip())
              char_ends.extend(token_char_ends)
              char_starts.extend(token_char_starts)
  
          batch_tokens.append(subtokens)
          batch_token_idxs.append(subtoken_idxs)
          batch_char_ends.append(char_ends)
          batch_char_starts.append(char_starts)
  
      return EncodedOutput(
          token_ids=batch_token_idxs,
          tokens=batch_tokens,
          token_ends=batch_char_ends,
          token_starts=batch_char_starts,
      )
  
file_path: finetune/finetune/base_models/gpt/encoder.py
imports: import re
  import json
  import os
  import warnings
  import codecs
  import numpy
  from ftfy.fixes import uncurl_quotes
  import finetune
  from finetune.encoding.input_encoder import get_spacy, EncodedOutput, BaseEncoder, get_pairs
label_desc: Use image node_id 4 for calling the GPTEncoder obj's underlying member method code with example usage: obj._encode(texts) and returns: EncodedOutput