text: import re
  import ftfy
  import json
  import spacy
  import torch
  
  from tqdm import tqdm
  
  
  def load_existing_data_loader(data_loader, path):
      old_data_loader = torch.load(path)
      for attr in data_loader.__dict__.keys():
          if attr not in old_data_loader.__dict__.keys():
              continue
          setattr(data_loader, attr, getattr(old_data_loader, attr))
  
  
  ################################################################################
  #
  # Code Below taken from HuggingFace pytorch-openai-lm repository
  #
  ################################################################################
  
  
  def get_pairs(word):
      """
      Return set of symbol pairs in a word.
      word is represented as tuple of symbols (symbols being variable-length strings)
      """
      pairs = set()
      prev_char = word[0]
      for char in word[1:]:
          pairs.add((prev_char, char))
          prev_char = char
      return pairs
  
  
  def text_standardize(text):
      """
      fixes some issues the spacy tokenizer had on books corpus
      also does some whitespace standardization
      """
      text = text.replace("—", "-")
      text = text.replace("–", "-")
      text = text.replace("―", "-")
      text = text.replace("…", "...")
      text = text.replace("´", "'")
      text = re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)', r" \1 ", text)
      text = re.sub(r"\s*\n\s*", " \n ", text)
      text = re.sub(r"[^\S\n]+", " ", text)
      return text.strip()
  
  
  class TextEncoder(object):
      """
      mostly a wrapper for a public python bpe tokenizer
      """
  
      def __init__(self, encoder_path, bpe_path):
          self.nlp = spacy.load("en_core_web_sm", disable=["parser", "tagger", "ner", "textcat", "lemmatizer"])
          self.encoder = json.load(open(encoder_path))
          self.decoder = {v: k for k, v in self.encoder.items()}
          merges = open(bpe_path, encoding="utf-8").read().split("\n")[1:-1]
          merges = [tuple(merge.split()) for merge in merges]
          self.bpe_ranks = dict(zip(merges, range(len(merges))))
          self.cache = {}
  
      def bpe(self, token):
          word = tuple(token[:-1]) + (token[-1] + "</w>",)
          if token in self.cache:
              return self.cache[token]
          pairs = get_pairs(word)
  
          if not pairs:
              return token + "</w>"
  
          while True:
              bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
              if bigram not in self.bpe_ranks:
                  break
              first, second = bigram
              new_word = []
              i = 0
              while i < len(word):
                  try:
                      j = word.index(first, i)
                      new_word.extend(word[i:j])
                      i = j
                  except ValueError:
                      new_word.extend(word[i:])
                      break
  
                  if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                      new_word.append(first + second)
                      i += 2
                  else:
                      new_word.append(word[i])
                      i += 1
              new_word = tuple(new_word)
              word = new_word
              if len(word) == 1:
                  break
              else:
                  pairs = get_pairs(word)
          word = " ".join(word)
          if word == "\n  </w>":
              word = "\n</w>"
          self.cache[token] = word
          return word
  
      def encode(self, texts, verbose=True):
          texts_tokens = []
          if verbose:
              texts = tqdm(texts, ncols=80, leave=False)
          for text in texts:
              text = self.nlp(text_standardize(ftfy.fix_text(text)))
              text_tokens = []
              for token in text:
                  text_tokens.extend([self.encoder.get(t, 0) for t in self.bpe(token.text.lower()).split(" ")])
              texts_tokens.append(text_tokens)
          return texts_tokens
  