text: import six
  from fastai.text.all import *
  
  
  class ByteTextTokenizer(Transform):
      """
          Encodes each byte to an id. For 8-bit strings only.
          Credit: https://github.com/tensorflow/tensor2tensor/blob/5f9dd2db6d7797162e53adf152310ed13e9fc711/tensor2tensor/data_generators/text_encoder.py#L176
      """
      def __init__(self, is_lm=True, add_bos=False, add_eos=False):
          self.ls_lm = is_lm
          self.add_bos, self.add_eos = add_bos, add_eos
          self.pad_token, self.eos_token, self.bos_token = '<pad>', '<eos>', '<bos>',
          self.pad_token_id, self.eos_token_id, self.bos_token_id = 0,1,2
          self.reserved_toks = [self.pad_token, self.eos_token, self.bos_token]  ## self.bos_token_id 
          self.reserved_tokens_bytes = [bytes(rtok, 'ascii') for rtok in self.reserved_toks]
          self.numres = len(self.reserved_toks)
          self.int2byte = six.int2byte
  
      @typedispatch
      def __call__(self, o:list, **kwargs):
          out = [c + self.numres for s in o for c in s.encode("utf-8")]
          if self.add_bos: out = [self.bos_token_id] + out
          if self.add_eos: out =  out + [self.eos_token_id]
          if self.ls_lm:return LMTensorText(out)
          else: return TensorText(out) 
          
      @typedispatch
      def __call__(self, o:str, **kwargs):
          out = [c + self.numres for c in o.encode("utf-8")]
          if self.add_bos: out = [self.bos_token_id] + out
          if self.add_eos: out =  out + [self.eos_token_id]
          if self.ls_lm:return LMTensorText(out)
          else: return TensorText(out) 
      
      def encodes(self,o):
          return self.__call__(o)
      
      def decodes(self, o:tuple):
          decoded_ids = ()
          for i in o:
              tmp_ls=[]
              for id_ in i:
                  if 0 <= id_ < self.numres: tmp_ls.append(self.reserved_tokens_bytes[int(id_)])
                  else: tmp_ls.append(self.int2byte(id_ - self.numres))
              decoded_ids = decoded_ids + (b"".join(tmp_ls).decode("utf-8", "replace"),)
          return TitledStr(decoded_ids)
      
      def decodes(self, o:list):
          decoded_ids = []
          for id_ in o:
              if 0 <= id_ < self.numres: decoded_ids.append(self.reserved_tokens_bytes[int(id_)])
              else: decoded_ids.append(self.int2byte(id_ - self.numres))
          return TitledStr(b"".join(decoded_ids).decode("utf-8", "replace"))
      
      def decodes(self, o:TensorText):
          return self.decodes(o.tolist())
      
      def decodes(self, o:LMTensorText):
          return self.decodes(o.tolist())
      
      @property
      def vocab_size(self): return 2**8 + self.numres