class_docstr: Convert a collection of raw documents to a document id matrix.
  
  Attributes:
      _use_char: boolean. Whether to use char feature.
      _num_norm: boolean. Whether to normalize text.
      _word_vocab: dict. A mapping of words to feature indices.
      _char_vocab: dict. A mapping of chars to feature indices.
      _label_vocab: dict. A mapping of labels to feature indices.
class_docstr_tok: ["Convert","a","collection","of","raw","documents","to","a","document","id","matrix",".","Attributes",":","_use_char",":","boolean",".","Whether","to","use","char","feature",".","_num_norm",":","boolean",".","Whether","to","normalize","text",".","_word_vocab",":","dict",".","A","mapping","of","words","to","feature","indices",".","_char_vocab",":","dict",".","A","mapping","of","chars","to","feature","indices",".","_label_vocab",":","dict",".","A","mapping","of","labels","to","feature","indices","."]
docstr: Should be called prior training
docstr_tok: ["Should","be","called","prior","training"]
code: def fix_tokenization(
      self,
      X,
      Y,
      maxlen=U.DEFAULT_TRANSFORMER_MAXLEN,
      num_special=U.DEFAULT_TRANSFORMER_NUM_SPECIAL,
  ):
      """
      Should be called prior training
      """
      if not self.transformer_is_activated():
          return X, Y
      ids2tok = self.te.tokenizer.convert_ids_to_tokens
      encode = self.te.tokenizer.encode_plus
      new_X = []
      new_Y = []
      for i, x in enumerate(X):
          new_x = []
          new_y = []
          seq_len = 0
          for j, s in enumerate(x):
              # subtokens = ids2tok(encode(s, add_special_tokens=False))
              encoded_input = encode(
                  s,
                  add_special_tokens=False,
                  return_offsets_mapping=True,
              )
              offsets = encoded_input["offset_mapping"]
              subtokens = encoded_input.tokens()
              token_len = len(subtokens)
              if (seq_len + token_len) > (maxlen - num_special):
                  break
              seq_len += token_len
  
              if len(s.split()) == 1:
                  hf_s = [s]
              else:
                  word_ids = encoded_input.word_ids()
                  hf_s = []
                  for k, subtoken in enumerate(subtokens):
                      word_id = word_ids[k]
                      currlen = len(hf_s)
                      if currlen == word_id + 1:
                          hf_s[word_id].append(offsets[k])
                      elif word_id + 1 > currlen:
                          hf_s.append([offsets[k]])
                  hf_s = [
                      s[entry[0][0] : entry[-1][1]] for entry in hf_s
                  ]
  
              new_x.extend(hf_s)
              if Y is not None:
                  tag = Y[i][j]
                  new_y.extend([tag])
                  if len(hf_s) > 1:
                      new_tag = tag
                      if tag.startswith("B-"):
                          new_tag = "I-" + tag[2:]
                      new_y.extend([new_tag] * (len(hf_s) - 1))
          new_X.append(new_x)
          new_Y.append(new_y)
      new_Y = None if Y is None else new_Y
      return new_X, new_Y
  
imports: from ....None import utils
  from ....imports import *
  from .utils import Vocabulary
label_desc: Use image node_id 9 for calling the IndexTransformer obj's underlying member method code with example usage: obj.fix_tokenization(X, Y, maxlen, num_special) and returns: new_X,  new_Y, X,  Y