code: def test_all_tokenizer_on_special_cases(caplog):
      caplog.set_level(logging.CRITICAL)
  
      lang_names = [
          "bert-base-cased",
          "roberta-base",
          "xlnet-base-cased",
      ]
  
      tokenizers = []
      for lang_name in lang_names:
          if "roberta" in lang_name:
              add_prefix_space = True
          else:
              add_prefix_space = False
          t = Tokenizer.load(
              lang_name,
              lower_case=False,
              add_prefix_space=add_prefix_space,
          )
          tokenizers.append(t)
  
      texts = [
          "This is a sentence",
          "Der entscheidende Pass",
          "力加勝北区ᴵᴺᵀᵃছজটডণত",
          "Thiso text is included tolod makelio sure Unicodeel is handled properly:",
          "This is a sentence...",
          "Let's see all on this text and. !23# neverseenwordspossible"
          "This      is a sentence with multiple spaces",
          """This is a sentence.
        With linebreak""",
          """Sentence with multiple
  
  
        newlines
        """,
          "and another one\n\n\nwithout space",
          "This is a sentence			with multiple tabs",
      ]
  
      expected_to_fail = [
          (1, 1),
          (1, 3),
          (1, 4),
          (1, 5),
          (1, 6),
          (1, 7),
          (2, 1),
          (2, 5),
      ]
  
      for i_tok, tokenizer in enumerate(tokenizers):
          for i_text, text in enumerate(texts):
              # Important: we don't assume to preserve whitespaces after tokenization.
              # This means: \t, \n " " etc will all resolve to a single " ".
              # This doesn't make a difference for BERT + XLNet but it does for roBERTa
  
              test_passed = True
  
              # 1. original tokenize function from transformer repo on full sentence
              standardized_whitespace_text = " ".join(
                  text.split()
              )  # remove multiple whitespaces
              tokenized = tokenizer.tokenize(
                  standardized_whitespace_text
              )
  
              # 2. Our tokenization method using a pretokenizer which can normalize multiple white spaces
              # This approach is used in NER
              pre_tokenizer = WhitespaceSplit()
              words_and_spans = pre_tokenizer.pre_tokenize_str(text)
              words = [x[0] for x in words_and_spans]
              word_spans = [x[1] for x in words_and_spans]
  
              encoded = tokenizer.encode_plus(
                  words,
                  is_split_into_words=True,
                  add_special_tokens=False,
              ).encodings[0]
  
              # verify that tokenization on full sequence is the same as the one on "whitespace tokenized words"
              if encoded.tokens != tokenized:
                  test_passed = False
  
              # token offsets are originally relative to the beginning of the word
              # These lines convert them so they are relative to the beginning of the sentence
              token_offsets = []
              for (
                  (start, end),
                  w_index,
              ) in zip(encoded.offsets, encoded.words):
                  word_start_ch = word_spans[w_index][0]
                  token_offsets.append(
                      (start + word_start_ch, end + word_start_ch)
                  )
              if getattr(tokenizer, "add_prefix_space", None):
                  token_offsets = [
                      (start - 1, end) for start, end in token_offsets
                  ]
  
              # verify that offsets align back to original text
              if text == "力加勝北区ᴵᴺᵀᵃছজটডণত":
                  # contains [UNK] that are impossible to match back to original text space
                  continue
              for tok, (start, end) in zip(
                  encoded.tokens, token_offsets
              ):
                  # subword-tokens have special chars depending on model type. In order to align with original text we need to get rid of them
                  tok = re.sub(r"^(##|Ġ|▁)", "", tok)
                  # tok = tokenizer.decode(tokenizer.convert_tokens_to_ids(tok))
                  original_tok = text[start:end]
                  if tok != original_tok:
                      test_passed = False
              if (i_tok, i_text) in expected_to_fail:
                  assert (
                      not test_passed
                  ), f"Behaviour of {tokenizer.__class__.__name__} has changed on text {text}'"
              else:
                  assert (
                      test_passed
                  ), f"Behaviour of {tokenizer.__class__.__name__} has changed on text {text}'"
  
file_path: FARM/test/test_tokenization.py
imports: import logging
  import pytest
  import re
  from transformers import BertTokenizer, BertTokenizerFast, RobertaTokenizer, RobertaTokenizerFast, XLNetTokenizer, XLNetTokenizerFast, ElectraTokenizerFast
  from tokenizers.pre_tokenizers import WhitespaceSplit
  from farm.modeling.tokenization import Tokenizer, tokenize_with_metadata, truncate_sequences
  import numpy
label_desc: Use image node_id 6 for calling a global function with example usage: test_all_tokenizer_on_special_cases(caplog) without return types