code: def check_diff(spm_diff, tok_diff, slow, fast):
      if spm_diff == list(reversed(tok_diff)):
          # AAA -> AA+A vs A+AA case.
          return True
      elif len(spm_diff) == len(tok_diff) and fast.decode(
          spm_diff
      ) == fast.decode(tok_diff):
          # Second order OK
          # Barrich -> Barr + ich vs Bar + rich
          return True
      spm_reencoded = slow.encode(slow.decode(spm_diff))
      tok_reencoded = fast.encode(fast.decode(spm_diff))
      if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
          # Type 3 error.
          # Snehagatha ->
          #       Sne, h, aga, th, a
          #       Sne, ha, gat, ha
          # Encoding the wrong with sp does not even recover what spm gave us
          # It fits tokenizer however...
          return True
      return False
  
file_path: transformers/scripts/check_tokenizers.py
imports: from collections import Counter
  import datasets
  import transformers
  from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
  from transformers.utils import logging
label_desc: Use image node_id 1 for calling a global function with example usage: check_diff(spm_diff, tok_diff, slow, fast) and returns: False, True, True, True