class_docstr: A class that regroups important test to make sure that we properly handle the special tokens.
class_docstr_tok: ["A","class","that","regroups","important","test","to","make","sure","that","we","properly","handle","the","special","tokens","."]
code: def test_integration_seqio(self):
      from datasets import load_dataset
      from seqio import SentencePieceVocabulary
  
      ds = load_dataset(
          "xnli", "all_languages", split="train+test+validation"
      )
  
      # TODO @ArthurZucker fix the 3 commented tests with #23909
      input_texts = [
          "Bonjour <extra_id_0>.",
          # "Bonjour<extra_id_0>.",  # this will fail. In T5 the special token has to be at the end.
          # because in T5 they add `_<extra_id_0>` to the vocab, not `<extra_id_0>`.
          "                   Hey <extra_id_0>I love you",
          # "Hey <extra_id_0> I love you", # this will fail, we strip left, to _I vs I
          # "Hey <extra_id_0>▁He", # this will fail for the same reason, we replace `_` then strip
      ]
  
      import tqdm
  
      # Test with umt5
      vocab_path = "gs://t5-data/vocabs/umt5.256000/sentencepiece.model"
      t5x_tokenizer = SentencePieceVocabulary(vocab_path, extra_ids=300)
      hf_tokenizer = T5Tokenizer.from_pretrained(
          "google/umt5-small", legacy=False
      )
      for text in input_texts:
          self.assertEqual(
              hf_tokenizer.encode(text, add_special_tokens=False),
              t5x_tokenizer.tokenizer.tokenize(text),
              f"{text}",
          )
      for texts in tqdm.tqdm(ds["premise"]):
          for text in texts:
              self.assertEqual(
                  hf_tokenizer.encode(text, add_special_tokens=False),
                  t5x_tokenizer.tokenizer.tokenize(text),
                  f"{text}",
              )
  
      # Test with T5
      hf_tokenizer = T5Tokenizer.from_pretrained("t5-small")
      vocab_path = (
          "gs://t5-data/vocabs/cc_all.32000/sentencepiece.model"
      )
      t5x_tokenizer = SentencePieceVocabulary(vocab_path, extra_ids=300)
      for text in input_texts:
          self.assertEqual(
              hf_tokenizer.encode(text, add_special_tokens=False),
              t5x_tokenizer.tokenizer.tokenize(text),
              f"{text}",
          )
      for texts in tqdm.tqdm(ds["premise"]):
          for text in texts:
              self.assertEqual(
                  hf_tokenizer.encode(text, add_special_tokens=False),
                  t5x_tokenizer.tokenizer.tokenize(text),
                  f"{text}",
              )
  
imports: import json
  import os
  import re
  import tempfile
  import unittest
  from transformers import SPIECE_UNDERLINE, AddedToken, BatchEncoding, T5Tokenizer, T5TokenizerFast
  from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_seqio, require_tokenizers, slow
  from transformers.utils import cached_property, is_tf_available, is_torch_available
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 6 for calling the CommonSpmIntegrationTests obj's underlying member method code with example usage: obj.test_integration_seqio() without return types