class_docstr: A class that regroups important test to make sure that we properly handle the special tokens.
class_docstr_tok: ["A","class","that","regroups","important","test","to","make","sure","that","we","properly","handle","the","special","tokens","."]
code: def test_special_tokens_strip(self):
      input_ids = self.tokenizer.encode(" <extra_id_0> ,")
      self.assertEqual(input_ids, [1001, 7, 3, 2])
      tokens = self.tokenizer.tokenize(" <extra_id_0> ,")
      # spaces are not longer eaten by rstrip and lstrip
      self.assertEqual(tokens, ["<extra_id_0>", "▁", ","])
  
      # test with a begin of word like `▁He`
      input_ids = self.tokenizer.encode("No <extra_id_0> He")
      self.assertEqual(input_ids, [284, 1001, 156, 2])
      # spaces are eaten by rstrip / lstrip, so this is expected. Don't strip otherwise you break
      tokens = self.tokenizer.tokenize("No <extra_id_0> He")
      self.assertEqual(tokens, ["▁No", "<extra_id_0>", "▁He"])
  
      # Make sure this does not happen if we don't strip
      tokenizer = T5Tokenizer(SAMPLE_VOCAB, extra_ids=0)
      tokenizer.add_special_tokens({"bos_token": AddedToken("<bos>")})
      input_ids = tokenizer.encode("No <bos> He")
      self.assertEqual(input_ids, [284, 1001, 156, 2])
      tokens = tokenizer.tokenize("No <bos> He")
      # the first `' '` after `'No'` is eaten by spm:
      self.assertEqual(
          tokenizer.sp_model.encode("No         ", out_type=str),
          ["▁No"],
      )
      self.assertEqual(tokens, ["▁No", "<bos>", "▁He"])
  
imports: import json
  import os
  import re
  import tempfile
  import unittest
  from transformers import SPIECE_UNDERLINE, AddedToken, BatchEncoding, T5Tokenizer, T5TokenizerFast
  from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_seqio, require_tokenizers, slow
  from transformers.utils import cached_property, is_tf_available, is_torch_available
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 5 for calling the CommonSpmIntegrationTests obj's underlying member method code with example usage: obj.test_special_tokens_strip() without return types