class_docstr: A class that regroups important test to make sure that we properly handle the special tokens.
class_docstr_tok: ["A","class","that","regroups","important","test","to","make","sure","that","we","properly","handle","the","special","tokens","."]
code: def test_special_tokens_strip(self):
      input_ids = self.tokenizer.encode(" <s> ,")
      self.assertEqual(input_ids, [1, 7, 3])
      tokens = self.tokenizer.tokenize(" <s> ,")
      # spaces are eaten by rstrip / lstrip + spm sp_model.encode("  ") = []
      self.assertEqual(tokens, ["<s>", "▁", ","])
  
      input_ids = self.tokenizer.encode("No <s> ▁He")
      self.assertEqual(input_ids, [284, 1, 156])
      tokens = self.tokenizer.tokenize("No <s> ▁He")
      self.assertEqual(tokens, ["▁No", "<s>", "▁He"])
  
imports: import os
  import pickle
  import shutil
  import tempfile
  import unittest
  from datasets import load_dataset
  from transformers import SPIECE_UNDERLINE, AddedToken, LlamaTokenizer, LlamaTokenizerFast, is_torch_available
  from transformers.convert_slow_tokenizer import convert_slow_tokenizer
  from transformers.testing_utils import get_tests_dir, nested_simplify, require_jinja, require_sentencepiece, require_tokenizers, require_torch, slow
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 5 for calling the CommonSpmIntegrationTests obj's underlying member method code with example usage: obj.test_special_tokens_strip() without return types