



"""simple docstring"""
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


A:     Dict										=	get_tests_dir("fixtures/test_sentencepiece.model")



@require_sentencepiece
@require_tokenizers
class    SCREAMING_SNAKE_CASE__						(      UpperCAmelCase__   ,    unittest.TestCase				):
   __lowerCAmelCase						:     str		=		XLMRobertaTokenizer
   __lowerCAmelCase						:     str		=		XLMRobertaTokenizerFast
   __lowerCAmelCase						:     int		=		True
   __lowerCAmelCase						:     Tuple		=		True

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Union[str, Any]:





    '''simple docstring'''




    super().setUp()

    # We have a SentencePiece fixture for testing
    UpperCAmelCase			:      Tuple			=			XLMRobertaTokenizer(_SCREAMING_SNAKE_CASE   ,  keep_accents=_SCREAMING_SNAKE_CASE		)
    tokenizer.save_pretrained(self.tmpdirname		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      Dict			=			"""<pad>"""
    UpperCAmelCase			:      List[str]			=			1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE		)   ,  _SCREAMING_SNAKE_CASE		)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE		)   ,  _SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Tuple:





    '''simple docstring'''




    UpperCAmelCase			:      str			=			list(self.get_tokenizer().get_vocab().keys()		)

    self.assertEqual(vocab_keys[0]   ,  """<s>"""		)
    self.assertEqual(vocab_keys[1]   ,  """<pad>"""		)
    self.assertEqual(vocab_keys[-1]   ,  """<mask>"""		)
    self.assertEqual(len(_SCREAMING_SNAKE_CASE		)   ,  1002		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    self.assertEqual(self.get_tokenizer().vocab_size   ,  1002		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    UpperCAmelCase			:      Union[str, Any]			=			XLMRobertaTokenizer(_SCREAMING_SNAKE_CASE   ,  keep_accents=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Optional[int]			=			tokenizer.tokenize("""This is a test"""		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]		)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)   ,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,  )

    UpperCAmelCase			:      Tuple			=			tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
    self.assertListEqual(
        _SCREAMING_SNAKE_CASE   ,  [
            SPIECE_UNDERLINE + """I""",
            SPIECE_UNDERLINE + """was""",
            SPIECE_UNDERLINE + """b""",
            """or""",
            """n""",
            SPIECE_UNDERLINE + """in""",
            SPIECE_UNDERLINE + """""",
            """9""",
            """2""",
            """0""",
            """0""",
            """0""",
            """,""",
            SPIECE_UNDERLINE + """and""",
            SPIECE_UNDERLINE + """this""",
            SPIECE_UNDERLINE + """is""",
            SPIECE_UNDERLINE + """f""",
            """al""",
            """s""",
            """é""",
            """.""",
        ]   ,  )
    UpperCAmelCase			:      int			=			tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(
        _SCREAMING_SNAKE_CASE   ,  [
            value + tokenizer.fairseq_offset
            for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
        ]   ,  )

    UpperCAmelCase			:      List[str]			=			tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(
        _SCREAMING_SNAKE_CASE   ,  [
            SPIECE_UNDERLINE + """I""",
            SPIECE_UNDERLINE + """was""",
            SPIECE_UNDERLINE + """b""",
            """or""",
            """n""",
            SPIECE_UNDERLINE + """in""",
            SPIECE_UNDERLINE + """""",
            """<unk>""",
            """2""",
            """0""",
            """0""",
            """0""",
            """,""",
            SPIECE_UNDERLINE + """and""",
            SPIECE_UNDERLINE + """this""",
            SPIECE_UNDERLINE + """is""",
            SPIECE_UNDERLINE + """f""",
            """al""",
            """s""",
            """<unk>""",
            """.""",
        ]   ,  )

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    if not self.test_slow_tokenizer:
     # as we don't have a slow version, we can't compare the outputs between slow and fast versions
     return

    UpperCAmelCase			:      Any			=			(self.rust_tokenizer_class, """hf-internal-testing/tiny-xlm-roberta""", {})
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"		):
      UpperCAmelCase			:      Tuple			=			self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,  **_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      Optional[Any]			=			self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,  **_SCREAMING_SNAKE_CASE		)

      UpperCAmelCase			:      Optional[Any]			=			tempfile.mkdtemp()

      UpperCAmelCase			:      Dict			=			tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      Dict			=			tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE		)

      # Checks it save with the same files + the tokenizer.json file for the fast one
      self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files		)		)
      UpperCAmelCase			:      Union[str, Any]			=			tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f		)
      self.assertSequenceEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

      # Checks everything loads correctly in the same way
      UpperCAmelCase			:      Dict			=			tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      List[Any]			=			tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE		)

      # Check special tokens are set accordingly on Rust and Python
      for key in tokenizer_pp.special_tokens_map:
       self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)		)
       # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
       # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

      shutil.rmtree(_SCREAMING_SNAKE_CASE		)

      # Save tokenizer rust, legacy_format=True
      UpperCAmelCase			:      str			=			tempfile.mkdtemp()

      UpperCAmelCase			:      Optional[Any]			=			tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE   ,  legacy_format=_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      List[str]			=			tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE		)

      # Checks it save with the same files
      self.assertSequenceEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

      # Checks everything loads correctly in the same way
      UpperCAmelCase			:      Tuple			=			tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      Dict			=			tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE		)

      # Check special tokens are set accordingly on Rust and Python
      for key in tokenizer_pp.special_tokens_map:
       self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)		)

      shutil.rmtree(_SCREAMING_SNAKE_CASE		)

      # Save tokenizer rust, legacy_format=False
      UpperCAmelCase			:      List[Any]			=			tempfile.mkdtemp()

      UpperCAmelCase			:      Optional[int]			=			tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE   ,  legacy_format=_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      Optional[int]			=			tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE		)

      # Checks it saved the tokenizer.json file
      self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files		)		)

      # Checks everything loads correctly in the same way
      UpperCAmelCase			:      str			=			tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE		)
      UpperCAmelCase			:      int			=			tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE		)

      # Check special tokens are set accordingly on Rust and Python
      for key in tokenizer_pp.special_tokens_map:
       self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)		)

      shutil.rmtree(_SCREAMING_SNAKE_CASE		)

   @cached_property
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Any:





    '''simple docstring'''




    return XLMRobertaTokenizer.from_pretrained("""xlm-roberta-base"""		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    with tempfile.NamedTemporaryFile() as f:
     shutil.copyfile(_SCREAMING_SNAKE_CASE   ,  f.name		)
     UpperCAmelCase			:      Union[str, Any]			=			XLMRobertaTokenizer(f.name   ,  keep_accents=_SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      Union[str, Any]			=			pickle.dumps(_SCREAMING_SNAKE_CASE		)
    pickle.loads(_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    if not self.test_rust_tokenizer:
     return

    UpperCAmelCase			:      Tuple			=			self.get_tokenizer()
    UpperCAmelCase			:      str			=			self.get_rust_tokenizer()

    UpperCAmelCase			:      List[Any]			=			"""I was born in 92000, and this is falsé."""

    UpperCAmelCase			:      Optional[int]			=			tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Any			=			rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Union[str, Any]			=			tokenizer.encode(_SCREAMING_SNAKE_CASE   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Any			=			rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      List[Any]			=			self.get_rust_tokenizer()
    UpperCAmelCase			:      Tuple			=			tokenizer.encode(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      int			=			rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

   @slow
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Union[str, Any]			=			"""Hello World!"""
    UpperCAmelCase			:      Tuple			=			[0, 35378, 6661, 38, 2]
    # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
    # xlmr.eval()
    # xlmr.encode(symbols)

    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE		)		)

   @slow
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			(
        """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
        """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
    )
    UpperCAmelCase			:      List[str]			=			[
        0,
        3293,
        83,
        10,
        4552,
        4989,
        7986,
        678,
        10,
        5915,
        111,
        179459,
        124850,
        4,
        6044,
        237,
        12,
        6,
        5,
        6,
        4,
        6780,
        705,
        15,
        1388,
        44,
        378,
        10114,
        711,
        152,
        20,
        6,
        5,
        22376,
        642,
        1221,
        15190,
        34153,
        450,
        5608,
        959,
        1119,
        57702,
        136,
        186,
        47,
        1098,
        29367,
        47,
        # 4426, # What fairseq tokenizes from "<unk>": "_<"
        # 3678, # What fairseq tokenizes from "<unk>": "unk"
        # 2740, # What fairseq tokenizes from "<unk>": ">"
        3,  # What we tokenize from "<unk>": "<unk>"
        6,  # Residue from the tokenization: an extra sentencepiece underline
        4,
        6044,
        237,
        6284,
        50901,
        528,
        31,
        90,
        34,
        927,
        2,
    ]
    # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
    # xlmr.eval()
    # xlmr.encode(symbols)

    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE		)		)




   @slow
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Any			=			{"""input_ids""": [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=_SCREAMING_SNAKE_CASE   ,  model_name="""xlm-roberta-base"""   ,  revision="""d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3"""   ,  )





"""simple docstring"""
import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class    SCREAMING_SNAKE_CASE__						(      UpperCAmelCase__   ,    unittest.TestCase				):
   __lowerCAmelCase						:     Any		=		XLMTokenizer
   __lowerCAmelCase						:     List[Any]		=		False

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    UpperCAmelCase			:      Optional[int]			=			[
        """l""",
        """o""",
        """w""",
        """e""",
        """r""",
        """s""",
        """t""",
        """i""",
        """d""",
        """n""",
        """w</w>""",
        """r</w>""",
        """t</w>""",
        """lo""",
        """low""",
        """er</w>""",
        """low</w>""",
        """lowest</w>""",
        """newer</w>""",
        """wider</w>""",
        """<unk>""",
    ]
    UpperCAmelCase			:      str			=			dict(zip(_SCREAMING_SNAKE_CASE   ,  range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
    UpperCAmelCase			:      List[Any]			=			["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

    UpperCAmelCase			:      Optional[Any]			=			os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["""vocab_file"""]		)
    UpperCAmelCase			:      Optional[Any]			=			os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["""merges_file"""]		)
    with open(self.vocab_file   ,  """w"""		) as fp:
     fp.write(json.dumps(_SCREAMING_SNAKE_CASE		)		)
    with open(self.merges_file   ,  """w"""		) as fp:
     fp.write("""\n""".join(_SCREAMING_SNAKE_CASE		)		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[int]			=			"""lower newer"""
    UpperCAmelCase			:      Dict			=			"""lower newer"""
    return input_text, output_text

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    UpperCAmelCase			:      Union[str, Any]			=			XLMTokenizer(self.vocab_file   ,  self.merges_file		)

    UpperCAmelCase			:      Optional[int]			=			"""lower"""
    UpperCAmelCase			:      Optional[int]			=			["""low""", """er</w>"""]
    UpperCAmelCase			:      List[Any]			=			tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Optional[int]			=			tokens + ["""<unk>"""]
    UpperCAmelCase			:      List[Any]			=			[14, 15, 20]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)   ,  _SCREAMING_SNAKE_CASE		)




   @slow
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			XLMTokenizer.from_pretrained("""xlm-mlm-en-2048"""		)

    UpperCAmelCase			:      int			=			tokenizer.encode("""sequence builders"""   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      List[Any]			=			tokenizer.encode("""multi-sequence build"""   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Tuple			=			tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      str			=			tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

    assert encoded_sentence == [0] + text + [1]
    assert encoded_pair == [0] + text + [1] + text_a + [1]
