






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class        lowerCAmelCase_				(      lowercase						,      unittest.TestCase	):







				"""simple docstring"""




				_snake_case				:   Tuple        =			BertTokenizer
				_snake_case				:   str        =			BertTokenizerFast
				_snake_case				:   Optional[Any]        =			True
				_snake_case				:   Optional[int]        =			True
				_snake_case				:   Dict        =			filter_non_english







				def 				__a   ( self     :List[str]     ):
					super().setUp()

					UpperCamelCase__ :str						=      [
					    """[UNK]""",
					    """[CLS]""",
					    """[SEP]""",
					    """[PAD]""",
					    """[MASK]""",
					    """want""",
					    """##want""",
					    """##ed""",
					    """wa""",
					    """un""",
					    """runn""",
					    """##ing""",
					    """,""",
					    """low""",
					    """lowest""",
					]
					UpperCamelCase__ :List[str]						=      os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["""vocab_file"""]     )
					with open(self.vocab_file  ,     """w"""  ,     encoding="""utf-8"""     ) as vocab_writer:
						vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )







				def 				__a   ( self     :Tuple  ,     lowerCamelCase__     :Dict     ):
					UpperCamelCase__ :Tuple						=      """UNwant\u00E9d,running"""
					UpperCamelCase__ :Tuple						=      """unwanted, running"""
					return input_text, output_text







				def 				__a   ( self     :int     ):
					UpperCamelCase__ :Optional[Any]						=      self.tokenizer_class(self.vocab_file     )

					UpperCamelCase__ :Union[str, Any]						=      tokenizer.tokenize("""UNwant\u00E9d,running"""     )
					self.assertListEqual(lowerCamelCase__  ,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]     )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__     )  ,     [9, 6, 7, 12, 10, 11]     )







				def 				__a   ( self     :Dict     ):
					if not self.test_rust_tokenizer:
						return

					UpperCamelCase__ :Tuple						=      self.get_tokenizer()
					UpperCamelCase__ :Dict						=      self.get_rust_tokenizer()

					UpperCamelCase__ :str						=      """UNwant\u00E9d,running"""

					UpperCamelCase__ :Optional[int]						=      tokenizer.tokenize(lowerCamelCase__     )
					UpperCamelCase__ :List[str]						=      rust_tokenizer.tokenize(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :Union[str, Any]						=      tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					UpperCamelCase__ :int						=      rust_tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :List[Any]						=      self.get_rust_tokenizer()
					UpperCamelCase__ :str						=      tokenizer.encode(lowerCamelCase__     )
					UpperCamelCase__ :Any						=      rust_tokenizer.encode(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					# With lower casing
					UpperCamelCase__ :Dict						=      self.get_tokenizer(do_lower_case=lowerCamelCase__     )
					UpperCamelCase__ :List[Any]						=      self.get_rust_tokenizer(do_lower_case=lowerCamelCase__     )

					UpperCamelCase__ :List[str]						=      """UNwant\u00E9d,running"""

					UpperCamelCase__ :List[Any]						=      tokenizer.tokenize(lowerCamelCase__     )
					UpperCamelCase__ :Union[str, Any]						=      rust_tokenizer.tokenize(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :Dict						=      tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					UpperCamelCase__ :Union[str, Any]						=      rust_tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :Union[str, Any]						=      self.get_rust_tokenizer()
					UpperCamelCase__ :Optional[Any]						=      tokenizer.encode(lowerCamelCase__     )
					UpperCamelCase__ :Any						=      rust_tokenizer.encode(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :str						=      BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )  ,     ["""ah""", """\u535A""", """\u63A8""", """zz"""]     )







				def 				__a   ( self     :Union[str, Any]     ):
					UpperCamelCase__ :str						=      BasicTokenizer(do_lower_case=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )  ,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""hello"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :Optional[int]						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""h\u00E9llo"""]     )







				def 				__a   ( self     :Optional[Any]     ):
					UpperCamelCase__ :Any						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""hello"""]     )







				def 				__a   ( self     :int     ):
					UpperCamelCase__ :int						=      BasicTokenizer(do_lower_case=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""hello"""]     )







				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :Optional[Any]						=      BasicTokenizer(do_lower_case=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )  ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )







				def 				__a   ( self     :List[Any]     ):
					UpperCamelCase__ :List[str]						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )







				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :List[Any]						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :str						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     never_split=["""[UNK]"""]     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )  ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :List[str]						=      BasicTokenizer()
					UpperCamelCase__ :List[str]						=      """a\n'll !!to?'d of, can't."""
					UpperCamelCase__ :Dict						=      ["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
					self.assertListEqual(tokenizer.tokenize(lowerCamelCase__     )  ,     lowerCamelCase__     )







				def 				__a   ( self     :Optional[Any]     ):
					UpperCamelCase__ :str						=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

					UpperCamelCase__ :Optional[Any]						=      {}
					for i, token in enumerate(lowerCamelCase__     ):
						UpperCamelCase__ :List[Any]						=      i
					UpperCamelCase__ :Optional[int]						=      WordpieceTokenizer(vocab=lowerCamelCase__  ,     unk_token="""[UNK]"""     )

					self.assertListEqual(tokenizer.tokenize(""""""     )  ,     []     )

					self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )  ,     ["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

					self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )  ,     ["""[UNK]""", """runn""", """##ing"""]     )







				def 				__a   ( self     :Optional[int]     ):
					self.assertTrue(_is_whitespace(""" """     )     )
					self.assertTrue(_is_whitespace("""\t"""     )     )
					self.assertTrue(_is_whitespace("""\r"""     )     )
					self.assertTrue(_is_whitespace("""\n"""     )     )
					self.assertTrue(_is_whitespace("""\u00A0"""     )     )

					self.assertFalse(_is_whitespace("""A"""     )     )
					self.assertFalse(_is_whitespace("""-"""     )     )







				def 				__a   ( self     :Optional[int]     ):
					self.assertTrue(_is_control("""\u0005"""     )     )

					self.assertFalse(_is_control("""A"""     )     )
					self.assertFalse(_is_control(""" """     )     )
					self.assertFalse(_is_control("""\t"""     )     )
					self.assertFalse(_is_control("""\r"""     )     )







				def 				__a   ( self     :List[Any]     ):
					self.assertTrue(_is_punctuation("""-"""     )     )
					self.assertTrue(_is_punctuation("""$"""     )     )
					self.assertTrue(_is_punctuation("""`"""     )     )
					self.assertTrue(_is_punctuation("""."""     )     )

					self.assertFalse(_is_punctuation("""A"""     )     )
					self.assertFalse(_is_punctuation(""" """     )     )







				def 				__a   ( self     :Tuple     ):
					UpperCamelCase__ :List[str]						=      self.get_tokenizer()
					UpperCamelCase__ :Dict						=      self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(lowerCamelCase__     ) for t in ["""Test""", """\xad""", """test"""]]  ,     [["""[UNK]"""], [], ["""[UNK]"""]]     )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(lowerCamelCase__     ) for t in ["""Test""", """\xad""", """test"""]]  ,     [["""[UNK]"""], [], ["""[UNK]"""]]     )







				@slow
				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :Optional[int]						=      self.tokenizer_class.from_pretrained("""bert-base-uncased"""     )

					UpperCamelCase__ :str						=      tokenizer.encode("""sequence builders"""  ,     add_special_tokens=lowerCamelCase__     )
					UpperCamelCase__ :int						=      tokenizer.encode("""multi-sequence build"""  ,     add_special_tokens=lowerCamelCase__     )

					UpperCamelCase__ :List[str]						=      tokenizer.build_inputs_with_special_tokens(lowerCamelCase__     )
					UpperCamelCase__ :Dict						=      tokenizer.build_inputs_with_special_tokens(lowerCamelCase__  ,     lowerCamelCase__     )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







				def 				__a   ( self     :int     ):
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
							UpperCamelCase__ :Any						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )

							UpperCamelCase__ :Optional[int]						=      f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
							UpperCamelCase__ :Any						=      tokenizer_r.encode_plus(
							    lowerCamelCase__  ,     return_attention_mask=lowerCamelCase__  ,     return_token_type_ids=lowerCamelCase__  ,     return_offsets_mapping=lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__  ,     )

							UpperCamelCase__ :Optional[Any]						=      tokenizer_r.do_lower_case if hasattr(lowerCamelCase__  ,     """do_lower_case"""     ) else False
							UpperCamelCase__ :Dict						=      (
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), """A"""),
							        ((1, 2), ""","""),
							        ((3, 5), """na"""),
							        ((5, 6), """##ï"""),
							        ((6, 8), """##ve"""),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), """Allen"""),
							        ((21, 23), """##NL"""),
							        ((23, 24), """##P"""),
							        ((25, 33), """sentence"""),
							        ((33, 34), """."""),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), """a"""),
							        ((1, 2), ""","""),
							        ((3, 8), """naive"""),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), """allen"""),
							        ((21, 23), """##nl"""),
							        ((23, 24), """##p"""),
							        ((25, 33), """sentence"""),
							        ((33, 34), """."""),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results]  ,     tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
							self.assertEqual([e[0] for e in expected_results]  ,     tokens["""offset_mapping"""]     )







				def 				__a   ( self     :Dict     ):
					UpperCamelCase__ :Any						=      ["""的""", """人""", """有"""]
					UpperCamelCase__ :Optional[Any]						=      """""".join(lowerCamelCase__     )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
							UpperCamelCase__ :str						=      True
							UpperCamelCase__ :int						=      self.tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )
							UpperCamelCase__ :List[str]						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )

							UpperCamelCase__ :Tuple						=      tokenizer_p.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
							UpperCamelCase__ :Optional[int]						=      tokenizer_r.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )

							UpperCamelCase__ :List[str]						=      tokenizer_r.convert_ids_to_tokens(lowerCamelCase__     )
							UpperCamelCase__ :List[Any]						=      tokenizer_p.convert_ids_to_tokens(lowerCamelCase__     )

							# it is expected that each Chinese character is not preceded by "##"
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

							UpperCamelCase__ :List[Any]						=      False
							UpperCamelCase__ :str						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )
							UpperCamelCase__ :int						=      self.tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )

							UpperCamelCase__ :Optional[int]						=      tokenizer_r.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
							UpperCamelCase__ :str						=      tokenizer_p.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )

							UpperCamelCase__ :Union[str, Any]						=      tokenizer_r.convert_ids_to_tokens(lowerCamelCase__     )
							UpperCamelCase__ :Dict						=      tokenizer_p.convert_ids_to_tokens(lowerCamelCase__     )

							# it is expected that only the first Chinese character is not preceded by "##".
							UpperCamelCase__ :Any						=      [
							    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__     )
							]
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )




'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


SCREAMING_SNAKE_CASE__     :						List[str]	=  logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
SCREAMING_SNAKE_CASE__     :						Dict	=  {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

SCREAMING_SNAKE_CASE__     :						Optional[int]	=  {
    '''facebook/bart-base''': 1_0_2_4,
    '''facebook/bart-large''': 1_0_2_4,
    '''facebook/bart-large-mnli''': 1_0_2_4,
    '''facebook/bart-large-cnn''': 1_0_2_4,
    '''facebook/bart-large-xsum''': 1_0_2_4,
    '''yjernite/bart_eli5''': 1_0_2_4,
}

class 							a__( snake_case__     ):
			a_	: Dict									=      VOCAB_FILES_NAMES
			a_	: Tuple									=      PRETRAINED_VOCAB_FILES_MAP
			a_	: Tuple									=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a_	: List[Any]									=      ['''input_ids''', '''attention_mask''']
			a_	: Union[str, Any]									=      BartTokenizer






			def __init__(     self					,   _UpperCAmelCase=None					,   _UpperCAmelCase=None					,   _UpperCAmelCase=None					,   _UpperCAmelCase="replace"					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="<unk>"					,   _UpperCAmelCase="<pad>"					,   _UpperCAmelCase="<mask>"					,   _UpperCAmelCase=False					,   _UpperCAmelCase=True					,   **_UpperCAmelCase					,   )							->							int:
					super().__init__(
					    _UpperCAmelCase					,   _UpperCAmelCase					,   tokenizer_file=_UpperCAmelCase					,   errors=_UpperCAmelCase					,   bos_token=_UpperCAmelCase					,   eos_token=_UpperCAmelCase					,   sep_token=_UpperCAmelCase					,   cls_token=_UpperCAmelCase					,   unk_token=_UpperCAmelCase					,   pad_token=_UpperCAmelCase					,   mask_token=_UpperCAmelCase					,   add_prefix_space=_UpperCAmelCase					,   trim_offsets=_UpperCAmelCase					,   **_UpperCAmelCase					,   )

					snake_case__	     =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
					if pre_tok_state.get('add_prefix_space'					,   _UpperCAmelCase						) != add_prefix_space:
							snake_case__	     =getattr(_UpperCAmelCase					,   pre_tok_state.pop('type'						)						)
							snake_case__	     =add_prefix_space
							snake_case__	     =pre_tok_class(**_UpperCAmelCase						)

					snake_case__	     =add_prefix_space

					# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
					snake_case__	     ='post_processor'
					snake_case__	     =getattr(self.backend_tokenizer					,   _UpperCAmelCase					,   _UpperCAmelCase						)
					if tokenizer_component_instance:
							snake_case__	     =json.loads(tokenizer_component_instance.__getstate__()						)

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
									snake_case__	     =tuple(state['sep']						)
							if "cls" in state:
									snake_case__	     =tuple(state['cls']						)

							snake_case__	     =False

							if state.get('add_prefix_space'					,   _UpperCAmelCase						) != add_prefix_space:
									snake_case__	     =add_prefix_space
									snake_case__	     =True

							if state.get('trim_offsets'					,   _UpperCAmelCase						) != trim_offsets:
									snake_case__	     =trim_offsets
									snake_case__	     =True

							if changes_to_apply:
									snake_case__	     =getattr(_UpperCAmelCase					,   state.pop('type'						)						)
									snake_case__	     =component_class(**_UpperCAmelCase						)
									setattr(self.backend_tokenizer					,   _UpperCAmelCase					,   _UpperCAmelCase						)






			@property
			def 						_lowercase  (     self						)							->							str:
					if self._mask_token is None:
							if self.verbose:
									logger.error('Using mask_token, but it is not set yet.'						)
							return None
					return str(self._mask_token						)






			@mask_token.setter
			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							int:
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else value
					snake_case__	     =value






			def 						_lowercase  (     self					,   *_UpperCAmelCase					,   **_UpperCAmelCase						)							->							BatchEncoding:
					snake_case__	     =kwargs.get('is_split_into_words'					,   _UpperCAmelCase						)

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
							    'to use it with pretokenized inputs.'						)

					return super()._batch_encode_plus(*_UpperCAmelCase					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   *_UpperCAmelCase					,   **_UpperCAmelCase						)							->							BatchEncoding:
					snake_case__	     =kwargs.get('is_split_into_words'					,   _UpperCAmelCase						)

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
							    'to use it with pretokenized inputs.'						)

					return super()._encode_plus(*_UpperCAmelCase					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							Tuple[str]:
					snake_case__	     =self._tokenizer.model.save(_UpperCAmelCase					,   name=_UpperCAmelCase						)
					return tuple(_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase=None						)							->							Optional[int]:
					snake_case__	     =[self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
							return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							List[int]:
					snake_case__	     =[self.sep_token_id]
					snake_case__	     =[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]




