






"""simple docstring"""
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
 snake_case : Dict     		=		"""pt"""
elif is_tf_available():
 snake_case : List[str]     		=		"""tf"""
else:
 snake_case : Optional[int]     		=		"""jax"""



class        UpperCamelCase__ (				a_							,	unittest.TestCase):







    """simple docstring"""
    __UpperCAmelCase           = ByTaTokenizer
    __UpperCAmelCase           = False







    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           super().setUp()
           __magic_name__      				=    ByTaTokenizer()
           tokenizer.save_pretrained(self.tmpdirname )







    @cached_property
    def 				a__			(		self :	str ):





           '''simple docstring'''

           return ByTaTokenizer.from_pretrained('google/byt5-small' )







    def 				a__			(		self :	List[str]				,    **UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           return self.tokenizer_class.from_pretrained(self.tmpdirname				,    **UpperCamelCase_ )







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	Tuple=False				,    UpperCamelCase_ :	Union[str, Any]=2_0				,    UpperCamelCase_ :	Tuple=5 ):





           '''simple docstring'''

           __magic_name__      				=    []
           for i in range(len(UpperCamelCase_ ) ):
                  try:
                         __magic_name__      				=    tokenizer.decode([i]				,    clean_up_tokenization_spaces=UpperCamelCase_ )
                  except UnicodeDecodeError:
                         pass
                  toks.append((i, tok) )

           __magic_name__      				=    list(filter(lambda UpperCamelCase_ : re.match(r'^[ a-zA-Z]+$'				,    t[1] )				,    UpperCamelCase_ ) )
           __magic_name__      				=    list(filter(lambda UpperCamelCase_ : [t[0]] == tokenizer.encode(t[1]				,    add_special_tokens=UpperCamelCase_ )				,    UpperCamelCase_ ) )
           if max_length is not None and len(UpperCamelCase_ ) > max_length:
                  __magic_name__      				=    toks[:max_length]
           if min_length is not None and len(UpperCamelCase_ ) < min_length and len(UpperCamelCase_ ) > 0:
                  while len(UpperCamelCase_ ) < min_length:
                         __magic_name__      				=    toks + toks
        # toks_str = [t[1] for t in toks]
           __magic_name__      				=    [t[0] for t in toks]

           # Ensure consistency
           __magic_name__      				=    tokenizer.decode(UpperCamelCase_				,    clean_up_tokenization_spaces=UpperCamelCase_ )
           if " " not in output_txt and len(UpperCamelCase_ ) > 1:
                  __magic_name__      				=    (
                      tokenizer.decode([toks_ids[0]]				,    clean_up_tokenization_spaces=UpperCamelCase_ )
                      + ' '
                      + tokenizer.decode(toks_ids[1:]				,    clean_up_tokenization_spaces=UpperCamelCase_ )
                  )
           if with_prefix_space:
                  __magic_name__      				=    ' ' + output_txt
           __magic_name__      				=    tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
           return output_txt, output_ids







    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           __magic_name__      				=    self.ta_base_tokenizer
           __magic_name__      				=    tokenizer(['hi</s>', 'I went to the gym</s>', '</s>'] )
           __magic_name__      				=    tokenizer(['hi', 'I went to the gym', ''] )
           self.assertListEqual(batch_with_eos_added['input_ids']				,    batch_without_eos_added['input_ids'] )







    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           __magic_name__      				=    self.ta_base_tokenizer
           __magic_name__      				=    'Unicode €.'
           __magic_name__      				=    tokenizer(UpperCamelCase_ )
           __magic_name__      				=    [8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
           self.assertEqual(encoded['input_ids']				,    UpperCamelCase_ )

           # decoding
           __magic_name__      				=    tokenizer.decode(UpperCamelCase_ )
           self.assertEqual(UpperCamelCase_				,    'Unicode €.</s>' )

           __magic_name__      				=    tokenizer('e è é ê ë' )
           __magic_name__      				=    [1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
           self.assertEqual(encoded['input_ids']				,    UpperCamelCase_ )
           # decoding
           __magic_name__      				=    tokenizer.decode(UpperCamelCase_ )
           self.assertEqual(UpperCamelCase_				,    'e è é ê ë</s>' )

           # encode/decode, but with `encode` instead of `__call__`
           self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë' ) )				,    'e è é ê ë</s>' )







    def 				a__			(		self :	Tuple ):





           '''simple docstring'''

           __magic_name__      				=    self.ta_base_tokenizer
           __magic_name__      				=    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
           # fmt: off
           __magic_name__      				=    [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
           # fmt: on
           __magic_name__      				=    tokenizer(UpperCamelCase_				,    padding=UpperCamelCase_				,    return_tensors=UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

           if FRAMEWORK != "jax":
                  __magic_name__      				=    list(batch.input_ids.numpy()[0] )
           else:
                  __magic_name__      				=    list(batch.input_ids.tolist()[0] )

           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )

           self.assertEqual((2, 3_7)				,    batch.input_ids.shape )
           self.assertEqual((2, 3_7)				,    batch.attention_mask.shape )







    def 				a__			(		self :	List[Any] ):





           '''simple docstring'''

           __magic_name__      				=    self.ta_base_tokenizer
           __magic_name__      				=    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
           __magic_name__      				=    tokenizer(UpperCamelCase_				,    padding=UpperCamelCase_				,    return_tensors=UpperCamelCase_ )
           # check if input_ids are returned and no decoder_input_ids
           self.assertIn('input_ids'				,    UpperCamelCase_ )
           self.assertIn('attention_mask'				,    UpperCamelCase_ )
           self.assertNotIn('decoder_input_ids'				,    UpperCamelCase_ )
           self.assertNotIn('decoder_attention_mask'				,    UpperCamelCase_ )







    def 				a__			(		self :	List[str] ):





           '''simple docstring'''

           __magic_name__      				=    self.ta_base_tokenizer
           __magic_name__      				=    [
               'Summary of the text.',
               'Another summary.',
           ]
           __magic_name__      				=    tokenizer(
               text_target=UpperCamelCase_				,    max_length=3_2				,    padding='max_length'				,    truncation=UpperCamelCase_				,    return_tensors=UpperCamelCase_ )
           self.assertEqual(3_2				,    targets['input_ids'].shape[1] )







    def 				a__			(		self :	int ):





           '''simple docstring'''

           __magic_name__      				=    self.ta_base_tokenizer
           __magic_name__      				=    ['A long paragraph for summarization. </s>']
           __magic_name__      				=    ['Summary of the text. </s>']
           # fmt: off
           __magic_name__      				=    [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
           __magic_name__      				=    [8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
           # fmt: on

           __magic_name__      				=    tokenizer(UpperCamelCase_				,    text_target=UpperCamelCase_ )

           self.assertEqual(UpperCamelCase_				,    batch['input_ids'][0] )
           self.assertEqual(UpperCamelCase_				,    batch['labels'][0] )







    def 				a__			(		self :	Any ):





           '''simple docstring'''

           __magic_name__      				=    self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
                         self.assertNotEqual(tokenizer.model_max_length				,    4_2 )

        # Now let's start the test
           __magic_name__      				=    self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         __magic_name__      				=    tempfile.mkdtemp()

                         __magic_name__      				=    ' He is very happy, UNwant\u00E9d,running'
                         __magic_name__      				=    tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
                         tokenizer.save_pretrained(UpperCamelCase_ )

                         __magic_name__      				=    tokenizer.__class__.from_pretrained(UpperCamelCase_ )
                         __magic_name__      				=    after_tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
                         self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )

                         shutil.rmtree(UpperCamelCase_ )

           __magic_name__      				=    self.get_tokenizers(model_max_length=4_2 )
           for tokenizer in tokenizers:
                  with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         __magic_name__      				=    tempfile.mkdtemp()

                         __magic_name__      				=    ' He is very happy, UNwant\u00E9d,running'
                         tokenizer.add_tokens(['bim', 'bambam'] )
                         __magic_name__      				=    tokenizer.additional_special_tokens
                         additional_special_tokens.append('new_additional_special_token' )
                         tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens} )
                         __magic_name__      				=    tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
                         tokenizer.save_pretrained(UpperCamelCase_ )

                         __magic_name__      				=    tokenizer.__class__.from_pretrained(UpperCamelCase_ )
                         __magic_name__      				=    after_tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
                         self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )
                         self.assertIn('new_additional_special_token'				,    after_tokenizer.additional_special_tokens )
                         self.assertEqual(after_tokenizer.model_max_length				,    4_2 )

                         __magic_name__      				=    tokenizer.__class__.from_pretrained(UpperCamelCase_				,    model_max_length=4_3 )
                         self.assertEqual(tokenizer.model_max_length				,    4_3 )

                         shutil.rmtree(UpperCamelCase_ )







    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           __magic_name__      				=    []
           if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

           if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(UpperCamelCase_ )

                         with open(os.path.join(UpperCamelCase_				,    'special_tokens_map.json' )				,    encoding='utf-8' ) as json_file:
                                __magic_name__      				=    json.load(UpperCamelCase_ )

                         with open(os.path.join(UpperCamelCase_				,    'tokenizer_config.json' )				,    encoding='utf-8' ) as json_file:
                                __magic_name__      				=    json.load(UpperCamelCase_ )

                         __magic_name__      				=    [f"""<extra_id_{i}>""" for i in range(1_2_5 )]

                         __magic_name__      				=    added_tokens_extra_ids + [
                             'an_additional_special_token'
                         ]
                         __magic_name__      				=    added_tokens_extra_ids + [
                             'an_additional_special_token'
                         ]

                         with open(os.path.join(UpperCamelCase_				,    'special_tokens_map.json' )				,    'w'				,    encoding='utf-8' ) as outfile:
                                json.dump(UpperCamelCase_				,    UpperCamelCase_ )
                         with open(os.path.join(UpperCamelCase_				,    'tokenizer_config.json' )				,    'w'				,    encoding='utf-8' ) as outfile:
                                json.dump(UpperCamelCase_				,    UpperCamelCase_ )

                         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                         # "special_tokens_map.json" files
                         __magic_name__      				=    tokenizer_class.from_pretrained(
                             UpperCamelCase_				,    )
                         self.assertIn(
                             'an_additional_special_token'				,    tokenizer_without_change_in_init.additional_special_tokens )
                         # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                         self.assertEqual(
                             ['an_additional_special_token']				,    tokenizer_without_change_in_init.convert_ids_to_tokens(
                                 tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token'] ) )				,    )

                         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                         __magic_name__      				=    added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'				,    lstrip=UpperCamelCase_ )]
                         __magic_name__      				=    tokenizer_class.from_pretrained(
                             UpperCamelCase_				,    additional_special_tokens=UpperCamelCase_				,    )

                         self.assertIn('a_new_additional_special_token'				,    tokenizer.additional_special_tokens )
                         self.assertEqual(
                             ['a_new_additional_special_token']				,    tokenizer.convert_ids_to_tokens(
                                 tokenizer.convert_tokens_to_ids(['a_new_additional_special_token'] ) )				,    )







    def 				a__			(		self :	int ):





           '''simple docstring'''

           __magic_name__      				=    []
           if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

           if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(UpperCamelCase_ )

                         __magic_name__      				=    tokenizer_class.from_pretrained(UpperCamelCase_ )

                         self.assertTrue(tokenizer.decode([2_5_5] ) == '' )







    def 				a__			(		self :	Union[str, Any] ):





           '''simple docstring'''

           pass







    def 				a__			(		self :	Tuple ):





           '''simple docstring'''

           pass







    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           pass







    def 				a__			(		self :	List[Any] ):





           '''simple docstring'''

           pass







    def 				a__			(		self :	Union[str, Any] ):





           '''simple docstring'''

           __magic_name__      				=    self.get_tokenizers(fast=UpperCamelCase_				,    do_lower_case=UpperCamelCase_ )
           for tokenizer in tokenizers:
                  with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
                         __magic_name__      				=    ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                         __magic_name__      				=    tokenizer.convert_tokens_to_string(UpperCamelCase_ )

                         self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    def 				a__			(		self :	Any ):





           '''simple docstring'''

           __magic_name__      				=    self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f"""{tokenizer.__class__.__name__}""" ):
                         __magic_name__      				=    [
                             'bos_token',
                             'eos_token',
                             'unk_token',
                             'sep_token',
                             'pad_token',
                             'cls_token',
                             'mask_token',
                         ]

                         __magic_name__      				=    0
                         __magic_name__      				=    tokenizer.convert_ids_to_tokens(
                             UpperCamelCase_				,    skip_special_tokens=UpperCamelCase_ )

                         for attr in attributes_list:
                                setattr(UpperCamelCase_				,    attr + '_id'				,    UpperCamelCase_ )
                                self.assertEqual(getattr(UpperCamelCase_				,    UpperCamelCase_ )				,    UpperCamelCase_ )
                                self.assertEqual(getattr(UpperCamelCase_				,    attr + '_id' )				,    UpperCamelCase_ )

                                setattr(UpperCamelCase_				,    attr + '_id'				,    UpperCamelCase_ )
                                self.assertEqual(getattr(UpperCamelCase_				,    UpperCamelCase_ )				,    UpperCamelCase_ )
                                self.assertEqual(getattr(UpperCamelCase_				,    attr + '_id' )				,    UpperCamelCase_ )

                         setattr(UpperCamelCase_				,    'additional_special_tokens_ids'				,    [] )
                         self.assertListEqual(getattr(UpperCamelCase_				,    'additional_special_tokens' )				,    [] )
                         self.assertListEqual(getattr(UpperCamelCase_				,    'additional_special_tokens_ids' )				,    [] )

                         setattr(UpperCamelCase_				,    'additional_special_tokens_ids'				,    [token_id_to_test_setters] )
                         self.assertListEqual(getattr(UpperCamelCase_				,    'additional_special_tokens' )				,    [token_to_test_setters] )
                         self.assertListEqual(getattr(UpperCamelCase_				,    'additional_special_tokens_ids' )				,    [token_id_to_test_setters] )







"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


snake_case : Optional[Any]     		=		logging.get_logger(__name__)

snake_case : str     		=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case : List[Any]     		=		{
    """vocab_file""": {"""mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"""},
    """tokenizer_file""": {
        """mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"""
    },
}

snake_case : Tuple     		=		{"""mobilebert-uncased""": 5_1_2}


snake_case : List[str]     		=		{}



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = VOCAB_FILES_NAMES
    __UpperCAmelCase           = PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase           = PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase           = MobileBertTokenizer







    def __init__(		self :	Optional[int]				,    UpperCamelCase_ :	Optional[int]=None				,    UpperCamelCase_ :	str=None				,    UpperCamelCase_ :	Tuple=True				,    UpperCamelCase_ :	Any="[UNK]"				,    UpperCamelCase_ :	str="[SEP]"				,    UpperCamelCase_ :	Any="[PAD]"				,    UpperCamelCase_ :	Dict="[CLS]"				,    UpperCamelCase_ :	str="[MASK]"				,    UpperCamelCase_ :	Union[str, Any]=True				,    UpperCamelCase_ :	Dict=None				,    **UpperCamelCase_ :	Dict				,    ):





           '''simple docstring'''

           super().__init__(
               UpperCamelCase_				,    tokenizer_file=UpperCamelCase_				,    do_lower_case=UpperCamelCase_				,    unk_token=UpperCamelCase_				,    sep_token=UpperCamelCase_				,    pad_token=UpperCamelCase_				,    cls_token=UpperCamelCase_				,    mask_token=UpperCamelCase_				,    tokenize_chinese_chars=UpperCamelCase_				,    strip_accents=UpperCamelCase_				,    **UpperCamelCase_				,    )

           __magic_name__      				=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'				,    UpperCamelCase_ ) != do_lower_case
               or normalizer_state.get('strip_accents'				,    UpperCamelCase_ ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,    UpperCamelCase_ ) != tokenize_chinese_chars
           ):
                  __magic_name__      				=    getattr(UpperCamelCase_				,    normalizer_state.pop('type' ) )
                  __magic_name__      				=    do_lower_case
                  __magic_name__      				=    strip_accents
                  __magic_name__      				=    tokenize_chinese_chars
                  __magic_name__      				=    normalizer_class(**UpperCamelCase_ )

           __magic_name__      				=    do_lower_case







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	List[Any]=None ):





           '''simple docstring'''

           __magic_name__      				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[int]				,    UpperCamelCase_ :	Optional[List[int]] = None ):





           '''simple docstring'''

           __magic_name__      				=    [self.sep_token_id]
           __magic_name__      				=    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Optional[str] = None ):





           '''simple docstring'''

           __magic_name__      				=    self._tokenizer.model.save(UpperCamelCase_				,    name=UpperCamelCase_ )
           return tuple(UpperCamelCase_ )