


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
 lowerCamelCase		   =			'pt'
elif is_tf_available():
 lowerCamelCase		   =			'tf'
else:
 lowerCamelCase		   =			'jax'




class A    (  UpperCamelCase_       ,				unittest.TestCase	):
      UpperCamelCase__				:    Optional[Any] =ByTaTokenizer
      UpperCamelCase__				:    Tuple =False






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Dict:
             """simple docstring"""




             super().setUp()
             _lowerCamelCase			:						int            =ByTaTokenizer()
             tokenizer.save_pretrained(self.tmpdirname					)






      @cached_property
      def 			lowerCamelCase			(   self		:	Tuple					)   ->		Optional[int]:
             """simple docstring"""




             return ByTaTokenizer.from_pretrained('google/byt5-small'					)






      def 			lowerCamelCase			(   self		:	str     ,  **lowercase_		:	Dict					)   ->		ByTaTokenizer:
             """simple docstring"""




             return self.tokenizer_class.from_pretrained(self.tmpdirname     ,  **lowercase_					)






      def 			lowerCamelCase			(   self		:	str     ,  lowercase_		:	List[str]     ,  lowercase_		:	Any=False     ,  lowercase_		:	List[str]=20     ,  lowercase_		:	Union[str, Any]=5					)   ->		Tuple[str, list]:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =[]
             for i in range(len(lowercase_					)					):
                    try:
                           _lowerCamelCase			:						Optional[Any]            =tokenizer.decode([i]     ,  clean_up_tokenization_spaces=lowercase_					)
                    except UnicodeDecodeError:
                           pass
                    toks.append((i, tok)					)

             _lowerCamelCase			:						int            =list(filter(lambda lowercase_					: re.match(R'^[ a-zA-Z]+$'     ,  t[1]					)     ,  lowercase_					)					)
             _lowerCamelCase			:						List[str]            =list(filter(lambda lowercase_					: [t[0]] == tokenizer.encode(t[1]     ,  add_special_tokens=lowercase_					)     ,  lowercase_					)					)
             if max_length is not None and len(lowercase_					) > max_length:
                    _lowerCamelCase			:						List[str]            =toks[:max_length]
             if min_length is not None and len(lowercase_					) < min_length and len(lowercase_					) > 0:
                    while len(lowercase_					) < min_length:
                           _lowerCamelCase			:						Optional[int]            =toks + toks
        # toks_str = [t[1] for t in toks]
             _lowerCamelCase			:						List[Any]            =[t[0] for t in toks]

             # Ensure consistency
             _lowerCamelCase			:						Optional[int]            =tokenizer.decode(lowercase_     ,  clean_up_tokenization_spaces=lowercase_					)
             if " " not in output_txt and len(lowercase_					) > 1:
                    _lowerCamelCase			:						Optional[Any]            =(
                        tokenizer.decode([toks_ids[0]]     ,  clean_up_tokenization_spaces=lowercase_					)
                        + ' '
                        + tokenizer.decode(toks_ids[1:]     ,  clean_up_tokenization_spaces=lowercase_					)
                    )
             if with_prefix_space:
                    _lowerCamelCase			:						Dict            =' ' + output_txt
             _lowerCamelCase			:						Optional[Any]            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             return output_txt, output_ids






      def 			lowerCamelCase			(   self		:	int					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            =self.ta_base_tokenizer
             _lowerCamelCase			:						List[Any]            =tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']					)
             _lowerCamelCase			:						str            =tokenizer(['hi', 'I went to the gym', '']					)
             self.assertListEqual(batch_with_eos_added['input_ids']     ,  batch_without_eos_added['input_ids']					)






      def 			lowerCamelCase			(   self		:	Tuple					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Optional[int]            =self.ta_base_tokenizer
             _lowerCamelCase			:						Any            ='Unicode €.'
             _lowerCamelCase			:						Optional[int]            =tokenizer(lowercase_					)
             _lowerCamelCase			:						str            =[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
             self.assertEqual(encoded['input_ids']     ,  lowercase_					)

             # decoding
             _lowerCamelCase			:						List[Any]            =tokenizer.decode(lowercase_					)
             self.assertEqual(lowercase_     ,  'Unicode €.</s>'					)

             _lowerCamelCase			:						Any            =tokenizer('e è é ê ë'					)
             _lowerCamelCase			:						int            =[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
             self.assertEqual(encoded['input_ids']     ,  lowercase_					)
             # decoding
             _lowerCamelCase			:						Optional[int]            =tokenizer.decode(lowercase_					)
             self.assertEqual(lowercase_     ,  'e è é ê ë</s>'					)

             # encode/decode, but with `encode` instead of `__call__`
             self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'					)					)     ,  'e è é ê ë</s>'					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.ta_base_tokenizer
             _lowerCamelCase			:						Union[str, Any]            =['A long paragraph for summarization.', 'Another paragraph for summarization.']
             # fmt: off
             _lowerCamelCase			:						Optional[Any]            =[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
             # fmt: on
             _lowerCamelCase			:						Any            =tokenizer(lowercase_     ,  padding=lowercase_     ,  return_tensors=lowercase_					)
             self.assertIsInstance(lowercase_     ,  lowercase_					)

             if FRAMEWORK != "jax":
                    _lowerCamelCase			:						List[Any]            =list(batch.input_ids.numpy()[0]					)
             else:
                    _lowerCamelCase			:						Optional[int]            =list(batch.input_ids.tolist()[0]					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             self.assertEqual((2, 37)     ,  batch.input_ids.shape					)
             self.assertEqual((2, 37)     ,  batch.attention_mask.shape					)






      def 			lowerCamelCase			(   self		:	str					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =self.ta_base_tokenizer
             _lowerCamelCase			:						Optional[Any]            =['A long paragraph for summarization.', 'Another paragraph for summarization.']
             _lowerCamelCase			:						Union[str, Any]            =tokenizer(lowercase_     ,  padding=lowercase_     ,  return_tensors=lowercase_					)
             # check if input_ids are returned and no decoder_input_ids
             self.assertIn('input_ids'     ,  lowercase_					)
             self.assertIn('attention_mask'     ,  lowercase_					)
             self.assertNotIn('decoder_input_ids'     ,  lowercase_					)
             self.assertNotIn('decoder_attention_mask'     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =self.ta_base_tokenizer
             _lowerCamelCase			:						int            =[
                 'Summary of the text.',
                 'Another summary.',
             ]
             _lowerCamelCase			:						Any            =tokenizer(
                 text_target=lowercase_     ,  max_length=32     ,  padding='max_length'     ,  truncation=lowercase_     ,  return_tensors=lowercase_					)
             self.assertEqual(32     ,  targets['input_ids'].shape[1]					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =self.ta_base_tokenizer
             _lowerCamelCase			:						str            =['A long paragraph for summarization. </s>']
             _lowerCamelCase			:						Tuple            =['Summary of the text. </s>']
             # fmt: off
             _lowerCamelCase			:						Any            =[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
             _lowerCamelCase			:						Optional[int]            =[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
             # fmt: on

             _lowerCamelCase			:						int            =tokenizer(lowercase_     ,  text_target=lowercase_					)

             self.assertEqual(lowercase_     ,  batch['input_ids'][0]					)
             self.assertEqual(lowercase_     ,  batch['labels'][0]					)






      def 			lowerCamelCase			(   self		:	Tuple					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Optional[int]            =self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
                           self.assertNotEqual(tokenizer.model_max_length     ,  42					)

        # Now let's start the test
             _lowerCamelCase			:						Tuple            =self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
                           # Isolate this from the other tests because we save additional tokens/etc
                           _lowerCamelCase			:						Optional[Any]            =tempfile.mkdtemp()

                           _lowerCamelCase			:						Dict            =' He is very happy, UNwant\u00E9d,running'
                           _lowerCamelCase			:						Any            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
                           tokenizer.save_pretrained(lowercase_					)

                           _lowerCamelCase			:						List[str]            =tokenizer.__class__.from_pretrained(lowercase_					)
                           _lowerCamelCase			:						Tuple            =after_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
                           self.assertListEqual(lowercase_     ,  lowercase_					)

                           shutil.rmtree(lowercase_					)

             _lowerCamelCase			:						Optional[Any]            =self.get_tokenizers(model_max_length=42					)
             for tokenizer in tokenizers:
                    with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
                           # Isolate this from the other tests because we save additional tokens/etc
                           _lowerCamelCase			:						Optional[int]            =tempfile.mkdtemp()

                           _lowerCamelCase			:						Dict            =' He is very happy, UNwant\u00E9d,running'
                           tokenizer.add_tokens(['bim', 'bambam']					)
                           _lowerCamelCase			:						int            =tokenizer.additional_special_tokens
                           additional_special_tokens.append('new_additional_special_token'					)
                           tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}					)
                           _lowerCamelCase			:						Tuple            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
                           tokenizer.save_pretrained(lowercase_					)

                           _lowerCamelCase			:						int            =tokenizer.__class__.from_pretrained(lowercase_					)
                           _lowerCamelCase			:						Optional[Any]            =after_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
                           self.assertListEqual(lowercase_     ,  lowercase_					)
                           self.assertIn('new_additional_special_token'     ,  after_tokenizer.additional_special_tokens					)
                           self.assertEqual(after_tokenizer.model_max_length     ,  42					)

                           _lowerCamelCase			:						Union[str, Any]            =tokenizer.__class__.from_pretrained(lowercase_     ,  model_max_length=43					)
                           self.assertEqual(tokenizer.model_max_length     ,  43					)

                           shutil.rmtree(lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						str            =[]
             if self.test_slow_tokenizer:
                    tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

             if self.test_rust_tokenizer:
                    tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

             for tokenizer_class, tokenizer_utils in tokenizer_list:
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           tokenizer_utils.save_pretrained(lowercase_					)

                           with open(os.path.join(lowercase_     ,  'special_tokens_map.json'					)     ,  encoding='utf-8'					) as json_file:
                                  _lowerCamelCase			:						Dict            =json.load(lowercase_					)

                           with open(os.path.join(lowercase_     ,  'tokenizer_config.json'					)     ,  encoding='utf-8'					) as json_file:
                                  _lowerCamelCase			:						int            =json.load(lowercase_					)

                           _lowerCamelCase			:						List[str]            =[F'''<extra_id_{i}>''' for i in range(125					)]

                           _lowerCamelCase			:						List[str]            =added_tokens_extra_ids + [
                               'an_additional_special_token'
                           ]
                           _lowerCamelCase			:						Any            =added_tokens_extra_ids + [
                               'an_additional_special_token'
                           ]

                           with open(os.path.join(lowercase_     ,  'special_tokens_map.json'					)     ,  'w'     ,  encoding='utf-8'					) as outfile:
                                  json.dump(lowercase_     ,  lowercase_					)
                           with open(os.path.join(lowercase_     ,  'tokenizer_config.json'					)     ,  'w'     ,  encoding='utf-8'					) as outfile:
                                  json.dump(lowercase_     ,  lowercase_					)

                           # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                           # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                           # "special_tokens_map.json" files
                           _lowerCamelCase			:						str            =tokenizer_class.from_pretrained(
                               lowercase_     ,  )
                           self.assertIn(
                               'an_additional_special_token'     ,  tokenizer_without_change_in_init.additional_special_tokens					)
                           # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                           self.assertEqual(
                               ['an_additional_special_token']     ,  tokenizer_without_change_in_init.convert_ids_to_tokens(
                                   tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']					)					)     ,  )

                           # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                           _lowerCamelCase			:						Tuple            =added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'     ,  lstrip=lowercase_					)]
                           _lowerCamelCase			:						Union[str, Any]            =tokenizer_class.from_pretrained(
                               lowercase_     ,  additional_special_tokens=lowercase_     ,  )

                           self.assertIn('a_new_additional_special_token'     ,  tokenizer.additional_special_tokens					)
                           self.assertEqual(
                               ['a_new_additional_special_token']     ,  tokenizer.convert_ids_to_tokens(
                                   tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']					)					)     ,  )






      def 			lowerCamelCase			(   self		:	int					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            =[]
             if self.test_slow_tokenizer:
                    tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

             if self.test_rust_tokenizer:
                    tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

             for tokenizer_class, tokenizer_utils in tokenizer_list:
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           tokenizer_utils.save_pretrained(lowercase_					)

                           _lowerCamelCase			:						Union[str, Any]            =tokenizer_class.from_pretrained(lowercase_					)

                           self.assertTrue(tokenizer.decode([255]					) == ''					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		List[Any]:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	str					)   ->		Optional[int]:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	Any					)   ->		Tuple:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	str					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						Any            =self.get_tokenizers(fast=lowercase_     ,  do_lower_case=lowercase_					)
             for tokenizer in tokenizers:
                    with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
                           _lowerCamelCase			:						Optional[Any]            =['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                           _lowerCamelCase			:						Dict            =tokenizer.convert_tokens_to_string(lowercase_					)

                           self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.get_tokenizers()
             for tokenizer in tokenizers:
                    with self.subTest(F'''{tokenizer.__class__.__name__}'''					):
                           _lowerCamelCase			:						Union[str, Any]            =[
                               'bos_token',
                               'eos_token',
                               'unk_token',
                               'sep_token',
                               'pad_token',
                               'cls_token',
                               'mask_token',
                           ]

                           _lowerCamelCase			:						List[Any]            =0
                           _lowerCamelCase			:						int            =tokenizer.convert_ids_to_tokens(
                               lowercase_     ,  skip_special_tokens=lowercase_					)

                           for attr in attributes_list:
                                  setattr(lowercase_     ,  attr + '_id'     ,  lowercase_					)
                                  self.assertEqual(getattr(lowercase_     ,  lowercase_					)     ,  lowercase_					)
                                  self.assertEqual(getattr(lowercase_     ,  attr + '_id'					)     ,  lowercase_					)

                                  setattr(lowercase_     ,  attr + '_id'     ,  lowercase_					)
                                  self.assertEqual(getattr(lowercase_     ,  lowercase_					)     ,  lowercase_					)
                                  self.assertEqual(getattr(lowercase_     ,  attr + '_id'					)     ,  lowercase_					)

                           setattr(lowercase_     ,  'additional_special_tokens_ids'     ,  []					)
                           self.assertListEqual(getattr(lowercase_     ,  'additional_special_tokens'					)     ,  []					)
                           self.assertListEqual(getattr(lowercase_     ,  'additional_special_tokens_ids'					)     ,  []					)

                           setattr(lowercase_     ,  'additional_special_tokens_ids'     ,  [token_id_to_test_setters]					)
                           self.assertListEqual(getattr(lowercase_     ,  'additional_special_tokens'					)     ,  [token_to_test_setters]					)
                           self.assertListEqual(getattr(lowercase_     ,  'additional_special_tokens_ids'					)     ,  [token_id_to_test_setters]					)






from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


lowerCamelCase		   =			logging.get_logger(__name__)

lowerCamelCase		   =			{
    'microsoft/focalnet-tiny': 'https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json',
}




class A    (  UpperCamelCase_       ,				UpperCamelCase_	):
      UpperCamelCase__				:    str ='focalnet'






      def __init__(   self		:	Optional[int]     ,  lowercase_		:	List[Any]=224     ,  lowercase_		:	str=4     ,  lowercase_		:	Any=3     ,  lowercase_		:	Dict=96     ,  lowercase_		:	List[str]=False     ,  lowercase_		:	Any=[192, 384, 768, 768]     ,  lowercase_		:	Dict=[2, 2, 6, 2]     ,  lowercase_		:	Optional[int]=[2, 2, 2, 2]     ,  lowercase_		:	List[Any]=[3, 3, 3, 3]     ,  lowercase_		:	Any="gelu"     ,  lowercase_		:	str=4.0     ,  lowercase_		:	List[Any]=0.0     ,  lowercase_		:	Tuple=0.1     ,  lowercase_		:	Optional[Any]=False     ,  lowercase_		:	Optional[int]=1E-4     ,  lowercase_		:	Dict=False     ,  lowercase_		:	Optional[Any]=False     ,  lowercase_		:	Dict=False     ,  lowercase_		:	List[Any]=0.02     ,  lowercase_		:	Any=1E-5     ,  lowercase_		:	Any=32     ,  lowercase_		:	List[str]=None     ,  lowercase_		:	int=None     ,  **lowercase_		:	Optional[Any]     ,  )   ->		Optional[Any]:
             """simple docstring"""




             super().__init__(**lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            =image_size
             _lowerCamelCase			:						str            =patch_size
             _lowerCamelCase			:						str            =num_channels
             _lowerCamelCase			:						Tuple            =embed_dim
             _lowerCamelCase			:						Dict            =use_conv_embed
             _lowerCamelCase			:						int            =hidden_sizes
             _lowerCamelCase			:						str            =depths
             _lowerCamelCase			:						int            =focal_levels
             _lowerCamelCase			:						List[Any]            =focal_windows
             _lowerCamelCase			:						List[Any]            =hidden_act
             _lowerCamelCase			:						int            =mlp_ratio
             _lowerCamelCase			:						Any            =hidden_dropout_prob
             _lowerCamelCase			:						Optional[Any]            =drop_path_rate
             _lowerCamelCase			:						int            =use_layerscale
             _lowerCamelCase			:						str            =layerscale_value
             _lowerCamelCase			:						Dict            =use_post_layernorm
             _lowerCamelCase			:						str            =use_post_layernorm_in_modulation
             _lowerCamelCase			:						List[Any]            =normalize_modulator
             _lowerCamelCase			:						Optional[Any]            =initializer_range
             _lowerCamelCase			:						List[str]            =layer_norm_eps
             _lowerCamelCase			:						Optional[int]            =encoder_stride
             _lowerCamelCase			:						List[str]            =['stem'] + [F'''stage{idx}''' for idx in range(1     ,  len(self.depths					) + 1					)]
             _lowerCamelCase      ,	_lowerCamelCase			:						Union[str, Any]            =get_aligned_output_features_output_indices(
                 out_features=lowercase_     ,  out_indices=lowercase_     ,  stage_names=self.stage_names					)


