
'''simple docstring'''





from __future__ import annotations

import inspect
import unittest
from math import floor

import numpy as np

from transformers import CvtConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import TFCvtForImageClassification, TFCvtModel
      from transformers.models.cvt.modeling_tf_cvt import TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					str				       =							self.config_class(**self.inputs_dict )
   self.parent.assertTrue(hasattr(snake_case__			,				"embed_dim" ) )
   self.parent.assertTrue(hasattr(snake_case__			,				"num_heads" ) )






class 							UpperCAmelCase_							:

 '''simple docstring'''







 def __init__(					self		:  Union[str, Any]			,				snake_case__		:  List[str]			,				snake_case__		:  Dict=13			,				snake_case__		:  List[Any]=64			,				snake_case__		:  str=3			,				snake_case__		:  List[str]=[16, 48, 96]			,				snake_case__		:  List[Any]=[1, 3, 6]			,				snake_case__		:  Tuple=[1, 2, 10]			,				snake_case__		:  Optional[int]=[7, 3, 3]			,				snake_case__		:  int=[4, 2, 2]			,				snake_case__		:  Dict=[2, 1, 1]			,				snake_case__		:  List[Any]=[2, 2, 2]			,				snake_case__		:  Optional[int]=[False, False, True]			,				snake_case__		:  Tuple=[0.0, 0.0, 0.0]			,				snake_case__		:  Optional[Any]=0.02			,				snake_case__		:  Optional[int]=1e-12			,				snake_case__		:  Optional[Any]=True			,				snake_case__		:  Union[str, Any]=True			,				snake_case__		:  Optional[int]=2			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					int				       =							parent
   UpperCAmelCase__      :					List[Any]				       =							batch_size
   UpperCAmelCase__      :					Union[str, Any]				       =							image_size
   UpperCAmelCase__      :					Tuple				       =							patch_sizes
   UpperCAmelCase__      :					List[str]				       =							patch_stride
   UpperCAmelCase__      :					Union[str, Any]				       =							patch_padding
   UpperCAmelCase__      :					Union[str, Any]				       =							is_training
   UpperCAmelCase__      :					Tuple				       =							use_labels
   UpperCAmelCase__      :					Dict				       =							num_labels
   UpperCAmelCase__      :					Optional[int]				       =							num_channels
   UpperCAmelCase__      :					Optional[int]				       =							embed_dim
   UpperCAmelCase__      :					Union[str, Any]				       =							num_heads
   UpperCAmelCase__      :					str				       =							stride_kv
   UpperCAmelCase__      :					Optional[Any]				       =							depth
   UpperCAmelCase__      :					Tuple				       =							cls_token
   UpperCAmelCase__      :					int				       =							attention_drop_rate
   UpperCAmelCase__      :					Union[str, Any]				       =							initializer_range
   UpperCAmelCase__      :					List[Any]				       =							layer_norm_eps







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   UpperCAmelCase__      :					str				       =							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

   UpperCAmelCase__      :					str				       =							None
   if self.use_labels:
     # create a random int32 tensor of given shape
     UpperCAmelCase__      :					Any				       =							ids_tensor([self.batch_size]			,				self.num_labels )

   UpperCAmelCase__      :					int				       =							self.get_config()
   return config, pixel_values, labels







 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   return CvtConfig(
       image_size=self.image_size			,				num_labels=self.num_labels			,				num_channels=self.num_channels			,				embed_dim=self.embed_dim			,				num_heads=self.num_heads			,				patch_sizes=self.patch_sizes			,				patch_padding=self.patch_padding			,				patch_stride=self.patch_stride			,				stride_kv=self.stride_kv			,				depth=self.depth			,				cls_token=self.cls_token			,				attention_drop_rate=self.attention_drop_rate			,				initializer_range=self.initializer_range			,				)







 def 							UpperCamelCase  (					self		:  Union[str, Any]			,				snake_case__		:  int			,				snake_case__		:  List[Any]			,				snake_case__		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[Any]				       =							TFCvtModel(config=snake_case__ )
   UpperCAmelCase__      :					Union[str, Any]				       =							model(snake_case__			,				training=snake_case__ )
   UpperCAmelCase__      :					Optional[int]				       =							(self.image_size, self.image_size)
   UpperCAmelCase__       ,			UpperCAmelCase__      :					Union[str, Any]				       =							image_size[0], image_size[1]
   for i in range(len(self.depth ) ):
     UpperCAmelCase__      :					List[Any]				       =							floor(((height + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1 )
     UpperCAmelCase__      :					Any				       =							floor(((width + 2 * self.patch_padding[i] - self.patch_sizes[i]) / self.patch_stride[i]) + 1 )
   self.parent.assertEqual(result.last_hidden_state.shape			,				(self.batch_size, self.embed_dim[-1], height, width) )







 def 							UpperCamelCase  (					self		:  Optional[Any]			,				snake_case__		:  Any			,				snake_case__		:  int			,				snake_case__		:  Optional[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[Any]				       =							self.num_labels
   UpperCAmelCase__      :					List[str]				       =							TFCvtForImageClassification(snake_case__ )
   UpperCAmelCase__      :					List[Any]				       =							model(snake_case__			,				labels=snake_case__			,				training=snake_case__ )
   self.parent.assertEqual(result.logits.shape			,				(self.batch_size, self.num_labels) )







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					int				       =							self.prepare_config_and_inputs()
   UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					Optional[Any]				       =							config_and_inputs
   UpperCAmelCase__      :					Dict				       =							{"pixel_values": pixel_values}
   return config, inputs_dict






@require_tf
class 							UpperCAmelCase_							(    A					,				A					,				unittest.TestCase				):

 '''simple docstring'''
 lowercase_					:    Dict				=							(TFCvtModel, TFCvtForImageClassification) if is_tf_available() else ()
 lowercase_					:    Dict				=							(
     {"feature-extraction": TFCvtModel, "image-classification": TFCvtForImageClassification}
     if is_tf_available()
     else {}
 )
 lowercase_					:    str				=							False
 lowercase_					:    List[str]				=							False
 lowercase_					:    Union[str, Any]				=							False
 lowercase_					:    Tuple				=							False
 lowercase_					:    int				=							False







 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							TFCvtModelTester(self )
   UpperCAmelCase__      :					str				       =							TFCvtConfigTester(self			,				config_class=snake_case__			,				has_text_modality=snake_case__			,				hidden_size=37 )







 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   self.config_tester.create_and_test_config_common_properties()
   self.config_tester.create_and_test_config_to_json_string()
   self.config_tester.create_and_test_config_to_json_file()
   self.config_tester.create_and_test_config_from_and_save_pretrained()
   self.config_tester.create_and_test_config_with_num_labels()
   self.config_tester.check_config_can_be_init_without_params()
   self.config_tester.check_config_arguments_init()







 @unittest.skip(reason="Cvt does not output attentions" )
 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   pass







 @unittest.skip(reason="Cvt does not use inputs_embeds" )
 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   pass







 @unittest.skip(reason="Cvt does not support input and output embeddings" )
 def 							UpperCamelCase  (					self		:  Optional[Any] ):



   '''simple docstring'''




   pass







 @unittest.skipIf(
     not is_tf_available() or len(tf.config.list_physical_devices("GPU" ) ) == 0			,				reason="TF does not support backprop for grouped convolutions on CPU."			,				)
 def 							UpperCamelCase  (					self		:  Tuple ):



   '''simple docstring'''




   super().test_dataset_conversion()







 @unittest.skipIf(
     not is_tf_available() or len(tf.config.list_physical_devices("GPU" ) ) == 0			,				reason="TF does not support backprop for grouped convolutions on CPU."			,				)
 @slow
 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   super().test_keras_fit()







 @unittest.skip(reason="Get `Failed to determine best cudnn convolution algo.` error after using TF 2.12+cuda 11.8" )
 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					int				       =							tf.keras.mixed_precision.Policy("mixed_float16" )
   tf.keras.mixed_precision.set_global_policy(snake_case__ )
   super().test_keras_fit()
   tf.keras.mixed_precision.set_global_policy("float32" )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   UpperCAmelCase__       ,			UpperCAmelCase__      :					Optional[int]				       =							self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
     UpperCAmelCase__      :					Optional[Any]				       =							model_class(snake_case__ )
     UpperCAmelCase__      :					Dict				       =							inspect.signature(model.call )
     # signature.parameters is an OrderedDict => so arg_names order is deterministic
     UpperCAmelCase__      :					Optional[int]				       =							[*signature.parameters.keys()]

     UpperCAmelCase__      :					str				       =							["pixel_values"]
     self.assertListEqual(arg_names[:1]			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   def check_hidden_states_output(snake_case__		:  str			,				snake_case__		:  Tuple			,				snake_case__		:  Dict ):
     UpperCAmelCase__      :					Optional[int]				       =							model_class(snake_case__ )

     UpperCAmelCase__      :					Dict				       =							model(**self._prepare_for_class(snake_case__			,				snake_case__ ) )
     UpperCAmelCase__      :					List[str]				       =							outputs.hidden_states

     UpperCAmelCase__      :					Optional[Any]				       =							len(self.model_tester.depth )
     self.assertEqual(len(snake_case__ )			,				snake_case__ )

     # verify the first hidden states (first block)
     self.assertListEqual(
         list(hidden_states[0].shape[-3:] )			,				[
             self.model_tester.embed_dim[0],
             self.model_tester.image_size // 4,
             self.model_tester.image_size // 4,
         ]			,				)

   UpperCAmelCase__       ,			UpperCAmelCase__      :					int				       =							self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
     UpperCAmelCase__      :					List[Any]				       =							True
     check_hidden_states_output(snake_case__			,				snake_case__			,				snake_case__ )

     # check that output_hidden_states also work using config
     del inputs_dict["output_hidden_states"]
     UpperCAmelCase__      :					Optional[Any]				       =							True

     check_hidden_states_output(snake_case__			,				snake_case__			,				snake_case__ )







 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					Union[str, Any]				       =							self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_model(*snake_case__ )







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					str				       =							self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_image_classification(*snake_case__ )







 @slow
 def 							UpperCamelCase  (					self		:  Optional[Any] ):



   '''simple docstring'''




   for model_name in TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     UpperCAmelCase__      :					List[Any]				       =							TFCvtModel.from_pretrained(snake_case__ )
     self.assertIsNotNone(snake_case__ )







def snake_case_     (					):
  UpperCAmelCase__      :					Optional[int]				       =							Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
  return image






@require_tf
@require_vision
class 							UpperCAmelCase_							(    unittest.TestCase				):

 '''simple docstring'''







 @cached_property
 def 							UpperCamelCase  (					self		:  Dict ):



   '''simple docstring'''




   return AutoImageProcessor.from_pretrained(TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0] )







 @slow
 def 							UpperCamelCase  (					self		:  Optional[int] ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[str]				       =							TFCvtForImageClassification.from_pretrained(TF_CVT_PRETRAINED_MODEL_ARCHIVE_LIST[0] )

   UpperCAmelCase__      :					Optional[Any]				       =							self.default_image_processor
   UpperCAmelCase__      :					Any				       =							prepare_img()
   UpperCAmelCase__      :					Optional[Any]				       =							image_processor(images=snake_case__			,				return_tensors="tf" )

   # forward pass
   UpperCAmelCase__      :					List[Any]				       =							model(**snake_case__ )

   # verify the logits
   UpperCAmelCase__      :					str				       =							tf.TensorShape((1, 10_00) )
   self.assertEqual(outputs.logits.shape			,				snake_case__ )

   UpperCAmelCase__      :					Optional[int]				       =							tf.constant([0.9285, 0.9015, -0.3150] )
   self.assertTrue(np.allclose(outputs.logits[0, :3].numpy()			,				snake_case__			,				atol=1e-4 ) )











import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__a:     Any   					=					logging.get_logger(__name__)


__a:     Dict   					=					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

__a:     int   					=					{
    '''vocab_file''': {
        '''facebook/blenderbot_small-90M''': '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''facebook/blenderbot_small-90M''': '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'''
    },
    '''tokenizer_config_file''': {
        '''facebook/blenderbot_small-90M''': (
            '''https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'''
        )
    },
}

__a:     str   					=					{'''facebook/blenderbot_small-90M''': 512}




def       _SCREAMING_SNAKE_CASE (							__snake_case )				->						List[str]:
       _UpperCAmelCase							=    set()
       _UpperCAmelCase							=    word[0]
       for char in word[1:]:
              pairs.add((prev_char, char) )
              _UpperCAmelCase							=    char

       _UpperCAmelCase							=    set(__snake_case )
       return pairs





class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  VOCAB_FILES_NAMES
  _lowerCamelCase    =  PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase    =  ['''input_ids''', '''attention_mask''']


  def __init__(      self		:			List[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			List[str]="__start__"       ,				lowerCamelCase		:			List[Any]="__end__"       ,				lowerCamelCase		:			Any="__unk__"       ,				lowerCamelCase		:			Optional[Any]="__null__"       ,				**lowerCamelCase		:			Optional[Any]       ,				)						->	Tuple:


         """simple docstring"""
         super().__init__(unk_token=lowerCamelCase       ,				bos_token=lowerCamelCase       ,				eos_token=lowerCamelCase       ,				pad_token=lowerCamelCase       ,				**lowerCamelCase							)

         with open(lowerCamelCase       ,				encoding="""utf-8"""							) as vocab_handle:
                _UpperCAmelCase							=    json.load(lowerCamelCase							)
         _UpperCAmelCase							=    {v: k for k, v in self.encoder.items()}
         with open(lowerCamelCase       ,				encoding="""utf-8"""							) as merges_handle:
                _UpperCAmelCase							=    merges_handle.read().split("""\n"""							)[1:-1]
         _UpperCAmelCase							=    [tuple(merge.split()							) for merge in merges]
         _UpperCAmelCase							=    dict(zip(lowerCamelCase       ,				range(len(lowerCamelCase							)							)							)							)
         _UpperCAmelCase							=    {}


  @property
  def  lowerCamelCase       (      self		:			str							)						->	int:


         """simple docstring"""
         return len(self.encoder							)


  def  lowerCamelCase       (      self		:			List[Any]							)						->	Dict:


         """simple docstring"""
         return dict(self.encoder       ,				**self.added_tokens_encoder							)


  def  lowerCamelCase       (      self		:			Optional[Any]       ,				lowerCamelCase		:			str							)						->	str:


         """simple docstring"""
         if token in self.cache:
                return self.cache[token]
         _UpperCAmelCase							=    re.sub("""([.,!?()])"""       ,				r""" \1"""       ,				lowerCamelCase							)
         _UpperCAmelCase							=    re.sub("""(')"""       ,				r""" \1 """       ,				lowerCamelCase							)
         _UpperCAmelCase							=    re.sub(r"""\s{2,}"""       ,				""" """       ,				lowerCamelCase							)
         if "\n" in token:
                _UpperCAmelCase							=    token.replace("""\n"""       ,				""" __newln__"""							)

         _UpperCAmelCase							=    token.split(""" """							)
         _UpperCAmelCase							=    []
         for token in tokens:
                if not len(lowerCamelCase							):
                       continue

                _UpperCAmelCase							=    token.lower()
                _UpperCAmelCase							=    tuple(lowerCamelCase							)
                _UpperCAmelCase							=    tuple(list(word[:-1]							) + [word[-1] + """</w>"""]							)
                _UpperCAmelCase							=    get_pairs(lowerCamelCase							)

                if not pairs:
                       words.append(lowerCamelCase							)
                       continue

                while True:
                       _UpperCAmelCase							=    min(lowerCamelCase       ,				key=lambda lowerCamelCase							: self.bpe_ranks.get(lowerCamelCase       ,				float("""inf"""							)							)							)
                       if bigram not in self.bpe_ranks:
                              break
                       _UpperCAmelCase	,			_UpperCAmelCase							=    bigram
                       _UpperCAmelCase							=    []
                       _UpperCAmelCase							=    0

                       while i < len(lowerCamelCase							):
                              try:
                                     _UpperCAmelCase							=    word.index(lowerCamelCase       ,				lowerCamelCase							)
                                     new_word.extend(word[i:j]							)
                                     _UpperCAmelCase							=    j
                              except ValueError:
                                     new_word.extend(word[i:]							)
                                     break

                              if word[i] == first and i < len(lowerCamelCase							) - 1 and word[i + 1] == second:
                                     new_word.append(first + second							)
                                     i += 2
                              else:
                                     new_word.append(word[i]							)
                                     i += 1
                       _UpperCAmelCase							=    tuple(lowerCamelCase							)
                       _UpperCAmelCase							=    new_word
                       if len(lowerCamelCase							) == 1:
                              break
                       else:
                              _UpperCAmelCase							=    get_pairs(lowerCamelCase							)
                _UpperCAmelCase							=    """@@ """.join(lowerCamelCase							)
                _UpperCAmelCase							=    word[:-4]

                _UpperCAmelCase							=    word
                words.append(lowerCamelCase							)
         return " ".join(lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Any       ,				lowerCamelCase		:			str							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    []

         _UpperCAmelCase							=    re.findall(r"""\S+\n?"""       ,				lowerCamelCase							)

         for token in words:
                split_tokens.extend(list(self.bpe(lowerCamelCase							).split(""" """							)							)							)
         return split_tokens


  def  lowerCamelCase       (      self		:			Tuple       ,				lowerCamelCase		:			str							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    token.lower()
         return self.encoder.get(lowerCamelCase       ,				self.encoder.get(self.unk_token							)							)


  def  lowerCamelCase       (      self		:			List[str]       ,				lowerCamelCase		:			int							)						->	str:


         """simple docstring"""
         return self.decoder.get(lowerCamelCase       ,				self.unk_token							)


  def  lowerCamelCase       (      self		:			Dict       ,				lowerCamelCase		:			List[str]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    """ """.join(lowerCamelCase							).replace("""@@ """       ,				""""""							).strip()
         return out_string





  def  lowerCamelCase       (      self		:			List[Any]       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			Optional[str] = None							)						->	Tuple[str]:


         """simple docstring"""
         if not os.path.isdir(lowerCamelCase							):
                logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
                return
         _UpperCAmelCase							=    os.path.join(
             lowerCamelCase       ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
         _UpperCAmelCase							=    os.path.join(
             lowerCamelCase       ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

         with open(lowerCamelCase       ,				"""w"""       ,				encoding="""utf-8"""							) as f:
                f.write(json.dumps(self.encoder       ,				indent=2       ,				sort_keys=lowerCamelCase       ,				ensure_ascii=lowerCamelCase							) + """\n"""							)

         _UpperCAmelCase							=    0
         with open(lowerCamelCase       ,				"""w"""       ,				encoding="""utf-8"""							) as writer:
                writer.write("""#version: 0.2\n"""							)
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,				key=lambda lowerCamelCase							: kv[1]							):
                       if index != token_index:
                              logger.warning(
                                  f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                  """ Please check that the tokenizer is not corrupted!"""							)
                              _UpperCAmelCase							=    token_index
                       writer.write(""" """.join(lowerCamelCase							) + """\n"""							)
                       index += 1

         return vocab_file, merge_file