


'''simple docstring'''







from __future__ import annotations

import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers import TFViTForImageClassification, TFViTModel


if is_vision_available():
       from PIL import Image

       from transformers import ViTImageProcessor
class 						UpperCamelCase__       :



     """simple docstring"""






     def __init__(			self   ,						snake_case__   ,						snake_case__=13   ,						snake_case__=30   ,						snake_case__=2   ,						snake_case__=3   ,						snake_case__=True   ,						snake_case__=True   ,						snake_case__=32   ,						snake_case__=2   ,						snake_case__=4   ,						snake_case__=37   ,						snake_case__="gelu"   ,						snake_case__=0.1   ,						snake_case__=0.1   ,						snake_case__=10   ,						snake_case__=0.02   ,						snake_case__=3   ,						snake_case__=None   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							parent
            _lowerCAmelCase     :				List[str]       =							batch_size
            _lowerCAmelCase     :				str       =							image_size
            _lowerCAmelCase     :				List[str]       =							patch_size
            _lowerCAmelCase     :				Tuple       =							num_channels
            _lowerCAmelCase     :				str       =							is_training
            _lowerCAmelCase     :				Dict       =							use_labels
            _lowerCAmelCase     :				Union[str, Any]       =							hidden_size
            _lowerCAmelCase     :				List[str]       =							num_hidden_layers
            _lowerCAmelCase     :				Dict       =							num_attention_heads
            _lowerCAmelCase     :				Union[str, Any]       =							intermediate_size
            _lowerCAmelCase     :				List[str]       =							hidden_act
            _lowerCAmelCase     :				Optional[Any]       =							hidden_dropout_prob
            _lowerCAmelCase     :				Tuple       =							attention_probs_dropout_prob
            _lowerCAmelCase     :				Any       =							type_sequence_label_size
            _lowerCAmelCase     :				int       =							initializer_range
            _lowerCAmelCase     :				Union[str, Any]       =							scope

            # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
            _lowerCAmelCase     :				List[Any]       =							(image_size // patch_size) ** 2
            _lowerCAmelCase     :				Tuple       =							num_patches + 1







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

            _lowerCAmelCase     :				Optional[Any]       =							None
            if self.use_labels:
                   _lowerCAmelCase     :				Any       =							ids_tensor([self.batch_size]   ,						self.type_sequence_label_size )

            _lowerCAmelCase     :				int       =							self.get_config()

            return config, pixel_values, labels







     def a						(			self ):

            '''simple docstring'''

            return ViTConfig(
                image_size=self.image_size   ,						patch_size=self.patch_size   ,						num_channels=self.num_channels   ,						hidden_size=self.hidden_size   ,						num_hidden_layers=self.num_hidden_layers   ,						num_attention_heads=self.num_attention_heads   ,						intermediate_size=self.intermediate_size   ,						hidden_act=self.hidden_act   ,						hidden_dropout_prob=self.hidden_dropout_prob   ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,						is_decoder=UpperCAmelCase_   ,						initializer_range=self.initializer_range   ,						)







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							TFViTModel(config=UpperCAmelCase_ )
            _lowerCAmelCase     :				str       =							model(UpperCAmelCase_   ,						training=UpperCAmelCase_ )
            self.parent.assertEqual(result.last_hidden_state.shape   ,						(self.batch_size, self.seq_length, self.hidden_size) )

            # Test with an image with different size than the one specified in config.
            _lowerCAmelCase     :				Tuple       =							self.image_size // 2
            _lowerCAmelCase     :				str       =							pixel_values[:, :, :image_size, :image_size]
            _lowerCAmelCase     :				Optional[int]       =							model(UpperCAmelCase_   ,						interpolate_pos_encoding=UpperCAmelCase_   ,						training=UpperCAmelCase_ )
            _lowerCAmelCase     :				List[Any]       =							(image_size // self.patch_size) ** 2 + 1
            self.parent.assertEqual(result.last_hidden_state.shape   ,						(self.batch_size, seq_length, self.hidden_size) )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							self.type_sequence_label_size
            _lowerCAmelCase     :				Dict       =							TFViTForImageClassification(UpperCAmelCase_ )
            _lowerCAmelCase     :				List[str]       =							model(UpperCAmelCase_   ,						labels=UpperCAmelCase_   ,						training=UpperCAmelCase_ )
            self.parent.assertEqual(result.logits.shape   ,						(self.batch_size, self.type_sequence_label_size) )

            # Test with an image with different size than the one specified in config.
            _lowerCAmelCase     :				Optional[int]       =							self.image_size // 2
            _lowerCAmelCase     :				List[Any]       =							pixel_values[:, :, :image_size, :image_size]
            _lowerCAmelCase     :				List[Any]       =							model(UpperCAmelCase_   ,						interpolate_pos_encoding=UpperCAmelCase_   ,						training=UpperCAmelCase_ )
            self.parent.assertEqual(result.logits.shape   ,						(self.batch_size, self.type_sequence_label_size) )

            # test greyscale images
            _lowerCAmelCase     :				Tuple       =							1
            _lowerCAmelCase     :				int       =							TFViTForImageClassification(UpperCAmelCase_ )
            _lowerCAmelCase     :				Optional[int]       =							floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
            _lowerCAmelCase     :				Dict       =							model(UpperCAmelCase_ )
            self.parent.assertEqual(result.logits.shape   ,						(self.batch_size, self.type_sequence_label_size) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Union[str, Any]       =							self.prepare_config_and_inputs()
            _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase     :				str       =							config_and_inputs
            _lowerCAmelCase     :				Union[str, Any]       =							{'pixel_values': pixel_values}
            return config, inputs_dict
@require_tf
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_     ,       SCREAMING_SNAKE_CASE_     ,       unittest.TestCase						):



     """simple docstring"""






     __magic_name__           =     (TFViTModel, TFViTForImageClassification) if is_tf_available() else ()
     __magic_name__           =     (
         {"feature-extraction": TFViTModel, "image-classification": TFViTForImageClassification}
         if is_tf_available()
         else {}
     )

     __magic_name__           =     False
     __magic_name__           =     False
     __magic_name__           =     False







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							TFViTModelTester(self )
            _lowerCAmelCase     :				Any       =							ConfigTester(self   ,						config_class=UpperCAmelCase_   ,						has_text_modality=UpperCAmelCase_   ,						hidden_size=37 )







     def a						(			self ):

            '''simple docstring'''

            self.config_tester.run_common_tests()







     @unittest.skip(reason='ViT does not use inputs_embeds' )
     def a						(			self ):

            '''simple docstring'''

            pass







     @unittest.skip(reason='ViT does not use inputs_embeds' )
     def a						(			self ):

            '''simple docstring'''

            pass







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase      ,  _lowerCAmelCase     :				Dict       =							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _lowerCAmelCase     :				Dict       =							model_class(UpperCAmelCase_ )
                   self.assertIsInstance(model.get_input_embeddings()   ,						(tf.keras.layers.Layer) )
                   _lowerCAmelCase     :				Dict       =							model.get_output_embeddings()
                   self.assertTrue(x is None or isinstance(UpperCAmelCase_   ,						tf.keras.layers.Layer ) )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase      ,  _lowerCAmelCase     :				List[Any]       =							self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _lowerCAmelCase     :				str       =							model_class(UpperCAmelCase_ )
                   _lowerCAmelCase     :				Optional[int]       =							inspect.signature(model.call )
                   # signature.parameters is an OrderedDict => so arg_names order is deterministic
                   _lowerCAmelCase     :				Any       =							[*signature.parameters.keys()]

                   _lowerCAmelCase     :				Optional[Any]       =							['pixel_values']
                   self.assertListEqual(arg_names[:1]   ,						UpperCAmelCase_ )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*UpperCAmelCase_ )







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[Any]       =							self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*UpperCAmelCase_ )







     @slow
     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							TFViTModel.from_pretrained('google/vit-base-patch16-224' )
            self.assertIsNotNone(UpperCAmelCase_ )






def       lowercase					():







       """simple docstring"""







       _lowerCAmelCase     :				Union[str, Any]       =							Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'       )
       return image
@require_tf
@require_vision
class 						UpperCamelCase__       (			unittest.TestCase						):



     """simple docstring"""






     @cached_property
     def a						(			self ):

            '''simple docstring'''

            return ViTImageProcessor.from_pretrained('google/vit-base-patch16-224' ) if is_vision_available() else None







     @slow
     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							TFViTForImageClassification.from_pretrained('google/vit-base-patch16-224' )

            _lowerCAmelCase     :				Optional[int]       =							self.default_image_processor
            _lowerCAmelCase     :				Dict       =							prepare_img()
            _lowerCAmelCase     :				Tuple       =							image_processor(images=UpperCAmelCase_   ,						return_tensors='tf' )

            # forward pass
            _lowerCAmelCase     :				int       =							model(**UpperCAmelCase_ )

            # verify the logits
            _lowerCAmelCase     :				Dict       =							tf.TensorShape((1, 1000) )
            self.assertEqual(outputs.logits.shape   ,						UpperCAmelCase_ )

            _lowerCAmelCase     :				str       =							tf.constant([-0.2744, 0.8215, -0.0836] )

            tf.debugging.assert_near(outputs.logits[0, :3]   ,						UpperCAmelCase_   ,						atol=1E-4 )






'''simple docstring'''





import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 					snake_case			(UpperCamelCase ):



       def __init__(					self  ,UpperCAmelCase_ = "▁"  ,UpperCAmelCase_ = True  ,UpperCAmelCase_ = "<unk>"  ,UpperCAmelCase_ = "</s>"  ,UpperCAmelCase_ = "<pad>"  ,)	->				Optional[int]:
           lowercase__									=    {
               "pad": {"id": 0, "token": pad_token},
               "eos": {"id": 1, "token": eos_token},
               "unk": {"id": 2, "token": unk_token},
           }

           lowercase__									=    [None] * len(self.special_tokens   )
           for token_dict in self.special_tokens.values():
               lowercase__									=    token_dict["token"]

           lowercase__									=    Tokenizer(Unigram()   )

           lowercase__									=    normalizers.Sequence(
               [
                   normalizers.Nmt(),
                   normalizers.NFKC(),
                   normalizers.Replace(Regex(" {2,}"   )  ," "   ),
                   normalizers.Lowercase(),
               ]   )
           lowercase__									=    pre_tokenizers.Sequence(
               [
                   pre_tokenizers.Metaspace(replacement=UpperCAmelCase_  ,add_prefix_space=UpperCAmelCase_   ),
                   pre_tokenizers.Digits(individual_digits=UpperCAmelCase_   ),
                   pre_tokenizers.Punctuation(),
               ]   )
           lowercase__									=    decoders.Metaspace(replacement=UpperCAmelCase_  ,add_prefix_space=UpperCAmelCase_   )

           lowercase__									=    TemplateProcessing(
               single=F'''$A {self.special_tokens['eos']['token']}'''  ,special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])]  ,)

           lowercase__									=    {
               "model": "SentencePieceUnigram",
               "replacement": replacement,
               "add_prefix_space": add_prefix_space,
           }

           super().__init__(UpperCAmelCase_  ,UpperCAmelCase_   )



       def 				_a   (					self  ,UpperCAmelCase_  ,UpperCAmelCase_ = 8_000  ,UpperCAmelCase_ = True  ,)	->				List[str]:
           lowercase__									=    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase_  ,special_tokens=self.special_tokens_list  ,show_progress=UpperCAmelCase_  ,)

           if isinstance(UpperCAmelCase_  ,UpperCAmelCase_   ):
               lowercase__									=    [files]
           self._tokenizer.train(UpperCAmelCase_  ,trainer=UpperCAmelCase_   )

           self.add_unk_id()



       def 				_a   (					self  ,UpperCAmelCase_  ,UpperCAmelCase_ = 8_000  ,UpperCAmelCase_ = True  ,)	->				Union[str, Any]:
           lowercase__									=    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase_  ,special_tokens=self.special_tokens_list  ,show_progress=UpperCAmelCase_  ,)

           self._tokenizer.train_from_iterator(UpperCAmelCase_  ,trainer=UpperCAmelCase_   )

           self.add_unk_id()







       def 				_a   (					self   )	->				str:
           lowercase__									=    json.loads(self._tokenizer.to_str()   )

           lowercase__									=    self.special_tokens["unk"]["id"]

           lowercase__									=    Tokenizer.from_str(json.dumps(UpperCAmelCase_   )   )




