
'''simple docstring'''


from __future__ import annotations

import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import TFViTForImageClassification, TFViTModel


if is_vision_available():
   from PIL import Image

   from transformers import ViTImageProcessor


class 						lowercase__	:






   '''simple docstring'''







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__=1_3 ,   lowerCamelCase__=3_0 ,   lowerCamelCase__=2 ,   lowerCamelCase__=3 ,   lowerCamelCase__=True ,   lowerCamelCase__=True ,   lowerCamelCase__=3_2 ,   lowerCamelCase__=2 ,   lowerCamelCase__=4 ,   lowerCamelCase__=3_7 ,   lowerCamelCase__="gelu" ,   lowerCamelCase__=0.1 ,   lowerCamelCase__=0.1 ,   lowerCamelCase__=1_0 ,   lowerCamelCase__=0.02 ,   lowerCamelCase__=3 ,   lowerCamelCase__=None ,   ):

      '''simple docstring'''







      UpperCamelCase       =  parent
      UpperCamelCase       =  batch_size
      UpperCamelCase       =  image_size
      UpperCamelCase       =  patch_size
      UpperCamelCase       =  num_channels
      UpperCamelCase       =  is_training
      UpperCamelCase       =  use_labels
      UpperCamelCase       =  hidden_size
      UpperCamelCase       =  num_hidden_layers
      UpperCamelCase       =  num_attention_heads
      UpperCamelCase       =  intermediate_size
      UpperCamelCase       =  hidden_act
      UpperCamelCase       =  hidden_dropout_prob
      UpperCamelCase       =  attention_probs_dropout_prob
      UpperCamelCase       =  type_sequence_label_size
      UpperCamelCase       =  initializer_range
      UpperCamelCase       =  scope

      # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
      UpperCamelCase       =  (image_size // patch_size) ** 2
      UpperCamelCase       =  num_patches + 1







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

      UpperCamelCase       =  None
      if self.use_labels:
         UpperCamelCase       =  ids_tensor([self.batch_size] ,   self.type_sequence_label_size   )

      UpperCamelCase       =  self.get_config()

      return config, pixel_values, labels







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return ViTConfig(
          image_size=self.image_size ,   patch_size=self.patch_size ,   num_channels=self.num_channels ,   hidden_size=self.hidden_size ,   num_hidden_layers=self.num_hidden_layers ,   num_attention_heads=self.num_attention_heads ,   intermediate_size=self.intermediate_size ,   hidden_act=self.hidden_act ,   hidden_dropout_prob=self.hidden_dropout_prob ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob ,   is_decoder=lowerCamelCase__ ,   initializer_range=self.initializer_range ,   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  TFViTModel(config=lowerCamelCase__   )
      UpperCamelCase       =  model(lowerCamelCase__ ,   training=lowerCamelCase__   )
      self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size)   )

      # Test with an image with different size than the one specified in config.
      UpperCamelCase       =  self.image_size // 2
      UpperCamelCase       =  pixel_values[:, :, :image_size, :image_size]
      UpperCamelCase       =  model(lowerCamelCase__ ,   interpolate_pos_encoding=lowerCamelCase__ ,   training=lowerCamelCase__   )
      UpperCamelCase       =  (image_size // self.patch_size) ** 2 + 1
      self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, seq_length, self.hidden_size)   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.type_sequence_label_size
      UpperCamelCase       =  TFViTForImageClassification(lowerCamelCase__   )
      UpperCamelCase       =  model(lowerCamelCase__ ,   labels=lowerCamelCase__ ,   training=lowerCamelCase__   )
      self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)   )

      # Test with an image with different size than the one specified in config.
      UpperCamelCase       =  self.image_size // 2
      UpperCamelCase       =  pixel_values[:, :, :image_size, :image_size]
      UpperCamelCase       =  model(lowerCamelCase__ ,   interpolate_pos_encoding=lowerCamelCase__ ,   training=lowerCamelCase__   )
      self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)   )

      # test greyscale images
      UpperCamelCase       =  1
      UpperCamelCase       =  TFViTForImageClassification(lowerCamelCase__   )
      UpperCamelCase       =  floats_tensor([self.batch_size, 1, self.image_size, self.image_size]   )
      UpperCamelCase       =  model(lowerCamelCase__   )
      self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.prepare_config_and_inputs()
      UpperCamelCase       =  config_and_inputs
      UpperCamelCase       =  {'pixel_values': pixel_values}
      return config, inputs_dict




@require_tf
class 						lowercase__	(  __A,					__A,					unittest.TestCase   ):






   '''simple docstring'''







   _snake_case		   =							(TFViTModel, TFViTForImageClassification) if is_tf_available() else ()
   _snake_case		   =							(
       {'''feature-extraction''': TFViTModel, '''image-classification''': TFViTForImageClassification}
       if is_tf_available()
       else {}
   )

   _snake_case		   =							False
   _snake_case		   =							False
   _snake_case		   =							False







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  TFViTModelTester(self   )
      UpperCamelCase       =  ConfigTester(self ,   config_class=lowerCamelCase__ ,   has_text_modality=lowerCamelCase__ ,   hidden_size=3_7   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      self.config_tester.run_common_tests()







   @unittest.skip(reason='''ViT does not use inputs_embeds'''   )
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      pass







   @unittest.skip(reason='''ViT does not use inputs_embeds'''   )
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      pass







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
         UpperCamelCase       =  model_class(lowerCamelCase__   )
         self.assertIsInstance(model.get_input_embeddings() ,   (tf.keras.layers.Layer)   )
         UpperCamelCase       =  model.get_output_embeddings()
         self.assertTrue(x is None or isinstance(lowerCamelCase__ ,   tf.keras.layers.Layer   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
         UpperCamelCase       =  model_class(lowerCamelCase__   )
         UpperCamelCase       =  inspect.signature(model.call   )
         # signature.parameters is an OrderedDict => so arg_names order is deterministic
         UpperCamelCase       =  [*signature.parameters.keys()]

         UpperCamelCase       =  ['pixel_values']
         self.assertListEqual(arg_names[:1] ,   lowerCamelCase__   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowerCamelCase__   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_image_classification(*lowerCamelCase__   )







   @slow
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  TFViTModel.from_pretrained('''google/vit-base-patch16-224'''   )
      self.assertIsNotNone(lowerCamelCase__   )




def  __snake_case     (    ):
   UpperCamelCase       =  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
   return image


@require_tf
@require_vision
class 						lowercase__	(  unittest.TestCase   ):






   '''simple docstring'''







   @cached_property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return ViTImageProcessor.from_pretrained('''google/vit-base-patch16-224'''   ) if is_vision_available() else None







   @slow
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  TFViTForImageClassification.from_pretrained('''google/vit-base-patch16-224'''   )

      UpperCamelCase       =  self.default_image_processor
      UpperCamelCase       =  prepare_img()
      UpperCamelCase       =  image_processor(images=lowerCamelCase__ ,   return_tensors='''tf'''   )

      # forward pass
      UpperCamelCase       =  model(**lowerCamelCase__   )

      # verify the logits
      UpperCamelCase       =  tf.TensorShape((1, 1_0_0_0)   )
      self.assertEqual(outputs.logits.shape ,   lowerCamelCase__   )

      UpperCamelCase       =  tf.constant([-0.2744, 0.8215, -0.0836]   )

      tf.debugging.assert_near(outputs.logits[0, :3] ,   lowerCamelCase__ ,   atol=1e-4   )




'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case_ :  Optional[int]     =					logging.get_logger(__name__)

snake_case_ :  int     =					'▁'

snake_case_ :  str     =					{'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

snake_case_ :  Optional[Any]     =					{
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

snake_case_ :  Dict     =					{'vinai/bartpho-syllable': 1_024}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="<mask>" ,   lowerCamelCase__ = None ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  AddedToken(lowerCamelCase__ ,   lstrip=lowerCamelCase__ ,   rstrip=lowerCamelCase__   ) if isinstance(lowerCamelCase__ ,   lowerCamelCase__   ) else mask_token

      UpperCamelCase       =  {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   sep_token=lowerCamelCase__ ,   cls_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   mask_token=lowerCamelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  monolingual_vocab_file
      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(str(lowerCamelCase__   )   )

      # Load the reduced vocab

      # Keep order of special tokens for backward compatibility
      UpperCamelCase       =  {}
      UpperCamelCase       =  0
      for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
         if str(lowerCamelCase__   ) not in self.fairseq_tokens_to_ids:
            UpperCamelCase       =  cnt
            cnt += 1
      with open(lowerCamelCase__ ,   '''r''' ,   encoding='''utf-8'''   ) as f:
         for line in f.readlines():
            UpperCamelCase       =  line.strip().split()[0]
            UpperCamelCase       =  len(self.fairseq_tokens_to_ids   )
      if str(lowerCamelCase__   ) not in self.fairseq_tokens_to_ids:
         UpperCamelCase       =  len(self.fairseq_tokens_to_ids   )

      UpperCamelCase       =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}







   def __getstate__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.__dict__.copy()
      UpperCamelCase       =  None
      UpperCamelCase       =  self.sp_model.serialized_model_proto()
      return state







   def __setstate__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''   ):
         UpperCamelCase       =  {}

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]
      UpperCamelCase       =  [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase__ ,   token_ids_a=lowerCamelCase__ ,   already_has_special_tokens=lowerCamelCase__   )

      if token_ids_a is None:
         return [1] + ([0] * len(lowerCamelCase__   )) + [1]
      return [1] + ([0] * len(lowerCamelCase__   )) + [1, 1] + ([0] * len(lowerCamelCase__   )) + [1]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      UpperCamelCase       =  [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return len(self.fairseq_ids_to_tokens   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {self.convert_ids_to_tokens(lowerCamelCase__   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.encode(lowerCamelCase__ ,   out_type=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
      else:
         return self.unk_token_id







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.fairseq_ids_to_tokens[index]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  ''''''.join(lowerCamelCase__   ).replace(lowerCamelCase__ ,   ''' '''   ).strip()
      return out_string







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file'''] ,   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.vocab_file   ):
         with open(lowerCamelCase__ ,   '''wb'''   ) as fi:
            UpperCamelCase       =  self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase__   )

      if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
          lowerCamelCase__   ) and os.path.isfile(self.monolingual_vocab_file   ):
         copyfile(self.monolingual_vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.monolingual_vocab_file   ):
         with open(lowerCamelCase__ ,   '''w''' ,   encoding='''utf-8'''   ) as fp:
            for token in self.fairseq_tokens_to_ids:
               if token not in self.all_special_tokens:
                  fp.write(f'{str(lowerCamelCase__   )} \n'   )

      return out_vocab_file, out_monolingual_vocab_file





