
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCamelCase_  :       Dict					      =     logging.get_logger(__name__)

lowerCamelCase_  :       List[Any]					      =     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase_  :       List[Any]					      =     {
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase_  :       str					      =     {
    """openai/clip-vit-base-patch32""": 77,
}

class 				_UpperCamelCase (    _A     ):






			'''simple docstring'''

			__UpperCamelCase					:  List[str]    					=       VOCAB_FILES_NAMES
			__UpperCamelCase					:  Optional[Any]    					=       PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase					:  Dict    					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase					:  Optional[int]    					=       ["""input_ids""", """attention_mask"""]
			__UpperCamelCase					:  Optional[Any]    					=       CLIPTokenizer

			def __init__(				self       :				int  ,		snake_case_       :				List[str]=None  ,		snake_case_       :				Tuple=None  ,		snake_case_       :				int=None  ,		snake_case_       :				Optional[int]="<|endoftext|>"  ,		snake_case_       :				Union[str, Any]="<|startoftext|>"  ,		snake_case_       :				Dict="<|endoftext|>"  ,		snake_case_       :				List[Any]="<|endoftext|>"  ,		**snake_case_       :				int  ,		):
							super().__init__(
							    snake_case_  ,		snake_case_  ,		tokenizer_file=snake_case_  ,		unk_token=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		pad_token=snake_case_  ,		**snake_case_  ,		)

							if not isinstance(self.backend_tokenizer.pre_tokenizer  ,		pre_tokenizers.Sequence					):
											raise ValueError(
											    """The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"""
											    """ heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"""
											    """ to be compatible with this version.The easiest way to do so is"""
											    """ `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"""
											    """ to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"""
											    """ transformers."""					)

							self._wrap_decode_method_backend_tokenizer()

			def 					lowerCAmelCase__		(				self       :				Optional[Any]					):
							UpperCamelCase_:							Dict		=  self.backend_tokenizer.decode

							def new_decode_method(*snake_case_       :				str  ,		**snake_case_       :				int					):
											UpperCamelCase_:							List[Any]		=  orig_decode_method(*snake_case_  ,		**snake_case_					)
											UpperCamelCase_:							Dict		=  text.replace(self.backend_tokenizer.model.end_of_word_suffix  ,		""" """					).strip()
											return text

							UpperCamelCase_:							Optional[int]		=  new_decode_method

			def 					lowerCAmelCase__		(				self       :				Union[str, Any]  ,		snake_case_       :				List[int]  ,		snake_case_       :				Optional[List[int]] = None					):
							UpperCamelCase_:							Tuple		=  [self.bos_token_id]
							UpperCamelCase_:							List[str]		=  [self.eos_token_id]

							if token_ids_a is None:
											return bos_token + token_ids_a + eos_token
							return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token

			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				List[int]  ,		snake_case_       :				Optional[List[int]] = None					):
							UpperCamelCase_:							Optional[int]		=  [self.bos_token_id]
							UpperCamelCase_:							Optional[Any]		=  [self.eos_token_id]

							if token_ids_a is None:
											return len(bos_token + token_ids_a + eos_token					) * [0]
							return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token					) * [0]




			def 					lowerCAmelCase__		(				self       :				List[str]  ,		snake_case_       :				str  ,		snake_case_       :				Optional[str] = None					):
							UpperCamelCase_:							Union[str, Any]		=  self._tokenizer.model.save(snake_case_  ,		name=snake_case_					)
							return tuple(snake_case_					)













"""simple docstring"""
from __future__ import annotations

import unittest

from transformers import MPNetConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers.models.mpnet.modeling_tf_mpnet import (
           TFMPNetForMaskedLM,
           TFMPNetForMultipleChoice,
           TFMPNetForQuestionAnswering,
           TFMPNetForSequenceClassification,
           TFMPNetForTokenClassification,
           TFMPNetModel,
       )







class 	a				:



 def __init__(			self							,    _snake_case							,    _snake_case=13							,    _snake_case=7							,    _snake_case=True							,    _snake_case=True							,    _snake_case=False							,    _snake_case=True							,    _snake_case=99							,    _snake_case=64							,    _snake_case=2							,    _snake_case=4							,    _snake_case=64							,    _snake_case="gelu"							,    _snake_case=0.1							,    _snake_case=0.1							,    _snake_case=5_12							,    _snake_case=16							,    _snake_case=2							,    _snake_case=0.02							,    _snake_case=3							,    _snake_case=4							,    _snake_case=None							,    ):







  """simple docstring"""






  lowerCAmelCase  							=      parent
  lowerCAmelCase  							=      batch_size
  lowerCAmelCase  							=      seq_length
  lowerCAmelCase  							=      is_training
  lowerCAmelCase  							=      use_input_mask
  lowerCAmelCase  							=      use_token_type_ids
  lowerCAmelCase  							=      use_labels
  lowerCAmelCase  							=      vocab_size
  lowerCAmelCase  							=      hidden_size
  lowerCAmelCase  							=      num_hidden_layers
  lowerCAmelCase  							=      num_attention_heads
  lowerCAmelCase  							=      intermediate_size
  lowerCAmelCase  							=      hidden_act
  lowerCAmelCase  							=      hidden_dropout_prob
  lowerCAmelCase  							=      attention_probs_dropout_prob
  lowerCAmelCase  							=      max_position_embeddings
  lowerCAmelCase  							=      type_vocab_size
  lowerCAmelCase  							=      type_sequence_label_size
  lowerCAmelCase  							=      initializer_range
  lowerCAmelCase  							=      num_labels
  lowerCAmelCase  							=      num_choices
  lowerCAmelCase  							=      scope



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      ids_tensor([self.batch_size, self.seq_length]							,    self.vocab_size			)

  lowerCAmelCase  							=      None
  if self.use_input_mask:
   lowerCAmelCase  							=      random_attention_mask([self.batch_size, self.seq_length]			)

  lowerCAmelCase  							=      None
  lowerCAmelCase  							=      None
  lowerCAmelCase  							=      None
  if self.use_labels:
   lowerCAmelCase  							=      ids_tensor([self.batch_size]							,    self.type_sequence_label_size			)
   lowerCAmelCase  							=      ids_tensor([self.batch_size, self.seq_length]							,    self.num_labels			)
   lowerCAmelCase  							=      ids_tensor([self.batch_size]							,    self.num_choices			)

  lowerCAmelCase  							=      MPNetConfig(
      vocab_size=self.vocab_size							,    hidden_size=self.hidden_size							,    num_hidden_layers=self.num_hidden_layers							,    num_attention_heads=self.num_attention_heads							,    intermediate_size=self.intermediate_size							,    hidden_act=self.hidden_act							,    hidden_dropout_prob=self.hidden_dropout_prob							,    attention_probs_dropout_prob=self.attention_probs_dropout_prob							,    max_position_embeddings=self.max_position_embeddings							,    initializer_range=self.initializer_range							,    )
  return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



 def 					UpperCamelCase__   (			self							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  lowerCAmelCase  							=      TFMPNetModel(config=_snake_case			)
  lowerCAmelCase  							=      {'input_ids': input_ids, 'attention_mask': input_mask}
  lowerCAmelCase  							=      model(_snake_case			)
  lowerCAmelCase  							=      [input_ids, input_mask]
  lowerCAmelCase  							=      model(_snake_case			)
  self.parent.assertEqual(result.last_hidden_state.shape							,    (self.batch_size, self.seq_length, self.hidden_size)			)



 def 					UpperCamelCase__   (			self							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  lowerCAmelCase  							=      TFMPNetForMaskedLM(config=_snake_case			)
  lowerCAmelCase  							=      {'input_ids': input_ids, 'attention_mask': input_mask}
  lowerCAmelCase  							=      model(_snake_case			)
  self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.vocab_size)			)



 def 					UpperCamelCase__   (			self							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  lowerCAmelCase  							=      TFMPNetForQuestionAnswering(config=_snake_case			)
  lowerCAmelCase  							=      {
      'input_ids': input_ids,
      'attention_mask': input_mask,
  }
  lowerCAmelCase  							=      model(_snake_case			)
  self.parent.assertEqual(result.start_logits.shape							,    (self.batch_size, self.seq_length)			)
  self.parent.assertEqual(result.end_logits.shape							,    (self.batch_size, self.seq_length)			)



 def 					UpperCamelCase__   (			self							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  lowerCAmelCase  							=      self.num_labels
  lowerCAmelCase  							=      TFMPNetForSequenceClassification(_snake_case			)
  lowerCAmelCase  							=      {'input_ids': input_ids, 'attention_mask': input_mask}
  lowerCAmelCase  							=      model(_snake_case			)
  self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.num_labels)			)



 def 					UpperCamelCase__   (			self							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  lowerCAmelCase  							=      self.num_choices
  lowerCAmelCase  							=      TFMPNetForMultipleChoice(_snake_case			)
  lowerCAmelCase  							=      tf.tile(tf.expand_dims(_snake_case							,    1			)							,    (1, self.num_choices, 1)			)
  lowerCAmelCase  							=      tf.tile(tf.expand_dims(_snake_case							,    1			)							,    (1, self.num_choices, 1)			)
  lowerCAmelCase  							=      {
      'input_ids': multiple_choice_inputs_ids,
      'attention_mask': multiple_choice_input_mask,
  }
  lowerCAmelCase  							=      model(_snake_case			)
  self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.num_choices)			)



 def 					UpperCamelCase__   (			self							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case							,    _snake_case			):







  """simple docstring"""






  lowerCAmelCase  							=      self.num_labels
  lowerCAmelCase  							=      TFMPNetForTokenClassification(_snake_case			)
  lowerCAmelCase  							=      {'input_ids': input_ids, 'attention_mask': input_mask}
  lowerCAmelCase  							=      model(_snake_case			)
  self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.num_labels)			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.prepare_config_and_inputs()
  ((lowerCAmelCase)							,(lowerCAmelCase)							,(lowerCAmelCase)							,(lowerCAmelCase)							,(lowerCAmelCase)							,(lowerCAmelCase))  							=      config_and_inputs
  lowerCAmelCase  							=      {'input_ids': input_ids, 'attention_mask': input_mask}
  return config, inputs_dict







@require_tf
class 	a				(					a__					,      a__					,      unittest.TestCase   ):
 snake_case__    	=    (
     (
         TFMPNetForMaskedLM,
         TFMPNetForMultipleChoice,
         TFMPNetForQuestionAnswering,
         TFMPNetForSequenceClassification,
         TFMPNetForTokenClassification,
         TFMPNetModel,
     )
     if is_tf_available()
     else ()
 )
 snake_case__    	=    (
     {
         '''feature-extraction''': TFMPNetModel,
         '''fill-mask''': TFMPNetForMaskedLM,
         '''question-answering''': TFMPNetForQuestionAnswering,
         '''text-classification''': TFMPNetForSequenceClassification,
         '''token-classification''': TFMPNetForTokenClassification,
         '''zero-shot''': TFMPNetForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 snake_case__    	=    False
 snake_case__    	=    False



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      TFMPNetModelTester(self			)
  lowerCAmelCase  							=      ConfigTester(self							,    config_class=_snake_case							,    hidden_size=37			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  self.config_tester.run_common_tests()



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_mpnet_model(*_snake_case			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_mpnet_for_masked_lm(*_snake_case			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_mpnet_for_question_answering(*_snake_case			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_mpnet_for_sequence_classification(*_snake_case			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_mpnet_for_multiple_choice(*_snake_case			)



 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_mpnet_for_token_classification(*_snake_case			)



 @slow
 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  for model_name in ["microsoft/mpnet-base"]:
   lowerCAmelCase  							=      TFMPNetModel.from_pretrained(_snake_case			)
   self.assertIsNotNone(_snake_case			)







@require_tf
class 	a				(					unittest.TestCase   ):



 @slow
 def 					UpperCamelCase__   (			self			):







  """simple docstring"""






  lowerCAmelCase  							=      TFMPNetModel.from_pretrained('microsoft/mpnet-base'			)
  lowerCAmelCase  							=      tf.constant([[0, 1, 2, 3, 4, 5]]			)
  lowerCAmelCase  							=      model(_snake_case			)[0]

  lowerCAmelCase  							=      [1, 6, 7_68]
  self.assertEqual(output.shape							,    _snake_case			)

  lowerCAmelCase  							=      tf.constant(
      [
          [
              [-0.1_067_172, 0.08_216_473, 0.0_024_543],
              [-0.03_465_879, 0.8_354_118, -0.03_252_288],
              [-0.06_569_476, -0.12_424_111, -0.0_494_436],
          ]
      ]			)
  tf.debugging.assert_near(output[:, :3, :3]							,    _snake_case							,    atol=1E-4			)





