





import inspect
import unittest
from typing import List

import numpy as np

from transformers import EfficientFormerConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFEfficientFormerForImageClassification,
         TFEfficientFormerForImageClassificationWithTeacher,
         TFEfficientFormerModel,
     )
     from transformers.models.efficientformer.modeling_tf_efficientformer import (
         TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
     )


if is_vision_available():
     from PIL import Image

     from transformers import EfficientFormerImageProcessor



class 							_UpperCamelCase					:

       def __init__(		self: Tuple					,   _SCREAMING_SNAKE_CASE: List[Any]					,   _SCREAMING_SNAKE_CASE: int = 13					,   _SCREAMING_SNAKE_CASE: int = 64					,   _SCREAMING_SNAKE_CASE: int = 2					,   _SCREAMING_SNAKE_CASE: int = 3					,   _SCREAMING_SNAKE_CASE: int = 3					,   _SCREAMING_SNAKE_CASE: bool = True					,   _SCREAMING_SNAKE_CASE: bool = True					,   _SCREAMING_SNAKE_CASE: int = 128					,   _SCREAMING_SNAKE_CASE: Tuple=[16, 32, 64, 128]					,   _SCREAMING_SNAKE_CASE: int = 7					,   _SCREAMING_SNAKE_CASE: int = 4					,   _SCREAMING_SNAKE_CASE: int = 37					,   _SCREAMING_SNAKE_CASE: str = "gelu"					,   _SCREAMING_SNAKE_CASE: float = 0.1					,   _SCREAMING_SNAKE_CASE: float = 0.1					,   _SCREAMING_SNAKE_CASE: int = 10					,   _SCREAMING_SNAKE_CASE: float = 0.02					,   _SCREAMING_SNAKE_CASE: int = 2					,   _SCREAMING_SNAKE_CASE: int = 1					,   _SCREAMING_SNAKE_CASE: int = 128					,   _SCREAMING_SNAKE_CASE: List[int] = [2, 2, 2, 2]					,   _SCREAMING_SNAKE_CASE: int = 2					,   _SCREAMING_SNAKE_CASE: int = 2					,   )       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =  parent
              UpperCamelCase_          =  batch_size
              UpperCamelCase_          =  image_size
              UpperCamelCase_          =  patch_size
              UpperCamelCase_          =  num_channels
              UpperCamelCase_          =  is_training
              UpperCamelCase_          =  use_labels
              UpperCamelCase_          =  hidden_size
              UpperCamelCase_          =  num_hidden_layers
              UpperCamelCase_          =  num_attention_heads
              UpperCamelCase_          =  intermediate_size
              UpperCamelCase_          =  hidden_act
              UpperCamelCase_          =  hidden_dropout_prob
              UpperCamelCase_          =  attention_probs_dropout_prob
              UpperCamelCase_          =  type_sequence_label_size
              UpperCamelCase_          =  initializer_range
              UpperCamelCase_          =  encoder_stride
              UpperCamelCase_          =  num_attention_outputs
              UpperCamelCase_          =  embed_dim
              UpperCamelCase_          =  embed_dim + 1
              UpperCamelCase_          =  resolution
              UpperCamelCase_          =  depths
              UpperCamelCase_          =  hidden_sizes
              UpperCamelCase_          =  dim
              UpperCamelCase_          =  mlp_expansion_ratio

       def lowercase (		self: Optional[int]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

              UpperCamelCase_          =  None
              if self.use_labels:
                     UpperCamelCase_          =  ids_tensor([self.batch_size]					,   self.type_sequence_label_size		)

              UpperCamelCase_          =  self.get_config()

              return config, pixel_values, labels

       def lowercase (		self: List[str]		)       -> List[str]:




              """simple docstring"""


              return EfficientFormerConfig(
                  image_size=self.image_size					,   patch_size=self.patch_size					,   num_channels=self.num_channels					,   hidden_size=self.hidden_size					,   num_hidden_layers=self.num_hidden_layers					,   num_attention_heads=self.num_attention_heads					,   intermediate_size=self.intermediate_size					,   hidden_act=self.hidden_act					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   is_decoder=_SCREAMING_SNAKE_CASE					,   initializer_range=self.initializer_range					,   encoder_stride=self.encoder_stride					,   resolution=self.resolution					,   depths=self.depths					,   hidden_sizes=self.hidden_sizes					,   dim=self.dim					,   mlp_expansion_ratio=self.mlp_expansion_ratio					,   )

       def lowercase (		self: int					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: List[Any]		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  TFEfficientFormerModel(config=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   training=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)		)

       def lowercase (		self: Tuple					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Any		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  self.type_sequence_label_size
              UpperCamelCase_          =  TFEfficientFormerForImageClassification(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE					,   training=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.type_sequence_label_size)		)

              # test greyscale images
              UpperCamelCase_          =  1
              UpperCamelCase_          =  TFEfficientFormerForImageClassification(_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  floats_tensor([self.batch_size, 1, self.image_size, self.image_size]		)
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.type_sequence_label_size)		)



       def lowercase (		self: Union[str, Any]		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  self.prepare_config_and_inputs()
              UpperCamelCase_       , UpperCamelCase_       , UpperCamelCase_          =  config_and_inputs
              UpperCamelCase_          =  {"pixel_values": pixel_values}
              return config, inputs_dict



@require_tf
class 							_UpperCamelCase					(	lowerCAmelCase_  ,						lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					Tuple         =   (
           (
               TFEfficientFormerModel,
               TFEfficientFormerForImageClassificationWithTeacher,
               TFEfficientFormerForImageClassification,
           )
           if is_tf_available()
           else ()
       )
       _UpperCamelCase				:					List[str]         =   (
           {
               '''feature-extraction''': TFEfficientFormerModel,
               '''image-classification''': (
                   TFEfficientFormerForImageClassification,
                   TFEfficientFormerForImageClassificationWithTeacher,
               ),
           }
           if is_tf_available()
           else {}
       )

       _UpperCamelCase				:					List[Any]         =   False

       _UpperCamelCase				:					Dict         =   False
       _UpperCamelCase				:					Union[str, Any]         =   False
       _UpperCamelCase				:					Optional[int]         =   False
       _UpperCamelCase				:					Tuple         =   False

       def lowercase (		self: Optional[int]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  TFEfficientFormerModelTester(self		)
              UpperCamelCase_          =  ConfigTester(
                  self					,   config_class=_SCREAMING_SNAKE_CASE					,   has_text_modality=_SCREAMING_SNAKE_CASE					,   hidden_size=37		)

       def lowercase (		self: Any		)       -> List[str]:




              """simple docstring"""


              self.config_tester.run_common_tests()

       @unittest.skip(reason="EfficientFormer does not use inputs_embeds"		)
       def lowercase (		self: Tuple		)       -> Any:




              """simple docstring"""


              pass

       @unittest.skip(reason="EfficientFormer does not support input and output embeddings"		)
       def lowercase (		self: List[str]		)       -> Union[str, Any]:




              """simple docstring"""


              pass

       def lowercase (		self: Union[str, Any]		)       -> int:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  inspect.signature(model.call		)
                     # signature.parameters is an OrderedDict => so arg_names order is deterministic
                     UpperCamelCase_          =  [*signature.parameters.keys()]

                     UpperCamelCase_          =  ["pixel_values"]
                     self.assertListEqual(arg_names[:1]					,   _SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Tuple		)       -> List[str]:




              """simple docstring"""


              def check_hidden_states_output(_SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Tuple		):
                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		)

                     UpperCamelCase_          =  model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)					,   training=_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

                     UpperCamelCase_          =  getattr(
                         self.model_tester					,   "expected_num_hidden_layers"					,   self.model_tester.num_hidden_layers + 1		)
                     self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)

                     if hasattr(self.model_tester					,   "encoder_seq_length"		):
                            UpperCamelCase_          =  self.model_tester.encoder_seq_length
                            if hasattr(self.model_tester					,   "chunk_length"		) and self.model_tester.chunk_length > 1:
                                   UpperCamelCase_          =  seq_length * self.model_tester.chunk_length
                     else:
                            UpperCamelCase_          =  self.model_tester.seq_length

                     self.assertListEqual(
                         list(hidden_states[-1].shape[-2:]		)					,   [seq_length, self.model_tester.hidden_size]					,   )

                     if config.is_encoder_decoder:
                            UpperCamelCase_          =  outputs.decoder_hidden_states

                            self.asseretIsInstance(_SCREAMING_SNAKE_CASE					,   (list, tuple)		)
                            self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  getattr(self.model_tester					,   "seq_length"					,   _SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  getattr(self.model_tester					,   "decoder_seq_length"					,   _SCREAMING_SNAKE_CASE		)

                            self.assertListEqual(
                                list(hidden_states[-1].shape[-2:]		)					,   [decoder_seq_length, self.model_tester.hidden_size]					,   )

              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     UpperCamelCase_          =  True
                     check_hidden_states_output(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

                     # check that output_hidden_states also work using config
                     del inputs_dict["output_hidden_states"]
                     UpperCamelCase_          =  True

                     check_hidden_states_output(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[int]=False		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  super()._prepare_for_class(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE					,   return_labels=_SCREAMING_SNAKE_CASE		)

              if return_labels:
                     if model_class.__name__ == "TFEfficientFormerForImageClassificationWithTeacher":
                            del inputs_dict["labels"]

              return inputs_dict

       def lowercase (		self: List[str]		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE		)

       @unittest.skip(reason="EfficientFormer does not implement masked image modeling yet"		)
       def lowercase (		self: Union[str, Any]		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_masked_image_modeling(*_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Tuple		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_image_classification(*_SCREAMING_SNAKE_CASE		)

       @slow
       def lowercase (		self: Dict		)       -> int:




              """simple docstring"""


              for model_name in TF_EFFICIENTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                     UpperCamelCase_          =  TFEfficientFormerModel.from_pretrained(_SCREAMING_SNAKE_CASE		)
                     self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: List[str]		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  True

              UpperCamelCase_          =  getattr(self.model_tester					,   "seq_length"					,   _SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  getattr(self.model_tester					,   "encoder_seq_length"					,   _SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  getattr(self.model_tester					,   "key_length"					,   _SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  getattr(self.model_tester					,   "chunk_length"					,   _SCREAMING_SNAKE_CASE		)

              if chunk_length is not None and hasattr(self.model_tester					,   "num_hashes"		):
                     UpperCamelCase_          =  encoder_seq_length * self.model_tester.num_hashes

              for model_class in self.all_model_classes:
                     UpperCamelCase_          =  True
                     UpperCamelCase_          =  False
                     UpperCamelCase_          =  True
                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		)

                     UpperCamelCase_          =  model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)					,   training=_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
                     self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   self.model_tester.num_attention_outputs		)

                     # check that output_attentions also work using config
                     del inputs_dict["output_attentions"]
                     UpperCamelCase_          =  True
                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)					,   training=_SCREAMING_SNAKE_CASE		)

                     UpperCamelCase_          =  outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
                     self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   self.model_tester.num_attention_outputs		)

                     if chunk_length is not None:
                            self.assertListEqual(
                                list(attentions[0].shape[-4:]		)					,   [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length]					,   )
                     else:
                            self.assertListEqual(
                                list(attentions[0].shape[-3:]		)					,   [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length]					,   )



       def lowercase (		self: Any		)       -> Dict:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     # Prepare our model
                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		)
                     # These are maximally general inputs for the model, with multiple None dimensions
                     # Hopefully this will catch any conditionals that fail for flexible shapes
                     UpperCamelCase_          =  {
                         key: tf.keras.Input(shape=val.shape[1:]					,   dtype=val.dtype					,   name=_SCREAMING_SNAKE_CASE		)
                         for key, val in model.input_signature.items()
                         if key in model.dummy_inputs
                     }
                     UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE		)
                     self.assertTrue(outputs_dict is not None		)




def  lowerCAmelCase_ (							)     ->		int:
       UpperCamelCase_          =  Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"  )
       return image







@require_tf
@require_vision
class 							_UpperCamelCase					(	unittest.TestCase     ):

       @cached_property
       def lowercase (		self: Any		)       -> Optional[Any]:




              """simple docstring"""


              return (
                  EfficientFormerImageProcessor.from_pretrained("snap-research/efficientformer-l1-300"		)
                  if is_vision_available()
                  else None
              )

       @slow
       def lowercase (		self: Optional[Any]		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  TFEfficientFormerForImageClassification.from_pretrained("snap-research/efficientformer-l1-300"		)
              UpperCamelCase_          =  self.default_image_processor
              UpperCamelCase_          =  prepare_img()
              UpperCamelCase_          =  image_processor(images=_SCREAMING_SNAKE_CASE					,   return_tensors="tf"		)
              # forward pass
              UpperCamelCase_          =  model(**_SCREAMING_SNAKE_CASE					,   training=_SCREAMING_SNAKE_CASE		)
              # verify the logits
              UpperCamelCase_          =  tf.TensorShape((1, 1000)		)
              self.assertEqual(outputs.logits.shape					,   _SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tf.constant([-0.05_55, 0.48_25, -0.08_52]		)
              self.assertTrue(np.allclose(outputs.logits[0, :3]					,   _SCREAMING_SNAKE_CASE					,   atol=1e-4		)		)



       @slow
       def lowercase (		self: str		)       -> Optional[Any]:




              """simple docstring"""


              UpperCamelCase_          =  TFEfficientFormerForImageClassificationWithTeacher.from_pretrained(
                  "snap-research/efficientformer-l1-300"		)
              UpperCamelCase_          =  self.default_image_processor
              UpperCamelCase_          =  prepare_img()
              UpperCamelCase_          =  image_processor(images=_SCREAMING_SNAKE_CASE					,   return_tensors="tf"		)
              # forward pass
              UpperCamelCase_          =  model(**_SCREAMING_SNAKE_CASE					,   training=_SCREAMING_SNAKE_CASE		)
              # verify the logits
              UpperCamelCase_          =  tf.TensorShape((1, 1000)		)
              self.assertEqual(outputs.logits.shape					,   _SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tf.constant([-0.13_12, 0.43_53, -1.04_99]		)
              self.assertTrue(np.allclose(outputs.logits[0, :3]					,   _SCREAMING_SNAKE_CASE					,   atol=1e-4		)		)








import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{'vocab_file': 'spiece.model'}

_UpperCAmelCase        =				{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

_UpperCAmelCase        =				{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
_UpperCAmelCase        =				0
_UpperCAmelCase        =				1
_UpperCAmelCase        =				2
_UpperCAmelCase        =				3
_UpperCAmelCase        =				4



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					List[str]         =   VOCAB_FILES_NAMES
       _UpperCamelCase				:					int         =   PRETRAINED_VOCAB_FILES_MAP
       _UpperCamelCase				:					Any         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _UpperCamelCase				:					Dict         =   '''left'''

       def __init__(		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Dict					,   _SCREAMING_SNAKE_CASE: Optional[int]=False					,   _SCREAMING_SNAKE_CASE: List[Any]=True					,   _SCREAMING_SNAKE_CASE: List[str]=False					,   _SCREAMING_SNAKE_CASE: Any="<s>"					,   _SCREAMING_SNAKE_CASE: Dict="</s>"					,   _SCREAMING_SNAKE_CASE: List[str]="<unk>"					,   _SCREAMING_SNAKE_CASE: Tuple="<sep>"					,   _SCREAMING_SNAKE_CASE: Optional[int]="<pad>"					,   _SCREAMING_SNAKE_CASE: int="<cls>"					,   _SCREAMING_SNAKE_CASE: Optional[Any]="<mask>"					,   _SCREAMING_SNAKE_CASE: List[str]=["<eop>", "<eod>"]					,   _SCREAMING_SNAKE_CASE: Optional[Dict[str, Any]] = None					,   **_SCREAMING_SNAKE_CASE: Optional[Any]					,   )       -> None:




              """simple docstring"""


              UpperCamelCase_          =  AddedToken(_SCREAMING_SNAKE_CASE					,   lstrip=_SCREAMING_SNAKE_CASE					,   rstrip=_SCREAMING_SNAKE_CASE		) if isinstance(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		) else mask_token

              UpperCamelCase_          =  {} if sp_model_kwargs is None else sp_model_kwargs

              super().__init__(
                  do_lower_case=_SCREAMING_SNAKE_CASE					,   remove_space=_SCREAMING_SNAKE_CASE					,   keep_accents=_SCREAMING_SNAKE_CASE					,   bos_token=_SCREAMING_SNAKE_CASE					,   eos_token=_SCREAMING_SNAKE_CASE					,   unk_token=_SCREAMING_SNAKE_CASE					,   sep_token=_SCREAMING_SNAKE_CASE					,   pad_token=_SCREAMING_SNAKE_CASE					,   cls_token=_SCREAMING_SNAKE_CASE					,   mask_token=_SCREAMING_SNAKE_CASE					,   additional_special_tokens=_SCREAMING_SNAKE_CASE					,   sp_model_kwargs=self.sp_model_kwargs					,   **_SCREAMING_SNAKE_CASE					,   )

              UpperCamelCase_          =  3

              UpperCamelCase_          =  do_lower_case
              UpperCamelCase_          =  remove_space
              UpperCamelCase_          =  keep_accents
              UpperCamelCase_          =  vocab_file

              UpperCamelCase_          =  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
              self.sp_model.Load(_SCREAMING_SNAKE_CASE		)

       @property
       def lowercase (		self: List[Any]		)       -> Dict:




              """simple docstring"""


              return len(self.sp_model		)

       def lowercase (		self: List[Any]		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
              vocab.update(self.added_tokens_encoder		)
              return vocab

       def __getstate__(		self: int		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  self.__dict__.copy()
              UpperCamelCase_          =  None
              return state

       def __setstate__(		self: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[int]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  d

              # for backward compatibility
              if not hasattr(self					,   "sp_model_kwargs"		):
                     UpperCamelCase_          =  {}

              UpperCamelCase_          =  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
              self.sp_model.Load(self.vocab_file		)

       def lowercase (		self: Any					,   _SCREAMING_SNAKE_CASE: List[str]		)       -> int:




              """simple docstring"""


              if self.remove_space:
                     UpperCamelCase_          =  " ".join(inputs.strip().split()		)
              else:
                     UpperCamelCase_          =  inputs
              UpperCamelCase_          =  outputs.replace("``"					,   "\""		).replace("''"					,   "\""		)

              if not self.keep_accents:
                     UpperCamelCase_          =  unicodedata.normalize("NFKD"					,   _SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  "".join([c for c in outputs if not unicodedata.combining(_SCREAMING_SNAKE_CASE		)]		)
              if self.do_lower_case:
                     UpperCamelCase_          =  outputs.lower()

              return outputs

       def lowercase (		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: str		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  self.preprocess_text(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  self.sp_model.encode(_SCREAMING_SNAKE_CASE					,   out_type=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  []
              for piece in pieces:
                     if len(_SCREAMING_SNAKE_CASE		) > 1 and piece[-1] == str(","		) and piece[-2].isdigit():
                            UpperCamelCase_          =  self.sp_model.EncodeAsPieces(piece[:-1].replace(_SCREAMING_SNAKE_CASE					,   ""		)		)
                            if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                   if len(cur_pieces[0]		) == 1:
                                          UpperCamelCase_          =  cur_pieces[1:]
                                   else:
                                          UpperCamelCase_          =  cur_pieces[0][1:]
                            cur_pieces.append(piece[-1]		)
                            new_pieces.extend(_SCREAMING_SNAKE_CASE		)
                     else:
                            new_pieces.append(_SCREAMING_SNAKE_CASE		)

              return new_pieces

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: str		)       -> Any:




              """simple docstring"""


              return self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: List[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]		)       -> str:




              """simple docstring"""


              return self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: List[Any]					,   _SCREAMING_SNAKE_CASE: Tuple		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  "".join(_SCREAMING_SNAKE_CASE		).replace(_SCREAMING_SNAKE_CASE					,   " "		).strip()
              return out_string

       def lowercase (		self: int					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: bool = False					,   _SCREAMING_SNAKE_CASE: bool = None					,   _SCREAMING_SNAKE_CASE: bool = True					,   **_SCREAMING_SNAKE_CASE: Dict					,   )       -> str:




              """simple docstring"""


              UpperCamelCase_          =  kwargs.pop("use_source_tokenizer"					,   _SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE					,   skip_special_tokens=_SCREAMING_SNAKE_CASE		)

              # To avoid mixing byte-level and unicode for byte-level BPT
              # we need to build string separately for added tokens and byte-level tokens
              # cf. https://github.com/huggingface/transformers/issues/1133
              UpperCamelCase_          =  []
              UpperCamelCase_          =  []
              for token in filtered_tokens:
                     if skip_special_tokens and token in self.all_special_ids:
                            continue
                     if token in self.added_tokens_encoder:
                            if current_sub_text:
                                   sub_texts.append(self.convert_tokens_to_string(_SCREAMING_SNAKE_CASE		)		)
                                   UpperCamelCase_          =  []
                            sub_texts.append(_SCREAMING_SNAKE_CASE		)
                     else:
                            current_sub_text.append(_SCREAMING_SNAKE_CASE		)
              if current_sub_text:
                     sub_texts.append(self.convert_tokens_to_string(_SCREAMING_SNAKE_CASE		)		)

              # Mimic the behavior of the Rust tokenizer:
              # By default, there are no spaces between special tokens
              UpperCamelCase_          =  "".join(_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  (
                  clean_up_tokenization_spaces
                  if clean_up_tokenization_spaces is not None
                  else self.clean_up_tokenization_spaces
              )
              if clean_up_tokenization_spaces:
                     UpperCamelCase_          =  self.clean_up_tokenization(_SCREAMING_SNAKE_CASE		)
                     return clean_text
              else:
                     return text

       def lowercase (		self: List[Any]					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None		)       -> List[int]:




              """simple docstring"""


              UpperCamelCase_          =  [self.sep_token_id]
              UpperCamelCase_          =  [self.cls_token_id]
              if token_ids_a is None:
                     return token_ids_a + sep + cls
              return token_ids_a + sep + token_ids_a + sep + cls

       def lowercase (		self: Dict					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None					,   _SCREAMING_SNAKE_CASE: bool = False		)       -> List[int]:




              """simple docstring"""


              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=_SCREAMING_SNAKE_CASE					,   token_ids_a=_SCREAMING_SNAKE_CASE					,   already_has_special_tokens=_SCREAMING_SNAKE_CASE		)

              if token_ids_a is not None:
                     return ([0] * len(_SCREAMING_SNAKE_CASE		)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE		)) + [1, 1]
              return ([0] * len(_SCREAMING_SNAKE_CASE		)) + [1, 1]

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None		)       -> List[int]:




              """simple docstring"""


              UpperCamelCase_          =  [self.sep_token_id]
              UpperCamelCase_          =  [2]

              if token_ids_a is None:
                     return len(token_ids_a + sep		) * [0] + cls_segment_id
              return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id



       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: Optional[str] = None		)       -> Tuple[str]:




              """simple docstring"""


              if not os.path.isdir(_SCREAMING_SNAKE_CASE		):
                     logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
                     return
              UpperCamelCase_          =  os.path.join(
                  _SCREAMING_SNAKE_CASE					,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

              if os.path.abspath(self.vocab_file		) != os.path.abspath(_SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
                     copyfile(self.vocab_file					,   _SCREAMING_SNAKE_CASE		)
              elif not os.path.isfile(self.vocab_file		):
                     with open(_SCREAMING_SNAKE_CASE					,   "wb"		) as fi:
                            UpperCamelCase_          =  self.sp_model.serialized_model_proto()
                            fi.write(_SCREAMING_SNAKE_CASE		)

              return (out_vocab_file,)

