





import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch
   from torch import nn

   from transformers import (
       FocalNetBackbone,
       FocalNetForImageClassification,
       FocalNetForMaskedImageModeling,
       FocalNetModel,
   )
   from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
   from PIL import Image

   from transformers import AutoImageProcessor




class 			__UpperCamelCase    :





     def __init__(     self				,							_UpperCamelCase				,							_UpperCamelCase=13				,							_UpperCamelCase=32				,							_UpperCamelCase=2				,							_UpperCamelCase=3				,							_UpperCamelCase=16				,							_UpperCamelCase=[32, 64, 128]				,							_UpperCamelCase=[1, 2, 1]				,							_UpperCamelCase=[2, 2, 4]				,							_UpperCamelCase=2				,							_UpperCamelCase=2.0				,							_UpperCamelCase=True				,							_UpperCamelCase=0.0				,							_UpperCamelCase=0.0				,							_UpperCamelCase=0.1				,							_UpperCamelCase="gelu"				,							_UpperCamelCase=False				,							_UpperCamelCase=True				,							_UpperCamelCase=0.02				,							_UpperCamelCase=1e-5				,							_UpperCamelCase=True				,							_UpperCamelCase=None				,							_UpperCamelCase=True				,							_UpperCamelCase=10				,							_UpperCamelCase=8				,							_UpperCamelCase=["stage1", "stage2"]				,							_UpperCamelCase=[1, 2]				,							):
        _UpperCAmelCase           =   parent
        _UpperCAmelCase           =   batch_size
        _UpperCAmelCase           =   image_size
        _UpperCAmelCase           =   patch_size
        _UpperCAmelCase           =   num_channels
        _UpperCAmelCase           =   embed_dim
        _UpperCAmelCase           =   hidden_sizes
        _UpperCAmelCase           =   depths
        _UpperCAmelCase           =   num_heads
        _UpperCAmelCase           =   window_size
        _UpperCAmelCase           =   mlp_ratio
        _UpperCAmelCase           =   qkv_bias
        _UpperCAmelCase           =   hidden_dropout_prob
        _UpperCAmelCase           =   attention_probs_dropout_prob
        _UpperCAmelCase           =   drop_path_rate
        _UpperCAmelCase           =   hidden_act
        _UpperCAmelCase           =   use_absolute_embeddings
        _UpperCAmelCase           =   patch_norm
        _UpperCAmelCase           =   layer_norm_eps
        _UpperCAmelCase           =   initializer_range
        _UpperCAmelCase           =   is_training
        _UpperCAmelCase           =   scope
        _UpperCAmelCase           =   use_labels
        _UpperCAmelCase           =   type_sequence_label_size
        _UpperCAmelCase           =   encoder_stride
        _UpperCAmelCase           =   out_features
        _UpperCAmelCase           =   out_indices





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

        _UpperCAmelCase           =   None
        if self.use_labels:
           _UpperCAmelCase           =   ids_tensor([self.batch_size]				,							self.type_sequence_label_size					)

        _UpperCAmelCase           =   self.get_config()

        return config, pixel_values, labels





     def 		UpperCamelCase(     self					):
        return FocalNetConfig(
            image_size=self.image_size				,							patch_size=self.patch_size				,							num_channels=self.num_channels				,							embed_dim=self.embed_dim				,							hidden_sizes=self.hidden_sizes				,							depths=self.depths				,							num_heads=self.num_heads				,							window_size=self.window_size				,							mlp_ratio=self.mlp_ratio				,							qkv_bias=self.qkv_bias				,							hidden_dropout_prob=self.hidden_dropout_prob				,							attention_probs_dropout_prob=self.attention_probs_dropout_prob				,							drop_path_rate=self.drop_path_rate				,							hidden_act=self.hidden_act				,							use_absolute_embeddings=self.use_absolute_embeddings				,							path_norm=self.patch_norm				,							layer_norm_eps=self.layer_norm_eps				,							initializer_range=self.initializer_range				,							encoder_stride=self.encoder_stride				,							out_features=self.out_features				,							out_indices=self.out_indices				,							)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   FocalNetModel(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase					)

        _UpperCAmelCase           =   ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths					) - 1))
        _UpperCAmelCase           =   int(config.embed_dim * 2 ** (len(config.depths					) - 1)					)

        self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, expected_seq_len, expected_dim)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   FocalNetBackbone(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase					)

        # verify feature maps
        self.parent.assertEqual(len(result.feature_maps					)				,							len(config.out_features					)					)
        self.parent.assertListEqual(list(result.feature_maps[0].shape					)				,							[self.batch_size, self.image_size, 8, 8]					)

        # verify channels
        self.parent.assertEqual(len(model.channels					)				,							len(config.out_features					)					)
        self.parent.assertListEqual(model.channels				,							config.hidden_sizes[:-1]					)

        # verify backbone works with out_features=None
        _UpperCAmelCase           =   None
        _UpperCAmelCase           =   FocalNetBackbone(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase					)

        # verify feature maps
        self.parent.assertEqual(len(result.feature_maps					)				,							1					)
        self.parent.assertListEqual(list(result.feature_maps[0].shape					)				,							[self.batch_size, self.image_size * 2, 4, 4]					)

        # verify channels
        self.parent.assertEqual(len(model.channels					)				,							1					)
        self.parent.assertListEqual(model.channels				,							[config.hidden_sizes[-1]]					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   FocalNetForMaskedImageModeling(config=_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase					)
        self.parent.assertEqual(
            result.reconstruction.shape				,							(self.batch_size, self.num_channels, self.image_size, self.image_size)					)

        # test greyscale images
        _UpperCAmelCase           =   1
        _UpperCAmelCase           =   FocalNetForMaskedImageModeling(_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()

        _UpperCAmelCase           =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]					)
        _UpperCAmelCase           =   model(_UpperCamelCase					)
        self.parent.assertEqual(result.reconstruction.shape				,							(self.batch_size, 1, self.image_size, self.image_size)					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   self.type_sequence_label_size
        _UpperCAmelCase           =   FocalNetForImageClassification(_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()
        _UpperCAmelCase           =   model(_UpperCamelCase				,							labels=_UpperCamelCase					)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.type_sequence_label_size)					)

        # test greyscale images
        _UpperCAmelCase           =   1
        _UpperCAmelCase           =   FocalNetForImageClassification(_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()

        _UpperCAmelCase           =   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]					)
        _UpperCAmelCase           =   model(_UpperCamelCase					)
        self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.type_sequence_label_size)					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.prepare_config_and_inputs()

        _UpperCAmelCase       ,      _UpperCAmelCase       ,      _UpperCAmelCase           =   config_and_inputs
        _UpperCAmelCase           =   {'''pixel_values''': pixel_values}
        return config, inputs_dict




@require_torch
class 			__UpperCamelCase    (							A__	, A__	, unittest.TestCase    ):
     __A	:  List[str]							  =   (
         (
             FocalNetModel,
             FocalNetForImageClassification,
             FocalNetForMaskedImageModeling,
             FocalNetBackbone,
         )
         if is_torch_available()
         else ()
     )
     __A	:  Union[str, Any]							  =   (
         {"""feature-extraction""": FocalNetModel, """image-classification""": FocalNetForImageClassification}
         if is_torch_available()
         else {}
     )
     __A	:  int							  =   False

     __A	:  int							  =   False
     __A	:  Dict							  =   False
     __A	:  str							  =   False
     __A	:  Tuple							  =   False





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   FocalNetModelTester(self					)
        _UpperCAmelCase           =   ConfigTester(self				,							config_class=_UpperCamelCase				,							embed_dim=37				,							has_text_modality=_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()





     def 		UpperCamelCase(     self					):
        return





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_backbone(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*_UpperCamelCase					)





     @unittest.skip(reason='''FocalNet does not use inputs_embeds'''					)
     def 		UpperCamelCase(     self					):
        pass





     @unittest.skip(reason='''FocalNet does not use feedforward chunking'''					)
     def 		UpperCamelCase(     self					):
        pass





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase       ,      _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes[:-1]:
           _UpperCAmelCase           =   model_class(_UpperCamelCase					)
           self.assertIsInstance(model.get_input_embeddings()				,							(nn.Module)					)
           _UpperCAmelCase           =   model.get_output_embeddings()
           self.assertTrue(x is None or isinstance(_UpperCamelCase				,							nn.Linear					)					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase       ,      _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes[:-1]:
           _UpperCAmelCase           =   model_class(_UpperCamelCase					)
           _UpperCAmelCase           =   inspect.signature(model.forward					)
           # signature.parameters is an OrderedDict => so arg_names order is deterministic
           _UpperCAmelCase           =   [*signature.parameters.keys()]

           _UpperCAmelCase           =   ['''pixel_values''']
           self.assertListEqual(arg_names[:1]				,							_UpperCamelCase					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        _UpperCAmelCase           =   model_class(_UpperCamelCase					)
        model.to(_UpperCamelCase					)
        model.eval()

        with torch.no_grad():
           _UpperCAmelCase           =   model(**self._prepare_for_class(_UpperCamelCase				,							_UpperCamelCase					)					)

        _UpperCAmelCase           =   outputs.hidden_states

        _UpperCAmelCase           =   getattr(
            self.model_tester				,							'''expected_num_hidden_layers'''				,							len(self.model_tester.depths					) + 1					)
        self.assertEqual(len(_UpperCamelCase					)				,							_UpperCamelCase					)

        # FocalNet has a different seq_length
        _UpperCAmelCase           =   (
            config.patch_size
            if isinstance(config.patch_size				,							collections.abc.Iterable					)
            else (config.patch_size, config.patch_size)
        )

        _UpperCAmelCase           =   (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

        self.assertListEqual(
            list(hidden_states[0].shape[-2:]					)				,							[num_patches, self.model_tester.embed_dim]				,							)

        _UpperCAmelCase           =   outputs.reshaped_hidden_states
        self.assertEqual(len(_UpperCamelCase					)				,							_UpperCamelCase					)

        _UpperCAmelCase       ,      _UpperCAmelCase       ,      _UpperCAmelCase       ,      _UpperCAmelCase           =   reshaped_hidden_states[0].shape
        _UpperCAmelCase           =   (
            reshaped_hidden_states[0].view(_UpperCamelCase				,							_UpperCamelCase				,							height * width					).permute(0				,							2				,							1					)
        )
        self.assertListEqual(
            list(reshaped_hidden_states.shape[-2:]					)				,							[num_patches, self.model_tester.embed_dim]				,							)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase       ,      _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs_for_common()

        _UpperCAmelCase           =   (
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size				,							collections.abc.Iterable					)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )

        for model_class in self.all_model_classes[:-1]:
           _UpperCAmelCase           =   True
           self.check_hidden_states_output(_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					)

           # check that output_hidden_states also work using config
           del inputs_dict["output_hidden_states"]
           _UpperCAmelCase           =   True

           self.check_hidden_states_output(_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase       ,      _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs_for_common()
        _UpperCAmelCase           =   3

        _UpperCAmelCase           =   (
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size				,							collections.abc.Iterable					)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )
        _UpperCAmelCase           =   (
            config.patch_size
            if isinstance(config.patch_size				,							collections.abc.Iterable					)
            else (config.patch_size, config.patch_size)
        )

        _UpperCAmelCase           =   image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
        _UpperCAmelCase           =   image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

        for model_class in self.all_model_classes[:-1]:
           _UpperCAmelCase           =   True
           self.check_hidden_states_output(_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							(padded_height, padded_width)					)

           # check that output_hidden_states also work using config
           del inputs_dict["output_hidden_states"]
           _UpperCAmelCase           =   True
           self.check_hidden_states_output(_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							(padded_height, padded_width)					)





     @slow
     def 		UpperCamelCase(     self					):
        for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           _UpperCAmelCase           =   FocalNetModel.from_pretrained(_UpperCamelCase					)
           self.assertIsNotNone(_UpperCamelCase					)





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase       ,      _UpperCAmelCase           =   self.model_tester.prepare_config_and_inputs_for_common()

        _UpperCAmelCase           =   _config_zero_init(_UpperCamelCase					)
        for model_class in self.all_model_classes:
           _UpperCAmelCase           =   model_class(config=_UpperCamelCase					)
           for name, param in model.named_parameters():
              if "embeddings" not in name and param.requires_grad:
                 self.assertIn(
                     ((param.data.mean() * 1e9).round() / 1e9).item()				,							[0.0, 1.0]				,							msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''				,							)




@require_vision
@require_torch
class 			__UpperCamelCase    (							unittest.TestCase    ):





     @cached_property
     def 		UpperCamelCase(     self					):
        # TODO update organization
        return AutoImageProcessor.from_pretrained('''microsoft/focalnet-tiny'''					) if is_vision_available() else None





     @slow
     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   FocalNetForImageClassification.from_pretrained('''microsoft/focalnet-tiny'''					).to(_UpperCamelCase					)
        _UpperCAmelCase           =   self.default_image_processor

        _UpperCAmelCase           =   Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''					)
        _UpperCAmelCase           =   image_processor(images=_UpperCamelCase				,							return_tensors='''pt'''					).to(_UpperCamelCase					)

        # forward pass
        with torch.no_grad():
           _UpperCAmelCase           =   model(**_UpperCamelCase					)

        # verify the logits
        _UpperCAmelCase           =   torch.Size((1, 1000)					)
        self.assertEqual(outputs.logits.shape				,							_UpperCamelCase					)
        _UpperCAmelCase           =   torch.tensor([0.2166, -0.4368, 0.2191]					).to(_UpperCamelCase					)
        self.assertTrue(torch.allclose(outputs.logits[0, :3]				,							_UpperCamelCase				,							atol=1e-4					)					)
        self.assertTrue(outputs.logits.argmax(dim=-1					).item()				,							281					)




@require_torch
class 			__UpperCamelCase    (							A__	, unittest.TestCase    ):
     __A	:  Optional[int]							  =   (FocalNetBackbone,) if is_torch_available() else ()
     __A	:  str							  =   FocalNetConfig

     __A	:  Any							  =   False





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   FocalNetModelTester(self					)






import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
   from ...tokenization_utils_base import TextInput
from ...utils import logging


UpperCAmelCase_           =	logging.get_logger(__name__)

UpperCAmelCase_           =	{"vocab_file": "spiece.model"}

UpperCAmelCase_           =	{
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
UpperCAmelCase_           =	{
    "t5-small": 5_12,
    "t5-base": 5_12,
    "t5-large": 5_12,
    "t5-3b": 5_12,
    "t5-11b": 5_12,
}

UpperCAmelCase_           =	"▁"




class 			__UpperCamelCase    (							A__    ):
     __A	:  Any							  =   VOCAB_FILES_NAMES
     __A	:  Union[str, Any]							  =   PRETRAINED_VOCAB_FILES_MAP
     __A	:  Any							  =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A	:  Tuple							  =   ["""input_ids""", """attention_mask"""]





     def __init__(     self				,							_UpperCamelCase				,							_UpperCamelCase="</s>"				,							_UpperCamelCase="<unk>"				,							_UpperCamelCase="<pad>"				,							_UpperCamelCase=100				,							_UpperCamelCase=None				,							_UpperCamelCase = None				,							_UpperCamelCase=True				,							**_UpperCamelCase				,							):
        # Add extra_ids to the special token list
        if extra_ids > 0 and additional_special_tokens is None:
           _UpperCAmelCase           =   [f'''<extra_id_{i}>''' for i in range(_UpperCamelCase					)]
        elif extra_ids > 0 and additional_special_tokens is not None:
           # Check that we have the right number of extra_id special tokens
           _UpperCAmelCase           =   len(set(filter(lambda _UpperCamelCase					: bool('''extra_id''' in str(_UpperCamelCase					)					)				,							_UpperCamelCase					)					)					)
           if extra_tokens != extra_ids:
              raise ValueError(
                  f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
                  ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                  ''' tokens'''					)
        if legacy:
           logger.warning_once(
               f'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
               ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''					)

        _UpperCAmelCase           =   legacy
        _UpperCAmelCase           =   {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            eos_token=_UpperCamelCase				,							unk_token=_UpperCamelCase				,							pad_token=_UpperCamelCase				,							extra_ids=_UpperCamelCase				,							additional_special_tokens=_UpperCamelCase				,							sp_model_kwargs=self.sp_model_kwargs				,							legacy=_UpperCamelCase				,							**_UpperCamelCase				,							)

        _UpperCAmelCase           =   vocab_file
        _UpperCAmelCase           =   extra_ids

        _UpperCAmelCase           =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
        self.sp_model.Load(_UpperCamelCase					)





     @staticmethod
     def 		UpperCamelCase(     _UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
           _UpperCAmelCase           =   TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
           if init_max_model_length is not None and init_max_model_length != max_model_length:
              return init_max_model_length
           elif init_max_model_length is None:
              warnings.warn(
                  '''This tokenizer was incorrectly instantiated with a model max length of'''
                  f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
                  ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                  ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                  f''' {pretrained_model_name_or_path} automatically truncating your input to'''
                  f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
                  f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
                  ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                  ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''				,							_UpperCamelCase				,							)

        return max_model_length





     @property
     def 		UpperCamelCase(     self					):
        return self.sp_model.get_piece_size() + self._extra_ids





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   {self.convert_ids_to_tokens(_UpperCamelCase					): i for i in range(self.vocab_size					)}
        vocab.update(self.added_tokens_encoder					)
        return vocab





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None				,							_UpperCamelCase = False					):
        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_UpperCamelCase				,							token_ids_a=_UpperCamelCase				,							already_has_special_tokens=_UpperCamelCase					)

        # normal case: some special tokens
        if token_ids_a is None:
           return ([0] * len(_UpperCamelCase					)) + [1]
        return ([0] * len(_UpperCamelCase					)) + [1] + ([0] * len(_UpperCamelCase					)) + [1]





     def 		UpperCamelCase(     self					):
        return list(
            set(filter(lambda _UpperCamelCase					: bool(re.search(R'''<extra_id_\d+>'''				,							_UpperCamelCase					)					) is not None				,							self.additional_special_tokens					)					)					)





     def 		UpperCamelCase(     self					):
        return [self._convert_token_to_id(_UpperCamelCase					) for token in self.get_sentinel_tokens()]





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        if len(_UpperCamelCase					) > 0 and token_ids[-1] == self.eos_token_id:
           warnings.warn(
               f'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
               ''' eos tokens being added.'''					)
           return token_ids
        else:
           return token_ids + [self.eos_token_id]





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None					):
        _UpperCAmelCase           =   [self.eos_token_id]

        if token_ids_a is None:
           return len(token_ids_a + eos					) * [0]
        return len(token_ids_a + eos + token_ids_a + eos					) * [0]





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None					):
        _UpperCAmelCase           =   self._add_eos_if_not_present(_UpperCamelCase					)
        if token_ids_a is None:
           return token_ids_a
        else:
           _UpperCAmelCase           =   self._add_eos_if_not_present(_UpperCamelCase					)
           return token_ids_a + token_ids_a





     def __getstate__(     self					):
        _UpperCAmelCase           =   self.__dict__.copy()
        _UpperCAmelCase           =   None
        return state





     def __setstate__(     self				,							_UpperCamelCase					):
        _UpperCAmelCase           =   d

        # for backward compatibility
        if not hasattr(self				,							'''sp_model_kwargs'''					):
           _UpperCAmelCase           =   {}

        _UpperCAmelCase           =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
        self.sp_model.Load(self.vocab_file					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							**_UpperCamelCase					):
        # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
        # the beginning of the text
        if not self.legacy:
           _UpperCAmelCase           =   SPIECE_UNDERLINE + text.replace(_UpperCamelCase				,							''' '''					)
        return super().tokenize(_UpperCamelCase				,							**_UpperCamelCase					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							**_UpperCamelCase					):
        if not self.legacy:
           _UpperCAmelCase           =   text.startswith(_UpperCamelCase					)
           if is_first:
              _UpperCAmelCase           =   text[1:]

        _UpperCAmelCase           =   self.sp_model.encode(_UpperCamelCase				,							out_type=_UpperCamelCase					)

        if not self.legacy and not is_first and not text.startswith(''' '''					) and tokens[0].startswith(_UpperCamelCase					):
           _UpperCAmelCase           =   ([tokens[0][1:]] if len(tokens[0]					) > 1 else []) + tokens[1:]
        return tokens





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        if token.startswith('''<extra_id_'''					):
           _UpperCAmelCase           =   re.match(R'''<extra_id_(\d+)>'''				,							_UpperCamelCase					)
           _UpperCAmelCase           =   int(match.group(1					)					)
           return self.vocab_size - num - 1
        return self.sp_model.piece_to_id(_UpperCamelCase					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        if index < self.sp_model.get_piece_size():
           _UpperCAmelCase           =   self.sp_model.IdToPiece(_UpperCamelCase					)
        else:
           _UpperCAmelCase           =   f'''<extra_id_{self.vocab_size - 1 - index}>'''
        return token





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        _UpperCAmelCase           =   []
        _UpperCAmelCase           =   ''''''
        _UpperCAmelCase           =   False
        for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
              if not prev_is_special:
                 out_string += " "
              out_string += self.sp_model.decode(_UpperCamelCase					) + token
              _UpperCAmelCase           =   True
              _UpperCAmelCase           =   []
           else:
              current_sub_tokens.append(_UpperCamelCase					)
              _UpperCAmelCase           =   False
        out_string += self.sp_model.decode(_UpperCamelCase					)
        return out_string.strip()





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None					):
        if not os.path.isdir(_UpperCamelCase					):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
           return
        _UpperCAmelCase           =   os.path.join(
            _UpperCamelCase				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

        if os.path.abspath(self.vocab_file					) != os.path.abspath(_UpperCamelCase					) and os.path.isfile(self.vocab_file					):
           copyfile(self.vocab_file				,							_UpperCamelCase					)
        elif not os.path.isfile(self.vocab_file					):
           with open(_UpperCamelCase				,							'''wb'''					) as fi:
              _UpperCAmelCase           =   self.sp_model.serialized_model_proto()
              fi.write(_UpperCamelCase					)

        return (out_vocab_file,)