
'''simple docstring'''



import inspect
import unittest

from datasets import load_dataset
from packaging import version

from transformers import BeitConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch
						from torch import nn

						from transformers import (
						    MODEL_MAPPING,
						    BeitForImageClassification,
						    BeitForMaskedImageModeling,
						    BeitForSemanticSegmentation,
						    BeitModel,
						)
						from transformers.models.beit.modeling_beit import BEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
						import PIL
						from PIL import Image

						from transformers import BeitImageProcessor




class     UpperCAmelCase_	:






						'''simple docstring'''






						def __init__(   self       ,  _lowercase       ,  _lowercase=100       ,  _lowercase=13       ,  _lowercase=30       ,  _lowercase=2       ,  _lowercase=3       ,  _lowercase=True       ,  _lowercase=True       ,  _lowercase=32       ,  _lowercase=4       ,  _lowercase=4       ,  _lowercase=37       ,  _lowercase="gelu"       ,  _lowercase=0.1       ,  _lowercase=0.1       ,  _lowercase=10       ,  _lowercase=0.02       ,  _lowercase=3       ,  _lowercase=None       ,  _lowercase=[0, 1, 2, 3]       ,  ):

												"""simple docstring"""



												_lowerCAmelCase    		=					parent
												_lowerCAmelCase    		=					100
												_lowerCAmelCase    		=					batch_size
												_lowerCAmelCase    		=					image_size
												_lowerCAmelCase    		=					patch_size
												_lowerCAmelCase    		=					num_channels
												_lowerCAmelCase    		=					is_training
												_lowerCAmelCase    		=					use_labels
												_lowerCAmelCase    		=					hidden_size
												_lowerCAmelCase    		=					num_hidden_layers
												_lowerCAmelCase    		=					num_attention_heads
												_lowerCAmelCase    		=					intermediate_size
												_lowerCAmelCase    		=					hidden_act
												_lowerCAmelCase    		=					hidden_dropout_prob
												_lowerCAmelCase    		=					attention_probs_dropout_prob
												_lowerCAmelCase    		=					type_sequence_label_size
												_lowerCAmelCase    		=					initializer_range
												_lowerCAmelCase    		=					scope
												_lowerCAmelCase    		=					out_indices
												_lowerCAmelCase    		=					num_labels

												# in BeiT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
												_lowerCAmelCase    		=					(image_size // patch_size) ** 2
												_lowerCAmelCase    		=					num_patches + 1




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

												_lowerCAmelCase    		=					None
												_lowerCAmelCase    		=					None
												if self.use_labels:
																		_lowerCAmelCase    		=					ids_tensor([self.batch_size]       ,  self.type_sequence_label_size )
																		_lowerCAmelCase    		=					ids_tensor([self.batch_size, self.image_size, self.image_size]       ,  self.num_labels )

												_lowerCAmelCase    		=					self.get_config()

												return config, pixel_values, labels, pixel_labels




						def    _lowercase      (   self ):

												"""simple docstring"""



												return BeitConfig(
												    vocab_size=self.vocab_size       ,  image_size=self.image_size       ,  patch_size=self.patch_size       ,  num_channels=self.num_channels       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  is_decoder=_lowercase       ,  initializer_range=self.initializer_range       ,  out_indices=self.out_indices       ,  )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitModel(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase )
												self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitForMaskedImageModeling(config=_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length - 1, self.vocab_size) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.type_sequence_label_size
												_lowerCAmelCase    		=					BeitForImageClassification(_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase       ,  labels=_lowercase )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.type_sequence_label_size) )

												# test greyscale images
												_lowerCAmelCase    		=					1
												_lowerCAmelCase    		=					BeitForImageClassification(_lowercase )
												model.to(_lowercase )
												model.eval()

												_lowerCAmelCase    		=					floats_tensor([self.batch_size, 1, self.image_size, self.image_size] )
												_lowerCAmelCase    		=					model(_lowercase       ,  labels=_lowercase )
												self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.type_sequence_label_size) )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.num_labels
												_lowerCAmelCase    		=					BeitForSemanticSegmentation(_lowercase )
												model.to(_lowercase )
												model.eval()
												_lowerCAmelCase    		=					model(_lowercase )
												self.parent.assertEqual(
												    result.logits.shape       ,  (self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2) )
												_lowerCAmelCase    		=					model(_lowercase       ,  labels=_lowercase )
												self.parent.assertEqual(
												    result.logits.shape       ,  (self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2) )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.prepare_config_and_inputs()
												_lowerCAmelCase    		=					config_and_inputs
												_lowerCAmelCase    		=					{'pixel_values': pixel_values}
												return config, inputs_dict






@require_torch
class     UpperCAmelCase_	(						__UpperCamelCase			,				__UpperCamelCase			,				unittest.TestCase					):






						'''simple docstring'''






						_lowercase						: Tuple										=  (
						    (BeitModel, BeitForImageClassification, BeitForMaskedImageModeling, BeitForSemanticSegmentation)
						    if is_torch_available()
						    else ()
						)
						_lowercase						: str										=  (
						    {
						        '''feature-extraction''': BeitModel,
						        '''image-classification''': BeitForImageClassification,
						        '''image-segmentation''': BeitForSemanticSegmentation,
						    }
						    if is_torch_available()
						    else {}
						)

						_lowercase						: str										=  False
						_lowercase						: Optional[int]										=  False
						_lowercase						: Tuple										=  False




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitModelTester(self )
												_lowerCAmelCase    		=					ConfigTester(self       ,  config_class=_lowercase       ,  has_text_modality=_lowercase       ,  hidden_size=37 )




						def    _lowercase      (   self ):

												"""simple docstring"""



												self.config_tester.run_common_tests()




						@unittest.skip(reason="""BEiT does not use inputs_embeds""" )
						def    _lowercase      (   self ):

												"""simple docstring"""



												pass




						@require_torch_multi_gpu
						@unittest.skip(reason="""BEiT has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`""" )
						def    _lowercase      (   self ):

												"""simple docstring"""



												pass




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs_for_common()

												for model_class in self.all_model_classes:
																		_lowerCAmelCase    		=					model_class(_lowercase )
																		self.assertIsInstance(model.get_input_embeddings()       ,  (nn.Module) )
																		_lowerCAmelCase    		=					model.get_output_embeddings()
																		self.assertTrue(x is None or isinstance(_lowercase       ,  nn.Linear ) )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs_for_common()

												for model_class in self.all_model_classes:
																		_lowerCAmelCase    		=					model_class(_lowercase )
																		_lowerCAmelCase    		=					inspect.signature(model.forward )
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		_lowerCAmelCase    		=					[*signature.parameters.keys()]

																		_lowerCAmelCase    		=					['pixel_values']
																		self.assertListEqual(arg_names[:1]       ,  _lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_image_classification(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_semantic_segmentation(*_lowercase )




						def    _lowercase      (   self ):

												"""simple docstring"""



												if not self.model_tester.is_training:
																		return

												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs_for_common()
												_lowerCAmelCase    		=					True

												for model_class in self.all_model_classes:
																		# we don't test BeitForMaskedImageModeling
																		if model_class in [*get_values(_lowercase ), BeitForMaskedImageModeling]:
																								continue

																		_lowerCAmelCase    		=					model_class(_lowercase )
																		model.to(_lowercase )
																		model.train()
																		_lowerCAmelCase    		=					self._prepare_for_class(_lowercase       ,  _lowercase       ,  return_labels=_lowercase )
																		_lowerCAmelCase    		=					model(**_lowercase ).loss
																		loss.backward()




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs_for_common()
												if not self.model_tester.is_training:
																		return

												_lowerCAmelCase    		=					False
												_lowerCAmelCase    		=					True

												for model_class in self.all_model_classes:
																		# we don't test BeitForMaskedImageModeling
																		if (
																		    model_class in [*get_values(_lowercase ), BeitForMaskedImageModeling]
																		    or not model_class.supports_gradient_checkpointing
																		):
																								continue

																		_lowerCAmelCase    		=					model_class(_lowercase )
																		model.gradient_checkpointing_enable()
																		model.to(_lowercase )
																		model.train()
																		_lowerCAmelCase    		=					self._prepare_for_class(_lowercase       ,  _lowercase       ,  return_labels=_lowercase )
																		_lowerCAmelCase    		=					model(**_lowercase ).loss
																		loss.backward()




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					self.model_tester.prepare_config_and_inputs_for_common()

												_lowerCAmelCase    		=					_config_zero_init(_lowercase )
												for model_class in self.all_model_classes:
																		_lowerCAmelCase    		=					model_class(config=_lowercase )
																		for name, param in model.named_parameters():
																								# we skip lambda parameters as these require special initial values
																								# determined by config.layer_scale_init_value
																								if "lambda" in name:
																														continue
																								if param.requires_grad:
																														self.assertIn(
																														    ((param.data.mean() * 1e9).round() / 1e9).item()       ,  [0.0, 1.0]       ,  msg=F'Parameter {name} of model {model_class} seems not properly initialized'       ,  )




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												for model_name in BEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		_lowerCAmelCase    		=					BeitModel.from_pretrained(_lowercase )
																		self.assertIsNotNone(_lowercase )






def A ():
						_lowerCAmelCase    		=					Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""						)
						return image





@require_torch
@require_vision
class     UpperCAmelCase_	(						unittest.TestCase					):






						'''simple docstring'''






						@cached_property
						def    _lowercase      (   self ):

												"""simple docstring"""



												return BeitImageProcessor.from_pretrained("""microsoft/beit-base-patch16-224""" ) if is_vision_available() else None




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitForMaskedImageModeling.from_pretrained("""microsoft/beit-base-patch16-224-pt22k""" ).to(_lowercase )

												_lowerCAmelCase    		=					self.default_image_processor
												_lowerCAmelCase    		=					prepare_img()
												_lowerCAmelCase    		=					image_processor(images=_lowercase       ,  return_tensors="""pt""" ).pixel_values.to(_lowercase )

												# prepare bool_masked_pos
												_lowerCAmelCase    		=					torch.ones((1, 196)       ,  dtype=torch.bool ).to(_lowercase )

												# forward pass
												with torch.no_grad():
																		_lowerCAmelCase    		=					model(pixel_values=_lowercase       ,  bool_masked_pos=_lowercase )
												_lowerCAmelCase    		=					outputs.logits

												# verify the logits
												_lowerCAmelCase    		=					torch.Size((1, 196, 8_192) )
												self.assertEqual(logits.shape       ,  _lowercase )

												_lowerCAmelCase    		=					torch.tensor(
												    [[-3.2437, 0.5072, -13.9_174], [-3.2456, 0.4948, -13.9_401], [-3.2033, 0.5121, -13.8_550]] ).to(_lowercase )

												self.assertTrue(torch.allclose(logits[bool_masked_pos][:3, :3]       ,  _lowercase       ,  atol=1e-2 ) )




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitForImageClassification.from_pretrained("""microsoft/beit-base-patch16-224""" ).to(_lowercase )

												_lowerCAmelCase    		=					self.default_image_processor
												_lowerCAmelCase    		=					prepare_img()
												_lowerCAmelCase    		=					image_processor(images=_lowercase       ,  return_tensors="""pt""" ).to(_lowercase )

												# forward pass
												with torch.no_grad():
																		_lowerCAmelCase    		=					model(**_lowercase )
												_lowerCAmelCase    		=					outputs.logits

												# verify the logits
												_lowerCAmelCase    		=					torch.Size((1, 1_000) )
												self.assertEqual(logits.shape       ,  _lowercase )

												_lowerCAmelCase    		=					torch.tensor([-1.2385, -1.0987, -1.0108] ).to(_lowercase )

												self.assertTrue(torch.allclose(logits[0, :3]       ,  _lowercase       ,  atol=1e-4 ) )

												_lowerCAmelCase    		=					281
												self.assertEqual(logits.argmax(-1 ).item()       ,  _lowercase )




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitForImageClassification.from_pretrained("""microsoft/beit-large-patch16-224-pt22k-ft22k""" ).to(
												    _lowercase )

												_lowerCAmelCase    		=					self.default_image_processor
												_lowerCAmelCase    		=					prepare_img()
												_lowerCAmelCase    		=					image_processor(images=_lowercase       ,  return_tensors="""pt""" ).to(_lowercase )

												# forward pass
												with torch.no_grad():
																		_lowerCAmelCase    		=					model(**_lowercase )
												_lowerCAmelCase    		=					outputs.logits

												# verify the logits
												_lowerCAmelCase    		=					torch.Size((1, 21_841) )
												self.assertEqual(logits.shape       ,  _lowercase )

												_lowerCAmelCase    		=					torch.tensor([1.6881, -0.2787, 0.5901] ).to(_lowercase )

												self.assertTrue(torch.allclose(logits[0, :3]       ,  _lowercase       ,  atol=1e-4 ) )

												_lowerCAmelCase    		=					2_396
												self.assertEqual(logits.argmax(-1 ).item()       ,  _lowercase )




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitForSemanticSegmentation.from_pretrained("""microsoft/beit-base-finetuned-ade-640-640""" )
												_lowerCAmelCase    		=					model.to(_lowercase )

												_lowerCAmelCase    		=					BeitImageProcessor(do_resize=_lowercase       ,  size=640       ,  do_center_crop=_lowercase )

												_lowerCAmelCase    		=					load_dataset("""hf-internal-testing/fixtures_ade20k"""       ,  split="""test""" )
												_lowerCAmelCase    		=					Image.open(ds[0]["""file"""] )
												_lowerCAmelCase    		=					image_processor(images=_lowercase       ,  return_tensors="""pt""" ).to(_lowercase )

												# forward pass
												with torch.no_grad():
																		_lowerCAmelCase    		=					model(**_lowercase )
												_lowerCAmelCase    		=					outputs.logits

												# verify the logits
												_lowerCAmelCase    		=					torch.Size((1, 150, 160, 160) )
												self.assertEqual(logits.shape       ,  _lowercase )

												_lowerCAmelCase    		=					version.parse(PIL.__version__ ) < version.parse("""9.0.0""" )

												if is_pillow_less_than_a:
																		_lowerCAmelCase    		=					torch.tensor(
																		    [
																		        [[-4.9225, -2.3954, -3.0522], [-2.8822, -1.0046, -1.7561], [-2.9549, -1.3228, -2.1347]],
																		        [[-5.8168, -3.4129, -4.0778], [-3.8651, -2.2214, -3.0277], [-3.8356, -2.4643, -3.3535]],
																		        [[-0.0078, 3.9952, 4.0754], [2.9856, 4.6944, 5.0035], [3.2413, 4.7813, 4.9969]],
																		    ]       ,  device=_lowercase       ,  )
												else:
																		_lowerCAmelCase    		=					torch.tensor(
																		    [
																		        [[-4.8960, -2.3688, -3.0355], [-2.8478, -0.9836, -1.7418], [-2.9449, -1.3332, -2.1456]],
																		        [[-5.8081, -3.4124, -4.1006], [-3.8561, -2.2081, -3.0323], [-3.8365, -2.4601, -3.3669]],
																		        [[-0.0309, 3.9868, 4.0540], [2.9640, 4.6877, 4.9976], [3.2081, 4.7690, 4.9942]],
																		    ]       ,  device=_lowercase       ,  )

												self.assertTrue(torch.allclose(logits[0, :3, :3, :3]       ,  _lowercase       ,  atol=1e-4 ) )




						@slow
						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					BeitForSemanticSegmentation.from_pretrained("""microsoft/beit-base-finetuned-ade-640-640""" )
												_lowerCAmelCase    		=					model.to(_lowercase )

												_lowerCAmelCase    		=					BeitImageProcessor(do_resize=_lowercase       ,  size=640       ,  do_center_crop=_lowercase )

												_lowerCAmelCase    		=					load_dataset("""hf-internal-testing/fixtures_ade20k"""       ,  split="""test""" )
												_lowerCAmelCase    		=					Image.open(ds[0]["""file"""] )
												_lowerCAmelCase    		=					image_processor(images=_lowercase       ,  return_tensors="""pt""" ).to(_lowercase )

												# forward pass
												with torch.no_grad():
																		_lowerCAmelCase    		=					model(**_lowercase )

												_lowerCAmelCase    		=					outputs.logits.detach().cpu()

												_lowerCAmelCase    		=					image_processor.post_process_semantic_segmentation(outputs=_lowercase       ,  target_sizes=[(500, 300)] )
												_lowerCAmelCase    		=					torch.Size((500, 300) )
												self.assertEqual(segmentation[0].shape       ,  _lowercase )

												_lowerCAmelCase    		=					image_processor.post_process_semantic_segmentation(outputs=_lowercase )
												_lowerCAmelCase    		=					torch.Size((160, 160) )
												self.assertEqual(segmentation[0].shape       ,  _lowercase )







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase							:			List[str]        =							logging.get_logger(__name__)

_UpperCAmelCase							:			List[str]        =							{"""vocab_file""": """spiece.model"""}

_UpperCAmelCase							:			Tuple        =							{
    """vocab_file""": {
        """bert_for_seq_generation""": (
            """https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model"""
        ),
    }
}

_UpperCAmelCase							:			List[str]        =							{"""bert_for_seq_generation""": 5_12}

class   lowerCAmelCase				(   __UpperCamelCase     ):
 UpperCAmelCase__						=						VOCAB_FILES_NAMES
 UpperCAmelCase__						=						PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__						=						[]
 UpperCAmelCase__						=						["""input_ids""", """attention_mask"""]
 def __init__( self    : Union[str, Any]     ,	UpperCAmelCase    : List[Any]     ,	UpperCAmelCase    : int="<s>"     ,	UpperCAmelCase    : Any="</s>"     ,	UpperCAmelCase    : List[Any]="<unk>"     ,	UpperCAmelCase    : Any="<pad>"     ,	UpperCAmelCase    : Dict="<::::>"     ,	UpperCAmelCase    : Optional[Dict[str, Any]] = None     ,	**UpperCAmelCase    : Any     ,	)     ->      None:
     lowerCamelCase__       : str   					=						{} if sp_model_kwargs is None else sp_model_kwargs

     # Add extra_ids to the special token list
     super().__init__(
         bos_token=UpperCAmelCase     ,	eos_token=UpperCAmelCase     ,	unk_token=UpperCAmelCase     ,	pad_token=UpperCAmelCase     ,	sep_token=UpperCAmelCase     ,	sp_model_kwargs=self.sp_model_kwargs     ,	**UpperCAmelCase     ,	)

     lowerCamelCase__       : List[str]   					=						vocab_file

     lowerCamelCase__       : List[str]   					=						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
     self.sp_model.Load(UpperCAmelCase      )
 @property
 def 	A_      ( self    : List[str]      )     ->      Optional[int]:
     return self.sp_model.get_piece_size()
 def 	A_      ( self    : Tuple      )     ->      Any:
     lowerCamelCase__       : Tuple   					=						{self.convert_ids_to_tokens(UpperCAmelCase      ): i for i in range(self.vocab_size      )}
     vocab.update(self.added_tokens_encoder      )
     return vocab
 def __getstate__( self    : Union[str, Any]      )     ->      Optional[Any]:
     lowerCamelCase__       : List[str]   					=						self.__dict__.copy()
     lowerCamelCase__       : List[Any]   					=						None
     return state
 def __setstate__( self    : Dict     ,	UpperCAmelCase    : Optional[int]      )     ->      List[Any]:
     lowerCamelCase__       : Optional[int]   					=						d

     # for backward compatibility
     if not hasattr(self     ,	'sp_model_kwargs'      ):
         lowerCamelCase__       : Dict   					=						{}

     lowerCamelCase__       : Optional[int]   					=						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
     self.sp_model.Load(self.vocab_file      )
 def 	A_      ( self    : Dict     ,	UpperCAmelCase    : str      )     ->      List[str]:
     return self.sp_model.encode(UpperCAmelCase     ,	out_type=UpperCAmelCase      )
 def 	A_      ( self    : str     ,	UpperCAmelCase    : Tuple      )     ->      Union[str, Any]:
     return self.sp_model.piece_to_id(UpperCAmelCase      )
 def 	A_      ( self    : Optional[Any]     ,	UpperCAmelCase    : Tuple      )     ->      Dict:
     lowerCamelCase__       : str   					=						self.sp_model.IdToPiece(UpperCAmelCase      )
     return token
 def 	A_      ( self    : Optional[Any]     ,	UpperCAmelCase    : int      )     ->      Union[str, Any]:
     lowerCamelCase__       : str   					=						[]
     lowerCamelCase__       : List[str]   					=						''
     for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
             out_string += self.sp_model.decode(UpperCAmelCase      ) + token
             lowerCamelCase__       : Tuple   					=						[]
         else:
             current_sub_tokens.append(UpperCAmelCase      )
     out_string += self.sp_model.decode(UpperCAmelCase      )
     return out_string.strip()







 def 	A_      ( self    : List[Any]     ,	UpperCAmelCase    : str     ,	UpperCAmelCase    : Optional[str] = None      )     ->      Tuple[str]:
     if not os.path.isdir(UpperCAmelCase      ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
         return
     lowerCamelCase__       : Any   					=						os.path.join(
         UpperCAmelCase     ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

     if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCAmelCase      ) and os.path.isfile(self.vocab_file      ):
         copyfile(self.vocab_file     ,	UpperCAmelCase      )
     elif not os.path.isfile(self.vocab_file      ):
         with open(UpperCAmelCase     ,	'wb'      ) as fi:
             lowerCamelCase__       : int   					=						self.sp_model.serialized_model_proto()
             fi.write(UpperCAmelCase      )

     return (out_vocab_file,)





