




'''simple docstring'''



from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel







def       _UpperCamelCase			(_lowerCamelCase				:  List[Any] ,						_lowerCamelCase				:  Optional[int] ,						_lowerCamelCase				:  str=None ,						_lowerCamelCase				:  Any=None						)-> List[Any]:




     '''simple docstring'''





     if attention_mask is None:
          __snake_case            =	tf.cast(tf.math.not_equal(_lowerCamelCase ,						config.pad_token_id						) ,						tf.inta						)
     return {"input_ids": input_ids, "attention_mask": attention_mask}





@require_tf
class      lowerCAmelCase :
       __lowercase	:   List[Any]										=      OPTConfig
       __lowercase	:   List[str]										=      {}
       __lowercase	:   Tuple										=      '''gelu'''


       def __init__( self     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE=13     ,	__SCREAMING_SNAKE_CASE=7     ,	__SCREAMING_SNAKE_CASE=True     ,	__SCREAMING_SNAKE_CASE=False     ,	__SCREAMING_SNAKE_CASE=99     ,	__SCREAMING_SNAKE_CASE=16     ,	__SCREAMING_SNAKE_CASE=2     ,	__SCREAMING_SNAKE_CASE=4     ,	__SCREAMING_SNAKE_CASE=4     ,	__SCREAMING_SNAKE_CASE="gelu"     ,	__SCREAMING_SNAKE_CASE=0.1     ,	__SCREAMING_SNAKE_CASE=0.1     ,	__SCREAMING_SNAKE_CASE=20     ,	__SCREAMING_SNAKE_CASE=2     ,	__SCREAMING_SNAKE_CASE=1     ,	__SCREAMING_SNAKE_CASE=0     ,	__SCREAMING_SNAKE_CASE=16     ,	__SCREAMING_SNAKE_CASE=16     ,	)   ->	Tuple:



            '''simple docstring'''




            __snake_case            =	parent
            __snake_case            =	batch_size
            __snake_case            =	seq_length
            __snake_case            =	is_training
            __snake_case            =	use_labels
            __snake_case            =	vocab_size
            __snake_case            =	hidden_size
            __snake_case            =	num_hidden_layers
            __snake_case            =	num_attention_heads
            __snake_case            =	intermediate_size
            __snake_case            =	hidden_act
            __snake_case            =	hidden_dropout_prob
            __snake_case            =	attention_probs_dropout_prob
            __snake_case            =	max_position_embeddings
            __snake_case            =	eos_token_id
            __snake_case            =	pad_token_id
            __snake_case            =	bos_token_id
            __snake_case            =	embed_dim
            __snake_case            =	word_embed_proj_dim
            __snake_case            =	False


       def    lowerCAmelCase   ( self	)   ->	str:



            '''simple docstring'''




            __snake_case            =	ids_tensor([self.batch_size, self.seq_length - 1]     ,	self.vocab_size	)
            __snake_case            =	tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size	)     ,	1	)
            __snake_case            =	tf.concat([input_ids, eos_tensor]     ,	axis=1	)

            __snake_case            =	self.config_cls(
                vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	ffn_dim=self.intermediate_size     ,	dropout=self.hidden_dropout_prob     ,	attention_dropout=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	eos_token_id=self.eos_token_id     ,	bos_token_id=self.bos_token_id     ,	pad_token_id=self.pad_token_id     ,	embed_dim=self.embed_dim     ,	word_embed_proj_dim=self.word_embed_proj_dim     ,	is_encoder_decoder=__SCREAMING_SNAKE_CASE     ,	**self.config_updates     ,	)
            __snake_case            =	prepare_opt_inputs_dict(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)
            return config, inputs_dict




       def    lowerCAmelCase   ( self     ,	__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)   ->	List[str]:



            '''simple docstring'''




            __snake_case            =	TFOPTModel(config=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	inputs_dict['''input_ids''']

            __snake_case            =	input_ids[:1, :]
            __snake_case            =	inputs_dict['''attention_mask'''][:1, :]
            __snake_case            =	1

            # first forward pass
            __snake_case            =	model(__SCREAMING_SNAKE_CASE     ,	attention_mask=__SCREAMING_SNAKE_CASE     ,	use_cache=__SCREAMING_SNAKE_CASE	)

            __snake_case       ,     __snake_case            =	outputs.to_tuple()

            # create hypothetical next token and extent to next_input_ids
            __snake_case            =	ids_tensor((self.batch_size, 3)     ,	config.vocab_size	)
            __snake_case            =	tf.cast(ids_tensor((self.batch_size, 3)     ,	2	)     ,	tf.inta	)

            # append to next input_ids and
            __snake_case            =	tf.concat([input_ids, next_tokens]     ,	axis=-1	)
            __snake_case            =	tf.concat([attention_mask, next_attn_mask]     ,	axis=-1	)

            __snake_case            =	model(__SCREAMING_SNAKE_CASE     ,	attention_mask=__SCREAMING_SNAKE_CASE	)[0]
            __snake_case            =	model(__SCREAMING_SNAKE_CASE     ,	attention_mask=__SCREAMING_SNAKE_CASE     ,	past_key_values=__SCREAMING_SNAKE_CASE	)[0]

            self.parent.assertEqual(next_tokens.shape[1]     ,	output_from_past.shape[1]	)

            # select random slice
            __snake_case            =	int(ids_tensor((1,)     ,	output_from_past.shape[-1]	)	)
            __snake_case            =	output_from_no_past[:, -3:, random_slice_idx]
            __snake_case            =	output_from_past[:, :, random_slice_idx]

            # test that outputs are equal for slice
            tf.debugging.assert_near(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE     ,	rtol=1E-3	)





@require_tf
class      lowerCAmelCase (			__lowerCAmelCase					, __lowerCAmelCase					, unittest.TestCase):
       __lowercase	:   Optional[int]										=      (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
       __lowercase	:   Tuple										=      (TFOPTForCausalLM,) if is_tf_available() else ()
       __lowercase	:   int										=      (
           {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
       )
       __lowercase	:   Tuple										=      False
       __lowercase	:   int										=      False
       __lowercase	:   Any										=      False
       __lowercase	:   Optional[Any]										=      10


       def    lowerCAmelCase   ( self	)   ->	Any:



            '''simple docstring'''




            __snake_case            =	TFOPTModelTester(self	)
            __snake_case            =	ConfigTester(self     ,	config_class=__SCREAMING_SNAKE_CASE	)


       def    lowerCAmelCase   ( self	)   ->	List[str]:



            '''simple docstring'''




            self.config_tester.run_common_tests()


       def    lowerCAmelCase   ( self	)   ->	Any:



            '''simple docstring'''




            __snake_case            =	self.model_tester.prepare_config_and_inputs_for_common()
            self.model_tester.check_decoder_model_past_large_inputs(*__SCREAMING_SNAKE_CASE	)




       def    lowerCAmelCase   ( self	)   ->	Tuple:



            '''simple docstring'''




            __snake_case       ,     __snake_case            =	self.model_tester.prepare_config_and_inputs_for_common()

            def _get_word_embedding_weight(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	):
                 if hasattr(__SCREAMING_SNAKE_CASE     ,	'''weight'''	):
                      return embedding_layer.weight
                 else:
                      # Here we build the word embeddings weights if not exists.
                      # And then we retry to get the attribute once built.
                      model.build()
                      if hasattr(__SCREAMING_SNAKE_CASE     ,	'''weight'''	):
                           return embedding_layer.weight
                      else:
                           return None

            for model_class in self.all_model_classes:
                 for size in [config.vocab_size - 10, config.vocab_size + 10]:
                      # build the embeddings
                      __snake_case            =	model_class(config=__SCREAMING_SNAKE_CASE	)
                      __snake_case            =	_get_word_embedding_weight(__SCREAMING_SNAKE_CASE     ,	model.get_input_embeddings()	)
                      __snake_case            =	_get_word_embedding_weight(__SCREAMING_SNAKE_CASE     ,	model.get_output_embeddings()	)

                      # reshape the embeddings
                      model.resize_token_embeddings(__SCREAMING_SNAKE_CASE	)
                      __snake_case            =	_get_word_embedding_weight(__SCREAMING_SNAKE_CASE     ,	model.get_input_embeddings()	)
                      __snake_case            =	_get_word_embedding_weight(__SCREAMING_SNAKE_CASE     ,	model.get_output_embeddings()	)

                      # check that the resized embeddings size matches the desired size.
                      __snake_case            =	size if size is not None else config.vocab_size

                      self.assertEqual(new_input_embeddings.shape[0]     ,	__SCREAMING_SNAKE_CASE	)

                      # check that weights remain the same after resizing
                      __snake_case            =	True
                      for pa, pa in zip(old_input_embeddings.value()     ,	new_input_embeddings.value()	):
                           if tf.math.reduce_sum(tf.math.abs(pa - pa	)	) > 0:
                                __snake_case            =	False
                      self.assertTrue(__SCREAMING_SNAKE_CASE	)

                      if old_output_embeddings is not None and new_output_embeddings is not None:
                           self.assertEqual(new_output_embeddings.shape[0]     ,	__SCREAMING_SNAKE_CASE	)

                           __snake_case            =	True
                           for pa, pa in zip(old_output_embeddings.value()     ,	new_output_embeddings.value()	):
                                if tf.math.reduce_sum(tf.math.abs(pa - pa	)	) > 0:
                                     __snake_case            =	False
                           self.assertTrue(__SCREAMING_SNAKE_CASE	)







def       _UpperCamelCase			(_lowerCamelCase				:  List[str]						)-> List[str]:




     '''simple docstring'''





     return tf.constant(_lowerCamelCase ,						dtype=tf.intaa						)





@require_tf
class      lowerCAmelCase (			unittest.TestCase):
       __lowercase	:   Dict										=      99


       def    lowerCAmelCase   ( self	)   ->	str:



            '''simple docstring'''




            __snake_case            =	tf.ones((4, 1)     ,	dtype=tf.intaa	) * 2
            __snake_case            =	tf.concat([ids_tensor((4, 6)     ,	self.vocab_size - 3	) + 3, eos_column_vector]     ,	axis=1	)
            __snake_case            =	input_ids.shape[0]
            __snake_case            =	OPTConfig(
                vocab_size=self.vocab_size     ,	hidden_size=24     ,	num_hidden_layers=2     ,	num_attention_heads=2     ,	ffn_dim=32     ,	max_position_embeddings=48     ,	eos_token_id=2     ,	pad_token_id=1     ,	bos_token_id=0     ,	)
            return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class      lowerCAmelCase (			unittest.TestCase):


       @slow
       def    lowerCAmelCase   ( self	)   ->	Dict:



            '''simple docstring'''




            __snake_case            =	TFOPTModel.from_pretrained('''facebook/opt-350m'''	)
            __snake_case            =	_long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]	)
            __snake_case            =	tf.not_equal(__SCREAMING_SNAKE_CASE     ,	model.config.pad_token_id	)
            with tf.GradientTape():
                 __snake_case            =	model(input_ids=__SCREAMING_SNAKE_CASE     ,	attention_mask=__SCREAMING_SNAKE_CASE	).last_hidden_state
            __snake_case            =	(1, 11, 512)
            self.assertEqual(output.shape     ,	__SCREAMING_SNAKE_CASE	)
            __snake_case            =	tf.constant(
                [[-0.2_873, -1.9_218, -0.3_033], [-1.2_710, -0.1_338, -0.1_902], [0.4_095, 0.1_214, -1.3_121]]	)
            self.assertTrue(np.allclose(output[:, :3, :3]     ,	__SCREAMING_SNAKE_CASE     ,	atol=4E-3	)	)

            __snake_case            =	tf.function(__SCREAMING_SNAKE_CASE     ,	jit_compile=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	xla_generate(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)[0]
            self.assertTrue(np.allclose(output[:, :3, :3]     ,	__SCREAMING_SNAKE_CASE     ,	atol=4E-2	)	)





@require_tf
@slow
class      lowerCAmelCase (			unittest.TestCase):


       def    lowerCAmelCase   ( self	)   ->	Optional[Any]:



            '''simple docstring'''




            super().setUp()
            __snake_case            =	'''facebook/opt-350m'''




       def    lowerCAmelCase   ( self	)   ->	Union[str, Any]:



            '''simple docstring'''




            __snake_case            =	TFOPTForCausalLM.from_pretrained(self.path_model	)
            __snake_case            =	GPTaTokenizer.from_pretrained(self.path_model	)

            __snake_case            =	[
                '''Today is a beautiful day and I want to''',
                '''In the city of''',
                '''Paris is the capital of France and''',
                '''Computers and mobile phones have taken''',
            ]
            # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
            __snake_case            =	tokenizer(__SCREAMING_SNAKE_CASE     ,	return_tensors='''tf'''     ,	padding=__SCREAMING_SNAKE_CASE     ,	add_special_tokens=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	tf.math.reduce_mean(model(inputs.input_ids     ,	attention_mask=inputs.attention_mask	)[0]     ,	axis=-1	)
            __snake_case            =	tf.constant(
                [
                    [1.3_851, -13.8_923, -10.5_229, -10.7_533, -0.2_309, -10.2_384, -0.5_365, -9.0_947, -5.1_670],
                    [-4.7_073, -10.6_276, -3.9_415, -21.5_242, -0.2_822, -0.2_822, -0.2_822, -0.2_822, -0.2_822],
                    [0.6_247, -3.4_229, -8.9_179, -1.4_297, -14.1_650, 1.4_146, -9.0_218, -0.2_703, -0.2_703],
                    [6.4_783, -1.9_913, -10.7_926, -2.3_336, 1.5_092, -0.9_974, -6.8_213, 1.3_477, 1.3_477],
                ]	)
            self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE     ,	atol=1E-4	)	)

            __snake_case            =	tf.function(__SCREAMING_SNAKE_CASE     ,	jit_compile=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	tf.math.reduce_mean(xla_generate(inputs.input_ids     ,	attention_mask=inputs.attention_mask	)[0]     ,	axis=-1	)
            self.assertTrue(np.allclose(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE     ,	atol=1E-4	)	)





@require_tf
@slow
class      lowerCAmelCase (			unittest.TestCase):


       @property
       def    lowerCAmelCase   ( self	)   ->	Dict:



            '''simple docstring'''




            return [
                "Today is a beautiful day and I want",
                "In the city of",
                "Paris is the capital of France and",
                "Computers and mobile phones have taken",
            ]


       def    lowerCAmelCase   ( self	)   ->	List[str]:



            '''simple docstring'''




            __snake_case            =	'''facebook/opt-125m'''

            __snake_case            =	[
                '''Today is a beautiful day and I want to''',
                '''In the city of New York, the city''',
                '''Paris is the capital of France and the capital''',
                '''Computers and mobile phones have taken over the''',
            ]

            __snake_case            =	[]
            __snake_case            =	GPTaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE	)
            __snake_case            =	TFOPTForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE	)

            for prompt in self.prompts:
                 __snake_case            =	tokenizer(__SCREAMING_SNAKE_CASE     ,	return_tensors='''tf'''	).input_ids

                 __snake_case            =	model.generate(__SCREAMING_SNAKE_CASE     ,	max_length=10	)

                 __snake_case            =	tokenizer.batch_decode(__SCREAMING_SNAKE_CASE     ,	skip_special_tokens=__SCREAMING_SNAKE_CASE	)
                 predicted_outputs += generated_string

            self.assertListEqual(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)


       def    lowerCAmelCase   ( self	)   ->	List[Any]:



            '''simple docstring'''




            __snake_case            =	'''facebook/opt-350m'''

            __snake_case            =	GPTaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE	)
            __snake_case            =	TFOPTForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE	)

            __snake_case            =	'''left'''

            # use different length sentences to test batching
            __snake_case            =	[
                '''Hello, my dog is a little''',
                '''Today, I''',
            ]

            __snake_case            =	tokenizer(__SCREAMING_SNAKE_CASE     ,	return_tensors='''tf'''     ,	padding=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	inputs['''input_ids''']

            __snake_case            =	model.generate(input_ids=__SCREAMING_SNAKE_CASE     ,	attention_mask=inputs['''attention_mask''']	)

            __snake_case            =	tokenizer(sentences[0]     ,	return_tensors='''tf'''	).input_ids
            __snake_case            =	model.generate(input_ids=__SCREAMING_SNAKE_CASE	)

            __snake_case            =	inputs_non_padded.shape[-1] - tf.math.reduce_sum(
                tf.cast(inputs['''attention_mask'''][-1]     ,	tf.intaa	)	)
            __snake_case            =	tokenizer(sentences[1]     ,	return_tensors='''tf'''	).input_ids
            __snake_case            =	model.generate(input_ids=__SCREAMING_SNAKE_CASE     ,	max_length=model.config.max_length - num_paddings	)

            __snake_case            =	tokenizer.batch_decode(__SCREAMING_SNAKE_CASE     ,	skip_special_tokens=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	tokenizer.decode(output_non_padded[0]     ,	skip_special_tokens=__SCREAMING_SNAKE_CASE	)
            __snake_case            =	tokenizer.decode(output_padded[0]     ,	skip_special_tokens=__SCREAMING_SNAKE_CASE	)

            __snake_case            =	[
                '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
                '''Today, I was in the middle of a conversation with a friend about the''',
            ]
            self.assertListEqual(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)
            self.assertListEqual(__SCREAMING_SNAKE_CASE     ,	[non_padded_sentence, padded_sentence]	)




       def    lowerCAmelCase   ( self	)   ->	Dict:



            '''simple docstring'''




            __snake_case            =	'''facebook/opt-350m'''

            __snake_case            =	[
                '''Today is a beautiful day and I want to''',
                '''In the city of San Francisco, the city''',
                '''Paris is the capital of France and the capital''',
                '''Computers and mobile phones have taken over the''',
            ]

            __snake_case            =	[]
            __snake_case            =	GPTaTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE	)
            __snake_case            =	TFOPTForCausalLM.from_pretrained(__SCREAMING_SNAKE_CASE	)

            for prompt in self.prompts:
                 __snake_case            =	tokenizer(__SCREAMING_SNAKE_CASE     ,	return_tensors='''tf'''	).input_ids

                 __snake_case            =	model.generate(__SCREAMING_SNAKE_CASE     ,	max_length=10	)

                 __snake_case            =	tokenizer.batch_decode(__SCREAMING_SNAKE_CASE     ,	skip_special_tokens=__SCREAMING_SNAKE_CASE	)
                 predicted_outputs += generated_string

            self.assertListEqual(__SCREAMING_SNAKE_CASE     ,	__SCREAMING_SNAKE_CASE	)











"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_							=    logging.get_logger(__name__)


a_							=    """▁"""

a_							=    {"""vocab_file""": """spiece.model"""}

a_							=    {
    """vocab_file""": {
        """google/reformer-crime-and-punishment""": (
            """https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"""
        )
    }
}

a_							=    {
    """google/reformer-crime-and-punishment""": 52_42_88,
}







class 					A_(SCREAMING_SNAKE_CASE_      ):







			"""simple docstring"""



			a_       :			Optional[int]												=			VOCAB_FILES_NAMES
			a_       :			int												=			PRETRAINED_VOCAB_FILES_MAP
			a_       :			str												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a_       :			Optional[Any]												=			["""input_ids""", """attention_mask"""]
			def __init__(	self				,   A				,   A="</s>"				,   A="<unk>"				,   A=[]				,   A = None				,   **A				,   ):
						_lowerCamelCase					:					Any	       =    {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=A				,   unk_token=A				,   additional_special_tokens=A				,   sp_model_kwargs=self.sp_model_kwargs				,   **A				,   )

						_lowerCamelCase					:					Dict	       =    vocab_file
						_lowerCamelCase					:					List[Any]	       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(A							)
			@property
			def    _lowerCAmelCase		(	self							):
						return self.sp_model.get_piece_size()
			def    _lowerCAmelCase		(	self							):
						_lowerCamelCase					:					int	       =    {self.convert_ids_to_tokens(A							): i for i in range(self.vocab_size							)}
						vocab.update(self.added_tokens_encoder							)
						return vocab
			def __getstate__(	self							):
						_lowerCamelCase					:					Optional[int]	       =    self.__dict__.copy()
						_lowerCamelCase					:					Union[str, Any]	       =    None
						return state
			def __setstate__(	self				,   A							):
						_lowerCamelCase					:					int	       =    d

						# for backward compatibility
						if not hasattr(self				,   'sp_model_kwargs'							):
									_lowerCamelCase					:					Union[str, Any]	       =    {}

						_lowerCamelCase					:					Dict	       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(self.vocab_file							)
			def    _lowerCAmelCase		(	self				,   A							):
						return self.sp_model.encode(A				,   out_type=A							)
			def    _lowerCAmelCase		(	self				,   A							):
						return self.sp_model.piece_to_id(A							)
			def    _lowerCAmelCase		(	self				,   A							):
						if index < self.sp_model.get_piece_size():
									_lowerCamelCase					:					Dict	       =    self.sp_model.IdToPiece(A							)
						return token
			def    _lowerCAmelCase		(	self				,   A							):
						_lowerCamelCase					:					Tuple	       =    []
						_lowerCamelCase					:					Optional[int]	       =    ''
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												out_string += self.sp_model.decode(A							) + token
												_lowerCamelCase					:					Dict	       =    []
									else:
												current_sub_tokens.append(A							)
						out_string += self.sp_model.decode(A							)
						return out_string.strip()

			def    _lowerCAmelCase		(	self				,   A				,   A = None							):
						if not os.path.isdir(A							):
									logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
									return
						_lowerCamelCase					:					Any	       =    os.path.join(
						    A				,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

						if os.path.abspath(self.vocab_file							) != os.path.abspath(A							) and os.path.isfile(self.vocab_file							):
									copyfile(self.vocab_file				,   A							)
						elif not os.path.isfile(self.vocab_file							):
									with open(A				,   'wb'							) as fi:
												_lowerCamelCase					:					List[str]	       =    self.sp_model.serialized_model_proto()
												fi.write(A							)

						return (out_vocab_file,)





