
'''simple docstring'''







import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotSmallConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
							import os

							# The slow tests are often failing with OOM error on GPU
							# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
							# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
							_lowercase			:  Union[str, Any]   							=	"platform"

							import jax
							import jax.numpy as jnp

							from transformers.models.blenderbot_small.modeling_flax_blenderbot_small import (
							    FlaxBlenderbotSmallForConditionalGeneration,
							    FlaxBlenderbotSmallModel,
							    shift_tokens_right,
							)
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			int				,				__SCREAMING_SNAKE_CASE :			Dict=None				,				__SCREAMING_SNAKE_CASE :			Any=None				,				__SCREAMING_SNAKE_CASE :			str=None				,				__SCREAMING_SNAKE_CASE :			Dict=None				,				__SCREAMING_SNAKE_CASE :			Optional[Any]=None				,				__SCREAMING_SNAKE_CASE :			Dict=None				,				):



						"""simple docstring"""





						if attention_mask is None:
												lowercase_     :					str													=	np.where(input_ids != config.pad_token_id				,				1				,				0							)
						if decoder_attention_mask is None:
												lowercase_     :					List[Any]													=	np.where(decoder_input_ids != config.pad_token_id				,				1				,				0							)
						if head_mask is None:
												lowercase_     :					str													=	np.ones((config.encoder_layers, config.encoder_attention_heads)							)
						if decoder_head_mask is None:
												lowercase_     :					List[Any]													=	np.ones((config.decoder_layers, config.decoder_attention_heads)							)
						if cross_attn_head_mask is None:
												lowercase_     :					Union[str, Any]													=	np.ones((config.decoder_layers, config.decoder_attention_heads)							)
						return {
						    "input_ids": input_ids,
						    "decoder_input_ids": decoder_input_ids,
						    "attention_mask": attention_mask,
						    "decoder_attention_mask": attention_mask,
						}






class 			lowerCAmelCase__       :






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=13    ,							__SCREAMING_SNAKE_CASE=7    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=False    ,							__SCREAMING_SNAKE_CASE=99    ,							__SCREAMING_SNAKE_CASE=16    ,							__SCREAMING_SNAKE_CASE=2    ,							__SCREAMING_SNAKE_CASE=4    ,							__SCREAMING_SNAKE_CASE=4    ,							__SCREAMING_SNAKE_CASE="gelu"    ,							__SCREAMING_SNAKE_CASE=0.1    ,							__SCREAMING_SNAKE_CASE=0.1    ,							__SCREAMING_SNAKE_CASE=32    ,							__SCREAMING_SNAKE_CASE=2    ,							__SCREAMING_SNAKE_CASE=1    ,							__SCREAMING_SNAKE_CASE=0    ,							__SCREAMING_SNAKE_CASE=0.02    ,							):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	parent
											lowercase_     :					int													=	batch_size
											lowercase_     :					Optional[Any]													=	seq_length
											lowercase_     :					Optional[Any]													=	is_training
											lowercase_     :					Tuple													=	use_labels
											lowercase_     :					Dict													=	vocab_size
											lowercase_     :					Any													=	hidden_size
											lowercase_     :					Union[str, Any]													=	num_hidden_layers
											lowercase_     :					Dict													=	num_attention_heads
											lowercase_     :					List[str]													=	intermediate_size
											lowercase_     :					str													=	hidden_act
											lowercase_     :					Union[str, Any]													=	hidden_dropout_prob
											lowercase_     :					Tuple													=	attention_probs_dropout_prob
											lowercase_     :					List[str]													=	max_position_embeddings
											lowercase_     :					List[Any]													=	eos_token_id
											lowercase_     :					Any													=	pad_token_id
											lowercase_     :					Dict													=	bos_token_id
											lowercase_     :					int													=	initializer_range






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	np.clip(ids_tensor([self.batch_size, self.seq_length - 1]    ,							self.vocab_size		)    ,							3    ,							self.vocab_size		)
											lowercase_     :					List[str]													=	np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)    ,							dtype=np.intaa		))    ,							-1		)

											lowercase_     :					List[str]													=	shift_tokens_right(__SCREAMING_SNAKE_CASE    ,							1    ,							2		)

											lowercase_     :					int													=	BlenderbotSmallConfig(
											    vocab_size=self.vocab_size    ,							d_model=self.hidden_size    ,							encoder_layers=self.num_hidden_layers    ,							decoder_layers=self.num_hidden_layers    ,							encoder_attention_heads=self.num_attention_heads    ,							decoder_attention_heads=self.num_attention_heads    ,							encoder_ffn_dim=self.intermediate_size    ,							decoder_ffn_dim=self.intermediate_size    ,							dropout=self.hidden_dropout_prob    ,							attention_dropout=self.attention_probs_dropout_prob    ,							max_position_embeddings=self.max_position_embeddings    ,							eos_token_id=self.eos_token_id    ,							bos_token_id=self.bos_token_id    ,							pad_token_id=self.pad_token_id    ,							initializer_range=self.initializer_range    ,							use_cache=__SCREAMING_SNAKE_CASE    ,							)
											lowercase_     :					List[str]													=	prepare_blenderbot_inputs_dict(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
											return config, inputs_dict






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_						,			lowercase_     :					Optional[Any]													=	self.prepare_config_and_inputs()
											return config, inputs_dict






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	20
											lowercase_     :					Tuple													=	model_class_name(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Union[str, Any]													=	model.encode(inputs_dict['''input_ids''']		)

											lowercase_						,			lowercase_     :					Dict													=	(
											    inputs_dict['''decoder_input_ids'''],
											    inputs_dict['''decoder_attention_mask'''],
											)

											lowercase_     :					int													=	model.init_cache(decoder_input_ids.shape[0]    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Optional[int]													=	jnp.ones((decoder_input_ids.shape[0], max_decoder_length)    ,							dtype='''i4'''		)

											lowercase_     :					Any													=	jnp.broadcast_to(
											    jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]    ,							(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)    ,							)
											lowercase_     :					Union[str, Any]													=	model.decode(
											    decoder_input_ids[:, :-1]    ,							__SCREAMING_SNAKE_CASE    ,							decoder_attention_mask=__SCREAMING_SNAKE_CASE    ,							past_key_values=__SCREAMING_SNAKE_CASE    ,							decoder_position_ids=__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Any													=	jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]    ,							dtype='''i4'''		)
											lowercase_     :					Union[str, Any]													=	model.decode(
											    decoder_input_ids[:, -1:]    ,							__SCREAMING_SNAKE_CASE    ,							decoder_attention_mask=__SCREAMING_SNAKE_CASE    ,							past_key_values=outputs_cache.past_key_values    ,							decoder_position_ids=__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					List[str]													=	model.decode(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					str													=	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
											self.parent.assertTrue(diff < 1E-3    ,							msg=F'''Max diff is {diff}'''		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	20
											lowercase_     :					str													=	model_class_name(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Any													=	model.encode(inputs_dict['''input_ids''']		)

											lowercase_						,			lowercase_     :					str													=	(
											    inputs_dict['''decoder_input_ids'''],
											    inputs_dict['''decoder_attention_mask'''],
											)

											lowercase_     :					List[str]													=	jnp.concatenate(
											    [
											        decoder_attention_mask,
											        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])		),
											    ]    ,							axis=-1    ,							)

											lowercase_     :					int													=	model.init_cache(decoder_input_ids.shape[0]    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Any													=	jnp.broadcast_to(
											    jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]    ,							(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)    ,							)

											lowercase_     :					List[Any]													=	model.decode(
											    decoder_input_ids[:, :-1]    ,							__SCREAMING_SNAKE_CASE    ,							decoder_attention_mask=__SCREAMING_SNAKE_CASE    ,							past_key_values=__SCREAMING_SNAKE_CASE    ,							decoder_position_ids=__SCREAMING_SNAKE_CASE    ,							)
											lowercase_     :					str													=	jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]    ,							dtype='''i4'''		)
											lowercase_     :					Optional[Any]													=	model.decode(
											    decoder_input_ids[:, -1:]    ,							__SCREAMING_SNAKE_CASE    ,							past_key_values=outputs_cache.past_key_values    ,							decoder_attention_mask=__SCREAMING_SNAKE_CASE    ,							decoder_position_ids=__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Optional[int]													=	model.decode(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							decoder_attention_mask=__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Optional[Any]													=	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
											self.parent.assertTrue(diff < 1E-3    ,							msg=F'''Max diff is {diff}'''		)







@require_flax
class 			lowerCAmelCase__       (  unittest.TestCase					):
					lowerCAmelCase_					       =					9_9






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	np.array(
											    [
											        [71, 82, 18, 33, 46, 91, 2],
											        [68, 34, 26, 58, 30, 82, 2],
											        [5, 97, 17, 39, 94, 40, 2],
											        [76, 83, 94, 25, 70, 78, 2],
											        [87, 59, 41, 35, 48, 66, 2],
											        [55, 13, 16, 58, 5, 2, 1],  # note padding
											        [64, 27, 31, 51, 12, 75, 2],
											        [52, 64, 86, 17, 83, 39, 2],
											        [48, 61, 9, 24, 71, 82, 2],
											        [26, 1, 60, 48, 22, 13, 2],
											        [21, 5, 62, 28, 14, 76, 2],
											        [45, 98, 37, 86, 59, 48, 2],
											        [70, 70, 50, 9, 28, 0, 2],
											    ]    ,							dtype=np.intaa    ,							)

											lowercase_     :					List[Any]													=	input_ids.shape[0]
											lowercase_     :					Union[str, Any]													=	BlenderbotSmallConfig(
											    vocab_size=self.vocab_size    ,							d_model=24    ,							encoder_layers=2    ,							decoder_layers=2    ,							encoder_attention_heads=2    ,							decoder_attention_heads=2    ,							encoder_ffn_dim=32    ,							decoder_ffn_dim=32    ,							max_position_embeddings=48    ,							eos_token_id=2    ,							pad_token_id=1    ,							bos_token_id=0    ,							)
											return config, input_ids, batch_size






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_						,			lowercase_						,			lowercase_     :					Union[str, Any]													=	self._get_config_and_data()
											lowercase_     :					Optional[int]													=	FlaxBlenderbotSmallForConditionalGeneration(__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Any													=	lm_model(input_ids=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Any													=	(batch_size, input_ids.shape[1], config.vocab_size)
											self.assertEqual(outputs['''logits'''].shape    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	BlenderbotSmallConfig(
											    vocab_size=self.vocab_size    ,							d_model=14    ,							encoder_layers=2    ,							decoder_layers=2    ,							encoder_attention_heads=2    ,							decoder_attention_heads=2    ,							encoder_ffn_dim=8    ,							decoder_ffn_dim=8    ,							max_position_embeddings=48    ,							)
											lowercase_     :					List[str]													=	FlaxBlenderbotSmallForConditionalGeneration(__SCREAMING_SNAKE_CASE		)
											lowercase_     :					int													=	np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]    ,							dtype=np.intaa		)
											lowercase_     :					int													=	np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]    ,							dtype=np.intaa		)
											lowercase_     :					str													=	lm_model(input_ids=__SCREAMING_SNAKE_CASE    ,							decoder_input_ids=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Optional[int]													=	(*summary.shape, config.vocab_size)
											self.assertEqual(outputs['''logits'''].shape    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					int													=	np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]    ,							dtype=np.intaa		)
											lowercase_     :					Optional[int]													=	shift_tokens_right(__SCREAMING_SNAKE_CASE    ,							1    ,							2		)
											lowercase_     :					int													=	np.equal(__SCREAMING_SNAKE_CASE    ,							1		).astype(np.floataa		).sum()
											lowercase_     :					Tuple													=	np.equal(__SCREAMING_SNAKE_CASE    ,							1		).astype(np.floataa		).sum()
											self.assertEqual(shifted.shape    ,							input_ids.shape		)
											self.assertEqual(__SCREAMING_SNAKE_CASE    ,							n_pad_before - 1		)
											self.assertTrue(np.equal(shifted[:, 0]    ,							2		).all()		)







@require_flax
class 			lowerCAmelCase__       (  lowerCamelCase_   ,						unittest.TestCase   ,						lowerCamelCase_					):
					lowerCAmelCase_					       =					True
					lowerCAmelCase_					       =					(
					    (
					        FlaxBlenderbotSmallModel,
					        FlaxBlenderbotSmallForConditionalGeneration,
					    )
					    if is_flax_available()
					    else ()
					)
					lowerCAmelCase_					       =					(FlaxBlenderbotSmallForConditionalGeneration,) if is_flax_available() else ()






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	FlaxBlenderbotSmallModelTester(self		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_						,			lowercase_     :					List[str]													=	self.model_tester.prepare_config_and_inputs()
											for model_class in self.all_model_classes:
																	self.model_tester.check_use_cache_forward(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_						,			lowercase_     :					Tuple													=	self.model_tester.prepare_config_and_inputs()
											for model_class in self.all_model_classes:
																	self.model_tester.check_use_cache_forward_with_attn_mask(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_						,			lowercase_     :					Dict													=	self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																	with self.subTest(model_class.__name__		):
																							lowercase_     :					List[Any]													=	self._prepare_for_class(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
																							lowercase_     :					int													=	model_class(__SCREAMING_SNAKE_CASE		)

																							@jax.jit
																							def encode_jitted(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None    ,							**__SCREAMING_SNAKE_CASE		):
																													return model.encode(input_ids=__SCREAMING_SNAKE_CASE    ,							attention_mask=__SCREAMING_SNAKE_CASE		)

																							with self.subTest('''JIT Enabled'''		):
																													lowercase_     :					Dict													=	encode_jitted(**__SCREAMING_SNAKE_CASE		).to_tuple()

																							with self.subTest('''JIT Disabled'''		):
																													with jax.disable_jit():
																																			lowercase_     :					List[Any]													=	encode_jitted(**__SCREAMING_SNAKE_CASE		).to_tuple()

																							self.assertEqual(len(__SCREAMING_SNAKE_CASE		)    ,							len(__SCREAMING_SNAKE_CASE		)		)
																							for jitted_output, output in zip(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																													self.assertEqual(jitted_output.shape    ,							output.shape		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_						,			lowercase_     :					Any													=	self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																	with self.subTest(model_class.__name__		):
																							lowercase_     :					int													=	model_class(__SCREAMING_SNAKE_CASE		)
																							lowercase_     :					List[Any]													=	model.encode(inputs_dict['''input_ids''']    ,							inputs_dict['''attention_mask''']		)

																							lowercase_     :					Optional[int]													=	{
																							    '''decoder_input_ids''': inputs_dict['''decoder_input_ids'''],
																							    '''decoder_attention_mask''': inputs_dict['''decoder_attention_mask'''],
																							    '''encoder_outputs''': encoder_outputs,
																							}

																							@jax.jit
																							def decode_jitted(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																													return model.decode(
																													    decoder_input_ids=__SCREAMING_SNAKE_CASE    ,							decoder_attention_mask=__SCREAMING_SNAKE_CASE    ,							encoder_outputs=__SCREAMING_SNAKE_CASE    ,							)

																							with self.subTest('''JIT Enabled'''		):
																													lowercase_     :					Union[str, Any]													=	decode_jitted(**__SCREAMING_SNAKE_CASE		).to_tuple()

																							with self.subTest('''JIT Disabled'''		):
																													with jax.disable_jit():
																																			lowercase_     :					str													=	decode_jitted(**__SCREAMING_SNAKE_CASE		).to_tuple()

																							self.assertEqual(len(__SCREAMING_SNAKE_CASE		)    ,							len(__SCREAMING_SNAKE_CASE		)		)
																							for jitted_output, output in zip(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																													self.assertEqual(jitted_output.shape    ,							output.shape		)






					@slow
					def      _snake_case   (       self		):


											"""simple docstring"""





											for model_class_name in self.all_model_classes:
																	lowercase_     :					Dict													=	model_class_name.from_pretrained('''facebook/blenderbot_small-90M'''		)
																	# FlaxBlenderbotForSequenceClassification expects eos token in input_ids
																	lowercase_     :					Optional[Any]													=	np.ones((1, 1)		) * model.config.eos_token_id
																	lowercase_     :					Optional[int]													=	model(__SCREAMING_SNAKE_CASE		)
																	self.assertIsNotNone(__SCREAMING_SNAKE_CASE		)





'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase			:  Optional[Any]   							=	logging.get_logger(__name__)

_lowercase			:  str   							=	"▁"

_lowercase			:  Optional[int]   							=	{"vocab_file": "sentencepiece.bpe.model"}

_lowercase			:  Dict   							=	{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

_lowercase			:  Optional[Any]   							=	{
    "facebook/xglm-564M": 2_0_4_8,
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE = None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Any													=	{} if sp_model_kwargs is None else sp_model_kwargs

											# Compatibility with the original tokenizer
											lowercase_     :					Optional[Any]													=	7
											lowercase_     :					List[Any]													=	[F'''<madeupword{i}>''' for i in range(self.num_madeup_words		)]

											lowercase_     :					Tuple													=	kwargs.get('''additional_special_tokens'''    ,							[]		)
											kwargs["additional_special_tokens"] += [
											    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							cls_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							sp_model_kwargs=self.sp_model_kwargs    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					int													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(str(__SCREAMING_SNAKE_CASE		)		)
											lowercase_     :					Dict													=	vocab_file

											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
											# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
											# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											lowercase_     :					List[Any]													=	1

											# Mimic fairseq token-to-id alignment for the first 4 token
											lowercase_     :					Optional[Any]													=	{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

											lowercase_     :					Dict													=	len(self.sp_model		)
											lowercase_     :					int													=	{F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words		)}
											self.fairseq_tokens_to_ids.update(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					List[str]													=	{v: k for k, v in self.fairseq_tokens_to_ids.items()}






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	self.__dict__.copy()
											lowercase_     :					Optional[Any]													=	None
											lowercase_     :					List[Any]													=	self.sp_model.serialized_model_proto()
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	d

											# for backward compatibility
											if not hasattr(self    ,							'''sp_model_kwargs'''		):
																	lowercase_     :					Optional[Any]													=	{}

											lowercase_     :					int													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return [self.sep_token_id] + token_ids_a
											lowercase_     :					Optional[Any]													=	[self.sep_token_id]
											return sep + token_ids_a + sep + sep + token_ids_a






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE    ,							token_ids_a=__SCREAMING_SNAKE_CASE    ,							already_has_special_tokens=__SCREAMING_SNAKE_CASE		)

											if token_ids_a is None:
																	return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		))
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE		))






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	[self.sep_token_id]

											if token_ids_a is None:
																	return len(sep + token_ids_a		) * [0]
											return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return len(self.sp_model		) + self.fairseq_offset + self.num_madeup_words






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE    ,							out_type=__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
											lowercase_     :					str													=	self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE		)

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Dict													=	''''''.join(__SCREAMING_SNAKE_CASE		).replace(__SCREAMING_SNAKE_CASE    ,							''' '''		).strip()
											return out_string






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_     :					str													=	os.path.join(
											    __SCREAMING_SNAKE_CASE    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
																	copyfile(self.vocab_file    ,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.vocab_file		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''wb'''		) as fi:
																							lowercase_     :					Optional[int]													=	self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)



