
'''simple docstring'''







from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    TFRoFormerForCausalLM,
	    TFRoFormerForMaskedLM,
	    TFRoFormerForMultipleChoice,
	    TFRoFormerForQuestionAnswering,
	    TFRoFormerForSequenceClassification,
	    TFRoFormerForTokenClassification,
	    TFRoFormerModel,
	)
	from transformers.models.roformer.modeling_tf_roformer import (
	    TFRoFormerSelfAttention,
	    TFRoFormerSinusoidalPositionalEmbedding,
	)






class 			lowerCAmelCase__       :






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=13    ,							__SCREAMING_SNAKE_CASE=7    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=99    ,							__SCREAMING_SNAKE_CASE=32    ,							__SCREAMING_SNAKE_CASE=2    ,							__SCREAMING_SNAKE_CASE=4    ,							__SCREAMING_SNAKE_CASE=37    ,							__SCREAMING_SNAKE_CASE="gelu"    ,							__SCREAMING_SNAKE_CASE=0.1    ,							__SCREAMING_SNAKE_CASE=0.1    ,							__SCREAMING_SNAKE_CASE=5_12    ,							__SCREAMING_SNAKE_CASE=16    ,							__SCREAMING_SNAKE_CASE=2    ,							__SCREAMING_SNAKE_CASE=0.02    ,							__SCREAMING_SNAKE_CASE=3    ,							__SCREAMING_SNAKE_CASE=4    ,							__SCREAMING_SNAKE_CASE=None    ,							):


											"""simple docstring"""





											lowercase_     :					int													=	parent
											lowercase_     :					List[str]													=	13
											lowercase_     :					Union[str, Any]													=	7
											lowercase_     :					Tuple													=	True
											lowercase_     :					List[Any]													=	True
											lowercase_     :					List[Any]													=	True
											lowercase_     :					Tuple													=	True
											lowercase_     :					List[Any]													=	99
											lowercase_     :					int													=	32
											lowercase_     :					Optional[int]													=	2
											lowercase_     :					Optional[int]													=	4
											lowercase_     :					Optional[Any]													=	37
											lowercase_     :					List[str]													=	'''gelu'''
											lowercase_     :					Union[str, Any]													=	0.1
											lowercase_     :					Union[str, Any]													=	0.1
											lowercase_     :					Any													=	5_12
											lowercase_     :					Union[str, Any]													=	16
											lowercase_     :					Optional[Any]													=	2
											lowercase_     :					Any													=	0.02
											lowercase_     :					List[Any]													=	3
											lowercase_     :					int													=	4
											lowercase_     :					Optional[Any]													=	None






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	ids_tensor([self.batch_size, self.seq_length]    ,							self.vocab_size		)

											lowercase_     :					Optional[Any]													=	None
											if self.use_input_mask:
																	lowercase_     :					List[str]													=	random_attention_mask([self.batch_size, self.seq_length]		)

											lowercase_     :					Tuple													=	None
											if self.use_token_type_ids:
																	lowercase_     :					Union[str, Any]													=	ids_tensor([self.batch_size, self.seq_length]    ,							self.type_vocab_size		)

											lowercase_     :					Dict													=	None
											lowercase_     :					List[str]													=	None
											lowercase_     :					Union[str, Any]													=	None
											if self.use_labels:
																	lowercase_     :					Union[str, Any]													=	ids_tensor([self.batch_size]    ,							self.type_sequence_label_size		)
																	lowercase_     :					str													=	ids_tensor([self.batch_size, self.seq_length]    ,							self.num_labels		)
																	lowercase_     :					Tuple													=	ids_tensor([self.batch_size]    ,							self.num_choices		)

											lowercase_     :					Union[str, Any]													=	RoFormerConfig(
											    vocab_size=self.vocab_size    ,							hidden_size=self.hidden_size    ,							num_hidden_layers=self.num_hidden_layers    ,							num_attention_heads=self.num_attention_heads    ,							intermediate_size=self.intermediate_size    ,							hidden_act=self.hidden_act    ,							hidden_dropout_prob=self.hidden_dropout_prob    ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,							max_position_embeddings=self.max_position_embeddings    ,							type_vocab_size=self.type_vocab_size    ,							initializer_range=self.initializer_range    ,							return_dict=__SCREAMING_SNAKE_CASE    ,							)

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	TFRoFormerModel(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					int													=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

											lowercase_     :					Dict													=	[input_ids, input_mask]
											lowercase_     :					Union[str, Any]													=	model(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Any													=	model(__SCREAMING_SNAKE_CASE		)

											self.parent.assertEqual(result.last_hidden_state.shape    ,							(self.batch_size, self.seq_length, self.hidden_size)		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	True
											lowercase_     :					Tuple													=	TFRoFormerForCausalLM(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					str													=	{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}
											lowercase_     :					Any													=	model(__SCREAMING_SNAKE_CASE		)['''logits''']
											self.parent.assertListEqual(
											    list(prediction_scores.numpy().shape		)    ,							[self.batch_size, self.seq_length, self.vocab_size]		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					str													=	TFRoFormerForMaskedLM(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Union[str, Any]													=	{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}
											lowercase_     :					Dict													=	model(__SCREAMING_SNAKE_CASE		)
											self.parent.assertEqual(result.logits.shape    ,							(self.batch_size, self.seq_length, self.vocab_size)		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	self.num_labels
											lowercase_     :					List[str]													=	TFRoFormerForSequenceClassification(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					str													=	{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}

											lowercase_     :					Union[str, Any]													=	model(__SCREAMING_SNAKE_CASE		)
											self.parent.assertEqual(result.logits.shape    ,							(self.batch_size, self.num_labels)		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	self.num_choices
											lowercase_     :					str													=	TFRoFormerForMultipleChoice(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Tuple													=	tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE    ,							1		)    ,							(1, self.num_choices, 1)		)
											lowercase_     :					Dict													=	tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE    ,							1		)    ,							(1, self.num_choices, 1)		)
											lowercase_     :					Optional[int]													=	tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE    ,							1		)    ,							(1, self.num_choices, 1)		)
											lowercase_     :					Dict													=	{
											    '''input_ids''': multiple_choice_inputs_ids,
											    '''attention_mask''': multiple_choice_input_mask,
											    '''token_type_ids''': multiple_choice_token_type_ids,
											}
											lowercase_     :					Tuple													=	model(__SCREAMING_SNAKE_CASE		)
											self.parent.assertEqual(result.logits.shape    ,							(self.batch_size, self.num_choices)		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	self.num_labels
											lowercase_     :					Optional[int]													=	TFRoFormerForTokenClassification(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Any													=	{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}
											lowercase_     :					Optional[int]													=	model(__SCREAMING_SNAKE_CASE		)
											self.parent.assertEqual(result.logits.shape    ,							(self.batch_size, self.seq_length, self.num_labels)		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	TFRoFormerForQuestionAnswering(config=__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Union[str, Any]													=	{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}

											lowercase_     :					Tuple													=	model(__SCREAMING_SNAKE_CASE		)
											self.parent.assertEqual(result.start_logits.shape    ,							(self.batch_size, self.seq_length)		)
											self.parent.assertEqual(result.end_logits.shape    ,							(self.batch_size, self.seq_length)		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[str]													=	self.prepare_config_and_inputs()
											(
											    (
											    lowercase_   
											)						,			(
											    lowercase_   
											)						,			(
											    lowercase_   
											)						,			(
											    lowercase_   
											)						,			(
											    lowercase_   
											)						,			(
											    lowercase_   
											)						,			(
											    lowercase_   
											)						,			   
											)     :					Any													=	config_and_inputs
											lowercase_     :					List[str]													=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
											return config, inputs_dict







@require_tf
class 			lowerCAmelCase__       (  snake_case_   ,						snake_case_   ,						unittest.TestCase					):
					lowerCAmelCase_					       =					(
					    (
					        TFRoFormerModel,
					        TFRoFormerForCausalLM,
					        TFRoFormerForMaskedLM,
					        TFRoFormerForQuestionAnswering,
					        TFRoFormerForSequenceClassification,
					        TFRoFormerForTokenClassification,
					        TFRoFormerForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					lowerCAmelCase_					       =					(
					    {
					        '''feature-extraction''': TFRoFormerModel,
					        '''fill-mask''': TFRoFormerForMaskedLM,
					        '''question-answering''': TFRoFormerForQuestionAnswering,
					        '''text-classification''': TFRoFormerForSequenceClassification,
					        '''text-generation''': TFRoFormerForCausalLM,
					        '''token-classification''': TFRoFormerForTokenClassification,
					        '''zero-shot''': TFRoFormerForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)

					lowerCAmelCase_					       =					False
					lowerCAmelCase_					       =					False






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if pipeline_test_casse_name == "TextGenerationPipelineTests":
																	return True

											return False






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	TFRoFormerModelTester(self		)
											lowercase_     :					Tuple													=	ConfigTester(self    ,							config_class=__SCREAMING_SNAKE_CASE    ,							hidden_size=37		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.config_tester.run_common_tests()






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					str													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					str													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_lm_head(*__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[str]													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[str]													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE		)






					@slow
					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''		)
											self.assertIsNotNone(__SCREAMING_SNAKE_CASE		)







@require_tf
class 			lowerCAmelCase__       (  unittest.TestCase					):






					@slow
					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					str													=	TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''		)
											lowercase_     :					List[Any]													=	tf.constant([[0, 1, 2, 3, 4, 5]]		)
											lowercase_     :					Dict													=	model(__SCREAMING_SNAKE_CASE		)[0]

											# TODO Replace vocab size
											lowercase_     :					str													=	5_00_00

											lowercase_     :					List[Any]													=	[1, 6, vocab_size]
											self.assertEqual(output.shape    ,							__SCREAMING_SNAKE_CASE		)

											print(output[:, :3, :3]		)

											# TODO Replace values below with what was printed above.
											lowercase_     :					Optional[int]													=	tf.constant(
											    [
											        [
											            [-0.12_053_341, -1.0_264_901, 0.29_221_946],
											            [-1.5_133_783, 0.197_433, 0.15_190_607],
											            [-5.0_135_403, -3.900_256, -0.84_038_764],
											        ]
											    ]		)
											tf.debugging.assert_near(output[:, :3, :3]    ,							__SCREAMING_SNAKE_CASE    ,							atol=1E-4		)







@require_tf
class 			lowerCAmelCase__       (  unittest.TestCase					):
					lowerCAmelCase_					       =					1e-4






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	tf.constant([[4, 10]]		)
											lowercase_     :					Any													=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=6    ,							embedding_dim=6		)

											lowercase_     :					int													=	emba(input_ids.shape		)
											lowercase_     :					List[str]													=	tf.constant(
											    [[0.0_000, 0.0_000, 0.0_000, 1.0_000, 1.0_000, 1.0_000], [0.8_415, 0.0_464, 0.0_022, 0.5_403, 0.9_989, 1.0_000]]		)

											tf.debugging.assert_near(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							atol=self.tolerance		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	tf.constant(
											    [
											        [0.0_000, 0.0_000, 0.0_000, 0.0_000, 0.0_000],
											        [0.8_415, 0.8_219, 0.8_020, 0.7_819, 0.7_617],
											        [0.9_093, 0.9_364, 0.9_581, 0.9_749, 0.9_870],
											    ]		)
											lowercase_     :					Optional[int]													=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_12    ,							embedding_dim=5_12		)
											emba([2, 16, 5_12]		)
											lowercase_     :					Union[str, Any]													=	emba.weight[:3, :5]

											tf.debugging.assert_near(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							atol=self.tolerance		)







@require_tf
class 			lowerCAmelCase__       (  unittest.TestCase					):
					lowerCAmelCase_					       =					1e-4






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	tf.reshape(tf.range(2 * 12 * 16 * 64    ,							dtype=tf.floataa		)    ,							shape=(2, 12, 16, 64)		) / 1_00

											lowercase_     :					int													=	-tf.reshape(tf.range(2 * 12 * 16 * 64    ,							dtype=tf.floataa		)    ,							shape=(2, 12, 16, 64)		) / 1_00

											lowercase_     :					Optional[Any]													=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=32    ,							embedding_dim=64		)
											lowercase_     :					List[Any]													=	embed_positions([2, 16, 7_68]		)[None, None, :, :]

											lowercase_						,			lowercase_     :					Optional[Any]													=	TFRoFormerSelfAttention.apply_rotary_position_embeddings(
											    __SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					int													=	tf.constant(
											    [
											        [0.0_000, 0.0_100, 0.0_200, 0.0_300, 0.0_400, 0.0_500, 0.0_600, 0.0_700],
											        [-0.2_012, 0.8_897, 0.0_263, 0.9_401, 0.2_074, 0.9_463, 0.3_481, 0.9_343],
											        [-1.7_057, 0.6_271, -1.2_145, 1.3_897, -0.6_303, 1.7_647, -0.1_173, 1.8_985],
											        [-2.1_731, -1.6_397, -2.7_358, 0.2_854, -2.1_840, 1.7_183, -1.3_018, 2.4_871],
											        [0.2_717, -3.6_173, -2.9_206, -2.1_988, -3.6_638, 0.3_858, -2.9_155, 2.2_980],
											        [3.9_859, -2.1_580, -0.7_984, -4.4_904, -4.1_181, -2.0_252, -4.4_782, 1.1_253],
											    ]		)
											lowercase_     :					Dict													=	tf.constant(
											    [
											        [0.0_000, -0.0_100, -0.0_200, -0.0_300, -0.0_400, -0.0_500, -0.0_600, -0.0_700],
											        [0.2_012, -0.8_897, -0.0_263, -0.9_401, -0.2_074, -0.9_463, -0.3_481, -0.9_343],
											        [1.7_057, -0.6_271, 1.2_145, -1.3_897, 0.6_303, -1.7_647, 0.1_173, -1.8_985],
											        [2.1_731, 1.6_397, 2.7_358, -0.2_854, 2.1_840, -1.7_183, 1.3_018, -2.4_871],
											        [-0.2_717, 3.6_173, 2.9_206, 2.1_988, 3.6_638, -0.3_858, 2.9_155, -2.2_980],
											        [-3.9_859, 2.1_580, 0.7_984, 4.4_904, 4.1_181, 2.0_252, 4.4_782, -1.1_253],
											    ]		)

											tf.debugging.assert_near(query_layer[0, 0, :6, :8]    ,							__SCREAMING_SNAKE_CASE    ,							atol=self.tolerance		)
											tf.debugging.assert_near(key_layer[0, 0, :6, :8]    ,							__SCREAMING_SNAKE_CASE    ,							atol=self.tolerance		)





'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase			:  Optional[Any]   							=	logging.get_logger(__name__)

_lowercase			:  str   							=	"▁"

_lowercase			:  Optional[int]   							=	{"vocab_file": "sentencepiece.bpe.model"}

_lowercase			:  Dict   							=	{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

_lowercase			:  Optional[Any]   							=	{
    "facebook/xglm-564M": 2_0_4_8,
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE = None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Any													=	{} if sp_model_kwargs is None else sp_model_kwargs

											# Compatibility with the original tokenizer
											lowercase_     :					Optional[Any]													=	7
											lowercase_     :					List[Any]													=	[F'''<madeupword{i}>''' for i in range(self.num_madeup_words		)]

											lowercase_     :					Tuple													=	kwargs.get('''additional_special_tokens'''    ,							[]		)
											kwargs["additional_special_tokens"] += [
											    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							cls_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							sp_model_kwargs=self.sp_model_kwargs    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					int													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(str(__SCREAMING_SNAKE_CASE		)		)
											lowercase_     :					Dict													=	vocab_file

											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
											# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
											# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											lowercase_     :					List[Any]													=	1

											# Mimic fairseq token-to-id alignment for the first 4 token
											lowercase_     :					Optional[Any]													=	{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

											lowercase_     :					Dict													=	len(self.sp_model		)
											lowercase_     :					int													=	{F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words		)}
											self.fairseq_tokens_to_ids.update(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					List[str]													=	{v: k for k, v in self.fairseq_tokens_to_ids.items()}






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	self.__dict__.copy()
											lowercase_     :					Optional[Any]													=	None
											lowercase_     :					List[Any]													=	self.sp_model.serialized_model_proto()
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	d

											# for backward compatibility
											if not hasattr(self    ,							'''sp_model_kwargs'''		):
																	lowercase_     :					Optional[Any]													=	{}

											lowercase_     :					int													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return [self.sep_token_id] + token_ids_a
											lowercase_     :					Optional[Any]													=	[self.sep_token_id]
											return sep + token_ids_a + sep + sep + token_ids_a






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE    ,							token_ids_a=__SCREAMING_SNAKE_CASE    ,							already_has_special_tokens=__SCREAMING_SNAKE_CASE		)

											if token_ids_a is None:
																	return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		))
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE		))






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	[self.sep_token_id]

											if token_ids_a is None:
																	return len(sep + token_ids_a		) * [0]
											return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return len(self.sp_model		) + self.fairseq_offset + self.num_madeup_words






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE    ,							out_type=__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
											lowercase_     :					str													=	self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE		)

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Dict													=	''''''.join(__SCREAMING_SNAKE_CASE		).replace(__SCREAMING_SNAKE_CASE    ,							''' '''		).strip()
											return out_string






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_     :					str													=	os.path.join(
											    __SCREAMING_SNAKE_CASE    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
																	copyfile(self.vocab_file    ,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.vocab_file		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''wb'''		) as fi:
																							lowercase_     :					Optional[int]													=	self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)



