



import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
				import torch

				from transformers import (
				    MraForMaskedLM,
				    MraForMultipleChoice,
				    MraForQuestionAnswering,
				    MraForSequenceClassification,
				    MraForTokenClassification,
				    MraModel,
				)
				from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST






class     __a			:



	def __init__(    self				:     int					,						snake_case_				:     int					,						snake_case_				:     str=2					,						snake_case_				:     Optional[Any]=8					,						snake_case_				:     str=True					,						snake_case_				:     int=True					,						snake_case_				:     Optional[Any]=True					,						snake_case_				:     int=True					,						snake_case_				:     List[str]=99					,						snake_case_				:     Optional[int]=16					,						snake_case_				:     List[Any]=5					,						snake_case_				:     Tuple=2					,						snake_case_				:     Optional[Any]=36					,						snake_case_				:     Optional[Any]="gelu"					,						snake_case_				:     Optional[int]=0.0					,						snake_case_				:     Optional[Any]=0.0					,						snake_case_				:     str=5_12					,						snake_case_				:     Optional[int]=16					,						snake_case_				:     Any=2					,						snake_case_				:     Optional[Any]=0.0_2					,						snake_case_				:     Any=3					,						snake_case_				:     Tuple=4					,						snake_case_				:     Optional[Any]=None					,						)->				Union[str, Any]:
						__lowerCAmelCase						 =parent
						__lowerCAmelCase						 =batch_size
						__lowerCAmelCase						 =seq_length
						__lowerCAmelCase						 =is_training
						__lowerCAmelCase						 =use_input_mask
						__lowerCAmelCase						 =use_token_type_ids
						__lowerCAmelCase						 =use_labels
						__lowerCAmelCase						 =vocab_size
						__lowerCAmelCase						 =hidden_size
						__lowerCAmelCase						 =num_hidden_layers
						__lowerCAmelCase						 =num_attention_heads
						__lowerCAmelCase						 =intermediate_size
						__lowerCAmelCase						 =hidden_act
						__lowerCAmelCase						 =hidden_dropout_prob
						__lowerCAmelCase						 =attention_probs_dropout_prob
						__lowerCAmelCase						 =max_position_embeddings
						__lowerCAmelCase						 =type_vocab_size
						__lowerCAmelCase						 =type_sequence_label_size
						__lowerCAmelCase						 =initializer_range
						__lowerCAmelCase						 =num_labels
						__lowerCAmelCase						 =num_choices
						__lowerCAmelCase						 =scope



	def 		UpperCamelCase				(    self				:     Union[str, Any])->				str:
						__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						self.vocab_size)

						__lowerCAmelCase						 =None
						if self.use_input_mask:
											__lowerCAmelCase						 =random_attention_mask([self.batch_size, self.seq_length])

						__lowerCAmelCase						 =None
						if self.use_token_type_ids:
											__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						self.type_vocab_size)

						__lowerCAmelCase						 =None
						__lowerCAmelCase						 =None
						__lowerCAmelCase						 =None
						if self.use_labels:
											__lowerCAmelCase						 =ids_tensor([self.batch_size]					,						self.type_sequence_label_size)
											__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						self.num_labels)
											__lowerCAmelCase						 =ids_tensor([self.batch_size]					,						self.num_choices)

						__lowerCAmelCase						 =self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



	def 		UpperCamelCase				(    self				:     List[str])->				Tuple:
						return MraConfig(
						    vocab_size=self.vocab_size					,						hidden_size=self.hidden_size					,						num_hidden_layers=self.num_hidden_layers					,						num_attention_heads=self.num_attention_heads					,						intermediate_size=self.intermediate_size					,						hidden_act=self.hidden_act					,						hidden_dropout_prob=self.hidden_dropout_prob					,						attention_probs_dropout_prob=self.attention_probs_dropout_prob					,						max_position_embeddings=self.max_position_embeddings					,						type_vocab_size=self.type_vocab_size					,						is_decoder=snake_case_					,						initializer_range=self.initializer_range					,						)



	def 		UpperCamelCase				(    self				:     List[Any])->				Dict:
						__lowerCAmelCase						 =self.get_config()
						__lowerCAmelCase						 =3_00
						return config



	def 		UpperCamelCase				(    self				:     Tuple)->				str:
						(
						    (
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						
						)						 =self.prepare_config_and_inputs()

						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
						__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						vocab_size=2)

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						    choice_labels,
						    encoder_hidden_states,
						    encoder_attention_mask,
						)



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     str					,						snake_case_				:     Tuple					,						snake_case_				:     Any					,						snake_case_				:     Dict					,						snake_case_				:     List[str]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Optional[Any])->				List[Any]:
						__lowerCAmelCase						 =MraModel(config=snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =model(snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_)
						__lowerCAmelCase						 =model(snake_case_					,						token_type_ids=snake_case_)
						__lowerCAmelCase						 =model(snake_case_)
						self.parent.assertEqual(result.last_hidden_state.shape					,						(self.batch_size, self.seq_length, self.hidden_size))



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     List[str]					,						snake_case_				:     List[str]					,						snake_case_				:     List[str]					,						snake_case_				:     List[str]					,						snake_case_				:     int					,						snake_case_				:     Dict					,						snake_case_				:     Tuple					,						snake_case_				:     List[str]					,						)->				Union[str, Any]:
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =MraModel(snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =model(
						    snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						encoder_hidden_states=snake_case_					,						encoder_attention_mask=snake_case_					,						)
						__lowerCAmelCase						 =model(
						    snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						encoder_hidden_states=snake_case_					,						)
						__lowerCAmelCase						 =model(snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_)
						self.parent.assertEqual(result.last_hidden_state.shape					,						(self.batch_size, self.seq_length, self.hidden_size))



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     Optional[int]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     str					,						snake_case_				:     Dict					,						snake_case_				:     List[str]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Union[str, Any])->				Union[str, Any]:
						__lowerCAmelCase						 =MraForMaskedLM(config=snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =model(snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						labels=snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.seq_length, self.vocab_size))



	def 		UpperCamelCase				(    self				:     List[Any]					,						snake_case_				:     Optional[Any]					,						snake_case_				:     int					,						snake_case_				:     List[str]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     List[Any]					,						snake_case_				:     Any					,						snake_case_				:     Dict)->				str:
						__lowerCAmelCase						 =MraForQuestionAnswering(config=snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =model(
						    snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						start_positions=snake_case_					,						end_positions=snake_case_					,						)
						self.parent.assertEqual(result.start_logits.shape					,						(self.batch_size, self.seq_length))
						self.parent.assertEqual(result.end_logits.shape					,						(self.batch_size, self.seq_length))



	def 		UpperCamelCase				(    self				:     List[Any]					,						snake_case_				:     Tuple					,						snake_case_				:     str					,						snake_case_				:     Optional[Any]					,						snake_case_				:     List[Any]					,						snake_case_				:     List[str]					,						snake_case_				:     Tuple					,						snake_case_				:     Optional[Any])->				Optional[int]:
						__lowerCAmelCase						 =self.num_labels
						__lowerCAmelCase						 =MraForSequenceClassification(snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =model(snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						labels=snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.num_labels))



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     List[str]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     List[str]					,						snake_case_				:     Dict					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     List[str]					,						snake_case_				:     Any)->				int:
						__lowerCAmelCase						 =self.num_labels
						__lowerCAmelCase						 =MraForTokenClassification(config=snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =model(snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						labels=snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.seq_length, self.num_labels))



	def 		UpperCamelCase				(    self				:     Tuple					,						snake_case_				:     Optional[Any]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Tuple					,						snake_case_				:     str					,						snake_case_				:     Optional[int]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     str)->				Optional[Any]:
						__lowerCAmelCase						 =self.num_choices
						__lowerCAmelCase						 =MraForMultipleChoice(config=snake_case_)
						model.to(snake_case_)
						model.eval()
						__lowerCAmelCase						 =input_ids.unsqueeze(1).expand(-1					,						self.num_choices					,						-1).contiguous()
						__lowerCAmelCase						 =token_type_ids.unsqueeze(1).expand(-1					,						self.num_choices					,						-1).contiguous()
						__lowerCAmelCase						 =input_mask.unsqueeze(1).expand(-1					,						self.num_choices					,						-1).contiguous()
						__lowerCAmelCase						 =model(
						    snake_case_					,						attention_mask=snake_case_					,						token_type_ids=snake_case_					,						labels=snake_case_					,						)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.num_choices))



	def 		UpperCamelCase				(    self				:     Tuple)->				List[Any]:
						__lowerCAmelCase						 =self.prepare_config_and_inputs()
						(
						    (
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						
						)						 =config_and_inputs
						__lowerCAmelCase						 ={"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
						return config, inputs_dict






@require_torch
class     __a			(  SCREAMING_SNAKE_CASE						,     unittest.TestCase					):
	SCREAMING_SNAKE_CASE  	=							(
	    (
	        MraModel,
	        MraForMaskedLM,
	        MraForMultipleChoice,
	        MraForQuestionAnswering,
	        MraForSequenceClassification,
	        MraForTokenClassification,
	    )
	    if is_torch_available()
	    else ()
	)
	SCREAMING_SNAKE_CASE  	=							False
	SCREAMING_SNAKE_CASE  	=							False
	SCREAMING_SNAKE_CASE  	=							False
	SCREAMING_SNAKE_CASE  	=							False

	SCREAMING_SNAKE_CASE  	=							()



	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Optional[int]:
						__lowerCAmelCase						 =MraModelTester(self)
						__lowerCAmelCase						 =ConfigTester(self					,						config_class=snake_case_					,						hidden_size=37)



	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Optional[Any]:
						self.config_tester.run_common_tests()



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Union[str, Any]:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*snake_case_)



	def 		UpperCamelCase				(    self				:     str)->				int:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
											__lowerCAmelCase						 =type
											self.model_tester.create_and_check_model(*snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Optional[Any]:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*snake_case_)



	def 		UpperCamelCase				(    self				:     List[str])->				str:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*snake_case_)



	def 		UpperCamelCase				(    self				:     Tuple)->				str:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*snake_case_)



	def 		UpperCamelCase				(    self				:     Union[str, Any])->				List[str]:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*snake_case_)



	def 		UpperCamelCase				(    self				:     str)->				Optional[Any]:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*snake_case_)



	@slow
	def 		UpperCamelCase				(    self				:     List[str])->				int:
						for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__lowerCAmelCase						 =MraModel.from_pretrained(snake_case_)
											self.assertIsNotNone(snake_case_)



	@unittest.skip(reason="""MRA does not output attentions""")
	def 		UpperCamelCase				(    self				:     Tuple)->				Tuple:
						return






@require_torch
class     __a			(  unittest.TestCase					):



	@slow
	def 		UpperCamelCase				(    self				:     Tuple)->				List[Any]:
						__lowerCAmelCase						 =MraModel.from_pretrained("""uw-madison/mra-base-512-4""")
						__lowerCAmelCase						 =torch.arange(2_56).unsqueeze(0)

						with torch.no_grad():
											__lowerCAmelCase						 =model(snake_case_)[0]

						__lowerCAmelCase						 =torch.Size((1, 2_56, 7_68))
						self.assertEqual(output.shape					,						snake_case_)

						__lowerCAmelCase						 =torch.tensor(
						    [[[-0.0_1_4_0, 0.0_8_3_0, -0.0_3_8_1], [0.1_5_4_6, 0.1_4_0_2, 0.0_2_2_0], [0.1_1_6_2, 0.0_8_5_1, 0.0_1_6_5]]])

						self.assertTrue(torch.allclose(output[:, :3, :3]					,						snake_case_					,						atol=1e-4))



	@slow
	def 		UpperCamelCase				(    self				:     Dict)->				int:
						__lowerCAmelCase						 =MraForMaskedLM.from_pretrained("""uw-madison/mra-base-512-4""")
						__lowerCAmelCase						 =torch.arange(2_56).unsqueeze(0)

						with torch.no_grad():
											__lowerCAmelCase						 =model(snake_case_)[0]

						__lowerCAmelCase						 =5_02_65

						__lowerCAmelCase						 =torch.Size((1, 2_56, vocab_size))
						self.assertEqual(output.shape					,						snake_case_)

						__lowerCAmelCase						 =torch.tensor(
						    [[[9.2_5_9_5, -3.6_0_3_8, 1_1.8_8_1_9], [9.3_8_6_9, -3.2_6_9_3, 1_1.0_9_5_6], [1_1.8_5_2_4, -3.4_9_3_8, 1_3.1_2_1_0]]])

						self.assertTrue(torch.allclose(output[:, :3, :3]					,						snake_case_					,						atol=1e-4))



	@slow
	def 		UpperCamelCase				(    self				:     Any)->				Any:
						__lowerCAmelCase						 =MraForMaskedLM.from_pretrained("""uw-madison/mra-base-4096-8-d3""")
						__lowerCAmelCase						 =torch.arange(40_96).unsqueeze(0)

						with torch.no_grad():
											__lowerCAmelCase						 =model(snake_case_)[0]

						__lowerCAmelCase						 =5_02_65

						__lowerCAmelCase						 =torch.Size((1, 40_96, vocab_size))
						self.assertEqual(output.shape					,						snake_case_)

						__lowerCAmelCase						 =torch.tensor(
						    [[[5.4_7_8_9, -2.3_5_6_4, 7.5_0_6_4], [7.9_0_6_7, -1.3_3_6_9, 9.9_6_6_8], [9.0_7_1_2, -1.8_1_0_6, 7.0_3_8_0]]])

						self.assertTrue(torch.allclose(output[:, :3, :3]					,						snake_case_					,						atol=1e-4))









import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase_													=      logging.get_logger(__name__)

lowercase_													=      '''▁'''

lowercase_													=      {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

lowercase_													=      {
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

lowercase_													=      {
    '''facebook/m2m100_418M''': 10_24,
}

# fmt: off
lowercase_													=      {
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}






class     __a			(  SCREAMING_SNAKE_CASE					):
	SCREAMING_SNAKE_CASE  	=							VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE  	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE  	=							PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE  	=							["input_ids", "attention_mask"]

	SCREAMING_SNAKE_CASE  	=							[]
	SCREAMING_SNAKE_CASE  	=							[]



	def __init__(    self				:     int					,						snake_case_				:     List[str]					,						snake_case_				:     Any					,						snake_case_				:     Union[str, Any]=None					,						snake_case_				:     Tuple=None					,						snake_case_				:     List[str]="<s>"					,						snake_case_				:     List[str]="</s>"					,						snake_case_				:     Optional[Any]="</s>"					,						snake_case_				:     Dict="<pad>"					,						snake_case_				:     Any="<unk>"					,						snake_case_				:     Tuple="m2m100"					,						snake_case_				:     Optional[Dict[str, Any]] = None					,						snake_case_				:     Dict=8					,						**snake_case_				:     List[Any]					,						)->				None:
						__lowerCAmelCase						 ={} if sp_model_kwargs is None else sp_model_kwargs

						__lowerCAmelCase						 =language_codes
						__lowerCAmelCase						 =FAIRSEQ_LANGUAGE_CODES[language_codes]
						__lowerCAmelCase						 ={lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

						__lowerCAmelCase						 =kwargs.get("""additional_special_tokens"""					,						[])
						kwargs["additional_special_tokens"] += [
						    self.get_lang_token(snake_case_)
						    for lang_code in fairseq_language_code
						    if self.get_lang_token(snake_case_) not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    src_lang=snake_case_					,						tgt_lang=snake_case_					,						bos_token=snake_case_					,						eos_token=snake_case_					,						sep_token=snake_case_					,						unk_token=snake_case_					,						pad_token=snake_case_					,						language_codes=snake_case_					,						sp_model_kwargs=self.sp_model_kwargs					,						num_madeup_words=snake_case_					,						**snake_case_					,						)

						__lowerCAmelCase						 =vocab_file
						__lowerCAmelCase						 =load_json(snake_case_)
						__lowerCAmelCase						 ={v: k for k, v in self.encoder.items()}
						__lowerCAmelCase						 =spm_file
						__lowerCAmelCase						 =load_spm(snake_case_					,						self.sp_model_kwargs)

						__lowerCAmelCase						 =len(self.encoder)

						__lowerCAmelCase						 ={
						    self.get_lang_token(snake_case_): self.encoder_size + i for i, lang_code in enumerate(snake_case_)
						}
						__lowerCAmelCase						 ={lang_code: self.encoder_size + i for i, lang_code in enumerate(snake_case_)}
						__lowerCAmelCase						 ={v: k for k, v in self.lang_token_to_id.items()}

						__lowerCAmelCase						 =src_lang if src_lang is not None else """en"""
						__lowerCAmelCase						 =tgt_lang
						__lowerCAmelCase						 =self.get_lang_id(self._src_lang)
						self.set_src_lang_special_tokens(self._src_lang)

						__lowerCAmelCase						 =num_madeup_words



	@property
	def 		UpperCamelCase				(    self				:     int)->				int:
						return len(self.encoder) + len(self.lang_token_to_id)



	@property
	def 		UpperCamelCase				(    self				:     Optional[Any])->				str:
						return self._src_lang



	@src_lang.setter
	def 		UpperCamelCase				(    self				:     List[Any]					,						snake_case_				:     str)->				None:
						__lowerCAmelCase						 =new_src_lang
						self.set_src_lang_special_tokens(self._src_lang)



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     str)->				List[str]:
						return self.sp_model.encode(snake_case_					,						out_type=snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[Any]					,						snake_case_				:     List[str])->				str:
						if token in self.lang_token_to_id:
											return self.lang_token_to_id[token]
						return self.encoder.get(snake_case_					,						self.encoder[self.unk_token])



	def 		UpperCamelCase				(    self				:     Dict					,						snake_case_				:     int)->				str:
						if index in self.id_to_lang_token:
											return self.id_to_lang_token[index]
						return self.decoder.get(snake_case_					,						self.unk_token)



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     Tuple)->				str:
						__lowerCAmelCase						 =[]
						__lowerCAmelCase						 =""""""
						for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
																out_string += self.sp_model.decode(snake_case_) + token
																__lowerCAmelCase						 =[]
											else:
																current_sub_tokens.append(snake_case_)
						out_string += self.sp_model.decode(snake_case_)
						return out_string.strip()



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     List[int]					,						snake_case_				:     Optional[List[int]] = None					,						snake_case_				:     bool = False)->				List[int]:

						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_					,						token_ids_a=snake_case_					,						already_has_special_tokens=snake_case_)

						__lowerCAmelCase						 =[1] * len(self.prefix_tokens)
						__lowerCAmelCase						 =[1] * len(self.suffix_tokens)
						if token_ids_a is None:
											return prefix_ones + ([0] * len(snake_case_)) + suffix_ones
						return prefix_ones + ([0] * len(snake_case_)) + ([0] * len(snake_case_)) + suffix_ones



	def 		UpperCamelCase				(    self				:     Any					,						snake_case_				:     List[int]					,						snake_case_				:     Optional[List[int]] = None)->				List[int]:
						if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Dict:
						__lowerCAmelCase						 ={self.convert_ids_to_tokens(snake_case_): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab



	def __getstate__(    self				:     Tuple)->				Dict:
						__lowerCAmelCase						 =self.__dict__.copy()
						__lowerCAmelCase						 =None
						return state



	def __setstate__(    self				:     List[Any]					,						snake_case_				:     Dict)->				None:
						__lowerCAmelCase						 =d

						# for backward compatibility
						if not hasattr(self					,						"""sp_model_kwargs"""):
											__lowerCAmelCase						 ={}

						__lowerCAmelCase						 =load_spm(self.spm_file					,						self.sp_model_kwargs)



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     str					,						snake_case_				:     Optional[str] = None)->				Tuple[str]:
						__lowerCAmelCase						 =Path(snake_case_)
						if not save_dir.is_dir():
											raise OSError(F"""{save_directory} should be a directory""")
						__lowerCAmelCase						 =save_dir / (
						    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
						)
						__lowerCAmelCase						 =save_dir / (
						    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
						)

						save_json(self.encoder					,						snake_case_)

						if os.path.abspath(self.spm_file) != os.path.abspath(snake_case_) and os.path.isfile(self.spm_file):
											copyfile(self.spm_file					,						snake_case_)
						elif not os.path.isfile(self.spm_file):
											with open(snake_case_					,						"""wb""") as fi:
																__lowerCAmelCase						 =self.sp_model.serialized_model_proto()
																fi.write(snake_case_)

						return (str(snake_case_), str(snake_case_))



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     List[str]					,						snake_case_				:     str = "en"					,						snake_case_				:     Optional[List[str]] = None					,						snake_case_				:     str = "ro"					,						**snake_case_				:     int					,						)->				BatchEncoding:
						__lowerCAmelCase						 =src_lang
						__lowerCAmelCase						 =tgt_lang
						self.set_src_lang_special_tokens(self.src_lang)
						return super().prepare_seqaseq_batch(snake_case_					,						snake_case_					,						**snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     int					,						snake_case_				:     Optional[str]					,						snake_case_				:     Optional[str]					,						**snake_case_				:     List[str])->				List[Any]:
						if src_lang is None or tgt_lang is None:
											raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""")
						__lowerCAmelCase						 =src_lang
						__lowerCAmelCase						 =self(snake_case_					,						add_special_tokens=snake_case_					,						**snake_case_)
						__lowerCAmelCase						 =self.get_lang_id(snake_case_)
						__lowerCAmelCase						 =tgt_lang_id
						return inputs



	def 		UpperCamelCase				(    self				:     Optional[int])->				Union[str, Any]:
						self.set_src_lang_special_tokens(self.src_lang)



	def 		UpperCamelCase				(    self				:     Dict)->				Dict:
						self.set_tgt_lang_special_tokens(self.tgt_lang)



	def 		UpperCamelCase				(    self				:     Union[str, Any]					,						snake_case_				:     str)->				None:
						__lowerCAmelCase						 =self.get_lang_token(snake_case_)
						__lowerCAmelCase						 =self.lang_token_to_id[lang_token]
						__lowerCAmelCase						 =[self.cur_lang_id]
						__lowerCAmelCase						 =[self.eos_token_id]



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     str)->				None:
						__lowerCAmelCase						 =self.get_lang_token(snake_case_)
						__lowerCAmelCase						 =self.lang_token_to_id[lang_token]
						__lowerCAmelCase						 =[self.cur_lang_id]
						__lowerCAmelCase						 =[self.eos_token_id]



	def 		UpperCamelCase				(    self				:     int					,						snake_case_				:     str)->				str:
						return self.lang_code_to_token[lang]



	def 		UpperCamelCase				(    self				:     List[str]					,						snake_case_				:     str)->				int:
						__lowerCAmelCase						 =self.get_lang_token(snake_case_)
						return self.lang_token_to_id[lang_token]






def        __lowerCAmelCase		(	__lowerCamelCase						:     str					, __lowerCamelCase						:     Dict[str, Any]						)	->    sentencepiece.SentencePieceProcessor:
					__lowerCAmelCase						 =sentencepiece.SentencePieceProcessor(**__lowerCamelCase						)
					spm.Load(str(__lowerCamelCase						)						)
					return spm






def        __lowerCAmelCase		(	__lowerCamelCase						:     str						)	->    Union[Dict, List]:
					with open(__lowerCamelCase					, """r"""						) as f:
										return json.load(__lowerCamelCase						)






def        __lowerCAmelCase		(	__lowerCamelCase						:     Any					, __lowerCamelCase						:     str						)	->    None:
					with open(__lowerCamelCase					, """w"""						) as f:
										json.dump(__lowerCamelCase					, __lowerCamelCase					, indent=2						)




