






import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotSmallConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
 import os

 # The slow tests are often failing with OOM error on GPU
 # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
 # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
 _a     : Any   	=      """platform"""

 import jax
 import jax.numpy as jnp

 from transformers.models.blenderbot_small.modeling_flax_blenderbot_small import (
     FlaxBlenderbotSmallForConditionalGeneration,
     FlaxBlenderbotSmallModel,
     shift_tokens_right,
 )


def 					snake_case__				(				UpperCAmelCase				:	Tuple     ,      UpperCAmelCase				:	List[Any]     ,      UpperCAmelCase				:	int=None     ,      UpperCAmelCase				:	List[Any]=None     ,      UpperCAmelCase				:	int=None     ,      UpperCAmelCase				:	Tuple=None     ,      UpperCAmelCase				:	int=None     ,      UpperCAmelCase				:	Optional[Any]=None     ,      ):





    if attention_mask is None:
        lowerCAmelCase__ :str   				=					np.where(input_ids != config.pad_token_id     ,      1     ,      0							)
    if decoder_attention_mask is None:
        lowerCAmelCase__ :int   				=					np.where(decoder_input_ids != config.pad_token_id     ,      1     ,      0							)
    if head_mask is None:
        lowerCAmelCase__ :Optional[int]   				=					np.ones((config.encoder_layers, config.encoder_attention_heads)							)
    if decoder_head_mask is None:
        lowerCAmelCase__ :Dict   				=					np.ones((config.decoder_layers, config.decoder_attention_heads)							)
    if cross_attn_head_mask is None:
        lowerCAmelCase__ :List[str]   				=					np.ones((config.decoder_layers, config.decoder_attention_heads)							)
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": attention_mask,
    }







class 						_UpperCAmelCase	:




      """simple docstring"""



      def __init__( self     ,						_lowerCAmelCase     ,						_lowerCAmelCase=13     ,						_lowerCAmelCase=7     ,						_lowerCAmelCase=True     ,						_lowerCAmelCase=False     ,						_lowerCAmelCase=99     ,						_lowerCAmelCase=16     ,						_lowerCAmelCase=2     ,						_lowerCAmelCase=4     ,						_lowerCAmelCase=4     ,						_lowerCAmelCase="gelu"     ,						_lowerCAmelCase=0.1     ,						_lowerCAmelCase=0.1     ,						_lowerCAmelCase=32     ,						_lowerCAmelCase=2     ,						_lowerCAmelCase=1     ,						_lowerCAmelCase=0     ,						_lowerCAmelCase=0.02     ,						):



          '''simple docstring'''
          lowerCAmelCase__ :Optional[Any]   				=					parent
          lowerCAmelCase__ :Any   				=					batch_size
          lowerCAmelCase__ :int   				=					seq_length
          lowerCAmelCase__ :Tuple   				=					is_training
          lowerCAmelCase__ :List[Any]   				=					use_labels
          lowerCAmelCase__ :List[Any]   				=					vocab_size
          lowerCAmelCase__ :Optional[Any]   				=					hidden_size
          lowerCAmelCase__ :str   				=					num_hidden_layers
          lowerCAmelCase__ :int   				=					num_attention_heads
          lowerCAmelCase__ :List[Any]   				=					intermediate_size
          lowerCAmelCase__ :Optional[Any]   				=					hidden_act
          lowerCAmelCase__ :List[Any]   				=					hidden_dropout_prob
          lowerCAmelCase__ :int   				=					attention_probs_dropout_prob
          lowerCAmelCase__ :str   				=					max_position_embeddings
          lowerCAmelCase__ :str   				=					eos_token_id
          lowerCAmelCase__ :Any   				=					pad_token_id
          lowerCAmelCase__ :Optional[int]   				=					bos_token_id
          lowerCAmelCase__ :List[str]   				=					initializer_range
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__ :str   				=					np.clip(ids_tensor([self.batch_size, self.seq_length - 1]     ,						self.vocab_size		)     ,						3     ,						self.vocab_size		)
          lowerCAmelCase__ :Optional[Any]   				=					np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)     ,						dtype=np.intaa		))     ,						-1		)

          lowerCAmelCase__ :int   				=					shift_tokens_right(__lowercase     ,						1     ,						2		)

          lowerCAmelCase__ :str   				=					BlenderbotSmallConfig(
              vocab_size=self.vocab_size     ,						d_model=self.hidden_size     ,						encoder_layers=self.num_hidden_layers     ,						decoder_layers=self.num_hidden_layers     ,						encoder_attention_heads=self.num_attention_heads     ,						decoder_attention_heads=self.num_attention_heads     ,						encoder_ffn_dim=self.intermediate_size     ,						decoder_ffn_dim=self.intermediate_size     ,						dropout=self.hidden_dropout_prob     ,						attention_dropout=self.attention_probs_dropout_prob     ,						max_position_embeddings=self.max_position_embeddings     ,						eos_token_id=self.eos_token_id     ,						bos_token_id=self.bos_token_id     ,						pad_token_id=self.pad_token_id     ,						initializer_range=self.initializer_range     ,						use_cache=__lowercase     ,						)
          lowerCAmelCase__ :Optional[Any]   				=					prepare_blenderbot_inputs_dict(__lowercase     ,						__lowercase     ,						__lowercase		)
          return config, inputs_dict
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__				,lowerCAmelCase__ :List[str]   				=					self.prepare_config_and_inputs()
          return config, inputs_dict
      def       snake_case_   ( self     ,						_lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__ :str   				=					20
          lowerCAmelCase__ :List[str]   				=					model_class_name(__lowercase		)

          lowerCAmelCase__ :int   				=					model.encode(inputs_dict["input_ids"]		)

          lowerCAmelCase__				,lowerCAmelCase__ :List[str]   				=					(
              inputs_dict["decoder_input_ids"],
              inputs_dict["decoder_attention_mask"],
          )

          lowerCAmelCase__ :int   				=					model.init_cache(decoder_input_ids.shape[0]     ,						__lowercase     ,						__lowercase		)
          lowerCAmelCase__ :Tuple   				=					jnp.ones((decoder_input_ids.shape[0], max_decoder_length)     ,						dtype="i4"		)

          lowerCAmelCase__ :Optional[int]   				=					jnp.broadcast_to(
              jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]     ,						(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)     ,						)
          lowerCAmelCase__ :int   				=					model.decode(
              decoder_input_ids[:, :-1]     ,						__lowercase     ,						decoder_attention_mask=__lowercase     ,						past_key_values=__lowercase     ,						decoder_position_ids=__lowercase     ,						)

          lowerCAmelCase__ :str   				=					jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]     ,						dtype="i4"		)
          lowerCAmelCase__ :Dict   				=					model.decode(
              decoder_input_ids[:, -1:]     ,						__lowercase     ,						decoder_attention_mask=__lowercase     ,						past_key_values=outputs_cache.past_key_values     ,						decoder_position_ids=__lowercase     ,						)

          lowerCAmelCase__ :Union[str, Any]   				=					model.decode(__lowercase     ,						__lowercase		)

          lowerCAmelCase__ :Dict   				=					np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
          self.parent.assertTrue(diff < 1e-3     ,						msg=f'''Max diff is {diff}'''		)



      def       snake_case_   ( self     ,						_lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__ :Tuple   				=					20
          lowerCAmelCase__ :Dict   				=					model_class_name(__lowercase		)

          lowerCAmelCase__ :Tuple   				=					model.encode(inputs_dict["input_ids"]		)

          lowerCAmelCase__				,lowerCAmelCase__ :str   				=					(
              inputs_dict["decoder_input_ids"],
              inputs_dict["decoder_attention_mask"],
          )

          lowerCAmelCase__ :Optional[int]   				=					jnp.concatenate(
              [
                  decoder_attention_mask,
                  jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])		),
              ]     ,						axis=-1     ,						)

          lowerCAmelCase__ :Optional[int]   				=					model.init_cache(decoder_input_ids.shape[0]     ,						__lowercase     ,						__lowercase		)
          lowerCAmelCase__ :Any   				=					jnp.broadcast_to(
              jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]     ,						(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)     ,						)

          lowerCAmelCase__ :Optional[int]   				=					model.decode(
              decoder_input_ids[:, :-1]     ,						__lowercase     ,						decoder_attention_mask=__lowercase     ,						past_key_values=__lowercase     ,						decoder_position_ids=__lowercase     ,						)
          lowerCAmelCase__ :int   				=					jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]     ,						dtype="i4"		)
          lowerCAmelCase__ :Optional[Any]   				=					model.decode(
              decoder_input_ids[:, -1:]     ,						__lowercase     ,						past_key_values=outputs_cache.past_key_values     ,						decoder_attention_mask=__lowercase     ,						decoder_position_ids=__lowercase     ,						)

          lowerCAmelCase__ :Any   				=					model.decode(__lowercase     ,						__lowercase     ,						decoder_attention_mask=__lowercase		)

          lowerCAmelCase__ :Tuple   				=					np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
          self.parent.assertTrue(diff < 1e-3     ,						msg=f'''Max diff is {diff}'''		)







@require_flax
class 						_UpperCAmelCase	(	unittest.TestCase		):




      """simple docstring"""



      A    =     99
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__ :Dict   				=					np.array(
              [
                  [71, 82, 18, 33, 46, 91, 2],
                  [68, 34, 26, 58, 30, 82, 2],
                  [5, 97, 17, 39, 94, 40, 2],
                  [76, 83, 94, 25, 70, 78, 2],
                  [87, 59, 41, 35, 48, 66, 2],
                  [55, 13, 16, 58, 5, 2, 1],  # note padding
                  [64, 27, 31, 51, 12, 75, 2],
                  [52, 64, 86, 17, 83, 39, 2],
                  [48, 61, 9, 24, 71, 82, 2],
                  [26, 1, 60, 48, 22, 13, 2],
                  [21, 5, 62, 28, 14, 76, 2],
                  [45, 98, 37, 86, 59, 48, 2],
                  [70, 70, 50, 9, 28, 0, 2],
              ]     ,						dtype=np.intaa     ,						)

          lowerCAmelCase__ :str   				=					input_ids.shape[0]
          lowerCAmelCase__ :Optional[Any]   				=					BlenderbotSmallConfig(
              vocab_size=self.vocab_size     ,						d_model=24     ,						encoder_layers=2     ,						decoder_layers=2     ,						encoder_attention_heads=2     ,						decoder_attention_heads=2     ,						encoder_ffn_dim=32     ,						decoder_ffn_dim=32     ,						max_position_embeddings=48     ,						eos_token_id=2     ,						pad_token_id=1     ,						bos_token_id=0     ,						)
          return config, input_ids, batch_size
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__				,lowerCAmelCase__				,lowerCAmelCase__ :Any   				=					self._get_config_and_data()
          lowerCAmelCase__ :int   				=					FlaxBlenderbotSmallForConditionalGeneration(__lowercase		)
          lowerCAmelCase__ :Any   				=					lm_model(input_ids=__lowercase		)
          lowerCAmelCase__ :List[Any]   				=					(batch_size, input_ids.shape[1], config.vocab_size)
          self.assertEqual(outputs["logits"].shape     ,						__lowercase		)
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__ :int   				=					BlenderbotSmallConfig(
              vocab_size=self.vocab_size     ,						d_model=14     ,						encoder_layers=2     ,						decoder_layers=2     ,						encoder_attention_heads=2     ,						decoder_attention_heads=2     ,						encoder_ffn_dim=8     ,						decoder_ffn_dim=8     ,						max_position_embeddings=48     ,						)
          lowerCAmelCase__ :Union[str, Any]   				=					FlaxBlenderbotSmallForConditionalGeneration(__lowercase		)
          lowerCAmelCase__ :Dict   				=					np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]     ,						dtype=np.intaa		)
          lowerCAmelCase__ :List[Any]   				=					np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]     ,						dtype=np.intaa		)
          lowerCAmelCase__ :Dict   				=					lm_model(input_ids=__lowercase     ,						decoder_input_ids=__lowercase		)
          lowerCAmelCase__ :Optional[int]   				=					(*summary.shape, config.vocab_size)
          self.assertEqual(outputs["logits"].shape     ,						__lowercase		)



      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__ :Tuple   				=					np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]     ,						dtype=np.intaa		)
          lowerCAmelCase__ :Any   				=					shift_tokens_right(__lowercase     ,						1     ,						2		)
          lowerCAmelCase__ :Optional[int]   				=					np.equal(__lowercase     ,						1		).astype(np.floataa		).sum()
          lowerCAmelCase__ :Dict   				=					np.equal(__lowercase     ,						1		).astype(np.floataa		).sum()
          self.assertEqual(shifted.shape     ,						input_ids.shape		)
          self.assertEqual(__lowercase     ,						n_pad_before - 1		)
          self.assertTrue(np.equal(shifted[:, 0]     ,						2		).all()		)







@require_flax
class 						_UpperCAmelCase	(	_A					,				unittest.TestCase					,				_A		):




      """simple docstring"""



      A    =     True
      A    =     (
          (
              FlaxBlenderbotSmallModel,
              FlaxBlenderbotSmallForConditionalGeneration,
          )
          if is_flax_available()
          else ()
      )
      A    =     (FlaxBlenderbotSmallForConditionalGeneration,) if is_flax_available() else ()
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__ :List[Any]   				=					FlaxBlenderbotSmallModelTester(self		)
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__				,lowerCAmelCase__ :Dict   				=					self.model_tester.prepare_config_and_inputs()
          for model_class in self.all_model_classes:
              self.model_tester.check_use_cache_forward(__lowercase     ,						__lowercase     ,						__lowercase		)
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__				,lowerCAmelCase__ :Dict   				=					self.model_tester.prepare_config_and_inputs()
          for model_class in self.all_model_classes:
              self.model_tester.check_use_cache_forward_with_attn_mask(__lowercase     ,						__lowercase     ,						__lowercase		)
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__				,lowerCAmelCase__ :List[Any]   				=					self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              with self.subTest(model_class.__name__		):
                  lowerCAmelCase__ :Tuple   				=					self._prepare_for_class(__lowercase     ,						__lowercase		)
                  lowerCAmelCase__ :List[Any]   				=					model_class(__lowercase		)

                  @jax.jit
                  def encode_jitted(_lowerCAmelCase     ,						_lowerCAmelCase=None     ,						**_lowerCAmelCase		):
                      return model.encode(input_ids=__lowercase     ,						attention_mask=__lowercase		)

                  with self.subTest("JIT Enabled"		):
                      lowerCAmelCase__ :Union[str, Any]   				=					encode_jitted(**__lowercase		).to_tuple()

                  with self.subTest("JIT Disabled"		):
                      with jax.disable_jit():
                          lowerCAmelCase__ :Tuple   				=					encode_jitted(**__lowercase		).to_tuple()

                  self.assertEqual(len(__lowercase		)     ,						len(__lowercase		)		)
                  for jitted_output, output in zip(__lowercase     ,						__lowercase		):
                      self.assertEqual(jitted_output.shape     ,						output.shape		)
      def       snake_case_   ( self		):



          '''simple docstring'''
          lowerCAmelCase__				,lowerCAmelCase__ :List[Any]   				=					self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              with self.subTest(model_class.__name__		):
                  lowerCAmelCase__ :Optional[Any]   				=					model_class(__lowercase		)
                  lowerCAmelCase__ :Any   				=					model.encode(inputs_dict["input_ids"]     ,						inputs_dict["attention_mask"]		)

                  lowerCAmelCase__ :List[str]   				=					{
                      "decoder_input_ids": inputs_dict["decoder_input_ids"],
                      "decoder_attention_mask": inputs_dict["decoder_attention_mask"],
                      "encoder_outputs": encoder_outputs,
                  }

                  @jax.jit
                  def decode_jitted(_lowerCAmelCase     ,						_lowerCAmelCase     ,						_lowerCAmelCase		):
                      return model.decode(
                          decoder_input_ids=__lowercase     ,						decoder_attention_mask=__lowercase     ,						encoder_outputs=__lowercase     ,						)

                  with self.subTest("JIT Enabled"		):
                      lowerCAmelCase__ :int   				=					decode_jitted(**__lowercase		).to_tuple()

                  with self.subTest("JIT Disabled"		):
                      with jax.disable_jit():
                          lowerCAmelCase__ :int   				=					decode_jitted(**__lowercase		).to_tuple()

                  self.assertEqual(len(__lowercase		)     ,						len(__lowercase		)		)
                  for jitted_output, output in zip(__lowercase     ,						__lowercase		):
                      self.assertEqual(jitted_output.shape     ,						output.shape		)



      @slow
      def       snake_case_   ( self		):



          '''simple docstring'''







          for model_class_name in self.all_model_classes:
              lowerCAmelCase__ :List[Any]   				=					model_class_name.from_pretrained("facebook/blenderbot_small-90M"		)
              # FlaxBlenderbotForSequenceClassification expects eos token in input_ids
              lowerCAmelCase__ :int   				=					np.ones((1, 1)		) * model.config.eos_token_id
              lowerCAmelCase__ :Tuple   				=					model(__lowercase		)
              self.assertIsNotNone(__lowercase		)






'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase					 =	logging.get_logger(__name__)

UpperCAmelCase					 =	'''▁'''

UpperCAmelCase					 =	{'''vocab_file''': '''sentencepiece.bpe.model'''}

UpperCAmelCase					 =	{
    '''vocab_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model'''
        ),
    }
}

UpperCAmelCase					 =	{
    '''facebook/nllb-200-distilled-600M''': 1024,
}

# fmt: off
UpperCAmelCase					 =	['''ace_Arab''', '''ace_Latn''', '''acm_Arab''', '''acq_Arab''', '''aeb_Arab''', '''afr_Latn''', '''ajp_Arab''', '''aka_Latn''', '''amh_Ethi''', '''apc_Arab''', '''arb_Arab''', '''ars_Arab''', '''ary_Arab''', '''arz_Arab''', '''asm_Beng''', '''ast_Latn''', '''awa_Deva''', '''ayr_Latn''', '''azb_Arab''', '''azj_Latn''', '''bak_Cyrl''', '''bam_Latn''', '''ban_Latn''', '''bel_Cyrl''', '''bem_Latn''', '''ben_Beng''', '''bho_Deva''', '''bjn_Arab''', '''bjn_Latn''', '''bod_Tibt''', '''bos_Latn''', '''bug_Latn''', '''bul_Cyrl''', '''cat_Latn''', '''ceb_Latn''', '''ces_Latn''', '''cjk_Latn''', '''ckb_Arab''', '''crh_Latn''', '''cym_Latn''', '''dan_Latn''', '''deu_Latn''', '''dik_Latn''', '''dyu_Latn''', '''dzo_Tibt''', '''ell_Grek''', '''eng_Latn''', '''epo_Latn''', '''est_Latn''', '''eus_Latn''', '''ewe_Latn''', '''fao_Latn''', '''pes_Arab''', '''fij_Latn''', '''fin_Latn''', '''fon_Latn''', '''fra_Latn''', '''fur_Latn''', '''fuv_Latn''', '''gla_Latn''', '''gle_Latn''', '''glg_Latn''', '''grn_Latn''', '''guj_Gujr''', '''hat_Latn''', '''hau_Latn''', '''heb_Hebr''', '''hin_Deva''', '''hne_Deva''', '''hrv_Latn''', '''hun_Latn''', '''hye_Armn''', '''ibo_Latn''', '''ilo_Latn''', '''ind_Latn''', '''isl_Latn''', '''ita_Latn''', '''jav_Latn''', '''jpn_Jpan''', '''kab_Latn''', '''kac_Latn''', '''kam_Latn''', '''kan_Knda''', '''kas_Arab''', '''kas_Deva''', '''kat_Geor''', '''knc_Arab''', '''knc_Latn''', '''kaz_Cyrl''', '''kbp_Latn''', '''kea_Latn''', '''khm_Khmr''', '''kik_Latn''', '''kin_Latn''', '''kir_Cyrl''', '''kmb_Latn''', '''kon_Latn''', '''kor_Hang''', '''kmr_Latn''', '''lao_Laoo''', '''lvs_Latn''', '''lij_Latn''', '''lim_Latn''', '''lin_Latn''', '''lit_Latn''', '''lmo_Latn''', '''ltg_Latn''', '''ltz_Latn''', '''lua_Latn''', '''lug_Latn''', '''luo_Latn''', '''lus_Latn''', '''mag_Deva''', '''mai_Deva''', '''mal_Mlym''', '''mar_Deva''', '''min_Latn''', '''mkd_Cyrl''', '''plt_Latn''', '''mlt_Latn''', '''mni_Beng''', '''khk_Cyrl''', '''mos_Latn''', '''mri_Latn''', '''zsm_Latn''', '''mya_Mymr''', '''nld_Latn''', '''nno_Latn''', '''nob_Latn''', '''npi_Deva''', '''nso_Latn''', '''nus_Latn''', '''nya_Latn''', '''oci_Latn''', '''gaz_Latn''', '''ory_Orya''', '''pag_Latn''', '''pan_Guru''', '''pap_Latn''', '''pol_Latn''', '''por_Latn''', '''prs_Arab''', '''pbt_Arab''', '''quy_Latn''', '''ron_Latn''', '''run_Latn''', '''rus_Cyrl''', '''sag_Latn''', '''san_Deva''', '''sat_Beng''', '''scn_Latn''', '''shn_Mymr''', '''sin_Sinh''', '''slk_Latn''', '''slv_Latn''', '''smo_Latn''', '''sna_Latn''', '''snd_Arab''', '''som_Latn''', '''sot_Latn''', '''spa_Latn''', '''als_Latn''', '''srd_Latn''', '''srp_Cyrl''', '''ssw_Latn''', '''sun_Latn''', '''swe_Latn''', '''swh_Latn''', '''szl_Latn''', '''tam_Taml''', '''tat_Cyrl''', '''tel_Telu''', '''tgk_Cyrl''', '''tgl_Latn''', '''tha_Thai''', '''tir_Ethi''', '''taq_Latn''', '''taq_Tfng''', '''tpi_Latn''', '''tsn_Latn''', '''tso_Latn''', '''tuk_Latn''', '''tum_Latn''', '''tur_Latn''', '''twi_Latn''', '''tzm_Tfng''', '''uig_Arab''', '''ukr_Cyrl''', '''umb_Latn''', '''urd_Arab''', '''uzn_Latn''', '''vec_Latn''', '''vie_Latn''', '''war_Latn''', '''wol_Latn''', '''xho_Latn''', '''ydd_Hebr''', '''yor_Latn''', '''yue_Hant''', '''zho_Hans''', '''zho_Hant''', '''zul_Latn''']


class 	lowerCAmelCase  (   A ):
       lowerCAmelCase_												=			VOCAB_FILES_NAMES
       lowerCAmelCase_												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase_												=			PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase_												=			["input_ids", "attention_mask"]

       lowerCAmelCase_												=			[]
       lowerCAmelCase_												=			[]







       def __init__(				self      :			Dict    ,   __lowercase      :			List[str]    ,   __lowercase      :			int="<s>"    ,   __lowercase      :			Any="</s>"    ,   __lowercase      :			Tuple="</s>"    ,   __lowercase      :			Dict="<s>"    ,   __lowercase      :			Dict="<unk>"    ,   __lowercase      :			List[Any]="<pad>"    ,   __lowercase      :			Dict="<mask>"    ,   __lowercase      :			Tuple=None    ,   __lowercase      :			List[str]=None    ,   __lowercase      :			Union[str, Any]=None    ,   __lowercase      :			Optional[Dict[str, Any]] = None    ,   __lowercase      :			List[str]=None    ,   __lowercase      :			List[str]=False    ,   **__lowercase      :			Union[str, Any]    ,   ):







            """simple docstring"""







            __lowercase      		=AddedToken(__lowercase    ,   lstrip=__lowercase    ,   rstrip=__lowercase   ) if isinstance(__lowercase    ,   __lowercase   ) else mask_token

            __lowercase      		={} if sp_model_kwargs is None else sp_model_kwargs
            __lowercase      		=legacy_behaviour

            super().__init__(
                bos_token=__lowercase    ,   eos_token=__lowercase    ,   unk_token=__lowercase    ,   sep_token=__lowercase    ,   cls_token=__lowercase    ,   pad_token=__lowercase    ,   mask_token=__lowercase    ,   tokenizer_file=__lowercase    ,   src_lang=__lowercase    ,   tgt_lang=__lowercase    ,   additional_special_tokens=__lowercase    ,   sp_model_kwargs=self.sp_model_kwargs    ,   legacy_behaviour=__lowercase    ,   **__lowercase    ,   )

            __lowercase      		=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.Load(str(__lowercase   )   )
            __lowercase      		=vocab_file
            # Original fairseq vocab and spm vocab must be "aligned":
            # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
            # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
            # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
            # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

            # Mimic fairseq token-to-id alignment for the first 4 token
            __lowercase      		={'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

            # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
            __lowercase      		=1

            __lowercase      		=len(self.sp_model   )
            __lowercase      		={
                code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__lowercase   )
            }
            __lowercase      		={v: k for k, v in self.lang_code_to_id.items()}
            __lowercase      		=len(self.sp_model   ) + len(self.lang_code_to_id   ) + self.fairseq_offset

            self.fairseq_tokens_to_ids.update(self.lang_code_to_id   )
            __lowercase      		={v: k for k, v in self.fairseq_tokens_to_ids.items()}
            __lowercase      		=list(self.lang_code_to_id.keys()   )

            if additional_special_tokens is not None:
                 # Only add those special tokens if they are not already there.
                 self._additional_special_tokens.extend(
                     [t for t in additional_special_tokens if t not in self._additional_special_tokens]   )

            __lowercase      		=src_lang if src_lang is not None else 'eng_Latn'
            __lowercase      		=self.lang_code_to_id[self._src_lang]
            __lowercase      		=tgt_lang
            self.set_src_lang_special_tokens(self._src_lang   )







       def __getstate__(				self      :			Tuple   ):







            """simple docstring"""







            __lowercase      		=self.__dict__.copy()
            __lowercase      		=None
            __lowercase      		=self.sp_model.serialized_model_proto()
            return state







       def __setstate__(				self      :			List[Any]    ,   __lowercase      :			List[Any]   ):







            """simple docstring"""







            __lowercase      		=d

            # for backward compatibility
            if not hasattr(self    ,   'sp_model_kwargs'   ):
                 __lowercase      		={}

            __lowercase      		=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )







       @property
       def       snake_case (				self      :			Dict   ):







            """simple docstring"""







            return len(self.sp_model   ) + len(self.lang_code_to_id   ) + self.fairseq_offset + 1  # Plus 1 for the mask token







       @property
       def       snake_case (				self      :			Tuple   ):







            """simple docstring"""







            return self._src_lang







       @src_lang.setter
       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			str   ):







            """simple docstring"""







            __lowercase      		=new_src_lang
            self.set_src_lang_special_tokens(self._src_lang   )







       def       snake_case (				self      :			List[str]    ,   __lowercase      :			List[int]    ,   __lowercase      :			Optional[List[int]] = None    ,   __lowercase      :			bool = False   ):







            """simple docstring"""







            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__lowercase    ,   token_ids_a=__lowercase    ,   already_has_special_tokens=__lowercase   )

            __lowercase      		=[1] * len(self.prefix_tokens   )
            __lowercase      		=[1] * len(self.suffix_tokens   )
            if token_ids_a is None:
                 return prefix_ones + ([0] * len(__lowercase   )) + suffix_ones
            return prefix_ones + ([0] * len(__lowercase   )) + ([0] * len(__lowercase   )) + suffix_ones







       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			List[int]    ,   __lowercase      :			Optional[List[int]] = None   ):







            """simple docstring"""







            if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







       def       snake_case (				self      :			int    ,   __lowercase      :			List[int]    ,   __lowercase      :			Optional[List[int]] = None   ):







            """simple docstring"""







            __lowercase      		=[self.sep_token_id]
            __lowercase      		=[self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep   ) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			Dict    ,   __lowercase      :			str    ,   __lowercase      :			Optional[str]    ,   __lowercase      :			Optional[str]    ,   **__lowercase      :			Optional[int]   ):







            """simple docstring"""







            if src_lang is None or tgt_lang is None:
                 raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
            __lowercase      		=src_lang
            __lowercase      		=self(__lowercase    ,   add_special_tokens=__lowercase    ,   return_tensors=__lowercase    ,   **__lowercase   )
            __lowercase      		=self.convert_tokens_to_ids(__lowercase   )
            __lowercase      		=tgt_lang_id
            return inputs







       def       snake_case (				self      :			Any   ):







            """simple docstring"""







            __lowercase      		={self.convert_ids_to_tokens(__lowercase   ): i for i in range(self.vocab_size   )}
            vocab.update(self.added_tokens_encoder   )
            return vocab







       def       snake_case (				self      :			Any    ,   __lowercase      :			str   ):







            """simple docstring"""







            return self.sp_model.encode(__lowercase    ,   out_type=__lowercase   )







       def       snake_case (				self      :			int    ,   __lowercase      :			int   ):







            """simple docstring"""







            if token in self.fairseq_tokens_to_ids:
                 return self.fairseq_tokens_to_ids[token]
            __lowercase      		=self.sp_model.PieceToId(__lowercase   )

            # Need to return unknown token if the SP model returned 0
            return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







       def       snake_case (				self      :			int    ,   __lowercase      :			Any   ):







            """simple docstring"""







            if index in self.fairseq_ids_to_tokens:
                 return self.fairseq_ids_to_tokens[index]
            return self.sp_model.IdToPiece(index - self.fairseq_offset   )







       def       snake_case (				self      :			str    ,   __lowercase      :			List[str]   ):







            """simple docstring"""







            __lowercase      		=''.join(__lowercase   ).replace(__lowercase    ,   ' '   ).strip()
            return out_string







       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			str    ,   __lowercase      :			Optional[str] = None   ):







            """simple docstring"""







            if not os.path.isdir(__lowercase   ):
                 logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
                 return
            __lowercase      		=os.path.join(
                __lowercase    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

            if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowercase   ) and os.path.isfile(self.vocab_file   ):
                 copyfile(self.vocab_file    ,   __lowercase   )
            elif not os.path.isfile(self.vocab_file   ):
                 with open(__lowercase    ,   'wb'   ) as fi:
                      __lowercase      		=self.sp_model.serialized_model_proto()
                      fi.write(__lowercase   )

            return (out_vocab_file,)







       def       snake_case (				self      :			List[str]    ,   __lowercase      :			List[str]    ,   __lowercase      :			str = "eng_Latn"    ,   __lowercase      :			Optional[List[str]] = None    ,   __lowercase      :			str = "fra_Latn"    ,   **__lowercase      :			List[str]    ,   ):







            """simple docstring"""







            __lowercase      		=src_lang
            __lowercase      		=tgt_lang
            return super().prepare_seqaseq_batch(__lowercase    ,   __lowercase    ,   **__lowercase   )







       def       snake_case (				self      :			Optional[int]   ):







            """simple docstring"""







            return self.set_src_lang_special_tokens(self.src_lang   )







       def       snake_case (				self      :			List[Any]   ):







            """simple docstring"""







            return self.set_tgt_lang_special_tokens(self.tgt_lang   )







       def       snake_case (				self      :			Optional[Any]    ,   __lowercase      :			List[str]   ):







            """simple docstring"""







            __lowercase      		=self.lang_code_to_id[src_lang]
            if self.legacy_behaviour:
                 __lowercase      		=[]
                 __lowercase      		=[self.eos_token_id, self.cur_lang_code]
            else:
                 __lowercase      		=[self.cur_lang_code]
                 __lowercase      		=[self.eos_token_id]







       def       snake_case (				self      :			Union[str, Any]    ,   __lowercase      :			str   ):







            """simple docstring"""







            __lowercase      		=self.lang_code_to_id[lang]
            if self.legacy_behaviour:
                 __lowercase      		=[]
                 __lowercase      		=[self.eos_token_id, self.cur_lang_code]
            else:
                 __lowercase      		=[self.cur_lang_code]
                 __lowercase      		=[self.eos_token_id]


