




'''simple docstring'''


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_      :  Dict				     =  get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCAmelCase_      :  Any				     =  25_6047
lowerCAmelCase_      :  Union[str, Any]				     =  25_6145







@require_sentencepiece
@require_tokenizers
class      SCREAMING_SNAKE_CASE    (						snake_case_							,      unittest.TestCase				):
      __magic_name__  :       str   							=					NllbTokenizer
      __magic_name__  :       Optional[Any]   							=					NllbTokenizerFast
      __magic_name__  :       str   							=					True
      __magic_name__  :       Union[str, Any]   							=					True
      __magic_name__  :       List[str]   							=					{}






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           super().setUp()

           # We have a SentencePiece fixture for testing
           a_     :							Optional[int]     						=  NllbTokenizer(lowercase__						,						keep_accents=lowercase__       )
           tokenizer.save_pretrained(self.tmpdirname       )






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           a_     :							Dict     						=  NllbTokenizer(lowercase__						,						keep_accents=lowercase__       )

           a_     :							Any     						=  tokenizer.tokenize("""This is a test"""       )
           self.assertListEqual(lowercase__						,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(lowercase__       )						,						[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,						)

           a_     :							Optional[int]     						=  tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
           self.assertListEqual(
               lowercase__						,						[
                   SPIECE_UNDERLINE + """I""",
                   SPIECE_UNDERLINE + """was""",
                   SPIECE_UNDERLINE + """b""",
                   """or""",
                   """n""",
                   SPIECE_UNDERLINE + """in""",
                   SPIECE_UNDERLINE + """""",
                   """9""",
                   """2""",
                   """0""",
                   """0""",
                   """0""",
                   """,""",
                   SPIECE_UNDERLINE + """and""",
                   SPIECE_UNDERLINE + """this""",
                   SPIECE_UNDERLINE + """is""",
                   SPIECE_UNDERLINE + """f""",
                   """al""",
                   """s""",
                   """é""",
                   """.""",
               ]						,						)
           a_     :							Tuple     						=  tokenizer.convert_tokens_to_ids(lowercase__       )
           self.assertListEqual(
               lowercase__						,						[
                   value + tokenizer.fairseq_offset
                   for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
               ]						,						)

           a_     :							List[Any]     						=  tokenizer.convert_ids_to_tokens(lowercase__       )
           self.assertListEqual(
               lowercase__						,						[
                   SPIECE_UNDERLINE + """I""",
                   SPIECE_UNDERLINE + """was""",
                   SPIECE_UNDERLINE + """b""",
                   """or""",
                   """n""",
                   SPIECE_UNDERLINE + """in""",
                   SPIECE_UNDERLINE + """""",
                   """<unk>""",
                   """2""",
                   """0""",
                   """0""",
                   """0""",
                   """,""",
                   SPIECE_UNDERLINE + """and""",
                   SPIECE_UNDERLINE + """this""",
                   SPIECE_UNDERLINE + """is""",
                   SPIECE_UNDERLINE + """f""",
                   """al""",
                   """s""",
                   """<unk>""",
                   """.""",
               ]						,						)






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							str     						=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                     a_     :							Union[str, Any]     						=  self.rust_tokenizer_class.from_pretrained(lowercase__						,						**lowercase__       )
                     a_     :							Union[str, Any]     						=  self.tokenizer_class.from_pretrained(lowercase__						,						**lowercase__       )

                     a_     :							Optional[int]     						=  tempfile.mkdtemp()

                     a_     :							List[str]     						=  tokenizer_r.save_pretrained(lowercase__       )
                     a_     :							str     						=  tokenizer_p.save_pretrained(lowercase__       )

                     # Checks it save with the same files + the tokenizer.json file for the fast one
                     self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files       )       )
                     a_     :							List[str]     						=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f       )
                     self.assertSequenceEqual(lowercase__						,						lowercase__       )

                     # Checks everything loads correctly in the same way
                     a_     :							int     						=  tokenizer_r.from_pretrained(lowercase__       )
                     a_     :							Union[str, Any]     						=  tokenizer_p.from_pretrained(lowercase__       )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                          self.assertTrue(hasattr(lowercase__						,						lowercase__       )       )

                     shutil.rmtree(lowercase__       )

                     # Save tokenizer rust, legacy_format=True
                     a_     :							Optional[Any]     						=  tempfile.mkdtemp()

                     a_     :							Optional[int]     						=  tokenizer_r.save_pretrained(lowercase__						,						legacy_format=lowercase__       )
                     a_     :							List[Any]     						=  tokenizer_p.save_pretrained(lowercase__       )

                     # Checks it save with the same files
                     self.assertSequenceEqual(lowercase__						,						lowercase__       )

                     # Checks everything loads correctly in the same way
                     a_     :							int     						=  tokenizer_r.from_pretrained(lowercase__       )
                     a_     :							Any     						=  tokenizer_p.from_pretrained(lowercase__       )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                          self.assertTrue(hasattr(lowercase__						,						lowercase__       )       )

                     shutil.rmtree(lowercase__       )

                     # Save tokenizer rust, legacy_format=False
                     a_     :							Optional[int]     						=  tempfile.mkdtemp()

                     a_     :							Tuple     						=  tokenizer_r.save_pretrained(lowercase__						,						legacy_format=lowercase__       )
                     a_     :							Optional[int]     						=  tokenizer_p.save_pretrained(lowercase__       )

                     # Checks it saved the tokenizer.json file
                     self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files       )       )

                     # Checks everything loads correctly in the same way
                     a_     :							Union[str, Any]     						=  tokenizer_r.from_pretrained(lowercase__       )
                     a_     :							Optional[Any]     						=  tokenizer_p.from_pretrained(lowercase__       )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                          self.assertTrue(hasattr(lowercase__						,						lowercase__       )       )

                     shutil.rmtree(lowercase__       )






      @require_torch
      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           if not self.test_seqaseq:
                return

           a_     :							Tuple     						=  self.get_tokenizers()
           for tokenizer in tokenizers:
                with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     # Longer text that will definitely require truncation.
                     a_     :							List[str]     						=  [
                         """ UN Chief Says There Is No Military Solution in Syria""",
                         """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                         """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                         """ will only worsen the violence and misery for millions of people.""",
                     ]
                     a_     :							Any     						=  [
                         """Şeful ONU declară că nu există o soluţie militară în Siria""",
                         """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                         """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                         """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                     ]
                     try:
                          a_     :							int     						=  tokenizer.prepare_seqaseq_batch(
                              src_texts=lowercase__						,						tgt_texts=lowercase__						,						max_length=3						,						max_target_length=10						,						return_tensors="""pt"""						,						src_lang="""eng_Latn"""						,						tgt_lang="""ron_Latn"""						,						)
                     except NotImplementedError:
                          return
                     self.assertEqual(batch.input_ids.shape[1]						,						3       )
                     self.assertEqual(batch.labels.shape[1]						,						10       )
                     # max_target_length will default to max_length if not specified
                     a_     :							Optional[Any]     						=  tokenizer.prepare_seqaseq_batch(
                         lowercase__						,						tgt_texts=lowercase__						,						max_length=3						,						return_tensors="""pt"""       )
                     self.assertEqual(batch.input_ids.shape[1]						,						3       )
                     self.assertEqual(batch.labels.shape[1]						,						3       )

                     a_     :							Tuple     						=  tokenizer.prepare_seqaseq_batch(
                         src_texts=lowercase__						,						max_length=3						,						max_target_length=10						,						return_tensors="""pt"""       )
                     self.assertEqual(batch_encoder_only.input_ids.shape[1]						,						3       )
                     self.assertEqual(batch_encoder_only.attention_mask.shape[1]						,						3       )
                     self.assertNotIn("""decoder_input_ids"""						,						lowercase__       )






      @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""       )
      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           pass






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                     a_     :							Optional[int]     						=  [AddedToken("""<special>"""						,						lstrip=lowercase__       )]

                     a_     :							Any     						=  self.rust_tokenizer_class.from_pretrained(
                         lowercase__						,						additional_special_tokens=lowercase__						,						**lowercase__       )
                     a_     :							Any     						=  tokenizer_r.encode("""Hey this is a <special> token"""       )

                     a_     :							int     						=  tokenizer_r.encode("""<special>"""						,						add_special_tokens=lowercase__       )[0]

                     self.assertTrue(special_token_id in r_output       )

                     if self.test_slow_tokenizer:
                          a_     :							List[str]     						=  self.rust_tokenizer_class.from_pretrained(
                              lowercase__						,						additional_special_tokens=lowercase__						,						**lowercase__						,						)
                          a_     :							str     						=  self.tokenizer_class.from_pretrained(
                              lowercase__						,						additional_special_tokens=lowercase__						,						**lowercase__       )

                          a_     :							Optional[int]     						=  tokenizer_p.encode("""Hey this is a <special> token"""       )

                          a_     :							List[str]     						=  tokenizer_cr.encode("""Hey this is a <special> token"""       )

                          self.assertEqual(lowercase__						,						lowercase__       )
                          self.assertEqual(lowercase__						,						lowercase__       )
                          self.assertTrue(special_token_id in p_output       )
                          self.assertTrue(special_token_id in cr_output       )







@require_torch
@require_sentencepiece
@require_tokenizers
class      SCREAMING_SNAKE_CASE    (						unittest.TestCase				):
      __magic_name__  :       List[Any]   							=					'''facebook/nllb-200-distilled-600M'''
      __magic_name__  :       Union[str, Any]   							=					[
          ''' UN Chief Says There Is No Military Solution in Syria''',
          ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
      ]
      __magic_name__  :       Any   							=					[
          '''Şeful ONU declară că nu există o soluţie militară în Siria''',
          '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
          ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
          ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
      ]
      __magic_name__  :       Optional[Any]   							=					[
          25_6047,
          1_6297,
          13_4408,
          8165,
          24_8066,
          1_4734,
          950,
          1135,
          10_5721,
          3573,
          83,
          2_7352,
          108,
          4_9486,
          2,
      ]






      @classmethod
      def 							lowercase_						( cls   :  Any       ):


           '''simple docstring'''

           a_     :							NllbTokenizer     						=  NllbTokenizer.from_pretrained(
               cls.checkpoint_name						,						src_lang="""eng_Latn"""						,						tgt_lang="""ron_Latn"""       )
           a_     :							str     						=  1
           return cls






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]						,						25_6001       )
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]						,						25_6002       )
           self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]						,						25_6057       )






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							Tuple     						=  self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
           self.assertListEqual(self.expected_src_tokens						,						lowercase__       )






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           self.assertIn(lowercase__						,						self.tokenizer.all_special_ids       )
           # fmt: off
           a_     :							int     						=  [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
           # fmt: on

           a_     :							str     						=  self.tokenizer.decode(lowercase__						,						skip_special_tokens=lowercase__       )
           a_     :							Any     						=  self.tokenizer.decode(generated_ids[1:]						,						skip_special_tokens=lowercase__       )
           self.assertEqual(lowercase__						,						lowercase__       )
           self.assertNotIn(self.tokenizer.eos_token						,						lowercase__       )






      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							List[Any]     						=  ["""this is gunna be a long sentence """ * 20]
           assert isinstance(src_text[0]						,						lowercase__       )
           a_     :							str     						=  10
           a_     :							List[str]     						=  self.tokenizer(lowercase__						,						max_length=lowercase__						,						truncation=lowercase__       ).input_ids[0]
           self.assertEqual(ids[-1]						,						2       )
           self.assertEqual(ids[0]						,						lowercase__       )
           self.assertEqual(len(lowercase__       )						,						lowercase__       )






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]       )						,						[25_6203, 3]       )






      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  tempfile.mkdtemp()
           a_     :							int     						=  self.tokenizer.fairseq_tokens_to_ids
           self.tokenizer.save_pretrained(lowercase__       )
           a_     :							Optional[Any]     						=  NllbTokenizer.from_pretrained(lowercase__       )
           self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,						lowercase__       )






      @require_torch
      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  self.tokenizer(
               self.src_text						,						text_target=self.tgt_text						,						padding=lowercase__						,						truncation=lowercase__						,						max_length=len(self.expected_src_tokens       )						,						return_tensors="""pt"""						,						)
           a_     :							Union[str, Any]     						=  shift_tokens_right(
               batch["""labels"""]						,						self.tokenizer.pad_token_id						,						self.tokenizer.lang_code_to_id["""ron_Latn"""]       )

           self.assertIsInstance(lowercase__						,						lowercase__       )

           self.assertEqual((2, 15)						,						batch.input_ids.shape       )
           self.assertEqual((2, 15)						,						batch.attention_mask.shape       )
           a_     :							Optional[Any]     						=  batch.input_ids.tolist()[0]
           self.assertListEqual(self.expected_src_tokens						,						lowercase__       )
           self.assertEqual(lowercase__						,						batch.decoder_input_ids[0, 0]       )  # EOS
           # Test that special tokens are reset
           self.assertEqual(self.tokenizer.prefix_tokens						,						[EN_CODE]       )
           self.assertEqual(self.tokenizer.suffix_tokens						,						[self.tokenizer.eos_token_id]       )






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							List[str]     						=  self.tokenizer(self.src_text						,						padding=lowercase__						,						truncation=lowercase__						,						max_length=3						,						return_tensors="""pt"""       )
           a_     :							Optional[Any]     						=  self.tokenizer(
               text_target=self.tgt_text						,						padding=lowercase__						,						truncation=lowercase__						,						max_length=10						,						return_tensors="""pt"""       )
           a_     :							List[str]     						=  targets["""input_ids"""]
           a_     :							Optional[int]     						=  shift_tokens_right(
               lowercase__						,						self.tokenizer.pad_token_id						,						decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]						,						)

           self.assertEqual(batch.input_ids.shape[1]						,						3       )
           self.assertEqual(batch.decoder_input_ids.shape[1]						,						10       )






      @require_torch
      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           a_     :							Tuple     						=  self.tokenizer._build_translation_inputs(
               """A test"""						,						return_tensors="""pt"""						,						src_lang="""eng_Latn"""						,						tgt_lang="""fra_Latn"""       )

           self.assertEqual(
               nested_simplify(lowercase__       )						,						{
                   # A, test, EOS, en_XX
                   """input_ids""": [[25_6047, 70, 7356, 2]],
                   """attention_mask""": [[1, 1, 1, 1]],
                   # ar_AR
                   """forced_bos_token_id""": 25_6057,
               }						,						)






      @require_torch
      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							int     						=  True
           a_     :							List[str]     						=  self.tokenizer(
               """UN Chief says there is no military solution in Syria"""						,						src_lang="""eng_Latn"""						,						tgt_lang="""fra_Latn"""       )
           self.assertEqual(
               inputs.input_ids						,						[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]       )

           a_     :							Union[str, Any]     						=  False
           a_     :							Union[str, Any]     						=  self.tokenizer(
               """UN Chief says there is no military solution in Syria"""						,						src_lang="""eng_Latn"""						,						tgt_lang="""fra_Latn"""       )
           self.assertEqual(
               inputs.input_ids						,						[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]       )






'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_      :  Optional[int]				     =  logging.get_logger(__name__)

lowerCAmelCase_      :  str				     =  '▁'

lowerCAmelCase_      :  List[str]				     =  {'vocab_file': 'sentencepiece.bpe.model'}

lowerCAmelCase_      :  Optional[Any]				     =  {
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

lowerCAmelCase_      :  List[str]				     =  {
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
lowerCAmelCase_      :  Optional[Any]				     =  ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']







class      SCREAMING_SNAKE_CASE    (						snake_case_				):
      __magic_name__  :       Any   							=					VOCAB_FILES_NAMES
      __magic_name__  :       List[Any]   							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __magic_name__  :       Optional[Any]   							=					PRETRAINED_VOCAB_FILES_MAP
      __magic_name__  :       Union[str, Any]   							=					['''input_ids''', '''attention_mask''']

      __magic_name__  :       List[int]   							=					[]
      __magic_name__  :       List[int]   							=					[]






      def __init__( self   :  int						,						lowercase__   :  Tuple						,						lowercase__   :  Optional[int]=None						,						lowercase__   :  str=None						,						lowercase__   :  Dict="</s>"						,						lowercase__   :  Tuple="</s>"						,						lowercase__   :  Any="<s>"						,						lowercase__   :  Tuple="<unk>"						,						lowercase__   :  List[Any]="<pad>"						,						lowercase__   :  Dict="<mask>"						,						lowercase__   :  Optional[Dict[str, Any]] = None						,						**lowercase__   :  Tuple						,						):


           '''simple docstring'''

           a_     :							Any     						=  AddedToken(lowercase__						,						lstrip=lowercase__						,						rstrip=lowercase__       ) if isinstance(lowercase__						,						lowercase__       ) else mask_token

           a_     :							Union[str, Any]     						=  {} if sp_model_kwargs is None else sp_model_kwargs

           a_     :							int     						=  kwargs.get("""additional_special_tokens"""						,						[]       )
           kwargs["additional_special_tokens"] += [
               code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               src_lang=lowercase__						,						tgt_lang=lowercase__						,						eos_token=lowercase__						,						unk_token=lowercase__						,						sep_token=lowercase__						,						cls_token=lowercase__						,						pad_token=lowercase__						,						mask_token=lowercase__						,						sp_model_kwargs=self.sp_model_kwargs						,						**lowercase__						,						)

           a_     :							Union[str, Any]     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs       )
           self.sp_model.Load(str(lowercase__       )       )
           a_     :							Union[str, Any]     						=  vocab_file

           # Original fairseq vocab and spm vocab must be "aligned":
           # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
           # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
           # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
           # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

           # Mimic fairseq token-to-id alignment for the first 4 token
           a_     :							Tuple     						=  {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

           # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
           a_     :							List[str]     						=  1

           a_     :							List[Any]     						=  len(self.sp_model       )
           a_     :							List[str]     						=  {
               code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(lowercase__       )
           }
           a_     :							str     						=  {v: k for k, v in self.lang_code_to_id.items()}
           a_     :							Any     						=  len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

           self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
           a_     :							int     						=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}

           a_     :							List[Any]     						=  src_lang if src_lang is not None else """en_XX"""
           a_     :							Any     						=  self.lang_code_to_id[self._src_lang]
           a_     :							Union[str, Any]     						=  tgt_lang
           self.set_src_lang_special_tokens(self._src_lang       )






      @property
      def 							lowercase_						( self   :  Union[str, Any]       ):


           '''simple docstring'''

           return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token






      @property
      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           return self._src_lang






      @src_lang.setter
      def 							lowercase_						( self   :  Optional[Any]						,						lowercase__   :  str       ):


           '''simple docstring'''

           a_     :							Tuple     						=  new_src_lang
           self.set_src_lang_special_tokens(self._src_lang       )






      def __getstate__( self   :  Any       ):


           '''simple docstring'''

           a_     :							str     						=  self.__dict__.copy()
           a_     :							Dict     						=  None
           return state






      def __setstate__( self   :  Optional[Any]						,						lowercase__   :  Dict       ):


           '''simple docstring'''

           a_     :							Any     						=  d

           # for backward compatibility
           if not hasattr(self						,						"""sp_model_kwargs"""       ):
                a_     :							Tuple     						=  {}

           a_     :							int     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs       )
           self.sp_model.Load(self.vocab_file       )






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Dict     						=  {self.convert_ids_to_tokens(lowercase__       ): i for i in range(self.vocab_size       )}
           vocab.update(self.added_tokens_encoder       )
           return vocab






      def 							lowercase_						( self   :  Optional[int]						,						lowercase__   :  str       ):


           '''simple docstring'''

           return self.sp_model.encode(lowercase__						,						out_type=lowercase__       )






      def 							lowercase_						( self   :  Tuple						,						lowercase__   :  str       ):


           '''simple docstring'''

           if token in self.fairseq_tokens_to_ids:
                return self.fairseq_tokens_to_ids[token]
           a_     :							List[str]     						=  self.sp_model.PieceToId(lowercase__       )

           # Need to return unknown token if the SP model returned 0
           return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






      def 							lowercase_						( self   :  Optional[int]						,						lowercase__   :  int       ):


           '''simple docstring'''

           if index in self.fairseq_ids_to_tokens:
                return self.fairseq_ids_to_tokens[index]
           return self.sp_model.IdToPiece(index - self.fairseq_offset       )






      def 							lowercase_						( self   :  Tuple						,						lowercase__   :  str       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  []
           a_     :							List[str]     						=  """"""
           a_     :							Dict     						=  False
           for token in tokens:
                # make sure that special tokens are not decoded using sentencepiece model
                if token in self.all_special_tokens:
                     if not prev_is_special:
                          out_string += " "
                     out_string += self.sp_model.decode(lowercase__       ) + token
                     a_     :							str     						=  True
                     a_     :							Optional[Any]     						=  []
                else:
                     current_sub_tokens.append(lowercase__       )
                     a_     :							Union[str, Any]     						=  False
           out_string += self.sp_model.decode(lowercase__       )
           return out_string.strip()






      def 							lowercase_						( self   :  Tuple						,						lowercase__   :  str						,						lowercase__   :  Optional[str] = None       ):


           '''simple docstring'''

           if not os.path.isdir(lowercase__       ):
                logger.error(F"Vocabulary path ({save_directory}) should be a directory"       )
                return
           a_     :							Dict     						=  os.path.join(
               lowercase__						,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

           if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowercase__       ) and os.path.isfile(self.vocab_file       ):
                copyfile(self.vocab_file						,						lowercase__       )
           elif not os.path.isfile(self.vocab_file       ):
                with open(lowercase__						,						"""wb"""       ) as fi:
                     a_     :							Dict     						=  self.sp_model.serialized_model_proto()
                     fi.write(lowercase__       )

           return (out_vocab_file,)






      def 							lowercase_						( self   :  List[str]						,						lowercase__   :  List[int]						,						lowercase__   :  Optional[List[int]] = None						,						lowercase__   :  bool = False       ):


           '''simple docstring'''

           if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=lowercase__						,						token_ids_a=lowercase__						,						already_has_special_tokens=lowercase__       )

           a_     :							Union[str, Any]     						=  [1] * len(self.prefix_tokens       )
           a_     :							List[Any]     						=  [1] * len(self.suffix_tokens       )
           if token_ids_a is None:
                return prefix_ones + ([0] * len(lowercase__       )) + suffix_ones
           return prefix_ones + ([0] * len(lowercase__       )) + ([0] * len(lowercase__       )) + suffix_ones






      def 							lowercase_						( self   :  List[str]						,						lowercase__   :  List[int]						,						lowercase__   :  Optional[List[int]] = None       ):


           '''simple docstring'''

           if token_ids_a is None:
                return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






      def 							lowercase_						( self   :  Any						,						lowercase__   :  Optional[int]						,						lowercase__   :  str						,						lowercase__   :  Optional[str]						,						lowercase__   :  Optional[str]						,						**lowercase__   :  Optional[int]       ):


           '''simple docstring'''

           if src_lang is None or tgt_lang is None:
                raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""       )
           a_     :							Union[str, Any]     						=  src_lang
           a_     :							Dict     						=  self(lowercase__						,						add_special_tokens=lowercase__						,						return_tensors=lowercase__						,						**lowercase__       )
           a_     :							Tuple     						=  self.convert_tokens_to_ids(lowercase__       )
           a_     :							Optional[Any]     						=  tgt_lang_id
           return inputs






      def 							lowercase_						( self   :  Optional[Any]						,						lowercase__   :  List[str]						,						lowercase__   :  str = "en_XX"						,						lowercase__   :  Optional[List[str]] = None						,						lowercase__   :  str = "ro_RO"						,						**lowercase__   :  Union[str, Any]						,						):


           '''simple docstring'''

           a_     :							str     						=  src_lang
           a_     :							Dict     						=  tgt_lang
           return super().prepare_seqaseq_batch(lowercase__						,						lowercase__						,						**lowercase__       )






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           return self.set_src_lang_special_tokens(self.src_lang       )






      def 							lowercase_						( self   :  Union[str, Any]       ):


           '''simple docstring'''

           return self.set_tgt_lang_special_tokens(self.tgt_lang       )






      def 							lowercase_						( self   :  Optional[int]						,						lowercase__   :  str       ):


           '''simple docstring'''

           a_     :							Any     						=  self.lang_code_to_id[src_lang]
           a_     :							Dict     						=  [self.cur_lang_code_id]
           a_     :							Any     						=  [self.eos_token_id]






      def 							lowercase_						( self   :  Any						,						lowercase__   :  str       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  self.lang_code_to_id[tgt_lang]
           a_     :							str     						=  [self.cur_lang_code_id]
           a_     :							Union[str, Any]     						=  [self.eos_token_id]
