


'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_a   :   Optional[int]												=   get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
       from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_a   :   Union[str, Any]												=   256_047
_a   :   str												=   256_145




@require_sentencepiece
@require_tokenizers
class 			_lowercase (	__lowercase						,					unittest.TestCase							):
   _SCREAMING_SNAKE_CASE  :  Dict          =	NllbTokenizer
   _SCREAMING_SNAKE_CASE  :  Tuple          =	NllbTokenizerFast
   _SCREAMING_SNAKE_CASE  :  str          =	True
   _SCREAMING_SNAKE_CASE  :  Dict          =	True
   _SCREAMING_SNAKE_CASE  :  Any          =	{}
   def a  (						self       :      List[str]      ) ->							Union[str, Any]:
        super().setUp()

        # We have a SentencePiece fixture for testing
        __snake_case           =						NllbTokenizer(SCREAMING_SNAKE_CASE_       ,   keep_accents=SCREAMING_SNAKE_CASE_      )
        tokenizer.save_pretrained(self.tmpdirname      )
   def a  (						self       :      int      ) ->							int:
        __snake_case           =						NllbTokenizer(SCREAMING_SNAKE_CASE_       ,   keep_accents=SCREAMING_SNAKE_CASE_      )

        __snake_case           =						tokenizer.tokenize('This is a test'      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['▁This', '▁is', '▁a', '▁t', 'est']      )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )       ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,   )

        __snake_case           =						tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_       ,   [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]       ,   )
        __snake_case           =						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_       ,   [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ]       ,   )

        __snake_case           =						tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_      )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_       ,   [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ]       ,   )
   def a  (						self       :      List[str]      ) ->							Tuple:
        __snake_case           =						(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                  __snake_case           =						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )

                  __snake_case           =						tempfile.mkdtemp()

                  __snake_case           =						tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  # Checks it save with the same files + the tokenizer.json file for the fast one
                  self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )
                  __snake_case           =						tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f      )
                  self.assertSequenceEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

                  # Checks everything loads correctly in the same way
                  __snake_case           =						tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_      )

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )      )

                  shutil.rmtree(SCREAMING_SNAKE_CASE_      )

                  # Save tokenizer rust, legacy_format=True
                  __snake_case           =						tempfile.mkdtemp()

                  __snake_case           =						tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_       ,   legacy_format=SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  # Checks it save with the same files
                  self.assertSequenceEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

                  # Checks everything loads correctly in the same way
                  __snake_case           =						tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_      )

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )      )

                  shutil.rmtree(SCREAMING_SNAKE_CASE_      )

                  # Save tokenizer rust, legacy_format=False
                  __snake_case           =						tempfile.mkdtemp()

                  __snake_case           =						tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_       ,   legacy_format=SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_      )

                  # Checks it saved the tokenizer.json file
                  self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )

                  # Checks everything loads correctly in the same way
                  __snake_case           =						tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_      )

                  # Check special tokens are set accordingly on Rust and Python
                  for key in tokenizer_pp.special_tokens_map:
                       self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )      )

                  shutil.rmtree(SCREAMING_SNAKE_CASE_      )
   @require_torch
   def a  (						self       :      Dict      ) ->							Optional[Any]:
        if not self.test_seqaseq:
             return

        __snake_case           =						self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                  # Longer text that will definitely require truncation.
                  __snake_case           =						[
                      ' UN Chief Says There Is No Military Solution in Syria',
                      ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                      ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                      ' will only worsen the violence and misery for millions of people.',
                  ]
                  __snake_case           =						[
                      'Şeful ONU declară că nu există o soluţie militară în Siria',
                      'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                      ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                      ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                  ]
                  try:
                       __snake_case           =						tokenizer.prepare_seqaseq_batch(
                           src_texts=SCREAMING_SNAKE_CASE_       ,   tgt_texts=SCREAMING_SNAKE_CASE_       ,   max_length=3       ,   max_target_length=10       ,   return_tensors='pt'       ,   src_lang='eng_Latn'       ,   tgt_lang='ron_Latn'       ,   )
                  except NotImplementedError:
                       return
                  self.assertEqual(batch.input_ids.shape[1]       ,   3      )
                  self.assertEqual(batch.labels.shape[1]       ,   10      )
                  # max_target_length will default to max_length if not specified
                  __snake_case           =						tokenizer.prepare_seqaseq_batch(
                      SCREAMING_SNAKE_CASE_       ,   tgt_texts=SCREAMING_SNAKE_CASE_       ,   max_length=3       ,   return_tensors='pt'      )
                  self.assertEqual(batch.input_ids.shape[1]       ,   3      )
                  self.assertEqual(batch.labels.shape[1]       ,   3      )

                  __snake_case           =						tokenizer.prepare_seqaseq_batch(
                      src_texts=SCREAMING_SNAKE_CASE_       ,   max_length=3       ,   max_target_length=10       ,   return_tensors='pt'      )
                  self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,   3      )
                  self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,   3      )
                  self.assertNotIn('decoder_input_ids'       ,   SCREAMING_SNAKE_CASE_      )
   @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'      )
   def a  (						self       :      Any      ) ->							Union[str, Any]:
        pass





   def a  (						self       :      str      ) ->							Union[str, Any]:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                  __snake_case           =						[AddedToken('<special>'       ,   lstrip=SCREAMING_SNAKE_CASE_      )]

                  __snake_case           =						self.rust_tokenizer_class.from_pretrained(
                      SCREAMING_SNAKE_CASE_       ,   additional_special_tokens=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )
                  __snake_case           =						tokenizer_r.encode('Hey this is a <special> token'      )

                  __snake_case           =						tokenizer_r.encode('<special>'       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )[0]

                  self.assertTrue(special_token_id in r_output      )

                  if self.test_slow_tokenizer:
                       __snake_case           =						self.rust_tokenizer_class.from_pretrained(
                           SCREAMING_SNAKE_CASE_       ,   additional_special_tokens=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_       ,   )
                       __snake_case           =						self.tokenizer_class.from_pretrained(
                           SCREAMING_SNAKE_CASE_       ,   additional_special_tokens=SCREAMING_SNAKE_CASE_       ,   **SCREAMING_SNAKE_CASE_      )

                       __snake_case           =						tokenizer_p.encode('Hey this is a <special> token'      )

                       __snake_case           =						tokenizer_cr.encode('Hey this is a <special> token'      )

                       self.assertEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
                       self.assertEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
                       self.assertTrue(special_token_id in p_output      )
                       self.assertTrue(special_token_id in cr_output      )




@require_torch
@require_sentencepiece
@require_tokenizers
class 			_lowercase (	unittest.TestCase							):
   _SCREAMING_SNAKE_CASE  :  Union[str, Any]          =	"facebook/nllb-200-distilled-600M"
   _SCREAMING_SNAKE_CASE  :  Optional[Any]          =	[
       " UN Chief Says There Is No Military Solution in Syria",
       " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
   ]
   _SCREAMING_SNAKE_CASE  :  Optional[int]          =	[
       "Şeful ONU declară că nu există o soluţie militară în Siria",
       "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
       " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
       " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
   ]
   _SCREAMING_SNAKE_CASE  :  List[str]          =	[
       2_5_6_0_4_7,
       1_6_2_9_7,
       1_3_4_4_0_8,
       8_1_6_5,
       2_4_8_0_6_6,
       1_4_7_3_4,
       9_5_0,
       1_1_3_5,
       1_0_5_7_2_1,
       3_5_7_3,
       8_3,
       2_7_3_5_2,
       1_0_8,
       4_9_4_8_6,
       2,
   ]
   @classmethod
   def a  (						cls       :      Union[str, Any]      ) ->							Union[str, Any]:
        __snake_case           =						NllbTokenizer.from_pretrained(
            cls.checkpoint_name       ,   src_lang='eng_Latn'       ,   tgt_lang='ron_Latn'      )
        __snake_case           =						1
        return cls
   def a  (						self       :      int      ) ->							str:
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']       ,   25_6001      )
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']       ,   25_6002      )
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']       ,   25_6057      )
   def a  (						self       :      Optional[int]      ) ->							int:
        __snake_case           =						self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
        self.assertListEqual(self.expected_src_tokens       ,   SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Tuple      ) ->							Union[str, Any]:
        self.assertIn(SCREAMING_SNAKE_CASE_       ,   self.tokenizer.all_special_ids      )
        # fmt: off
        __snake_case           =						[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
        # fmt: on

        __snake_case           =						self.tokenizer.decode(SCREAMING_SNAKE_CASE_       ,   skip_special_tokens=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.tokenizer.decode(generated_ids[1:]       ,   skip_special_tokens=SCREAMING_SNAKE_CASE_      )
        self.assertEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
        self.assertNotIn(self.tokenizer.eos_token       ,   SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      List[Any]      ) ->							Union[str, Any]:
        __snake_case           =						['this is gunna be a long sentence ' * 20]
        assert isinstance(src_text[0]       ,   SCREAMING_SNAKE_CASE_      )
        __snake_case           =						10
        __snake_case           =						self.tokenizer(SCREAMING_SNAKE_CASE_       ,   max_length=SCREAMING_SNAKE_CASE_       ,   truncation=SCREAMING_SNAKE_CASE_      ).input_ids[0]
        self.assertEqual(ids[-1]       ,   2      )
        self.assertEqual(ids[0]       ,   SCREAMING_SNAKE_CASE_      )
        self.assertEqual(len(SCREAMING_SNAKE_CASE_      )       ,   SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Any      ) ->							str:
        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']      )       ,   [25_6203, 3]      )
   def a  (						self       :      str      ) ->							Tuple:
        __snake_case           =						tempfile.mkdtemp()
        __snake_case           =						self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,   SCREAMING_SNAKE_CASE_      )
   @require_torch
   def a  (						self       :      str      ) ->							List[Any]:
        __snake_case           =						self.tokenizer(
            self.src_text       ,   text_target=self.tgt_text       ,   padding=SCREAMING_SNAKE_CASE_       ,   truncation=SCREAMING_SNAKE_CASE_       ,   max_length=len(self.expected_src_tokens      )       ,   return_tensors='pt'       ,   )
        __snake_case           =						shift_tokens_right(
            batch['labels']       ,   self.tokenizer.pad_token_id       ,   self.tokenizer.lang_code_to_id['ron_Latn']      )

        self.assertIsInstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        self.assertEqual((2, 15)       ,   batch.input_ids.shape      )
        self.assertEqual((2, 15)       ,   batch.attention_mask.shape      )
        __snake_case           =						batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens       ,   SCREAMING_SNAKE_CASE_      )
        self.assertEqual(SCREAMING_SNAKE_CASE_       ,   batch.decoder_input_ids[0, 0]      )  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens       ,   [EN_CODE]      )
        self.assertEqual(self.tokenizer.suffix_tokens       ,   [self.tokenizer.eos_token_id]      )
   def a  (						self       :      int      ) ->							List[Any]:
        __snake_case           =						self.tokenizer(self.src_text       ,   padding=SCREAMING_SNAKE_CASE_       ,   truncation=SCREAMING_SNAKE_CASE_       ,   max_length=3       ,   return_tensors='pt'      )
        __snake_case           =						self.tokenizer(
            text_target=self.tgt_text       ,   padding=SCREAMING_SNAKE_CASE_       ,   truncation=SCREAMING_SNAKE_CASE_       ,   max_length=10       ,   return_tensors='pt'      )
        __snake_case           =						targets['input_ids']
        __snake_case           =						shift_tokens_right(
            SCREAMING_SNAKE_CASE_       ,   self.tokenizer.pad_token_id       ,   decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,   )

        self.assertEqual(batch.input_ids.shape[1]       ,   3      )
        self.assertEqual(batch.decoder_input_ids.shape[1]       ,   10      )
   @require_torch
   def a  (						self       :      Union[str, Any]      ) ->							Any:
        __snake_case           =						self.tokenizer._build_translation_inputs(
            'A test'       ,   return_tensors='pt'       ,   src_lang='eng_Latn'       ,   tgt_lang='fra_Latn'      )

        self.assertEqual(
            nested_simplify(SCREAMING_SNAKE_CASE_      )       ,   {
                # A, test, EOS, en_XX
                'input_ids': [[25_6047, 70, 7356, 2]],
                'attention_mask': [[1, 1, 1, 1]],
                # ar_AR
                'forced_bos_token_id': 25_6057,
            }       ,   )





   @require_torch
   def a  (						self       :      List[str]      ) ->							Union[str, Any]:
        __snake_case           =						True
        __snake_case           =						self.tokenizer(
            'UN Chief says there is no military solution in Syria'       ,   src_lang='eng_Latn'       ,   tgt_lang='fra_Latn'      )
        self.assertEqual(
            inputs.input_ids       ,   [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]      )

        __snake_case           =						False
        __snake_case           =						self.tokenizer(
            'UN Chief says there is no military solution in Syria'       ,   src_lang='eng_Latn'       ,   tgt_lang='fra_Latn'      )
        self.assertEqual(
            inputs.input_ids       ,   [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]      )







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__		 =     logging.get_logger(__name__)

UpperCamelCase__		 =     '''▁'''

UpperCamelCase__		 =     {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
}

UpperCamelCase__		 =     {
    '''vocab_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'''
        ),
    },
    '''spm_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'''
        )
    },
}

UpperCamelCase__		 =     {
    '''facebook/s2t-small-librispeech-asr''': 1024,
}

UpperCamelCase__		 =     ['''pt''', '''fr''', '''ru''', '''nl''', '''ro''', '''it''', '''es''', '''de''']

UpperCamelCase__		 =     {'''mustc''': MUSTC_LANGS}

class     __snake_case    (				snake_case__  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							VOCAB_FILES_NAMES
       __SCREAMING_SNAKE_CASE				    =							PRETRAINED_VOCAB_FILES_MAP
       __SCREAMING_SNAKE_CASE				    =							MAX_MODEL_INPUT_SIZES
       __SCREAMING_SNAKE_CASE				    =							["input_ids", "attention_mask"]

       __SCREAMING_SNAKE_CASE				    =							[]


       def __init__(		self      , _UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase="<s>"      , _UpperCamelCase="</s>"      , _UpperCamelCase="<pad>"      , _UpperCamelCase="<unk>"      , _UpperCamelCase=False      , _UpperCamelCase=False      , _UpperCamelCase=None      , _UpperCamelCase=None      , _UpperCamelCase = None      , **_UpperCamelCase      , )      ->							None:






              """simple docstring"""


              __snake_case       	=							{} if sp_model_kwargs is None else sp_model_kwargs

              super().__init__(
                  bos_token=_UpperCamelCase      , eos_token=_UpperCamelCase      , unk_token=_UpperCamelCase      , pad_token=_UpperCamelCase      , do_upper_case=_UpperCamelCase      , do_lower_case=_UpperCamelCase      , tgt_lang=_UpperCamelCase      , lang_codes=_UpperCamelCase      , sp_model_kwargs=self.sp_model_kwargs      , **_UpperCamelCase      , )
              __snake_case       	=							do_upper_case
              __snake_case       	=							do_lower_case

              __snake_case       	=							load_json(_UpperCamelCase   )
              __snake_case       	=							{v: k for k, v in self.encoder.items()}
              __snake_case       	=							spm_file
              __snake_case       	=							load_spm(_UpperCamelCase      , self.sp_model_kwargs   )

              if lang_codes is not None:
                     __snake_case       	=							lang_codes
                     __snake_case       	=							LANGUAGES[lang_codes]
                     __snake_case       	=							[F'<lang:{lang}>' for lang in self.langs]
                     __snake_case       	=							{lang: self.sp_model.PieceToId(F'<lang:{lang}>'   ) for lang in self.langs}

                     __snake_case       	=							self.lang_tokens
                     __snake_case       	=							tgt_lang if tgt_lang is not None else self.langs[0]

                     self.set_tgt_lang_special_tokens(self._tgt_lang   )
              else:
                     __snake_case       	=							{}


       @property
       def 			a		(		self   )      ->							int:






              """simple docstring"""


              return len(self.encoder   )


       @property
       def 			a		(		self   )      ->							str:






              """simple docstring"""


              return self._tgt_lang


       @tgt_lang.setter
       def 			a		(		self      , _UpperCamelCase   )      ->							None:






              """simple docstring"""


              __snake_case       	=							new_tgt_lang
              self.set_tgt_lang_special_tokens(_UpperCamelCase   )


       def 			a		(		self      , _UpperCamelCase   )      ->							None:






              """simple docstring"""


              __snake_case       	=							self.lang_code_to_id[tgt_lang]
              __snake_case       	=							[lang_code_id]


       def 			a		(		self      , _UpperCamelCase   )      ->							List[str]:






              """simple docstring"""


              return self.sp_model.encode(_UpperCamelCase      , out_type=_UpperCamelCase   )


       def 			a		(		self      , _UpperCamelCase   )      ->							Optional[Any]:






              """simple docstring"""


              return self.encoder.get(_UpperCamelCase      , self.encoder[self.unk_token]   )


       def 			a		(		self      , _UpperCamelCase   )      ->							str:






              """simple docstring"""


              return self.decoder.get(_UpperCamelCase      , self.unk_token   )


       def 			a		(		self      , _UpperCamelCase   )      ->							str:






              """simple docstring"""


              __snake_case       	=							[]
              __snake_case       	=							""""""
              for token in tokens:
                     # make sure that special tokens are not decoded using sentencepiece model
                     if token in self.all_special_tokens:
                            __snake_case       	=							self.sp_model.decode(_UpperCamelCase   )
                            out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
                            __snake_case       	=							[]
                     else:
                            current_sub_tokens.append(_UpperCamelCase   )
              __snake_case       	=							self.sp_model.decode(_UpperCamelCase   )
              out_string += decoded.upper() if self.do_upper_case else decoded
              return out_string.strip()


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase=None   )      ->							List[int]:






              """simple docstring"""


              if token_ids_a is None:
                     return self.prefix_tokens + token_ids_a + [self.eos_token_id]
              # We don't expect to process pairs, but leave the pair logic for API consistency
              return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase = None      , _UpperCamelCase = False   )      ->							List[int]:






              """simple docstring"""


              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=_UpperCamelCase      , token_ids_a=_UpperCamelCase      , already_has_special_tokens=_UpperCamelCase   )

              __snake_case       	=							[1] * len(self.prefix_tokens   )
              __snake_case       	=							[1]
              if token_ids_a is None:
                     return prefix_ones + ([0] * len(_UpperCamelCase   )) + suffix_ones
              return prefix_ones + ([0] * len(_UpperCamelCase   )) + ([0] * len(_UpperCamelCase   )) + suffix_ones


       def 			a		(		self   )      ->							Dict:






              """simple docstring"""


              __snake_case       	=							self.encoder.copy()
              vocab.update(self.added_tokens_encoder   )
              return vocab


       def __getstate__(		self   )      ->							Dict:






              """simple docstring"""


              __snake_case       	=							self.__dict__.copy()
              __snake_case       	=							None
              return state


       def __setstate__(		self      , _UpperCamelCase   )      ->							None:






              """simple docstring"""


              __snake_case       	=							d

              # for backward compatibility
              if not hasattr(self      , """sp_model_kwargs"""   ):
                     __snake_case       	=							{}

              __snake_case       	=							load_spm(self.spm_file      , self.sp_model_kwargs   )






       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase = None   )      ->							Tuple[str]:






              """simple docstring"""


              __snake_case       	=							Path(_UpperCamelCase   )
              assert save_dir.is_dir(), F'{save_directory} should be a directory'
              __snake_case       	=							save_dir / (
                  (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
              )
              __snake_case       	=							save_dir / (
                  (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
              )

              save_json(self.encoder      , _UpperCamelCase   )

              if os.path.abspath(self.spm_file   ) != os.path.abspath(_UpperCamelCase   ) and os.path.isfile(self.spm_file   ):
                     copyfile(self.spm_file      , _UpperCamelCase   )
              elif not os.path.isfile(self.spm_file   ):
                     with open(_UpperCamelCase      , """wb"""   ) as fi:
                            __snake_case       	=							self.sp_model.serialized_model_proto()
                            fi.write(_UpperCamelCase   )

              return (str(_UpperCamelCase   ), str(_UpperCamelCase   ))







def 						lowerCamelCase__      ( __A    :str	,__A    :Dict[str, Any]		):


       """simple docstring"""




       __snake_case       	=							sentencepiece.SentencePieceProcessor(**__A		)
       spm.Load(str(__A		)		)
       return spm







def 						lowerCamelCase__      ( __A    :str		):


       """simple docstring"""




       with open(__A	,"""r"""		) as f:
              return json.load(__A		)







def 						lowerCamelCase__      ( __A    :Optional[Any]	,__A    :str		):


       """simple docstring"""




       with open(__A	,"""w"""		) as f:
              json.dump(__A	,__A	,indent=2		)

