

'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_mbart import MBartTokenizer
else:
      __lowerCamelCase									=     None


__lowerCamelCase									=     logging.get_logger(__name__)


__lowerCamelCase									=     {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase									=     {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-en-ro''': '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json''',
        '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json''',
    },
}

__lowerCamelCase									=     {
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
__lowerCamelCase									=     ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']


class        A__   (   _snake_case      ):
    lowercase						       =		VOCAB_FILES_NAMES
    lowercase						       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase						       =		PRETRAINED_VOCAB_FILES_MAP
    lowercase						       =		["input_ids", "attention_mask"]
    lowercase						       =		MBartTokenizer

    lowercase						       =		[]
    lowercase						       =		[]



    def __init__(       self		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					UpperCamelCase__="<s>"		,					UpperCamelCase__="</s>"		,					UpperCamelCase__="</s>"		,					UpperCamelCase__="<s>"		,					UpperCamelCase__="<unk>"		,					UpperCamelCase__="<pad>"		,					UpperCamelCase__="<mask>"		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					**UpperCamelCase__		,					)    ->   Tuple:





           '''simple docstring'''
           # Mask token behave like a normal word, i.e. include the space before it
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else mask_token

           super().__init__(
               vocab_file=UpperCamelCase__		,					tokenizer_file=UpperCamelCase__		,					bos_token=UpperCamelCase__		,					eos_token=UpperCamelCase__		,					sep_token=UpperCamelCase__		,					cls_token=UpperCamelCase__		,					unk_token=UpperCamelCase__		,					pad_token=UpperCamelCase__		,					mask_token=UpperCamelCase__		,					src_lang=UpperCamelCase__		,					tgt_lang=UpperCamelCase__		,					additional_special_tokens=UpperCamelCase__		,					**UpperCamelCase__		,					)

           A_						       =   vocab_file
           A_						       =   False if not self.vocab_file else True

           A_						       =   FAIRSEQ_LANGUAGE_CODES.copy()

           if additional_special_tokens is not None:
                  # Only add those special tokens if they are not already there.
                  _additional_special_tokens.extend(
                      [t for t in additional_special_tokens if t not in _additional_special_tokens]							)

           self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}							)
           A_						       =   {
               lang_code: self.convert_tokens_to_ids(UpperCamelCase__							) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           A_						       =   src_lang if src_lang is not None else """en_XX"""
           A_						       =   self.convert_tokens_to_ids(self._src_lang							)
           A_						       =   tgt_lang
           self.set_src_lang_special_tokens(self._src_lang							)



    @property
    def        snake_case_      (       self							)    ->   str:





           '''simple docstring'''
           return self._src_lang



    @src_lang.setter
    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   None:





           '''simple docstring'''
           A_						       =   new_src_lang
           self.set_src_lang_special_tokens(self._src_lang							)



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None							)    ->   List[int]:





           '''simple docstring'''
           if token_ids_a is None:
                  return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None							)    ->   List[int]:





           '''simple docstring'''
           A_						       =   [self.sep_token_id]
           A_						       =   [self.cls_token_id]

           if token_ids_a is None:
                  return len(cls + token_ids_a + sep							) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__		,					UpperCamelCase__		,					UpperCamelCase__		,					**UpperCamelCase__							)    ->   int:





           '''simple docstring'''
           if src_lang is None or tgt_lang is None:
                  raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""							)
           A_						       =   src_lang
           A_						       =   self(UpperCamelCase__		,					add_special_tokens=UpperCamelCase__		,					return_tensors=UpperCamelCase__		,					**UpperCamelCase__							)
           A_						       =   self.convert_tokens_to_ids(UpperCamelCase__							)
           A_						       =   tgt_lang_id
           return inputs



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = "en_XX"		,					UpperCamelCase__ = None		,					UpperCamelCase__ = "ro_RO"		,					**UpperCamelCase__		,					)    ->   BatchEncoding:





           '''simple docstring'''
           A_						       =   src_lang
           A_						       =   tgt_lang
           return super().prepare_seqaseq_batch(UpperCamelCase__		,					UpperCamelCase__		,					**UpperCamelCase__							)



    def        snake_case_      (       self							)    ->   Optional[int]:





           '''simple docstring'''
           return self.set_src_lang_special_tokens(self.src_lang							)



    def        snake_case_      (       self							)    ->   Optional[Any]:





           '''simple docstring'''
           return self.set_tgt_lang_special_tokens(self.tgt_lang							)



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   None:





           '''simple docstring'''
           A_						       =   self.convert_tokens_to_ids(UpperCamelCase__							)
           A_						       =   []
           A_						       =   [self.eos_token_id, self.cur_lang_code]

           A_						       =   self.convert_ids_to_tokens(self.prefix_tokens							)
           A_						       =   self.convert_ids_to_tokens(self.suffix_tokens							)

           A_						       =   processors.TemplateProcessing(
               single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str		,					pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str		,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		,					self.prefix_tokens + self.suffix_tokens							)							)		,					)



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   None:





           '''simple docstring'''
           A_						       =   self.convert_tokens_to_ids(UpperCamelCase__							)
           A_						       =   []
           A_						       =   [self.eos_token_id, self.cur_lang_code]

           A_						       =   self.convert_ids_to_tokens(self.prefix_tokens							)
           A_						       =   self.convert_ids_to_tokens(self.suffix_tokens							)

           A_						       =   processors.TemplateProcessing(
               single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str		,					pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str		,					special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str		,					self.prefix_tokens + self.suffix_tokens							)							)		,					)







    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None							)    ->   Tuple[str]:





           '''simple docstring'''
           if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                      """tokenizer."""							)

           if not os.path.isdir(UpperCamelCase__							):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''							)
                  return
           A_						       =   os.path.join(
               UpperCamelCase__		,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

           if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase__							):
                  copyfile(self.vocab_file		,					UpperCamelCase__							)

           return (out_vocab_file,)



'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


__lowerCamelCase									=     {
    '''configuration_ernie''': ['''ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ErnieConfig''', '''ErnieOnnxConfig'''],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __lowerCamelCase									=     [
          '''ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''ErnieForCausalLM''',
          '''ErnieForMaskedLM''',
          '''ErnieForMultipleChoice''',
          '''ErnieForNextSentencePrediction''',
          '''ErnieForPreTraining''',
          '''ErnieForQuestionAnswering''',
          '''ErnieForSequenceClassification''',
          '''ErnieForTokenClassification''',
          '''ErnieModel''',
          '''ErniePreTrainedModel''',
      ]

if TYPE_CHECKING:
      from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_ernie import (
                ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
                ErnieForCausalLM,
                ErnieForMaskedLM,
                ErnieForMultipleChoice,
                ErnieForNextSentencePrediction,
                ErnieForPreTraining,
                ErnieForQuestionAnswering,
                ErnieForSequenceClassification,
                ErnieForTokenClassification,
                ErnieModel,
                ErniePreTrainedModel,
            )

else:
      import sys

      __lowerCamelCase									=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
