






'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_mbart import MBartTokenizer
else:
 __UpperCamelCase      :       Optional[int]      			=      None


__UpperCamelCase      :       Any      			=      logging.get_logger(__name__)


__UpperCamelCase      :       Any      			=      {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

__UpperCamelCase      :       Optional[int]      			=      {
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

__UpperCamelCase      :       Tuple      			=      {
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
__UpperCamelCase      :       str      			=      ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):
      __a  					=VOCAB_FILES_NAMES
      __a  					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a  					=PRETRAINED_VOCAB_FILES_MAP
      __a  					=["input_ids", "attention_mask"]
      __a  					=MBartTokenizer

      __a  					=[]
      __a  					=[]




      def __init__(  self ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase="<s>" ,   lowerCamelCase="</s>" ,   lowerCamelCase="</s>" ,   lowerCamelCase="<s>" ,   lowerCamelCase="<unk>" ,   lowerCamelCase="<pad>" ,   lowerCamelCase="<mask>" ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase=None ,   **lowerCamelCase ,   )							->Dict:



         '''simple docstring'''
         # Mask token behave like a normal word, i.e. include the space before it
         __a				     =    AddedToken(lowerCamelCase ,   lstrip=lowerCamelCase ,   rstrip=lowerCamelCase      ) if isinstance(lowerCamelCase ,   lowerCamelCase      ) else mask_token

         super().__init__(
             vocab_file=lowerCamelCase ,   tokenizer_file=lowerCamelCase ,   bos_token=lowerCamelCase ,   eos_token=lowerCamelCase ,   sep_token=lowerCamelCase ,   cls_token=lowerCamelCase ,   unk_token=lowerCamelCase ,   pad_token=lowerCamelCase ,   mask_token=lowerCamelCase ,   src_lang=lowerCamelCase ,   tgt_lang=lowerCamelCase ,   additional_special_tokens=lowerCamelCase ,   **lowerCamelCase ,   )

         __a				     =    vocab_file
         __a				     =    False if not self.vocab_file else True

         __a				     =    FAIRSEQ_LANGUAGE_CODES.copy()

         if additional_special_tokens is not None:
            # Only add those special tokens if they are not already there.
            _additional_special_tokens.extend(
                [t for t in additional_special_tokens if t not in _additional_special_tokens]      )

         self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}      )
         __a				     =    {
             lang_code: self.convert_tokens_to_ids(lowerCamelCase      ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         __a				     =    src_lang if src_lang is not None else 'en_XX'
         __a				     =    self.convert_tokens_to_ids(self._src_lang      )
         __a				     =    tgt_lang
         self.set_src_lang_special_tokens(self._src_lang      )




      @property
      def  __UpperCamelCase							(  self      )							->str:



         '''simple docstring'''
         return self._src_lang




      @src_lang.setter
      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->None:



         '''simple docstring'''
         __a				     =    new_src_lang
         self.set_src_lang_special_tokens(self._src_lang      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->List[int]:



         '''simple docstring'''
         if token_ids_a is None:
            return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->List[int]:



         '''simple docstring'''
         __a				     =    [self.sep_token_id]
         __a				     =    [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase ,   lowerCamelCase ,   lowerCamelCase ,   **lowerCamelCase      )							->Any:



         '''simple docstring'''
         if src_lang is None or tgt_lang is None:
            raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'      )
         __a				     =    src_lang
         __a				     =    self(lowerCamelCase ,   add_special_tokens=lowerCamelCase ,   return_tensors=lowerCamelCase ,   **lowerCamelCase      )
         __a				     =    self.convert_tokens_to_ids(lowerCamelCase      )
         __a				     =    tgt_lang_id
         return inputs




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = "en_XX" ,   lowerCamelCase = None ,   lowerCamelCase = "ro_RO" ,   **lowerCamelCase ,   )							->BatchEncoding:



         '''simple docstring'''
         __a				     =    src_lang
         __a				     =    tgt_lang
         return super().prepare_seqaseq_batch(lowerCamelCase ,   lowerCamelCase ,   **lowerCamelCase      )




      def  __UpperCamelCase							(  self      )							->Optional[int]:



         '''simple docstring'''
         return self.set_src_lang_special_tokens(self.src_lang      )




      def  __UpperCamelCase							(  self      )							->str:



         '''simple docstring'''
         return self.set_tgt_lang_special_tokens(self.tgt_lang      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->None:



         '''simple docstring'''
         __a				     =    self.convert_tokens_to_ids(lowerCamelCase      )
         __a				     =    []
         __a				     =    [self.eos_token_id, self.cur_lang_code]

         __a				     =    self.convert_ids_to_tokens(self.prefix_tokens      )
         __a				     =    self.convert_ids_to_tokens(self.suffix_tokens      )

         __a				     =    processors.TemplateProcessing(
             single=prefix_tokens_str + ['$A'] + suffix_tokens_str ,   pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,   self.prefix_tokens + self.suffix_tokens      )      ) ,   )




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->None:



         '''simple docstring'''
         __a				     =    self.convert_tokens_to_ids(lowerCamelCase      )
         __a				     =    []
         __a				     =    [self.eos_token_id, self.cur_lang_code]

         __a				     =    self.convert_ids_to_tokens(self.prefix_tokens      )
         __a				     =    self.convert_ids_to_tokens(self.suffix_tokens      )

         __a				     =    processors.TemplateProcessing(
             single=prefix_tokens_str + ['$A'] + suffix_tokens_str ,   pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,   self.prefix_tokens + self.suffix_tokens      )      ) ,   )






      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->Tuple[str]:



         '''simple docstring'''
         if not self.can_save_slow_tokenizer:
            raise ValueError(
                'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                'tokenizer.'      )

         if not os.path.isdir(lowerCamelCase      ):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""      )
            return
         __a				     =    os.path.join(
             lowerCamelCase ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

         if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowerCamelCase      ):
            copyfile(self.vocab_file ,   lowerCamelCase      )

         return (out_vocab_file,)







'''simple docstring'''



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__UpperCamelCase      :       Dict      			=      logging.get_logger(__name__)

__UpperCamelCase      :       Optional[int]      			=      {
    """facebook/xmod-base""": """https://huggingface.co/facebook/xmod-base/resolve/main/config.json""",
    """facebook/xmod-large-prenorm""": """https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json""",
    """facebook/xmod-base-13-125k""": """https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json""",
    """facebook/xmod-base-30-125k""": """https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json""",
    """facebook/xmod-base-30-195k""": """https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json""",
    """facebook/xmod-base-60-125k""": """https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json""",
    """facebook/xmod-base-60-265k""": """https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json""",
    """facebook/xmod-base-75-125k""": """https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json""",
    """facebook/xmod-base-75-269k""": """https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json""",
}
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):
      __a  					="xmod"




      def __init__(  self ,   lowerCamelCase=3_0522 ,   lowerCamelCase=768 ,   lowerCamelCase=12 ,   lowerCamelCase=12 ,   lowerCamelCase=3072 ,   lowerCamelCase="gelu" ,   lowerCamelCase=0.1 ,   lowerCamelCase=0.1 ,   lowerCamelCase=512 ,   lowerCamelCase=2 ,   lowerCamelCase=0.02 ,   lowerCamelCase=1e-12 ,   lowerCamelCase=1 ,   lowerCamelCase=0 ,   lowerCamelCase=2 ,   lowerCamelCase="absolute" ,   lowerCamelCase=True ,   lowerCamelCase=None ,   lowerCamelCase=False ,   lowerCamelCase=2 ,   lowerCamelCase=False ,   lowerCamelCase=True ,   lowerCamelCase=True ,   lowerCamelCase=("en_XX",) ,   lowerCamelCase=None ,   **lowerCamelCase ,   )							->Any:



         '''simple docstring'''
         super().__init__(pad_token_id=lowerCamelCase ,   bos_token_id=lowerCamelCase ,   eos_token_id=lowerCamelCase ,   **lowerCamelCase      )

         __a				     =    vocab_size
         __a				     =    hidden_size
         __a				     =    num_hidden_layers
         __a				     =    num_attention_heads
         __a				     =    hidden_act
         __a				     =    intermediate_size
         __a				     =    hidden_dropout_prob
         __a				     =    attention_probs_dropout_prob
         __a				     =    max_position_embeddings
         __a				     =    type_vocab_size
         __a				     =    initializer_range
         __a				     =    layer_norm_eps
         __a				     =    position_embedding_type
         __a				     =    use_cache
         __a				     =    classifier_dropout
         __a				     =    pre_norm
         __a				     =    adapter_reduction_factor
         __a				     =    adapter_layer_norm
         __a				     =    adapter_reuse_layer_norm
         __a				     =    ln_before_adapter
         __a				     =    list(lowerCamelCase      )
         __a				     =    default_language







class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):




      @property
      def  __UpperCamelCase							(  self      )							->Mapping[str, Mapping[int, str]]:



         '''simple docstring'''
         if self.task == "multiple-choice":
            __a				     =    {0: 'batch', 1: 'choice', 2: 'sequence'}
         else:
            __a				     =    {0: 'batch', 1: 'sequence'}
         return OrderedDict(
             [
                 ('input_ids', dynamic_axis),
                 ('attention_mask', dynamic_axis),
             ]      )