
"""simple docstring"""

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_			    =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_			    =  {
    '''flaubert/flaubert_small_cased''': '''https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/config.json''',
    '''flaubert/flaubert_base_uncased''': '''https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/config.json''',
    '''flaubert/flaubert_base_cased''': '''https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/config.json''',
    '''flaubert/flaubert_large_cased''': '''https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/config.json''',
}
class    _UpperCAmelCase							(						SCREAMING_SNAKE_CASE_							):
      __SCREAMING_SNAKE_CASE      :							Union[str, Any]												=		"flaubert"
      __SCREAMING_SNAKE_CASE      :							List[Any]												=		{
          "hidden_size": "emb_dim",
          "num_attention_heads": "n_heads",
          "num_hidden_layers": "n_layers",
          "n_words": "vocab_size",  # For backward compatibility
      }




      def __init__(							self      ,				lowercase_=False      ,				lowercase_=0.0      ,				lowercase_=3_0_1_4_5      ,				lowercase_=2_0_4_8      ,				lowercase_=1_2      ,				lowercase_=1_6      ,				lowercase_=0.1      ,				lowercase_=0.1      ,				lowercase_=True      ,				lowercase_=False      ,				lowercase_=False      ,				lowercase_=False      ,				lowercase_=1      ,				lowercase_=True      ,				lowercase_=5_1_2      ,				lowercase_=2_0_4_8**-0.5      ,				lowercase_=1E-12      ,				lowercase_=0.0_2      ,				lowercase_=0      ,				lowercase_=1      ,				lowercase_=2      ,				lowercase_=3      ,				lowercase_=5      ,				lowercase_=True      ,				lowercase_="first"      ,				lowercase_=True      ,				lowercase_=None      ,				lowercase_=True      ,				lowercase_=0.1      ,				lowercase_=5      ,				lowercase_=5      ,				lowercase_=0      ,				lowercase_=0      ,				lowercase_=2      ,				lowercase_=0      ,				**lowercase_      ,				)  ->    int:
          UpperCAmelCase  	=    pre_norm
          UpperCAmelCase  	=    layerdrop
          UpperCAmelCase  	=    vocab_size
          UpperCAmelCase  	=    emb_dim
          UpperCAmelCase  	=    n_layers
          UpperCAmelCase  	=    n_heads
          UpperCAmelCase  	=    dropout
          UpperCAmelCase  	=    attention_dropout
          UpperCAmelCase  	=    gelu_activation
          UpperCAmelCase  	=    sinusoidal_embeddings
          UpperCAmelCase  	=    causal
          UpperCAmelCase  	=    asm
          UpperCAmelCase  	=    n_langs
          UpperCAmelCase  	=    use_lang_emb
          UpperCAmelCase  	=    layer_norm_eps
          UpperCAmelCase  	=    bos_index
          UpperCAmelCase  	=    eos_index
          UpperCAmelCase  	=    pad_index
          UpperCAmelCase  	=    unk_index
          UpperCAmelCase  	=    mask_index
          UpperCAmelCase  	=    is_encoder
          UpperCAmelCase  	=    max_position_embeddings
          UpperCAmelCase  	=    embed_init_std
          UpperCAmelCase  	=    init_std
          UpperCAmelCase  	=    summary_type
          UpperCAmelCase  	=    summary_use_proj
          UpperCAmelCase  	=    summary_activation
          UpperCAmelCase  	=    summary_proj_to_labels
          UpperCAmelCase  	=    summary_first_dropout
          UpperCAmelCase  	=    start_n_top
          UpperCAmelCase  	=    end_n_top
          UpperCAmelCase  	=    mask_token_id
          UpperCAmelCase  	=    lang_id

          if "n_words" in kwargs:
              UpperCAmelCase  	=    kwargs['n_words']

          super().__init__(pad_token_id=lowercase_      ,				bos_token_id=lowercase_      ,				**lowercase_							)

class    _UpperCAmelCase							(						SCREAMING_SNAKE_CASE_							):




      @property
      def 			a_					(							self							)  ->    Mapping[str, Mapping[int, str]]:
          if self.task == "multiple-choice":
              UpperCAmelCase  	=    {0: 'batch', 1: 'choice', 2: 'sequence'}
          else:
              UpperCAmelCase  	=    {0: 'batch', 1: 'sequence'}
          return OrderedDict(
              [
                  ('input_ids', dynamic_axis),
                  ('attention_mask', dynamic_axis),
              ]							)





"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_mbartaa import MBartaaTokenizer
else:
   SCREAMING_SNAKE_CASE_			    =  None


SCREAMING_SNAKE_CASE_			    =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_			    =  {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

SCREAMING_SNAKE_CASE_			    =  {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

SCREAMING_SNAKE_CASE_			    =  {
    '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
}

# fmt: off
SCREAMING_SNAKE_CASE_			    =  ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']
class    _UpperCAmelCase							(						SCREAMING_SNAKE_CASE_							):
      __SCREAMING_SNAKE_CASE      :							Tuple												=		VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE      :							Tuple												=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE      :							Tuple												=		PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE      :							Optional[Any]												=		["input_ids", "attention_mask"]
      __SCREAMING_SNAKE_CASE      :							Union[str, Any]												=		MBartaaTokenizer

      __SCREAMING_SNAKE_CASE      :							List[int]												=		[]
      __SCREAMING_SNAKE_CASE      :							List[int]												=		[]




      def __init__(							self      ,				lowercase_=None      ,				lowercase_=None      ,				lowercase_=None      ,				lowercase_=None      ,				lowercase_="</s>"      ,				lowercase_="</s>"      ,				lowercase_="<s>"      ,				lowercase_="<unk>"      ,				lowercase_="<pad>"      ,				lowercase_="<mask>"      ,				**lowercase_      ,				)  ->    Tuple:
          # Mask token behave like a normal word, i.e. include the space before it
          UpperCAmelCase  	=    AddedToken(lowercase_      ,				lstrip=lowercase_      ,				rstrip=lowercase_							) if isinstance(lowercase_      ,				lowercase_							) else mask_token

          UpperCAmelCase  	=    kwargs.get('additional_special_tokens'      ,				[]							)
          kwargs["additional_special_tokens"] += [
              code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              lowercase_      ,				src_lang=lowercase_      ,				tgt_lang=lowercase_      ,				tokenizer_file=lowercase_      ,				eos_token=lowercase_      ,				sep_token=lowercase_      ,				cls_token=lowercase_      ,				unk_token=lowercase_      ,				pad_token=lowercase_      ,				mask_token=lowercase_      ,				**lowercase_      ,				)

          UpperCAmelCase  	=    vocab_file
          UpperCAmelCase  	=    False if not self.vocab_file else True

          UpperCAmelCase  	=    {
              lang_code: self.convert_tokens_to_ids(lowercase_							) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          UpperCAmelCase  	=    src_lang if src_lang is not None else 'en_XX'
          UpperCAmelCase  	=    tgt_lang
          UpperCAmelCase  	=    self.lang_code_to_id[self._src_lang]
          self.set_src_lang_special_tokens(self._src_lang							)




      @property
      def 			a_					(							self							)  ->    str:
          return self._src_lang




      @src_lang.setter
      def 			a_					(							self      ,				lowercase_							)  ->    None:
          UpperCAmelCase  	=    new_src_lang
          self.set_src_lang_special_tokens(self._src_lang							)




      def 			a_					(							self      ,				lowercase_      ,				lowercase_ = None							)  ->    List[int]:
          if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




      def 			a_					(							self      ,				lowercase_      ,				lowercase_ = "en_XX"      ,				lowercase_ = None      ,				lowercase_ = "ro_RO"      ,				**lowercase_      ,				)  ->    BatchEncoding:
          UpperCAmelCase  	=    src_lang
          UpperCAmelCase  	=    tgt_lang
          return super().prepare_seqaseq_batch(lowercase_      ,				lowercase_      ,				**lowercase_							)




      def 			a_					(							self							)  ->    Optional[int]:
          return self.set_src_lang_special_tokens(self.src_lang							)




      def 			a_					(							self							)  ->    List[Any]:
          return self.set_tgt_lang_special_tokens(self.tgt_lang							)




      def 			a_					(							self      ,				lowercase_							)  ->    None:
          UpperCAmelCase  	=    self.convert_tokens_to_ids(lowercase_							)
          UpperCAmelCase  	=    [self.cur_lang_code_id]
          UpperCAmelCase  	=    [self.eos_token_id]

          UpperCAmelCase  	=    self.convert_ids_to_tokens(self.prefix_tokens							)
          UpperCAmelCase  	=    self.convert_ids_to_tokens(self.suffix_tokens							)

          UpperCAmelCase  	=    processors.TemplateProcessing(
              single=prefix_tokens_str + ['$A'] + suffix_tokens_str      ,				pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,				self.prefix_tokens + self.suffix_tokens							)							)      ,				)




      def 			a_					(							self      ,				lowercase_							)  ->    None:
          UpperCAmelCase  	=    self.convert_tokens_to_ids(lowercase_							)
          UpperCAmelCase  	=    [self.cur_lang_code_id]
          UpperCAmelCase  	=    [self.eos_token_id]

          UpperCAmelCase  	=    self.convert_ids_to_tokens(self.prefix_tokens							)
          UpperCAmelCase  	=    self.convert_ids_to_tokens(self.suffix_tokens							)

          UpperCAmelCase  	=    processors.TemplateProcessing(
              single=prefix_tokens_str + ['$A'] + suffix_tokens_str      ,				pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str      ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,				self.prefix_tokens + self.suffix_tokens							)							)      ,				)




      def 			a_					(							self      ,				lowercase_      ,				lowercase_      ,				lowercase_      ,				lowercase_      ,				**lowercase_							)  ->    int:
          if src_lang is None or tgt_lang is None:
              raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'							)
          UpperCAmelCase  	=    src_lang
          UpperCAmelCase  	=    self(lowercase_      ,				add_special_tokens=lowercase_      ,				return_tensors=lowercase_      ,				**lowercase_							)
          UpperCAmelCase  	=    self.convert_tokens_to_ids(lowercase_							)
          UpperCAmelCase  	=    tgt_lang_id
          return inputs




      def 			a_					(							self      ,				lowercase_      ,				lowercase_ = None							)  ->    Tuple[str]:
          if not self.can_save_slow_tokenizer:
              raise ValueError(
                  'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                  'tokenizer.'							)

          if not os.path.isdir(lowercase_							):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
              return
          UpperCAmelCase  	=    os.path.join(
              lowercase_      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

          if os.path.abspath(self.vocab_file							) != os.path.abspath(lowercase_							):
              copyfile(self.vocab_file      ,				lowercase_							)

          return (out_vocab_file,)



