
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_mbart import MBartTokenizer
else:
 __lowerCAmelCase					:			Any			      =		None


__lowerCAmelCase					:			str			      =		logging.get_logger(__name__)


__lowerCAmelCase					:			Optional[Any]			      =		{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__lowerCAmelCase					:			Any			      =		{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

__lowerCAmelCase					:			List[str]			      =		{
    "facebook/mbart-large-en-ro": 1_024,
    "facebook/mbart-large-cc25": 1_024,
}

# fmt: off
__lowerCAmelCase					:			int			      =		["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]


class 						__lowerCAmelCase	(	lowerCAmelCase_      ):



      """simple docstring"""







      A__	:			Dict        =    VOCAB_FILES_NAMES
      A__	:			str        =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A__	:			int        =    PRETRAINED_VOCAB_FILES_MAP
      A__	:			Tuple        =    ['''input_ids''', '''attention_mask''']
      A__	:			Dict        =    MBartTokenizer

      A__	:			List[int]        =    []
      A__	:			List[int]        =    []




      def __init__(      self		:       int				,       _snake_case		:       Union[str, Any]=None				,       _snake_case		:       Union[str, Any]=None				,       _snake_case		:       str="<s>"				,       _snake_case		:       List[str]="</s>"				,       _snake_case		:       List[Any]="</s>"				,       _snake_case		:       Optional[int]="<s>"				,       _snake_case		:       Optional[Any]="<unk>"				,       _snake_case		:       Optional[int]="<pad>"				,       _snake_case		:       Union[str, Any]="<mask>"				,       _snake_case		:       Optional[int]=None				,       _snake_case		:       Union[str, Any]=None				,       _snake_case		:       Any=None				,       **_snake_case		:       List[str]				,       ):
          # Mask token behave like a normal word, i.e. include the space before it
          __lowercase		:			str	    =       AddedToken(_snake_case				,       lstrip=_snake_case				,       rstrip=_snake_case				) if isinstance(_snake_case				,       _snake_case				) else mask_token

          super().__init__(
              vocab_file=_snake_case				,       tokenizer_file=_snake_case				,       bos_token=_snake_case				,       eos_token=_snake_case				,       sep_token=_snake_case				,       cls_token=_snake_case				,       unk_token=_snake_case				,       pad_token=_snake_case				,       mask_token=_snake_case				,       src_lang=_snake_case				,       tgt_lang=_snake_case				,       additional_special_tokens=_snake_case				,       **_snake_case				,       )

          __lowercase		:			Tuple	    =       vocab_file
          __lowercase		:			Tuple	    =       False if not self.vocab_file else True

          __lowercase		:			Dict	    =       FAIRSEQ_LANGUAGE_CODES.copy()

          if additional_special_tokens is not None:
              # Only add those special tokens if they are not already there.
              _additional_special_tokens.extend(
                  [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

          self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}				)
          __lowercase		:			str	    =       {
              lang_code: self.convert_tokens_to_ids(_snake_case				) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          __lowercase		:			Any	    =       src_lang if src_lang is not None else '''en_XX'''
          __lowercase		:			Any	    =       self.convert_tokens_to_ids(self._src_lang				)
          __lowercase		:			str	    =       tgt_lang
          self.set_src_lang_special_tokens(self._src_lang				)




      @property
      def 		snake_case_			(      self		:       Dict				):
          return self._src_lang




      @src_lang.setter
      def 		snake_case_			(      self		:       Dict				,       _snake_case		:       str				):
          __lowercase		:			Any	    =       new_src_lang
          self.set_src_lang_special_tokens(self._src_lang				)




      def 		snake_case_			(      self		:       Optional[Any]				,       _snake_case		:       List[int]				,       _snake_case		:       Optional[List[int]] = None				):
          if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




      def 		snake_case_			(      self		:       Union[str, Any]				,       _snake_case		:       List[int]				,       _snake_case		:       Optional[List[int]] = None				):
          __lowercase		:			int	    =       [self.sep_token_id]
          __lowercase		:			Optional[Any]	    =       [self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep				) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]




      def 		snake_case_			(      self		:       List[Any]				,       _snake_case		:       str				,       _snake_case		:       str				,       _snake_case		:       Optional[str]				,       _snake_case		:       Optional[str]				,       **_snake_case		:       Optional[Any]				):
          if src_lang is None or tgt_lang is None:
              raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''				)
          __lowercase		:			Tuple	    =       src_lang
          __lowercase		:			str	    =       self(_snake_case				,       add_special_tokens=_snake_case				,       return_tensors=_snake_case				,       **_snake_case				)
          __lowercase		:			Tuple	    =       self.convert_tokens_to_ids(_snake_case				)
          __lowercase		:			Optional[int]	    =       tgt_lang_id
          return inputs




      def 		snake_case_			(      self		:       List[str]				,       _snake_case		:       List[str]				,       _snake_case		:       str = "en_XX"				,       _snake_case		:       Optional[List[str]] = None				,       _snake_case		:       str = "ro_RO"				,       **_snake_case		:       List[str]				,       ):
          __lowercase		:			Union[str, Any]	    =       src_lang
          __lowercase		:			Optional[Any]	    =       tgt_lang
          return super().prepare_seqaseq_batch(_snake_case				,       _snake_case				,       **_snake_case				)




      def 		snake_case_			(      self		:       List[str]				):
          return self.set_src_lang_special_tokens(self.src_lang				)




      def 		snake_case_			(      self		:       List[str]				):
          return self.set_tgt_lang_special_tokens(self.tgt_lang				)




      def 		snake_case_			(      self		:       str				,       _snake_case		:       int				):
          __lowercase		:			List[Any]	    =       self.convert_tokens_to_ids(_snake_case				)
          __lowercase		:			str	    =       []
          __lowercase		:			int	    =       [self.eos_token_id, self.cur_lang_code]

          __lowercase		:			Tuple	    =       self.convert_ids_to_tokens(self.prefix_tokens				)
          __lowercase		:			Tuple	    =       self.convert_ids_to_tokens(self.suffix_tokens				)

          __lowercase		:			List[str]	    =       processors.TemplateProcessing(
              single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,       self.prefix_tokens + self.suffix_tokens				)				)				,       )




      def 		snake_case_			(      self		:       Union[str, Any]				,       _snake_case		:       str				):
          __lowercase		:			List[str]	    =       self.convert_tokens_to_ids(_snake_case				)
          __lowercase		:			Tuple	    =       []
          __lowercase		:			Dict	    =       [self.eos_token_id, self.cur_lang_code]

          __lowercase		:			Union[str, Any]	    =       self.convert_ids_to_tokens(self.prefix_tokens				)
          __lowercase		:			Optional[Any]	    =       self.convert_ids_to_tokens(self.suffix_tokens				)

          __lowercase		:			Any	    =       processors.TemplateProcessing(
              single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,       self.prefix_tokens + self.suffix_tokens				)				)				,       )




      def 		snake_case_			(      self		:       str				,       _snake_case		:       str				,       _snake_case		:       Optional[str] = None				):
          if not self.can_save_slow_tokenizer:
              raise ValueError(
                  '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                  '''tokenizer.'''				)

          if not os.path.isdir(_snake_case				):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory.'				)
              return
          __lowercase		:			List[Any]	    =       os.path.join(
              _snake_case				,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

          if os.path.abspath(self.vocab_file				) != os.path.abspath(_snake_case				):
              copyfile(self.vocab_file				,       _snake_case				)

          return (out_vocab_file,)




import warnings
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCAmelCase					:			Union[str, Any]			      =		logging.get_logger(__name__)

__lowerCAmelCase					:			Optional[int]			      =		{
    "nvidia/segformer-b0-finetuned-ade-512-512": (
        "https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512/resolve/main/config.json"
    ),
    # See all SegFormer models at https://huggingface.co/models?filter=segformer
}


class 						__lowerCAmelCase	(	lowerCAmelCase_      ):



      """simple docstring"""







      A__	:			Dict        =    '''segformer'''




      def __init__(      self		:       Optional[Any]				,       _snake_case		:       List[str]=3				,       _snake_case		:       Union[str, Any]=4				,       _snake_case		:       str=[2, 2, 2, 2]				,       _snake_case		:       Dict=[8, 4, 2, 1]				,       _snake_case		:       List[Any]=[32, 64, 160, 256]				,       _snake_case		:       Dict=[7, 3, 3, 3]				,       _snake_case		:       List[Any]=[4, 2, 2, 2]				,       _snake_case		:       Tuple=[1, 2, 5, 8]				,       _snake_case		:       Optional[Any]=[4, 4, 4, 4]				,       _snake_case		:       Optional[int]="gelu"				,       _snake_case		:       Optional[int]=0.0				,       _snake_case		:       Optional[int]=0.0				,       _snake_case		:       int=0.1				,       _snake_case		:       Any=0.02				,       _snake_case		:       Optional[int]=0.1				,       _snake_case		:       Union[str, Any]=1E-6				,       _snake_case		:       Tuple=256				,       _snake_case		:       str=255				,       **_snake_case		:       Any				,       ):
          super().__init__(**_snake_case				)

          if "reshape_last_stage" in kwargs and kwargs["reshape_last_stage"] is False:
              warnings.warn(
                  '''Reshape_last_stage is set to False in this config. This argument is deprecated and will soon be'''
                  ''' removed, as the behaviour will default to that of reshape_last_stage = True.'''				,       _snake_case				,       )

          __lowercase		:			Any	    =       num_channels
          __lowercase		:			Optional[Any]	    =       num_encoder_blocks
          __lowercase		:			List[str]	    =       depths
          __lowercase		:			str	    =       sr_ratios
          __lowercase		:			List[str]	    =       hidden_sizes
          __lowercase		:			List[str]	    =       patch_sizes
          __lowercase		:			Dict	    =       strides
          __lowercase		:			Optional[int]	    =       mlp_ratios
          __lowercase		:			List[str]	    =       num_attention_heads
          __lowercase		:			Optional[int]	    =       hidden_act
          __lowercase		:			List[Any]	    =       hidden_dropout_prob
          __lowercase		:			Optional[Any]	    =       attention_probs_dropout_prob
          __lowercase		:			List[Any]	    =       classifier_dropout_prob
          __lowercase		:			str	    =       initializer_range
          __lowercase		:			Optional[int]	    =       drop_path_rate
          __lowercase		:			List[str]	    =       layer_norm_eps
          __lowercase		:			List[str]	    =       decoder_hidden_size
          __lowercase		:			Union[str, Any]	    =       kwargs.get('''reshape_last_stage'''				,       _snake_case				)
          __lowercase		:			str	    =       semantic_loss_ignore_index






class 						__lowerCAmelCase	(	lowerCAmelCase_      ):



      """simple docstring"""







      A__	:			Union[str, Any]        =    version.parse('''1.11'''      )




      @property
      def 		snake_case_			(      self		:       Dict				):
          return OrderedDict(
              [
                  ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
              ]				)




      @property
      def 		snake_case_			(      self		:       int				):
          return 1E-4




      @property
      def 		snake_case_			(      self		:       List[Any]				):
          return 12


