

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase     :   Tuple										=  logging.get_logger(__name__)

lowerCAmelCase     :   Dict										=  '''▁'''

lowerCAmelCase     :   int										=  {'''vocab_file''': '''sentencepiece.bpe.model'''}

lowerCAmelCase     :   List[str]										=  {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

lowerCAmelCase     :   Dict										=  {
    '''facebook/mbart-large-50-one-to-many-mmt''': 10_24,
}

# fmt: off
lowerCAmelCase     :   Optional[Any]										=  ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']
class  _A		(  UpperCamelCase__):
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE    :      Any       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE    :      Optional[int]       =  PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE    :      List[str]       =  ["""input_ids""", """attention_mask"""]

  SCREAMING_SNAKE_CASE    :      List[int]       =  []
  SCREAMING_SNAKE_CASE    :      List[int]       =  []

  def __init__(				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE="</s>" ,					_SCREAMING_SNAKE_CASE="</s>" ,					_SCREAMING_SNAKE_CASE="<s>" ,					_SCREAMING_SNAKE_CASE="<unk>" ,					_SCREAMING_SNAKE_CASE="<pad>" ,					_SCREAMING_SNAKE_CASE="<mask>" ,					_SCREAMING_SNAKE_CASE = None ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     AddedToken(_SCREAMING_SNAKE_CASE ,					lstrip=_SCREAMING_SNAKE_CASE ,					rstrip=_SCREAMING_SNAKE_CASE					) if isinstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					) else mask_token

        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     {} if sp_model_kwargs is None else sp_model_kwargs

        SCREAMING_SNAKE_CASE_				:  Dict           =     kwargs.get('additional_special_tokens' ,					[]					)
        kwargs["additional_special_tokens"] += [
            code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=_SCREAMING_SNAKE_CASE ,					tgt_lang=_SCREAMING_SNAKE_CASE ,					eos_token=_SCREAMING_SNAKE_CASE ,					unk_token=_SCREAMING_SNAKE_CASE ,					sep_token=_SCREAMING_SNAKE_CASE ,					cls_token=_SCREAMING_SNAKE_CASE ,					pad_token=_SCREAMING_SNAKE_CASE ,					mask_token=_SCREAMING_SNAKE_CASE ,					sp_model_kwargs=self.sp_model_kwargs ,					**_SCREAMING_SNAKE_CASE ,					)

        SCREAMING_SNAKE_CASE_				:  List[str]           =     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
        self.sp_model.Load(str(_SCREAMING_SNAKE_CASE					)					)
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # Mimic fairseq token-to-id alignment for the first 4 token
        SCREAMING_SNAKE_CASE_				:  Tuple           =     {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        SCREAMING_SNAKE_CASE_				:  Any           =     1

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     len(self.sp_model					)
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     {
            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_SCREAMING_SNAKE_CASE					)
        }
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     {v: k for k, v in self.lang_code_to_id.items()}
        SCREAMING_SNAKE_CASE_				:  List[str]           =     len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset

        self.fairseq_tokens_to_ids.update(self.lang_code_to_id					)
        SCREAMING_SNAKE_CASE_				:  Any           =     {v: k for k, v in self.fairseq_tokens_to_ids.items()}

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     src_lang if src_lang is not None else 'en_XX'
        SCREAMING_SNAKE_CASE_				:  List[str]           =     self.lang_code_to_id[self._src_lang]
        SCREAMING_SNAKE_CASE_				:  Any           =     tgt_lang
        self.set_src_lang_special_tokens(self._src_lang					)

  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset + 1  # Plus 1 for the mask token

  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self._src_lang

  @src_lang.setter
  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Tuple           =     new_src_lang
        self.set_src_lang_special_tokens(self._src_lang					)

  def __getstate__(				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     self.__dict__.copy()
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     None
        return state

  def __setstate__(				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[str]           =     d

        # for backward compatibility
        if not hasattr(self ,					'sp_model_kwargs'					):
              SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     {}

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
        self.sp_model.Load(self.vocab_file					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE					): i for i in range(self.vocab_size					)}
        vocab.update(self.added_tokens_encoder					)
        return vocab

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        return self.sp_model.encode(_SCREAMING_SNAKE_CASE ,					out_type=_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
        SCREAMING_SNAKE_CASE_				:  int           =     self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE					)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Tuple           =     []
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     ''
        SCREAMING_SNAKE_CASE_				:  str           =     False
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    if not prev_is_special:
                          out_string += " "
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE					) + token
                    SCREAMING_SNAKE_CASE_				:  Optional[int]           =     True
                    SCREAMING_SNAKE_CASE_				:  Dict           =     []
              else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     False
        out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE					)
        return out_string.strip()

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        if not os.path.isdir(_SCREAMING_SNAKE_CASE					):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
              return
        SCREAMING_SNAKE_CASE_				:  Any           =     os.path.join(
            _SCREAMING_SNAKE_CASE ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

        if os.path.abspath(self.vocab_file					) != os.path.abspath(_SCREAMING_SNAKE_CASE					) and os.path.isfile(self.vocab_file					):
              copyfile(self.vocab_file ,					_SCREAMING_SNAKE_CASE					)
        elif not os.path.isfile(self.vocab_file					):
              with open(_SCREAMING_SNAKE_CASE ,					'wb'					) as fi:
                    SCREAMING_SNAKE_CASE_				:  List[str]           =     self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE					)

        return (out_vocab_file,)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None ,					_SCREAMING_SNAKE_CASE = False					):





        """simple docstring"""







        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_SCREAMING_SNAKE_CASE ,					token_ids_a=_SCREAMING_SNAKE_CASE ,					already_has_special_tokens=_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  int           =     [1] * len(self.prefix_tokens					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     [1] * len(self.suffix_tokens					)
        if token_ids_a is None:
              return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE					)) + suffix_ones
        return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE					)) + ([0] * len(_SCREAMING_SNAKE_CASE					)) + suffix_ones

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        if src_lang is None or tgt_lang is None:
              raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     src_lang
        SCREAMING_SNAKE_CASE_				:  int           =     self(_SCREAMING_SNAKE_CASE ,					add_special_tokens=_SCREAMING_SNAKE_CASE ,					return_tensors=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Tuple           =     self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     tgt_lang_id
        return inputs

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = "en_XX" ,					_SCREAMING_SNAKE_CASE = None ,					_SCREAMING_SNAKE_CASE = "ro_RO" ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Tuple           =     src_lang
        SCREAMING_SNAKE_CASE_				:  Any           =     tgt_lang
        return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self.set_src_lang_special_tokens(self.src_lang					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self.set_tgt_lang_special_tokens(self.tgt_lang					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.lang_code_to_id[src_lang]
        SCREAMING_SNAKE_CASE_				:  List[str]           =     [self.cur_lang_code_id]
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     [self.eos_token_id]



  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  int           =     self.lang_code_to_id[tgt_lang]
        SCREAMING_SNAKE_CASE_				:  Any           =     [self.cur_lang_code_id]
        SCREAMING_SNAKE_CASE_				:  Tuple           =     [self.eos_token_id]









import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase     :   Optional[int]						     =  logging.get_logger(__name__)

lowerCAmelCase     :   str						     =  {
    'microsoft/git-base': 'https://huggingface.co/microsoft/git-base/resolve/main/config.json',
}
class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      Optional[int]       =  '''git_vision_model'''

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=768 ,					_SCREAMING_SNAKE_CASE=3072 ,					_SCREAMING_SNAKE_CASE=12 ,					_SCREAMING_SNAKE_CASE=12 ,					_SCREAMING_SNAKE_CASE=3 ,					_SCREAMING_SNAKE_CASE=224 ,					_SCREAMING_SNAKE_CASE=16 ,					_SCREAMING_SNAKE_CASE="quick_gelu" ,					_SCREAMING_SNAKE_CASE=1e-5 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=0.02 ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        super().__init__(**_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  List[str]           =     hidden_size
        SCREAMING_SNAKE_CASE_				:  Any           =     intermediate_size
        SCREAMING_SNAKE_CASE_				:  List[str]           =     num_hidden_layers
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     num_attention_heads
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     num_channels
        SCREAMING_SNAKE_CASE_				:  Dict           =     patch_size
        SCREAMING_SNAKE_CASE_				:  Any           =     image_size
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     initializer_range
        SCREAMING_SNAKE_CASE_				:  Any           =     attention_dropout
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     layer_norm_eps
        SCREAMING_SNAKE_CASE_				:  Tuple           =     hidden_act



  @classmethod
  def  UpperCAmelCase    (				cls ,					_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        cls._set_token_in_kwargs(_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_       ,						SCREAMING_SNAKE_CASE_				:  str           =     cls.get_config_dict(_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

        # get the vision config dict if we are loading from GITConfig
        if config_dict.get('model_type'					) == "git":
              SCREAMING_SNAKE_CASE_				:  str           =     config_dict['vision_config']

        if "model_type" in config_dict and hasattr(cls ,					'model_type'					) and config_dict["model_type"] != cls.model_type:
              logger.warning(
                  f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                  f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."					)

        return cls.from_dict(_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      Optional[Any]       =  '''git'''

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=3_0522 ,					_SCREAMING_SNAKE_CASE=768 ,					_SCREAMING_SNAKE_CASE=6 ,					_SCREAMING_SNAKE_CASE=12 ,					_SCREAMING_SNAKE_CASE=3072 ,					_SCREAMING_SNAKE_CASE="gelu" ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=1024 ,					_SCREAMING_SNAKE_CASE=0.02 ,					_SCREAMING_SNAKE_CASE=1e-12 ,					_SCREAMING_SNAKE_CASE=0 ,					_SCREAMING_SNAKE_CASE="absolute" ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=101 ,					_SCREAMING_SNAKE_CASE=102 ,					_SCREAMING_SNAKE_CASE=None ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE ,					eos_token_id=_SCREAMING_SNAKE_CASE ,					pad_token_id=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

        if vision_config is None:
              SCREAMING_SNAKE_CASE_				:  Any           =     {}
              logger.info('vision_config is None. initializing the GitVisionConfig with default values.'					)

        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     GitVisionConfig(**_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  Dict           =     vocab_size
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     hidden_size
        SCREAMING_SNAKE_CASE_				:  Dict           =     num_hidden_layers
        SCREAMING_SNAKE_CASE_				:  List[str]           =     num_attention_heads
        SCREAMING_SNAKE_CASE_				:  List[str]           =     hidden_act
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     intermediate_size
        SCREAMING_SNAKE_CASE_				:  Any           =     hidden_dropout_prob
        SCREAMING_SNAKE_CASE_				:  Tuple           =     attention_probs_dropout_prob
        SCREAMING_SNAKE_CASE_				:  int           =     max_position_embeddings
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     initializer_range
        SCREAMING_SNAKE_CASE_				:  int           =     layer_norm_eps
        SCREAMING_SNAKE_CASE_				:  List[str]           =     position_embedding_type
        SCREAMING_SNAKE_CASE_				:  List[str]           =     use_cache
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tie_word_embeddings
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     num_image_with_embedding

        SCREAMING_SNAKE_CASE_				:  Tuple           =     bos_token_id
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     eos_token_id



  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[str]           =     copy.deepcopy(self.__dict__					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     self.vision_config.to_dict()
        SCREAMING_SNAKE_CASE_				:  int           =     self.__class__.model_type
        return output






