





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase_           =	{
    "configuration_git": ["GIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "GitConfig", "GitVisionConfig"],
    "processing_git": ["GitProcessor"],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_           =	[
       "GIT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "GitForCausalLM",
       "GitModel",
       "GitPreTrainedModel",
       "GitVisionModel",
   ]

if TYPE_CHECKING:
   from .configuration_git import GIT_PRETRAINED_CONFIG_ARCHIVE_MAP, GitConfig, GitVisionConfig
   from .processing_git import GitProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_git import (
          GIT_PRETRAINED_MODEL_ARCHIVE_LIST,
          GitForCausalLM,
          GitModel,
          GitPreTrainedModel,
          GitVisionModel,
      )

else:
   import sys

   UpperCAmelCase_           =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
   from ...tokenization_utils_base import TextInput
from ...utils import logging


UpperCAmelCase_           =	logging.get_logger(__name__)

UpperCAmelCase_           =	{"vocab_file": "spiece.model"}

UpperCAmelCase_           =	{
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
UpperCAmelCase_           =	{
    "t5-small": 5_12,
    "t5-base": 5_12,
    "t5-large": 5_12,
    "t5-3b": 5_12,
    "t5-11b": 5_12,
}

UpperCAmelCase_           =	"▁"




class 			__UpperCamelCase    (							A__    ):
     __A	:  Any							  =   VOCAB_FILES_NAMES
     __A	:  Union[str, Any]							  =   PRETRAINED_VOCAB_FILES_MAP
     __A	:  Any							  =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A	:  Tuple							  =   ["""input_ids""", """attention_mask"""]





     def __init__(     self				,							_UpperCamelCase				,							_UpperCamelCase="</s>"				,							_UpperCamelCase="<unk>"				,							_UpperCamelCase="<pad>"				,							_UpperCamelCase=100				,							_UpperCamelCase=None				,							_UpperCamelCase = None				,							_UpperCamelCase=True				,							**_UpperCamelCase				,							):
        # Add extra_ids to the special token list
        if extra_ids > 0 and additional_special_tokens is None:
           _UpperCAmelCase           =   [f'''<extra_id_{i}>''' for i in range(_UpperCamelCase					)]
        elif extra_ids > 0 and additional_special_tokens is not None:
           # Check that we have the right number of extra_id special tokens
           _UpperCAmelCase           =   len(set(filter(lambda _UpperCamelCase					: bool('''extra_id''' in str(_UpperCamelCase					)					)				,							_UpperCamelCase					)					)					)
           if extra_tokens != extra_ids:
              raise ValueError(
                  f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
                  ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                  ''' tokens'''					)
        if legacy:
           logger.warning_once(
               f'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
               ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''					)

        _UpperCAmelCase           =   legacy
        _UpperCAmelCase           =   {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            eos_token=_UpperCamelCase				,							unk_token=_UpperCamelCase				,							pad_token=_UpperCamelCase				,							extra_ids=_UpperCamelCase				,							additional_special_tokens=_UpperCamelCase				,							sp_model_kwargs=self.sp_model_kwargs				,							legacy=_UpperCamelCase				,							**_UpperCamelCase				,							)

        _UpperCAmelCase           =   vocab_file
        _UpperCAmelCase           =   extra_ids

        _UpperCAmelCase           =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
        self.sp_model.Load(_UpperCamelCase					)





     @staticmethod
     def 		UpperCamelCase(     _UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase					):
        if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
           _UpperCAmelCase           =   TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
           if init_max_model_length is not None and init_max_model_length != max_model_length:
              return init_max_model_length
           elif init_max_model_length is None:
              warnings.warn(
                  '''This tokenizer was incorrectly instantiated with a model max length of'''
                  f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
                  ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                  ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                  f''' {pretrained_model_name_or_path} automatically truncating your input to'''
                  f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
                  f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
                  ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                  ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''				,							_UpperCamelCase				,							)

        return max_model_length





     @property
     def 		UpperCamelCase(     self					):
        return self.sp_model.get_piece_size() + self._extra_ids





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   {self.convert_ids_to_tokens(_UpperCamelCase					): i for i in range(self.vocab_size					)}
        vocab.update(self.added_tokens_encoder					)
        return vocab





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None				,							_UpperCamelCase = False					):
        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_UpperCamelCase				,							token_ids_a=_UpperCamelCase				,							already_has_special_tokens=_UpperCamelCase					)

        # normal case: some special tokens
        if token_ids_a is None:
           return ([0] * len(_UpperCamelCase					)) + [1]
        return ([0] * len(_UpperCamelCase					)) + [1] + ([0] * len(_UpperCamelCase					)) + [1]





     def 		UpperCamelCase(     self					):
        return list(
            set(filter(lambda _UpperCamelCase					: bool(re.search(R'''<extra_id_\d+>'''				,							_UpperCamelCase					)					) is not None				,							self.additional_special_tokens					)					)					)





     def 		UpperCamelCase(     self					):
        return [self._convert_token_to_id(_UpperCamelCase					) for token in self.get_sentinel_tokens()]





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        if len(_UpperCamelCase					) > 0 and token_ids[-1] == self.eos_token_id:
           warnings.warn(
               f'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
               ''' eos tokens being added.'''					)
           return token_ids
        else:
           return token_ids + [self.eos_token_id]





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None					):
        _UpperCAmelCase           =   [self.eos_token_id]

        if token_ids_a is None:
           return len(token_ids_a + eos					) * [0]
        return len(token_ids_a + eos + token_ids_a + eos					) * [0]





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None					):
        _UpperCAmelCase           =   self._add_eos_if_not_present(_UpperCamelCase					)
        if token_ids_a is None:
           return token_ids_a
        else:
           _UpperCAmelCase           =   self._add_eos_if_not_present(_UpperCamelCase					)
           return token_ids_a + token_ids_a





     def __getstate__(     self					):
        _UpperCAmelCase           =   self.__dict__.copy()
        _UpperCAmelCase           =   None
        return state





     def __setstate__(     self				,							_UpperCamelCase					):
        _UpperCAmelCase           =   d

        # for backward compatibility
        if not hasattr(self				,							'''sp_model_kwargs'''					):
           _UpperCAmelCase           =   {}

        _UpperCAmelCase           =   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
        self.sp_model.Load(self.vocab_file					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							**_UpperCamelCase					):
        # Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
        # the beginning of the text
        if not self.legacy:
           _UpperCAmelCase           =   SPIECE_UNDERLINE + text.replace(_UpperCamelCase				,							''' '''					)
        return super().tokenize(_UpperCamelCase				,							**_UpperCamelCase					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							**_UpperCamelCase					):
        if not self.legacy:
           _UpperCAmelCase           =   text.startswith(_UpperCamelCase					)
           if is_first:
              _UpperCAmelCase           =   text[1:]

        _UpperCAmelCase           =   self.sp_model.encode(_UpperCamelCase				,							out_type=_UpperCamelCase					)

        if not self.legacy and not is_first and not text.startswith(''' '''					) and tokens[0].startswith(_UpperCamelCase					):
           _UpperCAmelCase           =   ([tokens[0][1:]] if len(tokens[0]					) > 1 else []) + tokens[1:]
        return tokens





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        if token.startswith('''<extra_id_'''					):
           _UpperCAmelCase           =   re.match(R'''<extra_id_(\d+)>'''				,							_UpperCamelCase					)
           _UpperCAmelCase           =   int(match.group(1					)					)
           return self.vocab_size - num - 1
        return self.sp_model.piece_to_id(_UpperCamelCase					)





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        if index < self.sp_model.get_piece_size():
           _UpperCAmelCase           =   self.sp_model.IdToPiece(_UpperCamelCase					)
        else:
           _UpperCAmelCase           =   f'''<extra_id_{self.vocab_size - 1 - index}>'''
        return token





     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        _UpperCAmelCase           =   []
        _UpperCAmelCase           =   ''''''
        _UpperCAmelCase           =   False
        for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
              if not prev_is_special:
                 out_string += " "
              out_string += self.sp_model.decode(_UpperCamelCase					) + token
              _UpperCAmelCase           =   True
              _UpperCAmelCase           =   []
           else:
              current_sub_tokens.append(_UpperCamelCase					)
              _UpperCAmelCase           =   False
        out_string += self.sp_model.decode(_UpperCamelCase					)
        return out_string.strip()





     def 		UpperCamelCase(     self				,							_UpperCamelCase				,							_UpperCamelCase = None					):
        if not os.path.isdir(_UpperCamelCase					):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
           return
        _UpperCAmelCase           =   os.path.join(
            _UpperCamelCase				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

        if os.path.abspath(self.vocab_file					) != os.path.abspath(_UpperCamelCase					) and os.path.isfile(self.vocab_file					):
           copyfile(self.vocab_file				,							_UpperCamelCase					)
        elif not os.path.isfile(self.vocab_file					):
           with open(_UpperCamelCase				,							'''wb'''					) as fi:
              _UpperCAmelCase           =   self.sp_model.serialized_model_proto()
              fi.write(_UpperCamelCase					)

        return (out_vocab_file,)