'''simple docstring'''




import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_ta import TaTokenizer
else:
						__lowercase        =    None


__lowercase        =    logging.get_logger(__name__)

__lowercase        =    {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__lowercase        =    {
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/tokenizer.json''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/tokenizer.json''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/tokenizer.json''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/tokenizer.json''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/tokenizer.json''',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__lowercase        =    {
    '''t5-small''': 5_1_2,
    '''t5-base''': 5_1_2,
    '''t5-large''': 5_1_2,
    '''t5-3b''': 5_1_2,
    '''t5-11b''': 5_1_2,
}






class 							a__(   lowerCAmelCase__      ):


			'''simple docstring'''






			UpperCAmelCase_     :	Optional[Any]						=			VOCAB_FILES_NAMES
			UpperCAmelCase_     :	List[Any]						=			PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_     :	str						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_     :	Union[str, Any]						=			['''input_ids''', '''attention_mask''']
			UpperCAmelCase_     :	Any						=			TaTokenizer

			UpperCAmelCase_     :	List[int]						=			[]



			def __init__(  self			,       __lowerCAmelCase=None			,       __lowerCAmelCase=None			,       __lowerCAmelCase="</s>"			,       __lowerCAmelCase="<unk>"			,       __lowerCAmelCase="<pad>"			,       __lowerCAmelCase=100			,       __lowerCAmelCase=None			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					if extra_ids > 0 and additional_special_tokens is None:
							lowerCAmelCase     =    [f"<extra_id_{i}>" for i in range(__lowerCAmelCase)]
					elif extra_ids > 0 and additional_special_tokens is not None:
							# Check that we have the right number of extra special tokens
							lowerCAmelCase     =    len(set(filter(lambda __lowerCAmelCase: bool("""extra_id_""" in str(__lowerCAmelCase))			,       __lowerCAmelCase)))
							if extra_tokens != extra_ids:
									raise ValueError(
									    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
									    """ provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"""
									    """ tokens""")

					super().__init__(
					    __lowerCAmelCase			,       tokenizer_file=__lowerCAmelCase			,       eos_token=__lowerCAmelCase			,       unk_token=__lowerCAmelCase			,       pad_token=__lowerCAmelCase			,       extra_ids=__lowerCAmelCase			,       additional_special_tokens=__lowerCAmelCase			,       **__lowerCAmelCase			,       )

					lowerCAmelCase     =    vocab_file
					lowerCAmelCase     =    False if not self.vocab_file else True
					lowerCAmelCase     =    extra_ids



			@staticmethod
			def 				a_			(  __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase):




					"""simple docstring"""


					if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
							lowerCAmelCase     =    TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
							if init_max_model_length is not None and init_max_model_length != max_model_length:
									return init_max_model_length
							elif init_max_model_length is None:
									warnings.warn(
									    """This tokenizer was incorrectly instantiated with a model max length of"""
									    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
									    """ behavior is kept to avoid breaking backwards compatibility when padding/encoding with"""
									    """ `truncation is True`.\n- Be aware that you SHOULD NOT rely on"""
									    f" {pretrained_model_name_or_path} automatically truncating your input to"
									    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
									    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
									    """ `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"""
									    """ instantiate this tokenizer with `model_max_length` set to your preferred value."""			,       __lowerCAmelCase			,       )

					return max_model_length



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None):




					"""simple docstring"""


					if not self.can_save_slow_tokenizer:
							raise ValueError(
							    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
							    """tokenizer.""")

					if not os.path.isdir(__lowerCAmelCase):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory")
							return
					lowerCAmelCase     =    os.path.join(
					    __lowerCAmelCase			,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

					if os.path.abspath(self.vocab_file) != os.path.abspath(__lowerCAmelCase):
							copyfile(self.vocab_file			,       __lowerCAmelCase)
							logger.info(f"Copy vocab file to {out_vocab_file}")

					return (out_vocab_file,)



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase     =    token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
							return self.prefix_tokens + token_ids_a
					else:
							lowerCAmelCase     =    token_ids_a + [self.eos_token_id]
							return self.prefix_tokens + token_ids_a + token_ids_a



			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase     =    [self.eos_token_id]

					if token_ids_a is None:
							return len(token_ids_a + eos) * [0]
					return len(token_ids_a + eos + token_ids_a + eos) * [0]



			def 				a_			(  self):




					"""simple docstring"""


					return list(
					    set(filter(lambda __lowerCAmelCase: bool(re.search(r"""<extra_id_\d+>"""			,       __lowerCAmelCase)) is not None			,       self.additional_special_tokens)))







			def 				a_			(  self):




					"""simple docstring"""


					return [self.convert_tokens_to_ids(__lowerCAmelCase) for token in self.get_sentinel_tokens()]





'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowercase        =    {
    '''configuration_chinese_clip''': [
        '''CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ChineseCLIPConfig''',
        '''ChineseCLIPOnnxConfig''',
        '''ChineseCLIPTextConfig''',
        '''ChineseCLIPVisionConfig''',
    ],
    '''processing_chinese_clip''': ['''ChineseCLIPProcessor'''],
}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__lowercase        =    ['''ChineseCLIPFeatureExtractor''']
						__lowercase        =    ['''ChineseCLIPImageProcessor''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__lowercase        =    [
						    '''CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''ChineseCLIPModel''',
						    '''ChineseCLIPPreTrainedModel''',
						    '''ChineseCLIPTextModel''',
						    '''ChineseCLIPVisionModel''',
						]

if TYPE_CHECKING:
						from .configuration_chinese_clip import (
						    CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    ChineseCLIPConfig,
						    ChineseCLIPOnnxConfig,
						    ChineseCLIPTextConfig,
						    ChineseCLIPVisionConfig,
						)
						from .processing_chinese_clip import ChineseCLIPProcessor

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_chinese_clip import (
												    CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
												    ChineseCLIPModel,
												    ChineseCLIPPreTrainedModel,
												    ChineseCLIPTextModel,
												    ChineseCLIPVisionModel,
												)

else:
						import sys

						__lowercase        =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




