from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCamelCase_				:		Optional[int]       						=       logging.get_logger(__name__)

lowerCamelCase_				:		Union[str, Any]       						=       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase_				:		Any       						=       {
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase_				:		Union[str, Any]       						=       {
    """openai/clip-vit-base-patch32""": 77,
}


class 						a__							(   __snake_case		):
			A__				:   str								=			VOCAB_FILES_NAMES
			A__				:   Optional[int]								=			PRETRAINED_VOCAB_FILES_MAP
			A__				:   int								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__				:   Dict								=			['input_ids', 'attention_mask']
			A__				:   Optional[int]								=			CLIPTokenizer



			def __init__(						self      ,						UpperCAmelCase=None      ,						UpperCAmelCase=None      ,						UpperCAmelCase=None      ,						UpperCAmelCase="<|endoftext|>"      ,						UpperCAmelCase="<|startoftext|>"      ,						UpperCAmelCase="<|endoftext|>"      ,						UpperCAmelCase="<|endoftext|>"      ,						**UpperCAmelCase      ,						) ->    List[Any]:
								super().__init__(
								    UpperCAmelCase      ,						UpperCAmelCase      ,						tokenizer_file=UpperCAmelCase      ,						unk_token=UpperCAmelCase      ,						bos_token=UpperCAmelCase      ,						eos_token=UpperCAmelCase      ,						pad_token=UpperCAmelCase      ,						**UpperCAmelCase      ,						)

								if not isinstance(self.backend_tokenizer.pre_tokenizer      ,						pre_tokenizers.Sequence    ):
													raise ValueError(
													    'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
													    ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
													    ' to be compatible with this version.The easiest way to do so is'
													    ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
													    ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
													    ' transformers.'    )

								self._wrap_decode_method_backend_tokenizer()



			def __SCREAMING_SNAKE_CASE							(						self    ) ->    Dict:
								__a							=   self.backend_tokenizer.decode

								def new_decode_method(*UpperCAmelCase      ,						**UpperCAmelCase    ):
													__a							=   orig_decode_method(*UpperCAmelCase      ,						**UpperCAmelCase    )
													__a							=   text.replace(self.backend_tokenizer.model.end_of_word_suffix      ,						' '    ).strip()
													return text

								__a							=   new_decode_method



			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase      ,						UpperCAmelCase = None    ) ->    List[int]:
								__a							=   [self.bos_token_id]
								__a							=   [self.eos_token_id]

								if token_ids_a is None:
													return bos_token + token_ids_a + eos_token
								return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token



			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase      ,						UpperCAmelCase = None    ) ->    List[int]:
								__a							=   [self.bos_token_id]
								__a							=   [self.eos_token_id]

								if token_ids_a is None:
													return len(bos_token + token_ids_a + eos_token    ) * [0]
								return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token    ) * [0]







			def __SCREAMING_SNAKE_CASE							(						self      ,						UpperCAmelCase      ,						UpperCAmelCase = None    ) ->    Tuple[str]:
								__a							=   self._tokenizer.model.save(UpperCAmelCase      ,						name=UpperCAmelCase    )
								return tuple(UpperCAmelCase    )





from typing import TypeVar

from .arrow_dataset import Dataset, _split_by_node_map_style_dataset
from .iterable_dataset import IterableDataset, _split_by_node_iterable_dataset


lowerCamelCase_				:		List[Any]       						=       TypeVar("""DatasetType""", Dataset, IterableDataset)
def 							lowerCAmelCase(	__lowerCamelCase      ,				__lowerCamelCase      ,				__lowerCamelCase						):
					if isinstance(__lowerCamelCase      ,				__lowerCamelCase						):
										return _split_by_node_map_style_dataset(__lowerCamelCase      ,				rank=__lowerCamelCase      ,				world_size=__lowerCamelCase						)
					else:
										return _split_by_node_iterable_dataset(__lowerCamelCase      ,				rank=__lowerCamelCase      ,				world_size=__lowerCamelCase						)




