




from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


a_      						=   logging.get_logger(__name__)

a_      						=   {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

a_      						=   {
    "vocab_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json",
    },
    "merges_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "openai/clip-vit-base-patch32": (
            "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"
        ),
    },
}

a_      						=   {
    "openai/clip-vit-base-patch32": 77,
}

class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      lowerCAmelCase__	:       Tuple			=    VOCAB_FILES_NAMES
      lowerCAmelCase__	:       Optional[int]			=    PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__	:       Optional[Any]			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__	:       Optional[Any]			=    ["input_ids", "attention_mask"]
      lowerCAmelCase__	:       Union[str, Any]			=    CLIPTokenizer







      def __init__(   self    :       Optional[Any]       ,				snake_case    :       List[Any]=None       ,				snake_case    :       int=None       ,				snake_case    :       Any=None       ,				snake_case    :       Optional[Any]="<|endoftext|>"       ,				snake_case    :       List[str]="<|startoftext|>"       ,				snake_case    :       Optional[Any]="<|endoftext|>"       ,				snake_case    :       int="<|endoftext|>"       ,				**snake_case    :       Any       ,				):
          super().__init__(
              snake_case       ,				snake_case       ,				tokenizer_file=snake_case       ,				unk_token=snake_case       ,				bos_token=snake_case       ,				eos_token=snake_case       ,				pad_token=snake_case       ,				**snake_case       ,				)

          if not isinstance(self.backend_tokenizer.pre_tokenizer       ,				pre_tokenizers.Sequence ):
              raise ValueError(
                  '''The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'''
                  ''' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'''
                  ''' to be compatible with this version.The easiest way to do so is'''
                  ''' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'''
                  ''' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'''
                  ''' transformers.''' )

          self._wrap_decode_method_backend_tokenizer()







      def 					snake_case		(   self    :       Optional[int] ):
          __UpperCamelCase     						=  self.backend_tokenizer.decode

          def new_decode_method(*snake_case    :       str       ,				**snake_case    :       Union[str, Any] ):
              __UpperCamelCase     						=  orig_decode_method(*snake_case       ,				**snake_case )
              __UpperCamelCase     						=  text.replace(self.backend_tokenizer.model.end_of_word_suffix       ,				''' ''' ).strip()
              return text

          __UpperCamelCase     						=  new_decode_method







      def 					snake_case		(   self    :       Optional[Any]       ,				snake_case    :       List[int]       ,				snake_case    :       Optional[List[int]] = None ):
          __UpperCamelCase     						=  [self.bos_token_id]
          __UpperCamelCase     						=  [self.eos_token_id]

          if token_ids_a is None:
              return bos_token + token_ids_a + eos_token
          return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token







      def 					snake_case		(   self    :       Tuple       ,				snake_case    :       List[int]       ,				snake_case    :       Optional[List[int]] = None ):
          __UpperCamelCase     						=  [self.bos_token_id]
          __UpperCamelCase     						=  [self.eos_token_id]

          if token_ids_a is None:
              return len(bos_token + token_ids_a + eos_token ) * [0]
          return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token ) * [0]







      def 					snake_case		(   self    :       Tuple       ,				snake_case    :       str       ,				snake_case    :       Optional[str] = None ):
          __UpperCamelCase     						=  self._tokenizer.model.save(snake_case       ,				name=snake_case )
          return tuple(snake_case )










"""simple docstring"""


import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


a_											=			get_logger(__name__)


if TYPE_CHECKING:
		from .builder import DatasetBuilder






def    a__       ( __lowercase   ,	__lowercase = None      )		->		Optional[Any]:
				_A         =	importlib.import_module(__lowercase      )

				# TODO(QL): always update the module to add subsequent new authentication
				if hasattr(__lowercase   ,	"_patched_for_streaming"      ) and module._patched_for_streaming:
								return

				def wrap_auth(__lowercase      ):
								@wraps(__lowercase      )
								def wrapper(*__lowercase   ,	**__lowercase      ):
												return function(*__lowercase   ,	use_auth_token=__lowercase   ,	**__lowercase      )

								_A         =	"wrap_auth"
								return wrapper

				# open files in a streaming fashion
				patch_submodule(__lowercase   ,	"open"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.listdir"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.walk"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"glob.glob"   ,	wrap_auth(__lowercase      )      ).start()
				# allow to navigate in remote zip files
				patch_submodule(__lowercase   ,	"os.path.join"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.dirname"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.basename"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.relpath"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.split"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.splitext"   ,	__lowercase      ).start()
				# allow checks on paths
				patch_submodule(__lowercase   ,	"os.path.exists"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.isdir"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.isfile"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.getsize"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"pathlib.Path"   ,	__lowercase      ).start()
				# file readers
				patch_submodule(__lowercase   ,	"gzip.open"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"numpy.load"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"pandas.read_csv"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"pandas.read_excel"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"scipy.io.loadmat"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"xml.etree.ElementTree.parse"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"xml.dom.minidom.parse"   ,	wrap_auth(__lowercase      )      ).start()
				_A         =	True






def    a__       ( __lowercase      )		->		str:
				extend_module_for_streaming(builder.__module__   ,	use_auth_token=builder.use_auth_token      )
				# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
				if not builder.__module__.startswith("datasets."      ):  # check that it's not a packaged builder like csv
								for imports in get_imports(inspect.getfile(builder.__class__      )      ):
												if imports[0] == "internal":
																_A         =	imports[1]
																_A         =	".".join(builder.__module__.split("."      )[:-1] + [internal_import_name]      )
																extend_module_for_streaming(__lowercase   ,	use_auth_token=builder.use_auth_token      )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
				from .builder import DatasetBuilder

				_A         =	[
				    cls.__module__
				    for cls in type(__lowercase      ).__mro__[1:]  # make sure it's not the same module we've already patched
				    if issubclass(cls   ,	__lowercase      ) and cls.__module__ != DatasetBuilder.__module__
				]  # check it's not a standard builder from datasets.builder
				for module in parent_builder_modules:
								extend_module_for_streaming(__lowercase   ,	use_auth_token=builder.use_auth_token      )