

"""simple docstring"""




import inspect
import tempfile
from collections import OrderedDict, UserDict
from collections.abc import MutableMapping
from contextlib import ExitStack, contextmanager
from dataclasses import fields
from enum import Enum
from typing import Any, ContextManager, List, Tuple

import numpy as np

from .import_utils import is_flax_available, is_tf_available, is_torch_available, is_torch_fx_proxy


if is_flax_available():
 import jax.numpy as jnp



class   lowercase_   (  __lowerCAmelCase					):



      def __get__(      self				:    int      , A__				:    Union[str, Any]      , A__				:    Union[str, Any]=None		)   ->					str:






        if obj is None:
          return self
        if self.fget is None:
          raise AttributeError('''unreadable attribute'''		)
        _snake_case						=							'''__cached_''' + self.fget.__name__
        _snake_case						=							getattr(lowerCamelCase__      , lowerCamelCase__      , lowerCamelCase__		)
        if cached is None:
          _snake_case						=							self.fget(lowerCamelCase__		)
          setattr(lowerCamelCase__      , lowerCamelCase__      , lowerCamelCase__		)
        return cached



def 				snake_case_(_UpperCamelCase					)							->     str:



  """simple docstring"""






  _snake_case						=							val.lower()
  if val in {"y", "yes", "t", "true", "on", "1"}:
    return 1
  if val in {"n", "no", "f", "false", "off", "0"}:
    return 0
  raise ValueError(F"""invalid truth value {val!r}"""					)



def 				snake_case_(_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  if is_torch_fx_proxy(_UpperCamelCase					):
    return True
  if is_torch_available():
    import torch

    if isinstance(_UpperCamelCase			,	torch.Tensor					):
      return True
  if is_tf_available():
    import tensorflow as tf

    if isinstance(_UpperCamelCase			,	tf.Tensor					):
      return True

  if is_flax_available():
    import jax.numpy as jnp
    from jax.core import Tracer

    if isinstance(_UpperCamelCase			,	(jnp.ndarray, Tracer)					):
      return True

  return isinstance(_UpperCamelCase			,	np.ndarray					)



def 				snake_case_(_UpperCamelCase					)							->     Optional[Any]:



  """simple docstring"""






  return isinstance(_UpperCamelCase			,	np.ndarray					)



def 				snake_case_(_UpperCamelCase					)							->     int:



  """simple docstring"""






  return _is_numpy(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     Optional[Any]:



  """simple docstring"""






  import torch

  return isinstance(_UpperCamelCase			,	torch.Tensor					)



def 				snake_case_(_UpperCamelCase					)							->     Any:



  """simple docstring"""






  return False if not is_torch_available() else _is_torch(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     Tuple:



  """simple docstring"""






  import torch

  return isinstance(_UpperCamelCase			,	torch.device					)



def 				snake_case_(_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  return False if not is_torch_available() else _is_torch_device(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     int:



  """simple docstring"""






  import torch

  if isinstance(_UpperCamelCase			,	_UpperCamelCase					):
    if hasattr(_UpperCamelCase			,	_UpperCamelCase					):
      _snake_case						=							getattr(_UpperCamelCase			,	_UpperCamelCase					)
    else:
      return False
  return isinstance(_UpperCamelCase			,	torch.dtype					)



def 				snake_case_(_UpperCamelCase					)							->     str:



  """simple docstring"""






  return False if not is_torch_available() else _is_torch_dtype(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     List[Any]:



  """simple docstring"""






  import tensorflow as tf

  return isinstance(_UpperCamelCase			,	tf.Tensor					)



def 				snake_case_(_UpperCamelCase					)							->     str:



  """simple docstring"""






  return False if not is_tf_available() else _is_tensorflow(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     int:



  """simple docstring"""






  import tensorflow as tf

  # the `is_symbolic_tensor` predicate is only available starting with TF 2.14
  if hasattr(_UpperCamelCase			,	'''is_symbolic_tensor'''					):
    return tf.is_symbolic_tensor(_UpperCamelCase					)
  return type(_UpperCamelCase					) == tf.Tensor



def 				snake_case_(_UpperCamelCase					)							->     Optional[Any]:



  """simple docstring"""






  return False if not is_tf_available() else _is_tf_symbolic_tensor(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     Tuple:



  """simple docstring"""






  import jax.numpy as jnp  # noqa: F811

  return isinstance(_UpperCamelCase			,	jnp.ndarray					)



def 				snake_case_(_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  return False if not is_flax_available() else _is_jax(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase					)							->     List[str]:



  """simple docstring"""






  if isinstance(_UpperCamelCase			,	(dict, UserDict)					):
    return {k: to_py_obj(_UpperCamelCase					) for k, v in obj.items()}
  elif isinstance(_UpperCamelCase			,	(list, tuple)					):
    return [to_py_obj(_UpperCamelCase					) for o in obj]
  elif is_tf_tensor(_UpperCamelCase					):
    return obj.numpy().tolist()
  elif is_torch_tensor(_UpperCamelCase					):
    return obj.detach().cpu().tolist()
  elif is_jax_tensor(_UpperCamelCase					):
    return np.asarray(_UpperCamelCase					).tolist()
  elif isinstance(_UpperCamelCase			,	(np.ndarray, np.number)					):  # tolist also works on 0d np arrays
    return obj.tolist()
  else:
    return obj



def 				snake_case_(_UpperCamelCase					)							->     Tuple:



  """simple docstring"""






  if isinstance(_UpperCamelCase			,	(dict, UserDict)					):
    return {k: to_numpy(_UpperCamelCase					) for k, v in obj.items()}
  elif isinstance(_UpperCamelCase			,	(list, tuple)					):
    return np.array(_UpperCamelCase					)
  elif is_tf_tensor(_UpperCamelCase					):
    return obj.numpy()
  elif is_torch_tensor(_UpperCamelCase					):
    return obj.detach().cpu().numpy()
  elif is_jax_tensor(_UpperCamelCase					):
    return np.asarray(_UpperCamelCase					)
  else:
    return obj



class   lowercase_   (  __lowerCAmelCase					):



      def 						UpperCamelCase_		(      self				:    List[Any]		)   ->					Dict:
        _snake_case						=							fields(self		)

        # Safety and consistency checks
        if not len(lowerCamelCase__		):
          raise ValueError(f"""{self.__class__.__name__} has no fields."""		)
        if not all(field.default is None for field in class_fields[1:]		):
          raise ValueError(f"""{self.__class__.__name__} should not have more than one required field."""		)

        _snake_case						=							getattr(self      , class_fields[0].name		)
        _snake_case						=							all(getattr(self      , field.name		) is None for field in class_fields[1:]		)

        if other_fields_are_none and not is_tensor(lowerCamelCase__		):
          if isinstance(lowerCamelCase__      , lowerCamelCase__		):
            _snake_case						=							first_field.items()
            _snake_case						=							True
          else:
            try:
              _snake_case						=							iter(lowerCamelCase__		)
              _snake_case						=							True
            except TypeError:
              _snake_case						=							False

                 # if we provided an iterator as first field and the iterator is a (key, value) iterator
                 # set the associated fields
          if first_field_iterator:
            for idx, element in enumerate(lowerCamelCase__		):
              if (
                  not isinstance(lowerCamelCase__      , (list, tuple)		)
                  or not len(lowerCamelCase__		) == 2
                  or not isinstance(element[0]      , lowerCamelCase__		)
              ):
                if idx == 0:
                  # If we do not have an iterator of key/values, set it as attribute
                  _snake_case						=							first_field
                else:
                  # If we have a mixed iterator, raise an error
                  raise ValueError(
                      f"""Cannot set key/value for {element}. It needs to be a tuple (key, value)."""		)
                break
              setattr(self      , element[0]      , element[1]		)
              if element[1] is not None:
                _snake_case						=							element[1]
          elif first_field is not None:
            _snake_case						=							first_field
        else:
          for field in class_fields:
            _snake_case						=							getattr(self      , field.name		)
            if v is not None:
              _snake_case						=							v



      def __delitem__(      self				:    Optional[int]      , *A__				:    str      , **A__				:    Optional[int]		)   ->					str:
        raise Exception(f"""You cannot use ``__delitem__`` on a {self.__class__.__name__} instance."""		)



      def 						UpperCamelCase_		(      self				:    List[Any]      , *A__				:    Any      , **A__				:    Any		)   ->					Optional[int]:
        raise Exception(f"""You cannot use ``setdefault`` on a {self.__class__.__name__} instance."""		)



      def 						UpperCamelCase_		(      self				:    List[str]      , *A__				:    Dict      , **A__				:    Dict		)   ->					Dict:
        raise Exception(f"""You cannot use ``pop`` on a {self.__class__.__name__} instance."""		)



      def 						UpperCamelCase_		(      self				:    Tuple      , *A__				:    Any      , **A__				:    int		)   ->					str:
        raise Exception(f"""You cannot use ``update`` on a {self.__class__.__name__} instance."""		)



      def __getitem__(      self				:    str      , A__				:    Optional[Any]		)   ->					Tuple:






        if isinstance(lowerCamelCase__      , lowerCamelCase__		):
          _snake_case						=							dict(self.items()		)
          return inner_dict[k]
        else:
          return self.to_tuple()[k]



      def __setattr__(      self				:    List[Any]      , A__				:    Optional[int]      , A__				:    Optional[Any]		)   ->					Optional[Any]:






        if name in self.keys() and value is not None:
          # Don't call self.__setitem__ to avoid recursion errors
          super().__setitem__(lowerCamelCase__      , lowerCamelCase__		)
        super().__setattr__(lowerCamelCase__      , lowerCamelCase__		)



      def __setitem__(      self				:    str      , A__				:    Optional[int]      , A__				:    Tuple		)   ->					List[str]:
        super().__setitem__(lowerCamelCase__      , lowerCamelCase__		)
        # Don't call self.__setattr__ to avoid recursion errors
        super().__setattr__(lowerCamelCase__      , lowerCamelCase__		)







      def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					Tuple[Any]:
        return tuple(self[k] for k in self.keys()		)



class   lowercase_   (  __lowerCAmelCase       ,							__lowerCAmelCase					):



      @classmethod
      def 						UpperCamelCase_		(      cls				:    List[Any]      , A__				:    List[Any]		)   ->					List[Any]:
        raise ValueError(
            f"""{value} is not a valid {cls.__name__}, please select one of {list(cls._valueamember_map_.keys()		)}"""		)



class   lowercase_   (  __lowerCAmelCase					):
      UpperCamelCase_      :		List[Any]										=    "longest"
      UpperCamelCase_      :		str										=    "max_length"
      UpperCamelCase_      :		Any										=    "do_not_pad"



class   lowercase_   (  __lowerCAmelCase					):
      UpperCamelCase_      :		str										=    "pt"
      UpperCamelCase_      :		List[Any]										=    "tf"
      UpperCamelCase_      :		Union[str, Any]										=    "np"
      UpperCamelCase_      :		Optional[Any]										=    "jax"



class   lowercase_   :



      def __init__(      self				:    Optional[int]      , A__				:    List[ContextManager]		)   ->					List[Any]:
        _snake_case						=							context_managers
        _snake_case						=							ExitStack()



      def __enter__(      self				:    List[str]		)   ->					List[str]:






        for context_manager in self.context_managers:
          self.stack.enter_context(lowerCamelCase__		)







      def __exit__(      self				:    Tuple      , *A__				:    Tuple      , **A__				:    List[str]		)   ->					Dict:
        self.stack.__exit__(*lowerCamelCase__      , **lowerCamelCase__		)



def 				snake_case_(_UpperCamelCase					)							->     Tuple:



  """simple docstring"""






  _snake_case						=							infer_framework(_UpperCamelCase					)
  if framework == "tf":
    _snake_case						=							inspect.signature(model_class.call					)  # TensorFlow models
  elif framework == "pt":
    _snake_case						=							inspect.signature(model_class.forward					)  # PyTorch models
  else:
    _snake_case						=							inspect.signature(model_class.__call__					)  # Flax models

  for p in signature.parameters:
    if p == "return_loss" and signature.parameters[p].default is True:
      return True

  return False



def 				snake_case_(_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  _snake_case						=							model_class.__name__
  _snake_case						=							infer_framework(_UpperCamelCase					)
  if framework == "tf":
    _snake_case						=							inspect.signature(model_class.call					)  # TensorFlow models
  elif framework == "pt":
    _snake_case						=							inspect.signature(model_class.forward					)  # PyTorch models
  else:
    _snake_case						=							inspect.signature(model_class.__call__					)  # Flax models

  if "QuestionAnswering" in model_name:
    return [p for p in signature.parameters if "label" in p or p in ("start_positions", "end_positions")]
  else:
    return [p for p in signature.parameters if "label" in p]



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase = ""			,	_UpperCamelCase = "."					)							->     str:



  """simple docstring"""






  def _flatten_dict(_UpperCamelCase			,	_UpperCamelCase=""			,	_UpperCamelCase="."					):
    for k, v in d.items():
      _snake_case						=							str(_UpperCamelCase					) + delimiter + str(_UpperCamelCase					) if parent_key else k
      if v and isinstance(_UpperCamelCase			,	_UpperCamelCase					):
        yield from flatten_dict(_UpperCamelCase			,	_UpperCamelCase			,	delimiter=_UpperCamelCase					).items()
      else:
        yield key, v

  return dict(_flatten_dict(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)					)



@contextmanager
def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase = False					)							->     Union[str, Any]:



  """simple docstring"""






  if use_temp_dir:
    with tempfile.TemporaryDirectory() as tmp_dir:
      yield tmp_dir
  else:
    yield working_dir



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase=None					)							->     Optional[int]:



  """simple docstring"""






  if is_numpy_array(_UpperCamelCase					):
    return np.transpose(_UpperCamelCase			,	axes=_UpperCamelCase					)
  elif is_torch_tensor(_UpperCamelCase					):
    return array.T if axes is None else array.permute(*_UpperCamelCase					)
  elif is_tf_tensor(_UpperCamelCase					):
    import tensorflow as tf

    return tf.transpose(_UpperCamelCase			,	perm=_UpperCamelCase					)
  elif is_jax_tensor(_UpperCamelCase					):
    return jnp.transpose(_UpperCamelCase			,	axes=_UpperCamelCase					)
  else:
    raise ValueError(F"""Type not supported for transpose: {type(_UpperCamelCase					)}."""					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  if is_numpy_array(_UpperCamelCase					):
    return np.reshape(_UpperCamelCase			,	_UpperCamelCase					)
  elif is_torch_tensor(_UpperCamelCase					):
    return array.reshape(*_UpperCamelCase					)
  elif is_tf_tensor(_UpperCamelCase					):
    import tensorflow as tf

    return tf.reshape(_UpperCamelCase			,	_UpperCamelCase					)
  elif is_jax_tensor(_UpperCamelCase					):
    return jnp.reshape(_UpperCamelCase			,	_UpperCamelCase					)
  else:
    raise ValueError(F"""Type not supported for reshape: {type(_UpperCamelCase					)}."""					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase=None					)							->     Union[str, Any]:



  """simple docstring"""






  if is_numpy_array(_UpperCamelCase					):
    return np.squeeze(_UpperCamelCase			,	axis=_UpperCamelCase					)
  elif is_torch_tensor(_UpperCamelCase					):
    return array.squeeze() if axis is None else array.squeeze(dim=_UpperCamelCase					)
  elif is_tf_tensor(_UpperCamelCase					):
    import tensorflow as tf

    return tf.squeeze(_UpperCamelCase			,	axis=_UpperCamelCase					)
  elif is_jax_tensor(_UpperCamelCase					):
    return jnp.squeeze(_UpperCamelCase			,	axis=_UpperCamelCase					)
  else:
    raise ValueError(F"""Type not supported for squeeze: {type(_UpperCamelCase					)}."""					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     Tuple:



  """simple docstring"""






  if is_numpy_array(_UpperCamelCase					):
    return np.expand_dims(_UpperCamelCase			,	_UpperCamelCase					)
  elif is_torch_tensor(_UpperCamelCase					):
    return array.unsqueeze(dim=_UpperCamelCase					)
  elif is_tf_tensor(_UpperCamelCase					):
    import tensorflow as tf

    return tf.expand_dims(_UpperCamelCase			,	axis=_UpperCamelCase					)
  elif is_jax_tensor(_UpperCamelCase					):
    return jnp.expand_dims(_UpperCamelCase			,	axis=_UpperCamelCase					)
  else:
    raise ValueError(F"""Type not supported for expand_dims: {type(_UpperCamelCase					)}."""					)



def 				snake_case_(_UpperCamelCase					)							->     Union[str, Any]:



  """simple docstring"""






  if is_numpy_array(_UpperCamelCase					):
    return np.size(_UpperCamelCase					)
  elif is_torch_tensor(_UpperCamelCase					):
    return array.numel()
  elif is_tf_tensor(_UpperCamelCase					):
    import tensorflow as tf

    return tf.size(_UpperCamelCase					)
  elif is_jax_tensor(_UpperCamelCase					):
    return array.size
  else:
    raise ValueError(F"""Type not supported for expand_dims: {type(_UpperCamelCase					)}."""					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     str:



  """simple docstring"""






  for key, value in auto_map.items():
    if isinstance(_UpperCamelCase			,	(tuple, list)					):
      _snake_case						=							[F"""{repo_id}--{v}""" if (v is not None and '''--''' not in v) else v for v in value]
    elif value is not None and "--" not in value:
      _snake_case						=							F"""{repo_id}--{value}"""

  return auto_map



def 				snake_case_(_UpperCamelCase					)							->     int:



  """simple docstring"""






  for base_class in inspect.getmro(_UpperCamelCase					):
    _snake_case						=							base_class.__module__
    _snake_case						=							base_class.__name__
    if module.startswith('''tensorflow'''					) or module.startswith('''keras'''					) or name == "TFPreTrainedModel":
      return "tf"
    elif module.startswith('''torch'''					) or name == "PreTrainedModel":
      return "pt"
    elif module.startswith('''flax'''					) or module.startswith('''jax'''					) or name == "FlaxPreTrainedModel":
      return "flax"
  else:
    raise TypeError(F"""Could not infer framework from class {model_class}."""					)




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class   lowercase_   (  __lowercase       ,							unittest.TestCase					):
      UpperCamelCase_      :		Union[str, Any]										=    CLIPTokenizer
      UpperCamelCase_      :		Optional[int]										=    CLIPTokenizerFast
      UpperCamelCase_      :		Dict										=    True
      UpperCamelCase_      :		Union[str, Any]										=    {}
      UpperCamelCase_      :		Optional[Any]										=    False



      def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					Dict:
        super().setUp()

        # fmt: off
        _snake_case						=							['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
        # fmt: on
        _snake_case						=							dict(zip(A__      , range(len(A__		)		)		)		)
        _snake_case						=							['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
        _snake_case						=							{'''unk_token''': '''<unk>'''}

        _snake_case						=							os.path.join(self.tmpdirname      , VOCAB_FILES_NAMES['''vocab_file''']		)
        _snake_case						=							os.path.join(self.tmpdirname      , VOCAB_FILES_NAMES['''merges_file''']		)
        with open(self.vocab_file      , '''w'''      , encoding='''utf-8'''		) as fp:
          fp.write(json.dumps(A__		) + '''\n'''		)
        with open(self.merges_file      , '''w'''      , encoding='''utf-8'''		) as fp:
          fp.write('''\n'''.join(A__		)		)



      def 						UpperCamelCase_		(      self				:    List[Any]      , **A__				:    int		)   ->					Union[str, Any]:
        kwargs.update(self.special_tokens_map		)
        return CLIPTokenizer.from_pretrained(self.tmpdirname      , **A__		)



      def 						UpperCamelCase_		(      self				:    Any      , **A__				:    Tuple		)   ->					Optional[Any]:
        kwargs.update(self.special_tokens_map		)
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname      , **A__		)



      def 						UpperCamelCase_		(      self				:    Optional[Any]      , A__				:    str		)   ->					str:
        _snake_case						=							'''lower newer'''
        _snake_case						=							'''lower newer'''
        return input_text, output_text



      def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					Optional[int]:
        _snake_case						=							CLIPTokenizer(self.vocab_file      , self.merges_file      , **self.special_tokens_map		)
        _snake_case						=							'''lower newer'''
        _snake_case						=							['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
        _snake_case						=							tokenizer.tokenize(A__		)
        self.assertListEqual(A__      , A__		)

        _snake_case						=							tokens + [tokenizer.unk_token]
        _snake_case						=							[10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A__		)      , A__		)



      @require_ftfy
      def 						UpperCamelCase_		(      self				:    Any		)   ->					Optional[int]:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
            _snake_case						=							self.tokenizer_class.from_pretrained(A__      , **A__		)
            _snake_case						=							self.rust_tokenizer_class.from_pretrained(A__      , **A__		)

            _snake_case						=							'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
            _snake_case						=							tokenizer_s.tokenize(A__		)
            _snake_case						=							tokenizer_r.tokenize(A__		)

            self.assertListEqual(A__      , A__		)

            # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
            # with Tilde) encoded in 2 different ways
            _snake_case						=							'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
            _snake_case						=							tokenizer_s.tokenize(A__		)
            _snake_case						=							tokenizer_r.tokenize(A__		)

            self.assertListEqual(A__      , A__		)

            # Test that the tokenization is identical on unicode of space type
            _snake_case						=							[
                '''\u0009''',  # (horizontal tab, '\t')
                '''\u000B''',  # (vertical tab)
                '''\u000C''',  # (form feed)
                '''\u0020''',  # (space, ' ')
                '''\u200E''',  # (left-to-right mark):w
                '''\u200F''',  # (right-to-left mark)
            ]
            for unicode_seq in spaces_unicodes:
              _snake_case						=							tokenizer_s.tokenize(A__		)
              _snake_case						=							tokenizer_r.tokenize(A__		)

              self.assertListEqual(A__      , A__		)

            # Test that the tokenization is identical on unicode of line break type
            _snake_case						=							[
                '''\u000A''',  # (line feed, '\n')
                '''\r\n''',  # (carriage return and line feed, '\r\n')
                '''\u000D''',  # (carriage return, '\r')
                '''\r''',  # (carriage return, '\r')
                '''\u000D''',  # (carriage return, '\r')
                '''\u2028''',  # (line separator)
                '''\u2029''',  # (paragraph separator)
                # "\u0085", # (next line)
            ]

            # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
            # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
            # space (and thus into an empty list).

            for unicode_seq in line_break_unicodes:
              _snake_case						=							tokenizer_s.tokenize(A__		)
              _snake_case						=							tokenizer_r.tokenize(A__		)

              self.assertListEqual(A__      , A__		)



      def 						UpperCamelCase_		(      self				:    List[Any]		)   ->					Optional[Any]:
        # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
            _snake_case						=							'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
            _snake_case						=							f"""{text_of_1_token} {text_of_1_token}"""

            _snake_case						=							self.rust_tokenizer_class.from_pretrained(
                A__      , use_fast=A__      , )
            _snake_case						=							tokenizer_r(A__      , return_offsets_mapping=A__      , add_special_tokens=A__		)
            self.assertEqual(encoding.offset_mapping[0]      , (0, len(A__		))		)
            self.assertEqual(
                encoding.offset_mapping[1]      , (len(A__		) + 1, len(A__		) + 1 + len(A__		))      , )

            _snake_case						=							f""" {text}"""

            _snake_case						=							self.rust_tokenizer_class.from_pretrained(
                A__      , use_fast=A__      , )
            _snake_case						=							tokenizer_r(A__      , return_offsets_mapping=A__      , add_special_tokens=A__		)
            self.assertEqual(encoding.offset_mapping[0]      , (1, 1 + len(A__		))		)
            self.assertEqual(
                encoding.offset_mapping[1]      , (1 + len(A__		) + 1, 1 + len(A__		) + 1 + len(A__		))      , )



      def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					Optional[Any]:
        # Test related to the breaking change introduced in transformers v4.17.0
        # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
        with self.assertRaises(A__		) as context:
          self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''		)

        self.assertTrue(
            context.exception.args[0].startswith(
                '''The `backend_tokenizer` provided does not match the expected format.'''		)		)



      @require_ftfy
      def 						UpperCamelCase_		(      self				:    Dict		)   ->					Union[str, Any]:
        super().test_tokenization_python_rust_equals()







      def 						UpperCamelCase_		(      self				:    str		)   ->					Optional[int]:
        # CLIP always lower cases letters
        pass

