




"""simple docstring"""







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    SCREAMING_SNAKE_CASE__       (       snake_case_    ,					unittest.TestCase       ):
						_a         =    CLIPTokenizer
						_a         =    CLIPTokenizerFast
						_a         =    True
						_a         =    {}
						_a         =    False


						def 	__lowercase			(				self					:      int       ):
										super().setUp()

										# fmt: off
										lowerCAmelCase						=					['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
										# fmt: on
										lowerCAmelCase						=					dict(zip(__lowercase    ,    range(len(__lowercase       )       )       )       )
										lowerCAmelCase						=					['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
										lowerCAmelCase						=					{'unk_token': '<unk>'}

										lowerCAmelCase						=					os.path.join(self.tmpdirname    ,    VOCAB_FILES_NAMES["""vocab_file"""]       )
										lowerCAmelCase						=					os.path.join(self.tmpdirname    ,    VOCAB_FILES_NAMES["""merges_file"""]       )
										with open(self.vocab_file    ,    """w"""    ,    encoding="""utf-8"""       ) as fp:
														fp.write(json.dumps(__lowercase       ) + """\n"""       )
										with open(self.merges_file    ,    """w"""    ,    encoding="""utf-8"""       ) as fp:
														fp.write("""\n""".join(__lowercase       )       )


						def 	__lowercase			(				self					:      Union[str, Any]    ,    **lowerCAmelCase					:      Any       ):
										kwargs.update(self.special_tokens_map       )
										return CLIPTokenizer.from_pretrained(self.tmpdirname    ,    **__lowercase       )


						def 	__lowercase			(				self					:      int    ,    **lowerCAmelCase					:      Any       ):
										kwargs.update(self.special_tokens_map       )
										return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,    **__lowercase       )


						def 	__lowercase			(				self					:      Dict    ,    lowerCAmelCase					:      int       ):
										lowerCAmelCase						=					'lower newer'
										lowerCAmelCase						=					'lower newer'
										return input_text, output_text


						def 	__lowercase			(				self					:      List[Any]       ):
										lowerCAmelCase						=					CLIPTokenizer(self.vocab_file    ,    self.merges_file    ,    **self.special_tokens_map       )
										lowerCAmelCase						=					'lower newer'
										lowerCAmelCase						=					['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
										lowerCAmelCase						=					tokenizer.tokenize(__lowercase       )
										self.assertListEqual(__lowercase    ,    __lowercase       )

										lowerCAmelCase						=					tokens + [tokenizer.unk_token]
										lowerCAmelCase						=					[10, 2, 16, 9, 3, 2, 16, 20]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase       )    ,    __lowercase       )


						@require_ftfy
						def 	__lowercase			(				self					:      int       ):







										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																		lowerCAmelCase						=					self.tokenizer_class.from_pretrained(__lowercase    ,    **__lowercase       )
																		lowerCAmelCase						=					self.rust_tokenizer_class.from_pretrained(__lowercase    ,    **__lowercase       )

																		lowerCAmelCase						=					'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
																		lowerCAmelCase						=					tokenizer_s.tokenize(__lowercase       )
																		lowerCAmelCase						=					tokenizer_r.tokenize(__lowercase       )

																		self.assertListEqual(__lowercase    ,    __lowercase       )

																		# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																		# with Tilde) encoded in 2 different ways
																		lowerCAmelCase						=					'xa\u0303y' + ' ' + 'x\xe3y'
																		lowerCAmelCase						=					tokenizer_s.tokenize(__lowercase       )
																		lowerCAmelCase						=					tokenizer_r.tokenize(__lowercase       )

																		self.assertListEqual(__lowercase    ,    __lowercase       )

																		# Test that the tokenization is identical on unicode of space type
																		lowerCAmelCase						=					[
																		    '\u0009',  # (horizontal tab, '\t')
																		    '\u000B',  # (vertical tab)
																		    '\u000C',  # (form feed)
																		    '\u0020',  # (space, ' ')
																		    '\u200E',  # (left-to-right mark):w
																		    '\u200F',  # (right-to-left mark)
																		]
																		for unicode_seq in spaces_unicodes:
																						lowerCAmelCase						=					tokenizer_s.tokenize(__lowercase       )
																						lowerCAmelCase						=					tokenizer_r.tokenize(__lowercase       )

																						self.assertListEqual(__lowercase    ,    __lowercase       )

																		# Test that the tokenization is identical on unicode of line break type
																		lowerCAmelCase						=					[
																		    '\u000A',  # (line feed, '\n')
																		    '\r\n',  # (carriage return and line feed, '\r\n')
																		    '\u000D',  # (carriage return, '\r')
																		    '\r',  # (carriage return, '\r')
																		    '\u000D',  # (carriage return, '\r')
																		    '\u2028',  # (line separator)
																		    '\u2029',  # (paragraph separator)
																		    # "\u0085", # (next line)
																		]

																		# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																		# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																		# space (and thus into an empty list).

																		for unicode_seq in line_break_unicodes:
																						lowerCAmelCase						=					tokenizer_s.tokenize(__lowercase       )
																						lowerCAmelCase						=					tokenizer_r.tokenize(__lowercase       )

																						self.assertListEqual(__lowercase    ,    __lowercase       )


						def 	__lowercase			(				self					:      Tuple       ):







										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																		lowerCAmelCase						=					'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
																		lowerCAmelCase						=					f'''{text_of_1_token} {text_of_1_token}'''

																		lowerCAmelCase						=					self.rust_tokenizer_class.from_pretrained(
																		    __lowercase    ,    use_fast=__lowercase    ,    )
																		lowerCAmelCase						=					tokenizer_r(__lowercase    ,    return_offsets_mapping=__lowercase    ,    add_special_tokens=__lowercase       )
																		self.assertEqual(encoding.offset_mapping[0]    ,    (0, len(__lowercase       ))       )
																		self.assertEqual(
																		    encoding.offset_mapping[1]    ,    (len(__lowercase       ) + 1, len(__lowercase       ) + 1 + len(__lowercase       ))    ,    )

																		lowerCAmelCase						=					f''' {text}'''

																		lowerCAmelCase						=					self.rust_tokenizer_class.from_pretrained(
																		    __lowercase    ,    use_fast=__lowercase    ,    )
																		lowerCAmelCase						=					tokenizer_r(__lowercase    ,    return_offsets_mapping=__lowercase    ,    add_special_tokens=__lowercase       )
																		self.assertEqual(encoding.offset_mapping[0]    ,    (1, 1 + len(__lowercase       ))       )
																		self.assertEqual(
																		    encoding.offset_mapping[1]    ,    (1 + len(__lowercase       ) + 1, 1 + len(__lowercase       ) + 1 + len(__lowercase       ))    ,    )


						def 	__lowercase			(				self					:      Any       ):







										with self.assertRaises(__lowercase       ) as context:
														self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""       )

										self.assertTrue(
										    context.exception.args[0].startswith(
										        """The `backend_tokenizer` provided does not match the expected format."""       )       )


						@require_ftfy
						def 	__lowercase			(				self					:      Union[str, Any]       ):
										super().test_tokenization_python_rust_equals()




						def 	__lowercase			(				self					:      Optional[int]       ):
										pass









'''simple docstring'''
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_			:  int										=   logging.get_logger(__name__)

UpperCAmelCase_			:  List[Any]										=   {
    'facebook/deit-base-distilled-patch16-224': (
        'https://huggingface.co/facebook/deit-base-patch16-224/resolve/main/config.json'
    ),
    # See all DeiT models at https://huggingface.co/models?filter=deit
}

class     _lowerCamelCase       (     snake_case_					):




 '''simple docstring'''
 __lowercase					:		Union[str, Any]     				=    '''deit'''


 def __init__(   self   ,   __lowercase=768   ,   __lowercase=12   ,   __lowercase=12   ,   __lowercase=3_072   ,   __lowercase="gelu"   ,   __lowercase=0.0   ,   __lowercase=0.0   ,   __lowercase=0.0_2   ,   __lowercase=1E-12   ,   __lowercase=224   ,   __lowercase=16   ,   __lowercase=3   ,   __lowercase=True   ,   __lowercase=16   ,   **__lowercase   ,   ):

  """simple docstring"""







  super().__init__(**__lowercase    )

  __A	:		Union[str, Any]	  =      hidden_size
  __A	:		int	  =      num_hidden_layers
  __A	:		str	  =      num_attention_heads
  __A	:		Optional[int]	  =      intermediate_size
  __A	:		List[str]	  =      hidden_act
  __A	:		str	  =      hidden_dropout_prob
  __A	:		str	  =      attention_probs_dropout_prob
  __A	:		int	  =      initializer_range
  __A	:		Tuple	  =      layer_norm_eps
  __A	:		List[Any]	  =      image_size
  __A	:		Dict	  =      patch_size
  __A	:		Optional[Any]	  =      num_channels
  __A	:		List[Any]	  =      qkv_bias
  __A	:		str	  =      encoder_stride


class     _lowerCamelCase       (     snake_case_					):




 '''simple docstring'''
 __lowercase					:		Optional[int]     				=    version.parse('''1.11'''					)


 @property
 def 			snake_case__ (   self    ):

  """simple docstring"""







  return OrderedDict(
      [
          ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
      ]    )







 @property
 def 			snake_case__ (   self    ):

  """simple docstring"""







  return 1E-4






