





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class        __SCREAMING_SNAKE_CASE      (			lowerCAmelCase_					,unittest.TestCase ):
       _UpperCAmelCase				:  Optional[Any]			   =   CLIPTokenizer
       _UpperCAmelCase				:  Dict			   =   CLIPTokenizerFast
       _UpperCAmelCase				:  int			   =   True
       _UpperCAmelCase				:  Optional[Any]			   =   {}
       _UpperCAmelCase				:  int			   =   False







       def 						__lowerCamelCase   (				self					:     Optional[int]      )       ->List[Any]:
          super().setUp()

          # fmt: off
          lowerCamelCase__       :						Optional[Any]        =							['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
          # fmt: on
          lowerCamelCase__       :						Optional[int]        =							dict(zip(A		, range(len(A      )      )      )      )
          lowerCamelCase__       :						Dict        =							['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
          lowerCamelCase__       :						Dict        =							{'''unk_token''': '''<unk>'''}

          lowerCamelCase__       :						Tuple        =							os.path.join(self.tmpdirname		, VOCAB_FILES_NAMES['''vocab_file''']      )
          lowerCamelCase__       :						int        =							os.path.join(self.tmpdirname		, VOCAB_FILES_NAMES['''merges_file''']      )
          with open(self.vocab_file		, '''w'''		, encoding='''utf-8'''      ) as fp:
             fp.write(json.dumps(A      ) + '''\n'''      )
          with open(self.merges_file		, '''w'''		, encoding='''utf-8'''      ) as fp:
             fp.write('''\n'''.join(A      )      )







       def 						__lowerCamelCase   (				self					:     str		, **A					:     List[str]      )       ->List[str]:
          kwargs.update(self.special_tokens_map      )
          return CLIPTokenizer.from_pretrained(self.tmpdirname		, **A      )







       def 						__lowerCamelCase   (				self					:     Union[str, Any]		, **A					:     Union[str, Any]      )       ->List[Any]:
          kwargs.update(self.special_tokens_map      )
          return CLIPTokenizerFast.from_pretrained(self.tmpdirname		, **A      )







       def 						__lowerCamelCase   (				self					:     str		, A					:     List[Any]      )       ->Optional[int]:
          lowerCamelCase__       :						str        =							'''lower newer'''
          lowerCamelCase__       :						Tuple        =							'''lower newer'''
          return input_text, output_text







       def 						__lowerCamelCase   (				self					:     Tuple      )       ->Dict:
          lowerCamelCase__       :						List[Any]        =							CLIPTokenizer(self.vocab_file		, self.merges_file		, **self.special_tokens_map      )
          lowerCamelCase__       :						Any        =							'''lower newer'''
          lowerCamelCase__       :						Dict        =							['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
          lowerCamelCase__       :						List[str]        =							tokenizer.tokenize(A      )
          self.assertListEqual(A		, A      )

          lowerCamelCase__       :						str        =							tokens + [tokenizer.unk_token]
          lowerCamelCase__       :						Any        =							[1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(A      )		, A      )







       @require_ftfy
       def 						__lowerCamelCase   (				self					:     Union[str, Any]      )       ->Optional[int]:
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                lowerCamelCase__       :						Dict        =							self.tokenizer_class.from_pretrained(A		, **A      )
                lowerCamelCase__       :						int        =							self.rust_tokenizer_class.from_pretrained(A		, **A      )

                lowerCamelCase__       :						Any        =							'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                lowerCamelCase__       :						int        =							tokenizer_s.tokenize(A      )
                lowerCamelCase__       :						Tuple        =							tokenizer_r.tokenize(A      )

                self.assertListEqual(A		, A      )

                # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                # with Tilde) encoded in 2 different ways
                lowerCamelCase__       :						Tuple        =							'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                lowerCamelCase__       :						Optional[Any]        =							tokenizer_s.tokenize(A      )
                lowerCamelCase__       :						int        =							tokenizer_r.tokenize(A      )

                self.assertListEqual(A		, A      )

                # Test that the tokenization is identical on unicode of space type
                lowerCamelCase__       :						Tuple        =							[
                    '''\u0009''',  # (horizontal tab, '\t')
                    '''\u000B''',  # (vertical tab)
                    '''\u000C''',  # (form feed)
                    '''\u0020''',  # (space, ' ')
                    '''\u200E''',  # (left-to-right mark):w
                    '''\u200F''',  # (right-to-left mark)
                ]
                for unicode_seq in spaces_unicodes:
                   lowerCamelCase__       :						Optional[int]        =							tokenizer_s.tokenize(A      )
                   lowerCamelCase__       :						Dict        =							tokenizer_r.tokenize(A      )

                   self.assertListEqual(A		, A      )

                # Test that the tokenization is identical on unicode of line break type
                lowerCamelCase__       :						Optional[Any]        =							[
                    '''\u000A''',  # (line feed, '\n')
                    '''\r\n''',  # (carriage return and line feed, '\r\n')
                    '''\u000D''',  # (carriage return, '\r')
                    '''\r''',  # (carriage return, '\r')
                    '''\u000D''',  # (carriage return, '\r')
                    '''\u2028''',  # (line separator)
                    '''\u2029''',  # (paragraph separator)
                    # "\u0085", # (next line)
                ]

                # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                # space (and thus into an empty list).

                for unicode_seq in line_break_unicodes:
                   lowerCamelCase__       :						int        =							tokenizer_s.tokenize(A      )
                   lowerCamelCase__       :						Optional[int]        =							tokenizer_r.tokenize(A      )

                   self.assertListEqual(A		, A      )







       def 						__lowerCamelCase   (				self					:     Optional[Any]      )       ->Optional[Any]:
          # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                lowerCamelCase__       :						Any        =							'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                lowerCamelCase__       :						List[str]        =							F"{text_of_1_token} {text_of_1_token}"

                lowerCamelCase__       :						List[Any]        =							self.rust_tokenizer_class.from_pretrained(
                    A		, use_fast=A		, )
                lowerCamelCase__       :						List[Any]        =							tokenizer_r(A		, return_offsets_mapping=A		, add_special_tokens=A      )
                self.assertEqual(encoding.offset_mapping[0]		, (0, len(A      ))      )
                self.assertEqual(
                    encoding.offset_mapping[1]		, (len(A      ) + 1, len(A      ) + 1 + len(A      ))		, )

                lowerCamelCase__       :						List[Any]        =							F" {text}"

                lowerCamelCase__       :						int        =							self.rust_tokenizer_class.from_pretrained(
                    A		, use_fast=A		, )
                lowerCamelCase__       :						Optional[int]        =							tokenizer_r(A		, return_offsets_mapping=A		, add_special_tokens=A      )
                self.assertEqual(encoding.offset_mapping[0]		, (1, 1 + len(A      ))      )
                self.assertEqual(
                    encoding.offset_mapping[1]		, (1 + len(A      ) + 1, 1 + len(A      ) + 1 + len(A      ))		, )







       def 						__lowerCamelCase   (				self					:     Any      )       ->List[Any]:
          # Test related to the breaking change introduced in transformers v4.17.0
          # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
          with self.assertRaises(A      ) as context:
             self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''      )

          self.assertTrue(
              context.exception.args[0].startswith(
                  '''The `backend_tokenizer` provided does not match the expected format.'''      )      )







       @require_ftfy
       def 						__lowerCamelCase   (				self					:     List[str]      )       ->Union[str, Any]:
          super().test_tokenization_python_rust_equals()







       def 						__lowerCamelCase   (				self					:     int      )       ->Optional[int]:
          # CLIP always lower cases letters
          pass










def 	_a						(   UpperCAmelCase					)    ->    int:







   """simple docstring"""

   if not isinstance(UpperCAmelCase      ,  UpperCAmelCase					):
      raise TypeError('''only integers accepted as input'''					)
   else:
      lowerCamelCase__       :						Any        =							str(abs(UpperCAmelCase					)					)
      lowerCamelCase__       :						Union[str, Any]        =							[list(UpperCAmelCase					) for char in range(len(UpperCAmelCase					)					)]
      for index in range(len(UpperCAmelCase					)					):
         num_transpositions[index].pop(UpperCAmelCase					)
      return max(
          int(''''''.join(list(UpperCAmelCase					)					)					) for transposition in num_transpositions					)


if __name__ == "__main__":
       __import__('doctest').testmod()



