






"""simple docstring"""





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 							snake_case							(			a__      ,  unittest.TestCase ):

      '''simple docstring'''



      _A     :       str         =       CLIPTokenizer
      _A     :       Tuple         =       CLIPTokenizerFast
      _A     :       List[str]         =       True
      _A     :       int         =       {}
      _A     :       int         =       False

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        super().setUp()

        # fmt: off
        __UpperCAmelCase    :						Dict       =						["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
        # fmt: on
        __UpperCAmelCase    :						Any       =						dict(zip(lowerCAmelCase__						,				range(len(lowerCAmelCase__ ) ) ) )
        __UpperCAmelCase    :						Optional[int]       =						["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
        __UpperCAmelCase    :						List[str]       =						{"unk_token": "<unk>"}

        __UpperCAmelCase    :						Dict       =						os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['''vocab_file'''] )
        __UpperCAmelCase    :						List[str]       =						os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['''merges_file'''] )
        with open(self.vocab_file						,				'''w'''						,				encoding='''utf-8''' ) as fp:
          fp.write(json.dumps(lowerCAmelCase__ ) + '''\n''' )
        with open(self.merges_file						,				'''w'''						,				encoding='''utf-8''' ) as fp:
          fp.write('''\n'''.join(lowerCAmelCase__ ) )

      def       A_					(							self       :      List[str]						,				**__lowercase       :      int ):


        '''simple docstring'''




        kwargs.update(self.special_tokens_map )
        return CLIPTokenizer.from_pretrained(self.tmpdirname						,				**lowerCAmelCase__ )

      def       A_					(							self       :      Optional[Any]						,				**__lowercase       :      Dict ):


        '''simple docstring'''




        kwargs.update(self.special_tokens_map )
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,				**lowerCAmelCase__ )

      def       A_					(							self       :      Any						,				__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						"lower newer"
        __UpperCAmelCase    :						int       =						"lower newer"
        return input_text, output_text

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						CLIPTokenizer(self.vocab_file						,				self.merges_file						,				**self.special_tokens_map )
        __UpperCAmelCase    :						List[Any]       =						"lower newer"
        __UpperCAmelCase    :						Optional[int]       =						["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
        __UpperCAmelCase    :						List[str]       =						tokenizer.tokenize(lowerCAmelCase__ )
        self.assertListEqual(lowerCAmelCase__						,				lowerCAmelCase__ )

        __UpperCAmelCase    :						List[Any]       =						tokens + [tokenizer.unk_token]
        __UpperCAmelCase    :						List[Any]       =						[10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__ )						,				lowerCAmelCase__ )

      @require_ftfy
      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
            __UpperCAmelCase    :						List[str]       =						self.tokenizer_class.from_pretrained(lowerCAmelCase__						,				**lowerCAmelCase__ )
            __UpperCAmelCase    :						List[Any]       =						self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__						,				**lowerCAmelCase__ )

            __UpperCAmelCase    :						Any       =						"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
            __UpperCAmelCase    :						str       =						tokenizer_s.tokenize(lowerCAmelCase__ )
            __UpperCAmelCase    :						Union[str, Any]       =						tokenizer_r.tokenize(lowerCAmelCase__ )

            self.assertListEqual(lowerCAmelCase__						,				lowerCAmelCase__ )

            # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
            # with Tilde) encoded in 2 different ways
            __UpperCAmelCase    :						List[Any]       =						"xa\u0303y" + " " + "x\xe3y"
            __UpperCAmelCase    :						str       =						tokenizer_s.tokenize(lowerCAmelCase__ )
            __UpperCAmelCase    :						Optional[int]       =						tokenizer_r.tokenize(lowerCAmelCase__ )

            self.assertListEqual(lowerCAmelCase__						,				lowerCAmelCase__ )

            # Test that the tokenization is identical on unicode of space type
            __UpperCAmelCase    :						Any       =						[
                "\u0009",  # (horizontal tab, '\t')
                "\u000B",  # (vertical tab)
                "\u000C",  # (form feed)
                "\u0020",  # (space, ' ')
                "\u200E",  # (left-to-right mark):w
                "\u200F",  # (right-to-left mark)
            ]
            for unicode_seq in spaces_unicodes:
              __UpperCAmelCase    :						Any       =						tokenizer_s.tokenize(lowerCAmelCase__ )
              __UpperCAmelCase    :						Dict       =						tokenizer_r.tokenize(lowerCAmelCase__ )

              self.assertListEqual(lowerCAmelCase__						,				lowerCAmelCase__ )

            # Test that the tokenization is identical on unicode of line break type
            __UpperCAmelCase    :						Any       =						[
                "\u000A",  # (line feed, '\n')
                "\r\n",  # (carriage return and line feed, '\r\n')
                "\u000D",  # (carriage return, '\r')
                "\r",  # (carriage return, '\r')
                "\u000D",  # (carriage return, '\r')
                "\u2028",  # (line separator)
                "\u2029",  # (paragraph separator)
                # "\u0085", # (next line)
            ]

            # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
            # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
            # space (and thus into an empty list).

            for unicode_seq in line_break_unicodes:
              __UpperCAmelCase    :						List[str]       =						tokenizer_s.tokenize(lowerCAmelCase__ )
              __UpperCAmelCase    :						Tuple       =						tokenizer_r.tokenize(lowerCAmelCase__ )

              self.assertListEqual(lowerCAmelCase__						,				lowerCAmelCase__ )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
            __UpperCAmelCase    :						Tuple       =						"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
            __UpperCAmelCase    :						Tuple       =						f'''{text_of_1_token} {text_of_1_token}'''

            __UpperCAmelCase    :						Union[str, Any]       =						self.rust_tokenizer_class.from_pretrained(
                lowerCAmelCase__						,				use_fast=lowerCAmelCase__						,				)
            __UpperCAmelCase    :						str       =						tokenizer_r(lowerCAmelCase__						,				return_offsets_mapping=lowerCAmelCase__						,				add_special_tokens=lowerCAmelCase__ )
            self.assertEqual(encoding.offset_mapping[0]						,				(0, len(lowerCAmelCase__ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(len(lowerCAmelCase__ ) + 1, len(lowerCAmelCase__ ) + 1 + len(lowerCAmelCase__ ))						,				)

            __UpperCAmelCase    :						Optional[Any]       =						f''' {text}'''

            __UpperCAmelCase    :						Optional[int]       =						self.rust_tokenizer_class.from_pretrained(
                lowerCAmelCase__						,				use_fast=lowerCAmelCase__						,				)
            __UpperCAmelCase    :						Any       =						tokenizer_r(lowerCAmelCase__						,				return_offsets_mapping=lowerCAmelCase__						,				add_special_tokens=lowerCAmelCase__ )
            self.assertEqual(encoding.offset_mapping[0]						,				(1, 1 + len(lowerCAmelCase__ )) )
            self.assertEqual(
                encoding.offset_mapping[1]						,				(1 + len(lowerCAmelCase__ ) + 1, 1 + len(lowerCAmelCase__ ) + 1 + len(lowerCAmelCase__ ))						,				)

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        with self.assertRaises(lowerCAmelCase__ ) as context:
          self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer''' )

        self.assertTrue(
            context.exception.args[0].startswith(
                '''The `backend_tokenizer` provided does not match the expected format.''' ) )

      @require_ftfy
      def       A_					(							self       :      Any ):


        '''simple docstring'''




        super().test_tokenization_python_rust_equals()







      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        pass






"""simple docstring"""





# flake8: noqa
# Lint as: python3
lowercase__				:Tuple				   =       [
    'VerificationMode',
    'Version',
    'disable_progress_bar',
    'enable_progress_bar',
    'is_progress_bar_enabled',
    'experimental',
]

from .info_utils import VerificationMode
from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
from .version import Version
from .experimental import experimental