




'''simple docstring'''







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class     _a						(			__a							,							unittest.TestCase				):
       __a       :  Dict            =      CLIPTokenizer
       __a       :  Any            =      CLIPTokenizerFast
       __a       :  Optional[Any]            =      True
       __a       :  List[str]            =      {}
       __a       :  List[str]            =      False
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           super().setUp()

           # fmt: off
           UpperCAmelCase     =	['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
           # fmt: on
           UpperCAmelCase     =	dict(zip(lowercase     ,       range(len(lowercase       )       )       )       )
           UpperCAmelCase     =	['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
           UpperCAmelCase     =	{'''unk_token''': '''<unk>'''}

           UpperCAmelCase     =	os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['''vocab_file''']       )
           UpperCAmelCase     =	os.path.join(self.tmpdirname     ,       VOCAB_FILES_NAMES['''merges_file''']       )
           with open(self.vocab_file     ,       '''w'''     ,       encoding='''utf-8'''       ) as fp:
               fp.write(json.dumps(lowercase       ) + '''\n'''       )
           with open(self.merges_file     ,       '''w'''     ,       encoding='''utf-8'''       ) as fp:
               fp.write('''\n'''.join(lowercase       )       )
       def 					A   (			self		:			str     ,       **lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizer.from_pretrained(self.tmpdirname     ,       **lowercase       )
       def 					A   (			self		:			Any     ,       **lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname     ,       **lowercase       )
       def 					A   (			self		:			List[Any]     ,       lowercase		:			Union[str, Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	'''lower newer'''
           UpperCAmelCase     =	'''lower newer'''
           return input_text, output_text
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	CLIPTokenizer(self.vocab_file     ,       self.merges_file     ,       **self.special_tokens_map       )
           UpperCAmelCase     =	'''lower newer'''
           UpperCAmelCase     =	['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
           UpperCAmelCase     =	tokenizer.tokenize(lowercase       )
           self.assertListEqual(lowercase     ,       lowercase       )

           UpperCAmelCase     =	tokens + [tokenizer.unk_token]
           UpperCAmelCase     =	[10, 2, 16, 9, 3, 2, 16, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase       )     ,       lowercase       )
       @require_ftfy
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                   UpperCAmelCase     =	self.tokenizer_class.from_pretrained(lowercase     ,       **lowercase       )
                   UpperCAmelCase     =	self.rust_tokenizer_class.from_pretrained(lowercase     ,       **lowercase       )

                   UpperCAmelCase     =	'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                   UpperCAmelCase     =	tokenizer_s.tokenize(lowercase       )
                   UpperCAmelCase     =	tokenizer_r.tokenize(lowercase       )

                   self.assertListEqual(lowercase     ,       lowercase       )

                   # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                   # with Tilde) encoded in 2 different ways
                   UpperCAmelCase     =	'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                   UpperCAmelCase     =	tokenizer_s.tokenize(lowercase       )
                   UpperCAmelCase     =	tokenizer_r.tokenize(lowercase       )

                   self.assertListEqual(lowercase     ,       lowercase       )

                   # Test that the tokenization is identical on unicode of space type
                   UpperCAmelCase     =	[
                       '''\u0009''',  # (horizontal tab, '\t')
                       '''\u000B''',  # (vertical tab)
                       '''\u000C''',  # (form feed)
                       '''\u0020''',  # (space, ' ')
                       '''\u200E''',  # (left-to-right mark):w
                       '''\u200F''',  # (right-to-left mark)
                   ]
                   for unicode_seq in spaces_unicodes:
                       UpperCAmelCase     =	tokenizer_s.tokenize(lowercase       )
                       UpperCAmelCase     =	tokenizer_r.tokenize(lowercase       )

                       self.assertListEqual(lowercase     ,       lowercase       )

                   # Test that the tokenization is identical on unicode of line break type
                   UpperCAmelCase     =	[
                       '''\u000A''',  # (line feed, '\n')
                       '''\r\n''',  # (carriage return and line feed, '\r\n')
                       '''\u000D''',  # (carriage return, '\r')
                       '''\r''',  # (carriage return, '\r')
                       '''\u000D''',  # (carriage return, '\r')
                       '''\u2028''',  # (line separator)
                       '''\u2029''',  # (paragraph separator)
                       # "\u0085", # (next line)
                   ]

                   # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                   # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                   # space (and thus into an empty list).

                   for unicode_seq in line_break_unicodes:
                       UpperCAmelCase     =	tokenizer_s.tokenize(lowercase       )
                       UpperCAmelCase     =	tokenizer_r.tokenize(lowercase       )

                       self.assertListEqual(lowercase     ,       lowercase       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                   UpperCAmelCase     =	'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                   UpperCAmelCase     =	f"{text_of_1_token} {text_of_1_token}"

                   UpperCAmelCase     =	self.rust_tokenizer_class.from_pretrained(
                       lowercase     ,       use_fast=lowercase     ,       )
                   UpperCAmelCase     =	tokenizer_r(lowercase     ,       return_offsets_mapping=lowercase     ,       add_special_tokens=lowercase       )
                   self.assertEqual(encoding.offset_mapping[0]     ,       (0, len(lowercase       ))       )
                   self.assertEqual(
                       encoding.offset_mapping[1]     ,       (len(lowercase       ) + 1, len(lowercase       ) + 1 + len(lowercase       ))     ,       )

                   UpperCAmelCase     =	f" {text}"

                   UpperCAmelCase     =	self.rust_tokenizer_class.from_pretrained(
                       lowercase     ,       use_fast=lowercase     ,       )
                   UpperCAmelCase     =	tokenizer_r(lowercase     ,       return_offsets_mapping=lowercase     ,       add_special_tokens=lowercase       )
                   self.assertEqual(encoding.offset_mapping[0]     ,       (1, 1 + len(lowercase       ))       )
                   self.assertEqual(
                       encoding.offset_mapping[1]     ,       (1 + len(lowercase       ) + 1, 1 + len(lowercase       ) + 1 + len(lowercase       ))     ,       )
       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           with self.assertRaises(lowercase       ) as context:
               self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''       )

           self.assertTrue(
               context.exception.args[0].startswith(
                   '''The `backend_tokenizer` provided does not match the expected format.'''       )       )
       @require_ftfy
       def 					A   (			self		:			int       ):



           '''simple docstring'''

           super().test_tokenization_python_rust_equals()







       def 					A   (			self		:			int       ):



           '''simple docstring'''

           pass










'''simple docstring'''







import warnings

from ...utils import logging
from .image_processing_clip import CLIPImageProcessor


A									=logging.get_logger(__name__)


class     _a						(			__a				):
       def __init__(			self		:			List[str]     ,       *lowercase		:			List[str]     ,       **lowercase		:			Optional[int]       ):



           '''simple docstring'''

           warnings.warn(
               '''The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
               ''' use CLIPImageProcessor instead.'''     ,       lowercase     ,       )
           super().__init__(*lowercase     ,       **lowercase       )




