'''simple docstring'''






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class lowercase_       (		_UpperCamelCase    ,    unittest.TestCase			):







   """simple docstring"""

   __lowerCAmelCase         =   CLIPTokenizer
   __lowerCAmelCase         =   CLIPTokenizerFast
   __lowerCAmelCase         =   True
   __lowerCAmelCase         =   {}
   __lowerCAmelCase         =   False
   def 	__UpperCAmelCase   (      self      :			Dict     )  ->			Tuple:
        super().setUp()

        # fmt: off
        _A    		=  ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
        # fmt: on
        _A    		=  dict(zip(UpperCamelCase__, range(len(UpperCamelCase__     )     )     )     )
        _A    		=  ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
        _A    		=  {'unk_token': '<unk>'}

        _A    		=  os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']     )
        _A    		=  os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file']     )
        with open(self.vocab_file, 'w', encoding='utf-8'     ) as fp:
             fp.write(json.dumps(UpperCamelCase__     ) + '\n'     )
        with open(self.merges_file, 'w', encoding='utf-8'     ) as fp:
             fp.write('\n'.join(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			List[Any], **UpperCamelCase__      :			Optional[Any]     )  ->			int:
        kwargs.update(self.special_tokens_map     )
        return CLIPTokenizer.from_pretrained(self.tmpdirname, **UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			str, **UpperCamelCase__      :			List[str]     )  ->			Tuple:
        kwargs.update(self.special_tokens_map     )
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname, **UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			str     )  ->			Tuple:
        _A    		=  'lower newer'
        _A    		=  'lower newer'
        return input_text, output_text
   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			List[Any]:
        _A    		=  CLIPTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map     )
        _A    		=  'lower newer'
        _A    		=  ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
        _A    		=  tokenizer.tokenize(UpperCamelCase__     )
        self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

        _A    		=  tokens + [tokenizer.unk_token]
        _A    		=  [10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__     ), UpperCamelCase__     )
   @require_ftfy
   def 	__UpperCAmelCase   (      self      :			int     )  ->			Any:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                  _A    		=  self.tokenizer_class.from_pretrained(UpperCamelCase__, **UpperCamelCase__     )
                  _A    		=  self.rust_tokenizer_class.from_pretrained(UpperCamelCase__, **UpperCamelCase__     )

                  _A    		=  'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
                  _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                  _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                  self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

                  # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                  # with Tilde) encoded in 2 different ways
                  _A    		=  'xa\u0303y' + ' ' + 'x\xe3y'
                  _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                  _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                  self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

                  # Test that the tokenization is identical on unicode of space type
                  _A    		=  [
                      '\u0009',  # (horizontal tab, '\t')
                      '\u000B',  # (vertical tab)
                      '\u000C',  # (form feed)
                      '\u0020',  # (space, ' ')
                      '\u200E',  # (left-to-right mark):w
                      '\u200F',  # (right-to-left mark)
                  ]
                  for unicode_seq in spaces_unicodes:
                       _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                       _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                       self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

                  # Test that the tokenization is identical on unicode of line break type
                  _A    		=  [
                      '\u000A',  # (line feed, '\n')
                      '\r\n',  # (carriage return and line feed, '\r\n')
                      '\u000D',  # (carriage return, '\r')
                      '\r',  # (carriage return, '\r')
                      '\u000D',  # (carriage return, '\r')
                      '\u2028',  # (line separator)
                      '\u2029',  # (paragraph separator)
                      # "\u0085", # (next line)
                  ]

                  # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                  # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                  # space (and thus into an empty list).

                  for unicode_seq in line_break_unicodes:
                       _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                       _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                       self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			str:
        # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                  _A    		=  'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                  _A    		=  f'{text_of_1_token} {text_of_1_token}'

                  _A    		=  self.rust_tokenizer_class.from_pretrained(
                      UpperCamelCase__, use_fast=UpperCamelCase__, )
                  _A    		=  tokenizer_r(UpperCamelCase__, return_offsets_mapping=UpperCamelCase__, add_special_tokens=UpperCamelCase__     )
                  self.assertEqual(encoding.offset_mapping[0], (0, len(UpperCamelCase__     ))     )
                  self.assertEqual(
                      encoding.offset_mapping[1], (len(UpperCamelCase__     ) + 1, len(UpperCamelCase__     ) + 1 + len(UpperCamelCase__     )), )

                  _A    		=  f' {text}'

                  _A    		=  self.rust_tokenizer_class.from_pretrained(
                      UpperCamelCase__, use_fast=UpperCamelCase__, )
                  _A    		=  tokenizer_r(UpperCamelCase__, return_offsets_mapping=UpperCamelCase__, add_special_tokens=UpperCamelCase__     )
                  self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(UpperCamelCase__     ))     )
                  self.assertEqual(
                      encoding.offset_mapping[1], (1 + len(UpperCamelCase__     ) + 1, 1 + len(UpperCamelCase__     ) + 1 + len(UpperCamelCase__     )), )
   def 	__UpperCAmelCase   (      self      :			Optional[Any]     )  ->			Dict:
        # Test related to the breaking change introduced in transformers v4.17.0
        # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
        with self.assertRaises(UpperCamelCase__     ) as context:
             self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'     )

        self.assertTrue(
            context.exception.args[0].startswith(
                'The `backend_tokenizer` provided does not match the expected format.'     )     )
   @require_ftfy
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			List[Any]:
        super().test_tokenization_python_rust_equals()

   def 	__UpperCAmelCase   (      self      :			str     )  ->			str:
        # CLIP always lower cases letters
        pass










'''simple docstring'''

import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class   UpperCAmelCase      (			UpperCamelCase__							):
 __lowercase  						=							["""image_processor""", """tokenizer"""]
 __lowercase  						=							"""CLIPImageProcessor"""
 __lowercase  						=							("""CLIPTokenizer""", """CLIPTokenizerFast""")
 def __init__(							self :List[Any]		,			lowercase_ :List[str]=None		,			lowercase_ :Any=None		,			**lowercase_ :Union[str, Any]				)->				Optional[Any]:
     A__							=     None
     if "feature_extractor" in kwargs:
         warnings.warn(
             "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
             " instead."		,			lowercase_		,			)
         A__							=     kwargs.pop("feature_extractor"				)

     A__							=     image_processor if image_processor is not None else feature_extractor
     if image_processor is None:
         raise ValueError("You need to specify an `image_processor`."				)
     if tokenizer is None:
         raise ValueError("You need to specify a `tokenizer`."				)

     super().__init__(lowercase_		,			lowercase_				)
 def __call__(							self :Optional[Any]		,			lowercase_ :Optional[int]=None		,			lowercase_ :Dict=None		,			lowercase_ :List[Any]=None		,			**lowercase_ :Optional[Any]				)->				Union[str, Any]:

     if text is None and images is None:
         raise ValueError("You have to specify either text or images. Both cannot be none."				)

     if text is not None:
         A__							=     self.tokenizer(lowercase_		,			return_tensors=lowercase_		,			**lowercase_				)

     if images is not None:
         A__							=     self.image_processor(lowercase_		,			return_tensors=lowercase_		,			**lowercase_				)

     if text is not None and images is not None:
         A__							=     image_features.pixel_values
         return encoding
     elif text is not None:
         return encoding
     else:
         return BatchEncoding(data=dict(**lowercase_				)		,			tensor_type=lowercase_				)
 def 	UpperCAmelCase_				(							self :int		,			*lowercase_ :Tuple		,			**lowercase_ :Any				)->				Dict:
     return self.tokenizer.batch_decode(*lowercase_		,			**lowercase_				)
 def 	UpperCAmelCase_				(							self :Any		,			*lowercase_ :Any		,			**lowercase_ :Any				)->				List[str]:
     return self.tokenizer.decode(*lowercase_		,			**lowercase_				)
 @property
 def 	UpperCAmelCase_				(							self :Union[str, Any]				)->				Optional[int]:
     A__							=     self.tokenizer.model_input_names
     A__							=     self.image_processor.model_input_names
     return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names				)				)
 @property
 def 	UpperCAmelCase_				(							self :List[Any]				)->				str:
     warnings.warn(
         "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."		,			lowercase_		,			)
     return self.image_processor_class



 @property
 def 	UpperCAmelCase_				(							self :List[str]				)->				List[Any]:
     warnings.warn(
         "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."		,			lowercase_		,			)
     return self.image_processor



