





from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase:	Optional[int] = logging.get_logger(__name__)


__UpperCamelCase:	Any = {
    """nielsr/canine-s""": 2_0_4_8,
}

# Unicode defines 1,114,112 total “codepoints”
__UpperCamelCase:	str = 1_1_1_4_1_1_2

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
__UpperCamelCase:	Optional[Any] = 0

__UpperCamelCase:	List[Any] = 0xE_000
__UpperCamelCase:	Any = 0xE_001
__UpperCamelCase:	Union[str, Any] = 0xE_002
__UpperCamelCase:	Optional[int] = 0xE_003
__UpperCamelCase:	List[Any] = 0xE_004

# Maps special codepoints to human-readable names.
__UpperCamelCase:	Tuple = {
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: """[CLS]""",
    SEP: """[SEP]""",
    BOS: """[BOS]""",
    MASK: """[MASK]""",
    PAD: """[PAD]""",
    RESERVED: """[RESERVED]""",
}

# Maps special codepoint human-readable names to their codepoint values.
__UpperCamelCase:	str = {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}


class       __lowerCAmelCase					(       snake_case__      ):


	'''simple docstring'''





	_A													=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

	def __init__(						self:						Tuple,					lowerCamelCase_:						str=chr(_UpperCamelCase			),					lowerCamelCase_:						Dict=chr(_UpperCamelCase			),					lowerCamelCase_:						Optional[Any]=chr(_UpperCamelCase			),					lowerCamelCase_:						Dict=chr(_UpperCamelCase			),					lowerCamelCase_:						Dict=chr(_UpperCamelCase			),					lowerCamelCase_:						Tuple=chr(_UpperCamelCase			),					lowerCamelCase_:						Tuple=False,					lowerCamelCase_:						List[str]=2048,					**lowerCamelCase_:						Union[str, Any],					):
			lowercase__					:      Optional[Any]      =		AddedToken(_UpperCamelCase,					lstrip=_UpperCamelCase,					rstrip=_UpperCamelCase			) if isinstance(_UpperCamelCase,					_UpperCamelCase			) else bos_token
			lowercase__					:      Optional[int]      =		AddedToken(_UpperCamelCase,					lstrip=_UpperCamelCase,					rstrip=_UpperCamelCase			) if isinstance(_UpperCamelCase,					_UpperCamelCase			) else eos_token
			lowercase__					:      Any      =		AddedToken(_UpperCamelCase,					lstrip=_UpperCamelCase,					rstrip=_UpperCamelCase			) if isinstance(_UpperCamelCase,					_UpperCamelCase			) else sep_token
			lowercase__					:      List[Any]      =		AddedToken(_UpperCamelCase,					lstrip=_UpperCamelCase,					rstrip=_UpperCamelCase			) if isinstance(_UpperCamelCase,					_UpperCamelCase			) else cls_token
			lowercase__					:      List[str]      =		AddedToken(_UpperCamelCase,					lstrip=_UpperCamelCase,					rstrip=_UpperCamelCase			) if isinstance(_UpperCamelCase,					_UpperCamelCase			) else pad_token

			# Mask token behave like a normal word, i.e. include the space before it
			lowercase__					:      Optional[Any]      =		AddedToken(_UpperCamelCase,					lstrip=_UpperCamelCase,					rstrip=_UpperCamelCase			) if isinstance(_UpperCamelCase,					_UpperCamelCase			) else mask_token

			super().__init__(
			    bos_token=_UpperCamelCase,					eos_token=_UpperCamelCase,					sep_token=_UpperCamelCase,					cls_token=_UpperCamelCase,					pad_token=_UpperCamelCase,					mask_token=_UpperCamelCase,					add_prefix_space=_UpperCamelCase,					model_max_length=_UpperCamelCase,					**_UpperCamelCase,					)

			# Creates a mapping for looking up the IDs of special symbols.
			lowercase__					:      List[str]      =		{}
			for codepoint, name in SPECIAL_CODEPOINTS.items():
					lowercase__					:      Optional[Any]      =		codepoint

			# Creates a mapping for looking up the string forms of special symbol IDs.
			lowercase__					:      List[Any]      =		{
			    codepoint: name for name, codepoint in self._special_codepoints.items()
			}

			lowercase__					:      int      =		UNICODE_VOCAB_SIZE
			lowercase__					:      Tuple      =		len(self._special_codepoints			)

	@property
	def    snake_case__(						self:						List[Any]			):
			return self._unicode_vocab_size

	def    snake_case__(						self:						List[Any],					lowerCamelCase_:						Dict			):
			return list(_UpperCamelCase			)

	def    snake_case__(						self:						List[Any],					lowerCamelCase_:						Optional[int]			):


			try:
					return ord(_UpperCamelCase			)
			except TypeError:
					raise ValueError(F"""invalid token: \'{token}\'"""			)

	def    snake_case__(						self:						Tuple,					lowerCamelCase_:						str			):


			try:
					if index in SPECIAL_CODEPOINTS:
							return SPECIAL_CODEPOINTS[index]
					return chr(_UpperCamelCase			)
			except TypeError:
					raise ValueError(F"""invalid id: {index}"""			)

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						Optional[int]			):
			return "".join(_UpperCamelCase			)

	def    snake_case__(						self:						List[Any],					lowerCamelCase_:						Union[str, Any],					lowerCamelCase_:						Optional[Any] = None			):
			lowercase__					:      str      =		[self.sep_token_id]
			lowercase__					:      str      =		[self.cls_token_id]

			lowercase__					:      Optional[int]      =		cls + token_ids_a + sep
			if token_ids_a is not None:
					result += token_ids_a + sep
			return result

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						Tuple = None,					lowerCamelCase_:						Any = False			):


			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=_UpperCamelCase,					token_ids_a=_UpperCamelCase,					already_has_special_tokens=_UpperCamelCase			)

			lowercase__					:      Dict      =		[1] + ([0] * len(_UpperCamelCase			)) + [1]
			if token_ids_a is not None:
					result += ([0] * len(_UpperCamelCase			)) + [1]
			return result

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						List[str],					lowerCamelCase_:						Any = None			):
			lowercase__					:      Dict      =		[self.sep_token_id]
			lowercase__					:      List[str]      =		[self.cls_token_id]

			lowercase__					:      Tuple      =		len(cls + token_ids_a + sep			) * [0]
			if token_ids_a is not None:
					result += len(token_ids_a + sep			) * [1]
			return result





	def    snake_case__(						self:						Union[str, Any],					lowerCamelCase_:						str,					lowerCamelCase_:						Tuple = None			):
			return ()





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
    from PIL import Image

    from transformers import CLIPImageProcessor, CLIPProcessor

@require_vision
class     __snake_case    (				unittest.TestCase  ):
       """simple docstring"""







       def 			a		(		self   )      ->							Optional[int]:






              """simple docstring"""


              __snake_case       	=							tempfile.mkdtemp()

              # fmt: off
              __snake_case       	=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
              # fmt: on
              __snake_case       	=							dict(zip(_UpperCamelCase      , range(len(_UpperCamelCase   )   )   )   )
              __snake_case       	=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
              __snake_case       	=							{"""unk_token""": """<unk>"""}

              __snake_case       	=							os.path.join(self.tmpdirname      , VOCAB_FILES_NAMES["""vocab_file"""]   )
              __snake_case       	=							os.path.join(self.tmpdirname      , VOCAB_FILES_NAMES["""merges_file"""]   )
              with open(self.vocab_file      , """w"""      , encoding="""utf-8"""   ) as fp:
                     fp.write(json.dumps(_UpperCamelCase   ) + """\n"""   )
              with open(self.merges_file      , """w"""      , encoding="""utf-8"""   ) as fp:
                     fp.write("""\n""".join(_UpperCamelCase   )   )

              __snake_case       	=							{
                  """do_resize""": True,
                  """size""": 20,
                  """do_center_crop""": True,
                  """crop_size""": 18,
                  """do_normalize""": True,
                  """image_mean""": [0.4814_5466, 0.457_8275, 0.4082_1073],
                  """image_std""": [0.2686_2954, 0.2613_0258, 0.2757_7711],
              }
              __snake_case       	=							os.path.join(self.tmpdirname      , _UpperCamelCase   )
              with open(self.image_processor_file      , """w"""      , encoding="""utf-8"""   ) as fp:
                     json.dump(_UpperCamelCase      , _UpperCamelCase   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							Optional[int]:






              """simple docstring"""


              return CLIPTokenizer.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							Union[str, Any]:






              """simple docstring"""


              return CLIPTokenizerFast.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							List[str]:






              """simple docstring"""


              return CLIPImageProcessor.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self   )      ->							int:






              """simple docstring"""


              shutil.rmtree(self.tmpdirname   )


       def 			a		(		self   )      ->							Optional[Any]:






              """simple docstring"""


              __snake_case       	=							[np.random.randint(2_55      , size=(3, 30, 4_00)      , dtype=np.uinta   )]

              __snake_case       	=							[Image.fromarray(np.moveaxis(_UpperCamelCase      , 0      , -1   )   ) for x in image_inputs]

              return image_inputs


       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizer()
              __snake_case       	=							self.get_rust_tokenizer()
              __snake_case       	=							self.get_image_processor()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )
              processor_slow.save_pretrained(self.tmpdirname   )
              __snake_case       	=							CLIPProcessor.from_pretrained(self.tmpdirname      , use_fast=_UpperCamelCase   )

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )
              processor_fast.save_pretrained(self.tmpdirname   )
              __snake_case       	=							CLIPProcessor.from_pretrained(self.tmpdirname   )

              self.assertEqual(processor_slow.tokenizer.get_vocab()      , tokenizer_slow.get_vocab()   )
              self.assertEqual(processor_fast.tokenizer.get_vocab()      , tokenizer_fast.get_vocab()   )
              self.assertEqual(tokenizer_slow.get_vocab()      , tokenizer_fast.get_vocab()   )
              self.assertIsInstance(processor_slow.tokenizer      , _UpperCamelCase   )
              self.assertIsInstance(processor_fast.tokenizer      , _UpperCamelCase   )

              self.assertEqual(processor_slow.image_processor.to_json_string()      , image_processor.to_json_string()   )
              self.assertEqual(processor_fast.image_processor.to_json_string()      , image_processor.to_json_string()   )
              self.assertIsInstance(processor_slow.image_processor      , _UpperCamelCase   )
              self.assertIsInstance(processor_fast.image_processor      , _UpperCamelCase   )


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              __snake_case       	=							CLIPProcessor(tokenizer=self.get_tokenizer()      , image_processor=self.get_image_processor()   )
              processor.save_pretrained(self.tmpdirname   )

              __snake_case       	=							self.get_tokenizer(bos_token="""(BOS)"""      , eos_token="""(EOS)"""   )
              __snake_case       	=							self.get_image_processor(do_normalize=_UpperCamelCase      , padding_value=1.0   )

              __snake_case       	=							CLIPProcessor.from_pretrained(
                  self.tmpdirname      , bos_token="""(BOS)"""      , eos_token="""(EOS)"""      , do_normalize=_UpperCamelCase      , padding_value=1.0   )

              self.assertEqual(processor.tokenizer.get_vocab()      , tokenizer_add_kwargs.get_vocab()   )
              self.assertIsInstance(processor.tokenizer      , _UpperCamelCase   )

              self.assertEqual(processor.image_processor.to_json_string()      , image_processor_add_kwargs.to_json_string()   )
              self.assertIsInstance(processor.image_processor      , _UpperCamelCase   )


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							self.prepare_image_inputs()

              __snake_case       	=							image_processor(_UpperCamelCase      , return_tensors="""np"""   )
              __snake_case       	=							processor(images=_UpperCamelCase      , return_tensors="""np"""   )

              for key in input_image_proc.keys():
                     self.assertAlmostEqual(input_image_proc[key].sum()      , input_processor[key].sum()      , delta=1E-2   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							"""lower newer"""

              __snake_case       	=							processor(text=_UpperCamelCase   )

              __snake_case       	=							tokenizer(_UpperCamelCase   )

              for key in encoded_tok.keys():
                     self.assertListEqual(encoded_tok[key]      , encoded_processor[key]   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							"""lower newer"""
              __snake_case       	=							self.prepare_image_inputs()

              __snake_case       	=							processor(text=_UpperCamelCase      , images=_UpperCamelCase   )

              self.assertListEqual(list(inputs.keys()   )      , ["""input_ids""", """attention_mask""", """pixel_values"""]   )

              # test if it raises when no input is passed
              with pytest.raises(_UpperCamelCase   ):
                     processor()


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

              __snake_case       	=							processor.batch_decode(_UpperCamelCase   )
              __snake_case       	=							tokenizer.batch_decode(_UpperCamelCase   )

              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )






       def 			a		(		self   )      ->							int:






              """simple docstring"""


              __snake_case       	=							self.get_image_processor()
              __snake_case       	=							self.get_tokenizer()

              __snake_case       	=							CLIPProcessor(tokenizer=_UpperCamelCase      , image_processor=_UpperCamelCase   )

              __snake_case       	=							"""lower newer"""
              __snake_case       	=							self.prepare_image_inputs()

              __snake_case       	=							processor(text=_UpperCamelCase      , images=_UpperCamelCase   )

              self.assertListEqual(list(inputs.keys()   )      , processor.model_input_names   )

