

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class __UpperCamelCase     (				_a					,unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   CLIPTokenizer
					__magic_name__        =   CLIPTokenizerFast
					__magic_name__        =   True
					__magic_name__        =   {}
					__magic_name__        =   False





					def    _UpperCAmelCase    (				self						):
						super().setUp()

						# fmt: off
						UpperCAmelCase__:	Tuple									= ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
						# fmt: on
						UpperCAmelCase__:	List[Any]									= dict(zip(lowerCamelCase__				,      range(len(lowerCamelCase__						)						)						)						)
						UpperCAmelCase__:	Tuple									= ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
						UpperCAmelCase__:	List[Any]									= {"unk_token": "<unk>"}

						UpperCAmelCase__:	Optional[Any]									= os.path.join(self.tmpdirname				,      VOCAB_FILES_NAMES["vocab_file"]						)
						UpperCAmelCase__:	Dict									= os.path.join(self.tmpdirname				,      VOCAB_FILES_NAMES["merges_file"]						)
						with open(self.vocab_file				,      "w"				,      encoding="utf-8"						) as fp:
							fp.write(json.dumps(lowerCamelCase__						) + "\n"						)
						with open(self.merges_file				,      "w"				,      encoding="utf-8"						) as fp:
							fp.write("\n".join(lowerCamelCase__						)						)





					def    _UpperCAmelCase    (				self				,      **lowerCamelCase__						):
						kwargs.update(self.special_tokens_map						)
						return CLIPTokenizer.from_pretrained(self.tmpdirname				,      **lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      **lowerCamelCase__						):
						kwargs.update(self.special_tokens_map						)
						return CLIPTokenizerFast.from_pretrained(self.tmpdirname				,      **lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	Dict									= "lower newer"
						UpperCAmelCase__:	Dict									= "lower newer"
						return input_text, output_text





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	List[Any]									= CLIPTokenizer(self.vocab_file				,      self.merges_file				,      **self.special_tokens_map						)
						UpperCAmelCase__:	Optional[Any]									= "lower newer"
						UpperCAmelCase__:	Optional[int]									= ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
						UpperCAmelCase__:	Optional[int]									= tokenizer.tokenize(lowerCamelCase__						)
						self.assertListEqual(lowerCamelCase__				,      lowerCamelCase__						)

						UpperCAmelCase__:	Any									= tokens + [tokenizer.unk_token]
						UpperCAmelCase__:	Dict									= [1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__						)				,      lowerCamelCase__						)





					@require_ftfy
					def    _UpperCAmelCase    (				self						):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
								UpperCAmelCase__:	Optional[int]									= self.tokenizer_class.from_pretrained(lowerCamelCase__				,      **lowerCamelCase__						)
								UpperCAmelCase__:	Optional[int]									= self.rust_tokenizer_class.from_pretrained(lowerCamelCase__				,      **lowerCamelCase__						)

								UpperCAmelCase__:	Optional[int]									= "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
								UpperCAmelCase__:	List[str]									= tokenizer_s.tokenize(lowerCamelCase__						)
								UpperCAmelCase__:	Optional[int]									= tokenizer_r.tokenize(lowerCamelCase__						)

								self.assertListEqual(lowerCamelCase__				,      lowerCamelCase__						)

								# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
								# with Tilde) encoded in 2 different ways
								UpperCAmelCase__:	Union[str, Any]									= "xa\u0303y" + " " + "x\xe3y"
								UpperCAmelCase__:	Any									= tokenizer_s.tokenize(lowerCamelCase__						)
								UpperCAmelCase__:	int									= tokenizer_r.tokenize(lowerCamelCase__						)

								self.assertListEqual(lowerCamelCase__				,      lowerCamelCase__						)

								# Test that the tokenization is identical on unicode of space type
								UpperCAmelCase__:	Tuple									= [
								    "\u0009",  # (horizontal tab, '\t')
								    "\u000B",  # (vertical tab)
								    "\u000C",  # (form feed)
								    "\u0020",  # (space, ' ')
								    "\u200E",  # (left-to-right mark):w
								    "\u200F",  # (right-to-left mark)
								]
								for unicode_seq in spaces_unicodes:
									UpperCAmelCase__:	List[Any]									= tokenizer_s.tokenize(lowerCamelCase__						)
									UpperCAmelCase__:	Any									= tokenizer_r.tokenize(lowerCamelCase__						)

									self.assertListEqual(lowerCamelCase__				,      lowerCamelCase__						)

								# Test that the tokenization is identical on unicode of line break type
								UpperCAmelCase__:	Any									= [
								    "\u000A",  # (line feed, '\n')
								    "\r\n",  # (carriage return and line feed, '\r\n')
								    "\u000D",  # (carriage return, '\r')
								    "\r",  # (carriage return, '\r')
								    "\u000D",  # (carriage return, '\r')
								    "\u2028",  # (line separator)
								    "\u2029",  # (paragraph separator)
								    # "\u0085", # (next line)
								]

								# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
								# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
								# space (and thus into an empty list).

								for unicode_seq in line_break_unicodes:
									UpperCAmelCase__:	List[str]									= tokenizer_s.tokenize(lowerCamelCase__						)
									UpperCAmelCase__:	Optional[Any]									= tokenizer_r.tokenize(lowerCamelCase__						)

									self.assertListEqual(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
								UpperCAmelCase__:	Optional[Any]									= "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
								UpperCAmelCase__:	Union[str, Any]									= F"{text_of_1_token} {text_of_1_token}"

								UpperCAmelCase__:	Union[str, Any]									= self.rust_tokenizer_class.from_pretrained(
								    lowerCamelCase__				,      use_fast=lowerCamelCase__				,      )
								UpperCAmelCase__:	Tuple									= tokenizer_r(lowerCamelCase__				,      return_offsets_mapping=lowerCamelCase__				,      add_special_tokens=lowerCamelCase__						)
								self.assertEqual(encoding.offset_mapping[0]				,      (0, len(lowerCamelCase__						))						)
								self.assertEqual(
								    encoding.offset_mapping[1]				,      (len(lowerCamelCase__						) + 1, len(lowerCamelCase__						) + 1 + len(lowerCamelCase__						))				,      )

								UpperCAmelCase__:	Dict									= F" {text}"

								UpperCAmelCase__:	Dict									= self.rust_tokenizer_class.from_pretrained(
								    lowerCamelCase__				,      use_fast=lowerCamelCase__				,      )
								UpperCAmelCase__:	Optional[int]									= tokenizer_r(lowerCamelCase__				,      return_offsets_mapping=lowerCamelCase__				,      add_special_tokens=lowerCamelCase__						)
								self.assertEqual(encoding.offset_mapping[0]				,      (1, 1 + len(lowerCamelCase__						))						)
								self.assertEqual(
								    encoding.offset_mapping[1]				,      (1 + len(lowerCamelCase__						) + 1, 1 + len(lowerCamelCase__						) + 1 + len(lowerCamelCase__						))				,      )





					def    _UpperCAmelCase    (				self						):
						# Test related to the breaking change introduced in transformers v4.17.0
						# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
						with self.assertRaises(lowerCamelCase__						) as context:
							self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"						)

						self.assertTrue(
						    context.exception.args[0].startswith(
						        "The `backend_tokenizer` provided does not match the expected format."						)						)





					@require_ftfy
					def    _UpperCAmelCase    (				self						):
						super().test_tokenization_python_rust_equals()





					def    _UpperCAmelCase    (				self						):
						# CLIP always lower cases letters
						pass


def 			_A	(      SCREAMING_SNAKE_CASE      ):
	UpperCAmelCase__:	Tuple									= int(SCREAMING_SNAKE_CASE      )
	if decimal in (0, 1):  # Exit cases for the recursion
		return str(SCREAMING_SNAKE_CASE      )
	UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= divmod(SCREAMING_SNAKE_CASE	,2      )
	return binary_recursive(SCREAMING_SNAKE_CASE      ) + str(SCREAMING_SNAKE_CASE      )



def 			_A	(      SCREAMING_SNAKE_CASE      ):
	UpperCAmelCase__:	List[str]									= str(SCREAMING_SNAKE_CASE      ).strip()
	if not number:
		raise ValueError("No input value was provided"      )
	UpperCAmelCase__:	Tuple									= "-" if number.startswith("-"      ) else ""
	UpperCAmelCase__:	Any									= number.lstrip("-"      )
	if not number.isnumeric():
		raise ValueError("Input value is not an integer"      )
	return f"{negative}0b{binary_recursive(int(SCREAMING_SNAKE_CASE      )      )}"


if __name__ == "__main__":
					from doctest import testmod

					testmod()