





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class       __lowerCAmelCase					(       _UpperCamelCase   ,    unittest.TestCase      ):


	'''simple docstring'''





	_A													=				CLIPTokenizer
	_A													=				CLIPTokenizerFast
	_A													=				True
	_A													=				{}
	_A													=				False

	def    snake_case__(						self:						Optional[Any]			):
			super().setUp()

			# fmt: off
			lowercase__					:      Optional[int]      =		['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
			# fmt: on
			lowercase__					:      Dict      =		dict(zip(lowerCamelCase_,					range(len(lowerCamelCase_			)			)			)			)
			lowercase__					:      Dict      =		['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
			lowercase__					:      Dict      =		{'unk_token': '<unk>'}

			lowercase__					:      List[Any]      =		os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES['vocab_file']			)
			lowercase__					:      List[str]      =		os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES['merges_file']			)
			with open(self.vocab_file,					'w',					encoding='utf-8'			) as fp:
					fp.write(json.dumps(lowerCamelCase_			) + '\n'			)
			with open(self.merges_file,					'w',					encoding='utf-8'			) as fp:
					fp.write('\n'.join(lowerCamelCase_			)			)

	def    snake_case__(						self:						Optional[int],					**lowerCamelCase_:						Tuple			):
			kwargs.update(self.special_tokens_map			)
			return CLIPTokenizer.from_pretrained(self.tmpdirname,					**lowerCamelCase_			)

	def    snake_case__(						self:						Tuple,					**lowerCamelCase_:						Optional[int]			):
			kwargs.update(self.special_tokens_map			)
			return CLIPTokenizerFast.from_pretrained(self.tmpdirname,					**lowerCamelCase_			)

	def    snake_case__(						self:						int,					lowerCamelCase_:						Dict			):
			lowercase__					:      List[str]      =		'lower newer'
			lowercase__					:      Optional[Any]      =		'lower newer'
			return input_text, output_text

	def    snake_case__(						self:						Optional[Any]			):
			lowercase__					:      Any      =		CLIPTokenizer(self.vocab_file,					self.merges_file,					**self.special_tokens_map			)
			lowercase__					:      List[Any]      =		'lower newer'
			lowercase__					:      Any      =		['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
			lowercase__					:      int      =		tokenizer.tokenize(lowerCamelCase_			)
			self.assertListEqual(lowerCamelCase_,					lowerCamelCase_			)

			lowercase__					:      Tuple      =		tokens + [tokenizer.unk_token]
			lowercase__					:      int      =		[10, 2, 16, 9, 3, 2, 16, 20]
			self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_			),					lowerCamelCase_			)

	@require_ftfy
	def    snake_case__(						self:						Tuple			):
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
							lowercase__					:      Union[str, Any]      =		self.tokenizer_class.from_pretrained(lowerCamelCase_,					**lowerCamelCase_			)
							lowercase__					:      Optional[Any]      =		self.rust_tokenizer_class.from_pretrained(lowerCamelCase_,					**lowerCamelCase_			)

							lowercase__					:      List[str]      =		'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
							lowercase__					:      Dict      =		tokenizer_s.tokenize(lowerCamelCase_			)
							lowercase__					:      List[str]      =		tokenizer_r.tokenize(lowerCamelCase_			)

							self.assertListEqual(lowerCamelCase_,					lowerCamelCase_			)

							# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
							# with Tilde) encoded in 2 different ways
							lowercase__					:      Dict      =		'xa\u0303y' + ' ' + 'x\xe3y'
							lowercase__					:      List[str]      =		tokenizer_s.tokenize(lowerCamelCase_			)
							lowercase__					:      List[Any]      =		tokenizer_r.tokenize(lowerCamelCase_			)

							self.assertListEqual(lowerCamelCase_,					lowerCamelCase_			)

							# Test that the tokenization is identical on unicode of space type
							lowercase__					:      int      =		[
							    '\u0009',  # (horizontal tab, '\t')
							    '\u000B',  # (vertical tab)
							    '\u000C',  # (form feed)
							    '\u0020',  # (space, ' ')
							    '\u200E',  # (left-to-right mark):w
							    '\u200F',  # (right-to-left mark)
							]
							for unicode_seq in spaces_unicodes:
									lowercase__					:      Dict      =		tokenizer_s.tokenize(lowerCamelCase_			)
									lowercase__					:      str      =		tokenizer_r.tokenize(lowerCamelCase_			)

									self.assertListEqual(lowerCamelCase_,					lowerCamelCase_			)

							# Test that the tokenization is identical on unicode of line break type
							lowercase__					:      int      =		[
							    '\u000A',  # (line feed, '\n')
							    '\r\n',  # (carriage return and line feed, '\r\n')
							    '\u000D',  # (carriage return, '\r')
							    '\r',  # (carriage return, '\r')
							    '\u000D',  # (carriage return, '\r')
							    '\u2028',  # (line separator)
							    '\u2029',  # (paragraph separator)
							    # "\u0085", # (next line)
							]

							# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
							# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
							# space (and thus into an empty list).

							for unicode_seq in line_break_unicodes:
									lowercase__					:      Tuple      =		tokenizer_s.tokenize(lowerCamelCase_			)
									lowercase__					:      int      =		tokenizer_r.tokenize(lowerCamelCase_			)

									self.assertListEqual(lowerCamelCase_,					lowerCamelCase_			)

	def    snake_case__(						self:						Optional[int]			):
			# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
							lowercase__					:      Optional[int]      =		'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
							lowercase__					:      Any      =		F"""{text_of_1_token} {text_of_1_token}"""

							lowercase__					:      str      =		self.rust_tokenizer_class.from_pretrained(
							    lowerCamelCase_,					use_fast=lowerCamelCase_,					)
							lowercase__					:      str      =		tokenizer_r(lowerCamelCase_,					return_offsets_mapping=lowerCamelCase_,					add_special_tokens=lowerCamelCase_			)
							self.assertEqual(encoding.offset_mapping[0],					(0, len(lowerCamelCase_			))			)
							self.assertEqual(
							    encoding.offset_mapping[1],					(len(lowerCamelCase_			) + 1, len(lowerCamelCase_			) + 1 + len(lowerCamelCase_			)),					)

							lowercase__					:      Tuple      =		F""" {text}"""

							lowercase__					:      int      =		self.rust_tokenizer_class.from_pretrained(
							    lowerCamelCase_,					use_fast=lowerCamelCase_,					)
							lowercase__					:      int      =		tokenizer_r(lowerCamelCase_,					return_offsets_mapping=lowerCamelCase_,					add_special_tokens=lowerCamelCase_			)
							self.assertEqual(encoding.offset_mapping[0],					(1, 1 + len(lowerCamelCase_			))			)
							self.assertEqual(
							    encoding.offset_mapping[1],					(1 + len(lowerCamelCase_			) + 1, 1 + len(lowerCamelCase_			) + 1 + len(lowerCamelCase_			)),					)

	def    snake_case__(						self:						Optional[int]			):
			# Test related to the breaking change introduced in transformers v4.17.0
			# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
			with self.assertRaises(lowerCamelCase_			) as context:
					self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'			)

			self.assertTrue(
			    context.exception.args[0].startswith(
			        'The `backend_tokenizer` provided does not match the expected format.'			)			)

	@require_ftfy
	def    snake_case__(						self:						Union[str, Any]			):
			super().test_tokenization_python_rust_equals()





	def    snake_case__(						self:						Optional[Any]			):
			# CLIP always lower cases letters
			pass









import enum
import shutil
import sys


__UpperCamelCase,	__UpperCamelCase:	Optional[int] = shutil.get_terminal_size()

__UpperCamelCase:	Optional[Any] = {"""UP""": """A""", """DOWN""": """B""", """RIGHT""": """C""", """LEFT""": """D"""}


class       __lowerCAmelCase					(       enum.Enum      ):


	'''simple docstring'''





	_A													=				0
	_A													=				1


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[Any]   ,    _lowercase     : Union[str, Any]=""			)   ->							int:




		'''simple docstring'''
		sys.stdout.write(str(_lowercase			) + end			)
		sys.stdout.flush()


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Optional[Any]   ,    _lowercase     : Union[str, Any]   ,    _lowercase     : Optional[int]=""			)   ->							str:




		'''simple docstring'''
		forceWrite(f"""\u001b[{color}m{content}\u001b[0m"""   ,    _lowercase			)


def       SCREAMING_SNAKE_CASE__  (       )   ->							int:




		'''simple docstring'''
		forceWrite('\r'			)


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : int   ,    _lowercase     : str			)   ->							List[Any]:




		'''simple docstring'''
		forceWrite(f"""\033[{num_lines}{CURSOR_TO_CHAR[direction.upper()]}"""			)


def       SCREAMING_SNAKE_CASE__  (       )   ->							Dict:




		'''simple docstring'''
		forceWrite(' ' * TERMINAL_WIDTH			)
		reset_cursor()





def       SCREAMING_SNAKE_CASE__  (       )   ->							List[str]:




		'''simple docstring'''
		reset_cursor()
		forceWrite('-' * TERMINAL_WIDTH			)


