





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		snake_case_   (							UpperCAmelCase_     ,     unittest.TestCase						):







	'''simple docstring'''
	__UpperCamelCase   =		CLIPTokenizer
	__UpperCamelCase   =		CLIPTokenizerFast
	__UpperCamelCase   =		True
	__UpperCamelCase   =		{}
	__UpperCamelCase   =		False




	def    UpperCAmelCase			(						self							:     str   )       -> int:

			'''simple docstring'''




			super().setUp()

			# fmt: off
			__lowercase											=     ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
			# fmt: on
			__lowercase											=     dict(zip(__lowerCamelCase					,      range(len(__lowerCamelCase   )   )   )   )
			__lowercase											=     ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
			__lowercase											=     {'unk_token': '<unk>'}

			__lowercase											=     os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES['vocab_file']   )
			__lowercase											=     os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES['merges_file']   )
			with open(self.vocab_file					,      'w'					,      encoding='utf-8'   ) as fp:
					fp.write(json.dumps(__lowerCamelCase   ) + '\n'   )
			with open(self.merges_file					,      'w'					,      encoding='utf-8'   ) as fp:
					fp.write('\n'.join(__lowerCamelCase   )   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      **__lowerCamelCase							:     int   )       -> Dict:

			'''simple docstring'''




			kwargs.update(self.special_tokens_map   )
			return CLIPTokenizer.from_pretrained(self.tmpdirname					,      **__lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Tuple					,      **__lowerCamelCase							:     List[str]   )       -> Union[str, Any]:

			'''simple docstring'''




			kwargs.update(self.special_tokens_map   )
			return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,      **__lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     str					,      __lowerCamelCase							:     Union[str, Any]   )       -> Union[str, Any]:

			'''simple docstring'''




			__lowercase											=     'lower newer'
			__lowercase											=     'lower newer'
			return input_text, output_text




	def    UpperCAmelCase			(						self							:     Dict   )       -> List[str]:

			'''simple docstring'''




			__lowercase											=     CLIPTokenizer(self.vocab_file					,      self.merges_file					,      **self.special_tokens_map   )
			__lowercase											=     'lower newer'
			__lowercase											=     ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
			__lowercase											=     tokenizer.tokenize(__lowerCamelCase   )
			self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase   )

			__lowercase											=     tokens + [tokenizer.unk_token]
			__lowercase											=     [10, 2, 16, 9, 3, 2, 16, 20]
			self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase   )					,      __lowerCamelCase   )




	@require_ftfy
	def    UpperCAmelCase			(						self							:     Any   )       -> Any:

			'''simple docstring'''




			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
							__lowercase											=     self.tokenizer_class.from_pretrained(__lowerCamelCase					,      **__lowerCamelCase   )
							__lowercase											=     self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,      **__lowerCamelCase   )

							__lowercase											=     'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
							__lowercase											=     tokenizer_s.tokenize(__lowerCamelCase   )
							__lowercase											=     tokenizer_r.tokenize(__lowerCamelCase   )

							self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase   )

							# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
							# with Tilde) encoded in 2 different ways
							__lowercase											=     'xa\u0303y' + ' ' + 'x\xe3y'
							__lowercase											=     tokenizer_s.tokenize(__lowerCamelCase   )
							__lowercase											=     tokenizer_r.tokenize(__lowerCamelCase   )

							self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase   )

							# Test that the tokenization is identical on unicode of space type
							__lowercase											=     [
							    '\u0009',  # (horizontal tab, '\t')
							    '\u000B',  # (vertical tab)
							    '\u000C',  # (form feed)
							    '\u0020',  # (space, ' ')
							    '\u200E',  # (left-to-right mark):w
							    '\u200F',  # (right-to-left mark)
							]
							for unicode_seq in spaces_unicodes:
									__lowercase											=     tokenizer_s.tokenize(__lowerCamelCase   )
									__lowercase											=     tokenizer_r.tokenize(__lowerCamelCase   )

									self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase   )

							# Test that the tokenization is identical on unicode of line break type
							__lowercase											=     [
							    '\u000A',  # (line feed, '\n')
							    '\r\n',  # (carriage return and line feed, '\r\n')
							    '\u000D',  # (carriage return, '\r')
							    '\r',  # (carriage return, '\r')
							    '\u000D',  # (carriage return, '\r')
							    '\u2028',  # (line separator)
							    '\u2029',  # (paragraph separator)
							    # "\u0085", # (next line)
							]

							# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
							# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
							# space (and thus into an empty list).

							for unicode_seq in line_break_unicodes:
									__lowercase											=     tokenizer_s.tokenize(__lowerCamelCase   )
									__lowercase											=     tokenizer_r.tokenize(__lowerCamelCase   )

									self.assertListEqual(__lowerCamelCase					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Union[str, Any]   )       -> Dict:

			'''simple docstring'''




			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
							__lowercase											=     'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
							__lowercase											=     F"{text_of_1_token} {text_of_1_token}"

							__lowercase											=     self.rust_tokenizer_class.from_pretrained(
							    __lowerCamelCase					,      use_fast=__lowerCamelCase					,      )
							__lowercase											=     tokenizer_r(__lowerCamelCase					,      return_offsets_mapping=__lowerCamelCase					,      add_special_tokens=__lowerCamelCase   )
							self.assertEqual(encoding.offset_mapping[0]					,      (0, len(__lowerCamelCase   ))   )
							self.assertEqual(
							    encoding.offset_mapping[1]					,      (len(__lowerCamelCase   ) + 1, len(__lowerCamelCase   ) + 1 + len(__lowerCamelCase   ))					,      )

							__lowercase											=     F" {text}"

							__lowercase											=     self.rust_tokenizer_class.from_pretrained(
							    __lowerCamelCase					,      use_fast=__lowerCamelCase					,      )
							__lowercase											=     tokenizer_r(__lowerCamelCase					,      return_offsets_mapping=__lowerCamelCase					,      add_special_tokens=__lowerCamelCase   )
							self.assertEqual(encoding.offset_mapping[0]					,      (1, 1 + len(__lowerCamelCase   ))   )
							self.assertEqual(
							    encoding.offset_mapping[1]					,      (1 + len(__lowerCamelCase   ) + 1, 1 + len(__lowerCamelCase   ) + 1 + len(__lowerCamelCase   ))					,      )




	def    UpperCAmelCase			(						self							:     Union[str, Any]   )       -> Optional[int]:

			'''simple docstring'''




			with self.assertRaises(__lowerCamelCase   ) as context:
					self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'   )

			self.assertTrue(
			    context.exception.args[0].startswith(
			        'The `backend_tokenizer` provided does not match the expected format.'   )   )




	@require_ftfy
	def    UpperCAmelCase			(						self							:     Any   )       -> Optional[Any]:

			'''simple docstring'''




			super().test_tokenization_python_rust_equals()






	def    UpperCAmelCase			(						self							:     Optional[Any]   )       -> Optional[int]:

			'''simple docstring'''




			pass











class 		snake_case_   :







	'''simple docstring'''




	def __init__(						self							:     Dict					,      __lowerCamelCase							:     int					,      __lowerCamelCase							:     Optional[Any]=None					,      __lowerCamelCase							:     Optional[Any]=None   )       -> Dict:

			'''simple docstring'''




			__lowercase											=     data
			__lowercase											=     previous
			__lowercase											=     next_node




	def __str__(						self							:     int   )       -> str:

			'''simple docstring'''




			return F"{self.data}"




	def    UpperCAmelCase			(						self							:     int   )       -> int:

			'''simple docstring'''




			return self.data




	def    UpperCAmelCase			(						self							:     Tuple   )       -> Optional[int]:

			'''simple docstring'''




			return self.next






	def    UpperCAmelCase			(						self							:     Optional[int]   )       -> Any:

			'''simple docstring'''




			return self.previous





class 		snake_case_   :







	'''simple docstring'''




	def __init__(						self							:     Tuple					,      __lowerCamelCase							:     Optional[Any]   )       -> str:

			'''simple docstring'''




			__lowercase											=     head




	def __iter__(						self							:     Union[str, Any]   )       -> List[Any]:

			'''simple docstring'''




			return self






	def    UpperCAmelCase			(						self							:     str   )       -> str:

			'''simple docstring'''




			if not self.current:
					raise StopIteration
			else:
					__lowercase											=     self.current.get_data()
					__lowercase											=     self.current.get_next()
					return value





class 		snake_case_   :







	'''simple docstring'''




	def __init__(						self							:     List[str]   )       -> Any:

			'''simple docstring'''




			__lowercase											=     None  # First node in list
			__lowercase											=     None  # Last node in list




	def __str__(						self							:     int   )       -> Any:

			'''simple docstring'''




			__lowercase											=     self.head
			__lowercase											=     []
			while current is not None:
					nodes.append(current.get_data()   )
					__lowercase											=     current.get_next()
			return " ".join(str(__lowerCamelCase   ) for node in nodes   )




	def __contains__(						self							:     Tuple					,      __lowerCamelCase							:     int   )       -> List[Any]:

			'''simple docstring'''




			__lowercase											=     self.head
			while current:
					if current.get_data() == value:
							return True
					__lowercase											=     current.get_next()
			return False




	def __iter__(						self							:     Optional[int]   )       -> List[str]:

			'''simple docstring'''




			return LinkedListIterator(self.head   )




	def    UpperCAmelCase			(						self							:     Any   )       -> str:

			'''simple docstring'''




			if self.head:
					return self.head.get_data()
			return None




	def    UpperCAmelCase			(						self							:     str   )       -> Union[str, Any]:

			'''simple docstring'''




			if self.tail:
					return self.tail.get_data()
			return None




	def    UpperCAmelCase			(						self							:     str					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			if self.head is None:
					__lowercase											=     node
					__lowercase											=     node
			else:
					self.insert_before_node(self.head					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Dict					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			if self.head is None:
					self.set_head(__lowerCamelCase   )
			else:
					self.insert_after_node(self.tail					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     int   )       -> None:

			'''simple docstring'''




			__lowercase											=     Node(__lowerCamelCase   )
			if self.head is None:
					self.set_head(__lowerCamelCase   )
			else:
					self.set_tail(__lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     int					,      __lowerCamelCase							:     Node					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			__lowercase											=     node
			__lowercase											=     node.previous

			if node.get_previous() is None:
					__lowercase											=     node_to_insert
			else:
					__lowercase											=     node_to_insert

			__lowercase											=     node_to_insert




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     Node					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			__lowercase											=     node
			__lowercase											=     node.next

			if node.get_next() is None:
					__lowercase											=     node_to_insert
			else:
					__lowercase											=     node_to_insert

			__lowercase											=     node_to_insert




	def    UpperCAmelCase			(						self							:     List[str]					,      __lowerCamelCase							:     int					,      __lowerCamelCase							:     int   )       -> None:

			'''simple docstring'''




			__lowercase											=     1
			__lowercase											=     Node(__lowerCamelCase   )
			__lowercase											=     self.head
			while node:
					if current_position == position:
							self.insert_before_node(__lowerCamelCase					,      __lowerCamelCase   )
							return
					current_position += 1
					__lowercase											=     node.next
			self.insert_after_node(self.tail					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     int   )       -> Node:

			'''simple docstring'''




			__lowercase											=     self.head
			while node:
					if node.get_data() == item:
							return node
					__lowercase											=     node.get_next()
			raise Exception('Node not found'   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     List[str]   )       -> Dict:

			'''simple docstring'''




			if (node := self.get_node(__lowerCamelCase   )) is not None:
					if node == self.head:
							__lowercase											=     self.head.get_next()

					if node == self.tail:
							__lowercase											=     self.tail.get_previous()

					self.remove_node_pointers(__lowerCamelCase   )




	@staticmethod
	def    UpperCAmelCase			(						__lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			if node.get_next():
					__lowercase											=     node.previous

			if node.get_previous():
					__lowercase											=     node.next

			__lowercase											=     None
			__lowercase											=     None






	def    UpperCAmelCase			(						self							:     str   )       -> List[str]:

			'''simple docstring'''




			return self.head is None





def  SCREAMING_SNAKE_CASE				(      )		->    None:
		pass


if __name__ == "__main__":
		import doctest

		doctest.testmod()




