


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class     A__				(		_UpperCamelCase ,  unittest.TestCase	):
						UpperCAmelCase   							=    CLIPTokenizer
						UpperCAmelCase   							=    CLIPTokenizerFast
						UpperCAmelCase   							=    True
						UpperCAmelCase   							=    {}
						UpperCAmelCase   							=    False




						def 				__UpperCamelCase					(							self       :   Optional[Any]	)				->  Optional[Any]:


										"""simple docstring"""





										super().setUp()

										# fmt: off
										_SCREAMING_SNAKE_CASE      	=['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
										# fmt: on
										_SCREAMING_SNAKE_CASE      	=dict(zip(lowerCamelCase_					,     range(len(lowerCamelCase_	)	)	)	)
										_SCREAMING_SNAKE_CASE      	=['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
										_SCREAMING_SNAKE_CASE      	={'unk_token': '<unk>'}

										_SCREAMING_SNAKE_CASE      	=os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']	)
										_SCREAMING_SNAKE_CASE      	=os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''merges_file''']	)
										with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''	) as fp:
														fp.write(json.dumps(lowerCamelCase_	) + '''\n'''	)
										with open(self.merges_file					,     '''w'''					,     encoding='''utf-8'''	) as fp:
														fp.write('''\n'''.join(lowerCamelCase_	)	)




						def 				__UpperCamelCase					(							self       :   Optional[int]					,     **_a       :   Tuple	)				->  str:


										"""simple docstring"""





										kwargs.update(self.special_tokens_map	)
										return CLIPTokenizer.from_pretrained(self.tmpdirname					,     **lowerCamelCase_	)




						def 				__UpperCamelCase					(							self       :   Tuple					,     **_a       :   Optional[int]	)				->  str:


										"""simple docstring"""





										kwargs.update(self.special_tokens_map	)
										return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,     **lowerCamelCase_	)




						def 				__UpperCamelCase					(							self       :   int					,     _a       :   Dict	)				->  str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE      	='lower newer'
										_SCREAMING_SNAKE_CASE      	='lower newer'
										return input_text, output_text




						def 				__UpperCamelCase					(							self       :   Optional[Any]	)				->  Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE      	=CLIPTokenizer(self.vocab_file					,     self.merges_file					,     **self.special_tokens_map	)
										_SCREAMING_SNAKE_CASE      	='lower newer'
										_SCREAMING_SNAKE_CASE      	=['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
										_SCREAMING_SNAKE_CASE      	=tokenizer.tokenize(lowerCamelCase_	)
										self.assertListEqual(lowerCamelCase_					,     lowerCamelCase_	)

										_SCREAMING_SNAKE_CASE      	=tokens + [tokenizer.unk_token]
										_SCREAMING_SNAKE_CASE      	=[10, 2, 16, 9, 3, 2, 16, 20]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_	)					,     lowerCamelCase_	)




						@require_ftfy
						def 				__UpperCamelCase					(							self       :   Tuple	)				->  Dict:


										"""simple docstring"""





										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"	):
																		_SCREAMING_SNAKE_CASE      	=self.tokenizer_class.from_pretrained(lowerCamelCase_					,     **lowerCamelCase_	)
																		_SCREAMING_SNAKE_CASE      	=self.rust_tokenizer_class.from_pretrained(lowerCamelCase_					,     **lowerCamelCase_	)

																		_SCREAMING_SNAKE_CASE      	='A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
																		_SCREAMING_SNAKE_CASE      	=tokenizer_s.tokenize(lowerCamelCase_	)
																		_SCREAMING_SNAKE_CASE      	=tokenizer_r.tokenize(lowerCamelCase_	)

																		self.assertListEqual(lowerCamelCase_					,     lowerCamelCase_	)

																		# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																		# with Tilde) encoded in 2 different ways
																		_SCREAMING_SNAKE_CASE      	='xa\u0303y' + ' ' + 'x\xe3y'
																		_SCREAMING_SNAKE_CASE      	=tokenizer_s.tokenize(lowerCamelCase_	)
																		_SCREAMING_SNAKE_CASE      	=tokenizer_r.tokenize(lowerCamelCase_	)

																		self.assertListEqual(lowerCamelCase_					,     lowerCamelCase_	)

																		# Test that the tokenization is identical on unicode of space type
																		_SCREAMING_SNAKE_CASE      	=[
																		    '\u0009',  # (horizontal tab, '\t')
																		    '\u000B',  # (vertical tab)
																		    '\u000C',  # (form feed)
																		    '\u0020',  # (space, ' ')
																		    '\u200E',  # (left-to-right mark):w
																		    '\u200F',  # (right-to-left mark)
																		]
																		for unicode_seq in spaces_unicodes:
																						_SCREAMING_SNAKE_CASE      	=tokenizer_s.tokenize(lowerCamelCase_	)
																						_SCREAMING_SNAKE_CASE      	=tokenizer_r.tokenize(lowerCamelCase_	)

																						self.assertListEqual(lowerCamelCase_					,     lowerCamelCase_	)

																		# Test that the tokenization is identical on unicode of line break type
																		_SCREAMING_SNAKE_CASE      	=[
																		    '\u000A',  # (line feed, '\n')
																		    '\r\n',  # (carriage return and line feed, '\r\n')
																		    '\u000D',  # (carriage return, '\r')
																		    '\r',  # (carriage return, '\r')
																		    '\u000D',  # (carriage return, '\r')
																		    '\u2028',  # (line separator)
																		    '\u2029',  # (paragraph separator)
																		    # "\u0085", # (next line)
																		]

																		# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																		# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																		# space (and thus into an empty list).

																		for unicode_seq in line_break_unicodes:
																						_SCREAMING_SNAKE_CASE      	=tokenizer_s.tokenize(lowerCamelCase_	)
																						_SCREAMING_SNAKE_CASE      	=tokenizer_r.tokenize(lowerCamelCase_	)

																						self.assertListEqual(lowerCamelCase_					,     lowerCamelCase_	)




						def 				__UpperCamelCase					(							self       :   Optional[int]	)				->  int:


										"""simple docstring"""





										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"	):
																		_SCREAMING_SNAKE_CASE      	='hello'  # `hello` is a token in the vocabulary of `pretrained_name`
																		_SCREAMING_SNAKE_CASE      	=f"{text_of_1_token} {text_of_1_token}"

																		_SCREAMING_SNAKE_CASE      	=self.rust_tokenizer_class.from_pretrained(
																		    lowerCamelCase_					,     use_fast=lowerCamelCase_					,     )
																		_SCREAMING_SNAKE_CASE      	=tokenizer_r(lowerCamelCase_					,     return_offsets_mapping=lowerCamelCase_					,     add_special_tokens=lowerCamelCase_	)
																		self.assertEqual(encoding.offset_mapping[0]					,     (0, len(lowerCamelCase_	))	)
																		self.assertEqual(
																		    encoding.offset_mapping[1]					,     (len(lowerCamelCase_	) + 1, len(lowerCamelCase_	) + 1 + len(lowerCamelCase_	))					,     )

																		_SCREAMING_SNAKE_CASE      	=f" {text}"

																		_SCREAMING_SNAKE_CASE      	=self.rust_tokenizer_class.from_pretrained(
																		    lowerCamelCase_					,     use_fast=lowerCamelCase_					,     )
																		_SCREAMING_SNAKE_CASE      	=tokenizer_r(lowerCamelCase_					,     return_offsets_mapping=lowerCamelCase_					,     add_special_tokens=lowerCamelCase_	)
																		self.assertEqual(encoding.offset_mapping[0]					,     (1, 1 + len(lowerCamelCase_	))	)
																		self.assertEqual(
																		    encoding.offset_mapping[1]					,     (1 + len(lowerCamelCase_	) + 1, 1 + len(lowerCamelCase_	) + 1 + len(lowerCamelCase_	))					,     )




						def 				__UpperCamelCase					(							self       :   Optional[int]	)				->  Tuple:


										"""simple docstring"""





										with self.assertRaises(lowerCamelCase_	) as context:
														self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''	)

										self.assertTrue(
										    context.exception.args[0].startswith(
										        '''The `backend_tokenizer` provided does not match the expected format.'''	)	)




						@require_ftfy
						def 				__UpperCamelCase					(							self       :   Union[str, Any]	)				->  Dict:


										"""simple docstring"""





										super().test_tokenization_python_rust_equals()




						def 				__UpperCamelCase					(							self       :   Optional[Any]	)				->  List[str]:


										"""simple docstring"""





										pass






from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import BaseOutput, is_torch_available, is_transformers_available


@dataclass
class       __lowerCAmelCase					(       _UpperCamelCase      ):


	'''simple docstring'''





	_A													=				42
	_A													=				42


if is_transformers_available() and is_torch_available():
			from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline


