





import tempfile
import unittest

import numpy as np

from diffusers import (
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionPipeline,
    PNDMScheduler,
)
from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
			import onnxruntime as ort






class    __A       (       __A     ,							unittest.TestCase					):

			'''simple docstring'''







			lowerCAmelCase_					=		"""hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline"""





			def 						__lowerCamelCase							(       self			,    __lowerCAmelCase=0				):



				'''simple docstring'''







				lowerCamelCase__														=						np.random.RandomState(lowerCAmelCase_				)
				lowerCamelCase__														=						{
				    "prompt": "A painting of a squirrel eating a burger",
				    "generator": generator,
				    "num_inference_steps": 2,
				    "guidance_scale": 7.5,
				    "output_type": "numpy",
				}
				return inputs





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				).images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 1_2_8, 1_2_8, 3)
				lowerCamelCase__														=						np.array([0.6_5072, 0.5_8492, 0.4_8219, 0.5_5521, 0.5_3180, 0.5_5939, 0.5_0697, 0.3_9800, 0.4_6455]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				lowerCamelCase__														=						PNDMScheduler.from_config(pipe.scheduler.config			,    skip_prk_steps=lowerCAmelCase_				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				).images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 1_2_8, 1_2_8, 3)
				lowerCamelCase__														=						np.array([0.6_5863, 0.5_9425, 0.4_9326, 0.5_6313, 0.5_3875, 0.5_6627, 0.5_1065, 0.3_9777, 0.4_6330]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				lowerCamelCase__														=						LMSDiscreteScheduler.from_config(pipe.scheduler.config				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				).images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 1_2_8, 1_2_8, 3)
				lowerCamelCase__														=						np.array([0.5_3755, 0.6_0786, 0.4_7402, 0.4_9488, 0.5_1869, 0.4_9819, 0.4_7985, 0.3_8957, 0.4_4279]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				lowerCamelCase__														=						EulerDiscreteScheduler.from_config(pipe.scheduler.config				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				).images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 1_2_8, 1_2_8, 3)
				lowerCamelCase__														=						np.array([0.5_3755, 0.6_0786, 0.4_7402, 0.4_9488, 0.5_1869, 0.4_9819, 0.4_7985, 0.3_8957, 0.4_4279]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				lowerCamelCase__														=						EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				).images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 1_2_8, 1_2_8, 3)
				lowerCamelCase__														=						np.array([0.5_3817, 0.6_0812, 0.4_7384, 0.4_9530, 0.5_1894, 0.4_9814, 0.4_7984, 0.3_8958, 0.4_4271]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				lowerCamelCase__														=						DPMSolverMultistepScheduler.from_config(pipe.scheduler.config				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				).images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 1_2_8, 1_2_8, 3)
				lowerCamelCase__														=						np.array([0.5_3895, 0.6_0808, 0.4_7933, 0.4_9608, 0.5_1886, 0.4_9950, 0.4_8053, 0.3_8957, 0.4_4200]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						3 * [inputs["prompt"]]

				# forward
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				)
				lowerCamelCase__														=						output.images[0, -3:, -3:, -1]

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						3 * [inputs.pop('''prompt'''				)]

				lowerCamelCase__														=						pipe.tokenizer(
				    lowerCAmelCase_			,    padding='''max_length'''			,    max_length=pipe.tokenizer.model_max_length			,    truncation=lowerCAmelCase_			,    return_tensors='''np'''			,    )
				lowerCamelCase__														=						text_inputs["input_ids"]

				lowerCamelCase__														=						pipe.text_encoder(input_ids=text_inputs.astype(np.intaa				)				)[0]

				lowerCamelCase__														=						prompt_embeds

				# forward
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				)
				lowerCamelCase__														=						output.images[0, -3:, -3:, -1]

				assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()				).max() < 1E-4





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,    provider='''CPUExecutionProvider'''				)
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						3 * ["this is a negative prompt"]
				lowerCamelCase__														=						negative_prompt
				lowerCamelCase__														=						3 * [inputs["prompt"]]

				# forward
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				)
				lowerCamelCase__														=						output.images[0, -3:, -3:, -1]

				lowerCamelCase__														=						self.get_dummy_inputs()
				lowerCamelCase__														=						3 * [inputs.pop('''prompt'''				)]

				lowerCamelCase__														=						[]
				for p in [prompt, negative_prompt]:
					lowerCamelCase__														=						pipe.tokenizer(
					    lowerCAmelCase_			,    padding='''max_length'''			,    max_length=pipe.tokenizer.model_max_length			,    truncation=lowerCAmelCase_			,    return_tensors='''np'''			,    )
					lowerCamelCase__														=						text_inputs["input_ids"]

					embeds.append(pipe.text_encoder(input_ids=text_inputs.astype(np.intaa				)				)[0]				)

				lowerCamelCase__														=						embeds

				# forward
				lowerCamelCase__														=						pipe(**lowerCAmelCase_				)
				lowerCamelCase__														=						output.images[0, -3:, -3:, -1]

				assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()				).max() < 1E-4






@nightly
@require_onnxruntime
@require_torch_gpu
class    __A       (       unittest.TestCase					):

			'''simple docstring'''







			@property
			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				return (
				    "CUDAExecutionProvider",
				    {
				        "gpu_mem_limit": "15000000000",  # 15GB
				        "arena_extend_strategy": "kSameAsRequested",
				    },
				)





			@property
			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						ort.SessionOptions()
				lowerCamelCase__														=						False
				return options





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(
				    '''CompVis/stable-diffusion-v1-4'''			,    revision='''onnx'''			,    safety_checker=lowerCAmelCase_			,    feature_extractor=lowerCAmelCase_			,    provider=self.gpu_provider			,    sess_options=self.gpu_options			,    )
				sd_pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						"A painting of a squirrel eating a burger"
				np.random.seed(0				)
				lowerCamelCase__														=						sd_pipe([prompt]			,    guidance_scale=6.0			,    num_inference_steps=1_0			,    output_type='''np'''				)
				lowerCamelCase__														=						output.images

				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 5_1_2, 5_1_2, 3)
				lowerCamelCase__														=						np.array([0.0452, 0.0390, 0.0087, 0.0350, 0.0617, 0.0364, 0.0544, 0.0523, 0.0720]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-3





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						DDIMScheduler.from_pretrained(
				    '''runwayml/stable-diffusion-v1-5'''			,    subfolder='''scheduler'''			,    revision='''onnx'''				)
				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(
				    '''runwayml/stable-diffusion-v1-5'''			,    revision='''onnx'''			,    scheduler=lowerCAmelCase_			,    safety_checker=lowerCAmelCase_			,    feature_extractor=lowerCAmelCase_			,    provider=self.gpu_provider			,    sess_options=self.gpu_options			,    )
				sd_pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						"open neural network exchange"
				lowerCamelCase__														=						np.random.RandomState(0				)
				lowerCamelCase__														=						sd_pipe([prompt]			,    guidance_scale=7.5			,    num_inference_steps=1_0			,    generator=lowerCAmelCase_			,    output_type='''np'''				)
				lowerCamelCase__														=						output.images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 5_1_2, 5_1_2, 3)
				lowerCamelCase__														=						np.array([0.2867, 0.1974, 0.1481, 0.7294, 0.7251, 0.6667, 0.4194, 0.5642, 0.6486]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-3





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						LMSDiscreteScheduler.from_pretrained(
				    '''runwayml/stable-diffusion-v1-5'''			,    subfolder='''scheduler'''			,    revision='''onnx'''				)
				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(
				    '''runwayml/stable-diffusion-v1-5'''			,    revision='''onnx'''			,    scheduler=lowerCAmelCase_			,    safety_checker=lowerCAmelCase_			,    feature_extractor=lowerCAmelCase_			,    provider=self.gpu_provider			,    sess_options=self.gpu_options			,    )
				sd_pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						"open neural network exchange"
				lowerCamelCase__														=						np.random.RandomState(0				)
				lowerCamelCase__														=						sd_pipe([prompt]			,    guidance_scale=7.5			,    num_inference_steps=1_0			,    generator=lowerCAmelCase_			,    output_type='''np'''				)
				lowerCamelCase__														=						output.images
				lowerCamelCase__														=						image[0, -3:, -3:, -1]

				assert image.shape == (1, 5_1_2, 5_1_2, 3)
				lowerCamelCase__														=						np.array([0.2306, 0.1959, 0.1593, 0.6549, 0.6394, 0.5408, 0.5065, 0.6010, 0.6161]				)

				assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-3





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						0

				def test_callback_fn(__lowerCAmelCase			,    __lowerCAmelCase			,    __lowerCAmelCase				) -> None:
					lowerCamelCase__														=						True
					nonlocal number_of_steps
					number_of_steps += 1
					if step == 0:
						assert latents.shape == (1, 4, 6_4, 6_4)
						lowerCamelCase__														=						latents[0, -3:, -3:, -1]
						lowerCamelCase__														=						np.array(
						    [-0.6772, -0.3835, -1.2456, 0.1905, -1.0974, 0.6967, -1.9353, 0.0178, 1.0167]				)

						assert np.abs(latents_slice.flatten() - expected_slice				).max() < 1E-3
					elif step == 5:
						assert latents.shape == (1, 4, 6_4, 6_4)
						lowerCamelCase__														=						latents[0, -3:, -3:, -1]
						lowerCamelCase__														=						np.array(
						    [-0.3351, 0.2241, -0.1837, -0.2325, -0.6577, 0.3393, -0.0241, 0.5899, 1.3875]				)

						assert np.abs(latents_slice.flatten() - expected_slice				).max() < 1E-3

				lowerCamelCase__														=						False

				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(
				    '''runwayml/stable-diffusion-v1-5'''			,    revision='''onnx'''			,    safety_checker=lowerCAmelCase_			,    feature_extractor=lowerCAmelCase_			,    provider=self.gpu_provider			,    sess_options=self.gpu_options			,    )
				pipe.set_progress_bar_config(disable=lowerCAmelCase_				)

				lowerCamelCase__														=						"Andromeda galaxy in a bottle"

				lowerCamelCase__														=						np.random.RandomState(0				)
				pipe(
				    prompt=lowerCAmelCase_			,    num_inference_steps=5			,    guidance_scale=7.5			,    generator=lowerCAmelCase_			,    callback=lowerCAmelCase_			,    callback_steps=1			,    )
				assert test_callback_fn.has_been_called
				assert number_of_steps == 6





			def 						__lowerCamelCase							(       self				):



				'''simple docstring'''







				lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(
				    '''runwayml/stable-diffusion-v1-5'''			,    revision='''onnx'''			,    safety_checker=lowerCAmelCase_			,    feature_extractor=lowerCAmelCase_			,    provider=self.gpu_provider			,    sess_options=self.gpu_options			,    )
				assert isinstance(lowerCAmelCase_			,    lowerCAmelCase_				)
				assert pipe.safety_checker is None

				lowerCamelCase__														=						pipe('''example prompt'''			,    num_inference_steps=2				).images[0]
				assert image is not None

				# check that there's no error when saving a pipeline with one of the models being None
				with tempfile.TemporaryDirectory() as tmpdirname:
					pipe.save_pretrained(lowerCAmelCase_				)
					lowerCamelCase__														=						OnnxStableDiffusionPipeline.from_pretrained(lowerCAmelCase_				)

				# sanity check that the pipeline still works
				assert pipe.safety_checker is None
				lowerCamelCase__														=						pipe('''example prompt'''			,    num_inference_steps=2				).images[0]
				assert image is not None











"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class      UpperCamelCase_			(__A							,		unittest.TestCase	):
	__magic_name__       =							BertTokenizer
	__magic_name__       =							BertTokenizerFast
	__magic_name__       =							True
	__magic_name__       =							True
	__magic_name__       =							filter_non_english





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     Optional[Any]:
				super().setUp()

				UpperCAmelCase_	:					Optional[int]     	=   [
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_	:					Optional[Any]     	=   os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["vocab_file"]		)
				with open(self.vocab_file      ,      "w"      ,      encoding="utf-8"		) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]      ,      lowerCAmelCase_   :					List[str]		)		->     List[str]:
				UpperCAmelCase_	:					str     	=   "UNwant\u00E9d,running"
				UpperCAmelCase_	:					Optional[int]     	=   "unwanted, running"
				return input_text, output_text





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Any:
				UpperCAmelCase_	:					Optional[int]     	=   self.tokenizer_class(self.vocab_file		)

				UpperCAmelCase_	:					int     	=   tokenizer.tokenize("UNwant\u00E9d,running"		)
				self.assertListEqual(lowerCAmelCase_      ,      ["un", "##want", "##ed", ",", "runn", "##ing"]		)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)      ,      [9, 6, 7, 12, 10, 11]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     List[Any]:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_	:					List[Any]     	=   self.get_tokenizer()
				UpperCAmelCase_	:					str     	=   self.get_rust_tokenizer()

				UpperCAmelCase_	:					str     	=   "UNwant\u00E9d,running"

				UpperCAmelCase_	:					Any     	=   tokenizer.tokenize(lowerCAmelCase_		)
				UpperCAmelCase_	:					List[Any]     	=   rust_tokenizer.tokenize(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					List[str]     	=   rust_tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Dict     	=   self.get_rust_tokenizer()
				UpperCAmelCase_	:					int     	=   tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_	:					Optional[Any]     	=   rust_tokenizer.encode(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				# With lower casing
				UpperCAmelCase_	:					int     	=   self.get_tokenizer(do_lower_case=lowerCAmelCase_		)
				UpperCAmelCase_	:					str     	=   self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_		)

				UpperCAmelCase_	:					Any     	=   "UNwant\u00E9d,running"

				UpperCAmelCase_	:					Dict     	=   tokenizer.tokenize(lowerCAmelCase_		)
				UpperCAmelCase_	:					Union[str, Any]     	=   rust_tokenizer.tokenize(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Optional[int]     	=   tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					Any     	=   rust_tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Union[str, Any]     	=   self.get_rust_tokenizer()
				UpperCAmelCase_	:					Dict     	=   tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_	:					Union[str, Any]     	=   rust_tokenizer.encode(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     List[str]:
				UpperCAmelCase_	:					Union[str, Any]     	=   BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)      ,      ["ah", "\u535A", "\u63A8", "zz"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple		)		->     Optional[int]:
				UpperCAmelCase_	:					Union[str, Any]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,      ["hello", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Optional[Any]:
				UpperCAmelCase_	:					Dict     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hällo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["h\u00E9llo"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     Any:
				UpperCAmelCase_	:					str     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hallo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					List[str]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hallo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					List[Any]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Union[str, Any]:
				UpperCAmelCase_	:					str     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     str:
				UpperCAmelCase_	:					Any     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     Dict:
				UpperCAmelCase_	:					Dict     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      never_split=["[UNK]"]		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)      ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple		)		->     Union[str, Any]:
				UpperCAmelCase_	:					Optional[int]     	=   BasicTokenizer()
				UpperCAmelCase_	:					int     	=   "a\n'll !!to?'d of, can't."
				UpperCAmelCase_	:					Union[str, Any]     	=   ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_		)      ,      lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Dict:
				UpperCAmelCase_	:					List[str]     	=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_	:					int     	=   {}
				for i, token in enumerate(lowerCAmelCase_		):
							UpperCAmelCase_	:					Tuple     	=   i
				UpperCAmelCase_	:					Dict     	=   WordpieceTokenizer(vocab=lowerCAmelCase_      ,      unk_token="[UNK]"		)

				self.assertListEqual(tokenizer.tokenize(""		)      ,      []		)

				self.assertListEqual(tokenizer.tokenize("unwanted running"		)      ,      ["un", "##want", "##ed", "runn", "##ing"]		)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"		)      ,      ["[UNK]", "runn", "##ing"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Any:
				self.assertTrue(_is_whitespace(" "		)		)
				self.assertTrue(_is_whitespace("\t"		)		)
				self.assertTrue(_is_whitespace("\r"		)		)
				self.assertTrue(_is_whitespace("\n"		)		)
				self.assertTrue(_is_whitespace("\u00A0"		)		)

				self.assertFalse(_is_whitespace("A"		)		)
				self.assertFalse(_is_whitespace("-"		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Any:
				self.assertTrue(_is_control("\u0005"		)		)

				self.assertFalse(_is_control("A"		)		)
				self.assertFalse(_is_control(" "		)		)
				self.assertFalse(_is_control("\t"		)		)
				self.assertFalse(_is_control("\r"		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     int:
				self.assertTrue(_is_punctuation("-"		)		)
				self.assertTrue(_is_punctuation("$"		)		)
				self.assertTrue(_is_punctuation("`"		)		)
				self.assertTrue(_is_punctuation("."		)		)

				self.assertFalse(_is_punctuation("A"		)		)
				self.assertFalse(_is_punctuation(" "		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     str:
				UpperCAmelCase_	:					List[Any]     	=   self.get_tokenizer()
				UpperCAmelCase_	:					List[Any]     	=   self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_		) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]		)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_		) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]		)





	@slow
	def 	_SCREAMING_SNAKE_CASE ( self   :					int		)		->     List[Any]:
				UpperCAmelCase_	:					str     	=   self.tokenizer_class.from_pretrained("bert-base-uncased"		)

				UpperCAmelCase_	:					Optional[Any]     	=   tokenizer.encode("sequence builders"      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					str     	=   tokenizer.encode("multi-sequence build"      ,      add_special_tokens=lowerCAmelCase_		)

				UpperCAmelCase_	:					str     	=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_		)
				UpperCAmelCase_	:					Tuple     	=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_      ,      lowerCAmelCase_		)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[int]		)		->     List[str]:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_	:					Optional[int]     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_	:					Dict     	=   tokenizer_r.encode_plus(
										    lowerCAmelCase_      ,      return_attention_mask=lowerCAmelCase_      ,      return_token_type_ids=lowerCAmelCase_      ,      return_offsets_mapping=lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_      ,      )

										UpperCAmelCase_	:					int     	=   tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_      ,      "do_lower_case"		) else False
										UpperCAmelCase_	:					Dict     	=   (
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]      ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
										self.assertEqual([e[0] for e in expected_results]      ,      tokens["offset_mapping"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     str:
				UpperCAmelCase_	:					Dict     	=   ["的", "人", "有"]
				UpperCAmelCase_	:					Union[str, Any]     	=   "".join(lowerCAmelCase_		)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_	:					Tuple     	=   True
										UpperCAmelCase_	:					Optional[Any]     	=   self.tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)
										UpperCAmelCase_	:					Dict     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer_p.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
										UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer_r.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)

										UpperCAmelCase_	:					str     	=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_		)
										UpperCAmelCase_	:					Optional[Any]     	=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_		)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   False
										UpperCAmelCase_	:					str     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)
										UpperCAmelCase_	:					List[str]     	=   self.tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   tokenizer_r.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
										UpperCAmelCase_	:					int     	=   tokenizer_p.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)

										UpperCAmelCase_	:					Optional[Any]     	=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_		)
										UpperCAmelCase_	:					List[str]     	=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_		)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_	:					Dict     	=   [
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_		)
										]
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)



