
import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline
from diffusers.pipelines.shap_e import ShapERenderer
from diffusers.utils import load_numpy, slow
from diffusers.utils.testing_utils import require_torch_gpu, torch_device

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference





class     __UpperCamelCase (							lowerCAmelCase__ ,						unittest.TestCase       ):






     """simple docstring"""







     lowerCAmelCase_							   =			ShapEPipeline
     lowerCAmelCase_							   =			['''prompt''']
     lowerCAmelCase_							   =			['''prompt''']
     lowerCAmelCase_							   =			[
         '''num_images_per_prompt''',
         '''num_inference_steps''',
         '''generator''',
         '''latents''',
         '''guidance_scale''',
         '''frame_size''',
         '''output_type''',
         '''return_dict''',
     ]
     lowerCAmelCase_							   =			False



     @property
     def  UpperCAmelCase__							(				self							:							Optional[int] ):







       """simple docstring"""




       return 32



     @property
     def  UpperCAmelCase__							(				self							:							Tuple ):







       """simple docstring"""




       return 32



     @property
     def  UpperCAmelCase__							(				self							:							Union[str, Any] ):







       """simple docstring"""




       return self.time_input_dim * 4



     @property
     def  UpperCAmelCase__							(				self							:							int ):







       """simple docstring"""




       return 8



     @property
     def  UpperCAmelCase__							(				self							:							Any ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip''' )
       return tokenizer



     @property
     def  UpperCAmelCase__							(				self							:							Any ):







       """simple docstring"""




       torch.manual_seed(0 )
       __SCREAMING_SNAKE_CASE      :	Dict					=		CLIPTextConfig(
           bos_token_id=0     ,						eos_token_id=2     ,						hidden_size=self.text_embedder_hidden_size     ,						projection_dim=self.text_embedder_hidden_size     ,						intermediate_size=37     ,						layer_norm_eps=1e-05     ,						num_attention_heads=4     ,						num_hidden_layers=5     ,						pad_token_id=1     ,						vocab_size=1000     ,						)
       return CLIPTextModelWithProjection(_A )



     @property
     def  UpperCAmelCase__							(				self							:							List[Any] ):







       """simple docstring"""




       torch.manual_seed(0 )

       __SCREAMING_SNAKE_CASE      :	Any					=		{
           '''num_attention_heads''': 2,
           '''attention_head_dim''': 16,
           '''embedding_dim''': self.time_input_dim,
           '''num_embeddings''': 32,
           '''embedding_proj_dim''': self.text_embedder_hidden_size,
           '''time_embed_dim''': self.time_embed_dim,
           '''num_layers''': 1,
           '''clip_embed_dim''': self.time_input_dim * 2,
           '''additional_embeddings''': 0,
           '''time_embed_act_fn''': '''gelu''',
           '''norm_in_type''': '''layer''',
           '''encoder_hid_proj_type''': None,
           '''added_emb_type''': None,
       }

       __SCREAMING_SNAKE_CASE      :	List[Any]					=		PriorTransformer(**_A )
       return model



     @property
     def  UpperCAmelCase__							(				self							:							str ):







       """simple docstring"""




       torch.manual_seed(0 )

       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		{
           '''param_shapes''': (
               (self.renderer_dim, 93),
               (self.renderer_dim, 8),
               (self.renderer_dim, 8),
               (self.renderer_dim, 8),
           ),
           '''d_latent''': self.time_input_dim,
           '''d_hidden''': self.renderer_dim,
           '''n_output''': 12,
           '''background''': (
               0.1,
               0.1,
               0.1,
           ),
       }
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		ShapERenderer(**_A )
       return model



     def  UpperCAmelCase__							(				self							:							int ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	str					=		self.dummy_prior
       __SCREAMING_SNAKE_CASE      :	int					=		self.dummy_text_encoder
       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		self.dummy_tokenizer
       __SCREAMING_SNAKE_CASE      :	List[Any]					=		self.dummy_renderer

       __SCREAMING_SNAKE_CASE      :	Tuple					=		HeunDiscreteScheduler(
           beta_schedule='''exp'''     ,						num_train_timesteps=1024     ,						prediction_type='''sample'''     ,						use_karras_sigmas=_A     ,						clip_sample=_A     ,						clip_sample_range=1.0     ,						)
       __SCREAMING_SNAKE_CASE      :	Dict					=		{
           '''prior''': prior,
           '''text_encoder''': text_encoder,
           '''tokenizer''': tokenizer,
           '''renderer''': renderer,
           '''scheduler''': scheduler,
       }

       return components



     def  UpperCAmelCase__							(				self							:							Optional[int]     ,						_A							:							Dict     ,						_A							:							Optional[int]=0 ):







       """simple docstring"""




       if str(_A ).startswith('''mps''' ):
         __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		torch.manual_seed(_A )
       else:
         __SCREAMING_SNAKE_CASE      :	Dict					=		torch.Generator(device=_A ).manual_seed(_A )
       __SCREAMING_SNAKE_CASE      :	int					=		{
           '''prompt''': '''horse''',
           '''generator''': generator,
           '''num_inference_steps''': 1,
           '''frame_size''': 32,
           '''output_type''': '''np''',
       }
       return inputs



     def  UpperCAmelCase__							(				self							:							List[Any] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Any					=		'''cpu'''

       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		self.get_dummy_components()

       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		self.pipeline_class(**_A )
       __SCREAMING_SNAKE_CASE      :	str					=		pipe.to(_A )

       pipe.set_progress_bar_config(disable=_A )

       __SCREAMING_SNAKE_CASE      :	Dict					=		pipe(**self.get_dummy_inputs(_A ) )
       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		output.images[0]
       __SCREAMING_SNAKE_CASE      :	Any					=		image[0, -3:, -3:, -1]

       assert image.shape == (20, 32, 32, 3)

       __SCREAMING_SNAKE_CASE      :	str					=		np.array(
           [
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
               0.00_03_92_16,
           ] )

       assert np.abs(image_slice.flatten() - expected_slice ).max() < 1e-2



     def  UpperCAmelCase__							(				self							:							str ):







       """simple docstring"""




       self._test_inference_batch_consistent(batch_sizes=[1, 2] )



     def  UpperCAmelCase__							(				self							:							Optional[Any] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	int					=		torch_device == '''cpu'''
       __SCREAMING_SNAKE_CASE      :	List[str]					=		True

       self._test_inference_batch_single_identical(
           batch_size=2     ,						test_max_difference=_A     ,						relax_max_difference=_A     ,						)




     def  UpperCAmelCase__							(				self							:							Any ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		self.get_dummy_components()
       __SCREAMING_SNAKE_CASE      :	Tuple					=		self.pipeline_class(**_A )
       __SCREAMING_SNAKE_CASE      :	Dict					=		pipe.to(_A )
       pipe.set_progress_bar_config(disable=_A )

       __SCREAMING_SNAKE_CASE      :	Tuple					=		1
       __SCREAMING_SNAKE_CASE      :	Any					=		2

       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		self.get_dummy_inputs(_A )

       for key in inputs.keys():
         if key in self.batch_params:
           __SCREAMING_SNAKE_CASE      :	str					=		batch_size * [inputs[key]]

       __SCREAMING_SNAKE_CASE      :	str					=		pipe(**_A     ,						num_images_per_prompt=_A )[0]

       assert images.shape[0] == batch_size * num_images_per_prompt






@slow
@require_torch_gpu
class     __UpperCamelCase (							unittest.TestCase       ):






     """simple docstring"""







     def  UpperCAmelCase__							(				self							:							int ):







       """simple docstring"""




       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()




     def  UpperCAmelCase__							(				self							:							str ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		load_numpy(
           '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
           '''/shap_e/test_shap_e_np_out.npy''' )
       __SCREAMING_SNAKE_CASE      :	str					=		ShapEPipeline.from_pretrained('''openai/shap-e''' )
       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		pipe.to(_A )
       pipe.set_progress_bar_config(disable=_A )

       __SCREAMING_SNAKE_CASE      :	Any					=		torch.Generator(device=_A ).manual_seed(0 )

       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		pipe(
           '''a shark'''     ,						generator=_A     ,						guidance_scale=15.0     ,						num_inference_steps=64     ,						frame_size=64     ,						output_type='''np'''     ,						).images[0]

       assert images.shape == (20, 64, 64, 3)

       assert_mean_pixel_difference(_A     ,						_A )








"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class      UpperCamelCase_			(__A							,		unittest.TestCase	):
	__magic_name__       =							BertTokenizer
	__magic_name__       =							BertTokenizerFast
	__magic_name__       =							True
	__magic_name__       =							True
	__magic_name__       =							filter_non_english





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     Optional[Any]:
				super().setUp()

				UpperCAmelCase_	:					Optional[int]     	=   [
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				UpperCAmelCase_	:					Optional[Any]     	=   os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["vocab_file"]		)
				with open(self.vocab_file      ,      "w"      ,      encoding="utf-8"		) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]      ,      lowerCAmelCase_   :					List[str]		)		->     List[str]:
				UpperCAmelCase_	:					str     	=   "UNwant\u00E9d,running"
				UpperCAmelCase_	:					Optional[int]     	=   "unwanted, running"
				return input_text, output_text





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Any:
				UpperCAmelCase_	:					Optional[int]     	=   self.tokenizer_class(self.vocab_file		)

				UpperCAmelCase_	:					int     	=   tokenizer.tokenize("UNwant\u00E9d,running"		)
				self.assertListEqual(lowerCAmelCase_      ,      ["un", "##want", "##ed", ",", "runn", "##ing"]		)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)      ,      [9, 6, 7, 12, 10, 11]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     List[Any]:
				if not self.test_rust_tokenizer:
							return

				UpperCAmelCase_	:					List[Any]     	=   self.get_tokenizer()
				UpperCAmelCase_	:					str     	=   self.get_rust_tokenizer()

				UpperCAmelCase_	:					str     	=   "UNwant\u00E9d,running"

				UpperCAmelCase_	:					Any     	=   tokenizer.tokenize(lowerCAmelCase_		)
				UpperCAmelCase_	:					List[Any]     	=   rust_tokenizer.tokenize(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					List[str]     	=   rust_tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Dict     	=   self.get_rust_tokenizer()
				UpperCAmelCase_	:					int     	=   tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_	:					Optional[Any]     	=   rust_tokenizer.encode(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				# With lower casing
				UpperCAmelCase_	:					int     	=   self.get_tokenizer(do_lower_case=lowerCAmelCase_		)
				UpperCAmelCase_	:					str     	=   self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_		)

				UpperCAmelCase_	:					Any     	=   "UNwant\u00E9d,running"

				UpperCAmelCase_	:					Dict     	=   tokenizer.tokenize(lowerCAmelCase_		)
				UpperCAmelCase_	:					Union[str, Any]     	=   rust_tokenizer.tokenize(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Optional[int]     	=   tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					Any     	=   rust_tokenizer.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

				UpperCAmelCase_	:					Union[str, Any]     	=   self.get_rust_tokenizer()
				UpperCAmelCase_	:					Dict     	=   tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_	:					Union[str, Any]     	=   rust_tokenizer.encode(lowerCAmelCase_		)
				self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     List[str]:
				UpperCAmelCase_	:					Union[str, Any]     	=   BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)      ,      ["ah", "\u535A", "\u63A8", "zz"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple		)		->     Optional[int]:
				UpperCAmelCase_	:					Union[str, Any]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,      ["hello", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Optional[Any]:
				UpperCAmelCase_	:					Dict     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hällo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["h\u00E9llo"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any		)		->     Any:
				UpperCAmelCase_	:					str     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hallo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					List[str]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["hallo", "!", "how", "are", "you", "?"]		)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,      ["hello"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					List[Any]     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Union[str, Any]:
				UpperCAmelCase_	:					str     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     str:
				UpperCAmelCase_	:					Any     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Union[str, Any]		)		->     Dict:
				UpperCAmelCase_	:					Dict     	=   BasicTokenizer(do_lower_case=lowerCAmelCase_      ,      never_split=["[UNK]"]		)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)      ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Tuple		)		->     Union[str, Any]:
				UpperCAmelCase_	:					Optional[int]     	=   BasicTokenizer()
				UpperCAmelCase_	:					int     	=   "a\n'll !!to?'d of, can't."
				UpperCAmelCase_	:					Union[str, Any]     	=   ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
				self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_		)      ,      lowerCAmelCase_		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]		)		->     Dict:
				UpperCAmelCase_	:					List[str]     	=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				UpperCAmelCase_	:					int     	=   {}
				for i, token in enumerate(lowerCAmelCase_		):
							UpperCAmelCase_	:					Tuple     	=   i
				UpperCAmelCase_	:					Dict     	=   WordpieceTokenizer(vocab=lowerCAmelCase_      ,      unk_token="[UNK]"		)

				self.assertListEqual(tokenizer.tokenize(""		)      ,      []		)

				self.assertListEqual(tokenizer.tokenize("unwanted running"		)      ,      ["un", "##want", "##ed", "runn", "##ing"]		)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"		)      ,      ["[UNK]", "runn", "##ing"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					Dict		)		->     Any:
				self.assertTrue(_is_whitespace(" "		)		)
				self.assertTrue(_is_whitespace("\t"		)		)
				self.assertTrue(_is_whitespace("\r"		)		)
				self.assertTrue(_is_whitespace("\n"		)		)
				self.assertTrue(_is_whitespace("\u00A0"		)		)

				self.assertFalse(_is_whitespace("A"		)		)
				self.assertFalse(_is_whitespace("-"		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Any:
				self.assertTrue(_is_control("\u0005"		)		)

				self.assertFalse(_is_control("A"		)		)
				self.assertFalse(_is_control(" "		)		)
				self.assertFalse(_is_control("\t"		)		)
				self.assertFalse(_is_control("\r"		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]		)		->     int:
				self.assertTrue(_is_punctuation("-"		)		)
				self.assertTrue(_is_punctuation("$"		)		)
				self.assertTrue(_is_punctuation("`"		)		)
				self.assertTrue(_is_punctuation("."		)		)

				self.assertFalse(_is_punctuation("A"		)		)
				self.assertFalse(_is_punctuation(" "		)		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     str:
				UpperCAmelCase_	:					List[Any]     	=   self.get_tokenizer()
				UpperCAmelCase_	:					List[Any]     	=   self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_		) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]		)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(lowerCAmelCase_		) for t in ["Test", "\xad", "test"]]      ,      [["[UNK]"], [], ["[UNK]"]]		)





	@slow
	def 	_SCREAMING_SNAKE_CASE ( self   :					int		)		->     List[Any]:
				UpperCAmelCase_	:					str     	=   self.tokenizer_class.from_pretrained("bert-base-uncased"		)

				UpperCAmelCase_	:					Optional[Any]     	=   tokenizer.encode("sequence builders"      ,      add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_	:					str     	=   tokenizer.encode("multi-sequence build"      ,      add_special_tokens=lowerCAmelCase_		)

				UpperCAmelCase_	:					str     	=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_		)
				UpperCAmelCase_	:					Tuple     	=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_      ,      lowerCAmelCase_		)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[int]		)		->     List[str]:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_	:					Optional[int]     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										UpperCAmelCase_	:					Dict     	=   tokenizer_r.encode_plus(
										    lowerCAmelCase_      ,      return_attention_mask=lowerCAmelCase_      ,      return_token_type_ids=lowerCAmelCase_      ,      return_offsets_mapping=lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_      ,      )

										UpperCAmelCase_	:					int     	=   tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_      ,      "do_lower_case"		) else False
										UpperCAmelCase_	:					Dict     	=   (
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results]      ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
										self.assertEqual([e[0] for e in expected_results]      ,      tokens["offset_mapping"]		)





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     str:
				UpperCAmelCase_	:					Dict     	=   ["的", "人", "有"]
				UpperCAmelCase_	:					Union[str, Any]     	=   "".join(lowerCAmelCase_		)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_	:					Tuple     	=   True
										UpperCAmelCase_	:					Optional[Any]     	=   self.tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)
										UpperCAmelCase_	:					Dict     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer_p.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
										UpperCAmelCase_	:					Union[str, Any]     	=   tokenizer_r.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)

										UpperCAmelCase_	:					str     	=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_		)
										UpperCAmelCase_	:					Optional[Any]     	=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_		)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   False
										UpperCAmelCase_	:					str     	=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)
										UpperCAmelCase_	:					List[str]     	=   self.tokenizer_class.from_pretrained(lowerCAmelCase_      ,      **lowerCAmelCase_		)

										UpperCAmelCase_	:					int     	=   tokenizer_r.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)
										UpperCAmelCase_	:					int     	=   tokenizer_p.encode(lowerCAmelCase_      ,      add_special_tokens=lowerCAmelCase_		)

										UpperCAmelCase_	:					Optional[Any]     	=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_		)
										UpperCAmelCase_	:					List[str]     	=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_		)

										# it is expected that only the first Chinese character is not preceded by "##".
										UpperCAmelCase_	:					Dict     	=   [
										    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_		)
										]
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)
										self.assertListEqual(lowerCAmelCase_      ,      lowerCAmelCase_		)



