




'''simple docstring'''



import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DiffusionPipeline,
    EulerDiscreteScheduler,
    StableDiffusionXLImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import floats_tensor, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()






class 							_a				(			SCREAMING_SNAKE_CASE ,      SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				Union[str, Any]        =						StableDiffusionXLImgaImgPipeline
       A						:				Any        =						TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'''height''', '''width'''}
       A						:				str        =						PipelineTesterMixin.required_optional_params - {'''latents'''}
       A						:				List[str]        =						TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
       A						:				Dict        =						IMAGE_TO_IMAGE_IMAGE_PARAMS
       A						:				int        =						IMAGE_TO_IMAGE_IMAGE_PARAMS







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             torch.manual_seed(0  )
             SCREAMING_SNAKE_CASE   :      Any    					=						UNetaDConditionModel(
                 block_out_channels=(32, 64),							layers_per_block=2,							sample_size=32,							in_channels=4,							out_channels=4,							down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D'),							up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D'),							attention_head_dim=(2, 4),							use_linear_projection=A,							addition_embed_type='text_time',							addition_time_embed_dim=8,							transformer_layers_per_block=(1, 2),							projection_class_embeddings_input_dim=80,							cross_attention_dim=64,							)
             SCREAMING_SNAKE_CASE   :      str    					=						EulerDiscreteScheduler(
                 beta_start=0.0_00_85,							beta_end=0.0_12,							steps_offset=1,							beta_schedule='scaled_linear',							timestep_spacing='leading',							)
             torch.manual_seed(0  )
             SCREAMING_SNAKE_CASE   :      Any    					=						AutoencoderKL(
                 block_out_channels=[32, 64],							in_channels=3,							out_channels=3,							down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D'],							up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D'],							latent_channels=4,							sample_size=128,							)
             torch.manual_seed(0  )
             SCREAMING_SNAKE_CASE   :      List[str]    					=						CLIPTextConfig(
                 bos_token_id=0,							eos_token_id=2,							hidden_size=32,							intermediate_size=37,							layer_norm_eps=1E-05,							num_attention_heads=4,							num_hidden_layers=5,							pad_token_id=1,							vocab_size=1_000,							hidden_act='gelu',							projection_dim=32,							)
             SCREAMING_SNAKE_CASE   :      int    					=						CLIPTextModel(A  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip',							local_files_only=A  )

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						CLIPTextModelWithProjection(A  )
             SCREAMING_SNAKE_CASE   :      Dict    					=						CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip',							local_files_only=A  )

             SCREAMING_SNAKE_CASE   :      List[str]    					=						{
                 'unet': unet,
                 'scheduler': scheduler,
                 'vae': vae,
                 'text_encoder': text_encoder,
                 'tokenizer': tokenizer,
                 'text_encoder_2': text_encoder_a,
                 'tokenizer_2': tokenizer_a,
                 # "safety_checker": None,
                 # "feature_extractor": None,
             }
             return components







       def        UpperCamelCase_							(      self,							A,							A=0  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						floats_tensor((1, 3, 32, 32),							rng=random.Random(A  )  ).to(A  )
             SCREAMING_SNAKE_CASE   :      str    					=						image / 2 + 0.5
             if str(A  ).startswith('mps'  ):
                   SCREAMING_SNAKE_CASE   :      Tuple    					=						torch.manual_seed(A  )
             else:
                   SCREAMING_SNAKE_CASE   :      Tuple    					=						torch.Generator(device=A  ).manual_seed(A  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						{
                 'prompt': 'A painting of a squirrel eating a burger',
                 'image': image,
                 'generator': generator,
                 'num_inference_steps': 2,
                 'guidance_scale': 5.0,
                 'output_type': 'numpy',
                 'strength': 0.75,
             }
             return inputs







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      int    					=						'cpu'  # ensure determinism for the device-dependent torch.Generator
             SCREAMING_SNAKE_CASE   :      str    					=						self.get_dummy_components()
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						StableDiffusionXLImgaImgPipeline(**A  )
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						sd_pipe.to(A  )
             sd_pipe.set_progress_bar_config(disable=A  )

             SCREAMING_SNAKE_CASE   :      List[str]    					=						self.get_dummy_inputs(A  )
             SCREAMING_SNAKE_CASE   :      Any    					=						sd_pipe(**A  ).images
             SCREAMING_SNAKE_CASE   :      Dict    					=						image[0, -3:, -3:, -1]

             assert image.shape == (1, 32, 32, 3)

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						np.array([0.46_56, 0.48_40, 0.44_39, 0.66_98, 0.55_74, 0.45_24, 0.57_99, 0.59_43, 0.51_65]  )

             assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1E-2







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().test_attention_slicing_forward_pass(expected_max_diff=3E-3  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().test_inference_batch_single_identical(expected_max_diff=3E-3  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.get_dummy_components()
             SCREAMING_SNAKE_CASE   :      List[str]    					=						StableDiffusionXLImgaImgPipeline(**A  )
             SCREAMING_SNAKE_CASE   :      str    					=						sd_pipe.to(A  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						sd_pipe.to(A  )
             sd_pipe.set_progress_bar_config(disable=A  )

             # forward without prompt embeds
             SCREAMING_SNAKE_CASE   :      List[str]    					=						self.get_dummy_inputs(A  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						3 * ['this is a negative prompt']
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						negative_prompt
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						3 * [inputs['prompt']]

             SCREAMING_SNAKE_CASE   :      int    					=						sd_pipe(**A  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						output.images[0, -3:, -3:, -1]

             # forward with prompt embeds
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.get_dummy_inputs(A  )
             SCREAMING_SNAKE_CASE   :      str    					=						3 * ['this is a negative prompt']
             SCREAMING_SNAKE_CASE   :      int    					=						3 * [inputs.pop('prompt'  )]

             (
                 (
                 SCREAMING_SNAKE_CASE
             )					,    (
                 SCREAMING_SNAKE_CASE
             )					,    (
                 SCREAMING_SNAKE_CASE
             )					,    (
                 SCREAMING_SNAKE_CASE
             )					,    
             )   :      Optional[Any]    					=						sd_pipe.encode_prompt(A,							negative_prompt=A  )

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						sd_pipe(
                 **A,							prompt_embeds=A,							negative_prompt_embeds=A,							pooled_prompt_embeds=A,							negative_pooled_prompt_embeds=A,							)
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						output.images[0, -3:, -3:, -1]

             # make sure that it's equal
             assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()  ).max() < 1E-4






@slow
@require_torch_gpu
class 							_a				(			unittest.TestCase ):





       '''simple docstring'''






       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().tearDown()
             gc.collect()
             torch.cuda.empty_cache()







       def        UpperCamelCase_							(      self,							A,							A="cpu",							A=torch.floataa,							A=0  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						torch.Generator(device=A  ).manual_seed(A  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						np.random.RandomState(A  ).standard_normal((1, 4, 64, 64)  )
             SCREAMING_SNAKE_CASE   :      str    					=						torch.from_numpy(A  ).to(device=A,							dtype=A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						{
                 'prompt': 'a photograph of an astronaut riding a horse',
                 'latents': latents,
                 'generator': generator,
                 'num_inference_steps': 3,
                 'guidance_scale': 7.5,
                 'output_type': 'numpy',
             }
             return inputs







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						DiffusionPipeline.from_pretrained('stabilityai/stable-diffusion-2-base'  )
             pipe.to(A  )
             pipe.set_progress_bar_config(disable=A  )

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.get_inputs(A  )
             SCREAMING_SNAKE_CASE   :      str    					=						pipe(**A  ).images
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						image[0, -3:, -3:, -1].flatten()

             assert image.shape == (1, 512, 512, 3)
             SCREAMING_SNAKE_CASE   :      Dict    					=						np.array([0.4_94_93, 0.4_78_96, 0.4_07_98, 0.5_42_14, 0.5_32_12, 0.4_82_02, 0.4_76_56, 0.4_63_29, 0.4_85_06]  )
             assert np.abs(image_slice - expected_slice  ).max() < 7E-3








'''simple docstring'''



import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							_a				(			SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				str        =						LongformerTokenizer
       A						:				List[str]        =						True
       A						:				Optional[int]        =						LongformerTokenizerFast
       A						:				Tuple        =						True







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             SCREAMING_SNAKE_CASE   :      Any    					=						[
                 'l',
                 'o',
                 'w',
                 'e',
                 'r',
                 's',
                 't',
                 'i',
                 'd',
                 'n',
                 '\u0120',
                 '\u0120l',
                 '\u0120n',
                 '\u0120lo',
                 '\u0120low',
                 'er',
                 '\u0120lowest',
                 '\u0120newer',
                 '\u0120wider',
                 '<unk>',
             ]
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						dict(zip(A,							range(len(A  )  )  )  )
             SCREAMING_SNAKE_CASE   :      str    					=						['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
             SCREAMING_SNAKE_CASE   :      Tuple    					=						{'unk_token': '<unk>'}

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						os.path.join(self.tmpdirname,							VOCAB_FILES_NAMES['vocab_file']  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						os.path.join(self.tmpdirname,							VOCAB_FILES_NAMES['merges_file']  )
             with open(self.vocab_file,							'w',							encoding='utf-8'  ) as fp:
                   fp.write(json.dumps(A  ) + '\n'  )
             with open(self.merges_file,							'w',							encoding='utf-8'  ) as fp:
                   fp.write('\n'.join(A  )  )







       def        UpperCamelCase_							(      self,							**A  ):







             '''simple docstring'''







             kwargs.update(self.special_tokens_map  )
             return self.tokenizer_class.from_pretrained(self.tmpdirname,							**A  )







       def        UpperCamelCase_							(      self,							**A  ):







             '''simple docstring'''







             kwargs.update(self.special_tokens_map  )
             return self.rust_tokenizer_class.from_pretrained(self.tmpdirname,							**A  )







       def        UpperCamelCase_							(      self,							A  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      int    					=						'lower newer'
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						'lower newer'
             return input_text, output_text







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.tokenizer_class(self.vocab_file,							self.merges_file,							**self.special_tokens_map  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						'lower newer'
             SCREAMING_SNAKE_CASE   :      List[str]    					=						['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.tokenize(A  )  # , add_prefix_space=True)
             self.assertListEqual(A,							A  )

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokens + [tokenizer.unk_token]
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(A  ),							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.get_tokenizer()

             self.assertListEqual(tokenizer.encode('Hello world!',							add_special_tokens=A  ),							[0, 31_414, 232, 328, 2]  )
             self.assertListEqual(
                 tokenizer.encode('Hello world! cécé herlolip 418',							add_special_tokens=A  ),							[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2],							)







       @slow
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      str    					=						self.tokenizer_class.from_pretrained('allenai/longformer-base-4096'  )

             SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.encode('sequence builders',							add_special_tokens=A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.encode('multi-sequence build',							add_special_tokens=A  )

             SCREAMING_SNAKE_CASE   :      int    					=						tokenizer.encode(
                 'sequence builders',							add_special_tokens=A,							add_prefix_space=A  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.encode(
                 'sequence builders',							'multi-sequence build',							add_special_tokens=A,							add_prefix_space=A  )

             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.build_inputs_with_special_tokens(A  )
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer.build_inputs_with_special_tokens(A,							A  )

             assert encoded_sentence == encoded_text_from_decode
             assert encoded_pair == encoded_pair_from_decode







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.get_tokenizer()

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						'Encode this sequence.'
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.byte_encoder[' '.encode('utf-8'  )[0]]

             # Testing encoder arguments
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.encode(A,							add_special_tokens=A,							add_prefix_space=A  )
             SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer.convert_ids_to_tokens(encoded[0]  )[0]
             self.assertNotEqual(A,							A  )

             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.encode(A,							add_special_tokens=A,							add_prefix_space=A  )
             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.convert_ids_to_tokens(encoded[0]  )[0]
             self.assertEqual(A,							A  )

             tokenizer.add_special_tokens({'bos_token': '<s>'}  )
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.encode(A,							add_special_tokens=A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.convert_ids_to_tokens(encoded[1]  )[0]
             self.assertNotEqual(A,							A  )

             # Testing spaces after special tokens
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						'<mask>'
             tokenizer.add_special_tokens(
                 {'mask_token': AddedToken(A,							lstrip=A,							rstrip=A  )}  )  # mask token has a left space
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.convert_tokens_to_ids(A  )

             SCREAMING_SNAKE_CASE   :      List[str]    					=						'Encode <mask> sequence'
             SCREAMING_SNAKE_CASE   :      List[str]    					=						'Encode <mask>sequence'

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer.encode(A  )
             SCREAMING_SNAKE_CASE   :      Tuple    					=						encoded.index(A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]  )[0]
             self.assertEqual(A,							A  )

             SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.encode(A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						encoded.index(A  )
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]  )[0]
             self.assertNotEqual(A,							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
                         SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.rust_tokenizer_class.from_pretrained(A,							**A  )
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						self.tokenizer_class.from_pretrained(A,							**A  )
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						'A, <mask> AllenNLP sentence.'
                         SCREAMING_SNAKE_CASE   :      Any    					=						tokenizer_r.encode_plus(A,							add_special_tokens=A,							return_token_type_ids=A  )
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer_p.encode_plus(A,							add_special_tokens=A,							return_token_type_ids=A  )

                         # token_type_ids should put 0 everywhere
                         self.assertEqual(sum(tokens_r['token_type_ids']  ),							sum(tokens_p['token_type_ids']  )  )

                         # attention_mask should put 1 everywhere, so sum over length should be 1
                         self.assertEqual(
                             sum(tokens_r['attention_mask']  ) / len(tokens_r['attention_mask']  ),							sum(tokens_p['attention_mask']  ) / len(tokens_p['attention_mask']  ),							)

                         SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']  )
                         SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']  )

                         # Rust correctly handles the space before the mask while python doesnt
                         self.assertSequenceEqual(tokens_p['input_ids'],							[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]  )
                         self.assertSequenceEqual(tokens_r['input_ids'],							[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]  )

                         self.assertSequenceEqual(
                             A,							['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']  )
                         self.assertSequenceEqual(
                             A,							['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             for trim_offsets, add_prefix_space in itertools.product([True, False],							repeat=2  ):
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.rust_tokenizer_class.from_pretrained(
                       self.tmpdirname,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )

                   SCREAMING_SNAKE_CASE   :      Tuple    					=						json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()  )
                   SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()  )

                   self.assertEqual(pre_tokenizer_state['add_prefix_space'],							A  )

                   self.assertEqual(post_processor_state['add_prefix_space'],							A  )
                   self.assertEqual(post_processor_state['trim_offsets'],							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
                         SCREAMING_SNAKE_CASE   :      str    					=						'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						F"{text_of_1_token} {text_of_1_token}"

                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ) + 1, len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ) + 1, len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      List[str]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ), len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Any    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(len(A  ), len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Any    					=						F" {text}"

                         # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                         #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                         # )
                         # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                         # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                         # self.assertEqual(
                         #     encoding.offset_mapping[1],
                         #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                         # )

                         SCREAMING_SNAKE_CASE   :      str    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(1, 1 + len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(1 + len(A  ) + 1, 1 + len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      str    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, 1 + len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(1 + len(A  ), 1 + len(A  ) + 1 + len(A  )),							)

                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.rust_tokenizer_class.from_pretrained(
                             A,							use_fast=A,							add_prefix_space=A,							trim_offsets=A  )
                         SCREAMING_SNAKE_CASE   :      List[Any]    					=						tokenizer_r(A,							return_offsets_mapping=A,							add_special_tokens=A  )
                         self.assertEqual(encoding.offset_mapping[0],							(0, 1 + len(A  ))  )
                         self.assertEqual(
                             encoding.offset_mapping[1],							(1 + len(A  ), 1 + len(A  ) + 1 + len(A  )),							)


