






# This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    ControlNetModel,
    DDIMScheduler,
    StableDiffusionControlNetImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from diffusers.utils import floats_tensor, load_image, load_numpy, randn_tensor, slow, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
)


enable_full_determinism()
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      UpperCAmelCase					,      UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  StableDiffusionControlNetImgaImgPipeline
  _lowerCamelCase    =  TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'''height''', '''width'''}
  _lowerCamelCase    =  TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
  _lowerCamelCase    =  IMAGE_TO_IMAGE_IMAGE_PARAMS.union({'''control_image'''}     )
  _lowerCamelCase    =  IMAGE_TO_IMAGE_IMAGE_PARAMS


  def  lowerCamelCase       (      self		:			str							)						->	List[str]:


         """simple docstring"""
         torch.manual_seed(0							)
         _UpperCAmelCase							=    UNetaDConditionModel(
             block_out_channels=(32, 64)       ,				layers_per_block=2       ,				sample_size=32       ,				in_channels=4       ,				out_channels=4       ,				down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")       ,				up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")       ,				cross_attention_dim=32       ,				)
         torch.manual_seed(0							)
         _UpperCAmelCase							=    ControlNetModel(
             block_out_channels=(32, 64)       ,				layers_per_block=2       ,				in_channels=4       ,				down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")       ,				cross_attention_dim=32       ,				conditioning_embedding_out_channels=(16, 32)       ,				)
         torch.manual_seed(0							)
         _UpperCAmelCase							=    DDIMScheduler(
             beta_start=0.0_0085       ,				beta_end=0.012       ,				beta_schedule="""scaled_linear"""       ,				clip_sample=lowerCamelCase       ,				set_alpha_to_one=lowerCamelCase       ,				)
         torch.manual_seed(0							)
         _UpperCAmelCase							=    AutoencoderKL(
             block_out_channels=[32, 64]       ,				in_channels=3       ,				out_channels=3       ,				down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]       ,				up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]       ,				latent_channels=4       ,				)
         torch.manual_seed(0							)
         _UpperCAmelCase							=    CLIPTextConfig(
             bos_token_id=0       ,				eos_token_id=2       ,				hidden_size=32       ,				intermediate_size=37       ,				layer_norm_eps=1E-05       ,				num_attention_heads=4       ,				num_hidden_layers=5       ,				pad_token_id=1       ,				vocab_size=1000       ,				)
         _UpperCAmelCase							=    CLIPTextModel(lowerCamelCase							)
         _UpperCAmelCase							=    CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""							)

         _UpperCAmelCase							=    {
             """unet""": unet,
             """controlnet""": controlnet,
             """scheduler""": scheduler,
             """vae""": vae,
             """text_encoder""": text_encoder,
             """tokenizer""": tokenizer,
             """safety_checker""": None,
             """feature_extractor""": None,
         }
         return components


  def  lowerCamelCase       (      self		:			Optional[int]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			int=0							)						->	Any:


         """simple docstring"""
         if str(lowerCamelCase							).startswith("""mps"""							):
                _UpperCAmelCase							=    torch.manual_seed(lowerCamelCase							)
         else:
                _UpperCAmelCase							=    torch.Generator(device=lowerCamelCase							).manual_seed(lowerCamelCase							)

         _UpperCAmelCase							=    2
         _UpperCAmelCase							=    randn_tensor(
             (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)       ,				generator=lowerCamelCase       ,				device=torch.device(lowerCamelCase							)       ,				)
         _UpperCAmelCase							=    floats_tensor(control_image.shape       ,				rng=random.Random(lowerCamelCase							)							).to(lowerCamelCase							)
         _UpperCAmelCase							=    image.cpu().permute(0       ,				2       ,				3       ,				1							)[0]
         _UpperCAmelCase							=    Image.fromarray(np.uinta(lowerCamelCase							)							).convert("""RGB"""							).resize((64, 64)							)
         _UpperCAmelCase							=    {
             """prompt""": """A painting of a squirrel eating a burger""",
             """generator""": generator,
             """num_inference_steps""": 2,
             """guidance_scale""": 6.0,
             """output_type""": """numpy""",
             """image""": image,
             """control_image""": control_image,
         }

         return inputs


  def  lowerCamelCase       (      self		:			Union[str, Any]							)						->	Union[str, Any]:


         """simple docstring"""
         return self._test_attention_slicing_forward_pass(expected_max_diff=2E-3							)


  @unittest.skipIf(
      torch_device != """cuda""" or not is_xformers_available()       ,				reason="""XFormers attention is only available with CUDA and `xformers` installed"""       ,				)
  def  lowerCamelCase       (      self		:			Optional[int]							)						->	List[str]:


         """simple docstring"""
         self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2E-3							)





  def  lowerCamelCase       (      self		:			Any							)						->	Tuple:


         """simple docstring"""
         self._test_inference_batch_single_identical(expected_max_diff=2E-3							)



class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  StableDiffusionControlNetImgaImgPipeline
  _lowerCamelCase    =  TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'''height''', '''width'''}
  _lowerCamelCase    =  TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
  _lowerCamelCase    =  frozenset([]     )  # TO_DO: add image_params once refactored VaeImageProcessor.preprocess


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	Optional[int]:


         """simple docstring"""
         torch.manual_seed(0							)
         _UpperCAmelCase							=    UNetaDConditionModel(
             block_out_channels=(32, 64)       ,				layers_per_block=2       ,				sample_size=32       ,				in_channels=4       ,				out_channels=4       ,				down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")       ,				up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")       ,				cross_attention_dim=32       ,				)
         torch.manual_seed(0							)

         def init_weights(lowerCamelCase		:			Dict							):
                if isinstance(lowerCamelCase       ,				torch.nn.Convad							):
                       torch.nn.init.normal(m.weight							)
                       m.bias.data.fill_(1.0							)

         _UpperCAmelCase							=    ControlNetModel(
             block_out_channels=(32, 64)       ,				layers_per_block=2       ,				in_channels=4       ,				down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")       ,				cross_attention_dim=32       ,				conditioning_embedding_out_channels=(16, 32)       ,				)
         controlneta.controlnet_down_blocks.apply(lowerCamelCase							)

         torch.manual_seed(0							)
         _UpperCAmelCase							=    ControlNetModel(
             block_out_channels=(32, 64)       ,				layers_per_block=2       ,				in_channels=4       ,				down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")       ,				cross_attention_dim=32       ,				conditioning_embedding_out_channels=(16, 32)       ,				)
         controlneta.controlnet_down_blocks.apply(lowerCamelCase							)

         torch.manual_seed(0							)
         _UpperCAmelCase							=    DDIMScheduler(
             beta_start=0.0_0085       ,				beta_end=0.012       ,				beta_schedule="""scaled_linear"""       ,				clip_sample=lowerCamelCase       ,				set_alpha_to_one=lowerCamelCase       ,				)
         torch.manual_seed(0							)
         _UpperCAmelCase							=    AutoencoderKL(
             block_out_channels=[32, 64]       ,				in_channels=3       ,				out_channels=3       ,				down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]       ,				up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]       ,				latent_channels=4       ,				)
         torch.manual_seed(0							)
         _UpperCAmelCase							=    CLIPTextConfig(
             bos_token_id=0       ,				eos_token_id=2       ,				hidden_size=32       ,				intermediate_size=37       ,				layer_norm_eps=1E-05       ,				num_attention_heads=4       ,				num_hidden_layers=5       ,				pad_token_id=1       ,				vocab_size=1000       ,				)
         _UpperCAmelCase							=    CLIPTextModel(lowerCamelCase							)
         _UpperCAmelCase							=    CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""							)

         _UpperCAmelCase							=    MultiControlNetModel([controlneta, controlneta]							)

         _UpperCAmelCase							=    {
             """unet""": unet,
             """controlnet""": controlnet,
             """scheduler""": scheduler,
             """vae""": vae,
             """text_encoder""": text_encoder,
             """tokenizer""": tokenizer,
             """safety_checker""": None,
             """feature_extractor""": None,
         }
         return components


  def  lowerCamelCase       (      self		:			Any       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Optional[int]=0							)						->	Dict:


         """simple docstring"""
         if str(lowerCamelCase							).startswith("""mps"""							):
                _UpperCAmelCase							=    torch.manual_seed(lowerCamelCase							)
         else:
                _UpperCAmelCase							=    torch.Generator(device=lowerCamelCase							).manual_seed(lowerCamelCase							)

         _UpperCAmelCase							=    2

         _UpperCAmelCase							=    [
             randn_tensor(
                 (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)       ,				generator=lowerCamelCase       ,				device=torch.device(lowerCamelCase							)       ,				),
             randn_tensor(
                 (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)       ,				generator=lowerCamelCase       ,				device=torch.device(lowerCamelCase							)       ,				),
         ]

         _UpperCAmelCase							=    floats_tensor(control_image[0].shape       ,				rng=random.Random(lowerCamelCase							)							).to(lowerCamelCase							)
         _UpperCAmelCase							=    image.cpu().permute(0       ,				2       ,				3       ,				1							)[0]
         _UpperCAmelCase							=    Image.fromarray(np.uinta(lowerCamelCase							)							).convert("""RGB"""							).resize((64, 64)							)
         _UpperCAmelCase							=    {
             """prompt""": """A painting of a squirrel eating a burger""",
             """generator""": generator,
             """num_inference_steps""": 2,
             """guidance_scale""": 6.0,
             """output_type""": """numpy""",
             """image""": image,
             """control_image""": control_image,
         }

         return inputs


  def  lowerCamelCase       (      self		:			Union[str, Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_dummy_components()
         _UpperCAmelCase							=    self.pipeline_class(**lowerCamelCase							)
         pipe.to(lowerCamelCase							)

         _UpperCAmelCase							=    10.0
         _UpperCAmelCase							=    4

         _UpperCAmelCase							=    self.get_dummy_inputs(lowerCamelCase							)
         _UpperCAmelCase							=    steps
         _UpperCAmelCase							=    scale
         _UpperCAmelCase							=    pipe(**lowerCamelCase							)[0]

         _UpperCAmelCase							=    self.get_dummy_inputs(lowerCamelCase							)
         _UpperCAmelCase							=    steps
         _UpperCAmelCase							=    scale
         _UpperCAmelCase							=    pipe(**lowerCamelCase       ,				control_guidance_start=0.1       ,				control_guidance_end=0.2							)[0]

         _UpperCAmelCase							=    self.get_dummy_inputs(lowerCamelCase							)
         _UpperCAmelCase							=    steps
         _UpperCAmelCase							=    scale
         _UpperCAmelCase							=    pipe(**lowerCamelCase       ,				control_guidance_start=[0.1, 0.3]       ,				control_guidance_end=[0.2, 0.7]							)[0]

         _UpperCAmelCase							=    self.get_dummy_inputs(lowerCamelCase							)
         _UpperCAmelCase							=    steps
         _UpperCAmelCase							=    scale
         _UpperCAmelCase							=    pipe(**lowerCamelCase       ,				control_guidance_start=0.4       ,				control_guidance_end=[0.5, 0.8]							)[0]

         # make sure that all outputs are different
         assert np.sum(np.abs(output_a - output_a							)							) > 1E-3
         assert np.sum(np.abs(output_a - output_a							)							) > 1E-3
         assert np.sum(np.abs(output_a - output_a							)							) > 1E-3


  def  lowerCamelCase       (      self		:			Tuple							)						->	Optional[Any]:


         """simple docstring"""
         return self._test_attention_slicing_forward_pass(expected_max_diff=2E-3							)


  @unittest.skipIf(
      torch_device != """cuda""" or not is_xformers_available()       ,				reason="""XFormers attention is only available with CUDA and `xformers` installed"""       ,				)
  def  lowerCamelCase       (      self		:			List[str]							)						->	List[str]:


         """simple docstring"""
         self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2E-3							)


  def  lowerCamelCase       (      self		:			Tuple							)						->	Dict:


         """simple docstring"""
         self._test_inference_batch_single_identical(expected_max_diff=2E-3							)





  def  lowerCamelCase       (      self		:			Dict							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_dummy_components()
         _UpperCAmelCase							=    self.pipeline_class(**lowerCamelCase							)
         pipe.to(lowerCamelCase							)
         pipe.set_progress_bar_config(disable=lowerCamelCase							)
         with tempfile.TemporaryDirectory() as tmpdir:
                try:
                       # save_pretrained is not implemented for Multi-ControlNet
                       pipe.save_pretrained(lowerCamelCase							)
                except NotImplementedError:
                       pass



@slow
@require_torch_gpu
class 		SCREAMING_SNAKE_CASE__	(  unittest.TestCase     ):

  '''simple docstring'''







  def  lowerCamelCase       (      self		:			Any							)						->	Dict:


         """simple docstring"""
         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()





  def  lowerCamelCase       (      self		:			Tuple							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    ControlNetModel.from_pretrained("""lllyasviel/sd-controlnet-canny"""							)

         _UpperCAmelCase							=    StableDiffusionControlNetImgaImgPipeline.from_pretrained(
             """runwayml/stable-diffusion-v1-5"""       ,				safety_checker=lowerCamelCase       ,				controlnet=lowerCamelCase							)
         pipe.enable_model_cpu_offload()
         pipe.set_progress_bar_config(disable=lowerCamelCase							)

         _UpperCAmelCase							=    torch.Generator(device="""cpu"""							).manual_seed(0							)
         _UpperCAmelCase							=    """evil space-punk bird"""
         _UpperCAmelCase							=    load_image(
             """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"""							).resize((512, 512)							)
         _UpperCAmelCase							=    load_image(
             """https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"""							).resize((512, 512)							)

         _UpperCAmelCase							=    pipe(
             lowerCamelCase       ,				lowerCamelCase       ,				control_image=lowerCamelCase       ,				generator=lowerCamelCase       ,				output_type="""np"""       ,				num_inference_steps=50       ,				strength=0.6       ,				)

         _UpperCAmelCase							=    output.images[0]

         assert image.shape == (512, 512, 3)

         _UpperCAmelCase							=    load_numpy(
             """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/img2img.npy"""							)

         assert np.abs(expected_image - image							).max() < 9E-2







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  LongformerTokenizer
  _lowerCamelCase    =  True
  _lowerCamelCase    =  LongformerTokenizerFast
  _lowerCamelCase    =  True


  def  lowerCamelCase       (      self		:			int							)						->	Optional[int]:


         """simple docstring"""
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         _UpperCAmelCase							=    [
             """l""",
             """o""",
             """w""",
             """e""",
             """r""",
             """s""",
             """t""",
             """i""",
             """d""",
             """n""",
             """\u0120""",
             """\u0120l""",
             """\u0120n""",
             """\u0120lo""",
             """\u0120low""",
             """er""",
             """\u0120lowest""",
             """\u0120newer""",
             """\u0120wider""",
             """<unk>""",
         ]
         _UpperCAmelCase							=    dict(zip(lowerCamelCase       ,				range(len(lowerCamelCase							)							)							)							)
         _UpperCAmelCase							=    ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
         _UpperCAmelCase							=    {"""unk_token""": """<unk>"""}

         _UpperCAmelCase							=    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""]							)
         _UpperCAmelCase							=    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""merges_file"""]							)
         with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8"""							) as fp:
                fp.write(json.dumps(lowerCamelCase							) + """\n"""							)
         with open(self.merges_file       ,				"""w"""       ,				encoding="""utf-8"""							) as fp:
                fp.write("""\n""".join(lowerCamelCase							)							)


  def  lowerCamelCase       (      self		:			List[Any]       ,				**lowerCamelCase		:			str							)						->	Tuple:


         """simple docstring"""
         kwargs.update(self.special_tokens_map							)
         return self.tokenizer_class.from_pretrained(self.tmpdirname       ,				**lowerCamelCase							)


  def  lowerCamelCase       (      self		:			int       ,				**lowerCamelCase		:			str							)						->	int:


         """simple docstring"""
         kwargs.update(self.special_tokens_map							)
         return self.rust_tokenizer_class.from_pretrained(self.tmpdirname       ,				**lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Union[str, Any]       ,				lowerCamelCase		:			Tuple							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    """lower newer"""
         _UpperCAmelCase							=    """lower newer"""
         return input_text, output_text


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.tokenizer_class(self.vocab_file       ,				self.merges_file       ,				**self.special_tokens_map							)
         _UpperCAmelCase							=    """lower newer"""
         _UpperCAmelCase							=    ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
         _UpperCAmelCase							=    tokenizer.tokenize(lowerCamelCase							)  # , add_prefix_space=True)
         self.assertListEqual(lowerCamelCase       ,				lowerCamelCase							)

         _UpperCAmelCase							=    tokens + [tokenizer.unk_token]
         _UpperCAmelCase							=    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase							)       ,				lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Dict							)						->	Tuple:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_tokenizer()

         self.assertListEqual(tokenizer.encode("""Hello world!"""       ,				add_special_tokens=lowerCamelCase							)       ,				[0, 3_1414, 232, 328, 2]							)
         self.assertListEqual(
             tokenizer.encode("""Hello world! cécé herlolip 418"""       ,				add_special_tokens=lowerCamelCase							)       ,				[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]       ,				)


  @slow
  def  lowerCamelCase       (      self		:			Dict							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""							)

         _UpperCAmelCase							=    tokenizer.encode("""sequence builders"""       ,				add_special_tokens=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.encode("""multi-sequence build"""       ,				add_special_tokens=lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.encode(
             """sequence builders"""       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.encode(
             """sequence builders"""       ,				"""multi-sequence build"""       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase       ,				lowerCamelCase							)

         assert encoded_sentence == encoded_text_from_decode
         assert encoded_pair == encoded_pair_from_decode


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_tokenizer()

         _UpperCAmelCase							=    """Encode this sequence."""
         _UpperCAmelCase							=    tokenizer.byte_encoder[""" """.encode("""utf-8"""							)[0]]

         # Testing encoder arguments
         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertNotEqual(lowerCamelCase       ,				lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				add_prefix_space=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
         self.assertEqual(lowerCamelCase       ,				lowerCamelCase							)

         tokenizer.add_special_tokens({"""bos_token""": """<s>"""}							)
         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
         self.assertNotEqual(lowerCamelCase       ,				lowerCamelCase							)

         # Testing spaces after special tokens
         _UpperCAmelCase							=    """<mask>"""
         tokenizer.add_special_tokens(
             {"""mask_token""": AddedToken(lowerCamelCase       ,				lstrip=lowerCamelCase       ,				rstrip=lowerCamelCase							)}							)  # mask token has a left space
         _UpperCAmelCase							=    tokenizer.convert_tokens_to_ids(lowerCamelCase							)

         _UpperCAmelCase							=    """Encode <mask> sequence"""
         _UpperCAmelCase							=    """Encode <mask>sequence"""

         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase							)
         _UpperCAmelCase							=    encoded.index(lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertEqual(lowerCamelCase       ,				lowerCamelCase							)

         _UpperCAmelCase							=    tokenizer.encode(lowerCamelCase							)
         _UpperCAmelCase							=    encoded.index(lowerCamelCase							)
         _UpperCAmelCase							=    tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
         self.assertNotEqual(lowerCamelCase       ,				lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	Union[str, Any]:


         """simple docstring"""
         pass


  def  lowerCamelCase       (      self		:			Any							)						->	Union[str, Any]:


         """simple docstring"""
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(lowerCamelCase       ,				**lowerCamelCase							)
                       _UpperCAmelCase							=    self.tokenizer_class.from_pretrained(lowerCamelCase       ,				**lowerCamelCase							)
                       _UpperCAmelCase							=    """A, <mask> AllenNLP sentence."""
                       _UpperCAmelCase							=    tokenizer_r.encode_plus(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				return_token_type_ids=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_p.encode_plus(lowerCamelCase       ,				add_special_tokens=lowerCamelCase       ,				return_token_type_ids=lowerCamelCase							)

                       # token_type_ids should put 0 everywhere
                       self.assertEqual(sum(tokens_r["""token_type_ids"""]							)       ,				sum(tokens_p["""token_type_ids"""]							)							)

                       # attention_mask should put 1 everywhere, so sum over length should be 1
                       self.assertEqual(
                           sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)       ,				sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)       ,				)

                       _UpperCAmelCase							=    tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
                       _UpperCAmelCase							=    tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)

                       # Rust correctly handles the space before the mask while python doesnt
                       self.assertSequenceEqual(tokens_p["""input_ids"""]       ,				[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]							)
                       self.assertSequenceEqual(tokens_r["""input_ids"""]       ,				[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]							)

                       self.assertSequenceEqual(
                           lowerCamelCase       ,				["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
                       self.assertSequenceEqual(
                           lowerCamelCase       ,				["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)


  def  lowerCamelCase       (      self		:			int							)						->	str:


         """simple docstring"""
         for trim_offsets, add_prefix_space in itertools.product([True, False]       ,				repeat=2							):
                _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                    self.tmpdirname       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)

                _UpperCAmelCase							=    json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
                _UpperCAmelCase							=    json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

                self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]       ,				lowerCamelCase							)

                self.assertEqual(post_processor_state["""add_prefix_space"""]       ,				lowerCamelCase							)
                self.assertEqual(post_processor_state["""trim_offsets"""]       ,				lowerCamelCase							)





  def  lowerCamelCase       (      self		:			Dict							)						->	Tuple:


         """simple docstring"""
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
         # `trim_offsets`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
                       _UpperCAmelCase							=    """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                       _UpperCAmelCase							=    f"""{text_of_1_token} {text_of_1_token}"""

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							) + 1, len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							) + 1, len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							), len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(len(lowerCamelCase							), len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    f""" {text}"""

                       # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                       #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                       # )
                       # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                       # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                       # self.assertEqual(
                       #     encoding.offset_mapping[1],
                       #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                       # )

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(1, 1 + len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(1 + len(lowerCamelCase							) + 1, 1 + len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, 1 + len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(1 + len(lowerCamelCase							), 1 + len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)

                       _UpperCAmelCase							=    self.rust_tokenizer_class.from_pretrained(
                           lowerCamelCase       ,				use_fast=lowerCamelCase       ,				add_prefix_space=lowerCamelCase       ,				trim_offsets=lowerCamelCase							)
                       _UpperCAmelCase							=    tokenizer_r(lowerCamelCase       ,				return_offsets_mapping=lowerCamelCase       ,				add_special_tokens=lowerCamelCase							)
                       self.assertEqual(encoding.offset_mapping[0]       ,				(0, 1 + len(lowerCamelCase							))							)
                       self.assertEqual(
                           encoding.offset_mapping[1]       ,				(1 + len(lowerCamelCase							), 1 + len(lowerCamelCase							) + 1 + len(lowerCamelCase							))       ,				)