


'''simple docstring'''
import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionPanoramaPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, skip_mps

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()




@skip_mps
class      lowerCamelCase_  (      __a	,			__a	,			unittest.TestCase ):
      lowerCAmelCase__							=	StableDiffusionPanoramaPipeline
      lowerCAmelCase__							=	TEXT_TO_IMAGE_PARAMS
      lowerCAmelCase__							=	TEXT_TO_IMAGE_BATCH_PARAMS
      lowerCAmelCase__							=	TEXT_TO_IMAGE_IMAGE_PARAMS
      lowerCAmelCase__							=	TEXT_TO_IMAGE_IMAGE_PARAMS






      def lowercase_				(   self							:       List[str]					):


          '''simple docstring'''


          torch.manual_seed(0					)
          UpperCAmelCase__     :						List[Any]						=							UNetaDConditionModel(
              block_out_channels=(32, 64)		,		layers_per_block=1		,		sample_size=32		,		in_channels=4		,		out_channels=4		,		down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')		,		up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')		,		cross_attention_dim=32		,		)
          UpperCAmelCase__     :						Optional[Any]						=							DDIMScheduler()
          torch.manual_seed(0					)
          UpperCAmelCase__     :						Optional[int]						=							AutoencoderKL(
              block_out_channels=[32, 64]		,		in_channels=3		,		out_channels=3		,		down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']		,		up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']		,		latent_channels=4		,		)
          torch.manual_seed(0					)
          UpperCAmelCase__     :						List[Any]						=							CLIPTextConfig(
              bos_token_id=0		,		eos_token_id=2		,		hidden_size=32		,		intermediate_size=37		,		layer_norm_eps=1e-05		,		num_attention_heads=4		,		num_hidden_layers=5		,		pad_token_id=1		,		vocab_size=1_000		,		)
          UpperCAmelCase__     :						Any						=							CLIPTextModel(_A					)
          UpperCAmelCase__     :						List[str]						=							CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''					)

          UpperCAmelCase__     :						Any						=							{
              '''unet''': unet,
              '''scheduler''': scheduler,
              '''vae''': vae,
              '''text_encoder''': text_encoder,
              '''tokenizer''': tokenizer,
              '''safety_checker''': None,
              '''feature_extractor''': None,
          }
          return components






      def lowercase_				(   self							:       Any		,		_A							:       Optional[Any]		,		_A							:       Dict=0					):


          '''simple docstring'''


          UpperCAmelCase__     :						Tuple						=							torch.manual_seed(_A					)
          UpperCAmelCase__     :						Optional[Any]						=							{
              '''prompt''': '''a photo of the dolomites''',
              '''generator''': generator,
              # Setting height and width to None to prevent OOMs on CPU.
              '''height''': None,
              '''width''': None,
              '''num_inference_steps''': 1,
              '''guidance_scale''': 6.0,
              '''output_type''': '''numpy''',
          }
          return inputs






      def lowercase_				(   self							:       Optional[Any]					):


          '''simple docstring'''


          UpperCAmelCase__     :						List[Any]						=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
          UpperCAmelCase__     :						Tuple						=							self.get_dummy_components()
          UpperCAmelCase__     :						str						=							StableDiffusionPanoramaPipeline(**_A					)
          UpperCAmelCase__     :						int						=							sd_pipe.to(_A					)
          sd_pipe.set_progress_bar_config(disable=_A					)

          UpperCAmelCase__     :						List[Any]						=							self.get_dummy_inputs(_A					)
          UpperCAmelCase__     :						int						=							sd_pipe(**_A					).images
          UpperCAmelCase__     :						Any						=							image[0, -3:, -3:, -1]
          assert image.shape == (1, 64, 64, 3)

          UpperCAmelCase__     :						Dict						=							np.array([0.6_1_8_6, 0.5_3_7_4, 0.4_9_1_5, 0.4_1_3_5, 0.4_1_1_4, 0.4_5_6_3, 0.5_1_2_8, 0.4_9_7_7, 0.4_7_5_7]					)

          assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-2






      def lowercase_				(   self							:       Tuple					):


          '''simple docstring'''


          super().test_inference_batch_consistent(batch_sizes=[1, 2]					)






      def lowercase_				(   self							:       int					):


          '''simple docstring'''


          super().test_inference_batch_single_identical(batch_size=2		,		expected_max_diff=3.25e-3					)






      def lowercase_				(   self							:       Union[str, Any]					):


          '''simple docstring'''


          UpperCAmelCase__     :						Dict						=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
          UpperCAmelCase__     :						Optional[int]						=							self.get_dummy_components()
          UpperCAmelCase__     :						Dict						=							StableDiffusionPanoramaPipeline(**_A					)
          UpperCAmelCase__     :						Optional[Any]						=							sd_pipe.to(_A					)
          sd_pipe.set_progress_bar_config(disable=_A					)

          UpperCAmelCase__     :						List[str]						=							self.get_dummy_inputs(_A					)
          UpperCAmelCase__     :						List[Any]						=							'''french fries'''
          UpperCAmelCase__     :						Tuple						=							sd_pipe(**_A		,		negative_prompt=_A					)
          UpperCAmelCase__     :						Tuple						=							output.images
          UpperCAmelCase__     :						Union[str, Any]						=							image[0, -3:, -3:, -1]

          assert image.shape == (1, 64, 64, 3)

          UpperCAmelCase__     :						List[Any]						=							np.array([0.6_1_8_7, 0.5_3_7_5, 0.4_9_1_5, 0.4_1_3_6, 0.4_1_1_4, 0.4_5_6_3, 0.5_1_2_8, 0.4_9_7_6, 0.4_7_5_7]					)

          assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-2






      def lowercase_				(   self							:       str					):


          '''simple docstring'''


          UpperCAmelCase__     :						Optional[int]						=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
          UpperCAmelCase__     :						Union[str, Any]						=							self.get_dummy_components()
          UpperCAmelCase__     :						str						=							StableDiffusionPanoramaPipeline(**_A					)
          UpperCAmelCase__     :						Optional[int]						=							sd_pipe.to(_A					)
          sd_pipe.set_progress_bar_config(disable=_A					)

          UpperCAmelCase__     :						str						=							self.get_dummy_inputs(_A					)
          UpperCAmelCase__     :						int						=							sd_pipe(**_A		,		view_batch_size=2					)
          UpperCAmelCase__     :						Optional[int]						=							output.images
          UpperCAmelCase__     :						List[Any]						=							image[0, -3:, -3:, -1]

          assert image.shape == (1, 64, 64, 3)

          UpperCAmelCase__     :						int						=							np.array([0.6_1_8_7, 0.5_3_7_5, 0.4_9_1_5, 0.4_1_3_6, 0.4_1_1_4, 0.4_5_6_3, 0.5_1_2_8, 0.4_9_7_6, 0.4_7_5_7]					)

          assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-2






      def lowercase_				(   self							:       List[Any]					):


          '''simple docstring'''


          UpperCAmelCase__     :						Tuple						=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
          UpperCAmelCase__     :						Optional[Any]						=							self.get_dummy_components()
          UpperCAmelCase__     :						List[Any]						=							EulerAncestralDiscreteScheduler(
              beta_start=0.0_0_0_8_5		,		beta_end=0.0_1_2		,		beta_schedule='''scaled_linear'''					)
          UpperCAmelCase__     :						List[str]						=							StableDiffusionPanoramaPipeline(**_A					)
          UpperCAmelCase__     :						Union[str, Any]						=							sd_pipe.to(_A					)
          sd_pipe.set_progress_bar_config(disable=_A					)

          UpperCAmelCase__     :						List[Any]						=							self.get_dummy_inputs(_A					)
          UpperCAmelCase__     :						Optional[int]						=							sd_pipe(**_A					).images
          UpperCAmelCase__     :						Tuple						=							image[0, -3:, -3:, -1]

          assert image.shape == (1, 64, 64, 3)

          UpperCAmelCase__     :						List[Any]						=							np.array([0.4_0_2_4, 0.6_5_1_0, 0.4_9_0_1, 0.5_3_7_8, 0.5_8_1_3, 0.5_6_2_2, 0.4_7_9_5, 0.4_4_6_7, 0.4_9_5_2]					)

          assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-2






      def lowercase_				(   self							:       Dict					):


          '''simple docstring'''


          UpperCAmelCase__     :						str						=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
          UpperCAmelCase__     :						Tuple						=							self.get_dummy_components()
          UpperCAmelCase__     :						List[str]						=							PNDMScheduler(
              beta_start=0.0_0_0_8_5		,		beta_end=0.0_1_2		,		beta_schedule='''scaled_linear'''		,		skip_prk_steps=_A					)
          UpperCAmelCase__     :						Optional[int]						=							StableDiffusionPanoramaPipeline(**_A					)
          UpperCAmelCase__     :						str						=							sd_pipe.to(_A					)
          sd_pipe.set_progress_bar_config(disable=_A					)

          UpperCAmelCase__     :						Tuple						=							self.get_dummy_inputs(_A					)
          UpperCAmelCase__     :						Optional[int]						=							sd_pipe(**_A					).images
          UpperCAmelCase__     :						Optional[Any]						=							image[0, -3:, -3:, -1]

          assert image.shape == (1, 64, 64, 3)

          UpperCAmelCase__     :						int						=							np.array([0.6_3_9_1, 0.6_2_9_1, 0.4_8_6_1, 0.5_1_3_4, 0.5_5_5_2, 0.4_5_7_8, 0.5_0_3_2, 0.5_0_2_3, 0.4_5_3_9]					)

          assert np.abs(image_slice.flatten() - expected_slice					).max() < 1e-2




@slow
@require_torch_gpu
class      lowerCamelCase_  (      unittest.TestCase ):






      def lowercase_				(   self							:       Dict					):


          '''simple docstring'''


          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()






      def lowercase_				(   self							:       int		,		_A							:       Tuple=0					):


          '''simple docstring'''


          UpperCAmelCase__     :						str						=							torch.manual_seed(_A					)
          UpperCAmelCase__     :						str						=							{
              '''prompt''': '''a photo of the dolomites''',
              '''generator''': generator,
              '''num_inference_steps''': 3,
              '''guidance_scale''': 7.5,
              '''output_type''': '''numpy''',
          }
          return inputs






      def lowercase_				(   self							:       Optional[Any]					):


          '''simple docstring'''


          UpperCAmelCase__     :						Tuple						=							'''stabilityai/stable-diffusion-2-base'''
          UpperCAmelCase__     :						Optional[Any]						=							DDIMScheduler.from_pretrained(_A		,		subfolder='''scheduler'''					)
          UpperCAmelCase__     :						Dict						=							StableDiffusionPanoramaPipeline.from_pretrained(_A		,		scheduler=_A		,		safety_checker=_A					)
          pipe.to(_A					)
          pipe.set_progress_bar_config(disable=_A					)
          pipe.enable_attention_slicing()

          UpperCAmelCase__     :						Tuple						=							self.get_inputs()
          UpperCAmelCase__     :						Dict						=							pipe(**_A					).images
          UpperCAmelCase__     :						List[Any]						=							image[0, -3:, -3:, -1].flatten()

          assert image.shape == (1, 512, 2_048, 3)

          UpperCAmelCase__     :						Optional[Any]						=							np.array(
              [
                  0.3_6_9_6_8_3_9_2,
                  0.2_7_0_2_5_3_7_2,
                  0.3_2_4_4_6_7_6_6,
                  0.2_8_3_7_9_3_8_7,
                  0.3_6_3_6_3_2_7_4,
                  0.3_0_7_3_3_3_4_7,
                  0.2_7_1_0_0_0_2_7,
                  0.2_7_0_5_4_1_2_5,
                  0.2_5_5_3_6_0_9_6,
              ]					)

          assert np.abs(expected_slice - image_slice					).max() < 1e-2






      def lowercase_				(   self							:       int					):


          '''simple docstring'''


          UpperCAmelCase__     :						Any						=							StableDiffusionPanoramaPipeline.from_pretrained(
              '''stabilityai/stable-diffusion-2-base'''		,		safety_checker=_A					)
          UpperCAmelCase__     :						Tuple						=							LMSDiscreteScheduler.from_config(pipe.scheduler.config					)
          pipe.to(_A					)
          pipe.set_progress_bar_config(disable=_A					)
          pipe.enable_attention_slicing()

          UpperCAmelCase__     :						Union[str, Any]						=							self.get_inputs()
          UpperCAmelCase__     :						Any						=							pipe(**_A					).images
          UpperCAmelCase__     :						Dict						=							image[0, -3:, -3:, -1].flatten()

          assert image.shape == (1, 512, 2_048, 3)

          UpperCAmelCase__     :						int						=							np.array(
              [
                  [
                      0.0,
                      0.0,
                      0.0,
                      0.0,
                      0.0,
                      0.0,
                      0.0,
                      0.0,
                      0.0,
                  ]
              ]					)

          assert np.abs(expected_slice - image_slice					).max() < 1e-3






      def lowercase_				(   self							:       Tuple					):


          '''simple docstring'''


          UpperCAmelCase__     :						List[str]						=							0

          def callback_fn(_A							:       int		,		_A							:       int		,		_A							:       torch.FloatTensor					) -> None:
              UpperCAmelCase__     :						str						=							True
              nonlocal number_of_steps
              number_of_steps += 1
              if step == 1:
                  UpperCAmelCase__     :						List[str]						=							latents.detach().cpu().numpy()
                  assert latents.shape == (1, 4, 64, 256)
                  UpperCAmelCase__     :						str						=							latents[0, -3:, -3:, -1]

                  UpperCAmelCase__     :						Tuple						=							np.array(
                      [
                          0.1_8_6_8_1_8_6_9,
                          0.3_3_9_0_7_8_1_6,
                          0.5_3_6_1_2_7_6,
                          0.1_4_4_3_2_8_6_5,
                          -0.0_2_8_5_6_6_1_1,
                          -0.7_3_9_4_1_1_2_3,
                          0.2_3_3_9_7_9_8_7,
                          0.4_7_3_2_2_6_8_2,
                          -0.3_7_8_2_3_1_6_4,
                      ]					)
                  assert np.abs(latents_slice.flatten() - expected_slice					).max() < 5e-2
              elif step == 2:
                  UpperCAmelCase__     :						Tuple						=							latents.detach().cpu().numpy()
                  assert latents.shape == (1, 4, 64, 256)
                  UpperCAmelCase__     :						int						=							latents[0, -3:, -3:, -1]

                  UpperCAmelCase__     :						Union[str, Any]						=							np.array(
                      [
                          0.1_8_5_3_9_6_4_5,
                          0.3_3_9_8_7_2_4_8,
                          0.5_3_7_8_5_5_9,
                          0.1_4_4_3_7_1_4_2,
                          -0.0_2_4_5_5_2_6_1,
                          -0.7_3_3_8_3_1_7,
                          0.2_3_9_9_0_7_5_5,
                          0.4_7_3_5_6_2_7_2,
                          -0.3_7_8_6_5_0_5,
                      ]					)

                  assert np.abs(latents_slice.flatten() - expected_slice					).max() < 5e-2

          UpperCAmelCase__     :						Union[str, Any]						=							False

          UpperCAmelCase__     :						Dict						=							'''stabilityai/stable-diffusion-2-base'''
          UpperCAmelCase__     :						List[Any]						=							DDIMScheduler.from_pretrained(_A		,		subfolder='''scheduler'''					)
          UpperCAmelCase__     :						List[str]						=							StableDiffusionPanoramaPipeline.from_pretrained(_A		,		scheduler=_A		,		safety_checker=_A					)
          UpperCAmelCase__     :						Tuple						=							pipe.to(_A					)
          pipe.set_progress_bar_config(disable=_A					)
          pipe.enable_attention_slicing()

          UpperCAmelCase__     :						Optional[Any]						=							self.get_inputs()
          pipe(**_A		,		callback=_A		,		callback_steps=1					)
          assert callback_fn.has_been_called
          assert number_of_steps == 3






      def lowercase_				(   self							:       Tuple					):


          '''simple docstring'''


          torch.cuda.empty_cache()
          torch.cuda.reset_max_memory_allocated()
          torch.cuda.reset_peak_memory_stats()

          UpperCAmelCase__     :						Tuple						=							'''stabilityai/stable-diffusion-2-base'''
          UpperCAmelCase__     :						List[Any]						=							DDIMScheduler.from_pretrained(_A		,		subfolder='''scheduler'''					)
          UpperCAmelCase__     :						Optional[int]						=							StableDiffusionPanoramaPipeline.from_pretrained(_A		,		scheduler=_A		,		safety_checker=_A					)
          UpperCAmelCase__     :						Tuple						=							pipe.to(_A					)
          pipe.set_progress_bar_config(disable=_A					)
          pipe.enable_attention_slicing(1					)
          pipe.enable_sequential_cpu_offload()

          UpperCAmelCase__     :						Dict						=							self.get_inputs()
          UpperCAmelCase__     :						Optional[Any]						=							pipe(**_A					)

          UpperCAmelCase__     :						Optional[int]						=							torch.cuda.max_memory_allocated()
          # make sure that less than 5.2 GB is allocated
          assert mem_bytes < 5.5 * 10**9








import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class       snake_case__ (				lowercase_						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					List[str]								=			GPTaTokenizer
       lowerCamelCase				:					Optional[int]								=			GPTaTokenizerFast
       lowerCamelCase				:					List[Any]								=			True
       lowerCamelCase				:					List[str]								=			{"add_prefix_space": True}
       lowerCamelCase				:					Optional[int]								=			False



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            __snake_case      :Optional[Any] 							=    [
                """l""",
                """o""",
                """w""",
                """e""",
                """r""",
                """s""",
                """t""",
                """i""",
                """d""",
                """n""",
                """\u0120""",
                """\u0120l""",
                """\u0120n""",
                """\u0120lo""",
                """\u0120low""",
                """er""",
                """\u0120lowest""",
                """\u0120newer""",
                """\u0120wider""",
                """<unk>""",
                """<|endoftext|>""",
            ]
            __snake_case      :Dict 							=    dict(zip(a__      ,    range(len(a__     )     )     )     )
            __snake_case      :List[Any] 							=    ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
            __snake_case      :List[str] 							=    {"""unk_token""": """<unk>"""}

            __snake_case      :Dict 							=    os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["""vocab_file"""]     )
            __snake_case      :Optional[Any] 							=    os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["""merges_file"""]     )
            with open(self.vocab_file      ,    """w"""      ,    encoding="""utf-8"""     ) as fp:
                 fp.write(json.dumps(a__     ) + """\n"""     )
            with open(self.merges_file      ,    """w"""      ,    encoding="""utf-8"""     ) as fp:
                 fp.write("""\n""".join(a__     )     )



       def        __lowercase    (				self      ,    **a__     )			->   Any:






            '''simple docstring'''





            kwargs.update(self.special_tokens_map     )
            return GPTaTokenizer.from_pretrained(self.tmpdirname      ,    **a__     )



       def        __lowercase    (				self      ,    **a__     )			->   Tuple:






            '''simple docstring'''





            kwargs.update(self.special_tokens_map     )
            return GPTaTokenizerFast.from_pretrained(self.tmpdirname      ,    **a__     )



       def        __lowercase    (				self      ,    a__     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    """lower newer"""
            __snake_case      :Any 							=    """lower newer"""
            return input_text, output_text



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    GPTaTokenizer(self.vocab_file      ,    self.merges_file      ,    **self.special_tokens_map     )
            __snake_case      :List[Any] 							=    """lower newer"""
            __snake_case      :int 							=    ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
            __snake_case      :Any 							=    tokenizer.tokenize(a__      ,    add_prefix_space=a__     )
            self.assertListEqual(a__      ,    a__     )

            __snake_case      :List[Any] 							=    tokens + [tokenizer.unk_token]
            __snake_case      :List[Any] 							=    [14, 15, 10, 9, 3, 2, 15, 19]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__     )      ,    a__     )



       def        __lowercase    (				self     )			->   Dict:






            '''simple docstring'''





            if not self.test_rust_tokenizer:
                 return

            __snake_case      :Union[str, Any] 							=    self.get_tokenizer()
            __snake_case      :Optional[Any] 							=    self.get_rust_tokenizer(add_prefix_space=a__     )

            __snake_case      :Optional[int] 							=    """lower newer"""

            # Testing tokenization
            __snake_case      :List[Any] 							=    tokenizer.tokenize(a__      ,    add_prefix_space=a__     )
            __snake_case      :int 							=    rust_tokenizer.tokenize(a__     )
            self.assertListEqual(a__      ,    a__     )

            # Testing conversion to ids without special tokens
            __snake_case      :List[str] 							=    tokenizer.encode(a__      ,    add_special_tokens=a__      ,    add_prefix_space=a__     )
            __snake_case      :int 							=    rust_tokenizer.encode(a__      ,    add_special_tokens=a__     )
            self.assertListEqual(a__      ,    a__     )

            # Testing conversion to ids with special tokens
            __snake_case      :Union[str, Any] 							=    self.get_rust_tokenizer(add_prefix_space=a__     )
            __snake_case      :Optional[int] 							=    tokenizer.encode(a__      ,    add_prefix_space=a__     )
            __snake_case      :Optional[int] 							=    rust_tokenizer.encode(a__     )
            self.assertListEqual(a__      ,    a__     )

            # Testing the unknown token
            __snake_case      :Dict 							=    tokens + [rust_tokenizer.unk_token]
            __snake_case      :Optional[int] 							=    [14, 15, 10, 9, 3, 2, 15, 19]
            self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a__     )      ,    a__     )



       def        __lowercase    (				self      ,    *a__      ,    **a__     )			->   Any:






            '''simple docstring'''





            pass



       def        __lowercase    (				self      ,    a__=15     )			->   Optional[Any]:






            '''simple docstring'''





            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                      __snake_case      :Optional[int] 							=    self.rust_tokenizer_class.from_pretrained(a__      ,    **a__     )

                      # Simple input
                      __snake_case      :List[str] 							=    """This is a simple input"""
                      __snake_case      :List[str] 							=    ["""This is a simple input 1""", """This is a simple input 2"""]
                      __snake_case      :Tuple 							=    ("""This is a simple input""", """This is a pair""")
                      __snake_case      :Union[str, Any] 							=    [
                          ("""This is a simple input 1""", """This is a simple input 2"""),
                          ("""This is a simple pair 1""", """This is a simple pair 2"""),
                      ]

                      # Simple input tests
                      self.assertRaises(a__      ,    tokenizer_r.encode      ,    a__      ,    max_length=a__      ,    padding="""max_length"""     )

                      # Simple input
                      self.assertRaises(a__      ,    tokenizer_r.encode_plus      ,    a__      ,    max_length=a__      ,    padding="""max_length"""     )

                      # Simple input
                      self.assertRaises(
                          a__      ,    tokenizer_r.batch_encode_plus      ,    a__      ,    max_length=a__      ,    padding="""max_length"""      ,    )

                      # Pair input
                      self.assertRaises(a__      ,    tokenizer_r.encode      ,    a__      ,    max_length=a__      ,    padding="""max_length"""     )

                      # Pair input
                      self.assertRaises(a__      ,    tokenizer_r.encode_plus      ,    a__      ,    max_length=a__      ,    padding="""max_length"""     )

                      # Pair input
                      self.assertRaises(
                          a__      ,    tokenizer_r.batch_encode_plus      ,    a__      ,    max_length=a__      ,    padding="""max_length"""      ,    )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    GPTaTokenizer.from_pretrained(self.tmpdirname      ,    pad_token="""<pad>"""     )

            # Simple input
            __snake_case      :Tuple 							=    """This is a simple input"""
            __snake_case      :Optional[int] 							=    ["""This is a simple input looooooooong""", """This is a simple input"""]
            __snake_case      :List[str] 							=    ("""This is a simple input""", """This is a pair""")
            __snake_case      :List[str] 							=    [
                ("""This is a simple input loooooong""", """This is a simple input"""),
                ("""This is a simple pair loooooong""", """This is a simple pair"""),
            ]

            __snake_case      :Tuple 							=    tokenizer.pad_token_id

            __snake_case      :int 							=    tokenizer(a__      ,    padding="""max_length"""      ,    max_length=30      ,    return_tensors="""np"""     )
            __snake_case      :Optional[Any] 							=    tokenizer(a__      ,    padding=a__      ,    truncate=a__      ,    return_tensors="""np"""     )
            __snake_case      :List[Any] 							=    tokenizer(*a__      ,    padding="""max_length"""      ,    max_length=60      ,    return_tensors="""np"""     )
            __snake_case      :Optional[int] 							=    tokenizer(a__      ,    padding=a__      ,    truncate=a__      ,    return_tensors="""np"""     )

            # s
            # test single string max_length padding
            self.assertEqual(out_s["""input_ids"""].shape[-1]      ,    30     )
            self.assertTrue(pad_token_id in out_s["""input_ids"""]     )
            self.assertTrue(0 in out_s["""attention_mask"""]     )

            # s2
            # test automatic padding
            self.assertEqual(out_sa["""input_ids"""].shape[-1]      ,    33     )
            # long slice doesn't have padding
            self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]     )
            self.assertFalse(0 in out_sa["""attention_mask"""][0]     )
            # short slice does have padding
            self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]     )
            self.assertTrue(0 in out_sa["""attention_mask"""][1]     )

            # p
            # test single pair max_length padding
            self.assertEqual(out_p["""input_ids"""].shape[-1]      ,    60     )
            self.assertTrue(pad_token_id in out_p["""input_ids"""]     )
            self.assertTrue(0 in out_p["""attention_mask"""]     )

            # p2
            # test automatic padding pair
            self.assertEqual(out_pa["""input_ids"""].shape[-1]      ,    52     )
            # long slice pair doesn't have padding
            self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]     )
            self.assertFalse(0 in out_pa["""attention_mask"""][0]     )
            # short slice pair does have padding
            self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]     )
            self.assertTrue(0 in out_pa["""attention_mask"""][1]     )



       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    """$$$"""
            __snake_case      :Optional[Any] 							=    GPTaTokenizer.from_pretrained(self.tmpdirname      ,    bos_token=a__      ,    add_bos_token=a__     )

            __snake_case      :List[Any] 							=    """This is a simple input"""
            __snake_case      :Tuple 							=    ["""This is a simple input 1""", """This is a simple input 2"""]

            __snake_case      :Tuple 							=    tokenizer.bos_token_id

            __snake_case      :List[str] 							=    tokenizer(a__     )
            __snake_case      :Any 							=    tokenizer(a__     )

            self.assertEqual(out_s.input_ids[0]      ,    a__     )
            self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids     )     )

            __snake_case      :int 							=    tokenizer.decode(out_s.input_ids     )
            __snake_case      :Tuple 							=    tokenizer.batch_decode(out_sa.input_ids     )

            self.assertEqual(decode_s.split()[0]      ,    a__     )
            self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa     )     )



       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            pass



       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :List[str] 							=    [self.get_tokenizer(do_lower_case=a__      ,    add_bos_token=a__     )]
            for tokenizer in tokenizers:
                 with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
                      __snake_case      :Tuple 							=    """Encode this."""
                      __snake_case      :Tuple 							=    """This one too please."""
                      __snake_case      :Union[str, Any] 							=    tokenizer.encode(a__      ,    add_special_tokens=a__     )
                      encoded_sequence += tokenizer.encode(a__      ,    add_special_tokens=a__     )
                      __snake_case      :List[str] 							=    tokenizer.encode_plus(
                          a__      ,    a__      ,    add_special_tokens=a__      ,    return_special_tokens_mask=a__      ,    )
                      __snake_case      :Union[str, Any] 							=    encoded_sequence_dict["""input_ids"""]
                      __snake_case      :Optional[Any] 							=    encoded_sequence_dict["""special_tokens_mask"""]
                      self.assertEqual(len(a__     )      ,    len(a__     )     )

                      __snake_case      :Optional[int] 							=    [
                          (x if not special_tokens_mask[i] else None) for i, x in enumerate(a__     )
                      ]
                      __snake_case      :str 							=    [x for x in filtered_sequence if x is not None]
                      self.assertEqual(a__      ,    a__     )




@require_tokenizers
class       snake_case__ (				unittest.TestCase):

       '''simple docstring'''



       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    AutoTokenizer.from_pretrained("""facebook/opt-350m"""      ,    from_slow=a__     )
            __snake_case      :Union[str, Any] 							=    """A photo of a cat"""

            __snake_case      :int 							=    tokenizer.encode(
                a__      ,    )
            self.assertEqual(a__      ,    [2, 2_50, 13_45, 9, 10, 47_58]     )
            tokenizer.save_pretrained("""test_opt"""     )

            __snake_case      :int 							=    AutoTokenizer.from_pretrained("""./test_opt"""     )
            __snake_case      :Tuple 							=    tokenizer.encode(
                a__      ,    )
            self.assertEqual(a__      ,    [2, 2_50, 13_45, 9, 10, 47_58]     )



       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            __snake_case      :Tuple 							=    AutoTokenizer.from_pretrained("""facebook/opt-350m"""      ,    use_slow=a__     )
            __snake_case      :str 							=    """A photo of a cat"""

            __snake_case      :List[str] 							=    tokenizer.encode(
                a__      ,    )
            # Same as above
            self.assertEqual(a__      ,    [2, 2_50, 13_45, 9, 10, 47_58]     )



       @unittest.skip("""This test is failing because of a bug in the fast tokenizer"""     )
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :int 							=    AutoTokenizer.from_pretrained("""facebook/opt-350m"""      ,    from_slow=a__     )

            __snake_case      :Optional[Any] 							=    """bos"""
            __snake_case      :int 							=    tokenizer.get_vocab()["""bos"""]

            __snake_case      :str 							=    """A photo of a cat"""
            __snake_case      :int 							=    tokenizer.encode(
                a__      ,    )
            # We changed the bos token
            self.assertEqual(a__      ,    [3_19_57, 2_50, 13_45, 9, 10, 47_58]     )
            tokenizer.save_pretrained("""./tok"""     )
            __snake_case      :Dict 							=    AutoTokenizer.from_pretrained("""./tok"""     )
            self.assertTrue(tokenizer.is_fast     )
            __snake_case      :List[Any] 							=    tokenizer.encode(
                a__      ,    )
            self.assertEqual(a__      ,    [3_19_57, 2_50, 13_45, 9, 10, 47_58]     )


