



'''simple docstring'''





import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    PNDMScheduler,
    StableDiffusionLDMaDPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import nightly, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS


enable_full_determinism()


class 				snake_case (unittest.TestCase   ):
       lowerCAmelCase__  :Optional[Any]						   =					StableDiffusionLDMaDPipeline
       lowerCAmelCase__  :Any						   =					TEXT_TO_IMAGE_PARAMS
       lowerCAmelCase__  :List[str]						   =					TEXT_TO_IMAGE_BATCH_PARAMS
       lowerCAmelCase__  :str						   =					TEXT_TO_IMAGE_IMAGE_PARAMS



       def 							_a      (				self      )  ->							Tuple:
           torch.manual_seed(0      )
           lowercase__		       =						UNetaDConditionModel(
               block_out_channels=(32, 64)     ,layers_per_block=2     ,sample_size=32     ,in_channels=4     ,out_channels=4     ,down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")     ,up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")     ,cross_attention_dim=32     ,)
           lowercase__		       =						DDIMScheduler(
               beta_start=0.0_00_85     ,beta_end=0.0_12     ,beta_schedule="scaled_linear"     ,clip_sample=_UpperCAmelCase     ,set_alpha_to_one=_UpperCAmelCase     ,)
           torch.manual_seed(0      )
           lowercase__		       =						AutoencoderKL(
               block_out_channels=[32, 64]     ,in_channels=6     ,out_channels=6     ,down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]     ,up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]     ,latent_channels=4     ,)
           torch.manual_seed(0      )
           lowercase__		       =						CLIPTextConfig(
               bos_token_id=0     ,eos_token_id=2     ,hidden_size=32     ,intermediate_size=37     ,layer_norm_eps=1E-0_5     ,num_attention_heads=4     ,num_hidden_layers=5     ,pad_token_id=1     ,vocab_size=1_000     ,)
           lowercase__		       =						CLIPTextModel(_UpperCAmelCase      )
           lowercase__		       =						CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"      )

           lowercase__		       =						{
               '''unet''': unet,
               '''scheduler''': scheduler,
               '''vae''': vae,
               '''text_encoder''': text_encoder,
               '''tokenizer''': tokenizer,
               '''safety_checker''': None,
               '''feature_extractor''': None,
           }
           return components



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_=0      )  ->							Dict:
           if str(_UpperCAmelCase      ).startswith("mps"      ):
               lowercase__		       =						torch.manual_seed(_UpperCAmelCase      )
           else:
               lowercase__		       =						torch.Generator(device=_UpperCAmelCase      ).manual_seed(_UpperCAmelCase      )
           lowercase__		       =						{
               '''prompt''': '''A painting of a squirrel eating a burger''',
               '''generator''': generator,
               '''num_inference_steps''': 2,
               '''guidance_scale''': 6.0,
               '''output_type''': '''numpy''',
           }
           return inputs



       def 							_a      (				self      )  ->							Optional[Any]:
           lowercase__		       =						'''cpu'''  # ensure determinism for the device-dependent torch.Generator

           lowercase__		       =						self.get_dummy_components()
           lowercase__		       =						StableDiffusionLDMaDPipeline(**_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe.to(_UpperCAmelCase      )
           ldmad_pipe.set_progress_bar_config(disable=_UpperCAmelCase      )

           lowercase__		       =						self.get_dummy_inputs(_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase      )
           lowercase__		       =						output.rgb, output.depth

           lowercase__		       =						rgb[0, -3:, -3:, -1]
           lowercase__		       =						depth[0, -3:, -1]

           assert rgb.shape == (1, 64, 64, 3)
           assert depth.shape == (1, 64, 64)

           lowercase__		       =						np.array(
               [0.37_33_81_76, 0.7_02_47, 0.74_20_31_93, 0.51_64_36_04, 0.58_25_67_93, 0.60_93_21_36, 0.4_18_10_95, 0.48_35_58_77, 0.46_53_52_62]      )
           lowercase__		       =						np.array([103.46_727, 85.812_004, 87.849_236]      )

           assert np.abs(image_slice_rgb.flatten() - expected_slice_rgb      ).max() < 1E-2
           assert np.abs(image_slice_depth.flatten() - expected_slice_depth      ).max() < 1E-2



       def 							_a      (				self      )  ->							Union[str, Any]:
           lowercase__		       =						self.get_dummy_components()
           lowercase__		       =						StableDiffusionLDMaDPipeline(**_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe.to(_UpperCAmelCase      )
           ldmad_pipe.set_progress_bar_config(disable=_UpperCAmelCase      )

           lowercase__		       =						self.get_dummy_inputs(_UpperCAmelCase      )
           lowercase__		       =						3 * [inputs['''prompt''']]

           # forward
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase      )
           lowercase__		       =						output.rgb, output.depth
           lowercase__		       =						rgb_slice_a[0, -3:, -3:, -1]
           lowercase__		       =						depth_slice_a[0, -3:, -1]

           lowercase__		       =						self.get_dummy_inputs(_UpperCAmelCase      )
           lowercase__		       =						3 * [inputs.pop("prompt"      )]

           lowercase__		       =						ldmad_pipe.tokenizer(
               _UpperCAmelCase     ,padding="max_length"     ,max_length=ldmad_pipe.tokenizer.model_max_length     ,truncation=_UpperCAmelCase     ,return_tensors="pt"     ,)
           lowercase__		       =						text_inputs['''input_ids'''].to(_UpperCAmelCase      )

           lowercase__		       =						ldmad_pipe.text_encoder(_UpperCAmelCase      )[0]

           lowercase__		       =						prompt_embeds

           # forward
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase      )
           lowercase__		       =						output.rgb, output.depth
           lowercase__		       =						rgb_slice_a[0, -3:, -3:, -1]
           lowercase__		       =						depth_slice_a[0, -3:, -1]

           assert np.abs(rgb_slice_a.flatten() - rgb_slice_a.flatten()      ).max() < 1E-4
           assert np.abs(depth_slice_a.flatten() - depth_slice_a.flatten()      ).max() < 1E-4







       def 							_a      (				self      )  ->							Dict:
           lowercase__		       =						'''cpu'''  # ensure determinism for the device-dependent torch.Generator
           lowercase__		       =						self.get_dummy_components()
           lowercase__		       =						PNDMScheduler(skip_prk_steps=_UpperCAmelCase      )
           lowercase__		       =						StableDiffusionLDMaDPipeline(**_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe.to(_UpperCAmelCase      )
           ldmad_pipe.set_progress_bar_config(disable=_UpperCAmelCase      )

           lowercase__		       =						self.get_dummy_inputs(_UpperCAmelCase      )
           lowercase__		       =						'''french fries'''
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase     ,negative_prompt=_UpperCAmelCase      )

           lowercase__		       =						output.rgb, output.depth
           lowercase__		       =						rgb[0, -3:, -3:, -1]
           lowercase__		       =						depth[0, -3:, -1]

           assert rgb.shape == (1, 64, 64, 3)
           assert depth.shape == (1, 64, 64)

           lowercase__		       =						np.array(
               [0.3_70_44, 0.71_81_15_03, 0.7_22_32_51, 0.48_60_36_75, 0.5_63_83_91, 0.6_36_49_48, 0.42_83_37_04, 0.4_90_13_15, 0.47_92_62_17]      )
           lowercase__		       =						np.array([107.84_738, 84.62_802, 89.962_135]      )
           assert np.abs(rgb_slice.flatten() - expected_slice_rgb      ).max() < 1E-2
           assert np.abs(depth_slice.flatten() - expected_slice_depth      ).max() < 1E-2





@slow
@require_torch_gpu
class 				snake_case (unittest.TestCase   ):



       def 							_a      (				self      )  ->							Any:
           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_="cpu"     ,UpperCAmelCase_=torch.floataa     ,UpperCAmelCase_=0      )  ->							str:
           lowercase__		       =						torch.Generator(device=_UpperCAmelCase      ).manual_seed(_UpperCAmelCase      )
           lowercase__		       =						np.random.RandomState(_UpperCAmelCase      ).standard_normal((1, 4, 64, 64)      )
           lowercase__		       =						torch.from_numpy(_UpperCAmelCase      ).to(device=_UpperCAmelCase     ,dtype=_UpperCAmelCase      )
           lowercase__		       =						{
               '''prompt''': '''a photograph of an astronaut riding a horse''',
               '''latents''': latents,
               '''generator''': generator,
               '''num_inference_steps''': 3,
               '''guidance_scale''': 7.5,
               '''output_type''': '''numpy''',
           }
           return inputs







       def 							_a      (				self      )  ->							Any:
           lowercase__		       =						StableDiffusionLDMaDPipeline.from_pretrained("Intel/ldm3d"      )
           lowercase__		       =						ldmad_pipe.to(_UpperCAmelCase      )
           ldmad_pipe.set_progress_bar_config(disable=_UpperCAmelCase      )

           lowercase__		       =						self.get_inputs(_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase      )
           lowercase__		       =						output.rgb, output.depth
           lowercase__		       =						rgb[0, -3:, -3:, -1].flatten()
           lowercase__		       =						rgb[0, -3:, -1].flatten()

           assert rgb.shape == (1, 512, 512, 3)
           assert depth.shape == (1, 512, 512)

           lowercase__		       =						np.array(
               [0.53_80_54_65, 0.56_70_73_05, 0.5_48_65_15, 0.57_01_22_36, 0.5_81_45_11, 0.56_25_34_87, 0.54_84_30_14, 0.55_09_22_63, 0.6_45_97_06]      )
           lowercase__		       =						np.array(
               [0.9_26_37_81, 0.6_67_86_72, 0.5_48_65_15, 0.92_20_21_45, 0.67_83_11_35, 0.56_25_34_87, 0.9_24_16_94, 0.7_55_14_78, 0.6_45_97_06]      )
           assert np.abs(rgb_slice - expected_slice_rgb      ).max() < 3E-3
           assert np.abs(depth_slice - expected_slice_depth      ).max() < 3E-3





@nightly
@require_torch_gpu
class 				snake_case (unittest.TestCase   ):



       def 							_a      (				self      )  ->							Tuple:
           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_="cpu"     ,UpperCAmelCase_=torch.floataa     ,UpperCAmelCase_=0      )  ->							Dict:
           lowercase__		       =						torch.Generator(device=_UpperCAmelCase      ).manual_seed(_UpperCAmelCase      )
           lowercase__		       =						np.random.RandomState(_UpperCAmelCase      ).standard_normal((1, 4, 64, 64)      )
           lowercase__		       =						torch.from_numpy(_UpperCAmelCase      ).to(device=_UpperCAmelCase     ,dtype=_UpperCAmelCase      )
           lowercase__		       =						{
               '''prompt''': '''a photograph of an astronaut riding a horse''',
               '''latents''': latents,
               '''generator''': generator,
               '''num_inference_steps''': 50,
               '''guidance_scale''': 7.5,
               '''output_type''': '''numpy''',
           }
           return inputs



       def 							_a      (				self      )  ->							Union[str, Any]:
           lowercase__		       =						StableDiffusionLDMaDPipeline.from_pretrained("Intel/ldm3d"      ).to(_UpperCAmelCase      )
           ldmad_pipe.set_progress_bar_config(disable=_UpperCAmelCase      )

           lowercase__		       =						self.get_inputs(_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase      )
           lowercase__		       =						output.rgb, output.depth

           lowercase__		       =						0.49_55_86
           lowercase__		       =						0.33_79_55_15
           lowercase__		       =						112.48_518
           lowercase__		       =						98.489_746
           assert np.abs(expected_rgb_mean - rgb.mean()      ) < 1E-3
           assert np.abs(expected_rgb_std - rgb.std()      ) < 1E-3
           assert np.abs(expected_depth_mean - depth.mean()      ) < 1E-3
           assert np.abs(expected_depth_std - depth.std()      ) < 1E-3







       def 							_a      (				self      )  ->							int:
           lowercase__		       =						StableDiffusionLDMaDPipeline.from_pretrained("Intel/ldm3d-4c"      ).to(_UpperCAmelCase      )
           ldmad_pipe.set_progress_bar_config(disable=_UpperCAmelCase      )

           lowercase__		       =						self.get_inputs(_UpperCAmelCase      )
           lowercase__		       =						ldmad_pipe(**_UpperCAmelCase      )
           lowercase__		       =						output.rgb, output.depth

           lowercase__		       =						0.4_19_41_27
           lowercase__		       =						0.35_37_55_86
           lowercase__		       =						0.5_63_85_02
           lowercase__		       =						0.34_68_61_03

           assert rgb.shape == (1, 512, 512, 3)
           assert depth.shape == (1, 512, 512, 1)
           assert np.abs(expected_rgb_mean - rgb.mean()      ) < 1E-3
           assert np.abs(expected_rgb_std - rgb.std()      ) < 1E-3
           assert np.abs(expected_depth_mean - depth.mean()      ) < 1E-3
           assert np.abs(expected_depth_std - depth.std()      ) < 1E-3




'''simple docstring'''





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 				snake_case (UpperCamelCase		,      unittest.TestCase   ):
       lowerCAmelCase__  :Optional[int]						   =					CodeGenTokenizer
       lowerCAmelCase__  :List[Any]						   =					CodeGenTokenizerFast
       lowerCAmelCase__  :str						   =					True
       lowerCAmelCase__  :Tuple						   =					{"add_prefix_space": True}
       lowerCAmelCase__  :Dict						   =					False



       def 							_a      (				self      )  ->							Union[str, Any]:
           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           lowercase__		       =						[
               "l",
               "o",
               "w",
               "e",
               "r",
               "s",
               "t",
               "i",
               "d",
               "n",
               "\u0120",
               "\u0120l",
               "\u0120n",
               "\u0120lo",
               "\u0120low",
               "er",
               "\u0120lowest",
               "\u0120newer",
               "\u0120wider",
               "<unk>",
               "<|endoftext|>",
           ]
           lowercase__		       =						dict(zip(UpperCAmelCase_     ,range(len(UpperCAmelCase_      )      )      )      )
           lowercase__		       =						["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
           lowercase__		       =						{"unk_token": "<unk>"}

           lowercase__		       =						os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["vocab_file"]      )
           lowercase__		       =						os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["merges_file"]      )
           with open(self.vocab_file     ,"w"     ,encoding="utf-8"      ) as fp:
               fp.write(json.dumps(UpperCAmelCase_      ) + "\n"      )
           with open(self.merges_file     ,"w"     ,encoding="utf-8"      ) as fp:
               fp.write("\n".join(UpperCAmelCase_      )      )



       def 							_a      (				self     ,**UpperCAmelCase_      )  ->							int:
           kwargs.update(self.special_tokens_map      )
           return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,**UpperCAmelCase_      )



       def 							_a      (				self     ,**UpperCAmelCase_      )  ->							Tuple:
           kwargs.update(self.special_tokens_map      )
           return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,**UpperCAmelCase_      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						"lower newer"
           lowercase__		       =						"lower newer"
           return input_text, output_text



       def 							_a      (				self      )  ->							Optional[int]:
           lowercase__		       =						CodeGenTokenizer(self.vocab_file     ,self.merges_file     ,**self.special_tokens_map      )
           lowercase__		       =						"lower newer"
           lowercase__		       =						["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
           lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						tokens + [tokenizer.unk_token]
           lowercase__		       =						[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							int:
           if not self.test_rust_tokenizer:
               return

           lowercase__		       =						self.get_tokenizer()
           lowercase__		       =						self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )

           lowercase__		       =						"lower newer"

           # Testing tokenization
           lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.tokenize(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # Testing conversion to ids without special tokens
           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # Testing conversion to ids with special tokens
           lowercase__		       =						self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # Testing the unknown token
           lowercase__		       =						tokens + [rust_tokenizer.unk_token]
           lowercase__		       =						[14, 15, 10, 9, 3, 2, 15, 19]
           self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )     ,UpperCAmelCase_      )



       def 							_a      (				self     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							Any:
           # It's very difficult to mix/test pretokenization with byte-level
           # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
           pass



       def 							_a      (				self     ,UpperCAmelCase_=15      )  ->							Union[str, Any]:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                   lowercase__		       =						self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_     ,**UpperCAmelCase_      )

                   # Simple input
                   lowercase__		       =						"This is a simple input"
                   lowercase__		       =						["This is a simple input 1", "This is a simple input 2"]
                   lowercase__		       =						("This is a simple input", "This is a pair")
                   lowercase__		       =						[
                       ("This is a simple input 1", "This is a simple input 2"),
                       ("This is a simple pair 1", "This is a simple pair 2"),
                   ]

                   # Simple input tests
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Simple input
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Simple input
                   self.assertRaises(
                       UpperCAmelCase_     ,tokenizer_r.batch_encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"     ,)

                   # Pair input
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Pair input
                   self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

                   # Pair input
                   self.assertRaises(
                       UpperCAmelCase_     ,tokenizer_r.batch_encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"     ,)



       def 							_a      (				self      )  ->							Optional[Any]:
           lowercase__		       =						CodeGenTokenizer.from_pretrained(self.tmpdirname     ,pad_token="<pad>"      )

           # Simple input
           lowercase__		       =						"This is a simple input"
           lowercase__		       =						["This is a simple input looooooooong", "This is a simple input"]
           lowercase__		       =						("This is a simple input", "This is a pair")
           lowercase__		       =						[
               ("This is a simple input loooooong", "This is a simple input"),
               ("This is a simple pair loooooong", "This is a simple pair"),
           ]

           lowercase__		       =						tokenizer.pad_token_id

           lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding="max_length"     ,max_length=30     ,return_tensors="np"      )
           lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding=UpperCAmelCase_     ,truncate=UpperCAmelCase_     ,return_tensors="np"      )
           lowercase__		       =						tokenizer(*UpperCAmelCase_     ,padding="max_length"     ,max_length=60     ,return_tensors="np"      )
           lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding=UpperCAmelCase_     ,truncate=UpperCAmelCase_     ,return_tensors="np"      )

           # s
           # test single string max_length padding
           self.assertEqual(out_s["input_ids"].shape[-1]     ,30      )
           self.assertTrue(pad_token_id in out_s["input_ids"]      )
           self.assertTrue(0 in out_s["attention_mask"]      )

           # s2
           # test automatic padding
           self.assertEqual(out_sa["input_ids"].shape[-1]     ,33      )
           # long slice doesn't have padding
           self.assertFalse(pad_token_id in out_sa["input_ids"][0]      )
           self.assertFalse(0 in out_sa["attention_mask"][0]      )
           # short slice does have padding
           self.assertTrue(pad_token_id in out_sa["input_ids"][1]      )
           self.assertTrue(0 in out_sa["attention_mask"][1]      )

           # p
           # test single pair max_length padding
           self.assertEqual(out_p["input_ids"].shape[-1]     ,60      )
           self.assertTrue(pad_token_id in out_p["input_ids"]      )
           self.assertTrue(0 in out_p["attention_mask"]      )

           # p2
           # test automatic padding pair
           self.assertEqual(out_pa["input_ids"].shape[-1]     ,52      )
           # long slice pair doesn't have padding
           self.assertFalse(pad_token_id in out_pa["input_ids"][0]      )
           self.assertFalse(0 in out_pa["attention_mask"][0]      )
           # short slice pair does have padding
           self.assertTrue(pad_token_id in out_pa["input_ids"][1]      )
           self.assertTrue(0 in out_pa["attention_mask"][1]      )



       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						"$$$"
           lowercase__		       =						CodeGenTokenizer.from_pretrained(self.tmpdirname     ,bos_token=UpperCAmelCase_     ,add_bos_token=UpperCAmelCase_      )

           lowercase__		       =						"This is a simple input"
           lowercase__		       =						["This is a simple input 1", "This is a simple input 2"]

           lowercase__		       =						tokenizer.bos_token_id

           lowercase__		       =						tokenizer(UpperCAmelCase_      )
           lowercase__		       =						tokenizer(UpperCAmelCase_      )

           self.assertEqual(out_s.input_ids[0]     ,UpperCAmelCase_      )
           self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

           lowercase__		       =						tokenizer.decode(out_s.input_ids      )
           lowercase__		       =						tokenizer.batch_decode(out_sa.input_ids      )

           self.assertEqual(decode_s.split()[0]     ,UpperCAmelCase_      )
           self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )



       @slow
       def 							_a      (				self      )  ->							List[Any]:
           lowercase__		       =						CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"      )

           lowercase__		       =						"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
           lowercase__		       =						"\nif len_a > len_b:      result = a\nelse:      result = b"

           lowercase__		       =						tokenizer.encode(UpperCAmelCase_      )
           lowercase__		       =						["^#", re.escape("<|endoftext|>"      ), "^'''", "^\"\"\"", "\n\n\n"]
           lowercase__		       =						tokenizer.decode(UpperCAmelCase_     ,truncate_before_pattern=UpperCAmelCase_      )
           self.assertEqual(UpperCAmelCase_     ,UpperCAmelCase_      )







       def 							_a      (				self      )  ->							Any:
           pass




