


'''simple docstring'''



import gc
import random
import unittest

import numpy as np
import torch
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import (
    DiffusionPipeline,
    UnCLIPImageVariationPipeline,
    UnCLIPScheduler,
    UNetaDConditionModel,
    UNetaDModel,
)
from diffusers.pipelines.unclip.text_proj import UnCLIPTextProjModel
from diffusers.utils import floats_tensor, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, load_image, require_torch_gpu, skip_mps

from ..pipeline_params import IMAGE_VARIATION_BATCH_PARAMS, IMAGE_VARIATION_PARAMS
from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()





class 							__UpperCamelCase    (		lowerCAmelCase_		,				unittest.TestCase ):
 A_           =   UnCLIPImageVariationPipeline
 A_           =   IMAGE_VARIATION_PARAMS - {"height", "width", "guidance_scale"}
 A_           =   IMAGE_VARIATION_BATCH_PARAMS

 A_           =   [
     "generator",
     "return_dict",
     "decoder_num_inference_steps",
     "super_res_num_inference_steps",
 ]
 A_           =   False




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return 32




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return 32




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return self.time_input_dim




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return self.time_input_dim * 4




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return 100




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	List[Any]    =  CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'							)
      return tokenizer




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      torch.manual_seed(0							)
      __a      :	Any    =  CLIPTextConfig(
          bos_token_id=0						,    eos_token_id=2						,    hidden_size=self.text_embedder_hidden_size						,    projection_dim=self.text_embedder_hidden_size						,    intermediate_size=37						,    layer_norm_eps=1E-0_5						,    num_attention_heads=4						,    num_hidden_layers=5						,    pad_token_id=1						,    vocab_size=1000						,    )
      return CLIPTextModelWithProjection(lowercase_							)




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      torch.manual_seed(0							)
      __a      :	Optional[Any]    =  CLIPVisionConfig(
          hidden_size=self.text_embedder_hidden_size						,    projection_dim=self.text_embedder_hidden_size						,    num_hidden_layers=5						,    num_attention_heads=4						,    image_size=32						,    intermediate_size=37						,    patch_size=1						,    )
      return CLIPVisionModelWithProjection(lowercase_							)




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      torch.manual_seed(0							)

      __a      :	List[str]    =  {
          'clip_embeddings_dim': self.text_embedder_hidden_size,
          'time_embed_dim': self.time_embed_dim,
          'cross_attention_dim': self.cross_attention_dim,
      }

      __a      :	Optional[int]    =  UnCLIPTextProjModel(**lowercase_							)
      return model




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      torch.manual_seed(0							)

      __a      :	Any    =  {
          'sample_size': 32,
          # RGB in channels
          'in_channels': 3,
          # Out channels is double in channels because predicts mean and variance
          'out_channels': 6,
          'down_block_types': ('ResnetDownsampleBlock2D', 'SimpleCrossAttnDownBlock2D'),
          'up_block_types': ('SimpleCrossAttnUpBlock2D', 'ResnetUpsampleBlock2D'),
          'mid_block_type': 'UNetMidBlock2DSimpleCrossAttn',
          'block_out_channels': (self.block_out_channels_a, self.block_out_channels_a * 2),
          'layers_per_block': 1,
          'cross_attention_dim': self.cross_attention_dim,
          'attention_head_dim': 4,
          'resnet_time_scale_shift': 'scale_shift',
          'class_embed_type': 'identity',
      }

      __a      :	List[str]    =  UNetaDConditionModel(**lowercase_							)
      return model




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return {
          "sample_size": 64,
          "layers_per_block": 1,
          "down_block_types": ("ResnetDownsampleBlock2D", "ResnetDownsampleBlock2D"),
          "up_block_types": ("ResnetUpsampleBlock2D", "ResnetUpsampleBlock2D"),
          "block_out_channels": (self.block_out_channels_a, self.block_out_channels_a * 2),
          "in_channels": 6,
          "out_channels": 3,
      }




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      torch.manual_seed(0							)

      __a      :	Union[str, Any]    =  UNetaDModel(**self.dummy_super_res_kwargs							)
      return model




 @property
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      torch.manual_seed(1							)

      __a      :	Union[str, Any]    =  UNetaDModel(**self.dummy_super_res_kwargs							)
      return model




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	str    =  self.dummy_decoder
      __a      :	int    =  self.dummy_text_proj
      __a      :	List[Any]    =  self.dummy_text_encoder
      __a      :	str    =  self.dummy_tokenizer
      __a      :	Tuple    =  self.dummy_super_res_first
      __a      :	List[Any]    =  self.dummy_super_res_last

      __a      :	List[str]    =  UnCLIPScheduler(
          variance_type='learned_range'						,    prediction_type='epsilon'						,    num_train_timesteps=1000						,    )

      __a      :	Optional[int]    =  UnCLIPScheduler(
          variance_type='fixed_small_log'						,    prediction_type='epsilon'						,    num_train_timesteps=1000						,    )

      __a      :	List[str]    =  CLIPImageProcessor(crop_size=32						,    size=32							)

      __a      :	Optional[Any]    =  self.dummy_image_encoder

      return {
          "decoder": decoder,
          "text_encoder": text_encoder,
          "tokenizer": tokenizer,
          "text_proj": text_proj,
          "feature_extractor": feature_extractor,
          "image_encoder": image_encoder,
          "super_res_first": super_res_first,
          "super_res_last": super_res_last,
          "decoder_scheduler": decoder_scheduler,
          "super_res_scheduler": super_res_scheduler,
      }




 def 				__UpperCAmelCase							(       self						,    __a						,    __a=0						,    __a=True							):

      '''simple docstring'''





      __a      :	Tuple    =  floats_tensor((1, 3, 32, 32)						,    rng=random.Random(lowercase_							)							).to(lowercase_							)
      if str(lowercase_							).startswith('mps'							):
           __a      :	Tuple    =  torch.manual_seed(lowercase_							)
      else:
           __a      :	int    =  torch.Generator(device=lowercase_							).manual_seed(lowercase_							)

      if pil_image:
           __a      :	int    =  input_image * 0.5 + 0.5
           __a      :	Optional[Any]    =  input_image.clamp(0						,    1							)
           __a      :	List[Any]    =  input_image.cpu().permute(0						,    2						,    3						,    1							).float().numpy()
           __a      :	List[Any]    =  DiffusionPipeline.numpy_to_pil(lowercase_							)[0]

      return {
          "image": input_image,
          "generator": generator,
          "decoder_num_inference_steps": 2,
          "super_res_num_inference_steps": 2,
          "output_type": "np",
      }




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	str    =  'cpu'

      __a      :	List[Any]    =  self.get_dummy_components()

      __a      :	str    =  self.pipeline_class(**lowercase_							)
      __a      :	Dict    =  pipe.to(lowercase_							)

      pipe.set_progress_bar_config(disable=lowercase_							)

      __a      :	str    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)

      __a      :	str    =  pipe(**lowercase_							)
      __a      :	str    =  output.images

      __a      :	Tuple    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)

      __a      :	List[str]    =  pipe(
          **lowercase_						,    return_dict=lowercase_						,    )[0]

      __a      :	List[Any]    =  image[0, -3:, -3:, -1]
      __a      :	Any    =  image_from_tuple[0, -3:, -3:, -1]

      assert image.shape == (1, 64, 64, 3)

      __a      :	Dict    =  np.array(
          [
              0.9997,
              0.0002,
              0.9997,
              0.9997,
              0.9969,
              0.0023,
              0.9997,
              0.9969,
              0.9970,
          ]							)

      assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2
      assert np.abs(image_from_tuple_slice.flatten() - expected_slice							).max() < 1E-2




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	List[Any]    =  'cpu'

      __a      :	str    =  self.get_dummy_components()

      __a      :	List[Any]    =  self.pipeline_class(**lowercase_							)
      __a      :	Optional[int]    =  pipe.to(lowercase_							)

      pipe.set_progress_bar_config(disable=lowercase_							)

      __a      :	str    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)

      __a      :	List[str]    =  pipe(**lowercase_							)
      __a      :	List[Any]    =  output.images

      __a      :	int    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)

      __a      :	Dict    =  pipe(
          **lowercase_						,    return_dict=lowercase_						,    )[0]

      __a      :	Any    =  image[0, -3:, -3:, -1]
      __a      :	Dict    =  image_from_tuple[0, -3:, -3:, -1]

      assert image.shape == (1, 64, 64, 3)

      __a      :	List[str]    =  np.array([0.9997, 0.0003, 0.9997, 0.9997, 0.9970, 0.0024, 0.9997, 0.9971, 0.9971]							)

      assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2
      assert np.abs(image_from_tuple_slice.flatten() - expected_slice							).max() < 1E-2




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	List[str]    =  'cpu'

      __a      :	Optional[int]    =  self.get_dummy_components()

      __a      :	List[str]    =  self.pipeline_class(**lowercase_							)
      __a      :	str    =  pipe.to(lowercase_							)

      pipe.set_progress_bar_config(disable=lowercase_							)

      __a      :	Optional[Any]    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)
      __a      :	List[Any]    =  [
          pipeline_inputs['image'],
          pipeline_inputs['image'],
      ]

      __a      :	str    =  pipe(**lowercase_							)
      __a      :	Any    =  output.images

      __a      :	str    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)
      __a      :	List[str]    =  [
          tuple_pipeline_inputs['image'],
          tuple_pipeline_inputs['image'],
      ]

      __a      :	List[Any]    =  pipe(
          **lowercase_						,    return_dict=lowercase_						,    )[0]

      __a      :	int    =  image[0, -3:, -3:, -1]
      __a      :	Dict    =  image_from_tuple[0, -3:, -3:, -1]

      assert image.shape == (2, 64, 64, 3)

      __a      :	str    =  np.array(
          [
              0.9997,
              0.9989,
              0.0008,
              0.0021,
              0.9960,
              0.0018,
              0.0014,
              0.0002,
              0.9933,
          ]							)

      assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2
      assert np.abs(image_from_tuple_slice.flatten() - expected_slice							).max() < 1E-2




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	Optional[Any]    =  torch.device('cpu'							)





      class 							__UpperCamelCase    :
       A_           =   1



      __a      :	int    =  self.get_dummy_components()

      __a      :	Optional[int]    =  self.pipeline_class(**lowercase_							)
      __a      :	Tuple    =  pipe.to(lowercase_							)

      pipe.set_progress_bar_config(disable=lowercase_							)

      __a      :	Union[str, Any]    =  torch.Generator(device=lowercase_							).manual_seed(0							)
      __a      :	Union[str, Any]    =  pipe.decoder.dtype
      __a      :	Optional[int]    =  1

      __a      :	int    =  (
          batch_size,
          pipe.decoder.config.in_channels,
          pipe.decoder.config.sample_size,
          pipe.decoder.config.sample_size,
      )
      __a      :	str    =  pipe.prepare_latents(
          lowercase_						,    dtype=lowercase_						,    device=lowercase_						,    generator=lowercase_						,    latents=lowercase_						,    scheduler=DummyScheduler()							)

      __a      :	Optional[int]    =  (
          batch_size,
          pipe.super_res_first.config.in_channels // 2,
          pipe.super_res_first.config.sample_size,
          pipe.super_res_first.config.sample_size,
      )
      __a      :	Tuple    =  pipe.prepare_latents(
          lowercase_						,    dtype=lowercase_						,    device=lowercase_						,    generator=lowercase_						,    latents=lowercase_						,    scheduler=DummyScheduler()							)

      __a      :	Any    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)

      __a      :	Dict    =  pipe(
          **lowercase_						,    decoder_latents=lowercase_						,    super_res_latents=lowercase_							).images

      __a      :	List[Any]    =  self.get_dummy_inputs(lowercase_						,    pil_image=lowercase_							)
      # Don't pass image, instead pass embedding
      __a      :	List[str]    =  pipeline_inputs.pop('image'							)
      __a      :	Tuple    =  pipe.image_encoder(lowercase_							).image_embeds

      __a      :	Tuple    =  pipe(
          **lowercase_						,    decoder_latents=lowercase_						,    super_res_latents=lowercase_						,    image_embeddings=lowercase_						,    ).images

      # make sure passing text embeddings manually is identical
      assert np.abs(img_out_a - img_out_a							).max() < 1E-4




 @skip_mps
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	Any    =  torch_device == 'cpu'

      # Check is relaxed because there is not a torch 2.0 sliced attention added kv processor
      __a      :	Dict    =  1E-2

      self._test_attention_slicing_forward_pass(
          test_max_difference=lowercase_						,    expected_max_diff=lowercase_							)




 @skip_mps
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	Any    =  torch_device == 'cpu'
      __a      :	Tuple    =  True
      __a      :	Optional[int]    =  [
          'decoder_num_inference_steps',
          'super_res_num_inference_steps',
      ]

      self._test_inference_batch_single_identical(
          test_max_difference=lowercase_						,    relax_max_difference=lowercase_						,    additional_params_copy_to_batched_inputs=lowercase_						,    )




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	List[str]    =  [
          'decoder_num_inference_steps',
          'super_res_num_inference_steps',
      ]

      if torch_device == "mps":
           # TODO: MPS errors with larger batch sizes
           __a      :	Optional[int]    =  [2, 3]
           self._test_inference_batch_consistent(
               batch_sizes=lowercase_						,    additional_params_copy_to_batched_inputs=lowercase_						,    )
      else:
           self._test_inference_batch_consistent(
               additional_params_copy_to_batched_inputs=lowercase_							)




 @skip_mps
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return super().test_dict_tuple_outputs_equivalent()




 @skip_mps
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return super().test_save_load_local()





 @skip_mps
 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      return super().test_save_load_optional_components()





@slow
@require_torch_gpu
class 							__UpperCamelCase    (		unittest.TestCase ):




 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()





 def 				__UpperCAmelCase							(       self							):

      '''simple docstring'''





      __a      :	List[str]    =  load_image(
          'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/unclip/cat.png'							)
      __a      :	List[str]    =  load_numpy(
          'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
          '/unclip/karlo_v1_alpha_cat_variation_fp16.npy'							)

      __a      :	str    =  UnCLIPImageVariationPipeline.from_pretrained(
          'kakaobrain/karlo-v1-alpha-image-variations'						,    torch_dtype=torch.floataa							)
      __a      :	str    =  pipeline.to(lowercase_							)
      pipeline.set_progress_bar_config(disable=lowercase_							)

      __a      :	Union[str, Any]    =  torch.Generator(device='cpu'							).manual_seed(0							)
      __a      :	List[str]    =  pipeline(
          lowercase_						,    generator=lowercase_						,    output_type='np'						,    )

      __a      :	Union[str, Any]    =  output.images[0]

      assert image.shape == (256, 256, 3)

      assert_mean_pixel_difference(lowercase_						,    lowercase_						,    15							)







'''simple docstring'''



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							__UpperCamelCase    (		lowerCAmelCase_		,				unittest.TestCase ):
	A_           =   CodeGenTokenizer
	A_           =   CodeGenTokenizerFast
	A_           =   True
	A_           =   {"add_prefix_space": True}
	A_           =   False




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						__a      :	Tuple    =  [
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    '\u0120',
						    '\u0120l',
						    '\u0120n',
						    '\u0120lo',
						    '\u0120low',
						    'er',
						    '\u0120lowest',
						    '\u0120newer',
						    '\u0120wider',
						    '<unk>',
						    '<|endoftext|>',
						]
						__a      :	Union[str, Any]    =  dict(zip(__a						,    range(len(__a							)							)							)							)
						__a      :	Tuple    =  ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
						__a      :	Dict    =  {'unk_token': '<unk>'}

						__a      :	List[Any]    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['vocab_file']							)
						__a      :	List[str]    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['merges_file']							)
						with open(self.vocab_file						,    'w'						,    encoding='utf-8'							) as fp:
											fp.write(json.dumps(__a							) + '\n'							)
						with open(self.merges_file						,    'w'						,    encoding='utf-8'							) as fp:
											fp.write('\n'.join(__a							)							)




	def 				__UpperCAmelCase							(       self						,    **__a							):

						'''simple docstring'''





						kwargs.update(self.special_tokens_map							)
						return CodeGenTokenizer.from_pretrained(self.tmpdirname						,    **__a							)




	def 				__UpperCAmelCase							(       self						,    **__a							):

						'''simple docstring'''





						kwargs.update(self.special_tokens_map							)
						return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,    **__a							)




	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						__a      :	Tuple    =  'lower newer'
						__a      :	Tuple    =  'lower newer'
						return input_text, output_text




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  CodeGenTokenizer(self.vocab_file						,    self.merges_file						,    **self.special_tokens_map							)
						__a      :	str    =  'lower newer'
						__a      :	Tuple    =  ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
						__a      :	Dict    =  tokenizer.tokenize(__a						,    add_prefix_space=__a							)
						self.assertListEqual(__a						,    __a							)

						__a      :	List[str]    =  tokens + [tokenizer.unk_token]
						__a      :	Any    =  [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)						,    __a							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						if not self.test_rust_tokenizer:
											return

						__a      :	List[Any]    =  self.get_tokenizer()
						__a      :	List[str]    =  self.get_rust_tokenizer(add_prefix_space=__a							)

						__a      :	Any    =  'lower newer'

						# Testing tokenization
						__a      :	Dict    =  tokenizer.tokenize(__a						,    add_prefix_space=__a							)
						__a      :	Dict    =  rust_tokenizer.tokenize(__a							)
						self.assertListEqual(__a						,    __a							)

						# Testing conversion to ids without special tokens
						__a      :	int    =  tokenizer.encode(__a						,    add_special_tokens=__a						,    add_prefix_space=__a							)
						__a      :	Tuple    =  rust_tokenizer.encode(__a						,    add_special_tokens=__a							)
						self.assertListEqual(__a						,    __a							)

						# Testing conversion to ids with special tokens
						__a      :	Tuple    =  self.get_rust_tokenizer(add_prefix_space=__a							)
						__a      :	Union[str, Any]    =  tokenizer.encode(__a						,    add_prefix_space=__a							)
						__a      :	int    =  rust_tokenizer.encode(__a							)
						self.assertListEqual(__a						,    __a							)

						# Testing the unknown token
						__a      :	Any    =  tokens + [rust_tokenizer.unk_token]
						__a      :	Tuple    =  [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__a							)						,    __a							)




	def 				__UpperCAmelCase							(       self						,    *__a						,    **__a							):

						'''simple docstring'''





						pass




	def 				__UpperCAmelCase							(       self						,    __a=15							):

						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a      :	Optional[int]    =  self.rust_tokenizer_class.from_pretrained(__a						,    **__a							)

																# Simple input
																__a      :	List[Any]    =  'This is a simple input'
																__a      :	Tuple    =  ['This is a simple input 1', 'This is a simple input 2']
																__a      :	Tuple    =  ('This is a simple input', 'This is a pair')
																__a      :	str    =  [
																    ('This is a simple input 1', 'This is a simple input 2'),
																    ('This is a simple pair 1', 'This is a simple pair 2'),
																]

																# Simple input tests
																self.assertRaises(__a						,    tokenizer_r.encode						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Simple input
																self.assertRaises(__a						,    tokenizer_r.encode_plus						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Simple input
																self.assertRaises(
																    __a						,    tokenizer_r.batch_encode_plus						,    __a						,    max_length=__a						,    padding='max_length'						,    )

																# Pair input
																self.assertRaises(__a						,    tokenizer_r.encode						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Pair input
																self.assertRaises(__a						,    tokenizer_r.encode_plus						,    __a						,    max_length=__a						,    padding='max_length'							)

																# Pair input
																self.assertRaises(
																    __a						,    tokenizer_r.batch_encode_plus						,    __a						,    max_length=__a						,    padding='max_length'						,    )




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  CodeGenTokenizer.from_pretrained(self.tmpdirname						,    pad_token='<pad>'							)

						# Simple input
						__a      :	str    =  'This is a simple input'
						__a      :	Any    =  ['This is a simple input looooooooong', 'This is a simple input']
						__a      :	Optional[int]    =  ('This is a simple input', 'This is a pair')
						__a      :	Optional[Any]    =  [
						    ('This is a simple input loooooong', 'This is a simple input'),
						    ('This is a simple pair loooooong', 'This is a simple pair'),
						]

						__a      :	int    =  tokenizer.pad_token_id

						__a      :	List[Any]    =  tokenizer(__a						,    padding='max_length'						,    max_length=30						,    return_tensors='np'							)
						__a      :	Union[str, Any]    =  tokenizer(__a						,    padding=__a						,    truncate=__a						,    return_tensors='np'							)
						__a      :	Optional[Any]    =  tokenizer(*__a						,    padding='max_length'						,    max_length=60						,    return_tensors='np'							)
						__a      :	List[Any]    =  tokenizer(__a						,    padding=__a						,    truncate=__a						,    return_tensors='np'							)

						# s
						# test single string max_length padding
						self.assertEqual(out_s['input_ids'].shape[-1]						,    30							)
						self.assertTrue(pad_token_id in out_s['input_ids']							)
						self.assertTrue(0 in out_s['attention_mask']							)

						# s2
						# test automatic padding
						self.assertEqual(out_sa['input_ids'].shape[-1]						,    33							)
						# long slice doesn't have padding
						self.assertFalse(pad_token_id in out_sa['input_ids'][0]							)
						self.assertFalse(0 in out_sa['attention_mask'][0]							)
						# short slice does have padding
						self.assertTrue(pad_token_id in out_sa['input_ids'][1]							)
						self.assertTrue(0 in out_sa['attention_mask'][1]							)

						# p
						# test single pair max_length padding
						self.assertEqual(out_p['input_ids'].shape[-1]						,    60							)
						self.assertTrue(pad_token_id in out_p['input_ids']							)
						self.assertTrue(0 in out_p['attention_mask']							)

						# p2
						# test automatic padding pair
						self.assertEqual(out_pa['input_ids'].shape[-1]						,    52							)
						# long slice pair doesn't have padding
						self.assertFalse(pad_token_id in out_pa['input_ids'][0]							)
						self.assertFalse(0 in out_pa['attention_mask'][0]							)
						# short slice pair does have padding
						self.assertTrue(pad_token_id in out_pa['input_ids'][1]							)
						self.assertTrue(0 in out_pa['attention_mask'][1]							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  '$$$'
						__a      :	List[str]    =  CodeGenTokenizer.from_pretrained(self.tmpdirname						,    bos_token=__a						,    add_bos_token=__a							)

						__a      :	Union[str, Any]    =  'This is a simple input'
						__a      :	List[Any]    =  ['This is a simple input 1', 'This is a simple input 2']

						__a      :	List[Any]    =  tokenizer.bos_token_id

						__a      :	List[str]    =  tokenizer(__a							)
						__a      :	Optional[Any]    =  tokenizer(__a							)

						self.assertEqual(out_s.input_ids[0]						,    __a							)
						self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

						__a      :	Any    =  tokenizer.decode(out_s.input_ids							)
						__a      :	Union[str, Any]    =  tokenizer.batch_decode(out_sa.input_ids							)

						self.assertEqual(decode_s.split()[0]						,    __a							)
						self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)




	@slow
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'							)

						__a      :	Optional[int]    =  '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
						__a      :	Tuple    =  '\nif len_a > len_b:      result = a\nelse:      result = b'

						__a      :	Optional[int]    =  tokenizer.encode(__a							)
						__a      :	Union[str, Any]    =  ['^#', re.escape('<|endoftext|>'							), '^\'\'\'', '^"""', '\n\n\n']
						__a      :	Tuple    =  tokenizer.decode(__a						,    truncate_before_pattern=__a							)
						self.assertEqual(__a						,    __a							)





	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						pass



