






'''simple docstring'''
import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_flax_cross_test,
    require_flax,
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import is_flax_available, is_torch_available, is_vision_available

from ...test_modeling_flax_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_flax_bert import FlaxBertModelTester
from ..clip.test_modeling_flax_clip import FlaxCLIPVisionModelTester
from ..vit.test_modeling_flax_vit import FlaxViTModelTester


if is_flax_available():
  from transformers import (
      FlaxBertModel,
      FlaxCLIPVisionModel,
      FlaxVisionTextDualEncoderModel,
      FlaxViTModel,
      VisionTextDualEncoderConfig,
      VisionTextDualEncoderProcessor,
  )
  from transformers.modeling_flax_pytorch_utils import (
      convert_pytorch_state_dict_to_flax,
      load_flax_weights_in_pytorch_model,
  )


if is_torch_available():
  import torch

  from transformers import VisionTextDualEncoderModel

if is_vision_available():
  from PIL import Image





def 			_a(	UpperCamelCase__					:    Optional[Any]				):




      '''simple docstring'''






      if isinstance(lowerCAmelCase__,						collections.abc.Iterable				):
            return x
      return (x, x)






@require_flax
class   __SCREAMING_SNAKE_CASE   :




 def   __magic_name__						(   self						:				Any       ,   __lowercase						:				Tuple       ,   __lowercase						:				Union[str, Any]   )						->    Any:
       pass




 def   __magic_name__						(   self						:				Union[str, Any]   )						->    Optional[Any]:
       pass




 def   __magic_name__						(   self						:				Tuple   )						->    Optional[int]:
       pass




 def   __magic_name__						(   self						:				Union[str, Any]       ,   __lowercase						:				np.ndarray       ,   __lowercase						:				np.ndarray       ,   __lowercase						:				float   )						->    Any:
       SCREAMING_SNAKE_CASE__  :					Tuple						=np.abs((a - b)   ).max()
       self.assertLessEqual(__lowercase       ,   __lowercase       ,   F"Difference between torch and flax is {diff} (>= {tol})."   )




 def   __magic_name__						(   self						:				Optional[Any]       ,   __lowercase						:				Dict       ,   __lowercase						:				Any       ,   __lowercase						:				str       ,   __lowercase						:				Dict       ,   __lowercase						:				Tuple=None       ,   **__lowercase						:				Optional[int]   )						->    Union[str, Any]:
       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=VisionTextDualEncoderConfig.from_vision_text_configs(__lowercase       ,   __lowercase   )

       SCREAMING_SNAKE_CASE__  :					int						=FlaxVisionTextDualEncoderModel(__lowercase   )

       SCREAMING_SNAKE_CASE__  :					str						=model(input_ids=__lowercase       ,   pixel_values=__lowercase       ,   attention_mask=__lowercase   )

       self.assertEqual(output['''text_embeds'''].shape       ,   (input_ids.shape[0], config.projection_dim)   )
       self.assertEqual(output['''image_embeds'''].shape       ,   (pixel_values.shape[0], config.projection_dim)   )




 def   __magic_name__						(   self						:				List[str]       ,   __lowercase						:				str       ,   __lowercase						:				List[str]       ,   __lowercase						:				Optional[int]       ,   __lowercase						:				str       ,   __lowercase						:				Optional[int]=None       ,   **__lowercase						:				Optional[int]   )						->    Dict:
       SCREAMING_SNAKE_CASE__  :					int						=self.get_vision_text_model(__lowercase       ,   __lowercase   )
       SCREAMING_SNAKE_CASE__  :					Dict						={'vision_model': vision_model, 'text_model': text_model}
       SCREAMING_SNAKE_CASE__  :					Any						=FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__lowercase   )

       SCREAMING_SNAKE_CASE__  :					Tuple						=model(input_ids=__lowercase       ,   pixel_values=__lowercase       ,   attention_mask=__lowercase   )

       self.assertEqual(output['''text_embeds'''].shape       ,   (input_ids.shape[0], model.config.projection_dim)   )
       self.assertEqual(output['''image_embeds'''].shape       ,   (pixel_values.shape[0], model.config.projection_dim)   )




 def   __magic_name__						(   self						:				Optional[int]       ,   __lowercase						:				int       ,   __lowercase						:				str       ,   __lowercase						:				List[str]       ,   __lowercase						:				List[Any]       ,   __lowercase						:				Union[str, Any]=None       ,   **__lowercase						:				Tuple   )						->    Optional[Any]:
       SCREAMING_SNAKE_CASE__  :					Any						=self.get_vision_text_model(__lowercase       ,   __lowercase   )
       SCREAMING_SNAKE_CASE__  :					Optional[int]						={'vision_model': vision_model, 'text_model': text_model}
       SCREAMING_SNAKE_CASE__  :					Dict						=FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__lowercase   )

       SCREAMING_SNAKE_CASE__  :					Tuple						=model(input_ids=__lowercase       ,   pixel_values=__lowercase       ,   attention_mask=__lowercase   )
       SCREAMING_SNAKE_CASE__  :					Tuple						=output[0]

       with tempfile.TemporaryDirectory() as tmpdirname:
             model.save_pretrained(__lowercase   )
             SCREAMING_SNAKE_CASE__  :					Optional[int]						=FlaxVisionTextDualEncoderModel.from_pretrained(__lowercase   )

             SCREAMING_SNAKE_CASE__  :					Dict						=model(input_ids=__lowercase       ,   pixel_values=__lowercase       ,   attention_mask=__lowercase   )
             SCREAMING_SNAKE_CASE__  :					str						=after_output[0]
             SCREAMING_SNAKE_CASE__  :					Tuple						=np.amax(np.abs(out_a - out_a   )   )
             self.assertLessEqual(__lowercase       ,   1e-3   )




 def   __magic_name__						(   self						:				int       ,   __lowercase						:				Tuple       ,   __lowercase						:				Dict       ,   __lowercase						:				List[str]       ,   __lowercase						:				Union[str, Any]       ,   __lowercase						:				Union[str, Any]=None       ,   **__lowercase						:				List[str]   )						->    List[str]:
       SCREAMING_SNAKE_CASE__  :					List[Any]						=self.get_vision_text_model(__lowercase       ,   __lowercase   )
       SCREAMING_SNAKE_CASE__  :					str						={'vision_model': vision_model, 'text_model': text_model}
       SCREAMING_SNAKE_CASE__  :					str						=FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__lowercase   )

       SCREAMING_SNAKE_CASE__  :					List[Any]						=model(
           input_ids=__lowercase       ,   pixel_values=__lowercase       ,   attention_mask=__lowercase       ,   output_attentions=__lowercase   )

       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=output.vision_model_output.attentions
       self.assertEqual(len(__lowercase   )       ,   vision_config.num_hidden_layers   )

       # in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
       SCREAMING_SNAKE_CASE__  :					str						=to_atuple(vision_model.config.image_size   )
       SCREAMING_SNAKE_CASE__  :					Dict						=to_atuple(vision_model.config.patch_size   )
       SCREAMING_SNAKE_CASE__  :					List[str]						=(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
       SCREAMING_SNAKE_CASE__  :					Tuple						=num_patches + 1
       self.assertEqual(vision_attentions[0].shape[-3:]       ,   (vision_config.num_attention_heads, seq_len, seq_len)   )

       SCREAMING_SNAKE_CASE__  :					Optional[Any]						=output.text_model_output.attentions
       self.assertEqual(len(__lowercase   )       ,   text_config.num_hidden_layers   )

       self.assertEqual(
           text_attentions[0].shape[-3:]       ,   (text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])       ,   )




 def   __magic_name__						(   self						:				Union[str, Any]       ,   __lowercase						:				Union[str, Any]       ,   __lowercase						:				Any       ,   __lowercase						:				Optional[Any]   )						->    int:
       pt_model.to(__lowercase   )
       pt_model.eval()

       # prepare inputs
       SCREAMING_SNAKE_CASE__  :					Optional[int]						=inputs_dict
       SCREAMING_SNAKE_CASE__  :					int						={k: torch.tensor(v.tolist()   ) for k, v in flax_inputs.items()}

       with torch.no_grad():
             SCREAMING_SNAKE_CASE__  :					Any						=pt_model(**__lowercase   ).to_tuple()

       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=fx_model(**__lowercase   ).to_tuple()
       self.assertEqual(len(__lowercase   )       ,   len(__lowercase   )       ,   '''Output lengths differ between Flax and PyTorch'''   )
       for fx_output, pt_output in zip(fx_outputs[:4]       ,   pt_outputs[:4]   ):
             self.assert_almost_equals(__lowercase       ,   pt_output.numpy()       ,   4e-2   )

       # PT -> Flax
       with tempfile.TemporaryDirectory() as tmpdirname:
             pt_model.save_pretrained(__lowercase   )
             SCREAMING_SNAKE_CASE__  :					Optional[Any]						=FlaxVisionTextDualEncoderModel.from_pretrained(__lowercase       ,   from_pt=__lowercase   )

       SCREAMING_SNAKE_CASE__  :					str						=fx_model_loaded(**__lowercase   ).to_tuple()
       self.assertEqual(len(__lowercase   )       ,   len(__lowercase   )       ,   '''Output lengths differ between Flax and PyTorch'''   )
       for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4]       ,   pt_outputs[:4]   ):
             self.assert_almost_equals(__lowercase       ,   pt_output.numpy()       ,   4e-2   )

       # Flax -> PT
       with tempfile.TemporaryDirectory() as tmpdirname:
             fx_model.save_pretrained(__lowercase   )
             SCREAMING_SNAKE_CASE__  :					Optional[int]						=VisionTextDualEncoderModel.from_pretrained(__lowercase       ,   from_flax=__lowercase   )

       pt_model_loaded.to(__lowercase   )
       pt_model_loaded.eval()

       with torch.no_grad():
             SCREAMING_SNAKE_CASE__  :					Optional[Any]						=pt_model_loaded(**__lowercase   ).to_tuple()

       self.assertEqual(len(__lowercase   )       ,   len(__lowercase   )       ,   '''Output lengths differ between Flax and PyTorch'''   )
       for fx_output, pt_output_loaded in zip(fx_outputs[:4]       ,   pt_outputs_loaded[:4]   ):
             self.assert_almost_equals(__lowercase       ,   pt_output_loaded.numpy()       ,   4e-2   )




 def   __magic_name__						(   self						:				Dict       ,   __lowercase						:				Dict       ,   __lowercase						:				Optional[Any]       ,   __lowercase						:				List[str]   )						->    Union[str, Any]:
       SCREAMING_SNAKE_CASE__  :					Tuple						=VisionTextDualEncoderConfig.from_vision_text_configs(__lowercase       ,   __lowercase   )

       SCREAMING_SNAKE_CASE__  :					List[str]						=VisionTextDualEncoderModel(__lowercase   )
       SCREAMING_SNAKE_CASE__  :					int						=FlaxVisionTextDualEncoderModel(__lowercase   )

       SCREAMING_SNAKE_CASE__  :					Any						=convert_pytorch_state_dict_to_flax(pt_model.state_dict()       ,   __lowercase   )
       SCREAMING_SNAKE_CASE__  :					Any						=fx_state

       self.check_pt_flax_equivalence(__lowercase       ,   __lowercase       ,   __lowercase   )




 def   __magic_name__						(   self						:				int       ,   __lowercase						:				Any       ,   __lowercase						:				Any       ,   __lowercase						:				Optional[int]   )						->    List[str]:
       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=VisionTextDualEncoderConfig.from_vision_text_configs(__lowercase       ,   __lowercase   )

       SCREAMING_SNAKE_CASE__  :					List[str]						=VisionTextDualEncoderModel(__lowercase   )
       SCREAMING_SNAKE_CASE__  :					List[str]						=FlaxVisionTextDualEncoderModel(__lowercase   )

       SCREAMING_SNAKE_CASE__  :					Optional[Any]						=load_flax_weights_in_pytorch_model(__lowercase       ,   fx_model.params   )

       self.check_pt_flax_equivalence(__lowercase       ,   __lowercase       ,   __lowercase   )




 def   __magic_name__						(   self						:				Union[str, Any]   )						->    Dict:
       SCREAMING_SNAKE_CASE__  :					Any						=self.prepare_config_and_inputs()
       self.check_model_from_pretrained_configs(**__lowercase   )




 def   __magic_name__						(   self						:				List[str]   )						->    Tuple:
       SCREAMING_SNAKE_CASE__  :					Optional[int]						=self.prepare_config_and_inputs()
       self.check_vision_text_dual_encoder_from_pretrained(**__lowercase   )




 def   __magic_name__						(   self						:				Union[str, Any]   )						->    Any:
       SCREAMING_SNAKE_CASE__  :					Optional[int]						=self.prepare_config_and_inputs()
       self.check_save_load(**__lowercase   )




 def   __magic_name__						(   self						:				Tuple   )						->    Tuple:
       SCREAMING_SNAKE_CASE__  :					List[Any]						=self.prepare_config_and_inputs()
       self.check_vision_text_output_attention(**__lowercase   )




 @is_pt_flax_cross_test
 def   __magic_name__						(   self						:				Optional[int]   )						->    Union[str, Any]:
       SCREAMING_SNAKE_CASE__  :					List[Any]						=self.prepare_config_and_inputs()
       SCREAMING_SNAKE_CASE__  :					Any						=config_inputs_dict.pop('''vision_config'''   )
       SCREAMING_SNAKE_CASE__  :					Any						=config_inputs_dict.pop('''text_config'''   )

       SCREAMING_SNAKE_CASE__  :					List[str]						=config_inputs_dict

       self.check_equivalence_pt_to_flax(__lowercase       ,   __lowercase       ,   __lowercase   )
       self.check_equivalence_flax_to_pt(__lowercase       ,   __lowercase       ,   __lowercase   )




 @slow
 def   __magic_name__						(   self						:				Any   )						->    List[Any]:
       SCREAMING_SNAKE_CASE__  :					List[Any]						=self.get_pretrained_model_and_inputs()

       SCREAMING_SNAKE_CASE__  :					List[Any]						=model_a(**__lowercase   )
       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=outputs[0]

       with tempfile.TemporaryDirectory() as tmp_dirname:
             model_a.save_pretrained(__lowercase   )
             SCREAMING_SNAKE_CASE__  :					Dict						=FlaxVisionTextDualEncoderModel.from_pretrained(__lowercase   )

             SCREAMING_SNAKE_CASE__  :					str						=model_a(**__lowercase   )
             SCREAMING_SNAKE_CASE__  :					List[str]						=after_outputs[0]
             SCREAMING_SNAKE_CASE__  :					Tuple						=np.amax(np.abs(out_a - out_a   )   )
             self.assertLessEqual(__lowercase       ,   1e-5   )






@require_flax
class   __SCREAMING_SNAKE_CASE   (      snake_case__						,					unittest.TestCase					):




 def   __magic_name__						(   self						:				List[str]   )						->    Union[str, Any]:
       SCREAMING_SNAKE_CASE__  :					Any						=FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
           '''hf-internal-testing/tiny-random-vit'''       ,   '''hf-internal-testing/tiny-bert'''       ,   vision_from_pt=__lowercase       ,   text_from_pt=__lowercase       ,   )
       SCREAMING_SNAKE_CASE__  :					Optional[Any]						=13
       SCREAMING_SNAKE_CASE__  :					List[Any]						=floats_tensor(
           [
               batch_size,
               model.config.vision_config.num_channels,
               model.config.vision_config.image_size,
               model.config.vision_config.image_size,
           ]   )
       SCREAMING_SNAKE_CASE__  :					int						=ids_tensor([batch_size, 4]       ,   model.config.text_config.vocab_size   )
       SCREAMING_SNAKE_CASE__  :					str						=random_attention_mask([batch_size, 4]   )
       SCREAMING_SNAKE_CASE__  :					Dict						={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

       return model, inputs




 def   __magic_name__						(   self						:				Optional[int]       ,   __lowercase						:				Tuple       ,   __lowercase						:				Any   )						->    Tuple:
       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=FlaxViTModel(__lowercase   )
       SCREAMING_SNAKE_CASE__  :					List[Any]						=FlaxBertModel(__lowercase   )
       return vision_model, text_model




 def   __magic_name__						(   self						:				List[Any]   )						->    Tuple:
       SCREAMING_SNAKE_CASE__  :					List[str]						=FlaxViTModelTester(self   )
       SCREAMING_SNAKE_CASE__  :					List[Any]						=FlaxBertModelTester(self   )
       SCREAMING_SNAKE_CASE__  :					str						=vit_model_tester.prepare_config_and_inputs()
       SCREAMING_SNAKE_CASE__  :					List[str]						=bert_model_tester.prepare_config_and_inputs()

       SCREAMING_SNAKE_CASE__  :					Optional[int]						=vision_config_and_inputs

       SCREAMING_SNAKE_CASE__  :					Dict						=text_config_and_inputs

       # make sure that cross attention layers are added
       return {
           "text_config": text_config,
           "vision_config": vision_config,
           "pixel_values": pixel_values,
           "attention_mask": attention_mask,
           "input_ids": input_ids,
           "token_type_ids": token_type_ids,
       }






@require_torch
class   __SCREAMING_SNAKE_CASE   (      snake_case__						,					unittest.TestCase					):




 def   __magic_name__						(   self						:				Optional[int]   )						->    List[str]:
       SCREAMING_SNAKE_CASE__  :					Tuple						=FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
           '''hf-internal-testing/tiny-random-clip'''       ,   '''hf-internal-testing/tiny-bert'''       ,   vision_from_pt=__lowercase       ,   text_from_pt=__lowercase       ,   )
       SCREAMING_SNAKE_CASE__  :					Optional[Any]						=13
       SCREAMING_SNAKE_CASE__  :					str						=floats_tensor(
           [
               batch_size,
               model.config.vision_config.num_channels,
               model.config.vision_config.image_size,
               model.config.vision_config.image_size,
           ]   )
       SCREAMING_SNAKE_CASE__  :					str						=ids_tensor([batch_size, 4]       ,   model.config.text_config.vocab_size   )
       SCREAMING_SNAKE_CASE__  :					List[Any]						=random_attention_mask([batch_size, 4]   )
       SCREAMING_SNAKE_CASE__  :					str						={'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

       return model, inputs




 def   __magic_name__						(   self						:				int       ,   __lowercase						:				Optional[int]       ,   __lowercase						:				str   )						->    Union[str, Any]:
       SCREAMING_SNAKE_CASE__  :					List[str]						=FlaxCLIPVisionModel(__lowercase   )
       SCREAMING_SNAKE_CASE__  :					Tuple						=FlaxBertModel(__lowercase   )
       return vision_model, text_model




 def   __magic_name__						(   self						:				Any   )						->    List[str]:
       SCREAMING_SNAKE_CASE__  :					int						=FlaxCLIPVisionModelTester(self   )
       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=FlaxBertModelTester(self   )
       SCREAMING_SNAKE_CASE__  :					Optional[Any]						=clip_model_tester.prepare_config_and_inputs()
       SCREAMING_SNAKE_CASE__  :					Any						=bert_model_tester.prepare_config_and_inputs()

       SCREAMING_SNAKE_CASE__  :					Union[str, Any]						=vision_config_and_inputs

       SCREAMING_SNAKE_CASE__  :					str						=text_config_and_inputs

       # make sure that cross attention layers are added
       return {
           "text_config": text_config,
           "vision_config": vision_config,
           "pixel_values": pixel_values,
           "attention_mask": attention_mask,
           "input_ids": input_ids,
           "token_type_ids": token_type_ids,
       }






@require_flax
@require_vision
class   __SCREAMING_SNAKE_CASE   (      unittest.TestCase					):




 @slow
 def   __magic_name__						(   self						:				Optional[int]   )						->    Optional[int]:
       SCREAMING_SNAKE_CASE__  :					str						=FlaxVisionTextDualEncoderModel.from_pretrained('''clip-italian/clip-italian'''       ,   logit_scale_init_value=1.0   )
       SCREAMING_SNAKE_CASE__  :					Dict						=VisionTextDualEncoderProcessor.from_pretrained('''clip-italian/clip-italian'''   )

       SCREAMING_SNAKE_CASE__  :					List[Any]						=Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''   )
       SCREAMING_SNAKE_CASE__  :					List[str]						=processor(
           text=['''una foto di un gatto''', '''una foto di un cane''']       ,   images=__lowercase       ,   padding=__lowercase       ,   return_tensors='''np'''   )

       SCREAMING_SNAKE_CASE__  :					Tuple						=model(**__lowercase   )

       # verify the logits
       self.assertEqual(outputs.logits_per_image.shape       ,   (inputs.pixel_values.shape[0], inputs.input_ids.shape[0])   )
       self.assertEqual(
           outputs.logits_per_text.shape       ,   (inputs.input_ids.shape[0], inputs.pixel_values.shape[0])       ,   )

       SCREAMING_SNAKE_CASE__  :					Any						=np.array([[1.2284727, 0.3104122]]   )

       self.assertTrue(np.allclose(outputs.logits_per_image       ,   __lowercase       ,   atol=1e-3   )   )






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						UpperCamelCase_  (  snake_case__  ,     unittest.TestCase							):
					_a      :						Optional[Any]						    =					CodeGenTokenizer
					_a      :						Optional[Any]						    =					CodeGenTokenizerFast
					_a      :						Union[str, Any]						    =					True
					_a      :						Dict						    =					{'add_prefix_space': True}
					_a      :						Dict						    =					False



					def   __a     (		self     :     Union[str, Any]						):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							lowerCamelCase_	:     Optional[Any] 						=      [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							    '<|endoftext|>',
							]
							lowerCamelCase_	:     List[Any] 						=      dict(zip(lowerCamelCase       ,      range(len(lowerCamelCase						)						)						)						)
							lowerCamelCase_	:     Tuple 						=      ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							lowerCamelCase_	:     Optional[int] 						=      {'unk_token': '<unk>'}

							lowerCamelCase_	:     Optional[Any] 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['vocab_file']						)
							lowerCamelCase_	:     Tuple 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['merges_file']						)
							with open(self.vocab_file       ,      'w'       ,      encoding='utf-8'						) as fp:
									fp.write(json.dumps(lowerCamelCase						) + '\n'						)
							with open(self.merges_file       ,      'w'       ,      encoding='utf-8'						) as fp:
									fp.write('\n'.join(lowerCamelCase						)						)



					def   __a     (		self     :     Optional[int]       ,      **lowerCamelCase     :     Dict						):
							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     List[Any]       ,      **lowerCamelCase     :     int						):
							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     Tuple       ,      lowerCamelCase     :     List[str]						):
							lowerCamelCase_	:     List[str] 						=      'lower newer'
							lowerCamelCase_	:     str 						=      'lower newer'
							return input_text, output_text



					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     List[Any] 						=      CodeGenTokenizer(self.vocab_file       ,      self.merges_file       ,      **self.special_tokens_map						)
							lowerCamelCase_	:     Union[str, Any] 						=      'lower newer'
							lowerCamelCase_	:     List[str] 						=      ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
							lowerCamelCase_	:     Dict 						=      tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							lowerCamelCase_	:     Dict 						=      tokens + [tokenizer.unk_token]
							lowerCamelCase_	:     Dict 						=      [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      lowerCamelCase						)



					def   __a     (		self     :     str						):
							if not self.test_rust_tokenizer:
									return

							lowerCamelCase_	:     Dict 						=      self.get_tokenizer()
							lowerCamelCase_	:     Optional[Any] 						=      self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)

							lowerCamelCase_	:     int 						=      'lower newer'

							# Testing tokenization
							lowerCamelCase_	:     List[str] 						=      tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     List[Any] 						=      rust_tokenizer.tokenize(lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing conversion to ids without special tokens
							lowerCamelCase_	:     List[str] 						=      tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     str 						=      rust_tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing conversion to ids with special tokens
							lowerCamelCase_	:     Dict 						=      self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     Tuple 						=      tokenizer.encode(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     Union[str, Any] 						=      rust_tokenizer.encode(lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing the unknown token
							lowerCamelCase_	:     Optional[int] 						=      tokens + [rust_tokenizer.unk_token]
							lowerCamelCase_	:     int 						=      [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      lowerCamelCase						)



					def   __a     (		self     :     Tuple       ,      *lowerCamelCase     :     Dict       ,      **lowerCamelCase     :     Tuple						):
							# It's very difficult to mix/test pretokenization with byte-level
							# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
							pass



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     Any=15						):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
											lowerCamelCase_	:     Optional[int] 						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase       ,      **lowerCamelCase						)

											# Simple input
											lowerCamelCase_	:     Any 						=      'This is a simple input'
											lowerCamelCase_	:     Optional[int] 						=      ['This is a simple input 1', 'This is a simple input 2']
											lowerCamelCase_	:     Optional[Any] 						=      ('This is a simple input', 'This is a pair')
											lowerCamelCase_	:     str 						=      [
											    ('This is a simple input 1', 'This is a simple input 2'),
											    ('This is a simple pair 1', 'This is a simple pair 2'),
											]

											# Simple input tests
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Simple input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Simple input
											self.assertRaises(
											    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'       ,      )

											# Pair input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Pair input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Pair input
											self.assertRaises(
											    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'       ,      )



					def   __a     (		self     :     Optional[Any]						):
							lowerCamelCase_	:     Optional[int] 						=      CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      pad_token='<pad>'						)

							# Simple input
							lowerCamelCase_	:     Tuple 						=      'This is a simple input'
							lowerCamelCase_	:     Tuple 						=      ['This is a simple input looooooooong', 'This is a simple input']
							lowerCamelCase_	:     Any 						=      ('This is a simple input', 'This is a pair')
							lowerCamelCase_	:     Tuple 						=      [
							    ('This is a simple input loooooong', 'This is a simple input'),
							    ('This is a simple pair loooooong', 'This is a simple pair'),
							]

							lowerCamelCase_	:     str 						=      tokenizer.pad_token_id

							lowerCamelCase_	:     int 						=      tokenizer(lowerCamelCase       ,      padding='max_length'       ,      max_length=30       ,      return_tensors='np'						)
							lowerCamelCase_	:     List[Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors='np'						)
							lowerCamelCase_	:     Tuple 						=      tokenizer(*lowerCamelCase       ,      padding='max_length'       ,      max_length=60       ,      return_tensors='np'						)
							lowerCamelCase_	:     Union[str, Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors='np'						)

							# s
							# test single string max_length padding
							self.assertEqual(out_s['input_ids'].shape[-1]       ,      30						)
							self.assertTrue(pad_token_id in out_s['input_ids']						)
							self.assertTrue(0 in out_s['attention_mask']						)

							# s2
							# test automatic padding
							self.assertEqual(out_sa['input_ids'].shape[-1]       ,      33						)
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa['input_ids'][0]						)
							self.assertFalse(0 in out_sa['attention_mask'][0]						)
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa['input_ids'][1]						)
							self.assertTrue(0 in out_sa['attention_mask'][1]						)

							# p
							# test single pair max_length padding
							self.assertEqual(out_p['input_ids'].shape[-1]       ,      60						)
							self.assertTrue(pad_token_id in out_p['input_ids']						)
							self.assertTrue(0 in out_p['attention_mask']						)

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa['input_ids'].shape[-1]       ,      52						)
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa['input_ids'][0]						)
							self.assertFalse(0 in out_pa['attention_mask'][0]						)
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa['input_ids'][1]						)
							self.assertTrue(0 in out_pa['attention_mask'][1]						)



					def   __a     (		self     :     str						):
							lowerCamelCase_	:     List[str] 						=      '$$$'
							lowerCamelCase_	:     Dict 						=      CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      bos_token=lowerCamelCase       ,      add_bos_token=lowerCamelCase						)

							lowerCamelCase_	:     Optional[Any] 						=      'This is a simple input'
							lowerCamelCase_	:     Optional[Any] 						=      ['This is a simple input 1', 'This is a simple input 2']

							lowerCamelCase_	:     List[str] 						=      tokenizer.bos_token_id

							lowerCamelCase_	:     Optional[Any] 						=      tokenizer(lowerCamelCase						)
							lowerCamelCase_	:     Optional[int] 						=      tokenizer(lowerCamelCase						)

							self.assertEqual(out_s.input_ids[0]       ,      lowerCamelCase						)
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

							lowerCamelCase_	:     str 						=      tokenizer.decode(out_s.input_ids						)
							lowerCamelCase_	:     Any 						=      tokenizer.batch_decode(out_sa.input_ids						)

							self.assertEqual(decode_s.split()[0]       ,      lowerCamelCase						)
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)



					@slow
					def   __a     (		self     :     List[str]						):
							lowerCamelCase_	:     Optional[int] 						=      CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'						)

							lowerCamelCase_	:     Tuple 						=      '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
							lowerCamelCase_	:     Any 						=      '\nif len_a > len_b:      result = a\nelse:      result = b'

							lowerCamelCase_	:     int 						=      tokenizer.encode(lowerCamelCase						)
							lowerCamelCase_	:     Optional[int] 						=      ['^#', re.escape('<|endoftext|>'						), '^\'\'\'', '^"""', '\n\n\n']
							lowerCamelCase_	:     str 						=      tokenizer.decode(lowerCamelCase       ,      truncate_before_pattern=lowerCamelCase						)
							self.assertEqual(lowerCamelCase       ,      lowerCamelCase						)





					def   __a     (		self     :     Optional[Any]						):
							pass
