

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 				__UpperCamelCase				(					_A  ,						unittest.TestCase			):
 _lowercase  :   List[str]       				=  CodeGenTokenizer
 _lowercase  :   Optional[Any]       				=  CodeGenTokenizerFast
 _lowercase  :   Tuple       				=  True
 _lowercase  :   Union[str, Any]       				=  {"add_prefix_space": True}
 _lowercase  :   Dict       				=  False







 def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

      '''simple docstring'''





      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      __magic_name__							=    [
          "l",
          "o",
          "w",
          "e",
          "r",
          "s",
          "t",
          "i",
          "d",
          "n",
          "\u0120",
          "\u0120l",
          "\u0120n",
          "\u0120lo",
          "\u0120low",
          "er",
          "\u0120lowest",
          "\u0120newer",
          "\u0120wider",
          "<unk>",
          "<|endoftext|>",
      ]
      __magic_name__							=    dict(zip(__UpperCamelCase  , range(len(__UpperCamelCase      )      )      )      )
      __magic_name__							=    ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
      __magic_name__							=    {"unk_token": "<unk>"}

      __magic_name__							=    os.path.join(self.tmpdirname  , VOCAB_FILES_NAMES['vocab_file']      )
      __magic_name__							=    os.path.join(self.tmpdirname  , VOCAB_FILES_NAMES['merges_file']      )
      with open(self.vocab_file  , 'w'  , encoding='utf-8'      ) as fp:
           fp.write(json.dumps(__UpperCamelCase      ) + '\n'      )
      with open(self.merges_file  , 'w'  , encoding='utf-8'      ) as fp:
           fp.write('\n'.join(__UpperCamelCase      )      )







 def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]  , **__UpperCamelCase: List[Any]      ):

      '''simple docstring'''





      kwargs.update(self.special_tokens_map      )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname  , **__UpperCamelCase      )







 def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , **__UpperCamelCase: int      ):

      '''simple docstring'''





      kwargs.update(self.special_tokens_map      )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname  , **__UpperCamelCase      )







 def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]  , __UpperCamelCase: List[str]      ):

      '''simple docstring'''





      __magic_name__							=    "lower newer"
      __magic_name__							=    "lower newer"
      return input_text, output_text







 def 	_SCREAMING_SNAKE_CASE							(   self: List[str]      ):

      '''simple docstring'''





      __magic_name__							=    CodeGenTokenizer(self.vocab_file  , self.merges_file  , **self.special_tokens_map      )
      __magic_name__							=    "lower newer"
      __magic_name__							=    ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
      __magic_name__							=    tokenizer.tokenize(__UpperCamelCase  , add_prefix_space=__UpperCamelCase      )
      self.assertListEqual(__UpperCamelCase  , __UpperCamelCase      )

      __magic_name__							=    tokens + [tokenizer.unk_token]
      __magic_name__							=    [14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase      )  , __UpperCamelCase      )







 def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

      '''simple docstring'''





      if not self.test_rust_tokenizer:
           return

      __magic_name__							=    self.get_tokenizer()
      __magic_name__							=    self.get_rust_tokenizer(add_prefix_space=__UpperCamelCase      )

      __magic_name__							=    "lower newer"

      # Testing tokenization
      __magic_name__							=    tokenizer.tokenize(__UpperCamelCase  , add_prefix_space=__UpperCamelCase      )
      __magic_name__							=    rust_tokenizer.tokenize(__UpperCamelCase      )
      self.assertListEqual(__UpperCamelCase  , __UpperCamelCase      )

      # Testing conversion to ids without special tokens
      __magic_name__							=    tokenizer.encode(__UpperCamelCase  , add_special_tokens=__UpperCamelCase  , add_prefix_space=__UpperCamelCase      )
      __magic_name__							=    rust_tokenizer.encode(__UpperCamelCase  , add_special_tokens=__UpperCamelCase      )
      self.assertListEqual(__UpperCamelCase  , __UpperCamelCase      )

      # Testing conversion to ids with special tokens
      __magic_name__							=    self.get_rust_tokenizer(add_prefix_space=__UpperCamelCase      )
      __magic_name__							=    tokenizer.encode(__UpperCamelCase  , add_prefix_space=__UpperCamelCase      )
      __magic_name__							=    rust_tokenizer.encode(__UpperCamelCase      )
      self.assertListEqual(__UpperCamelCase  , __UpperCamelCase      )

      # Testing the unknown token
      __magic_name__							=    tokens + [rust_tokenizer.unk_token]
      __magic_name__							=    [14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__UpperCamelCase      )  , __UpperCamelCase      )







 def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , *__UpperCamelCase: Tuple  , **__UpperCamelCase: Optional[Any]      ):

      '''simple docstring'''





      pass







 def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , __UpperCamelCase: str=15      ):

      '''simple docstring'''





      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                __magic_name__							=    self.rust_tokenizer_class.from_pretrained(__UpperCamelCase  , **__UpperCamelCase      )

                # Simple input
                __magic_name__							=    "This is a simple input"
                __magic_name__							=    ["This is a simple input 1", "This is a simple input 2"]
                __magic_name__							=    ("This is a simple input", "This is a pair")
                __magic_name__							=    [
                    ("This is a simple input 1", "This is a simple input 2"),
                    ("This is a simple pair 1", "This is a simple pair 2"),
                ]

                # Simple input tests
                self.assertRaises(__UpperCamelCase  , tokenizer_r.encode  , __UpperCamelCase  , max_length=__UpperCamelCase  , padding='max_length'      )

                # Simple input
                self.assertRaises(__UpperCamelCase  , tokenizer_r.encode_plus  , __UpperCamelCase  , max_length=__UpperCamelCase  , padding='max_length'      )

                # Simple input
                self.assertRaises(
                    __UpperCamelCase  , tokenizer_r.batch_encode_plus  , __UpperCamelCase  , max_length=__UpperCamelCase  , padding='max_length'  , )

                # Pair input
                self.assertRaises(__UpperCamelCase  , tokenizer_r.encode  , __UpperCamelCase  , max_length=__UpperCamelCase  , padding='max_length'      )

                # Pair input
                self.assertRaises(__UpperCamelCase  , tokenizer_r.encode_plus  , __UpperCamelCase  , max_length=__UpperCamelCase  , padding='max_length'      )

                # Pair input
                self.assertRaises(
                    __UpperCamelCase  , tokenizer_r.batch_encode_plus  , __UpperCamelCase  , max_length=__UpperCamelCase  , padding='max_length'  , )







 def 	_SCREAMING_SNAKE_CASE							(   self: Any      ):

      '''simple docstring'''





      __magic_name__							=    CodeGenTokenizer.from_pretrained(self.tmpdirname  , pad_token='<pad>'      )

      # Simple input
      __magic_name__							=    "This is a simple input"
      __magic_name__							=    ["This is a simple input looooooooong", "This is a simple input"]
      __magic_name__							=    ("This is a simple input", "This is a pair")
      __magic_name__							=    [
          ("This is a simple input loooooong", "This is a simple input"),
          ("This is a simple pair loooooong", "This is a simple pair"),
      ]

      __magic_name__							=    tokenizer.pad_token_id

      __magic_name__							=    tokenizer(__UpperCamelCase  , padding='max_length'  , max_length=30  , return_tensors='np'      )
      __magic_name__							=    tokenizer(__UpperCamelCase  , padding=__UpperCamelCase  , truncate=__UpperCamelCase  , return_tensors='np'      )
      __magic_name__							=    tokenizer(*__UpperCamelCase  , padding='max_length'  , max_length=60  , return_tensors='np'      )
      __magic_name__							=    tokenizer(__UpperCamelCase  , padding=__UpperCamelCase  , truncate=__UpperCamelCase  , return_tensors='np'      )

      # s
      # test single string max_length padding
      self.assertEqual(out_s['input_ids'].shape[-1]  , 30      )
      self.assertTrue(pad_token_id in out_s['input_ids']      )
      self.assertTrue(0 in out_s['attention_mask']      )

      # s2
      # test automatic padding
      self.assertEqual(out_sa['input_ids'].shape[-1]  , 33      )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa['input_ids'][0]      )
      self.assertFalse(0 in out_sa['attention_mask'][0]      )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa['input_ids'][1]      )
      self.assertTrue(0 in out_sa['attention_mask'][1]      )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p['input_ids'].shape[-1]  , 60      )
      self.assertTrue(pad_token_id in out_p['input_ids']      )
      self.assertTrue(0 in out_p['attention_mask']      )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa['input_ids'].shape[-1]  , 52      )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa['input_ids'][0]      )
      self.assertFalse(0 in out_pa['attention_mask'][0]      )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa['input_ids'][1]      )
      self.assertTrue(0 in out_pa['attention_mask'][1]      )







 def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]      ):

      '''simple docstring'''





      __magic_name__							=    "$$$"
      __magic_name__							=    CodeGenTokenizer.from_pretrained(self.tmpdirname  , bos_token=__UpperCamelCase  , add_bos_token=__UpperCamelCase      )

      __magic_name__							=    "This is a simple input"
      __magic_name__							=    ["This is a simple input 1", "This is a simple input 2"]

      __magic_name__							=    tokenizer.bos_token_id

      __magic_name__							=    tokenizer(__UpperCamelCase      )
      __magic_name__							=    tokenizer(__UpperCamelCase      )

      self.assertEqual(out_s.input_ids[0]  , __UpperCamelCase      )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

      __magic_name__							=    tokenizer.decode(out_s.input_ids      )
      __magic_name__							=    tokenizer.batch_decode(out_sa.input_ids      )

      self.assertEqual(decode_s.split()[0]  , __UpperCamelCase      )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )







 @slow
 def 	_SCREAMING_SNAKE_CASE							(   self: Any      ):

      '''simple docstring'''





      __magic_name__							=    CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'      )

      __magic_name__							=    "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
      __magic_name__							=    "\nif len_a > len_b:      result = a\nelse:      result = b"

      __magic_name__							=    tokenizer.encode(__UpperCamelCase      )
      __magic_name__							=    ["^#", re.escape('<|endoftext|>'      ), "^'''", "^\"\"\"", "\n\n\n"]
      __magic_name__							=    tokenizer.decode(__UpperCamelCase  , truncate_before_pattern=__UpperCamelCase      )
      self.assertEqual(__UpperCamelCase  , __UpperCamelCase      )







 def 	_SCREAMING_SNAKE_CASE							(   self: str      ):

      '''simple docstring'''





      pass







import unittest

from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow


if is_flax_available():
						import jax

						from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
						from transformers.models.bert.modeling_flax_bert import FlaxBertModel
						from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel

@require_flax
class 				__UpperCamelCase				(					unittest.TestCase			):







	@slow
	def 	_SCREAMING_SNAKE_CASE							(   self: Any      ):

						'''simple docstring'''





						for model_name in ["bert-base-cased", "bert-large-uncased"]:
											with self.subTest(__UpperCamelCase      ):
																__magic_name__							=    AutoConfig.from_pretrained(__UpperCamelCase      )
																self.assertIsNotNone(__UpperCamelCase      )
																self.assertIsInstance(__UpperCamelCase  , __UpperCamelCase      )

																__magic_name__							=    FlaxAutoModel.from_pretrained(__UpperCamelCase      )
																self.assertIsNotNone(__UpperCamelCase      )
																self.assertIsInstance(__UpperCamelCase  , __UpperCamelCase      )







	@slow
	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple      ):

						'''simple docstring'''





						for model_name in ["roberta-base", "roberta-large"]:
											with self.subTest(__UpperCamelCase      ):
																__magic_name__							=    AutoConfig.from_pretrained(__UpperCamelCase      )
																self.assertIsNotNone(__UpperCamelCase      )
																self.assertIsInstance(__UpperCamelCase  , __UpperCamelCase      )

																__magic_name__							=    FlaxAutoModel.from_pretrained(__UpperCamelCase      )
																self.assertIsNotNone(__UpperCamelCase      )
																self.assertIsInstance(__UpperCamelCase  , __UpperCamelCase      )







	@slow
	def 	_SCREAMING_SNAKE_CASE							(   self: Dict      ):

						'''simple docstring'''





						for model_name in ["bert-base-cased", "bert-large-uncased"]:
											__magic_name__							=    AutoTokenizer.from_pretrained(__UpperCamelCase      )
											__magic_name__							=    FlaxBertModel.from_pretrained(__UpperCamelCase      )
											__magic_name__							=    tokenizer('Do you support jax jitted function?'  , return_tensors=TensorType.JAX      )

											@jax.jit
											def eval(**__UpperCamelCase: Optional[Any]      ):
																return model(**__UpperCamelCase      )

											eval(**__UpperCamelCase      ).block_until_ready()







	@slow
	def 	_SCREAMING_SNAKE_CASE							(   self: int      ):

						'''simple docstring'''





						for model_name in ["roberta-base", "roberta-large"]:
											__magic_name__							=    AutoTokenizer.from_pretrained(__UpperCamelCase      )
											__magic_name__							=    FlaxRobertaModel.from_pretrained(__UpperCamelCase      )
											__magic_name__							=    tokenizer('Do you support jax jitted function?'  , return_tensors=TensorType.JAX      )

											@jax.jit
											def eval(**__UpperCamelCase: Any      ):
																return model(**__UpperCamelCase      )

											eval(**__UpperCamelCase      ).block_until_ready()







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple      ):

						'''simple docstring'''





						with self.assertRaisesRegex(
						    __UpperCamelCase  , 'bert-base is not a local folder and is not a valid model identifier'      ):
											__magic_name__							=    FlaxAutoModel.from_pretrained('bert-base'      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						with self.assertRaisesRegex(
						    __UpperCamelCase  , r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'      ):
											__magic_name__							=    FlaxAutoModel.from_pretrained(__UpperCamelCase  , revision='aaaaaa'      )







	def 	_SCREAMING_SNAKE_CASE							(   self: str      ):

						'''simple docstring'''





						with self.assertRaisesRegex(
						    __UpperCamelCase  , 'hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack'  , ):
											__magic_name__							=    FlaxAutoModel.from_pretrained('hf-internal-testing/config-no-model'      )







	def 	_SCREAMING_SNAKE_CASE							(   self: int      ):

						'''simple docstring'''





						with self.assertRaisesRegex(__UpperCamelCase  , 'Use `from_pt=True` to load this model'      ):
											__magic_name__							=    FlaxAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'      )




