





"""simple docstring"""
import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class lowerCamelCase__	(     UpperCamelCase_	,					unittest.TestCase					):
   __UpperCAmelCase										=       CodeGenTokenizer
   __UpperCAmelCase										=       CodeGenTokenizerFast
   __UpperCAmelCase										=       True
   __UpperCAmelCase										=       {'add_prefix_space': True}
   __UpperCAmelCase										=       False



   def  _UpperCAmelCase						(							self						)			->     Union[str, Any]:







      """simple docstring"""
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      lowercase     :      Any	=							[
          'l',
          'o',
          'w',
          'e',
          'r',
          's',
          't',
          'i',
          'd',
          'n',
          '\u0120',
          '\u0120l',
          '\u0120n',
          '\u0120lo',
          '\u0120low',
          'er',
          '\u0120lowest',
          '\u0120newer',
          '\u0120wider',
          '<unk>',
          '<|endoftext|>',
      ]
      lowercase     :      Dict	=							dict(zip(lowercase_   , range(len(lowercase_						)						)						)						)
      lowercase     :      Tuple	=							['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
      lowercase     :      Optional[int]	=							{'unk_token': '<unk>'}

      lowercase     :      List[Any]	=							os.path.join(self.tmpdirname   , VOCAB_FILES_NAMES["""vocab_file"""]						)
      lowercase     :      Union[str, Any]	=							os.path.join(self.tmpdirname   , VOCAB_FILES_NAMES["""merges_file"""]						)
      with open(self.vocab_file   , """w"""   , encoding="""utf-8"""						) as fp:
         fp.write(json.dumps(lowercase_						) + """\n"""						)
      with open(self.merges_file   , """w"""   , encoding="""utf-8"""						) as fp:
         fp.write("""\n""".join(lowercase_						)						)



   def  _UpperCAmelCase						(							self   , **snake_case						)			->     Optional[Any]:







      """simple docstring"""
      kwargs.update(self.special_tokens_map						)
      return CodeGenTokenizer.from_pretrained(self.tmpdirname   , **lowercase_						)



   def  _UpperCAmelCase						(							self   , **snake_case						)			->     Tuple:







      """simple docstring"""
      kwargs.update(self.special_tokens_map						)
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname   , **lowercase_						)



   def  _UpperCAmelCase						(							self   , snake_case						)			->     Optional[int]:







      """simple docstring"""
      lowercase     :      Dict	=							'lower newer'
      lowercase     :      List[str]	=							'lower newer'
      return input_text, output_text



   def  _UpperCAmelCase						(							self						)			->     int:







      """simple docstring"""
      lowercase     :      List[Any]	=							CodeGenTokenizer(self.vocab_file   , self.merges_file   , **self.special_tokens_map						)
      lowercase     :      Tuple	=							'lower newer'
      lowercase     :      str	=							['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
      lowercase     :      Tuple	=							tokenizer.tokenize(lowercase_   , add_prefix_space=lowercase_						)
      self.assertListEqual(lowercase_   , lowercase_						)

      lowercase     :      Optional[int]	=							tokens + [tokenizer.unk_token]
      lowercase     :      Union[str, Any]	=							[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_						)   , lowercase_						)



   def  _UpperCAmelCase						(							self						)			->     int:







      """simple docstring"""




      if not self.test_rust_tokenizer:
         return

      lowercase     :      Tuple	=							self.get_tokenizer()
      lowercase     :      Dict	=							self.get_rust_tokenizer(add_prefix_space=lowercase_						)

      lowercase     :      Union[str, Any]	=							'lower newer'

      # Testing tokenization
      lowercase     :      Optional[Any]	=							tokenizer.tokenize(lowercase_   , add_prefix_space=lowercase_						)
      lowercase     :      List[str]	=							rust_tokenizer.tokenize(lowercase_						)
      self.assertListEqual(lowercase_   , lowercase_						)

      # Testing conversion to ids without special tokens
      lowercase     :      int	=							tokenizer.encode(lowercase_   , add_special_tokens=lowercase_   , add_prefix_space=lowercase_						)
      lowercase     :      List[str]	=							rust_tokenizer.encode(lowercase_   , add_special_tokens=lowercase_						)
      self.assertListEqual(lowercase_   , lowercase_						)

      # Testing conversion to ids with special tokens
      lowercase     :      Any	=							self.get_rust_tokenizer(add_prefix_space=lowercase_						)
      lowercase     :      Dict	=							tokenizer.encode(lowercase_   , add_prefix_space=lowercase_						)
      lowercase     :      List[str]	=							rust_tokenizer.encode(lowercase_						)
      self.assertListEqual(lowercase_   , lowercase_						)

      # Testing the unknown token
      lowercase     :      List[str]	=							tokens + [rust_tokenizer.unk_token]
      lowercase     :      List[Any]	=							[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_						)   , lowercase_						)



   def  _UpperCAmelCase						(							self   , *snake_case   , **snake_case						)			->     str:







      """simple docstring"""
      pass



   def  _UpperCAmelCase						(							self   , snake_case=1_5						)			->     List[Any]:







      """simple docstring"""




      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
            lowercase     :      List[Any]	=							self.rust_tokenizer_class.from_pretrained(lowercase_   , **lowercase_						)

            # Simple input
            lowercase     :      Optional[Any]	=							'This is a simple input'
            lowercase     :      Optional[Any]	=							['This is a simple input 1', 'This is a simple input 2']
            lowercase     :      Tuple	=							('This is a simple input', 'This is a pair')
            lowercase     :      Optional[Any]	=							[
                ('This is a simple input 1', 'This is a simple input 2'),
                ('This is a simple pair 1', 'This is a simple pair 2'),
            ]

            # Simple input tests
            self.assertRaises(lowercase_   , tokenizer_r.encode   , lowercase_   , max_length=lowercase_   , padding="""max_length"""						)

            # Simple input
            self.assertRaises(lowercase_   , tokenizer_r.encode_plus   , lowercase_   , max_length=lowercase_   , padding="""max_length"""						)

            # Simple input
            self.assertRaises(
                lowercase_   , tokenizer_r.batch_encode_plus   , lowercase_   , max_length=lowercase_   , padding="""max_length"""   , )

            # Pair input
            self.assertRaises(lowercase_   , tokenizer_r.encode   , lowercase_   , max_length=lowercase_   , padding="""max_length"""						)

            # Pair input
            self.assertRaises(lowercase_   , tokenizer_r.encode_plus   , lowercase_   , max_length=lowercase_   , padding="""max_length"""						)

            # Pair input
            self.assertRaises(
                lowercase_   , tokenizer_r.batch_encode_plus   , lowercase_   , max_length=lowercase_   , padding="""max_length"""   , )



   def  _UpperCAmelCase						(							self						)			->     Union[str, Any]:







      """simple docstring"""
      lowercase     :      List[Any]	=							CodeGenTokenizer.from_pretrained(self.tmpdirname   , pad_token="""<pad>"""						)

      # Simple input
      lowercase     :      Dict	=							'This is a simple input'
      lowercase     :      Optional[int]	=							['This is a simple input looooooooong', 'This is a simple input']
      lowercase     :      List[Any]	=							('This is a simple input', 'This is a pair')
      lowercase     :      Tuple	=							[
          ('This is a simple input loooooong', 'This is a simple input'),
          ('This is a simple pair loooooong', 'This is a simple pair'),
      ]

      lowercase     :      str	=							tokenizer.pad_token_id

      lowercase     :      List[Any]	=							tokenizer(lowercase_   , padding="""max_length"""   , max_length=3_0   , return_tensors="""np"""						)
      lowercase     :      Tuple	=							tokenizer(lowercase_   , padding=lowercase_   , truncate=lowercase_   , return_tensors="""np"""						)
      lowercase     :      Any	=							tokenizer(*lowercase_   , padding="""max_length"""   , max_length=6_0   , return_tensors="""np"""						)
      lowercase     :      Any	=							tokenizer(lowercase_   , padding=lowercase_   , truncate=lowercase_   , return_tensors="""np"""						)

      # s
      # test single string max_length padding
      self.assertEqual(out_s["""input_ids"""].shape[-1]   , 3_0						)
      self.assertTrue(pad_token_id in out_s["""input_ids"""]						)
      self.assertTrue(0 in out_s["""attention_mask"""]						)

      # s2
      # test automatic padding
      self.assertEqual(out_sa["""input_ids"""].shape[-1]   , 3_3						)
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]						)
      self.assertFalse(0 in out_sa["""attention_mask"""][0]						)
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]						)
      self.assertTrue(0 in out_sa["""attention_mask"""][1]						)

      # p
      # test single pair max_length padding
      self.assertEqual(out_p["""input_ids"""].shape[-1]   , 6_0						)
      self.assertTrue(pad_token_id in out_p["""input_ids"""]						)
      self.assertTrue(0 in out_p["""attention_mask"""]						)

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa["""input_ids"""].shape[-1]   , 5_2						)
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]						)
      self.assertFalse(0 in out_pa["""attention_mask"""][0]						)
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]						)
      self.assertTrue(0 in out_pa["""attention_mask"""][1]						)



   def  _UpperCAmelCase						(							self						)			->     Any:







      """simple docstring"""
      lowercase     :      int	=							'$$$'
      lowercase     :      Union[str, Any]	=							CodeGenTokenizer.from_pretrained(self.tmpdirname   , bos_token=lowercase_   , add_bos_token=lowercase_						)

      lowercase     :      Optional[Any]	=							'This is a simple input'
      lowercase     :      Union[str, Any]	=							['This is a simple input 1', 'This is a simple input 2']

      lowercase     :      Union[str, Any]	=							tokenizer.bos_token_id

      lowercase     :      Optional[int]	=							tokenizer(lowercase_						)
      lowercase     :      List[str]	=							tokenizer(lowercase_						)

      self.assertEqual(out_s.input_ids[0]   , lowercase_						)
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

      lowercase     :      List[Any]	=							tokenizer.decode(out_s.input_ids						)
      lowercase     :      Optional[Any]	=							tokenizer.batch_decode(out_sa.input_ids						)

      self.assertEqual(decode_s.split()[0]   , lowercase_						)
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)



   @slow
   def  _UpperCAmelCase						(							self						)			->     Optional[int]:







      """simple docstring"""
      lowercase     :      List[Any]	=							CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""						)

      lowercase     :      Union[str, Any]	=							'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
      lowercase     :      Optional[int]	=							'\nif len_a > len_b:      result = a\nelse:      result = b'

      lowercase     :      List[str]	=							tokenizer.encode(lowercase_						)
      lowercase     :      str	=							['^#', re.escape("""<|endoftext|>"""						), '^\'\'\'', '^"""', '\n\n\n']
      lowercase     :      str	=							tokenizer.decode(lowercase_   , truncate_before_pattern=lowercase_						)
      self.assertEqual(lowercase_   , lowercase_						)





   def  _UpperCAmelCase						(							self						)			->     Optional[int]:







      """simple docstring"""
      pass






from timeit import timeit

lowerCamelCase		   =			{
    'MALAYALAM': True,
    'String': False,
    'rotor': True,
    'level': True,
    'A': True,
    'BB': True,
    'ABC': False,
    'amanaplanacanalpanama': True,  # "a man a plan a canal panama"
}
# Ensure our test data is valid
assert all((key == key[::-1]) is value for key, value in test_data.items())




def       a_ (       SCREAMING_SNAKE_CASE__   : str	):
       '''simple docstring'''





       _lowerCamelCase			:						Optional[int]            =0
       _lowerCamelCase			:						Union[str, Any]            =len(SCREAMING_SNAKE_CASE__	) - 1
       while start_i < end_i:
              if s[start_i] == s[end_i]:
                     start_i += 1
                     end_i -= 1
              else:
                     return False
       return True




def       a_ (       SCREAMING_SNAKE_CASE__   : str	):
       '''simple docstring'''





       _lowerCamelCase			:						List[str]            =len(SCREAMING_SNAKE_CASE__	) // 2
       _lowerCamelCase			:						Optional[Any]            =len(SCREAMING_SNAKE_CASE__	)

       # We need to traverse till half of the length of string
       # as we can get access of the i'th last element from
       # i'th index.
       # eg: [0,1,2,3,4,5] => 4th index can be accessed
       # with the help of 1st index (i==n-i-1)
       # where n is length of string
       return all(s[i] == s[n - i - 1] for i in range(SCREAMING_SNAKE_CASE__	)	)




def       a_ (       SCREAMING_SNAKE_CASE__   : str	):
       '''simple docstring'''





       if len(SCREAMING_SNAKE_CASE__	) <= 2:
              return True
       if s[0] == s[len(SCREAMING_SNAKE_CASE__	) - 1]:
              return is_palindrome_recursive(s[1:-1]	)
       else:
              return False




def       a_ (       SCREAMING_SNAKE_CASE__   : str	):
       '''simple docstring'''





       return s == s[::-1]




def       a_ (       SCREAMING_SNAKE_CASE__   : str	):
       '''simple docstring'''





       _lowerCamelCase			:						Optional[int]            =F'''all({name}(key) is value for key, value in test_data.items())'''
       _lowerCamelCase			:						List[Any]            =F'''from __main__ import test_data, {name}'''
       _lowerCamelCase			:						Any            =500_000
       _lowerCamelCase			:						Dict            =timeit(stmt=SCREAMING_SNAKE_CASE__						,     setup=SCREAMING_SNAKE_CASE__						,     number=SCREAMING_SNAKE_CASE__	)
       print(F'''{name:<35} finished {number:,} runs in {result:.5f} seconds'''	)


if __name__ == "__main__":
 for key, value in test_data.items():
  assert is_palindrome(key) is is_palindrome_recursive(key)
  assert is_palindrome(key) is is_palindrome_slice(key)
  print(F"""{key:21} {value}""")
 print('a man a plan a canal panama')

 # finished 500,000 runs in 0.46793 seconds
 benchmark_function('is_palindrome_slice')
 # finished 500,000 runs in 0.85234 seconds
 benchmark_function('is_palindrome')
 # finished 500,000 runs in 1.32028 seconds
 benchmark_function('is_palindrome_recursive')
 # finished 500,000 runs in 2.08679 seconds
 benchmark_function('is_palindrome_traversal')


