



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class     _UpperCAmelCase			(       A__			,      unittest.TestCase     ):
     UpperCamelCase__		    =				CodeGenTokenizer
     UpperCamelCase__		    =				CodeGenTokenizerFast
     UpperCamelCase__		    =				True
     UpperCamelCase__		    =				{'''add_prefix_space''': True}
     UpperCamelCase__		    =				False




     def 		snake_case_		(				self):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         A__		 =					[
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''\u0120''',
             '''\u0120l''',
             '''\u0120n''',
             '''\u0120lo''',
             '''\u0120low''',
             '''er''',
             '''\u0120lowest''',
             '''\u0120newer''',
             '''\u0120wider''',
             '''<unk>''',
             '''<|endoftext|>''',
         ]
         A__		 =					dict(zip(a__ ,						range(len(a__))))
         A__		 =					['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
         A__		 =					{'''unk_token''': '''<unk>'''}

         A__		 =					os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''vocab_file'''])
         A__		 =					os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''merges_file'''])
         with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8''') as fp:
             fp.write(json.dumps(a__) + '''\n''')
         with open(self.merges_file ,						'''w''' ,						encoding='''utf-8''') as fp:
             fp.write('''\n'''.join(a__))




     def 		snake_case_		(				self ,						**a__):
         kwargs.update(self.special_tokens_map)
         return CodeGenTokenizer.from_pretrained(self.tmpdirname ,						**a__)




     def 		snake_case_		(				self ,						**a__):
         kwargs.update(self.special_tokens_map)
         return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,						**a__)




     def 		snake_case_		(				self ,						a__):
         A__		 =					'''lower newer'''
         A__		 =					'''lower newer'''
         return input_text, output_text




     def 		snake_case_		(				self):
         A__		 =					CodeGenTokenizer(self.vocab_file ,						self.merges_file ,						**self.special_tokens_map)
         A__		 =					'''lower newer'''
         A__		 =					['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
         A__		 =					tokenizer.tokenize(a__ ,						add_prefix_space=a__)
         self.assertListEqual(a__ ,						a__)

         A__		 =					tokens + [tokenizer.unk_token]
         A__		 =					[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__) ,						a__)




     def 		snake_case_		(				self):
         if not self.test_rust_tokenizer:
             return

         A__		 =					self.get_tokenizer()
         A__		 =					self.get_rust_tokenizer(add_prefix_space=a__)

         A__		 =					'''lower newer'''

         # Testing tokenization
         A__		 =					tokenizer.tokenize(a__ ,						add_prefix_space=a__)
         A__		 =					rust_tokenizer.tokenize(a__)
         self.assertListEqual(a__ ,						a__)

         # Testing conversion to ids without special tokens
         A__		 =					tokenizer.encode(a__ ,						add_special_tokens=a__ ,						add_prefix_space=a__)
         A__		 =					rust_tokenizer.encode(a__ ,						add_special_tokens=a__)
         self.assertListEqual(a__ ,						a__)

         # Testing conversion to ids with special tokens
         A__		 =					self.get_rust_tokenizer(add_prefix_space=a__)
         A__		 =					tokenizer.encode(a__ ,						add_prefix_space=a__)
         A__		 =					rust_tokenizer.encode(a__)
         self.assertListEqual(a__ ,						a__)

         # Testing the unknown token
         A__		 =					tokens + [rust_tokenizer.unk_token]
         A__		 =					[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a__) ,						a__)




     def 		snake_case_		(				self ,						*a__ ,						**a__):
         # It's very difficult to mix/test pretokenization with byte-level
         # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
         pass




     def 		snake_case_		(				self ,						a__=1_5):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 A__		 =					self.rust_tokenizer_class.from_pretrained(a__ ,						**a__)

                 # Simple input
                 A__		 =					'''This is a simple input'''
                 A__		 =					['''This is a simple input 1''', '''This is a simple input 2''']
                 A__		 =					('''This is a simple input''', '''This is a pair''')
                 A__		 =					[
                     ('''This is a simple input 1''', '''This is a simple input 2'''),
                     ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                 ]

                 # Simple input tests
                 self.assertRaises(a__ ,						tokenizer_r.encode ,						a__ ,						max_length=a__ ,						padding='''max_length''')

                 # Simple input
                 self.assertRaises(a__ ,						tokenizer_r.encode_plus ,						a__ ,						max_length=a__ ,						padding='''max_length''')

                 # Simple input
                 self.assertRaises(
                     a__ ,						tokenizer_r.batch_encode_plus ,						a__ ,						max_length=a__ ,						padding='''max_length''' ,						)

                 # Pair input
                 self.assertRaises(a__ ,						tokenizer_r.encode ,						a__ ,						max_length=a__ ,						padding='''max_length''')

                 # Pair input
                 self.assertRaises(a__ ,						tokenizer_r.encode_plus ,						a__ ,						max_length=a__ ,						padding='''max_length''')

                 # Pair input
                 self.assertRaises(
                     a__ ,						tokenizer_r.batch_encode_plus ,						a__ ,						max_length=a__ ,						padding='''max_length''' ,						)




     def 		snake_case_		(				self):
         A__		 =					CodeGenTokenizer.from_pretrained(self.tmpdirname ,						pad_token='''<pad>''')

         # Simple input
         A__		 =					'''This is a simple input'''
         A__		 =					['''This is a simple input looooooooong''', '''This is a simple input''']
         A__		 =					('''This is a simple input''', '''This is a pair''')
         A__		 =					[
             ('''This is a simple input loooooong''', '''This is a simple input'''),
             ('''This is a simple pair loooooong''', '''This is a simple pair'''),
         ]

         A__		 =					tokenizer.pad_token_id

         A__		 =					tokenizer(a__ ,						padding='''max_length''' ,						max_length=3_0 ,						return_tensors='''np''')
         A__		 =					tokenizer(a__ ,						padding=a__ ,						truncate=a__ ,						return_tensors='''np''')
         A__		 =					tokenizer(*a__ ,						padding='''max_length''' ,						max_length=6_0 ,						return_tensors='''np''')
         A__		 =					tokenizer(a__ ,						padding=a__ ,						truncate=a__ ,						return_tensors='''np''')

         # s
         # test single string max_length padding
         self.assertEqual(out_s['''input_ids'''].shape[-1] ,						3_0)
         self.assertTrue(pad_token_id in out_s['''input_ids'''])
         self.assertTrue(0 in out_s['''attention_mask'''])

         # s2
         # test automatic padding
         self.assertEqual(out_sa['''input_ids'''].shape[-1] ,						3_3)
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa['''input_ids'''][0])
         self.assertFalse(0 in out_sa['''attention_mask'''][0])
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa['''input_ids'''][1])
         self.assertTrue(0 in out_sa['''attention_mask'''][1])

         # p
         # test single pair max_length padding
         self.assertEqual(out_p['''input_ids'''].shape[-1] ,						6_0)
         self.assertTrue(pad_token_id in out_p['''input_ids'''])
         self.assertTrue(0 in out_p['''attention_mask'''])

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa['''input_ids'''].shape[-1] ,						5_2)
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa['''input_ids'''][0])
         self.assertFalse(0 in out_pa['''attention_mask'''][0])
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa['''input_ids'''][1])
         self.assertTrue(0 in out_pa['''attention_mask'''][1])




     def 		snake_case_		(				self):
         A__		 =					'''$$$'''
         A__		 =					CodeGenTokenizer.from_pretrained(self.tmpdirname ,						bos_token=a__ ,						add_bos_token=a__)

         A__		 =					'''This is a simple input'''
         A__		 =					['''This is a simple input 1''', '''This is a simple input 2''']

         A__		 =					tokenizer.bos_token_id

         A__		 =					tokenizer(a__)
         A__		 =					tokenizer(a__)

         self.assertEqual(out_s.input_ids[0] ,						a__)
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

         A__		 =					tokenizer.decode(out_s.input_ids)
         A__		 =					tokenizer.batch_decode(out_sa.input_ids)

         self.assertEqual(decode_s.split()[0] ,						a__)
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))




     @slow
     def 		snake_case_		(				self):
         A__		 =					CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono''')

         A__		 =					'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
         A__		 =					'''\nif len_a > len_b:      result = a\nelse:      result = b'''

         A__		 =					tokenizer.encode(a__)
         A__		 =					['''^#''', re.escape('''<|endoftext|>'''), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
         A__		 =					tokenizer.decode(a__ ,						truncate_before_pattern=a__)
         self.assertEqual(a__ ,						a__)




     def 		snake_case_		(				self):
         pass





# Copyright 2022 The HuggingFace Team and The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_lowercase						  =		{
    "configuration_cpmant": ["CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP", "CpmAntConfig"],
    "tokenization_cpmant": ["CpmAntTokenizer"],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase						  =		[
       "CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "CpmAntForCausalLM",
       "CpmAntModel",
       "CpmAntPreTrainedModel",
   ]


if TYPE_CHECKING:
   from .configuration_cpmant import CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP, CpmAntConfig
   from .tokenization_cpmant import CpmAntTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_cpmant import (
          CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST,
          CpmAntForCausalLM,
          CpmAntModel,
          CpmAntPreTrainedModel,
      )


else:
   import sys

   _lowercase						  =		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
