


'''simple docstring'''

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class     __lowerCAmelCase		(lowercase_    ,	unittest.TestCase	):





      '''simple docstring'''






      lowerCAmelCase__   :	Optional[int]       =				CodeGenTokenizer
      lowerCAmelCase__   :	Dict       =				CodeGenTokenizerFast
      lowerCAmelCase__   :	Union[str, Any]       =				True
      lowerCAmelCase__   :	List[Any]       =				{"""add_prefix_space""": True}
      lowerCAmelCase__   :	List[str]       =				False



      def  UpperCamelCase__						(self       :       Tuple					):

             '''simple docstring'''







             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             lowercase__      =					[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''\u0120''',
                 '''\u0120l''',
                 '''\u0120n''',
                 '''\u0120lo''',
                 '''\u0120low''',
                 '''er''',
                 '''\u0120lowest''',
                 '''\u0120newer''',
                 '''\u0120wider''',
                 '''<unk>''',
                 '''<|endoftext|>''',
             ]
             lowercase__      =					dict(zip(UpperCamelCase  ,		range(len(UpperCamelCase					)					)					)					)
             lowercase__      =					['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
             lowercase__      =					{'''unk_token''': '''<unk>'''}

             lowercase__      =					os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES['''vocab_file''']					)
             lowercase__      =					os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES['''merges_file''']					)
             with open(self.vocab_file  ,		'''w'''  ,		encoding='''utf-8'''					) as fp:
                    fp.write(json.dumps(UpperCamelCase					) + '''\n'''					)
             with open(self.merges_file  ,		'''w'''  ,		encoding='''utf-8'''					) as fp:
                    fp.write('''\n'''.join(UpperCamelCase					)					)



      def  UpperCamelCase__						(self       :       List[str]  ,		**UpperCamelCase       :       str					):

             '''simple docstring'''







             kwargs.update(self.special_tokens_map					)
             return CodeGenTokenizer.from_pretrained(self.tmpdirname  ,		**UpperCamelCase					)



      def  UpperCamelCase__						(self       :       int  ,		**UpperCamelCase       :       int					):

             '''simple docstring'''







             kwargs.update(self.special_tokens_map					)
             return CodeGenTokenizerFast.from_pretrained(self.tmpdirname  ,		**UpperCamelCase					)



      def  UpperCamelCase__						(self       :       List[Any]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					'''lower newer'''
             lowercase__      =					'''lower newer'''
             return input_text, output_text



      def  UpperCamelCase__						(self       :       Union[str, Any]					):

             '''simple docstring'''







             lowercase__      =					CodeGenTokenizer(self.vocab_file  ,		self.merges_file  ,		**self.special_tokens_map					)
             lowercase__      =					'''lower newer'''
             lowercase__      =					['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
             lowercase__      =					tokenizer.tokenize(UpperCamelCase  ,		add_prefix_space=UpperCamelCase					)
             self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

             lowercase__      =					tokens + [tokenizer.unk_token]
             lowercase__      =					[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase					)  ,		UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Optional[int]					):

             '''simple docstring'''







             if not self.test_rust_tokenizer:
                    return

             lowercase__      =					self.get_tokenizer()
             lowercase__      =					self.get_rust_tokenizer(add_prefix_space=UpperCamelCase					)

             lowercase__      =					'''lower newer'''

             # Testing tokenization
             lowercase__      =					tokenizer.tokenize(UpperCamelCase  ,		add_prefix_space=UpperCamelCase					)
             lowercase__      =					rust_tokenizer.tokenize(UpperCamelCase					)
             self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

             # Testing conversion to ids without special tokens
             lowercase__      =					tokenizer.encode(UpperCamelCase  ,		add_special_tokens=UpperCamelCase  ,		add_prefix_space=UpperCamelCase					)
             lowercase__      =					rust_tokenizer.encode(UpperCamelCase  ,		add_special_tokens=UpperCamelCase					)
             self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

             # Testing conversion to ids with special tokens
             lowercase__      =					self.get_rust_tokenizer(add_prefix_space=UpperCamelCase					)
             lowercase__      =					tokenizer.encode(UpperCamelCase  ,		add_prefix_space=UpperCamelCase					)
             lowercase__      =					rust_tokenizer.encode(UpperCamelCase					)
             self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

             # Testing the unknown token
             lowercase__      =					tokens + [rust_tokenizer.unk_token]
             lowercase__      =					[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase					)  ,		UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Union[str, Any]  ,		*UpperCamelCase       :       str  ,		**UpperCamelCase       :       Union[str, Any]					):

             '''simple docstring'''







             pass



      def  UpperCamelCase__						(self       :       Union[str, Any]  ,		UpperCamelCase       :       Optional[Any]=15					):

             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                           lowercase__      =					self.rust_tokenizer_class.from_pretrained(UpperCamelCase  ,		**UpperCamelCase					)

                           # Simple input
                           lowercase__      =					'''This is a simple input'''
                           lowercase__      =					['''This is a simple input 1''', '''This is a simple input 2''']
                           lowercase__      =					('''This is a simple input''', '''This is a pair''')
                           lowercase__      =					[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(UpperCamelCase  ,		tokenizer_r.encode  ,		UpperCamelCase  ,		max_length=UpperCamelCase  ,		padding='''max_length'''					)

                           # Simple input
                           self.assertRaises(UpperCamelCase  ,		tokenizer_r.encode_plus  ,		UpperCamelCase  ,		max_length=UpperCamelCase  ,		padding='''max_length'''					)

                           # Simple input
                           self.assertRaises(
                               UpperCamelCase  ,		tokenizer_r.batch_encode_plus  ,		UpperCamelCase  ,		max_length=UpperCamelCase  ,		padding='''max_length'''  ,		)

                           # Pair input
                           self.assertRaises(UpperCamelCase  ,		tokenizer_r.encode  ,		UpperCamelCase  ,		max_length=UpperCamelCase  ,		padding='''max_length'''					)

                           # Pair input
                           self.assertRaises(UpperCamelCase  ,		tokenizer_r.encode_plus  ,		UpperCamelCase  ,		max_length=UpperCamelCase  ,		padding='''max_length'''					)

                           # Pair input
                           self.assertRaises(
                               UpperCamelCase  ,		tokenizer_r.batch_encode_plus  ,		UpperCamelCase  ,		max_length=UpperCamelCase  ,		padding='''max_length'''  ,		)



      def  UpperCamelCase__						(self       :       Union[str, Any]					):

             '''simple docstring'''







             lowercase__      =					CodeGenTokenizer.from_pretrained(self.tmpdirname  ,		pad_token='''<pad>'''					)

             # Simple input
             lowercase__      =					'''This is a simple input'''
             lowercase__      =					['''This is a simple input looooooooong''', '''This is a simple input''']
             lowercase__      =					('''This is a simple input''', '''This is a pair''')
             lowercase__      =					[
                 ('''This is a simple input loooooong''', '''This is a simple input'''),
                 ('''This is a simple pair loooooong''', '''This is a simple pair'''),
             ]

             lowercase__      =					tokenizer.pad_token_id

             lowercase__      =					tokenizer(UpperCamelCase  ,		padding='''max_length'''  ,		max_length=30  ,		return_tensors='''np'''					)
             lowercase__      =					tokenizer(UpperCamelCase  ,		padding=UpperCamelCase  ,		truncate=UpperCamelCase  ,		return_tensors='''np'''					)
             lowercase__      =					tokenizer(*UpperCamelCase  ,		padding='''max_length'''  ,		max_length=60  ,		return_tensors='''np'''					)
             lowercase__      =					tokenizer(UpperCamelCase  ,		padding=UpperCamelCase  ,		truncate=UpperCamelCase  ,		return_tensors='''np'''					)

             # s
             # test single string max_length padding
             self.assertEqual(out_s['''input_ids'''].shape[-1]  ,		30					)
             self.assertTrue(pad_token_id in out_s['''input_ids''']					)
             self.assertTrue(0 in out_s['''attention_mask''']					)

             # s2
             # test automatic padding
             self.assertEqual(out_sa['''input_ids'''].shape[-1]  ,		33					)
             # long slice doesn't have padding
             self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]					)
             self.assertFalse(0 in out_sa['''attention_mask'''][0]					)
             # short slice does have padding
             self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]					)
             self.assertTrue(0 in out_sa['''attention_mask'''][1]					)

             # p
             # test single pair max_length padding
             self.assertEqual(out_p['''input_ids'''].shape[-1]  ,		60					)
             self.assertTrue(pad_token_id in out_p['''input_ids''']					)
             self.assertTrue(0 in out_p['''attention_mask''']					)

             # p2
             # test automatic padding pair
             self.assertEqual(out_pa['''input_ids'''].shape[-1]  ,		52					)
             # long slice pair doesn't have padding
             self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]					)
             self.assertFalse(0 in out_pa['''attention_mask'''][0]					)
             # short slice pair does have padding
             self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]					)
             self.assertTrue(0 in out_pa['''attention_mask'''][1]					)



      def  UpperCamelCase__						(self       :       Tuple					):

             '''simple docstring'''







             lowercase__      =					'''$$$'''
             lowercase__      =					CodeGenTokenizer.from_pretrained(self.tmpdirname  ,		bos_token=UpperCamelCase  ,		add_bos_token=UpperCamelCase					)

             lowercase__      =					'''This is a simple input'''
             lowercase__      =					['''This is a simple input 1''', '''This is a simple input 2''']

             lowercase__      =					tokenizer.bos_token_id

             lowercase__      =					tokenizer(UpperCamelCase					)
             lowercase__      =					tokenizer(UpperCamelCase					)

             self.assertEqual(out_s.input_ids[0]  ,		UpperCamelCase					)
             self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids					)					)

             lowercase__      =					tokenizer.decode(out_s.input_ids					)
             lowercase__      =					tokenizer.batch_decode(out_sa.input_ids					)

             self.assertEqual(decode_s.split()[0]  ,		UpperCamelCase					)
             self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa					)					)



      @slow
      def  UpperCamelCase__						(self       :       List[Any]					):

             '''simple docstring'''







             lowercase__      =					CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''					)

             lowercase__      =					'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
             lowercase__      =					'''\nif len_a > len_b:      result = a\nelse:      result = b'''

             lowercase__      =					tokenizer.encode(UpperCamelCase					)
             lowercase__      =					['''^#''', re.escape('''<|endoftext|>'''					), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
             lowercase__      =					tokenizer.decode(UpperCamelCase  ,		truncate_before_pattern=UpperCamelCase					)
             self.assertEqual(UpperCamelCase  ,		UpperCamelCase					)






      def  UpperCamelCase__						(self       :       str					):

             '''simple docstring'''







             pass








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase				:  Tuple    		=      {
    'configuration_distilbert': [
        'DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'DistilBertConfig',
        'DistilBertOnnxConfig',
    ],
    'tokenization_distilbert': ['DistilBertTokenizer'],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:  str    		=      ['DistilBertTokenizerFast']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:  List[str]    		=      [
     'DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'DistilBertForMaskedLM',
     'DistilBertForMultipleChoice',
     'DistilBertForQuestionAnswering',
     'DistilBertForSequenceClassification',
     'DistilBertForTokenClassification',
     'DistilBertModel',
     'DistilBertPreTrainedModel',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:  Union[str, Any]    		=      [
     'TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFDistilBertForMaskedLM',
     'TFDistilBertForMultipleChoice',
     'TFDistilBertForQuestionAnswering',
     'TFDistilBertForSequenceClassification',
     'TFDistilBertForTokenClassification',
     'TFDistilBertMainLayer',
     'TFDistilBertModel',
     'TFDistilBertPreTrainedModel',
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:  str    		=      [
     'FlaxDistilBertForMaskedLM',
     'FlaxDistilBertForMultipleChoice',
     'FlaxDistilBertForQuestionAnswering',
     'FlaxDistilBertForSequenceClassification',
     'FlaxDistilBertForTokenClassification',
     'FlaxDistilBertModel',
     'FlaxDistilBertPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_distilbert import (
     DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
     DistilBertConfig,
     DistilBertOnnxConfig,
 )
 from .tokenization_distilbert import DistilBertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_distilbert_fast import DistilBertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_distilbert import (
      DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      DistilBertForMaskedLM,
      DistilBertForMultipleChoice,
      DistilBertForQuestionAnswering,
      DistilBertForSequenceClassification,
      DistilBertForTokenClassification,
      DistilBertModel,
      DistilBertPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_distilbert import (
      TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFDistilBertForMaskedLM,
      TFDistilBertForMultipleChoice,
      TFDistilBertForQuestionAnswering,
      TFDistilBertForSequenceClassification,
      TFDistilBertForTokenClassification,
      TFDistilBertMainLayer,
      TFDistilBertModel,
      TFDistilBertPreTrainedModel,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_distilbert import (
      FlaxDistilBertForMaskedLM,
      FlaxDistilBertForMultipleChoice,
      FlaxDistilBertForQuestionAnswering,
      FlaxDistilBertForSequenceClassification,
      FlaxDistilBertForTokenClassification,
      FlaxDistilBertModel,
      FlaxDistilBertPreTrainedModel,
  )

else:
 import sys

 lowerCamelCase				:  int    		=      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




