





"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a_			=      {
    """configuration_nezha""": ["""NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """NezhaConfig"""],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_			=      [
           """NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """NezhaForNextSentencePrediction""",
           """NezhaForMaskedLM""",
           """NezhaForPreTraining""",
           """NezhaForMultipleChoice""",
           """NezhaForQuestionAnswering""",
           """NezhaForSequenceClassification""",
           """NezhaForTokenClassification""",
           """NezhaModel""",
           """NezhaPreTrainedModel""",
       ]


if TYPE_CHECKING:
       from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_nezha import (
                  NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
                  NezhaForMaskedLM,
                  NezhaForMultipleChoice,
                  NezhaForNextSentencePrediction,
                  NezhaForPreTraining,
                  NezhaForQuestionAnswering,
                  NezhaForSequenceClassification,
                  NezhaForTokenClassification,
                  NezhaModel,
                  NezhaPreTrainedModel,
              )


else:
       import sys

       a_			=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)











"""simple docstring"""



import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		__snake_case (		SCREAMING_SNAKE_CASE__							,							unittest.TestCase					):





 """simple docstring"""


 _lowerCamelCase				=					MgpstrTokenizer
 _lowerCamelCase				=					False
 _lowerCamelCase				=					{}
 _lowerCamelCase				=					False

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     super().setUp()

     # fmt: off
     __A   :						int									=					['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
     # fmt: on
     __A   :						Dict									=					dict(zip(__lowerCamelCase ,						range(len(__lowerCamelCase  )  )  )  )

     __A   :						Optional[Any]									=					os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''vocab_file''']  )
     with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8'''  ) as fp:
         fp.write(json.dumps(__lowerCamelCase  ) + '''\n'''  )

 def  UpperCamelCase__(   self ,						**__lowerCamelCase  ):



     '''simple docstring'''




     return MgpstrTokenizer.from_pretrained(self.tmpdirname ,						**__lowerCamelCase  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     __A   :						List[str]									=					'''tester'''
     __A   :						Dict									=					'''tester'''
     return input_text, output_text

 @unittest.skip('''MGP-STR always lower cases letters.'''  )
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     pass

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						List[Any]									=					self.get_tokenizers(do_lower_case=__lowerCamelCase  )
     for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""  ):
             __A   :						Union[str, Any]									=					'''[SPECIAL_TOKEN]'''

             tokenizer.add_special_tokens({'''cls_token''': special_token}  )
             __A   :						Optional[Any]									=					tokenizer.encode([special_token] ,						add_special_tokens=__lowerCamelCase  )
             self.assertEqual(len(__lowerCamelCase  ) ,						1  )

             __A   :						List[Any]									=					tokenizer.decode(__lowerCamelCase ,						skip_special_tokens=__lowerCamelCase  )
             self.assertTrue(special_token not in decoded  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Tuple									=					self.get_tokenizers()
     for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""  ):
             __A			,	__A   :						str									=					self.get_input_output_texts(__lowerCamelCase  )

             __A   :						Union[str, Any]									=					tokenizer.tokenize(__lowerCamelCase  )
             __A   :						Union[str, Any]									=					tokenizer.convert_tokens_to_ids(__lowerCamelCase  )
             __A   :						Union[str, Any]									=					tokenizer.encode(__lowerCamelCase ,						add_special_tokens=__lowerCamelCase  )
             self.assertListEqual(__lowerCamelCase ,						__lowerCamelCase  )

             __A   :						Optional[Any]									=					tokenizer.convert_ids_to_tokens(__lowerCamelCase  )
             self.assertNotEqual(len(__lowerCamelCase  ) ,						0  )
             __A   :						Union[str, Any]									=					tokenizer.decode(__lowerCamelCase  )
             self.assertIsInstance(__lowerCamelCase ,						__lowerCamelCase  )

             self.assertEqual(text_a.replace(''' ''' ,						''''''  ) ,						__lowerCamelCase  )

 @unittest.skip('''MGP-STR tokenizer only handles one sequence.'''  )
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     pass







 @unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''  )
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     pass




