




'''simple docstring'''



import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class 							_a				(			SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				Optional[int]        =						CpmAntTokenizer
       A						:				Dict        =						False







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().setUp()

             SCREAMING_SNAKE_CASE   :      List[str]    					=						[
                 '<d>',
                 '</d>',
                 '<s>',
                 '</s>',
                 '</_>',
                 '<unk>',
                 '<pad>',
                 '</n>',
                 '我',
                 '是',
                 'C',
                 'P',
                 'M',
                 'A',
                 'n',
                 't',
             ]
             SCREAMING_SNAKE_CASE   :      Any    					=						os.path.join(self.tmpdirname,							VOCAB_FILES_NAMES['vocab_file']  )
             with open(self.vocab_file,							'w',							encoding='utf-8'  ) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )







       @tooslow
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b'  )
             SCREAMING_SNAKE_CASE   :      Tuple    					=						'今天天气真好！'
             SCREAMING_SNAKE_CASE   :      Tuple    					=						['今天', '天气', '真', '好', '！']
             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.tokenize(A  )
             self.assertListEqual(A,							A  )
             SCREAMING_SNAKE_CASE   :      int    					=						'今天天气真好！'
             SCREAMING_SNAKE_CASE   :      Dict    					=						[tokenizer.bos_token] + tokens

             SCREAMING_SNAKE_CASE   :      List[Any]    					=						[6, 9_802, 14_962, 2_082, 831, 244]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(A  ),							A  )

             SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.decode(A  )
             self.assertEqual(A,							A  )









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


a__: str							   =						{
    'configuration_mega': ['MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegaConfig', 'MegaOnnxConfig'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__: Union[str, Any]							   =						[
           'MEGA_PRETRAINED_MODEL_ARCHIVE_LIST',
           'MegaForCausalLM',
           'MegaForMaskedLM',
           'MegaForMultipleChoice',
           'MegaForQuestionAnswering',
           'MegaForSequenceClassification',
           'MegaForTokenClassification',
           'MegaModel',
           'MegaPreTrainedModel',
       ]

if TYPE_CHECKING:
       from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_mega import (
                  MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
                  MegaForCausalLM,
                  MegaForMaskedLM,
                  MegaForMultipleChoice,
                  MegaForQuestionAnswering,
                  MegaForSequenceClassification,
                  MegaForTokenClassification,
                  MegaModel,
                  MegaPreTrainedModel,
              )

else:
       import sys

       a__: Union[str, Any]							   =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





