



'''simple docstring'''





import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin



@require_jieba
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= CpmAntTokenizer
 A__				= False
 def A_   (   self						:							int  )      ->   List[Any]:







       '''simple docstring'''
       super().setUp()

       __snake_case :   int													=    [
           '<d>',
           '</d>',
           '<s>',
           '</s>',
           '</_>',
           '<unk>',
           '<pad>',
           '</n>',
           '我',
           '是',
           'C',
           'P',
           'M',
           'A',
           'n',
           't',
       ]
       __snake_case :   List[Any]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )





 @tooslow
 def A_   (   self						:							Any  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   str													=    CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b'  )
       __snake_case :   Optional[int]													=    '今天天气真好！'
       __snake_case :   Tuple													=    ['今天', '天气', '真', '好', '！']
       __snake_case :   Optional[int]													=    tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )
       __snake_case :   Union[str, Any]													=    '今天天气真好！'
       __snake_case :   Dict													=    [tokenizer.bos_token] + tokens

       __snake_case :   Optional[Any]													=    [6, 9802, 14962, 2082, 831, 244]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     __a  )

       __snake_case :   Dict													=    tokenizer.decode(__a  )
       self.assertEqual(__a      ,     __a  )








'''simple docstring'''





from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__    :   List[str]           =					{
    '''configuration_autoformer''': [
        '''AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''AutoformerConfig''',
    ],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A__    :   Optional[Any]           =					[
        '''AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''AutoformerForPrediction''',
        '''AutoformerModel''',
        '''AutoformerPreTrainedModel''',
    ]


if TYPE_CHECKING:
    from .configuration_autoformer import (
        AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
        AutoformerConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_autoformer import (
            AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
            AutoformerForPrediction,
            AutoformerModel,
            AutoformerPreTrainedModel,
        )

else:
    import sys

    A__    :   Optional[int]           =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



