





import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin




@require_jieba
class 	UpperCamelCase_							(  SCREAMING_SNAKE_CASE_					,	unittest.TestCase     ):
							lowerCamelCase_							     =       CpmAntTokenizer
							lowerCamelCase_							     =       False


							def   _snake_case	(				self     :int		) -> List[Any]:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__								=   [
									    '<d>',
									    '</d>',
									    '<s>',
									    '</s>',
									    '</_>',
									    '<unk>',
									    '<pad>',
									    '</n>',
									    '我',
									    '是',
									    'C',
									    'P',
									    'M',
									    'A',
									    'n',
									    't',
									]
									SCREAMING_SNAKE_CASE__								=   os.path.join(self.tmpdirname				,					VOCAB_FILES_NAMES["""vocab_file"""]		)
									with open(self.vocab_file				,					"""w"""				,					encoding="""utf-8"""		) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)


							@tooslow
							def   _snake_case	(				self     :Any		) -> Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   CpmAntTokenizer.from_pretrained("""openbmb/cpm-ant-10b"""		)
									SCREAMING_SNAKE_CASE__								=   '今天天气真好！'
									SCREAMING_SNAKE_CASE__								=   ['今天', '天气', '真', '好', '！']
									SCREAMING_SNAKE_CASE__								=   tokenizer.tokenize(__a		)
									self.assertListEqual(__a				,					__a		)
									SCREAMING_SNAKE_CASE__								=   '今天天气真好！'
									SCREAMING_SNAKE_CASE__								=   [tokenizer.bos_token] + tokens

									SCREAMING_SNAKE_CASE__								=   [6, 9802, 1_4962, 2082, 831, 244]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__a		)				,					__a		)

									SCREAMING_SNAKE_CASE__								=   tokenizer.decode(__a		)
									self.assertEqual(__a				,					__a		)




'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__    :   List[Any]           =					{
    '''configuration_luke''': ['''LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''LukeConfig'''],
    '''tokenization_luke''': ['''LukeTokenizer'''],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A__    :   Union[str, Any]           =					[
        '''LUKE_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''LukeForEntityClassification''',
        '''LukeForEntityPairClassification''',
        '''LukeForEntitySpanClassification''',
        '''LukeForMultipleChoice''',
        '''LukeForQuestionAnswering''',
        '''LukeForSequenceClassification''',
        '''LukeForTokenClassification''',
        '''LukeForMaskedLM''',
        '''LukeModel''',
        '''LukePreTrainedModel''',
    ]


if TYPE_CHECKING:
    from .configuration_luke import LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP, LukeConfig
    from .tokenization_luke import LukeTokenizer

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_luke import (
            LUKE_PRETRAINED_MODEL_ARCHIVE_LIST,
            LukeForEntityClassification,
            LukeForEntityPairClassification,
            LukeForEntitySpanClassification,
            LukeForMaskedLM,
            LukeForMultipleChoice,
            LukeForQuestionAnswering,
            LukeForSequenceClassification,
            LukeForTokenClassification,
            LukeModel,
            LukePreTrainedModel,
        )

else:
    import sys

    A__    :   List[str]           =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



