






"""simple docstring"""






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin





@require_jieba
class       snake_case_       (    _lowerCamelCase    ,							unittest.TestCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Optional[int]   			=					CpmAntTokenizer
	SCREAMING_SNAKE_CASE_:							Dict   			=					False


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					super().setUp()

					A__							 =       [
					    '<d>',
					    '</d>',
					    '<s>',
					    '</s>',
					    '</_>',
					    '<unk>',
					    '<pad>',
					    '</n>',
					    '我',
					    '是',
					    'C',
					    'P',
					    'M',
					    'A',
					    'n',
					    't',
					]
					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






	@tooslow
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b'						)
					A__							 =       '今天天气真好！'
					A__							 =       ['今天', '天气', '真', '好', '！']
					A__							 =       tokenizer.tokenize(__a						)
					self.assertListEqual(__a  ,					__a						)
					A__							 =       '今天天气真好！'
					A__							 =       [tokenizer.bos_token] + tokens

					A__							 =       [6, 9802, 1_4962, 2082, 831, 244]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__a						)  ,					__a						)

					A__							 =       tokenizer.decode(__a						)
					self.assertEqual(__a  ,					__a						)










"""simple docstring"""






from ..utils import DummyObject, requires_backends





class       snake_case_       (    metaclass=_lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							List[str]   			=					["""note_seq"""]


	def __init__(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					requires_backends(self  ,					['note_seq']						)


	@classmethod
	def       _UpperCAmelCase						(      cls  ,					*__a  ,					**__a						):


					"""simple docstring"""

					requires_backends(cls  ,					['note_seq']						)






	@classmethod
	def       _UpperCAmelCase						(      cls  ,					*__a  ,					**__a						):


					"""simple docstring"""

					requires_backends(cls  ,					['note_seq']						)


