






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin
@require_jieba
class  _UpperCAmelCase   (			UpperCamelCase__						,unittest.TestCase      ):

				"""simple docstring"""




				lowercase__											=			CpmAntTokenizer
				lowercase__											=			False




				def 							lowercase__  (	self		:       Optional[int]    ):

								'''simple docstring'''
								super().setUp()

								lowercase__  					=     [
								    '''<d>''',
								    '''</d>''',
								    '''<s>''',
								    '''</s>''',
								    '''</_>''',
								    '''<unk>''',
								    '''<pad>''',
								    '''</n>''',
								    '''我''',
								    '''是''',
								    '''C''',
								    '''P''',
								    '''M''',
								    '''A''',
								    '''n''',
								    '''t''',
								]
								lowercase__  					=     os.path.join(self.tmpdirname,    VOCAB_FILES_NAMES['''vocab_file''']    )
								with open(self.vocab_file,    '''w''',    encoding='''utf-8'''    ) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )




				@tooslow
				def 							lowercase__  (	self		:       Union[str, Any]    ):

								'''simple docstring'''
								lowercase__  					=     CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''    )
								lowercase__  					=     '''今天天气真好！'''
								lowercase__  					=     ['''今天''', '''天气''', '''真''', '''好''', '''！''']
								lowercase__  					=     tokenizer.tokenize(lowercase_    )
								self.assertListEqual(lowercase_,    lowercase_    )
								lowercase__  					=     '''今天天气真好！'''
								lowercase__  					=     [tokenizer.bos_token] + tokens

								lowercase__  					=     [6, 9_802, 14_962, 2_082, 831, 244]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_    ),    lowercase_    )

								lowercase__  					=     tokenizer.decode(lowercase_    )
								self.assertEqual(lowercase_,    lowercase_    )











'''simple docstring'''

from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
  if not (is_transformers_available() and is_torch_available()):
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  from ...utils.dummy_torch_and_transformers_objects import KandinskyPipeline, KandinskyPriorPipeline
else:
  from .pipeline_kandinsky import KandinskyPipeline
  from .pipeline_kandinsky_imgaimg import KandinskyImgaImgPipeline
  from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline
  from .pipeline_kandinsky_prior import KandinskyPriorPipeline, KandinskyPriorPipelineOutput
  from .text_encoder import MultilingualCLIP



