


'''simple docstring'''




import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


_a   :   Optional[Any]												=   logging.getLogger()




@unittest.skip("Temporarily disable the doc tests."							)
@require_torch
@require_tf
@slow
class 			_lowercase (	unittest.TestCase							):
   def a  (						self       :      Optional[int]       ,   SCREAMING_SNAKE_CASE_       :      Path       ,   SCREAMING_SNAKE_CASE_       :      Union[str, None] = None       ,   SCREAMING_SNAKE_CASE_       :      Union[List[str], None] = None       ,   SCREAMING_SNAKE_CASE_       :      Union[str, List[str], None] = None       ,   SCREAMING_SNAKE_CASE_       :      bool = True       ,   ) ->							Any:
        __snake_case           =						[file for file in os.listdir(SCREAMING_SNAKE_CASE_      ) if os.path.isfile(os.path.join(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )      )]

        if identifier is not None:
             __snake_case           =						[file for file in files if identifier in file]

        if n_identifier is not None:
             if isinstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      ):
                  for n_ in n_identifier:
                       __snake_case           =						[file for file in files if n_ not in file]
             else:
                  __snake_case           =						[file for file in files if n_identifier not in file]

        __snake_case           =						ignore_files or []
        ignore_files.append('__init__.py'      )
        __snake_case           =						[file for file in files if file not in ignore_files]

        for file in files:
             # Open all files
             print('Testing'       ,   SCREAMING_SNAKE_CASE_      )

             if only_modules:
                  __snake_case           =						file.split('.'      )[0]
                  try:
                       __snake_case           =						getattr(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
                       __snake_case           =						doctest.DocTestSuite(SCREAMING_SNAKE_CASE_      )
                       __snake_case           =						unittest.TextTestRunner().run(SCREAMING_SNAKE_CASE_      )
                       self.assertIs(len(result.failures      )       ,   0      )
                  except AttributeError:
                       logger.info(f'{module_identifier} is not a module.'      )
             else:
                  __snake_case           =						doctest.testfile(str('..' / directory / file      )       ,   optionflags=doctest.ELLIPSIS      )
                  self.assertIs(result.failed       ,   0      )
   def a  (						self       :      List[Any]      ) ->							int:
        __snake_case           =						Path('src/transformers'      )
        __snake_case           =						'modeling'
        __snake_case           =						[
            'modeling_ctrl.py',
            'modeling_tf_ctrl.py',
        ]
        self.analyze_directory(SCREAMING_SNAKE_CASE_       ,   identifier=SCREAMING_SNAKE_CASE_       ,   ignore_files=SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      List[Any]      ) ->							str:
        __snake_case           =						Path('src/transformers'      )
        __snake_case           =						'tokenization'
        self.analyze_directory(SCREAMING_SNAKE_CASE_       ,   identifier=SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Optional[Any]      ) ->							Tuple:
        __snake_case           =						Path('src/transformers'      )
        __snake_case           =						'configuration'
        self.analyze_directory(SCREAMING_SNAKE_CASE_       ,   identifier=SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      List[Any]      ) ->							int:
        __snake_case           =						Path('src/transformers'      )
        __snake_case           =						['configuration', 'modeling', 'tokenization']
        self.analyze_directory(SCREAMING_SNAKE_CASE_       ,   n_identifier=SCREAMING_SNAKE_CASE_      )





   def a  (						self       :      int      ) ->							Tuple:
        __snake_case           =						Path('docs/source'      )
        __snake_case           =						['favicon.ico']
        self.analyze_directory(SCREAMING_SNAKE_CASE_       ,   ignore_files=SCREAMING_SNAKE_CASE_       ,   only_modules=SCREAMING_SNAKE_CASE_      )








"""simple docstring"""






import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class      A(  lowerCamelCase__				,							unittest.TestCase		):

		"""simple docstring"""





		A    =							CpmAntTokenizer
		A    =							False
		def   _UpperCamelCase( self   )      ->	Union[str, Any]:

					"""simple docstring"""



					super().setUp()

					_UpperCamelCase					:str					     =	[
					    '''<d>''',
					    '''</d>''',
					    '''<s>''',
					    '''</s>''',
					    '''</_>''',
					    '''<unk>''',
					    '''<pad>''',
					    '''</n>''',
					    '''我''',
					    '''是''',
					    '''C''',
					    '''P''',
					    '''M''',
					    '''A''',
					    '''n''',
					    '''t''',
					]
					_UpperCamelCase					:List[str]					     =	os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES['''vocab_file''']   )
					with open(self.vocab_file					,				'''w'''					,				encoding='''utf-8'''   ) as vocab_writer:
								vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )





		@tooslow
		def   _UpperCamelCase( self   )      ->	Tuple:

					"""simple docstring"""



					_UpperCamelCase					:Any					     =	CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''   )
					_UpperCamelCase					:List[str]					     =	'''今天天气真好！'''
					_UpperCamelCase					:Tuple					     =	['''今天''', '''天气''', '''真''', '''好''', '''！''']
					_UpperCamelCase					:Tuple					     =	tokenizer.tokenize(SCREAMING_SNAKE_CASE__   )
					self.assertListEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:Dict					     =	'''今天天气真好！'''
					_UpperCamelCase					:Optional[int]					     =	[tokenizer.bos_token] + tokens

					_UpperCamelCase					:List[str]					     =	[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__   )					,				SCREAMING_SNAKE_CASE__   )

					_UpperCamelCase					:str					     =	tokenizer.decode(SCREAMING_SNAKE_CASE__   )
					self.assertEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )

