"""simple docstring"""




import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


__lowerCamelCase											=  get_tests_dir("fixtures")
__lowerCamelCase											=  get_tests_dir("fixtures/dummy_feature_extractor_config.json")
__lowerCamelCase											=  get_tests_dir("fixtures/dummy-config.json")
class     UpperCamelCase__(			unittest.TestCase ):





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							0





						def 				snake_case__	(       self     )					->				Dict:
													A__	     =							AutoFeatureExtractor.from_pretrained('facebook/wav2vec2-base-960h'     )
													self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase     )
													self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				List[str]:
													with tempfile.TemporaryDirectory() as tmpdirname:
																				A__	     =							WavaVecaConfig()

																				# remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
																				A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase     ).to_dict()

																				config_dict.pop('feature_extractor_type'     )
																				A__	     =							WavaVecaFeatureExtractor(**__UpperCAmelCase     )

																				# save in new folder
																				model_config.save_pretrained(__UpperCAmelCase     )
																				config.save_pretrained(__UpperCAmelCase     )

																				A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase     )

																				# make sure private variable is not incorrectly saved
																				A__	     =							json.loads(config.to_json_string()     )
																				self.assertTrue('_processor_class' not in dict_as_saved     )

													self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase     )
													self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Union[str, Any]:
													with self.assertRaisesRegex(
													    __UpperCAmelCase		,'bert-base is not a local folder and is not a valid model identifier'     ):
																				A__	     =							AutoFeatureExtractor.from_pretrained('bert-base'     )





						def 				snake_case__	(       self     )					->				str:
													with self.assertRaisesRegex(
													    __UpperCAmelCase		,R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'     ):
																				A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase		,revision='aaaaaa'     )





						def 				snake_case__	(       self     )					->				Tuple:
													with self.assertRaisesRegex(
													    __UpperCAmelCase		,'hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'		,):
																				A__	     =							AutoFeatureExtractor.from_pretrained('hf-internal-testing/config-no-model'     )





						def 				snake_case__	(       self     )					->				Optional[Any]:
													# If remote code is not set, we will time out when asking whether to load the model.
													with self.assertRaises(__UpperCAmelCase     ):
																				A__	     =							AutoFeatureExtractor.from_pretrained(
																				    'hf-internal-testing/test_dynamic_feature_extractor'     )
													# If remote code is disabled, we can't load this config.
													with self.assertRaises(__UpperCAmelCase     ):
																				A__	     =							AutoFeatureExtractor.from_pretrained(
																				    'hf-internal-testing/test_dynamic_feature_extractor'		,trust_remote_code=__UpperCAmelCase     )

													A__	     =							AutoFeatureExtractor.from_pretrained(
													    'hf-internal-testing/test_dynamic_feature_extractor'		,trust_remote_code=__UpperCAmelCase     )
													self.assertEqual(feature_extractor.__class__.__name__		,'NewFeatureExtractor'     )

													# Test feature extractor can be reloaded.
													with tempfile.TemporaryDirectory() as tmp_dir:
																				feature_extractor.save_pretrained(__UpperCAmelCase     )
																				A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase		,trust_remote_code=__UpperCAmelCase     )
													self.assertEqual(reloaded_feature_extractor.__class__.__name__		,'NewFeatureExtractor'     )





						def 				snake_case__	(       self     )					->				List[Any]:
													try:
																				AutoConfig.register('custom'		,__UpperCAmelCase     )
																				AutoFeatureExtractor.register(__UpperCAmelCase		,__UpperCAmelCase     )
																				# Trying to register something existing in the Transformers library will raise an error
																				with self.assertRaises(__UpperCAmelCase     ):
																											AutoFeatureExtractor.register(__UpperCAmelCase		,__UpperCAmelCase     )

																				# Now that the config is registered, it can be used as any other config with the auto-API
																				A__	     =							CustomFeatureExtractor.from_pretrained(__UpperCAmelCase     )
																				with tempfile.TemporaryDirectory() as tmp_dir:
																											feature_extractor.save_pretrained(__UpperCAmelCase     )
																											A__	     =							AutoFeatureExtractor.from_pretrained(__UpperCAmelCase     )
																											self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )

													finally:
																				if "custom" in CONFIG_MAPPING._extra_content:
																											del CONFIG_MAPPING._extra_content["custom"]
																				if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																											del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]





						def 				snake_case__	(       self     )					->				Optional[Any]:
													class     UpperCamelCase__(			__A ):
																			lowerCAmelCase__  :						Tuple													=		True

													try:
																				AutoConfig.register('custom'		,__UpperCAmelCase     )
																				AutoFeatureExtractor.register(__UpperCAmelCase		,__UpperCAmelCase     )
																				# If remote code is not set, the default is to use local
																				A__	     =							AutoFeatureExtractor.from_pretrained(
																				    'hf-internal-testing/test_dynamic_feature_extractor'     )
																				self.assertEqual(feature_extractor.__class__.__name__		,'NewFeatureExtractor'     )
																				self.assertTrue(feature_extractor.is_local     )

																				# If remote code is disabled, we load the local one.
																				A__	     =							AutoFeatureExtractor.from_pretrained(
																				    'hf-internal-testing/test_dynamic_feature_extractor'		,trust_remote_code=__UpperCAmelCase     )
																				self.assertEqual(feature_extractor.__class__.__name__		,'NewFeatureExtractor'     )
																				self.assertTrue(feature_extractor.is_local     )

																				# If remote is enabled, we load from the Hub
																				A__	     =							AutoFeatureExtractor.from_pretrained(
																				    'hf-internal-testing/test_dynamic_feature_extractor'		,trust_remote_code=__UpperCAmelCase     )
																				self.assertEqual(feature_extractor.__class__.__name__		,'NewFeatureExtractor'     )
																				self.assertTrue(not hasattr(__UpperCAmelCase		,'is_local'     )     )

													finally:
																				if "custom" in CONFIG_MAPPING._extra_content:
																											del CONFIG_MAPPING._extra_content["custom"]
																				if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																											del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]






"""simple docstring"""




import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin
@require_jieba
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						Optional[int]													=		CpmAntTokenizer
						lowerCAmelCase__  :						List[str]													=		False





						def 				snake_case__	(       self     )					->				Dict:
													super().setUp()

													A__	     =							[
													    '<d>',
													    '</d>',
													    '<s>',
													    '</s>',
													    '</_>',
													    '<unk>',
													    '<pad>',
													    '</n>',
													    '我',
													    '是',
													    'C',
													    'P',
													    'M',
													    'A',
													    'n',
													    't',
													]
													A__	     =							os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['vocab_file']     )
													with open(self.vocab_file		,'w'		,encoding='utf-8'     ) as vocab_writer:
																				vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )





						@tooslow
						def 				snake_case__	(       self     )					->				List[Any]:
													A__	     =							CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b'     )
													A__	     =							'今天天气真好！'
													A__	     =							['今天', '天气', '真', '好', '！']
													A__	     =							tokenizer.tokenize(__UpperCAmelCase     )
													self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )
													A__	     =							'今天天气真好！'
													A__	     =							[tokenizer.bos_token] + tokens

													A__	     =							[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
													self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase     )		,__UpperCAmelCase     )

													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,__UpperCAmelCase     )





