






import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer

def        UpperCAmelCase__		(						lowerCamelCase_       :       List[Any]				):
			# This defines a "chinese character" as anything in the CJK Unicode block:
			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
			#
			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
			# despite its name. The modern Korean Hangul alphabet is a different block,
			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
			# space-separated words, so they are not treated specially and handled
			# like the all of the other languages.
			if (
			    (cp >= 0x4e00 and cp <= 0x9fff)
			    or (cp >= 0x3400 and cp <= 0x4dbf)  #
			    or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
			    or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
			    or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
			    or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
			    or (cp >= 0xf900 and cp <= 0xfaff)
			    or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
			):  #
						return True

			return False

def        UpperCAmelCase__		(						lowerCamelCase_       :       str				):
			# word like '180' or '身高' or '神'
			for char in word:
						__a						:  Optional[int]									=     ord(lowerCamelCase_				)
						if not _is_chinese_char(lowerCamelCase_				):
									return 0
			return 1

def        UpperCAmelCase__		(						lowerCamelCase_       :       List[str]				):
			__a						:  Union[str, Any]									=     set()

			for token in tokens:
						__a						:  Any									=     len(lowerCamelCase_				) > 1 and is_chinese(lowerCamelCase_				)
						if chinese_word:
									word_set.add(lowerCamelCase_				)
			__a						:  str									=     list(lowerCamelCase_				)
			return word_list

def        UpperCAmelCase__		(						lowerCamelCase_       :       List[str]   ,  lowerCamelCase_       :       set()				):
			if not chinese_word_set:
						return bert_tokens
			__a						:  Tuple									=     max([len(lowerCamelCase_				) for w in chinese_word_set]				)

			__a						:  List[str]									=     bert_tokens
			__a						,					__a						:  Tuple									=     0, len(lowerCamelCase_				)
			while start < end:
						__a						:  List[str]									=     True
						if is_chinese(bert_word[start]				):
									__a						:  List[str]									=     min(end - start   ,  lowerCamelCase_				)
									for i in range(lowerCamelCase_   ,  1   ,  -1				):
												__a						:  Optional[int]									=     ''.join(bert_word[start : start + i]				)
												if whole_word in chinese_word_set:
															for j in range(start + 1   ,  start + i				):
																		__a						:  Optional[Any]									=     '##' + bert_word[j]
															__a						:  List[Any]									=     start + i
															__a						:  Tuple									=     False
															break
						if single_word:
									start += 1
			return bert_word

def        UpperCAmelCase__		(						lowerCamelCase_       :       List[str]   ,  lowerCamelCase_       :       LTP   ,  lowerCamelCase_       :       BertTokenizer				):
			__a						:  Union[str, Any]									=     []

			for i in range(0   ,  len(lowerCamelCase_				)   ,  1_0_0				):
						__a						:  List[Any]									=     ltp_tokenizer.pipeline(lines[i : i + 1_0_0]   ,  tasks=['cws']				).cws
						__a						:  Optional[int]									=     [get_chinese_word(lowerCamelCase_				) for r in res]
						ltp_res.extend(lowerCamelCase_				)
			assert len(lowerCamelCase_				) == len(lowerCamelCase_				)

			__a						:  Optional[Any]									=     []
			for i in range(0   ,  len(lowerCamelCase_				)   ,  1_0_0				):
						__a						:  Union[str, Any]									=     bert_tokenizer(lines[i : i + 1_0_0]   ,  add_special_tokens=lowerCamelCase_   ,  truncation=lowerCamelCase_   ,  max_length=5_1_2				)
						bert_res.extend(res['input_ids']				)
			assert len(lowerCamelCase_				) == len(lowerCamelCase_				)

			__a						:  Union[str, Any]									=     []
			for input_ids, chinese_word in zip(lowerCamelCase_   ,  lowerCamelCase_				):
						__a						:  str									=     []
						for id in input_ids:
									__a						:  Optional[int]									=     bert_tokenizer._convert_id_to_token(lowerCamelCase_				)
									input_tokens.append(lowerCamelCase_				)
						__a						:  int									=     add_sub_symbol(lowerCamelCase_   ,  lowerCamelCase_				)
						__a						:  List[Any]									=     []
						# We only save pos of chinese subwords start with ##, which mean is part of a whole word.
						for i, token in enumerate(lowerCamelCase_				):
									if token[:2] == "##":
												__a						:  Dict									=     token[2:]
												# save chinese tokens' pos
												if len(lowerCamelCase_				) == 1 and _is_chinese_char(ord(lowerCamelCase_				)				):
															ref_id.append(lowerCamelCase_				)
						ref_ids.append(lowerCamelCase_				)

			assert len(lowerCamelCase_				) == len(lowerCamelCase_				)

			return ref_ids

def        UpperCAmelCase__		(						lowerCamelCase_       :       str				):
			# For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
			# If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
			with open(args.file_name   ,  'r'   ,  encoding='utf-8'				) as f:
						__a						:  int									=     f.readlines()
			__a						:  Optional[Any]									=     [line.strip() for line in data if len(lowerCamelCase_				) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
			__a						:  int									=     LTP(args.ltp				)  # faster in GPU device
			__a						:  Dict									=     BertTokenizer.from_pretrained(args.bert				)

			__a						:  Union[str, Any]									=     prepare_ref(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_				)

			with open(args.save_path   ,  'w'   ,  encoding='utf-8'				) as f:
						__a						:  Union[str, Any]									=     [json.dumps(lowerCamelCase_				) + '\n' for ref in ref_ids]
						f.writelines(lowerCamelCase_				)


if __name__ == "__main__":
							SCREAMING_SNAKE_CASE__				     =			argparse.ArgumentParser(description='''prepare_chinese_ref''')
							parser.add_argument(
							    '''--file_name''',
							    required=False,
							    type=str,
							    default='''./resources/chinese-demo.txt''',
							    help='''file need process, same as training data in lm''',
							)
							parser.add_argument(
							    '''--ltp''',
							    required=False,
							    type=str,
							    default='''./resources/ltp''',
							    help='''resources for LTP tokenizer, usually a path''',
							)
							parser.add_argument(
							    '''--bert''',
							    required=False,
							    type=str,
							    default='''./resources/robert''',
							    help='''resources for Bert tokenizer''',
							)
							parser.add_argument(
							    '''--save_path''',
							    required=False,
							    type=str,
							    default='''./resources/ref.txt''',
							    help='''path to save res''',
							)

							SCREAMING_SNAKE_CASE__				     =			parser.parse_args()
							main(args)









import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


lowerCamelCase		   =			get_tests_dir('fixtures/dummy_feature_extractor_config.json')
lowerCamelCase		   =			get_tests_dir('fixtures/vocab.json')
lowerCamelCase		   =			get_tests_dir('fixtures')




class A    (  unittest.TestCase	):
      UpperCamelCase__				:    Dict =['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






      def 			lowerCamelCase			(   self		:	str					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =0






      def 			lowerCamelCase			(   self		:	str					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'					)
             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Tuple:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						int            =WavaVecaConfig()
                    _lowerCamelCase			:						Dict            =AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'					)

                    # save in new folder
                    model_config.save_pretrained(lowercase_					)
                    processor.save_pretrained(lowercase_					)

                    _lowerCamelCase			:						Any            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Union[str, Any]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    # copy relevant files
                    copyfile(lowercase_     ,  os.path.join(lowercase_     ,  lowercase_					)					)
                    copyfile(lowercase_     ,  os.path.join(lowercase_     ,  'vocab.json'					)					)

                    _lowerCamelCase			:						Union[str, Any]            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		Optional[Any]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						List[Any]            =WavaVecaFeatureExtractor()
                    _lowerCamelCase			:						List[str]            =AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'					)

                    _lowerCamelCase			:						str            =WavaVecaProcessor(lowercase_     ,  lowercase_					)

                    # save in new folder
                    processor.save_pretrained(lowercase_					)

                    # drop `processor_class` in tokenizer
                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'r'					) as f:
                           _lowerCamelCase			:						Optional[int]            =json.load(lowercase_					)
                           config_dict.pop('processor_class'					)

                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'w'					) as f:
                           f.write(json.dumps(lowercase_					)					)

                    _lowerCamelCase			:						Optional[int]            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[int]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						Optional[Any]            =WavaVecaFeatureExtractor()
                    _lowerCamelCase			:						Tuple            =AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'					)

                    _lowerCamelCase			:						Dict            =WavaVecaProcessor(lowercase_     ,  lowercase_					)

                    # save in new folder
                    processor.save_pretrained(lowercase_					)

                    # drop `processor_class` in feature extractor
                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'r'					) as f:
                           _lowerCamelCase			:						Union[str, Any]            =json.load(lowercase_					)
                           config_dict.pop('processor_class'					)

                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'w'					) as f:
                           f.write(json.dumps(lowercase_					)					)

                    _lowerCamelCase			:						Optional[int]            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		List[str]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						Optional[Any]            =WavaVecaConfig(processor_class='Wav2Vec2Processor'					)
                    model_config.save_pretrained(lowercase_					)
                    # copy relevant files
                    copyfile(lowercase_     ,  os.path.join(lowercase_     ,  'vocab.json'					)					)
                    # create emtpy sample processor
                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'w'					) as f:
                           f.write('{}'					)

                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Optional[int]:
             """simple docstring"""




             with self.assertRaises(lowercase_					):
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'					)
             # If remote code is disabled, we can't load this config.
             with self.assertRaises(lowercase_					):
                    _lowerCamelCase			:						Union[str, Any]            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)

             _lowerCamelCase			:						List[str]            =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)
             self.assertTrue(processor.special_attribute_present					)
             self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)

             _lowerCamelCase			:						int            =processor.feature_extractor
             self.assertTrue(feature_extractor.special_attribute_present					)
             self.assertEqual(feature_extractor.__class__.__name__     ,  'NewFeatureExtractor'					)

             _lowerCamelCase			:						Optional[int]            =processor.tokenizer
             self.assertTrue(tokenizer.special_attribute_present					)
             if is_tokenizers_available():
                    self.assertEqual(tokenizer.__class__.__name__     ,  'NewTokenizerFast'					)

                    # Test we can also load the slow version
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_     ,  use_fast=lowercase_					)
                    _lowerCamelCase			:						Optional[int]            =new_processor.tokenizer
                    self.assertTrue(new_tokenizer.special_attribute_present					)
                    self.assertEqual(new_tokenizer.__class__.__name__     ,  'NewTokenizer'					)
             else:
                    self.assertEqual(tokenizer.__class__.__name__     ,  'NewTokenizer'					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		Optional[int]:
             """simple docstring"""




             try:
                    AutoConfig.register('custom'     ,  lowercase_					)
                    AutoFeatureExtractor.register(lowercase_     ,  lowercase_					)
                    AutoTokenizer.register(lowercase_     ,  slow_tokenizer_class=lowercase_					)
                    AutoProcessor.register(lowercase_     ,  lowercase_					)
                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(lowercase_					):
                           AutoProcessor.register(lowercase_     ,  lowercase_					)

                    # Now that the config is registered, it can be used as any other config with the auto-API
                    _lowerCamelCase			:						str            =CustomFeatureExtractor.from_pretrained(lowercase_					)

                    with tempfile.TemporaryDirectory() as tmp_dir:
                           _lowerCamelCase			:						str            =os.path.join(lowercase_     ,  'vocab.txt'					)
                           with open(lowercase_     ,  'w'     ,  encoding='utf-8'					) as vocab_writer:
                                  vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
                           _lowerCamelCase			:						List[Any]            =CustomTokenizer(lowercase_					)

                    _lowerCamelCase			:						Optional[int]            =CustomProcessor(lowercase_     ,  lowercase_					)

                    with tempfile.TemporaryDirectory() as tmp_dir:
                           processor.save_pretrained(lowercase_					)
                           _lowerCamelCase			:						List[Any]            =AutoProcessor.from_pretrained(lowercase_					)
                           self.assertIsInstance(lowercase_     ,  lowercase_					)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in PROCESSOR_MAPPING._extra_content:
                           del PROCESSOR_MAPPING._extra_content[CustomConfig]






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		List[str]:
             """simple docstring"""




             class A    (  UpperCamelCase_	):
                   UpperCamelCase__				:    Optional[Any] =False




             class A    (  UpperCamelCase_	):
                   UpperCamelCase__				:    int =False




             class A    (  UpperCamelCase_	):
                   UpperCamelCase__				:    Union[str, Any] ='AutoFeatureExtractor'
                   UpperCamelCase__				:    str ='AutoTokenizer'
                   UpperCamelCase__				:    List[Any] =False

             try:
                    AutoConfig.register('custom'     ,  lowercase_					)
                    AutoFeatureExtractor.register(lowercase_     ,  lowercase_					)
                    AutoTokenizer.register(lowercase_     ,  slow_tokenizer_class=lowercase_					)
                    AutoProcessor.register(lowercase_     ,  lowercase_					)
                    # If remote code is not set, the default is to use local classes.
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'					)
                    self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)
                    self.assertFalse(processor.special_attribute_present					)
                    self.assertFalse(processor.feature_extractor.special_attribute_present					)
                    self.assertFalse(processor.tokenizer.special_attribute_present					)

                    # If remote code is disabled, we load the local ones.
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)
                    self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)
                    self.assertFalse(processor.special_attribute_present					)
                    self.assertFalse(processor.feature_extractor.special_attribute_present					)
                    self.assertFalse(processor.tokenizer.special_attribute_present					)

                    # If remote is enabled, we load from the Hub.
                    _lowerCamelCase			:						str            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)
                    self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)
                    self.assertTrue(processor.special_attribute_present					)
                    self.assertTrue(processor.feature_extractor.special_attribute_present					)
                    self.assertTrue(processor.tokenizer.special_attribute_present					)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in PROCESSOR_MAPPING._extra_content:
                           del PROCESSOR_MAPPING._extra_content[CustomConfig]






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-bert'					)
             self.assertEqual(processor.__class__.__name__     ,  'BertTokenizerFast'					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Any            =AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-convnext'					)
             self.assertEqual(processor.__class__.__name__     ,  'ConvNextImageProcessor'					)




@is_staging_test
class A    (  unittest.TestCase	):
      UpperCamelCase__				:    List[Any] =['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






      @classmethod
      def 			lowerCamelCase			(   cls		:	int					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =TOKEN
             HfFolder.save_token(lowercase_					)






      @classmethod
      def 			lowerCamelCase			(   cls		:	Optional[int]					)   ->		Union[str, Any]:
             """simple docstring"""




             try:
                    delete_repo(token=cls._token     ,  repo_id='test-processor'					)
             except HTTPError:
                    pass

             try:
                    delete_repo(token=cls._token     ,  repo_id='valid_org/test-processor-org'					)
             except HTTPError:
                    pass

             try:
                    delete_repo(token=cls._token     ,  repo_id='test-dynamic-processor'					)
             except HTTPError:
                    pass






      def 			lowerCamelCase			(   self		:	str					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						Tuple            =WavaVecaProcessor.from_pretrained(lowercase_					)
             with tempfile.TemporaryDirectory() as tmp_dir:
                    processor.save_pretrained(
                        os.path.join(lowercase_     ,  'test-processor'					)     ,  push_to_hub=lowercase_     ,  use_auth_token=self._token					)

                    _lowerCamelCase			:						Union[str, Any]            =WavaVecaProcessor.from_pretrained(F'''{USER}/test-processor'''					)
                    for k, v in processor.feature_extractor.__dict__.items():
                           self.assertEqual(lowercase_     ,  getattr(new_processor.feature_extractor     ,  lowercase_					)					)
                    self.assertDictEqual(new_processor.tokenizer.get_vocab()     ,  processor.tokenizer.get_vocab()					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						int            =WavaVecaProcessor.from_pretrained(lowercase_					)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    processor.save_pretrained(
                        os.path.join(lowercase_     ,  'test-processor-org'					)     ,  push_to_hub=lowercase_     ,  use_auth_token=self._token     ,  organization='valid_org'     ,  )

                    _lowerCamelCase			:						str            =WavaVecaProcessor.from_pretrained('valid_org/test-processor-org'					)
                    for k, v in processor.feature_extractor.__dict__.items():
                           self.assertEqual(lowercase_     ,  getattr(new_processor.feature_extractor     ,  lowercase_					)					)
                    self.assertDictEqual(new_processor.tokenizer.get_vocab()     ,  processor.tokenizer.get_vocab()					)






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		str:
             """simple docstring"""




             CustomFeatureExtractor.register_for_auto_class()
             CustomTokenizer.register_for_auto_class()
             CustomProcessor.register_for_auto_class()

             _lowerCamelCase			:						Optional[Any]            =CustomFeatureExtractor.from_pretrained(lowercase_					)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    _lowerCamelCase			:						Dict            =os.path.join(lowercase_     ,  'vocab.txt'					)
                    with open(lowercase_     ,  'w'     ,  encoding='utf-8'					) as vocab_writer:
                           vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
                    _lowerCamelCase			:						Any            =CustomTokenizer(lowercase_					)

             _lowerCamelCase			:						List[Any]            =CustomProcessor(lowercase_     ,  lowercase_					)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    create_repo(F'''{USER}/test-dynamic-processor'''     ,  token=self._token					)
                    _lowerCamelCase			:						List[str]            =Repository(lowercase_     ,  clone_from=F'''{USER}/test-dynamic-processor'''     ,  token=self._token					)
                    processor.save_pretrained(lowercase_					)

                    # This has added the proper auto_map field to the feature extractor config
                    self.assertDictEqual(
                        processor.feature_extractor.auto_map     ,  {
                            'AutoFeatureExtractor': 'custom_feature_extraction.CustomFeatureExtractor',
                            'AutoProcessor': 'custom_processing.CustomProcessor',
                        }     ,  )

                    # This has added the proper auto_map field to the tokenizer config
                    with open(os.path.join(lowercase_     ,  'tokenizer_config.json'					)					) as f:
                           _lowerCamelCase			:						Union[str, Any]            =json.load(lowercase_					)
                    self.assertDictEqual(
                        tokenizer_config['auto_map']     ,  {
                            'AutoTokenizer': ['custom_tokenization.CustomTokenizer', None],
                            'AutoProcessor': 'custom_processing.CustomProcessor',
                        }     ,  )

                    # The code has been copied from fixtures
                    self.assertTrue(os.path.isfile(os.path.join(lowercase_     ,  'custom_feature_extraction.py'					)					)					)
                    self.assertTrue(os.path.isfile(os.path.join(lowercase_     ,  'custom_tokenization.py'					)					)					)
                    self.assertTrue(os.path.isfile(os.path.join(lowercase_     ,  'custom_processing.py'					)					)					)

                    repo.push_to_hub()

             _lowerCamelCase			:						Tuple            =AutoProcessor.from_pretrained(F'''{USER}/test-dynamic-processor'''     ,  trust_remote_code=lowercase_					)
             # Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
             self.assertEqual(new_processor.__class__.__name__     ,  'CustomProcessor'					)


