


import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase		   =			logging.get_logger(__name__)

lowerCamelCase		   =			'▁'

lowerCamelCase		   =			{'vocab_file': 'prophetnet.tokenizer'}

lowerCamelCase		   =			{
    'vocab_file': {
        'microsoft/xprophetnet-large-wiki100-cased': (
            'https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'
        ),
    }
}

lowerCamelCase		   =			{
    'microsoft/xprophetnet-large-wiki100-cased': {'do_lower_case': False},
}

lowerCamelCase		   =			{
    'microsoft/xprophetnet-large-wiki100-cased': 5_12,
}




def       a_ (       SCREAMING_SNAKE_CASE__   : Any	):
       '''simple docstring'''





       _lowerCamelCase			:						List[str]            =collections.OrderedDict()
       with open(SCREAMING_SNAKE_CASE__						,     'r'						,     encoding='utf-8'	) as reader:
              _lowerCamelCase			:						Dict            =reader.readlines()
       for index, token in enumerate(SCREAMING_SNAKE_CASE__	):
              _lowerCamelCase			:						int            =token.rstrip('\n'	)
              _lowerCamelCase			:						Tuple            =index
       return vocab




class A    (  UpperCamelCase_	):
      UpperCamelCase__				:    str =VOCAB_FILES_NAMES
      UpperCamelCase__				:    int =PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__				:    List[Any] =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__				:    int =['input_ids', 'attention_mask']






      def __init__(   self		:	Tuple     ,  lowercase_		:	Union[str, Any]     ,  lowercase_		:	List[str]="[SEP]"     ,  lowercase_		:	Any="[SEP]"     ,  lowercase_		:	Optional[int]="[SEP]"     ,  lowercase_		:	Optional[Any]="[UNK]"     ,  lowercase_		:	int="[PAD]"     ,  lowercase_		:	Union[str, Any]="[CLS]"     ,  lowercase_		:	Optional[int]="[MASK]"     ,  lowercase_		:	Optional[Dict[str, Any]] = None     ,  **lowercase_		:	str     ,  )   ->		None:
             """simple docstring"""




             _lowerCamelCase			:						int            ={} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=lowercase_     ,  eos_token=lowercase_     ,  sep_token=lowercase_     ,  unk_token=lowercase_     ,  pad_token=lowercase_     ,  cls_token=lowercase_     ,  mask_token=lowercase_     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **lowercase_     ,  )

             try:
                    import sentencepiece as spm
             except ImportError:
                    logger.warning(
                        'You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'
                        ' pip install sentencepiece'					)
                    raise

             _lowerCamelCase			:						Optional[int]            =spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(str(lowercase_					)					)
             _lowerCamelCase			:						Dict            =vocab_file

             # Original fairseq vocab and spm vocab must be "aligned":
             # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
             # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
             # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
             # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

             # put special tokens and [unused] tokens into the vocab
             _lowerCamelCase			:						int            ={'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[UNK]': 3, '[MASK]': 4}

             for i in range(10					):
                    _lowerCamelCase			:						Dict            =F'''[unused{i}]'''
                    _lowerCamelCase			:						List[str]            =5 + i

             # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
             _lowerCamelCase			:						Any            =12
             _lowerCamelCase			:						Dict            ={v: k for k, v in self.fairseq_tokens_to_ids.items()}
             for k in self.fairseq_tokens_to_ids.keys():
                    self.unique_no_split_tokens.append(lowercase_					)






      def __getstate__(   self		:	int					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =self.__dict__.copy()
             _lowerCamelCase			:						Dict            =None
             return state






      def __setstate__(   self		:	str     ,  lowercase_		:	Union[str, Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =d
             try:
                    import sentencepiece as spm
             except ImportError:
                    logger.warning(
                        'You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece'
                        ' pip install sentencepiece'					)
                    raise

             # for backward compatibility
             if not hasattr(self     ,  'sp_model_kwargs'					):
                    _lowerCamelCase			:						Any            ={}

             _lowerCamelCase			:						Optional[Any]            =spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(self.vocab_file					)






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	List[int]     ,  lowercase_		:	Optional[List[int]] = None     ,  lowercase_		:	bool = False					)   ->		List[int]:
             """simple docstring"""




             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=lowercase_     ,  token_ids_a=lowercase_     ,  already_has_special_tokens=lowercase_					)

             if token_ids_a is None:
                    return ([0] * len(lowercase_					)) + [1]
             return ([0] * len(lowercase_					)) + [1] + ([0] * len(lowercase_					)) + [1]






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	List[int]     ,  lowercase_		:	Optional[List[int]] = None					)   ->		List[int]:
             """simple docstring"""




             _lowerCamelCase			:						Any            =[self.sep_token_id]

             if token_ids_a is None:
                    return len(token_ids_a + sep					) * [0]
             return len(token_ids_a + sep + sep + token_ids_a + sep					) * [0]






      @property
      def 			lowerCamelCase			(   self		:	Any					)   ->		Optional[Any]:
             """simple docstring"""




             return len(self.sp_model					) + self.fairseq_offset






      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            ={self.convert_ids_to_tokens(lowercase_					): i for i in range(self.vocab_size					)}
             vocab.update(self.added_tokens_encoder					)
             return vocab






      def 			lowerCamelCase			(   self		:	List[Any]     ,  lowercase_		:	str					)   ->		str:
             """simple docstring"""




             return self.sp_model.encode(lowercase_     ,  out_type=lowercase_					)






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	Dict					)   ->		str:
             """simple docstring"""




             if token in self.fairseq_tokens_to_ids:
                    return self.fairseq_tokens_to_ids[token]
             _lowerCamelCase			:						Optional[int]            =self.sp_model.PieceToId(lowercase_					)

             # Need to return unknown token if the SP model returned 0
             return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






      def 			lowerCamelCase			(   self		:	str     ,  lowercase_		:	Dict					)   ->		Any:
             """simple docstring"""




             if index in self.fairseq_ids_to_tokens:
                    return self.fairseq_ids_to_tokens[index]
             return self.sp_model.IdToPiece(index - self.fairseq_offset					)






      def 			lowerCamelCase			(   self		:	Any     ,  lowercase_		:	List[str]					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						int            =''.join(lowercase_					).replace(lowercase_     ,  ' '					).strip()
             return out_string






      def 			lowerCamelCase			(   self		:	int     ,  lowercase_		:	str     ,  lowercase_		:	Optional[str] = None					)   ->		Tuple[str]:
             """simple docstring"""




             if not os.path.isdir(lowercase_					):
                    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
                    return
             _lowerCamelCase			:						Dict            =os.path.join(
                 lowercase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

             if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase_					) and os.path.isfile(self.vocab_file					):
                    copyfile(self.vocab_file     ,  lowercase_					)
             elif not os.path.isfile(self.vocab_file					):
                    with open(lowercase_     ,  'wb'					) as fi:
                           _lowerCamelCase			:						Dict            =self.sp_model.serialized_model_proto()
                           fi.write(lowercase_					)

             return (out_vocab_file,)






      def 			lowerCamelCase			(   self		:	List[str]     ,  lowercase_		:	List[int]     ,  lowercase_		:	Optional[List[int]] = None					)   ->		List[int]:
             """simple docstring"""




             if token_ids_a is None:
                    return token_ids_a + [self.sep_token_id]
             _lowerCamelCase			:						Any            =[self.sep_token_id]
             return token_ids_a + sep + token_ids_a + sep






import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


lowerCamelCase		   =			get_tests_dir('fixtures/dummy_feature_extractor_config.json')
lowerCamelCase		   =			get_tests_dir('fixtures/vocab.json')
lowerCamelCase		   =			get_tests_dir('fixtures')




class A    (  unittest.TestCase	):
      UpperCamelCase__				:    Dict =['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






      def 			lowerCamelCase			(   self		:	str					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =0






      def 			lowerCamelCase			(   self		:	str					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'					)
             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Tuple:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						int            =WavaVecaConfig()
                    _lowerCamelCase			:						Dict            =AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h'					)

                    # save in new folder
                    model_config.save_pretrained(lowercase_					)
                    processor.save_pretrained(lowercase_					)

                    _lowerCamelCase			:						Any            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Union[str, Any]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    # copy relevant files
                    copyfile(lowercase_     ,  os.path.join(lowercase_     ,  lowercase_					)					)
                    copyfile(lowercase_     ,  os.path.join(lowercase_     ,  'vocab.json'					)					)

                    _lowerCamelCase			:						Union[str, Any]            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		Optional[Any]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						List[Any]            =WavaVecaFeatureExtractor()
                    _lowerCamelCase			:						List[str]            =AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'					)

                    _lowerCamelCase			:						str            =WavaVecaProcessor(lowercase_     ,  lowercase_					)

                    # save in new folder
                    processor.save_pretrained(lowercase_					)

                    # drop `processor_class` in tokenizer
                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'r'					) as f:
                           _lowerCamelCase			:						Optional[int]            =json.load(lowercase_					)
                           config_dict.pop('processor_class'					)

                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'w'					) as f:
                           f.write(json.dumps(lowercase_					)					)

                    _lowerCamelCase			:						Optional[int]            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[int]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						Optional[Any]            =WavaVecaFeatureExtractor()
                    _lowerCamelCase			:						Tuple            =AutoTokenizer.from_pretrained('facebook/wav2vec2-base-960h'					)

                    _lowerCamelCase			:						Dict            =WavaVecaProcessor(lowercase_     ,  lowercase_					)

                    # save in new folder
                    processor.save_pretrained(lowercase_					)

                    # drop `processor_class` in feature extractor
                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'r'					) as f:
                           _lowerCamelCase			:						Union[str, Any]            =json.load(lowercase_					)
                           config_dict.pop('processor_class'					)

                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'w'					) as f:
                           f.write(json.dumps(lowercase_					)					)

                    _lowerCamelCase			:						Optional[int]            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		List[str]:
             """simple docstring"""




             with tempfile.TemporaryDirectory() as tmpdirname:
                    _lowerCamelCase			:						Optional[Any]            =WavaVecaConfig(processor_class='Wav2Vec2Processor'					)
                    model_config.save_pretrained(lowercase_					)
                    # copy relevant files
                    copyfile(lowercase_     ,  os.path.join(lowercase_     ,  'vocab.json'					)					)
                    # create emtpy sample processor
                    with open(os.path.join(lowercase_     ,  lowercase_					)     ,  'w'					) as f:
                           f.write('{}'					)

                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained(lowercase_					)

             self.assertIsInstance(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Optional[int]:
             """simple docstring"""




             with self.assertRaises(lowercase_					):
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'					)
             # If remote code is disabled, we can't load this config.
             with self.assertRaises(lowercase_					):
                    _lowerCamelCase			:						Union[str, Any]            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)

             _lowerCamelCase			:						List[str]            =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)
             self.assertTrue(processor.special_attribute_present					)
             self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)

             _lowerCamelCase			:						int            =processor.feature_extractor
             self.assertTrue(feature_extractor.special_attribute_present					)
             self.assertEqual(feature_extractor.__class__.__name__     ,  'NewFeatureExtractor'					)

             _lowerCamelCase			:						Optional[int]            =processor.tokenizer
             self.assertTrue(tokenizer.special_attribute_present					)
             if is_tokenizers_available():
                    self.assertEqual(tokenizer.__class__.__name__     ,  'NewTokenizerFast'					)

                    # Test we can also load the slow version
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_     ,  use_fast=lowercase_					)
                    _lowerCamelCase			:						Optional[int]            =new_processor.tokenizer
                    self.assertTrue(new_tokenizer.special_attribute_present					)
                    self.assertEqual(new_tokenizer.__class__.__name__     ,  'NewTokenizer'					)
             else:
                    self.assertEqual(tokenizer.__class__.__name__     ,  'NewTokenizer'					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		Optional[int]:
             """simple docstring"""




             try:
                    AutoConfig.register('custom'     ,  lowercase_					)
                    AutoFeatureExtractor.register(lowercase_     ,  lowercase_					)
                    AutoTokenizer.register(lowercase_     ,  slow_tokenizer_class=lowercase_					)
                    AutoProcessor.register(lowercase_     ,  lowercase_					)
                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(lowercase_					):
                           AutoProcessor.register(lowercase_     ,  lowercase_					)

                    # Now that the config is registered, it can be used as any other config with the auto-API
                    _lowerCamelCase			:						str            =CustomFeatureExtractor.from_pretrained(lowercase_					)

                    with tempfile.TemporaryDirectory() as tmp_dir:
                           _lowerCamelCase			:						str            =os.path.join(lowercase_     ,  'vocab.txt'					)
                           with open(lowercase_     ,  'w'     ,  encoding='utf-8'					) as vocab_writer:
                                  vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
                           _lowerCamelCase			:						List[Any]            =CustomTokenizer(lowercase_					)

                    _lowerCamelCase			:						Optional[int]            =CustomProcessor(lowercase_     ,  lowercase_					)

                    with tempfile.TemporaryDirectory() as tmp_dir:
                           processor.save_pretrained(lowercase_					)
                           _lowerCamelCase			:						List[Any]            =AutoProcessor.from_pretrained(lowercase_					)
                           self.assertIsInstance(lowercase_     ,  lowercase_					)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in PROCESSOR_MAPPING._extra_content:
                           del PROCESSOR_MAPPING._extra_content[CustomConfig]






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		List[str]:
             """simple docstring"""




             class A    (  UpperCamelCase_	):
                   UpperCamelCase__				:    Optional[Any] =False




             class A    (  UpperCamelCase_	):
                   UpperCamelCase__				:    int =False




             class A    (  UpperCamelCase_	):
                   UpperCamelCase__				:    Union[str, Any] ='AutoFeatureExtractor'
                   UpperCamelCase__				:    str ='AutoTokenizer'
                   UpperCamelCase__				:    List[Any] =False

             try:
                    AutoConfig.register('custom'     ,  lowercase_					)
                    AutoFeatureExtractor.register(lowercase_     ,  lowercase_					)
                    AutoTokenizer.register(lowercase_     ,  slow_tokenizer_class=lowercase_					)
                    AutoProcessor.register(lowercase_     ,  lowercase_					)
                    # If remote code is not set, the default is to use local classes.
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained('hf-internal-testing/test_dynamic_processor'					)
                    self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)
                    self.assertFalse(processor.special_attribute_present					)
                    self.assertFalse(processor.feature_extractor.special_attribute_present					)
                    self.assertFalse(processor.tokenizer.special_attribute_present					)

                    # If remote code is disabled, we load the local ones.
                    _lowerCamelCase			:						int            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)
                    self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)
                    self.assertFalse(processor.special_attribute_present					)
                    self.assertFalse(processor.feature_extractor.special_attribute_present					)
                    self.assertFalse(processor.tokenizer.special_attribute_present					)

                    # If remote is enabled, we load from the Hub.
                    _lowerCamelCase			:						str            =AutoProcessor.from_pretrained(
                        'hf-internal-testing/test_dynamic_processor'     ,  trust_remote_code=lowercase_					)
                    self.assertEqual(processor.__class__.__name__     ,  'NewProcessor'					)
                    self.assertTrue(processor.special_attribute_present					)
                    self.assertTrue(processor.feature_extractor.special_attribute_present					)
                    self.assertTrue(processor.tokenizer.special_attribute_present					)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]
                    if CustomConfig in PROCESSOR_MAPPING._extra_content:
                           del PROCESSOR_MAPPING._extra_content[CustomConfig]






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-bert'					)
             self.assertEqual(processor.__class__.__name__     ,  'BertTokenizerFast'					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Any            =AutoProcessor.from_pretrained('hf-internal-testing/tiny-random-convnext'					)
             self.assertEqual(processor.__class__.__name__     ,  'ConvNextImageProcessor'					)




@is_staging_test
class A    (  unittest.TestCase	):
      UpperCamelCase__				:    List[Any] =['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






      @classmethod
      def 			lowerCamelCase			(   cls		:	int					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =TOKEN
             HfFolder.save_token(lowercase_					)






      @classmethod
      def 			lowerCamelCase			(   cls		:	Optional[int]					)   ->		Union[str, Any]:
             """simple docstring"""




             try:
                    delete_repo(token=cls._token     ,  repo_id='test-processor'					)
             except HTTPError:
                    pass

             try:
                    delete_repo(token=cls._token     ,  repo_id='valid_org/test-processor-org'					)
             except HTTPError:
                    pass

             try:
                    delete_repo(token=cls._token     ,  repo_id='test-dynamic-processor'					)
             except HTTPError:
                    pass






      def 			lowerCamelCase			(   self		:	str					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						Tuple            =WavaVecaProcessor.from_pretrained(lowercase_					)
             with tempfile.TemporaryDirectory() as tmp_dir:
                    processor.save_pretrained(
                        os.path.join(lowercase_     ,  'test-processor'					)     ,  push_to_hub=lowercase_     ,  use_auth_token=self._token					)

                    _lowerCamelCase			:						Union[str, Any]            =WavaVecaProcessor.from_pretrained(F'''{USER}/test-processor'''					)
                    for k, v in processor.feature_extractor.__dict__.items():
                           self.assertEqual(lowercase_     ,  getattr(new_processor.feature_extractor     ,  lowercase_					)					)
                    self.assertDictEqual(new_processor.tokenizer.get_vocab()     ,  processor.tokenizer.get_vocab()					)






      def 			lowerCamelCase			(   self		:	str					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						int            =WavaVecaProcessor.from_pretrained(lowercase_					)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    processor.save_pretrained(
                        os.path.join(lowercase_     ,  'test-processor-org'					)     ,  push_to_hub=lowercase_     ,  use_auth_token=self._token     ,  organization='valid_org'     ,  )

                    _lowerCamelCase			:						str            =WavaVecaProcessor.from_pretrained('valid_org/test-processor-org'					)
                    for k, v in processor.feature_extractor.__dict__.items():
                           self.assertEqual(lowercase_     ,  getattr(new_processor.feature_extractor     ,  lowercase_					)					)
                    self.assertDictEqual(new_processor.tokenizer.get_vocab()     ,  processor.tokenizer.get_vocab()					)






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		str:
             """simple docstring"""




             CustomFeatureExtractor.register_for_auto_class()
             CustomTokenizer.register_for_auto_class()
             CustomProcessor.register_for_auto_class()

             _lowerCamelCase			:						Optional[Any]            =CustomFeatureExtractor.from_pretrained(lowercase_					)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    _lowerCamelCase			:						Dict            =os.path.join(lowercase_     ,  'vocab.txt'					)
                    with open(lowercase_     ,  'w'     ,  encoding='utf-8'					) as vocab_writer:
                           vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
                    _lowerCamelCase			:						Any            =CustomTokenizer(lowercase_					)

             _lowerCamelCase			:						List[Any]            =CustomProcessor(lowercase_     ,  lowercase_					)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    create_repo(F'''{USER}/test-dynamic-processor'''     ,  token=self._token					)
                    _lowerCamelCase			:						List[str]            =Repository(lowercase_     ,  clone_from=F'''{USER}/test-dynamic-processor'''     ,  token=self._token					)
                    processor.save_pretrained(lowercase_					)

                    # This has added the proper auto_map field to the feature extractor config
                    self.assertDictEqual(
                        processor.feature_extractor.auto_map     ,  {
                            'AutoFeatureExtractor': 'custom_feature_extraction.CustomFeatureExtractor',
                            'AutoProcessor': 'custom_processing.CustomProcessor',
                        }     ,  )

                    # This has added the proper auto_map field to the tokenizer config
                    with open(os.path.join(lowercase_     ,  'tokenizer_config.json'					)					) as f:
                           _lowerCamelCase			:						Union[str, Any]            =json.load(lowercase_					)
                    self.assertDictEqual(
                        tokenizer_config['auto_map']     ,  {
                            'AutoTokenizer': ['custom_tokenization.CustomTokenizer', None],
                            'AutoProcessor': 'custom_processing.CustomProcessor',
                        }     ,  )

                    # The code has been copied from fixtures
                    self.assertTrue(os.path.isfile(os.path.join(lowercase_     ,  'custom_feature_extraction.py'					)					)					)
                    self.assertTrue(os.path.isfile(os.path.join(lowercase_     ,  'custom_tokenization.py'					)					)					)
                    self.assertTrue(os.path.isfile(os.path.join(lowercase_     ,  'custom_processing.py'					)					)					)

                    repo.push_to_hub()

             _lowerCamelCase			:						Tuple            =AutoProcessor.from_pretrained(F'''{USER}/test-dynamic-processor'''     ,  trust_remote_code=lowercase_					)
             # Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
             self.assertEqual(new_processor.__class__.__name__     ,  'CustomProcessor'					)


