

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase  :      int			 ="""▁"""

_lowerCAmelCase  :      int			 ={"""vocab_file""": """spiece.model"""}

_lowerCAmelCase  :      List[Any]			 ={
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""}
}

_lowerCAmelCase  :      str			 ={
    """google/pegasus-xsum""": 5_12,
}


_lowerCAmelCase  :      Optional[Any]			 =logging.get_logger(__name__)

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   VOCAB_FILES_NAMES

					__magic_name__        =   VOCAB_FILES_NAMES
					__magic_name__        =   PRETRAINED_VOCAB_FILES_MAP
					__magic_name__        =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__magic_name__        =   ["input_ids", "attention_mask"]





					def __init__(				self				,      lowerCamelCase__				,      lowerCamelCase__="<pad>"				,      lowerCamelCase__="</s>"				,      lowerCamelCase__="<unk>"				,      lowerCamelCase__="<mask_2>"				,      lowerCamelCase__="<mask_1>"				,      lowerCamelCase__=None				,      lowerCamelCase__=1_0_3				,      lowerCamelCase__ = None				,      **lowerCamelCase__				,      ):
						UpperCAmelCase__:	Tuple									= offset
						if additional_special_tokens is not None:
							if not isinstance(lowerCamelCase__				,      lowerCamelCase__						):
								raise TypeError(
								    F"additional_special_tokens should be of type {type(lowerCamelCase__						)}, but is"
								    F" {type(lowerCamelCase__						)}"						)

							UpperCAmelCase__:	List[str]									= (
							    ([mask_token_sent] + additional_special_tokens)
							    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
							    else additional_special_tokens
							)
							# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
							additional_special_tokens_extended += [
							    F"<unk_{i}>" for i in range(len(lowerCamelCase__						)				,      self.offset - 1						)
							]

							if len(set(lowerCamelCase__						)						) != len(lowerCamelCase__						):
								raise ValueError(
								    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
								    F" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."						)
							UpperCAmelCase__:	int									= additional_special_tokens_extended
						else:
							UpperCAmelCase__:	Any									= [mask_token_sent] if mask_token_sent is not None else []
							additional_special_tokens += [F"<unk_{i}>" for i in range(2				,      self.offset						)]

						UpperCAmelCase__:	List[str]									= {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=lowerCamelCase__				,      unk_token=lowerCamelCase__				,      mask_token=lowerCamelCase__				,      pad_token=lowerCamelCase__				,      mask_token_sent=lowerCamelCase__				,      offset=lowerCamelCase__				,      additional_special_tokens=lowerCamelCase__				,      sp_model_kwargs=self.sp_model_kwargs				,      **lowerCamelCase__				,      )
						UpperCAmelCase__:	Optional[Any]									= mask_token_sent
						UpperCAmelCase__:	str									= vocab_file
						UpperCAmelCase__:	Any									= spm.SentencePieceProcessor(**self.sp_model_kwargs						)
						self.sp_model.Load(lowerCamelCase__						)

						# add special tokens to encoder dict
						UpperCAmelCase__:	Dict[int, str]									= {
						    0: self.pad_token,
						    1: self.eos_token,
						}

						if self.mask_token_sent is not None:
							self.encoder.update(
							    {
							        2: self.mask_token_sent,
							        3: self.mask_token,
							    }						)

						if self.offset > 0:
							# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
							# mask_token_sent is already added to list -> so start at 1
							self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1				,      self.offset - 1						)}						)

						UpperCAmelCase__:	Dict[str, int]									= {v: k for k, v in self.encoder.items()}





					@property
					def    _UpperCAmelCase    (				self						):
						return len(self.sp_model						) + self.offset





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Any									= {self.convert_ids_to_tokens(lowerCamelCase__						): i for i in range(self.vocab_size						)}
						vocab.update(self.added_tokens_encoder						)
						return vocab





					def __getstate__(				self						):
						UpperCAmelCase__:	Optional[int]									= self.__dict__.copy()
						UpperCAmelCase__:	Any									= None
						return state





					def __setstate__(				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	List[Any]									= d

						# for backward compatibility
						if not hasattr(self				,      "sp_model_kwargs"						):
							UpperCAmelCase__:	List[str]									= {}

						UpperCAmelCase__:	str									= spm.SentencePieceProcessor(**self.sp_model_kwargs						)
						self.sp_model.Load(self.vocab_file						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						return self.sp_model.encode(lowerCamelCase__				,      out_type=lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						if token in self.decoder:
							return self.decoder[token]
						elif token in self.added_tokens_decoder:
							return self.added_tokens_decoder[token]
						UpperCAmelCase__:	Union[str, Any]									= self.sp_model.piece_to_id(lowerCamelCase__						)
						return sp_id + self.offset





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						if index in self.encoder:
							return self.encoder[index]
						elif index in self.added_tokens_encoder:
							return self.added_tokens_encoder[index]
						else:
							UpperCAmelCase__:	Optional[Any]									= self.sp_model.IdToPiece(index - self.offset						)
						return token





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	Union[str, Any]									= []
						UpperCAmelCase__:	List[str]									= ""
						for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
								out_string += self.sp_model.decode(lowerCamelCase__						) + token
								UpperCAmelCase__:	int									= []
							else:
								current_sub_tokens.append(lowerCamelCase__						)
						out_string += self.sp_model.decode(lowerCamelCase__						)
						return out_string.strip()





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__=False						):
						return 1





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	Dict									= set(self.all_special_ids						)  # call it once instead of inside list comp
						all_special_ids.remove(self.unk_token_id						)  # <unk> is only sometimes special

						return [1 if x in all_special_ids else 0 for x in seq]





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__ = None				,      lowerCamelCase__ = False						):
						if already_has_special_tokens:
							return self._special_token_mask(lowerCamelCase__						)
						elif token_ids_a is None:
							return self._special_token_mask(lowerCamelCase__						) + [1]
						else:
							return self._special_token_mask(token_ids_a + token_ids_a						) + [1]





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__=None						):
						if token_ids_a is None:
							return token_ids_a + [self.eos_token_id]
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return token_ids_a + token_ids_a + [self.eos_token_id]





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__ = None						):
						if not os.path.isdir(lowerCamelCase__						):
							logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
							return
						UpperCAmelCase__:	Union[str, Any]									= os.path.join(
						    lowerCamelCase__				,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

						if os.path.abspath(self.vocab_file						) != os.path.abspath(lowerCamelCase__						) and os.path.isfile(self.vocab_file						):
							copyfile(self.vocab_file				,      lowerCamelCase__						)
						elif not os.path.isfile(self.vocab_file						):
							with open(lowerCamelCase__				,      "wb"						) as fi:
								UpperCAmelCase__:	Union[str, Any]									= self.sp_model.serialized_model_proto()
								fi.write(lowerCamelCase__						)

						return (out_vocab_file,)


import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_lowerCAmelCase  :      List[str]			 =get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
_lowerCAmelCase  :      List[str]			 =get_tests_dir("""fixtures/vocab.json""")
_lowerCAmelCase  :      Dict			 =get_tests_dir("""fixtures""")

class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Union[str, Any]									= 0





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Optional[int]									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)
						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Any									= WavaVecaConfig()
							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)

							# save in new folder
							model_config.save_pretrained(lowerCamelCase__						)
							processor.save_pretrained(lowerCamelCase__						)

							UpperCAmelCase__:	int									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							# copy relevant files
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      lowerCamelCase__						)						)
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      "vocab.json"						)						)

							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Optional[Any]									= WavaVecaFeatureExtractor()
							UpperCAmelCase__:	List[Any]									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

							UpperCAmelCase__:	Dict									= WavaVecaProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							# save in new folder
							processor.save_pretrained(lowerCamelCase__						)

							# drop `processor_class` in tokenizer
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "r"						) as f:
								UpperCAmelCase__:	Optional[int]									= json.load(lowerCamelCase__						)
								config_dict.pop("processor_class"						)

							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write(json.dumps(lowerCamelCase__						)						)

							UpperCAmelCase__:	Any									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	str									= WavaVecaFeatureExtractor()
							UpperCAmelCase__:	Tuple									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

							UpperCAmelCase__:	List[Any]									= WavaVecaProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							# save in new folder
							processor.save_pretrained(lowerCamelCase__						)

							# drop `processor_class` in feature extractor
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "r"						) as f:
								UpperCAmelCase__:	str									= json.load(lowerCamelCase__						)
								config_dict.pop("processor_class"						)

							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write(json.dumps(lowerCamelCase__						)						)

							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Union[str, Any]									= WavaVecaConfig(processor_class="Wav2Vec2Processor"						)
							model_config.save_pretrained(lowerCamelCase__						)
							# copy relevant files
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      "vocab.json"						)						)
							# create emtpy sample processor
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write("{}"						)

							UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						# If remote code is not set, we will time out when asking whether to load the model.
						with self.assertRaises(lowerCamelCase__						):
							UpperCAmelCase__:	Optional[Any]									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
						# If remote code is disabled, we can't load this config.
						with self.assertRaises(lowerCamelCase__						):
							UpperCAmelCase__:	Optional[int]									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)

						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
						self.assertTrue(processor.special_attribute_present						)
						self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)

						UpperCAmelCase__:	Dict									= processor.feature_extractor
						self.assertTrue(feature_extractor.special_attribute_present						)
						self.assertEqual(feature_extractor.__class__.__name__				,      "NewFeatureExtractor"						)

						UpperCAmelCase__:	Any									= processor.tokenizer
						self.assertTrue(tokenizer.special_attribute_present						)
						if is_tokenizers_available():
							self.assertEqual(tokenizer.__class__.__name__				,      "NewTokenizerFast"						)

							# Test we can also load the slow version
							UpperCAmelCase__:	Dict									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__				,      use_fast=lowerCamelCase__						)
							UpperCAmelCase__:	str									= new_processor.tokenizer
							self.assertTrue(new_tokenizer.special_attribute_present						)
							self.assertEqual(new_tokenizer.__class__.__name__				,      "NewTokenizer"						)
						else:
							self.assertEqual(tokenizer.__class__.__name__				,      "NewTokenizer"						)





					def    _UpperCAmelCase    (				self						):
						try:
							AutoConfig.register("custom"				,      lowerCamelCase__						)
							AutoFeatureExtractor.register(lowerCamelCase__				,      lowerCamelCase__						)
							AutoTokenizer.register(lowerCamelCase__				,      slow_tokenizer_class=lowerCamelCase__						)
							AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)
							# Trying to register something existing in the Transformers library will raise an error
							with self.assertRaises(lowerCamelCase__						):
								AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)

							# Now that the config is registered, it can be used as any other config with the auto-API
							UpperCAmelCase__:	Dict									= CustomFeatureExtractor.from_pretrained(lowerCamelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								UpperCAmelCase__:	Tuple									= os.path.join(lowerCamelCase__				,      "vocab.txt"						)
								with open(lowerCamelCase__				,      "w"				,      encoding="utf-8"						) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
								UpperCAmelCase__:	str									= CustomTokenizer(lowerCamelCase__						)

							UpperCAmelCase__:	Optional[int]									= CustomProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								processor.save_pretrained(lowerCamelCase__						)
								UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)
								self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)

						finally:
							if "custom" in CONFIG_MAPPING._extra_content:
								del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
								del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
								del PROCESSOR_MAPPING._extra_content[CustomConfig]





					def    _UpperCAmelCase    (				self						):

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   False

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   False

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   "AutoFeatureExtractor"
											__magic_name__        =   "AutoTokenizer"
											__magic_name__        =   False

						try:
							AutoConfig.register("custom"				,      lowerCamelCase__						)
							AutoFeatureExtractor.register(lowerCamelCase__				,      lowerCamelCase__						)
							AutoTokenizer.register(lowerCamelCase__				,      slow_tokenizer_class=lowerCamelCase__						)
							AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)
							# If remote code is not set, the default is to use local classes.
							UpperCAmelCase__:	Any									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote code is disabled, we load the local ones.
							UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote is enabled, we load from the Hub.
							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertTrue(processor.special_attribute_present						)
							self.assertTrue(processor.feature_extractor.special_attribute_present						)
							self.assertTrue(processor.tokenizer.special_attribute_present						)

						finally:
							if "custom" in CONFIG_MAPPING._extra_content:
								del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
								del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
								del PROCESSOR_MAPPING._extra_content[CustomConfig]





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert"						)
						self.assertEqual(processor.__class__.__name__				,      "BertTokenizerFast"						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext"						)
						self.assertEqual(processor.__class__.__name__				,      "ConvNextImageProcessor"						)

@is_staging_test
class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						UpperCAmelCase__:	Dict									= TOKEN
						HfFolder.save_token(lowerCamelCase__						)





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						try:
							delete_repo(token=cls._token				,      repo_id="test-processor"						)
						except HTTPError:
							pass

						try:
							delete_repo(token=cls._token				,      repo_id="valid_org/test-processor-org"						)
						except HTTPError:
							pass

						try:
							delete_repo(token=cls._token				,      repo_id="test-dynamic-processor"						)
						except HTTPError:
							pass





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Optional[Any]									= WavaVecaProcessor.from_pretrained(lowerCamelCase__						)
						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(lowerCamelCase__				,      "test-processor"						)				,      push_to_hub=lowerCamelCase__				,      use_auth_token=self._token						)

							UpperCAmelCase__:	Union[str, Any]									= WavaVecaProcessor.from_pretrained(F"{USER}/test-processor"						)
							for k, v in processor.feature_extractor.__dict__.items():
								self.assertEqual(lowerCamelCase__				,      getattr(new_processor.feature_extractor				,      lowerCamelCase__						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()				,      processor.tokenizer.get_vocab()						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Dict									= WavaVecaProcessor.from_pretrained(lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(lowerCamelCase__				,      "test-processor-org"						)				,      push_to_hub=lowerCamelCase__				,      use_auth_token=self._token				,      organization="valid_org"				,      )

							UpperCAmelCase__:	Optional[int]									= WavaVecaProcessor.from_pretrained("valid_org/test-processor-org"						)
							for k, v in processor.feature_extractor.__dict__.items():
								self.assertEqual(lowerCamelCase__				,      getattr(new_processor.feature_extractor				,      lowerCamelCase__						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()				,      processor.tokenizer.get_vocab()						)





					def    _UpperCAmelCase    (				self						):
						CustomFeatureExtractor.register_for_auto_class()
						CustomTokenizer.register_for_auto_class()
						CustomProcessor.register_for_auto_class()

						UpperCAmelCase__:	List[str]									= CustomFeatureExtractor.from_pretrained(lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							UpperCAmelCase__:	Optional[int]									= os.path.join(lowerCamelCase__				,      "vocab.txt"						)
							with open(lowerCamelCase__				,      "w"				,      encoding="utf-8"						) as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
							UpperCAmelCase__:	Optional[Any]									= CustomTokenizer(lowerCamelCase__						)

						UpperCAmelCase__:	str									= CustomProcessor(lowerCamelCase__				,      lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							create_repo(F"{USER}/test-dynamic-processor"				,      token=self._token						)
							UpperCAmelCase__:	int									= Repository(lowerCamelCase__				,      clone_from=F"{USER}/test-dynamic-processor"				,      token=self._token						)
							processor.save_pretrained(lowerCamelCase__						)

							# This has added the proper auto_map field to the feature extractor config
							self.assertDictEqual(
							    processor.feature_extractor.auto_map				,      {
							        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
							        "AutoProcessor": "custom_processing.CustomProcessor",
							    }				,      )

							# This has added the proper auto_map field to the tokenizer config
							with open(os.path.join(lowerCamelCase__				,      "tokenizer_config.json"						)						) as f:
								UpperCAmelCase__:	Tuple									= json.load(lowerCamelCase__						)
							self.assertDictEqual(
							    tokenizer_config["auto_map"]				,      {
							        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
							        "AutoProcessor": "custom_processing.CustomProcessor",
							    }				,      )

							# The code has been copied from fixtures
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_feature_extraction.py"						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_tokenization.py"						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_processing.py"						)						)						)

							repo.push_to_hub()

						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(F"{USER}/test-dynamic-processor"				,      trust_remote_code=lowerCamelCase__						)
						# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
						self.assertEqual(new_processor.__class__.__name__				,      "CustomProcessor"						)