

from typing import Dict

import numpy as np

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, GenericTensor, Pipeline, PipelineException


if is_tf_available():
					import tensorflow as tf

					from ..tf_utils import stable_softmax


if is_torch_available():
					import torch


_lowerCAmelCase  :      List[Any]			 =logging.get_logger(__name__)

@add_end_docstrings(
    _a					,r"\n        top_k (`int`, defaults to 5):\n            The number of predictions to return.\n        targets (`str` or `List[str]`, *optional*):\n            When passed, the model will limit the scores to the passed targets instead of looking up in the whole\n            vocab. If the provided targets are not in the model vocab, they will be tokenized and the first resulting\n            token will be used (with a warning, and that might be slower).\n\n    "					,)
class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						if self.framework == "tf":
							UpperCAmelCase__:	Tuple									= tf.where(input_ids == self.tokenizer.mask_token_id						).numpy()
						elif self.framework == "pt":
							UpperCAmelCase__:	Any									= torch.nonzero(input_ids == self.tokenizer.mask_token_id				,      as_tuple=lowerCamelCase__						)
						else:
							raise ValueError("Unsupported framework"						)
						return masked_index





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	List[Any]									= self.get_masked_index(lowerCamelCase__						)
						UpperCAmelCase__:	Optional[int]									= np.prod(masked_index.shape						)
						if numel < 1:
							raise PipelineException(
							    "fill-mask"				,      self.model.base_model_prefix				,      F"No mask_token ({self.tokenizer.mask_token}) found on the input"				,      )





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						if isinstance(lowerCamelCase__				,      lowerCamelCase__						):
							for model_input in model_inputs:
								self._ensure_exactly_one_mask_token(model_input["input_ids"][0]						)
						else:
							for input_ids in model_inputs["input_ids"]:
								self._ensure_exactly_one_mask_token(lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__=None				,      **lowerCamelCase__						):
						if return_tensors is None:
							UpperCAmelCase__:	Optional[int]									= self.framework
						UpperCAmelCase__:	Tuple									= self.tokenizer(lowerCamelCase__				,      return_tensors=lowerCamelCase__						)
						self.ensure_exactly_one_mask_token(lowerCamelCase__						)
						return model_inputs





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__						):
						UpperCAmelCase__:	Any									= self.model(**lowerCamelCase__						)
						UpperCAmelCase__:	List[str]									= model_inputs["input_ids"]
						return model_outputs





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__=5				,      lowerCamelCase__=None						):
						# Cap top_k if there are targets
						if target_ids is not None and target_ids.shape[0] < top_k:
							UpperCAmelCase__:	Union[str, Any]									= target_ids.shape[0]
						UpperCAmelCase__:	Tuple									= model_outputs["input_ids"][0]
						UpperCAmelCase__:	Tuple									= model_outputs["logits"]

						if self.framework == "tf":
							UpperCAmelCase__:	List[str]									= tf.where(input_ids == self.tokenizer.mask_token_id						).numpy()[:, 0]

							UpperCAmelCase__:	Tuple									= outputs.numpy()

							UpperCAmelCase__:	int									= outputs[0, masked_index, :]
							UpperCAmelCase__:	Optional[Any]									= stable_softmax(lowerCamelCase__				,      axis=-1						)
							if target_ids is not None:
								UpperCAmelCase__:	List[Any]									= tf.gather_nd(tf.squeeze(lowerCamelCase__				,      0						)				,      target_ids.reshape(-1				,      1						)						)
								UpperCAmelCase__:	Any									= tf.expand_dims(lowerCamelCase__				,      0						)

							UpperCAmelCase__:	List[Any]									= tf.math.top_k(lowerCamelCase__				,      k=lowerCamelCase__						)
							UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= topk.values.numpy(), topk.indices.numpy()
						else:
							UpperCAmelCase__:	Any									= torch.nonzero(input_ids == self.tokenizer.mask_token_id				,      as_tuple=lowerCamelCase__						).squeeze(-1						)
							# Fill mask pipeline supports only one ${mask_token} per sample

							UpperCAmelCase__:	List[Any]									= outputs[0, masked_index, :]
							UpperCAmelCase__:	List[Any]									= logits.softmax(dim=-1						)
							if target_ids is not None:
								UpperCAmelCase__:	List[Any]									= probs[..., target_ids]

							UpperCAmelCase__							,						UpperCAmelCase__:	Optional[int]									= probs.topk(lowerCamelCase__						)

						UpperCAmelCase__:	Tuple									= []
						UpperCAmelCase__:	Union[str, Any]									= values.shape[0] == 1
						for i, (_values, _predictions) in enumerate(zip(values.tolist()				,      predictions.tolist()						)						):
							UpperCAmelCase__:	str									= []
							for v, p in zip(_values				,      _predictions						):
								# Copy is important since we're going to modify this array in place
								UpperCAmelCase__:	List[Any]									= input_ids.numpy().copy()
								if target_ids is not None:
									UpperCAmelCase__:	str									= target_ids[p].tolist()

								UpperCAmelCase__:	Union[str, Any]									= p
								# Filter padding out:
								UpperCAmelCase__:	List[str]									= tokens[np.where(tokens != self.tokenizer.pad_token_id						)]
								# Originally we skip special tokens to give readable output.
								# For multi masks though, the other [MASK] would be removed otherwise
								# making the output look odd, so we add them back
								UpperCAmelCase__:	int									= self.tokenizer.decode(lowerCamelCase__				,      skip_special_tokens=lowerCamelCase__						)
								UpperCAmelCase__:	List[Any]									= {"score": v, "token": p, "token_str": self.tokenizer.decode([p]						), "sequence": sequence}
								row.append(lowerCamelCase__						)
							result.append(lowerCamelCase__						)
						if single_mask:
							return result[0]
						return result





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__=None						):
						if isinstance(lowerCamelCase__				,      lowerCamelCase__						):
							UpperCAmelCase__:	Any									= [targets]
						try:
							UpperCAmelCase__:	Dict									= self.tokenizer.get_vocab()
						except Exception:
							UpperCAmelCase__:	Optional[int]									= {}
						UpperCAmelCase__:	List[str]									= []
						for target in targets:
							UpperCAmelCase__:	int									= vocab.get(lowerCamelCase__				,      lowerCamelCase__						)
							if id_ is None:
								UpperCAmelCase__:	int									= self.tokenizer(
								    lowerCamelCase__				,      add_special_tokens=lowerCamelCase__				,      return_attention_mask=lowerCamelCase__				,      return_token_type_ids=lowerCamelCase__				,      max_length=1				,      truncation=lowerCamelCase__				,      )["input_ids"]
								if len(lowerCamelCase__						) == 0:
									logger.warning(
									    F"The specified target token `{target}` does not exist in the model vocabulary. "
									    "We cannot replace it with anything meaningful, ignoring it"						)
									continue
								UpperCAmelCase__:	Union[str, Any]									= input_ids[0]
								# XXX: If users encounter this pass
								# it becomes pretty slow, so let's make sure
								# The warning enables them to fix the input to
								# get faster performance.
								logger.warning(
								    F"The specified target token `{target}` does not exist in the model vocabulary. "
								    F"Replacing with `{self.tokenizer.convert_ids_to_tokens(id_						)}`."						)
							target_ids.append(id_						)
						UpperCAmelCase__:	Union[str, Any]									= list(set(lowerCamelCase__						)						)
						if len(lowerCamelCase__						) == 0:
							raise ValueError("At least one target must be provided when passed."						)
						UpperCAmelCase__:	Optional[Any]									= np.array(lowerCamelCase__						)
						return target_ids





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__=None				,      lowerCamelCase__=None						):
						UpperCAmelCase__:	Union[str, Any]									= {}

						if targets is not None:
							UpperCAmelCase__:	str									= self.get_target_ids(lowerCamelCase__				,      lowerCamelCase__						)
							UpperCAmelCase__:	int									= target_ids

						if top_k is not None:
							UpperCAmelCase__:	Any									= top_k

						if self.tokenizer.mask_token_id is None:
							raise PipelineException(
							    "fill-mask"				,      self.model.base_model_prefix				,      "The tokenizer does not define a `mask_token`."						)
						return {}, {}, postprocess_params





					def __call__(				self				,      lowerCamelCase__				,      *lowerCamelCase__				,      **lowerCamelCase__						):
						UpperCAmelCase__:	Optional[int]									= super().__call__(lowerCamelCase__				,      **lowerCamelCase__						)
						if isinstance(lowerCamelCase__				,      lowerCamelCase__						) and len(lowerCamelCase__						) == 1:
							return outputs[0]
						return outputs


import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


_lowerCAmelCase  :      List[str]			 =get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
_lowerCAmelCase  :      List[str]			 =get_tests_dir("""fixtures/vocab.json""")
_lowerCAmelCase  :      Dict			 =get_tests_dir("""fixtures""")

class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Union[str, Any]									= 0





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Optional[int]									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)
						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Any									= WavaVecaConfig()
							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h"						)

							# save in new folder
							model_config.save_pretrained(lowerCamelCase__						)
							processor.save_pretrained(lowerCamelCase__						)

							UpperCAmelCase__:	int									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							# copy relevant files
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      lowerCamelCase__						)						)
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      "vocab.json"						)						)

							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Optional[Any]									= WavaVecaFeatureExtractor()
							UpperCAmelCase__:	List[Any]									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

							UpperCAmelCase__:	Dict									= WavaVecaProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							# save in new folder
							processor.save_pretrained(lowerCamelCase__						)

							# drop `processor_class` in tokenizer
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "r"						) as f:
								UpperCAmelCase__:	Optional[int]									= json.load(lowerCamelCase__						)
								config_dict.pop("processor_class"						)

							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write(json.dumps(lowerCamelCase__						)						)

							UpperCAmelCase__:	Any									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	str									= WavaVecaFeatureExtractor()
							UpperCAmelCase__:	Tuple									= AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h"						)

							UpperCAmelCase__:	List[Any]									= WavaVecaProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							# save in new folder
							processor.save_pretrained(lowerCamelCase__						)

							# drop `processor_class` in feature extractor
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "r"						) as f:
								UpperCAmelCase__:	str									= json.load(lowerCamelCase__						)
								config_dict.pop("processor_class"						)

							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write(json.dumps(lowerCamelCase__						)						)

							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						with tempfile.TemporaryDirectory() as tmpdirname:
							UpperCAmelCase__:	Union[str, Any]									= WavaVecaConfig(processor_class="Wav2Vec2Processor"						)
							model_config.save_pretrained(lowerCamelCase__						)
							# copy relevant files
							copyfile(lowerCamelCase__				,      os.path.join(lowerCamelCase__				,      "vocab.json"						)						)
							# create emtpy sample processor
							with open(os.path.join(lowerCamelCase__				,      lowerCamelCase__						)				,      "w"						) as f:
								f.write("{}"						)

							UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(lowerCamelCase__						)

						self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						# If remote code is not set, we will time out when asking whether to load the model.
						with self.assertRaises(lowerCamelCase__						):
							UpperCAmelCase__:	Optional[Any]									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
						# If remote code is disabled, we can't load this config.
						with self.assertRaises(lowerCamelCase__						):
							UpperCAmelCase__:	Optional[int]									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)

						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
						self.assertTrue(processor.special_attribute_present						)
						self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)

						UpperCAmelCase__:	Dict									= processor.feature_extractor
						self.assertTrue(feature_extractor.special_attribute_present						)
						self.assertEqual(feature_extractor.__class__.__name__				,      "NewFeatureExtractor"						)

						UpperCAmelCase__:	Any									= processor.tokenizer
						self.assertTrue(tokenizer.special_attribute_present						)
						if is_tokenizers_available():
							self.assertEqual(tokenizer.__class__.__name__				,      "NewTokenizerFast"						)

							# Test we can also load the slow version
							UpperCAmelCase__:	Dict									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__				,      use_fast=lowerCamelCase__						)
							UpperCAmelCase__:	str									= new_processor.tokenizer
							self.assertTrue(new_tokenizer.special_attribute_present						)
							self.assertEqual(new_tokenizer.__class__.__name__				,      "NewTokenizer"						)
						else:
							self.assertEqual(tokenizer.__class__.__name__				,      "NewTokenizer"						)





					def    _UpperCAmelCase    (				self						):
						try:
							AutoConfig.register("custom"				,      lowerCamelCase__						)
							AutoFeatureExtractor.register(lowerCamelCase__				,      lowerCamelCase__						)
							AutoTokenizer.register(lowerCamelCase__				,      slow_tokenizer_class=lowerCamelCase__						)
							AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)
							# Trying to register something existing in the Transformers library will raise an error
							with self.assertRaises(lowerCamelCase__						):
								AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)

							# Now that the config is registered, it can be used as any other config with the auto-API
							UpperCAmelCase__:	Dict									= CustomFeatureExtractor.from_pretrained(lowerCamelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								UpperCAmelCase__:	Tuple									= os.path.join(lowerCamelCase__				,      "vocab.txt"						)
								with open(lowerCamelCase__				,      "w"				,      encoding="utf-8"						) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
								UpperCAmelCase__:	str									= CustomTokenizer(lowerCamelCase__						)

							UpperCAmelCase__:	Optional[int]									= CustomProcessor(lowerCamelCase__				,      lowerCamelCase__						)

							with tempfile.TemporaryDirectory() as tmp_dir:
								processor.save_pretrained(lowerCamelCase__						)
								UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(lowerCamelCase__						)
								self.assertIsInstance(lowerCamelCase__				,      lowerCamelCase__						)

						finally:
							if "custom" in CONFIG_MAPPING._extra_content:
								del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
								del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
								del PROCESSOR_MAPPING._extra_content[CustomConfig]





					def    _UpperCAmelCase    (				self						):

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   False

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   False

						class __UpperCamelCase     (				_a		):






											'''simple docstring'''



											__magic_name__        =   "AutoFeatureExtractor"
											__magic_name__        =   "AutoTokenizer"
											__magic_name__        =   False

						try:
							AutoConfig.register("custom"				,      lowerCamelCase__						)
							AutoFeatureExtractor.register(lowerCamelCase__				,      lowerCamelCase__						)
							AutoTokenizer.register(lowerCamelCase__				,      slow_tokenizer_class=lowerCamelCase__						)
							AutoProcessor.register(lowerCamelCase__				,      lowerCamelCase__						)
							# If remote code is not set, the default is to use local classes.
							UpperCAmelCase__:	Any									= AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor"						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote code is disabled, we load the local ones.
							UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertFalse(processor.special_attribute_present						)
							self.assertFalse(processor.feature_extractor.special_attribute_present						)
							self.assertFalse(processor.tokenizer.special_attribute_present						)

							# If remote is enabled, we load from the Hub.
							UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained(
							    "hf-internal-testing/test_dynamic_processor"				,      trust_remote_code=lowerCamelCase__						)
							self.assertEqual(processor.__class__.__name__				,      "NewProcessor"						)
							self.assertTrue(processor.special_attribute_present						)
							self.assertTrue(processor.feature_extractor.special_attribute_present						)
							self.assertTrue(processor.tokenizer.special_attribute_present						)

						finally:
							if "custom" in CONFIG_MAPPING._extra_content:
								del CONFIG_MAPPING._extra_content["custom"]
							if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
								del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
							if CustomConfig in TOKENIZER_MAPPING._extra_content:
								del TOKENIZER_MAPPING._extra_content[CustomConfig]
							if CustomConfig in PROCESSOR_MAPPING._extra_content:
								del PROCESSOR_MAPPING._extra_content[CustomConfig]





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-bert"						)
						self.assertEqual(processor.__class__.__name__				,      "BertTokenizerFast"						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Tuple									= AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-convnext"						)
						self.assertEqual(processor.__class__.__name__				,      "ConvNextImageProcessor"						)

@is_staging_test
class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						UpperCAmelCase__:	Dict									= TOKEN
						HfFolder.save_token(lowerCamelCase__						)





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						try:
							delete_repo(token=cls._token				,      repo_id="test-processor"						)
						except HTTPError:
							pass

						try:
							delete_repo(token=cls._token				,      repo_id="valid_org/test-processor-org"						)
						except HTTPError:
							pass

						try:
							delete_repo(token=cls._token				,      repo_id="test-dynamic-processor"						)
						except HTTPError:
							pass





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Optional[Any]									= WavaVecaProcessor.from_pretrained(lowerCamelCase__						)
						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(lowerCamelCase__				,      "test-processor"						)				,      push_to_hub=lowerCamelCase__				,      use_auth_token=self._token						)

							UpperCAmelCase__:	Union[str, Any]									= WavaVecaProcessor.from_pretrained(F"{USER}/test-processor"						)
							for k, v in processor.feature_extractor.__dict__.items():
								self.assertEqual(lowerCamelCase__				,      getattr(new_processor.feature_extractor				,      lowerCamelCase__						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()				,      processor.tokenizer.get_vocab()						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Dict									= WavaVecaProcessor.from_pretrained(lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							processor.save_pretrained(
							    os.path.join(lowerCamelCase__				,      "test-processor-org"						)				,      push_to_hub=lowerCamelCase__				,      use_auth_token=self._token				,      organization="valid_org"				,      )

							UpperCAmelCase__:	Optional[int]									= WavaVecaProcessor.from_pretrained("valid_org/test-processor-org"						)
							for k, v in processor.feature_extractor.__dict__.items():
								self.assertEqual(lowerCamelCase__				,      getattr(new_processor.feature_extractor				,      lowerCamelCase__						)						)
							self.assertDictEqual(new_processor.tokenizer.get_vocab()				,      processor.tokenizer.get_vocab()						)





					def    _UpperCAmelCase    (				self						):
						CustomFeatureExtractor.register_for_auto_class()
						CustomTokenizer.register_for_auto_class()
						CustomProcessor.register_for_auto_class()

						UpperCAmelCase__:	List[str]									= CustomFeatureExtractor.from_pretrained(lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							UpperCAmelCase__:	Optional[int]									= os.path.join(lowerCamelCase__				,      "vocab.txt"						)
							with open(lowerCamelCase__				,      "w"				,      encoding="utf-8"						) as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
							UpperCAmelCase__:	Optional[Any]									= CustomTokenizer(lowerCamelCase__						)

						UpperCAmelCase__:	str									= CustomProcessor(lowerCamelCase__				,      lowerCamelCase__						)

						with tempfile.TemporaryDirectory() as tmp_dir:
							create_repo(F"{USER}/test-dynamic-processor"				,      token=self._token						)
							UpperCAmelCase__:	int									= Repository(lowerCamelCase__				,      clone_from=F"{USER}/test-dynamic-processor"				,      token=self._token						)
							processor.save_pretrained(lowerCamelCase__						)

							# This has added the proper auto_map field to the feature extractor config
							self.assertDictEqual(
							    processor.feature_extractor.auto_map				,      {
							        "AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
							        "AutoProcessor": "custom_processing.CustomProcessor",
							    }				,      )

							# This has added the proper auto_map field to the tokenizer config
							with open(os.path.join(lowerCamelCase__				,      "tokenizer_config.json"						)						) as f:
								UpperCAmelCase__:	Tuple									= json.load(lowerCamelCase__						)
							self.assertDictEqual(
							    tokenizer_config["auto_map"]				,      {
							        "AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
							        "AutoProcessor": "custom_processing.CustomProcessor",
							    }				,      )

							# The code has been copied from fixtures
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_feature_extraction.py"						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_tokenization.py"						)						)						)
							self.assertTrue(os.path.isfile(os.path.join(lowerCamelCase__				,      "custom_processing.py"						)						)						)

							repo.push_to_hub()

						UpperCAmelCase__:	Union[str, Any]									= AutoProcessor.from_pretrained(F"{USER}/test-dynamic-processor"				,      trust_remote_code=lowerCamelCase__						)
						# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
						self.assertEqual(new_processor.__class__.__name__				,      "CustomProcessor"						)