"""simple docstring"""






import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from timm.data import resolve_data_config
from timm.data.transforms_factory import create_transform

from transformers import (
    BitConfig,
    ViTHybridConfig,
    ViTHybridForImageClassification,
    ViTHybridImageProcessor,
    ViTHybridModel,
)
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCAmelCase       :			Optional[Any]   	=logging.get_logger(__name__)


def      UpperCAmelCase__			(							lowerCAmelCase__	:Optional[Any]   ,						lowerCAmelCase__	:int=False						)					->       Union[str, Any]:
      '''simple docstring'''


      lowercase								=      []

      # fmt: off
      # stem:
      rename_keys.append(("""cls_token""", """vit.embeddings.cls_token""")						)
      rename_keys.append(("""pos_embed""", """vit.embeddings.position_embeddings""")						)

      rename_keys.append(("""patch_embed.proj.weight""", """vit.embeddings.patch_embeddings.projection.weight""")						)
      rename_keys.append(("""patch_embed.proj.bias""", """vit.embeddings.patch_embeddings.projection.bias""")						)

      # backbone
      rename_keys.append(("""patch_embed.backbone.stem.conv.weight""", """vit.embeddings.patch_embeddings.backbone.bit.embedder.convolution.weight""")						)
      rename_keys.append(("""patch_embed.backbone.stem.norm.weight""", """vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.weight""")						)
      rename_keys.append(("""patch_embed.backbone.stem.norm.bias""", """vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.bias""")						)

      for stage_idx in range(len(config.backbone_config.depths						)						):
            for layer_idx in range(config.backbone_config.depths[stage_idx]						):
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv1.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv1.weight')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.weight')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.bias', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.bias')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv2.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv2.weight')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.weight')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.bias', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.bias')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv3.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv3.weight')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.weight')						)
                  rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.bias', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.bias')						)

            rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.conv.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.conv.weight')						)
            rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.weight', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.weight')						)
            rename_keys.append((f'patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.bias', f'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.bias')						)

      # transformer encoder
      for i in range(config.num_hidden_layers						):
            # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
            rename_keys.append((f'blocks.{i}.norm1.weight', f'vit.encoder.layer.{i}.layernorm_before.weight')						)
            rename_keys.append((f'blocks.{i}.norm1.bias', f'vit.encoder.layer.{i}.layernorm_before.bias')						)
            rename_keys.append((f'blocks.{i}.attn.proj.weight', f'vit.encoder.layer.{i}.attention.output.dense.weight')						)
            rename_keys.append((f'blocks.{i}.attn.proj.bias', f'vit.encoder.layer.{i}.attention.output.dense.bias')						)
            rename_keys.append((f'blocks.{i}.norm2.weight', f'vit.encoder.layer.{i}.layernorm_after.weight')						)
            rename_keys.append((f'blocks.{i}.norm2.bias', f'vit.encoder.layer.{i}.layernorm_after.bias')						)
            rename_keys.append((f'blocks.{i}.mlp.fc1.weight', f'vit.encoder.layer.{i}.intermediate.dense.weight')						)
            rename_keys.append((f'blocks.{i}.mlp.fc1.bias', f'vit.encoder.layer.{i}.intermediate.dense.bias')						)
            rename_keys.append((f'blocks.{i}.mlp.fc2.weight', f'vit.encoder.layer.{i}.output.dense.weight')						)
            rename_keys.append((f'blocks.{i}.mlp.fc2.bias', f'vit.encoder.layer.{i}.output.dense.bias')						)

      if base_model:
            # layernorm + pooler
            rename_keys.extend(
                [
                    ("""norm.weight""", """layernorm.weight"""),
                    ("""norm.bias""", """layernorm.bias"""),
                    ("""pre_logits.fc.weight""", """pooler.dense.weight"""),
                    ("""pre_logits.fc.bias""", """pooler.dense.bias"""),
                ]						)

            # if just the base model, we should remove "vit" from all keys that start with "vit"
            lowercase								=      [(pair[0], pair[1][4:]) if pair[1].startswith("""vit"""						) else pair for pair in rename_keys]
      else:
            # layernorm + classification head
            rename_keys.extend(
                [
                    ("""norm.weight""", """vit.layernorm.weight"""),
                    ("""norm.bias""", """vit.layernorm.bias"""),
                    ("""head.weight""", """classifier.weight"""),
                    ("""head.bias""", """classifier.bias"""),
                ]						)
      # fmt: on

      return rename_keys


def      UpperCAmelCase__			(							lowerCAmelCase__	:Dict   ,						lowerCAmelCase__	:int   ,						lowerCAmelCase__	:Optional[int]=False						)					->       Dict:
      '''simple docstring'''


      for i in range(config.num_hidden_layers						):
            if base_model:
                  lowercase								=      """"""
            else:
                  lowercase								=      """vit."""
            # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
            lowercase								=      state_dict.pop(f'blocks.{i}.attn.qkv.weight'						)
            lowercase								=      state_dict.pop(f'blocks.{i}.attn.qkv.bias'						)
            # next, add query, keys and values (in that order) to the state dict
            lowercase								=      in_proj_weight[
                : config.hidden_size, :
            ]
            lowercase								=      in_proj_bias[: config.hidden_size]
            lowercase								=      in_proj_weight[
                config.hidden_size : config.hidden_size * 2, :
            ]
            lowercase								=      in_proj_bias[
                config.hidden_size : config.hidden_size * 2
            ]
            lowercase								=      in_proj_weight[
                -config.hidden_size :, :
            ]
            lowercase								=      in_proj_bias[-config.hidden_size :]


def      UpperCAmelCase__			(							lowerCAmelCase__	:int						)					->       Optional[int]:
      '''simple docstring'''


      lowercase								=      ["""head.weight""", """head.bias"""]
      for k in ignore_keys:
            state_dict.pop(lowerCAmelCase__   ,						lowerCAmelCase__						)


def      UpperCAmelCase__			(							lowerCAmelCase__	:Any   ,						lowerCAmelCase__	:Any   ,						lowerCAmelCase__	:Dict						)					->       Optional[Any]:
      '''simple docstring'''


      lowercase								=      dct.pop(lowerCAmelCase__						)
      lowercase								=      val


def      UpperCAmelCase__			(							)					->       Any:
      '''simple docstring'''


      lowercase								=      """http://images.cocodataset.org/val2017/000000039769.jpg"""
      lowercase								=      Image.open(requests.get(lowerCAmelCase__   ,						stream=lowerCAmelCase__						).raw						)
      return im


@torch.no_grad()
def      UpperCAmelCase__			(							lowerCAmelCase__	:Union[str, Any]   ,						lowerCAmelCase__	:List[str]   ,						lowerCAmelCase__	:Tuple=False						)					->       int:
      '''simple docstring'''


      lowercase								=      BitConfig(
          global_padding="""same"""   ,						layer_type="""bottleneck"""   ,						depths=(3, 4, 9)   ,						out_features=["""stage3"""]   ,						embedding_dynamic_padding=lowerCAmelCase__   ,						)
      lowercase								=      ViTHybridConfig(backbone_config=lowerCAmelCase__   ,						image_size=3_8_4   ,						num_labels=1_0_0_0						)
      lowercase								=      False

      # load original model from timm
      lowercase								=      timm.create_model(lowerCAmelCase__   ,						pretrained=lowerCAmelCase__						)
      timm_model.eval()

      # load state_dict of original model, remove and rename some keys
      lowercase								=      timm_model.state_dict()
      if base_model:
            remove_classification_head_(lowerCAmelCase__						)
      lowercase								=      create_rename_keys(lowerCAmelCase__   ,						lowerCAmelCase__						)
      for src, dest in rename_keys:
            rename_key(lowerCAmelCase__   ,						lowerCAmelCase__   ,						lowerCAmelCase__						)
      read_in_q_k_v(lowerCAmelCase__   ,						lowerCAmelCase__   ,						lowerCAmelCase__						)

      lowercase								=      """huggingface/label-files"""
      lowercase								=      """imagenet-1k-id2label.json"""
      lowercase								=      json.load(open(hf_hub_download(lowerCAmelCase__   ,						lowerCAmelCase__   ,						repo_type="""dataset"""						)   ,						"""r"""						)						)
      lowercase								=      {int(lowerCAmelCase__						): v for k, v in idalabel.items()}
      lowercase								=      idalabel
      lowercase								=      {v: k for k, v in idalabel.items()}

      # load HuggingFace model
      if vit_name[-5:] == "in21k":
            lowercase								=      ViTHybridModel(lowerCAmelCase__						).eval()
      else:
            lowercase								=      ViTHybridForImageClassification(lowerCAmelCase__						).eval()
      model.load_state_dict(lowerCAmelCase__						)

      # create image processor
      lowercase								=      create_transform(**resolve_data_config({}   ,						model=lowerCAmelCase__						)						)
      lowercase								=      transform.transforms

      lowercase								=      {
          """bilinear""": PILImageResampling.BILINEAR,
          """bicubic""": PILImageResampling.BICUBIC,
          """nearest""": PILImageResampling.NEAREST,
      }

      lowercase								=      ViTHybridImageProcessor(
          do_resize=lowerCAmelCase__   ,						size={"""shortest_edge""": timm_transforms[0].size}   ,						resample=pillow_resamplings[timm_transforms[0].interpolation.value]   ,						do_center_crop=lowerCAmelCase__   ,						crop_size={"""height""": timm_transforms[1].size[0], """width""": timm_transforms[1].size[1]}   ,						do_normalize=lowerCAmelCase__   ,						image_mean=timm_transforms[-1].mean.tolist()   ,						image_std=timm_transforms[-1].std.tolist()   ,						)

      lowercase								=      prepare_img()
      lowercase								=      transform(lowerCAmelCase__						).unsqueeze(0						)
      lowercase								=      processor(lowerCAmelCase__   ,						return_tensors="""pt"""						).pixel_values

      # verify pixel values
      assert torch.allclose(lowerCAmelCase__   ,						lowerCAmelCase__						)

      # verify logits
      with torch.no_grad():
            lowercase								=      model(lowerCAmelCase__						)
            lowercase								=      outputs.logits

      print("""Predicted class:"""   ,						logits.argmax(-1						).item()						)
      if base_model:
            lowercase								=      timm_model.forward_features(lowerCAmelCase__						)
            assert timm_pooled_output.shape == outputs.pooler_output.shape
            assert torch.allclose(lowerCAmelCase__   ,						outputs.pooler_output   ,						atol=1e-3						)
      else:
            lowercase								=      timm_model(lowerCAmelCase__						)
            assert timm_logits.shape == outputs.logits.shape
            assert torch.allclose(lowerCAmelCase__   ,						outputs.logits   ,						atol=1e-3						)
      print("""Looks ok!"""						)

      if pytorch_dump_folder_path is not None:
            Path(lowerCAmelCase__						).mkdir(exist_ok=lowerCAmelCase__						)
            print(f'Saving model {vit_name} to {pytorch_dump_folder_path}'						)
            model.save_pretrained(lowerCAmelCase__						)
            print(f'Saving processor to {pytorch_dump_folder_path}'						)
            processor.save_pretrained(lowerCAmelCase__						)

      if push_to_hub:
            print(f'Pushing model and processor to the hub {vit_name}'						)
            model.push_to_hub(f'ybelkada/{vit_name}'						)
            processor.push_to_hub(f'ybelkada/{vit_name}'						)


if __name__ == "__main__":
       __lowerCAmelCase       :			List[str]   	=argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           """--vit_name""",
           default="""vit_base_r50_s16_384""",
           type=str,
           help="""Name of the hybrid ViT timm model you'd like to convert.""",
       )
       parser.add_argument(
           """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
       )
       parser.add_argument(
           """--push_to_hub""", action="""store_true""", help="""Whether to upload the model to the HuggingFace hub."""
       )

       __lowerCAmelCase       :			str   	=parser.parse_args()
       convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path, args.push_to_hub)




"""simple docstring"""






import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from huggingface_hub import HfFolder, Repository, create_repo, delete_repo
from requests.exceptions import HTTPError

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoProcessor,
    AutoTokenizer,
    BertTokenizer,
    ProcessorMixin,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
)
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402
from test_module.custom_processing import CustomProcessor  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


__lowerCAmelCase       :			str   	=get_tests_dir("""fixtures/dummy_feature_extractor_config.json""")
__lowerCAmelCase       :			Tuple   	=get_tests_dir("""fixtures/vocab.json""")
__lowerCAmelCase       :			Tuple   	=get_tests_dir("""fixtures""")





class 					_A		(			unittest.TestCase      ):
      snake_case__     :  List[Any]				    =      ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']

      def 							A__			(		self		):







            """simple docstring"""



            lowercase								=      0

      def 							A__			(		self		):







            """simple docstring"""



            lowercase								=      AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""		)
            self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

      def 							A__			(		self		):







            """simple docstring"""



            with tempfile.TemporaryDirectory() as tmpdirname:
                  lowercase								=      WavaVecaConfig()
                  lowercase								=      AutoProcessor.from_pretrained("""facebook/wav2vec2-base-960h"""		)

                  # save in new folder
                  model_config.save_pretrained(__lowerCAmelCase		)
                  processor.save_pretrained(__lowerCAmelCase		)

                  lowercase								=      AutoProcessor.from_pretrained(__lowerCAmelCase		)

            self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

      def 							A__			(		self		):







            """simple docstring"""



            with tempfile.TemporaryDirectory() as tmpdirname:
                  # copy relevant files
                  copyfile(__lowerCAmelCase						,			os.path.join(__lowerCAmelCase						,			__lowerCAmelCase		)		)
                  copyfile(__lowerCAmelCase						,			os.path.join(__lowerCAmelCase						,			"""vocab.json"""		)		)

                  lowercase								=      AutoProcessor.from_pretrained(__lowerCAmelCase		)

            self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

      def 							A__			(		self		):







            """simple docstring"""



            with tempfile.TemporaryDirectory() as tmpdirname:
                  lowercase								=      WavaVecaFeatureExtractor()
                  lowercase								=      AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""		)

                  lowercase								=      WavaVecaProcessor(__lowerCAmelCase						,			__lowerCAmelCase		)

                  # save in new folder
                  processor.save_pretrained(__lowerCAmelCase		)

                  # drop `processor_class` in tokenizer
                  with open(os.path.join(__lowerCAmelCase						,			__lowerCAmelCase		)						,			"""r"""		) as f:
                        lowercase								=      json.load(__lowerCAmelCase		)
                        config_dict.pop("""processor_class"""		)

                  with open(os.path.join(__lowerCAmelCase						,			__lowerCAmelCase		)						,			"""w"""		) as f:
                        f.write(json.dumps(__lowerCAmelCase		)		)

                  lowercase								=      AutoProcessor.from_pretrained(__lowerCAmelCase		)

            self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

      def 							A__			(		self		):







            """simple docstring"""



            with tempfile.TemporaryDirectory() as tmpdirname:
                  lowercase								=      WavaVecaFeatureExtractor()
                  lowercase								=      AutoTokenizer.from_pretrained("""facebook/wav2vec2-base-960h"""		)

                  lowercase								=      WavaVecaProcessor(__lowerCAmelCase						,			__lowerCAmelCase		)

                  # save in new folder
                  processor.save_pretrained(__lowerCAmelCase		)

                  # drop `processor_class` in feature extractor
                  with open(os.path.join(__lowerCAmelCase						,			__lowerCAmelCase		)						,			"""r"""		) as f:
                        lowercase								=      json.load(__lowerCAmelCase		)
                        config_dict.pop("""processor_class"""		)

                  with open(os.path.join(__lowerCAmelCase						,			__lowerCAmelCase		)						,			"""w"""		) as f:
                        f.write(json.dumps(__lowerCAmelCase		)		)

                  lowercase								=      AutoProcessor.from_pretrained(__lowerCAmelCase		)

            self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

      def 							A__			(		self		):







            """simple docstring"""



            with tempfile.TemporaryDirectory() as tmpdirname:
                  lowercase								=      WavaVecaConfig(processor_class="""Wav2Vec2Processor"""		)
                  model_config.save_pretrained(__lowerCAmelCase		)
                  # copy relevant files
                  copyfile(__lowerCAmelCase						,			os.path.join(__lowerCAmelCase						,			"""vocab.json"""		)		)
                  # create emtpy sample processor
                  with open(os.path.join(__lowerCAmelCase						,			__lowerCAmelCase		)						,			"""w"""		) as f:
                        f.write("""{}"""		)

                  lowercase								=      AutoProcessor.from_pretrained(__lowerCAmelCase		)

            self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

      def 							A__			(		self		):







            """simple docstring"""



            with self.assertRaises(__lowerCAmelCase		):
                  lowercase								=      AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""		)
            # If remote code is disabled, we can't load this config.
            with self.assertRaises(__lowerCAmelCase		):
                  lowercase								=      AutoProcessor.from_pretrained(
                      """hf-internal-testing/test_dynamic_processor"""						,			trust_remote_code=__lowerCAmelCase		)

            lowercase								=      AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""						,			trust_remote_code=__lowerCAmelCase		)
            self.assertTrue(processor.special_attribute_present		)
            self.assertEqual(processor.__class__.__name__						,			"""NewProcessor"""		)

            lowercase								=      processor.feature_extractor
            self.assertTrue(feature_extractor.special_attribute_present		)
            self.assertEqual(feature_extractor.__class__.__name__						,			"""NewFeatureExtractor"""		)

            lowercase								=      processor.tokenizer
            self.assertTrue(tokenizer.special_attribute_present		)
            if is_tokenizers_available():
                  self.assertEqual(tokenizer.__class__.__name__						,			"""NewTokenizerFast"""		)

                  # Test we can also load the slow version
                  lowercase								=      AutoProcessor.from_pretrained(
                      """hf-internal-testing/test_dynamic_processor"""						,			trust_remote_code=__lowerCAmelCase						,			use_fast=__lowerCAmelCase		)
                  lowercase								=      new_processor.tokenizer
                  self.assertTrue(new_tokenizer.special_attribute_present		)
                  self.assertEqual(new_tokenizer.__class__.__name__						,			"""NewTokenizer"""		)
            else:
                  self.assertEqual(tokenizer.__class__.__name__						,			"""NewTokenizer"""		)

      def 							A__			(		self		):







            """simple docstring"""



            try:
                  AutoConfig.register("""custom"""						,			__lowerCAmelCase		)
                  AutoFeatureExtractor.register(__lowerCAmelCase						,			__lowerCAmelCase		)
                  AutoTokenizer.register(__lowerCAmelCase						,			slow_tokenizer_class=__lowerCAmelCase		)
                  AutoProcessor.register(__lowerCAmelCase						,			__lowerCAmelCase		)
                  # Trying to register something existing in the Transformers library will raise an error
                  with self.assertRaises(__lowerCAmelCase		):
                        AutoProcessor.register(__lowerCAmelCase						,			__lowerCAmelCase		)

                  # Now that the config is registered, it can be used as any other config with the auto-API
                  lowercase								=      CustomFeatureExtractor.from_pretrained(__lowerCAmelCase		)

                  with tempfile.TemporaryDirectory() as tmp_dir:
                        lowercase								=      os.path.join(__lowerCAmelCase						,			"""vocab.txt"""		)
                        with open(__lowerCAmelCase						,			"""w"""						,			encoding="""utf-8"""		) as vocab_writer:
                              vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]		)		)
                        lowercase								=      CustomTokenizer(__lowerCAmelCase		)

                  lowercase								=      CustomProcessor(__lowerCAmelCase						,			__lowerCAmelCase		)

                  with tempfile.TemporaryDirectory() as tmp_dir:
                        processor.save_pretrained(__lowerCAmelCase		)
                        lowercase								=      AutoProcessor.from_pretrained(__lowerCAmelCase		)
                        self.assertIsInstance(__lowerCAmelCase						,			__lowerCAmelCase		)

            finally:
                  if "custom" in CONFIG_MAPPING._extra_content:
                        del CONFIG_MAPPING._extra_content["custom"]
                  if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                        del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
                  if CustomConfig in TOKENIZER_MAPPING._extra_content:
                        del TOKENIZER_MAPPING._extra_content[CustomConfig]
                  if CustomConfig in PROCESSOR_MAPPING._extra_content:
                        del PROCESSOR_MAPPING._extra_content[CustomConfig]

      def 							A__			(		self		):







            """simple docstring"""



            class 					_A		(			lowerCAmelCase      ):
                  snake_case__     :  Optional[Any]				    =      False






            class 					_A		(			lowerCAmelCase      ):
                  snake_case__     :  int				    =      False






            class 					_A		(			lowerCAmelCase      ):
                  snake_case__     :  List[Any]				    =      'AutoFeatureExtractor'
                  snake_case__     :  List[str]				    =      'AutoTokenizer'
                  snake_case__     :  List[str]				    =      False

            try:
                  AutoConfig.register("""custom"""						,			__lowerCAmelCase		)
                  AutoFeatureExtractor.register(__lowerCAmelCase						,			__lowerCAmelCase		)
                  AutoTokenizer.register(__lowerCAmelCase						,			slow_tokenizer_class=__lowerCAmelCase		)
                  AutoProcessor.register(__lowerCAmelCase						,			__lowerCAmelCase		)
                  # If remote code is not set, the default is to use local classes.
                  lowercase								=      AutoProcessor.from_pretrained("""hf-internal-testing/test_dynamic_processor"""		)
                  self.assertEqual(processor.__class__.__name__						,			"""NewProcessor"""		)
                  self.assertFalse(processor.special_attribute_present		)
                  self.assertFalse(processor.feature_extractor.special_attribute_present		)
                  self.assertFalse(processor.tokenizer.special_attribute_present		)

                  # If remote code is disabled, we load the local ones.
                  lowercase								=      AutoProcessor.from_pretrained(
                      """hf-internal-testing/test_dynamic_processor"""						,			trust_remote_code=__lowerCAmelCase		)
                  self.assertEqual(processor.__class__.__name__						,			"""NewProcessor"""		)
                  self.assertFalse(processor.special_attribute_present		)
                  self.assertFalse(processor.feature_extractor.special_attribute_present		)
                  self.assertFalse(processor.tokenizer.special_attribute_present		)

                  # If remote is enabled, we load from the Hub.
                  lowercase								=      AutoProcessor.from_pretrained(
                      """hf-internal-testing/test_dynamic_processor"""						,			trust_remote_code=__lowerCAmelCase		)
                  self.assertEqual(processor.__class__.__name__						,			"""NewProcessor"""		)
                  self.assertTrue(processor.special_attribute_present		)
                  self.assertTrue(processor.feature_extractor.special_attribute_present		)
                  self.assertTrue(processor.tokenizer.special_attribute_present		)

            finally:
                  if "custom" in CONFIG_MAPPING._extra_content:
                        del CONFIG_MAPPING._extra_content["custom"]
                  if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                        del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
                  if CustomConfig in TOKENIZER_MAPPING._extra_content:
                        del TOKENIZER_MAPPING._extra_content[CustomConfig]
                  if CustomConfig in PROCESSOR_MAPPING._extra_content:
                        del PROCESSOR_MAPPING._extra_content[CustomConfig]

      def 							A__			(		self		):







            """simple docstring"""



            lowercase								=      AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-bert"""		)
            self.assertEqual(processor.__class__.__name__						,			"""BertTokenizerFast"""		)







      def 							A__			(		self		):







            """simple docstring"""



            lowercase								=      AutoProcessor.from_pretrained("""hf-internal-testing/tiny-random-convnext"""		)
            self.assertEqual(processor.__class__.__name__						,			"""ConvNextImageProcessor"""		)






@is_staging_test
class 					_A		(			unittest.TestCase      ):
      snake_case__     :  Dict				    =      ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']

      @classmethod
      def 							A__			(		cls		):







            """simple docstring"""



            lowercase								=      TOKEN
            HfFolder.save_token(__lowerCAmelCase		)

      @classmethod
      def 							A__			(		cls		):







            """simple docstring"""



            try:
                  delete_repo(token=cls._token						,			repo_id="""test-processor"""		)
            except HTTPError:
                  pass

            try:
                  delete_repo(token=cls._token						,			repo_id="""valid_org/test-processor-org"""		)
            except HTTPError:
                  pass

            try:
                  delete_repo(token=cls._token						,			repo_id="""test-dynamic-processor"""		)
            except HTTPError:
                  pass

      def 							A__			(		self		):







            """simple docstring"""



            lowercase								=      WavaVecaProcessor.from_pretrained(__lowerCAmelCase		)
            with tempfile.TemporaryDirectory() as tmp_dir:
                  processor.save_pretrained(
                      os.path.join(__lowerCAmelCase						,			"""test-processor"""		)						,			push_to_hub=__lowerCAmelCase						,			use_auth_token=self._token		)

                  lowercase								=      WavaVecaProcessor.from_pretrained(f'{USER}/test-processor'		)
                  for k, v in processor.feature_extractor.__dict__.items():
                        self.assertEqual(__lowerCAmelCase						,			getattr(new_processor.feature_extractor						,			__lowerCAmelCase		)		)
                  self.assertDictEqual(new_processor.tokenizer.get_vocab()						,			processor.tokenizer.get_vocab()		)

      def 							A__			(		self		):







            """simple docstring"""



            lowercase								=      WavaVecaProcessor.from_pretrained(__lowerCAmelCase		)

            with tempfile.TemporaryDirectory() as tmp_dir:
                  processor.save_pretrained(
                      os.path.join(__lowerCAmelCase						,			"""test-processor-org"""		)						,			push_to_hub=__lowerCAmelCase						,			use_auth_token=self._token						,			organization="""valid_org"""						,			)

                  lowercase								=      WavaVecaProcessor.from_pretrained("""valid_org/test-processor-org"""		)
                  for k, v in processor.feature_extractor.__dict__.items():
                        self.assertEqual(__lowerCAmelCase						,			getattr(new_processor.feature_extractor						,			__lowerCAmelCase		)		)
                  self.assertDictEqual(new_processor.tokenizer.get_vocab()						,			processor.tokenizer.get_vocab()		)







      def 							A__			(		self		):







            """simple docstring"""



            CustomFeatureExtractor.register_for_auto_class()
            CustomTokenizer.register_for_auto_class()
            CustomProcessor.register_for_auto_class()

            lowercase								=      CustomFeatureExtractor.from_pretrained(__lowerCAmelCase		)

            with tempfile.TemporaryDirectory() as tmp_dir:
                  lowercase								=      os.path.join(__lowerCAmelCase						,			"""vocab.txt"""		)
                  with open(__lowerCAmelCase						,			"""w"""						,			encoding="""utf-8"""		) as vocab_writer:
                        vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]		)		)
                  lowercase								=      CustomTokenizer(__lowerCAmelCase		)

            lowercase								=      CustomProcessor(__lowerCAmelCase						,			__lowerCAmelCase		)

            with tempfile.TemporaryDirectory() as tmp_dir:
                  create_repo(f'{USER}/test-dynamic-processor'						,			token=self._token		)
                  lowercase								=      Repository(__lowerCAmelCase						,			clone_from=f'{USER}/test-dynamic-processor'						,			token=self._token		)
                  processor.save_pretrained(__lowerCAmelCase		)

                  # This has added the proper auto_map field to the feature extractor config
                  self.assertDictEqual(
                      processor.feature_extractor.auto_map						,			{
                          """AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor""",
                          """AutoProcessor""": """custom_processing.CustomProcessor""",
                      }						,			)

                  # This has added the proper auto_map field to the tokenizer config
                  with open(os.path.join(__lowerCAmelCase						,			"""tokenizer_config.json"""		)		) as f:
                        lowercase								=      json.load(__lowerCAmelCase		)
                  self.assertDictEqual(
                      tokenizer_config["""auto_map"""]						,			{
                          """AutoTokenizer""": ["""custom_tokenization.CustomTokenizer""", None],
                          """AutoProcessor""": """custom_processing.CustomProcessor""",
                      }						,			)

                  # The code has been copied from fixtures
                  self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase						,			"""custom_feature_extraction.py"""		)		)		)
                  self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase						,			"""custom_tokenization.py"""		)		)		)
                  self.assertTrue(os.path.isfile(os.path.join(__lowerCAmelCase						,			"""custom_processing.py"""		)		)		)

                  repo.push_to_hub()

            lowercase								=      AutoProcessor.from_pretrained(f'{USER}/test-dynamic-processor'						,			trust_remote_code=__lowerCAmelCase		)
            # Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
            self.assertEqual(new_processor.__class__.__name__						,			"""CustomProcessor"""		)



