



"""simple docstring"""




import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextConfig, ConvNextForImageClassification, ConvNextImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
_lowerCAmelCase				:					Dict							=							logging.get_logger(__name__)

def 						__snake_case						(      SCREAMING_SNAKE_CASE__	:						List[str]				)  ->   Optional[Any]:





     '''simple docstring'''

     _UpperCAmelCase			:	str								=    ConvNextConfig()

     if "tiny" in checkpoint_url:
          _UpperCAmelCase			:	int								=    [3, 3, 9, 3]
          _UpperCAmelCase			:	Any								=    [96, 192, 384, 768]
     if "small" in checkpoint_url:
          _UpperCAmelCase			:	Dict								=    [3, 3, 27, 3]
          _UpperCAmelCase			:	int								=    [96, 192, 384, 768]
     if "base" in checkpoint_url:
          _UpperCAmelCase			:	str								=    [3, 3, 27, 3]
          _UpperCAmelCase			:	int								=    [128, 256, 512, 1_024]
     if "large" in checkpoint_url:
          _UpperCAmelCase			:	int								=    [3, 3, 27, 3]
          _UpperCAmelCase			:	int								=    [192, 384, 768, 1_536]
     if "xlarge" in checkpoint_url:
          _UpperCAmelCase			:	List[Any]								=    [3, 3, 27, 3]
          _UpperCAmelCase			:	Optional[Any]								=    [256, 512, 1_024, 2_048]

     if "1k" in checkpoint_url:
          _UpperCAmelCase			:	Optional[int]								=    1_000
          _UpperCAmelCase			:	int								=    'imagenet-1k-id2label.json'
          _UpperCAmelCase			:	Union[str, Any]								=    (1, 1_000)
     else:
          _UpperCAmelCase			:	Union[str, Any]								=    21_841
          _UpperCAmelCase			:	Optional[int]								=    'imagenet-22k-id2label.json'
          _UpperCAmelCase			:	int								=    (1, 21_841)

     _UpperCAmelCase			:	Dict								=    'huggingface/label-files'
     _UpperCAmelCase			:	List[Any]								=    num_labels
     _UpperCAmelCase			:	Optional[Any]								=    json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE__						,  SCREAMING_SNAKE_CASE__						,  repo_type="dataset"				)						,  "r"				)				)
     _UpperCAmelCase			:	Any								=    {int(SCREAMING_SNAKE_CASE__				): v for k, v in idalabel.items()}
     if "1k" not in checkpoint_url:
          # this dataset contains 21843 labels but the model only has 21841
          # we delete the classes as mentioned in https://github.com/google-research/big_transfer/issues/18
          del idalabel[9_205]
          del idalabel[15_027]
     _UpperCAmelCase			:	str								=    idalabel
     _UpperCAmelCase			:	int								=    {v: k for k, v in idalabel.items()}
     _UpperCAmelCase			:	Union[str, Any]								=    hidden_sizes
     _UpperCAmelCase			:	Optional[int]								=    depths

     return config, expected_shape

def 						__snake_case						(      SCREAMING_SNAKE_CASE__	:						int				)  ->   List[str]:





     '''simple docstring'''

     if "downsample_layers.0.0" in name:
          _UpperCAmelCase			:	Dict								=    name.replace("downsample_layers.0.0"						,  "embeddings.patch_embeddings"				)
     if "downsample_layers.0.1" in name:
          _UpperCAmelCase			:	Union[str, Any]								=    name.replace("downsample_layers.0.1"						,  "embeddings.norm"				)  # we rename to layernorm later on
     if "downsample_layers.1.0" in name:
          _UpperCAmelCase			:	int								=    name.replace("downsample_layers.1.0"						,  "stages.1.downsampling_layer.0"				)
     if "downsample_layers.1.1" in name:
          _UpperCAmelCase			:	Tuple								=    name.replace("downsample_layers.1.1"						,  "stages.1.downsampling_layer.1"				)
     if "downsample_layers.2.0" in name:
          _UpperCAmelCase			:	Tuple								=    name.replace("downsample_layers.2.0"						,  "stages.2.downsampling_layer.0"				)
     if "downsample_layers.2.1" in name:
          _UpperCAmelCase			:	Any								=    name.replace("downsample_layers.2.1"						,  "stages.2.downsampling_layer.1"				)
     if "downsample_layers.3.0" in name:
          _UpperCAmelCase			:	Union[str, Any]								=    name.replace("downsample_layers.3.0"						,  "stages.3.downsampling_layer.0"				)
     if "downsample_layers.3.1" in name:
          _UpperCAmelCase			:	int								=    name.replace("downsample_layers.3.1"						,  "stages.3.downsampling_layer.1"				)
     if "stages" in name and "downsampling_layer" not in name:
          # stages.0.0. for instance should be renamed to stages.0.layers.0.
          _UpperCAmelCase			:	Any								=    name[: len("stages.0"				)] + '.layers' + name[len("stages.0"				) :]
     if "stages" in name:
          _UpperCAmelCase			:	Optional[Any]								=    name.replace("stages"						,  "encoder.stages"				)
     if "norm" in name:
          _UpperCAmelCase			:	Optional[int]								=    name.replace("norm"						,  "layernorm"				)
     if "gamma" in name:
          _UpperCAmelCase			:	int								=    name.replace("gamma"						,  "layer_scale_parameter"				)
     if "head" in name:
          _UpperCAmelCase			:	Dict								=    name.replace("head"						,  "classifier"				)

     return name

def 						__snake_case						(      )  ->   int:





     '''simple docstring'''

     _UpperCAmelCase			:	int								=    'http://images.cocodataset.org/val2017/000000039769.jpg'
     _UpperCAmelCase			:	Any								=    Image.open(requests.get(SCREAMING_SNAKE_CASE__						,  stream=SCREAMING_SNAKE_CASE__				).raw				)
     return im

@torch.no_grad()
def 						__snake_case						(      SCREAMING_SNAKE_CASE__	:						str						,  SCREAMING_SNAKE_CASE__	:						Optional[Any]				)  ->   int:





     '''simple docstring'''

     _UpperCAmelCase			:	Tuple								=    get_convnext_config(SCREAMING_SNAKE_CASE__				)
     # load original state_dict from URL
     _UpperCAmelCase			:	Dict								=    torch.hub.load_state_dict_from_url(SCREAMING_SNAKE_CASE__				)['model']
     # rename keys
     for key in state_dict.copy().keys():
          _UpperCAmelCase			:	Any								=    state_dict.pop(SCREAMING_SNAKE_CASE__				)
          _UpperCAmelCase			:	Any								=    val
     # add prefix to all keys expect classifier head
     for key in state_dict.copy().keys():
          _UpperCAmelCase			:	Any								=    state_dict.pop(SCREAMING_SNAKE_CASE__				)
          if not key.startswith("classifier"				):
               _UpperCAmelCase			:	List[str]								=    'convnext.' + key
          _UpperCAmelCase			:	List[str]								=    val

     # load HuggingFace model
     _UpperCAmelCase			:	Optional[int]								=    ConvNextForImageClassification(SCREAMING_SNAKE_CASE__				)
     model.load_state_dict(SCREAMING_SNAKE_CASE__				)
     model.eval()

     # Check outputs on an image, prepared by ConvNextImageProcessor
     _UpperCAmelCase			:	Any								=    224 if '224' in checkpoint_url else 384
     _UpperCAmelCase			:	Optional[Any]								=    ConvNextImageProcessor(size=SCREAMING_SNAKE_CASE__				)
     _UpperCAmelCase			:	str								=    image_processor(images=prepare_img()						,  return_tensors="pt"				).pixel_values

     _UpperCAmelCase			:	int								=    model(SCREAMING_SNAKE_CASE__				).logits

     # note: the logits below were obtained without center cropping
     if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth":
          _UpperCAmelCase			:	Union[str, Any]								=    torch.tensor([-0.1_210, -0.6_605, 0.1_918]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_small_1k_224_ema.pth":
          _UpperCAmelCase			:	List[str]								=    torch.tensor([-0.4_473, -0.1_847, -0.6_365]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_224_ema.pth":
          _UpperCAmelCase			:	Union[str, Any]								=    torch.tensor([0.4_525, 0.7_539, 0.0_308]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_1k_384.pth":
          _UpperCAmelCase			:	Union[str, Any]								=    torch.tensor([0.3_561, 0.6_350, -0.0_384]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_224_ema.pth":
          _UpperCAmelCase			:	Union[str, Any]								=    torch.tensor([0.4_174, -0.0_989, 0.1_489]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_1k_384.pth":
          _UpperCAmelCase			:	Any								=    torch.tensor([0.2_513, -0.1_349, -0.1_613]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_224.pth":
          _UpperCAmelCase			:	List[str]								=    torch.tensor([1.2_980, 0.3_631, -0.1_198]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_224.pth":
          _UpperCAmelCase			:	Tuple								=    torch.tensor([1.2_963, 0.1_227, 0.1_723]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_224.pth":
          _UpperCAmelCase			:	Optional[Any]								=    torch.tensor([1.7_956, 0.8_390, 0.2_820]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_224.pth":
          _UpperCAmelCase			:	int								=    torch.tensor([-0.2_822, -0.0_502, -0.0_878]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_base_22k_1k_384.pth":
          _UpperCAmelCase			:	Dict								=    torch.tensor([-0.5_672, -0.0_730, -0.4_348]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_224.pth":
          _UpperCAmelCase			:	str								=    torch.tensor([0.2_681, 0.2_365, 0.6_246]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_large_22k_1k_384.pth":
          _UpperCAmelCase			:	List[str]								=    torch.tensor([-0.2_642, 0.3_931, 0.5_116]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_224_ema.pth":
          _UpperCAmelCase			:	Dict								=    torch.tensor([-0.6_677, -0.1_873, -0.8_379]				)
     elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnext_xlarge_22k_1k_384_ema.pth":
          _UpperCAmelCase			:	List[str]								=    torch.tensor([-0.7_749, -0.2_967, -0.6_444]				)
     else:
          raise ValueError(f'Unknown URL: {checkpoint_url}'				)

     assert torch.allclose(logits[0, :3]						,  SCREAMING_SNAKE_CASE__						,  atol=1E-3				)
     assert logits.shape == expected_shape

     Path(SCREAMING_SNAKE_CASE__				).mkdir(exist_ok=SCREAMING_SNAKE_CASE__				)
     print(f'Saving model to {pytorch_dump_folder_path}'				)
     model.save_pretrained(SCREAMING_SNAKE_CASE__				)
     print(f'Saving image processor to {pytorch_dump_folder_path}'				)
     image_processor.save_pretrained(SCREAMING_SNAKE_CASE__				)

     print("Pushing model to the hub..."				)
     _UpperCAmelCase			:	Tuple								=    'convnext'
     if "tiny" in checkpoint_url:
          model_name += "-tiny"
     elif "small" in checkpoint_url:
          model_name += "-small"
     elif "base" in checkpoint_url:
          model_name += "-base"
     elif "xlarge" in checkpoint_url:
          model_name += "-xlarge"
     elif "large" in checkpoint_url:
          model_name += "-large"
     if "224" in checkpoint_url:
          model_name += "-224"
     elif "384" in checkpoint_url:
          model_name += "-384"
     if "22k" in checkpoint_url and "1k" not in checkpoint_url:
          model_name += "-22k"
     if "22k" in checkpoint_url and "1k" in checkpoint_url:
          model_name += "-22k-1k"

     model.push_to_hub(
         repo_path_or_name=Path(SCREAMING_SNAKE_CASE__						,  SCREAMING_SNAKE_CASE__				)						,  organization="nielsr"						,  commit_message="Add model"						,  )


if __name__ == "__main__":
 _lowerCAmelCase				:					List[str]							=							argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--checkpoint_url",
     default="https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth",
     type=str,
     help="URL of the original ConvNeXT checkpoint you\'d like to convert.",
 )
 parser.add_argument(
     "--pytorch_dump_folder_path",
     default=None,
     type=str,
     required=True,
     help="Path to the output PyTorch model directory.",
 )

 _lowerCAmelCase				:					List[str]							=							parser.parse_args()
 convert_convnext_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)




import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_           =	get_tests_dir('fixtures/spiece.model')


@require_sentencepiece
@require_tokenizers
class        A_					(       __UpperCamelCase					,      unittest.TestCase    ):




      '''simple docstring'''


      __snake_case					       =   ErnieMTokenizer
      __snake_case					       =   False
      __snake_case					       =   True
      __snake_case					       =   False
      __snake_case					       =   False




      def      _snake_case	(				self:       Optional[int]				):
       super().setUp()

       # We have a SentencePiece fixture for testing
       __lowerCamelCase		:							List[str]							=						ErnieMTokenizer(a						,      unk_token='<unk>'						,      pad_token='<pad>'				)
       tokenizer.save_pretrained(self.tmpdirname				)




      def      _snake_case	(				self:       Dict						,      a:       List[Any]				):
       __lowerCamelCase		:							Optional[Any]							=						'this is a test'
       __lowerCamelCase		:							Optional[int]							=						'this is a test'
       return input_text, output_text




      def      _snake_case	(				self:       Tuple				):
       __lowerCamelCase		:							List[str]							=						'<pad>'
       __lowerCamelCase		:							Union[str, Any]							=						0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(a				)						,      a				)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(a				)						,      a				)




      def      _snake_case	(				self:       List[Any]				):
       __lowerCamelCase		:							List[str]							=						list(self.get_tokenizer().get_vocab().keys()				)

       self.assertEqual(vocab_keys[0]						,      '<pad>'				)
       self.assertEqual(vocab_keys[1]						,      '<unk>'				)
       self.assertEqual(vocab_keys[-1]						,      '▁eloquent'				)
       self.assertEqual(len(a				)						,      3_0000				)




      def      _snake_case	(				self:       Tuple				):
       self.assertEqual(self.get_tokenizer().vocab_size						,      3_0000				)




      def      _snake_case	(				self:       int				):
       if not self.test_rust_tokenizer:
        return

       __lowerCamelCase		:							Any							=						self.get_tokenizer()
       __lowerCamelCase		:							Optional[Any]							=						self.get_rust_tokenizer()

       __lowerCamelCase		:							Optional[int]							=						'I was born in 92000, and this is falsé.'

       __lowerCamelCase		:							Optional[Any]							=						tokenizer.tokenize(a				)
       __lowerCamelCase		:							Dict							=						rust_tokenizer.tokenize(a				)
       self.assertListEqual(a						,      a				)

       __lowerCamelCase		:							List[str]							=						tokenizer.encode(a						,      add_special_tokens=a				)
       __lowerCamelCase		:							List[Any]							=						rust_tokenizer.encode(a						,      add_special_tokens=a				)
       self.assertListEqual(a						,      a				)

       __lowerCamelCase		:							str							=						self.get_rust_tokenizer()
       __lowerCamelCase		:							str							=						tokenizer.encode(a				)
       __lowerCamelCase		:							List[str]							=						rust_tokenizer.encode(a				)
       self.assertListEqual(a						,      a				)




      def      _snake_case	(				self:       Optional[Any]				):
       __lowerCamelCase		:							Union[str, Any]							=						ErnieMTokenizer(a						,      do_lower_case=a						,      unk_token='<unk>'						,      pad_token='<pad>'				)

       __lowerCamelCase		:							List[str]							=						tokenizer.tokenize('This is a test'				)
       self.assertListEqual(a						,      ['▁this', '▁is', '▁a', '▁test']				)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)						,      [48, 25, 21, 1289]				)

       __lowerCamelCase		:							Optional[Any]							=						tokenizer.tokenize('I was born in 92000, and this is falsé.'				)
       # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
       self.assertListEqual(
           a						,      ['▁i', '▁was', '▁born', '▁in', '9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']				)
       __lowerCamelCase		:							Any							=						tokenizer.convert_tokens_to_ids(a				)
       self.assertListEqual(a						,      [31, 23, 386, 19, 518, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]				)

       __lowerCamelCase		:							int							=						tokenizer.convert_ids_to_tokens(a				)
       self.assertListEqual(
           a						,      ['▁i', '▁was', '▁born', '▁in', '9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']						,      )




      def      _snake_case	(				self:       str				):
       __lowerCamelCase		:							Tuple							=						ErnieMTokenizer(a						,      unk_token='<unk>'						,      pad_token='<pad>'				)

       __lowerCamelCase		:							Tuple							=						tokenizer.encode('sequence builders'				)
       __lowerCamelCase		:							List[Any]							=						tokenizer.encode('multi-sequence build'				)

       __lowerCamelCase		:							Optional[Any]							=						tokenizer.build_inputs_with_special_tokens(a				)
       __lowerCamelCase		:							Tuple							=						tokenizer.build_inputs_with_special_tokens(a						,      a				)

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
           tokenizer.sep_token_id
       ] + text_a + [tokenizer.sep_token_id]




      @slow
      def      _snake_case	(				self:       List[Any]				):
       # fmt: off
       __lowerCamelCase		:							int							=						{'input_ids': [[0, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 9, 304, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 5, 5, 5, 16, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 6460, 1328, 4589, 42, 12_2009, 11_5774, 23, 3559, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [0, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=a						,      model_name='susnato/ernie-m-base_pytorch'						,      sequences=[
               'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
               'general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
               'Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained '
               'models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.',
               'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
               'conditioning on both left and right context in all layers.',
               'The quick brown fox jumps over the lazy dog.',
           ]						,      )

