






import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      XLMForMultipleChoice,
      XLMForQuestionAnswering,
      XLMForQuestionAnsweringSimple,
      XLMForSequenceClassification,
      XLMForTokenClassification,
      XLMModel,
      XLMWithLMHeadModel,
  )
  from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST
class 		SCREAMING_SNAKE_CASE__	:

  '''simple docstring'''







  def __init__(      self		:			List[str]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Union[str, Any]=13       ,				lowerCamelCase		:			Any=7       ,				lowerCamelCase		:			int=True       ,				lowerCamelCase		:			Dict=True       ,				lowerCamelCase		:			Optional[Any]=True       ,				lowerCamelCase		:			int=True       ,				lowerCamelCase		:			int=True       ,				lowerCamelCase		:			Tuple=False       ,				lowerCamelCase		:			Tuple=False       ,				lowerCamelCase		:			Optional[int]=False       ,				lowerCamelCase		:			List[str]=2       ,				lowerCamelCase		:			Any=99       ,				lowerCamelCase		:			Optional[int]=0       ,				lowerCamelCase		:			Dict=32       ,				lowerCamelCase		:			Optional[int]=5       ,				lowerCamelCase		:			Union[str, Any]=4       ,				lowerCamelCase		:			str=0.1       ,				lowerCamelCase		:			Any=0.1       ,				lowerCamelCase		:			Dict=512       ,				lowerCamelCase		:			Dict=2       ,				lowerCamelCase		:			Any=0.02       ,				lowerCamelCase		:			int=2       ,				lowerCamelCase		:			Optional[int]=4       ,				lowerCamelCase		:			Optional[int]="last"       ,				lowerCamelCase		:			List[Any]=True       ,				lowerCamelCase		:			List[str]=None       ,				lowerCamelCase		:			Tuple=0       ,				)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    parent
         _UpperCAmelCase							=    batch_size
         _UpperCAmelCase							=    seq_length
         _UpperCAmelCase							=    is_training
         _UpperCAmelCase							=    use_input_lengths
         _UpperCAmelCase							=    use_token_type_ids
         _UpperCAmelCase							=    use_labels
         _UpperCAmelCase							=    gelu_activation
         _UpperCAmelCase							=    sinusoidal_embeddings
         _UpperCAmelCase							=    causal
         _UpperCAmelCase							=    asm
         _UpperCAmelCase							=    n_langs
         _UpperCAmelCase							=    vocab_size
         _UpperCAmelCase							=    n_special
         _UpperCAmelCase							=    hidden_size
         _UpperCAmelCase							=    num_hidden_layers
         _UpperCAmelCase							=    num_attention_heads
         _UpperCAmelCase							=    hidden_dropout_prob
         _UpperCAmelCase							=    attention_probs_dropout_prob
         _UpperCAmelCase							=    max_position_embeddings
         _UpperCAmelCase							=    type_sequence_label_size
         _UpperCAmelCase							=    initializer_range
         _UpperCAmelCase							=    num_labels
         _UpperCAmelCase							=    num_choices
         _UpperCAmelCase							=    summary_type
         _UpperCAmelCase							=    use_proj
         _UpperCAmelCase							=    scope
         _UpperCAmelCase							=    bos_token_id


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size							)
         _UpperCAmelCase							=    random_attention_mask([self.batch_size, self.seq_length]							)

         _UpperCAmelCase							=    None
         if self.use_input_lengths:
                _UpperCAmelCase							=    (
                    ids_tensor([self.batch_size]       ,				vocab_size=2							) + self.seq_length - 2
                )  # small variation of seq_length

         _UpperCAmelCase							=    None
         if self.use_token_type_ids:
                _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				self.n_langs							)

         _UpperCAmelCase							=    None
         _UpperCAmelCase							=    None
         _UpperCAmelCase							=    None
         if self.use_labels:
                _UpperCAmelCase							=    ids_tensor([self.batch_size]       ,				self.type_sequence_label_size							)
                _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels							)
                _UpperCAmelCase							=    ids_tensor([self.batch_size]       ,				2							).float()
                _UpperCAmelCase							=    ids_tensor([self.batch_size]       ,				self.num_choices							)

         _UpperCAmelCase							=    self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )


  def  lowerCamelCase       (      self		:			List[str]							)						->	Union[str, Any]:


         """simple docstring"""
         return XLMConfig(
             vocab_size=self.vocab_size       ,				n_special=self.n_special       ,				emb_dim=self.hidden_size       ,				n_layers=self.num_hidden_layers       ,				n_heads=self.num_attention_heads       ,				dropout=self.hidden_dropout_prob       ,				attention_dropout=self.attention_probs_dropout_prob       ,				gelu_activation=self.gelu_activation       ,				sinusoidal_embeddings=self.sinusoidal_embeddings       ,				asm=self.asm       ,				causal=self.causal       ,				n_langs=self.n_langs       ,				max_position_embeddings=self.max_position_embeddings       ,				initializer_range=self.initializer_range       ,				summary_type=self.summary_type       ,				use_proj=self.use_proj       ,				num_labels=self.num_labels       ,				bos_token_id=self.bos_token_id       ,				)


  def  lowerCamelCase       (      self		:			str       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			str       ,				)						->	Dict:


         """simple docstring"""
         _UpperCAmelCase							=    XLMModel(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(lowerCamelCase       ,				lengths=lowerCamelCase       ,				langs=lowerCamelCase							)
         _UpperCAmelCase							=    model(lowerCamelCase       ,				langs=lowerCamelCase							)
         _UpperCAmelCase							=    model(lowerCamelCase							)
         self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)							)


  def  lowerCamelCase       (      self		:			int       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Optional[Any]       ,				)						->	Tuple:


         """simple docstring"""
         _UpperCAmelCase							=    XLMWithLMHeadModel(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()

         _UpperCAmelCase							=    model(lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase							)
         self.parent.assertEqual(result.loss.shape       ,				()							)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)							)


  def  lowerCamelCase       (      self		:			Any       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Optional[Any]       ,				)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    XLMForQuestionAnsweringSimple(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()

         _UpperCAmelCase							=    model(lowerCamelCase							)

         _UpperCAmelCase							=    model(lowerCamelCase       ,				start_positions=lowerCamelCase       ,				end_positions=lowerCamelCase							)
         _UpperCAmelCase							=    outputs
         self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)							)


  def  lowerCamelCase       (      self		:			str       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Dict       ,				)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    XLMForQuestionAnswering(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()

         _UpperCAmelCase							=    model(lowerCamelCase							)

         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				start_positions=lowerCamelCase       ,				end_positions=lowerCamelCase       ,				cls_index=lowerCamelCase       ,				is_impossible=lowerCamelCase       ,				p_mask=lowerCamelCase       ,				)

         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				start_positions=lowerCamelCase       ,				end_positions=lowerCamelCase       ,				cls_index=lowerCamelCase       ,				is_impossible=lowerCamelCase       ,				)

         ((_UpperCAmelCase)	,			)							=    result_with_labels.to_tuple()

         _UpperCAmelCase							=    model(lowerCamelCase       ,				start_positions=lowerCamelCase       ,				end_positions=lowerCamelCase							)

         ((_UpperCAmelCase)	,			)							=    result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape       ,				()							)
         self.parent.assertEqual(result.start_top_log_probs.shape       ,				(self.batch_size, model.config.start_n_top)							)
         self.parent.assertEqual(result.start_top_index.shape       ,				(self.batch_size, model.config.start_n_top)							)
         self.parent.assertEqual(
             result.end_top_log_probs.shape       ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
         self.parent.assertEqual(
             result.end_top_index.shape       ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
         self.parent.assertEqual(result.cls_logits.shape       ,				(self.batch_size,)							)


  def  lowerCamelCase       (      self		:			Optional[Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			List[Any]       ,				)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    XLMForSequenceClassification(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()

         _UpperCAmelCase							=    model(lowerCamelCase							)
         _UpperCAmelCase							=    model(lowerCamelCase       ,				labels=lowerCamelCase							)
         self.parent.assertEqual(result.loss.shape       ,				()							)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.type_sequence_label_size)							)


  def  lowerCamelCase       (      self		:			Optional[Any]       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[Any]       ,				)						->	Dict:


         """simple docstring"""
         _UpperCAmelCase							=    self.num_labels
         _UpperCAmelCase							=    XLMForTokenClassification(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()

         _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				labels=lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)							)


  def  lowerCamelCase       (      self		:			int       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			List[str]       ,				)						->	Optional[int]:


         """simple docstring"""
         _UpperCAmelCase							=    self.num_choices
         _UpperCAmelCase							=    XLMForMultipleChoice(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    input_ids.unsqueeze(1							).expand(-1       ,				self.num_choices       ,				-1							).contiguous()
         _UpperCAmelCase							=    token_type_ids.unsqueeze(1							).expand(-1       ,				self.num_choices       ,				-1							).contiguous()
         _UpperCAmelCase							=    input_mask.unsqueeze(1							).expand(-1       ,				self.num_choices       ,				-1							).contiguous()
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase       ,				)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)							)





  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.prepare_config_and_inputs()
         (
             (
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			
         )							=    config_and_inputs
         _UpperCAmelCase							=    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """lengths""": input_lengths}
         return config, inputs_dict



@require_torch
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      UpperCAmelCase					,      UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  (
      (
          XLMModel,
          XLMWithLMHeadModel,
          XLMForQuestionAnswering,
          XLMForSequenceClassification,
          XLMForQuestionAnsweringSimple,
          XLMForTokenClassification,
          XLMForMultipleChoice,
      )
      if is_torch_available()
      else ()
  )
  _lowerCamelCase    =  (
      (XLMWithLMHeadModel,) if is_torch_available() else ()
  )  # TODO (PVP): Check other models whether language generation is also applicable
  _lowerCamelCase    =  (
      {
          '''feature-extraction''': XLMModel,
          '''fill-mask''': XLMWithLMHeadModel,
          '''question-answering''': XLMForQuestionAnsweringSimple,
          '''text-classification''': XLMForSequenceClassification,
          '''text-generation''': XLMWithLMHeadModel,
          '''token-classification''': XLMForTokenClassification,
          '''zero-shot''': XLMForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )


  def  lowerCamelCase       (      self		:			Dict       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Tuple							)						->	List[Any]:


         """simple docstring"""
         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith("""Fast"""							)
         ):
                # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                return True

         return False


  def  lowerCamelCase       (      self		:			Dict       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Union[str, Any]=False							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    super()._prepare_for_class(lowerCamelCase       ,				lowerCamelCase       ,				return_labels=lowerCamelCase							)

         if return_labels:
                if model_class.__name__ == "XLMForQuestionAnswering":
                       _UpperCAmelCase							=    torch.zeros(
                           self.model_tester.batch_size       ,				dtype=torch.long       ,				device=lowerCamelCase							)
                       _UpperCAmelCase							=    torch.zeros(
                           self.model_tester.batch_size       ,				dtype=torch.long       ,				device=lowerCamelCase							)

         return inputs_dict


  def  lowerCamelCase       (      self		:			Dict							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    XLMModelTester(self							)
         _UpperCAmelCase							=    ConfigTester(self       ,				config_class=lowerCamelCase       ,				emb_dim=37							)


  def  lowerCamelCase       (      self		:			str							)						->	int:


         """simple docstring"""
         self.config_tester.run_common_tests()


  def  lowerCamelCase       (      self		:			str							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_model(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			int							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_lm_head(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Tuple							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_simple_qa(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Tuple							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_qa(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			int							)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_sequence_classif(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_token_classif(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			List[str]							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_multiple_choice(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Union[str, Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Any=False       ,				lowerCamelCase		:			List[Any]=1							)						->	Optional[int]:


         """simple docstring"""
         self.assertIsInstance(lowerCamelCase       ,				lowerCamelCase							)
         self.assertListEqual(
             [isinstance(lowerCamelCase       ,				lowerCamelCase							) for iter_attentions in attentions]       ,				[True] * len(lowerCamelCase							)							)
         self.assertEqual(len(lowerCamelCase							)       ,				(max_length - min_length) * num_beam_groups							)

         for idx, iter_attentions in enumerate(lowerCamelCase							):
                # adds PAD dummy token
                _UpperCAmelCase							=    min_length + idx + 1
                _UpperCAmelCase							=    min_length + idx + 1

                _UpperCAmelCase							=    (
                    batch_size * num_beam_groups,
                    config.num_attention_heads,
                    tgt_len,
                    src_len,
                )
                # check attn size
                self.assertListEqual(
                    [layer_attention.shape for layer_attention in iter_attentions]       ,				[expected_shape] * len(lowerCamelCase							)							)


  def  lowerCamelCase       (      self		:			Dict       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Any=False       ,				lowerCamelCase		:			Union[str, Any]=1							)						->	str:


         """simple docstring"""
         self.assertIsInstance(lowerCamelCase       ,				lowerCamelCase							)
         self.assertListEqual(
             [isinstance(lowerCamelCase       ,				lowerCamelCase							) for iter_hidden_states in hidden_states]       ,				[True] * len(lowerCamelCase							)       ,				)
         self.assertEqual(len(lowerCamelCase							)       ,				(max_length - min_length) * num_beam_groups							)

         for idx, iter_hidden_states in enumerate(lowerCamelCase							):
                # adds PAD dummy token
                _UpperCAmelCase							=    min_length + idx + 1
                _UpperCAmelCase							=    (batch_size * num_beam_groups, seq_len, config.hidden_size)
                # check hidden size
                self.assertListEqual(
                    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]       ,				[expected_shape] * len(lowerCamelCase							)       ,				)
         pass





  @slow
  def  lowerCamelCase       (      self		:			int							)						->	Optional[Any]:


         """simple docstring"""
         for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                _UpperCAmelCase							=    XLMModel.from_pretrained(lowerCamelCase							)
                self.assertIsNotNone(lowerCamelCase							)



@require_torch
class 		SCREAMING_SNAKE_CASE__	(  unittest.TestCase     ):

  '''simple docstring'''







  @slow
  def  lowerCamelCase       (      self		:			List[Any]							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    XLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048"""							)
         model.to(lowerCamelCase							)
         _UpperCAmelCase							=    torch.tensor([[14, 447]]       ,				dtype=torch.long       ,				device=lowerCamelCase							)  # the president
         _UpperCAmelCase							=    [
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
         ]  # the president the president the president the president the president the president the president the president the president the president
         # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
         _UpperCAmelCase							=    model.generate(lowerCamelCase       ,				do_sample=lowerCamelCase							)
         self.assertListEqual(output_ids[0].cpu().numpy().tolist()       ,				lowerCamelCase							)






'''simple docstring'''




import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DeiTImageProcessor, ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
_SCREAMING_SNAKE_CASE							     =			logging.get_logger(__name__)





def 	__lowerCamelCase			(      __lowerCAmelCase       :							Optional[int]      ,    __lowerCAmelCase       :							List[Any]=False		)		->  List[str]:
   snake_case 						=   []
   for i in range(config.num_hidden_layers		):
      # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
      rename_keys.append((F'''blocks.{i}.norm1.weight''', F'''vit.encoder.layer.{i}.layernorm_before.weight''')		)
      rename_keys.append((F'''blocks.{i}.norm1.bias''', F'''vit.encoder.layer.{i}.layernorm_before.bias''')		)
      rename_keys.append((F'''blocks.{i}.attn.proj.weight''', F'''vit.encoder.layer.{i}.attention.output.dense.weight''')		)
      rename_keys.append((F'''blocks.{i}.attn.proj.bias''', F'''vit.encoder.layer.{i}.attention.output.dense.bias''')		)
      rename_keys.append((F'''blocks.{i}.norm2.weight''', F'''vit.encoder.layer.{i}.layernorm_after.weight''')		)
      rename_keys.append((F'''blocks.{i}.norm2.bias''', F'''vit.encoder.layer.{i}.layernorm_after.bias''')		)
      rename_keys.append((F'''blocks.{i}.mlp.fc1.weight''', F'''vit.encoder.layer.{i}.intermediate.dense.weight''')		)
      rename_keys.append((F'''blocks.{i}.mlp.fc1.bias''', F'''vit.encoder.layer.{i}.intermediate.dense.bias''')		)
      rename_keys.append((F'''blocks.{i}.mlp.fc2.weight''', F'''vit.encoder.layer.{i}.output.dense.weight''')		)
      rename_keys.append((F'''blocks.{i}.mlp.fc2.bias''', F'''vit.encoder.layer.{i}.output.dense.bias''')		)

   # projection layer + position embeddings
   rename_keys.extend(
       [
           ("""cls_token""", """vit.embeddings.cls_token"""),
           ("""patch_embed.proj.weight""", """vit.embeddings.patch_embeddings.projection.weight"""),
           ("""patch_embed.proj.bias""", """vit.embeddings.patch_embeddings.projection.bias"""),
           ("""pos_embed""", """vit.embeddings.position_embeddings"""),
       ]		)

   if base_model:
      # layernorm + pooler
      rename_keys.extend(
          [
              ("""norm.weight""", """layernorm.weight"""),
              ("""norm.bias""", """layernorm.bias"""),
              ("""pre_logits.fc.weight""", """pooler.dense.weight"""),
              ("""pre_logits.fc.bias""", """pooler.dense.bias"""),
          ]		)

      # if just the base model, we should remove "vit" from all keys that start with "vit"
      snake_case 						=   [(pair[0], pair[1][4:]) if pair[1].startswith("""vit"""		) else pair for pair in rename_keys]
   else:
      # layernorm + classification head
      rename_keys.extend(
          [
              ("""norm.weight""", """vit.layernorm.weight"""),
              ("""norm.bias""", """vit.layernorm.bias"""),
              ("""head.weight""", """classifier.weight"""),
              ("""head.bias""", """classifier.bias"""),
          ]		)

   return rename_keys





def 	__lowerCamelCase			(      __lowerCAmelCase       :							Dict      ,    __lowerCAmelCase       :							Optional[Any]      ,    __lowerCAmelCase       :							int=False		)		->  Union[str, Any]:
   for i in range(config.num_hidden_layers		):
      if base_model:
         snake_case 						=   """"""
      else:
         snake_case 						=   """vit."""
      # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
      snake_case 						=   state_dict.pop(F'''blocks.{i}.attn.qkv.weight'''		)
      snake_case 						=   state_dict.pop(F'''blocks.{i}.attn.qkv.bias'''		)
      # next, add query, keys and values (in that order) to the state dict
      snake_case 						=   in_proj_weight[
          : config.hidden_size, :
      ]
      snake_case 						=   in_proj_bias[: config.hidden_size]
      snake_case 						=   in_proj_weight[
          config.hidden_size : config.hidden_size * 2, :
      ]
      snake_case 						=   in_proj_bias[
          config.hidden_size : config.hidden_size * 2
      ]
      snake_case 						=   in_proj_weight[
          -config.hidden_size :, :
      ]
      snake_case 						=   in_proj_bias[-config.hidden_size :]





def 	__lowerCamelCase			(      __lowerCAmelCase       :							List[str]		)		->  List[str]:
   snake_case 						=   ["""head.weight""", """head.bias"""]
   for k in ignore_keys:
      state_dict.pop(__lowerCAmelCase      ,    __lowerCAmelCase		)





def 	__lowerCamelCase			(      __lowerCAmelCase       :							Optional[Any]      ,    __lowerCAmelCase       :							int      ,    __lowerCAmelCase       :							Tuple		)		->  int:
   snake_case 						=   dct.pop(__lowerCAmelCase		)
   snake_case 						=   val





def 	__lowerCamelCase			(      )		->  Optional[Any]:
   snake_case 						=   """http://images.cocodataset.org/val2017/000000039769.jpg"""
   snake_case 						=   Image.open(requests.get(__lowerCAmelCase      ,    stream=__lowerCAmelCase		).raw		)
   return im





@torch.no_grad()
def 	__lowerCamelCase			(      __lowerCAmelCase       :							List[str]      ,    __lowerCAmelCase       :							Any		)		->  Dict:
   snake_case 						=   ViTConfig()
   snake_case 						=   False
   # dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
   if vit_name[-5:] == "in21k":
      snake_case 						=   True
      snake_case 						=   int(vit_name[-12:-10]		)
      snake_case 						=   int(vit_name[-9:-6]		)
   else:
      snake_case 						=   10_00
      snake_case 						=   """huggingface/label-files"""
      snake_case 						=   """imagenet-1k-id2label.json"""
      snake_case 						=   json.load(open(hf_hub_download(__lowerCAmelCase      ,    __lowerCAmelCase      ,    repo_type="""dataset"""		)      ,    """r"""		)		)
      snake_case 						=   {int(__lowerCAmelCase		): v for k, v in idalabel.items()}
      snake_case 						=   idalabel
      snake_case 						=   {v: k for k, v in idalabel.items()}
      snake_case 						=   int(vit_name[-6:-4]		)
      snake_case 						=   int(vit_name[-3:]		)
   # size of the architecture
   if "deit" in vit_name:
      if vit_name[9:].startswith("""tiny"""		):
         snake_case 						=   1_92
         snake_case 						=   7_68
         snake_case 						=   12
         snake_case 						=   3
      elif vit_name[9:].startswith("""small"""		):
         snake_case 						=   3_84
         snake_case 						=   15_36
         snake_case 						=   12
         snake_case 						=   6
      else:
         pass
   else:
      if vit_name[4:].startswith("""small"""		):
         snake_case 						=   7_68
         snake_case 						=   23_04
         snake_case 						=   8
         snake_case 						=   8
      elif vit_name[4:].startswith("""base"""		):
         pass
      elif vit_name[4:].startswith("""large"""		):
         snake_case 						=   10_24
         snake_case 						=   40_96
         snake_case 						=   24
         snake_case 						=   16
      elif vit_name[4:].startswith("""huge"""		):
         snake_case 						=   12_80
         snake_case 						=   51_20
         snake_case 						=   32
         snake_case 						=   16

    # load original model from timm
   snake_case 						=   timm.create_model(__lowerCAmelCase      ,    pretrained=__lowerCAmelCase		)
   timm_model.eval()

   # load state_dict of original model, remove and rename some keys
   snake_case 						=   timm_model.state_dict()
   if base_model:
      remove_classification_head_(__lowerCAmelCase		)
   snake_case 						=   create_rename_keys(__lowerCAmelCase      ,    __lowerCAmelCase		)
   for src, dest in rename_keys:
      rename_key(__lowerCAmelCase      ,    __lowerCAmelCase      ,    __lowerCAmelCase		)
   read_in_q_k_v(__lowerCAmelCase      ,    __lowerCAmelCase      ,    __lowerCAmelCase		)

   # load HuggingFace model
   if vit_name[-5:] == "in21k":
      snake_case 						=   ViTModel(__lowerCAmelCase		).eval()
   else:
      snake_case 						=   ViTForImageClassification(__lowerCAmelCase		).eval()
   model.load_state_dict(__lowerCAmelCase		)

   # Check outputs on an image, prepared by ViTImageProcessor/DeiTImageProcessor
   if "deit" in vit_name:
      snake_case 						=   DeiTImageProcessor(size=config.image_size		)
   else:
      snake_case 						=   ViTImageProcessor(size=config.image_size		)
   snake_case 						=   image_processor(images=prepare_img()      ,    return_tensors="""pt"""		)
   snake_case 						=   encoding["""pixel_values"""]
   snake_case 						=   model(__lowerCAmelCase		)

   if base_model:
      snake_case 						=   timm_model.forward_features(__lowerCAmelCase		)
      assert timm_pooled_output.shape == outputs.pooler_output.shape
      assert torch.allclose(__lowerCAmelCase      ,    outputs.pooler_output      ,    atol=1e-3		)
   else:
      snake_case 						=   timm_model(__lowerCAmelCase		)
      assert timm_logits.shape == outputs.logits.shape
      assert torch.allclose(__lowerCAmelCase      ,    outputs.logits      ,    atol=1e-3		)

   Path(__lowerCAmelCase		).mkdir(exist_ok=__lowerCAmelCase		)
   print(F'''Saving model {vit_name} to {pytorch_dump_folder_path}'''		)
   model.save_pretrained(__lowerCAmelCase		)
   print(F'''Saving image processor to {pytorch_dump_folder_path}'''		)
   image_processor.save_pretrained(__lowerCAmelCase		)


if __name__ == "__main__":
 _SCREAMING_SNAKE_CASE							     =			argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--vit_name",
     default="vit_base_patch16_224",
     type=str,
     help="Name of the ViT timm model you'd like to convert.",
 )
 parser.add_argument(
     "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
 )

 _SCREAMING_SNAKE_CASE							     =			parser.parse_args()
 convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path)

