'''simple docstring'''


import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig


def 					_lowerCamelCase    (      lowerCamelCase_					:							Dict	,     lowerCamelCase_					:							Optional[int]	,     lowerCamelCase_					:							Union[str, Any]=0   ):



     """simple docstring"""





     if name is None:
          UpperCAmelCase_							:   int   =      None
     else:
          UpperCAmelCase_							:   str   =      '.' * max(0	,     spaces - 2   ) + '# {:' + str(50 - spaces   ) + 's}'
          UpperCAmelCase_							:   int   =      fmt.format(lowerCamelCase_   )

     # Print and recurse (if needed).
     if isinstance(lowerCamelCase_	,     lowerCamelCase_   ):
          if msg is not None:
               print(lowerCamelCase_   )
          for k in val.keys():
               recursive_print(lowerCamelCase_	,     val[k]	,     spaces + 2   )
     elif isinstance(lowerCamelCase_	,     torch.Tensor   ):
          print(lowerCamelCase_	,     ':'	,     val.size()   )
     else:
          print(lowerCamelCase_	,     ':'	,     lowerCamelCase_   )


def 					_lowerCamelCase    (      lowerCamelCase_					:							List[str]	,     lowerCamelCase_					:							int	,     lowerCamelCase_					:							Union[str, Any]	,     lowerCamelCase_					:							Optional[Any]   ):



     """simple docstring"""





     UpperCAmelCase_							:   List[Any]   =      param.size()
     # other versions store [num_heads * num_splits * hidden_size, :]
     UpperCAmelCase_							:   str   =      (num_heads, num_splits, hidden_size) + input_shape[1:]
     UpperCAmelCase_							:   Dict   =      param.view(*lowerCamelCase_   )
     UpperCAmelCase_							:   int   =      param.transpose(0	,     1   ).contiguous()
     UpperCAmelCase_							:   Tuple   =      param.view(*lowerCamelCase_   )
     return param


def 					_lowerCamelCase    (      lowerCamelCase_					:							List[Any]	,     lowerCamelCase_					:							str   ):



     """simple docstring"""





     UpperCAmelCase_							:   Dict   =      config.n_positions
     UpperCAmelCase_							:   List[Any]   =      config.n_layer
     UpperCAmelCase_							:   Union[str, Any]   =      config.vocab_size
     UpperCAmelCase_							:   Union[str, Any]   =      config.n_head
     UpperCAmelCase_							:   Optional[int]   =      config.n_embd // config.n_head

     UpperCAmelCase_							:   Union[str, Any]   =      sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
     UpperCAmelCase_							:   List[Any]   =      {
         'transformer.wte.weight': word_embeddings,
         'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
         'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
         'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
     }

     UpperCAmelCase_							:   Any   =      'model.language_model.encoder.layers.'
     for i in range(lowerCamelCase_   ):
          UpperCAmelCase_							:   Any   =      torch.tril(torch.ones((n_positions, n_positions)	,     dtype=torch.bool   )   )
          UpperCAmelCase_							:   List[str]   =      causal_mask.view(1	,     1	,     lowerCamelCase_	,     lowerCamelCase_   )
          UpperCAmelCase_							:   Any   =      causal_mask
          UpperCAmelCase_							:   Tuple   =      torch.tensor(-1e4	,     dtype=torch.bfloataa   )

          UpperCAmelCase_							:   Any   =      sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
          UpperCAmelCase_							:   str   =      sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

          UpperCAmelCase_							:   Union[str, Any]   =      sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
          UpperCAmelCase_							:   Tuple   =      fix_query_key_value_ordering(lowerCamelCase_	,     3	,     lowerCamelCase_	,     lowerCamelCase_   )
          UpperCAmelCase_							:   Dict   =      vala.transpose(0	,     1   ).contiguous()

          UpperCAmelCase_							:   int   =      sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
          UpperCAmelCase_							:   int   =      fix_query_key_value_ordering(lowerCamelCase_	,     3	,     lowerCamelCase_	,     lowerCamelCase_   )
          UpperCAmelCase_							:   Union[str, Any]   =      vala

          UpperCAmelCase_							:   Tuple   =      sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
              0	,     1   )
          UpperCAmelCase_							:   int   =      sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
          UpperCAmelCase_							:   List[str]   =      sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
          UpperCAmelCase_							:   int   =      sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
          UpperCAmelCase_							:   str   =      sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0	,     1   )
          UpperCAmelCase_							:   Optional[Any]   =      sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
          UpperCAmelCase_							:   Optional[Any]   =      sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
              0	,     1   )
          UpperCAmelCase_							:   List[Any]   =      sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

     # For LM head, transformers' wants the matrix to weight embeddings.
     UpperCAmelCase_							:   List[Any]   =      word_embeddings

     return sd_hf


def 					_lowerCamelCase    (      lowerCamelCase_					:							Any	,     lowerCamelCase_					:							List[Any]   ):



     """simple docstring"""





     UpperCAmelCase_							:   Optional[int]   =      64000
     UpperCAmelCase_							:   Optional[Any]   =      config_megatron['encoder_seq_length']
     UpperCAmelCase_							:   List[str]   =      config_megatron['hidden_size']
     UpperCAmelCase_							:   Tuple   =      config_megatron['num_layers']
     UpperCAmelCase_							:   Dict   =      config_megatron['num_attention_heads']
     UpperCAmelCase_							:   Optional[int]   =      config_megatron['ffn_hidden_size']
     UpperCAmelCase_							:   int   =      'gelu'
     UpperCAmelCase_							:   str   =      0.1
     UpperCAmelCase_							:   Optional[Any]   =      0.1
     UpperCAmelCase_							:   Any   =      0.1
     UpperCAmelCase_							:   str   =      config_megatron['layernorm_epsilon']  # 1e-5
     UpperCAmelCase_							:   Dict   =      config_megatron['init_method_std']  # 0.02
     UpperCAmelCase_							:   Union[str, Any]   =      config_megatron['apply_query_key_layer_scaling']  # True
     UpperCAmelCase_							:   Union[str, Any]   =      True
     UpperCAmelCase_							:   int   =      True

     # This identifies the 6.7B (7B) model which uses a different tokenizer
     if config_megatron["hidden_size"] == 4096:
          UpperCAmelCase_							:   Any   =      1  # <|endoftext|>
          UpperCAmelCase_							:   List[str]   =      1  # <|endoftext|>
          UpperCAmelCase_							:   Optional[int]   =      0  # <unk>
     else:
          UpperCAmelCase_							:   Tuple   =      2  # <s>
          UpperCAmelCase_							:   List[Any]   =      3  # <|endoftext|>
          UpperCAmelCase_							:   Optional[Any]   =      0  # <pad>

     return config_hf


def 					_lowerCamelCase    (      lowerCamelCase_					:							Any   ):



     """simple docstring"""





     print(lowerCamelCase_   )

     UpperCAmelCase_							:   str   =      args.checkpoint_path
     UpperCAmelCase_							:   List[Any]   =      args.save_path
     if isfile(lowerCamelCase_   ):
          raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''   )

     # Load the model.
     UpperCAmelCase_							:   Any   =      torch.load(lowerCamelCase_	,     map_location='cpu'   )

     # Load the config.
     UpperCAmelCase_							:   Optional[Any]   =      checkpoint['hyper_parameters']['cfg']
     UpperCAmelCase_							:   int   =      GPTaConfig()
     UpperCAmelCase_							:   List[str]   =      copy_config(config_hf=lowerCamelCase_	,     config_megatron=lowerCamelCase_   )
     UpperCAmelCase_							:   List[str]   =      ['GPT2LMHeadModel']

     UpperCAmelCase_							:   int   =      checkpoint['state_dict']

     # Convert.
     print('Converting'   )
     UpperCAmelCase_							:   List[str]   =      convert_megatron_checkpoint(lowerCamelCase_	,     lowerCamelCase_   )

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(lowerCamelCase_	,     lowerCamelCase_   )

     UpperCAmelCase_							:   Optional[int]   =      'GPTSw3Tokenizer'

     # Store the config to file.
     print('Saving config'   )
     config_hf.save_pretrained(lowerCamelCase_   )

     # Store the state_dict to file.
     UpperCAmelCase_							:   str   =      os.path.join(lowerCamelCase_	,     'pytorch_model.bin'   )
     print(F'''Saving checkpoint to "{output_checkpoint_file}"'''   )
     torch.save(lowerCamelCase_	,     lowerCamelCase_   )


if __name__ == "__main__":
 snake_case__     :			int      			=  argparse.ArgumentParser()
 parser.add_argument(
     '''--checkpoint_path''',
     type=str,
     required=True,
     help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
 )
 parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
 parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
 snake_case__     :			int      			=  parser.parse_args()
 main(_args)







'''simple docstring'''


from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFXLMForMultipleChoice,
     TFXLMForQuestionAnsweringSimple,
     TFXLMForSequenceClassification,
     TFXLMForTokenClassification,
     TFXLMModel,
     TFXLMWithLMHeadModel,
     XLMConfig,
 )






class      __SCREAMING_SNAKE_CASE						:







  '''simple docstring'''






  def __init__( self      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   str   =      parent
       UpperCAmelCase_							:   Tuple   =      1_3
       UpperCAmelCase_							:   Dict   =      7
       UpperCAmelCase_							:   Union[str, Any]   =      True
       UpperCAmelCase_							:   Optional[Any]   =      True
       UpperCAmelCase_							:   Tuple   =      True
       UpperCAmelCase_							:   Dict   =      True
       UpperCAmelCase_							:   List[Any]   =      True
       UpperCAmelCase_							:   Union[str, Any]   =      False
       UpperCAmelCase_							:   Any   =      False
       UpperCAmelCase_							:   List[str]   =      False
       UpperCAmelCase_							:   int   =      2
       UpperCAmelCase_							:   Union[str, Any]   =      9_9
       UpperCAmelCase_							:   Dict   =      0
       UpperCAmelCase_							:   List[str]   =      3_2
       UpperCAmelCase_							:   str   =      2
       UpperCAmelCase_							:   int   =      4
       UpperCAmelCase_							:   Dict   =      0.1
       UpperCAmelCase_							:   int   =      0.1
       UpperCAmelCase_							:   List[str]   =      5_1_2
       UpperCAmelCase_							:   Union[str, Any]   =      1_6
       UpperCAmelCase_							:   List[Any]   =      2
       UpperCAmelCase_							:   int   =      0.02
       UpperCAmelCase_							:   Union[str, Any]   =      3
       UpperCAmelCase_							:   Union[str, Any]   =      4
       UpperCAmelCase_							:   Optional[int]   =      'last'
       UpperCAmelCase_							:   List[Any]   =      True
       UpperCAmelCase_							:   Union[str, Any]   =      None
       UpperCAmelCase_							:   int   =      0


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Tuple   =      ids_tensor([self.batch_size, self.seq_length]      ,		self.vocab_size						)
       UpperCAmelCase_							:   Tuple   =      random_attention_mask([self.batch_size, self.seq_length]      ,		dtype=tf.floataa						)

       UpperCAmelCase_							:   Optional[Any]   =      None
       if self.use_input_lengths:
            UpperCAmelCase_							:   Any   =      (
                ids_tensor([self.batch_size]      ,		vocab_size=2						) + self.seq_length - 2
            )  # small variation of seq_length

       UpperCAmelCase_							:   Optional[Any]   =      None
       if self.use_token_type_ids:
            UpperCAmelCase_							:   Tuple   =      ids_tensor([self.batch_size, self.seq_length]      ,		self.n_langs						)

       UpperCAmelCase_							:   Any   =      None
       UpperCAmelCase_							:   Union[str, Any]   =      None
       UpperCAmelCase_							:   Optional[int]   =      None
       if self.use_labels:
            UpperCAmelCase_							:   Union[str, Any]   =      ids_tensor([self.batch_size]      ,		self.type_sequence_label_size						)
            UpperCAmelCase_							:   Any   =      ids_tensor([self.batch_size, self.seq_length]      ,		self.num_labels						)
            UpperCAmelCase_							:   Union[str, Any]   =      ids_tensor([self.batch_size]      ,		2      ,		dtype=tf.floataa						)
            UpperCAmelCase_							:   Any   =      ids_tensor([self.batch_size]      ,		self.num_choices						)

       UpperCAmelCase_							:   int   =      XLMConfig(
           vocab_size=self.vocab_size      ,		n_special=self.n_special      ,		emb_dim=self.hidden_size      ,		n_layers=self.num_hidden_layers      ,		n_heads=self.num_attention_heads      ,		dropout=self.hidden_dropout_prob      ,		attention_dropout=self.attention_probs_dropout_prob      ,		gelu_activation=self.gelu_activation      ,		sinusoidal_embeddings=self.sinusoidal_embeddings      ,		asm=self.asm      ,		causal=self.causal      ,		n_langs=self.n_langs      ,		max_position_embeddings=self.max_position_embeddings      ,		initializer_range=self.initializer_range      ,		summary_type=self.summary_type      ,		use_proj=self.use_proj      ,		bos_token_id=self.bos_token_id      ,		)

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   Tuple   =      TFXLMModel(config=snake_case_						)
       UpperCAmelCase_							:   int   =      {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
       UpperCAmelCase_							:   Tuple   =      model(snake_case_						)

       UpperCAmelCase_							:   Any   =      [input_ids, input_mask]
       UpperCAmelCase_							:   Optional[Any]   =      model(snake_case_						)
       self.parent.assertEqual(result.last_hidden_state.shape      ,		(self.batch_size, self.seq_length, self.hidden_size)						)


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      TFXLMWithLMHeadModel(snake_case_						)

       UpperCAmelCase_							:   Union[str, Any]   =      {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
       UpperCAmelCase_							:   Optional[Any]   =      model(snake_case_						)

       UpperCAmelCase_							:   Tuple   =      outputs

       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.vocab_size)						)


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      TFXLMForQuestionAnsweringSimple(snake_case_						)

       UpperCAmelCase_							:   Optional[int]   =      {'input_ids': input_ids, 'lengths': input_lengths}

       UpperCAmelCase_							:   Optional[int]   =      model(snake_case_						)

       self.parent.assertEqual(result.start_logits.shape      ,		(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape      ,		(self.batch_size, self.seq_length)						)


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   List[str]   =      TFXLMForSequenceClassification(snake_case_						)

       UpperCAmelCase_							:   Dict   =      {'input_ids': input_ids, 'lengths': input_lengths}

       UpperCAmelCase_							:   Tuple   =      model(snake_case_						)

       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.type_sequence_label_size)						)


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   int   =      self.num_labels
       UpperCAmelCase_							:   List[str]   =      TFXLMForTokenClassification(config=snake_case_						)
       UpperCAmelCase_							:   str   =      {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
       UpperCAmelCase_							:   Optional[int]   =      model(snake_case_						)
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.seq_length, self.num_labels)						)


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		):




       '''simple docstring'''

       UpperCAmelCase_							:   List[Any]   =      self.num_choices
       UpperCAmelCase_							:   Optional[Any]   =      TFXLMForMultipleChoice(config=snake_case_						)
       UpperCAmelCase_							:   Tuple   =      tf.tile(tf.expand_dims(snake_case_      ,		1						)      ,		(1, self.num_choices, 1)						)
       UpperCAmelCase_							:   Dict   =      tf.tile(tf.expand_dims(snake_case_      ,		1						)      ,		(1, self.num_choices, 1)						)
       UpperCAmelCase_							:   str   =      tf.tile(tf.expand_dims(snake_case_      ,		1						)      ,		(1, self.num_choices, 1)						)
       UpperCAmelCase_							:   Tuple   =      {
           'input_ids': multiple_choice_inputs_ids,
           'attention_mask': multiple_choice_input_mask,
           'token_type_ids': multiple_choice_token_type_ids,
       }
       UpperCAmelCase_							:   int   =      model(snake_case_						)
       self.parent.assertEqual(result.logits.shape      ,		(self.batch_size, self.num_choices)						)






  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[int]   =      self.prepare_config_and_inputs()
       (
           (
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			(
           UpperCAmelCase_
       )   ,			
       )							:   Optional[int]   =      config_and_inputs
       UpperCAmelCase_							:   Any   =      {
           'input_ids': input_ids,
           'token_type_ids': token_type_ids,
           'langs': token_type_ids,
           'lengths': input_lengths,
       }
       return config, inputs_dict







@require_tf
class      __SCREAMING_SNAKE_CASE						(       lowerCamelCase_						,	lowerCamelCase_						,	unittest.TestCase       ):







  '''simple docstring'''






  lowerCamelCase_		:List[Any]						    =		(
      (
          TFXLMModel,
          TFXLMWithLMHeadModel,
          TFXLMForSequenceClassification,
          TFXLMForQuestionAnsweringSimple,
          TFXLMForTokenClassification,
          TFXLMForMultipleChoice,
      )
      if is_tf_available()
      else ()
  )
  lowerCamelCase_		:str						    =		(
      (TFXLMWithLMHeadModel,) if is_tf_available() else ()
  )  # TODO (PVP): Check other models whether language generation is also applicable
  lowerCamelCase_		:int						    =		(
      {
          '''feature-extraction''': TFXLMModel,
          '''fill-mask''': TFXLMWithLMHeadModel,
          '''question-answering''': TFXLMForQuestionAnsweringSimple,
          '''text-classification''': TFXLMForSequenceClassification,
          '''text-generation''': TFXLMWithLMHeadModel,
          '''token-classification''': TFXLMForTokenClassification,
          '''zero-shot''': TFXLMForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )
  lowerCamelCase_		:List[Any]						    =		False
  lowerCamelCase_		:Dict						    =		False


  def _UpperCamelCase     ( self      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_      ,		snake_case_						):




       '''simple docstring'''

       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith('Fast'						)
       ):
            # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
            # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
            # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
            return True

       return False


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   str   =      TFXLMModelTester(self						)
       UpperCAmelCase_							:   int   =      ConfigTester(self      ,		config_class=snake_case_      ,		emb_dim=3_7						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       self.config_tester.run_common_tests()


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Any   =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_model(*snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Optional[int]   =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_lm_head(*snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   List[Any]   =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_qa(*snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Tuple   =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_sequence_classif(*snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_token_classification(*snake_case_						)


  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Union[str, Any]   =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_multiple_choice(*snake_case_						)






  @slow
  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            UpperCAmelCase_							:   int   =      TFXLMModel.from_pretrained(snake_case_						)
            self.assertIsNotNone(snake_case_						)







@require_tf
class      __SCREAMING_SNAKE_CASE						(       unittest.TestCase       ):







  '''simple docstring'''






  @slow
  def _UpperCamelCase     ( self						):




       '''simple docstring'''

       UpperCAmelCase_							:   Any   =      TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'						)
       UpperCAmelCase_							:   List[str]   =      tf.convert_to_tensor([[1_4, 4_4_7]]      ,		dtype=tf.intaa						)  # the president
       UpperCAmelCase_							:   Optional[int]   =      [
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
           1_4,
           4_4_7,
       ]  # the president the president the president the president the president the president the president the president the president the president
       # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
       UpperCAmelCase_							:   Optional[int]   =      model.generate(snake_case_      ,		do_sample=snake_case_						)
       self.assertListEqual(output_ids[0].numpy().tolist()      ,		snake_case_						)






