






import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           MegatronBertForCausalLM,
           MegatronBertForMaskedLM,
           MegatronBertForMultipleChoice,
           MegatronBertForNextSentencePrediction,
           MegatronBertForPreTraining,
           MegatronBertForQuestionAnswering,
           MegatronBertForSequenceClassification,
           MegatronBertForTokenClassification,
           MegatronBertModel,
       )







class 		_A						:

  def __init__(    self						:	List[Any]   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Any=13   ,				lowerCamelCase__						:	List[Any]=7   ,				lowerCamelCase__						:	str=True   ,				lowerCamelCase__						:	Optional[int]=True   ,				lowerCamelCase__						:	Any=True   ,				lowerCamelCase__						:	int=True   ,				lowerCamelCase__						:	Optional[Any]=99   ,				lowerCamelCase__						:	List[Any]=64   ,				lowerCamelCase__						:	Tuple=32   ,				lowerCamelCase__						:	str=5   ,				lowerCamelCase__						:	Optional[Any]=4   ,				lowerCamelCase__						:	Optional[Any]=37   ,				lowerCamelCase__						:	Optional[int]="gelu"   ,				lowerCamelCase__						:	Tuple=0.1   ,				lowerCamelCase__						:	int=0.1   ,				lowerCamelCase__						:	Optional[int]=5_12   ,				lowerCamelCase__						:	Tuple=16   ,				lowerCamelCase__						:	List[Any]=2   ,				lowerCamelCase__						:	Dict=0.02   ,				lowerCamelCase__						:	Any=3   ,				lowerCamelCase__						:	Optional[Any]=4   ,				lowerCamelCase__						:	str=None   ,				):
     """simple docstring"""

     __UpperCamelCase	:			Any					=					parent
     __UpperCamelCase	:			Tuple					=					batch_size
     __UpperCamelCase	:			Dict					=					seq_length
     __UpperCamelCase	:			List[Any]					=					is_training
     __UpperCamelCase	:			str					=					use_input_mask
     __UpperCamelCase	:			List[Any]					=					use_token_type_ids
     __UpperCamelCase	:			List[Any]					=					use_labels
     __UpperCamelCase	:			Union[str, Any]					=					vocab_size
     __UpperCamelCase	:			Union[str, Any]					=					hidden_size
     __UpperCamelCase	:			Optional[Any]					=					embedding_size
     __UpperCamelCase	:			Dict					=					num_hidden_layers
     __UpperCamelCase	:			Optional[int]					=					num_attention_heads
     __UpperCamelCase	:			List[str]					=					intermediate_size
     __UpperCamelCase	:			Optional[Any]					=					hidden_act
     __UpperCamelCase	:			Any					=					hidden_dropout_prob
     __UpperCamelCase	:			str					=					attention_probs_dropout_prob
     __UpperCamelCase	:			Union[str, Any]					=					max_position_embeddings
     __UpperCamelCase	:			int					=					type_vocab_size
     __UpperCamelCase	:			Any					=					type_sequence_label_size
     __UpperCamelCase	:			str					=					initializer_range
     __UpperCamelCase	:			int					=					num_labels
     __UpperCamelCase	:			Union[str, Any]					=					num_choices
     __UpperCamelCase	:			int					=					scope

  def    a					(    self						:	Dict			):
     """simple docstring"""

     __UpperCamelCase	:			str					=					ids_tensor([self.batch_size, self.seq_length]   ,				self.vocab_size			)

     __UpperCamelCase	:			Optional[int]					=					None
     if self.use_input_mask:
        __UpperCamelCase	:			Optional[Any]					=					random_attention_mask([self.batch_size, self.seq_length]			)

     __UpperCamelCase	:			str					=					None
     if self.use_token_type_ids:
        __UpperCamelCase	:			Optional[Any]					=					ids_tensor([self.batch_size, self.seq_length]   ,				self.type_vocab_size			)

     __UpperCamelCase	:			List[str]					=					None
     __UpperCamelCase	:			List[str]					=					None
     __UpperCamelCase	:			Dict					=					None
     if self.use_labels:
        __UpperCamelCase	:			Tuple					=					ids_tensor([self.batch_size]   ,				self.type_sequence_label_size			)
        __UpperCamelCase	:			str					=					ids_tensor([self.batch_size, self.seq_length]   ,				self.num_labels			)
        __UpperCamelCase	:			str					=					ids_tensor([self.batch_size]   ,				self.num_choices			)

     __UpperCamelCase	:			List[str]					=					self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

  def    a					(    self						:	int			):
     """simple docstring"""

     return MegatronBertConfig(
         vocab_size=self.vocab_size   ,				hidden_size=self.hidden_size   ,				num_hidden_layers=self.num_hidden_layers   ,				num_attention_heads=self.num_attention_heads   ,				intermediate_size=self.intermediate_size   ,				embedding_size=self.embedding_size   ,				hidden_act=self.hidden_act   ,				hidden_dropout_prob=self.hidden_dropout_prob   ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,				max_position_embeddings=self.max_position_embeddings   ,				type_vocab_size=self.type_vocab_size   ,				is_decoder=lowerCamelCase__   ,				initializer_range=self.initializer_range   ,				)

  def    a					(    self						:	Dict   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	int   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	int			):
     """simple docstring"""

     __UpperCamelCase	:			Tuple					=					MegatronBertModel(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			str					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__			)
     __UpperCamelCase	:			str					=					model(lowerCamelCase__   ,				token_type_ids=lowerCamelCase__			)
     __UpperCamelCase	:			List[Any]					=					model(lowerCamelCase__			)

     self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.seq_length, self.hidden_size)			)
     self.parent.assertEqual(result.pooler_output.shape   ,				(self.batch_size, self.hidden_size)			)

  def    a					(    self						:	Tuple   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	str			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[int]					=					MegatronBertForMaskedLM(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Union[str, Any]					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size)			)

  def    a					(    self						:	int   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	Optional[Any]			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					MegatronBertForCausalLM(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			str					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size)			)

  def    a					(    self						:	Tuple   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	Optional[Any]   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	int   ,				lowerCamelCase__						:	Tuple			):
     """simple docstring"""

     __UpperCamelCase	:			Dict					=					MegatronBertForNextSentencePrediction(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			str					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__   ,				)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, 2)			)

  def    a					(    self						:	Any   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	int   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	int   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Tuple			):
     """simple docstring"""

     __UpperCamelCase	:			Dict					=					MegatronBertForPreTraining(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Optional[int]					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__   ,				next_sentence_label=lowerCamelCase__   ,				)
     self.parent.assertEqual(result.prediction_logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size)			)
     self.parent.assertEqual(result.seq_relationship_logits.shape   ,				(self.batch_size, 2)			)

  def    a					(    self						:	Tuple   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Optional[int]			):
     """simple docstring"""

     __UpperCamelCase	:			str					=					MegatronBertForQuestionAnswering(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Optional[int]					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				start_positions=lowerCamelCase__   ,				end_positions=lowerCamelCase__   ,				)
     self.parent.assertEqual(result.start_logits.shape   ,				(self.batch_size, self.seq_length)			)
     self.parent.assertEqual(result.end_logits.shape   ,				(self.batch_size, self.seq_length)			)

  def    a					(    self						:	Any   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	List[str]   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	Optional[Any]   ,				lowerCamelCase__						:	str			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					self.num_labels
     __UpperCamelCase	:			Optional[int]					=					MegatronBertForSequenceClassification(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Dict					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.num_labels)			)

  def    a					(    self						:	Union[str, Any]   ,				lowerCamelCase__						:	int   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	List[str]   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			Any					=					self.num_labels
     __UpperCamelCase	:			Optional[int]					=					MegatronBertForTokenClassification(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Optional[int]					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.num_labels)			)

  def    a					(    self						:	List[Any]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	List[str]			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[int]					=					self.num_choices
     __UpperCamelCase	:			List[str]					=					MegatronBertForMultipleChoice(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Any					=					input_ids.unsqueeze(1			).expand(-1   ,				self.num_choices   ,				-1			).contiguous()
     __UpperCamelCase	:			List[Any]					=					token_type_ids.unsqueeze(1			).expand(-1   ,				self.num_choices   ,				-1			).contiguous()
     __UpperCamelCase	:			int					=					input_mask.unsqueeze(1			).expand(-1   ,				self.num_choices   ,				-1			).contiguous()
     __UpperCamelCase	:			Dict					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				labels=lowerCamelCase__   ,				)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.num_choices)			)

  def    a					(    self						:	List[Any]			):
     """simple docstring"""

     __UpperCamelCase	:			List[str]					=					self.prepare_config_and_inputs()
     (
         (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     
     )	:			str					=					config_and_inputs
     __UpperCamelCase	:			List[str]					=					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
     return config, inputs_dict







@require_torch
class 		_A						( UpperCAmelCase_	,				UpperCAmelCase_	,				unittest.TestCase ):
  lowercase_						:			Union[str, Any]						=					(
      (
          MegatronBertModel,
          MegatronBertForMaskedLM,
          MegatronBertForCausalLM,
          MegatronBertForMultipleChoice,
          MegatronBertForNextSentencePrediction,
          MegatronBertForPreTraining,
          MegatronBertForQuestionAnswering,
          MegatronBertForSequenceClassification,
          MegatronBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  lowercase_						:			Union[str, Any]						=					(
      {
          '''feature-extraction''': MegatronBertModel,
          '''fill-mask''': MegatronBertForMaskedLM,
          '''question-answering''': MegatronBertForQuestionAnswering,
          '''text-classification''': MegatronBertForSequenceClassification,
          '''text-generation''': MegatronBertForCausalLM,
          '''token-classification''': MegatronBertForTokenClassification,
          '''zero-shot''': MegatronBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  lowercase_						:			str						=					True
  # test_resize_embeddings = False
  lowercase_						:			Optional[Any]						=					False

  def    a					(    self						:	List[str]   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	List[str]   ,				lowerCamelCase__						:	List[Any]=False			):
     """simple docstring"""

     __UpperCamelCase	:			List[str]					=					super()._prepare_for_class(lowerCamelCase__   ,				lowerCamelCase__   ,				return_labels=lowerCamelCase__			)

     if return_labels:
        if model_class in get_values(lowerCamelCase__			):
           __UpperCamelCase	:			int					=					torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)   ,				dtype=torch.long   ,				device=lowerCamelCase__			)
           __UpperCamelCase	:			Union[str, Any]					=					torch.zeros(
               self.model_tester.batch_size   ,				dtype=torch.long   ,				device=lowerCamelCase__			)
     return inputs_dict

  def    a					(    self						:	str			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[Any]					=					MegatronBertModelTester(self			)
     __UpperCamelCase	:			Tuple					=					ConfigTester(self   ,				config_class=lowerCamelCase__   ,				hidden_size=37			)

  def    a					(    self						:	int			):
     """simple docstring"""

     self.config_tester.run_common_tests()

  def    a					(    self						:	Optional[int]			):
     """simple docstring"""

     __UpperCamelCase	:			Any					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_model(*lowerCamelCase__			)

  def    a					(    self						:	Optional[int]			):
     """simple docstring"""

     __UpperCamelCase	:			Union[str, Any]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_masked_lm(*lowerCamelCase__			)

  def    a					(    self						:	str			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[int]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*lowerCamelCase__			)

  def    a					(    self						:	Any			):
     """simple docstring"""

     __UpperCamelCase	:			Union[str, Any]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*lowerCamelCase__			)

  def    a					(    self						:	Tuple			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_pretraining(*lowerCamelCase__			)

  def    a					(    self						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_question_answering(*lowerCamelCase__			)

  def    a					(    self						:	int			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*lowerCamelCase__			)

  def    a					(    self						:	Optional[Any]			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_megatron_bert_for_token_classification(*lowerCamelCase__			)







def 				__lowerCamelCase     (							__lowerCAmelCase							:					str							)     ->  Any:
   return torch.tensor(
       __lowerCAmelCase					,							dtype=torch.long					,							device=__lowerCAmelCase					,							)


UpperCamelCase    							=      1e-4







@require_torch
@require_sentencepiece
@require_tokenizers
class 		_A						( unittest.TestCase ):

  @slow
  @unittest.skip("""Model is not available."""			)
  def    a					(    self						:	Optional[Any]			):
     """simple docstring"""

     __UpperCamelCase	:			Any					=					"""nvidia/megatron-bert-uncased-345m"""
     if "MYDIR" in os.environ:
        __UpperCamelCase	:			List[str]					=					os.path.join(os.environ["""MYDIR"""]   ,				lowerCamelCase__			)
     __UpperCamelCase	:			int					=					MegatronBertModel.from_pretrained(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.half()
     __UpperCamelCase	:			int					=					_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]			)
     with torch.no_grad():
        __UpperCamelCase	:			Tuple					=					model(lowerCamelCase__			)[0]
     __UpperCamelCase	:			Tuple					=					torch.Size((1, 9, 10_24)			)
     self.assertEqual(output.shape   ,				lowerCamelCase__			)

     __UpperCamelCase	:			Any					=					[-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
     for ii in range(3			):
        for jj in range(3			):
           __UpperCamelCase	:			Any					=					output[0, ii, jj]
           __UpperCamelCase	:			Dict					=					expected[3 * ii + jj]
           __UpperCamelCase	:			str					=					"""ii={} jj={} a={} b={}""".format(lowerCamelCase__   ,				lowerCamelCase__   ,				lowerCamelCase__   ,				lowerCamelCase__			)
           self.assertTrue(math.isclose(lowerCamelCase__   ,				lowerCamelCase__   ,				rel_tol=lowerCamelCase__   ,				abs_tol=lowerCamelCase__			)   ,				msg=lowerCamelCase__			)











import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


UpperCamelCase    							=      logging.get_logger(__name__)


def 				__lowerCamelCase     (							__lowerCAmelCase							:					Any					,							__lowerCAmelCase							:					Any					,							__lowerCAmelCase							:					Any					,							__lowerCAmelCase							:					List[Any]=False							)     ->  Union[str, Any]:
   try:
      import torch  # noqa: F401
   except ImportError:
      logger.error(
          """Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see"""
          """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
          """ instructions."""							)
      raise

   if not is_sharded:
      __UpperCamelCase	:			Tuple					=					os.path.abspath(__lowerCAmelCase							)
      logger.info(f'Loading PyTorch weights from {pt_path}'							)

      __UpperCamelCase	:			int					=					torch.load(__lowerCAmelCase					,							map_location="""cpu"""							)
      logger.info(f'PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()							):,} parameters.'							)

      __UpperCamelCase	:			List[Any]					=					convert_pytorch_state_dict_to_flax(__lowerCAmelCase					,							__lowerCAmelCase							)
   else:
      # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
      __UpperCamelCase	:			List[str]					=					convert_pytorch_sharded_state_dict_to_flax(__lowerCAmelCase					,							__lowerCAmelCase							)
   return flax_state_dict


def 				__lowerCamelCase     (							__lowerCAmelCase							:					Tuple[str]					,							__lowerCAmelCase							:					np.ndarray					,							__lowerCAmelCase							:					Dict[str, jnp.ndarray]					,							__lowerCAmelCase							:					str					,							)     ->  (Tuple[str], np.ndarray):

   def is_key_or_prefix_key_in_dict(__lowerCAmelCase							:					Tuple[str]							) -> bool:
      return len(set(__lowerCAmelCase							) & {key, (model_prefix,) + key}							) > 0

   # layer norm
   __UpperCamelCase	:			Optional[int]					=					pt_tuple_key[:-1] + ("""scale""",)
   if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(__lowerCAmelCase							):
      return renamed_pt_tuple_key, pt_tensor

   # batch norm layer mean
   __UpperCamelCase	:			Optional[Any]					=					pt_tuple_key[:-1] + ("""mean""",)
   if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(__lowerCAmelCase							):
      return renamed_pt_tuple_key, pt_tensor

   # batch norm layer var
   __UpperCamelCase	:			Optional[Any]					=					pt_tuple_key[:-1] + ("""var""",)
   if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(__lowerCAmelCase							):
      return renamed_pt_tuple_key, pt_tensor

   # embedding
   __UpperCamelCase	:			Union[str, Any]					=					pt_tuple_key[:-1] + ("""embedding""",)
   if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(__lowerCAmelCase							):
      return renamed_pt_tuple_key, pt_tensor

   # conv layer
   __UpperCamelCase	:			Tuple					=					pt_tuple_key[:-1] + ("""kernel""",)
   if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(__lowerCAmelCase							):
      __UpperCamelCase	:			Optional[int]					=					pt_tensor.transpose(2					,							3					,							1					,							0							)
      return renamed_pt_tuple_key, pt_tensor

   # linear layer
   __UpperCamelCase	:			int					=					pt_tuple_key[:-1] + ("""kernel""",)
   if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(__lowerCAmelCase							):
      __UpperCamelCase	:			Optional[int]					=					pt_tensor.T
      return renamed_pt_tuple_key, pt_tensor

   # old PyTorch layer norm weight
   __UpperCamelCase	:			Tuple					=					pt_tuple_key[:-1] + ("""weight""",)
   if pt_tuple_key[-1] == "gamma":
      return renamed_pt_tuple_key, pt_tensor

   # old PyTorch layer norm bias
   __UpperCamelCase	:			str					=					pt_tuple_key[:-1] + ("""bias""",)
   if pt_tuple_key[-1] == "beta":
      return renamed_pt_tuple_key, pt_tensor

   # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
   __UpperCamelCase	:			Optional[int]					=					None
   if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
      __UpperCamelCase	:			Tuple					=					pt_tuple_key[-2] + """_g"""
   elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
      __UpperCamelCase	:			Dict					=					pt_tuple_key[-2] + """_v"""
   if name is not None:
      __UpperCamelCase	:			Optional[Any]					=					pt_tuple_key[:-3] + (name,)
      return renamed_pt_tuple_key, pt_tensor

   return pt_tuple_key, pt_tensor


def 				__lowerCamelCase     (							__lowerCAmelCase							:					int					,							__lowerCAmelCase							:					int							)     ->  Union[str, Any]:
   # convert pytorch tensor to numpy
   __UpperCamelCase	:			Optional[Any]					=					{k: v.numpy() for k, v in pt_state_dict.items()}

   __UpperCamelCase	:			List[Any]					=					flax_model.base_model_prefix

   # use params dict if the model contains batch norm layers
   if "params" in flax_model.params:
      __UpperCamelCase	:			Union[str, Any]					=					flax_model.params["""params"""]
   else:
      __UpperCamelCase	:			List[Any]					=					flax_model.params
   __UpperCamelCase	:			List[Any]					=					flatten_dict(__lowerCAmelCase							)

   # add batch_stats keys,values to dict
   if "batch_stats" in flax_model.params:
      __UpperCamelCase	:			List[str]					=					flatten_dict(flax_model.params["""batch_stats"""]							)
      random_flax_state_dict.update(__lowerCAmelCase							)

   __UpperCamelCase	:			List[Any]					=					{}

   __UpperCamelCase	:			int					=					(model_prefix not in flax_model_params) and (
       model_prefix in {k.split("""."""							)[0] for k in pt_state_dict.keys()}
   )
   __UpperCamelCase	:			int					=					(model_prefix in flax_model_params) and (
       model_prefix not in {k.split("""."""							)[0] for k in pt_state_dict.keys()}
   )

   # Need to change some parameters name to match Flax names
   for pt_key, pt_tensor in pt_state_dict.items():
      __UpperCamelCase	:			List[str]					=					tuple(pt_key.split("""."""							)							)

      # remove base model prefix if necessary
      __UpperCamelCase	:			Optional[Any]					=					pt_tuple_key[0] == model_prefix
      if load_model_with_head_into_base_model and has_base_model_prefix:
         __UpperCamelCase	:			List[str]					=					pt_tuple_key[1:]

      # Correctly rename weight parameters
      __UpperCamelCase	,     __UpperCamelCase	:			Optional[int]					=					rename_key_and_reshape_tensor(
          __lowerCAmelCase					,							__lowerCAmelCase					,							__lowerCAmelCase					,							__lowerCAmelCase							)

      # add model prefix if necessary
      __UpperCamelCase	:			Optional[Any]					=					(model_prefix,) + flax_key in random_flax_state_dict
      if load_base_model_into_model_with_head and require_base_model_prefix:
         __UpperCamelCase	:			Dict					=					(model_prefix,) + flax_key

      if flax_key in random_flax_state_dict:
         if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
            raise ValueError(
                f'PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '
                f'{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'							)

        # add batch stats if the model contains batchnorm layers
      if "batch_stats" in flax_model.params:
         if "mean" in flax_key[-1] or "var" in flax_key[-1]:
            __UpperCamelCase	:			int					=					jnp.asarray(__lowerCAmelCase							)
            continue
         # remove num_batches_tracked key
         if "num_batches_tracked" in flax_key[-1]:
            flax_state_dict.pop(__lowerCAmelCase					,							__lowerCAmelCase							)
            continue

         # also add unexpected weight so that warning is thrown
         __UpperCamelCase	:			Optional[Any]					=					jnp.asarray(__lowerCAmelCase							)

      else:
         # also add unexpected weight so that warning is thrown
         __UpperCamelCase	:			Optional[int]					=					jnp.asarray(__lowerCAmelCase							)

   return unflatten_dict(__lowerCAmelCase							)


def 				__lowerCamelCase     (							__lowerCAmelCase							:					List[str]					,							__lowerCAmelCase							:					int							)     ->  Dict:
   import torch

   # Load the index
   __UpperCamelCase	:			List[str]					=					{}
   for shard_file in shard_filenames:
      # load using msgpack utils
      __UpperCamelCase	:			List[str]					=					torch.load(__lowerCAmelCase							)
      __UpperCamelCase	:			int					=					{k: v.numpy() for k, v in pt_state_dict.items()}

      __UpperCamelCase	:			List[str]					=					flax_model.base_model_prefix

      # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
      if "batch_stats" in flax_model.params:
         __UpperCamelCase	:			Optional[Any]					=					flax_model.params["""params"""]

         __UpperCamelCase	:			int					=					flatten_dict(__lowerCAmelCase							)
         random_flax_state_dict.update(flatten_dict(flax_model.params["""batch_stats"""]							)							)
      else:
         __UpperCamelCase	:			Dict					=					flax_model.params
         __UpperCamelCase	:			int					=					flatten_dict(__lowerCAmelCase							)

      __UpperCamelCase	:			Union[str, Any]					=					(model_prefix not in flax_model_params) and (
          model_prefix in {k.split("""."""							)[0] for k in pt_state_dict.keys()}
      )
      __UpperCamelCase	:			Any					=					(model_prefix in flax_model_params) and (
          model_prefix not in {k.split("""."""							)[0] for k in pt_state_dict.keys()}
      )
      # Need to change some parameters name to match Flax names
      for pt_key, pt_tensor in pt_state_dict.items():
         __UpperCamelCase	:			Any					=					tuple(pt_key.split("""."""							)							)

         # remove base model prefix if necessary
         __UpperCamelCase	:			Dict					=					pt_tuple_key[0] == model_prefix
         if load_model_with_head_into_base_model and has_base_model_prefix:
            __UpperCamelCase	:			str					=					pt_tuple_key[1:]

         # Correctly rename weight parameters
         __UpperCamelCase	,     __UpperCamelCase	:			Optional[Any]					=					rename_key_and_reshape_tensor(
             __lowerCAmelCase					,							__lowerCAmelCase					,							__lowerCAmelCase					,							__lowerCAmelCase							)
         # add model prefix if necessary
         __UpperCamelCase	:			Tuple					=					(model_prefix,) + flax_key in random_flax_state_dict
         if load_base_model_into_model_with_head and require_base_model_prefix:
            __UpperCamelCase	:			Optional[Any]					=					(model_prefix,) + flax_key

         if flax_key in random_flax_state_dict:
            if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
               raise ValueError(
                   f'PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '
                   f'{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'							)

            # add batch stats if the model contains batchnorm layers
         if "batch_stats" in flax_model.params:
            if "mean" in flax_key[-1]:
               __UpperCamelCase	:			List[Any]					=					jnp.asarray(__lowerCAmelCase							)
               continue
            if "var" in flax_key[-1]:
               __UpperCamelCase	:			Optional[Any]					=					jnp.asarray(__lowerCAmelCase							)
               continue
            # remove num_batches_tracked key
            if "num_batches_tracked" in flax_key[-1]:
               flax_state_dict.pop(__lowerCAmelCase					,							__lowerCAmelCase							)
               continue

            # also add unexpected weight so that warning is thrown
            __UpperCamelCase	:			int					=					jnp.asarray(__lowerCAmelCase							)

         else:
            # also add unexpected weight so that warning is thrown
            __UpperCamelCase	:			int					=					jnp.asarray(__lowerCAmelCase							)
   return unflatten_dict(__lowerCAmelCase							)


def 				__lowerCamelCase     (							__lowerCAmelCase							:					Any					,							__lowerCAmelCase							:					Union[str, Any]							)     ->  Union[str, Any]:
   __UpperCamelCase	:			List[Any]					=					os.path.abspath(__lowerCAmelCase							)
   logger.info(f'Loading Flax weights from {flax_checkpoint_path}'							)

   # import correct flax class
   __UpperCamelCase	:			str					=					getattr(__lowerCAmelCase					,							"""Flax""" + model.__class__.__name__							)

   # load flax weight dict
   with open(__lowerCAmelCase					,							"""rb"""							) as state_f:
      try:
         __UpperCamelCase	:			Optional[int]					=					from_bytes(__lowerCAmelCase					,							state_f.read()							)
      except UnpicklingError:
         raise EnvironmentError(f'Unable to convert {flax_checkpoint_path} to Flax deserializable object. '							)

   return load_flax_weights_in_pytorch_model(__lowerCAmelCase					,							__lowerCAmelCase							)



def 				__lowerCamelCase     (							__lowerCAmelCase							:					Optional[int]					,							__lowerCAmelCase							:					Optional[int]							)     ->  List[Any]:

   try:
      import torch  # noqa: F401
   except ImportError:
      logger.error(
          """Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see"""
          """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
          """ instructions."""							)
      raise

   # check if we have bf16 weights
   __UpperCamelCase	:			Tuple					=					flatten_dict(jax.tree_util.tree_map(lambda __lowerCAmelCase							: x.dtype == jnp.bfloataa					,							__lowerCAmelCase							)							).values()
   if any(__lowerCAmelCase							):
      # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
      # and bf16 is not fully supported in PT yet.
      logger.warning(
          """Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` """
          """before loading those in PyTorch model."""							)
      __UpperCamelCase	:			List[Any]					=					jax.tree_util.tree_map(
          lambda __lowerCAmelCase							: params.astype(np.floataa							) if params.dtype == jnp.bfloataa else params					,							__lowerCAmelCase							)

   __UpperCamelCase	:			Tuple					=					flatten_dict(__lowerCAmelCase							)
   __UpperCamelCase	:			int					=					pt_model.state_dict()

   __UpperCamelCase	:			Tuple					=					(pt_model.base_model_prefix in flax_state) and (
       pt_model.base_model_prefix not in {k.split("""."""							)[0] for k in pt_model_dict.keys()}
   )
   __UpperCamelCase	:			Any					=					(pt_model.base_model_prefix not in flax_state) and (
       pt_model.base_model_prefix in {k.split("""."""							)[0] for k in pt_model_dict.keys()}
   )

   # keep track of unexpected & missing keys
   __UpperCamelCase	:			Dict					=					[]
   __UpperCamelCase	:			List[str]					=					set(pt_model_dict.keys()							)

   for flax_key_tuple, flax_tensor in flax_state_dict.items():
      __UpperCamelCase	:			Union[str, Any]					=					flax_key_tuple[0] == pt_model.base_model_prefix
      __UpperCamelCase	:			Optional[Any]					=					""".""".join((pt_model.base_model_prefix,) + flax_key_tuple							) in pt_model_dict

      # adapt flax_key to prepare for loading from/to base model only
      if load_model_with_head_into_base_model and has_base_model_prefix:
         __UpperCamelCase	:			List[Any]					=					flax_key_tuple[1:]
      elif load_base_model_into_model_with_head and require_base_model_prefix:
         __UpperCamelCase	:			Tuple					=					(pt_model.base_model_prefix,) + flax_key_tuple

      # rename flax weights to PyTorch format
      if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(__lowerCAmelCase							) not in pt_model_dict:
         # conv layer
         __UpperCamelCase	:			Optional[int]					=					flax_key_tuple[:-1] + ("""weight""",)
         __UpperCamelCase	:			Dict					=					jnp.transpose(__lowerCAmelCase					,							(3, 2, 0, 1)							)
      elif flax_key_tuple[-1] == "kernel" and ".".join(__lowerCAmelCase							) not in pt_model_dict:
         # linear layer
         __UpperCamelCase	:			Optional[int]					=					flax_key_tuple[:-1] + ("""weight""",)
         __UpperCamelCase	:			List[Any]					=					flax_tensor.T
      elif flax_key_tuple[-1] in ["scale", "embedding"]:
         __UpperCamelCase	:			Union[str, Any]					=					flax_key_tuple[:-1] + ("""weight""",)

      # adding batch stats from flax batch norm to pt
      elif "mean" in flax_key_tuple[-1]:
         __UpperCamelCase	:			List[str]					=					flax_key_tuple[:-1] + ("""running_mean""",)
      elif "var" in flax_key_tuple[-1]:
         __UpperCamelCase	:			Tuple					=					flax_key_tuple[:-1] + ("""running_var""",)

      if "batch_stats" in flax_state:
         __UpperCamelCase	:			Optional[int]					=					""".""".join(flax_key_tuple[1:]							)  # Remove the params/batch_stats header
      else:
         __UpperCamelCase	:			Optional[int]					=					""".""".join(__lowerCAmelCase							)

      # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
      __UpperCamelCase	:			Union[str, Any]					=					{}
      # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
      for key in pt_model_dict:
         __UpperCamelCase	:			int					=					key.split("""."""							)
         __UpperCamelCase	:			str					=					None
         if key_components[-3::2] == ["parametrizations", "original0"]:
            __UpperCamelCase	:			Union[str, Any]					=					key_components[-2] + """_g"""
         elif key_components[-3::2] == ["parametrizations", "original1"]:
            __UpperCamelCase	:			Tuple					=					key_components[-2] + """_v"""
         if name is not None:
            __UpperCamelCase	:			Optional[Any]					=					key_components[:-3] + [name]
            __UpperCamelCase	:			int					=					""".""".join(__lowerCAmelCase							)
            __UpperCamelCase	:			List[str]					=					key

      if flax_key in special_pt_names:
         __UpperCamelCase	:			Any					=					special_pt_names[flax_key]

      if flax_key in pt_model_dict:
         if flax_tensor.shape != pt_model_dict[flax_key].shape:
            raise ValueError(
                f'Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '
                f'to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'							)
         else:
            # add weight to pytorch dict
            __UpperCamelCase	:			str					=					np.asarray(__lowerCAmelCase							) if not isinstance(__lowerCAmelCase					,							np.ndarray							) else flax_tensor
            __UpperCamelCase	:			Optional[int]					=					torch.from_numpy(__lowerCAmelCase							)
            # remove from missing keys
            missing_keys.remove(__lowerCAmelCase							)
      else:
         # weight is not expected by PyTorch model
         unexpected_keys.append(__lowerCAmelCase							)

   pt_model.load_state_dict(__lowerCAmelCase							)

   # re-transform missing_keys to list
   __UpperCamelCase	:			Union[str, Any]					=					list(__lowerCAmelCase							)

   if len(__lowerCAmelCase							) > 0:
      logger.warning(
          """Some weights of the Flax model were not used when initializing the PyTorch model"""
          f' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'
          f' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'
          """ (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This"""
          f' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'
          """ to be exactly identical (e.g. initializing a BertForSequenceClassification model from a"""
          """ FlaxBertForSequenceClassification model)."""							)
   else:
      logger.warning(f'All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n'							)
   if len(__lowerCAmelCase							) > 0:
      logger.warning(
          f'Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'
          f' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'
          """ use it for predictions and inference."""							)
   else:
      logger.warning(
          f'All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n'
          """If your task is similar to the task the model of the checkpoint was trained on, """
          f'you can already use {pt_model.__class__.__name__} for predictions without further training.'							)

   return pt_model



