

import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MPNetForMaskedLM,
         MPNetForMultipleChoice,
         MPNetForQuestionAnswering,
         MPNetForSequenceClassification,
         MPNetForTokenClassification,
         MPNetModel,
     )


class 			A						:







    """simple docstring"""





    def __init__(				self			:	Dict,lowercase_			:	Any,lowercase_			:	Optional[Any]=1_3,lowercase_			:	str=7,lowercase_			:	str=True,lowercase_			:	Dict=True,lowercase_			:	Optional[int]=False,lowercase_			:	Any=True,lowercase_			:	Any=9_9,lowercase_			:	Union[str, Any]=6_4,lowercase_			:	Any=5,lowercase_			:	int=4,lowercase_			:	Tuple=6_4,lowercase_			:	str="gelu",lowercase_			:	List[Any]=0.1,lowercase_			:	Union[str, Any]=0.1,lowercase_			:	int=5_1_2,lowercase_			:	List[Any]=1_6,lowercase_			:	Union[str, Any]=2,lowercase_			:	Union[str, Any]=0.02,lowercase_			:	Optional[Any]=3,lowercase_			:	Union[str, Any]=4,lowercase_			:	Any=None,)->   Optional[int]:





        '''simple docstring'''







        A__   				=       parent
        A__   				=       batch_size
        A__   				=       seq_length
        A__   				=       is_training
        A__   				=       use_input_mask
        A__   				=       use_token_type_ids
        A__   				=       use_labels
        A__   				=       vocab_size
        A__   				=       hidden_size
        A__   				=       num_hidden_layers
        A__   				=       num_attention_heads
        A__   				=       intermediate_size
        A__   				=       hidden_act
        A__   				=       hidden_dropout_prob
        A__   				=       attention_probs_dropout_prob
        A__   				=       max_position_embeddings
        A__   				=       type_vocab_size
        A__   				=       type_sequence_label_size
        A__   				=       initializer_range
        A__   				=       num_labels
        A__   				=       num_choices
        A__   				=       scope


    def 						snake_case__					(				self			:	Tuple		)->   Any:





        '''simple docstring'''







        return MPNetConfig.from_pretrained('microsoft/mpnet-base'		)


    def 						snake_case__					(				self			:	Any		)->   int:





        '''simple docstring'''







        A__   				=       ids_tensor([self.batch_size, self.seq_length],self.vocab_size		)

        A__   				=       None
        if self.use_input_mask:
            A__   				=       random_attention_mask([self.batch_size, self.seq_length]		)

        A__   				=       None
        A__   				=       None
        A__   				=       None
        if self.use_labels:
            A__   				=       ids_tensor([self.batch_size],self.type_sequence_label_size		)
            A__   				=       ids_tensor([self.batch_size, self.seq_length],self.num_labels		)
            A__   				=       ids_tensor([self.batch_size],self.num_choices		)

        A__   				=       self.get_config()
        return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


    def 						snake_case__					(				self			:	List[str]		)->   Any:





        '''simple docstring'''







        return MPNetConfig(
            vocab_size=self.vocab_size,hidden_size=self.hidden_size,num_hidden_layers=self.num_hidden_layers,num_attention_heads=self.num_attention_heads,intermediate_size=self.intermediate_size,hidden_act=self.hidden_act,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,max_position_embeddings=self.max_position_embeddings,initializer_range=self.initializer_range,)


    def 						snake_case__					(				self			:	Dict,lowercase_			:	Optional[Any],lowercase_			:	Any,lowercase_			:	str,lowercase_			:	List[Any],lowercase_			:	Dict,lowercase_			:	Tuple		)->   Optional[Any]:





        '''simple docstring'''







        A__   				=       MPNetModel(config=lowerCAmelCase__		)
        model.to(lowerCAmelCase__		)
        model.eval()
        A__   				=       model(lowerCAmelCase__,lowerCAmelCase__		)
        A__   				=       model(lowerCAmelCase__		)
        self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.hidden_size)		)
        self.parent.assertEqual(result.pooler_output.shape,(self.batch_size, self.hidden_size)		)


    def 						snake_case__					(				self			:	Dict,lowercase_			:	Tuple,lowercase_			:	Tuple,lowercase_			:	Optional[Any],lowercase_			:	List[Any],lowercase_			:	str,lowercase_			:	int		)->   int:





        '''simple docstring'''







        A__   				=       MPNetForQuestionAnswering(config=lowerCAmelCase__		)
        model.to(lowerCAmelCase__		)
        model.eval()
        A__   				=       model(
            lowerCAmelCase__,attention_mask=lowerCAmelCase__,start_positions=lowerCAmelCase__,end_positions=lowerCAmelCase__,)
        self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)		)


    def 						snake_case__					(				self			:	List[str],lowercase_			:	List[str],lowercase_			:	List[str],lowercase_			:	Any,lowercase_			:	Optional[Any],lowercase_			:	List[str],lowercase_			:	int		)->   Any:





        '''simple docstring'''







        A__   				=       self.num_labels
        A__   				=       MPNetForSequenceClassification(lowerCAmelCase__		)
        model.to(lowerCAmelCase__		)
        model.eval()
        A__   				=       model(lowerCAmelCase__,attention_mask=lowerCAmelCase__,labels=lowerCAmelCase__		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)		)


    def 						snake_case__					(				self			:	Any,lowercase_			:	Union[str, Any],lowercase_			:	Union[str, Any],lowercase_			:	int,lowercase_			:	Tuple,lowercase_			:	Optional[Any],lowercase_			:	int		)->   str:





        '''simple docstring'''







        A__   				=       self.num_choices
        A__   				=       MPNetForMultipleChoice(config=lowerCAmelCase__		)
        model.to(lowerCAmelCase__		)
        model.eval()
        A__   				=       input_ids.unsqueeze(1		).expand(-1,self.num_choices,-1		).contiguous()
        A__   				=       input_mask.unsqueeze(1		).expand(-1,self.num_choices,-1		).contiguous()
        A__   				=       model(
            lowerCAmelCase__,attention_mask=lowerCAmelCase__,labels=lowerCAmelCase__,)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_choices)		)


    def 						snake_case__					(				self			:	List[Any],lowercase_			:	Optional[int],lowercase_			:	str,lowercase_			:	Union[str, Any],lowercase_			:	List[str],lowercase_			:	str,lowercase_			:	List[Any]		)->   Union[str, Any]:





        '''simple docstring'''







        A__   				=       self.num_labels
        A__   				=       MPNetForTokenClassification(config=lowerCAmelCase__		)
        model.to(lowerCAmelCase__		)
        model.eval()
        A__   				=       model(lowerCAmelCase__,attention_mask=lowerCAmelCase__,labels=lowerCAmelCase__		)
        self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)		)


    def 						snake_case__					(				self			:	str		)->   Dict:





        '''simple docstring'''







        A__   				=       self.prepare_config_and_inputs()
        (A__)   				=       config_and_inputs
        A__   				=       {"input_ids": input_ids, "attention_mask": input_mask}
        return config, inputs_dict




@require_torch
class 			A						(							_UpperCAmelCase   ,							_UpperCAmelCase   ,							unittest.TestCase			):







    """simple docstring"""





    lowerCamelCase :    Tuple		     =							(
        (
            MPNetForMaskedLM,
            MPNetForMultipleChoice,
            MPNetForQuestionAnswering,
            MPNetForSequenceClassification,
            MPNetForTokenClassification,
            MPNetModel,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase :    int		     =							(
        {
            '''feature-extraction''': MPNetModel,
            '''fill-mask''': MPNetForMaskedLM,
            '''question-answering''': MPNetForQuestionAnswering,
            '''text-classification''': MPNetForSequenceClassification,
            '''token-classification''': MPNetForTokenClassification,
            '''zero-shot''': MPNetForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowerCamelCase :    Dict		     =							False
    lowerCamelCase :    Optional[int]		     =							True


    def 						snake_case__					(				self			:	Optional[int]		)->   int:





        '''simple docstring'''







        A__   				=       MPNetModelTester(self		)
        A__   				=       ConfigTester(self,config_class=lowerCAmelCase__,hidden_size=3_7		)


    def 						snake_case__					(				self			:	Union[str, Any]		)->   List[Any]:





        '''simple docstring'''







        self.config_tester.run_common_tests()


    def 						snake_case__					(				self			:	Dict		)->   Union[str, Any]:





        '''simple docstring'''







        A__   				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_model(*lowerCAmelCase__		)


    def 						snake_case__					(				self			:	Tuple		)->   Optional[int]:





        '''simple docstring'''







        A__   				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_sequence_classification(*lowerCAmelCase__		)


    def 						snake_case__					(				self			:	Dict		)->   Dict:





        '''simple docstring'''







        A__   				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_multiple_choice(*lowerCAmelCase__		)


    def 						snake_case__					(				self			:	Any		)->   Dict:





        '''simple docstring'''







        A__   				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_token_classification(*lowerCAmelCase__		)


    def 						snake_case__					(				self			:	Optional[int]		)->   str:





        '''simple docstring'''







        A__   				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_question_answering(*lowerCAmelCase__		)




@require_torch
class 			A						(							unittest.TestCase			):







    """simple docstring"""





    @slow
    def 						snake_case__					(				self			:	str		)->   Union[str, Any]:





        '''simple docstring'''







        A__   				=       MPNetModel.from_pretrained('microsoft/mpnet-base'		)
        A__   				=       torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]		)
        A__   				=       model(lowerCAmelCase__		)[0]
        A__   				=       torch.Size((1, 1_1, 7_6_8)		)
        self.assertEqual(output.shape,lowerCAmelCase__		)
        A__   				=       torch.tensor(
            [[[-0.0_550, 0.1_943, -0.0_740], [-0.0_562, 0.2_211, -0.0_579], [-0.0_437, 0.3_337, -0.0_641]]]		)
        # compare the actual values for a slice.
        self.assertTrue(torch.allclose(output[:, :3, :3],lowerCAmelCase__,atol=1E-4		)		)




import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)


class 			A						(							nn.Module			):







    """simple docstring"""





    def __init__(				self			:	Union[str, Any]		)->   List[str]:





        '''simple docstring'''







        super().__init__()
        A__   				=       nn.Linear(3,4		)
        A__   				=       nn.BatchNormad(4		)
        A__   				=       nn.Linear(4,5		)


    def 						snake_case__					(				self			:	Dict,lowercase_			:	Union[str, Any]		)->   Tuple:





        '''simple docstring'''







        return self.lineara(self.batchnorm(self.lineara(lowercase_		)		)		)




class 			A						(							unittest.TestCase			):







    """simple docstring"""





    def 						snake_case__					(				self			:	List[str]		)->   Any:





        '''simple docstring'''







        A__   				=       ModelForTest()
        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,model.state_dict()		)
            A__   				=       os.path.join(lowercase_,'index.json'		)
            self.assertTrue(os.path.isfile(lowercase_		)		)
            # TODO: add tests on what is inside the index

            for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
                A__   				=       os.path.join(lowercase_,F'{key}.dat'		)
                self.assertTrue(os.path.isfile(lowercase_		)		)
                # TODO: add tests on the fact weights are properly loaded


    def 						snake_case__					(				self			:	List[Any]		)->   Optional[int]:





        '''simple docstring'''







        A__   				=       [torch.floataa, torch.floataa, torch.bfloataa]

        for dtype in dtypes:
            A__   				=       torch.randn(2,3,dtype=lowercase_		)
            with TemporaryDirectory() as tmp_dir:
                A__   				=       offload_weight(lowercase_,'weight',lowercase_,{}		)
                A__   				=       os.path.join(lowercase_,'weight.dat'		)
                self.assertTrue(os.path.isfile(lowercase_		)		)
                self.assertDictEqual(lowercase_,{'weight': {'shape': [2, 3], 'dtype': str(lowercase_		).split('.'		)[1]}}		)

                A__   				=       load_offloaded_weight(lowercase_,index['weight']		)
                self.assertTrue(torch.equal(lowercase_,lowercase_		)		)


    def 						snake_case__					(				self			:	Optional[int]		)->   Optional[Any]:





        '''simple docstring'''







        A__   				=       ModelForTest()
        A__   				=       model.state_dict()
        A__   				=       {k: v for k, v in state_dict.items() if 'linear2' not in k}
        A__   				=       {k: v for k, v in state_dict.items() if 'linear2' in k}

        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,lowercase_		)
            A__   				=       OffloadedWeightsLoader(state_dict=lowercase_,save_folder=lowercase_		)

            # Every key is there with the right value
            self.assertEqual(sorted(lowercase_		),sorted(state_dict.keys()		)		)
            for key, param in state_dict.items():
                self.assertTrue(torch.allclose(lowercase_,weight_map[key]		)		)

        A__   				=       {k: v for k, v in state_dict.items() if 'weight' in k}
        A__   				=       {k: v for k, v in state_dict.items() if 'weight' not in k}

        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,lowercase_		)
            A__   				=       OffloadedWeightsLoader(state_dict=lowercase_,save_folder=lowercase_		)

            # Every key is there with the right value
            self.assertEqual(sorted(lowercase_		),sorted(state_dict.keys()		)		)
            for key, param in state_dict.items():
                self.assertTrue(torch.allclose(lowercase_,weight_map[key]		)		)

        with TemporaryDirectory() as tmp_dir:
            offload_state_dict(lowercase_,lowercase_		)
            # Duplicates are removed
            A__   				=       OffloadedWeightsLoader(state_dict=lowercase_,save_folder=lowercase_		)

            # Every key is there with the right value
            self.assertEqual(sorted(lowercase_		),sorted(state_dict.keys()		)		)
            for key, param in state_dict.items():
                self.assertTrue(torch.allclose(lowercase_,weight_map[key]		)		)


    def 						snake_case__					(				self			:	int		)->   Union[str, Any]:





        '''simple docstring'''







        A__   				=       {'a.1': 0, 'a.10': 1, 'a.2': 2}
        A__   				=       extract_submodules_state_dict(lowercase_,['a.1', 'a.2']		)
        self.assertDictEqual(lowercase_,{'a.1': 0, 'a.2': 2}		)

        A__   				=       {'a.1.a': 0, 'a.10.a': 1, 'a.2.a': 2}
        A__   				=       extract_submodules_state_dict(lowercase_,['a.1', 'a.2']		)
        self.assertDictEqual(lowercase_,{'a.1.a': 0, 'a.2.a': 2}		)

