





import unittest

from transformers import SqueezeBertConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
     SqueezeBertForMaskedLM,
     SqueezeBertForMultipleChoice,
     SqueezeBertForQuestionAnswering,
     SqueezeBertForSequenceClassification,
     SqueezeBertForTokenClassification,
     SqueezeBertModel,
 )







class      lowerCamelCase(						__snake_case	):
     '''simple docstring'''


     def __init__(    self				,     snake_case_				,     snake_case_=13				,     snake_case_=7				,     snake_case_=True				,     snake_case_=True				,     snake_case_=False				,     snake_case_=True				,     snake_case_=99				,     snake_case_=32				,     snake_case_=5				,     snake_case_=4				,     snake_case_=64				,     snake_case_="gelu"				,     snake_case_=0.1				,     snake_case_=0.1				,     snake_case_=512				,     snake_case_=16				,     snake_case_=2				,     snake_case_=0.02				,     snake_case_=3				,     snake_case_=4				,     snake_case_=None				,     snake_case_=2				,     snake_case_=2				,     snake_case_=2				,     snake_case_=2				,     snake_case_=4				,     snake_case_=1				,     ):
         _A     =							parent
         _A     =							batch_size
         _A     =							seq_length
         _A     =							is_training
         _A     =							use_input_mask
         _A     =							use_token_type_ids
         _A     =							use_labels
         _A     =							vocab_size
         _A     =							hidden_size
         _A     =							num_hidden_layers
         _A     =							num_attention_heads
         _A     =							intermediate_size
         _A     =							hidden_act
         _A     =							hidden_dropout_prob
         _A     =							attention_probs_dropout_prob
         _A     =							max_position_embeddings
         _A     =							type_vocab_size
         _A     =							type_sequence_label_size
         _A     =							initializer_range
         _A     =							num_labels
         _A     =							num_choices
         _A     =							scope
         _A     =							q_groups
         _A     =							k_groups
         _A     =							v_groups
         _A     =							post_attention_groups
         _A     =							intermediate_groups
         _A     =							output_groups




     def        lowerCAmelCase__		(    self							):
         _A     =							ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size							)

         _A     =							None
         if self.use_input_mask:
             _A     =							random_attention_mask([self.batch_size, self.seq_length]							)

         _A     =							None
         _A     =							None
         _A     =							None
         if self.use_labels:
             _A     =							ids_tensor([self.batch_size]				,     self.type_sequence_label_size							)
             _A     =							ids_tensor([self.batch_size, self.seq_length]				,     self.num_labels							)
             _A     =							ids_tensor([self.batch_size]				,     self.num_choices							)

         _A     =							self.get_config()

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




     def        lowerCAmelCase__		(    self							):
         return SqueezeBertConfig(
             embedding_size=self.hidden_size				,     vocab_size=self.vocab_size				,     hidden_size=self.hidden_size				,     num_hidden_layers=self.num_hidden_layers				,     num_attention_heads=self.num_attention_heads				,     intermediate_size=self.intermediate_size				,     hidden_act=self.hidden_act				,     attention_probs_dropout_prob=self.hidden_dropout_prob				,     attention_dropout=self.attention_probs_dropout_prob				,     max_position_embeddings=self.max_position_embeddings				,     initializer_range=self.initializer_range				,     q_groups=self.q_groups				,     k_groups=self.k_groups				,     v_groups=self.v_groups				,     post_attention_groups=self.post_attention_groups				,     intermediate_groups=self.intermediate_groups				,     output_groups=self.output_groups				,     )




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_							):
         _A     =							SqueezeBertModel(config=snake_case_							)
         model.to(snake_case_							)
         model.eval()
         _A     =							model(snake_case_				,     snake_case_							)
         _A     =							model(snake_case_							)
         self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size)							)




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_							):
         _A     =							SqueezeBertForMaskedLM(config=snake_case_							)
         model.to(snake_case_							)
         model.eval()
         _A     =							model(snake_case_				,     attention_mask=snake_case_				,     labels=snake_case_							)
         self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.vocab_size)							)




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_							):
         _A     =							SqueezeBertForQuestionAnswering(config=snake_case_							)
         model.to(snake_case_							)
         model.eval()
         _A     =							model(
             snake_case_				,     attention_mask=snake_case_				,     start_positions=snake_case_				,     end_positions=snake_case_							)
         self.parent.assertEqual(result.start_logits.shape				,     (self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape				,     (self.batch_size, self.seq_length)							)




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_							):
         _A     =							self.num_labels
         _A     =							SqueezeBertForSequenceClassification(snake_case_							)
         model.to(snake_case_							)
         model.eval()
         _A     =							model(snake_case_				,     attention_mask=snake_case_				,     labels=snake_case_							)
         self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_labels)							)




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_							):
         _A     =							self.num_labels
         _A     =							SqueezeBertForTokenClassification(config=snake_case_							)
         model.to(snake_case_							)
         model.eval()

         _A     =							model(snake_case_				,     attention_mask=snake_case_				,     labels=snake_case_							)
         self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.num_labels)							)




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_				,     snake_case_							):
         _A     =							self.num_choices
         _A     =							SqueezeBertForMultipleChoice(config=snake_case_							)
         model.to(snake_case_							)
         model.eval()
         _A     =							input_ids.unsqueeze(1							).expand(-1				,     self.num_choices				,     -1							).contiguous()
         _A     =							input_mask.unsqueeze(1							).expand(-1				,     self.num_choices				,     -1							).contiguous()
         _A     =							model(
             snake_case_				,     attention_mask=snake_case_				,     labels=snake_case_				,     )
         self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_choices)							)




     def        lowerCAmelCase__		(    self							):
         _A     =							self.prepare_config_and_inputs()
         ((_A),			(_A),			(_A),			(_A),			(_A),			(_A))     =							config_and_inputs
         _A     =							{'input_ids': input_ids, 'attention_mask': input_mask}
         return config, inputs_dict







@require_torch
class      lowerCamelCase(						__snake_case			, __snake_case			, unittest.TestCase	):
     '''simple docstring'''


     __magic_name__   =			(
         (
             SqueezeBertModel,
             SqueezeBertForMaskedLM,
             SqueezeBertForMultipleChoice,
             SqueezeBertForQuestionAnswering,
             SqueezeBertForSequenceClassification,
             SqueezeBertForTokenClassification,
         )
         if is_torch_available()
         else None
     )
     __magic_name__   =			(
         {
             'feature-extraction': SqueezeBertModel,
             'fill-mask': SqueezeBertForMaskedLM,
             'question-answering': SqueezeBertForQuestionAnswering,
             'text-classification': SqueezeBertForSequenceClassification,
             'token-classification': SqueezeBertForTokenClassification,
             'zero-shot': SqueezeBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     __magic_name__   =			False
     __magic_name__   =			True
     __magic_name__   =			False




     def        lowerCAmelCase__		(    self							):
         _A     =							SqueezeBertModelTester(self							)
         _A     =							ConfigTester(self				,     config_class=snake_case_				,     dim=37							)




     def        lowerCAmelCase__		(    self							):
         self.config_tester.run_common_tests()




     def        lowerCAmelCase__		(    self							):
         _A     =							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_model(*snake_case_							)




     def        lowerCAmelCase__		(    self							):
         _A     =							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_masked_lm(*snake_case_							)




     def        lowerCAmelCase__		(    self							):
         _A     =							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_question_answering(*snake_case_							)




     def        lowerCAmelCase__		(    self							):
         _A     =							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_sequence_classification(*snake_case_							)




     def        lowerCAmelCase__		(    self							):
         _A     =							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_token_classification(*snake_case_							)




     def        lowerCAmelCase__		(    self							):
         _A     =							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_multiple_choice(*snake_case_							)




     @slow
     def        lowerCAmelCase__		(    self							):
         for model_name in SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             _A     =							SqueezeBertModel.from_pretrained(snake_case_							)
             self.assertIsNotNone(snake_case_							)







@require_sentencepiece
@require_tokenizers
@require_torch
class      lowerCamelCase(						unittest.TestCase	):
     '''simple docstring'''


     @slow
     def        lowerCAmelCase__		(    self							):
         _A     =							SqueezeBertForSequenceClassification.from_pretrained('squeezebert/squeezebert-mnli'							)

         _A     =							torch.tensor([[1, 2_9414, 232, 328, 740, 1140, 1_2695, 69, 13, 1588, 2]]							)
         _A     =							model(snake_case_							)[0]
         _A     =							torch.Size((1, 3)							)
         self.assertEqual(output.shape				,     snake_case_							)
         _A     =							torch.tensor([[0.6401, -0.0349, -0.6041]]							)
         self.assertTrue(torch.allclose(snake_case_				,     snake_case_				,     atol=1E-4							)							)







"""simple docstring"""


import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
		import torch.distributed.checkpoint as dist_cp
		from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
		from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
		from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
		from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


_lowercase				 =					get_logger(__name__)






def    _snake_case					(		snake_case__ :  Optional[Any]						,   snake_case__ :  List[Any]						,   snake_case__ :  int						,   snake_case__ :  int						,   snake_case__ :  str=0  ):
	os.makedirs(snake_case__						,   exist_ok=snake_case__  )
	with FSDP.state_dict_type(
	    snake_case__						,   fsdp_plugin.state_dict_type						,   fsdp_plugin.state_dict_config						,   fsdp_plugin.optim_state_dict_config  ):
		A     = model.state_dict()
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			A     = F'{MODEL_NAME}.bin' if model_index == 0 else F'{MODEL_NAME}_{model_index}.bin'
			A     = os.path.join(snake_case__						,   snake_case__  )
			if accelerator.process_index == 0:
				logger.info(F'Saving model to {output_model_file}'  )
				torch.save(snake_case__						,   snake_case__  )
				logger.info(F'Model saved to {output_model_file}'  )
		elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
			A     = (
			    F'{MODEL_NAME}_rank{accelerator.process_index}.bin'
			    if model_index == 0
			    else F'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
			)
			A     = os.path.join(snake_case__						,   snake_case__  )
			logger.info(F'Saving model to {output_model_file}'  )
			torch.save(snake_case__						,   snake_case__  )
			logger.info(F'Model saved to {output_model_file}'  )
		elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
			A     = os.path.join(snake_case__						,   F'{MODEL_NAME}_{model_index}'  )
			os.makedirs(snake_case__						,   exist_ok=snake_case__  )
			logger.info(F'Saving model to {ckpt_dir}'  )
			A     = {'model': state_dict}

			dist_cp.save_state_dict(
			    state_dict=snake_case__						,   storage_writer=dist_cp.FileSystemWriter(snake_case__  )						,   planner=DefaultSavePlanner()						,   )
			logger.info(F'Model saved to {ckpt_dir}'  )






def    _snake_case					(		snake_case__ :  int						,   snake_case__ :  List[str]						,   snake_case__ :  str						,   snake_case__ :  str						,   snake_case__ :  Any=0  ):
	accelerator.wait_for_everyone()
	with FSDP.state_dict_type(
	    snake_case__						,   fsdp_plugin.state_dict_type						,   fsdp_plugin.state_dict_config						,   fsdp_plugin.optim_state_dict_config  ):
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			if type(snake_case__  ) != FSDP and accelerator.process_index != 0:
				if not fsdp_plugin.sync_module_states:
					raise ValueError(
					    'Set the `sync_module_states` flag to `True` so that model states are synced across processes when '
					    'initializing FSDP object'  )
				return
			A     = F'{MODEL_NAME}.bin' if model_index == 0 else F'{MODEL_NAME}_{model_index}.bin'
			A     = os.path.join(snake_case__						,   snake_case__  )
			logger.info(F'Loading model from {input_model_file}'  )
			A     = torch.load(snake_case__  )
			logger.info(F'Model loaded from {input_model_file}'  )
		elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
			A     = (
			    F'{MODEL_NAME}_rank{accelerator.process_index}.bin'
			    if model_index == 0
			    else F'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
			)
			A     = os.path.join(snake_case__						,   snake_case__  )
			logger.info(F'Loading model from {input_model_file}'  )
			A     = torch.load(snake_case__  )
			logger.info(F'Model loaded from {input_model_file}'  )
		elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
			A     = (
			    os.path.join(snake_case__						,   F'{MODEL_NAME}_{model_index}'  )
			    if F'{MODEL_NAME}' not in input_dir
			    else input_dir
			)
			logger.info(F'Loading model from {ckpt_dir}'  )
			A     = {'model': model.state_dict()}
			dist_cp.load_state_dict(
			    state_dict=snake_case__						,   storage_reader=dist_cp.FileSystemReader(snake_case__  )						,   planner=DefaultLoadPlanner()						,   )
			A     = state_dict['model']
			logger.info(F'Model loaded from {ckpt_dir}'  )
		model.load_state_dict(snake_case__  )






def    _snake_case					(		snake_case__ :  Tuple						,   snake_case__ :  Optional[int]						,   snake_case__ :  Optional[int]						,   snake_case__ :  Dict						,   snake_case__ :  List[str]						,   snake_case__ :  Any=0  ):
	os.makedirs(snake_case__						,   exist_ok=snake_case__  )
	with FSDP.state_dict_type(
	    snake_case__						,   fsdp_plugin.state_dict_type						,   fsdp_plugin.state_dict_config						,   fsdp_plugin.optim_state_dict_config  ):
		A     = FSDP.optim_state_dict(snake_case__						,   snake_case__  )
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			if accelerator.process_index == 0:
				A     = (
				    F'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else F'{OPTIMIZER_NAME}_{optimizer_index}.bin'
				)
				A     = os.path.join(snake_case__						,   snake_case__  )
				logger.info(F'Saving Optimizer state to {output_optimizer_file}'  )
				torch.save(snake_case__						,   snake_case__  )
				logger.info(F'Optimizer state saved in {output_optimizer_file}'  )
		else:
			A     = os.path.join(snake_case__						,   F'{OPTIMIZER_NAME}_{optimizer_index}'  )
			os.makedirs(snake_case__						,   exist_ok=snake_case__  )
			logger.info(F'Saving Optimizer state to {ckpt_dir}'  )
			dist_cp.save_state_dict(
			    state_dict={'optimizer': optim_state}						,   storage_writer=dist_cp.FileSystemWriter(snake_case__  )						,   planner=DefaultSavePlanner()						,   )
			logger.info(F'Optimizer state saved in {ckpt_dir}'  )






def    _snake_case					(		snake_case__ :  str						,   snake_case__ :  Optional[int]						,   snake_case__ :  List[str]						,   snake_case__ :  List[str]						,   snake_case__ :  int						,   snake_case__ :  Optional[int]=0  ):
	accelerator.wait_for_everyone()
	with FSDP.state_dict_type(
	    snake_case__						,   fsdp_plugin.state_dict_type						,   fsdp_plugin.state_dict_config						,   fsdp_plugin.optim_state_dict_config  ):
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			A     = None
			# below check should work but currently it isn't working (mostly opytorch issue),
			# in the meantime disabling it at the cost of excess memory usage
			# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
			A     = (
			    F'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else F'{OPTIMIZER_NAME}_{optimizer_index}.bin'
			)
			A     = os.path.join(snake_case__						,   snake_case__  )
			logger.info(F'Loading Optimizer state from {input_optimizer_file}'  )
			A     = torch.load(snake_case__  )
			logger.info(F'Optimizer state loaded from {input_optimizer_file}'  )
		else:
			A     = (
			    os.path.join(snake_case__						,   F'{OPTIMIZER_NAME}_{optimizer_index}'  )
			    if F'{OPTIMIZER_NAME}' not in input_dir
			    else input_dir
			)
			logger.info(F'Loading Optimizer from {ckpt_dir}'  )
			A     = load_sharded_optimizer_state_dict(
			    model_state_dict=model.state_dict()						,   optimizer_key='optimizer'						,   storage_reader=dist_cp.FileSystemReader(snake_case__  )						,   )
			A     = optim_state['optimizer']
			logger.info(F'Optimizer loaded from {ckpt_dir}'  )
		A     = FSDP.optim_state_dict_to_load(snake_case__						,   snake_case__						,   snake_case__  )
		optimizer.load_state_dict(snake_case__  )