
"""simple docstring"""
import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    FlaubertForMultipleChoice,
			    FlaubertForQuestionAnswering,
			    FlaubertForQuestionAnsweringSimple,
			    FlaubertForSequenceClassification,
			    FlaubertForTokenClassification,
			    FlaubertModel,
			    FlaubertWithLMHeadModel,
			)
			from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							def __init__( self    ,				snake_case_    ,				snake_case_=13    ,				snake_case_=7    ,				snake_case_=True    ,				snake_case_=True    ,				snake_case_=True    ,				snake_case_=True    ,				snake_case_=True    ,				snake_case_=False    ,				snake_case_=False    ,				snake_case_=False    ,				snake_case_=2    ,				snake_case_=99    ,				snake_case_=0    ,				snake_case_=32    ,				snake_case_=5    ,				snake_case_=4    ,				snake_case_=0.1    ,				snake_case_=0.1    ,				snake_case_=512    ,				snake_case_=12    ,				snake_case_=2    ,				snake_case_=0.02    ,				snake_case_=3    ,				snake_case_=4    ,				snake_case_="last"    ,				snake_case_=None    ,				snake_case_=None    ,				)       ->  Tuple:
													__lowerCAmelCase     =  parent
													__lowerCAmelCase     =  batch_size
													__lowerCAmelCase     =  seq_length
													__lowerCAmelCase     =  is_training
													__lowerCAmelCase     =  use_input_lengths
													__lowerCAmelCase     =  use_token_type_ids
													__lowerCAmelCase     =  use_labels
													__lowerCAmelCase     =  gelu_activation
													__lowerCAmelCase     =  sinusoidal_embeddings
													__lowerCAmelCase     =  causal
													__lowerCAmelCase     =  asm
													__lowerCAmelCase     =  n_langs
													__lowerCAmelCase     =  vocab_size
													__lowerCAmelCase     =  n_special
													__lowerCAmelCase     =  hidden_size
													__lowerCAmelCase     =  num_hidden_layers
													__lowerCAmelCase     =  num_attention_heads
													__lowerCAmelCase     =  hidden_dropout_prob
													__lowerCAmelCase     =  attention_probs_dropout_prob
													__lowerCAmelCase     =  max_position_embeddings
													__lowerCAmelCase     =  type_vocab_size
													__lowerCAmelCase     =  type_sequence_label_size
													__lowerCAmelCase     =  initializer_range
													__lowerCAmelCase     =  num_labels
													__lowerCAmelCase     =  num_choices
													__lowerCAmelCase     =  summary_type
													__lowerCAmelCase     =  use_proj
													__lowerCAmelCase     =  scope




							def    A__						( self				)       ->  Optional[int]:
													__lowerCAmelCase     =  ids_tensor([self.batch_size, self.seq_length]    ,				self.vocab_size				)
													__lowerCAmelCase     =  random_attention_mask([self.batch_size, self.seq_length]				)

													__lowerCAmelCase     =  None
													if self.use_input_lengths:
																			__lowerCAmelCase     =  (
																			    ids_tensor([self.batch_size]    ,				vocab_size=2				) + self.seq_length - 2
																			)  # small variation of seq_length

													__lowerCAmelCase     =  None
													if self.use_token_type_ids:
																			__lowerCAmelCase     =  ids_tensor([self.batch_size, self.seq_length]    ,				self.n_langs				)

													__lowerCAmelCase     =  None
													__lowerCAmelCase     =  None
													__lowerCAmelCase     =  None
													if self.use_labels:
																			__lowerCAmelCase     =  ids_tensor([self.batch_size]    ,				self.type_sequence_label_size				)
																			__lowerCAmelCase     =  ids_tensor([self.batch_size, self.seq_length]    ,				self.num_labels				)
																			__lowerCAmelCase     =  ids_tensor([self.batch_size]    ,				2				).float()
																			__lowerCAmelCase     =  ids_tensor([self.batch_size]    ,				self.num_choices				)

													__lowerCAmelCase     =  self.get_config()

													return (
													    config,
													    input_ids,
													    token_type_ids,
													    input_lengths,
													    sequence_labels,
													    token_labels,
													    is_impossible_labels,
													    choice_labels,
													    input_mask,
													)




							def    A__						( self				)       ->  Optional[int]:
													return FlaubertConfig(
													    vocab_size=self.vocab_size    ,				n_special=self.n_special    ,				emb_dim=self.hidden_size    ,				n_layers=self.num_hidden_layers    ,				n_heads=self.num_attention_heads    ,				dropout=self.hidden_dropout_prob    ,				attention_dropout=self.attention_probs_dropout_prob    ,				gelu_activation=self.gelu_activation    ,				sinusoidal_embeddings=self.sinusoidal_embeddings    ,				asm=self.asm    ,				causal=self.causal    ,				n_langs=self.n_langs    ,				max_position_embeddings=self.max_position_embeddings    ,				initializer_range=self.initializer_range    ,				summary_type=self.summary_type    ,				use_proj=self.use_proj    ,				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  Optional[int]:
													__lowerCAmelCase     =  FlaubertModel(config=snake_case_				)
													model.to(snake_case_				)
													model.eval()
													__lowerCAmelCase     =  model(snake_case_    ,				lengths=snake_case_    ,				langs=snake_case_				)
													__lowerCAmelCase     =  model(snake_case_    ,				langs=snake_case_				)
													__lowerCAmelCase     =  model(snake_case_				)
													self.parent.assertEqual(result.last_hidden_state.shape    ,				(self.batch_size, self.seq_length, self.hidden_size)				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  Optional[int]:
													__lowerCAmelCase     =  FlaubertWithLMHeadModel(snake_case_				)
													model.to(snake_case_				)
													model.eval()

													__lowerCAmelCase     =  model(snake_case_    ,				token_type_ids=snake_case_    ,				labels=snake_case_				)
													self.parent.assertEqual(result.loss.shape    ,				()				)
													self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.seq_length, self.vocab_size)				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  Tuple:
													__lowerCAmelCase     =  FlaubertForQuestionAnsweringSimple(snake_case_				)
													model.to(snake_case_				)
													model.eval()

													__lowerCAmelCase     =  model(snake_case_				)

													__lowerCAmelCase     =  model(snake_case_    ,				start_positions=snake_case_    ,				end_positions=snake_case_				)
													self.parent.assertEqual(result.start_logits.shape    ,				(self.batch_size, self.seq_length)				)
													self.parent.assertEqual(result.end_logits.shape    ,				(self.batch_size, self.seq_length)				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  Tuple:
													__lowerCAmelCase     =  FlaubertForQuestionAnswering(snake_case_				)
													model.to(snake_case_				)
													model.eval()

													__lowerCAmelCase     =  model(snake_case_				)

													__lowerCAmelCase     =  model(
													    snake_case_    ,				start_positions=snake_case_    ,				end_positions=snake_case_    ,				cls_index=snake_case_    ,				is_impossible=snake_case_    ,				p_mask=snake_case_    ,				)

													__lowerCAmelCase     =  model(
													    snake_case_    ,				start_positions=snake_case_    ,				end_positions=snake_case_    ,				cls_index=snake_case_    ,				is_impossible=snake_case_    ,				)

													((__lowerCAmelCase)   ,     )     =  result_with_labels.to_tuple()

													__lowerCAmelCase     =  model(snake_case_    ,				start_positions=snake_case_    ,				end_positions=snake_case_				)

													((__lowerCAmelCase)   ,     )     =  result_with_labels.to_tuple()

													self.parent.assertEqual(result_with_labels.loss.shape    ,				()				)
													self.parent.assertEqual(result.start_top_log_probs.shape    ,				(self.batch_size, model.config.start_n_top)				)
													self.parent.assertEqual(result.start_top_index.shape    ,				(self.batch_size, model.config.start_n_top)				)
													self.parent.assertEqual(
													    result.end_top_log_probs.shape    ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)				)
													self.parent.assertEqual(
													    result.end_top_index.shape    ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)				)
													self.parent.assertEqual(result.cls_logits.shape    ,				(self.batch_size,)				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  List[str]:
													__lowerCAmelCase     =  FlaubertForSequenceClassification(snake_case_				)
													model.to(snake_case_				)
													model.eval()

													__lowerCAmelCase     =  model(snake_case_				)
													__lowerCAmelCase     =  model(snake_case_    ,				labels=snake_case_				)

													self.parent.assertEqual(result.loss.shape    ,				()				)
													self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.type_sequence_label_size)				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  Tuple:
													__lowerCAmelCase     =  self.num_labels
													__lowerCAmelCase     =  FlaubertForTokenClassification(snake_case_				)
													model.to(snake_case_				)
													model.eval()

													__lowerCAmelCase     =  model(snake_case_    ,				attention_mask=snake_case_    ,				labels=snake_case_				)
													self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.seq_length, self.num_labels)				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				)       ->  Dict:
													__lowerCAmelCase     =  self.num_choices
													__lowerCAmelCase     =  FlaubertForMultipleChoice(config=snake_case_				)
													model.to(snake_case_				)
													model.eval()
													__lowerCAmelCase     =  input_ids.unsqueeze(1				).expand(-1    ,				self.num_choices    ,				-1				).contiguous()
													__lowerCAmelCase     =  token_type_ids.unsqueeze(1				).expand(-1    ,				self.num_choices    ,				-1				).contiguous()
													__lowerCAmelCase     =  input_mask.unsqueeze(1				).expand(-1    ,				self.num_choices    ,				-1				).contiguous()
													__lowerCAmelCase     =  model(
													    snake_case_    ,				attention_mask=snake_case_    ,				token_type_ids=snake_case_    ,				labels=snake_case_    ,				)
													self.parent.assertEqual(result.logits.shape    ,				(self.batch_size, self.num_choices)				)




							def    A__						( self				)       ->  List[Any]:
													__lowerCAmelCase     =  self.prepare_config_and_inputs()
													(
													    (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     (
													    __lowerCAmelCase
													)   ,     
													)     =  config_and_inputs
													__lowerCAmelCase     =  {
													    """input_ids""": input_ids,
													    """token_type_ids""": token_type_ids,
													    """lengths""": input_lengths,
													    """attention_mask""": input_mask,
													}
													return config, inputs_dict
@require_torch
class       lowerCAmelCase_		(				A__				,				A__				,				unittest.TestCase ):



							'''simple docstring'''





							_snake_case						      =		(
							    (
							        FlaubertModel,
							        FlaubertWithLMHeadModel,
							        FlaubertForQuestionAnswering,
							        FlaubertForQuestionAnsweringSimple,
							        FlaubertForSequenceClassification,
							        FlaubertForTokenClassification,
							        FlaubertForMultipleChoice,
							    )
							    if is_torch_available()
							    else ()
							)
							_snake_case						      =		(
							    {
							        '''feature-extraction''': FlaubertModel,
							        '''fill-mask''': FlaubertWithLMHeadModel,
							        '''question-answering''': FlaubertForQuestionAnsweringSimple,
							        '''text-classification''': FlaubertForSequenceClassification,
							        '''token-classification''': FlaubertForTokenClassification,
							        '''zero-shot''': FlaubertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_    ,				snake_case_				)       ->  Dict:
													if (
													    pipeline_test_casse_name == "QAPipelineTests"
													    and tokenizer_name is not None
													    and not tokenizer_name.endswith("""Fast"""				)
													):
																			# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
																			# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
																			# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
																			return True

													return False




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_=False				)       ->  List[Any]:
													__lowerCAmelCase     =  super()._prepare_for_class(snake_case_    ,				snake_case_    ,				return_labels=snake_case_				)

													if return_labels:
																			if model_class.__name__ == "FlaubertForQuestionAnswering":
																									__lowerCAmelCase     =  torch.zeros(
																									    self.model_tester.batch_size    ,				dtype=torch.long    ,				device=snake_case_				)
																									__lowerCAmelCase     =  torch.zeros(
																									    self.model_tester.batch_size    ,				dtype=torch.long    ,				device=snake_case_				)

													return inputs_dict




							def    A__						( self				)       ->  Union[str, Any]:
													__lowerCAmelCase     =  FlaubertModelTester(self				)
													__lowerCAmelCase     =  ConfigTester(self    ,				config_class=snake_case_    ,				emb_dim=37				)




							def    A__						( self				)       ->  int:
													self.config_tester.run_common_tests()




							def    A__						( self				)       ->  List[str]:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_model(*snake_case_				)




							def    A__						( self				)       ->  Optional[Any]:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_lm_head(*snake_case_				)




							def    A__						( self				)       ->  Optional[int]:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_simple_qa(*snake_case_				)




							def    A__						( self				)       ->  Any:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_qa(*snake_case_				)




							def    A__						( self				)       ->  Union[str, Any]:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_sequence_classif(*snake_case_				)




							def    A__						( self				)       ->  Any:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_token_classif(*snake_case_				)




							def    A__						( self				)       ->  Tuple:
													__lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_flaubert_multiple_choice(*snake_case_				)




							@slow
							def    A__						( self				)       ->  int:
													for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__lowerCAmelCase     =  FlaubertModel.from_pretrained(snake_case_				)
																			self.assertIsNotNone(snake_case_				)




							@slow
							@require_torch_gpu
							def    A__						( self				)       ->  List[str]:
													__lowerCAmelCase   ,     __lowerCAmelCase     =  self.model_tester.prepare_config_and_inputs_for_common()
													for model_class in self.all_model_classes:
																			# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
																			if model_class == FlaubertForMultipleChoice:
																									return

																			__lowerCAmelCase     =  True
																			__lowerCAmelCase     =  model_class(config=snake_case_				)

																			__lowerCAmelCase     =  self._prepare_for_class(snake_case_    ,				snake_case_				)
																			__lowerCAmelCase     =  torch.jit.trace(
																			    snake_case_    ,				(inputs_dict["""input_ids"""].to("""cpu"""				), inputs_dict["""attention_mask"""].to("""cpu"""				))				)

																			with tempfile.TemporaryDirectory() as tmp:
																									torch.jit.save(snake_case_    ,				os.path.join(snake_case_    ,				"""traced_model.pt"""				)				)
																									__lowerCAmelCase     =  torch.jit.load(os.path.join(snake_case_    ,				"""traced_model.pt"""				)    ,				map_location=snake_case_				)
																									loaded(inputs_dict["""input_ids"""].to(snake_case_				)    ,				inputs_dict["""attention_mask"""].to(snake_case_				)				)
@require_torch
class       lowerCAmelCase_		(				unittest.TestCase ):



							'''simple docstring'''





							@slow
							def    A__						( self				)       ->  Union[str, Any]:
													__lowerCAmelCase     =  FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""				)
													__lowerCAmelCase     =  torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]				)
													with torch.no_grad():
																			__lowerCAmelCase     =  model(snake_case_				)[0]
													__lowerCAmelCase     =  torch.Size((1, 11, 768)				)
													self.assertEqual(output.shape    ,				snake_case_				)
													__lowerCAmelCase     =  torch.tensor(
													    [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]				)

													self.assertTrue(torch.allclose(output[:, :3, :3]    ,				snake_case_    ,				atol=1e-4				)				)


"""simple docstring"""
import operator as op


SCREAMING_SNAKE_CASE_					     =	'''scaler.pt'''
SCREAMING_SNAKE_CASE_					     =	'''pytorch_model'''
SCREAMING_SNAKE_CASE_					     =	'''random_states'''
SCREAMING_SNAKE_CASE_					     =	'''optimizer'''
SCREAMING_SNAKE_CASE_					     =	'''scheduler'''
SCREAMING_SNAKE_CASE_					     =	'''pytorch_model.bin'''
SCREAMING_SNAKE_CASE_					     =	'''pytorch_model.bin.index.json'''
SCREAMING_SNAKE_CASE_					     =	'''model.safetensors'''
SCREAMING_SNAKE_CASE_					     =	'''model.safetensors.index.json'''
SCREAMING_SNAKE_CASE_					     =	'''1.10.2'''
SCREAMING_SNAKE_CASE_					     =	'''py38'''
SCREAMING_SNAKE_CASE_					     =	'''4.17.0'''
SCREAMING_SNAKE_CASE_					     =	['''ml.p3.16xlarge''', '''ml.p3dn.24xlarge''', '''ml.p4dn.24xlarge''']
SCREAMING_SNAKE_CASE_					     =	['''FULL_SHARD''', '''SHARD_GRAD_OP''', '''NO_SHARD''', '''HYBRID_SHARD''', '''HYBRID_SHARD_ZERO2''']
SCREAMING_SNAKE_CASE_					     =	['''TRANSFORMER_BASED_WRAP''', '''SIZE_BASED_WRAP''', '''NO_WRAP''']
SCREAMING_SNAKE_CASE_					     =	['''BACKWARD_PRE''', '''BACKWARD_POST''', '''NO_PREFETCH''']
SCREAMING_SNAKE_CASE_					     =	['''FULL_STATE_DICT''', '''LOCAL_STATE_DICT''', '''SHARDED_STATE_DICT''']
SCREAMING_SNAKE_CASE_					     =	'''2.0.1'''
SCREAMING_SNAKE_CASE_					     =	['''pdsh''', '''standard''', '''openmpi''', '''mvapich''']
SCREAMING_SNAKE_CASE_					     =	['''default''', '''reduce-overhead''', '''max-autotune''']

SCREAMING_SNAKE_CASE_					     =	{'''>''': op.gt, '''>=''': op.ge, '''==''': op.eq, '''!=''': op.ne, '''<=''': op.le, '''<''': op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
SCREAMING_SNAKE_CASE_					     =	[
    '''nnodes''',
    '''nproc_per_node''',
    '''rdzv_backend''',
    '''rdzv_endpoint''',
    '''rdzv_id''',
    '''rdzv_conf''',
    '''standalone''',
    '''max_restarts''',
    '''monitor_interval''',
    '''start_method''',
    '''role''',
    '''module''',
    '''m''',
    '''no_python''',
    '''run_path''',
    '''log_dir''',
    '''r''',
    '''redirects''',
    '''t''',
    '''tee''',
    '''node_rank''',
    '''master_addr''',
    '''master_port''',
]

SCREAMING_SNAKE_CASE_					     =	['''DEEPSPEED''', '''MULTI_GPU''', '''FSDP''', '''MEGATRON_LM''']
SCREAMING_SNAKE_CASE_					     =	['''DEEPSPEED''', '''MULTI_XPU''', '''FSDP''']
