


import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    FlaubertForMultipleChoice,
		    FlaubertForQuestionAnswering,
		    FlaubertForQuestionAnsweringSimple,
		    FlaubertForSequenceClassification,
		    FlaubertForTokenClassification,
		    FlaubertModel,
		    FlaubertWithLMHeadModel,
		)
		from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class 					lowerCAmelCase_ (       _a):




						def __init__(	self						:   str   ,     __A						:   Union[str, Any]   ,     __A						:   Any=13   ,     __A						:   List[str]=7   ,     __A						:   int=True   ,     __A						:   int=True   ,     __A						:   int=True   ,     __A						:   List[Any]=True   ,     __A						:   str=True   ,     __A						:   List[str]=False   ,     __A						:   Optional[int]=False   ,     __A						:   Optional[Any]=False   ,     __A						:   List[str]=2   ,     __A						:   Any=99   ,     __A						:   Union[str, Any]=0   ,     __A						:   List[str]=32   ,     __A						:   List[str]=5   ,     __A						:   int=4   ,     __A						:   List[str]=0.1   ,     __A						:   Tuple=0.1   ,     __A						:   int=512   ,     __A						:   List[str]=12   ,     __A						:   Optional[int]=2   ,     __A						:   Optional[int]=0.02   ,     __A						:   List[Any]=3   ,     __A						:   int=4   ,     __A						:   Union[str, Any]="last"   ,     __A						:   List[str]=None   ,     __A						:   str=None   ,     )      ->Optional[Any]:






									"""simple docstring"""






									a__ :Optional[int]						  =    parent
									a__ :List[Any]						  =    batch_size
									a__ :Dict						  =    seq_length
									a__ :Optional[Any]						  =    is_training
									a__ :Tuple						  =    use_input_lengths
									a__ :List[str]						  =    use_token_type_ids
									a__ :str						  =    use_labels
									a__ :str						  =    gelu_activation
									a__ :int						  =    sinusoidal_embeddings
									a__ :Any						  =    causal
									a__ :Optional[Any]						  =    asm
									a__ :List[Any]						  =    n_langs
									a__ :Dict						  =    vocab_size
									a__ :int						  =    n_special
									a__ :Optional[int]						  =    hidden_size
									a__ :str						  =    num_hidden_layers
									a__ :Tuple						  =    num_attention_heads
									a__ :Union[str, Any]						  =    hidden_dropout_prob
									a__ :Optional[int]						  =    attention_probs_dropout_prob
									a__ :str						  =    max_position_embeddings
									a__ :Any						  =    type_vocab_size
									a__ :Union[str, Any]						  =    type_sequence_label_size
									a__ :Union[str, Any]						  =    initializer_range
									a__ :Tuple						  =    num_labels
									a__ :List[Any]						  =    num_choices
									a__ :List[Any]						  =    summary_type
									a__ :List[str]						  =    use_proj
									a__ :Optional[int]						  =    scope




						def  _snake_case					(	self						:   Dict						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :List[str]						  =    ids_tensor([self.batch_size, self.seq_length]   ,     self.vocab_size						)
									a__ :List[Any]						  =    random_attention_mask([self.batch_size, self.seq_length]						)

									a__ :Dict						  =    None
									if self.use_input_lengths:
												a__ :Union[str, Any]						  =    (
												    ids_tensor([self.batch_size]   ,     vocab_size=2						) + self.seq_length - 2
												)  # small variation of seq_length

									a__ :Optional[Any]						  =    None
									if self.use_token_type_ids:
												a__ :List[Any]						  =    ids_tensor([self.batch_size, self.seq_length]   ,     self.n_langs						)

									a__ :Dict						  =    None
									a__ :Tuple						  =    None
									a__ :str						  =    None
									if self.use_labels:
												a__ :List[str]						  =    ids_tensor([self.batch_size]   ,     self.type_sequence_label_size						)
												a__ :Union[str, Any]						  =    ids_tensor([self.batch_size, self.seq_length]   ,     self.num_labels						)
												a__ :str						  =    ids_tensor([self.batch_size]   ,     2						).float()
												a__ :List[str]						  =    ids_tensor([self.batch_size]   ,     self.num_choices						)

									a__ :Tuple						  =    self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_lengths,
									    sequence_labels,
									    token_labels,
									    is_impossible_labels,
									    choice_labels,
									    input_mask,
									)




						def  _snake_case					(	self						:   str						)      ->List[str]:






									"""simple docstring"""






									return FlaubertConfig(
									    vocab_size=self.vocab_size   ,     n_special=self.n_special   ,     emb_dim=self.hidden_size   ,     n_layers=self.num_hidden_layers   ,     n_heads=self.num_attention_heads   ,     dropout=self.hidden_dropout_prob   ,     attention_dropout=self.attention_probs_dropout_prob   ,     gelu_activation=self.gelu_activation   ,     sinusoidal_embeddings=self.sinusoidal_embeddings   ,     asm=self.asm   ,     causal=self.causal   ,     n_langs=self.n_langs   ,     max_position_embeddings=self.max_position_embeddings   ,     initializer_range=self.initializer_range   ,     summary_type=self.summary_type   ,     use_proj=self.use_proj   ,     )




						def  _snake_case					(	self						:   Any   ,     __A						:   List[str]   ,     __A						:   Union[str, Any]   ,     __A						:   Any   ,     __A						:   str   ,     __A						:   Tuple   ,     __A						:   Any   ,     __A						:   Any   ,     __A						:   List[str]   ,     __A						:   List[str]   ,     )      ->Union[str, Any]:






									"""simple docstring"""






									a__ :int						  =    FlaubertModel(config=__A						)
									model.to(__A						)
									model.eval()
									a__ :Optional[Any]						  =    model(__A   ,     lengths=__A   ,     langs=__A						)
									a__ :Union[str, Any]						  =    model(__A   ,     langs=__A						)
									a__ :List[Any]						  =    model(__A						)
									self.parent.assertEqual(result.last_hidden_state.shape   ,     (self.batch_size, self.seq_length, self.hidden_size)						)




						def  _snake_case					(	self						:   Union[str, Any]   ,     __A						:   int   ,     __A						:   Any   ,     __A						:   Dict   ,     __A						:   Tuple   ,     __A						:   Optional[Any]   ,     __A						:   Any   ,     __A						:   List[str]   ,     __A						:   Union[str, Any]   ,     __A						:   List[Any]   ,     )      ->Dict:






									"""simple docstring"""






									a__ :List[str]						  =    FlaubertWithLMHeadModel(__A						)
									model.to(__A						)
									model.eval()

									a__ :str						  =    model(__A   ,     token_type_ids=__A   ,     labels=__A						)
									self.parent.assertEqual(result.loss.shape   ,     ()						)
									self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.vocab_size)						)




						def  _snake_case					(	self						:   Optional[Any]   ,     __A						:   str   ,     __A						:   Tuple   ,     __A						:   int   ,     __A						:   Optional[Any]   ,     __A						:   int   ,     __A						:   Dict   ,     __A						:   Optional[int]   ,     __A						:   List[Any]   ,     __A						:   int   ,     )      ->str:






									"""simple docstring"""






									a__ :List[str]						  =    FlaubertForQuestionAnsweringSimple(__A						)
									model.to(__A						)
									model.eval()

									a__ :Dict						  =    model(__A						)

									a__ :int						  =    model(__A   ,     start_positions=__A   ,     end_positions=__A						)
									self.parent.assertEqual(result.start_logits.shape   ,     (self.batch_size, self.seq_length)						)
									self.parent.assertEqual(result.end_logits.shape   ,     (self.batch_size, self.seq_length)						)




						def  _snake_case					(	self						:   List[str]   ,     __A						:   Tuple   ,     __A						:   Tuple   ,     __A						:   Union[str, Any]   ,     __A						:   str   ,     __A						:   Any   ,     __A						:   Optional[Any]   ,     __A						:   List[str]   ,     __A						:   Union[str, Any]   ,     __A						:   Optional[int]   ,     )      ->Any:






									"""simple docstring"""






									a__ :Dict						  =    FlaubertForQuestionAnswering(__A						)
									model.to(__A						)
									model.eval()

									a__ :Optional[int]						  =    model(__A						)

									a__ :Optional[int]						  =    model(
									    __A   ,     start_positions=__A   ,     end_positions=__A   ,     cls_index=__A   ,     is_impossible=__A   ,     p_mask=__A   ,     )

									a__ :Dict						  =    model(
									    __A   ,     start_positions=__A   ,     end_positions=__A   ,     cls_index=__A   ,     is_impossible=__A   ,     )

									((a__)  ,  ) :Optional[Any]						  =    result_with_labels.to_tuple()

									a__ :Any						  =    model(__A   ,     start_positions=__A   ,     end_positions=__A						)

									((a__)  ,  ) :str						  =    result_with_labels.to_tuple()

									self.parent.assertEqual(result_with_labels.loss.shape   ,     ()						)
									self.parent.assertEqual(result.start_top_log_probs.shape   ,     (self.batch_size, model.config.start_n_top)						)
									self.parent.assertEqual(result.start_top_index.shape   ,     (self.batch_size, model.config.start_n_top)						)
									self.parent.assertEqual(
									    result.end_top_log_probs.shape   ,     (self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
									self.parent.assertEqual(
									    result.end_top_index.shape   ,     (self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
									self.parent.assertEqual(result.cls_logits.shape   ,     (self.batch_size,)						)




						def  _snake_case					(	self						:   int   ,     __A						:   Union[str, Any]   ,     __A						:   Optional[Any]   ,     __A						:   str   ,     __A						:   Optional[Any]   ,     __A						:   Any   ,     __A						:   int   ,     __A						:   Union[str, Any]   ,     __A						:   List[Any]   ,     __A						:   Tuple   ,     )      ->Optional[int]:






									"""simple docstring"""






									a__ :Optional[Any]						  =    FlaubertForSequenceClassification(__A						)
									model.to(__A						)
									model.eval()

									a__ :List[str]						  =    model(__A						)
									a__ :Optional[Any]						  =    model(__A   ,     labels=__A						)

									self.parent.assertEqual(result.loss.shape   ,     ()						)
									self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.type_sequence_label_size)						)




						def  _snake_case					(	self						:   Any   ,     __A						:   List[str]   ,     __A						:   Any   ,     __A						:   str   ,     __A						:   List[Any]   ,     __A						:   Tuple   ,     __A						:   Any   ,     __A						:   Optional[Any]   ,     __A						:   Optional[int]   ,     __A						:   Tuple   ,     )      ->Dict:






									"""simple docstring"""






									a__ :Any						  =    self.num_labels
									a__ :str						  =    FlaubertForTokenClassification(__A						)
									model.to(__A						)
									model.eval()

									a__ :Any						  =    model(__A   ,     attention_mask=__A   ,     labels=__A						)
									self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.num_labels)						)




						def  _snake_case					(	self						:   Dict   ,     __A						:   Any   ,     __A						:   Dict   ,     __A						:   Any   ,     __A						:   int   ,     __A						:   Union[str, Any]   ,     __A						:   Union[str, Any]   ,     __A						:   int   ,     __A						:   List[str]   ,     __A						:   Tuple   ,     )      ->List[Any]:






									"""simple docstring"""






									a__ :Dict						  =    self.num_choices
									a__ :List[Any]						  =    FlaubertForMultipleChoice(config=__A						)
									model.to(__A						)
									model.eval()
									a__ :Dict						  =    input_ids.unsqueeze(1						).expand(-1   ,     self.num_choices   ,     -1						).contiguous()
									a__ :str						  =    token_type_ids.unsqueeze(1						).expand(-1   ,     self.num_choices   ,     -1						).contiguous()
									a__ :Union[str, Any]						  =    input_mask.unsqueeze(1						).expand(-1   ,     self.num_choices   ,     -1						).contiguous()
									a__ :List[str]						  =    model(
									    __A   ,     attention_mask=__A   ,     token_type_ids=__A   ,     labels=__A   ,     )
									self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.num_choices)						)






						def  _snake_case					(	self						:   Optional[int]						)      ->Tuple:






									"""simple docstring"""






									a__ :Optional[Any]						  =    self.prepare_config_and_inputs()
									(
									    (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  (
									    a__
									)  ,  
									) :Tuple						  =    config_and_inputs
									a__ :Dict						  =    {
									    "input_ids": input_ids,
									    "token_type_ids": token_type_ids,
									    "lengths": input_lengths,
									    "attention_mask": input_mask,
									}
									return config, inputs_dict






@require_torch
class 					lowerCAmelCase_ (       _a			,_a			,unittest.TestCase):
						lowerCamelCase_         =     (
						    (
						        FlaubertModel,
						        FlaubertWithLMHeadModel,
						        FlaubertForQuestionAnswering,
						        FlaubertForQuestionAnsweringSimple,
						        FlaubertForSequenceClassification,
						        FlaubertForTokenClassification,
						        FlaubertForMultipleChoice,
						    )
						    if is_torch_available()
						    else ()
						)
						lowerCamelCase_         =     (
						    {
						        'feature-extraction': FlaubertModel,
						        'fill-mask': FlaubertWithLMHeadModel,
						        'question-answering': FlaubertForQuestionAnsweringSimple,
						        'text-classification': FlaubertForSequenceClassification,
						        'token-classification': FlaubertForTokenClassification,
						        'zero-shot': FlaubertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)




						def  _snake_case					(	self						:   Dict   ,     __A						:   int   ,     __A						:   Optional[int]   ,     __A						:   Optional[Any]   ,     __A						:   List[Any]   ,     __A						:   int						)      ->List[Any]:






									"""simple docstring"""






									if (
									    pipeline_test_casse_name == "QAPipelineTests"
									    and tokenizer_name is not None
									    and not tokenizer_name.endswith("Fast"						)
									):
												# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
												# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
												# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
												return True

									return False




						def  _snake_case					(	self						:   List[Any]   ,     __A						:   List[Any]   ,     __A						:   str   ,     __A						:   List[str]=False						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :Dict						  =    super()._prepare_for_class(__A   ,     __A   ,     return_labels=__A						)

									if return_labels:
												if model_class.__name__ == "FlaubertForQuestionAnswering":
															a__ :Optional[int]						  =    torch.zeros(
															    self.model_tester.batch_size   ,     dtype=torch.long   ,     device=__A						)
															a__ :Optional[int]						  =    torch.zeros(
															    self.model_tester.batch_size   ,     dtype=torch.long   ,     device=__A						)

									return inputs_dict




						def  _snake_case					(	self						:   Any						)      ->Any:






									"""simple docstring"""






									a__ :List[Any]						  =    FlaubertModelTester(self						)
									a__ :str						  =    ConfigTester(self   ,     config_class=__A   ,     emb_dim=37						)




						def  _snake_case					(	self						:   Optional[Any]						)      ->List[Any]:






									"""simple docstring"""






									self.config_tester.run_common_tests()




						def  _snake_case					(	self						:   Any						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :Optional[int]						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_model(*__A						)




						def  _snake_case					(	self						:   Dict						)      ->str:






									"""simple docstring"""






									a__ :Optional[Any]						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_lm_head(*__A						)




						def  _snake_case					(	self						:   List[str]						)      ->int:






									"""simple docstring"""






									a__ :Optional[int]						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_simple_qa(*__A						)




						def  _snake_case					(	self						:   List[Any]						)      ->Optional[Any]:






									"""simple docstring"""






									a__ :List[Any]						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_qa(*__A						)




						def  _snake_case					(	self						:   str						)      ->Optional[Any]:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_sequence_classif(*__A						)




						def  _snake_case					(	self						:   List[Any]						)      ->str:






									"""simple docstring"""






									a__ :Dict						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_token_classif(*__A						)




						def  _snake_case					(	self						:   str						)      ->Dict:






									"""simple docstring"""






									a__ :int						  =    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_flaubert_multiple_choice(*__A						)




						@slow
						def  _snake_case					(	self						:   str						)      ->Any:






									"""simple docstring"""






									for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												a__ :Any						  =    FlaubertModel.from_pretrained(__A						)
												self.assertIsNotNone(__A						)






						@slow
						@require_torch_gpu
						def  _snake_case					(	self						:   List[Any]						)      ->Any:






									"""simple docstring"""






									a__  ,  a__ :Optional[Any]						  =    self.model_tester.prepare_config_and_inputs_for_common()
									for model_class in self.all_model_classes:
												# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
												if model_class == FlaubertForMultipleChoice:
															return

												a__ :Any						  =    True
												a__ :Any						  =    model_class(config=__A						)

												a__ :List[str]						  =    self._prepare_for_class(__A   ,     __A						)
												a__ :List[str]						  =    torch.jit.trace(
												    __A   ,     (inputs_dict["input_ids"].to("cpu"						), inputs_dict["attention_mask"].to("cpu"						))						)

												with tempfile.TemporaryDirectory() as tmp:
															torch.jit.save(__A   ,     os.path.join(__A   ,     "traced_model.pt"						)						)
															a__ :Dict						  =    torch.jit.load(os.path.join(__A   ,     "traced_model.pt"						)   ,     map_location=__A						)
															loaded(inputs_dict["input_ids"].to(__A						)   ,     inputs_dict["attention_mask"].to(__A						)						)






@require_torch
class 					lowerCAmelCase_ (       unittest.TestCase):




						@slow
						def  _snake_case					(	self						:   Union[str, Any]						)      ->Optional[Any]:






									"""simple docstring"""






									a__ :str						  =    FlaubertModel.from_pretrained("flaubert/flaubert_base_cased"						)
									a__ :Tuple						  =    torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]						)
									with torch.no_grad():
												a__ :Tuple						  =    model(__A						)[0]
									a__ :Optional[Any]						  =    torch.Size((1, 11, 768)						)
									self.assertEqual(output.shape   ,     __A						)
									a__ :Optional[int]						  =    torch.tensor(
									    [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]						)

									self.assertTrue(torch.allclose(output[:, :3, :3]   ,     __A   ,     atol=1E-4						)						)






import datasets

from .evaluate import evaluate


snake_case__			 =						'''\
@inproceedings{Rajpurkar2016SQuAD10,
  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
  booktitle={EMNLP},
  year={2016}
}
'''

snake_case__			 =						'''
This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).

Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
from the corresponding reading passage, or the question might be unanswerable.
'''

snake_case__			 =						'''
Computes SQuAD scores (F1 and EM).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair as given in the references (see below)
        - \'prediction_text\': the text of the answer
    references: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair (see above),
        - \'answers\': a Dict in the SQuAD dataset format
            {
                \'text\': list of possible texts for the answer, as a list of strings
                \'answer_start\': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
Examples:

    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> squad_metric = datasets.load_metric("squad")
    >>> results = squad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 100.0, \'f1\': 100.0}
'''


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,_KWARGS_DESCRIPTION)
class 					lowerCAmelCase_ (       datasets.Metric):




						def  _snake_case					(	self						:   List[str]						)      ->Union[str, Any]:






									"""simple docstring"""






									return datasets.MetricInfo(
									    description=_DESCRIPTION   ,     citation=_CITATION   ,     inputs_description=_KWARGS_DESCRIPTION   ,     features=datasets.Features(
									        {
									            "predictions": {"id": datasets.Value("string"						), "prediction_text": datasets.Value("string"						)},
									            "references": {
									                "id": datasets.Value("string"						),
									                "answers": datasets.features.Sequence(
									                    {
									                        "text": datasets.Value("string"						),
									                        "answer_start": datasets.Value("int32"						),
									                    }						),
									            },
									        }						)   ,     codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]   ,     reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]   ,     )






						def  _snake_case					(	self						:   Dict   ,     __A						:   List[Any]   ,     __A						:   Optional[int]						)      ->str:






									"""simple docstring"""






									a__ :Optional[int]						  =    {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
									a__ :Optional[Any]						  =    [
									    {
									        "paragraphs": [
									            {
									                "qas": [
									                    {
									                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
									                        "id": ref["id"],
									                    }
									                    for ref in references
									                ]
									            }
									        ]
									    }
									]
									a__ :Union[str, Any]						  =    evaluate(dataset=__A   ,     predictions=__A						)
									return score


