'''simple docstring'''




from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMForMultipleChoice,
      TFXLMForQuestionAnsweringSimple,
      TFXLMForSequenceClassification,
      TFXLMForTokenClassification,
      TFXLMModel,
      TFXLMWithLMHeadModel,
      XLMConfig,
  )




class 	__UpperCAmelCase				:





       '''simple docstring'''



       def __init__(      self    ,     _SCREAMING_SNAKE_CASE    ,     )		->     str:
           A_     =   parent
           A_     =   13
           A_     =   7
           A_     =   True
           A_     =   True
           A_     =   True
           A_     =   True
           A_     =   True
           A_     =   False
           A_     =   False
           A_     =   False
           A_     =   2
           A_     =   99
           A_     =   0
           A_     =   32
           A_     =   2
           A_     =   4
           A_     =   0.1
           A_     =   0.1
           A_     =   512
           A_     =   16
           A_     =   2
           A_     =   0.02
           A_     =   3
           A_     =   4
           A_     =   '''last'''
           A_     =   True
           A_     =   None
           A_     =   0

       def 					__A       (      self )		->     Union[str, Any]:
           A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     self.vocab_size )
           A_     =   random_attention_mask([self.batch_size, self.seq_length]    ,     dtype=tf.floataa )

           A_     =   None
           if self.use_input_lengths:
               A_     =   (
                   ids_tensor([self.batch_size]    ,     vocab_size=2 ) + self.seq_length - 2
               )  # small variation of seq_length

           A_     =   None
           if self.use_token_type_ids:
               A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     self.n_langs )

           A_     =   None
           A_     =   None
           A_     =   None
           if self.use_labels:
               A_     =   ids_tensor([self.batch_size]    ,     self.type_sequence_label_size )
               A_     =   ids_tensor([self.batch_size, self.seq_length]    ,     self.num_labels )
               A_     =   ids_tensor([self.batch_size]    ,     2    ,     dtype=tf.floataa )
               A_     =   ids_tensor([self.batch_size]    ,     self.num_choices )

           A_     =   XLMConfig(
               vocab_size=self.vocab_size    ,     n_special=self.n_special    ,     emb_dim=self.hidden_size    ,     n_layers=self.num_hidden_layers    ,     n_heads=self.num_attention_heads    ,     dropout=self.hidden_dropout_prob    ,     attention_dropout=self.attention_probs_dropout_prob    ,     gelu_activation=self.gelu_activation    ,     sinusoidal_embeddings=self.sinusoidal_embeddings    ,     asm=self.asm    ,     causal=self.causal    ,     n_langs=self.n_langs    ,     max_position_embeddings=self.max_position_embeddings    ,     initializer_range=self.initializer_range    ,     summary_type=self.summary_type    ,     use_proj=self.use_proj    ,     bos_token_id=self.bos_token_id    ,     )

           return (
               config,
               input_ids,
               token_type_ids,
               input_lengths,
               sequence_labels,
               token_labels,
               is_impossible_labels,
               choice_labels,
               input_mask,
           )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     Union[str, Any]:
           A_     =   TFXLMModel(config=_SCREAMING_SNAKE_CASE )
           A_     =   {'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
           A_     =   model(_SCREAMING_SNAKE_CASE )

           A_     =   [input_ids, input_mask]
           A_     =   model(_SCREAMING_SNAKE_CASE )
           self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.hidden_size) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     List[Any]:
           A_     =   TFXLMWithLMHeadModel(_SCREAMING_SNAKE_CASE )

           A_     =   {'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
           A_     =   model(_SCREAMING_SNAKE_CASE )

           A_     =   outputs

           self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.vocab_size) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     int:
           A_     =   TFXLMForQuestionAnsweringSimple(_SCREAMING_SNAKE_CASE )

           A_     =   {'''input_ids''': input_ids, '''lengths''': input_lengths}

           A_     =   model(_SCREAMING_SNAKE_CASE )

           self.parent.assertEqual(result.start_logits.shape    ,     (self.batch_size, self.seq_length) )
           self.parent.assertEqual(result.end_logits.shape    ,     (self.batch_size, self.seq_length) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     Any:
           A_     =   TFXLMForSequenceClassification(_SCREAMING_SNAKE_CASE )

           A_     =   {'''input_ids''': input_ids, '''lengths''': input_lengths}

           A_     =   model(_SCREAMING_SNAKE_CASE )

           self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.type_sequence_label_size) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     Optional[int]:
           A_     =   self.num_labels
           A_     =   TFXLMForTokenClassification(config=_SCREAMING_SNAKE_CASE )
           A_     =   {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
           A_     =   model(_SCREAMING_SNAKE_CASE )
           self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.num_labels) )

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     )		->     List[str]:
           A_     =   self.num_choices
           A_     =   TFXLMForMultipleChoice(config=_SCREAMING_SNAKE_CASE )
           A_     =   tf.tile(tf.expand_dims(_SCREAMING_SNAKE_CASE    ,     1 )    ,     (1, self.num_choices, 1) )
           A_     =   tf.tile(tf.expand_dims(_SCREAMING_SNAKE_CASE    ,     1 )    ,     (1, self.num_choices, 1) )
           A_     =   tf.tile(tf.expand_dims(_SCREAMING_SNAKE_CASE    ,     1 )    ,     (1, self.num_choices, 1) )
           A_     =   {
               '''input_ids''': multiple_choice_inputs_ids,
               '''attention_mask''': multiple_choice_input_mask,
               '''token_type_ids''': multiple_choice_token_type_ids,
           }
           A_     =   model(_SCREAMING_SNAKE_CASE )
           self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_choices) )

       def 					__A       (      self )		->     str:
           A_     =   self.prepare_config_and_inputs()
           (
               (
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,(
               A_
           )   ,
           )     =   config_and_inputs
           A_     =   {
               '''input_ids''': input_ids,
               '''token_type_ids''': token_type_ids,
               '''langs''': token_type_ids,
               '''lengths''': input_lengths,
           }
           return config, inputs_dict




@require_tf
class 	__UpperCAmelCase				(				_UpperCamelCase						,      _UpperCamelCase						,      unittest.TestCase     ):





       '''simple docstring'''



       __lowercase					:					str    =				(
           (
               TFXLMModel,
               TFXLMWithLMHeadModel,
               TFXLMForSequenceClassification,
               TFXLMForQuestionAnsweringSimple,
               TFXLMForTokenClassification,
               TFXLMForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       __lowercase					:					str    =				(
           (TFXLMWithLMHeadModel,) if is_tf_available() else ()
       )  # TODO (PVP): Check other models whether language generation is also applicable
       __lowercase					:					Tuple    =				(
           {
               'feature-extraction': TFXLMModel,
               'fill-mask': TFXLMWithLMHeadModel,
               'question-answering': TFXLMForQuestionAnsweringSimple,
               'text-classification': TFXLMForSequenceClassification,
               'text-generation': TFXLMWithLMHeadModel,
               'token-classification': TFXLMForTokenClassification,
               'zero-shot': TFXLMForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       __lowercase					:					Tuple    =				False
       __lowercase					:					Union[str, Any]    =				False

       def 					__A       (      self    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE )		->     List[str]:
           if (
               pipeline_test_casse_name == "QAPipelineTests"
               and tokenizer_name is not None
               and not tokenizer_name.endswith('''Fast''' )
           ):
               # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
               # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
               # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
               return True

           return False

       def 					__A       (      self )		->     Dict:
           A_     =   TFXLMModelTester(self )
           A_     =   ConfigTester(self    ,     config_class=_SCREAMING_SNAKE_CASE    ,     emb_dim=37 )

       def 					__A       (      self )		->     Tuple:
           self.config_tester.run_common_tests()

       def 					__A       (      self )		->     Optional[Any]:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_xlm_model(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     Any:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_xlm_lm_head(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     List[Any]:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_xlm_qa(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     int:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_xlm_sequence_classif(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     int:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_xlm_for_token_classification(*_SCREAMING_SNAKE_CASE )

       def 					__A       (      self )		->     Optional[Any]:
           A_     =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_xlm_for_multiple_choice(*_SCREAMING_SNAKE_CASE )

       @slow
       def 					__A       (      self )		->     List[Any]:
           for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               A_     =   TFXLMModel.from_pretrained(_SCREAMING_SNAKE_CASE )
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE )




@require_tf
class 	__UpperCAmelCase				(				unittest.TestCase     ):





       '''simple docstring'''



       @slow
       def 					__A       (      self )		->     Union[str, Any]:
           A_     =   TFXLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048''' )
           A_     =   tf.convert_to_tensor([[14, 447]]    ,     dtype=tf.intaa )  # the president
           A_     =   [
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
               14,
               447,
           ]  # the president the president the president the president the president the president the president the president the president the president
           # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
           A_     =   model.generate(_SCREAMING_SNAKE_CASE    ,     do_sample=_SCREAMING_SNAKE_CASE )
           self.assertListEqual(output_ids[0].numpy().tolist()    ,     _SCREAMING_SNAKE_CASE )







'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available


__snake_case   :      Optional[int]						    =  {}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case   :      Optional[Any]						    =  ['MBart50Tokenizer']

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case   :      Dict						    =  ['MBart50TokenizerFast']


if TYPE_CHECKING:
  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_mbartaa import MBartaaTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_mbartaa_fast import MBartaaTokenizerFast

else:
  import sys

  __snake_case   :      Tuple						    =  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






