


'''simple docstring'''




import gc
import unittest

import torch
from parameterized import parameterized

from diffusers import AutoencoderKL
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()




class 			_lowercase (	__lowercase						,					__lowercase						,					unittest.TestCase							):
   _SCREAMING_SNAKE_CASE  :  List[str]          =	AutoencoderKL
   _SCREAMING_SNAKE_CASE  :  Union[str, Any]          =	"sample"
   _SCREAMING_SNAKE_CASE  :  Union[str, Any]          =	1e-2
   @property
   def a  (						self       :      List[str]      ) ->							Optional[int]:
        __snake_case           =						4
        __snake_case           =						3
        __snake_case           =						(32, 32)

        __snake_case           =						floats_tensor((batch_size, num_channels) + sizes      ).to(SCREAMING_SNAKE_CASE_      )

        return {"sample": image}
   @property
   def a  (						self       :      List[Any]      ) ->							List[Any]:
        return (3, 32, 32)
   @property
   def a  (						self       :      int      ) ->							int:
        return (3, 32, 32)
   def a  (						self       :      Tuple      ) ->							Union[str, Any]:
        __snake_case           =						{
            'block_out_channels': [32, 64],
            'in_channels': 3,
            'out_channels': 3,
            'down_block_types': ['DownEncoderBlock2D', 'DownEncoderBlock2D'],
            'up_block_types': ['UpDecoderBlock2D', 'UpDecoderBlock2D'],
            'latent_channels': 4,
        }
        __snake_case           =						self.dummy_input
        return init_dict, inputs_dict
   def a  (						self       :      Optional[Any]      ) ->							Any:
        pass
   def a  (						self       :      Tuple      ) ->							List[Any]:
        pass
   @unittest.skipIf(torch_device == 'mps'       ,   'Gradient checkpointing skipped on MPS'      )
   def a  (						self       :      List[str]      ) ->							int:
        # enable deterministic behavior for gradient checkpointing
        __snake_case      ,       __snake_case           =						self.prepare_init_args_and_inputs_for_common()
        __snake_case           =						self.model_class(**SCREAMING_SNAKE_CASE_      )
        model.to(SCREAMING_SNAKE_CASE_      )

        assert not model.is_gradient_checkpointing and model.training

        __snake_case           =						model(**SCREAMING_SNAKE_CASE_      ).sample
        # run the backwards pass on the model. For backwards pass, for simplicity purpose,
        # we won't calculate the loss and rather backprop on out.sum()
        model.zero_grad()

        __snake_case           =						torch.randn_like(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						(out - labels).mean()
        loss.backward()

        # re-instantiate the model now enabling gradient checkpointing
        __snake_case           =						self.model_class(**SCREAMING_SNAKE_CASE_      )
        # clone model
        model_a.load_state_dict(model.state_dict()      )
        model_a.to(SCREAMING_SNAKE_CASE_      )
        model_a.enable_gradient_checkpointing()

        assert model_a.is_gradient_checkpointing and model_a.training

        __snake_case           =						model_a(**SCREAMING_SNAKE_CASE_      ).sample
        # run the backwards pass on the model. For backwards pass, for simplicity purpose,
        # we won't calculate the loss and rather backprop on out.sum()
        model_a.zero_grad()
        __snake_case           =						(out_a - labels).mean()
        loss_a.backward()

        # compare the output and parameters gradients
        self.assertTrue((loss - loss_a).abs() < 1e-5      )
        __snake_case           =						dict(model.named_parameters()      )
        __snake_case           =						dict(model_a.named_parameters()      )
        for name, param in named_params.items():
             self.assertTrue(torch_all_close(param.grad.data       ,   named_params_a[name].grad.data       ,   atol=5e-5      )      )
   def a  (						self       :      int      ) ->							int:
        __snake_case      ,       __snake_case           =						AutoencoderKL.from_pretrained('fusing/autoencoder-kl-dummy'       ,   output_loading_info=SCREAMING_SNAKE_CASE_      )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )
        self.assertEqual(len(loading_info['missing_keys']      )       ,   0      )

        model.to(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						model(**self.dummy_input      )

        assert image is not None, "Make sure output is not None"





   def a  (						self       :      Optional[int]      ) ->							List[str]:
        __snake_case           =						AutoencoderKL.from_pretrained('fusing/autoencoder-kl-dummy'      )
        __snake_case           =						model.to(SCREAMING_SNAKE_CASE_      )
        model.eval()

        if torch_device == "mps":
             __snake_case           =						torch.manual_seed(0      )
        else:
             __snake_case           =						torch.Generator(device=SCREAMING_SNAKE_CASE_      ).manual_seed(0      )

        __snake_case           =						torch.randn(
            1       ,   model.config.in_channels       ,   model.config.sample_size       ,   model.config.sample_size       ,   generator=torch.manual_seed(0      )       ,   )
        __snake_case           =						image.to(SCREAMING_SNAKE_CASE_      )
        with torch.no_grad():
             __snake_case           =						model(SCREAMING_SNAKE_CASE_       ,   sample_posterior=SCREAMING_SNAKE_CASE_       ,   generator=SCREAMING_SNAKE_CASE_      ).sample

        __snake_case           =						output[0, -1, -3:, -3:].flatten().cpu()

        # Since the VAE Gaussian prior's generator is seeded on the appropriate device,
        # the expected output slices are not the same for CPU and GPU.
        if torch_device == "mps":
             __snake_case           =						torch.tensor(
                 [
                     -4.0_078e-01,
                     -3.8_323e-04,
                     -1.2_681e-01,
                     -1.1_462e-01,
                     2.0_095e-01,
                     1.0_893e-01,
                     -8.8_247e-02,
                     -3.0_361e-01,
                     -9.8_644e-03,
                 ]      )
        elif torch_device == "cpu":
             __snake_case           =						torch.tensor(
                 [-0.1_3_5_2, 0.0_8_7_8, 0.0_4_1_9, -0.0_8_1_8, -0.1_0_6_9, 0.0_6_8_8, -0.1_4_5_8, -0.4_4_4_6, -0.0_0_2_6]      )
        else:
             __snake_case           =						torch.tensor(
                 [-0.2_4_2_1, 0.4_6_4_2, 0.2_5_0_7, -0.0_4_3_8, 0.0_6_8_2, 0.3_1_6_0, -0.2_0_1_8, -0.0_7_2_7, 0.2_4_8_5]      )

        self.assertTrue(torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   rtol=1e-2      )      )




@slow
class 			_lowercase (	unittest.TestCase							):
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      List[Any]       ,   SCREAMING_SNAKE_CASE_       :      Any      ) ->							Union[str, Any]:
        return f'gaussian_noise_s={seed}_shape={"_".join([str(SCREAMING_SNAKE_CASE_      ) for s in shape]      )}.npy'
   def a  (						self       :      Optional[Any]      ) ->							Optional[int]:
        # clean up the VRAM after each test
        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()
   def a  (						self       :      int       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]=0       ,   SCREAMING_SNAKE_CASE_       :      int=(4, 3, 512, 512)       ,   SCREAMING_SNAKE_CASE_       :      str=False      ) ->							int:
        __snake_case           =						torch.floataa if fpaa else torch.floataa
        __snake_case           =						torch.from_numpy(load_hf_numpy(self.get_file_format(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )      )      ).to(SCREAMING_SNAKE_CASE_      ).to(SCREAMING_SNAKE_CASE_      )
        return image
   def a  (						self       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      Tuple="CompVis/stable-diffusion-v1-4"       ,   SCREAMING_SNAKE_CASE_       :      Union[str, Any]=False      ) ->							List[str]:
        __snake_case           =						'fp16' if fpaa else None
        __snake_case           =						torch.floataa if fpaa else torch.floataa

        __snake_case           =						AutoencoderKL.from_pretrained(
            SCREAMING_SNAKE_CASE_       ,   subfolder='vae'       ,   torch_dtype=SCREAMING_SNAKE_CASE_       ,   revision=SCREAMING_SNAKE_CASE_       ,   )
        model.to(SCREAMING_SNAKE_CASE_      ).eval()

        return model
   def a  (						self       :      List[str]       ,   SCREAMING_SNAKE_CASE_       :      Tuple=0      ) ->							Union[str, Any]:
        if torch_device == "mps":
             return torch.manual_seed(SCREAMING_SNAKE_CASE_      )
        return torch.Generator(device=SCREAMING_SNAKE_CASE_      ).manual_seed(SCREAMING_SNAKE_CASE_      )
   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.1_6_0_3, 0.9_8_7_8, -0.0_4_9_5, -0.0_7_9_0, -0.2_7_0_9, 0.8_3_7_5, -0.2_0_6_0, -0.0_8_2_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
           [47, [-0.2_3_7_6, 0.1_1_6_8, 0.1_3_3_2, -0.4_8_4_0, -0.2_5_0_8, -0.0_7_9_1, -0.0_4_9_3, -0.4_0_8_9], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
           # fmt: on
       ]      )
   def a  (						self       :      Dict       ,   SCREAMING_SNAKE_CASE_       :      Dict       ,   SCREAMING_SNAKE_CASE_       :      Dict       ,   SCREAMING_SNAKE_CASE_       :      Optional[int]      ) ->							Optional[Any]:
        __snake_case           =						self.get_sd_vae_model()
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.get_generator(SCREAMING_SNAKE_CASE_      )

        with torch.no_grad():
             __snake_case           =						model(SCREAMING_SNAKE_CASE_       ,   generator=SCREAMING_SNAKE_CASE_       ,   sample_posterior=SCREAMING_SNAKE_CASE_      ).sample

        assert sample.shape == image.shape

        __snake_case           =						sample[-1, -2:, -2:, :2].flatten().float().cpu()
        __snake_case           =						torch.tensor(expected_slice_mps if torch_device == 'mps' else expected_slice      )

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=3e-3      )
   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.0_5_1_3, 0.0_2_8_9, 1.3_7_9_9, 0.2_1_6_6, -0.2_5_7_3, -0.0_8_7_1, 0.5_1_0_3, -0.0_9_9_9]],
           [47, [-0.4_1_2_8, -0.1_3_2_0, -0.3_7_0_4, 0.1_9_6_5, -0.4_1_1_6, -0.2_3_3_2, -0.3_3_4_0, 0.2_2_4_7]],
           # fmt: on
       ]      )
   @require_torch_gpu
   def a  (						self       :      Optional[int]       ,   SCREAMING_SNAKE_CASE_       :      Union[str, Any]       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]      ) ->							Union[str, Any]:
        __snake_case           =						self.get_sd_vae_model(fpaa=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_       ,   fpaa=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.get_generator(SCREAMING_SNAKE_CASE_      )

        with torch.no_grad():
             __snake_case           =						model(SCREAMING_SNAKE_CASE_       ,   generator=SCREAMING_SNAKE_CASE_       ,   sample_posterior=SCREAMING_SNAKE_CASE_      ).sample

        assert sample.shape == image.shape

        __snake_case           =						sample[-1, -2:, :2, -2:].flatten().float().cpu()
        __snake_case           =						torch.tensor(SCREAMING_SNAKE_CASE_      )

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=1e-2      )
   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.1_6_0_9, 0.9_8_6_6, -0.0_4_8_7, -0.0_7_7_7, -0.2_7_1_6, 0.8_3_6_8, -0.2_0_5_5, -0.0_8_1_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
           [47, [-0.2_3_7_7, 0.1_1_4_7, 0.1_3_3_3, -0.4_8_4_1, -0.2_5_0_6, -0.0_8_0_5, -0.0_4_9_1, -0.4_0_8_5], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
           # fmt: on
       ]      )
   def a  (						self       :      str       ,   SCREAMING_SNAKE_CASE_       :      int       ,   SCREAMING_SNAKE_CASE_       :      Union[str, Any]       ,   SCREAMING_SNAKE_CASE_       :      Dict      ) ->							List[Any]:
        __snake_case           =						self.get_sd_vae_model()
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_      )

        with torch.no_grad():
             __snake_case           =						model(SCREAMING_SNAKE_CASE_      ).sample

        assert sample.shape == image.shape

        __snake_case           =						sample[-1, -2:, -2:, :2].flatten().float().cpu()
        __snake_case           =						torch.tensor(expected_slice_mps if torch_device == 'mps' else expected_slice      )

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=3e-3      )
   @parameterized.expand(
       [
           # fmt: off
           [13, [-0.2_0_5_1, -0.1_8_0_3, -0.2_3_1_1, -0.2_1_1_4, -0.3_2_9_2, -0.3_5_7_4, -0.2_9_5_3, -0.3_3_2_3]],
           [37, [-0.2_6_3_2, -0.2_6_2_5, -0.2_1_9_9, -0.2_7_4_1, -0.4_5_3_9, -0.4_9_9_0, -0.3_7_2_0, -0.4_9_2_5]],
           # fmt: on
       ]      )
   @require_torch_gpu
   def a  (						self       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      int       ,   SCREAMING_SNAKE_CASE_       :      Union[str, Any]      ) ->							int:
        __snake_case           =						self.get_sd_vae_model()
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_       ,   shape=(3, 4, 64, 64)      )

        with torch.no_grad():
             __snake_case           =						model.decode(SCREAMING_SNAKE_CASE_      ).sample

        assert list(sample.shape      ) == [3, 3, 512, 512]

        __snake_case           =						sample[-1, -2:, :2, -2:].flatten().cpu()
        __snake_case           =						torch.tensor(SCREAMING_SNAKE_CASE_      )

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=1e-3      )
   @parameterized.expand(
       [
           # fmt: off
           [27, [-0.0_3_6_9, 0.0_2_0_7, -0.0_7_7_6, -0.0_6_8_2, -0.1_7_4_7, -0.1_9_3_0, -0.1_4_6_5, -0.2_0_3_9]],
           [16, [-0.1_6_2_8, -0.2_1_3_4, -0.2_7_4_7, -0.2_6_4_2, -0.3_7_7_4, -0.4_4_0_4, -0.3_6_8_7, -0.4_2_7_7]],
           # fmt: on
       ]      )
   @require_torch_gpu
   def a  (						self       :      int       ,   SCREAMING_SNAKE_CASE_       :      List[Any]       ,   SCREAMING_SNAKE_CASE_       :      List[Any]      ) ->							str:
        __snake_case           =						self.get_sd_vae_model(fpaa=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_       ,   shape=(3, 4, 64, 64)       ,   fpaa=SCREAMING_SNAKE_CASE_      )

        with torch.no_grad():
             __snake_case           =						model.decode(SCREAMING_SNAKE_CASE_      ).sample

        assert list(sample.shape      ) == [3, 3, 512, 512]

        __snake_case           =						sample[-1, -2:, :2, -2:].flatten().float().cpu()
        __snake_case           =						torch.tensor(SCREAMING_SNAKE_CASE_      )

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=5e-3      )
   @parameterized.expand([(13,), (16,), (27,)]      )
   @require_torch_gpu
   @unittest.skipIf(not is_xformers_available()       ,   reason='xformers is not required when using PyTorch 2.0.'      )
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      int      ) ->							Tuple:
        __snake_case           =						self.get_sd_vae_model(fpaa=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_       ,   shape=(3, 4, 64, 64)       ,   fpaa=SCREAMING_SNAKE_CASE_      )

        with torch.no_grad():
             __snake_case           =						model.decode(SCREAMING_SNAKE_CASE_      ).sample

        model.enable_xformers_memory_efficient_attention()
        with torch.no_grad():
             __snake_case           =						model.decode(SCREAMING_SNAKE_CASE_      ).sample

        assert list(sample.shape      ) == [3, 3, 512, 512]

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=1e-1      )
   @parameterized.expand([(13,), (16,), (37,)]      )
   @require_torch_gpu
   @unittest.skipIf(not is_xformers_available()       ,   reason='xformers is not required when using PyTorch 2.0.'      )
   def a  (						self       :      List[Any]       ,   SCREAMING_SNAKE_CASE_       :      int      ) ->							str:
        __snake_case           =						self.get_sd_vae_model()
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_       ,   shape=(3, 4, 64, 64)      )

        with torch.no_grad():
             __snake_case           =						model.decode(SCREAMING_SNAKE_CASE_      ).sample

        model.enable_xformers_memory_efficient_attention()
        with torch.no_grad():
             __snake_case           =						model.decode(SCREAMING_SNAKE_CASE_      ).sample

        assert list(sample.shape      ) == [3, 3, 512, 512]

        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=1e-2      )





   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.3_0_0_1, 0.0_9_1_8, -2.6_9_8_4, -3.9_7_2_0, -3.2_0_9_9, -5.0_3_5_3, 1.7_3_3_8, -0.2_0_6_5, 3.4_2_6_7]],
           [47, [-1.5_0_3_0, -4.3_8_7_1, -6.0_3_5_5, -9.1_1_5_7, -1.6_6_6_1, -2.7_8_5_3, 2.1_6_0_7, -5.0_8_2_3, 2.5_6_3_3]],
           # fmt: on
       ]      )
   def a  (						self       :      int       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      Optional[Any]      ) ->							Union[str, Any]:
        __snake_case           =						self.get_sd_vae_model()
        __snake_case           =						self.get_sd_image(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						self.get_generator(SCREAMING_SNAKE_CASE_      )

        with torch.no_grad():
             __snake_case           =						model.encode(SCREAMING_SNAKE_CASE_      ).latent_dist
             __snake_case           =						dist.sample(generator=SCREAMING_SNAKE_CASE_      )

        assert list(sample.shape      ) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]]

        __snake_case           =						sample[0, -1, -3:, -3:].flatten().cpu()
        __snake_case           =						torch.tensor(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						3e-3 if torch_device != 'mps' else 1e-2
        assert torch_all_close(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_       ,   atol=SCREAMING_SNAKE_CASE_      )








import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    FlaubertForMultipleChoice,
	    FlaubertForQuestionAnswering,
	    FlaubertForQuestionAnsweringSimple,
	    FlaubertForSequenceClassification,
	    FlaubertForTokenClassification,
	    FlaubertModel,
	    FlaubertWithLMHeadModel,
	)
	from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					def __init__(self       :		Optional[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		Any=13 ,   _UpperCAmelCase       :		str=7 ,   _UpperCAmelCase       :		str=True ,   _UpperCAmelCase       :		List[Any]=True ,   _UpperCAmelCase       :		Optional[int]=True ,   _UpperCAmelCase       :		List[Any]=True ,   _UpperCAmelCase       :		Tuple=True ,   _UpperCAmelCase       :		Optional[int]=False ,   _UpperCAmelCase       :		Any=False ,   _UpperCAmelCase       :		List[Any]=False ,   _UpperCAmelCase       :		Tuple=2 ,   _UpperCAmelCase       :		Optional[int]=99 ,   _UpperCAmelCase       :		Any=0 ,   _UpperCAmelCase       :		Dict=32 ,   _UpperCAmelCase       :		Tuple=5 ,   _UpperCAmelCase       :		Union[str, Any]=4 ,   _UpperCAmelCase       :		Union[str, Any]=0.1 ,   _UpperCAmelCase       :		Optional[Any]=0.1 ,   _UpperCAmelCase       :		List[str]=512 ,   _UpperCAmelCase       :		Tuple=12 ,   _UpperCAmelCase       :		int=2 ,   _UpperCAmelCase       :		Tuple=0.02 ,   _UpperCAmelCase       :		Union[str, Any]=3 ,   _UpperCAmelCase       :		Optional[Any]=4 ,   _UpperCAmelCase       :		Dict="last" ,   _UpperCAmelCase       :		Any=None ,   _UpperCAmelCase       :		int=None ,   )		->  int:




							"""simple docstring"""


							lowercase__	     =						parent
							lowercase__	     =						batch_size
							lowercase__	     =						seq_length
							lowercase__	     =						is_training
							lowercase__	     =						use_input_lengths
							lowercase__	     =						use_token_type_ids
							lowercase__	     =						use_labels
							lowercase__	     =						gelu_activation
							lowercase__	     =						sinusoidal_embeddings
							lowercase__	     =						causal
							lowercase__	     =						asm
							lowercase__	     =						n_langs
							lowercase__	     =						vocab_size
							lowercase__	     =						n_special
							lowercase__	     =						hidden_size
							lowercase__	     =						num_hidden_layers
							lowercase__	     =						num_attention_heads
							lowercase__	     =						hidden_dropout_prob
							lowercase__	     =						attention_probs_dropout_prob
							lowercase__	     =						max_position_embeddings
							lowercase__	     =						type_vocab_size
							lowercase__	     =						type_sequence_label_size
							lowercase__	     =						initializer_range
							lowercase__	     =						num_labels
							lowercase__	     =						num_choices
							lowercase__	     =						summary_type
							lowercase__	     =						use_proj
							lowercase__	     =						scope

					def     lowerCamelCase__	(self       :		List[str]						)		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						ids_tensor([self.batch_size, self.seq_length] ,   self.vocab_size						)
							lowercase__	     =						random_attention_mask([self.batch_size, self.seq_length]						)

							lowercase__	     =						None
							if self.use_input_lengths:
									lowercase__	     =						(
									    ids_tensor([self.batch_size] ,   vocab_size=2						) + self.seq_length - 2
									)  # small variation of seq_length

							lowercase__	     =						None
							if self.use_token_type_ids:
									lowercase__	     =						ids_tensor([self.batch_size, self.seq_length] ,   self.n_langs						)

							lowercase__	     =						None
							lowercase__	     =						None
							lowercase__	     =						None
							if self.use_labels:
									lowercase__	     =						ids_tensor([self.batch_size] ,   self.type_sequence_label_size						)
									lowercase__	     =						ids_tensor([self.batch_size, self.seq_length] ,   self.num_labels						)
									lowercase__	     =						ids_tensor([self.batch_size] ,   2						).float()
									lowercase__	     =						ids_tensor([self.batch_size] ,   self.num_choices						)

							lowercase__	     =						self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_lengths,
							    sequence_labels,
							    token_labels,
							    is_impossible_labels,
							    choice_labels,
							    input_mask,
							)

					def     lowerCamelCase__	(self       :		int						)		->  Dict:




							"""simple docstring"""


							return FlaubertConfig(
							    vocab_size=self.vocab_size ,   n_special=self.n_special ,   emb_dim=self.hidden_size ,   n_layers=self.num_hidden_layers ,   n_heads=self.num_attention_heads ,   dropout=self.hidden_dropout_prob ,   attention_dropout=self.attention_probs_dropout_prob ,   gelu_activation=self.gelu_activation ,   sinusoidal_embeddings=self.sinusoidal_embeddings ,   asm=self.asm ,   causal=self.causal ,   n_langs=self.n_langs ,   max_position_embeddings=self.max_position_embeddings ,   initializer_range=self.initializer_range ,   summary_type=self.summary_type ,   use_proj=self.use_proj ,   )

					def     lowerCamelCase__	(self       :		Tuple ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Tuple ,   )		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertModel(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						model(_UpperCAmelCase ,   lengths=_UpperCAmelCase ,   langs=_UpperCAmelCase						)
							lowercase__	     =						model(_UpperCAmelCase ,   langs=_UpperCAmelCase						)
							lowercase__	     =						model(_UpperCAmelCase						)
							self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size)						)

					def     lowerCamelCase__	(self       :		Union[str, Any] ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		List[str] ,   )		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						FlaubertWithLMHeadModel(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase ,   token_type_ids=_UpperCAmelCase ,   labels=_UpperCAmelCase						)
							self.parent.assertEqual(result.loss.shape ,   ()						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size)						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Union[str, Any] ,   )		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertForQuestionAnsweringSimple(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase						)

							lowercase__	     =						model(_UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase						)
							self.parent.assertEqual(result.start_logits.shape ,   (self.batch_size, self.seq_length)						)
							self.parent.assertEqual(result.end_logits.shape ,   (self.batch_size, self.seq_length)						)

					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Optional[int] ,   )		->  List[Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertForQuestionAnswering(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase						)

							lowercase__	     =						model(
							    _UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase ,   cls_index=_UpperCAmelCase ,   is_impossible=_UpperCAmelCase ,   p_mask=_UpperCAmelCase ,   )

							lowercase__	     =						model(
							    _UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase ,   cls_index=_UpperCAmelCase ,   is_impossible=_UpperCAmelCase ,   )

							((lowercase__)     ,      )	     =						result_with_labels.to_tuple()

							lowercase__	     =						model(_UpperCAmelCase ,   start_positions=_UpperCAmelCase ,   end_positions=_UpperCAmelCase						)

							((lowercase__)     ,      )	     =						result_with_labels.to_tuple()

							self.parent.assertEqual(result_with_labels.loss.shape ,   ()						)
							self.parent.assertEqual(result.start_top_log_probs.shape ,   (self.batch_size, model.config.start_n_top)						)
							self.parent.assertEqual(result.start_top_index.shape ,   (self.batch_size, model.config.start_n_top)						)
							self.parent.assertEqual(
							    result.end_top_log_probs.shape ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
							self.parent.assertEqual(
							    result.end_top_index.shape ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
							self.parent.assertEqual(result.cls_logits.shape ,   (self.batch_size,)						)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Any ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		List[str] ,   )		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						FlaubertForSequenceClassification(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase						)
							lowercase__	     =						model(_UpperCAmelCase ,   labels=_UpperCAmelCase						)

							self.parent.assertEqual(result.loss.shape ,   ()						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.type_sequence_label_size)						)

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		Union[str, Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Dict ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Optional[Any] ,   )		->  str:




							"""simple docstring"""


							lowercase__	     =						self.num_labels
							lowercase__	     =						FlaubertForTokenClassification(_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()

							lowercase__	     =						model(_UpperCAmelCase ,   attention_mask=_UpperCAmelCase ,   labels=_UpperCAmelCase						)
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.num_labels)						)

					def     lowerCamelCase__	(self       :		List[str] ,   _UpperCAmelCase       :		Optional[Any] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[str] ,   )		->  List[str]:




							"""simple docstring"""


							lowercase__	     =						self.num_choices
							lowercase__	     =						FlaubertForMultipleChoice(config=_UpperCAmelCase						)
							model.to(_UpperCAmelCase						)
							model.eval()
							lowercase__	     =						input_ids.unsqueeze(1						).expand(-1 ,   self.num_choices ,   -1						).contiguous()
							lowercase__	     =						token_type_ids.unsqueeze(1						).expand(-1 ,   self.num_choices ,   -1						).contiguous()
							lowercase__	     =						input_mask.unsqueeze(1						).expand(-1 ,   self.num_choices ,   -1						).contiguous()
							lowercase__	     =						model(
							    _UpperCAmelCase ,   attention_mask=_UpperCAmelCase ,   token_type_ids=_UpperCAmelCase ,   labels=_UpperCAmelCase ,   )
							self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.num_choices)						)


					def     lowerCamelCase__	(self       :		Dict						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.prepare_config_and_inputs()
							(
							    (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      (
							    lowercase__
							)     ,      
							)	     =						config_and_inputs
							lowercase__	     =						{
							    """input_ids""": input_ids,
							    """token_type_ids""": token_type_ids,
							    """lengths""": input_lengths,
							    """attention_mask""": input_mask,
							}
							return config, inputs_dict






@require_torch
class 				A       (			UpperCAmelCase__					,    UpperCAmelCase__					,    unittest.TestCase				):
					'''simple docstring'''






					A__     =					(
					    (
					        FlaubertModel,
					        FlaubertWithLMHeadModel,
					        FlaubertForQuestionAnswering,
					        FlaubertForQuestionAnsweringSimple,
					        FlaubertForSequenceClassification,
					        FlaubertForTokenClassification,
					        FlaubertForMultipleChoice,
					    )
					    if is_torch_available()
					    else ()
					)
					A__     =					(
					    {
					        '''feature-extraction''': FlaubertModel,
					        '''fill-mask''': FlaubertWithLMHeadModel,
					        '''question-answering''': FlaubertForQuestionAnsweringSimple,
					        '''text-classification''': FlaubertForSequenceClassification,
					        '''token-classification''': FlaubertForTokenClassification,
					        '''zero-shot''': FlaubertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		Tuple ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[Any] ,   _UpperCAmelCase       :		int						)		->  str:




							"""simple docstring"""


							if (
							    pipeline_test_casse_name == "QAPipelineTests"
							    and tokenizer_name is not None
							    and not tokenizer_name.endswith("""Fast"""						)
							):
									# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
									# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
									# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
									return True

							return False

					def     lowerCamelCase__	(self       :		Any ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		List[str] ,   _UpperCAmelCase       :		Union[str, Any]=False						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						super()._prepare_for_class(_UpperCAmelCase ,   _UpperCAmelCase ,   return_labels=_UpperCAmelCase						)

							if return_labels:
									if model_class.__name__ == "FlaubertForQuestionAnswering":
											lowercase__	     =						torch.zeros(
											    self.model_tester.batch_size ,   dtype=torch.long ,   device=_UpperCAmelCase						)
											lowercase__	     =						torch.zeros(
											    self.model_tester.batch_size ,   dtype=torch.long ,   device=_UpperCAmelCase						)

							return inputs_dict

					def     lowerCamelCase__	(self       :		str						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						FlaubertModelTester(self						)
							lowercase__	     =						ConfigTester(self ,   config_class=_UpperCAmelCase ,   emb_dim=37						)

					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  Tuple:




							"""simple docstring"""


							self.config_tester.run_common_tests()

					def     lowerCamelCase__	(self       :		List[Any]						)		->  Optional[int]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_model(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		List[Any]						)		->  str:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_lm_head(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  Tuple:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_simple_qa(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		str						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_qa(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		List[str]						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_sequence_classif(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Union[str, Any]						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_token_classif(*_UpperCAmelCase						)

					def     lowerCamelCase__	(self       :		Optional[Any]						)		->  Optional[Any]:




							"""simple docstring"""


							lowercase__	     =						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_flaubert_multiple_choice(*_UpperCAmelCase						)

					@slow
					def     lowerCamelCase__	(self       :		int						)		->  List[Any]:




							"""simple docstring"""


							for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__	     =						FlaubertModel.from_pretrained(_UpperCAmelCase						)
									self.assertIsNotNone(_UpperCAmelCase						)


					@slow
					@require_torch_gpu
					def     lowerCamelCase__	(self       :		Dict						)		->  int:




							"""simple docstring"""


							lowercase__     ,      lowercase__	     =						self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
									# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
									if model_class == FlaubertForMultipleChoice:
											return

									lowercase__	     =						True
									lowercase__	     =						model_class(config=_UpperCAmelCase						)

									lowercase__	     =						self._prepare_for_class(_UpperCAmelCase ,   _UpperCAmelCase						)
									lowercase__	     =						torch.jit.trace(
									    _UpperCAmelCase ,   (inputs_dict["""input_ids"""].to("""cpu"""						), inputs_dict["""attention_mask"""].to("""cpu"""						))						)

									with tempfile.TemporaryDirectory() as tmp:
											torch.jit.save(_UpperCAmelCase ,   os.path.join(_UpperCAmelCase ,   """traced_model.pt"""						)						)
											lowercase__	     =						torch.jit.load(os.path.join(_UpperCAmelCase ,   """traced_model.pt"""						) ,   map_location=_UpperCAmelCase						)
											loaded(inputs_dict["""input_ids"""].to(_UpperCAmelCase						) ,   inputs_dict["""attention_mask"""].to(_UpperCAmelCase						)						)






@require_torch
class 				A       (			unittest.TestCase				):
					'''simple docstring'''






					@slow
					def     lowerCamelCase__	(self       :		Tuple						)		->  Dict:




							"""simple docstring"""


							lowercase__	     =						FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""						)
							lowercase__	     =						torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]						)
							with torch.no_grad():
									lowercase__	     =						model(_UpperCAmelCase						)[0]
							lowercase__	     =						torch.Size((1, 11, 768)						)
							self.assertEqual(output.shape ,   _UpperCAmelCase						)
							lowercase__	     =						torch.tensor(
							    [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]						)

							self.assertTrue(torch.allclose(output[:, :3, :3] ,   _UpperCAmelCase ,   atol=1E-4						)						)

