

'''simple docstring'''





import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import XLMRobertaTokenizerFast

from diffusers import DDIMScheduler, KandinskyInpaintPipeline, KandinskyPriorPipeline, UNetaDConditionModel, VQModel
from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()





class 		__SCREAMING_SNAKE_CASE							(			lowercase__						,							unittest.TestCase      ):
				lowerCamelCase_		       =    KandinskyInpaintPipeline
				lowerCamelCase_		       =    ['prompt', 'image_embeds', 'negative_image_embeds', 'image', 'mask_image']
				lowerCamelCase_		       =    [
				    'prompt',
				    'negative_prompt',
				    'image_embeds',
				    'negative_image_embeds',
				    'image',
				    'mask_image',
				]
				lowerCamelCase_		       =    [
				    'generator',
				    'height',
				    'width',
				    'latents',
				    'guidance_scale',
				    'negative_prompt',
				    'num_inference_steps',
				    'return_dict',
				    'guidance_scale',
				    'num_images_per_prompt',
				    'output_type',
				    'return_dict',
				]
				lowerCamelCase_		       =    False







				@property
				def  lowerCamelCase_	(       self     :		Optional[Any]   ):
										'''simple docstring'''
										return 32







				@property
				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										return 32







				@property
				def  lowerCamelCase_	(       self     :		str   ):
										'''simple docstring'''
										return self.time_input_dim







				@property
				def  lowerCamelCase_	(       self     :		Optional[int]   ):
										'''simple docstring'''
										return self.time_input_dim * 4







				@property
				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										return 100







				@property
				def  lowerCamelCase_	(       self     :		Any   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=XLMRobertaTokenizerFast.from_pretrained('''YiYiXu/tiny-random-mclip-base'''   )
										return tokenizer







				@property
				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										torch.manual_seed(0   )
										lowercase :							int    		=MCLIPConfig(
										    numDims=self.cross_attention_dim		,					transformerDimensions=self.text_embedder_hidden_size		,					hidden_size=self.text_embedder_hidden_size		,					intermediate_size=37		,					num_attention_heads=4		,					num_hidden_layers=5		,					vocab_size=1005		,					)

										lowercase :							Union[str, Any]    		=MultilingualCLIP(UpperCAmelCase__   )
										lowercase :							int    		=text_encoder.eval()

										return text_encoder







				@property
				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										torch.manual_seed(0   )

										lowercase :							Optional[Any]    		={
										    '''in_channels''': 9,
										    # Out channels is double in channels because predicts mean and variance
										    '''out_channels''': 8,
										    '''addition_embed_type''': '''text_image''',
										    '''down_block_types''': ('''ResnetDownsampleBlock2D''', '''SimpleCrossAttnDownBlock2D'''),
										    '''up_block_types''': ('''SimpleCrossAttnUpBlock2D''', '''ResnetUpsampleBlock2D'''),
										    '''mid_block_type''': '''UNetMidBlock2DSimpleCrossAttn''',
										    '''block_out_channels''': (self.block_out_channels_a, self.block_out_channels_a * 2),
										    '''layers_per_block''': 1,
										    '''encoder_hid_dim''': self.text_embedder_hidden_size,
										    '''encoder_hid_dim_type''': '''text_image_proj''',
										    '''cross_attention_dim''': self.cross_attention_dim,
										    '''attention_head_dim''': 4,
										    '''resnet_time_scale_shift''': '''scale_shift''',
										    '''class_embed_type''': None,
										}

										lowercase :							Optional[int]    		=UNetaDConditionModel(**UpperCAmelCase__   )
										return model







				@property
				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										return {
										    "block_out_channels": [32, 64],
										    "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"],
										    "in_channels": 3,
										    "latent_channels": 4,
										    "layers_per_block": 1,
										    "norm_num_groups": 8,
										    "norm_type": "spatial",
										    "num_vq_embeddings": 12,
										    "out_channels": 3,
										    "up_block_types": [
										        "AttnUpDecoderBlock2D",
										        "UpDecoderBlock2D",
										    ],
										    "vq_embed_dim": 4,
										}







				@property
				def  lowerCamelCase_	(       self     :		Dict   ):
										'''simple docstring'''
										torch.manual_seed(0   )
										lowercase :							str    		=VQModel(**self.dummy_movq_kwargs   )
										return model







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							int    		=self.dummy_text_encoder
										lowercase :							int    		=self.dummy_tokenizer
										lowercase :							Optional[int]    		=self.dummy_unet
										lowercase :							Optional[int]    		=self.dummy_movq

										lowercase :							Any    		=DDIMScheduler(
										    num_train_timesteps=1000		,					beta_schedule='''linear'''		,					beta_start=0.0_00_85		,					beta_end=0.0_12		,					clip_sample=UpperCAmelCase__		,					set_alpha_to_one=UpperCAmelCase__		,					steps_offset=1		,					prediction_type='''epsilon'''		,					thresholding=UpperCAmelCase__		,					)

										lowercase :							Dict    		={
										    '''text_encoder''': text_encoder,
										    '''tokenizer''': tokenizer,
										    '''unet''': unet,
										    '''scheduler''': scheduler,
										    '''movq''': movq,
										}

										return components







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Any=0   ):
										'''simple docstring'''
										lowercase :							Optional[Any]    		=floats_tensor((1, self.cross_attention_dim)		,					rng=random.Random(UpperCAmelCase__   )   ).to(UpperCAmelCase__   )
										lowercase :							Tuple    		=floats_tensor((1, self.cross_attention_dim)		,					rng=random.Random(seed + 1   )   ).to(UpperCAmelCase__   )
										# create init_image
										lowercase :							Tuple    		=floats_tensor((1, 3, 64, 64)		,					rng=random.Random(UpperCAmelCase__   )   ).to(UpperCAmelCase__   )
										lowercase :							Any    		=image.cpu().permute(0		,					2		,					3		,					1   )[0]
										lowercase :							List[Any]    		=Image.fromarray(np.uinta(UpperCAmelCase__   )   ).convert('''RGB'''   ).resize((256, 256)   )
										# create mask
										lowercase :							List[str]    		=np.ones((64, 64)		,					dtype=np.floataa   )
										lowercase :							List[str]    		=0

										if str(UpperCAmelCase__   ).startswith('''mps'''   ):
																lowercase :							Optional[int]    		=torch.manual_seed(UpperCAmelCase__   )
										else:
																lowercase :							Dict    		=torch.Generator(device=UpperCAmelCase__   ).manual_seed(UpperCAmelCase__   )
										lowercase :							Optional[int]    		={
										    '''prompt''': '''horse''',
										    '''image''': init_image,
										    '''mask_image''': mask,
										    '''image_embeds''': image_embeds,
										    '''negative_image_embeds''': negative_image_embeds,
										    '''generator''': generator,
										    '''height''': 64,
										    '''width''': 64,
										    '''num_inference_steps''': 2,
										    '''guidance_scale''': 4.0,
										    '''output_type''': '''np''',
										}
										return inputs







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Dict    		='''cpu'''

										lowercase :							Optional[int]    		=self.get_dummy_components()

										lowercase :							Any    		=self.pipeline_class(**UpperCAmelCase__   )
										lowercase :							Any    		=pipe.to(UpperCAmelCase__   )

										pipe.set_progress_bar_config(disable=UpperCAmelCase__   )

										lowercase :							Tuple    		=pipe(**self.get_dummy_inputs(UpperCAmelCase__   )   )
										lowercase :							Any    		=output.images

										lowercase :							List[str]    		=pipe(
										    **self.get_dummy_inputs(UpperCAmelCase__   )		,					return_dict=UpperCAmelCase__		,					)[0]

										lowercase :							Dict    		=image[0, -3:, -3:, -1]
										lowercase :							str    		=image_from_tuple[0, -3:, -3:, -1]

										print(F'''image.shape {image.shape}'''   )

										assert image.shape == (1, 64, 64, 3)

										lowercase :							List[Any]    		=np.array(
										    [0.8_32_69_19, 0.73_79_04_67, 0.20_91_85_81, 0.9_30_96_12, 0.5_51_17_91, 0.43_71_33_28, 0.5_51_33_21, 0.49_92_29_34, 0.59_49_77_86]   )

										assert (
										    np.abs(image_slice.flatten() - expected_slice   ).max() < 1E-2
										), F''' expected_slice {expected_slice}, but got {image_slice.flatten()}'''
										assert (
										    np.abs(image_from_tuple_slice.flatten() - expected_slice   ).max() < 1E-2
										), F''' expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}'''







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										super().test_inference_batch_single_identical(expected_max_diff=3E-3   )





@slow
@require_torch_gpu
class 		__SCREAMING_SNAKE_CASE							(			unittest.TestCase      ):







				def  lowerCamelCase_	(       self     :		Optional[int]   ):
										'''simple docstring'''
										# clean up the VRAM after each test
										super().tearDown()
										gc.collect()
										torch.cuda.empty_cache()







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase :							str    		=load_numpy(
										    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
										    '''/kandinsky/kandinsky_inpaint_cat_with_hat_fp16.npy'''   )

										lowercase :							List[str]    		=load_image(
										    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main''' '''/kandinsky/cat.png'''   )
										lowercase :							Optional[Any]    		=np.ones((768, 768)		,					dtype=np.floataa   )
										lowercase :							Tuple    		=0

										lowercase :							Tuple    		='''a hat'''

										lowercase :							Tuple    		=KandinskyPriorPipeline.from_pretrained(
										    '''kandinsky-community/kandinsky-2-1-prior'''		,					torch_dtype=torch.floataa   )
										pipe_prior.to(UpperCAmelCase__   )

										lowercase :							List[str]    		=KandinskyInpaintPipeline.from_pretrained(
										    '''kandinsky-community/kandinsky-2-1-inpaint'''		,					torch_dtype=torch.floataa   )
										lowercase :							List[str]    		=pipeline.to(UpperCAmelCase__   )
										pipeline.set_progress_bar_config(disable=UpperCAmelCase__   )

										lowercase :							Dict    		=torch.Generator(device='''cpu'''   ).manual_seed(0   )
										lowercase		,	lowercase :							Optional[Any]    		=pipe_prior(
										    UpperCAmelCase__		,					generator=UpperCAmelCase__		,					num_inference_steps=5		,					negative_prompt=''''''		,					).to_tuple()

										lowercase :							List[str]    		=pipeline(
										    UpperCAmelCase__		,					image=UpperCAmelCase__		,					mask_image=UpperCAmelCase__		,					image_embeds=UpperCAmelCase__		,					negative_image_embeds=UpperCAmelCase__		,					generator=UpperCAmelCase__		,					num_inference_steps=100		,					height=768		,					width=768		,					output_type='''np'''		,					)

										lowercase :							List[Any]    		=output.images[0]

										assert image.shape == (768, 768, 3)

										assert_mean_pixel_difference(UpperCAmelCase__		,					UpperCAmelCase__   )








'''simple docstring'''





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    AlbertForMaskedLM,
							    AlbertForMultipleChoice,
							    AlbertForPreTraining,
							    AlbertForQuestionAnswering,
							    AlbertForSequenceClassification,
							    AlbertForTokenClassification,
							    AlbertModel,
							)
							from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class 		__SCREAMING_SNAKE_CASE							:







				def __init__(       self     :		Dict		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		str=13		,					UpperCAmelCase__     :		Tuple=7		,					UpperCAmelCase__     :		Dict=True		,					UpperCAmelCase__     :		Optional[int]=True		,					UpperCAmelCase__     :		Union[str, Any]=True		,					UpperCAmelCase__     :		Tuple=True		,					UpperCAmelCase__     :		Optional[Any]=99		,					UpperCAmelCase__     :		Any=16		,					UpperCAmelCase__     :		Optional[Any]=36		,					UpperCAmelCase__     :		str=6		,					UpperCAmelCase__     :		Any=6		,					UpperCAmelCase__     :		Optional[int]=6		,					UpperCAmelCase__     :		Any=37		,					UpperCAmelCase__     :		List[str]="gelu"		,					UpperCAmelCase__     :		Union[str, Any]=0.1		,					UpperCAmelCase__     :		Optional[int]=0.1		,					UpperCAmelCase__     :		List[str]=512		,					UpperCAmelCase__     :		List[str]=16		,					UpperCAmelCase__     :		Optional[int]=2		,					UpperCAmelCase__     :		Any=0.02		,					UpperCAmelCase__     :		Union[str, Any]=3		,					UpperCAmelCase__     :		List[Any]=4		,					UpperCAmelCase__     :		Tuple=None		,					):
										'''simple docstring'''
										lowercase :							str    		=parent
										lowercase :							int    		=batch_size
										lowercase :							Any    		=seq_length
										lowercase :							int    		=is_training
										lowercase :							str    		=use_input_mask
										lowercase :							int    		=use_token_type_ids
										lowercase :							Dict    		=use_labels
										lowercase :							int    		=vocab_size
										lowercase :							str    		=embedding_size
										lowercase :							Union[str, Any]    		=hidden_size
										lowercase :							Tuple    		=num_hidden_layers
										lowercase :							Any    		=num_hidden_groups
										lowercase :							Union[str, Any]    		=num_attention_heads
										lowercase :							Any    		=intermediate_size
										lowercase :							Tuple    		=hidden_act
										lowercase :							Optional[int]    		=hidden_dropout_prob
										lowercase :							Union[str, Any]    		=attention_probs_dropout_prob
										lowercase :							List[Any]    		=max_position_embeddings
										lowercase :							int    		=type_vocab_size
										lowercase :							int    		=type_sequence_label_size
										lowercase :							Any    		=initializer_range
										lowercase :							List[Any]    		=num_labels
										lowercase :							int    		=num_choices
										lowercase :							Optional[int]    		=scope







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							int    		=ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size   )

										lowercase :							Optional[int]    		=None
										if self.use_input_mask:
																lowercase :							List[str]    		=random_attention_mask([self.batch_size, self.seq_length]   )

										lowercase :							Dict    		=None
										if self.use_token_type_ids:
																lowercase :							List[str]    		=ids_tensor([self.batch_size, self.seq_length]		,					self.type_vocab_size   )

										lowercase :							Tuple    		=None
										lowercase :							Any    		=None
										lowercase :							Dict    		=None
										if self.use_labels:
																lowercase :							int    		=ids_tensor([self.batch_size]		,					self.type_sequence_label_size   )
																lowercase :							str    		=ids_tensor([self.batch_size, self.seq_length]		,					self.num_labels   )
																lowercase :							Optional[Any]    		=ids_tensor([self.batch_size]		,					self.num_choices   )

										lowercase :							Any    		=self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										return AlbertConfig(
										    vocab_size=self.vocab_size		,					hidden_size=self.hidden_size		,					num_hidden_layers=self.num_hidden_layers		,					num_attention_heads=self.num_attention_heads		,					intermediate_size=self.intermediate_size		,					hidden_act=self.hidden_act		,					hidden_dropout_prob=self.hidden_dropout_prob		,					attention_probs_dropout_prob=self.attention_probs_dropout_prob		,					max_position_embeddings=self.max_position_embeddings		,					type_vocab_size=self.type_vocab_size		,					initializer_range=self.initializer_range		,					num_hidden_groups=self.num_hidden_groups		,					)







				def  lowerCamelCase_	(       self     :		Union[str, Any]		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		str   ):
										'''simple docstring'''
										lowercase :							int    		=AlbertModel(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							int    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__   )
										lowercase :							Dict    		=model(UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__   )
										lowercase :							int    		=model(UpperCAmelCase__   )
										self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size)   )
										self.parent.assertEqual(result.pooler_output.shape		,					(self.batch_size, self.hidden_size)   )







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Tuple   ):
										'''simple docstring'''
										lowercase :							Tuple    		=AlbertForPreTraining(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							int    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__		,					sentence_order_label=UpperCAmelCase__		,					)
										self.parent.assertEqual(result.prediction_logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)   )
										self.parent.assertEqual(result.sop_logits.shape		,					(self.batch_size, config.num_labels)   )







				def  lowerCamelCase_	(       self     :		List[Any]		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		int   ):
										'''simple docstring'''
										lowercase :							Tuple    		=AlbertForMaskedLM(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							str    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)   )







				def  lowerCamelCase_	(       self     :		Any		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							List[str]    		=AlbertForQuestionAnswering(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[str]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					start_positions=UpperCAmelCase__		,					end_positions=UpperCAmelCase__		,					)
										self.parent.assertEqual(result.start_logits.shape		,					(self.batch_size, self.seq_length)   )
										self.parent.assertEqual(result.end_logits.shape		,					(self.batch_size, self.seq_length)   )







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		Union[str, Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							Optional[Any]    		=self.num_labels
										lowercase :							Any    		=AlbertForSequenceClassification(UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							Dict    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_labels)   )







				def  lowerCamelCase_	(       self     :		Optional[int]		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Optional[Any]		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Dict   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=self.num_labels
										lowercase :							str    		=AlbertForTokenClassification(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							int    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__   )
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.num_labels)   )







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		Optional[int]		,					UpperCAmelCase__     :		Union[str, Any]		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		List[str]		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=self.num_choices
										lowercase :							List[Any]    		=AlbertForMultipleChoice(config=UpperCAmelCase__   )
										model.to(UpperCAmelCase__   )
										model.eval()
										lowercase :							List[Any]    		=input_ids.unsqueeze(1   ).expand(-1		,					self.num_choices		,					-1   ).contiguous()
										lowercase :							int    		=token_type_ids.unsqueeze(1   ).expand(-1		,					self.num_choices		,					-1   ).contiguous()
										lowercase :							int    		=input_mask.unsqueeze(1   ).expand(-1		,					self.num_choices		,					-1   ).contiguous()
										lowercase :							List[str]    		=model(
										    UpperCAmelCase__		,					attention_mask=UpperCAmelCase__		,					token_type_ids=UpperCAmelCase__		,					labels=UpperCAmelCase__		,					)
										self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_choices)   )







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase :							Union[str, Any]    		=self.prepare_config_and_inputs()
										(
										    (
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	(
										    lowercase
										)		,	
										) :							Dict    		=config_and_inputs
										lowercase :							Optional[Any]    		={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict





@require_torch
class 		__SCREAMING_SNAKE_CASE							(			lowercase__						,							lowercase__						,							unittest.TestCase      ):
				lowerCamelCase_		       =    (
				    (
				        AlbertModel,
				        AlbertForPreTraining,
				        AlbertForMaskedLM,
				        AlbertForMultipleChoice,
				        AlbertForSequenceClassification,
				        AlbertForTokenClassification,
				        AlbertForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				lowerCamelCase_		       =    (
				    {
				        'feature-extraction': AlbertModel,
				        'fill-mask': AlbertForMaskedLM,
				        'question-answering': AlbertForQuestionAnswering,
				        'text-classification': AlbertForSequenceClassification,
				        'token-classification': AlbertForTokenClassification,
				        'zero-shot': AlbertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				lowerCamelCase_		       =    True







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		Tuple		,					UpperCAmelCase__     :		int=False   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=super()._prepare_for_class(UpperCAmelCase__		,					UpperCAmelCase__		,					return_labels=UpperCAmelCase__   )

										if return_labels:
																if model_class in get_values(UpperCAmelCase__   ):
																						lowercase :							Any    		=torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.seq_length)		,					dtype=torch.long		,					device=UpperCAmelCase__   )
																						lowercase :							Any    		=torch.zeros(
																						    self.model_tester.batch_size		,					dtype=torch.long		,					device=UpperCAmelCase__   )
										return inputs_dict







				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							Tuple    		=AlbertModelTester(self   )
										lowercase :							Optional[Any]    		=ConfigTester(self		,					config_class=UpperCAmelCase__		,					hidden_size=37   )







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										self.config_tester.run_common_tests()







				def  lowerCamelCase_	(       self     :		Tuple   ):
										'''simple docstring'''
										lowercase :							Optional[Any]    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		List[str]   ):
										'''simple docstring'''
										lowercase :							Dict    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		Dict   ):
										'''simple docstring'''
										lowercase :							Dict    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		str   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		int   ):
										'''simple docstring'''
										lowercase :							str    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		Optional[int]   ):
										'''simple docstring'''
										lowercase :							int    		=self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		List[Any]   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																lowercase :							Tuple    		=type
																self.model_tester.create_and_check_model(*UpperCAmelCase__   )







				@slow
				def  lowerCamelCase_	(       self     :		Optional[Any]   ):
										'''simple docstring'''
										for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																lowercase :							str    		=AlbertModel.from_pretrained(UpperCAmelCase__   )
																self.assertIsNotNone(UpperCAmelCase__   )





@require_torch
class 		__SCREAMING_SNAKE_CASE							(			unittest.TestCase      ):







				@slow
				def  lowerCamelCase_	(       self     :		Tuple   ):
										'''simple docstring'''
										lowercase :							int    		=AlbertModel.from_pretrained('''albert-base-v2'''   )
										lowercase :							Optional[int]    		=torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]   )
										lowercase :							Any    		=torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
										with torch.no_grad():
																lowercase :							Any    		=model(UpperCAmelCase__		,					attention_mask=UpperCAmelCase__   )[0]
										lowercase :							int    		=torch.Size((1, 11, 768)   )
										self.assertEqual(output.shape		,					UpperCAmelCase__   )
										lowercase :							Union[str, Any]    		=torch.tensor(
										    [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]   )

										self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,					UpperCAmelCase__		,					atol=1E-4   )   )





