




"""simple docstring"""







import inspect
from typing import Optional, Union

import numpy as np
import PIL
import torch
from torch.nn import functional as F
from torchvision import transforms
from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DiffusionPipeline,
    DPMSolverMultistepScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
from diffusers.utils import (
    PIL_INTERPOLATION,
    randn_tensor,
)

def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				)			->Any:






		"""simple docstring"""


		if isinstance(_lowerCamelCase,  torch.Tensor				):
				return image
		elif isinstance(_lowerCamelCase,  PIL.Image.Image				):
				__lowercase				:		List[Any]												=				[image]

		if isinstance(image[0],  PIL.Image.Image				):
				__lowercase				:		Optional[int]												=				[np.array(i.resize((w, h),  resample=PIL_INTERPOLATION["lanczos"]				)				)[None, :] for i in image]
				__lowercase				:		Optional[int]												=				np.concatenate(_lowerCamelCase,  axis=0				)
				__lowercase				:		List[Any]												=				np.array(_lowerCamelCase				).astype(np.floataa				) / 2_5_5.0
				__lowercase				:		Optional[int]												=				image.transpose(0,  3,  1,  2				)
				__lowercase				:		Dict												=				2.0 * image - 1.0
				__lowercase				:		Any												=				torch.from_numpy(_lowerCamelCase				)
		elif isinstance(image[0],  torch.Tensor				):
				__lowercase				:		List[str]												=				torch.cat(_lowerCamelCase,  dim=0				)
		return image

def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase=0.9_9_9_5				)			->Any:






		"""simple docstring"""


		if not isinstance(_lowerCamelCase,  np.ndarray				):
				__lowercase				:		Optional[int]												=				True
				__lowercase				:		Union[str, Any]												=				va.device
				__lowercase				:		str												=				va.cpu().numpy()
				__lowercase				:		Optional[int]												=				va.cpu().numpy()

		__lowercase				:		int												=				np.sum(va * va / (np.linalg.norm(_lowerCamelCase				) * np.linalg.norm(_lowerCamelCase				))				)
		if np.abs(_lowerCamelCase				) > DOT_THRESHOLD:
				__lowercase				:		Dict												=				(1 - t) * va + t * va
		else:
				__lowercase				:		Tuple												=				np.arccos(_lowerCamelCase				)
				__lowercase				:		List[Any]												=				np.sin(_lowerCamelCase				)
				__lowercase				:		Any												=				theta_a * t
				__lowercase				:		int												=				np.sin(_lowerCamelCase				)
				__lowercase				:		List[Any]												=				np.sin(theta_a - theta_t				) / sin_theta_a
				__lowercase				:		Tuple												=				sin_theta_t / sin_theta_a
				__lowercase				:		Tuple												=				sa * va + sa * va

		if inputs_are_torch:
				__lowercase				:		List[Any]												=				torch.from_numpy(_lowerCamelCase				).to(_lowerCamelCase				)

		return va

def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase				)			->Optional[Any]:






		"""simple docstring"""


		__lowercase				:		Tuple												=				F.normalize(_lowerCamelCase,  dim=-1				)
		__lowercase				:		Union[str, Any]												=				F.normalize(_lowerCamelCase,  dim=-1				)
		return (x - y).norm(dim=-1				).div(2				).arcsin().pow(2				).mul(2				)

def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase				)			->Dict:






		"""simple docstring"""


		for param in model.parameters():
				__lowercase				:		Dict												=				value





class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				def __init__(     self					:	Union[str, Any]    ,   lowercase__					:	AutoencoderKL    ,   lowercase__					:	CLIPTextModel    ,   lowercase__					:	CLIPModel    ,   lowercase__					:	CLIPTokenizer    ,   lowercase__					:	UNetaDConditionModel    ,   lowercase__					:	Union[PNDMScheduler, LMSDiscreteScheduler, DDIMScheduler, DPMSolverMultistepScheduler]    ,   lowercase__					:	CLIPFeatureExtractor    ,   lowercase__					:	List[str]=None    ,   lowercase__					:	Tuple=None    ,   lowercase__					:	Tuple=None    ,   ):
						super().__init__()
						self.register_modules(
						    vae=lowercase__    ,   text_encoder=lowercase__    ,   clip_model=lowercase__    ,   tokenizer=lowercase__    ,   unet=lowercase__    ,   scheduler=lowercase__    ,   feature_extractor=lowercase__    ,   coca_model=lowercase__    ,   coca_tokenizer=lowercase__    ,   coca_transform=lowercase__    ,   )
						__lowercase				:		Any												=				(
						    feature_extractor.size
						    if isinstance(feature_extractor.size    ,   lowercase__						)
						    else feature_extractor.size["shortest_edge"]
						)
						__lowercase				:		List[str]												=				transforms.Normalize(mean=feature_extractor.image_mean    ,   std=feature_extractor.image_std						)
						set_requires_grad(self.text_encoder    ,   lowercase__						)
						set_requires_grad(self.clip_model    ,   lowercase__						)



				def       snake_case    (     self					:	Tuple    ,   lowercase__					:	Optional[Union[str, int]] = "auto"						):




						if slice_size == "auto":
								# half the attention head size is usually a good trade-off between
								# speed and memory
								__lowercase				:		Optional[int]												=				self.unet.config.attention_head_dim // 2
						self.unet.set_attention_slice(lowercase__						)



				def       snake_case    (     self					:	Optional[int]						):
						self.enable_attention_slicing(lowercase__						)



				def       snake_case    (     self					:	str						):
						set_requires_grad(self.vae    ,   lowercase__						)



				def       snake_case    (     self					:	List[Any]						):
						set_requires_grad(self.vae    ,   lowercase__						)



				def       snake_case    (     self					:	List[str]						):
						set_requires_grad(self.unet    ,   lowercase__						)



				def       snake_case    (     self					:	Union[str, Any]						):
						set_requires_grad(self.unet    ,   lowercase__						)



				def       snake_case    (     self					:	Optional[Any]    ,   lowercase__					:	Optional[Any]    ,   lowercase__					:	Dict    ,   lowercase__					:	Tuple						):
						__lowercase				:		List[str]												=				min(int(num_inference_steps * strength						)    ,   lowercase__						)

						__lowercase				:		List[str]												=				max(num_inference_steps - init_timestep    ,   0						)
						__lowercase				:		List[str]												=				self.scheduler.timesteps[t_start:]

						return timesteps, num_inference_steps - t_start



				def       snake_case    (     self					:	Union[str, Any]    ,   lowercase__					:	Optional[int]    ,   lowercase__					:	Tuple    ,   lowercase__					:	Union[str, Any]    ,   lowercase__					:	Optional[Any]    ,   lowercase__					:	List[Any]    ,   lowercase__					:	List[str]=None						):




						if not isinstance(lowercase__    ,   torch.Tensor						):
								raise ValueError(f'`image` has to be of type `torch.Tensor` but is {type(lowercase__						)}'						)

						__lowercase				:		str												=				image.to(device=lowercase__    ,   dtype=lowercase__						)

						if isinstance(lowercase__    ,   lowercase__						):
								__lowercase				:		Tuple												=				[
								    self.vae.encode(image[i : i + 1]						).latent_dist.sample(generator[i]						) for i in range(lowercase__						)
								]
								__lowercase				:		Optional[Any]												=				torch.cat(lowercase__    ,   dim=0						)
						else:
								__lowercase				:		List[str]												=				self.vae.encode(lowercase__						).latent_dist.sample(lowercase__						)

						# Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
						__lowercase				:		int												=				0.1_8_2_1_5 * init_latents
						__lowercase				:		Any												=				init_latents.repeat_interleave(lowercase__    ,   dim=0						)

						__lowercase				:		Dict												=				randn_tensor(init_latents.shape    ,   generator=lowercase__    ,   device=lowercase__    ,   dtype=lowercase__						)

						# get latents
						__lowercase				:		List[str]												=				self.scheduler.add_noise(lowercase__    ,   lowercase__    ,   lowercase__						)
						__lowercase				:		Optional[int]												=				init_latents

						return latents



				def       snake_case    (     self					:	List[str]    ,   lowercase__					:	Any						):
						__lowercase				:		int												=				self.coca_transform(lowercase__						).unsqueeze(0						)
						with torch.no_grad(), torch.cuda.amp.autocast():
								__lowercase				:		Any												=				self.coca_model.generate(transformed_image.to(device=self.device    ,   dtype=self.coca_model.dtype						)						)
						__lowercase				:		str												=				self.coca_tokenizer.decode(generated[0].cpu().numpy()						)
						return generated.split("<end_of_text>"						)[0].replace("<start_of_text>"    ,   ""						).rstrip(" .,"						)



				def       snake_case    (     self					:	int    ,   lowercase__					:	Optional[int]    ,   lowercase__					:	Any						):
						__lowercase				:		str												=				self.feature_extractor.preprocess(lowercase__						)
						__lowercase				:		List[str]												=				torch.from_numpy(clip_image_input["pixel_values"][0]						).unsqueeze(0						).to(self.device						).half()
						__lowercase				:		Tuple												=				self.clip_model.get_image_features(lowercase__						)
						__lowercase				:		Dict												=				image_embeddings_clip / image_embeddings_clip.norm(p=2    ,   dim=-1    ,   keepdim=lowercase__						)
						__lowercase				:		Optional[int]												=				image_embeddings_clip.repeat_interleave(lowercase__    ,   dim=0						)
						return image_embeddings_clip



				@torch.enable_grad()
				def       snake_case    (     self					:	Dict    ,   lowercase__					:	Tuple    ,   lowercase__					:	Optional[int]    ,   lowercase__					:	Optional[int]    ,   lowercase__					:	Any    ,   lowercase__					:	Union[str, Any]    ,   lowercase__					:	List[Any]    ,   lowercase__					:	Tuple    ,   ):
						__lowercase				:		Tuple												=				latents.detach().requires_grad_()

						__lowercase				:		Dict												=				self.scheduler.scale_model_input(lowercase__    ,   lowercase__						)

						# predict the noise residual
						__lowercase				:		int												=				self.unet(lowercase__    ,   lowercase__    ,   encoder_hidden_states=lowercase__						).sample

						if isinstance(self.scheduler    ,   (PNDMScheduler, DDIMScheduler, DPMSolverMultistepScheduler)						):
								__lowercase				:		str												=				self.scheduler.alphas_cumprod[timestep]
								__lowercase				:		Optional[int]												=				1 - alpha_prod_t
								# compute predicted original sample from predicted noise also called
								# "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
								__lowercase				:		Tuple												=				(latents - beta_prod_t ** 0.5 * noise_pred) / alpha_prod_t ** 0.5

								__lowercase				:		Any												=				torch.sqrt(lowercase__						)
								__lowercase				:		Optional[Any]												=				pred_original_sample * (fac) + latents * (1 - fac)
						elif isinstance(self.scheduler    ,   lowercase__						):
								__lowercase				:		Optional[int]												=				self.scheduler.sigmas[index]
								__lowercase				:		str												=				latents - sigma * noise_pred
						else:
								raise ValueError(f'scheduler type {type(self.scheduler						)} not supported'						)

						# Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
						__lowercase				:		List[str]												=				1 / 0.1_8_2_1_5 * sample
						__lowercase				:		List[str]												=				self.vae.decode(lowercase__						).sample
						__lowercase				:		Union[str, Any]												=				(image / 2 + 0.5).clamp(0    ,   1						)

						__lowercase				:		Any												=				transforms.Resize(self.feature_extractor_size						)(lowercase__						)
						__lowercase				:		Tuple												=				self.normalize(lowercase__						).to(latents.dtype						)

						__lowercase				:		str												=				self.clip_model.get_image_features(lowercase__						)
						__lowercase				:		Optional[int]												=				image_embeddings_clip / image_embeddings_clip.norm(p=2    ,   dim=-1    ,   keepdim=lowercase__						)

						__lowercase				:		List[str]												=				spherical_dist_loss(lowercase__    ,   lowercase__						).mean() * clip_guidance_scale

						__lowercase				:		List[str]												=				-torch.autograd.grad(lowercase__    ,   lowercase__						)[0]

						if isinstance(self.scheduler    ,   lowercase__						):
								__lowercase				:		Tuple												=				latents.detach() + grads * (sigma**2)
								__lowercase				:		str												=				noise_pred_original
						else:
								__lowercase				:		List[Any]												=				noise_pred_original - torch.sqrt(lowercase__						) * grads
						return noise_pred, latents



				@torch.no_grad()
				def __call__(     self					:	Union[str, Any]    ,   lowercase__					:	Union[torch.FloatTensor, PIL.Image.Image]    ,   lowercase__					:	Union[torch.FloatTensor, PIL.Image.Image]    ,   lowercase__					:	Optional[str] = None    ,   lowercase__					:	Optional[str] = None    ,   lowercase__					:	Optional[int] = 5_1_2    ,   lowercase__					:	Optional[int] = 5_1_2    ,   lowercase__					:	float = 0.6    ,   lowercase__					:	Optional[int] = 5_0    ,   lowercase__					:	Optional[float] = 7.5    ,   lowercase__					:	Optional[int] = 1    ,   lowercase__					:	float = 0.0    ,   lowercase__					:	Optional[float] = 1_0_0    ,   lowercase__					:	Optional[torch.Generator] = None    ,   lowercase__					:	Optional[str] = "pil"    ,   lowercase__					:	bool = True    ,   lowercase__					:	float = 0.8    ,   lowercase__					:	float = 0.1    ,   lowercase__					:	float = 0.1    ,   ):




						if isinstance(lowercase__    ,   lowercase__						) and len(lowercase__						) != batch_size:
								raise ValueError(f'You have passed {batch_size} batch_size, but only {len(lowercase__						)} generators.'						)

						if height % 8 != 0 or width % 8 != 0:
								raise ValueError(f'`height` and `width` have to be divisible by 8 but are {height} and {width}.'						)

						if isinstance(lowercase__    ,   torch.Generator						) and batch_size > 1:
								__lowercase				:		Tuple												=				[generator] + [None] * (batch_size - 1)

						__lowercase				:		Any												=				[
						    ("model", self.coca_model is None),
						    ("tokenizer", self.coca_tokenizer is None),
						    ("transform", self.coca_transform is None),
						]
						__lowercase				:		Union[str, Any]												=				[x[0] for x in coca_is_none if x[1]]
						__lowercase				:		Dict												=				", ".join(lowercase__						)
						# generate prompts with coca model if prompt is None
						if content_prompt is None:
								if len(lowercase__						):
										raise ValueError(
										    f'Content prompt is None and CoCa [{coca_is_none_str}] is None.'
										    f'Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline.'						)
								__lowercase				:		Union[str, Any]												=				self.get_image_description(lowercase__						)
						if style_prompt is None:
								if len(lowercase__						):
										raise ValueError(
										    f'Style prompt is None and CoCa [{coca_is_none_str}] is None.'
										    f' Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline.'						)
								__lowercase				:		str												=				self.get_image_description(lowercase__						)

						# get prompt text embeddings for content and style
						__lowercase				:		Dict												=				self.tokenizer(
						    lowercase__    ,   padding="max_length"    ,   max_length=self.tokenizer.model_max_length    ,   truncation=lowercase__    ,   return_tensors="pt"    ,   )
						__lowercase				:		Tuple												=				self.text_encoder(content_text_input.input_ids.to(self.device						)						)[0]

						__lowercase				:		Union[str, Any]												=				self.tokenizer(
						    lowercase__    ,   padding="max_length"    ,   max_length=self.tokenizer.model_max_length    ,   truncation=lowercase__    ,   return_tensors="pt"    ,   )
						__lowercase				:		List[Any]												=				self.text_encoder(style_text_input.input_ids.to(self.device						)						)[0]

						__lowercase				:		Union[str, Any]												=				slerp(lowercase__    ,   lowercase__    ,   lowercase__						)

						# duplicate text embeddings for each generation per prompt
						__lowercase				:		Union[str, Any]												=				text_embeddings.repeat_interleave(lowercase__    ,   dim=0						)

						# set timesteps
						__lowercase				:		Optional[int]												=				"offset" in set(inspect.signature(self.scheduler.set_timesteps						).parameters.keys()						)
						__lowercase				:		str												=				{}
						if accepts_offset:
								__lowercase				:		List[str]												=				1

						self.scheduler.set_timesteps(lowercase__    ,   **lowercase__						)
						# Some schedulers like PNDM have timesteps as arrays
						# It's more optimized to move all timesteps to correct device beforehand
						self.scheduler.timesteps.to(self.device						)

						__lowercase						,__lowercase				:		List[str]												=				self.get_timesteps(lowercase__    ,   lowercase__    ,   self.device						)
						__lowercase				:		int												=				timesteps[:1].repeat(lowercase__						)

						# Preprocess image
						__lowercase				:		Optional[int]												=				preprocess(lowercase__    ,   lowercase__    ,   lowercase__						)
						__lowercase				:		Union[str, Any]												=				self.prepare_latents(
						    lowercase__    ,   lowercase__    ,   lowercase__    ,   text_embeddings.dtype    ,   self.device    ,   lowercase__						)

						__lowercase				:		Dict												=				preprocess(lowercase__    ,   lowercase__    ,   lowercase__						)
						__lowercase				:		str												=				self.prepare_latents(
						    lowercase__    ,   lowercase__    ,   lowercase__    ,   text_embeddings.dtype    ,   self.device    ,   lowercase__						)

						__lowercase				:		Any												=				slerp(lowercase__    ,   lowercase__    ,   lowercase__						)

						if clip_guidance_scale > 0:
								__lowercase				:		Dict												=				self.get_clip_image_embeddings(lowercase__    ,   lowercase__						)
								__lowercase				:		Tuple												=				self.get_clip_image_embeddings(lowercase__    ,   lowercase__						)
								__lowercase				:		Dict												=				slerp(
								    lowercase__    ,   lowercase__    ,   lowercase__						)

						# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
						# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
						# corresponds to doing no classifier free guidance.
						__lowercase				:		Optional[Any]												=				guidance_scale > 1.0
						# get unconditional embeddings for classifier free guidance
						if do_classifier_free_guidance:
								__lowercase				:		str												=				content_text_input.input_ids.shape[-1]
								__lowercase				:		str												=				self.tokenizer([""]    ,   padding="max_length"    ,   max_length=lowercase__    ,   return_tensors="pt"						)
								__lowercase				:		Optional[int]												=				self.text_encoder(uncond_input.input_ids.to(self.device						)						)[0]
								# duplicate unconditional embeddings for each generation per prompt
								__lowercase				:		int												=				uncond_embeddings.repeat_interleave(lowercase__    ,   dim=0						)

								# For classifier free guidance, we need to do two forward passes.
								# Here we concatenate the unconditional and text embeddings into a single batch
								# to avoid doing two forward passes
								__lowercase				:		Tuple												=				torch.cat([uncond_embeddings, text_embeddings]						)

						# get the initial random noise unless the user supplied it

						# Unlike in other pipelines, latents need to be generated in the target device
						# for 1-to-1 results reproducibility with the CompVis implementation.
						# However this currently doesn't work in `mps`.
						__lowercase				:		List[str]												=				(batch_size, self.unet.config.in_channels, height // 8, width // 8)
						__lowercase				:		str												=				text_embeddings.dtype
						if latents is None:
								if self.device.type == "mps":
										# randn does not work reproducibly on mps
										__lowercase				:		Any												=				torch.randn(lowercase__    ,   generator=lowercase__    ,   device="cpu"    ,   dtype=lowercase__						).to(
										    self.device						)
								else:
										__lowercase				:		List[str]												=				torch.randn(lowercase__    ,   generator=lowercase__    ,   device=self.device    ,   dtype=lowercase__						)
						else:
								if latents.shape != latents_shape:
										raise ValueError(f'Unexpected latents shape, got {latents.shape}, expected {latents_shape}'						)
								__lowercase				:		Tuple												=				latents.to(self.device						)

						# scale the initial noise by the standard deviation required by the scheduler
						__lowercase				:		str												=				latents * self.scheduler.init_noise_sigma

						# prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
						# eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
						# eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
						# and should be between [0, 1]
						__lowercase				:		Union[str, Any]												=				"eta" in set(inspect.signature(self.scheduler.step						).parameters.keys()						)
						__lowercase				:		int												=				{}
						if accepts_eta:
								__lowercase				:		int												=				eta

						# check if the scheduler accepts generator
						__lowercase				:		str												=				"generator" in set(inspect.signature(self.scheduler.step						).parameters.keys()						)
						if accepts_generator:
								__lowercase				:		Any												=				generator

						with self.progress_bar(total=lowercase__						):
								for i, t in enumerate(lowercase__						):
										# expand the latents if we are doing classifier free guidance
										__lowercase				:		Tuple												=				torch.cat([latents] * 2						) if do_classifier_free_guidance else latents
										__lowercase				:		Union[str, Any]												=				self.scheduler.scale_model_input(lowercase__    ,   lowercase__						)

										# predict the noise residual
										__lowercase				:		Tuple												=				self.unet(lowercase__    ,   lowercase__    ,   encoder_hidden_states=lowercase__						).sample

										# perform classifier free guidance
										if do_classifier_free_guidance:
												__lowercase						,__lowercase				:		Tuple												=				noise_pred.chunk(2						)
												__lowercase				:		List[str]												=				noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)

										# perform clip guidance
										if clip_guidance_scale > 0:
												__lowercase				:		Optional[int]												=				(
												    text_embeddings.chunk(2						)[1] if do_classifier_free_guidance else text_embeddings
												)
												__lowercase						,__lowercase				:		Dict												=				self.cond_fn(
												    lowercase__    ,   lowercase__    ,   lowercase__    ,   lowercase__    ,   lowercase__    ,   lowercase__    ,   lowercase__    ,   )

										# compute the previous noisy sample x_t -> x_t-1
										__lowercase				:		Tuple												=				self.scheduler.step(lowercase__    ,   lowercase__    ,   lowercase__    ,   **lowercase__						).prev_sample

        # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
						__lowercase				:		int												=				1 / 0.1_8_2_1_5 * latents
						__lowercase				:		Optional[int]												=				self.vae.decode(lowercase__						).sample

						__lowercase				:		Optional[int]												=				(image / 2 + 0.5).clamp(0    ,   1						)
						__lowercase				:		List[str]												=				image.cpu().permute(0    ,   2    ,   3    ,   1						).numpy()

						if output_type == "pil":
								__lowercase				:		int												=				self.numpy_to_pil(lowercase__						)

						if not return_dict:
								return (image, None)

						return StableDiffusionPipelineOutput(images=lowercase__    ,   nsfw_content_detected=lowercase__						)










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A   :  str					=      logging.get_logger(__name__)

__A   :  str					=      {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}




class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				__UpperCAmelCase :     Tuple       =      "visual_bert"



				def __init__(     self					:	List[Any]    ,   lowercase__					:	Dict=3_0_5_2_2    ,   lowercase__					:	Any=7_6_8    ,   lowercase__					:	Dict=5_1_2    ,   lowercase__					:	Tuple=1_2    ,   lowercase__					:	Any=1_2    ,   lowercase__					:	Union[str, Any]=3_0_7_2    ,   lowercase__					:	Tuple="gelu"    ,   lowercase__					:	Dict=0.1    ,   lowercase__					:	Tuple=0.1    ,   lowercase__					:	str=5_1_2    ,   lowercase__					:	str=2    ,   lowercase__					:	Any=0.0_2    ,   lowercase__					:	Any=1e-12    ,   lowercase__					:	Any=False    ,   lowercase__					:	List[str]=True    ,   lowercase__					:	Dict=1    ,   lowercase__					:	List[Any]=0    ,   lowercase__					:	Tuple=2    ,   **lowercase__					:	Any    ,   ):
						super().__init__(pad_token_id=lowercase__    ,   bos_token_id=lowercase__    ,   eos_token_id=lowercase__    ,   **lowercase__						)

						__lowercase				:		Optional[int]												=				vocab_size
						__lowercase				:		Any												=				max_position_embeddings
						__lowercase				:		Optional[Any]												=				hidden_size
						__lowercase				:		List[str]												=				visual_embedding_dim
						__lowercase				:		Dict												=				num_hidden_layers
						__lowercase				:		Any												=				num_attention_heads
						__lowercase				:		str												=				intermediate_size
						__lowercase				:		Dict												=				hidden_act
						__lowercase				:		Any												=				hidden_dropout_prob
						__lowercase				:		Optional[int]												=				attention_probs_dropout_prob
						__lowercase				:		Dict												=				initializer_range
						__lowercase				:		str												=				type_vocab_size
						__lowercase				:		List[Any]												=				layer_norm_eps
						__lowercase				:		Tuple												=				bypass_transformer
						__lowercase				:		List[str]												=				special_visual_initialize
