






"""simple docstring"""



import gc
import unittest

import numpy as np
import torch
import torch.nn.functional as F
from transformers import (
    ClapTextConfig,
    ClapTextModelWithProjection,
    RobertaTokenizer,
    SpeechTaHifiGan,
    SpeechTaHifiGanConfig,
)

from diffusers import (
    AudioLDMPipeline,
    AutoencoderKL,
    DDIMScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    UNetaDConditionModel,
)
from diffusers.utils import is_xformers_available, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from ..pipeline_params import TEXT_TO_AUDIO_BATCH_PARAMS, TEXT_TO_AUDIO_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()






class 							__SCREAMING_SNAKE_CASE  (			UpperCamelCase    ,						unittest.TestCase						):
							snake_case_       =							AudioLDMPipeline
							snake_case_       =							TEXT_TO_AUDIO_PARAMS
							snake_case_       =							TEXT_TO_AUDIO_BATCH_PARAMS
							snake_case_       =							frozenset(
							    [
							        'num_inference_steps',
							        'num_waveforms_per_prompt',
							        'generator',
							        'latents',
							        'output_type',
							        'return_dict',
							        'callback',
							        'callback_steps',
							    ]						)


							def   _UpperCamelCase    (     self :  Any							):
													'''simple docstring'''





													torch.manual_seed(0							)
													A__  :					List[str]											=					UNetaDConditionModel(
													    block_out_channels=(32, 64)     ,      layers_per_block=2     ,      sample_size=32     ,      in_channels=4     ,      out_channels=4     ,      down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")     ,      up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")     ,      cross_attention_dim=(32, 64)     ,      class_embed_type="""simple_projection"""     ,      projection_class_embeddings_input_dim=32     ,      class_embeddings_concat=snake_case     ,      )
													A__  :					int											=					DDIMScheduler(
													    beta_start=0.00085     ,      beta_end=0.012     ,      beta_schedule="""scaled_linear"""     ,      clip_sample=snake_case     ,      set_alpha_to_one=snake_case     ,      )
													torch.manual_seed(0							)
													A__  :					int											=					AutoencoderKL(
													    block_out_channels=[32, 64]     ,      in_channels=1     ,      out_channels=1     ,      down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]     ,      up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]     ,      latent_channels=4     ,      )
													torch.manual_seed(0							)
													A__  :					List[str]											=					ClapTextConfig(
													    bos_token_id=0     ,      eos_token_id=2     ,      hidden_size=32     ,      intermediate_size=37     ,      layer_norm_eps=1e-05     ,      num_attention_heads=4     ,      num_hidden_layers=5     ,      pad_token_id=1     ,      vocab_size=1000     ,      projection_dim=32     ,      )
													A__  :					Dict											=					ClapTextModelWithProjection(snake_case							)
													A__  :					Optional[int]											=					RobertaTokenizer.from_pretrained("""hf-internal-testing/tiny-random-roberta"""     ,      model_max_length=77							)

													A__  :					str											=					SpeechTaHifiGanConfig(
													    model_in_dim=8     ,      sampling_rate=1_6000     ,      upsample_initial_channel=16     ,      upsample_rates=[2, 2]     ,      upsample_kernel_sizes=[4, 4]     ,      resblock_kernel_sizes=[3, 7]     ,      resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5]]     ,      normalize_before=snake_case     ,      )

													A__  :					List[Any]											=					SpeechTaHifiGan(snake_case							)

													A__  :					Union[str, Any]											=					{
													    """unet""": unet,
													    """scheduler""": scheduler,
													    """vae""": vae,
													    """text_encoder""": text_encoder,
													    """tokenizer""": tokenizer,
													    """vocoder""": vocoder,
													}
													return components


							def   _UpperCamelCase    (     self :  int     ,      snake_case :  Union[str, Any]     ,      snake_case :  Any=0							):
													'''simple docstring'''





													if str(snake_case							).startswith("""mps"""							):
																			A__  :					Optional[int]											=					torch.manual_seed(snake_case							)
													else:
																			A__  :					Optional[Any]											=					torch.Generator(device=snake_case							).manual_seed(snake_case							)
													A__  :					Union[str, Any]											=					{
													    """prompt""": """A hammer hitting a wooden surface""",
													    """generator""": generator,
													    """num_inference_steps""": 2,
													    """guidance_scale""": 6.0,
													}
													return inputs


							def   _UpperCamelCase    (     self :  List[Any]							):
													'''simple docstring'''





													A__  :					Any											=					"""cpu"""  # ensure determinism for the device-dependent torch.Generator

													A__  :					Tuple											=					self.get_dummy_components()
													A__  :					Tuple											=					AudioLDMPipeline(**snake_case							)
													A__  :					Any											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					Dict											=					self.get_dummy_inputs(snake_case							)
													A__  :					Dict											=					audioldm_pipe(**snake_case							)
													A__  :					Union[str, Any]											=					output.audios[0]

													assert audio.ndim == 1
													assert len(snake_case							) == 256

													A__  :					Union[str, Any]											=					audio[:10]
													A__  :					Optional[Any]											=					np.array(
													    [-0.0050, 0.0050, -0.0060, 0.0033, -0.0026, 0.0033, -0.0027, 0.0033, -0.0028, 0.0033]							)

													assert np.abs(audio_slice - expected_slice							).max() < 1e-2


							def   _UpperCamelCase    (     self :  str							):
													'''simple docstring'''





													A__  :					List[str]											=					self.get_dummy_components()
													A__  :					Union[str, Any]											=					AudioLDMPipeline(**snake_case							)
													A__  :					Tuple											=					audioldm_pipe.to(snake_case							)
													A__  :					Optional[Any]											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					int											=					self.get_dummy_inputs(snake_case							)
													A__  :					Tuple											=					3 * [inputs["""prompt"""]]

													# forward
													A__  :					Dict											=					audioldm_pipe(**snake_case							)
													A__  :					List[str]											=					output.audios[0]

													A__  :					Dict											=					self.get_dummy_inputs(snake_case							)
													A__  :					Optional[int]											=					3 * [inputs.pop("""prompt"""							)]

													A__  :					Tuple											=					audioldm_pipe.tokenizer(
													    snake_case     ,      padding="""max_length"""     ,      max_length=audioldm_pipe.tokenizer.model_max_length     ,      truncation=snake_case     ,      return_tensors="""pt"""     ,      )
													A__  :					Union[str, Any]											=					text_inputs["""input_ids"""].to(snake_case							)

													A__  :					int											=					audioldm_pipe.text_encoder(
													    snake_case     ,      )
													A__  :					List[Any]											=					prompt_embeds.text_embeds
													# additional L_2 normalization over each hidden-state
													A__  :					Tuple											=					F.normalize(snake_case     ,      dim=-1							)

													A__  :					Dict											=					prompt_embeds

													# forward
													A__  :					List[Any]											=					audioldm_pipe(**snake_case							)
													A__  :					Dict											=					output.audios[0]

													assert np.abs(audio_a - audio_a							).max() < 1e-2


							def   _UpperCamelCase    (     self :  Dict							):
													'''simple docstring'''





													A__  :					int											=					self.get_dummy_components()
													A__  :					List[str]											=					AudioLDMPipeline(**snake_case							)
													A__  :					Tuple											=					audioldm_pipe.to(snake_case							)
													A__  :					List[str]											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					str											=					self.get_dummy_inputs(snake_case							)
													A__  :					int											=					3 * ["""this is a negative prompt"""]
													A__  :					Union[str, Any]											=					negative_prompt
													A__  :					Tuple											=					3 * [inputs["""prompt"""]]

													# forward
													A__  :					Union[str, Any]											=					audioldm_pipe(**snake_case							)
													A__  :					Tuple											=					output.audios[0]

													A__  :					Tuple											=					self.get_dummy_inputs(snake_case							)
													A__  :					Optional[int]											=					3 * [inputs.pop("""prompt"""							)]

													A__  :					int											=					[]
													for p in [prompt, negative_prompt]:
																			A__  :					Union[str, Any]											=					audioldm_pipe.tokenizer(
																			    snake_case     ,      padding="""max_length"""     ,      max_length=audioldm_pipe.tokenizer.model_max_length     ,      truncation=snake_case     ,      return_tensors="""pt"""     ,      )
																			A__  :					int											=					text_inputs["""input_ids"""].to(snake_case							)

																			A__  :					int											=					audioldm_pipe.text_encoder(
																			    snake_case     ,      )
																			A__  :					int											=					text_embeds.text_embeds
																			# additional L_2 normalization over each hidden-state
																			A__  :					Union[str, Any]											=					F.normalize(snake_case     ,      dim=-1							)

																			embeds.append(snake_case							)

													A__					,					A__  :					Any											=					embeds

													# forward
													A__  :					int											=					audioldm_pipe(**snake_case							)
													A__  :					str											=					output.audios[0]

													assert np.abs(audio_a - audio_a							).max() < 1e-2


							def   _UpperCamelCase    (     self :  Any							):
													'''simple docstring'''





													A__  :					str											=					"""cpu"""  # ensure determinism for the device-dependent torch.Generator
													A__  :					Optional[Any]											=					self.get_dummy_components()
													A__  :					List[Any]											=					PNDMScheduler(skip_prk_steps=snake_case							)
													A__  :					int											=					AudioLDMPipeline(**snake_case							)
													A__  :					str											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					Union[str, Any]											=					self.get_dummy_inputs(snake_case							)
													A__  :					List[str]											=					"""egg cracking"""
													A__  :					Dict											=					audioldm_pipe(**snake_case     ,      negative_prompt=snake_case							)
													A__  :					Union[str, Any]											=					output.audios[0]

													assert audio.ndim == 1
													assert len(snake_case							) == 256

													A__  :					Optional[int]											=					audio[:10]
													A__  :					Tuple											=					np.array(
													    [-0.0051, 0.0050, -0.0060, 0.0034, -0.0026, 0.0033, -0.0027, 0.0033, -0.0028, 0.0032]							)

													assert np.abs(audio_slice - expected_slice							).max() < 1e-2


							def   _UpperCamelCase    (     self :  Dict							):
													'''simple docstring'''





													A__  :					Optional[int]											=					"""cpu"""  # ensure determinism for the device-dependent torch.Generator
													A__  :					int											=					self.get_dummy_components()
													A__  :					Optional[int]											=					PNDMScheduler(skip_prk_steps=snake_case							)
													A__  :					Optional[int]											=					AudioLDMPipeline(**snake_case							)
													A__  :					Optional[Any]											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					str											=					"""A hammer hitting a wooden surface"""

													# test num_waveforms_per_prompt=1 (default)
													A__  :					str											=					audioldm_pipe(snake_case     ,      num_inference_steps=2							).audios

													assert audios.shape == (1, 256)

													# test num_waveforms_per_prompt=1 (default) for batch of prompts
													A__  :					List[Any]											=					2
													A__  :					Optional[Any]											=					audioldm_pipe([prompt] * batch_size     ,      num_inference_steps=2							).audios

													assert audios.shape == (batch_size, 256)

													# test num_waveforms_per_prompt for single prompt
													A__  :					int											=					2
													A__  :					Dict											=					audioldm_pipe(snake_case     ,      num_inference_steps=2     ,      num_waveforms_per_prompt=snake_case							).audios

													assert audios.shape == (num_waveforms_per_prompt, 256)

													# test num_waveforms_per_prompt for batch of prompts
													A__  :					Optional[Any]											=					2
													A__  :					int											=					audioldm_pipe(
													    [prompt] * batch_size     ,      num_inference_steps=2     ,      num_waveforms_per_prompt=snake_case							).audios

													assert audios.shape == (batch_size * num_waveforms_per_prompt, 256)


							def   _UpperCamelCase    (     self :  List[Any]							):
													'''simple docstring'''





													A__  :					Optional[Any]											=					"""cpu"""  # ensure determinism for the device-dependent torch.Generator
													A__  :					int											=					self.get_dummy_components()
													A__  :					List[Any]											=					AudioLDMPipeline(**snake_case							)
													A__  :					Optional[Any]											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)
													A__  :					Union[str, Any]											=					audioldm_pipe.vocoder.config.sampling_rate

													A__  :					Optional[Any]											=					self.get_dummy_inputs(snake_case							)
													A__  :					int											=					audioldm_pipe(audio_length_in_s=0.016     ,      **snake_case							)
													A__  :					Optional[int]											=					output.audios[0]

													assert audio.ndim == 1
													assert len(snake_case							) / vocoder_sampling_rate == 0.016

													A__  :					List[str]											=					audioldm_pipe(audio_length_in_s=0.032     ,      **snake_case							)
													A__  :					Union[str, Any]											=					output.audios[0]

													assert audio.ndim == 1
													assert len(snake_case							) / vocoder_sampling_rate == 0.032


							def   _UpperCamelCase    (     self :  Dict							):
													'''simple docstring'''





													A__  :					List[str]											=					self.get_dummy_components()
													A__  :					Tuple											=					AudioLDMPipeline(**snake_case							)
													A__  :					List[str]											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					Union[str, Any]											=					["""hey"""]

													A__  :					Optional[int]											=					audioldm_pipe(snake_case     ,      num_inference_steps=1							)
													A__  :					Union[str, Any]											=					output.audios.shape
													assert audio_shape == (1, 256)

													A__  :					int											=					audioldm_pipe.vocoder.config
													config.model_in_dim *= 2
													A__  :					Optional[Any]											=					SpeechTaHifiGan(snake_case							).to(snake_case							)
													A__  :					str											=					audioldm_pipe(snake_case     ,      num_inference_steps=1							)
													A__  :					Any											=					output.audios.shape
													# waveform shape is unchanged, we just have 2x the number of mel channels in the spectrogram
													assert audio_shape == (1, 256)


							def   _UpperCamelCase    (     self :  Optional[Any]							):
													'''simple docstring'''





													self._test_attention_slicing_forward_pass(test_mean_pixel_difference=snake_case							)


							def   _UpperCamelCase    (     self :  List[Any]							):
													'''simple docstring'''





													self._test_inference_batch_single_identical(test_mean_pixel_difference=snake_case							)


							@unittest.skipIf(
							    torch_device != """cuda""" or not is_xformers_available()     ,      reason="""XFormers attention is only available with CUDA and `xformers` installed"""     ,      )
							def   _UpperCamelCase    (     self :  List[str]							):
													'''simple docstring'''





													self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=snake_case							)






@slow
class 							__SCREAMING_SNAKE_CASE  (			unittest.TestCase						):


							def   _UpperCamelCase    (     self :  Optional[Any]							):
													'''simple docstring'''





													super().tearDown()
													gc.collect()
													torch.cuda.empty_cache()


							def   _UpperCamelCase    (     self :  Optional[Any]     ,      snake_case :  List[str]     ,      snake_case :  Tuple="cpu"     ,      snake_case :  Optional[int]=torch.floataa     ,      snake_case :  Optional[Any]=0							):
													'''simple docstring'''





													A__  :					Union[str, Any]											=					torch.Generator(device=snake_case							).manual_seed(snake_case							)
													A__  :					List[str]											=					np.random.RandomState(snake_case							).standard_normal((1, 8, 128, 16)							)
													A__  :					List[Any]											=					torch.from_numpy(snake_case							).to(device=snake_case     ,      dtype=snake_case							)
													A__  :					Dict											=					{
													    """prompt""": """A hammer hitting a wooden surface""",
													    """latents""": latents,
													    """generator""": generator,
													    """num_inference_steps""": 3,
													    """guidance_scale""": 2.5,
													}
													return inputs


							def   _UpperCamelCase    (     self :  Optional[int]							):
													'''simple docstring'''





													A__  :					Union[str, Any]											=					AudioLDMPipeline.from_pretrained("""cvssp/audioldm"""							)
													A__  :					Any											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					Optional[int]											=					self.get_inputs(snake_case							)
													A__  :					Tuple											=					25
													A__  :					Optional[int]											=					audioldm_pipe(**snake_case							).audios[0]

													assert audio.ndim == 1
													assert len(snake_case							) == 8_1920

													A__  :					Union[str, Any]											=					audio[7_7230:7_7240]
													A__  :					List[Any]											=					np.array(
													    [-0.4884, -0.4607, 0.0023, 0.5007, 0.5896, 0.5151, 0.3813, -0.0208, -0.3687, -0.4315]							)
													A__  :					int											=					np.abs(expected_slice - audio_slice							).max()
													assert max_diff < 1e-2


							def   _UpperCamelCase    (     self :  Any							):
													'''simple docstring'''





													A__  :					List[Any]											=					AudioLDMPipeline.from_pretrained("""cvssp/audioldm"""							)
													A__  :					str											=					LMSDiscreteScheduler.from_config(audioldm_pipe.scheduler.config							)
													A__  :					Optional[int]											=					audioldm_pipe.to(snake_case							)
													audioldm_pipe.set_progress_bar_config(disable=snake_case							)

													A__  :					List[Any]											=					self.get_inputs(snake_case							)
													A__  :					Optional[int]											=					audioldm_pipe(**snake_case							).audios[0]

													assert audio.ndim == 1
													assert len(snake_case							) == 8_1920

													A__  :					Any											=					audio[2_7780:2_7790]
													A__  :					Dict											=					np.array([-0.2131, -0.0873, -0.0124, -0.0189, 0.0569, 0.1373, 0.1883, 0.2886, 0.3297, 0.2212]							)
													A__  :					List[Any]											=					np.abs(expected_slice - audio_slice							).max()
													assert max_diff < 3e-2














"""simple docstring"""



import math
import unittest

from transformers import BioGptConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    BioGptForCausalLM,
						    BioGptForSequenceClassification,
						    BioGptForTokenClassification,
						    BioGptModel,
						    BioGptTokenizer,
						)
						from transformers.models.biogpt.modeling_biogpt import BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST






class 							__SCREAMING_SNAKE_CASE  :


							def __init__(     self :  Union[str, Any]     ,      snake_case :  str     ,      snake_case :  List[Any]=13     ,      snake_case :  Union[str, Any]=7     ,      snake_case :  Optional[Any]=True     ,      snake_case :  str=True     ,      snake_case :  Dict=False     ,      snake_case :  Union[str, Any]=True     ,      snake_case :  Optional[Any]=99     ,      snake_case :  str=32     ,      snake_case :  Tuple=5     ,      snake_case :  List[str]=4     ,      snake_case :  Optional[int]=37     ,      snake_case :  str="gelu"     ,      snake_case :  Tuple=0.1     ,      snake_case :  Optional[int]=0.1     ,      snake_case :  int=512     ,      snake_case :  List[str]=16     ,      snake_case :  str=2     ,      snake_case :  Optional[int]=0.02     ,      snake_case :  str=3     ,      snake_case :  Dict=4     ,      snake_case :  Optional[Any]=None     ,      ):
													'''simple docstring'''





													A__  :					int											=					parent
													A__  :					Union[str, Any]											=					batch_size
													A__  :					Optional[int]											=					seq_length
													A__  :					List[Any]											=					is_training
													A__  :					List[str]											=					use_input_mask
													A__  :					Optional[Any]											=					use_token_type_ids
													A__  :					List[Any]											=					use_labels
													A__  :					Union[str, Any]											=					vocab_size
													A__  :					List[Any]											=					hidden_size
													A__  :					Any											=					num_hidden_layers
													A__  :					Any											=					num_attention_heads
													A__  :					Optional[int]											=					intermediate_size
													A__  :					Any											=					hidden_act
													A__  :					Tuple											=					hidden_dropout_prob
													A__  :					Dict											=					attention_probs_dropout_prob
													A__  :					Optional[int]											=					max_position_embeddings
													A__  :					Tuple											=					type_vocab_size
													A__  :					Union[str, Any]											=					type_sequence_label_size
													A__  :					List[str]											=					initializer_range
													A__  :					Any											=					num_labels
													A__  :					Any											=					num_choices
													A__  :					int											=					scope


							def   _UpperCamelCase    (     self :  int							):
													'''simple docstring'''





													A__  :					str											=					ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size							)

													A__  :					Tuple											=					None
													if self.use_input_mask:
																			A__  :					List[str]											=					random_attention_mask([self.batch_size, self.seq_length]							)

													A__  :					Union[str, Any]											=					None
													if self.use_token_type_ids:
																			A__  :					str											=					ids_tensor([self.batch_size, self.seq_length]     ,      self.type_vocab_size							)

													A__  :					int											=					None
													A__  :					int											=					None
													A__  :					List[str]											=					None
													if self.use_labels:
																			A__  :					List[Any]											=					ids_tensor([self.batch_size]     ,      self.type_sequence_label_size							)
																			A__  :					Optional[int]											=					ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels							)
																			A__  :					Dict											=					ids_tensor([self.batch_size]     ,      self.num_choices							)

													A__  :					Union[str, Any]											=					self.get_config()

													return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def   _UpperCamelCase    (     self :  Tuple							):
													'''simple docstring'''





													return BioGptConfig(
													    vocab_size=self.vocab_size     ,      hidden_size=self.hidden_size     ,      num_hidden_layers=self.num_hidden_layers     ,      num_attention_heads=self.num_attention_heads     ,      intermediate_size=self.intermediate_size     ,      hidden_act=self.hidden_act     ,      hidden_dropout_prob=self.hidden_dropout_prob     ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,      max_position_embeddings=self.max_position_embeddings     ,      type_vocab_size=self.type_vocab_size     ,      is_decoder=snake_case     ,      initializer_range=self.initializer_range     ,      )


							def   _UpperCamelCase    (     self :  Optional[int]     ,      snake_case :  Dict     ,      snake_case :  Any     ,      snake_case :  Dict     ,      snake_case :  Any     ,      snake_case :  Union[str, Any]     ,      snake_case :  Tuple     ,      snake_case :  Optional[Any]							):
													'''simple docstring'''





													A__  :					Optional[Any]											=					BioGptModel(config=snake_case							)
													model.to(snake_case							)
													model.eval()
													A__  :					List[Any]											=					model(snake_case     ,      attention_mask=snake_case							)
													A__  :					Dict											=					model(snake_case							)
													self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)							)


							def   _UpperCamelCase    (     self :  Optional[int]     ,      snake_case :  Dict     ,      snake_case :  Optional[int]     ,      snake_case :  List[str]     ,      snake_case :  str     ,      snake_case :  Optional[Any]     ,      snake_case :  List[str]     ,      snake_case :  List[Any]     ,      snake_case :  Tuple     ,      snake_case :  Optional[Any]     ,      ):
													'''simple docstring'''





													A__  :					List[str]											=					BioGptForCausalLM(config=snake_case							)
													model.to(snake_case							)
													model.eval()
													A__  :					Union[str, Any]											=					model(snake_case     ,      attention_mask=snake_case     ,      token_type_ids=snake_case     ,      labels=snake_case							)
													self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)							)


							def   _UpperCamelCase    (     self :  Any     ,      snake_case :  str     ,      snake_case :  Tuple     ,      snake_case :  int     ,      snake_case :  Optional[Any]     ,      snake_case :  Any     ,      *snake_case :  Dict							):
													'''simple docstring'''





													A__  :					Union[str, Any]											=					BioGptModel(config=snake_case							)
													model.to(snake_case							)
													model.eval()

													# create attention mask
													A__  :					List[Any]											=					torch.ones(input_ids.shape     ,      dtype=torch.long     ,      device=snake_case							)
													A__  :					Any											=					self.seq_length // 2
													A__  :					str											=					0

													# first forward pass
													A__					,					A__  :					List[Any]											=					model(snake_case     ,      attention_mask=snake_case							).to_tuple()

													# create hypothetical next token and extent to next_input_ids
													A__  :					int											=					ids_tensor((self.batch_size, 1)     ,      config.vocab_size							)

													# change a random masked slice from input_ids
													A__  :					List[str]											=					ids_tensor((1,)     ,      snake_case							).item() + 1
													A__  :					Optional[int]											=					ids_tensor((self.batch_size, 1)     ,      config.vocab_size							).squeeze(-1							)
													A__  :					int											=					random_other_next_tokens

													# append to next input_ids and attn_mask
													A__  :					str											=					torch.cat([input_ids, next_tokens]     ,      dim=-1							)
													A__  :					List[Any]											=					torch.cat(
													    [attn_mask, torch.ones((attn_mask.shape[0], 1)     ,      dtype=torch.long     ,      device=snake_case							)]     ,      dim=1     ,      )

													# get two different outputs
													A__  :					Union[str, Any]											=					model(snake_case     ,      attention_mask=snake_case							)["""last_hidden_state"""]
													A__  :					Optional[int]											=					model(snake_case     ,      past_key_values=snake_case     ,      attention_mask=snake_case							)["""last_hidden_state"""]

													# select random slice
													A__  :					List[Any]											=					ids_tensor((1,)     ,      output_from_past.shape[-1]							).item()
													A__  :					List[str]											=					output_from_no_past[:, -1, random_slice_idx].detach()
													A__  :					Any											=					output_from_past[:, 0, random_slice_idx].detach()

													# test that outputs are equal for slice
													self.parent.assertTrue(torch.allclose(snake_case     ,      snake_case     ,      atol=1e-3							)							)


							def   _UpperCamelCase    (     self :  int     ,      snake_case :  Optional[Any]     ,      snake_case :  Optional[Any]     ,      snake_case :  str     ,      snake_case :  int     ,      snake_case :  Optional[Any]     ,      *snake_case :  str							):
													'''simple docstring'''





													A__  :					Dict											=					BioGptModel(config=snake_case							).to(snake_case							).eval()

													A__  :					Tuple											=					torch.ones(input_ids.shape     ,      dtype=torch.long     ,      device=snake_case							)

													# first forward pass
													A__  :					Dict											=					model(snake_case     ,      attention_mask=snake_case     ,      use_cache=snake_case							)

													A__					,					A__  :					List[Any]											=					outputs.to_tuple()

													# create hypothetical multiple next token and extent to next_input_ids
													A__  :					Union[str, Any]											=					ids_tensor((self.batch_size, 3)     ,      config.vocab_size							)
													A__  :					int											=					ids_tensor((self.batch_size, 3)     ,      2							)

													# append to next input_ids and
													A__  :					str											=					torch.cat([input_ids, next_tokens]     ,      dim=-1							)
													A__  :					Optional[int]											=					torch.cat([attention_mask, next_attn_mask]     ,      dim=-1							)

													A__  :					Any											=					model(snake_case     ,      attention_mask=snake_case							)["""last_hidden_state"""]
													A__  :					Union[str, Any]											=					model(snake_case     ,      attention_mask=snake_case     ,      past_key_values=snake_case							)[
													    """last_hidden_state"""
													]

													# select random slice
													A__  :					int											=					ids_tensor((1,)     ,      output_from_past.shape[-1]							).item()
													A__  :					Any											=					output_from_no_past[:, -3:, random_slice_idx].detach()
													A__  :					List[Any]											=					output_from_past[:, :, random_slice_idx].detach()

													self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

													# test that outputs are equal for slice
													self.parent.assertTrue(torch.allclose(snake_case     ,      snake_case     ,      atol=1e-3							)							)


							def   _UpperCamelCase    (     self :  Optional[Any]     ,      snake_case :  Tuple     ,      snake_case :  Union[str, Any]     ,      snake_case :  Optional[Any]     ,      snake_case :  Any     ,      snake_case :  Tuple     ,      *snake_case :  Union[str, Any]     ,      snake_case :  Union[str, Any]=False							):
													'''simple docstring'''





													A__  :					Tuple											=					BioGptForCausalLM(snake_case							)
													model.to(snake_case							)
													if gradient_checkpointing:
																			model.gradient_checkpointing_enable()

													A__  :					Optional[Any]											=					model(snake_case     ,      labels=snake_case							)
													self.parent.assertEqual(result.loss.shape     ,      ()							)
													self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)							)
													result.loss.backward()


							def   _UpperCamelCase    (     self :  int     ,      snake_case :  Optional[Any]     ,      *snake_case :  Optional[int]							):
													'''simple docstring'''





													A__  :					int											=					BioGptModel(snake_case							)
													A__  :					Union[str, Any]											=					model.config.initializer_range / math.sqrt(2 * model.config.num_hidden_layers							)
													for key in model.state_dict().keys():
																			if "c_proj" in key and "weight" in key:
																									self.parent.assertLessEqual(abs(torch.std(model.state_dict()[key]							) - model_std							)     ,      0.001							)
																									self.parent.assertLessEqual(abs(torch.mean(model.state_dict()[key]							) - 0.0							)     ,      0.01							)


							def   _UpperCamelCase    (     self :  Any     ,      snake_case :  Dict     ,      snake_case :  Tuple     ,      snake_case :  int     ,      snake_case :  Union[str, Any]     ,      snake_case :  Dict     ,      *snake_case :  Any							):
													'''simple docstring'''





													A__  :					Union[str, Any]											=					self.num_labels
													A__  :					int											=					BioGptForTokenClassification(snake_case							)
													model.to(snake_case							)
													model.eval()
													A__  :					str											=					model(snake_case     ,      attention_mask=snake_case     ,      token_type_ids=snake_case							)
													self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)							)


							def   _UpperCamelCase    (     self :  int							):
													'''simple docstring'''





													A__  :					List[str]											=					self.prepare_config_and_inputs()
													(
													    (
													    A__
													)					,					(
													    A__
													)					,					(
													    A__
													)					,					(
													    A__
													)					,					(
													    A__
													)					,					(
													    A__
													)					,					(
													    A__
													)					,					
													)  :					str											=					config_and_inputs
													A__  :					Union[str, Any]											=					{"""input_ids""": input_ids, """attention_mask""": input_mask}
													return config, inputs_dict






@require_torch
class 							__SCREAMING_SNAKE_CASE  (			UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						unittest.TestCase						):
							snake_case_       =							(
							    (BioGptModel, BioGptForCausalLM, BioGptForSequenceClassification, BioGptForTokenClassification)
							    if is_torch_available()
							    else ()
							)
							snake_case_       =							(BioGptForCausalLM,) if is_torch_available() else ()
							snake_case_       =							(
							    {
							        'feature-extraction': BioGptModel,
							        'text-classification': BioGptForSequenceClassification,
							        'text-generation': BioGptForCausalLM,
							        'token-classification': BioGptForTokenClassification,
							        'zero-shot': BioGptForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							snake_case_       =							False


							def   _UpperCamelCase    (     self :  Union[str, Any]							):
													'''simple docstring'''





													A__  :					List[str]											=					BioGptModelTester(self							)
													A__  :					List[Any]											=					ConfigTester(self     ,      config_class=snake_case     ,      hidden_size=37							)


							def   _UpperCamelCase    (     self :  int							):
													'''simple docstring'''





													self.config_tester.run_common_tests()


							def   _UpperCamelCase    (     self :  List[Any]							):
													'''simple docstring'''





													A__  :					str											=					self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*snake_case							)


							def   _UpperCamelCase    (     self :  Any							):
													'''simple docstring'''





													A__  :					int											=					self.model_tester.prepare_config_and_inputs()
													for type in ["absolute", "relative_key", "relative_key_query"]:
																			A__  :					str											=					type
																			self.model_tester.create_and_check_model(*snake_case							)


							def   _UpperCamelCase    (     self :  Union[str, Any]							):
													'''simple docstring'''





													A__  :					str											=					self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_biogpt_model_attention_mask_past(*snake_case							)


							def   _UpperCamelCase    (     self :  Optional[Any]							):
													'''simple docstring'''





													A__  :					Dict											=					self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_forward_and_backwards(*snake_case     ,      gradient_checkpointing=snake_case							)


							def   _UpperCamelCase    (     self :  Optional[int]							):
													'''simple docstring'''





													A__  :					Optional[int]											=					self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_biogpt_model_past_large_inputs(*snake_case							)


							def   _UpperCamelCase    (     self :  int							):
													'''simple docstring'''





													A__  :					Any											=					self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_biogpt_weight_initialization(*snake_case							)


							def   _UpperCamelCase    (     self :  List[str]							):
													'''simple docstring'''





													A__  :					List[str]											=					self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_biogpt_for_token_classification(*snake_case							)


							@slow
							def   _UpperCamelCase    (     self :  str							):
													'''simple docstring'''





													A__  :					Tuple											=					BioGptForCausalLM.from_pretrained("""microsoft/biogpt"""							)
													model.to(snake_case							)
													A__  :					Optional[int]											=					BioGptTokenizer.from_pretrained("""microsoft/biogpt"""							)

													A__  :					Any											=					"""left"""

													# Define PAD Token = EOS Token = 50256
													A__  :					Optional[int]											=					tokenizer.eos_token
													A__  :					Dict											=					model.config.eos_token_id

													# use different length sentences to test batching
													A__  :					Union[str, Any]											=					[
													    """Hello, my dog is a little""",
													    """Today, I""",
													]

													A__  :					List[str]											=					tokenizer(snake_case     ,      return_tensors="""pt"""     ,      padding=snake_case							)
													A__  :					str											=					inputs["""input_ids"""].to(snake_case							)

													A__  :					Dict											=					model.generate(
													    input_ids=snake_case     ,      attention_mask=inputs["""attention_mask"""].to(snake_case							)     ,      )

													A__  :					Optional[int]											=					tokenizer(sentences[0]     ,      return_tensors="""pt"""							).input_ids.to(snake_case							)
													A__  :					Any											=					model.generate(input_ids=snake_case							)

													A__  :					List[str]											=					inputs_non_padded.shape[-1] - inputs["""attention_mask"""][-1].long().sum().cpu().item()
													A__  :					str											=					tokenizer(sentences[1]     ,      return_tensors="""pt"""							).input_ids.to(snake_case							)
													A__  :					Dict											=					model.generate(input_ids=snake_case     ,      max_length=model.config.max_length - num_paddings							)

													A__  :					Optional[Any]											=					tokenizer.batch_decode(snake_case     ,      skip_special_tokens=snake_case							)
													A__  :					List[Any]											=					tokenizer.decode(output_non_padded[0]     ,      skip_special_tokens=snake_case							)
													A__  :					str											=					tokenizer.decode(output_padded[0]     ,      skip_special_tokens=snake_case							)

													A__  :					Optional[int]											=					[
													    """Hello, my dog is a little bit bigger than a little bit.""",
													    """Today, I have a good idea of how to use the information""",
													]
													self.assertListEqual(snake_case     ,      snake_case							)
													self.assertListEqual(snake_case     ,      [non_padded_sentence, padded_sentence]							)


							@slow
							def   _UpperCamelCase    (     self :  Optional[Any]							):
													'''simple docstring'''





													for model_name in BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			A__  :					Optional[Any]											=					BioGptModel.from_pretrained(snake_case							)
																			self.assertIsNotNone(snake_case							)


							def   _UpperCamelCase    (     self :  str							):
													'''simple docstring'''





													A__					,					A__  :					Dict											=					self.model_tester.prepare_config_and_inputs_for_common()
													A__  :					Optional[int]											=					3
													A__  :					List[Any]											=					input_dict["""input_ids"""]
													A__  :					Dict											=					input_ids.ne(1							).to(snake_case							)
													A__  :					Optional[Any]											=					ids_tensor([self.model_tester.batch_size]     ,      self.model_tester.type_sequence_label_size							)
													A__  :					Union[str, Any]											=					BioGptForSequenceClassification(snake_case							)
													model.to(snake_case							)
													model.eval()
													A__  :					int											=					model(snake_case     ,      attention_mask=snake_case     ,      labels=snake_case							)
													self.assertEqual(result.logits.shape     ,      (self.model_tester.batch_size, self.model_tester.num_labels)							)


							def   _UpperCamelCase    (     self :  int							):
													'''simple docstring'''





													A__					,					A__  :					str											=					self.model_tester.prepare_config_and_inputs_for_common()
													A__  :					Any											=					3
													A__  :					List[Any]											=					"""multi_label_classification"""
													A__  :					Dict											=					input_dict["""input_ids"""]
													A__  :					Tuple											=					input_ids.ne(1							).to(snake_case							)
													A__  :					Any											=					ids_tensor(
													    [self.model_tester.batch_size, config.num_labels]     ,      self.model_tester.type_sequence_label_size							).to(torch.float							)
													A__  :					Tuple											=					BioGptForSequenceClassification(snake_case							)
													model.to(snake_case							)
													model.eval()
													A__  :					List[str]											=					model(snake_case     ,      attention_mask=snake_case     ,      labels=snake_case							)
													self.assertEqual(result.logits.shape     ,      (self.model_tester.batch_size, self.model_tester.num_labels)							)






@require_torch
class 							__SCREAMING_SNAKE_CASE  (			unittest.TestCase						):


							@slow
							def   _UpperCamelCase    (     self :  List[str]							):
													'''simple docstring'''





													A__  :					Optional[Any]											=					BioGptForCausalLM.from_pretrained("""microsoft/biogpt"""							)
													A__  :					str											=					torch.tensor([[2, 4805, 9, 656, 21]]							)
													A__  :					Dict											=					model(snake_case							)[0]

													A__  :					Tuple											=					4_2384

													A__  :					str											=					torch.Size((1, 5, vocab_size)							)
													self.assertEqual(output.shape     ,      snake_case							)

													A__  :					str											=					torch.tensor(
													    [[[-9.5236, -9.8918, 10.4557], [-11.0469, -9.6423, 8.1022], [-8.8664, -7.8826, 5.5325]]]							)

													self.assertTrue(torch.allclose(output[:, :3, :3]     ,      snake_case     ,      atol=1e-4							)							)


							@slow
							def   _UpperCamelCase    (     self :  Dict							):
													'''simple docstring'''





													A__  :					Tuple											=					BioGptTokenizer.from_pretrained("""microsoft/biogpt"""							)
													A__  :					Any											=					BioGptForCausalLM.from_pretrained("""microsoft/biogpt"""							)
													model.to(snake_case							)

													torch.manual_seed(0							)
													A__  :					Tuple											=					tokenizer("""COVID-19 is"""     ,      return_tensors="""pt"""							).to(snake_case							)
													A__  :					Optional[int]											=					model.generate(
													    **snake_case     ,      min_length=100     ,      max_length=1024     ,      num_beams=5     ,      early_stopping=snake_case     ,      )
													A__  :					Optional[int]											=					tokenizer.decode(output_ids[0]     ,      skip_special_tokens=snake_case							)

													A__  :					List[str]											=					(
													    """COVID-19 is a global pandemic caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2), the"""
													    """ causative agent of coronavirus disease 2019 (COVID-19), which has spread to more than 200 countries and"""
													    """ territories, including the United States (US), Canada, Australia, New Zealand, the United Kingdom (UK),"""
													    """ and the United States of America (USA), as of March 11, 2020, with more than 800,000 confirmed cases and"""
													    """ more than 800,000 deaths."""
													)
													self.assertEqual(snake_case     ,      snake_case							)






