






import math
import unittest

from transformers import BioGptConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      BioGptForCausalLM,
      BioGptForSequenceClassification,
      BioGptForTokenClassification,
      BioGptModel,
      BioGptTokenizer,
  )
  from transformers.models.biogpt.modeling_biogpt import BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST


class       __lowercase      :







     '''simple docstring'''




     def __init__(				self       :			Dict ,	UpperCamelCase_       :			int ,	UpperCamelCase_       :			Union[str, Any]=13 ,	UpperCamelCase_       :			str=7 ,	UpperCamelCase_       :			List[Any]=True ,	UpperCamelCase_       :			List[Any]=True ,	UpperCamelCase_       :			List[Any]=False ,	UpperCamelCase_       :			str=True ,	UpperCamelCase_       :			int=99 ,	UpperCamelCase_       :			List[str]=32 ,	UpperCamelCase_       :			Optional[Any]=5 ,	UpperCamelCase_       :			List[Any]=4 ,	UpperCamelCase_       :			Tuple=37 ,	UpperCamelCase_       :			Any="gelu" ,	UpperCamelCase_       :			List[Any]=0.1 ,	UpperCamelCase_       :			Any=0.1 ,	UpperCamelCase_       :			Dict=512 ,	UpperCamelCase_       :			Dict=16 ,	UpperCamelCase_       :			Tuple=2 ,	UpperCamelCase_       :			Optional[Any]=0.02 ,	UpperCamelCase_       :			List[str]=3 ,	UpperCamelCase_       :			Optional[int]=4 ,	UpperCamelCase_       :			Optional[Any]=None ,	):


         """simple docstring"""



         __A         =	parent
         __A         =	batch_size
         __A         =	seq_length
         __A         =	is_training
         __A         =	use_input_mask
         __A         =	use_token_type_ids
         __A         =	use_labels
         __A         =	vocab_size
         __A         =	hidden_size
         __A         =	num_hidden_layers
         __A         =	num_attention_heads
         __A         =	intermediate_size
         __A         =	hidden_act
         __A         =	hidden_dropout_prob
         __A         =	attention_probs_dropout_prob
         __A         =	max_position_embeddings
         __A         =	type_vocab_size
         __A         =	type_sequence_label_size
         __A         =	initializer_range
         __A         =	num_labels
         __A         =	num_choices
         __A         =	scope




     def 		lowerCAmelCase_     (				self       :			Optional[int]				):


         """simple docstring"""



         __A         =	ids_tensor([self.batch_size, self.seq_length] ,	self.vocab_size				)

         __A         =	None
         if self.use_input_mask:
             __A         =	random_attention_mask([self.batch_size, self.seq_length]				)

         __A         =	None
         if self.use_token_type_ids:
             __A         =	ids_tensor([self.batch_size, self.seq_length] ,	self.type_vocab_size				)

         __A         =	None
         __A         =	None
         __A         =	None
         if self.use_labels:
             __A         =	ids_tensor([self.batch_size] ,	self.type_sequence_label_size				)
             __A         =	ids_tensor([self.batch_size, self.seq_length] ,	self.num_labels				)
             __A         =	ids_tensor([self.batch_size] ,	self.num_choices				)

         __A         =	self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




     def 		lowerCAmelCase_     (				self       :			str				):


         """simple docstring"""



         return BioGptConfig(
             vocab_size=self.vocab_size ,	hidden_size=self.hidden_size ,	num_hidden_layers=self.num_hidden_layers ,	num_attention_heads=self.num_attention_heads ,	intermediate_size=self.intermediate_size ,	hidden_act=self.hidden_act ,	hidden_dropout_prob=self.hidden_dropout_prob ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob ,	max_position_embeddings=self.max_position_embeddings ,	type_vocab_size=self.type_vocab_size ,	is_decoder=__a ,	initializer_range=self.initializer_range ,	)




     def 		lowerCAmelCase_     (				self       :			List[str] ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			Optional[Any] ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			Union[str, Any]				):


         """simple docstring"""



         __A         =	BioGptModel(config=__a				)
         model.to(__a				)
         model.eval()
         __A         =	model(__a ,	attention_mask=__a				)
         __A         =	model(__a				)
         self.parent.assertEqual(result.last_hidden_state.shape ,	(self.batch_size, self.seq_length, self.hidden_size)				)




     def 		lowerCAmelCase_     (				self       :			List[Any] ,	UpperCamelCase_       :			Tuple ,	UpperCamelCase_       :			Optional[Any] ,	UpperCamelCase_       :			Union[str, Any] ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			Tuple ,	UpperCamelCase_       :			Optional[Any] ,	UpperCamelCase_       :			str ,	):


         """simple docstring"""



         __A         =	BioGptForCausalLM(config=__a				)
         model.to(__a				)
         model.eval()
         __A         =	model(__a ,	attention_mask=__a ,	token_type_ids=__a ,	labels=__a				)
         self.parent.assertEqual(result.logits.shape ,	(self.batch_size, self.seq_length, self.vocab_size)				)




     def 		lowerCAmelCase_     (				self       :			Union[str, Any] ,	UpperCamelCase_       :			Union[str, Any] ,	UpperCamelCase_       :			str ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			str ,	UpperCamelCase_       :			str ,	*UpperCamelCase_       :			Optional[int]				):


         """simple docstring"""



         __A         =	BioGptModel(config=__a				)
         model.to(__a				)
         model.eval()

         # create attention mask
         __A         =	torch.ones(input_ids.shape ,	dtype=torch.long ,	device=__a				)
         __A         =	self.seq_length // 2
         __A         =	0

         # first forward pass
         __A    ,	__A         =	model(__a ,	attention_mask=__a				).to_tuple()

         # create hypothetical next token and extent to next_input_ids
         __A         =	ids_tensor((self.batch_size, 1) ,	config.vocab_size				)

         # change a random masked slice from input_ids
         __A         =	ids_tensor((1,) ,	__a				).item() + 1
         __A         =	ids_tensor((self.batch_size, 1) ,	config.vocab_size				).squeeze(-1				)
         __A         =	random_other_next_tokens

         # append to next input_ids and attn_mask
         __A         =	torch.cat([input_ids, next_tokens] ,	dim=-1				)
         __A         =	torch.cat(
             [attn_mask, torch.ones((attn_mask.shape[0], 1) ,	dtype=torch.long ,	device=__a				)] ,	dim=1 ,	)

         # get two different outputs
         __A         =	model(__a ,	attention_mask=__a				)["""last_hidden_state"""]
         __A         =	model(__a ,	past_key_values=__a ,	attention_mask=__a				)["""last_hidden_state"""]

         # select random slice
         __A         =	ids_tensor((1,) ,	output_from_past.shape[-1]				).item()
         __A         =	output_from_no_past[:, -1, random_slice_idx].detach()
         __A         =	output_from_past[:, 0, random_slice_idx].detach()

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(__a ,	__a ,	atol=1e-3				)				)




     def 		lowerCAmelCase_     (				self       :			List[Any] ,	UpperCamelCase_       :			int ,	UpperCamelCase_       :			Any ,	UpperCamelCase_       :			List[str] ,	UpperCamelCase_       :			Optional[int] ,	UpperCamelCase_       :			int ,	*UpperCamelCase_       :			Tuple				):


         """simple docstring"""



         __A         =	BioGptModel(config=__a				).to(__a				).eval()

         __A         =	torch.ones(input_ids.shape ,	dtype=torch.long ,	device=__a				)

         # first forward pass
         __A         =	model(__a ,	attention_mask=__a ,	use_cache=__a				)

         __A    ,	__A         =	outputs.to_tuple()

         # create hypothetical multiple next token and extent to next_input_ids
         __A         =	ids_tensor((self.batch_size, 3) ,	config.vocab_size				)
         __A         =	ids_tensor((self.batch_size, 3) ,	2				)

         # append to next input_ids and
         __A         =	torch.cat([input_ids, next_tokens] ,	dim=-1				)
         __A         =	torch.cat([attention_mask, next_attn_mask] ,	dim=-1				)

         __A         =	model(__a ,	attention_mask=__a				)["""last_hidden_state"""]
         __A         =	model(__a ,	attention_mask=__a ,	past_key_values=__a				)[
             """last_hidden_state"""
         ]

         # select random slice
         __A         =	ids_tensor((1,) ,	output_from_past.shape[-1]				).item()
         __A         =	output_from_no_past[:, -3:, random_slice_idx].detach()
         __A         =	output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(__a ,	__a ,	atol=1e-3				)				)




     def 		lowerCAmelCase_     (				self       :			str ,	UpperCamelCase_       :			Union[str, Any] ,	UpperCamelCase_       :			Dict ,	UpperCamelCase_       :			str ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			int ,	*UpperCamelCase_       :			Any ,	UpperCamelCase_       :			Optional[int]=False				):


         """simple docstring"""



         __A         =	BioGptForCausalLM(__a				)
         model.to(__a				)
         if gradient_checkpointing:
             model.gradient_checkpointing_enable()

         __A         =	model(__a ,	labels=__a				)
         self.parent.assertEqual(result.loss.shape ,	()				)
         self.parent.assertEqual(result.logits.shape ,	(self.batch_size, self.seq_length, self.vocab_size)				)
         result.loss.backward()




     def 		lowerCAmelCase_     (				self       :			str ,	UpperCamelCase_       :			Optional[int] ,	*UpperCamelCase_       :			str				):


         """simple docstring"""



         __A         =	BioGptModel(__a				)
         __A         =	model.config.initializer_range / math.sqrt(2 * model.config.num_hidden_layers				)
         for key in model.state_dict().keys():
             if "c_proj" in key and "weight" in key:
                 self.parent.assertLessEqual(abs(torch.std(model.state_dict()[key]				) - model_std				) ,	0.001				)
                 self.parent.assertLessEqual(abs(torch.mean(model.state_dict()[key]				) - 0.0				) ,	0.01				)




     def 		lowerCAmelCase_     (				self       :			Optional[int] ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			Tuple ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			str ,	UpperCamelCase_       :			Optional[int] ,	*UpperCamelCase_       :			Any				):


         """simple docstring"""



         __A         =	self.num_labels
         __A         =	BioGptForTokenClassification(__a				)
         model.to(__a				)
         model.eval()
         __A         =	model(__a ,	attention_mask=__a ,	token_type_ids=__a				)
         self.parent.assertEqual(result.logits.shape ,	(self.batch_size, self.seq_length, self.num_labels)				)





     def 		lowerCAmelCase_     (				self       :			Tuple				):


         """simple docstring"""



         __A         =	self.prepare_config_and_inputs()
         (
             (
             __A						
         )    ,	(
             __A						
         )    ,	(
             __A						
         )    ,	(
             __A						
         )    ,	(
             __A						
         )    ,	(
             __A						
         )    ,	(
             __A						
         )    ,							
         )         =	config_and_inputs
         __A         =	{"""input_ids""": input_ids, """attention_mask""": input_mask}
         return config, inputs_dict


@require_torch
class       __lowercase      (	lowercase_			,      lowercase_			,      lowercase_			,      unittest.TestCase  ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE   						=				(
         (BioGptModel, BioGptForCausalLM, BioGptForSequenceClassification, BioGptForTokenClassification)
         if is_torch_available()
         else ()
     )
     SCREAMING_SNAKE_CASE   						=				(BioGptForCausalLM,) if is_torch_available() else ()
     SCREAMING_SNAKE_CASE   						=				(
         {
             "feature-extraction": BioGptModel,
             "text-classification": BioGptForSequenceClassification,
             "text-generation": BioGptForCausalLM,
             "token-classification": BioGptForTokenClassification,
             "zero-shot": BioGptForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     SCREAMING_SNAKE_CASE   						=				False




     def 		lowerCAmelCase_     (				self       :			Dict				):


         """simple docstring"""



         __A         =	BioGptModelTester(self				)
         __A         =	ConfigTester(self ,	config_class=__a ,	hidden_size=37				)




     def 		lowerCAmelCase_     (				self       :			Dict				):


         """simple docstring"""



         self.config_tester.run_common_tests()




     def 		lowerCAmelCase_     (				self       :			List[Any]				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__a				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
             __A         =	type
             self.model_tester.create_and_check_model(*__a				)




     def 		lowerCAmelCase_     (				self       :			str				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_biogpt_model_attention_mask_past(*__a				)




     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_forward_and_backwards(*__a ,	gradient_checkpointing=__a				)




     def 		lowerCAmelCase_     (				self       :			List[str]				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_biogpt_model_past_large_inputs(*__a				)




     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_biogpt_weight_initialization(*__a				)




     def 		lowerCAmelCase_     (				self       :			Union[str, Any]				):


         """simple docstring"""



         __A         =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_biogpt_for_token_classification(*__a				)




     @slow
     def 		lowerCAmelCase_     (				self       :			List[str]				):


         """simple docstring"""



         __A         =	BioGptForCausalLM.from_pretrained("""microsoft/biogpt"""				)
         model.to(__a				)
         __A         =	BioGptTokenizer.from_pretrained("""microsoft/biogpt"""				)

         __A         =	"""left"""

         # Define PAD Token = EOS Token = 50256
         __A         =	tokenizer.eos_token
         __A         =	model.config.eos_token_id

         # use different length sentences to test batching
         __A         =	[
             """Hello, my dog is a little""",
             """Today, I""",
         ]

         __A         =	tokenizer(__a ,	return_tensors="""pt""" ,	padding=__a				)
         __A         =	inputs["""input_ids"""].to(__a				)

         __A         =	model.generate(
             input_ids=__a ,	attention_mask=inputs["""attention_mask"""].to(__a				) ,	)

         __A         =	tokenizer(sentences[0] ,	return_tensors="""pt"""				).input_ids.to(__a				)
         __A         =	model.generate(input_ids=__a				)

         __A         =	inputs_non_padded.shape[-1] - inputs["""attention_mask"""][-1].long().sum().cpu().item()
         __A         =	tokenizer(sentences[1] ,	return_tensors="""pt"""				).input_ids.to(__a				)
         __A         =	model.generate(input_ids=__a ,	max_length=model.config.max_length - num_paddings				)

         __A         =	tokenizer.batch_decode(__a ,	skip_special_tokens=__a				)
         __A         =	tokenizer.decode(output_non_padded[0] ,	skip_special_tokens=__a				)
         __A         =	tokenizer.decode(output_padded[0] ,	skip_special_tokens=__a				)

         __A         =	[
             """Hello, my dog is a little bit bigger than a little bit.""",
             """Today, I have a good idea of how to use the information""",
         ]
         self.assertListEqual(__a ,	__a				)
         self.assertListEqual(__a ,	[non_padded_sentence, padded_sentence]				)




     @slow
     def 		lowerCAmelCase_     (				self       :			Dict				):


         """simple docstring"""



         for model_name in BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __A         =	BioGptModel.from_pretrained(__a				)
             self.assertIsNotNone(__a				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A    ,	__A         =	self.model_tester.prepare_config_and_inputs_for_common()
         __A         =	3
         __A         =	input_dict["""input_ids"""]
         __A         =	input_ids.ne(1				).to(__a				)
         __A         =	ids_tensor([self.model_tester.batch_size] ,	self.model_tester.type_sequence_label_size				)
         __A         =	BioGptForSequenceClassification(__a				)
         model.to(__a				)
         model.eval()
         __A         =	model(__a ,	attention_mask=__a ,	labels=__a				)
         self.assertEqual(result.logits.shape ,	(self.model_tester.batch_size, self.model_tester.num_labels)				)





     def 		lowerCAmelCase_     (				self       :			str				):


         """simple docstring"""



         __A    ,	__A         =	self.model_tester.prepare_config_and_inputs_for_common()
         __A         =	3
         __A         =	"""multi_label_classification"""
         __A         =	input_dict["""input_ids"""]
         __A         =	input_ids.ne(1				).to(__a				)
         __A         =	ids_tensor(
             [self.model_tester.batch_size, config.num_labels] ,	self.model_tester.type_sequence_label_size				).to(torch.float				)
         __A         =	BioGptForSequenceClassification(__a				)
         model.to(__a				)
         model.eval()
         __A         =	model(__a ,	attention_mask=__a ,	labels=__a				)
         self.assertEqual(result.logits.shape ,	(self.model_tester.batch_size, self.model_tester.num_labels)				)


@require_torch
class       __lowercase      (	unittest.TestCase  ):







     '''simple docstring'''




     @slow
     def 		lowerCAmelCase_     (				self       :			Optional[int]				):


         """simple docstring"""



         __A         =	BioGptForCausalLM.from_pretrained("""microsoft/biogpt"""				)
         __A         =	torch.tensor([[2, 4_805, 9, 656, 21]]				)
         __A         =	model(__a				)[0]

         __A         =	42_384

         __A         =	torch.Size((1, 5, vocab_size)				)
         self.assertEqual(output.shape ,	__a				)

         __A         =	torch.tensor(
             [[[-9.5236, -9.8918, 10.4557], [-11.0469, -9.6423, 8.1022], [-8.8664, -7.8826, 5.5325]]]				)

         self.assertTrue(torch.allclose(output[:, :3, :3] ,	__a ,	atol=1e-4				)				)





     @slow
     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         __A         =	BioGptTokenizer.from_pretrained("""microsoft/biogpt"""				)
         __A         =	BioGptForCausalLM.from_pretrained("""microsoft/biogpt"""				)
         model.to(__a				)

         torch.manual_seed(0				)
         __A         =	tokenizer("""COVID-19 is""" ,	return_tensors="""pt"""				).to(__a				)
         __A         =	model.generate(
             **__a ,	min_length=100 ,	max_length=1_024 ,	num_beams=5 ,	early_stopping=__a ,	)
         __A         =	tokenizer.decode(output_ids[0] ,	skip_special_tokens=__a				)

         __A         =	(
             """COVID-19 is a global pandemic caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2), the"""
             """ causative agent of coronavirus disease 2019 (COVID-19), which has spread to more than 200 countries and"""
             """ territories, including the United States (US), Canada, Australia, New Zealand, the United Kingdom (UK),"""
             """ and the United States of America (USA), as of March 11, 2020, with more than 800,000 confirmed cases and"""
             """ more than 800,000 deaths."""
         )
         self.assertEqual(__a ,	__a				)












import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available
from . import BaseDiffusersCLICommand



def 			_SCREAMING_SNAKE_CASE						( __lowercase   :			Any    )						->      Optional[int]:





    """simple docstring"""

    return EnvironmentCommand()






class       __lowercase      (	lowercase_  ):







     '''simple docstring'''




     @staticmethod
     def 		lowerCAmelCase_     (				UpperCamelCase_       :			ArgumentParser				):


         """simple docstring"""



         __A         =	parser.add_parser("""env"""				)
         download_parser.set_defaults(func=UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	huggingface_hub.__version__

         __A         =	"""not installed"""
         __A         =	"""NA"""
         if is_torch_available():
             import torch

             __A         =	torch.__version__
             __A         =	torch.cuda.is_available()

         __A         =	"""not installed"""
         if is_transformers_available():
             import transformers

             __A         =	transformers.__version__

         __A         =	"""not installed"""
         if is_accelerate_available():
             import accelerate

             __A         =	accelerate.__version__

         __A         =	"""not installed"""
         if is_xformers_available():
             import xformers

             __A         =	xformers.__version__

         __A         =	{
             """`diffusers` version""": version,
             """Platform""": platform.platform(),
             """Python version""": platform.python_version(),
             """PyTorch version (GPU?)""": F"{pt_version} ({pt_cuda_available})",
             """Huggingface_hub version""": hub_version,
             """Transformers version""": transformers_version,
             """Accelerate version""": accelerate_version,
             """xFormers version""": xformers_version,
             """Using GPU in script?""": """<fill in>""",
             """Using distributed or parallel set-up in script?""": """<fill in>""",
         }

         print("""\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n"""				)
         print(self.format_dict(UpperCamelCase_				)				)

         return info





     @staticmethod
     def 		lowerCAmelCase_     (				UpperCamelCase_       :			str				):


         """simple docstring"""



         return "\n".join([F"- {prop}: {val}" for prop, val in d.items()]				) + "\n"




