





import argparse
import math
import os
from copy import deepcopy

import torch
from audio_diffusion.models import DiffusionAttnUnetaD
from diffusion import sampling
from torch import nn

from diffusers import DanceDiffusionPipeline, IPNDMScheduler, UNetaDModel


_UpperCAmelCase        =				{
    'gwf-440k': {
        'url': 'https://model-server.zqevans2.workers.dev/gwf-440k.ckpt',
        'sample_rate': 4_8_0_0_0,
        'sample_size': 6_5_5_3_6,
    },
    'jmann-small-190k': {
        'url': 'https://model-server.zqevans2.workers.dev/jmann-small-190k.ckpt',
        'sample_rate': 4_8_0_0_0,
        'sample_size': 6_5_5_3_6,
    },
    'jmann-large-580k': {
        'url': 'https://model-server.zqevans2.workers.dev/jmann-large-580k.ckpt',
        'sample_rate': 4_8_0_0_0,
        'sample_size': 1_3_1_0_7_2,
    },
    'maestro-uncond-150k': {
        'url': 'https://model-server.zqevans2.workers.dev/maestro-uncond-150k.ckpt',
        'sample_rate': 1_6_0_0_0,
        'sample_size': 6_5_5_3_6,
    },
    'unlocked-uncond-250k': {
        'url': 'https://model-server.zqevans2.workers.dev/unlocked-uncond-250k.ckpt',
        'sample_rate': 1_6_0_0_0,
        'sample_size': 6_5_5_3_6,
    },
    'honk-140k': {
        'url': 'https://model-server.zqevans2.workers.dev/honk-140k.ckpt',
        'sample_rate': 1_6_0_0_0,
        'sample_size': 6_5_5_3_6,
    },
}




def  lowerCAmelCase_ (							UpperCamelCase_					,		UpperCamelCase_  )     ->		Dict:
       return torch.atana(A_					,		A_  ) / math.pi * 2




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		List[str]:
       UpperCamelCase_          =  torch.sin(t * math.pi / 2  ) ** 2
       UpperCamelCase_          =  (1 - sigma**2) ** 0.5
       return alpha_sigma_to_t(A_					,		A_  )



class 							_UpperCamelCase					(	_A     ):
       pass



class 							_UpperCamelCase					(	nn.Module     ):

       def __init__(		self: Tuple					,   _SCREAMING_SNAKE_CASE: str		)       -> Optional[Any]:




              """simple docstring"""


              super().__init__()

              UpperCamelCase_          =  DiffusionAttnUnetaD(UpperCamelCase__					,   n_attn_layers=4		)
              UpperCamelCase_          =  deepcopy(self.diffusion		)
              UpperCamelCase_          =  torch.quasirandom.SobolEngine(1					,   scramble=UpperCamelCase__		)




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		str:
       UpperCamelCase_          =  MODELS_MAP[model_name]["url"]
       os.system(F'''wget {url} ./'''  )

       return F'''./{model_name}.ckpt'''


_UpperCAmelCase        =				{
    '1': 'resnets.0',
    '2': 'attentions.0',
    '3': 'resnets.1',
    '4': 'attentions.1',
    '5': 'resnets.2',
    '6': 'attentions.2',
}
_UpperCAmelCase        =				{
    '8': 'resnets.0',
    '9': 'attentions.0',
    '10': 'resnets.1',
    '11': 'attentions.1',
    '12': 'resnets.2',
    '13': 'attentions.2',
}
_UpperCAmelCase        =				{
    '1': 'resnets.0',
    '2': 'attentions.0',
    '3': 'resnets.1',
    '4': 'attentions.1',
    '5': 'resnets.2',
    '6': 'attentions.2',
    '8': 'resnets.3',
    '9': 'attentions.3',
    '10': 'resnets.4',
    '11': 'attentions.4',
    '12': 'resnets.5',
    '13': 'attentions.5',
}
_UpperCAmelCase        =				{
    '0': 'resnets.0',
    '1': 'resnets.1',
    '2': 'resnets.2',
    '4': 'resnets.0',
    '5': 'resnets.1',
    '6': 'resnets.2',
}

_UpperCAmelCase        =				{
    'skip': 'conv_skip',
    'main.0': 'conv_1',
    'main.1': 'group_norm_1',
    'main.3': 'conv_2',
    'main.4': 'group_norm_2',
}

_UpperCAmelCase        =				{
    'norm': 'group_norm',
    'qkv_proj': ['query', 'key', 'value'],
    'out_proj': ['proj_attn'],
}




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		Union[str, Any]:





       if name.startswith("skip"  ):
              return name.replace("skip"					,		RES_CONV_MAP["skip"]  )

       # name has to be of format main.{digit}
       if not name.startswith("main."  ):
              raise ValueError(F'''ResConvBlock error with {name}'''  )

       return name.replace(name[:6]					,		RES_CONV_MAP[name[:6]]  )




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		str:





       for key, value in ATTN_MAP.items():
              if name.startswith(A_  ) and not isinstance(A_					,		A_  ):
                     return name.replace(A_					,		A_  )
              elif name.startswith(A_  ):
                     return [name.replace(A_					,		A_  ) for v in value]
       raise ValueError(F'''Attn error with {name}'''  )




def  lowerCAmelCase_ (							UpperCamelCase_					,		UpperCamelCase_=13  )     ->		int:
       UpperCamelCase_          =  input_string

       if string.split("."  )[0] == "timestep_embed":
              return string.replace("timestep_embed"					,		"time_proj"  )

       UpperCamelCase_          =  0
       if string.startswith("net.3."  ):
              depth += 1
              UpperCamelCase_          =  string[6:]
       elif string.startswith("net."  ):
              UpperCamelCase_          =  string[4:]

       while string.startswith("main.7."  ):
              depth += 1
              UpperCamelCase_          =  string[7:]

       if string.startswith("main."  ):
              UpperCamelCase_          =  string[5:]

       # mid block
       if string[:2].isdigit():
              UpperCamelCase_          =  string[:2]
              UpperCamelCase_          =  string[2:]
       else:
              UpperCamelCase_          =  string[0]
              UpperCamelCase_          =  string[1:]

       if depth == max_depth:
              UpperCamelCase_          =  MID_NUM_TO_LAYER[layer_num]
              UpperCamelCase_          =  "mid_block"
       elif depth > 0 and int(A_  ) < 7:
              UpperCamelCase_          =  DOWN_NUM_TO_LAYER[layer_num]
              UpperCamelCase_          =  F'''down_blocks.{depth}'''
       elif depth > 0 and int(A_  ) > 7:
              UpperCamelCase_          =  UP_NUM_TO_LAYER[layer_num]
              UpperCamelCase_          =  F'''up_blocks.{max_depth - depth - 1}'''
       elif depth == 0:
              UpperCamelCase_          =  DEPTH_0_TO_LAYER[layer_num]
              UpperCamelCase_          =  F'''up_blocks.{max_depth - 1}''' if int(A_  ) > 3 else "down_blocks.0"

       if not string_left.startswith("."  ):
              raise ValueError(F'''Naming error with {input_string} and string_left: {string_left}.'''  )

       UpperCamelCase_          =  string_left[1:]

       if "resnets" in new_layer:
              UpperCamelCase_          =  convert_resconv_naming(A_  )
       elif "attentions" in new_layer:
              UpperCamelCase_          =  convert_attn_naming(A_  )
              UpperCamelCase_          =  new_string_left

       if not isinstance(A_					,		A_  ):
              UpperCamelCase_          =  prefix + "." + new_layer + "." + string_left
       else:
              UpperCamelCase_          =  [prefix + "." + new_layer + "." + s for s in string_left]
       return new_string




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		Dict:
       UpperCamelCase_          =  {}
       for k, v in state_dict.items():
              if k.endswith("kernel"  ):
                     # up- and downsample layers, don't have trainable weights
                     continue

              UpperCamelCase_          =  rename(A_  )

              # check if we need to transform from Conv => Linear for attention
              if isinstance(A_					,		A_  ):
                     UpperCamelCase_          =  transform_conv_attns(A_					,		A_					,		A_  )
              else:
                     UpperCamelCase_          =  v

       return new_state_dict




def  lowerCAmelCase_ (							UpperCamelCase_					,		UpperCamelCase_					,		UpperCamelCase_  )     ->		Union[str, Any]:





       if len(A_  ) == 1:
              if len(v.shape  ) == 3:
                     # weight
                     UpperCamelCase_          =  v[:, :, 0]
              else:
                     # bias
                     UpperCamelCase_          =  v
       else:
              # qkv matrices
              UpperCamelCase_          =  v.shape[0]
              UpperCamelCase_          =  trippled_shape // 3
              for i in range(3  ):
                     if len(v.shape  ) == 3:
                            UpperCamelCase_          =  v[i * single_shape : (i + 1) * single_shape, :, 0]
                     else:
                            UpperCamelCase_          =  v[i * single_shape : (i + 1) * single_shape]
       return new_state_dict




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		Optional[int]:
       UpperCamelCase_          =  torch.device("cuda" if torch.cuda.is_available() else "cpu"  )

       UpperCamelCase_          =  args.model_path.split("/"  )[-1].split("."  )[0]
       if not os.path.isfile(args.model_path  ):
              assert (
                  model_name == args.model_path
              ), F'''Make sure to provide one of the official model names {MODELS_MAP.keys()}'''
              UpperCamelCase_          =  download(A_  )

       UpperCamelCase_          =  MODELS_MAP[model_name]["sample_rate"]
       UpperCamelCase_          =  MODELS_MAP[model_name]["sample_size"]

       UpperCamelCase_          =  Object()
       UpperCamelCase_          =  sample_size
       UpperCamelCase_          =  sample_rate
       UpperCamelCase_          =  0

       UpperCamelCase_          =  UNetaDModel(sample_size=A_					,		sample_rate=A_  )
       UpperCamelCase_          =  diffusers_model.state_dict()

       UpperCamelCase_          =  DiffusionUncond(A_  )
       orig_model.load_state_dict(torch.load(args.model_path					,		map_location=A_  )["state_dict"]  )
       UpperCamelCase_          =  orig_model.diffusion_ema.eval()
       UpperCamelCase_          =  orig_model.state_dict()
       UpperCamelCase_          =  rename_orig_weights(A_  )

       UpperCamelCase_          =  set(renamed_state_dict.keys()  ) - set(diffusers_state_dict.keys()  )
       UpperCamelCase_          =  set(diffusers_state_dict.keys()  ) - set(renamed_state_dict.keys()  )

       assert len(A_  ) == 0, F'''Problem with {renamed_minus_diffusers}'''
       assert all(k.endswith("kernel"  ) for k in list(A_  )  ), F'''Problem with {diffusers_minus_renamed}'''

       for key, value in renamed_state_dict.items():
              assert (
                  diffusers_state_dict[key].squeeze().shape == value.squeeze().shape
              ), F'''Shape for {key} doesn\'t match. Diffusers: {diffusers_state_dict[key].shape} vs. {value.shape}'''
              if key == "time_proj.weight":
                     UpperCamelCase_          =  value.squeeze()

              UpperCamelCase_          =  value

       diffusers_model.load_state_dict(A_  )

       UpperCamelCase_          =  100
       UpperCamelCase_          =  33

       UpperCamelCase_          =  IPNDMScheduler(num_train_timesteps=A_  )

       UpperCamelCase_          =  torch.manual_seed(A_  )
       UpperCamelCase_          =  torch.randn([1, 2, config.sample_size]					,		generator=A_  ).to(A_  )

       UpperCamelCase_          =  torch.linspace(1					,		0					,		steps + 1					,		device=A_  )[:-1]
       UpperCamelCase_          =  get_crash_schedule(A_  )

       UpperCamelCase_          =  DanceDiffusionPipeline(unet=A_					,		scheduler=A_  )

       UpperCamelCase_          =  torch.manual_seed(33  )
       UpperCamelCase_          =  pipe(num_inference_steps=A_					,		generator=A_  ).audios

       UpperCamelCase_          =  sampling.iplms_sample(A_					,		A_					,		A_					,		{}  )
       UpperCamelCase_          =  generated.clamp(-1					,		1  )

       UpperCamelCase_          =  (generated - audio).abs().sum()
       UpperCamelCase_          =  (generated - audio).abs().max()

       if args.save:
              pipe.save_pretrained(args.checkpoint_path  )

       print("Diff sum"					,		A_  )
       print("Diff max"					,		A_  )

       assert diff_max < 1e-3, F'''Diff max: {diff_max} is too much :-/'''

       print(F'''Conversion for {model_name} successful!'''  )


if __name__ == "__main__":
     _UpperCAmelCase        =				argparse.ArgumentParser()

     parser.add_argument('--model_path', default=None, type=str, required=True, help='Path to the model to convert.')
     parser.add_argument(
         '--save', default=True, type=bool, required=False, help='Whether to save the converted model or not.'
     )
     parser.add_argument('--checkpoint_path', default=None, type=str, required=True, help='Path to the output model.')
     _UpperCAmelCase        =				parser.parse_args()

     main(args)








import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         FalconForCausalLM,
         FalconForQuestionAnswering,
         FalconForSequenceClassification,
         FalconForTokenClassification,
         FalconModel,
     )



class 							_UpperCamelCase					:

       def __init__(		self: Dict					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]=3					,   _SCREAMING_SNAKE_CASE: Any=7					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=True					,   _SCREAMING_SNAKE_CASE: Optional[int]=True					,   _SCREAMING_SNAKE_CASE: List[Any]=False					,   _SCREAMING_SNAKE_CASE: str=True					,   _SCREAMING_SNAKE_CASE: Any=99					,   _SCREAMING_SNAKE_CASE: List[str]=32					,   _SCREAMING_SNAKE_CASE: List[str]=5					,   _SCREAMING_SNAKE_CASE: Optional[int]=4					,   _SCREAMING_SNAKE_CASE: str=37					,   _SCREAMING_SNAKE_CASE: Optional[int]="gelu"					,   _SCREAMING_SNAKE_CASE: str=0.1					,   _SCREAMING_SNAKE_CASE: Any=0.1					,   _SCREAMING_SNAKE_CASE: Optional[Any]=512					,   _SCREAMING_SNAKE_CASE: Optional[Any]=16					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=2					,   _SCREAMING_SNAKE_CASE: Tuple=0.02					,   _SCREAMING_SNAKE_CASE: Optional[Any]=3					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=4					,   _SCREAMING_SNAKE_CASE: Tuple=None					,   )       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  parent
              UpperCamelCase_          =  batch_size
              UpperCamelCase_          =  seq_length
              UpperCamelCase_          =  is_training
              UpperCamelCase_          =  use_input_mask
              UpperCamelCase_          =  use_token_type_ids
              UpperCamelCase_          =  use_labels
              UpperCamelCase_          =  vocab_size
              UpperCamelCase_          =  hidden_size
              UpperCamelCase_          =  num_hidden_layers
              UpperCamelCase_          =  num_attention_heads
              UpperCamelCase_          =  intermediate_size
              UpperCamelCase_          =  hidden_act
              UpperCamelCase_          =  hidden_dropout_prob
              UpperCamelCase_          =  attention_probs_dropout_prob
              UpperCamelCase_          =  max_position_embeddings
              UpperCamelCase_          =  type_vocab_size
              UpperCamelCase_          =  type_sequence_label_size
              UpperCamelCase_          =  initializer_range
              UpperCamelCase_          =  num_labels
              UpperCamelCase_          =  num_choices
              UpperCamelCase_          =  scope

       def lowercase (		self: Any		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  ids_tensor([self.batch_size, self.seq_length]					,   self.vocab_size		)

              UpperCamelCase_          =  None
              if self.use_input_mask:
                     UpperCamelCase_          =  random_attention_mask([self.batch_size, self.seq_length]		)

              UpperCamelCase_          =  None

              UpperCamelCase_          =  None
              UpperCamelCase_          =  None
              UpperCamelCase_          =  None
              if self.use_labels:
                     UpperCamelCase_          =  ids_tensor([self.batch_size]					,   self.type_sequence_label_size		)
                     UpperCamelCase_          =  ids_tensor([self.batch_size, self.seq_length]					,   self.num_labels		)
                     UpperCamelCase_          =  ids_tensor([self.batch_size]					,   self.num_choices		)

              UpperCamelCase_          =  self.get_config()

              return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

       def lowercase (		self: Any		)       -> Union[str, Any]:




              """simple docstring"""


              return FalconConfig(
                  vocab_size=self.vocab_size					,   hidden_size=self.hidden_size					,   num_hidden_layers=self.num_hidden_layers					,   num_attention_heads=self.num_attention_heads					,   intermediate_size=self.intermediate_size					,   hidden_act=self.hidden_act					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   max_position_embeddings=self.max_position_embeddings					,   type_vocab_size=self.type_vocab_size					,   is_decoder=_SCREAMING_SNAKE_CASE					,   initializer_range=self.initializer_range					,   pad_token_id=1					,   new_decoder_architecture=_SCREAMING_SNAKE_CASE					,   )

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: Dict					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Dict					,   _SCREAMING_SNAKE_CASE: Union[str, Any]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  FalconModel(config=_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)		)

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: Tuple					,   )       -> str:




              """simple docstring"""


              UpperCamelCase_          =  True
              UpperCamelCase_          =  FalconModel(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)		)

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: int					,   )       -> int:




              """simple docstring"""


              UpperCamelCase_          =  FalconForCausalLM(config=_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)		)

       def lowercase (		self: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: Optional[int]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Union[str, Any]					,   )       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =  True
              UpperCamelCase_          =  True
              UpperCamelCase_          =  FalconForCausalLM(config=_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()

              # first forward pass
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   use_cache=_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  outputs.past_key_values

              # create hypothetical multiple next token and extent to next_input_ids
              UpperCamelCase_          =  ids_tensor((self.batch_size, 3)					,   config.vocab_size		)
              UpperCamelCase_          =  ids_tensor((self.batch_size, 3)					,   vocab_size=2		)

              # append to next input_ids and
              UpperCamelCase_          =  torch.cat([input_ids, next_tokens]					,   dim=-1		)
              UpperCamelCase_          =  torch.cat([input_mask, next_mask]					,   dim=-1		)

              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   output_hidden_states=_SCREAMING_SNAKE_CASE					,   )["hidden_states"][0]
              UpperCamelCase_          =  model(
                  _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   encoder_hidden_states=_SCREAMING_SNAKE_CASE					,   encoder_attention_mask=_SCREAMING_SNAKE_CASE					,   past_key_values=_SCREAMING_SNAKE_CASE					,   output_hidden_states=_SCREAMING_SNAKE_CASE					,   )["hidden_states"][0]

              # select random slice
              UpperCamelCase_          =  ids_tensor((1,)					,   output_from_past.shape[-1]		).item()
              UpperCamelCase_          =  output_from_no_past[:, -3:, random_slice_idx].detach()
              UpperCamelCase_          =  output_from_past[:, :, random_slice_idx].detach()

              self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

              # test that outputs are equal for slice
              self.parent.assertTrue(torch.allclose(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE					,   atol=1e-3		)		)



       def lowercase (		self: Tuple		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  self.prepare_config_and_inputs()
              (
                  (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , (
                  UpperCamelCase_
              )       , 
              )          =  config_and_inputs
              UpperCamelCase_          =  {"input_ids": input_ids, "attention_mask": input_mask}
              return config, inputs_dict



@require_torch
class 							_UpperCamelCase					(	lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					Optional[Any]         =   (
           (
               FalconModel,
               FalconForCausalLM,
               FalconForSequenceClassification,
               FalconForTokenClassification,
               FalconForQuestionAnswering,
           )
           if is_torch_available()
           else ()
       )
       _UpperCamelCase				:					Union[str, Any]         =   (FalconForCausalLM,) if is_torch_available() else ()
       _UpperCamelCase				:					Optional[int]         =   (
           {
               '''feature-extraction''': FalconModel,
               '''text-classification''': FalconForSequenceClassification,
               '''text-generation''': FalconForCausalLM,
               '''question-answering''': FalconForQuestionAnswering,
               '''token-classification''': FalconForTokenClassification,
               '''zero-shot''': FalconForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       _UpperCamelCase				:					Any         =   False
       _UpperCamelCase				:					int         =   False

       def lowercase (		self: str		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  FalconModelTester(self		)
              UpperCamelCase_          =  ConfigTester(self					,   config_class=_SCREAMING_SNAKE_CASE					,   hidden_size=37		)

       def lowercase (		self: Optional[Any]		)       -> Optional[int]:




              """simple docstring"""


              self.config_tester.run_common_tests()

       def lowercase (		self: str		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: List[Any]		)       -> Dict:




              """simple docstring"""


              UpperCamelCase_       , *UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs()
              for alibi in [True, False]:
                     UpperCamelCase_          =  alibi
                     self.model_tester.create_and_check_model(_SCREAMING_SNAKE_CASE					,   *_SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Any		)       -> str:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  3
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  input_ids.ne(1		).to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  ids_tensor([self.model_tester.batch_size]					,   self.model_tester.type_sequence_label_size		)
              UpperCamelCase_          =  FalconForSequenceClassification(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.assertEqual(result.logits.shape					,   (self.model_tester.batch_size, self.model_tester.num_labels)		)

       def lowercase (		self: Dict		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  3
              UpperCamelCase_          =  "single_label_classification"
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  input_ids.ne(1		).to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  ids_tensor([self.model_tester.batch_size]					,   self.model_tester.type_sequence_label_size		)
              UpperCamelCase_          =  FalconForSequenceClassification(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.assertEqual(result.logits.shape					,   (self.model_tester.batch_size, self.model_tester.num_labels)		)

       def lowercase (		self: List[str]		)       -> int:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  FalconForCausalLM(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   use_cache=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  input_ids.shape[0]
              UpperCamelCase_          =  model._convert_to_rw_cache(result.past_key_values		)
              UpperCamelCase_          =  model._convert_cache_to_standard_format(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)
              for layer in range(len(_SCREAMING_SNAKE_CASE		)		):
                     for tensor_idx in range(2		):
                            self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3		)
                            self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4		)
                            self.assertTrue(
                                torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]		)		)

       def lowercase (		self: Union[str, Any]		)       -> Any:




              """simple docstring"""


              UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()
              UpperCamelCase_          =  3
              UpperCamelCase_          =  "multi_label_classification"
              UpperCamelCase_          =  input_dict["input_ids"]
              UpperCamelCase_          =  input_ids.ne(1		).to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  ids_tensor(
                  [self.model_tester.batch_size, config.num_labels]					,   self.model_tester.type_sequence_label_size		).to(torch.float		)
              UpperCamelCase_          =  FalconForSequenceClassification(_SCREAMING_SNAKE_CASE		)
              model.to(_SCREAMING_SNAKE_CASE		)
              model.eval()
              UpperCamelCase_          =  model(_SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   labels=_SCREAMING_SNAKE_CASE		)
              self.assertEqual(result.logits.shape					,   (self.model_tester.batch_size, self.model_tester.num_labels)		)



       def lowercase (		self: List[Any]		)       -> List[Any]:




              """simple docstring"""


              for model_class in self.all_generative_model_classes:
                     UpperCamelCase_       , UpperCamelCase_          =  self.model_tester.prepare_config_and_inputs_for_common()

                     # If it doesn't support cache, pass the test
                     if not hasattr(_SCREAMING_SNAKE_CASE					,   "use_cache"		):
                            return

                     UpperCamelCase_          =  model_class(_SCREAMING_SNAKE_CASE		).to(_SCREAMING_SNAKE_CASE		)
                     if "use_cache" not in inputs:
                            UpperCamelCase_          =  True
                     UpperCamelCase_          =  model(**_SCREAMING_SNAKE_CASE		)

                     # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
                     if "past_key_values" not in outputs:
                            return

                     UpperCamelCase_          =  (
                         getattr(_SCREAMING_SNAKE_CASE					,   "decoder_layers"					,   _SCREAMING_SNAKE_CASE		)
                         or getattr(_SCREAMING_SNAKE_CASE					,   "num_decoder_layers"					,   _SCREAMING_SNAKE_CASE		)
                         or config.num_hidden_layers
                     )
                     UpperCamelCase_          =  getattr(_SCREAMING_SNAKE_CASE					,   "num_kv_heads"					,   config.num_attention_heads		)
                     UpperCamelCase_          =  getattr(_SCREAMING_SNAKE_CASE					,   "d_model"					,   config.hidden_size		)
                     UpperCamelCase_          =  embed_dim // num_attention_heads

                     UpperCamelCase_          =  outputs["past_key_values"]
                     self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)

                     UpperCamelCase_       , UpperCamelCase_          =  inputs["input_ids"].shape
                     for i in range(_SCREAMING_SNAKE_CASE		):
                            if config.new_decoder_architecture:
                                   UpperCamelCase_          =  config.num_attention_heads
                            elif config.multi_query:
                                   UpperCamelCase_          =  1
                            self.assertEqual(len(past_kv[0]		)					,   2		)  # K V for the decoder = 2
                            self.assertEqual(
                                past_kv[i][0].shape					,   (batch_size, num_attention_heads, seq_length, per_head_embed_dim)		)
                            self.assertEqual(
                                past_kv[i][1].shape					,   (batch_size, num_attention_heads, seq_length, per_head_embed_dim)		)



@require_torch
class 							_UpperCamelCase					(	unittest.TestCase     ):

       @slow
       def lowercase (		self: Tuple		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b"		)
              UpperCamelCase_          =  FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b"		)
              model.eval()
              model.to(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tokenizer("My favorite food is"					,   return_tensors="pt"		).to(_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  (
                  "My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."
              )

              UpperCamelCase_          =  model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=19		)
              UpperCamelCase_          =  tokenizer.batch_decode(_SCREAMING_SNAKE_CASE		)[0]

              self.assertEqual(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

       @slow
       def lowercase (		self: Dict		)       -> Dict:




              """simple docstring"""


              for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
                     UpperCamelCase_          =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  FalconForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
                     model.eval()
                     model.to(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  tokenizer("My favorite food is"					,   return_tensors="pt"		).to(_SCREAMING_SNAKE_CASE		)

                     # We just test that these run without errors - the models are randomly initialized
                     # and so the actual text outputs will be garbage
                     model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=4		)
                     model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=4		)
                     model.generate(**_SCREAMING_SNAKE_CASE					,   num_beams=2					,   max_new_tokens=4		)



       @slow
       def lowercase (		self: Union[str, Any]		)       -> Any:




              """simple docstring"""


              with torch.no_grad():
                     for repo in [
                         "Rocketknight1/falcon-rw-1b",
                         "Rocketknight1/tiny-random-falcon-7b",
                         "Rocketknight1/tiny-random-falcon-40b",
                     ]:
                            UpperCamelCase_          =  AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  FalconForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE		)
                            model.eval()
                            model.to(device=_SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  tokenizer("My favorite food is"					,   return_tensors="pt"		).to(_SCREAMING_SNAKE_CASE		)

                            # Test results are the same with and without cache
                            UpperCamelCase_          =  model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=20					,   use_cache=_SCREAMING_SNAKE_CASE		)
                            UpperCamelCase_          =  model.generate(**_SCREAMING_SNAKE_CASE					,   do_sample=_SCREAMING_SNAKE_CASE					,   max_new_tokens=20					,   use_cache=_SCREAMING_SNAKE_CASE		)
                            self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0		)

