

import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        FalconForCausalLM,
        FalconForQuestionAnswering,
        FalconForSequenceClassification,
        FalconForTokenClassification,
        FalconModel,
    )





class 			_SCREAMING_SNAKE_CASE :







   '''simple docstring'''




   def __init__(       self					:			str     , __lowerCamelCase					:			List[Any]     , __lowerCamelCase					:			int=3     , __lowerCamelCase					:			List[str]=7     , __lowerCamelCase					:			int=True     , __lowerCamelCase					:			Dict=True     , __lowerCamelCase					:			Optional[Any]=False     , __lowerCamelCase					:			str=True     , __lowerCamelCase					:			Optional[int]=99     , __lowerCamelCase					:			Any=32     , __lowerCamelCase					:			Tuple=5     , __lowerCamelCase					:			str=4     , __lowerCamelCase					:			str=37     , __lowerCamelCase					:			List[Any]="gelu"     , __lowerCamelCase					:			List[Any]=0.1     , __lowerCamelCase					:			int=0.1     , __lowerCamelCase					:			Any=512     , __lowerCamelCase					:			Tuple=16     , __lowerCamelCase					:			Optional[Any]=2     , __lowerCamelCase					:			Optional[int]=0.02     , __lowerCamelCase					:			List[Any]=3     , __lowerCamelCase					:			Dict=4     , __lowerCamelCase					:			Optional[Any]=None     , ):
       SCREAMING_SNAKE_CASE										=   parent
       SCREAMING_SNAKE_CASE										=   batch_size
       SCREAMING_SNAKE_CASE										=   seq_length
       SCREAMING_SNAKE_CASE										=   is_training
       SCREAMING_SNAKE_CASE										=   use_input_mask
       SCREAMING_SNAKE_CASE										=   use_token_type_ids
       SCREAMING_SNAKE_CASE										=   use_labels
       SCREAMING_SNAKE_CASE										=   vocab_size
       SCREAMING_SNAKE_CASE										=   hidden_size
       SCREAMING_SNAKE_CASE										=   num_hidden_layers
       SCREAMING_SNAKE_CASE										=   num_attention_heads
       SCREAMING_SNAKE_CASE										=   intermediate_size
       SCREAMING_SNAKE_CASE										=   hidden_act
       SCREAMING_SNAKE_CASE										=   hidden_dropout_prob
       SCREAMING_SNAKE_CASE										=   attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE										=   max_position_embeddings
       SCREAMING_SNAKE_CASE										=   type_vocab_size
       SCREAMING_SNAKE_CASE										=   type_sequence_label_size
       SCREAMING_SNAKE_CASE										=   initializer_range
       SCREAMING_SNAKE_CASE										=   num_labels
       SCREAMING_SNAKE_CASE										=   num_choices
       SCREAMING_SNAKE_CASE										=   scope





   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   ids_tensor([self.batch_size, self.seq_length]     , self.vocab_size			)

       SCREAMING_SNAKE_CASE										=   None
       if self.use_input_mask:
           SCREAMING_SNAKE_CASE										=   random_attention_mask([self.batch_size, self.seq_length]			)

       SCREAMING_SNAKE_CASE										=   None

       SCREAMING_SNAKE_CASE										=   None
       SCREAMING_SNAKE_CASE										=   None
       SCREAMING_SNAKE_CASE										=   None
       if self.use_labels:
           SCREAMING_SNAKE_CASE										=   ids_tensor([self.batch_size]     , self.type_sequence_label_size			)
           SCREAMING_SNAKE_CASE										=   ids_tensor([self.batch_size, self.seq_length]     , self.num_labels			)
           SCREAMING_SNAKE_CASE										=   ids_tensor([self.batch_size]     , self.num_choices			)

       SCREAMING_SNAKE_CASE										=   self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





   def      _snake_case		(       self					:			str			):
       return FalconConfig(
           vocab_size=self.vocab_size     , hidden_size=self.hidden_size     , num_hidden_layers=self.num_hidden_layers     , num_attention_heads=self.num_attention_heads     , intermediate_size=self.intermediate_size     , hidden_act=self.hidden_act     , hidden_dropout_prob=self.hidden_dropout_prob     , attention_probs_dropout_prob=self.attention_probs_dropout_prob     , max_position_embeddings=self.max_position_embeddings     , type_vocab_size=self.type_vocab_size     , is_decoder=__lowerCamelCase     , initializer_range=self.initializer_range     , pad_token_id=1     , new_decoder_architecture=__lowerCamelCase     , )





   def      _snake_case		(       self					:			Union[str, Any]     , __lowerCamelCase					:			List[str]     , __lowerCamelCase					:			List[Any]     , __lowerCamelCase					:			List[str]     , __lowerCamelCase					:			Tuple     , __lowerCamelCase					:			Any     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   FalconModel(config=__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase			)
       self.parent.assertEqual(result.last_hidden_state.shape     , (self.batch_size, self.seq_length, self.hidden_size)			)





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Tuple     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Optional[int]     , __lowerCamelCase					:			Union[str, Any]     , __lowerCamelCase					:			int     , __lowerCamelCase					:			Union[str, Any]     , __lowerCamelCase					:			List[Any]     , ):
       SCREAMING_SNAKE_CASE										=   True
       SCREAMING_SNAKE_CASE										=   FalconModel(__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(
           __lowerCamelCase     , attention_mask=__lowerCamelCase     , encoder_hidden_states=__lowerCamelCase     , encoder_attention_mask=__lowerCamelCase     , )
       SCREAMING_SNAKE_CASE										=   model(
           __lowerCamelCase     , attention_mask=__lowerCamelCase     , encoder_hidden_states=__lowerCamelCase     , )
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase			)
       self.parent.assertEqual(result.last_hidden_state.shape     , (self.batch_size, self.seq_length, self.hidden_size)			)





   def      _snake_case		(       self					:			List[str]     , __lowerCamelCase					:			Union[str, Any]     , __lowerCamelCase					:			Optional[int]     , __lowerCamelCase					:			Optional[int]     , __lowerCamelCase					:			List[Any]     , __lowerCamelCase					:			Tuple     , __lowerCamelCase					:			Tuple     , __lowerCamelCase					:			int     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			List[Any]     , ):
       SCREAMING_SNAKE_CASE										=   FalconForCausalLM(config=__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase     , labels=__lowerCamelCase			)
       self.parent.assertEqual(result.logits.shape     , (self.batch_size, self.seq_length, self.vocab_size)			)





   def      _snake_case		(       self					:			str     , __lowerCamelCase					:			Union[str, Any]     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			Union[str, Any]     , __lowerCamelCase					:			str     , __lowerCamelCase					:			int     , __lowerCamelCase					:			Optional[Any]     , __lowerCamelCase					:			Dict     , __lowerCamelCase					:			List[Any]     , ):
       SCREAMING_SNAKE_CASE										=   True
       SCREAMING_SNAKE_CASE										=   True
       SCREAMING_SNAKE_CASE										=   FalconForCausalLM(config=__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()

       # first forward pass
       SCREAMING_SNAKE_CASE										=   model(
           __lowerCamelCase     , attention_mask=__lowerCamelCase     , encoder_hidden_states=__lowerCamelCase     , encoder_attention_mask=__lowerCamelCase     , use_cache=__lowerCamelCase     , )
       SCREAMING_SNAKE_CASE										=   outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       SCREAMING_SNAKE_CASE										=   ids_tensor((self.batch_size, 3)     , config.vocab_size			)
       SCREAMING_SNAKE_CASE										=   ids_tensor((self.batch_size, 3)     , vocab_size=2			)

       # append to next input_ids and
       SCREAMING_SNAKE_CASE										=   torch.cat([input_ids, next_tokens]     , dim=-1			)
       SCREAMING_SNAKE_CASE										=   torch.cat([input_mask, next_mask]     , dim=-1			)

       SCREAMING_SNAKE_CASE										=   model(
           __lowerCamelCase     , attention_mask=__lowerCamelCase     , encoder_hidden_states=__lowerCamelCase     , encoder_attention_mask=__lowerCamelCase     , output_hidden_states=__lowerCamelCase     , )["hidden_states"][0]
       SCREAMING_SNAKE_CASE										=   model(
           __lowerCamelCase     , attention_mask=__lowerCamelCase     , encoder_hidden_states=__lowerCamelCase     , encoder_attention_mask=__lowerCamelCase     , past_key_values=__lowerCamelCase     , output_hidden_states=__lowerCamelCase     , )["hidden_states"][0]

       # select random slice
       SCREAMING_SNAKE_CASE										=   ids_tensor((1,)     , output_from_past.shape[-1]			).item()
       SCREAMING_SNAKE_CASE										=   output_from_no_past[:, -3:, random_slice_idx].detach()
       SCREAMING_SNAKE_CASE										=   output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(__lowerCamelCase     , __lowerCamelCase     , atol=1e-3			)			)





   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   self.prepare_config_and_inputs()
       (
           (
           SCREAMING_SNAKE_CASE
       )							,	(
           SCREAMING_SNAKE_CASE
       )							,	(
           SCREAMING_SNAKE_CASE
       )							,	(
           SCREAMING_SNAKE_CASE
       )							,	(
           SCREAMING_SNAKE_CASE
       )							,	(
           SCREAMING_SNAKE_CASE
       )							,	(
           SCREAMING_SNAKE_CASE
       )							,	
       )										=   config_and_inputs
       SCREAMING_SNAKE_CASE										=   {"input_ids": input_ids, "attention_mask": input_mask}
       return config, inputs_dict





@require_torch
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       __snake_case					,       __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      (
       (
           FalconModel,
           FalconForCausalLM,
           FalconForSequenceClassification,
           FalconForTokenClassification,
           FalconForQuestionAnswering,
       )
       if is_torch_available()
       else ()
   )
   lowerCamelCase__				=      (FalconForCausalLM,) if is_torch_available() else ()
   lowerCamelCase__				=      (
       {
           "feature-extraction": FalconModel,
           "text-classification": FalconForSequenceClassification,
           "text-generation": FalconForCausalLM,
           "question-answering": FalconForQuestionAnswering,
           "token-classification": FalconForTokenClassification,
           "zero-shot": FalconForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   lowerCamelCase__				=      False
   lowerCamelCase__				=      False





   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   FalconModelTester(self			)
       SCREAMING_SNAKE_CASE										=   ConfigTester(self     , config_class=__lowerCamelCase     , hidden_size=37			)





   def      _snake_case		(       self					:			int			):
       self.config_tester.run_common_tests()





   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__lowerCamelCase			)





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE							,	*SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs()
       for alibi in [True, False]:
           SCREAMING_SNAKE_CASE										=   alibi
           self.model_tester.create_and_check_model(__lowerCamelCase     , *__lowerCamelCase			)





   def      _snake_case		(       self					:			str			):
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs_for_common()
       SCREAMING_SNAKE_CASE										=   3
       SCREAMING_SNAKE_CASE										=   input_dict["input_ids"]
       SCREAMING_SNAKE_CASE										=   input_ids.ne(1			).to(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   ids_tensor([self.model_tester.batch_size]     , self.model_tester.type_sequence_label_size			)
       SCREAMING_SNAKE_CASE										=   FalconForSequenceClassification(__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase     , labels=__lowerCamelCase			)
       self.assertEqual(result.logits.shape     , (self.model_tester.batch_size, self.model_tester.num_labels)			)





   def      _snake_case		(       self					:			Any			):
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs_for_common()
       SCREAMING_SNAKE_CASE										=   3
       SCREAMING_SNAKE_CASE										=   "single_label_classification"
       SCREAMING_SNAKE_CASE										=   input_dict["input_ids"]
       SCREAMING_SNAKE_CASE										=   input_ids.ne(1			).to(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   ids_tensor([self.model_tester.batch_size]     , self.model_tester.type_sequence_label_size			)
       SCREAMING_SNAKE_CASE										=   FalconForSequenceClassification(__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase     , labels=__lowerCamelCase			)
       self.assertEqual(result.logits.shape     , (self.model_tester.batch_size, self.model_tester.num_labels)			)





   def      _snake_case		(       self					:			Any			):
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs_for_common()
       SCREAMING_SNAKE_CASE										=   input_dict["input_ids"]
       SCREAMING_SNAKE_CASE										=   FalconForCausalLM(__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , use_cache=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   input_ids.shape[0]
       SCREAMING_SNAKE_CASE										=   model._convert_to_rw_cache(result.past_key_values			)
       SCREAMING_SNAKE_CASE										=   model._convert_cache_to_standard_format(__lowerCamelCase     , __lowerCamelCase			)
       for layer in range(len(__lowerCamelCase			)			):
           for tensor_idx in range(2			):
               self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3			)
               self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4			)
               self.assertTrue(
                   torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]			)			)





   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs_for_common()
       SCREAMING_SNAKE_CASE										=   3
       SCREAMING_SNAKE_CASE										=   "multi_label_classification"
       SCREAMING_SNAKE_CASE										=   input_dict["input_ids"]
       SCREAMING_SNAKE_CASE										=   input_ids.ne(1			).to(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   ids_tensor(
           [self.model_tester.batch_size, config.num_labels]     , self.model_tester.type_sequence_label_size			).to(torch.float			)
       SCREAMING_SNAKE_CASE										=   FalconForSequenceClassification(__lowerCamelCase			)
       model.to(__lowerCamelCase			)
       model.eval()
       SCREAMING_SNAKE_CASE										=   model(__lowerCamelCase     , attention_mask=__lowerCamelCase     , labels=__lowerCamelCase			)
       self.assertEqual(result.logits.shape     , (self.model_tester.batch_size, self.model_tester.num_labels)			)





   def      _snake_case		(       self					:			List[str]			):
       # Falcon can have different numbers of KV-heads than the number of query heads, so we need
       # to override this test to use the right head counts.
       for model_class in self.all_generative_model_classes:
           SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.model_tester.prepare_config_and_inputs_for_common()

           # If it doesn't support cache, pass the test
           if not hasattr(__lowerCamelCase     , "use_cache"			):
               return

           SCREAMING_SNAKE_CASE										=   model_class(__lowerCamelCase			).to(__lowerCamelCase			)
           if "use_cache" not in inputs:
               SCREAMING_SNAKE_CASE										=   True
           SCREAMING_SNAKE_CASE										=   model(**__lowerCamelCase			)

           # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
           if "past_key_values" not in outputs:
               return

           SCREAMING_SNAKE_CASE										=   (
               getattr(__lowerCamelCase     , "decoder_layers"     , __lowerCamelCase			)
               or getattr(__lowerCamelCase     , "num_decoder_layers"     , __lowerCamelCase			)
               or config.num_hidden_layers
           )
           SCREAMING_SNAKE_CASE										=   getattr(__lowerCamelCase     , "num_kv_heads"     , config.num_attention_heads			)
           SCREAMING_SNAKE_CASE										=   getattr(__lowerCamelCase     , "d_model"     , config.hidden_size			)
           SCREAMING_SNAKE_CASE										=   embed_dim // num_attention_heads

           SCREAMING_SNAKE_CASE										=   outputs["past_key_values"]
           self.assertEqual(len(__lowerCamelCase			)     , __lowerCamelCase			)

           SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   inputs["input_ids"].shape
           for i in range(__lowerCamelCase			):
               if config.new_decoder_architecture:
                   SCREAMING_SNAKE_CASE										=   config.num_attention_heads
               elif config.multi_query:
                   SCREAMING_SNAKE_CASE										=   1
               self.assertEqual(len(past_kv[0]			)     , 2			)  # K V for the decoder = 2
               self.assertEqual(
                   past_kv[i][0].shape     , (batch_size, num_attention_heads, seq_length, per_head_embed_dim)			)
               self.assertEqual(
                   past_kv[i][1].shape     , (batch_size, num_attention_heads, seq_length, per_head_embed_dim)			)





@require_torch
class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   @slow
   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b"			)
       SCREAMING_SNAKE_CASE										=   FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b"			)
       model.eval()
       model.to(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer("My favorite food is"     , return_tensors="pt"			).to(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   (
           "My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."
       )

       SCREAMING_SNAKE_CASE										=   model.generate(**__lowerCamelCase     , do_sample=__lowerCamelCase     , max_new_tokens=19			)
       SCREAMING_SNAKE_CASE										=   tokenizer.batch_decode(__lowerCamelCase			)[0]

       self.assertEqual(__lowerCamelCase     , __lowerCamelCase			)





   @slow
   def      _snake_case		(       self					:			Union[str, Any]			):
       # The big models are way too big for the CI, so we use tiny random models that resemble their
       # architectures but with much smaller and fewer layers
       for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
           SCREAMING_SNAKE_CASE										=   AutoTokenizer.from_pretrained(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   FalconForCausalLM.from_pretrained(__lowerCamelCase			)
           model.eval()
           model.to(__lowerCamelCase			)
           SCREAMING_SNAKE_CASE										=   tokenizer("My favorite food is"     , return_tensors="pt"			).to(__lowerCamelCase			)

           # We just test that these run without errors - the models are randomly initialized
           # and so the actual text outputs will be garbage
           model.generate(**__lowerCamelCase     , do_sample=__lowerCamelCase     , max_new_tokens=4			)
           model.generate(**__lowerCamelCase     , do_sample=__lowerCamelCase     , max_new_tokens=4			)
           model.generate(**__lowerCamelCase     , num_beams=2     , max_new_tokens=4			)





   @slow
   def      _snake_case		(       self					:			int			):
       # The big models are way too big for the CI, so we use tiny random models that resemble their
       # architectures but with much smaller and fewer layers
       with torch.no_grad():
           for repo in [
               "Rocketknight1/falcon-rw-1b",
               "Rocketknight1/tiny-random-falcon-7b",
               "Rocketknight1/tiny-random-falcon-40b",
           ]:
               SCREAMING_SNAKE_CASE										=   AutoTokenizer.from_pretrained(__lowerCamelCase			)
               SCREAMING_SNAKE_CASE										=   FalconForCausalLM.from_pretrained(__lowerCamelCase			)
               model.eval()
               model.to(device=__lowerCamelCase			)
               SCREAMING_SNAKE_CASE										=   tokenizer("My favorite food is"     , return_tensors="pt"			).to(__lowerCamelCase			)

               # Test results are the same with and without cache
               SCREAMING_SNAKE_CASE										=   model.generate(**__lowerCamelCase     , do_sample=__lowerCamelCase     , max_new_tokens=20     , use_cache=__lowerCamelCase			)
               SCREAMING_SNAKE_CASE										=   model.generate(**__lowerCamelCase     , do_sample=__lowerCamelCase     , max_new_tokens=20     , use_cache=__lowerCamelCase			)
               self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0			)


def 	__a     (      A__			:  int = 1000       ):
    SCREAMING_SNAKE_CASE										=   3
    SCREAMING_SNAKE_CASE										=   0
    while a < n:
        if a % 3 == 0 or a % 5 == 0:
            result += a
        elif a % 15 == 0:
            result -= a
        a += 1
    return result


if __name__ == "__main__":
    print(f'{solution() = }')